ferret 0.3.2 → 0.9.0

data/ext/q_phrase.c

@@ -0,0 +1,657 @@
+#include <string.h>
+#include "search.h"
+
+/***************************************************************************
+ *
+ * PhraseWeight
+ *
+ ***************************************************************************/
+
+Scorer *phw_scorer(Weight *self, IndexReader *ir)
+{
+  Scorer *phsc;
+  PhraseQuery *phq = (PhraseQuery *)self->query->data;
+  int i;
+  if (phq->t_cnt == 0) return NULL; // optimize zero-term case
+
+  TermDocEnum **tps = ALLOC_N(TermDocEnum *, phq->t_cnt);
+
+  for (i = 0; i < phq->t_cnt; i++) {
+    tps[i] = ir_term_positions_for(ir, phq->terms[i]);
+    if (tps[i] == NULL) {
+      // free everything we just created and return NULL
+      int j;
+      for (j = 0; j < i; j++) {
+        tps[i]->close(tps[i]);
+      }
+      free(tps);
+      return NULL;
+    }
+  }
+
+  if (phq->slop == 0) {       // optimize exact case
+    phsc = exact_phrase_scorer_create(self, tps, phq->positions, phq->t_cnt,
+                                      self->similarity,
+                                      ir->get_norms(ir, phq->field));
+  } else {
+    phsc = sloppy_phrase_scorer_create(self, tps, phq->positions, phq->t_cnt,
+                                       self->similarity,
+                                       phq->slop,
+                                       ir->get_norms(ir, phq->field));
+  }
+  free(tps);
+  return phsc;
+}
+
+Explanation *phw_explain(Weight *self, IndexReader *ir, int doc_num)
+{
+  char *query_str = self->query->to_s(self->query, "");
+  PhraseQuery *phq = (PhraseQuery *)self->query->data;
+  int i;
+  char *doc_freqs = NULL;
+  int len = 0, pos = 0;
+
+  Explanation *expl = expl_create(0.0,
+      epstrdup("weight(%s in %d), product of:",
+        strlen(query_str) + 20,
+        query_str, doc_num));
+
+  for (i = 0; i < phq->t_cnt; i++) {
+    len += strlen(phq->terms[i]->text) + 30;
+  }
+  doc_freqs = ALLOC_N(char, len);
+  for (i = 0; i < phq->t_cnt; i++) {
+    Term *term = phq->terms[i];
+    sprintf(doc_freqs + pos, "%s=%d, ", term->text, ir->doc_freq(ir, term));
+    pos += strlen(doc_freqs + pos);
+  }
+  pos -= 2; // remove ", " from the end
+  doc_freqs[pos] = 0;
+
+  Explanation *idf_expl1 = expl_create(self->idf,
+      epstrdup("idf(%s:<%s>)", strlen(phq->field) + pos, phq->field, doc_freqs));
+  Explanation *idf_expl2 = expl_create(self->idf,
+      epstrdup("idf(%s:<%s>)", strlen(phq->field) + pos, phq->field, doc_freqs));
+  free(doc_freqs);
+  
+  // explain query weight
+  Explanation *query_expl = expl_create(0.0, 
+      epstrdup("query_weight(%s), product of:", strlen(query_str), query_str));
+
+  if (self->query->boost != 1.0) {
+    expl_add_detail(query_expl, expl_create(self->query->boost, estrdup("boost")));
+  }
+  expl_add_detail(query_expl, idf_expl1);
+  
+  Explanation *qnorm_expl = expl_create(self->qnorm, estrdup("query_norm"));
+  expl_add_detail(query_expl, qnorm_expl);
+  
+  query_expl->value = self->query->boost * self->idf * self->qnorm;
+
+  expl_add_detail(expl, query_expl);
+ 
+  // explain field weight
+  Explanation *field_expl = expl_create(0.0, 
+      epstrdup("field_weight(%s in %d), product of:",
+        strlen(query_str) + 20, query_str, doc_num));
+  free(query_str);
+
+  Scorer *scorer = self->scorer(self, ir);
+  Explanation *tf_expl = scorer->explain(scorer, doc_num);
+  scorer->destroy(scorer);
+  expl_add_detail(field_expl, tf_expl);
+  expl_add_detail(field_expl, idf_expl2);
+
+  uchar *field_norms = ir->get_norms(ir, phq->field);
+  float field_norm = (field_norms != NULL)
+    ? sim_decode_norm(self->similarity, field_norms[doc_num])
+    : 0.0;
+  Explanation *field_norm_expl = expl_create(field_norm,
+    epstrdup("field_norm(field=%s, doc=%d)",
+        strlen(phq->field) + 20, phq->field, doc_num));
+
+  expl_add_detail(field_expl, field_norm_expl);
+
+  field_expl->value = tf_expl->value * self->idf * field_norm;
+  
+  // combine them
+  if (query_expl->value == 1.0) {
+    expl_destoy(expl);
+    return field_expl;
+  } else {
+    expl->value = (query_expl->value * field_expl->value);
+    expl_add_detail(expl, field_expl);
+    return expl;
+  }
+}
+
+char *phw_to_s(Weight *self)
+{
+  char dbuf[32];
+  dbl_to_s(dbuf, self->value);
+  return epstrdup("PhraseWeight(%s)", strlen(dbuf), dbuf);
+}
+
+Weight *phw_create(Query *query, Searcher *searcher)
+{
+  PhraseQuery *phq = (PhraseQuery *)query->data;
+  Weight *self = ALLOC(Weight);
+  ZEROSET(self, Weight, 1);
+  self->get_query = &w_get_query;
+  self->get_value = &w_get_value;
+  self->normalize = &w_normalize;
+  self->scorer    = &phw_scorer;
+  self->explain   = &phw_explain;
+  self->to_s      = &phw_to_s;
+  self->destroy   = &free;
+  self->sum_of_squared_weights = &w_sum_of_squared_weights;
+
+  self->similarity = query->get_similarity(query, searcher);
+  self->query = query;
+  self->value = query->boost;
+  self->idf = sim_idf_phrase(self->similarity, phq->terms, phq->t_cnt, searcher);
+
+  return self;
+}
+
+/***************************************************************************
+ *
+ * PhraseQuery
+ *
+ ***************************************************************************/
+
+#define GET_PHQ PhraseQuery *phq = (PhraseQuery *)self->data
+
+void phq_extract_terms(Query *self, Array *terms)
+{
+  GET_PHQ;
+  int i;
+  for (i = 0; i < phq->t_cnt; i++) {
+    ary_append(terms, phq->terms[i]);
+  }
+}
+
+char *phq_to_s(Query *self, char *field)
+{
+  GET_PHQ;
+  int i, j, buf_index = 0, len = 0, pos, last_pos = -1;
+  char *buffer;
+  if (!phq->t_cnt) return NULL;
+  len = strlen(phq->field) + 1;
+  for (i = 0; i < phq->t_cnt; i++) {
+    len += strlen(phq->terms[i]->text) + 1;
+  }
+  // add space for extra characters and boost and slop
+  len += 100 + 3 * phq->positions[phq->t_cnt - 1];
+
+  buffer = ALLOC_N(char, len);
+
+  if (strcmp(field, phq->field) != 0) {
+    len = strlen(phq->field);
+    memcpy(buffer, phq->field, len);
+    buffer[len] = ':';
+    buf_index += len + 1;
+  }
+  buffer[buf_index++] = '"';
+
+  for (i = 0; i < phq->t_cnt; i++) {
+    Term *term = phq->terms[i];
+    pos = phq->positions[i];
+    for (j = last_pos; j < pos - 1; j++) {
+      memcpy(buffer + buf_index, "<> ", 3);
+      buf_index += 3;
+    }
+    last_pos = pos;
+
+    len = strlen(term->text);
+    memcpy(buffer + buf_index, term->text, len);
+    buf_index += len;
+    buffer[buf_index++] = ' ';
+  }
+  if (buffer[buf_index-1] == ' ') buf_index--;
+  buffer[buf_index++] = '"';
+  buffer[buf_index] = 0;
+  if (phq->slop != 0) {
+    sprintf(buffer + buf_index, "~%d", phq->slop);
+    buf_index += strlen(buffer + buf_index);
+  }
+  if (self->boost != 1.0) {
+    char dbuf[32];
+    dbl_to_s(dbuf, self->boost);
+    sprintf(buffer + buf_index, "^%s", dbuf);
+  }
+  return buffer;
+}
+
+void phq_destroy(void *p)
+{
+  Query *self = (Query *)p;
+
+  GET_PHQ;
+  int i;
+  if (self->destroy_all) {
+    for (i = 0; i < phq->t_cnt; i++) {
+      term_destroy(phq->terms[i]);
+    }
+  }
+  free(phq->terms);
+  free(phq->positions);
+  free(phq);
+
+  q_destroy(self);
+}
+
+Query *phq_rewrite(Query *self, IndexReader *ir)
+{
+  GET_PHQ;
+  if (phq->t_cnt == 1) { // optimize one-term case
+    Term *term = phq->terms[0];
+    Query *tq = tq_create(term_clone(term));
+    tq->boost = self->boost;
+    if (self->rewritten) self->rewritten->destroy(self->rewritten);
+    return self->rewritten = tq;
+  } else {
+    return self;
+  }
+}
+
+void phq_add_term(Query *self, Term *term, int pos_inc)
+{
+  GET_PHQ;
+  int position, index = phq->t_cnt;
+  if (index >= phq->t_capa) {
+    phq->t_capa *= 2;
+    REALLOC_N(phq->terms, Term *, phq->t_capa);
+    REALLOC_N(phq->positions, int, phq->t_capa);
+  }
+  if (index == 0) {
+    position = 0;
+    phq->field = term->field;
+  } else {
+    position = phq->positions[index - 1] + pos_inc;
+    if (strcmp(term->field, phq->field) != 0) {
+      eprintf(ARG_ERROR, "All phrase terms must be in the same field. Current phrase is %s, tried to add %s\n", phq->field, term->field);
+    }
+  }
+  phq->terms[index] = term;
+  phq->positions[index] = position;
+  phq->t_cnt++;
+}
+
+Query *phq_create()
+{
+  Query *self = q_create();
+  PhraseQuery *phq = ALLOC(PhraseQuery);
+  ZEROSET(phq, PhraseQuery, 1);
+  phq->t_capa = PHQ_INIT_CAPA;
+  phq->terms = ALLOC_N(Term *, PHQ_INIT_CAPA);
+  phq->positions = ALLOC_N(int, PHQ_INIT_CAPA);
+  self->data = phq;
+
+  self->create_weight = &phw_create;
+  self->extract_terms = &phq_extract_terms;
+  self->to_s = &phq_to_s;
+  self->destroy = &phq_destroy;
+  self->rewrite = &phq_rewrite;
+  self->type = PHRASE_QUERY;
+  return self;
+}
+
+/***************************************************************************
+ *
+ * PhraseScorer
+ *
+ ***************************************************************************/
+
+/***************************************************************************
+ * PhrasePosition
+ ***************************************************************************/
+
+bool pp_next(PhrasePosition *self)
+{
+  TermDocEnum *tpe = self->tpe;
+  if (!tpe->next(tpe)) {
+    tpe->close(tpe); // close stream
+    self->tpe = NULL;
+    self->doc = INT_MAX;   // sentinel value
+    return false;
+  }
+  self->doc = tpe->doc_num(tpe);
+  self->position = 0;
+  return true;
+}
+
+bool pp_skip_to(PhrasePosition *self, int doc_num)
+{
+  TermDocEnum *tpe = self->tpe;
+  if (!tpe->skip_to(tpe, doc_num)) {
+    tpe->close(tpe); // close stream
+    self->tpe = NULL;
+    self->doc = INT_MAX;   // sentinel value
+    return false;
+  }
+  self->doc = tpe->doc_num(tpe);
+  self->position = 0;
+  return true;
+}
+
+bool pp_next_position(PhrasePosition *self)
+{
+  TermDocEnum *tpe = self->tpe;
+  self->count -= 1;
+  if (self->count >= 0) {// read subsequent pos's
+    self->position = tpe->next_position(tpe) - self->offset;
+    return true;
+  } else {
+    return false;
+  }
+}
+
+bool pp_first_position(PhrasePosition *self)
+{
+  TermDocEnum *tpe = self->tpe;
+  self->count = tpe->freq(tpe); // read first pos
+  return pp_next_position(self);
+}
+
+char *pp_to_s(PhrasePosition *self)
+{
+  return epstrdup("pp->(doc => %d, position => %d)", 40, self->doc, self->position);
+}
+
+inline int pp_cmp(const void *const p1, const void *const p2)
+{
+  PhrasePosition *pp1 = *(PhrasePosition **)p1;
+  PhrasePosition *pp2 = *(PhrasePosition **)p2;
+  int cmp = pp1->doc - pp2->doc;
+  if (cmp == 0) {
+    return pp1->position - pp2->position;
+  } else {
+    return cmp;
+  }
+}
+bool pp_less_than(void *p1, void *p2)
+{
+  PhrasePosition *pp1 = (PhrasePosition *)p1;
+  PhrasePosition *pp2 = (PhrasePosition *)p2;
+  if (pp1->doc == pp2->doc) {
+    return pp1->position < pp2->position;
+  } else {
+    return pp1->doc < pp2->doc;
+  }
+}
+
+void pp_destroy(void *p)
+{
+  PhrasePosition *pp = (PhrasePosition *)p;
+  if (pp->tpe) pp->tpe->close(pp->tpe);
+  free(pp);
+}
+
+PhrasePosition *pp_create(TermDocEnum *tpe, int offset)
+{
+  PhrasePosition *self = ALLOC(PhrasePosition);
+  self->tpe = tpe;
+  self->count = self->doc = self->position = -1;
+  self->offset = offset;
+  return self;
+}
+
+/***************************************************************************
+ * PhraseScorer
+ ***************************************************************************/
+
+#define GET_PHSC PhraseScorer *phsc = (PhraseScorer *)self->data;
+
+
+void phsc_init(PhraseScorer *phsc)
+{
+  int i;
+  for (i = phsc->pp_cnt - 1; i >= 0; i--) {
+    if (!(phsc->more = pp_next(phsc->phrase_pos[i]))) break;
+  }
+
+  if (phsc->more) {
+    qsort(phsc->phrase_pos, phsc->pp_cnt, sizeof(PhrasePosition *), &pp_cmp);
+    phsc->pp_first = 0;
+    phsc->pp_last = phsc->pp_cnt - 1;
+  }
+}
+
+#define FIRST_TO_LAST() \
+      last = first;\
+      phsc->pp_last = phsc->pp_first;\
+      phsc->pp_first = (phsc->pp_first + 1) % phsc->pp_cnt;\
+      first = phsc->phrase_pos[phsc->pp_first];
+
+bool phsc_do_next(Scorer *self)
+{
+  GET_PHSC;
+  PhrasePosition *first = phsc->phrase_pos[phsc->pp_first];
+  PhrasePosition *last = phsc->phrase_pos[phsc->pp_last];
+
+  while (phsc->more) {
+    while (phsc->more && first->doc < last->doc) {   // find doc w/ all the terms
+      phsc->more = pp_skip_to(first, last->doc); // skip first upto last
+      FIRST_TO_LAST();                               // and move it to the end
+    }
+
+    if (phsc->more) {
+      // found a doc with all of the terms
+      phsc->freq = phsc->phrase_freq(self);  // check for phrase
+      if (phsc->freq == 0.0) {               // no match
+        first = phsc->phrase_pos[phsc->pp_first];
+        last = phsc->phrase_pos[phsc->pp_last];
+        phsc->more = pp_next(last);          // trigger further scanning
+      } else {
+        self->doc = first->doc;
+        return true;                         // found a match
+      }
+    }
+  }
+  return false;
+}
+
+float phsc_score(Scorer *self)
+{
+  GET_PHSC;
+  float raw = sim_tf(self->similarity, phsc->freq) * phsc->value; // raw score
+  // normalize
+  return raw * sim_decode_norm(self->similarity,
+      phsc->norms[phsc->phrase_pos[phsc->pp_first]->doc]);
+}
+
+bool phsc_next(Scorer *self)
+{
+  GET_PHSC;
+  if (phsc->first_time) {
+    phsc_init(phsc);
+    phsc->first_time = false;
+  } else if (phsc->more) {
+    phsc->more = pp_next(phsc->phrase_pos[phsc->pp_last]); // trigger further scanning
+  }
+  return phsc_do_next(self);
+}
+
+bool phsc_skip_to(Scorer *self, int doc_num)
+{
+  GET_PHSC;
+  int i;
+  for (i = phsc->pp_cnt - 1; i >= 0; i--) {
+    if (!(phsc->more = pp_skip_to(phsc->phrase_pos[i], doc_num))) break;
+  }
+
+  if (phsc->more) {
+    qsort(phsc->phrase_pos, phsc->pp_cnt, sizeof(PhrasePosition *), &pp_cmp);
+    phsc->pp_first = 0;
+    phsc->pp_last = phsc->pp_cnt - 1;
+  }
+  return phsc_do_next(self);
+}
+
+Explanation *phsc_explain(Scorer *self, int doc_num)
+{
+  char dbuf[32];
+  GET_PHSC;
+  while (phsc_next(self) && self->doc < doc_num)
+    ;
+
+  float phrase_freq = (self->doc == doc_num) ? phsc->freq : 0.0;
+  dbl_to_s(dbuf, phrase_freq);
+  return expl_create(sim_tf(self->similarity, phrase_freq), 
+      epstrdup("tf(phrase_freq=%s)", strlen(dbuf), dbuf));
+}
+
+void phsc_destroy(void *p)
+{
+  Scorer *self = (Scorer *)p;
+  GET_PHSC;
+  int i;
+  for (i = phsc->pp_cnt - 1; i >= 0; i--) {
+    pp_destroy(phsc->phrase_pos[i]);
+  }
+  free(phsc->phrase_pos);
+  scorer_destroy(self);
+}
+
+Scorer *phsc_create(Weight *weight, TermDocEnum **term_pos_enum,
+    int *positions, int t_cnt, Similarity *similarity, uchar *norms)
+{
+  int i;
+  Scorer *self = scorer_create(similarity);
+  PhraseScorer *phsc = ALLOC(PhraseScorer);
+  ZEROSET(phsc, PhraseScorer, 1);
+
+  phsc->weight = weight;
+  phsc->norms = norms;
+  phsc->value = weight->value;
+
+  phsc->phrase_pos = ALLOC_N(PhrasePosition *, t_cnt);
+  for (i = 0; i < t_cnt; i++) {
+    phsc->phrase_pos[i] = pp_create(term_pos_enum[i], positions[i]);
+  }
+  phsc->pp_first = 0;
+  phsc->pp_last = t_cnt - 1;
+  phsc->pp_cnt = t_cnt;
+
+  phsc->slop = 0;
+
+  phsc->first_time = true;
+  phsc->more = true;
+
+  self->data = phsc;
+  self->score = &phsc_score;
+  self->next = &phsc_next;
+  self->skip_to = &phsc_skip_to;
+  self->explain = &phsc_explain;
+  self->destroy = &phsc_destroy;
+
+  return self;
+}
+
+/***************************************************************************
+ * ExactPhraseScorer
+ ***************************************************************************/
+
+float ephsc_phrase_freq(Scorer *self)
+{
+  GET_PHSC;
+  // sort list with pq
+  int i;
+  float freq = 0.0;
+  PhrasePosition *first;
+  PhrasePosition *last;
+
+  for (i = phsc->pp_cnt - 1; i >= 0; i--) {
+    pp_first_position(phsc->phrase_pos[i]);
+  }
+  qsort(phsc->phrase_pos, phsc->pp_cnt, sizeof(PhrasePosition *), &pp_cmp);
+  phsc->pp_first = 0;
+  phsc->pp_last = phsc->pp_cnt - 1;
+
+  first = phsc->phrase_pos[0];
+  last = phsc->phrase_pos[phsc->pp_last];
+
+  do { // find position w/ all terms
+    while (first->position < last->position) { // scan forward in first
+      do {
+        if (! pp_next_position(first)) return freq;
+      } while (first->position < last->position);
+      FIRST_TO_LAST();
+    }
+    freq += 1.0; // all equal: a match
+  } while (pp_next_position(last));
+
+  return freq;
+}
+
+Scorer *exact_phrase_scorer_create(Weight *weight, TermDocEnum **term_pos_enum,
+    int *positions, int t_cnt, Similarity *similarity, uchar *norms)
+{
+  Scorer *self =
+    phsc_create(weight, term_pos_enum, positions, t_cnt, similarity, norms);
+  GET_PHSC;
+  phsc->phrase_freq = &ephsc_phrase_freq;
+  return self;
+}
+
+/***************************************************************************
+ * SloppyPhraseScorer
+ ***************************************************************************/
+
+float sphsc_phrase_freq(Scorer *self)
+{
+  GET_PHSC;
+  PhrasePosition *pp;
+  PriorityQueue *pq = pq_create(phsc->pp_cnt, &pp_less_than);
+
+  int last_pos = 0, pos, next_pos, start, match_length, i;
+  bool done = false;
+  float freq = 0.0;
+
+  for (i = phsc->pp_cnt - 1; i >= 0; i--) {
+    pp = phsc->phrase_pos[i];
+    pp_first_position(pp);
+    if (pp->position > last_pos) last_pos = pp->position;
+    pq_push(pq, pp);
+  }
+
+  do {
+    pp = pq_pop(pq);
+    pos = start = pp->position;
+    next_pos = ((PhrasePosition *)pq_top(pq))->position;
+    while (pos <= next_pos) {
+      start = pos;       // advance pp to min window
+      if (!pp_next_position(pp)) {
+        done = true;     // ran out of a term -- done
+        break;
+      }
+      pos = pp->position;
+    }
+
+    match_length = last_pos - start;
+    if (match_length <= phsc->slop) {
+      freq += sim_sloppy_freq(self->similarity, match_length); // score match
+    }
+
+    if (pp->position > last_pos) {
+      last_pos = pp->position;
+    }
+    pq_push(pq, pp); // restore pq
+  } while (!done);
+
+  pq_destroy(pq);
+  return freq;
+}
+    
+Scorer *sloppy_phrase_scorer_create(Weight *weight, TermDocEnum **term_pos_enum,
+    int *positions, int t_cnt, Similarity *similarity, int slop, uchar *norms)
+{
+  Scorer *self =
+    phsc_create(weight, term_pos_enum, positions, t_cnt, similarity, norms);
+  GET_PHSC;
+  phsc->slop = slop;
+  phsc->phrase_freq = &sphsc_phrase_freq;
+  return self;
+}
+