ferret 0.3.2 → 0.9.0

data/ext/q_term.c

@@ -0,0 +1,308 @@
+#include <string.h>
+#include "search.h"
+
+/***************************************************************************
+ *
+ * TermWeight
+ *
+ ***************************************************************************/
+
+Scorer *tw_scorer(Weight *self, IndexReader *ir)
+{
+  Term *term = ((TermQuery *)self->query->data)->term;
+  TermDocEnum *tde = ir_term_docs_for(ir,term);
+  if (!tde) return NULL;
+
+  return tsc_create(self, tde, ir->get_norms_always(ir, term->field));
+}
+
+Explanation *tw_explain(Weight *self, IndexReader *ir, int doc_num)
+{
+  char *query_str = self->query->to_s(self->query, "");
+  TermQuery *tq = (TermQuery *)self->query->data;
+  Term *term = tq->term;
+  char *field_name = term->field;
+
+  Explanation *expl = expl_create(0.0,
+      epstrdup("weight(%s in %d), product of:",
+        strlen(query_str) + 20,
+        query_str, doc_num));
+
+  // We need two of these as it's included in both the query explanation
+  // and the field explanation
+  Explanation *idf_expl1 = expl_create(self->idf,
+      epstrdup("idf(doc_freq=%d)", 20, ir->doc_freq(ir, tq->term)));
+  Explanation *idf_expl2 = expl_create(self->idf,
+      epstrdup("idf(doc_freq=%d)", 20, ir->doc_freq(ir, tq->term)));
+
+  // explain query weight
+  Explanation *query_expl = expl_create(0.0,
+      epstrdup("query_weight(%s), product of:", strlen(query_str), query_str));
+  free(query_str);
+
+  if (self->query->boost != 1.0) {
+    expl_add_detail(query_expl, expl_create(self->query->boost, estrdup("boost")));
+  }
+
+  expl_add_detail(query_expl, idf_expl1);
+
+  Explanation *qnorm_expl = expl_create(self->qnorm, estrdup("query_norm"));
+  expl_add_detail(query_expl, qnorm_expl);
+
+  query_expl->value = self->query->boost * idf_expl1->value * qnorm_expl->value;
+
+  expl_add_detail(expl, query_expl);
+
+  // explain field weight
+  Explanation *field_expl = expl_create(0.0, 
+      epstrdup("field_weight(%s:%s in %d), product of:",
+        strlen(field_name) + strlen(term->text) + 20,
+        field_name, term->text, doc_num));
+
+  Scorer *scorer = self->scorer(self, ir);
+  Explanation *tf_expl = scorer->explain(scorer, doc_num);
+  scorer->destroy(scorer);
+  expl_add_detail(field_expl, tf_expl);
+  expl_add_detail(field_expl, idf_expl2);
+
+  uchar *field_norms = ir->get_norms(ir, field_name);
+  float field_norm = (field_norms ? sim_decode_norm(self->similarity, field_norms[doc_num]) : 0.0);
+  Explanation *field_norm_expl = expl_create(field_norm,
+      epstrdup("field_norm(field=%s, doc=%d)",
+        strlen(field_name) + 20, field_name, doc_num));
+  expl_add_detail(field_expl, field_norm_expl);
+
+  field_expl->value = tf_expl->value * idf_expl2->value * field_norm_expl->value;
+
+  // combine them
+  if (query_expl->value == 1.0) {
+    expl_destoy(expl);
+    return field_expl;
+  } else {
+    expl->value = (query_expl->value * field_expl->value);
+    expl_add_detail(expl, field_expl);
+    return expl;
+  }
+}
+
+char *tw_to_s(Weight *self)
+{
+  char dbuf[32];
+  dbl_to_s(dbuf, self->value);
+  return epstrdup("TermWeight(%#.5g)", strlen(dbuf), dbuf);
+}
+
+void tw_destroy(void *p)
+{
+  free(p);
+}
+
+Weight *tw_create(Query *query, Searcher *searcher)
+{
+  Weight *self = ALLOC(Weight);
+  ZEROSET(self, Weight, 1);
+  self->get_query = &w_get_query;
+  self->get_value = &w_get_value;
+  self->normalize = &w_normalize;
+  self->scorer    = &tw_scorer;
+  self->explain   = &tw_explain;
+  self->to_s      = &tw_to_s;
+  self->destroy   = &tw_destroy;
+  self->sum_of_squared_weights = &w_sum_of_squared_weights;
+
+  self->similarity = query->get_similarity(query, searcher);
+  self->idf = sim_idf(self->similarity,
+      searcher->doc_freq(searcher, ((TermQuery *)query->data)->term),
+      searcher->max_doc(searcher)); // compute idf
+  self->query = query;
+  self->value = 0.0;
+
+  return self;
+}
+
+/***************************************************************************
+ *
+ * TermQuery
+ *
+ ***************************************************************************/
+
+void tq_destroy(void *p)
+{
+  Query *q = (Query *)p;
+  TermQuery *tq = q->data;
+  term_destroy(tq->term);
+  free(tq);
+  q_destroy(q);
+}
+
+char *tq_to_s(Query *self, char *field)
+{
+  Term *term = ((TermQuery *)self->data)->term;
+  int flen = strlen(term->field);
+  int tlen = strlen(term->text);
+  char *buffer = ALLOC_N(char, 34 + flen + tlen);
+  char *bp = buffer;
+  if (strcmp(field, term->field) != 0) {
+    memcpy(bp, term->field, sizeof(char) * flen);
+    bp[flen] = ':';
+    bp += flen + 1;
+  }
+  memcpy(bp, term->text, tlen);
+  bp += tlen;
+  *bp = 0;
+  if (self->boost != 1.0) {
+    char dbuf[32];
+    dbl_to_s(dbuf, self->boost);
+    sprintf(bp, "^%s", dbuf);
+  }
+  return buffer;
+}
+
+void tq_extract_terms(Query *self, Array *terms)
+{
+  Term *term = ((TermQuery *)self->data)->term;
+  ary_append(terms, term);
+}
+
+Query *tq_create(Term *term)
+{
+  Query *self = q_create();
+  TermQuery *tq = ALLOC(TermQuery);
+  tq->term = term;
+  self->type = TERM_QUERY;
+  self->data = tq;
+  self->create_weight = &tw_create;
+  self->extract_terms = &tq_extract_terms;
+  self->to_s = &tq_to_s;
+  self->destroy = &tq_destroy;
+
+  return self;
+}
+
+/***************************************************************************
+ *
+ * TermScorer
+ *
+ ***************************************************************************/
+
+float tsc_score(Scorer *self)
+{
+  TermScorer *ts = (TermScorer *)self->data;
+  int freq = ts->freqs[ts->pointer];
+  float score;
+  // compute tf(f)*weight
+  if (freq < SCORE_CACHE_SIZE) {    // check cache
+    score = ts->score_cache[freq];  // cache hit
+  } else {
+    score = sim_tf(self->similarity, freq) * ts->weight_value; // cache miss
+  }
+  // normalize for field
+  score *= sim_decode_norm(self->similarity, ts->norms[self->doc]);
+  return score;
+}
+
+bool tsc_next(Scorer *self)
+{
+  TermScorer *ts = (TermScorer *)self->data;
+
+  ts->pointer++;
+  if (ts->pointer >= ts->pointer_max) {
+    // refill buffer
+    ts->pointer_max = ts->tde->read(ts->tde, ts->docs, ts->freqs, TDE_READ_SIZE);
+    if (ts->pointer_max != 0) {
+      ts->pointer = 0;
+    } else {
+      ts->tde->close(ts->tde); // close stream
+      ts->tde = NULL;
+      return false;
+    }
+  }
+  self->doc = ts->docs[ts->pointer];
+  return true;
+}
+
+bool tsc_skip_to(Scorer *self, int doc_num)
+{
+  TermScorer *ts = (TermScorer *)self->data;
+
+  // first scan in cache
+  while (++(ts->pointer) < ts->pointer_max) {
+    if (ts->docs[ts->pointer] >= doc_num) {
+      self->doc = ts->docs[ts->pointer];
+      return true;
+    }
+  }
+
+  // not found in cache, seek underlying stream
+  TermDocEnum *tde = ts->tde;
+  bool result = tde->skip_to(tde, doc_num);
+  if (result) {
+    ts->pointer_max = 1;
+    ts->pointer = 0;
+    ts->docs[0] = self->doc = tde->doc_num(tde);
+    ts->freqs[0] = tde->freq(tde);
+    return true;
+  } else {
+    return false;
+  }
+}
+
+Explanation *tsc_explain(Scorer *self, int doc_num)
+{
+  TermScorer *ts = (TermScorer *)self->data;
+  Query *query = ts->weight->get_query(ts->weight);
+  Term *term = ((TermQuery *)query->data)->term;
+  int tf = 0;
+  TermDocEnum *tde = ts->tde;
+  while (ts->pointer < ts->pointer_max) {
+    if (ts->docs[ts->pointer] == doc_num)
+      tf = ts->freqs[ts->pointer];
+    ts->pointer++;
+  }
+  if (tf == 0) {
+    while (tde->next(tde)) {
+      if (tde->doc_num(tde) == doc_num) 
+        tf = tde->freq(tde);
+    }
+  }
+  tde->close(tde);
+  ts->tde = NULL;
+  Explanation *tf_explanation = expl_create(sim_tf(self->similarity, tf),
+      epstrdup("tf(term_freq(%s:%s)=%ld)",
+        strlen(term->field) + strlen(term->text) + 20,
+        term->field, term->text, tf));
+
+  return tf_explanation;
+}
+
+void tsc_destroy(void *p)
+{
+  Scorer *self = (Scorer *)p;
+  TermScorer *ts = (TermScorer *)self->data;
+  if (ts->tde) ts->tde->close(ts->tde);
+  scorer_destroy(p);
+}
+
+Scorer *tsc_create(Weight *weight, TermDocEnum *tde, uchar *norms)
+{
+  int i;
+  Scorer *self = scorer_create(weight->similarity);
+  TermScorer *ts = ALLOC(TermScorer);
+  ZEROSET(ts, TermScorer, 1);
+  self->data = ts;
+  ts->weight = weight;
+  ts->tde = tde;
+  ts->norms = norms;
+  ts->weight_value = weight->value;
+
+  for (i = 0; i < SCORE_CACHE_SIZE; i++) {
+    ts->score_cache[i] = sim_tf(self->similarity, i) * ts->weight_value;
+  }
+
+  self->score = &tsc_score;
+  self->next = &tsc_next;
+  self->skip_to = &tsc_skip_to;
+  self->explain = &tsc_explain;
+  self->destroy = &tsc_destroy;
+  return self;
+}

data/ext/q_wildcard.c

@@ -0,0 +1,146 @@
+#include <string.h>
+#include "search.h"
+
+/****************************************************************************
+ *
+ * WildCardQuery
+ *
+ ****************************************************************************/
+
+char *wcq_to_s(Query *self, char *field) 
+{
+  char *buffer, *bptr;
+  Term *term = (Term *)self->data;
+  int tlen = strlen(term->text);
+  int flen = strlen(term->field);
+  bptr = buffer = ALLOC_N(char, tlen + flen + 35);
+
+  if (strcmp(term->field, field) != 0) {
+    sprintf(bptr, "%s:", term->field);
+    bptr += strlen(term->field) + 1;
+  }
+  sprintf(bptr, "%s", term->text);
+  bptr = buffer + strlen(buffer);
+  if (self->boost != 1.0) {
+    *bptr = '^';
+    dbl_to_s(++bptr, self->boost);
+  }
+
+  return buffer;
+}
+
+bool wc_match(char *pattern, char *text)
+{
+  char *p = pattern, *t = text, *xt; 
+
+  /* include '\0' as we need to match empty string */
+  char *text_last = t + strlen(t);
+
+  for (;; p++, t++) {
+
+    /* end of text so make sure end of pattern doesn't matter */
+    if (*t == '\0') {
+      while (*p) {
+        if (*p != WILD_STRING) return false;
+        p++;
+      }
+      return true;
+    }
+
+    /* If we've gone past the end of the pattern, return false. */
+    if (*p == '\0') return false;
+
+    /* Match a single character, so continue. */
+    if (*p == WILD_CHAR) continue;
+
+    if (*p == WILD_STRING) {
+      // Look at the character beyond the '*'.
+      p++;
+      // Examine the string, starting at the last character.
+      for (xt = text_last; xt >= t; xt--) {
+        if (wc_match(p, xt)) return true;
+      }
+      return false;
+    }
+    if (*p != *t)
+      return false;
+  }
+
+  return false;
+}
+
+Query *wcq_rewrite(Query *self, IndexReader *ir)
+{
+  Query *q;
+  Query *tq;
+
+  Term *term = (Term *)self->data;
+  char *text = term->text;
+  char *field = term->field;
+  char *first_star = index(text, WILD_STRING);
+  char *first_ques = index(text, WILD_CHAR);
+  if (!first_star && !first_ques) {
+    q = tq_create(term_clone(term));
+  } else {
+    TermEnum *te;
+    Term prefix_term;
+    char *prefix = NULL;
+
+    char *pattern = (first_ques && first_star > first_ques)
+      ? first_ques : first_star;
+
+    int prefix_len = pattern - text;
+
+    prefix_term.field = field;
+    prefix_term.text = (char *)EMPTY_STRING;
+    if (prefix_len > 0) {
+      prefix = ALLOC_N(char, prefix_len + 1);
+      strncpy(prefix, text, prefix_len);
+      prefix_term.text = prefix;
+      prefix_term.text[prefix_len] = '\0';
+    }
+    te = ir->terms_from(ir, &prefix_term);
+
+    q = bq_create(true);
+    if (te) {
+      TermBuffer *tb = te->tb_curr;
+      do { 
+        if (strcmp(tb->field, field) != 0 ||
+            (prefix && strncmp(tb->text, prefix, prefix_len) != 0))
+          break;
+
+        if (wc_match(pattern, tb->text + prefix_len)) {
+          tq = tq_create(term_create(tb->field, tb->text)); /* found match */
+          tq->boost = self->boost;                          /* set boost */
+          bq_add_query(q, tq, BC_SHOULD);                   /* add query */
+        }
+      } while ((tb = te->next(te)) != NULL);
+      te->close(te);
+    }
+    free(prefix);
+  }
+
+  if (self->rewritten) self->rewritten->destroy(self->rewritten);
+  return self->rewritten = q;
+}
+
+void wcq_destroy(void *p)
+{
+  Query *self = (Query *)p;
+  if (self->destroy_all) term_destroy((Term *)self->data);
+  q_destroy(self);
+}
+
+Query *wcq_create(Term *term)
+{
+  Query *self = q_create();
+
+  self->data = term;
+  self->type = WILD_CARD_QUERY;
+  self->create_weight = NULL;
+  self->to_s = &wcq_to_s;
+  self->rewrite = &wcq_rewrite;
+  self->destroy = &wcq_destroy;
+
+  return self;
+}