jk-ferret 0.11.8.2

data/ext/q_term.c

@@ -0,0 +1,337 @@
+#include "symbol.h"
+#include <string.h>
+#include "search.h"
+#include "internal.h"
+
+#define TQ(query) ((TermQuery *)(query))
+#define TSc(scorer) ((TermScorer *)(scorer))
+
+/***************************************************************************
+ *
+ * TermScorer
+ *
+ ***************************************************************************/
+
+#define SCORE_CACHE_SIZE 32
+#define TDE_READ_SIZE 32
+
+typedef struct TermScorer
+{
+    Scorer          super;
+    int             docs[TDE_READ_SIZE];
+    int             freqs[TDE_READ_SIZE];
+    int             pointer;
+    int             pointer_max;
+    float           score_cache[SCORE_CACHE_SIZE];
+    Weight         *weight;
+    TermDocEnum    *tde;
+    uchar          *norms;
+    float           weight_value;
+} TermScorer;
+
+static float tsc_score(Scorer *self)
+{
+    TermScorer *ts = TSc(self);
+    int freq = ts->freqs[ts->pointer];
+    float score;
+    /* compute tf(f)*weight */
+    if (freq < SCORE_CACHE_SIZE) {    /* check cache */
+        score = ts->score_cache[freq];  /* cache hit */
+    }
+    else {
+        /* cache miss */
+        score = sim_tf(self->similarity, (float)freq) * ts->weight_value;
+    }
+    /* normalize for field */
+    score *= sim_decode_norm(self->similarity, ts->norms[self->doc]);
+    return score;
+}
+
+static bool tsc_next(Scorer *self)
+{
+    TermScorer *ts = TSc(self);
+
+    ts->pointer++;
+    if (ts->pointer >= ts->pointer_max) {
+        /* refill buffer */
+        ts->pointer_max = ts->tde->read(ts->tde, ts->docs, ts->freqs,
+                                        TDE_READ_SIZE);
+        if (ts->pointer_max != 0) {
+            ts->pointer = 0;
+        }
+        else {
+            return false;
+        }
+    }
+    self->doc = ts->docs[ts->pointer];
+    return true;
+}
+
+static bool tsc_skip_to(Scorer *self, int doc_num)
+{
+    TermScorer *ts = TSc(self);
+    TermDocEnum *tde = ts->tde;
+
+    /* first scan in cache */
+    while (++(ts->pointer) < ts->pointer_max) {
+        if (ts->docs[ts->pointer] >= doc_num) {
+            self->doc = ts->docs[ts->pointer];
+            return true;
+        }
+    }
+
+    /* not found in cache, seek underlying stream */
+    if (tde->skip_to(tde, doc_num)) {
+        ts->pointer_max = 1;
+        ts->pointer = 0;
+        ts->docs[0] = self->doc = tde->doc_num(tde);
+        ts->freqs[0] = tde->freq(tde);
+        return true;
+    }
+    else {
+        return false;
+    }
+}
+
+static Explanation *tsc_explain(Scorer *self, int doc_num)
+{
+    TermScorer *ts = TSc(self);
+    Query *query = ts->weight->get_query(ts->weight);
+    int tf = 0;
+
+    tsc_skip_to(self, doc_num);
+    if (self->doc == doc_num) {
+        tf = ts->freqs[ts->pointer];
+    }
+    return expl_new(sim_tf(self->similarity, (float)tf),
+                    "tf(term_freq(%s:%s)=%d)",
+                    S(TQ(query)->field), TQ(query)->term, tf);
+}
+
+static void tsc_destroy(Scorer *self)
+{
+    TSc(self)->tde->close(TSc(self)->tde);
+    scorer_destroy_i(self);
+}
+
+static Scorer *tsc_new(Weight *weight, TermDocEnum *tde, uchar *norms)
+{
+    int i;
+    Scorer *self            = scorer_new(TermScorer, weight->similarity);
+    TSc(self)->weight       = weight;
+    TSc(self)->tde          = tde;
+    TSc(self)->norms        = norms;
+    TSc(self)->weight_value = weight->value;
+
+    for (i = 0; i < SCORE_CACHE_SIZE; i++) {
+        TSc(self)->score_cache[i]
+            = sim_tf(self->similarity, (float)i) * TSc(self)->weight_value;
+    }
+
+    self->score             = &tsc_score;
+    self->next              = &tsc_next;
+    self->skip_to           = &tsc_skip_to;
+    self->explain           = &tsc_explain;
+    self->destroy           = &tsc_destroy;
+    return self;
+}
+
+/***************************************************************************
+ *
+ * TermWeight
+ *
+ ***************************************************************************/
+
+static Scorer *tw_scorer(Weight *self, IndexReader *ir)
+{
+    TermQuery *tq = TQ(self->query);
+    TermDocEnum *tde = ir_term_docs_for(ir, tq->field, tq->term);
+    /* ir_term_docs_for should always return a TermDocEnum */
+    assert(NULL != tde);
+
+    return tsc_new(self, tde, ir_get_norms(ir, tq->field));
+}
+
+static Explanation *tw_explain(Weight *self, IndexReader *ir, int doc_num)
+{
+    Explanation *qnorm_expl;
+    Explanation *field_expl;
+    Scorer *scorer;
+    Explanation *tf_expl;
+    uchar *field_norms;
+    float field_norm;
+    Explanation *field_norm_expl;
+
+    char *query_str = self->query->to_s(self->query, NULL);
+    TermQuery *tq = TQ(self->query);
+    char *term = tq->term;
+
+    Explanation *expl = expl_new(0.0, "weight(%s in %d), product of:",
+                                 query_str, doc_num);
+
+    /* We need two of these as it's included in both the query explanation
+     * and the field explanation */
+    Explanation *idf_expl1 = expl_new(self->idf, "idf(doc_freq=%d)",
+                                      ir_doc_freq(ir, tq->field, term));
+    Explanation *idf_expl2 = expl_new(self->idf, "idf(doc_freq=%d)",
+                                      ir_doc_freq(ir, tq->field, term));
+
+    /* explain query weight */
+    Explanation *query_expl = expl_new(0.0, "query_weight(%s), product of:",
+                                       query_str);
+    free(query_str);
+
+    if (self->query->boost != 1.0) {
+        expl_add_detail(query_expl, expl_new(self->query->boost, "boost"));
+    }
+
+    expl_add_detail(query_expl, idf_expl1);
+
+    qnorm_expl = expl_new(self->qnorm, "query_norm");
+    expl_add_detail(query_expl, qnorm_expl);
+
+    query_expl->value = self->query->boost
+        * idf_expl1->value * qnorm_expl->value;
+
+    expl_add_detail(expl, query_expl);
+
+    /* explain field weight */
+    field_expl = expl_new(0.0, "field_weight(%s:%s in %d), product of:",
+                          S(tq->field), term, doc_num);
+
+    scorer = self->scorer(self, ir);
+    tf_expl = scorer->explain(scorer, doc_num);
+    scorer->destroy(scorer);
+    expl_add_detail(field_expl, tf_expl);
+    expl_add_detail(field_expl, idf_expl2);
+
+    field_norms = ir_get_norms(ir, tq->field);
+    field_norm = (field_norms 
+                  ? sim_decode_norm(self->similarity, field_norms[doc_num]) 
+                  : (float)0.0);
+    field_norm_expl = expl_new(field_norm, "field_norm(field=%s, doc=%d)",
+                               S(tq->field), doc_num);
+
+    expl_add_detail(field_expl, field_norm_expl);
+
+    field_expl->value = tf_expl->value * idf_expl2->value
+        * field_norm_expl->value;
+
+    /* combine them */
+    if (query_expl->value == 1.0) {
+        expl_destroy(expl);
+        return field_expl;
+    } else {
+        expl->value = (query_expl->value * field_expl->value);
+        expl_add_detail(expl, field_expl);
+        return expl;
+    }
+}
+
+static char *tw_to_s(Weight *self)
+{
+    return strfmt("TermWeight(%f)", self->value);
+}
+
+static Weight *tw_new(Query *query, Searcher *searcher)
+{
+    Weight *self    = w_new(Weight, query);
+    self->scorer    = &tw_scorer;
+    self->explain   = &tw_explain;
+    self->to_s      = &tw_to_s;
+
+    self->similarity = query->get_similarity(query, searcher);
+    self->idf = sim_idf(self->similarity,
+                        searcher->doc_freq(searcher,
+                                           TQ(query)->field,
+                                           TQ(query)->term),
+                        searcher->max_doc(searcher)); /* compute idf */
+
+    return self;
+}
+
+/***************************************************************************
+ *
+ * TermQuery
+ *
+ ***************************************************************************/
+
+static void tq_destroy(Query *self)
+{
+    free(TQ(self)->term);
+    q_destroy_i(self);
+}
+
+static char *tq_to_s(Query *self, Symbol default_field)
+{
+    const char *field = S(TQ(self)->field);
+    size_t flen = strlen(field);
+    size_t tlen = strlen(TQ(self)->term);
+    char *buffer = ALLOC_N(char, 34 + flen + tlen);
+    char *b = buffer;
+    if (default_field != TQ(self)->field) {
+        memcpy(b, field, sizeof(char) * flen);
+        b[flen] = ':';
+        b += flen + 1;
+    }
+    memcpy(b, TQ(self)->term, tlen);
+    b += tlen;
+    *b = 0;
+    if (self->boost != 1.0) {
+        *b = '^';
+        dbl_to_s(b+1, self->boost);
+    }
+    return buffer;
+}
+
+static void tq_extract_terms(Query *self, HashSet *terms)
+{
+    hs_add(terms, term_new(TQ(self)->field, TQ(self)->term));
+}
+
+static unsigned long tq_hash(Query *self)
+{
+    return str_hash(TQ(self)->term) ^ sym_hash(TQ(self)->field);
+}
+
+static int tq_eq(Query *self, Query *o)
+{
+    return (strcmp(TQ(self)->term, TQ(o)->term) == 0) 
+        && (TQ(self)->field == TQ(o)->field);
+}
+
+static MatchVector *tq_get_matchv_i(Query *self, MatchVector *mv,
+                                    TermVector *tv)
+{
+    if (tv->field == TQ(self)->field) {
+        int i;
+        TVTerm *tv_term = tv_get_tv_term(tv, TQ(self)->term);
+        if (tv_term) {
+            for (i = 0; i < tv_term->freq; i++) {
+                int pos = tv_term->positions[i];
+                matchv_add(mv, pos, pos);
+            }
+        }
+    }
+    return mv;
+}
+
+Query *tq_new(Symbol field, const char *term)
+{
+    Query *self             = q_new(TermQuery);
+
+    TQ(self)->field         = field;
+    TQ(self)->term          = estrdup(term);
+
+    self->type              = TERM_QUERY;
+    self->extract_terms     = &tq_extract_terms;
+    self->to_s              = &tq_to_s;
+    self->hash              = &tq_hash;
+    self->eq                = &tq_eq;
+
+    self->destroy_i         = &tq_destroy;
+    self->create_weight_i   = &tw_new;
+    self->get_matchv_i      = &tq_get_matchv_i;
+
+    return self;
+}

data/ext/q_wildcard.c

@@ -0,0 +1,167 @@
+#include <string.h>
+#include "search.h"
+#include "symbol.h"
+#include "internal.h"
+
+/****************************************************************************
+ *
+ * WildCardQuery
+ *
+ ****************************************************************************/
+
+#define WCQ(query) ((WildCardQuery *)(query))
+
+static char *wcq_to_s(Query *self, Symbol default_field)
+{
+    char *buffer, *bptr;
+    const char *field_str = S(WCQ(self)->field);
+    const char *pattern = WCQ(self)->pattern;
+    bptr = buffer = ALLOC_N(char, strlen(pattern) + strlen(field_str) + 35);
+
+    if (WCQ(self)->field != default_field) {
+        bptr += sprintf(bptr, "%s:", field_str);
+    }
+    bptr += sprintf(bptr, "%s", pattern);
+
+    if (self->boost != 1.0) {
+        *bptr = '^';
+        dbl_to_s(++bptr, self->boost);
+    }
+
+    return buffer;
+}
+
+bool wc_match(const char *pattern, const char *text)
+{
+    const char *p = pattern, *t = text, *xt;
+
+    /* include '\0' as we need to match empty string */
+    const char *text_last = t + strlen(t);
+
+    for (;; p++, t++) {
+
+        /* end of text so make sure end of pattern doesn't matter */
+        if (*t == '\0') {
+            while (*p) {
+                if (*p != WILD_STRING) {
+                    return false;
+                }
+                p++;
+            }
+            return true;
+        }
+
+        /* If we've gone past the end of the pattern, return false. */
+        if (*p == '\0') {
+            return false;
+        }
+
+        /* Match a single character, so continue. */
+        if (*p == WILD_CHAR) {
+            continue;
+        }
+
+        if (*p == WILD_STRING) {
+            /* Look at the character beyond the '*'. */
+            p++;
+            /* Examine the string, starting at the last character. */
+            for (xt = text_last; xt >= t; xt--) {
+                if (wc_match(p, xt)) return true;
+            }
+            return false;
+        }
+        if (*p != *t) {
+            return false;
+        }
+    }
+
+    return false;
+}
+
+static Query *wcq_rewrite(Query *self, IndexReader *ir)
+{
+    Query *q;
+    const char *pattern = WCQ(self)->pattern;
+    const char *first_star = strchr(pattern, WILD_STRING);
+    const char *first_ques = strchr(pattern, WILD_CHAR);
+
+    if (NULL == first_star && NULL == first_ques) {
+        q = tq_new(WCQ(self)->field, pattern);
+        q->boost = self->boost;
+    }
+    else {
+        const int field_num = fis_get_field_num(ir->fis, WCQ(self)->field);
+        q = multi_tq_new_conf(WCQ(self)->field, MTQMaxTerms(self), 0.0);
+
+        if (field_num >= 0) {
+            TermEnum *te;
+            char prefix[MAX_WORD_SIZE] = "";
+            int prefix_len;
+
+            pattern = (first_ques && (!first_star || first_star > first_ques))
+                ? first_ques : first_star;
+
+            prefix_len = (int)(pattern - WCQ(self)->pattern);
+
+            if (prefix_len > 0) {
+                memcpy(prefix, WCQ(self)->pattern, prefix_len);
+                prefix[prefix_len] = '\0';
+            }
+
+            te = ir->terms_from(ir, field_num, prefix);
+
+            if (te != NULL) {
+                const char *term = te->curr_term;
+                const char *pat_term = term + prefix_len;
+                do {
+                    if (prefix[0] && strncmp(term, prefix, prefix_len) != 0) {
+                        break;
+                    }
+
+                    if (wc_match(pattern, pat_term)) {
+                        multi_tq_add_term(q, term);
+                    }
+                } while (te->next(te) != NULL);
+                te->close(te);
+            }
+        }
+    }
+
+    return q;
+}
+
+static void wcq_destroy(Query *self)
+{
+    free(WCQ(self)->pattern);
+    q_destroy_i(self);
+}
+
+static unsigned long wcq_hash(Query *self)
+{
+    return sym_hash(WCQ(self)->field) ^ str_hash(WCQ(self)->pattern);
+}
+
+static int wcq_eq(Query *self, Query *o)
+{
+    return (strcmp(WCQ(self)->pattern, WCQ(o)->pattern) == 0)
+        && (WCQ(self)->field == WCQ(o)->field);
+}
+
+Query *wcq_new(Symbol field, const char *pattern)
+{
+    Query *self = q_new(WildCardQuery);
+
+    WCQ(self)->field        = field;
+    WCQ(self)->pattern      = estrdup(pattern);
+    MTQMaxTerms(self)       = WILD_CARD_QUERY_MAX_TERMS;
+
+    self->type              = WILD_CARD_QUERY;
+    self->rewrite           = &wcq_rewrite;
+    self->to_s              = &wcq_to_s;
+    self->hash              = &wcq_hash;
+    self->eq                = &wcq_eq;
+    self->destroy_i         = &wcq_destroy;
+    self->create_weight_i   = &q_create_weight_unsup;
+
+    return self;
+}