isomorfeus-ferret 0.12.0

data/ext/isomorfeus_ferret_ext/frt_q_const_score.c

@@ -0,0 +1,157 @@
+#include "frt_search.h"
+#include <string.h>
+
+/***************************************************************************
+ *
+ * ConstantScoreScorer
+ *
+ ***************************************************************************/
+
+#define CScQ(query) ((FrtConstantScoreQuery *)(query))
+#define CScSc(scorer) ((ConstantScoreScorer *)(scorer))
+
+typedef struct ConstantScoreScorer
+{
+    FrtScorer      super;
+    FrtBitVector  *bv;
+    float       score;
+} ConstantScoreScorer;
+
+static float cssc_score(FrtScorer *self)
+{
+    return CScSc(self)->score;
+}
+
+static bool cssc_next(FrtScorer *self)
+{
+    return ((self->doc = frt_bv_scan_next(CScSc(self)->bv)) >= 0);
+}
+
+static bool cssc_skip_to(FrtScorer *self, int doc_num)
+{
+    return ((self->doc = frt_bv_scan_next_from(CScSc(self)->bv, doc_num)) >= 0);
+}
+
+static FrtExplanation *cssc_explain(FrtScorer *self, int doc_num)
+{
+    (void)self; (void)doc_num;
+    return frt_expl_new(1.0, "ConstantScoreScorer");
+}
+
+static FrtScorer *cssc_new(FrtWeight *weight, FrtIndexReader *ir)
+{
+    FrtScorer *self    = frt_scorer_new(ConstantScoreScorer, weight->similarity);
+    FrtFilter *filter  = CScQ(weight->query)->filter;
+
+    CScSc(self)->score  = weight->value;
+    CScSc(self)->bv     = frt_filt_get_bv(filter, ir);
+
+    self->score     = &cssc_score;
+    self->next      = &cssc_next;
+    self->skip_to   = &cssc_skip_to;
+    self->explain   = &cssc_explain;
+    self->destroy   = &frt_scorer_destroy_i;
+    return self;
+}
+
+/***************************************************************************
+ *
+ * ConstantScoreWeight
+ *
+ ***************************************************************************/
+
+static char *csw_to_s(FrtWeight *self)
+{
+    return frt_strfmt("ConstantScoreWeight(%f)", self->value);
+}
+
+static FrtExplanation *csw_explain(FrtWeight *self, FrtIndexReader *ir, int doc_num)
+{
+    FrtFilter *filter = CScQ(self->query)->filter;
+    FrtExplanation *expl;
+    char *filter_str = filter->to_s(filter);
+    FrtBitVector *bv = frt_filt_get_bv(filter, ir);
+
+    if (frt_bv_get(bv, doc_num)) {
+        expl = frt_expl_new(self->value, "ConstantScoreQuery(%s), product of:", filter_str);
+        frt_expl_add_detail(expl, frt_expl_new(self->query->boost, "boost"));
+        frt_expl_add_detail(expl, frt_expl_new(self->qnorm, "query_norm"));
+    } else {
+        expl = frt_expl_new(self->value, "ConstantScoreQuery(%s), does not match id %d", filter_str, doc_num);
+    }
+    free(filter_str);
+    return expl;
+}
+
+static FrtWeight *csw_new(FrtQuery *query, FrtSearcher *searcher)
+{
+    FrtWeight *self        = w_new(FrtWeight, query);
+
+    self->scorer        = &cssc_new;
+    self->explain       = &csw_explain;
+    self->to_s          = &csw_to_s;
+
+    self->similarity    = query->get_similarity(query, searcher);
+    self->idf           = 1.0f;
+
+    return self;
+}
+
+/***************************************************************************
+ *
+ * ConstantScoreQuery
+ *
+ ***************************************************************************/
+
+static char *csq_to_s(FrtQuery *self, FrtSymbol default_field)
+{
+    FrtFilter *filter = CScQ(self)->filter;
+    char *filter_str = filter->to_s(filter);
+    char *buffer;
+    (void)default_field;
+    if (self->boost == 1.0) {
+        buffer = frt_strfmt("ConstantScore(%s)", filter_str);
+    }
+    else {
+        buffer = frt_strfmt("ConstantScore(%s)^%f", filter_str, self->boost);
+    }
+    free(filter_str);
+    return buffer;;
+}
+
+static void csq_destroy(FrtQuery *self)
+{
+    frt_filt_deref(CScQ(self)->filter);
+    frt_q_destroy_i(self);
+}
+
+static unsigned long long csq_hash(FrtQuery *self)
+{
+    return frt_filt_hash(CScQ(self)->filter);
+}
+
+static int csq_eq(FrtQuery *self, FrtQuery *o)
+{
+    return frt_filt_eq(CScQ(self)->filter, CScQ(o)->filter);
+}
+
+FrtQuery *frt_csq_new_nr(FrtFilter *filter)
+{
+    FrtQuery *self = frt_q_new(FrtConstantScoreQuery);
+    CScQ(self)->filter = filter;
+
+    self->type              = CONSTANT_QUERY;
+    self->to_s              = &csq_to_s;
+    self->hash              = &csq_hash;
+    self->eq                = &csq_eq;
+    self->destroy_i         = &csq_destroy;
+    self->create_weight_i   = &csw_new;
+
+    return self;
+}
+
+FrtQuery *frt_csq_new(FrtFilter *filter)
+{
+    FRT_REF(filter);
+    return frt_csq_new_nr(filter);
+}

data/ext/isomorfeus_ferret_ext/frt_q_filtered_query.c

@@ -0,0 +1,209 @@
+#include "frt_search.h"
+#include <string.h>
+
+/***************************************************************************
+ *
+ * FilteredQueryScorer
+ *
+ ***************************************************************************/
+
+#define FQSc(scorer) ((FilteredQueryScorer *)(scorer))
+#define FQQ(query) ((FrtFilteredQuery *)(query))
+
+typedef struct FilteredQueryScorer
+{
+    FrtScorer      super;
+    FrtScorer     *sub_scorer;
+    FrtBitVector  *bv;
+} FilteredQueryScorer;
+
+static float fqsc_score(FrtScorer *self)
+{
+    FrtScorer *sub_sc = FQSc(self)->sub_scorer;
+    return sub_sc->score(sub_sc);
+}
+
+static bool fqsc_next(FrtScorer *self)
+{
+    FrtScorer *sub_sc = FQSc(self)->sub_scorer;
+    FrtBitVector *bv = FQSc(self)->bv;
+    while (sub_sc->next(sub_sc)) {
+        self->doc = sub_sc->doc;
+        if (frt_bv_get(bv, self->doc)) return true;
+    }
+    return false;
+}
+
+static bool fqsc_skip_to(FrtScorer *self, int doc_num)
+{
+    FrtScorer *sub_sc = FQSc(self)->sub_scorer;
+    FrtBitVector *bv = FQSc(self)->bv;
+    if (sub_sc->skip_to(sub_sc, doc_num)) {
+        do {
+            self->doc = sub_sc->doc;
+            if (frt_bv_get(bv, self->doc)) {
+                return true;
+            }
+        } while (sub_sc->next(sub_sc));
+    }
+    return false;
+}
+
+static FrtExplanation *fqsc_explain(FrtScorer *self, int doc_num)
+{
+    FrtScorer *sub_sc = FQSc(self)->sub_scorer;
+    return sub_sc->explain(sub_sc, doc_num);
+}
+
+static void fqsc_destroy(FrtScorer *self)
+{
+    FilteredQueryScorer *fqsc = FQSc(self);
+    fqsc->sub_scorer->destroy(fqsc->sub_scorer);
+    frt_scorer_destroy_i(self);
+}
+
+static FrtScorer *fqsc_new(FrtScorer *scorer, FrtBitVector *bv, FrtSimilarity *sim)
+{
+    FrtScorer *self            = frt_scorer_new(FilteredQueryScorer, sim);
+
+    FQSc(self)->sub_scorer  = scorer;
+    FQSc(self)->bv          = bv;
+
+    self->score   = &fqsc_score;
+    self->next    = &fqsc_next;
+    self->skip_to = &fqsc_skip_to;
+    self->explain = &fqsc_explain;
+    self->destroy = &fqsc_destroy;
+
+    return self;
+}
+
+/***************************************************************************
+ *
+ * FrtWeight
+ *
+ ***************************************************************************/
+
+#define FQW(weight) ((FilteredQueryWeight *)(weight))
+typedef struct FilteredQueryWeight
+{
+    FrtWeight  super;
+    FrtWeight *sub_weight;
+} FilteredQueryWeight;
+
+static char *fqw_to_s(FrtWeight *self)
+{
+    return frt_strfmt("FilteredQueryWeight(%f)", self->value);
+}
+
+static float fqw_sum_of_squared_weights(FrtWeight *self)
+{
+    FrtWeight *sub_weight = FQW(self)->sub_weight;
+    return sub_weight->sum_of_squared_weights(sub_weight);
+}
+
+static void fqw_normalize(FrtWeight *self, float normalization_factor)
+{
+    FrtWeight *sub_weight = FQW(self)->sub_weight;
+    sub_weight->normalize(sub_weight, normalization_factor);
+}
+
+static float fqw_get_value(FrtWeight *self)
+{
+    FrtWeight *sub_weight = FQW(self)->sub_weight;
+    return sub_weight->get_value(sub_weight);
+}
+
+static FrtExplanation *fqw_explain(FrtWeight *self, FrtIndexReader *ir, int doc_num)
+{
+    FrtWeight *sub_weight = FQW(self)->sub_weight;
+    return sub_weight->explain(sub_weight, ir, doc_num);
+}
+
+static FrtScorer *fqw_scorer(FrtWeight *self, FrtIndexReader *ir)
+{
+    FrtWeight *sub_weight = FQW(self)->sub_weight;
+    FrtScorer *scorer = sub_weight->scorer(sub_weight, ir);
+    FrtFilter *filter = FQQ(self->query)->filter;
+
+    return fqsc_new(scorer, frt_filt_get_bv(filter, ir), self->similarity);
+}
+
+static void fqw_destroy(FrtWeight *self)
+{
+    FrtWeight *sub_weight = FQW(self)->sub_weight;
+    sub_weight->destroy(sub_weight);
+    frt_w_destroy(self);
+}
+
+static FrtWeight *fqw_new(FrtQuery *query, FrtWeight *sub_weight, FrtSimilarity *sim)
+{
+    FrtWeight *self = w_new(FilteredQueryWeight, query);
+
+    FQW(self)->sub_weight           = sub_weight;
+
+    self->get_value                 = &fqw_get_value;
+    self->normalize                 = &fqw_normalize;
+    self->scorer                    = &fqw_scorer;
+    self->explain                   = &fqw_explain;
+    self->to_s                      = &fqw_to_s;
+    self->destroy                   = &fqw_destroy;
+    self->sum_of_squared_weights    = &fqw_sum_of_squared_weights;
+
+    self->similarity                = sim;
+    self->idf                       = 1.0f;
+    self->value                     = sub_weight->value;
+
+    return self;
+}
+
+/***************************************************************************
+ *
+ * FilteredQuery
+ *
+ ***************************************************************************/
+
+static char *fq_to_s(FrtQuery *self, FrtSymbol default_field)
+{
+    FrtFilteredQuery *fq = FQQ(self);
+    char *filter_str = fq->filter->to_s(fq->filter);
+    char *query_str = fq->query->to_s(fq->query, default_field);
+    char *buffer;
+    if (self->boost == 1.0) {
+        buffer = frt_strfmt("FilteredQuery(query:%s, filter:%s)", query_str, filter_str);
+    } else {
+        buffer = frt_strfmt("FilteredQuery(query:%s, filter:%s)^%f", query_str, filter_str, self->boost);
+    }
+    free(filter_str);
+    free(query_str);
+    return buffer;;
+}
+
+static void fq_destroy(FrtQuery *self)
+{
+    frt_filt_deref(FQQ(self)->filter);
+    frt_q_deref(FQQ(self)->query);
+    frt_q_destroy_i(self);
+}
+
+static FrtWeight *fq_new_weight(FrtQuery *self, FrtSearcher *searcher)
+{
+    FrtQuery *sub_query = FQQ(self)->query;
+    return fqw_new(self, frt_q_weight(sub_query, searcher),
+                      searcher->similarity);
+}
+
+FrtQuery *frt_fq_new(FrtQuery *query, FrtFilter *filter)
+{
+    FrtQuery *self = frt_q_new(FrtFilteredQuery);
+
+    FQQ(self)->query        = query;
+    FQQ(self)->filter       = filter;
+
+    self->type              = FILTERED_QUERY;
+    self->to_s              = &fq_to_s;
+    self->destroy_i         = &fq_destroy;
+    self->create_weight_i   = &fq_new_weight;
+
+    return self;
+}

data/ext/isomorfeus_ferret_ext/frt_q_fuzzy.c

@@ -0,0 +1,281 @@
+#include <string.h>
+#include "frt_search.h"
+#include "frt_helper.h"
+
+/****************************************************************************
+ *
+ * FuzzyStuff
+ *
+ * The main method here is the fuzq_score_mn method which scores a term
+ * against another term. The other methods all act in support.
+ *
+ * To learn more about the fuzzy scoring algorithm see;
+ *
+ *     http://en.wikipedia.org/wiki/Levenshtein_distance
+ *
+ ****************************************************************************/
+
+/**
+ * Calculate the maximum nomber of allowed edits (or maximum edit distance)
+ * for a word to be a match.
+ *
+ * Note that fuzq->text_len and m are both the lengths text *after* the prefix
+ * so `FRT_MIN(fuzq->text_len, m) + fuzq->pre_len)` actually gets the byte length
+ * of the shorter string out of the query string and the index term being
+ * compared.
+ */
+static int fuzq_calculate_max_distance(FrtFuzzyQuery *fuzq, int m)
+{
+    return (int)((1.0 - fuzq->min_sim) * (FRT_MIN(fuzq->text_len, m) + fuzq->pre_len));
+}
+
+/**
+ * The max-distance formula gets used a lot - it needs to be calculated for
+ * every possible match in the index - so we cache the results for all
+ * lengths up to the FRT_TYPICAL_LONGEST_WORD limit. For words longer than this we
+ * calculate the value live.
+ */
+static void fuzq_initialize_max_distances(FrtFuzzyQuery *fuzq)
+{
+    int i;
+    for (i = 0; i < FRT_TYPICAL_LONGEST_WORD; i++) {
+        fuzq->max_distances[i] = fuzq_calculate_max_distance(fuzq, i);
+    }
+}
+
+/**
+ * Return the cached max-distance value if the word is within the
+ * FRT_TYPICAL_LONGEST_WORD limit.
+ */
+static int fuzq_get_max_distance(FrtFuzzyQuery *fuzq, int m)
+{
+    if (m < FRT_TYPICAL_LONGEST_WORD)
+        return fuzq->max_distances[m];
+    return fuzq_calculate_max_distance(fuzq, m);
+}
+
+/**
+ * Calculate the similarity score for the +target+ against the query.
+ *
+ * @params fuzq The Fuzzy Query
+ * @params target *the term to compare against minus the prefix
+ * @params m the string length of +target+
+ * @params n the string length of the query string minus length of the prefix
+ */
+static float fuzq_score_mn(FrtFuzzyQuery *fuzq,
+                                  const char *target,
+                                  const int m, const int n)
+{
+    int i, j, prune;
+    int *d_curr, *d_prev;
+    const char *text = fuzq->text;
+    const int max_distance = fuzq_get_max_distance(fuzq, m);
+
+    /* Just adding the characters of m to n or vice-versa results in
+     * too many edits for example "pre" length is 3 and "prefixes"
+     * length is 8. We can see that given this optimal circumstance,
+     * the edit distance cannot be less than 5 which is 8-3 or more
+     * precisesly Math.abs(3-8). If our maximum edit distance is 4,
+     * then we can discard this word without looking at it. */
+    if (max_distance < FRT_ABS(m-n)) {
+        return 0.0f;
+    }
+
+    d_curr = fuzq->da;
+    d_prev = d_curr + n + 1;
+
+    /* init array */
+    for (j = 0; j <= n; j++) {
+        d_curr[j] = j;
+    }
+
+    /* start computing edit distance */
+    for (i = 0; i < m;) {
+        char s_i = target[i];
+        /* swap d_current into d_prev */
+        int *d_tmp = d_prev;
+        d_prev = d_curr;
+        d_curr = d_tmp;
+        prune = (d_curr[0] = ++i) > max_distance;
+
+        for (j = 0; j < n; j++) {
+            d_curr[j + 1] = (s_i == text[j])
+                ? FRT_MIN3(d_prev[j + 1] + 1, d_curr[j] + 1, d_prev[j])
+                : FRT_MIN3(d_prev[j + 1], d_curr[j], d_prev[j]) + 1;
+            if (prune && d_curr[j + 1] <= max_distance) {
+                prune = false;
+            }
+        }
+        if (prune) {
+            return 0.0f;
+        }
+    }
+
+    /* this will return less than 0.0 when the edit distance is greater
+     * than the number of characters in the shorter word.  but this was
+     * the formula that was previously used in FuzzyTermEnum, so it has
+     * not been changed (even though min_sim must be greater than 0.0) */
+    return (float)(1.0f - ((float)d_curr[n] / (float) (fuzq->pre_len + FRT_MIN(n, m))));
+}
+
+/**
+ * The following algorithm is taken from Bob Carpenter's FuzzyTermEnum
+ * implentation here;
+ *
+ * http://mail-archives.apache.org/mod_mbox/lucene-java-dev/200606.mbox/%3c448F0E8C.3050901@alias-i.com%3e
+ */
+float frt_fuzq_score(FrtFuzzyQuery *fuzq, const char *target)
+{
+    const int m = (int)strlen(target);
+    const int n = fuzq->text_len;
+
+    /* we don't have anything to compare.  That means if we just add
+     * the letters for m we get the new word */
+    if (m == 0 || n == 0) {
+        if (fuzq->pre_len == 0)
+            return 0.0f;
+        return 1.0f - ((float) (m+n) / fuzq->pre_len);
+    }
+
+    return fuzq_score_mn(fuzq, target, m, n);
+}
+
+/****************************************************************************
+ *
+ * FuzzyQuery
+ *
+ ****************************************************************************/
+
+#define FzQ(query) ((FrtFuzzyQuery *)(query))
+
+static char *fuzq_to_s(FrtQuery *self, FrtSymbol curr_field)
+{
+    char *buffer, *bptr;
+    char *term = FzQ(self)->term;
+    FrtSymbol field = FzQ(self)->field;
+    const char *field_name = rb_id2name(field);
+    bptr = buffer = FRT_ALLOC_N(char, strlen(term) + strlen(field_name) + 70);
+
+    if (curr_field != field) {
+        bptr += sprintf(bptr, "%s:", field_name);
+    }
+
+    bptr += sprintf(bptr, "%s~", term);
+    if (FzQ(self)->min_sim != 0.5) {
+        frt_dbl_to_s(bptr, FzQ(self)->min_sim);
+        bptr += strlen(bptr);
+    }
+
+    if (self->boost != 1.0) {
+        *bptr = '^';
+        frt_dbl_to_s(++bptr, self->boost);
+    }
+
+    return buffer;
+}
+
+static FrtQuery *fuzq_rewrite(FrtQuery *self, FrtIndexReader *ir)
+{
+    FrtQuery *q;
+    FrtFuzzyQuery *fuzq = FzQ(self);
+
+    int pre_len = fuzq->pre_len;
+    char *prefix = NULL;
+    const char *term = fuzq->term;
+    const int field_num = frt_fis_get_field_num(ir->fis, fuzq->field);
+    FrtTermEnum *te;
+
+    if (field_num < 0) {
+        return frt_bq_new(true);
+    }
+    if (fuzq->pre_len >= (int)strlen(term)) {
+        return frt_tq_new(fuzq->field, term);
+    }
+
+    q = frt_multi_tq_new_conf(fuzq->field, FrtMTQMaxTerms(self), fuzq->min_sim);
+    if (pre_len > 0) {
+        prefix = FRT_ALLOC_N(char, pre_len + 1);
+        strncpy(prefix, term, pre_len);
+        prefix[pre_len] = '\0';
+        te = ir->terms_from(ir, field_num, prefix);
+    }
+    else {
+        te = ir->terms(ir, field_num);
+    }
+
+    assert(NULL != te);
+
+    fuzq->scale_factor = (float)(1.0 / (1.0 - fuzq->min_sim));
+    fuzq->text = term + pre_len;
+    fuzq->text_len = (int)strlen(fuzq->text);
+    FRT_REALLOC_N(fuzq->da, int, fuzq->text_len * 2 + 2);
+    fuzq_initialize_max_distances(fuzq);
+
+    do {
+        const char *curr_term = te->curr_term;
+        const char *curr_suffix = curr_term + pre_len;
+        float score = 0.0f;
+
+        if (prefix && strncmp(curr_term, prefix, pre_len) != 0)
+            break;
+
+        score = frt_fuzq_score(fuzq, curr_suffix);
+        frt_multi_tq_add_term_boost(q, curr_term, score);
+    } while (te->next(te) != NULL);
+
+    te->close(te);
+    if (prefix) free(prefix);
+    return q;
+}
+
+static void fuzq_destroy(FrtQuery *self)
+{
+    free(FzQ(self)->term);
+    free(FzQ(self)->da);
+    frt_q_destroy_i(self);
+}
+
+static unsigned long long fuzq_hash(FrtQuery *self)
+{
+    return frt_str_hash(FzQ(self)->term) ^ frt_str_hash(rb_id2name(FzQ(self)->field))
+        ^ frt_float2int(FzQ(self)->min_sim) ^ FzQ(self)->pre_len;
+}
+
+static int fuzq_eq(FrtQuery *self, FrtQuery *o)
+{
+    FrtFuzzyQuery *fq1 = FzQ(self);
+    FrtFuzzyQuery *fq2 = FzQ(o);
+
+    return (strcmp(fq1->term, fq2->term) == 0)
+        && (fq1->field == fq2->field)
+        && (fq1->pre_len == fq2->pre_len)
+        && (fq1->min_sim == fq2->min_sim);
+}
+
+FrtQuery *frt_fuzq_new_conf(FrtSymbol field, const char *term,
+                     float min_sim, int pre_len, int max_terms)
+{
+    FrtQuery *self = frt_q_new(FrtFuzzyQuery);
+
+    FzQ(self)->field      = field;
+    FzQ(self)->term       = frt_estrdup(term);
+    FzQ(self)->pre_len    = pre_len ? pre_len : FRT_DEF_PRE_LEN;
+    FzQ(self)->min_sim    = min_sim ? min_sim : FRT_DEF_MIN_SIM;
+    FzQ(self)->da         = NULL;
+    FrtMTQMaxTerms(self)     = max_terms ? max_terms : FRT_DEF_MAX_TERMS;
+
+    self->type            = FUZZY_QUERY;
+    self->to_s            = &fuzq_to_s;
+    self->hash            = &fuzq_hash;
+    self->eq              = &fuzq_eq;
+    self->rewrite         = &fuzq_rewrite;
+    self->destroy_i       = &fuzq_destroy;
+    self->create_weight_i = &frt_q_create_weight_unsup;
+
+    return self;
+}
+
+FrtQuery *frt_fuzq_new(FrtSymbol field, const char *term)
+{
+    return frt_fuzq_new_conf(field, term, 0.0f, 0, 0);
+}