isomorfeus-ferret 0.12.0

data/ext/isomorfeus_ferret_ext/frt_similarity.c

@@ -0,0 +1,150 @@
+#include "frt_global.h"
+#include "frt_similarity.h"
+#include "frt_search.h"
+#include "frt_array.h"
+#include "frt_helper.h"
+#include <math.h>
+#include <stdlib.h>
+#include <string.h>
+
+/****************************************************************************
+ *
+ * Term
+ *
+ ****************************************************************************/
+
+FrtTerm *frt_term_new(FrtSymbol field, const char *text)
+{
+    FrtTerm *t = FRT_ALLOC(FrtTerm);
+    t->field = field;
+    t->text = frt_estrdup(text);
+    return t;
+}
+
+void frt_term_destroy(FrtTerm *self)
+{
+    free(self->text);
+    free(self);
+}
+
+int frt_term_eq(const void *t1, const void *t2)
+{
+    return ((strcmp(((FrtTerm *)t1)->text, ((FrtTerm *)t2)->text) == 0) &&
+        (((FrtTerm *)t1)->field == ((FrtTerm *)t2)->field));
+}
+
+unsigned long long frt_term_hash(const void *t)
+{
+    return frt_str_hash(((FrtTerm *)t)->text) * frt_str_hash(rb_id2name(((FrtTerm *)t)->field));
+}
+
+/****************************************************************************
+ *
+ * Similarity
+ *
+ ****************************************************************************/
+
+static float simdef_length_norm(FrtSimilarity *s, FrtSymbol field, int num_terms)
+{
+    (void)s;
+    (void)field;
+    return (float)(1.0 / sqrt(num_terms));
+}
+
+static float simdef_query_norm(struct FrtSimilarity *s, float sum_of_squared_weights)
+{
+    (void)s;
+    return (float)(1.0 / sqrt(sum_of_squared_weights));
+}
+
+static float simdef_tf(struct FrtSimilarity *s, float freq)
+{
+    (void)s;
+    return (float)sqrt(freq);
+}
+
+static float simdef_sloppy_freq(struct FrtSimilarity *s, int distance)
+{
+    (void)s;
+    return (float)(1.0 / (double)(distance + 1));
+}
+
+static float simdef_idf_term(struct FrtSimilarity *s, FrtSymbol field, char *term,
+                      FrtSearcher *searcher)
+{
+    return s->idf(s, searcher->doc_freq(searcher, field, term),
+                  searcher->max_doc(searcher));
+}
+
+static float simdef_idf_phrase(struct FrtSimilarity *s, FrtSymbol field,
+                        FrtPhrasePosition *positions,
+                        int pp_cnt, FrtSearcher *searcher)
+{
+    float idf = 0.0f;
+    int i, j;
+    for (i = 0; i < pp_cnt; i++) {
+        char **terms = positions[i].terms;
+        for (j = frt_ary_size(terms) - 1; j >= 0; j--) {
+            idf += frt_sim_idf_term(s, field, terms[j], searcher);
+        }
+    }
+    return idf;
+}
+
+static float simdef_idf(struct FrtSimilarity *s, int doc_freq, int num_docs)
+{
+    (void)s;
+    return (float)(log((float)num_docs/(float)(doc_freq+1)) + 1.0);
+}
+
+static float simdef_coord(struct FrtSimilarity *s, int overlap, int max_overlap)
+{
+    (void)s;
+    return (float)((double)overlap / (double)max_overlap);
+}
+
+static float simdef_decode_norm(struct FrtSimilarity *s, frt_uchar b)
+{
+    return s->norm_table[b];
+}
+
+static frt_uchar simdef_encode_norm(struct FrtSimilarity *s, float f)
+{
+    (void)s;
+    return frt_float2byte(f);
+}
+
+static void simdef_destroy(FrtSimilarity *s)
+{
+    (void)s;
+    /* nothing to do here */
+}
+
+static FrtSimilarity default_similarity = {
+    NULL,
+    {0},
+    &simdef_length_norm,
+    &simdef_query_norm,
+    &simdef_tf,
+    &simdef_sloppy_freq,
+    &simdef_idf_term,
+    &simdef_idf_phrase,
+    &simdef_idf,
+    &simdef_coord,
+    &simdef_decode_norm,
+    &simdef_encode_norm,
+    &simdef_destroy
+};
+
+FrtSimilarity *frt_sim_create_default()
+{
+    int i;
+    if (!default_similarity.data) {
+        for (i = 0; i < 256; i++) {
+            default_similarity.norm_table[i] = frt_byte2float((unsigned char)i);
+        }
+
+        default_similarity.data = &default_similarity;
+    }
+    return &default_similarity;
+}

data/ext/isomorfeus_ferret_ext/frt_similarity.h

@@ -0,0 +1,79 @@
+#ifndef FRT_SIMILARITY_H
+#define FRT_SIMILARITY_H
+
+typedef struct FrtSearcher FrtSearcher;
+
+/****************************************************************************
+ *
+ * FrtTerm
+ *
+ ****************************************************************************/
+
+typedef struct FrtTerm
+{
+    FrtSymbol field;
+    char *text;
+} FrtTerm;
+
+extern FrtTerm *frt_term_new(FrtSymbol field, const char *text);
+extern void frt_term_destroy(FrtTerm *self);
+extern int frt_term_eq(const void *t1, const void *t2);
+extern unsigned long long frt_term_hash(const void *t);
+
+/***************************************************************************
+ *
+ * FrtPhrasePosition
+ *
+ ***************************************************************************/
+
+typedef struct FrtPhrasePosition
+{
+    int pos;
+    char **terms;
+} FrtPhrasePosition;
+
+/***************************************************************************
+ *
+ * FrtSimilarity
+ *
+ ***************************************************************************/
+
+typedef struct FrtSimilarity FrtSimilarity;
+
+struct FrtSimilarity
+{
+    void *data;
+    float norm_table[256];
+    float (*length_norm)(FrtSimilarity *self, FrtSymbol field, int num_terms);
+    float (*query_norm)(FrtSimilarity *self, float sum_of_squared_weights);
+    float (*tf)(FrtSimilarity *self, float freq);
+    float (*sloppy_freq)(FrtSimilarity *self, int distance);
+    float (*idf_term)(FrtSimilarity *self, FrtSymbol field, char *term,
+                      FrtSearcher *searcher);
+    float (*idf_phrase)(FrtSimilarity *self, FrtSymbol field,
+                        FrtPhrasePosition *positions,
+                        int pp_cnt, FrtSearcher *searcher);
+    float (*idf)(FrtSimilarity *self, int doc_freq, int num_docs);
+    float (*coord)(FrtSimilarity *self, int overlap, int max_overlap);
+    float (*decode_norm)(FrtSimilarity *self, unsigned char b);
+    unsigned char (*encode_norm)(FrtSimilarity *self, float f);
+    void  (*destroy)(FrtSimilarity *self);
+};
+
+#define frt_sim_length_norm(msim, field, num_terms) msim->length_norm(msim, field, num_terms)
+#define frt_sim_query_norm(msim, sosw) msim->query_norm(msim, sosw)
+#define frt_sim_tf(msim, freq) msim->tf(msim, freq)
+#define frt_sim_sloppy_freq(msim, distance) msim->sloppy_freq(msim, distance)
+#define frt_sim_idf_term(msim, field, term, searcher)\
+    msim->idf_term(msim, field, term, searcher)
+#define frt_sim_idf_phrase(msim, field, positions, pos_cnt, searcher)\
+    msim->idf_phrase(msim, field, positions, pos_cnt, searcher)
+#define frt_sim_idf(msim, doc_freq, num_docs) msim->idf(msim, doc_freq, num_docs)
+#define frt_sim_coord(msim, overlap, max_overlap) msim->coord(msim, overlap, max_overlap)
+#define frt_sim_decode_norm(msim, b) msim->decode_norm(msim, b)
+#define frt_sim_encode_norm(msim, f) msim->encode_norm(msim, f)
+#define frt_sim_destroy(msim) msim->destroy(msim)
+
+FrtSimilarity *frt_sim_create_default();
+
+#endif