jk-ferret 0.11.8.2

data/ext/ferret.h

@@ -0,0 +1,94 @@
+#ifndef __FERRET_H_
+#define __FERRET_H_
+
+#include "global.h"
+#include "hashset.h"
+#include "document.h"
+#include "symbol.h"
+#include "internal.h"
+
+/* IDs */
+extern ID id_new;
+extern ID id_call;
+extern ID id_hash;
+extern ID id_eql;
+extern ID id_capacity;
+extern ID id_less_than;
+extern ID id_lt;
+extern ID id_is_directory;
+extern ID id_close;
+extern ID id_cclass;
+extern ID id_data;
+
+/* Symbols */
+extern VALUE sym_yes;
+extern VALUE sym_no;
+extern VALUE sym_true;
+extern VALUE sym_false;
+extern VALUE sym_path;
+extern VALUE sym_dir;
+
+/* Modules */
+extern VALUE mFerret;
+extern VALUE mIndex;
+extern VALUE mSearch;
+extern VALUE mStore;
+extern VALUE mStringHelper;
+extern VALUE mSpans;
+
+/* Classes */
+extern VALUE cDirectory;
+extern VALUE cLockError;
+extern VALUE cTerm;
+
+/* Ferret Inits */
+extern void Init_Utils();
+extern void Init_Analysis();
+extern void Init_Store();
+extern void Init_Index();
+extern void Init_Search();
+extern void Init_QueryParser();
+
+//extern void object_add(void *key, VALUE obj);
+#define object_add(key, obj) object_add2(key, obj,  __FILE__, __LINE__)
+extern void object_add2(void *key, VALUE obj, const char *file, int line);
+//extern void object_set(void *key, VALUE obj);
+#define object_set(key, obj) object_set2(key, obj,  __FILE__, __LINE__)
+extern void object_set2(void *key, VALUE obj, const char *file, int line);
+//extern void object_del(void *key);
+#define object_del(key) object_del2(key,  __FILE__, __LINE__)
+extern void object_del2(void *key, const char *file, int line);
+extern void frb_gc_mark(void *key);
+extern VALUE object_get(void *key);
+extern VALUE frb_data_alloc(VALUE klass);
+extern void frb_deref_free(void *p);
+extern void frb_create_dir(VALUE rpath);
+extern VALUE frb_hs_to_rb_ary(HashSet *hs);
+extern void *frb_rb_data_ptr(VALUE val);
+extern Symbol frb_field(VALUE rfield);
+extern VALUE frb_get_term(Symbol field, const char *term);
+extern char *json_concat_string(char *s, char *field);
+extern char *rs2s(VALUE rstr);
+extern char *rstrdup(VALUE rstr);
+extern Symbol rintern(VALUE rstr);
+#define Frt_Make_Struct(klass)\
+  rb_data_object_alloc(klass,NULL,(RUBY_DATA_FUNC)NULL,(RUBY_DATA_FUNC)NULL)
+
+#define Frt_Wrap_Struct(self,mmark,mfree,mdata)\
+  do {\
+    ((struct RData *)(self))->data = mdata;\
+    ((struct RData *)(self))->dmark = (RUBY_DATA_FUNC)mmark;\
+    ((struct RData *)(self))->dfree = (RUBY_DATA_FUNC)mfree;\
+  } while (0)
+
+#define Frt_Unwrap_Struct(self)\
+  do {\
+    ((struct RData *)(self))->data = NULL;\
+    ((struct RData *)(self))->dmark = NULL;\
+    ((struct RData *)(self))->dfree = NULL;\
+  } while (0)
+
+#endif
+
+#define frb_mark_cclass(klass) rb_ivar_set(klass, id_cclass, Qtrue)
+#define frb_is_cclass(obj) (rb_ivar_get(CLASS_OF(obj), id_cclass) == Qtrue)

data/ext/field_index.c

@@ -0,0 +1,262 @@
+#include <string.h>
+#include "field_index.h"
+#include "internal.h"
+
+/***************************************************************************
+ *
+ * FieldIndex
+ *
+ ***************************************************************************/
+
+static unsigned long field_index_hash(const void *p)
+{
+    FieldIndex *self = (FieldIndex *)p;
+    return sym_hash(self->field) ^ (unsigned long)(self->klass);
+}
+
+static int field_index_eq(const void *p1, const void *p2)
+{
+    FieldIndex *fi1 = (FieldIndex *)p1;
+    FieldIndex *fi2 = (FieldIndex *)p2;
+    return (fi1->field == fi2->field) &&
+        (fi1->klass->type == fi2->klass->type);
+}
+
+static void field_index_destroy(void *p)
+{
+    FieldIndex *self = (FieldIndex *)p;
+    if (self->index) {
+        self->klass->destroy_index(self->index);
+    }
+    free(self);
+}
+
+FieldIndex *field_index_get(IndexReader *ir, Symbol field,
+                            const FieldIndexClass *klass)
+{
+    int length = 0;
+    TermEnum *volatile te = NULL;
+    TermDocEnum *volatile tde = NULL;
+    FieldInfo *fi = fis_get_field(ir->fis, field);
+    const volatile int field_num = fi ? fi->number : -1;
+    FieldIndex *volatile self = NULL;
+    FieldIndex key;
+
+    if (field_num < 0) {
+        RAISE(ARG_ERROR,
+              "Cannot sort by field \"%s\". It doesn't exist in the index.",
+              S(field));
+    }
+
+    if (!ir->field_index_cache) {
+        ir->field_index_cache = h_new(&field_index_hash, &field_index_eq,
+                                      NULL, &field_index_destroy);
+    }
+
+    key.field = field;
+    key.klass = klass;
+    self = (FieldIndex *)h_get(ir->field_index_cache, &key);
+
+    if (self == NULL) {
+        self = ALLOC(FieldIndex);
+        self->klass = klass;
+        /* FieldIndex only lives as long as the IndexReader lives so we can
+         * just use the field_infos field string */
+        self->field = fi->name;
+
+        length = ir->max_doc(ir);
+        if (length > 0) {
+            TRY
+            {
+                void *index;
+                tde = ir->term_docs(ir);
+                te = ir->terms(ir, field_num);
+                index = self->index = klass->create_index(length);
+                while (te->next(te)) {
+                    tde->seek_te(tde, te);
+                    klass->handle_term(index, tde, te->curr_term);
+                }
+            }
+            XFINALLY
+                tde->close(tde);
+                te->close(te);
+            XENDTRY
+        }
+        h_set(ir->field_index_cache, self, self);
+    }
+
+    return self;
+}
+
+/******************************************************************************
+ * ByteFieldIndex < FieldIndex
+ *
+ * The ByteFieldIndex holds an array of integers for each document in the
+ * index where the integer represents the sort value for the document.  This
+ * index should only be used for sorting and not as a field cache of the
+ * column's value.
+ ******************************************************************************/
+static void byte_handle_term(void *index_ptr,
+                             TermDocEnum *tde,
+                             const char *text)
+{
+    long *index = (long *)index_ptr;
+    long val = index[-1]++;
+    (void)text;
+    while (tde->next(tde)) {
+        index[tde->doc_num(tde)] = val;
+    }
+}
+
+static void *byte_create_index(int size)
+{
+    long *index = ALLOC_AND_ZERO_N(long, size + 1);
+    index[0] = 1;
+    return &index[1];
+}
+
+static void byte_destroy_index(void *p)
+{
+    long *index = (long *)p;
+    free(&index[-1]);
+}
+
+const FieldIndexClass BYTE_FIELD_INDEX_CLASS = {
+    "byte",
+    &byte_create_index,
+    &byte_destroy_index,
+    &byte_handle_term
+};
+
+/******************************************************************************
+ * IntegerFieldIndex < FieldIndex
+ ******************************************************************************/
+static void *integer_create_index(int size)
+{
+    return ALLOC_AND_ZERO_N(long, size);
+}
+
+static void integer_handle_term(void *index_ptr,
+                                TermDocEnum *tde,
+                                const char *text)
+{
+    long *index = (long *)index_ptr;
+    long val;
+    sscanf(text, "%ld", &val);
+    while (tde->next(tde)) {
+        index[tde->doc_num(tde)] = val;
+    }
+}
+
+const FieldIndexClass INTEGER_FIELD_INDEX_CLASS = {
+    "integer",
+    &integer_create_index,
+    &free,
+    &integer_handle_term
+};
+
+long get_integer_value(FieldIndex *field_index, long doc_num)
+{
+    if (field_index->klass == &INTEGER_FIELD_INDEX_CLASS && doc_num >= 0) {
+        return ((long *)field_index->index)[doc_num];
+    }
+    return 0l;
+}
+
+
+/******************************************************************************
+ * FloatFieldIndex < FieldIndex
+ ******************************************************************************/
+#define VALUES_ARRAY_START_SIZE 8
+static void *float_create_index(int size)
+{
+    return ALLOC_AND_ZERO_N(float, size);
+}
+
+static void float_handle_term(void *index_ptr,
+                              TermDocEnum *tde,
+                              const char *text)
+{
+    float *index = (float *)index_ptr;
+    float val;
+    sscanf(text, "%g", &val);
+    while (tde->next(tde)) {
+        index[tde->doc_num(tde)] = val;
+    }
+}
+
+const FieldIndexClass FLOAT_FIELD_INDEX_CLASS = {
+    "float",
+    &float_create_index,
+    &free,
+    &float_handle_term
+};
+
+float get_float_value(FieldIndex *field_index, long doc_num)
+{
+    if (field_index->klass == &FLOAT_FIELD_INDEX_CLASS && doc_num >= 0) {
+        return ((float *)field_index->index)[doc_num];
+    }
+    return 0.0f;
+}
+
+/******************************************************************************
+ * StringFieldIndex < FieldIndex
+ ******************************************************************************/
+ 
+static void *string_create_index(int size)
+{
+    StringIndex *self = ALLOC_AND_ZERO(StringIndex);
+    self->size = size;
+    self->index = ALLOC_AND_ZERO_N(long, size);
+    self->v_capa = VALUES_ARRAY_START_SIZE;
+    self->v_size = 1; /* leave the first value as NULL */
+    self->values = ALLOC_AND_ZERO_N(char *, VALUES_ARRAY_START_SIZE);
+    return self;
+}
+
+static void string_destroy_index(void *p)
+{
+    StringIndex *self = (StringIndex *)p;
+    int i;
+    free(self->index);
+    for (i = 0; i < self->v_size; i++) {
+        free(self->values[i]);
+    }
+    free(self->values);
+    free(self);
+}
+
+static void string_handle_term(void *index_ptr,
+                               TermDocEnum *tde,
+                               const char *text)
+{
+    StringIndex *index = (StringIndex *)index_ptr;
+    if (index->v_size >= index->v_capa) {
+        index->v_capa *= 2;
+        index->values = REALLOC_N(index->values, char *, index->v_capa);
+    }
+    index->values[index->v_size] = estrdup(text);
+    while (tde->next(tde)) {
+        index->index[tde->doc_num(tde)] = index->v_size;
+    }
+    index->v_size++;
+}
+
+const FieldIndexClass STRING_FIELD_INDEX_CLASS = {
+    "string",
+    &string_create_index,
+    &string_destroy_index,
+    &string_handle_term
+};
+
+const char *get_string_value(FieldIndex *field_index, long doc_num)
+{
+    if (field_index->klass == &STRING_FIELD_INDEX_CLASS) {
+        StringIndex *string_index = (StringIndex *)field_index->index;
+        if (doc_num >= 0 && doc_num < string_index->size) {
+            return string_index->values[string_index->index[doc_num]];
+        }
+    }
+    return NULL;
+}

data/ext/field_index.h

@@ -0,0 +1,52 @@
+#ifndef FRT_FIELD_INDEX_H
+#define FRT_FIELD_INDEX_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "index.h"
+
+/***************************************************************************
+ *
+ * FrtFieldIndex
+ *
+ ***************************************************************************/
+
+typedef struct FrtStringIndex {
+    int size;
+    long *index;
+    char **values;
+    int v_size;
+    int v_capa;
+} FrtStringIndex;
+
+typedef struct FrtFieldIndexClass {
+    const char *type;
+    void *(*create_index)(int size);
+    void  (*destroy_index)(void *p);
+    void  (*handle_term)(void *index, FrtTermDocEnum *tde, const char *text);
+} FrtFieldIndexClass;
+
+typedef struct FrtFieldIndex {
+    FrtSymbol field;
+    const FrtFieldIndexClass *klass;
+    void *index;
+} FrtFieldIndex;
+
+extern const FrtFieldIndexClass FRT_INTEGER_FIELD_INDEX_CLASS;
+extern const FrtFieldIndexClass   FRT_FLOAT_FIELD_INDEX_CLASS;
+extern const FrtFieldIndexClass  FRT_STRING_FIELD_INDEX_CLASS;
+extern const FrtFieldIndexClass    FRT_BYTE_FIELD_INDEX_CLASS;
+
+extern FrtFieldIndex *frt_field_index_get(FrtIndexReader *ir, FrtSymbol field,
+                                   const FrtFieldIndexClass *klass);
+extern const char *frt_get_string_value(FrtFieldIndex *field_index, long doc_num);
+extern float frt_get_float_value(FrtFieldIndex *field_index, long doc_num);
+extern long frt_get_integer_value(FrtFieldIndex *field_index, long doc_num);
+
+#ifdef __cplusplus
+} // extern "C"
+#endif
+
+#endif

data/ext/filter.c

@@ -0,0 +1,157 @@
+#include "search.h"
+#include "symbol.h"
+#include <string.h>
+#include "internal.h"
+
+/***************************************************************************
+ *
+ * Filter
+ *
+ ***************************************************************************/
+
+void filt_destroy_i(Filter *filt)
+{
+    h_destroy(filt->cache);
+    free(filt);
+}
+void filt_deref(Filter *filt)
+{
+    if (--(filt->ref_cnt) == 0) {
+        filt->destroy_i(filt);
+    }
+}
+
+BitVector *filt_get_bv(Filter *filt, IndexReader *ir)
+{
+    CacheObject *co = (CacheObject *)h_get(filt->cache, ir);
+
+    if (!co) {
+        BitVector *bv;
+        if (!ir->cache) {
+            ir_add_cache(ir);
+        }
+        bv = filt->get_bv_i(filt, ir);
+        co = co_create(filt->cache, ir->cache, filt, ir,
+                       (free_ft)&bv_destroy, (void *)bv);
+    }
+    return (BitVector *)co->obj;
+}
+
+static char *filt_to_s_i(Filter *filt)
+{
+    return estrdup(S(filt->name));
+}
+
+static unsigned long filt_hash_default(Filter *filt)
+{
+    (void)filt;
+    return 0;
+}
+
+static int filt_eq_default(Filter *filt, Filter *o)
+{
+    (void)filt; (void)o;
+    return false;
+}
+
+Filter *filt_create(size_t size, Symbol name)
+{
+    Filter *filt    = (Filter *)emalloc(size);
+    filt->cache     = co_hash_create();
+    filt->name      = name;
+    filt->to_s      = &filt_to_s_i;
+    filt->hash      = &filt_hash_default;
+    filt->eq        = &filt_eq_default;
+    filt->destroy_i = &filt_destroy_i;
+    filt->ref_cnt   = 1;
+    return filt;
+}
+
+unsigned long filt_hash(Filter *filt)
+{
+    return sym_hash(filt->name) ^ filt->hash(filt);
+}
+
+int filt_eq(Filter *filt, Filter *o)
+{
+    return ((filt == o)
+            || ((filt->name == o->name)
+                && (filt->eq == o->eq)
+                && (filt->eq(filt, o))));
+}
+
+/***************************************************************************
+ *
+ * QueryFilter
+ *
+ ***************************************************************************/
+
+#define QF(filt) ((QueryFilter *)(filt))
+typedef struct QueryFilter
+{
+    Filter super;
+    Query *query;
+} QueryFilter;
+
+static char *qfilt_to_s(Filter *filt)
+{
+    Query *query = QF(filt)->query;
+    char *query_str = query->to_s(query, NULL);
+    char *filter_str = strfmt("QueryFilter< %s >", query_str);
+    free(query_str);
+    return filter_str;
+}
+
+static BitVector *qfilt_get_bv_i(Filter *filt, IndexReader *ir)
+{
+    BitVector *bv = bv_new_capa(ir->max_doc(ir));
+    Searcher *sea = isea_new(ir);
+    Weight *weight = q_weight(QF(filt)->query, sea);
+    Scorer *scorer = weight->scorer(weight, ir);
+    if (scorer) {
+        while (scorer->next(scorer)) {
+            bv_set(bv, scorer->doc);
+        }
+        scorer->destroy(scorer);
+    }
+    weight->destroy(weight);
+    free(sea);
+    return bv;
+}
+
+static unsigned long qfilt_hash(Filter *filt)
+{
+    return q_hash(QF(filt)->query);
+}
+
+static int qfilt_eq(Filter *filt, Filter *o)
+{
+    return q_eq(QF(filt)->query, QF(o)->query);
+}
+
+static void qfilt_destroy_i(Filter *filt)
+{
+    Query *query = QF(filt)->query;
+    q_deref(query);
+    filt_destroy_i(filt);
+}
+
+Filter *qfilt_new_nr(Query *query)
+{
+    Filter *filt = filt_new(QueryFilter);
+
+    QF(filt)->query = query;
+
+    filt->get_bv_i  = &qfilt_get_bv_i;
+    filt->hash      = &qfilt_hash;
+    filt->eq        = &qfilt_eq;
+    filt->to_s      = &qfilt_to_s;
+    filt->destroy_i = &qfilt_destroy_i;
+    return filt;
+}
+
+Filter *qfilt_new(Query *query)
+{
+    REF(query);
+    return qfilt_new_nr(query);
+}