ferret 0.10.4 → 0.10.5

data/Rakefile

@@ -112,7 +112,7 @@ EXT_SRC.each do |fn|
       mv dest_fn + ".out", dest_fn
     end
   end
-end
+end if File.exists?("../c")
 
 desc "Build the extension"
 task :ext => ["ext/#{EXT}"] + SRC

data/ext/analysis.c

@@ -843,9 +843,14 @@ static Token *std_next(TokenStream *ts)
             /* strip possesive */
             if ((t[-1] == 's' || t[-1] == 'S') && t[-2] == '\'') {
                 t -= 2;
+                tk_set_ts(&(CTS(ts)->token), start, t, ts->text, 1);
+                CTS(ts)->token.end += 2;
+            }
+            else {
+                tk_set_ts(&(CTS(ts)->token), start, t, ts->text, 1);
             }
 
-            return tk_set_ts(&(CTS(ts)->token), start, t, ts->text, 1);
+            return &(CTS(ts)->token);
         }
 
         if (*t == '&') {        /* apostrophe case. */
@@ -1434,6 +1439,7 @@ Analyzer *per_field_analyzer_new(Analyzer *default_a)
 
     a->destroy_i = &pfa_destroy_i;
     a->get_ts    = pfa_get_ts;
+    a->ref_cnt   = 1;
     
     return a;
 }

data/ext/bitvector.c

@@ -14,6 +14,7 @@ BitVector *bv_new_capa(int capa)
     bv->count = 0;
     bv->curr_bit = -1;
     bv->extends_as_ones = 0;
+    bv->ref_cnt = 1;
     return bv;
 }
 
@@ -24,8 +25,10 @@ BitVector *bv_new()
 
 void bv_destroy(BitVector * bv)
 {
-    free(bv->bits);
-    free(bv);
+    if (--(bv->ref_cnt) == 0) {
+        free(bv->bits);
+        free(bv);
+    }
 }
 
 void bv_set(BitVector * bv, int bit)

data/ext/bitvector.h

@@ -23,6 +23,7 @@ typedef struct BitVector
     int curr_bit;
 
     bool extends_as_ones : 1;
+    int ref_cnt;
 } BitVector;
 
 /**

data/ext/ferret.c

@@ -10,10 +10,12 @@ static HashTable *object_map;
 
 /* IDs */
 ID id_new;
+ID id_call;
+ID id_eql;
+ID id_hash;
 ID id_capacity;
 ID id_less_than;
 ID id_lt;
-ID id_call;
 ID id_is_directory;
 ID id_close;
 ID id_cclass;
@@ -36,6 +38,7 @@ VALUE mStringHelper;
 VALUE mSpans;
 
 /* Classes */
+VALUE cTerm;
 /*
 */
 
@@ -98,13 +101,6 @@ VALUE frt_data_alloc(VALUE klass)
     return Frt_Make_Struct(klass);
 }
 
-VALUE frt_define_class_under(VALUE module, char *name, VALUE super)
-{
-    VALUE klass = rb_define_class_under(module, name, super);
-    rb_ivar_set(klass, id_cclass, Qtrue);
-    return klass;
-}
-
 void frt_deref_free(void *p)
 {
     object_del(p);
@@ -233,6 +229,54 @@ void FRT_EXIT(const char *err_type, const char *fmt, ...)
     va_end(args);
 }
 
+/****************************************************************************
+ *
+ * Term Methods
+ *
+ ****************************************************************************/
+static ID id_field;
+static ID id_text;
+
+VALUE frt_get_term(const char *field, const char *text)
+{
+    return rb_struct_new(cTerm,
+                         ID2SYM(rb_intern(field)),
+                         rb_str_new2(text),
+                         NULL);
+}
+
+static VALUE frt_term_to_s(VALUE self)
+{
+    VALUE rstr;
+    VALUE rfield = rb_funcall(self, id_field, 0); 
+    VALUE rtext = rb_funcall(self, id_text, 0); 
+    char *field = StringValuePtr(rfield);
+    char *text = StringValuePtr(rtext);
+    char *term_str = ALLOC_N(char,
+                             5 + RSTRING(rfield)->len + RSTRING(rtext)->len);
+    sprintf(term_str, "%s:%s", field, text);
+    rstr = rb_str_new2(term_str);
+    free(term_str);
+    return rstr;
+}
+/*
+ *  Document-class: Ferret::Term
+ *
+ *  == Summary
+ *
+ *  A Term holds a term from a document and its field name (as a Symbol).
+ */
+void Init_Term(void)
+{
+    const char *term_class = "Term";
+    cTerm = rb_struct_define(term_class, "field", "text", NULL);
+    rb_set_class_path(cTerm, mFerret, term_class);
+    rb_const_set(mFerret, rb_intern(term_class), cTerm);
+    rb_define_method(cTerm, "to_s", frt_term_to_s, 0);
+    id_field = rb_intern("field");
+    id_text = rb_intern("text");
+}
+
 /*
  *  Document-module: Ferret
  *
@@ -241,6 +285,7 @@ void FRT_EXIT(const char *err_type, const char *fmt, ...)
 void Init_Ferret(void)
 {
     mFerret = rb_define_module("Ferret");
+    Init_Term();
 }
 
 void Init_ferret_ext(void)
@@ -254,6 +299,8 @@ void Init_ferret_ext(void)
     /* IDs */
     id_new = rb_intern("new");
     id_call = rb_intern("call");
+    id_eql = rb_intern("eql?");
+    id_hash = rb_intern("hash");
 
     id_capacity = rb_intern("capacity");
     id_less_than = rb_intern("less_than");

data/ext/ferret.h

@@ -7,10 +7,12 @@
 
 /* IDs */
 extern ID id_new;
+extern ID id_call;
+extern ID id_hash;
+extern ID id_eql;
 extern ID id_capacity;
 extern ID id_less_than;
 extern ID id_lt;
-extern ID id_call;
 extern ID id_is_directory;
 extern ID id_close;
 extern ID id_cclass;
@@ -35,6 +37,7 @@ extern VALUE mSpans;
 /* Classes */
 extern VALUE cDirectory;
 extern VALUE cLockError;
+extern VALUE cTerm;
 
 /* Ferret Inits */
 extern void Init_Utils();
@@ -61,7 +64,7 @@ extern void frt_create_dir(VALUE rpath);
 extern VALUE frt_hs_to_rb_ary(HashSet *hs);
 extern void *frt_rb_data_ptr(VALUE val);
 extern char * frt_field(VALUE rfield);
-extern VALUE frt_define_class_under(VALUE module, char *name, VALUE super);
+extern VALUE frt_get_term(const char *field, const char *term);
 
 #define Frt_Make_Struct(klass)\
   rb_data_object_alloc(klass,NULL,(RUBY_DATA_FUNC)NULL,(RUBY_DATA_FUNC)NULL)
@@ -81,3 +84,6 @@ extern VALUE frt_define_class_under(VALUE module, char *name, VALUE super);
   } while (0)
 
 #endif
+
+#define frt_mark_cclass(klass) rb_ivar_set(klass, id_cclass, Qtrue)
+#define frt_is_cclass(obj) (rb_ivar_get(CLASS_OF(obj), id_cclass) == Qtrue)

data/ext/index.c

@@ -2899,10 +2899,9 @@ void ir_set_norm(IndexReader *ir, int doc_num, const char *field, uchar val)
     }
 }
 
-uchar *ir_get_norms(IndexReader *ir, const char *field)
+uchar *ir_get_norms_i(IndexReader *ir, int field_num)
 {
     uchar *norms = NULL;
-    int field_num = fis_get_field_num(ir->fis, field);
     if (field_num >= 0) {
         norms = ir->get_norms(ir, field_num);
     }
@@ -2915,6 +2914,12 @@ uchar *ir_get_norms(IndexReader *ir, const char *field)
     return norms;
 }
 
+uchar *ir_get_norms(IndexReader *ir, const char *field)
+{
+    int field_num = fis_get_field_num(ir->fis, field);
+    return ir_get_norms_i(ir, field_num);
+}
+
 uchar *ir_get_norms_into(IndexReader *ir, const char *field, uchar *buf)
 {
     int field_num = fis_get_field_num(ir->fis, field);
@@ -3286,6 +3291,7 @@ static BitVector *bv_read(Store *store, char *name)
     bv->size = (int)is_read_vint(is);
     bv->capa = (bv->size >> 5) + 1;
     bv->bits = ALLOC_AND_ZERO_N(f_u32, bv->capa);
+    bv->ref_cnt = 1;
     for (i = (bv->size >> 5); i >= 0; i--) {
         bv->bits[i] = is_read_u32(is);
     }
@@ -5022,17 +5028,6 @@ int iw_doc_count(IndexWriter *iw)
     return doc_cnt;
 }
 
-void iw_delete_term(IndexWriter *iw, const char *field, const char *term)
-{
-    int field_num = fis_get_field_num(iw->fis, field);
-    if (field_num >= 0) {
-        DelTerm *dt = ALLOC(DelTerm);
-        dt->field_num = field_num;
-        dt->term = estrdup(term);
-        ary_push(iw->del_terms, dt);
-    }
-}
-
 static void delete_files(char **file_names, Store *store)
 {
     int i;
@@ -5277,37 +5272,9 @@ void iw_add_doc(IndexWriter *iw, Document *doc)
 
 static void iw_commit_i(IndexWriter *iw)
 {
-    /* optimized term deletion method */
-    const int del_term_cnt = ary_size(iw->del_terms);
     if (iw->dw && iw->dw->doc_num > 0) {
         iw_flush_ram_segment(iw);
     }
-    if (del_term_cnt) {
-        DelTerm **del_terms = iw->del_terms;
-        int i;
-        SegmentInfos *sis = iw->sis;
-        const int seg_cnt = sis->size;
-        for (i = 0; i < seg_cnt; i++) {
-            int j;
-            IndexReader *ir = sr_open(sis, iw->fis, i, false);
-            TermDocEnum *tde = ir->term_docs(ir);
-            for (j = 0; j < del_term_cnt; j++) {
-                DelTerm *del_term = del_terms[j];
-                stde_seek(tde, del_term->field_num, del_term->term);
-                while (tde->next(tde)) {
-                    sr_delete_doc_i(ir, STDE(tde)->doc_num);
-                }
-            }
-            tde_destroy(tde);
-            sr_commit_i(ir);
-            ir_close(ir);
-        }
-        ary_each_rev(del_terms, i) {
-            free(del_terms[i]->term);
-            free(del_terms[i]);
-        }
-        ary_size(del_terms) = 0;
-    }
 }
 
 void iw_commit(IndexWriter *iw)
@@ -5317,6 +5284,32 @@ void iw_commit(IndexWriter *iw)
     mutex_unlock(&iw->mutex);
 }
 
+void iw_delete_term(IndexWriter *iw, const char *field, const char *term)
+{
+    int field_num = fis_get_field_num(iw->fis, field);
+    if (field_num >= 0) {
+        int i;
+        mutex_lock(&iw->mutex);
+        iw_commit_i(iw);
+        do {
+            SegmentInfos *sis = iw->sis;
+            const int seg_cnt = sis->size;
+            for (i = 0; i < seg_cnt; i++) {
+                IndexReader *ir = sr_open(sis, iw->fis, i, false);
+                TermDocEnum *tde = ir->term_docs(ir);
+                stde_seek(tde, field_num, term);
+                while (tde->next(tde)) {
+                    sr_delete_doc_i(ir, STDE(tde)->doc_num);
+                }
+                tde_destroy(tde);
+                sr_commit_i(ir);
+                ir_close(ir);
+            }
+        } while (0);
+        mutex_unlock(&iw->mutex);
+    }
+}
+
 static void iw_optimize_i(IndexWriter *iw)
 {
     int min_segment;
@@ -5351,7 +5344,6 @@ void iw_close(IndexWriter *iw)
     sis_destroy(iw->sis);
     fis_deref(iw->fis);
     sim_destroy(iw->similarity);
-    ary_free(iw->del_terms);
 
     iw->write_lock->release(iw->write_lock);
     iw->store->close_lock(iw->write_lock);
@@ -5392,7 +5384,6 @@ IndexWriter *iw_open(Store *store, Analyzer *analyzer, const Config *config)
     XENDTRY
 
     iw->similarity = sim_create_default();
-    iw->del_terms = ary_new_type(DelTerm *);
     iw->analyzer = analyzer ? analyzer : mb_standard_analyzer_new(true);
 
     REF(store);

data/ext/index.h

@@ -744,6 +744,7 @@ extern void ir_undelete_all(IndexReader *ir);
 extern int ir_doc_freq(IndexReader *ir, const char *field, const char *term);
 extern void ir_set_norm(IndexReader *ir, int doc_num, const char *field,
                         uchar val);
+extern uchar *ir_get_norms_i(IndexReader *ir, int field_num);
 extern uchar *ir_get_norms(IndexReader *ir, const char *field);
 extern uchar *ir_get_norms_into(IndexReader *ir, const char *field, uchar *buf);
 extern void ir_destroy(IndexReader *self);
@@ -868,7 +869,6 @@ struct IndexWriter
     FieldInfos *fis;
     DocWriter *dw;
     Similarity *similarity;
-    DelTerm **del_terms;
     Lock *write_lock;
 };
 

data/ext/q_boolean.c

@@ -388,7 +388,7 @@ static bool csc_skip_to(Scorer *self, int doc_num)
         csc_sort_scorers(csc);
     }
 
-    more = csc->more;
+    csc->more = more;
     return csc_do_next(self);
 }
 

data/ext/q_multi_term.c

@@ -357,7 +357,7 @@ static Scorer *multi_tw_scorer(Weight *self, IndexReader *ir)
         te->close(te);
         if (tdew_cnt) {
             multi_tsc = multi_tsc_new(self, MTQ(self->query)->field, tdew_a,
-                                      tdew_cnt, ir->get_norms(ir, field_num));
+                                      tdew_cnt, ir_get_norms_i(ir, field_num));
         }
         else {
             free(tdew_a);
@@ -559,6 +559,17 @@ static void multi_tq_destroy_i(Query *self)
     q_destroy_i(self);
 }
 
+static void multi_tq_extract_terms(Query *self, HashSet *terms)
+{
+    int i;
+    char *field = MTQ(self)->field;
+    PriorityQueue *boosted_terms = MTQ(self)->boosted_terms;
+    for (i = boosted_terms->size; i > 0; i--) {
+        BoostedTerm *bt = (BoostedTerm *)boosted_terms->heap[i];
+        hs_add(terms, term_new(field, bt->term));
+    }
+}
+
 static ulong multi_tq_hash(Query *self)
 {
     int i;
@@ -631,6 +642,7 @@ Query *multi_tq_new_conf(const char *field, int max_terms, float min_boost)
 
     self->type               = MULTI_TERM_QUERY;
     self->to_s               = &multi_tq_to_s;
+    self->extract_terms      = &multi_tq_extract_terms;
     self->hash               = &multi_tq_hash;
     self->eq                 = &multi_tq_eq;
     self->destroy_i          = &multi_tq_destroy_i;

data/ext/q_parser.c

@@ -170,8 +170,8 @@ static Phrase *ph_first_word(char *word);
 static Phrase *ph_add_word(Phrase *self, char *word);
 static Phrase *ph_add_multi_word(Phrase *self, char *word);
 
-static Query *get_range_q(const char *field, const char *from, const char *to,
-                          bool inc_lower, bool inc_upper);
+static Query *get_r_q(QParser *qp, char *field, char *from, char *to,
+                      bool inc_lower, bool inc_upper);
 
 #define FLDS(q, func) do {\
     char *field;\
@@ -1405,62 +1405,62 @@ yyreduce:
 
   case 39:
 #line 146 "src/q_parser.y"
-    { FLDS((yyval.query), get_range_q(field, (yyvsp[-2].str),  (yyvsp[-1].str),  true,  true)); }
+    { FLDS((yyval.query), get_r_q(qp, field, (yyvsp[-2].str),  (yyvsp[-1].str),  true,  true)); }
     break;
 
   case 40:
 #line 147 "src/q_parser.y"
-    { FLDS((yyval.query), get_range_q(field, (yyvsp[-2].str),  (yyvsp[-1].str),  true,  false)); }
+    { FLDS((yyval.query), get_r_q(qp, field, (yyvsp[-2].str),  (yyvsp[-1].str),  true,  false)); }
     break;
 
   case 41:
 #line 148 "src/q_parser.y"
-    { FLDS((yyval.query), get_range_q(field, (yyvsp[-2].str),  (yyvsp[-1].str),  false, true)); }
+    { FLDS((yyval.query), get_r_q(qp, field, (yyvsp[-2].str),  (yyvsp[-1].str),  false, true)); }
     break;
 
   case 42:
 #line 149 "src/q_parser.y"
-    { FLDS((yyval.query), get_range_q(field, (yyvsp[-2].str),  (yyvsp[-1].str),  false, false)); }
+    { FLDS((yyval.query), get_r_q(qp, field, (yyvsp[-2].str),  (yyvsp[-1].str),  false, false)); }
     break;
 
   case 43:
 #line 150 "src/q_parser.y"
-    { FLDS((yyval.query), get_range_q(field, NULL,(yyvsp[-1].str),  false, false)); }
+    { FLDS((yyval.query), get_r_q(qp, field, NULL,(yyvsp[-1].str),  false, false)); }
     break;
 
   case 44:
 #line 151 "src/q_parser.y"
-    { FLDS((yyval.query), get_range_q(field, NULL,(yyvsp[-1].str),  false, true)); }
+    { FLDS((yyval.query), get_r_q(qp, field, NULL,(yyvsp[-1].str),  false, true)); }
     break;
 
   case 45:
 #line 152 "src/q_parser.y"
-    { FLDS((yyval.query), get_range_q(field, (yyvsp[-1].str),  NULL,true,  false)); }
+    { FLDS((yyval.query), get_r_q(qp, field, (yyvsp[-1].str),  NULL,true,  false)); }
     break;
 
   case 46:
 #line 153 "src/q_parser.y"
-    { FLDS((yyval.query), get_range_q(field, (yyvsp[-1].str),  NULL,false, false)); }
+    { FLDS((yyval.query), get_r_q(qp, field, (yyvsp[-1].str),  NULL,false, false)); }
     break;
 
   case 47:
 #line 154 "src/q_parser.y"
-    { FLDS((yyval.query), get_range_q(field, NULL,(yyvsp[0].str),  false, false)); }
+    { FLDS((yyval.query), get_r_q(qp, field, NULL,(yyvsp[0].str),  false, false)); }
     break;
 
   case 48:
 #line 155 "src/q_parser.y"
-    { FLDS((yyval.query), get_range_q(field, NULL,(yyvsp[0].str),  false, true)); }
+    { FLDS((yyval.query), get_r_q(qp, field, NULL,(yyvsp[0].str),  false, true)); }
     break;
 
   case 49:
 #line 156 "src/q_parser.y"
-    { FLDS((yyval.query), get_range_q(field, (yyvsp[0].str),  NULL,true,  false)); }
+    { FLDS((yyval.query), get_r_q(qp, field, (yyvsp[0].str),  NULL,true,  false)); }
     break;
 
   case 50:
 #line 157 "src/q_parser.y"
-    { FLDS((yyval.query), get_range_q(field, (yyvsp[0].str),  NULL,false, false)); }
+    { FLDS((yyval.query), get_r_q(qp, field, (yyvsp[0].str),  NULL,false, false)); }
     break;
 
 
@@ -1756,7 +1756,7 @@ static int get_word(YYSTYPE *lvalp, QParser *qp)
     while (!strchr(not_word, (c=*qp->qstrp++))) {
         switch (c) {
             case '\\':
-                if ((c=*qp->qstrp) == ' ' && c != '\t' && c != '\0') {
+                if ((c=*qp->qstrp) == '\0') {
                     *bufp++ = '\\';
                 }
                 else {
@@ -2248,10 +2248,25 @@ static Query *get_phrase_q(QParser *qp, Phrase *phrase, char *slop_str)
     return q;
 }
 
-static Query *get_range_q(const char *field, const char *from, const char *to,
-                          bool inc_lower, bool inc_upper)
+static Query *get_r_q(QParser *qp, char *field, char *from, char *to,
+                      bool inc_lower, bool inc_upper)
 {
-    return rq_new(field, from, to, inc_lower, inc_upper);
+    Query *rq;
+    if (from) {
+        TokenStream *stream = get_cached_ts(qp, field, from);
+        Token *token = ts_next(stream);
+        from = token ? estrdup(token->text) : NULL;
+    }
+    if (to) {
+        TokenStream *stream = get_cached_ts(qp, field, to);
+        Token *token = ts_next(stream);
+        to = token ? estrdup(token->text) : NULL;
+    }
+
+    rq = rq_new(field, from, to, inc_lower, inc_upper);
+    if (from) free(from);
+    if (to) free(to);
+    return rq;
 }
 
 void qp_destroy(QParser *self)