ferret 0.10.9 → 0.10.10

data/ext/index.c

@@ -1552,7 +1552,7 @@ TermEnum *ste_new(InStream *is, SegmentFieldIndex *sfi)
 
 typedef struct TermEnumWrapper
 {
-    int base;
+    int index;
     TermEnum *te;
     int *doc_map;
     IndexReader *ir;
@@ -1567,13 +1567,16 @@ typedef struct MultiTermEnum
     TermEnumWrapper *tews;
     int size;
     int **field_num_map;
+    int ti_cnt;
+    TermInfo *tis;
+    int *ti_indexes;
 } MultiTermEnum;
 
 static bool tew_lt(const TermEnumWrapper *tew1, const TermEnumWrapper *tew2)
 {
     int cmpres = strcmp(tew1->term, tew2->term);
     if (cmpres == 0) {
-        return tew1->base < tew2->base;
+        return tew1->index < tew2->index;
     }
     else {
         return cmpres < 0;
@@ -1617,10 +1620,10 @@ static void tew_destroy(TermEnumWrapper *tew)
     tew->te->close(tew->te);
 }
 
-TermEnumWrapper *tew_setup(TermEnumWrapper *tew, int base, TermEnum *te,
+TermEnumWrapper *tew_setup(TermEnumWrapper *tew, int index, TermEnum *te,
                            IndexReader *ir)
 {
-    tew->base = base;
+    tew->index = index;
     tew->ir = ir;
     tew->te = te;
     tew->term = te->curr_term;
@@ -1646,9 +1649,12 @@ static char *mte_next(TermEnum *te)
 
     te->curr_ti.doc_freq = 0;
 
+    MTE(te)->ti_cnt = 0;
     while ((top != NULL) && (strcmp(te->curr_term, top->term) == 0)) {
         pq_pop(MTE(te)->tew_queue);
         te->curr_ti.doc_freq += top->te->curr_ti.doc_freq;/* increment freq */
+        MTE(te)->ti_indexes[MTE(te)->ti_cnt] = top->index;
+        MTE(te)->tis[MTE(te)->ti_cnt++] = top->te->curr_ti;
         if (tew_next(top)) {
             pq_push(MTE(te)->tew_queue, top); /* restore queue */
         }
@@ -1711,6 +1717,8 @@ static void mte_close(TermEnum *te)
         tew_destroy(&(MTE(te)->tews[i]));
     }
     free(MTE(te)->tews);
+    free(MTE(te)->tis);
+    free(MTE(te)->ti_indexes);
     pq_destroy(MTE(te)->tew_queue);
     free(te);
 }
@@ -1718,7 +1726,6 @@ static void mte_close(TermEnum *te)
 TermEnum *mte_new(MultiReader *mr, int field_num, const char *term)
 {
     IndexReader **readers   = mr->sub_readers;
-    int *starts             = mr->starts;
     int r_cnt               = mr->r_cnt;
     int i;
     IndexReader *reader;
@@ -1731,6 +1738,8 @@ TermEnum *mte_new(MultiReader *mr, int field_num, const char *term)
     TE(mte)->close      = &mte_close;
 
     mte->size           = r_cnt;
+    mte->tis            = ALLOC_AND_ZERO_N(TermInfo, r_cnt);
+    mte->ti_indexes     = ALLOC_AND_ZERO_N(int, r_cnt);
     mte->tews           = ALLOC_AND_ZERO_N(TermEnumWrapper, r_cnt);
     mte->tew_queue      = pq_new(r_cnt, (lt_ft)&tew_lt, (free_ft)NULL);
     mte->field_num_map  = mr->field_num_map;
@@ -1750,7 +1759,7 @@ TermEnum *mte_new(MultiReader *mr, int field_num, const char *term)
                 sub_te = reader->terms(reader, fnum);
             }
 
-            tew = tew_setup(&(mte->tews[i]), starts[i], sub_te, reader);
+            tew = tew_setup(&(mte->tews[i]), i, sub_te, reader);
             if (((term == NULL) && tew_next(tew))
                 || (tew->term && (tew->term[0] != '\0'))) {
                 pq_push(mte->tew_queue, tew);          /* initialize queue */
@@ -1759,7 +1768,7 @@ TermEnum *mte_new(MultiReader *mr, int field_num, const char *term)
             /* add the term_enum_wrapper just in case */
             sub_te = reader->terms(reader, 0);
             sub_te->field_num = -1;
-            tew_setup(&(mte->tews[i]), starts[i], sub_te, reader);
+            tew_setup(&(mte->tews[i]), i, sub_te, reader);
         }
     }
 
@@ -2386,45 +2395,29 @@ typedef struct MultiTermDocEnum
 {
     TermDocEnum tde;
     int *starts;
-    char *term;
-    int field_num;
     int base;
     int ptr;
     int ir_cnt;
-    int **field_num_map;
+    char *state;
+    TermEnum *te;
     IndexReader **irs;
     TermDocEnum **irs_tde;
     TermDocEnum *curr_tde;
-    TermDocEnum *(*reader_tde_i)(IndexReader *ir);
 } MultiTermDocEnum;
 
-static TermDocEnum *mtde_reader_tde_i(IndexReader *ir)
-{
-    return ir->term_docs(ir);
-}
-
-static TermDocEnum *mtde_get_tde_i(MultiTermDocEnum *mtde, int i)
+static TermDocEnum *mtde_next_tde(MultiTermDocEnum *mtde)
 {
-    if (mtde->term == NULL) {
-        return NULL;
+    mtde->ptr++;
+    while (mtde->ptr < mtde->ir_cnt && !mtde->state[mtde->ptr]) {
+        mtde->ptr++;
+    }
+    if (mtde->ptr >= mtde->ir_cnt) {
+        return mtde->curr_tde = NULL;
     }
     else {
-        int fnum = mtde->field_num_map
-            ? mtde->field_num_map[i][mtde->field_num]
-            : mtde->field_num;
-
-        if (fnum >= 0) {
-            TermDocEnum *tde = mtde->irs_tde[i];
-            if (tde == NULL) {
-                tde = mtde->irs_tde[i] = mtde->reader_tde_i(mtde->irs[i]);
-            }
-
-            tde->seek(tde, fnum, mtde->term);
-            return tde;
-        }
-        else {
-            return NULL;
-        }
+        TermDocEnum *tde = mtde->curr_tde = mtde->irs_tde[mtde->ptr];
+        mtde->base = mtde->starts[mtde->ptr];
+        return tde;
     }
 }
 
@@ -2435,30 +2428,35 @@ static TermDocEnum *mtde_get_tde_i(MultiTermDocEnum *mtde, int i)
     }\
 } while (0)
 
-static void mtde_seek(TermDocEnum *tde, int field_num, const char *term)
+static void mtde_seek_te(TermDocEnum *tde, TermEnum *te)
 {
+    int i;
     MultiTermDocEnum *mtde = MTDE(tde);
-    if (mtde->term != NULL) {
-        free(mtde->term);
+    memset(mtde->state, 0, mtde->ir_cnt);
+    for (i = MTE(te)->ti_cnt - 1; i >= 0; i--) {
+        int index = MTE(te)->ti_indexes[i];
+        TermDocEnum *tde = mtde->irs_tde[index];
+        mtde->state[index] = 1;
+        if (tde->close == stde_close) {
+            stde_seek_ti(STDE(tde), MTE(te)->tis + i);
+        } else if (tde->close == stpe_close) {
+            stpe_seek_ti(STDE(tde), MTE(te)->tis + i);
+        } else {
+            tde->seek(tde, MTE(te)->tews[index].te->field_num, te->curr_term);
+        }
     }
-    mtde->term = estrdup(term);
-    mtde->field_num = field_num;
     mtde->base = 0;
-    mtde->ptr = 0;
-    mtde->curr_tde = NULL;
+    mtde->ptr = -1;
+    mtde_next_tde(mtde);
 }
 
-static void mtde_seek_te(TermDocEnum *tde, TermEnum *te)
+static void mtde_seek(TermDocEnum *tde, int field_num, const char *term)
 {
     MultiTermDocEnum *mtde = MTDE(tde);
-    if (mtde->term != NULL) {
-        free(mtde->term);
-    }
-    mtde->term = estrdup(te->curr_term);
-    mtde->field_num = te->field_num;
-    mtde->base = 0;
-    mtde->ptr = 0;
-    mtde->curr_tde = NULL;
+    TermEnum *te = mtde->te;
+    te->set_field(te, field_num);
+    te->skip_to(te, term);
+    mtde_seek_te(tde, te);
 }
 
 static int mtde_doc_num(TermDocEnum *tde)
@@ -2479,10 +2477,7 @@ static bool mtde_next(TermDocEnum *tde)
     if (mtde->curr_tde != NULL && mtde->curr_tde->next(mtde->curr_tde)) {
         return true;
     }
-    else if (mtde->ptr < mtde->ir_cnt) {
-        mtde->base = mtde->starts[mtde->ptr];
-        mtde->curr_tde = mtde_get_tde_i(mtde, mtde->ptr);
-        mtde->ptr++;
+    else if (mtde_next_tde(mtde)) {
         return mtde_next(tde);
     }
     else {
@@ -2495,19 +2490,11 @@ static int mtde_read(TermDocEnum *tde, int *docs, int *freqs, int req_num)
     int i, end = 0, last_end = 0, b;
     MultiTermDocEnum *mtde = MTDE(tde);
     while (true) {
-        while (mtde->curr_tde == NULL) {
-            if (mtde->ptr < mtde->ir_cnt) { /* try next segment */
-                mtde->base = mtde->starts[mtde->ptr];
-                mtde->curr_tde = mtde_get_tde_i(mtde, mtde->ptr++);
-            }
-            else {
-                return end;
-            }
-        }
+        if (mtde->curr_tde == NULL) return end;
         end += mtde->curr_tde->read(mtde->curr_tde, docs + last_end,
                                     freqs + last_end, req_num - last_end);
         if (end == last_end) {              /* none left in segment */
-            mtde->curr_tde = NULL;
+            if (!mtde_next_tde(mtde)) return end;
         }
         else {                            /* got some */
             b = mtde->base;                 /* adjust doc numbers */
@@ -2528,19 +2515,15 @@ static bool mtde_skip_to(TermDocEnum *tde, int target_doc_num)
 {
     MultiTermDocEnum *mtde = MTDE(tde);
     TermDocEnum *curr_tde;
-    while (mtde->ptr < mtde->ir_cnt) {
-        curr_tde = mtde->curr_tde;
-        if (curr_tde && (target_doc_num < mtde->starts[mtde->ptr]) &&
+    while (NULL != (curr_tde = mtde->curr_tde)) {
+        if (target_doc_num < mtde->starts[mtde->ptr + 1] &&
             (curr_tde->skip_to(curr_tde, target_doc_num - mtde->base))) {
             return true;
         }
 
-        mtde->base = mtde->starts[mtde->ptr];
-        mtde->curr_tde = mtde_get_tde_i(mtde, mtde->ptr);
-        mtde->ptr++;
+        mtde_next_tde(mtde);
     }
 
-    curr_tde = mtde->curr_tde;
     if (curr_tde) {
         return curr_tde->skip_to(curr_tde, target_doc_num - mtde->base);
     }
@@ -2554,20 +2537,18 @@ static void mtde_close(TermDocEnum *tde)
     MultiTermDocEnum *mtde = MTDE(tde);
     TermDocEnum *tmp_tde;
     int i = mtde->ir_cnt;
+    mtde->te->close(mtde->te);
     while (i > 0) {
         i--;
-        if ((tmp_tde = mtde->irs_tde[i]) != NULL) {
-            tmp_tde->close(tmp_tde);
-        }
-    }
-    if (mtde->term != NULL) {
-        free(mtde->term);
+        tmp_tde = mtde->irs_tde[i];
+        tmp_tde->close(tmp_tde);
     }
     free(mtde->irs_tde);
+    free(mtde->state);
     free(tde);
 }
 
-TermDocEnum *mtde_new(MultiReader *mr)
+TermDocEnum *mtxe_new(MultiReader *mr)
 {
     MultiTermDocEnum *mtde  = ALLOC_AND_ZERO(MultiTermDocEnum);
     TermDocEnum *tde        = TDE(mtde);
@@ -2578,28 +2559,34 @@ TermDocEnum *mtde_new(MultiReader *mr)
     tde->next               = &mtde_next;
     tde->read               = &mtde_read;
     tde->skip_to            = &mtde_skip_to;
-    tde->next_position      = NULL;
     tde->close              = &mtde_close;
 
+    mtde->state             = ALLOC_AND_ZERO_N(char, mr->r_cnt);
+    mtde->te                = ((IndexReader *)mr)->terms((IndexReader *)mr, 0);
     mtde->starts            = mr->starts;
     mtde->ir_cnt            = mr->r_cnt;
     mtde->irs               = mr->sub_readers;
-    mtde->field_num_map     = mr->field_num_map;
     mtde->irs_tde           = ALLOC_AND_ZERO_N(TermDocEnum *, mr->r_cnt);
-    mtde->reader_tde_i      = &mtde_reader_tde_i;
 
     return tde;
 }
 
+TermDocEnum *mtde_new(MultiReader *mr)
+{
+    int i;
+    TermDocEnum *tde        = mtxe_new(mr);
+    tde->next_position      = NULL;
+    for (i = mr->r_cnt - 1; i >= 0; i--) {
+        IndexReader *ir = mr->sub_readers[i];
+        MTDE(tde)->irs_tde[i] = ir->term_docs(ir);
+    }
+    return tde;
+}
+
 /****************************************************************************
  * MultiTermPosEnum
  ****************************************************************************/
 
-TermDocEnum *mtpe_reader_tde_i(IndexReader *ir)
-{
-    return ir->term_positions(ir);
-}
-
 int mtpe_next_position(TermDocEnum *tde)
 {
     CHECK_CURR_TDE("next_position");
@@ -2608,9 +2595,13 @@ int mtpe_next_position(TermDocEnum *tde)
 
 TermDocEnum *mtpe_new(MultiReader *mr)
 {
-    TermDocEnum *tde        = mtde_new(mr);
+    int i;
+    TermDocEnum *tde        = mtxe_new(mr);
     tde->next_position      = &mtpe_next_position;
-    MTDE(tde)->reader_tde_i = &mtpe_reader_tde_i;
+    for (i = mr->r_cnt - 1; i >= 0; i--) {
+        IndexReader *ir = mr->sub_readers[i];
+        MTDE(tde)->irs_tde[i] = ir->term_positions(ir);
+    }
     return tde;
 }
 

data/ext/index.h

@@ -378,6 +378,7 @@ struct TermDocEnum
 {
     void (*seek)(TermDocEnum *tde, int field_num, const char *term);
     void (*seek_te)(TermDocEnum *tde, TermEnum *te);
+    void (*seek_ti)(TermDocEnum *tde, TermInfo *ti);
     int  (*doc_num)(TermDocEnum *tde);
     int  (*freq)(TermDocEnum *tde);
     bool (*next)(TermDocEnum *tde);

data/ext/q_fuzzy.c

@@ -264,5 +264,5 @@ Query *fuzq_new_conf(const char *field, const char *term,
 
 Query *fuzq_new(const char *field, const char *term)
 {
-    return fuzq_new_conf(term, field, 0.0f, 0, 0);
+    return fuzq_new_conf(field, term, 0.0f, 0, 0);
 }

data/ext/r_index.c

@@ -564,6 +564,19 @@ frt_fis_to_s(VALUE self)
     free(fis_s);
     return rfis_s;
 }
+
+/*
+ *  call-seq:
+ *     fis.size -> int
+ *     
+ *  Return the number of fields in the FieldInfos object.
+ */
+static VALUE
+frt_fis_size(VALUE self)
+{
+    FieldInfos *fis = (FieldInfos *)DATA_PTR(self);
+    return INT2FIX(fis->size);
+}
  
 /*
  *  call-seq:
@@ -2225,7 +2238,7 @@ frt_ir_get_doc(int argc, VALUE *argv, VALUE self)
             pos = (pos < 0) ? (max + pos) : pos;
             if (pos < 0 || pos >= max) {
                 rb_raise(rb_eArgError, ":%d is out of range [%d..%d] for "
-                         "IndexWriter#[]", pos, 0, max,
+                         "IndexReader#[]", pos, 0, max,
                          rb_id2name(SYM2ID(argv)));
             }
             return frt_get_lazy_doc(ir->get_lazy_doc(ir, pos));
@@ -2425,6 +2438,25 @@ frt_ir_terms_from(VALUE self, VALUE rfield, VALUE rterm)
                                           StringValuePtr(rterm)));
 }
 
+/*
+ *  call-seq:
+ *     index_reader.term_count(field) -> int
+ *
+ *  Same return a count of the number of terms in the field
+ */
+static VALUE
+frt_ir_term_count(VALUE self, VALUE rfield)
+{
+    IndexReader *ir = (IndexReader *)DATA_PTR(self);
+    TermEnum *te = ir_terms(ir, frt_field(rfield));
+    int count = 0;
+    while (te->next(te)) {
+        count++;
+    }
+    te->close(te);
+    return INT2FIX(count);
+}
+
 /*
  *  call-seq:
  *     index_reader.fields -> array of field-names
@@ -2483,6 +2515,19 @@ frt_ir_tk_fields(VALUE self)
     return rfield_names;
 }
 
+/*
+ *  call-seq:
+ *     index_reader.version -> int
+ *
+ *  Returns the current version of the index reader.
+ */
+static VALUE
+frt_ir_version(VALUE self)
+{
+    IndexReader *ir = (IndexReader *)DATA_PTR(self);
+    return INT2FIX(ir->sis->version);
+}
+
 /****************************************************************************
  *
  * Init Functions
@@ -2708,6 +2753,7 @@ Init_FieldInfos(void)
     rb_define_method(cFieldInfos, "add_field",  frt_fis_add_field, -1);
     rb_define_method(cFieldInfos, "each",       frt_fis_each, 0);
     rb_define_method(cFieldInfos, "to_s",       frt_fis_to_s, 0);
+    rb_define_method(cFieldInfos, "size",       frt_fis_size, 0);
     rb_define_method(cFieldInfos, "create_index",
                                                 frt_fis_create_index, 1);
     rb_define_method(cFieldInfos, "fields",     frt_fis_get_fields, 0);
@@ -3188,6 +3234,7 @@ Init_IndexReader(void)
 {
     cIndexReader = rb_define_class_under(mIndex, "IndexReader", rb_cObject);
     rb_define_alloc_func(cIndexReader, frt_data_alloc);
+    /*rb_define_singleton_method(cIndexReader, "version", frt_class_ir_version, 0); */
     rb_define_method(cIndexReader, "initialize",    frt_ir_init, 1);
     rb_define_method(cIndexReader, "set_norm",      frt_ir_set_norm, 3);
     rb_define_method(cIndexReader, "norms",         frt_ir_norms, 1);
@@ -3212,10 +3259,12 @@ Init_IndexReader(void)
     rb_define_method(cIndexReader, "doc_freq",      frt_ir_doc_freq, 2);
     rb_define_method(cIndexReader, "terms",         frt_ir_terms, 1);
     rb_define_method(cIndexReader, "terms_from",    frt_ir_terms_from, 2);
+    rb_define_method(cIndexReader, "term_count",    frt_ir_term_count, 1);
     rb_define_method(cIndexReader, "fields",        frt_ir_fields, 0);
     rb_define_method(cIndexReader, "field_names",   frt_ir_fields, 0);
     rb_define_method(cIndexReader, "field_infos",   frt_ir_field_infos, 0);
     rb_define_method(cIndexReader, "tokenized_fields", frt_ir_tk_fields, 0);
+    rb_define_method(cIndexReader, "version",       frt_ir_version, 0);
 }
 
 /* rdoc hack

data/ext/r_search.c

@@ -104,6 +104,7 @@ static ID id_score;
 static ID id_hits;
 static ID id_total_hits;
 static ID id_max_score;
+static ID id_searcher;
 
 /* Search */
 static VALUE sym_offset;
@@ -152,7 +153,7 @@ frt_get_hit(Hit *hit)
  ****************************************************************************/
 
 static VALUE
-frt_get_td(TopDocs *td)
+frt_get_td(TopDocs *td, VALUE rsearcher)
 {
     int i;
     VALUE rtop_docs;
@@ -167,6 +168,7 @@ frt_get_td(TopDocs *td)
                               INT2FIX(td->total_hits),
                               hit_ary,
                               rb_float_new((double)td->max_score),
+                              rsearcher,
                               NULL);
     td_destroy(td);
     return rtop_docs;
@@ -174,20 +176,26 @@ frt_get_td(TopDocs *td)
 
 /*
  *  call-seq:
- *     top_doc.to_s -> string
+ *     top_doc.to_s(field = :id) -> string
  *
  *  Returns a string represention of the top_doc in readable format.
  */
 static VALUE
-frt_td_to_s(VALUE self)
+frt_td_to_s(int argc, VALUE *argv, VALUE self)
 {
     int i;
     VALUE rhits = rb_funcall(self, id_hits, 0);
+    Searcher *sea = (Searcher *)DATA_PTR(rb_funcall(self, id_searcher, 0));
     const int len = RARRAY(rhits)->len;
     char *str = ALLOC_N(char, len * 64 + 100);
     char *s = str;
+    char *field = "id";
     VALUE rstr;
 
+    if (argc) {
+        field = frt_field(argv[0]);
+    }
+
     sprintf(s, "TopDocs: total_hits = %d, max_score = %f [\n",
             FIX2INT(rb_funcall(self, id_total_hits, 0)),
             NUM2DBL(rb_funcall(self, id_max_score, 0)));
@@ -195,10 +203,18 @@ frt_td_to_s(VALUE self)
 
     for (i = 0; i < len; i++) {
         VALUE rhit = RARRAY(rhits)->ptr[i];
-        sprintf(s, "\t%d: %f\n",
-                FIX2INT(rb_funcall(rhit, id_doc, 0)),
+        int doc_id = FIX2INT(rb_funcall(rhit, id_doc, 0));
+        char *value = "";
+        LazyDoc *lzd = sea->get_lazy_doc(sea, doc_id);
+        LazyDocField *lzdf = h_get(lzd->field_dict, field);
+        if (NULL != lzdf) {
+            value = lazy_df_get_data(lzdf, 0);
+        }
+
+        sprintf(s, "\t%d \"%s\": %f\n", doc_id, value,
                 NUM2DBL(rb_funcall(rhit, id_score, 0)));
         s += strlen(s);
+        lazy_doc_close(lzd);
     }
 
     sprintf(s, "]\n");
@@ -2388,7 +2404,7 @@ frt_sea_search(int argc, VALUE *argv, VALUE self)
     Query *query;
     rb_scan_args(argc, argv, "11", &rquery, &roptions);
     Data_Get_Struct(rquery, Query, query);
-    return frt_get_td(frt_sea_search_internal(query, roptions, sea));
+    return frt_get_td(frt_sea_search_internal(query, roptions, sea), self);
 }
 
 /*
@@ -2760,13 +2776,15 @@ Init_TopDocs(void)
                                 "total_hits",
                                 "hits",
                                 "max_score",
+                                "searcher",
                                 NULL);
     rb_set_class_path(cTopDocs, mSearch, td_class);
     rb_const_set(mSearch, rb_intern(td_class), cTopDocs);
-    rb_define_method(cTopDocs, "to_s", frt_td_to_s, 0);
+    rb_define_method(cTopDocs, "to_s", frt_td_to_s, -1);
     id_hits = rb_intern("hits");
     id_total_hits = rb_intern("total_hits");
     id_max_score = rb_intern("max_score");
+    id_searcher = rb_intern("searcher");
 }
 
 /*

data/ext/search.c

@@ -122,11 +122,12 @@ static void hit_pq_down(PriorityQueue *pq)
 static Hit *hit_pq_pop(PriorityQueue *pq)
 {
     if (pq->size > 0) {
-        Hit *result = (Hit *)pq->heap[1]; /* save first value */
-        pq->heap[1] = pq->heap[pq->size]; /* move last to first */
-        pq->heap[pq->size] = NULL;
+        Hit **heap = (Hit **)pq->heap;
+        Hit *result = heap[1];    /* save first value */
+        heap[1] = heap[pq->size]; /* move last to first */
+        heap[pq->size] = NULL;
         pq->size--;
-        hit_pq_down(pq);                  /* adjust heap */
+        hit_pq_down(pq);          /* adjust heap */
         return result;
     }
     else {
@@ -1079,8 +1080,8 @@ static TopDocs *isea_search_w(Searcher *self,
         for (i = num_docs - 1; i >= 0; i--) {
             score_docs[i] = hq_pop(hq);
             /*
-            hit = score_docs[i] = pq_pop(hq);
-            printf("hit = %d-->%f\n", hit->doc, hit->score);
+            printf("score_docs[i][%d] = [%ld] => %d-->%f\n", i,
+                   score_docs[i], score_docs[i]->doc, score_docs[i]->score);
             */
         }
     }

data/ext/sort.c

@@ -426,8 +426,8 @@ int sf_string_compare(void *index, Hit *hit1, Hit *hit2)
     char *s2 = ((StringIndex *)index)->values[
         ((StringIndex *)index)->index[hit2->doc]];
 
-    if (s1 == NULL) return s1 ? -1 : 0;
-    if (s2 == NULL) return 1;
+    if (s1 == NULL) return s2 ? 1 : 0;
+    if (s2 == NULL) return -1;
 
 #ifdef POSH_OS_WIN32
     return strcmp(s1, s2);
@@ -874,8 +874,8 @@ bool fdshq_lt(FieldDoc *fd1, FieldDoc *fd2)
                 do {
                     char *s1 = cmps1[i].val.s;
                     char *s2 = cmps2[i].val.s;
-                    if (s1 == NULL) c = s2 ? -1 : 0;
-                    else if (s2 == NULL) c = 1;
+                    if (s1 == NULL) c = s2 ? 1 : 0;
+                    else if (s2 == NULL) c = -1;
 #ifdef POSH_OS_WIN32
                     else c = strcmp(s1, s2);
 #else

data/lib/ferret/index.rb

@@ -179,11 +179,13 @@ module Ferret::Index
     #                    Alternatively you may want to use the HTML entity
     #                    … or the UTF-8 string "\342\200\246".
     def highlight(query, doc_id, options = {})
-      ensure_searcher_open()
-      @searcher.highlight(do_process_query(query),
-                          doc_id,
-                          options[:field]||@options[:default_field],
-                          options)
+      @dir.synchronize do
+        ensure_searcher_open()
+        @searcher.highlight(do_process_query(query),
+                            doc_id,
+                            options[:field]||@options[:default_field],
+                            options)
+      end
     end
 
     # Closes this index by closing its associated reader and writer objects.
@@ -273,9 +275,14 @@ module Ferret::Index
         end
         ensure_writer_open()
 
-        old_analyzer = @writer.analyzer if analyzer
-        @writer.add_document(doc)
-        @writer.analyzer = old_analyzer if analyzer
+        if analyzer
+          old_analyzer = @writer.analyzer
+          @writer.analyzer = analyzer
+          @writer.add_document(doc)
+          @writer.analyzer = old_analyzer
+        else
+          @writer.add_document(doc)
+        end
 
         flush() if @auto_flush
       end

data/lib/ferret_version.rb

@@ -1,3 +1,3 @@
 module Ferret
-  VERSION = '0.10.9'
+  VERSION = '0.10.10'
 end

data/test/threading/number_to_spoken.rb

@@ -0,0 +1,132 @@
+# Author: Matthew D Moss
+#
+# Writtern for ruby quiz #25
+#
+class JapaneseTranslator
+    # My knowledge of counting Japanese is limited, so this may not
+    # be entirely correct; in particular, I don't know what rules
+    # to follow after 'hyaku man' (1,000,000).
+    # I also combine a digit with its group, such as 'gohyaku' rather
+    # than 'go hyaku'; I just like reading it better that way.
+
+    DIGITS = %w(zero ichi ni san yon go roku nana hachi kyu)
+    GROUPS = %w(nothingtoseeheremovealong ju hyaku sen)
+    MAN = 10000
+
+    def to_spoken(val)
+        case val <=> 0
+        when -1
+            '- ' + to_spoken(-val)
+        when 0
+            DIGITS[0]
+        else
+            group(val, 0)
+        end
+    end
+
+    private
+
+    def group(val, level)
+        if val >= MAN
+            group(val / MAN, 0) + 'man ' + group(val % MAN, 0)
+        else
+            case val
+            when 0
+                ''
+            when 1
+                level == 0 ? DIGITS[val] : GROUPS[level]
+            when 2...10
+                DIGITS[val] + (GROUPS[level] if level > 0).to_s
+            else
+                group(val / 10, level+1) + ' ' + group(val % 10, level)
+            end
+        end
+    end
+end
+
+
+class USEnglishTranslator
+    # Formal, US English. Optional 'and'. Will not produce things
+    # such as 'twelve hundred' but rather 'one thousand two hundred'.
+    # The use of 'and' is incomplete; it is sometimes missed.
+
+    DIGITS = %w(zero one two three four five six seven eight nine)
+    TEENS  = %w(ten eleven twelve thirteen fourteen fifteen sixteen
+                seventeen eighteen nineteen)
+    TENS   = %w(hello world twenty thirty forty fifty sixty seventy
+                eighty ninety)
+    GROUPS = %w(thousand million billion trillion quadrillion
+                quintillion sextillion septillion octillion nonillion
+                decillion)
+    K = 1000
+
+    def initialize(conjunction = true)
+        @conjunction = conjunction
+    end
+
+    def to_spoken(val)
+        case val <=> 0
+        when -1
+            'negative ' + to_spoken(-val)
+        when 0
+            DIGITS[0]
+        else
+            group(val, 0).flatten.join(' ')
+        end
+    end
+
+    private
+
+    def group(val, level)
+        x = group(val / K, level + 1) << GROUPS[level] if val >= K
+        x.to_a << under_1000(val % K, level)
+    end
+
+    def under_1000(val, level)
+        x = [DIGITS[val / 100]] << 'hundred' if val >= 100
+        x.to_a << under_100(val % 100, (level == 0 and not x.nil?))
+    end
+
+    def under_100(val, junction)
+        x = [('and' if @conjunction and junction)]    # wyf?
+        case val
+        when 0
+            []
+        when 1...10
+            x << DIGITS[val]
+        when 10...20
+            x << TEENS[val - 10]
+        else
+            d = val % 10
+            x << (TENS[val / 10] + ('-' + DIGITS[d] if d != 0).to_s)
+        end
+    end
+end
+
+
+class Integer
+    def to_spoken(translator = USEnglishTranslator.new)
+        translator.to_spoken(self).squeeze(' ').strip
+    end
+end
+
+if $0 == __FILE__
+    SAMPLES = [ 0, 1, 2, 5, 10, 11, 14, 18, 20, 21, 29, 33, 42, 50, 87, 99,
+                100, 101, 110, 167, 199, 200, 201, 276, 300, 314, 500, 610,
+                1000, 1039, 1347, 2309, 3098, 23501, 32767, 70000, 5480283,
+                2435489238, 234100090000, -42, -2001 ]
+
+    TRANSLATORS = { 'US English' => USEnglishTranslator.new,
+                    'Japanese'   => JapaneseTranslator.new }
+
+
+    # main
+    TRANSLATORS.each do |lang, translator|
+        puts
+        puts lang
+        puts '-' * lang.length
+        SAMPLES.each do |val|
+            puts "%12d => %s" % [val, val.to_spoken(translator)]
+        end
+    end
+end

data/test/threading/thread_safety_index_test.rb

@@ -0,0 +1,78 @@
+require File.dirname(__FILE__) + "/../test_helper"
+require File.dirname(__FILE__) + "/number_to_spoken.rb"
+require 'thread'
+
+class IndexThreadSafetyTest < Test::Unit::TestCase
+  include Ferret::Index
+
+  INDEX_DIR = File.expand_path(File.join(File.dirname(__FILE__), "index"))
+  ITERATIONS = 1000
+  NUM_THREADS = 2
+  ANALYZER = Ferret::Analysis::StandardAnalyzer.new()
+
+  def setup
+    index = Index.new(:path => INDEX_DIR,
+                      :create => true,
+                      :analyzer => ANALYZER,
+                      :default_field => :content)
+    index.close
+  end
+
+  def indexing_thread()
+    index = Index.new(:path => INDEX_DIR,
+                      :analyzer => ANALYZER,
+                      :default_field => :content)
+
+    ITERATIONS.times do
+      choice = rand()
+
+      if choice > 0.98
+        do_optimize(index)
+      elsif choice > 0.7
+        do_delete_doc(index)
+      elsif choice > 0.5
+        do_search(index)
+      else
+        do_add_doc(index)
+      end
+    end
+  end 
+
+  def do_optimize(index)
+    puts "Optimizing the index"
+    index.optimize
+  end
+
+  def do_delete_doc(index)
+    return if index.size == 0
+    doc_num = rand(index.size)
+    puts "Deleting #{doc_num} from index which has#{index.has_deletions? ? "" : " no"} deletions"
+    puts "document was already deleted" if (index.deleted?(doc_num))
+    index.delete(doc_num)
+  end
+
+  def do_add_doc(index)
+    n = rand(0xFFFFFFFF)
+    d = {:id => n, :content => n.to_spoken}
+    puts("Adding #{n}")
+    index << d
+  end
+  
+  def do_search(index)
+    n = rand(0xFFFFFFFF)
+    puts("Searching for #{n}")
+    hits = index.search_each(n.to_spoken, :num_docs => 3) do |d, s|
+      puts "Hit for #{n}: #{index[d][:id]} - #{s}"
+    end
+    puts("Searched for #{n}: total = #{hits}")
+  end
+
+  def test_threading
+    threads = []
+    NUM_THREADS.times do
+      threads << Thread.new { indexing_thread }
+    end
+
+    threads.each {|t| t.join}
+  end
+end

data/test/threading/thread_safety_test.rb

@@ -0,0 +1,137 @@
+require File.dirname(__FILE__) + "/../test_helper"
+require File.dirname(__FILE__) + "/../utils/number_to_spoken.rb"
+require 'thread'
+
+class ThreadSafetyTest
+  include Ferret::Index
+  include Ferret::Search
+  include Ferret::Store
+  include Ferret::Document
+
+  def initialize(options)
+    @options = options
+  end
+
+  INDEX_DIR = File.expand_path(File.join(File.dirname(__FILE__), "index"))
+  ANALYZER = Ferret::Analysis::Analyzer.new()
+  ITERATIONS = 19
+  @@searcher = nil
+  
+  def run_index_thread(writer)
+    reopen_interval = 30 + rand(60)
+
+    use_compound_file = false
+    
+    (400*ITERATIONS).times do |i|
+      d = Document.new()
+      n = rand(0xFFFFFFFF)
+      d << Field.new("id", n.to_s, Field::Store::YES, Field::Index::UNTOKENIZED)
+      d << Field.new("contents", n.to_spoken, Field::Store::NO, Field::Index::TOKENIZED)
+      puts("Adding #{n}")
+      
+      # Switch between single and multiple file segments
+      use_compound_file = (rand < 0.5)
+      writer.use_compound_file = use_compound_file
+      
+      writer << d
+
+      if (i % reopen_interval == 0) 
+        writer.close()
+        writer = IndexWriter.new(INDEX_DIR, :analyzer => ANALYZER)
+      end
+    end
+    
+    writer.close()
+  rescue => e
+    puts e
+    puts e.backtrace
+    raise e
+  end
+
+  def run_search_thread(use_global)
+    reopen_interval = 10 + rand(20)
+
+    unless use_global
+      searcher = IndexSearcher.new(INDEX_DIR)
+    end
+
+    (50*ITERATIONS).times do |i|
+      search_for(rand(0xFFFFFFFF), (searcher.nil? ? @@searcher : searcher))
+      if (i%reopen_interval == 0) 
+        if (searcher == nil) 
+          @@searcher = IndexSearcher.new(INDEX_DIR)
+        else 
+          searcher.close()
+          searcher = IndexSearcher.new(INDEX_DIR)
+        end
+      end
+    end
+  rescue => e
+    puts e
+    puts e.backtrace
+    raise e
+  end
+
+  def search_for(n, searcher)
+    puts("Searching for #{n}")
+    hits =
+      searcher.search(Ferret::QueryParser.parse(n.to_spoken, "contents", :analyzer => ANALYZER),
+                      :num_docs => 3)
+    puts("Search for #{n}: total = #{hits.size}")
+    hits.each do |d, s|
+      puts "Hit for #{n}: #{searcher.reader.get_document(d)["id"]} - #{s}"
+    end
+  end
+
+  def run_test_threads
+
+    threads = []
+    unless @options[:read_only]
+      writer = IndexWriter.new(INDEX_DIR, :analyzer => ANALYZER,
+                               :create => !@options[:add])
+      
+      threads << Thread.new { run_index_thread(writer) }
+      
+      sleep(1)
+    end
+      
+    threads << Thread.new { run_search_thread(false)}
+
+    @@searcher = IndexSearcher.new(INDEX_DIR) 
+    threads << Thread.new { run_search_thread(true)}
+
+    threads << Thread.new { run_search_thread(true)}
+
+    threads.each {|t| t.join}
+  end
+end
+
+
+if $0 == __FILE__
+  require 'optparse'
+
+  OPTIONS = {
+    :all        => false,
+    :read_only  => false,
+  }
+
+  ARGV.options do |opts|
+    script_name = File.basename($0)
+    opts.banner = "Usage: ruby #{script_name} [options]"
+
+    opts.separator ""
+
+    opts.on("-r", "--read-only", "Read Only.") { OPTIONS[:all] = true }
+    opts.on("-a", "--all", "All.") { OPTIONS[:read_only] = true }
+
+    opts.separator ""
+
+    opts.on("-h", "--help",
+            "Show this help message.") { puts opts; exit }
+
+    opts.parse!
+  end
+
+  tst = ThreadSafetyTest.new(OPTIONS)
+  tst.run_test_threads
+end

data/test/unit/index/tc_index.rb

@@ -766,4 +766,12 @@ class IndexTest < Test::Unit::TestCase
 
     index.close
   end
+
+  def test_changing_analyzer
+    index = Ferret::I.new
+    a = Ferret::Analysis::WhiteSpaceAnalyzer.new(false)
+    index.add_document({:content => "Content With Capitals"}, a)
+    tv = index.reader.term_vector(0, :content)
+    assert_equal("Capitals", tv.terms[0].text)
+  end
 end

data/test/unit/search/tc_search_and_sort.rb

@@ -16,8 +16,8 @@ class SearchAndSortTest < Test::Unit::TestCase
       {:x => "findall", :string => "c", :int => "5", :float => "0.1"},     #  3   3
       {:x => "findall", :string => "e", :int => "2", :float => "0.001"},   #  5   1
       {:x => "findall", :string => "g", :int => "1", :float => "1.0"},     #  3   3
-      {:x => "findall", :string => "i", :int => "3", :float => "0.0001"},  #  6   2
-      {:x => "findall", :string => "j", :int => "4", :float => "10.0"},    #  4   0
+      {:x => "findall", :string => nil, :int => "3", :float => "0.0001"},  #  6   2
+      {:x => "findall", :string => "",  :int => "4", :float => "10.0"},    #  4   0
       {:x => "findall", :string => "h", :int => "5", :float => "0.00001"}, #  7   3
       {:x => "findall", :string => "f", :int => "2", :float => "100.0"},   #  5   1
       {:x => "findall", :string => "d", :int => "3", :float => "1000.0"},  #  6   2
@@ -145,7 +145,7 @@ class SearchAndSortTest < Test::Unit::TestCase
 
     ## str
     sf_str = SortField.new(:string, {:type => :string})
-    do_test_top_docs(is, q, [0,9,1,8,2,7,3,6,4,5], [sf_str, SortField::SCORE])
+    do_test_top_docs(is, q, [0,9,1,8,2,7,3,6,5,4], [sf_str, SortField::SCORE])
     do_test_top_docs(is, q, [0,9,1,8,2,7,3,6,4,5], "string")
 
     ## auto

metadata

@@ -3,8 +3,8 @@ rubygems_version: 0.9.0
 specification_version: 1
 name: ferret
 version: !ruby/object:Gem::Version 
-  version: 0.10.9
-date: 2006-09-27 00:00:00 +09:00
+  version: 0.10.10
+date: 2006-10-08 00:00:00 +09:00
 summary: Ruby indexing library.
 require_paths: 
 - lib
@@ -198,6 +198,9 @@ files:
 - test/unit/search/tc_search_and_sort.rb
 - test/unit/search/tm_searcher.rb
 - test/unit/query_parser/tc_query_parser.rb
+- test/threading/thread_safety_index_test.rb
+- test/threading/thread_safety_test.rb
+- test/threading/number_to_spoken.rb
 test_files: []
 
 rdoc_options: