ferret 0.11.3 → 0.11.4

data/ext/q_span.c

@@ -2340,7 +2340,7 @@ static Query *spanprq_rewrite(Query *self, IndexReader *ir)
 {
     const char *field = SpQ(self)->field;
     const int field_num = fis_get_field_num(ir->fis, field);
-    Query *volatile q = spanmtq_new_conf(field, SPAN_PREFIX_QUERY_MAX_TERMS);
+    Query *volatile q = spanmtq_new_conf(field, SpPfxQ(self)->max_terms);
     q->boost = self->boost;        /* set the boost */
 
     if (field_num >= 0) {
@@ -2388,6 +2388,7 @@ Query *spanprq_new(const char *field, const char *prefix)
 
     SpQ(self)->field        = estrdup(field);
     SpPfxQ(self)->prefix    = estrdup(prefix);
+    SpPfxQ(self)->max_terms = SPAN_PREFIX_QUERY_MAX_TERMS;
 
     self->type              = SPAN_PREFIX_QUERY;
     self->rewrite           = &spanprq_rewrite;

data/ext/r_analysis.c

@@ -560,7 +560,6 @@ static TokenStream *
 cwrts_reset(TokenStream *ts, char *text)
 {
     ts->t = ts->text = text;
-    Xj
     rb_funcall(CWTS(ts)->rts, id_reset, 1, rb_str_new2(text));
     return ts;
 }
@@ -820,7 +819,9 @@ static VALUE
 frt_letter_tokenizer_init(int argc, VALUE *argv, VALUE self) 
 {
     TS_ARGS(false);
+#ifndef POSH_OS_WIN32
     if (!frt_locale) frt_locale = setlocale(LC_CTYPE, "");
+#endif
     return get_wrapped_ts(self, rstr, mb_letter_tokenizer_new(lower));
 }
 
@@ -849,7 +850,9 @@ static VALUE
 frt_whitespace_tokenizer_init(int argc, VALUE *argv, VALUE self) 
 {
     TS_ARGS(false);
+#ifndef POSH_OS_WIN32
     if (!frt_locale) frt_locale = setlocale(LC_CTYPE, "");
+#endif
     return get_wrapped_ts(self, rstr, mb_whitespace_tokenizer_new(lower));
 }
 
@@ -877,7 +880,9 @@ frt_a_standard_tokenizer_init(VALUE self, VALUE rstr)
 static VALUE
 frt_standard_tokenizer_init(VALUE self, VALUE rstr) 
 {
+#ifndef POSH_OS_WIN32
     if (!frt_locale) frt_locale = setlocale(LC_CTYPE, "");
+#endif
     return get_wrapped_ts(self, rstr, mb_standard_tokenizer_new());
 }
 
@@ -917,7 +922,9 @@ static VALUE
 frt_lowercase_filter_init(VALUE self, VALUE rsub_ts) 
 {
     TokenStream *ts = frt_get_cwrapped_rts(rsub_ts);
+#ifndef POSH_OS_WIN32
     if (!frt_locale) frt_locale = setlocale(LC_CTYPE, "");
+#endif
     ts = mb_lowercase_filter_new(ts);
     object_add(&(TkFilt(ts)->sub_ts), rsub_ts);
 
@@ -1257,7 +1264,9 @@ frt_white_space_analyzer_init(int argc, VALUE *argv, VALUE self)
 {
     Analyzer *a;
     GET_LOWER(false);
+#ifndef POSH_OS_WIN32
     if (!frt_locale) frt_locale = setlocale(LC_CTYPE, "");
+#endif
     a = mb_whitespace_analyzer_new(lower);
     Frt_Wrap_Struct(self, NULL, &frt_analyzer_free, a);
     object_add(a, self);
@@ -1300,7 +1309,9 @@ frt_letter_analyzer_init(int argc, VALUE *argv, VALUE self)
 {
     Analyzer *a;
     GET_LOWER(true);
+#ifndef POSH_OS_WIN32
     if (!frt_locale) frt_locale = setlocale(LC_CTYPE, "");
+#endif
     a = mb_letter_analyzer_new(lower);
     Frt_Wrap_Struct(self, NULL, &frt_analyzer_free, a);
     object_add(a, self);
@@ -1372,7 +1383,9 @@ frt_standard_analyzer_init(int argc, VALUE *argv, VALUE self)
     bool lower;
     VALUE rlower, rstop_words;
     Analyzer *a;
+#ifndef POSH_OS_WIN32
     if (!frt_locale) frt_locale = setlocale(LC_CTYPE, "");
+#endif
     rb_scan_args(argc, argv, "02", &rstop_words, &rlower);
     lower = ((rlower == Qnil) ? true : RTEST(rlower));
     if (rstop_words != Qnil) {

data/ext/r_index.c

@@ -196,6 +196,19 @@ frt_fi_init(int argc, VALUE *argv, VALUE self)
     return self;
 }
 
+/*
+ *  call-seq:
+ *     fi.name -> symbol
+ *
+ *  Return the name of the field
+ */
+static VALUE
+frt_fi_name(VALUE self)
+{
+    FieldInfo *fi = (FieldInfo *)DATA_PTR(self);
+    return ID2SYM(rb_intern(fi->name));
+}
+
 /*
  *  call-seq:
  *     fi.stored? -> bool
@@ -800,6 +813,86 @@ frt_te_set_field(VALUE self, VALUE rfield)
     return self;
 }
 
+/*
+ *  call-seq:
+ *     term_enum.to_json() -> string
+ *
+ *  Returns a JSON representation of the term enum. You can speed this up by
+ *  having the method return arrays instead of objects, simply by passing an
+ *  argument to the to_json method. For example;
+ *
+ *    term_enum.to_json() #=> 
+ *    # [
+ *    #   {"term":"apple","frequency":12},
+ *    #   {"term":"banana","frequency":2},
+ *    #   {"term":"cantaloupe","frequency":12}
+ *    # ]
+ *
+ *    term_enum.to_json(:fast) #=> 
+ *    # [
+ *    #   ["apple",12],
+ *    #   ["banana",2],
+ *    #   ["cantaloupe",12]
+ *    # ]
+ */
+static VALUE
+frt_te_to_json(int argc, VALUE *argv, VALUE self)
+{
+    TermEnum *te = (TermEnum *)DATA_PTR(self);
+    VALUE rjson;
+    char *json, *jp;
+    char *term;
+    int capa = 65536;
+    jp = json = ALLOC_N(char, capa);
+    *(jp++) = '[';
+
+    if (argc > 0) {
+        while (NULL != (term = te->next(te))) {
+            /* enough room for for term after converting " to '"' and frequency
+             * plus some extra for good measure */
+            *(jp++) = '[';
+            if (te->curr_term_len * 3 + (jp - json) + 100 > capa) {
+                capa <<= 1;
+                REALLOC_N(json, char, capa);
+            }
+            jp = json_concat_string(jp, term);
+            *(jp++) = ',';
+            sprintf(jp, "%d", te->curr_ti.doc_freq);
+            jp += strlen(jp);
+            *(jp++) = ']';
+            *(jp++) = ',';
+        }
+    }
+    else {
+        while (NULL != (term = te->next(te))) {
+            /* enough room for for term after converting " to '"' and frequency
+             * plus some extra for good measure */
+            if (te->curr_term_len * 3 + (jp - json) + 100 > capa) {
+                capa <<= 1;
+                REALLOC_N(json, char, capa);
+            }
+            *(jp++) = '{';
+            memcpy(jp, "\"term\":", 7);
+            jp += 7;
+            jp = json_concat_string(jp, term);
+            *(jp++) = ',';
+            memcpy(jp, "\"frequency\":", 12);
+            jp += 12;
+            sprintf(jp, "%d", te->curr_ti.doc_freq);
+            jp += strlen(jp);
+            *(jp++) = '}';
+            *(jp++) = ',';
+        }
+    }
+    if (*(jp-1) == ',') jp--;
+    *(jp++) = ']';
+    *jp = '\0';
+
+    rjson = rb_str_new2(json);
+    free(json);
+    return rjson;
+}
+
 /****************************************************************************
  *
  * TermDocEnum Methods
@@ -960,6 +1053,89 @@ frt_tde_each(VALUE self)
     return INT2FIX(doc_cnt);
 }
 
+/*
+ *  call-seq:
+ *     term_doc_enum.to_json() -> string
+ *
+ *  Returns a json representation of the term doc enum. It will also add the
+ *  term positions if they are available. You can speed this up by having the
+ *  method return arrays instead of objects, simply by passing an argument to
+ *  the to_json method. For example;
+ *
+ *    term_doc_enum.to_json() #=> 
+ *    # [
+ *    #   {"document":1,"frequency":12},
+ *    #   {"document":11,"frequency":1},
+ *    #   {"document":29,"frequency":120},
+ *    #   {"document":30,"frequency":3}
+ *    # ]
+ *
+ *    term_doc_enum.to_json(:fast) #=> 
+ *    # [
+ *    #   [1,12],
+ *    #   [11,1],
+ *    #   [29,120],
+ *    #   [30,3]
+ *    # ]
+ */
+static VALUE
+frt_tde_to_json(int argc, VALUE *argv, VALUE self)
+{
+    TermDocEnum *tde = (TermDocEnum *)DATA_PTR(self);
+    VALUE rjson;
+    char *json, *jp;
+    int capa = 65536;
+    char *format;
+    char close = (argc > 0) ? ']' : '}';
+    bool do_positions = tde->next_position != NULL;
+    jp = json = ALLOC_N(char, capa);
+    *(jp++) = '[';
+
+    if (do_positions) {
+        if (argc == 0) {
+            format = "{\"document\":%d,\"frequency\":%d,\"positions\":[";
+        }
+        else {
+            format = "[%d,%d,[";
+        }
+    }
+    else {
+        if (argc == 0) {
+            format = "{\"document\":%d,\"frequency\":%d},";
+        }
+        else {
+            format = "[%d,%d],";
+        }
+    }
+    while (tde->next(tde)) {
+        /* 100 chars should be enough room for an extra entry */
+        if ((jp - json) + 100 + tde->freq(tde) * 20 > capa) {
+            capa <<= 1;
+            REALLOC_N(json, char, capa);
+        }
+        sprintf(jp, format, tde->doc_num(tde), tde->freq(tde));
+        jp += strlen(jp);
+        if (do_positions) {
+            int pos;
+            while (0 <= (pos = tde->next_position(tde))) {
+                sprintf(jp, "%d,", pos);
+                jp += strlen(jp);
+            }
+            if (*(jp - 1) == ',') jp--;
+            *(jp++) = ']';
+            *(jp++) = close;
+            *(jp++) = ',';
+        }
+    }
+    if (*(jp - 1) == ',') jp--;
+    *(jp++) = ']';
+    *jp = '\0';
+
+    rjson = rb_str_new2(json);
+    free(json);
+    return rjson;
+}
+
 /*
  *  call-seq:
  *     term_doc_enum.each_position {|pos| do_something } -> term_doc_enum
@@ -2678,6 +2854,7 @@ Init_FieldInfo(void)
     rb_define_alloc_func(cFieldInfo, frt_data_alloc);
 
     rb_define_method(cFieldInfo, "initialize",  frt_fi_init, -1);
+    rb_define_method(cFieldInfo, "name",        frt_fi_name, 0);
     rb_define_method(cFieldInfo, "stored?",     frt_fi_is_stored, 0);
     rb_define_method(cFieldInfo, "compressed?", frt_fi_is_compressed, 0);
     rb_define_method(cFieldInfo, "indexed?",    frt_fi_is_indexed, 0);
@@ -2793,6 +2970,7 @@ Init_TermEnum(void)
     rb_define_method(cTermEnum, "each",     frt_te_each, 0);
     rb_define_method(cTermEnum, "field=",   frt_te_set_field, 1);
     rb_define_method(cTermEnum, "set_field",frt_te_set_field, 1);
+    rb_define_method(cTermEnum, "to_json",  frt_te_to_json, -1);
 }
 
 /*
@@ -2844,6 +3022,7 @@ Init_TermDocEnum(void)
     rb_define_method(cTermDocEnum, "each",           frt_tde_each, 0);
     rb_define_method(cTermDocEnum, "each_position",  frt_tde_each_position, 0);
     rb_define_method(cTermDocEnum, "skip_to",        frt_tde_skip_to, 1);
+    rb_define_method(cTermDocEnum, "to_json",        frt_tde_to_json, -1);
 }
 
 /* rdochack

data/ext/r_search.c

@@ -224,32 +224,7 @@ frt_td_to_s(int argc, VALUE *argv, VALUE self)
     return rstr;
 }
 
-/*
- * Json Exportation - Loading each LazyDoc and formatting them into json
- * This code is designed to get a VERY FAST json string, the goal was speed,
- * not sexyness.
- * Jeremie 'ahFeel' BORDIER
- * ahFeel@rift.Fr
- */
 __inline char *
-json_concat_string(char *s, char *field)
-{
-    *(s++) = '"';
-	while (*field) {
-		if (*field == '\"') {
-            *(s++) = '\'';
-            *(s++) = *(field++);
-            *(s++) = '\'';
-        }
-        else {
-            *(s++) = *(field++);
-        }
-    }
-    *(s++) = '"';
-    return s;
-}
-
-inline char *
 frt_lzd_load_to_json(LazyDoc *lzd, char **str, char *s, int *slen)
 {
 	int i, j;
@@ -260,7 +235,7 @@ frt_lzd_load_to_json(LazyDoc *lzd, char **str, char *s, int *slen)
 	for (i = 0; i < lzd->size; i++) {
 		f = lzd->fields[i];
         /* 3 times length of field to make space for quoted quotes ('"') and
-         * 4 x field length to make space for '"' around fields and ','
+         * 4 times field elements to make space for '"' around fields and ','
          * between fields. Add 100 for '[', ']' and good safety.
          */
         len += strlen(f->name) + f->len * 3 + 100 + 4 * f->size;
@@ -1632,15 +1607,22 @@ frt_spanmtq_init(VALUE self, VALUE rfield, VALUE rterms)
 
 /*
  *  call-seq:
- *     SpanPrefixQuery.new(field, prefix) -> query
+ *     SpanPrefixQuery.new(field, prefix, max_terms = 256) -> query
  *
  *  Create a new SpanPrefixQuery which matches all documents with the prefix
  *  +prefix+ in the field +field+.
  */
 static VALUE
-frt_spanprq_init(VALUE self, VALUE rfield, VALUE rprefix)
+frt_spanprq_init(int argc, VALUE *argv, VALUE self)
 {
-    Query *q = spanprq_new(frt_field(rfield), StringValuePtr(rprefix));
+    VALUE rfield, rprefix, rmax_terms;
+    int max_terms = SPAN_PREFIX_QUERY_MAX_TERMS;
+    Query *q;
+    if (rb_scan_args(argc, argv, "21", &rfield, &rprefix, &rmax_terms) == 3) {
+        max_terms = FIX2INT(rmax_terms);
+    }
+    q = spanprq_new(frt_field(rfield), StringValuePtr(rprefix));
+    ((SpanPrefixQuery *)q)->max_terms = max_terms;
     Frt_Wrap_Struct(self, NULL, &frt_q_free, q);
     object_add(q, self);
     return self;
@@ -3556,7 +3538,7 @@ Init_SpanPrefixQuery(void)
     cSpanPrefixQuery = rb_define_class_under(mSpans, "SpanPrefixQuery", cQuery);
     rb_define_alloc_func(cSpanPrefixQuery, frt_data_alloc);
 
-    rb_define_method(cSpanPrefixQuery, "initialize", frt_spanprq_init, 2);
+    rb_define_method(cSpanPrefixQuery, "initialize", frt_spanprq_init, -1);
 }
 
 /*

data/ext/search.c

@@ -1041,6 +1041,7 @@ static TopDocs *isea_search_w(Searcher *self,
 
     scorer = weight->scorer(weight, ISEA(self)->ir);
     if (!scorer || 0 == ISEA(self)->ir->num_docs(ISEA(self)->ir)) {
+        if (scorer) scorer->destroy(scorer);
         return td_new(0, 0, NULL, 0.0);
     }
 

data/ext/search.h

@@ -285,6 +285,7 @@ typedef struct BooleanQuery
 } BooleanQuery;
 
 extern Query *bq_new(bool coord_disabled);
+extern Query *bq_new_max(bool coord_disabled, int max);
 extern BooleanClause *bq_add_query(Query *self, Query *sub_query,
                                    enum BC_TYPE occur);
 extern BooleanClause *bq_add_query_nr(Query *self, Query *sub_query,
@@ -571,6 +572,7 @@ typedef struct SpanPrefixQuery
 {
     SpanQuery   super;
     char       *prefix;
+    int         max_terms;
 } SpanPrefixQuery;
 
 extern Query *spanprq_new(const char *field, const char *prefix);
@@ -868,6 +870,8 @@ typedef struct QParser
     bool handle_parse_errors : 1;
     bool allow_any_fields : 1;
     bool close_def_fields : 1;
+    bool destruct : 1;
+    bool recovering : 1;
 } QParser;
 
 extern QParser *qp_new(HashSet *all_fields, HashSet *def_fields,

data/ext/store.c

@@ -443,6 +443,30 @@ char *is_read_string(InStream *is)
     return str;
 }
 
+char *is_read_string_safe(InStream *is)
+{
+    register int length = (int) is_read_vint(is);
+    char *str = ALLOC_N(char, length + 1);
+    str[length] = '\0';
+
+    TRY
+        if (is->buf.pos > (is->buf.len - length)) {
+            register int i;
+            for (i = 0; i < length; i++) {
+                str[i] = is_read_byte(is);
+            }
+        }
+        else {                      /* unchecked optimization */
+            memcpy(str, is->buf.buf + is->buf.pos, length);
+            is->buf.pos += length;
+        }
+    XCATCHALL
+        free(str);
+    XENDTRY
+
+    return str;
+}
+
 void os_write_i32(OutStream *os, f_i32 num)
 {
     os_write_byte(os, (uchar)((num >> 24) & 0xFF));