ferret 0.10.6 → 0.10.7

data/ext/q_phrase.c

@@ -1015,11 +1015,11 @@ static Query *phq_rewrite(Query *self, IndexReader *ir)
     }
 }
 
-static ulong phq_hash(Query *self)
+static unsigned long phq_hash(Query *self)
 {
     int i, j;
     PhraseQuery *phq = PhQ(self);
-    ulong hash = str_hash(phq->field);
+    unsigned long hash = str_hash(phq->field);
     for (i = 0; i < phq->pos_cnt; i++) {
         char **terms = phq->positions[i].terms;
         for (j = ary_size(terms) - 1; j >= 0; j--) {

data/ext/q_prefix.c

@@ -69,7 +69,7 @@ static void prq_destroy(Query *self)
     q_destroy_i(self);
 }
 
-static ulong prq_hash(Query *self)
+static unsigned long prq_hash(Query *self)
 {
     return str_hash(PfxQ(self)->field) ^ str_hash(PfxQ(self)->prefix);
 }

data/ext/q_range.c

@@ -74,7 +74,7 @@ static void range_destroy(Range *range)
     free(range);
 }
 
-static ulong range_hash(Range *filt)
+static unsigned long range_hash(Range *filt)
 {
     return filt->include_lower | (filt->include_upper << 1)
         | ((str_hash(filt->field)
@@ -219,7 +219,7 @@ static BitVector *rfilt_get_bv_i(Filter *filt, IndexReader *ir)
     return bv;
 }
 
-static ulong rfilt_hash(Filter *filt)
+static unsigned long rfilt_hash(Filter *filt)
 {
     return range_hash(RF(filt)->range);
 }
@@ -278,7 +278,7 @@ static Query *rq_rewrite(Query *self, IndexReader *ir)
     return csq_new_nr(filter);
 }
 
-static ulong rq_hash(Query *self)
+static unsigned long rq_hash(Query *self)
 {
     return range_hash(RQ(self)->range);
 }

data/ext/q_span.c

@@ -17,7 +17,7 @@
 
 #define SpQ(query) ((SpanQuery *)(query))
 
-static ulong spanq_hash(Query *self)
+static unsigned long spanq_hash(Query *self)
 {
     return str_hash(SpQ(self)->field);
 }
@@ -1355,7 +1355,7 @@ static HashSet *spantq_get_terms(Query *self)
     return terms;
 }
 
-static ulong spantq_hash(Query *self)
+static unsigned long spantq_hash(Query *self)
 {
     return spanq_hash(self) ^ str_hash(SpTQ(self)->term);
 }
@@ -1430,7 +1430,7 @@ static void spanfq_destroy_i(Query *self)
     spanq_destroy_i(self);
 }
 
-static ulong spanfq_hash(Query *self)
+static unsigned long spanfq_hash(Query *self)
 {
     return spanq_hash(self) ^ SpFQ(self)->match->hash(SpFQ(self)->match)
         ^ SpFQ(self)->end;
@@ -1573,10 +1573,10 @@ static void spanoq_destroy_i(Query *self)
     spanq_destroy_i(self);
 }
 
-static ulong spanoq_hash(Query *self)
+static unsigned long spanoq_hash(Query *self)
 {
     int i;
-    ulong hash = spanq_hash(self);
+    unsigned long hash = spanq_hash(self);
     SpanOrQuery *soq = SpOQ(self);
 
     for (i = 0; i < soq->c_cnt; i++) {
@@ -1756,10 +1756,10 @@ static void spannq_destroy(Query *self)
     spanq_destroy_i(self);
 }
 
-static ulong spannq_hash(Query *self)
+static unsigned long spannq_hash(Query *self)
 {
     int i;
-    ulong hash = spanq_hash(self);
+    unsigned long hash = spanq_hash(self);
     SpanNearQuery *snq = SpNQ(self);
 
     for (i = 0; i < snq->c_cnt; i++) {
@@ -1907,7 +1907,7 @@ static void spanxq_destroy(Query *self)
     spanq_destroy_i(self);
 }
 
-static ulong spanxq_hash(Query *self)
+static unsigned long spanxq_hash(Query *self)
 {
     SpanNotQuery *sxq = SpXQ(self);
     return spanq_hash(self) ^ sxq->inc->hash(sxq->inc)

data/ext/q_term.c

@@ -289,7 +289,7 @@ static void tq_extract_terms(Query *self, HashSet *terms)
     hs_add(terms, term_new(TQ(self)->field, TQ(self)->term));
 }
 
-static ulong tq_hash(Query *self)
+static unsigned long tq_hash(Query *self)
 {
     return str_hash(TQ(self)->term) ^ str_hash(TQ(self)->field);
 }

data/ext/q_wildcard.c

@@ -140,7 +140,7 @@ static void wcq_destroy(Query *self)
     q_destroy_i(self);
 }
 
-static ulong wcq_hash(Query *self)
+static unsigned long wcq_hash(Query *self)
 {
     return str_hash(WCQ(self)->field) ^ str_hash(WCQ(self)->pattern);
 }

data/ext/r_analysis.c

@@ -4,6 +4,8 @@
 #include "ferret.h"
 #include "analysis.h"
 
+static char *frt_locale = NULL;
+
 static VALUE mAnalysis;
 
 static VALUE cToken;
@@ -808,6 +810,7 @@ static VALUE
 frt_letter_tokenizer_init(int argc, VALUE *argv, VALUE self) 
 {
     TS_ARGS(false);
+    if (!frt_locale) frt_locale = setlocale(LC_CTYPE, "");
     return get_wrapped_ts(self, rstr, mb_letter_tokenizer_new(lower));
 }
 
@@ -836,6 +839,7 @@ static VALUE
 frt_whitespace_tokenizer_init(int argc, VALUE *argv, VALUE self) 
 {
     TS_ARGS(false);
+    if (!frt_locale) frt_locale = setlocale(LC_CTYPE, "");
     return get_wrapped_ts(self, rstr, mb_whitespace_tokenizer_new(lower));
 }
 
@@ -863,6 +867,7 @@ frt_a_standard_tokenizer_init(VALUE self, VALUE rstr)
 static VALUE
 frt_standard_tokenizer_init(VALUE self, VALUE rstr) 
 {
+    if (!frt_locale) frt_locale = setlocale(LC_CTYPE, "");
     return get_wrapped_ts(self, rstr, mb_standard_tokenizer_new());
 }
 
@@ -902,6 +907,7 @@ static VALUE
 frt_lowercase_filter_init(VALUE self, VALUE rsub_ts) 
 {
     TokenStream *ts = frt_get_cwrapped_rts(rsub_ts);
+    if (!frt_locale) frt_locale = setlocale(LC_CTYPE, "");
     ts = mb_lowercase_filter_new(ts);
     object_add(&(TkFilt(ts)->sub_ts), rsub_ts);
 
@@ -1150,6 +1156,7 @@ frt_white_space_analyzer_init(int argc, VALUE *argv, VALUE self)
 {
     Analyzer *a;
     GET_LOWER(false);
+    if (!frt_locale) frt_locale = setlocale(LC_CTYPE, "");
     a = mb_whitespace_analyzer_new(lower);
     Frt_Wrap_Struct(self, NULL, &frt_analyzer_free, a);
     object_add(a, self);
@@ -1192,6 +1199,7 @@ frt_letter_analyzer_init(int argc, VALUE *argv, VALUE self)
 {
     Analyzer *a;
     GET_LOWER(true);
+    if (!frt_locale) frt_locale = setlocale(LC_CTYPE, "");
     a = mb_letter_analyzer_new(lower);
     Frt_Wrap_Struct(self, NULL, &frt_analyzer_free, a);
     object_add(a, self);
@@ -1263,6 +1271,7 @@ frt_standard_analyzer_init(int argc, VALUE *argv, VALUE self)
     bool lower;
     VALUE rlower, rstop_words;
     Analyzer *a;
+    if (!frt_locale) frt_locale = setlocale(LC_CTYPE, "");
     rb_scan_args(argc, argv, "02", &rstop_words, &rlower);
     lower = ((rlower == Qnil) ? true : RTEST(rlower));
     if (rstop_words != Qnil) {
@@ -1390,8 +1399,6 @@ frt_re_analyzer_init(int argc, VALUE *argv, VALUE self)
  *
  ****************************************************************************/
 
-static char *frt_locale = NULL;
-
 /*
  *  call-seq:
  *     Ferret.locale -> locale_str
@@ -1415,7 +1422,7 @@ static VALUE frt_get_locale(VALUE self, VALUE locale)
 static VALUE frt_set_locale(VALUE self, VALUE locale)
 {
     char *l = ((locale == Qnil) ? NULL : RSTRING(rb_obj_as_string(locale))->ptr);
-    frt_locale = setlocale(LC_ALL, l);
+    frt_locale = setlocale(LC_CTYPE, l);
     return frt_locale ? rb_str_new2(frt_locale) : Qnil;
 }
 
@@ -2188,7 +2195,6 @@ Init_Analysis(void)
     rb_define_const(mFerret, "OBJECT_SPACE", object_space);
 
     /*** * * Locale stuff * * ***/
-    frt_locale = setlocale(LC_ALL, "");
     rb_define_singleton_method(mFerret, "locale=", frt_set_locale, 1);
     rb_define_singleton_method(mFerret, "locale", frt_get_locale, 0);
 

data/ext/r_index.c

@@ -240,9 +240,11 @@ frt_fi_is_indexed(VALUE self)
  *     fi.tokenized? -> bool
  *
  *  Return true if the field is tokenized. Tokenizing is the process of
- *  breaking the field up into tokens. That is "the quick brown fox" becomes
- *  ["the", "quick", "brown", "fox"]  This is only possible if the field in
- *  indexed.
+ *  breaking the field up into tokens. That is "the quick brown fox" becomes:
+ *
+ *    ["the", "quick", "brown", "fox"]
+ *
+ *  A field can only be tokenized if it is indexed.
  */
 static VALUE
 frt_fi_is_tokenized(VALUE self)
@@ -595,7 +597,8 @@ frt_fis_create_index(VALUE self, VALUE rdir)
  *  call-seq:
  *     fis.fields -> symbol array
  *
- *  Return a list of the the field names (as symbols) in the index.
+ *  Return a list of the field names (as symbols) of all the fieldcs in the
+ *  index.
  */
 static VALUE
 frt_fis_get_fields(VALUE self)
@@ -609,6 +612,26 @@ frt_fis_get_fields(VALUE self)
     return rfield_names;
 }
 
+/*
+ *  call-seq:
+ *     fis.tokenized_fields -> symbol array
+ *
+ *  Return a list of the field names (as symbols) of all the tokenized fields
+ *  in the index.
+ */
+static VALUE
+frt_fis_get_tk_fields(VALUE self)
+{
+    FieldInfos *fis = (FieldInfos *)DATA_PTR(self);
+    VALUE rfield_names = rb_ary_new();
+    int i;
+    for (i = 0; i < fis->size; i++) {
+        if (!fi_is_tokenized(fis->fields[i])) continue;
+        rb_ary_push(rfield_names, ID2SYM(rb_intern(fis->fields[i]->name)));
+    }
+    return rfield_names;
+}
+
 /****************************************************************************
  *
  * TermEnum Methods
@@ -2375,7 +2398,7 @@ frt_ir_terms_from(VALUE self, VALUE rfield, VALUE rterm)
 
 /*
  *  call-seq:
- *     index_reader.field_names -> array of field-names
+ *     index_reader.fields -> array of field-names
  *
  *  Returns an array of field names in the index. This can be used to pass to
  *  the QueryParser so that the QueryParser knows how to expand the "*"
@@ -2383,7 +2406,7 @@ frt_ir_terms_from(VALUE self, VALUE rfield, VALUE rterm)
  *  gathered from the FieldInfos object.
  */
 static VALUE
-frt_ir_field_names(VALUE self)
+frt_ir_fields(VALUE self)
 {
     IndexReader *ir = (IndexReader *)DATA_PTR(self);
     FieldInfos *fis = ir->fis;
@@ -2408,6 +2431,29 @@ frt_ir_field_infos(VALUE self)
     return frt_get_field_infos(ir->fis);
 }
 
+/*
+ *  call-seq:
+ *     index_reader.tokenized_fields -> array of field-names
+ *
+ *  Returns an array of field names of all of the tokenized fields in the
+ *  index. This can be used to pass to the QueryParser so that the QueryParser
+ *  knows how to expand the "*" wild-card to all fields in the index. A list
+ *  of field names can also be gathered from the FieldInfos object.
+ */
+static VALUE
+frt_ir_tk_fields(VALUE self)
+{
+    IndexReader *ir = (IndexReader *)DATA_PTR(self);
+    FieldInfos *fis = ir->fis;
+    VALUE rfield_names = rb_ary_new();
+    int i;
+    for (i = 0; i < fis->size; i++) {
+        if (!fi_is_tokenized(fis->fields[i])) continue;
+        rb_ary_push(rfield_names, ID2SYM(rb_intern(fis->fields[i]->name)));
+    }
+    return rfield_names;
+}
+
 /****************************************************************************
  *
  * Init Functions
@@ -2515,6 +2561,16 @@ frt_ir_field_infos(VALUE self)
  *                  |                         |
  *                  | :with_positions_offsets | Store term-vectors with
  *                  | (default)               | positions and offsets.
+ *     -------------|-------------------------|------------------------------
+ *     :boost       | Float                   | The boost property is used to
+ *                  |                         | set the default boost for a
+ *                  |                         | field. This boost value will
+ *                  |                         | used for all instances of the
+ *                  |                         | field in the index unless 
+ *                  |                         | otherwise specified when you
+ *                  |                         | create the field. All values
+ *                  |                         | should be positive.
+ *                  |                         | 
  *
  *  == Examples
  *
@@ -2625,7 +2681,8 @@ Init_FieldInfos(void)
     rb_define_method(cFieldInfos, "to_s",       frt_fis_to_s, 0);
     rb_define_method(cFieldInfos, "create_index",
                                                 frt_fis_create_index, 1);
-    rb_define_method(cFieldInfos, "fields",     frt_fis_get_fields, -1);
+    rb_define_method(cFieldInfos, "fields",     frt_fis_get_fields, 0);
+    rb_define_method(cFieldInfos, "tokenized_fields", frt_fis_get_tk_fields, 0);
 }
 
 /*
@@ -2717,21 +2774,33 @@ Init_TermDocEnum(void)
     rb_define_method(cTermDocEnum, "skip_to",        frt_tde_skip_to, 1);
 }
 
+/* rdochack
+cTermVector = rb_define_class_under(mIndex, "TermVector", rb_cObject);
+*/
+
 /*
  *  Document-class: Ferret::Index::TermVector::TVOffsets
  *
  *  == Summary
  *
  *  Holds the start and end byte-offsets of a term in a field. For example, if
- *  the field was "the quick brown fox" then the start and end offsets of
- *  ["the", "quick", "brown", "fox"] would be [(0,3), (4,9), (10,15), (16,19)]
- *  respectively. See the Analysis module for more information on setting the
- *  offsets.
+ *  the field was "the quick brown fox" then the start and end offsets of:
+ *
+ *    ["the", "quick", "brown", "fox"]
+ *
+ *  Would be:
+ *
+ *    [(0,3), (4,9), (10,15), (16,19)]
+ *
+ *  See the Analysis module for more information on setting the offsets.
  */
 static void
 Init_TVOffsets(void)
 {
     const char *tv_offsets_class = "TVOffsets";
+    /* rdochack
+    cTVOffsets = rb_define_class_under(cTermVector, "TVOffsets", rb_cObject);
+    */
     cTVOffsets = rb_struct_define(tv_offsets_class, "start", "end", NULL);
     rb_set_class_path(cTVOffsets, cTermVector, tv_offsets_class);
     rb_const_set(mIndex, rb_intern(tv_offsets_class), cTVOffsets);
@@ -2756,6 +2825,9 @@ static void
 Init_TVTerm(void)
 {
     const char *tv_term_class = "TVTerm";
+    /* rdochack
+    cTVTerm = rb_define_class_under(cTermVector, "TVTerm", rb_cObject);
+    */
     cTVTerm = rb_struct_define(tv_term_class, "text", "positions", NULL);
     rb_set_class_path(cTVTerm, cTermVector, tv_term_class);
     rb_const_set(mIndex, rb_intern(tv_term_class), cTVTerm);
@@ -2795,6 +2867,9 @@ static void
 Init_TermVector(void)
 {
     const char *tv_class = "TermVector";
+    /* rdochack
+    cTermVector = rb_define_class_under(mIndex, "TermVector", rb_cObject);
+    */
     cTermVector = rb_struct_define(tv_class,
                                    "field", "terms", "offsets", NULL);
     rb_set_class_path(cTermVector, mIndex, tv_class);
@@ -3108,8 +3183,10 @@ Init_IndexReader(void)
     rb_define_method(cIndexReader, "doc_freq",      frt_ir_doc_freq, 2);
     rb_define_method(cIndexReader, "terms",         frt_ir_terms, 1);
     rb_define_method(cIndexReader, "terms_from",    frt_ir_terms_from, 2);
-    rb_define_method(cIndexReader, "field_names",   frt_ir_field_names, 0);
+    rb_define_method(cIndexReader, "fields",        frt_ir_fields, 0);
+    rb_define_method(cIndexReader, "field_names",   frt_ir_fields, 0);
     rb_define_method(cIndexReader, "field_infos",   frt_ir_field_infos, 0);
+    rb_define_method(cIndexReader, "tokenized_fields", frt_ir_tk_fields, 0);
 }
 
 /* rdoc hack

data/ext/r_qparser.c

@@ -6,7 +6,9 @@ VALUE cQueryParseException;
 
 extern VALUE sym_analyzer;
 static VALUE sym_wild_card_downcase;
+static VALUE sym_fields;
 static VALUE sym_all_fields;
+static VALUE sym_tkz_fields;
 static VALUE sym_default_field;
 static VALUE sym_validate_fields;
 static VALUE sym_or_default;
@@ -42,9 +44,12 @@ static HashSet *
 frt_get_fields(VALUE rfields)
 {
     VALUE rval;
-    HashSet *fields = hs_new_str(&free);
+    HashSet *fields;
     char *s, *p, *str;
 
+    if (rfields == Qnil) return NULL;
+
+    fields = hs_new_str(&free);
     if (TYPE(rfields) == T_ARRAY) {
         int i;
         for (i = 0; i < RARRAY(rfields)->len; i++) {
@@ -87,9 +92,12 @@ frt_get_fields(VALUE rfields)
  *  :wild_card_downcase::   Default: true. Specifies whether wild-card queries
  *                          should be downcased or not since they are not
  *                          passed through the parser
- *  :all_fields::           Default: []. Lets the query parser know what
+ *  :fields::               Default: []. Lets the query parser know what
  *                          fields are available for searching, particularly
  *                          when the "*" is specified as the search field
+ *  :tokenized_fields::     Default: :fields. Lets the query parser know which
+ *                          fields are tokenized so it knows which fields to
+ *                          run the analyzer over.
  *  :validate_fields::      Default: false. Set to true if you want an
  *                          exception to be raised if there is an attempt to
  *                          search a non-existent field
@@ -118,6 +126,7 @@ frt_qp_init(int argc, VALUE *argv, VALUE self)
     bool has_options = false;
 
     HashSet *all_fields = NULL;
+    HashSet *tkz_fields = NULL;
     HashSet *def_fields = NULL;
     QParser *qp;
 
@@ -133,6 +142,12 @@ frt_qp_init(int argc, VALUE *argv, VALUE self)
             if (Qnil != (rval = rb_hash_aref(roptions, sym_all_fields))) {
                 all_fields = frt_get_fields(rval);
             }
+            if (Qnil != (rval = rb_hash_aref(roptions, sym_fields))) {
+                all_fields = frt_get_fields(rval);
+            }
+            if (Qnil != (rval = rb_hash_aref(roptions, sym_tkz_fields))) {
+                tkz_fields = frt_get_fields(rval);
+            }
         } else {
             def_fields = frt_get_fields(roptions);
         }
@@ -145,7 +160,7 @@ frt_qp_init(int argc, VALUE *argv, VALUE self)
         analyzer = mb_standard_analyzer_new(true);
     }
 
-    qp = qp_new(all_fields, def_fields, analyzer);
+    qp = qp_new(all_fields, def_fields, tkz_fields, analyzer);
     qp->allow_any_fields = true;
     qp->clean_str = true;
     /* handle options */
@@ -255,6 +270,48 @@ frt_qp_set_fields(VALUE self, VALUE rfields)
     return self;
 }
 
+/*
+ *  call-seq:
+ *     query_parser.tokenized_fields -> Array of Symbols
+ *
+ *  Returns the list of all tokenized_fields that the QueryParser knows about.
+ */
+static VALUE
+frt_qp_get_tkz_fields(VALUE self)
+{
+    GET_QP;
+    int i;
+    HashSet *fields = qp->tokenized_fields;
+    if (fields) {
+        VALUE rfields = rb_ary_new();
+
+        for (i = 0; i < fields->size; i++) {
+            rb_ary_push(rfields, ID2SYM(rb_intern((char *)fields->elems[i])));
+        } 
+
+        return rfields;
+    }
+    else {
+        return Qnil;
+    }
+}
+
+/*
+ *  call-seq:
+ *     query_parser.tokenized_fields = fields -> self
+ *
+ *  Set the list of tokenized_fields. These tokenized_fields are tokenized in
+ *  the queries. If this is set to Qnil then all fields will be tokenized. 
+ */
+static VALUE
+frt_qp_set_tkz_fields(VALUE self, VALUE rfields)
+{
+    GET_QP;
+    if (qp->tokenized_fields) hs_destroy(qp->tokenized_fields);
+    qp->tokenized_fields = frt_get_fields(rfields);
+    return self;
+}
+
 /****************************************************************************
  *
  * Init function
@@ -483,7 +540,9 @@ Init_QueryParser(void)
 {
     /* hash keys */
     sym_wild_card_downcase = ID2SYM(rb_intern("wild_card_downcase"));
-    sym_all_fields = ID2SYM(rb_intern("fields"));
+    sym_fields = ID2SYM(rb_intern("fields"));
+    sym_all_fields = ID2SYM(rb_intern("all_fields"));
+    sym_tkz_fields = ID2SYM(rb_intern("tokenized_fields"));
     sym_default_field = ID2SYM(rb_intern("default_field"));
     sym_validate_fields = ID2SYM(rb_intern("validate_fields"));
     sym_or_default = ID2SYM(rb_intern("or_default"));
@@ -500,6 +559,10 @@ Init_QueryParser(void)
     rb_define_method(cQueryParser, "parse", frt_qp_parse, 1);
     rb_define_method(cQueryParser, "fields", frt_qp_get_fields, 0);
     rb_define_method(cQueryParser, "fields=", frt_qp_set_fields, 1);
+    rb_define_method(cQueryParser, "tokenized_fields",
+                     frt_qp_get_tkz_fields, 0);
+    rb_define_method(cQueryParser, "tokenized_fields=",
+                     frt_qp_set_tkz_fields, 1);
 
     Init_QueryParseException();
 }