ferret 0.2.2 → 0.3.0

data/ext/term_infos_reader.c

@@ -0,0 +1,54 @@
+#include "ferret.h"
+
+static ID frt_id_index_terms;
+/****************************************************************************
+ *
+ * TermInfosReader Methods
+ *
+ ****************************************************************************/
+
+static VALUE
+frt_tir_get_index_offset(VALUE self, VALUE rterm)
+{
+  VALUE index_terms = rb_ivar_get(self, frt_id_index_terms);
+
+  register int lo = 0;            // binary search @index_terms[]
+  register int hi = RARRAY(index_terms)->len - 1;
+  register int mid, delta;
+
+  Term *term, *tmp_term; 
+  Data_Get_Struct(rterm, Term, term);
+
+  while (hi >= lo) {
+    mid = (lo + hi) >> 1;
+
+    Data_Get_Struct(RARRAY(index_terms)->ptr[mid], Term, tmp_term);
+    delta = frt_term_cmp(term, tmp_term);
+    if (delta < 0) {
+      hi = mid - 1;
+    } else if (delta > 0) {
+      lo = mid + 1;
+    } else {
+      return INT2FIX(mid);
+    }
+  }
+  return INT2FIX(hi);
+}
+
+/****************************************************************************
+ *
+ * Init Function
+ *
+ ****************************************************************************/
+
+void
+Init_term_infos_reader(void)
+{
+  /* IDs */
+  frt_id_index_terms = rb_intern("@index_terms");
+
+  /* TermInfosReader */
+  cTermInfosReader = rb_define_class_under(mIndex, "TermInfosReader", rb_cObject);
+
+  rb_define_method(cTermInfosReader, "get_index_offset", frt_tir_get_index_offset, 1);
+}

data/ext/terminfo.c

@@ -0,0 +1,160 @@
+#include "ferret.h"
+
+
+/****************************************************************************
+ *
+ * TermInfo Methods
+ *
+ ****************************************************************************/
+
+void
+frt_ti_free(void *p)
+{
+  free(p);
+}
+
+static VALUE
+frt_ti_alloc(VALUE klass)
+{
+  TermInfo *ti = (TermInfo *)ALLOC(TermInfo);
+  VALUE rbuffer = Data_Wrap_Struct(klass, NULL, frt_ti_free, ti);
+  return rbuffer;
+}
+
+#define GET_TI TermInfo *ti; Data_Get_Struct(self, TermInfo, ti)
+inline VALUE 
+frt_ti_set(int argc, VALUE *argv, VALUE self)
+{
+  VALUE df, fp, pp, so;
+  GET_TI;
+  MEMZERO(ti, TermInfo, 1);
+  rb_scan_args(argc, argv, "04", &df, &fp, &pp, &so);
+  switch (argc) {
+    case 4:
+      ti->skip_offset = FIX2INT(so);
+    case 3:
+      ti->prox_pointer = FIX2INT(pp);
+    case 2:
+      ti->freq_pointer = FIX2INT(fp);
+    case 1:
+      ti->doc_freq = FIX2INT(df);
+    case 0:
+      break;
+  }
+  return Qnil;
+}
+
+static VALUE 
+frt_ti_init(int argc, VALUE *argv, VALUE self)
+{
+  frt_ti_set(argc, argv, self);
+  return self;
+}
+
+static VALUE 
+frt_ti_init_copy(VALUE self, VALUE rother)
+{
+  TermInfo *other_ti; 
+  GET_TI;
+  Data_Get_Struct(rother, TermInfo, other_ti);
+  MEMCPY(ti, other_ti, TermInfo, 1);
+  return self;
+}
+
+static VALUE
+frt_ti_eql(VALUE self, VALUE rother)
+{
+  TermInfo *other_ti; 
+  GET_TI;
+  if (NIL_P(rother)) return Qfalse;
+  Data_Get_Struct(rother, TermInfo, other_ti);
+  return (MEMCMP(ti, other_ti, TermInfo, 1) == 0) ? Qtrue : Qfalse;
+}
+
+static VALUE
+frt_ti_get_df(VALUE self)
+{
+  GET_TI;
+  return INT2FIX(ti->doc_freq);
+}
+
+static VALUE
+frt_ti_get_fp(VALUE self)
+{
+  GET_TI;
+  return INT2FIX(ti->freq_pointer);
+}
+
+static VALUE
+frt_ti_get_pp(VALUE self)
+{
+  GET_TI;
+  return INT2FIX(ti->prox_pointer);
+}
+
+static VALUE
+frt_ti_get_so(VALUE self)
+{
+  GET_TI;
+  return INT2FIX(ti->skip_offset);
+}
+
+static VALUE
+frt_ti_set_df(VALUE self, VALUE val)
+{
+  GET_TI;
+  ti->doc_freq = FIX2INT(val);
+  return Qnil;
+}
+
+static VALUE
+frt_ti_set_fp(VALUE self, VALUE val)
+{
+  GET_TI;
+  ti->freq_pointer = FIX2INT(val);
+  return Qnil;
+}
+
+static VALUE
+frt_ti_set_pp(VALUE self, VALUE val)
+{
+  GET_TI;
+  ti->prox_pointer = FIX2INT(val);
+  return Qnil;
+}
+
+static VALUE
+frt_ti_set_so(VALUE self, VALUE val)
+{
+  GET_TI;
+  ti->skip_offset = FIX2INT(val);
+  return Qnil;
+}
+
+/****************************************************************************
+ *
+ * Init Function
+ *
+ ****************************************************************************/
+
+void
+Init_term_info(void)
+{
+  /* TermInfo */
+  cTermInfo = rb_define_class_under(mIndex, "TermInfo", rb_cObject);
+  rb_define_alloc_func(cTermInfo, frt_ti_alloc);
+
+  rb_define_method(cTermInfo, "initialize", frt_ti_init, -1);
+  rb_define_method(cTermInfo, "set_values!", frt_ti_set, -1);
+  rb_define_method(cTermInfo, "initialize_copy", frt_ti_init_copy, 1);
+  rb_define_method(cTermInfo, "set!", frt_ti_init_copy, 1);
+  rb_define_method(cTermInfo, "==", frt_ti_eql, 1);
+  rb_define_method(cTermInfo, "doc_freq", frt_ti_get_df, 0);
+  rb_define_method(cTermInfo, "doc_freq=", frt_ti_set_df, 1);
+  rb_define_method(cTermInfo, "freq_pointer", frt_ti_get_fp, 0);
+  rb_define_method(cTermInfo, "freq_pointer=", frt_ti_set_fp, 1);
+  rb_define_method(cTermInfo, "prox_pointer", frt_ti_get_pp, 0);
+  rb_define_method(cTermInfo, "prox_pointer=", frt_ti_set_pp, 1);
+  rb_define_method(cTermInfo, "skip_offset", frt_ti_get_so, 0);
+  rb_define_method(cTermInfo, "skip_offset=", frt_ti_set_so, 1);
+}

data/ext/token.c

@@ -0,0 +1,93 @@
+#include "ferret.h"
+
+/****************************************************************************
+ *
+ * Token Methods
+ *
+ ****************************************************************************/
+
+ID id_tk_text, id_tk_pos_inc, id_tk_start_offset, id_tk_end_offset, id_tk_type;
+ID id_tk_pos_inc_set;
+
+static VALUE
+frt_token_pos_inc (VALUE self, VALUE pI)
+{
+  if(FIX2INT(pI) < 0)
+    rb_raise(rb_eArgError, "position_increment < 0");
+  rb_ivar_set(self, id_tk_pos_inc, pI);
+  return self;
+}
+
+static VALUE
+frt_token_init(int argc, VALUE *argv, VALUE self)
+{
+  VALUE text, start_offset, end_offset, type, pos_inc;
+  rb_scan_args(argc, argv, "32", &text,
+      &start_offset, &end_offset, &type, &pos_inc);
+  rb_ivar_set(self, id_tk_text, text);
+  rb_ivar_set(self, id_tk_start_offset, start_offset);
+  rb_ivar_set(self, id_tk_end_offset, end_offset);
+  if (argc < 4) {
+    rb_ivar_set(self, id_tk_type, rb_str_new("word", 4));
+  } else {
+    rb_ivar_set(self, id_tk_type, type);
+  }
+  if (argc < 5) {
+    rb_ivar_set(self, id_tk_pos_inc, INT2FIX(1));
+  } else {
+    rb_ivar_set(self, id_tk_pos_inc, pos_inc);
+  }
+  return self;
+}
+
+static VALUE
+frt_token_eql(VALUE self, VALUE other)
+{
+  VALUE rself_text, rother_text;
+  char *self_text, *other_text;
+  if (!rb_respond_to(other, id_tk_pos_inc_set))
+    return Qfalse;
+  rself_text = rb_ivar_get(self, id_tk_text);
+  rother_text = rb_ivar_get(other, id_tk_text);
+  self_text = StringValuePtr(rself_text);
+  other_text = StringValuePtr(rother_text);
+  if (rb_ivar_get(self, id_tk_start_offset) == rb_ivar_get(other, id_tk_start_offset) &&
+      rb_ivar_get(self, id_tk_end_offset) == rb_ivar_get(other, id_tk_end_offset) &&
+      (strcmp(self_text, other_text) == 0))
+    return Qtrue;
+  else
+    return Qfalse;
+}
+
+/****************************************************************************
+ *
+ * Init Function
+ *
+ ****************************************************************************/
+
+void
+Init_token(void)
+{
+  /* IDs */
+  id_tk_text = rb_intern("@term_text");
+  id_tk_start_offset = rb_intern("@start_offset");
+  id_tk_end_offset = rb_intern("@end_offset");
+  id_tk_type = rb_intern("@type");
+  id_tk_pos_inc = rb_intern("@position_increment");
+  id_tk_pos_inc_set = rb_intern("position_increment=");
+
+
+  /* IndexWriter */
+  cToken = rb_define_class_under(mAnalysis, "Token", rb_cObject);
+
+  rb_define_method(cToken, "initialize",   frt_token_init, -1);
+  rb_define_method(cToken, "position_increment=", frt_token_pos_inc, 1);
+  rb_define_method(cToken, "==", frt_token_eql, 1);
+  rb_define_method(cToken, "eql", frt_token_eql, 1);
+  
+  rb_define_attr(cToken, "term_text", 1, 1);
+  rb_define_attr(cToken, "position_increment", 1, 0);
+  rb_define_attr(cToken, "start_offset", 1, 0);
+  rb_define_attr(cToken, "end_offset", 1, 0);
+  rb_define_attr(cToken, "type", 1, 1);
+}

data/lib/ferret.rb

@@ -22,7 +22,7 @@
 #++
 # :include: ../TUTORIAL
 module Ferret
-  VERSION = '0.2.2'
+  VERSION = '0.3.0'
 end
 
 require 'ferret/utils'

data/lib/ferret/analysis/analyzers.rb

@@ -17,6 +17,24 @@ module Ferret::Analysis
     def token_stream(field, string)
       return LowerCaseTokenizer.new(string)
     end
+
+    # Invoked before indexing a Field instance if
+    # terms have already been added to that field.  This allows custom
+    # analyzers to place an automatic position increment gap between
+    # Field instances using the same field name.  The default value
+    # position increment gap is 0.  With a 0 position increment gap and
+    # the typical default token position increment of 1, all terms in a field,
+    # including across Field instances, are in successive positions, allowing
+    # exact PhraseQuery matches, for instance, across Field instance boundaries.
+    #
+    # field_name::             Field name being indexed.
+    # position_increment_gap:: added to the next token emitted from
+    #                          #token_stream(String,Reader)
+    #
+    def position_increment_gap(field_name)
+      return 0
+    end
+
   end
 
   # An Analyzer that uses WhiteSpaceTokenizer.

data/lib/ferret/analysis/standard_tokenizer.rb

@@ -18,7 +18,21 @@ module Ferret::Analysis
     ACRONYM    = /#{ALPHA}\.(#{ALPHA}\.)+/
     P          = /[_\/.,-]/
     HASDIGIT   = /\w*\d\w*/
+    TOKEN_RE   = /[[:alpha:]]+(('[[:alpha:]]+)+
+                              |\.([[:alpha:]]\.)+
+                              |(@|\&)\w+([-.]\w+)*
+                              )
+                 |\w+(([\-._]\w+)*\@\w+([-.]\w+)+
+                     |#{P}#{HASDIGIT}(#{P}\w+#{P}#{HASDIGIT})*(#{P}\w+)?
+                     |(\.\w+)+
+                     |
+                     )
+                 /x
 
+    ACRONYM_WORD    = /^#{ACRONYM}$/
+    APOSTROPHE_WORD = /^#{APOSTROPHE}$/
+    DOT             = /\./
+    APOSTROPHE_S    = /'[sS]$/
     protected
 
       # Collects only characters which are not spaces tabs or carraige returns
@@ -27,24 +41,15 @@ module Ferret::Analysis
         # This is a simplified version of the original Lucene standard
         # tokenizer.  I think it works better. I hope so anyway. Any way to
         # do this more neatly?
-        /[[:alpha:]]+(('[[:alpha:]]+)+
-                     |\.([[:alpha:]]\.)+
-                     |(@|\&)\w+([-.]\w+)*
-                     )
-        |\w+(([\-._]\w+)*\@\w+([-.]\w+)+
-            |#{P}#{HASDIGIT}(#{P}\w+#{P}#{HASDIGIT})*(#{P}\w+)?
-            |(\.\w+)+
-            |
-            )
-        /x
+        TOKEN_RE
       end
 
       # stem the 's and remove the '.'s from acronyms
       def normalize(str)
-        if str =~ /^#{ACRONYM}$/
-          str.gsub!(/\./, '')
-        elsif str =~ /^#{APOSTROPHE}$/
-          str.gsub!(/'[sS]$/, '')
+        if str =~ ACRONYM_WORD
+          str.gsub!(DOT, '')
+        elsif str =~ APOSTROPHE_WORD
+          str.gsub!(APOSTROPHE_S, '')
         end
         str
       end

data/lib/ferret/analysis/token.rb

@@ -35,9 +35,16 @@ module Ferret::Analysis
       @position_increment = pos_inc
     end
 
+    def set!(txt, so, eo)
+      @term_text = txt
+      @start_offset = so
+      @end_offset = eo
+      self
+    end
+
     def eql?(o)
       return (o.instance_of?(Token) and @start_offset == o.start_offset and
-              @end_offset == o.end_offset and @term_text = o.term_text)
+              @end_offset == o.end_offset and @term_text == o.term_text)
     end
     alias :== :eql?
 

data/lib/ferret/analysis/tokenizers.rb

@@ -36,6 +36,7 @@ module Ferret::Analysis
     # input:: must have a read(count) method which returns an array or string
     #         of _count_ chars.
     def initialize(input)
+      #@token_buffer = Token.new("", 0, 0)
       if input.is_a? String
         @ss = StringScanner.new(input)
       else
@@ -53,6 +54,7 @@ module Ferret::Analysis
         return nil
       end
 
+      #return @token_buffer.set!(normalize(term), term_start, term_end)
       return Token.new(normalize(term), term_start, term_end)
     end
 
@@ -62,8 +64,9 @@ module Ferret::Analysis
 
     protected
       # returns the regular expression used to find the next token
+      TOKEN_RE = /[[:alpha:]]+/
       def token_re
-        /[[:alpha:]]+/
+        TOKEN_RE
       end
 
       # Called on each token to normalize it before it is added to the
@@ -80,8 +83,9 @@ module Ferret::Analysis
     protected
       # Collects only characters which satisfy the regular expression
       # _/[[:alpha:]]+/_.
-      def token_re()
-        /[[:alpha:]]+/
+      TOKEN_RE = /[[:alpha:]]+/
+      def token_re
+        TOKEN_RE
       end
   end
 
@@ -100,8 +104,9 @@ module Ferret::Analysis
   class WhiteSpaceTokenizer < RegExpTokenizer
     protected
       # Collects only characters which are not spaces tabs or carraige returns
-      def token_re()
-        /\S+/
+      TOKEN_RE = /\S+/
+      def token_re
+        TOKEN_RE
       end
   end
 end