ferret 0.2.2 → 0.3.0

data/ext/Makefile

@@ -71,8 +71,8 @@ extout_prefix =
 target_prefix = 
 LOCAL_LIBS = 
 LIBS = $(LIBRUBYARG_SHARED)  -lpthread -ldl -lcrypt -lm   -lc
-SRCS = index_io.c term_buffer.c ram_directory.c priority_queue.c string_helper.c segment_merge_queue.c ferret.c term.c util.c
-OBJS = index_io.o term_buffer.o ram_directory.o priority_queue.o string_helper.o segment_merge_queue.o ferret.o term.o util.o
+SRCS = index_io.c term_buffer.c ram_directory.c priority_queue.c string_helper.c segment_merge_queue.c ferret.c term.c util.c token.c segment_term_enum.c terminfo.c term_infos_reader.c similarity.c
+OBJS = index_io.o term_buffer.o ram_directory.o priority_queue.o string_helper.o segment_merge_queue.o ferret.o term.o util.o token.o segment_term_enum.o terminfo.o term_infos_reader.o similarity.o
 TARGET = ferret_ext
 DLLIB = $(TARGET).so
 STATIC_LIB = 

data/ext/ferret.c

@@ -1,13 +1,18 @@
 #include "ferret.h"
 
 /* IDs */
-ID frt_newobj;
+ID id_new;
+ID id_close;
+ID id_size;
+ID id_iv_size;
 
 /* Modules */
 VALUE mFerret;
 VALUE mStore;
 VALUE mIndex;
 VALUE mUtils;
+VALUE mAnalysis;
+VALUE mSearch;
 VALUE mStringHelper;
 
 /* Classes */
@@ -22,27 +27,47 @@ VALUE cRAMIndexOut;
 VALUE cRAMIndexIn;
 VALUE cTerm;
 VALUE cTermBuffer;
+VALUE cTermInfo;
+VALUE cToken;
 VALUE cPriorityQueue;
 VALUE cSegmentMergeQueue;
+VALUE cSegmentTermEnum;
+VALUE cTermEnum;
+VALUE cTermInfosReader;
+VALUE cSimilarity;
+VALUE cDefaultSimilarity;
 
 void
 Init_ferret_ext(void)
 {
   /* IDs */
-	frt_newobj = rb_intern("new");
+  id_new = rb_intern("new");
+  id_close = rb_intern("close");
+  id_size = rb_intern("size");
+  id_iv_size = rb_intern("@size");
 
   /* Modules */
   mFerret = rb_define_module("Ferret");
   mStore = rb_define_module_under(mFerret, "Store");
   mIndex = rb_define_module_under(mFerret, "Index");
   mUtils = rb_define_module_under(mFerret, "Utils");
+  mAnalysis = rb_define_module_under(mFerret, "Analysis");
+  mSearch = rb_define_module_under(mFerret, "Search");
+
+  /* Classes */
+  cTermEnum = rb_define_class_under(mIndex, "TermEnum", rb_cObject);
 
   /* Inits */
   Init_indexio();
   Init_term();
   Init_term_buffer();
+  Init_term_info();
+  Init_term_infos_reader();
+  Init_token();
   Init_priority_queue();
   Init_segment_merge_queue();
+  Init_segment_term_enum();
   Init_ram_directory();
   Init_string_helper();
+  Init_similarity();
 }

data/ext/ferret.h

@@ -15,24 +15,23 @@ typedef struct IndexBuffer {
 } IndexBuffer;
 
 typedef struct Term {
-	char *field;
-	char *text;
-	int  flen;
-	int  tlen;
+  VALUE field;
+  char *text;
+  int  tlen;
 } Term;
 
 typedef struct PriorityQueue {
-	VALUE *heap;
-	int len;
-	int size;
+  VALUE *heap;
+  int len;
+  int size;
 } PriorityQueue;
 
-typedef struct TermBuffer {
-	char *field;
-	char *text;
-	int flen;
-	int tlen;
-} TermBuffer;
+typedef struct TermInfo {
+  int doc_freq;
+  long freq_pointer;
+  long prox_pointer;
+  int skip_offset;
+} TermInfo;
 
 typedef struct RAMFile {
   void  **buffers;
@@ -42,14 +41,38 @@ typedef struct RAMFile {
   int   length;
 } RAMFile;
 
+typedef struct SegmentTermEnum {
+  VALUE input;
+  IndexBuffer *buf;
+  VALUE field_infos;
+  VALUE rtb_curr;
+  Term *tb_curr;
+  VALUE rtb_prev;
+  Term *tb_prev;
+  TermInfo *ti;
+  int is_index;
+  int size;
+  int position;
+  int index_pointer;
+  int index_interval;
+  int skip_interval;
+  int format;
+  int format_m1skip_interval;
+} SegmentTermEnum;
+
 /* IDs */
-extern ID frt_newobj;
+extern ID id_new;
+extern ID id_close;
+extern ID id_size;
+extern ID id_iv_size;
 
 /* Modules */
 extern VALUE mFerret;
 extern VALUE mStore;
 extern VALUE mIndex;
 extern VALUE mUtils;
+extern VALUE mAnalysis;
+extern VALUE mSearch;
 extern VALUE mStringHelper;
 
 /* Classes */
@@ -64,22 +87,42 @@ extern VALUE cRAMIndexOut;
 extern VALUE cRAMIndexIn;
 extern VALUE cTerm;
 extern VALUE cTermBuffer;
+extern VALUE cTermInfo;
+extern VALUE cToken;
 extern VALUE cPriorityQueue;
 extern VALUE cSegmentMergeQueue;
+extern VALUE cTermEnum;
+extern VALUE cTermInfosReader;
+extern VALUE cSegmentTermEnum;
+extern VALUE cSimilarity;
+extern VALUE cDefaultSimilarity;
 
 /* Ferret Inits */
 extern void Init_indexio();
 extern void Init_term();
-extern void Init_priority_queue();
+extern void Init_term_info();
+extern void Init_term_infos_reader();
 extern void Init_term_buffer();
+extern void Init_priority_queue();
+extern void Init_token();
 extern void Init_segment_merge_queue();
+extern void Init_segment_term_enum();
 extern void Init_ram_directory();
 extern void Init_string_helper();
+extern void Init_similarity();
 
 /* External functions */
 extern int frt_hash(register char *p, register int len);
-extern unsigned long long frt_read_vint(VALUE self);
+extern unsigned long long frt_read_vint(VALUE self, IndexBuffer *my_buf);
+extern VALUE frt_indexin_read_long(VALUE self);
+extern VALUE frt_indexin_read_int(VALUE self);
+extern VALUE frt_indexin_seek(VALUE self, VALUE pos);
+extern VALUE frt_termbuffer_to_term(VALUE self);
 extern void frt_read_chars(VALUE self, char *buf, int offset, int len);
 extern void frt_write_bytes(VALUE self, byte_t *buf, int len);
 extern int frt_term_compare_to_int(VALUE self, VALUE rother);
+extern VALUE frt_termbuffer_init_copy(VALUE self, VALUE rother);
+extern VALUE frt_termbuffer_read(VALUE self, VALUE input, VALUE info);
+extern inline int frt_term_cmp(Term *t1, Term *t2);
+
 #endif

data/ext/index_io.c

@@ -12,43 +12,37 @@ void
 frt_indexbuffer_free(void *p)
 {
   IndexBuffer *my_buf = (IndexBuffer *)p;
-  free((void *)my_buf->buffer);
+  free(my_buf->buffer);
   free(p);
 }
 
 static VALUE
 frt_indexbuffer_alloc(VALUE klass)
 {
-  byte_t *buffer;
   IndexBuffer *my_buf;
 
-  my_buf = (IndexBuffer *)ALLOC(IndexBuffer);
-  buffer = (byte_t *)ALLOC_N(byte_t, BUFFER_SIZE);
-
-  my_buf->start = 0;
-  my_buf->pos = 0;
-  my_buf->len = 0;
-  my_buf->buffer = buffer;
+  my_buf = ALLOC(IndexBuffer);
+  MEMZERO(my_buf, IndexBuffer, 1);
+  my_buf->buffer = ALLOC_N(byte_t, BUFFER_SIZE);
 
   return Data_Wrap_Struct(klass, NULL, frt_indexbuffer_free, my_buf);
 }
 
+#define GET_MY_BUF IndexBuffer *my_buf; Data_Get_Struct(self, IndexBuffer, my_buf)
 static VALUE
 frt_indexin_init_copy(VALUE self, VALUE orig)
 {
   IndexBuffer *orig_buf;
-  IndexBuffer *my_buf;
   int len;
+  GET_MY_BUF;
   if (self == orig)
     return self;
 
-  Data_Get_Struct(self, IndexBuffer, my_buf);
   Data_Get_Struct(orig, IndexBuffer, orig_buf);
 
   len = orig_buf->len;
   my_buf->len = len;
   my_buf->pos = orig_buf->pos;
-  my_buf->len = orig_buf->len;
   my_buf->start = orig_buf->start;
 
   MEMCPY(my_buf->buffer, orig_buf->buffer, byte_t, len);
@@ -59,12 +53,11 @@ frt_indexin_init_copy(VALUE self, VALUE orig)
 static VALUE
 frt_indexin_refill(VALUE self)
 {
-  IndexBuffer *my_buf;
   long start;
+  VALUE rStr;
   int stop, len_to_read;
   int input_len = FIX2INT(rb_funcall(self, frt_length, 0, NULL));
-
-  Data_Get_Struct(self, IndexBuffer, my_buf);
+  GET_MY_BUF;
 
   start = my_buf->start + my_buf->pos;
   stop = start + BUFFER_SIZE;
@@ -77,7 +70,7 @@ frt_indexin_refill(VALUE self)
     rb_raise(rb_eEOFError, "IndexInput: Read past End of File");
   }
 
-  VALUE rStr = rb_str_new((char *)my_buf->buffer, BUFFER_SIZE);
+  rStr = rb_str_new((char *)my_buf->buffer, BUFFER_SIZE);
   rb_funcall(self, frt_read_internal, 3,
       rStr, INT2FIX(0), INT2FIX(len_to_read));
 
@@ -91,23 +84,19 @@ frt_indexin_refill(VALUE self)
   return Qnil;
 }
 
-byte_t
-frt_read_byte(VALUE self)
+static inline byte_t
+frt_read_byte(VALUE self, IndexBuffer *my_buf)
 {
-  IndexBuffer *my_buf;
-  Data_Get_Struct(self, IndexBuffer, my_buf);
-
   if (my_buf->pos >= my_buf->len)
     frt_indexin_refill(self);
-
-  byte_t res = my_buf->buffer[my_buf->pos++];
-  return res;
+  return my_buf->buffer[my_buf->pos++];
 }
 
 static VALUE
 frt_indexin_read_byte(VALUE self)
 {
-  return INT2FIX(frt_read_byte(self));
+  GET_MY_BUF;
+  return INT2FIX(frt_read_byte(self, my_buf));
 }
   
 static VALUE
@@ -122,16 +111,17 @@ static VALUE
 frt_read_bytes(VALUE self, VALUE rbuffer, int offset, int len)
 {
   int i;
-  IndexBuffer *my_buf;
   VALUE rbuf = StringValue(rbuffer);
 
+  GET_MY_BUF;
+
   if (RSTRING(rbuf)->len < (offset + len)) {
     rb_str_resize(rbuf, offset + len);
   }
   if ((len + offset) < BUFFER_SIZE) {
     rb_str_modify(rbuf);
     for (i = offset; i < offset + len; i++) {
-      RSTRING(rbuf)->ptr[i] = frt_read_byte(self);
+      RSTRING(rbuf)->ptr[i] = frt_read_byte(self, my_buf);
     }
   } else {
     VALUE start = frt_indexin_pos(self);
@@ -139,8 +129,6 @@ frt_read_bytes(VALUE self, VALUE rbuffer, int offset, int len)
     rb_funcall(self, frt_read_internal, 3,
         rbuf, INT2FIX(offset), INT2FIX(len));
 
-    Data_Get_Struct(self, IndexBuffer, my_buf);
-
     my_buf->start = my_buf->start + len;
     my_buf->pos = 0;
     my_buf->len = 0;              /* trigger refill() on read() */
@@ -160,12 +148,12 @@ frt_indexin_read_bytes(VALUE self, VALUE rbuf, VALUE roffset, VALUE rlen)
   return frt_read_bytes(self, rbuf, offset, len);
 }
 
-static VALUE
+VALUE
 frt_indexin_seek(VALUE self, VALUE rpos)
 {
   int pos = FIX2INT(rpos);
-  IndexBuffer *my_buf;
-  Data_Get_Struct(self, IndexBuffer, my_buf);
+
+  GET_MY_BUF;
 
   if ((pos >= my_buf->start) && (pos < (my_buf->start + my_buf->len))) {
     my_buf->pos = pos - my_buf->start;  /* seek within buffer */
@@ -178,61 +166,65 @@ frt_indexin_seek(VALUE self, VALUE rpos)
   return Qnil;
 }
 
-static VALUE
+VALUE
 frt_indexin_read_int(VALUE self)
 {
-  return LONG2NUM(((long)frt_read_byte(self) << 24) |
-                  ((long)frt_read_byte(self) << 16) |
-                  ((long)frt_read_byte(self) << 8) |
-                   (long)frt_read_byte(self));
+  GET_MY_BUF;
+  return LONG2NUM(((long)frt_read_byte(self, my_buf) << 24) |
+                  ((long)frt_read_byte(self, my_buf) << 16) |
+                  ((long)frt_read_byte(self, my_buf) << 8) |
+                   (long)frt_read_byte(self, my_buf));
 }
 
-static VALUE
+VALUE
 frt_indexin_read_long(VALUE self)
 {
-  return LL2NUM(((long long)frt_read_byte(self) << 56) |
-                ((long long)frt_read_byte(self) << 48) |
-                ((long long)frt_read_byte(self) << 40) |
-                ((long long)frt_read_byte(self) << 32) |
-                ((long long)frt_read_byte(self) << 24) |
-                ((long long)frt_read_byte(self) << 16) |
-                ((long long)frt_read_byte(self) << 8) |
-                 (long long)frt_read_byte(self));
+  GET_MY_BUF;
+  return LL2NUM(((long long)frt_read_byte(self, my_buf) << 56) |
+                ((long long)frt_read_byte(self, my_buf) << 48) |
+                ((long long)frt_read_byte(self, my_buf) << 40) |
+                ((long long)frt_read_byte(self, my_buf) << 32) |
+                ((long long)frt_read_byte(self, my_buf) << 24) |
+                ((long long)frt_read_byte(self, my_buf) << 16) |
+                ((long long)frt_read_byte(self, my_buf) << 8) |
+                 (long long)frt_read_byte(self, my_buf));
 }
 
 static VALUE
 frt_indexin_read_uint(VALUE self)
 {
-  return ULONG2NUM(((unsigned long)frt_read_byte(self) << 24) |
-                   ((unsigned long)frt_read_byte(self) << 16) |
-                   ((unsigned long)frt_read_byte(self) << 8) |
-                    (unsigned long)frt_read_byte(self));
+  GET_MY_BUF;
+  return ULONG2NUM(((unsigned long)frt_read_byte(self, my_buf) << 24) |
+                   ((unsigned long)frt_read_byte(self, my_buf) << 16) |
+                   ((unsigned long)frt_read_byte(self, my_buf) << 8) |
+                    (unsigned long)frt_read_byte(self, my_buf));
 }
 
 static VALUE
 frt_indexin_read_ulong(VALUE self)
 {
-  return ULL2NUM(((unsigned long long)frt_read_byte(self) << 56) |
-                 ((unsigned long long)frt_read_byte(self) << 48) |
-                 ((unsigned long long)frt_read_byte(self) << 40) |
-                 ((unsigned long long)frt_read_byte(self) << 32) |
-                 ((unsigned long long)frt_read_byte(self) << 24) |
-                 ((unsigned long long)frt_read_byte(self) << 16) |
-                 ((unsigned long long)frt_read_byte(self) << 8) |
-                  (unsigned long long)frt_read_byte(self));
+  GET_MY_BUF;
+  return ULL2NUM(((unsigned long long)frt_read_byte(self, my_buf) << 56) |
+                 ((unsigned long long)frt_read_byte(self, my_buf) << 48) |
+                 ((unsigned long long)frt_read_byte(self, my_buf) << 40) |
+                 ((unsigned long long)frt_read_byte(self, my_buf) << 32) |
+                 ((unsigned long long)frt_read_byte(self, my_buf) << 24) |
+                 ((unsigned long long)frt_read_byte(self, my_buf) << 16) |
+                 ((unsigned long long)frt_read_byte(self, my_buf) << 8) |
+                  (unsigned long long)frt_read_byte(self, my_buf));
 }
 
 unsigned long long
-frt_read_vint(VALUE self)
+frt_read_vint(VALUE self, IndexBuffer *my_buf)
 {
   register unsigned long long i, b;
   register int shift = 7;
 
-  b = frt_read_byte(self);
+  b = frt_read_byte(self, my_buf);
   i = b & 0x7F; /* 0x7F = 0b01111111 */
   
   while ((b & 0x80) != 0) {/* 0x80 = 0b10000000 */
-    b = frt_read_byte(self);
+    b = frt_read_byte(self, my_buf);
     i |= (b & 0x7F) << shift;
     shift += 7;
   }
@@ -243,27 +235,34 @@ frt_read_vint(VALUE self)
 static VALUE
 frt_indexin_read_vint(VALUE self)
 {
-  return ULL2NUM(frt_read_vint(self));
+  GET_MY_BUF;
+  return ULL2NUM(frt_read_vint(self, my_buf));
 }
 
 void
 frt_read_chars(VALUE self, char* buffer, int off, int len) 
 {
-	/* byte_t b, b1, b2; */
-	int end, i;
+  /* byte_t b, b1, b2; */
+  int end, i;
+
+  GET_MY_BUF;
 
-	end = off + len;
+  end = off + len;
 
-	for(i = off; i < end; i++) {
-		buffer[i] = frt_read_byte(self);
+
+  for(i = off; i < end; i++) {
+      buffer[i] = frt_read_byte(self, my_buf);
   }
 }
 
 static VALUE
 frt_indexin_read_string(VALUE self)
 {
-  int length = (int)frt_read_vint(self);
-  char *str = (char *)ALLOC_N(char, length);
+  int length;
+  char *str;
+  GET_MY_BUF;
+  length = (int)frt_read_vint(self, my_buf);
+  str = ALLOC_N(char, length);
 
   frt_read_chars(self, str, 0, length);
 
@@ -279,8 +278,7 @@ frt_indexin_read_string(VALUE self)
 static VALUE
 frt_indexout_flush(VALUE self)
 {
-  IndexBuffer *my_buf;
-  Data_Get_Struct(self, IndexBuffer, my_buf);
+  GET_MY_BUF;
 
   rb_funcall(self, frt_flush_buffer, 2,
       rb_str_new((char *)my_buf->buffer, BUFFER_SIZE), INT2FIX(my_buf->pos));
@@ -294,8 +292,7 @@ frt_indexout_flush(VALUE self)
 static VALUE
 frt_write_byte(VALUE self, byte_t b)
 {
-  IndexBuffer *my_buf;
-  Data_Get_Struct(self, IndexBuffer, my_buf);
+  GET_MY_BUF;
 
   my_buf->buffer[my_buf->pos++] = b;
 
@@ -336,16 +333,14 @@ frt_indexout_write_bytes(VALUE self, VALUE rbuffer, VALUE rlen)
 static VALUE
 frt_indexout_pos(VALUE self)
 {
-  IndexBuffer *my_buf;
-  Data_Get_Struct(self, IndexBuffer, my_buf);
+  GET_MY_BUF;
   return INT2FIX(my_buf->start + my_buf->pos);
 }
 
 static VALUE
 frt_indexout_seek(VALUE self, VALUE pos)
 {
-  IndexBuffer *my_buf;
-  Data_Get_Struct(self, IndexBuffer, my_buf);
+  GET_MY_BUF;
 
   frt_indexout_flush(self);
   my_buf->start = FIX2INT(pos);