ferret 0.1.1 → 0.1.2

data/README

@@ -107,7 +107,7 @@ be true about Ferret. Apart from the bits about it being in Java.
 
 [<b>David Balmain</b>] Port to Ruby
 
-[<b>Doug Cutting and friends</b>] Original Java Lucene
+[The Apache Software Foundation (Doug Cutting and friends)] Original Apache Lucene
 
 == License
 

data/TODO

@@ -5,5 +5,8 @@ Send suggestions for this list to mailto:dbalmain@gmail.com
 === To Do
 
 * Add the ability to persist an in memory index to Ferret::Index::Index
-* Add unicode support
 
+=== Done
+
+* Add UTF-8 support
+* Multi Field Query

data/TUTORIAL

@@ -28,7 +28,9 @@ For more options when creating an Index refer to Ferret::Index::Index.
 
 === Adding Documents
 
-To add a document you can simply add a string or an array of strings.
+To add a document you can simply add a string or an array of strings. This will
+store all the strings in the "" (ie empty string) field (unless you specify the
+default field when you create the index).
 
   index << "This is a new document to be indexed"
   index << ["And here", "is another", "new document", "to be indexed"]
@@ -108,6 +110,12 @@ document type;
 
 NOTE: documents are indexed from 0.
 
+The default field is an empty string when you use the simple string document so
+to access those strings you'll have type;
+
+  index << "This is a document"
+  index[0][""]
+
 Let's go back to the database example above. If we store all of our documents
 with an id then we can access that field using the id. As long as we called
 our id field "id" we can do this

data/ext/ferret.c

@@ -1,18 +1,43 @@
 #include "ferret.h"
 
+/* IDs */
+ID frt_newobj;
+
+/* Modules */
+VALUE mFerret;
+VALUE mStore;
+VALUE mIndex;
+VALUE mUtils;
+VALUE mStringHelper;
+
+/* Classes */
+VALUE cRAMDirectory;
+VALUE cIndexIn;
+VALUE cBufferedIndexIn;
+VALUE cFSIndexIn;
+VALUE cIndexOut;
+VALUE cBufferedIndexOut;
+VALUE cFSIndexOut;
+VALUE cRAMIndexOut;
+VALUE cRAMIndexIn;
+VALUE cTerm;
+VALUE cTermBuffer;
+VALUE cPriorityQueue;
+VALUE cSegmentMergeQueue;
+
 void
 Init_ferret_ext(void)
 {
-  // IDs
+  /* IDs */
 	frt_newobj = rb_intern("new");
 
-  // Modules
+  /* Modules */
   mFerret = rb_define_module("Ferret");
   mStore = rb_define_module_under(mFerret, "Store");
   mIndex = rb_define_module_under(mFerret, "Index");
   mUtils = rb_define_module_under(mFerret, "Utils");
 
-  // Inits
+  /* Inits */
   Init_indexio();
   Init_term();
   Init_term_buffer();

data/ext/ferret.h

@@ -42,32 +42,32 @@ typedef struct RAMFile {
   int   length;
 } RAMFile;
 
-// IDs
-ID frt_newobj;
+/* IDs */
+extern ID frt_newobj;
 
-// Modules
-VALUE mFerret;
-VALUE mStore;
-VALUE mIndex;
-VALUE mUtils;
-VALUE mStringHelper;
+/* Modules */
+extern VALUE mFerret;
+extern VALUE mStore;
+extern VALUE mIndex;
+extern VALUE mUtils;
+extern VALUE mStringHelper;
 
-// Classes
-VALUE cRAMDirectory;
-VALUE cIndexIn;
-VALUE cBufferedIndexIn;
-VALUE cFSIndexIn;
-VALUE cIndexOut;
-VALUE cBufferedIndexOut;
-VALUE cFSIndexOut;
-VALUE cRAMIndexOut;
-VALUE cRAMIndexIn;
-VALUE cTerm;
-VALUE cTermBuffer;
-VALUE cPriorityQueue;
-VALUE cSegmentMergeQueue;
+/* Classes */
+extern VALUE cRAMDirectory;
+extern VALUE cIndexIn;
+extern VALUE cBufferedIndexIn;
+extern VALUE cFSIndexIn;
+extern VALUE cIndexOut;
+extern VALUE cBufferedIndexOut;
+extern VALUE cFSIndexOut;
+extern VALUE cRAMIndexOut;
+extern VALUE cRAMIndexIn;
+extern VALUE cTerm;
+extern VALUE cTermBuffer;
+extern VALUE cPriorityQueue;
+extern VALUE cSegmentMergeQueue;
 
-// Ferret Inits
+/* Ferret Inits */
 extern void Init_indexio();
 extern void Init_term();
 extern void Init_priority_queue();
@@ -76,7 +76,7 @@ extern void Init_segment_merge_queue();
 extern void Init_ram_directory();
 extern void Init_string_helper();
 
-// External functions
+/* External functions */
 extern int frt_hash(register char *p, register int len);
 extern unsigned long long frt_read_vint(VALUE self);
 extern void frt_read_chars(VALUE self, char *buf, int offset, int len);

data/ext/index_io.c

@@ -82,7 +82,7 @@ frt_indexin_refill(VALUE self)
       rStr, INT2FIX(0), INT2FIX(len_to_read));
 
   memcpy(my_buf->buffer, RSTRING(rStr)->ptr, BUFFER_SIZE);
-  //my_buf->buffer = StringValuePtr(rStr);
+  /* my_buf->buffer = StringValuePtr(rStr); */
 
   my_buf->len = len_to_read;
   my_buf->start = start;
@@ -143,7 +143,7 @@ frt_read_bytes(VALUE self, VALUE rbuffer, int offset, int len)
 
     my_buf->start = my_buf->start + len;
     my_buf->pos = 0;
-    my_buf->len = 0;              // trigger refill() on read()
+    my_buf->len = 0;              /* trigger refill() on read() */
   }
 
   return rbuf;
@@ -168,11 +168,11 @@ frt_indexin_seek(VALUE self, VALUE rpos)
   Data_Get_Struct(self, IndexBuffer, my_buf);
 
   if ((pos >= my_buf->start) && (pos < (my_buf->start + my_buf->len))) {
-    my_buf->pos = pos - my_buf->start;  // seek within buffer
+    my_buf->pos = pos - my_buf->start;  /* seek within buffer */
   } else {
     my_buf->start = pos;
     my_buf->pos = 0;
-    my_buf->len = 0;              // trigger refill() on read()
+    my_buf->len = 0;                    /* trigger refill() on read() */
     rb_funcall(self, frt_seek_internal, 1, rpos);
   }
   return Qnil;
@@ -229,9 +229,9 @@ frt_read_vint(VALUE self)
   register int shift = 7;
 
   b = frt_read_byte(self);
-  i = b & 0x7F; // 0x7F = 0b01111111
+  i = b & 0x7F; /* 0x7F = 0b01111111 */
   
-  while ((b & 0x80) != 0) {// 0x80 = 0b10000000
+  while ((b & 0x80) != 0) {/* 0x80 = 0b10000000 */
     b = frt_read_byte(self);
     i |= (b & 0x7F) << shift;
     shift += 7;
@@ -249,7 +249,7 @@ frt_indexin_read_vint(VALUE self)
 void
 frt_read_chars(VALUE self, char* buffer, int off, int len) 
 {
-	//byte_t b, b1, b2;
+	/* byte_t b, b1, b2; */
 	int end, i;
 
 	end = off + len;
@@ -257,21 +257,6 @@ frt_read_chars(VALUE self, char* buffer, int off, int len)
 	for(i = off; i < end; i++) {
 		buffer[i] = frt_read_byte(self);
   }
-//	for(i = off; i < end; i++){
-//		b = frt_read_byte(self);
-//		if((b & 0x80) == 0){
-//			buffer[i] = (char)(b & 0x7F);
-//		} else {
-//			if((b & 0xE0) != 0xE0){
-//				b1 = frt_read_byte(self);
-//				buffer[i] = (char)(((b & 0x1F) << 6) | (b1 & 0x3F));
-//			} else{
-//				b1 = frt_read_byte(self);
-//				b2 = frt_read_byte(self);
-//				buffer[i] = (char)(((b & 0x0F) << 12) | ((b1 & 0x3F) << 6) | (b2 & 0x3F));
-//			}
-//		}
-//	}
 }
 
 static VALUE
@@ -412,7 +397,7 @@ static VALUE
 frt_indexout_write_ulong(VALUE self, VALUE rulong)
 {
   unsigned long long l;
-  l = rb_num2ull(rulong); // ruby 1.8 doesn't have NUM2ULL. Added in 1.9
+  l = rb_num2ull(rulong); /* ruby 1.8 doesn't have NUM2ULL. Added in 1.9 */
   frt_write_byte(self, (l >> 56) & 0xFF);
   frt_write_byte(self, (l >> 48) & 0xFF);
   frt_write_byte(self, (l >> 40) & 0xFF);
@@ -492,13 +477,13 @@ frt_indexout_write_string(VALUE self, VALUE rstr)
 void
 Init_indexio(void)
 {
-  // IDs
+  /* IDs */
   frt_length = rb_intern("length");
   frt_flush_buffer = rb_intern("flush_buffer");
   frt_read_internal = rb_intern("read_internal");
   frt_seek_internal = rb_intern("seek_internal");
 
-  // IndexInput
+  /* IndexInput */
   cIndexIn = rb_define_class_under(mStore, "IndexInput", rb_cObject);
   cBufferedIndexIn = rb_define_class_under(mStore, "BufferedIndexInput", cIndexIn);
   rb_define_alloc_func(cBufferedIndexIn, frt_indexbuffer_alloc);
@@ -518,7 +503,7 @@ Init_indexio(void)
   rb_define_method(cBufferedIndexIn, "read_string", frt_indexin_read_string, 0);
   rb_define_method(cBufferedIndexIn, "read_chars", frt_indexin_read_bytes, 3);
 
-  // IndexOutput
+  /* IndexOutput */
   cIndexOut = rb_define_class_under(mStore, "IndexOutput", rb_cObject);
   cBufferedIndexOut = rb_define_class_under(mStore, "BufferedIndexOutput", cIndexOut);
   rb_define_alloc_func(cBufferedIndexOut, frt_indexbuffer_alloc);
@@ -538,6 +523,6 @@ Init_indexio(void)
   rb_define_method(cBufferedIndexOut, "write_chars", frt_indexout_write_chars, 3);
   rb_define_method(cBufferedIndexOut, "write_string", frt_indexout_write_string, 1);
 
-  // FSIndexInput
-  //cFSIndexIn = rb_define_class_under(mStore, "FSIndexInput", cBufferedIndexIn);
+  /* FSIndexInput */
+  /*cFSIndexIn = rb_define_class_under(mStore, "FSIndexInput", cBufferedIndexIn); */
 }

data/ext/ram_directory.c

@@ -101,7 +101,7 @@ frt_rio_flush_buffer(VALUE self, VALUE rsrc, VALUE rlen)
   int buffer_number, buffer_offset, bytes_in_buffer, bytes_to_copy;
   int src_offset;
   int len = FIX2INT(rlen);
-  //char *src = StringValuePtr(rsrc);
+  /* char *src = StringValuePtr(rsrc); */
   int pointer = FIX2INT(rb_iv_get(self, "pointer"));
  
 	VALUE file = rb_iv_get(self, "file");
@@ -275,26 +275,26 @@ frt_rii_close(VALUE self)
 void
 Init_ram_directory(void)
 {
-  // IDs
+  /* IDs */
   flush = rb_intern("flush");
   seek = rb_intern("seek");
 
-  // RAMDirectory
+  /* RAMDirectory */
   VALUE cDirectory = rb_define_class_under(mStore, "Directory", rb_cObject);
   cRAMDirectory = rb_define_class_under(mStore, "RAMDirectory", cDirectory);
 
-  // RAMFile
+  /* RAMFile */
   VALUE cRAMFile = rb_define_class_under(cRAMDirectory, "RAMFile", rb_cObject);
   rb_define_alloc_func(cRAMFile, frt_rf_alloc);
 
-  // Methods
+  /* Methods */
   rb_define_method(cRAMFile, "length", frt_rf_length, 0);
 
-  // RAMIndexOutput
+  /* RAMIndexOutput */
   cRAMIndexOut = rb_define_class_under(cRAMDirectory, "RAMIndexOutput", cBufferedIndexOut);
-  //rb_define_alloc_func(cRAMIndexOut, frt_ramio_alloc);
+  /*rb_define_alloc_func(cRAMIndexOut, frt_ramio_alloc); */
 
-  // Methods
+  /* Methods */
   rb_define_method(cRAMIndexOut, "initialize", frt_rio_init, 1);
   rb_define_method(cRAMIndexOut, "length", frt_rio_length, 0);
   rb_define_method(cRAMIndexOut, "flush_buffer", frt_rio_flush_buffer, 2);
@@ -303,11 +303,11 @@ Init_ram_directory(void)
   rb_define_method(cRAMIndexOut, "close", frt_rio_close, 0);
   rb_define_method(cRAMIndexOut, "write_to", frt_rio_write_to, 1);
 
-  // RAMIndexInput
+  /* RAMIndexInput */
   cRAMIndexIn = rb_define_class_under(cRAMDirectory, "RAMIndexInput", cBufferedIndexIn);
-  //rb_define_alloc_func(cRAMIndexIn, frt_ramio_alloc);
+  /*rb_define_alloc_func(cRAMIndexIn, frt_ramio_alloc); */
 
-  // Methods
+  /* Methods */
   rb_define_method(cRAMIndexIn, "initialize", frt_rii_init, 1);
   rb_define_method(cRAMIndexIn, "length", frt_rii_length, 0);
   rb_define_method(cRAMIndexIn, "read_internal", frt_rii_read_internal, 3);

data/ext/segment_merge_queue.c

@@ -30,11 +30,11 @@ frt_smq_less_than(VALUE self, VALUE rsti1, VALUE rsti2)
 void
 Init_segment_merge_queue(void)
 {
-  // IDs
+  /* IDs */
   eq = rb_intern("==");
   lt = rb_intern("<");
 
-  // SegmentMergeQueue
+  /* SegmentMergeQueue */
 	cSegmentMergeQueue = rb_define_class_under(mIndex, "SegmentMergeQueue", cPriorityQueue);
 
 	rb_define_method(cSegmentMergeQueue, "less_than", frt_smq_less_than, 2);

data/ext/string_helper.c

@@ -35,7 +35,7 @@ frt_sh_string_difference(VALUE self, VALUE rstr1, VALUE rstr2)
 void
 Init_string_helper(void)
 {
-	// StringHelper
+	/* StringHelper */
   mStringHelper = rb_define_module_under(mUtils, "StringHelper");
 
 	rb_define_method(mStringHelper, "string_difference", frt_sh_string_difference, 2);

data/ext/term.c

@@ -145,13 +145,15 @@ frt_term_compare_to(VALUE self, VALUE rother)
 		} else
 			comp = mylen > olen ? 1 : -1;
 	}
-	//comp = strcmp(term->field, other->field);
-	//if(comp == 0)
-	//	comp = strcmp(term->text, other->text);
+  /*
+	comp = strcmp(term->field, other->field);
+	if(comp == 0)
+		comp = strcmp(term->text, other->text);
+  */
 	return INT2FIX(comp);
 }
 
-// keep in synch with fuction above
+/* keep in synch with fuction above */
 int
 frt_term_compare_to_int(VALUE self, VALUE rother)
 {
@@ -175,9 +177,11 @@ frt_term_compare_to_int(VALUE self, VALUE rother)
 		} else
 			comp = mylen > olen ? 1 : -1;
 	}
-	//comp = strcmp(term->field, other->field);
-	//if(comp == 0)
-	//	comp = strcmp(term->text, other->text);
+  /*
+	comp = strcmp(term->field, other->field);
+	if(comp == 0)
+		comp = strcmp(term->text, other->text);
+  */
 	return comp;
 }
 
@@ -214,11 +218,13 @@ frt_term_eq(VALUE self, VALUE rother)
 }
 
 
-//static VALUE
-//frt_term_compare_to(VALUE self, VALUE other)
-//{
-//	return INT2FIX(frt_term_compare_to_int(self, other));
-//}
+/*
+static VALUE
+frt_term_compare_to(VALUE self, VALUE other)
+{
+	return INT2FIX(frt_term_compare_to_int(self, other));
+}
+*/
 
 static VALUE
 frt_term_hash(VALUE self)
@@ -238,7 +244,7 @@ frt_term_hash(VALUE self)
 void
 Init_term(void)
 {
-	//Term
+	/* Term */
 	cTerm = rb_define_class_under(mIndex, "Term", rb_cObject);
 	rb_define_alloc_func(cTerm, frt_term_alloc);
 	rb_include_module(cTerm, rb_mComparable);

data/ext/term_buffer.c

@@ -267,15 +267,15 @@ frt_termbuffer_hash(VALUE self)
 
 void 
 Init_term_buffer(void) {
-  // IDs
+  /* IDs */
 	field_name = rb_intern("name");
 
-	// TermBuffer
+	/* TermBuffer */
 	cTermBuffer = rb_define_class_under(mIndex, "TermBuffer", rb_cObject);
 	rb_define_alloc_func(cTermBuffer, frt_termbuffer_alloc);
 	rb_include_module(cTermBuffer, rb_mComparable);
 
-  // Methods
+  /* Methods */
 	rb_define_method(cTermBuffer, "initialize", frt_termbuffer_init, 0);
 	rb_define_method(cTermBuffer, "initialize_copy", frt_termbuffer_init_copy, 1);
 	rb_define_method(cTermBuffer, "text", frt_termbuffer_get_text, 0);

data/lib/ferret.rb

@@ -22,7 +22,7 @@
 #++
 # :include: ../TUTORIAL
 module Ferret
-  VERSION = '0.1.1'
+  VERSION = '0.1.2'
 end
 
 require 'ferret/utils'

data/lib/ferret/analysis/standard_tokenizer.rb

@@ -12,7 +12,7 @@ module Ferret::Analysis
   # words correctly as well as tokenizing things like email addresses, web
   # addresses, phone numbers, etc.
 
-  class StandardTokenizer < RETokenizer
+  class StandardTokenizer < RegExpTokenizer
     ALPHA      = /[[:alpha:]]+/
     APOSTROPHE = /#{ALPHA}('#{ALPHA})+/
     ACRONYM    = /#{ALPHA}\.(#{ALPHA}\.)+/