ferret 0.9.1 → 0.9.2

data/README

@@ -12,17 +12,18 @@ search for things in them later.
 
 == Installation
 
-If you have gems installed you can simple do;
+If you have gems installed you can simply do;
 
   gem install ferret
 
-Otherwise, de-compress the archive and enter its top directory.
+Otherwise, you will need Rake installed. De-compress the archive and enter its top directory.
 
-  tar zxpvf ferret-0.1.tar.gz
-  cd ferret-0.1
+  tar zxpvf ferret-<version>.tar.gz
+  cd ferret-<version>
 
-Run the setup config;
+Run the following;
 
+  $ rake ext
   $ ruby setup.rb config
   $ ruby setup.rb setup
   # ruby setup.rb install

data/Rakefile

@@ -33,16 +33,25 @@ $VERBOSE = nil
 
 EXT = "ferret_ext.so"
 EXT_SRC = FileList["src/**/*.[ch]"]
+if (/mswin/ =~ RUBY_PLATFORM)
+  EXT_SRC.delete('src/io/nix_io.c')
+end
 
 EXT_SRC_DEST = EXT_SRC.map {|fn| File.join("ext", File.basename(fn))}
 SRC = (FileList["ext/*.[ch]"] + EXT_SRC_DEST).uniq
 
-CLEAN.include(FileList['**/*.o', 'InstalledFiles', '.config'])
+CLEAN.include(FileList['**/*.o', '**/*.obj', 'InstalledFiles', '.config'])
 CLOBBER.include(FileList['**/*.so'], 'ext/Makefile', EXT_SRC_DEST)
+POLISH = Rake::FileList.new.include(FileList['**/*.so'], 'ext/Makefile')
+
+desc "Clean specifically for the release."
+task :polish => [:clean] do
+  POLISH.each { |fn| rm_r fn rescue nil }
+end
 
-task :default => :all_tests
+task :default => :test_all
 desc "Run all tests"
-task :all_tests => [ :test_runits, :test_cunits, :test_functional ]
+task :test_all => [ :test_runits, :test_cunits, :test_functional ]
 
 desc "Generate API documentation, and show coding stats"
 task :doc => [ :stats, :appdoc ]
@@ -121,7 +130,13 @@ task :ext => ["ext/#{EXT}"] + SRC
 file "ext/#{EXT}" => ["ext/Makefile"] do
   cp "ext/inc/lang.h", "ext/lang.h"
   cp "ext/inc/except.h", "ext/except.h"
-  sh "cd ext; make"
+  cd "ext"
+  if (/mswin/ =~ RUBY_PLATFORM) and ENV['make'].nil?
+    sh "nmake"
+  else
+    sh "make"
+  end
+  cd ".."
 end
 
 file "ext/lang.h" => ["ext/inc/lang.h"] do
@@ -132,7 +147,9 @@ file "ext/except.h" => ["ext/inc/except.h"] do
 end
 
 file "ext/Makefile" => SRC do
-  sh "cd ext; ruby extconf.rb"
+  cd "ext"
+  `ruby extconf.rb`
+  cd ".."
 end
 
 # Make Parsers ---------------------------------------------------------------
@@ -158,6 +175,9 @@ PKG_FILES = FileList[
   'Rakefile'
 ]
 PKG_FILES.exclude('**/*.o')
+PKG_FILES.include('ext/termdocs.c')
+PKG_FILES.exclude('**/Makefile')
+PKG_FILES.exclude('ext/ferret_ext.so')
 
 
 if ! defined?(Gem)
@@ -233,12 +253,13 @@ end
 # Creating a release
 
 desc "Make a new release"
-task :prerelease => [:all_tests, :clobber]
-task :repackage => EXT_SRC_DEST
-task :package => EXT_SRC_DEST
-task :tag => [:prerelease]
-task :update_version => [:prerelease]
-task :release do #=> [:tag, :update_version, :package] do
+task :release => [
+  :prerelease,
+  :polish,
+  :test_all,
+  :update_version,
+  :package,
+  :tag] do
   announce 
   announce "**************************************************************"
   announce "* Release #{PKG_VERSION} Complete."
@@ -288,6 +309,7 @@ def reversion(fn)
       end
     end
   end
+  mv fn + ".new", fn
 end
 
 task :update_version => [:prerelease] do
@@ -300,9 +322,8 @@ task :update_version => [:prerelease] do
     if ENV['RELTEST']
       announce "Release Task Testing, skipping commiting of new version"
     else
-      mv "lib/rferret.rb.new", "lib/rferret.rb"
+      sh %{svn ci -m "Updated to version #{PKG_VERSION}" lib/rferret.rb}
     end
-    sh %{svn ci -m "Updated to version #{PKG_VERSION}" lib/rferret.rb}
   end
 end
 

data/TODO

@@ -5,6 +5,7 @@ Send suggestions for this list to mailto:dbalmain@gmail.com
 === To Do
 
 * Make a dll for people on Windows
+* fix rb_obj_as_string to StringValue()
 * pure ruby ConstantScoreQuery
 
 === Done

data/TUTORIAL

@@ -22,7 +22,7 @@ search for later. If you'd like to use a different analyzer you can specify it
 here, eg;
 
   index = Index::Index.new(:path => '/path/to/index',
-                           :analyzer => WhiteSpaceAnalyzer.new)
+                           :analyzer => Analysis::WhiteSpaceAnalyzer.new)
 
 For more options when creating an Index refer to Ferret::Index::Index.
 

data/ext/analysis.c

@@ -1,10 +1,11 @@
-#include <analysis.h>
+#include "analysis.h"
+#include "hash.h"
+#include "libstemmer.h"
 #include <string.h>
 #include <ctype.h>
 #include <wctype.h>
 #include <wchar.h>
-#include "hash.h"
-#include "libstemmer.h"
+
 
 /****************************************************************************
  *
@@ -22,9 +23,16 @@ void tk_destroy(void *p)
   free(p);
 }
 
-inline Token *tk_set(Token *tk, char *text, int tlen, int start, int end, int pos_inc)
+inline Token *tk_set(Token *tk, 
+					 char *text, 
+					 int tlen, 
+					 int start, 
+					 int end, 
+					 int pos_inc)
 {
-  if (tlen >= MAX_WORD_SIZE) tlen = MAX_WORD_SIZE - 1;
+  if (tlen >= MAX_WORD_SIZE) {
+    tlen = MAX_WORD_SIZE - 1;
+  }
   memcpy(tk->text, text, sizeof(char) * tlen);
   tk->text[tlen] = '\0';
   tk->start = start;
@@ -33,14 +41,23 @@ inline Token *tk_set(Token *tk, char *text, int tlen, int start, int end, int po
   return tk;
 }
 
-inline Token *tk_set_ts(Token *tk, char *start, char *end, char *text, int pos_inc)
+inline Token *tk_set_ts(Token *tk, 
+						char *start, 
+						char *end, 
+						char *text, 
+						int pos_inc)
 {
-  return tk_set(tk, start, end - start, start - text, end - text, pos_inc);
+  return tk_set(tk, start, (int)(end - start), 
+	  (int)(start - text), (int)(end - text), pos_inc);
 }
 
-inline Token *tk_set_no_len(Token *tk, char *text, int start, int end, int pos_inc)
+inline Token *tk_set_no_len(Token *tk, 
+							char *text, 
+							int start, 
+							int end, 
+							int pos_inc)
 {
-  return tk_set(tk, text, strlen(text), start, end, pos_inc);
+  return tk_set(tk, text, (int)strlen(text), start, end, pos_inc);
 }
 
 int tk_eq(Token *tk1, Token *tk2)
@@ -75,9 +92,14 @@ int tk_cmp(Token *tk1, Token *tk2)
  *
  ****************************************************************************/
 
-void ts_standard_destroy(void *p)
+void ts_deref(void *p)
 {
   TokenStream *ts = (TokenStream *)p;
+  if (--ts->ref_cnt <= 0) ts->destroy(ts);
+}
+
+void ts_standard_destroy(TokenStream *ts)
+{
   tk_destroy(ts->token);
   free(ts);
 }
@@ -89,13 +111,11 @@ void ts_reset(TokenStream *ts, char *text)
 
 TokenStream *ts_create()
 {
-  TokenStream *ts = ALLOC(TokenStream);
-  ts->text = NULL;
+  TokenStream *ts = ALLOC_AND_ZERO_N(TokenStream, 1);
   ts->token = tk_create();
   ts->destroy = &ts_standard_destroy;
   ts->reset = &ts_reset;
-  ts->sub_ts = NULL;
-  ts->clone_i = NULL;
+  ts->ref_cnt = 1;
   return ts;
 }
 
@@ -109,6 +129,7 @@ TokenStream *ts_clone(TokenStream *orig_ts)
   }
   if (orig_ts->sub_ts) ts->sub_ts = ts_clone(orig_ts->sub_ts);
   if (orig_ts->clone_i) orig_ts->clone_i(orig_ts, ts);
+  ts->ref_cnt = 1;
   return ts;
 }
 
@@ -116,7 +137,7 @@ TokenStream *ts_clone(TokenStream *orig_ts)
 static char * const ENC_ERR_MSG = "Error decoding input string. "
                           "Check that you have the locale set correctly";
 #define MB_NEXT_CHAR \
-  if ((i = mbrtowc(&wchr, t, MB_CUR_MAX, (mbstate_t *)ts->data)) < 0)\
+  if ((i = (int)mbrtowc(&wchr, t, MB_CUR_MAX, (mbstate_t *)ts->data)) < 0)\
     RAISE(IO_ERROR, ENC_ERR_MSG)
 
 inline Token *w_tk_set(Token *tk, wchar_t *text, int start, int end, int pos_inc)
@@ -128,9 +149,8 @@ inline Token *w_tk_set(Token *tk, wchar_t *text, int start, int end, int pos_inc
   return tk;
 }
 
-void mb_ts_standard_destroy(void *p)
+void mb_ts_standard_destroy(TokenStream *ts)
 {
-  TokenStream *ts = (TokenStream *)p;
   tk_destroy(ts->token);
   free(ts->data);
   free(ts);
@@ -150,14 +170,13 @@ void mb_ts_clone_i(TokenStream *orig_ts, TokenStream *new_ts)
 
 TokenStream *mb_ts_create()
 {
-  TokenStream *ts = ALLOC(TokenStream);
+  TokenStream *ts = ALLOC_AND_ZERO_N(TokenStream, 1);
   ts->data = ALLOC(mbstate_t);
-  ts->text = NULL;
   ts->token = tk_create();
   ts->destroy = &mb_ts_standard_destroy;
   ts->reset = &mb_ts_reset;
   ts->clone_i = &mb_ts_clone_i;
-  ts->sub_ts = NULL;
+  ts->ref_cnt = 1;
   return ts;
 }
 
@@ -167,11 +186,16 @@ TokenStream *mb_ts_create()
  *
  ****************************************************************************/
 
-void a_standard_destroy(void *p)
+void a_deref(void *p)
 {
   Analyzer *a = (Analyzer *)p;
-  ts_destroy(a->current_ts);
-  free(p);
+  if (--a->ref_cnt <= 0) a->destroy(a);
+}
+
+void a_standard_destroy(Analyzer *a)
+{
+  if (a->current_ts) ts_deref(a->current_ts);
+  free(a);
 }
 
 TokenStream *a_standard_get_ts(Analyzer *a, char *field, char *text)
@@ -180,7 +204,8 @@ TokenStream *a_standard_get_ts(Analyzer *a, char *field, char *text)
   return a->current_ts; 
 }
 
-Analyzer *analyzer_create(void *data, TokenStream *ts, void (*destroy)(void *),
+Analyzer *analyzer_create(void *data, TokenStream *ts,
+    void (*destroy)(Analyzer *a),
     TokenStream *(*get_ts)(Analyzer *a, char *field, char *text))
 {
   Analyzer *a = ALLOC(Analyzer);
@@ -188,6 +213,7 @@ Analyzer *analyzer_create(void *data, TokenStream *ts, void (*destroy)(void *),
   a->current_ts = ts;
   a->destroy = (destroy ? destroy : &a_standard_destroy);
   a->get_ts = (get_ts ? get_ts : &a_standard_get_ts);
+  a->ref_cnt = 1;
   return a;
 }
 
@@ -284,7 +310,7 @@ Token *mb_wst_next_lc(TokenStream *ts)
     MB_NEXT_CHAR;
   }
   *w = 0;
-  w_tk_set(ts->token, wbuf, start - ts->text, t - ts->text, 1);
+  w_tk_set(ts->token, wbuf, (int)(start - ts->text), (int)(t - ts->text), 1);
   ts->t = t;
   return ts->token;
 }
@@ -409,7 +435,7 @@ Token *mb_lt_next_lc(TokenStream *ts)
     MB_NEXT_CHAR;
   }
   *w = 0;
-  w_tk_set(ts->token, wbuf, start - ts->text, t - ts->text, 1);
+  w_tk_set(ts->token, wbuf, (int)(start - ts->text), (int)(t - ts->text), 1);
   ts->t = t;
   return ts->token;
 }
@@ -472,7 +498,7 @@ int mb_std_get_alpha(TokenStream *ts, char *token)
     if ((i = mbtowc(&w, t, MB_CUR_MAX)) < 0) RAISE(IO_ERROR, ENC_ERR_MSG);
   }
 
-  i = t - ts->t;
+  i = (int)(t - ts->t);
   if (i > MAX_WORD_SIZE) i = MAX_WORD_SIZE - 1;
   memcpy(token, ts->t, i);
   return i;
@@ -500,7 +526,7 @@ int mb_std_get_alnum(char *text, char *token, TokenStream *ts)
     if ((i = mbtowc(&w, t, MB_CUR_MAX)) < 0) RAISE(IO_ERROR, ENC_ERR_MSG);
   }
 
-  i = t - ts->t;
+  i = (int)(t - ts->t);
   if (i > MAX_WORD_SIZE) i = MAX_WORD_SIZE - 1;
   memcpy(token, ts->t, i);
   return i;
@@ -599,7 +625,7 @@ int std_get_apostrophe(char *input)
   while (isalpha(*t) || *t == '\'')
     t++;
 
-  return t - input;
+  return (int)(t - input);
 }
 
 int mb_std_get_apostrophe(char *input)
@@ -613,7 +639,7 @@ int mb_std_get_apostrophe(char *input)
     t += i;
     if ((i = mbtowc(&w, t, MB_CUR_MAX)) < 0) RAISE(IO_ERROR, ENC_ERR_MSG);
   }
-  return t - input;
+  return (int)(t - input);
 }
 
 int std_get_url(char *input, char *token, int i)
@@ -654,7 +680,7 @@ int mb_std_get_company_name(char *input, TokenStream *ts)
     MB_NEXT_CHAR;
   }
 
-  return t - input;
+  return (int)(t - input);
 }
 
 bool std_advance_to_start(TokenStream *ts)
@@ -723,7 +749,7 @@ Token *std_next(TokenStream *ts)
     if (*t == '\'') { // apostrophe case. 
       t += std_tz->get_apostrophe(t);
       ts->t = t;
-      len = t - start;
+      len = (int)(t - start);
       // strip possesive
       if ((t[-1] == 's' || t[-1] == 'S') && t[-2] == '\'') t -= 2;
 
@@ -760,13 +786,14 @@ Token *std_next(TokenStream *ts)
            memcmp(token, "file", 4) == 0)) {
         len = std_get_url(t, token, 0); // dispose of first part of the URL
       } else { //still treat as url but keep the first part
-        token_i = t - start;
+        token_i = (int)(t - start);
         memcpy(token, start, token_i * sizeof(char));
         len = token_i + std_get_url(t, token, token_i); // keep start
       }
       ts->t = t + len;
       token[len] = 0;
-      tk_set(ts->token, token, len, start - ts->text, ts->t - ts->text, 1);
+      tk_set(ts->token, token, len, (int)(start - ts->text), 
+		  (int)(ts->t - ts->text), 1);
       return ts->token;
     }
 
@@ -806,7 +833,8 @@ Token *std_next(TokenStream *ts)
             token_i++;
           }
         }
-        tk_set(ts->token, token, token_i, start - ts->text, t - ts->text, 1);
+        tk_set(ts->token, token, token_i, (int)(start - ts->text), 
+			(int)(t - ts->text), 1);
       } else { // just return the url as is
         tk_set_ts(ts->token, start, t, ts->text, 1);
       }
@@ -819,9 +847,8 @@ Token *std_next(TokenStream *ts)
   return ts->token;
 }
 
-void std_ts_destroy(void *p)
+void std_ts_destroy(TokenStream *ts)
 {
-  TokenStream *ts = (TokenStream *)p;
   free(ts->data);
   ts_standard_destroy(ts);
 }
@@ -871,19 +898,18 @@ void filter_reset(TokenStream *ts, char *text)
   ts->sub_ts->reset(ts->sub_ts, text);
 }
 
-void filter_destroy(void *p)
+void filter_destroy(TokenStream *tf)
 {
-  TokenStream *tf = (TokenStream *)p;
-  if (tf->destroy_sub) tf->sub_ts->destroy(tf->sub_ts);
+  ts_deref(tf->sub_ts);
   if (tf->token != NULL) tk_destroy(tf->token);
   free(tf);
 }
 
-void sf_destroy(void *p)
+void sf_destroy(TokenStream *tf)
 {
-  HshTable *words = (HshTable *)((TokenStream *)p)->data;
+  HshTable *words = (HshTable *)tf->data;
   h_destroy(words);
-  filter_destroy(p);
+  filter_destroy(tf);
 }
 
 void sf_clone_i_i(void *key, void *value, void *arg)
@@ -917,10 +943,10 @@ TokenStream *stop_filter_create_with_words_len(TokenStream *ts,
 {
   int i;
   char *w;
+  HshTable *wordtable = h_new_str(&free, (free_ft)NULL);
   TokenStream *tf = ALLOC(TokenStream);
   tf->sub_ts = ts;
-  tf->destroy_sub = true;
-  HshTable *wordtable = h_new_str(&free, NULL);
+
   for (i = 0; i < len; i++) {
     w = estrdup(words[i]);
     h_set(wordtable, w, w);
@@ -931,16 +957,16 @@ TokenStream *stop_filter_create_with_words_len(TokenStream *ts,
   tf->reset = &filter_reset;
   tf->destroy = &sf_destroy;
   tf->clone_i = &sf_clone_i;
+  tf->ref_cnt = 1;
   return tf;
 }
 
 TokenStream *stop_filter_create_with_words(TokenStream *ts, const char **words)
 {
   char *w;
+  HshTable *wordtable = h_new_str(&free, (free_ft)NULL);
   TokenStream *tf = ALLOC(TokenStream);
   tf->sub_ts = ts;
-  tf->destroy_sub = true;
-  HshTable *wordtable = h_new_str(&free, NULL);
   while (*words) {
     w = estrdup(*words);
     h_set(wordtable, w, w);
@@ -952,6 +978,7 @@ TokenStream *stop_filter_create_with_words(TokenStream *ts, const char **words)
   tf->reset = &filter_reset;
   tf->destroy = &sf_destroy;
   tf->clone_i = &sf_clone_i;
+  tf->ref_cnt = 1;
   return tf;
 }
 
@@ -968,7 +995,7 @@ Token *mb_lcf_next(TokenStream *ts)
   Token *tk = ts->sub_ts->next(ts->sub_ts);
   if (tk == NULL) return tk;
 
-  i = mbstowcs(wbuf, tk->text, MAX_WORD_SIZE);
+  i = (int)mbstowcs(wbuf, tk->text, MAX_WORD_SIZE);
   w = wbuf;
   while (*w != 0) {
     *w = towlower(*w);
@@ -986,8 +1013,8 @@ TokenStream *mb_lowercase_filter_create(TokenStream *ts)
   tf->reset = &filter_reset;
   tf->destroy = &filter_destroy;
   tf->sub_ts = ts;
-  tf->destroy_sub = true;
   tf->clone_i = NULL;
+  tf->ref_cnt = 1;
   return tf;
 }
 
@@ -1011,8 +1038,8 @@ TokenStream *lowercase_filter_create(TokenStream *ts)
   tf->reset = &filter_reset;
   tf->destroy = &filter_destroy;
   tf->sub_ts = ts;
-  tf->destroy_sub = true;
   tf->clone_i = NULL;
+  tf->ref_cnt = 1;
   return tf;
 }
 
@@ -1022,15 +1049,14 @@ typedef struct StemFilter {
   char *charenc;
 } StemFilter;
 
-void stemf_destroy(void *p)
+void stemf_destroy(TokenStream *tf)
 {
-  TokenStream *ts = (TokenStream *)p;
-  StemFilter *stemf = (StemFilter *)ts->data;
+  StemFilter *stemf = (StemFilter *)tf->data;
   sb_stemmer_delete(stemf->stemmer);
   free(stemf->algorithm);
   free(stemf->charenc);
   free(stemf);
-  filter_destroy(ts);
+  filter_destroy(tf);
 }
 
 Token *stemf_next(TokenStream *ts)
@@ -1040,7 +1066,7 @@ Token *stemf_next(TokenStream *ts)
   struct sb_stemmer *stemmer = ((StemFilter *)ts->data)->stemmer;
   Token *tk = ts->sub_ts->next(ts->sub_ts);
   if (tk == NULL) return tk;
-  stemmed = sb_stemmer_stem(stemmer, (sb_symbol *)tk->text, strlen(tk->text));
+  stemmed = sb_stemmer_stem(stemmer, (sb_symbol *)tk->text, (int)strlen(tk->text));
   len = sb_stemmer_length(stemmer);
   if (len >= MAX_WORD_SIZE) len = MAX_WORD_SIZE - 1;
   memcpy(tk->text, stemmed, len);
@@ -1074,7 +1100,7 @@ TokenStream *stem_filter_create(TokenStream *ts, const char * algorithm,
   tf->destroy = &stemf_destroy;
   tf->clone_i = &stemf_clone_i;
   tf->sub_ts = ts;
-  tf->destroy_sub = true;
+  tf->ref_cnt = 1;
   return tf;
 }
 
@@ -1148,19 +1174,12 @@ Analyzer *mb_standard_analyzer_create(bool lowercase)
  *
  ****************************************************************************/
 
-typedef struct PerFieldAnalyzer {
-  HshTable *dict;
-  Analyzer *def;
-  bool destroy_subs : 1;
-} PerFieldAnalyzer;
-
-void pfa_destroy(void *p)
+void pfa_destroy(Analyzer *self)
 {
-  Analyzer *self = (Analyzer *)p;
   PerFieldAnalyzer *pfa = (PerFieldAnalyzer *)self->data;
   h_destroy(pfa->dict);
 
-  if (pfa->destroy_subs) a_destroy(pfa->def);
+  a_deref(pfa->def);
   free(pfa);
   free(self);
 }
@@ -1176,7 +1195,7 @@ TokenStream *pfa_get_ts(Analyzer *self, char *field, char *text)
 void pfa_sub_a_destroy(void *p)
 {
   Analyzer *a = (Analyzer *)p;
-  a->destroy(a);
+  a_deref(a);
 }
 
 void pfa_add_field(Analyzer *self, char *field, Analyzer *analyzer)
@@ -1185,13 +1204,11 @@ void pfa_add_field(Analyzer *self, char *field, Analyzer *analyzer)
   h_set(pfa->dict, estrdup(field), analyzer);
 }
 
-Analyzer *per_field_analyzer_create(Analyzer *def, bool destroy_subs)
+Analyzer *per_field_analyzer_create(Analyzer *def)
 {
   PerFieldAnalyzer *pfa = ALLOC(PerFieldAnalyzer);
   pfa->def = def;
-  pfa->destroy_subs = destroy_subs;
-  pfa->dict = destroy_subs ? h_new_str(&free, &pfa_sub_a_destroy)
-                           : h_new_str(&free, NULL);
+  pfa->dict = h_new_str(&free, &pfa_sub_a_destroy);
   return analyzer_create(pfa, NULL, &pfa_destroy, &pfa_get_ts);
 }