ferret 0.3.2 → 0.9.0

data/CHANGELOG

@@ -0,0 +1,9 @@
+20060316:
+  * changed Token#term_text to Token#text
+  * changed Token#position_increment to Term#pos_inc
+  * changed order of args to Token.new. Now Term.new(text, start_offset,
+    end_offset, pos_inc=1, type="text"). NOTE: type does nothing.
+  * changed TermVectorOffsetInfo#start_offset to TermVectorOffsetInfo#start
+  * changed TermVectorOffsetInfo#end_offset to TermVectorOffsetInfo#end
+  * added :id_field option to Index::Index class.
+

data/Rakefile

@@ -9,7 +9,7 @@ require 'rake/testtask'
 require 'rake/rdoctask'
 require 'rake/clean'
 require 'rake_utils/code_statistics'
-require 'lib/ferret'
+require 'lib/rferret'
 
 begin
   require 'rubygems'
@@ -30,18 +30,32 @@ def announce(msg='')
 end
 
 $VERBOSE = nil
+
+EXT = "ferret_ext.so"
+EXT_SRC = FileList["src/*/*.[ch]"]
+EXT_SRC_DEST = EXT_SRC.map {|fn| File.join("ext", File.basename(fn))}
+SRC = (FileList["ext/*.[ch]"] + EXT_SRC_DEST).uniq
+
 CLEAN.include(FileList['**/*.o', 'InstalledFiles', '.config'])
 CLOBBER.include(FileList['**/*.so'], 'ext/Makefile')
 
 task :default => :all_tests
 desc "Run all tests"
-task :all_tests => [ :test_units, :test_functional ]
+task :all_tests => [ :test_runits, :test_cunits, :test_functional ]
 
 desc "Generate API documentation, and show coding stats"
 task :doc => [ :stats, :appdoc ]
 
-desc "run unit tests in test/unit"
-Rake::TestTask.new("test_units" => :parsers) do |t|
+desc "run unit tests in test/unit for pure ruby ferret"
+Rake::TestTask.new("test_runits" => :parsers) do |t|
+  t.ruby_opts = ["-r 'lib/rferret'"]
+  t.libs << "test/unit"
+  t.pattern = 'test/unit/ts_*.rb'
+  t.verbose = true
+end
+
+desc "run unit tests in test/unit for C ferret"
+Rake::TestTask.new("test_cunits" => :ext) do |t|
   t.libs << "test/unit"
   t.pattern = 'test/unit/t[cs]_*.rb'
   t.verbose = true
@@ -84,22 +98,28 @@ rd = Rake::RDocTask.new("appdoc") do |rdoc|
   rdoc.rdoc_files.include('lib/**/*.rb')
 end
 
-EXT = "ferret_ext.so"
+EXT_SRC.each do |fn|
+  dest_fn = File.join("ext", File.basename(fn))
+  file dest_fn => fn do |t|
+    cp fn, dest_fn
+  end
+end
 
 desc "Build the extension"
-task :ext => "ext/#{EXT}"
+task :ext => ["ext/#{EXT}"] + SRC
 
-file "ext/#{EXT}" => "ext/Makefile" do
+file "ext/#{EXT}" => ["ext/Makefile"] do
+  cp "ext/inc/lang.h", "ext/lang.h"
   sh "cd ext; make"
 end
 
-file "ext/Makefile" do
+file "ext/Makefile" => SRC do
   sh "cd ext; ruby extconf.rb"
 end
 
 # Make Parsers ---------------------------------------------------------------
 
-RACC_SRC = FileList["**/*.y"]
+RACC_SRC = FileList["lib/**/*.y"]
 RACC_OUT = RACC_SRC.collect { |fn| fn.sub(/\.y$/, '.tab.rb') }
 
 task :parsers => RACC_OUT
@@ -195,8 +215,9 @@ end
 # Creating a release
 
 desc "Make a new release"
-task :prerelease => [:clobber, :all_tests, :parsers]
-task :package => [:prerelease]
+task :prerelease => [:all_tests, :clobber]
+task :repackage => EXT_SRC_DEST
+task :package => EXT_SRC_DEST
 task :tag => [:prerelease]
 task :update_version => [:prerelease]
 task :release => [:tag, :update_version, :package] do
@@ -229,7 +250,7 @@ task :prerelease do
   end
 
   # Are all source files checked in?
-  data = `svn -q status`
+  data = `svn -q --ignore-externals status`
   unless data =~ /^$/
     fail "'svn -q status' is not clean ... do you have unchecked-in files?"
   end
@@ -237,28 +258,33 @@ task :prerelease do
   announce "No outstanding checkins found ... OK"
 end
 
+def reversion(fn)
+  open(fn) do |ferret_in|
+    open(fn + ".new", "w") do |ferret_out|
+      ferret_in.each do |line|
+        if line =~ /^  VERSION\s*=\s*/
+          ferret_out.puts "  VERSION = '#{PKG_VERSION}'"
+        else
+          ferret_out.puts line
+        end
+      end
+    end
+  end
+end
+
 task :update_version => [:prerelease] do
   if PKG_VERSION == CURRENT_VERSION
     announce "No version change ... skipping version update"
   else
     announce "Updating Ferret version to #{PKG_VERSION}"
-    open("lib/ferret.rb") do |ferret_in|
-      open("lib/ferret.rb.new", "w") do |ferret_out|
-        ferret_in.each do |line|
-          if line =~ /^  VERSION\s*=\s*/
-            ferret_out.puts "  VERSION = '#{PKG_VERSION}'"
-          else
-            ferret_out.puts line
-          end
-        end
-      end
-    end
+    reversion("lib/ferret.rb")
+    reversion("lib/rferret.rb")
     if ENV['RELTEST']
       announce "Release Task Testing, skipping commiting of new version"
     else
-      mv "lib/ferret.rb.new", "lib/ferret.rb"
+      mv "lib/rferret.rb.new", "lib/rferret.rb"
     end
-    sh %{svn ci -m "Updated to version #{PKG_VERSION}" lib/ferret.rb}
+    sh %{svn ci -m "Updated to version #{PKG_VERSION}" lib/rferret.rb}
   end
 end
 

data/ext/analysis.c

@@ -0,0 +1,553 @@
+#include <analysis.h>
+#include <string.h>
+#include <ctype.h>
+#include <hash.h>
+
+Token *tk_create()
+{
+  return ALLOC(Token);
+}
+
+void tk_destroy(void *p)
+{
+  free(p);
+}
+
+inline Token *tk_set(Token *tk, char *text, int tlen, int start, int end, int pos_inc)
+{
+  if (tlen >= MAX_WORD_SIZE) tlen = MAX_WORD_SIZE - 1;
+  memcpy(tk->text, text, sizeof(char) * tlen);
+  tk->text[tlen] = '\0';
+  tk->start = start;
+  tk->end = end;
+  tk->pos_inc = pos_inc;
+  return tk;
+}
+
+inline Token *tk_set_no_len(Token *tk, char *text, int start, int end, int pos_inc)
+{
+  return tk_set(tk, text, strlen(text), start, end, pos_inc);
+}
+
+int tk_eq(Token *tk1, Token *tk2)
+{
+  if (strcmp((char *)tk1->text, (char *)tk2->text) == 0 &&
+      tk1->start == tk2->start && tk1->end == tk2->end)
+    return true;
+  else
+    return false;
+}
+
+int tk_cmp(Token *tk1, Token *tk2)
+{
+  int cmp;
+  if (tk1->start > tk2->start) {
+    cmp = 1;
+  } else if (tk1->start < tk2->start) {
+    cmp = -1;
+  } else {
+    if (tk1->end > tk2->end) {
+      cmp = 1;
+    } else if (tk1->end < tk2->end) {
+      cmp = -1;
+    } else {
+      cmp = strcmp((char *)tk1->text, (char *)tk2->text);
+    }
+  }
+  return cmp;
+}
+
+void ts_standard_destroy(void *p)
+{
+  TokenStream *ts = (TokenStream *)p;
+  tk_destroy(ts->token);
+  free(p);
+}
+
+void ts_reset(TokenStream *ts, char *text)
+{
+  ts->text = text;
+  ts->pos = 0;
+}
+
+TokenStream *ts_create()
+{
+  TokenStream *ts = ALLOC(TokenStream);
+  ts->pos = -1;
+  ts->text = NULL;
+  ts->token = tk_create();
+  ts->destroy = &ts_standard_destroy;
+  ts->reset = &ts_reset;
+  return ts;
+}
+
+Token *wst_next(TokenStream *ts)
+{
+  int i = ts->pos;
+  int start, end;
+  char *text = ts->text;
+
+  while (text[i] != '\0' && isspace(text[i]))
+    i++;
+  if (text[i] == '\0')
+    return NULL;
+
+  start = i;
+  while (text[i] != '\0' && !isspace(text[i]))
+    i++;
+  ts->pos = end = i;
+  tk_set(ts->token, text+start, end-start, start, end, 1);
+  return ts->token;
+}
+
+TokenStream *whitespace_tokenizer_create()
+{
+  TokenStream *ts = ts_create();
+  ts->next = &wst_next;
+  return ts;
+}
+
+Token *lt_next(TokenStream *ts)
+{
+  int i = ts->pos;
+  int start, end;
+  char *text = ts->text;
+
+  while (text[i] != '\0' && !isalpha(text[i]))
+    i++;
+  if (text[i] == '\0')
+    return NULL;
+
+  start = i;
+  while (text[i] != '\0' && isalpha(text[i]))
+    i++;
+  ts->pos = end = i;
+  tk_set(ts->token, text+start, end-start, start, end, 1);
+  return ts->token;
+}
+
+TokenStream *letter_tokenizer_create()
+{
+  TokenStream *ts = ts_create();
+  ts->next = &lt_next;
+  return ts;
+}
+
+void a_standard_destroy(void *p)
+{
+  Analyzer *a = (Analyzer *)p;
+  ts_destroy(a->current_ts);
+  free(p);
+}
+
+TokenStream *a_standard_get_ts(Analyzer *a, char *field, char *text)
+{
+  a->current_ts->reset(a->current_ts, text);
+  return a->current_ts; 
+}
+
+Analyzer *whitespace_analyzer_create()
+{
+  Analyzer *a = ALLOC(Analyzer);
+  a->data = NULL;
+  a->current_ts = whitespace_tokenizer_create();
+  a->destroy = &a_standard_destroy;
+  a->get_ts = &a_standard_get_ts;
+  return a;
+}
+
+int std_get_alpha(char *input, char *token)
+{
+  int i = 0;
+  while (input[i] != '\0' && isalpha(input[i])) {
+    token[i] = input[i];
+    i++;
+  }
+  return i;
+}
+
+int std_get_alnum(char *input, char *token)
+{
+  int i = 0;
+  while (input[i] != '\0' && isalnum(input[i])) {
+    token[i] = input[i];
+    i++;
+  }
+  return i;
+}
+
+int isnumpunc(char c)
+{
+  return (c == '.' || c == ',' || c == '\\' || c == '/' || c == '_' || c == '-');
+}
+
+int isurlpunc(char c)
+{
+  return (c == '.' || c == '/' || c == '-' || c == '_');
+}
+
+int isurlc(char c)
+{
+  return (c == '.' || c == '/' || c == '-' || c == '_' || isalnum(c));
+}
+
+int isurlxatpunc(char c)
+{
+  return (c == '.' || c == '/' || c == '-' || c == '_' || c == '@');
+}
+
+int isurlxatc(char c)
+{
+  return (c == '.' || c == '/' || c == '-' || c == '_' || c == '@' || isalnum(c));
+}
+
+int isstdtokchar(char c)
+{
+  if (isspace(c)) return false; // most common so check first.
+  if (isalnum(c) || isnumpunc(c) || c == '&' ||
+      c == '@' || c == '\'' || c == ':')
+    return true;
+  return false;
+}
+
+/* (alnum)((punc)(alnum))+ where every second sequence of alnum must contain at
+ * least one digit.
+ * (alnum) = [a-zA-Z0-9]
+ * (punc) = [_\/.,-]
+ */
+int std_get_number(char *input)
+{
+  int i = 0;
+  int count = 0;
+  int last_seen_digit = 2;
+  int seen_digit = false;
+
+  while (last_seen_digit >= 0) {
+    while ((input[i] != '\0') && isalnum(input[i])) {
+      if ((last_seen_digit < 2) && isdigit(input[i])) last_seen_digit = 2;
+      if ((seen_digit == false) && isdigit(input[i])) seen_digit = true;
+      i++;
+    }
+    last_seen_digit--;
+    if (!isnumpunc(input[i]) || !isalnum(input[i+1])) {
+      
+      if (last_seen_digit >= 0)
+        count = i;
+      break;
+    }
+    count = i;
+    i++;
+  }
+  if (seen_digit)
+    return count;
+  else
+    return 0;
+}
+
+int std_get_apostrophe(char *input)
+{
+  int i = 0;
+
+  while (isalpha(input[i]) || input[i] == '\'')
+    i++;
+
+  return i;
+}
+
+int std_get_url(char *input, char *token)
+{
+  int i = 0;
+
+  while (isurlc(input[i])) {
+    if (isurlpunc(input[i]) && isurlpunc(input[i-1]))
+      break; // can't have to puncs in a row
+    token[i] = input[i];
+    i++;
+  }
+
+  //strip trailing puncs
+  while (isurlpunc(input[i-1])) i--;
+
+  return i;
+}
+
+/* Company names can contain '@' and '&' like AT&T and Excite@Home. Let's
+ */
+int std_get_company_name(char *input)
+{
+  int i = 0;
+  while (isalpha(input[i]) || input[i] == '@' || input[i] == '&')
+    i++;
+
+  return i;
+}
+
+Token *std_next(TokenStream *ts)
+{
+  int i = ts->pos, j;
+  int start;
+  char *text = ts->text;
+  char token[MAX_WORD_SIZE];
+  int token_i = 0;
+  int len;
+  int num_end = 0;
+  int is_acronym;
+  int seen_at_symbol;
+  
+  while (text[i] != '\0' && !isalnum(text[i]))
+    i++;
+  if (text[i] == '\0')
+    return NULL;
+
+  start = i;
+  if (isdigit(text[i])) {
+    i += std_get_number(text + i);
+    ts->pos = i;
+    tk_set(ts->token, text+start, i - start, start, ts->pos, 1);
+  } else {
+    token_i = std_get_alpha(text + i, token);
+    i += token_i;
+
+    if (!isstdtokchar(text[i])) {
+      // very common case, ie a plain word, so check and return
+      tk_set(ts->token, text+start, i-start, start, i, 1);
+      ts->pos = i;
+      return ts->token;
+    }
+
+    if (text[i] == '\'') { // apostrophe case. 
+      i += std_get_apostrophe(text + i);
+      ts->pos = i;
+      len = i - start;
+      // strip possesive
+      if ((text[i-1] == 's' || text[i-1] == 'S') && text[i-2] == '\'')
+        len -= 2;
+      tk_set(ts->token, text+start, len, start, i, 1);
+      return ts->token;
+    }
+    if (text[i] == '&') { // apostrophe case. 
+      i += std_get_company_name(text + i);
+      ts->pos = i;
+      tk_set(ts->token, text+start, i - start, start, i, 1);
+      return ts->token;
+    }
+
+    if (isdigit(text[i]) || isnumpunc(text[i])) { // possibly a number
+      num_end = start + std_get_number(text + start);
+      if (!isstdtokchar(text[num_end])) { // we won't find a longer token
+        ts->pos = num_end;
+        tk_set(ts->token, text+start, num_end-start, start, ts->pos, 1);
+        return ts->token;
+      }
+      // else there may be a longer token so check
+    }
+
+    if (text[i] == ':' && text[i+1] == '/' && text[i+2] == '/') {
+      // check for a known url start
+      token[token_i] = '\0';
+      i += 3;
+      while (text[i] == '/') i++;
+      if (isalpha(text[i]) &&
+          (strcmp(token, "ftp") == 0 ||
+           strcmp(token, "http") == 0 ||
+           strcmp(token, "https") == 0 ||
+           strcmp(token, "file") == 0)) {
+        len = std_get_url(text + i, token); // dispose of first part of the URL
+      } else { //still treat as url but keep the first part
+        token_i = i - start;
+        memcpy(token, text + start, token_i * sizeof(char));
+        len = token_i + std_get_url(text + i, token + token_i); // keep start
+      }
+      ts->pos = i + len;
+      token[len] = 0;
+      tk_set(ts->token, token, len, start, ts->pos, 1);
+      return ts->token;
+    }
+
+    // now see how int a url we can find.
+    is_acronym = true;
+    seen_at_symbol = false;
+    while (isurlxatc(text[i])) {
+      if (is_acronym && !isalpha(text[i]) && (text[i] != '.')) {
+        is_acronym = false;
+      }
+      if (isurlxatpunc(text[i]) && isurlxatpunc(text[i-1]))
+        break; // can't have to punctuation characters in a row
+      if (text[i] == '@') {
+        if (seen_at_symbol)
+          break; // we can only have one @ symbol
+        else
+          seen_at_symbol = true;
+      }
+      i++;
+    }
+    while (isurlxatpunc(text[i-1])) i--; // strip trailing punctuation
+    if (i > num_end) {
+      ts->pos = i;
+
+      if (is_acronym) { // check that it is one letter followed by one '.'
+        for (j = start; j < i-1; j++) {
+          if (isalpha(text[j]) && (text[j+1] != '.')) is_acronym = false;
+        }
+      }
+      if (is_acronym) {// strip '.'s
+        for (j = start + token_i; j < i; j++) {
+          if (text[j] != '.') {
+            token[token_i] = text[j];
+            token_i++;
+          }
+        }
+        tk_set(ts->token, token, token_i, start, ts->pos, 1);
+      } else { // just return the url as is
+        tk_set(ts->token, text+start, i-start, start, ts->pos, 1);
+      }
+    } else { // return the number
+      ts->pos = num_end;
+      tk_set(ts->token, text+start, num_end-start, start, ts->pos, 1);
+    }
+  }
+
+  return ts->token;
+}
+
+TokenStream *standard_tokenizer_create()
+{
+  TokenStream *ts = ts_create();
+  ts->next = &std_next;
+  return ts;
+}
+
+const char *ENGLISH_STOP_WORDS[] = {
+  "a", "an", "and", "are", "as", "at", "be", "but", "by",
+  "for", "if", "in", "into", "is", "it",
+  "no", "not", "of", "on", "or", "s", "such",
+  "t", "that", "the", "their", "then", "there", "these",
+  "they", "this", "to", "was", "will", "with"
+};
+
+void filter_reset(TokenStream *ts, char *text)
+{
+  ts->sub_ts->reset(ts->sub_ts, text);
+}
+
+void filter_destroy(void *p)
+{
+  TokenStream *ts = (TokenStream *)p;
+  ts->sub_ts->destroy(ts->sub_ts);
+  if (ts->token != NULL) tk_destroy(ts->token);
+  free(ts);
+}
+
+void sf_destroy(void *p)
+{
+  HshTable *words = (HshTable *)((TokenStream *)p)->data;
+  h_destroy(words);
+  filter_destroy(p);
+}
+
+Token *sf_next(TokenStream *ts)
+{
+  int pos_inc = 1;
+  HshTable *words = (HshTable *)ts->data;
+  Token *tk = ts->sub_ts->next(ts->sub_ts);
+  while ((tk != NULL) && (h_get(words, tk->text) != NULL)) {
+    tk = ts->sub_ts->next(ts->sub_ts);
+    pos_inc++;
+  }
+  if (tk != NULL) tk->pos_inc = pos_inc;
+  return tk;
+}
+
+TokenStream *stop_filter_create_with_words(TokenStream *ts, char **words, int len)
+{
+  int i;
+  TokenStream *tf = ALLOC(TokenStream);
+  tf->sub_ts = ts;
+  HshTable *wordtable = h_new_str(NULL, NULL);
+  for (i = 0; i < len; i++) {
+    h_set(wordtable, words[i], words[i]);
+  }
+  tf->data = wordtable;
+  tf->token = NULL;
+  tf->next = &sf_next;
+  tf->reset = &filter_reset;
+  tf->destroy = &sf_destroy;
+  return tf;
+}
+
+TokenStream *stop_filter_create(TokenStream *ts)
+{
+  return stop_filter_create_with_words(ts, 
+      (char **)ENGLISH_STOP_WORDS, NELEMS(ENGLISH_STOP_WORDS));
+}
+
+Token *lcf_next(TokenStream *ts)
+{
+  int i = 0;
+  Token *tk = ts->sub_ts->next(ts->sub_ts);
+  if (tk == NULL) return tk;
+  while (tk->text[i] != '\0') {
+    tk->text[i] = tolower(tk->text[i]);
+    i++;
+  }
+  return tk;
+}
+
+TokenStream *lowercase_filter_create(TokenStream *ts)
+{
+  TokenStream *tf = ALLOC(TokenStream);
+  tf->token = NULL;
+  tf->next = &lcf_next;
+  tf->reset = &filter_reset;
+  tf->destroy = &filter_destroy;
+  tf->sub_ts = ts;
+  return tf;
+}
+
+Analyzer *letter_analyzer_create()
+{
+  Analyzer *a = ALLOC(Analyzer);
+  a->data = NULL;
+  a->current_ts = lowercase_filter_create(letter_tokenizer_create());
+  a->destroy = &a_standard_destroy;
+  a->get_ts = &a_standard_get_ts;
+  return a;
+}
+
+
+Analyzer *standard_analyzer_create_with_words(char **words, int len)
+{
+  Analyzer *a = ALLOC(Analyzer);
+  a->data = NULL;
+  a->current_ts = 
+    stop_filter_create_with_words(
+        lowercase_filter_create(standard_tokenizer_create()), words, len);
+  a->destroy = &a_standard_destroy;
+  a->get_ts = &a_standard_get_ts;
+  return a;
+}
+
+Analyzer *standard_analyzer_create()
+{
+  return standard_analyzer_create_with_words(
+        (char **)ENGLISH_STOP_WORDS, NELEMS(ENGLISH_STOP_WORDS));
+}
+
+#ifdef ALONE
+int main(int argc, char **argv)
+{
+  char buf[10000];
+  Analyzer *a = standard_analyzer_create();
+  TokenStream *ts;
+  Token *tk;
+  while (fgets(buf, 9999, stdin) != NULL) {
+    ts = a->get_ts(a, "hello", buf);
+    ts->pos = 0;
+    while ((tk = ts->next(ts)) != NULL) {
+      printf("<%s:%ld:%ld> ", tk->text, tk->start, tk->end);
+    }
+    printf("\n");
+  }
+  return 0;
+}
+#endif