ferret 0.9.6 → 0.10.0

data/ext/w32_io.c

@@ -1,150 +0,0 @@
-#ifdef WIN32
-
-#include "global.h"
-#include "store.h"
-#include <stdio.h>
-#include <io.h>
-#include <errno.h>
-#include <string.h>
-
-/**
- * Create a filepath for a file in the store using the operating systems
- * default file seperator.
- */
-char *join_path(char *buf, const char *base, const char *filename)
-{
-  sprintf(buf, "%s\\%s", base, filename);
-  return buf;
-}
-
-bool exists(char *path)
-{
-  int fd = _open(path, 0);
-  if (fd < 0) {
-    if (errno != ENOENT) {
-      RAISE(IO_ERROR, strerror(errno));
-    }
-    return false;
-  }
-  _close(fd);
-  return true;
-}
-
-int fcount(char *path)
-{
-  char buf[MAX_FILE_PATH];
-  struct _finddata_t fd;
-  intptr_t d;
-  int cnt = 0;
-
-  join_path(buf, path, "*");
-
-  if ((d = _findfirst(buf, &fd)) < 0) {
-    RAISE(IO_ERROR, strerror(errno));
-  }
-
-  do {
-    if (fd.name[0] != '.') {
-      cnt++;
-    }
-  } while (_findnext(d, &fd) == 0);
-  _findclose(d);
-
-  return cnt;
-}
-
-void dir_each(char *path, void (*func)(char *fname, void *arg), void *arg)
-{
-  char buf[MAX_FILE_PATH];
-  struct _finddata_t fd;
-  intptr_t d;
-  join_path(buf, path, "*");
-
-  if ((d = _findfirst(buf, &fd)) < 0) {
-    RAISE(IO_ERROR, strerror(errno));
-  }
-
-  while (_findnext(d, &fd) == 0) {
-    if (fd.name[0] != '.' && !file_is_lock(fd.name)) {
-      func(fd.name, arg);
-    }
-  }
-  _findclose(d);
-}
-
-/**
- * Clear all the locks in the store.
- *
- * @param store the store to clear the locks from
- * @throws IO_ERROR if there is an error opening the directory
- */
-void fs_clear_locks(Store *store)
-{
-  char buf[MAX_FILE_PATH];
-  struct _finddata_t fd;
-  intptr_t d;
-  join_path(buf, store->dir.path, "*");
-
-  if ((d = _findfirst(buf, &fd)) < 0) {
-    RAISE(IO_ERROR, strerror(errno));
-  }
-
-  while (_findnext(d, &fd) == 0) {
-    if (file_is_lock(fd.name)) {
-      remove(join_path(buf, store->dir.path, fd.name));
-    }
-  }
-  _findclose(d);
-}
-
-/**
- * Clear all files from the store except the lock files.
- *
- * @param store the store to clear all the files from
- * @throws IO_ERROR if there is an error deleting the files
- */
-void fs_clear(Store *store)
-{
-  char buf[MAX_FILE_PATH];
-  struct _finddata_t fd;
-  intptr_t d;
-  join_path(buf, store->dir.path, "*");
-
-  if ((d = _findfirst(buf, &fd)) < 0) {
-    RAISE(IO_ERROR, strerror(errno));
-  }
-
-  while (_findnext(d, &fd) == 0) {
-    if (fd.name[0] != '.' && !file_is_lock(fd.name)) {
-      remove(join_path(buf, store->dir.path, fd.name));
-    }
-  }
-  _findclose(d);
-}
-
-/**
- * Clear all files from the store including the lock files.
- *
- * @param store the store to clear all the files from
- * @throws IO_ERROR if there is an error deleting the files
- */
-void fs_clear_all(Store *store)
-{
-  char buf[MAX_FILE_PATH];
-  struct _finddata_t fd;
-  intptr_t d;
-  join_path(buf, store->dir.path, "*");
-
-  if ((d = _findfirst(buf, &fd)) < 0) {
-    RAISE(IO_ERROR, strerror(errno));
-  }
-
-  while (_findnext(d, &fd) == 0) {
-    if (fd.name[0] != '.') {
-      remove(join_path(buf, store->dir.path, fd.name));
-    }
-  }
-  _findclose(d);
-}
-
-#endif

data/lib/ferret/analysis.rb

@@ -1,11 +0,0 @@
-# Documentation for Analysis
-module Ferret::Analysis
-end
-
-require 'ferret/analysis/token'
-require 'ferret/analysis/token_stream'
-require 'ferret/analysis/tokenizers'
-require 'ferret/analysis/standard_tokenizer'
-require 'ferret/analysis/token_filters'
-require 'ferret/analysis/word_list_loader'
-require 'ferret/analysis/analyzers'

data/lib/ferret/analysis/analyzers.rb

@@ -1,112 +0,0 @@
-module Ferret::Analysis
-  # An Analyzer builds TokenStreams, which analyze text.  It thus represents
-  # a policy for extracting index terms from text.
-  #
-  # Typical implementations first build a Tokenizer, which breaks the stream
-  # of characters from the Reader into raw Tokens. One or more TokenFilter s
-  # may then be applied to the output of the Tokenizer.
-  #
-  # The default Analyzer just creates a LowerCaseTokenizer which converts
-  # all text to lowercase tokens. See LowerCaseTokenizer for more details.
-  class Analyzer
-    # Creates a TokenStream which tokenizes all the text in the provided
-    # Reader. Override to allow Analyzer to choose strategy based on
-    # document and/or field.
-    # string:: the string representing the text in the field
-    # field:: name of the field. Not required.
-    def token_stream(field, string)
-      return LowerCaseTokenizer.new(string)
-    end
-
-    # Invoked before indexing a Field instance if
-    # terms have already been added to that field.  This allows custom
-    # analyzers to place an automatic position increment gap between
-    # Field instances using the same field name.  The default value
-    # position increment gap is 0.  With a 0 position increment gap and
-    # the typical default token position increment of 1, all terms in a field,
-    # including across Field instances, are in successive positions, allowing
-    # exact PhraseQuery matches, for instance, across Field instance boundaries.
-    #
-    # field_name::             Field name being indexed.
-    # pos_inc_gap:: added to the next token emitted from
-    #                          #token_stream(String,Reader)
-    #
-    def pos_inc_gap(field_name)
-      return 0
-    end
-
-  end
-
-  # An Analyzer that uses WhiteSpaceTokenizer.
-  class WhiteSpaceAnalyzer < Analyzer
-    def token_stream(field, string)
-      return WhiteSpaceTokenizer.new(string)
-    end
-  end
-
-  # Filters LetterTokenizer with LowerCaseFilter and StopFilter.
-  class StopAnalyzer < Analyzer
-
-    # An array containing some common English words that are not usually useful
-    # for searching.
-    ENGLISH_STOP_WORDS = [
-      "a", "an", "and", "are", "as", "at", "be", "but", "by", "for", "if",
-      "in", "into", "is", "it", "no", "not", "of", "on", "or", "s", "such",
-      "t", "that", "the", "their", "then", "there", "these",
-      "they", "this", "to", "was", "will", "with"
-    ]
-
-    # Builds an analyzer which removes words in the provided array.
-    def initialize(stop_words = ENGLISH_STOP_WORDS)
-      @stop_words = stop_words
-    end
-
-    # Filters LowerCaseTokenizer with StopFilter.
-    def token_stream(field, string)
-      return StopFilter.new(LowerCaseTokenizer.new(string), @stop_words)
-    end
-  end
-
-  # An Analyzer that filters LetterTokenizer with LowerCaseFilter.
-  # This analyzer subclasses the StopAnalyzer so you can add your own
-  # stoplist the same way. See StopAnalyzer.
-  class StandardAnalyzer < StopAnalyzer
-    def token_stream(field, string)
-      return StopFilter.new(LowerCaseFilter.new(StandardTokenizer.new(string)), @stop_words)
-    end
-  end
-
-
-  # This analyzer is used to facilitate scenarios where different
-  # fields require different analysis techniques.  Use #add_analyzer
-  # to add a non-default analyzer on a field name basis.
-  # See tc_per_field_analyzer_wrapper for example usage.
-  class PerFieldAnalyzerWrapper < Analyzer
-
-    # Constructs with default analyzer.
-    #
-    # default_analyzer:: Any fields not specifically defined to use a
-    #                    different analyzer will use the one provided here.
-    def initialize(default_analyzer)
-      @default_analyzer = default_analyzer
-      @analyzers = {}
-    end
-
-    # Defines an analyzer to use for the specified field.
-    #
-    # field:: field name requiring a non-default analyzer.
-    # analyzer:: non-default analyzer to use for field
-    def add_analyzer(field, analyzer)
-      @analyzers[field] = analyzer
-    end
-
-    def token_stream(field, string)
-      analyzer = @analyzers[field]
-      if (analyzer == nil)
-        analyzer = @default_analyzer
-      end
-
-      return analyzer.token_stream(field, string)
-    end
-  end
-end

data/lib/ferret/analysis/standard_tokenizer.rb

@@ -1,71 +0,0 @@
-if __FILE__ == $0
-  module Ferret
-  end
-  $:.unshift File.dirname(__FILE__)
-  require 'token_stream'
-  require 'tokenizers'
-  require 'token'
-end
-
-module Ferret::Analysis
-  # The standard tokenizer is an advanced tokenizer which tokenizes morst
-  # words correctly as well as tokenizing things like email addresses, web
-  # addresses, phone numbers, etc.
-
-  class StandardTokenizer < RegExpTokenizer
-    ALPHA      = /[[:alpha:]_-]+/
-    APOSTROPHE = /#{ALPHA}('#{ALPHA})+/
-    ACRONYM    = /#{ALPHA}\.(#{ALPHA}\.)+/
-    P          = /[_\/.,-]/
-    HASDIGIT   = /\w*\d\w*/
-    TOKEN_RE   = /#{ALPHA}+(('#{ALPHA}+)+
-                              |\.(#{ALPHA}\.)+
-                              |(@|\&)\w+([-.]\w+)*
-                              |:\/\/\w+([-.\/]\w+)*
-                              )
-                 |\w+(([\-._]\w+)*\@\w+([-.]\w+)+
-                     |#{P}#{HASDIGIT}(#{P}\w+#{P}#{HASDIGIT})*(#{P}\w+)?
-                     |(\.\w+)+
-                     |
-                     )
-                 /x
-
-    ACRONYM_WORD    = /^#{ACRONYM}$/
-    APOSTROPHE_WORD = /^#{APOSTROPHE}$/
-    DOT             = /\./
-    APOSTROPHE_S    = /'[sS]$/
-    protected
-
-      # Collects only characters which are not spaces tabs or carraige returns
-      def token_re()
-        #/#{NUM}|#{EMAIL}|#{ACRONYM}\w*|#{C0MPANY}|#{APOSTROPHE}|\w+/
-        # This is a simplified version of the original Lucene standard
-        # tokenizer.  I think it works better. I hope so anyway. Any way to
-        # do this more neatly?
-        TOKEN_RE
-      end
-
-      # stem the 's and remove the '.'s from acronyms
-      def normalize(str)
-        if str =~ ACRONYM_WORD
-          str.gsub!(DOT, '')
-        elsif str =~ APOSTROPHE_WORD
-          str.gsub!(APOSTROPHE_S, '')
-        end
-        str
-      end
-  end
-end
-
-# Add this so we can play around with the standard tokenizer
-if __FILE__ == $0
-  st = "\033[7m"
-  en = "\033[m"
-
-  $stdin.each do |line|
-    stk = Ferret::Analysis::StandardTokenizer.new(line)
-    while tk = stk.next()
-      puts "    <" + tk.text + "> from " + tk.start_offset.to_s + " to " + tk.end_offset.to_s
-    end
-  end
-end

data/lib/ferret/analysis/token.rb

@@ -1,100 +0,0 @@
-module Ferret::Analysis
-  # A Token is an occurence of a term from the text of a field.  It consists
-  # of a term's text, the start and end offset of the term in the text of the
-  # field, and a type string.
-  #
-  # The start and end offsets permit applications to re-associate a token with
-  # its source text, e.g., to display highlighted query terms in a document
-  # browser, or to show matching text fragments in a KWIC (KeyWord In Context)
-  # display, etc.
-  #
-  # The type is an interned string, assigned by a lexical analyzer (a.k.a.
-  # tokenizer), naming the lexical or syntactic class that the token belongs
-  # to.  For example an end of sentence marker token might be implemented with
-  # type "eos".  The default token type is "word".
-  #
-  # start_offset:: is the position of the first character corresponding to
-  #                this token in the source text
-  # end_offset:: is equal to one greater than the position of the last
-  #              character corresponding of this token Note that the
-  #              difference between @end_offset and @start_offset may not be
-  #              equal to @text.length(), as the term text may have been
-  #              altered by a stemmer or some other filter.
-  class Token
-    include Comparable
-    attr_accessor :text
-    attr_reader :pos_inc, :start_offset, :end_offset, :type
-
-    # Constructs a Token with the given term text, and start & end offsets.
-    # The type defaults to "word."
-    def initialize(txt, so, eo, pos_inc=1, typ="word")
-      @text = txt
-      @start_offset = so
-      @end_offset = eo
-      @type = typ # lexical type
-      @pos_inc = pos_inc
-    end
-
-    def set!(txt, so, eo)
-      @text = txt
-      @start_offset = so
-      @end_offset = eo
-      self
-    end
-
-    def eql?(o)
-      return (o.instance_of?(Token) and @start_offset == o.start_offset and
-              @end_offset == o.end_offset and @text == o.text)
-    end
-    alias :== :eql?
-
-    # Tokens are sorted by the position in the text at which they occur, ie
-    # the start_offset. If two tokens have the same start offset, (see
-    # pos_inc=) then, they are sorted by the end_offset and then
-    # lexically by the token text.
-    def <=>(o)
-      r = @start_offset <=> o.start_offset
-      return r if r != 0
-      r = @end_offset <=> o.end_offset
-      return r if r != 0
-      r = @text <=> o.text
-      return r
-    end
-
-    # Set the position increment.  This determines the position of this token
-    # relative to the previous Token in a TokenStream, used in phrase
-    # searching.
-    #
-    # The default value is one.
-    #
-    # Some common uses for this are:
-    #
-    # * Set it to zero to put multiple terms in the same position.  This is
-    #   useful if, e.g., a word has multiple stems.  Searches for phrases
-    #   including either stem will match.  In this case, all but the first
-    #   stem's increment should be set to zero: the increment of the first
-    #   instance should be one.  Repeating a token with an increment of zero
-    #   can also be used to boost the scores of matches on that token.
-    #
-    # * Set it to values greater than one to inhibit exact phrase matches.
-    #   If, for example, one does not want phrases to match across removed
-    #   stop words, then one could build a stop word filter that removes stop
-    #   words and also sets the increment to the number of stop words removed
-    #   before each non-stop word.  Then exact phrase queries will only match
-    #   when the terms occur with no intervening stop words.
-    def pos_inc=(pos_inc)
-      if (pos_inc < 0)
-        raise ArgumentError, "Increment must be zero or greater: " + pos_inc
-      end
-      @pos_inc = pos_inc
-    end
-
-    # Returns a string representation of the token with all the attributes.
-    def to_s
-      buf = "#{text}:#{start_offset}->#{end_offset}"
-      buf << "(pos_inc=#{@pos_inc})" if (@pos_inc != 1)
-      buf << "(type=#{@type})" if (@type != "word")
-      buf
-    end
-  end
-end