ferret 0.1.0

data/lib/ferret/analysis/standard_tokenizer.rb

@@ -0,0 +1,65 @@
+if __FILE__ == $0
+  module Ferret
+  end
+  $:.unshift File.dirname(__FILE__)
+  require 'token_stream'
+  require 'tokenizers'
+  require 'token'
+end
+
+module Ferret::Analysis
+  # The standard tokenizer is an advanced tokenizer which tokenizes morst
+  # words correctly as well as tokenizing things like email addresses, web
+  # addresses, phone numbers, etc.
+
+  class StandardTokenizer < RETokenizer
+    ALPHA      = /[[:alpha:]]+/
+    APOSTROPHE = /#{ALPHA}('#{ALPHA})+/
+    ACRONYM    = /#{ALPHA}\.(#{ALPHA}\.)+/
+    P          = /[_\/.,-]/
+    HASDIGIT   = /\w*\d\w*/
+
+    protected
+
+      # Collects only characters which are not spaces tabs or carraige returns
+      def token_re()
+        #/#{NUM}|#{EMAIL}|#{ACRONYM}\w*|#{C0MPANY}|#{APOSTROPHE}|\w+/
+        # This is a simplified version of the original Lucene standard
+        # tokenizer.  I think it works better. I hope so anyway. Any way to
+        # do this more neatly?
+        /[[:alpha:]]+(('[[:alpha:]]+)+
+                     |\.([[:alpha:]]\.)+
+                     |(@|\&)\w+([-.]\w+)*
+                     )
+        |\w+(([\-._]\w+)*\@\w+([-.]\w+)+
+            |#{P}#{HASDIGIT}(#{P}\w+#{P}#{HASDIGIT})*(#{P}\w+)?
+            |(\.\w+)+
+            |
+            )
+        /x
+      end
+
+      # stem the 's and remove the '.'s from acronyms
+      def normalize(str)
+        if str =~ /^#{ACRONYM}$/
+          str.gsub!(/\./, '')
+        elsif str =~ /^#{APOSTROPHE}$/
+          str.gsub!(/'[sS]$/, '')
+        end
+        str
+      end
+  end
+end
+
+# Add this so we can play around with the standard tokenizer
+if __FILE__ == $0
+  st = "\033[7m"
+  en = "\033[m"
+
+  $stdin.each do |line|
+    stk = Ferret::Analysis::StandardTokenizer.new(line)
+    while tk = stk.next()
+      puts "    <" + tk.term_text + "> from " + tk.start_offset.to_s + " to " + tk.end_offset.to_s
+    end
+  end
+end

data/lib/ferret/analysis/token.rb

@@ -0,0 +1,79 @@
+module Ferret::Analysis
+  # A Token is an occurence of a term from the text of a field.  It consists
+  # of a term's text, the start and end offset of the term in the text of the
+  # field, and a type string.
+  #
+  # The start and end offsets permit applications to re-associate a token with
+  # its source text, e.g., to display highlighted query terms in a document
+  # browser, or to show matching text fragments in a KWIC (KeyWord In Context)
+  # display, etc.
+  #
+  # The type is an interned string, assigned by a lexical analyzer (a.k.a.
+  # tokenizer), naming the lexical or syntactic class that the token belongs
+  # to.  For example an end of sentence marker token might be implemented with
+  # type "eos".  The default token type is "word".
+  #
+  # start_offset:: is the position of the first character corresponding to
+  #                this token in the source text
+  # end_offset:: is equal to one greater than the position of the last
+  #              character corresponding of this token Note that the
+  #              difference between @end_offset and @start_offset may not be
+  #              equal to @term_text.length(), as the term text may have been
+  #              altered by a stemmer or some other filter.
+  class Token
+    include Comparable
+    attr_accessor :term_text
+    attr_reader :position_increment, :start_offset, :end_offset, :type
+
+    # Constructs a Token with the given term text, and start & end offsets.
+    # The type defaults to "word."
+    def initialize(txt, so, eo, typ="word", pos_inc=1)
+      @term_text = txt
+      @start_offset = so
+      @end_offset = eo
+      @type = typ # lexical type
+      @position_increment = pos_inc
+    end
+
+    # Tokens are sorted by the position in the text at which they occur, ie
+    # the start_offset. If two tokens have the same start offset, (see
+    # position_increment=) then, they are sorted by the end_offset and then
+    # lexically by the token text.
+    def <=>(o)
+      r = @start_offset <=> o.start_offset
+      return r if r != 0
+      r = @end_offset <=> o.end_offset
+      return r if r != 0
+      r = @term_text <=> o.term_text
+      return r
+    end
+
+    # Set the position increment.  This determines the position of this token
+    # relative to the previous Token in a TokenStream, used in phrase
+    # searching.
+    #
+    # The default value is one.
+    #
+    # Some common uses for this are:
+    #
+    # * Set it to zero to put multiple terms in the same position.  This is
+    #   useful if, e.g., a word has multiple stems.  Searches for phrases
+    #   including either stem will match.  In this case, all but the first
+    #   stem's increment should be set to zero: the increment of the first
+    #   instance should be one.  Repeating a token with an increment of zero
+    #   can also be used to boost the scores of matches on that token.
+    #
+    # * Set it to values greater than one to inhibit exact phrase matches.
+    #   If, for example, one does not want phrases to match across removed
+    #   stop words, then one could build a stop word filter that removes stop
+    #   words and also sets the increment to the number of stop words removed
+    #   before each non-stop word.  Then exact phrase queries will only match
+    #   when the terms occur with no intervening stop words.
+    def position_increment=(pos_inc)
+      if (pos_inc < 0)
+        raise ArgumentError, "Increment must be zero or greater: " + pos_inc
+      end
+      @position_increment = pos_inc
+    end
+  end
+end

data/lib/ferret/analysis/token_filters.rb

@@ -0,0 +1,86 @@
+module Ferret::Analysis
+  # A TokenFilter is a TokenStream whose input is another token stream.
+  #
+  # This is an abstract class.
+  class TokenFilter < TokenStream
+    # Close the input TokenStream.
+    def close()
+      @input.close()
+    end
+
+    protected
+      # Construct a token stream filtering the given input.
+      def initialize(input)
+        @input = input
+      end
+  end
+
+  # Normalizes token text to lower case.
+  class LowerCaseFilter < TokenFilter
+    def next()
+      t = @input.next()
+
+      if (t == nil)
+        return nil
+      end
+
+      t.term_text = t.term_text.downcase()
+
+      return t
+    end
+  end
+
+  # Removes stop words from a token stream. To will need to pass your own
+  # set of stopwords to use this stop filter. If you with to use the default
+  # list of stopwords then use the StopAnalyzer.
+  class StopFilter < TokenFilter
+    # Constructs a filter which removes words from the input
+    # TokenStream that are named in the array of words.
+    def initialize(input, stop_set)
+      super(input);
+      @stop_set = stop_set
+    end
+
+    def StopFilter.new_with_file(input, path)
+      ws = WordListLoader.word_set_from_file(path)
+      return StopFilter.new(input, ws)
+    end
+
+    # Returns the next input Token whose termText() is not a stop word.
+    def next()
+      # return the first non-stop word found
+      while token = @input.next()
+        return token if ! @stop_set.include?(token.term_text)
+      end
+      return nil
+    end
+  end
+
+  # Transforms the token stream as per the Porter stemming algorithm.
+  # Note: the input to the stemming filter must already be in lower case,
+  # so you will need to use LowerCaseFilter or LowerCaseTokenizer further
+  # down the Tokenizer chain in order for this to work properly!
+  # 
+  # To use this filter with other analyzers, you'll want to write an
+  # Analyzer class that sets up the TokenStream chain as you want it.
+  # To use this with LowerCaseTokenizer, for example, you'd write an
+  # analyzer like this:
+  #
+  #   def MyAnalyzer < Analyzer
+  #     def token_stream(field, reader)
+  #       return PorterStemFilter.new(LowerCaseTokenizer.new(reader))
+  #     end
+  #   end
+  class PorterStemFilter < TokenFilter
+    # Returns the next input Token, after being stemmed
+    def next()
+      token = @input.next()
+      if (token == nil)
+        return nil
+      else
+        token.term_text = Stemmable.stem_porter(token.term_text)
+      end
+      token
+    end
+  end
+end

data/lib/ferret/analysis/token_stream.rb

@@ -0,0 +1,26 @@
+module Ferret::Analysis
+  # A TokenStream enumerates the sequence of tokens, either from
+  # fields of a document or from query text.
+  #
+  # This is an abstract class.  Concrete subclasses are:
+  # * Tokenizer, a TokenStream whose input is a Reader; and
+  # * TokenFilter, a TokenStream whose input is another TokenStream.
+  class TokenStream
+    # Returns the next token in the stream, or null at EOS.
+    def next
+      raise NotImplementedError
+    end
+
+    # Releases resources associated with this stream.
+    def close
+      raise NotImplementedError
+    end
+
+    # Iterates through the tokens in the field
+    def each # :yields: token
+      while (n = self.next())
+        yield n
+      end
+    end
+  end
+end

data/lib/ferret/analysis/tokenizers.rb

@@ -0,0 +1,107 @@
+require 'strscan'
+
+module Ferret::Analysis
+  # A Tokenizer is a TokenStream whose input is a Reader.
+  #
+  # This is an abstract class.
+  class Tokenizer < TokenStream
+    # By default, closes the input Reader.
+    def close()
+      @input.close()
+    end
+
+    protected
+      # Construct a token stream processing the given input.
+      def initialize(input)
+        @input = input
+      end
+  end
+
+  # An abstract base class for simple regular expression oriented
+  # tokenizers. Very powerful tokenizers can be created using this class as
+  # can be seen from the StandardTokenizer class. Bellow is an example of a
+  # simple implementation of a LetterTokenizer using an RETokenizer.
+  # Basically, a token is a sequence of alphabetic characters separated by
+  # one or more non-alphabetic characters.
+  #
+  #   class LetterTokenizer < RETokenizer
+  #       def token_re()
+  #         /[a-zA-Z]+/
+  #       end
+  #   end
+  class RETokenizer < Tokenizer
+
+    # Initialize with an IO implementing input such as a file.
+    #
+    # input:: must have a read(count) method which returns an array or string
+    #         of _count_ chars.
+    def initialize(input)
+      if input.is_a? String
+        @ss = StringScanner.new(input)
+      else
+        @ss = StringScanner.new(input.read())
+      end
+    end
+
+    # Returns the next token in the stream, or null at EOS.
+    def next()
+      if @ss.scan_until(token_re)
+        term = @ss.matched
+        term_end = @ss.pos
+        term_start = term_end - term.size
+      else
+        return nil
+      end
+
+      return Token.new(normalize(term), term_start, term_end)
+    end
+
+    def close()
+      @ss = nil
+    end
+
+    protected
+      # returns the regular expression used to find the next token
+      def token_re
+        /[a-zA-Z]+/
+      end
+
+      # Called on each token to normalize it before it is added to the
+      # token.  The default implementation does nothing.  Subclasses may
+      # use this to, e.g., lowercase tokens.
+      def normalize(str) return str end
+  end
+
+  
+  # A LetterTokenizer is a tokenizer that divides text at non-letters.
+  # That's to say, it defines tokens as maximal strings of adjacent letters,
+  # as defined by the regular expression _/[a-zA-Z]+/_.
+  class LetterTokenizer < RETokenizer
+    protected
+      # Collects only characters which satisfy the regular expression
+    # _/[a-zA-Z]+/_.
+      def token_re()
+        /[a-zA-Z]+/
+      end
+  end
+
+  # LowerCaseTokenizer performs the function of LetterTokenizer
+  # and LowerCaseFilter together.  It divides text at non-letters and converts
+  # them to lower case.
+  class LowerCaseTokenizer < LetterTokenizer
+    protected
+      def normalize(str)
+        return str.downcase
+      end
+  end
+
+  # A WhiteSpaceTokenizer is a tokenizer that divides text at whiteSpace.
+  # Adjacent sequences of non-WhiteSpace characters form tokens.
+  class WhiteSpaceTokenizer < RETokenizer
+    protected
+      # Collects only characters which are not spaces tabs or carraige returns
+      def token_re()
+        /\S+/
+      end
+  end
+end

data/lib/ferret/analysis/word_list_loader.rb

@@ -0,0 +1,27 @@
+require 'set'
+module Ferret::Analysis
+  # Loader for text files that represent a list of stopwords.
+  module WordListLoader
+    # Loads a text file and adds every line as an entry to a HashSet (omitting
+    # leading and trailing whitespace). Every line of the file should contain only 
+    # one word. The words need to be in lowercase if you make use of an
+    # Analyzer which uses LowerCaseFilter (like GermanAnalyzer).
+    # 
+    # path:: path to file containing the wordlist
+    # return:: A HashSet with the file's words
+    def WordListLoader.word_set_from_file(path)
+      result = Set.new()
+      File.open(path) do |word_file|
+        # we have to strip the end of line characters
+        word_file.each {|line| result << line[0..-2] }
+      end
+      return result
+    end
+
+    def WordListLoader.word_set_from_array(word_array)
+      result = Set.new()
+      word_array.each {|word| result << word }
+      return result
+    end
+  end
+end

data/lib/ferret/document.rb

@@ -0,0 +1,2 @@
+require 'ferret/document/field'
+require 'ferret/document/document'

data/lib/ferret/document/document.rb

@@ -0,0 +1,152 @@
+module Ferret::Document
+  # Documents are the unit of indexing and search.
+  #
+  # A Document is a set of fields.  Each field has a name and a textual
+  # value.  A field may be Field#stored?() with the document, in which case
+  # it is returned with search hits on the document.  Thus each document
+  # should typically contain one or more stored fields which uniquely
+  # identify it.
+  #
+  # Note that fields which are _not_ Field#stored?() are _not_ available in
+  # documents retrieved from the index, e.g. with Hits#doc, Searcher#doc or
+  # IndexReader#document.
+  #
+  # Several fields may be added with the same name.  In this case, if the
+  # fields are indexed, their text is treated as though appended for the
+  # purposes of search.
+  #
+  # Note that add like the remove_field(s) methods only makes sense prior to
+  # adding a document to an index. These methods cannot be used to change
+  # the content of an existing index! In order to achieve this, a document
+  # has to be deleted from an index and a new changed version of that
+  # document has to be added.
+  class Document
+    attr_accessor :boost
+
+    # Constructs a new document with no fields.
+    def initialize()
+      # Values are multiplied into the value of Field#boost of each field in
+      # this document.  Thus, this method in effect sets a default boost for
+      # the fields of this document.
+      #
+      # The default value is 1.0.
+      #
+      # Note: This value is not stored directly with the document in the
+      # index.  Documents returned from IndexReader#document and Hits#doc
+      # may thus not have the same value present as when this document was
+      # indexed.
+      @boost = 1.0
+      @fields = {}
+    end
+
+    # Returns an array of all fields. Note that it is possible for two
+    # fields to appear with the same field name. These will be concatenated
+    # in the index.
+    def all_fields
+      @fields.values.flatten
+    end
+
+    # Returns the number of distinct fields held within the document. This
+    # counts fields which have multiple entries as one.
+    def field_count()
+      return @fields.size
+    end
+
+    # Returns the number of entries held within the document. This counts
+    # all sections so for fields which have multiple entries, each entry
+    # is counted
+    def entry_count()
+      return @fields.values.flatten.size
+    end
+
+    # Adds a field to a document.  Several fields may be added with the same
+    # name.  In this case, if the fields are indexed, their text is treated
+    # as though appended for the purposes of search.
+    #
+    # Note that add like the remove_field(s) methods only makes sense prior
+    # to adding a document to an index. These methods cannot be used to
+    # change the content of an existing index! In order to achieve this, a
+    # document has to be deleted from an index and a new changed version of
+    # that document has to be added.
+    def add_field(field)
+      (@fields[field.name] ||= []) << field
+    end
+    alias :<< :add_field
+
+    # Removes the first field of this name if it exists.
+    def remove_field(name)
+      @fields[name].delete_at(0)
+    end
+
+    # Removes all fields with the given name from the document.
+    #
+    # If there is no field with the specified name, the document remains
+    # unchanged.
+    #
+    # Note that the remove_field(s) methods like the add method only make
+    # sense prior to adding a document to an index. These methods cannot be
+    # used to change the content of an existing index! In order to achieve
+    # this, a document has to be deleted from an index and a new changed
+    # version of that document has to be added.
+    def remove_fields(name)
+      @fields.delete(name)
+    end
+
+    # Returns the first field with the given name.
+    # This method can return _nil_.
+    #
+    # name:: the name of the field
+    # Return:: a _Field_ array
+    def field(name)
+      @fields[name] ? @fields[name][0] : nil
+    end
+
+    # Returns an array of all fields with the given name.
+    # This method can return _nil_.
+    #
+    # name:: the name of the field
+    # Return:: a _Field_ array
+    def fields(name)
+      @fields[name]
+    end
+
+    # Returns an array of values of the field specified as the method
+    # parameter.  This method can return _nil_.
+    #
+    # name:: the name of the field
+    # Return:: a _String_ of field values
+    def values(name)
+      return nil if @fields[name].nil?
+      @fields[name].map {|f| f.data if not f.binary? }.join(" ")
+    end
+    alias :[] :values
+
+    # Sets the data in field +field+ to +text+. If there is more than one
+    # field of that name then it will set the data in the first field of that
+    # name.
+    def []=(field_name, data)
+      field = field(field_name)
+      raise ArgumentError, "Field does not exist" unless field
+      field.data = data
+    end
+
+    # Returns an array of binaries of the field specified as the method
+    # parameter.  This method can return _nil_.
+    #
+    # name:: the name of the field
+    # Return:: a _String_ of field values
+    def binaries(name)
+      binaries = []
+      @fields[name].each {|f| binaries << f.data if f.binary? }
+      return binaries
+    end
+
+    # Prints the fields of a document for human consumption.#/
+    def to_s()
+      field_str = ""
+      @fields.each_key { |name| field_str += name + " " }
+      field_str[-1] = ">"
+      return "Document<" + field_str
+    end
+  end
+end