ferret 0.9.6 → 0.10.0

data/lib/ferret/analysis/token_filters.rb

@@ -1,86 +0,0 @@
-module Ferret::Analysis
-  # A TokenFilter is a TokenStream whose input is another token stream.
-  #
-  # This is an abstract class.
-  class TokenFilter < TokenStream
-    # Close the input TokenStream.
-    def close()
-      @input.close()
-    end
-
-    protected
-      # Construct a token stream filtering the given input.
-      def initialize(input)
-        @input = input
-      end
-  end
-
-  # Normalizes token text to lower case.
-  class LowerCaseFilter < TokenFilter
-    def next()
-      t = @input.next()
-
-      if (t == nil)
-        return nil
-      end
-
-      t.text = t.text.downcase()
-
-      return t
-    end
-  end
-
-  # Removes stop words from a token stream. To will need to pass your own
-  # set of stopwords to use this stop filter. If you with to use the default
-  # list of stopwords then use the StopAnalyzer.
-  class StopFilter < TokenFilter
-    # Constructs a filter which removes words from the input
-    # TokenStream that are named in the array of words.
-    def initialize(input, stop_set)
-      super(input);
-      @stop_set = stop_set
-    end
-
-    def StopFilter.new_with_file(input, path)
-      ws = WordListLoader.word_set_from_file(path)
-      return StopFilter.new(input, ws)
-    end
-
-    # Returns the next input Token whose termText() is not a stop word.
-    def next()
-      # return the first non-stop word found
-      while token = @input.next()
-        return token if ! @stop_set.include?(token.text)
-      end
-      return nil
-    end
-  end
-
-  # Transforms the token stream as per the Porter stemming algorithm.
-  # Note: the input to the stemming filter must already be in lower case,
-  # so you will need to use LowerCaseFilter or LowerCaseTokenizer further
-  # down the Tokenizer chain in order for this to work properly!
-  # 
-  # To use this filter with other analyzers, you'll want to write an
-  # Analyzer class that sets up the TokenStream chain as you want it.
-  # To use this with LowerCaseTokenizer, for example, you'd write an
-  # analyzer like this:
-  #
-  #   def MyAnalyzer < Analyzer
-  #     def token_stream(field, reader)
-  #       return PorterStemFilter.new(LowerCaseTokenizer.new(reader))
-  #     end
-  #   end
-  class PorterStemFilter < TokenFilter
-    # Returns the next input Token, after being stemmed
-    def next()
-      token = @input.next()
-      if (token == nil)
-        return nil
-      else
-        token.text = Stemmable.stem_porter(token.text)
-      end
-      token
-    end
-  end
-end

data/lib/ferret/analysis/token_stream.rb

@@ -1,26 +0,0 @@
-module Ferret::Analysis
-  # A TokenStream enumerates the sequence of tokens, either from
-  # fields of a document or from query text.
-  #
-  # This is an abstract class.  Concrete subclasses are:
-  # * Tokenizer, a TokenStream whose input is a Reader; and
-  # * TokenFilter, a TokenStream whose input is another TokenStream.
-  class TokenStream
-    # Returns the next token in the stream, or null at EOS.
-    def next
-      raise NotImplementedError
-    end
-
-    # Releases resources associated with this stream.
-    def close
-      raise NotImplementedError
-    end
-
-    # Iterates through the tokens in the field
-    def each # :yields: token
-      while (n = self.next())
-        yield n
-      end
-    end
-  end
-end

data/lib/ferret/analysis/tokenizers.rb

@@ -1,112 +0,0 @@
-require 'strscan'
-
-module Ferret::Analysis
-  # A Tokenizer is a TokenStream whose input is a Reader.
-  #
-  # This is an abstract class.
-  class Tokenizer < TokenStream
-    # By default, closes the input Reader.
-    def close()
-      @input.close()
-    end
-
-    protected
-      # Construct a token stream processing the given input.
-      def initialize(input)
-        @input = input
-      end
-  end
-
-  # An abstract base class for simple regular expression oriented
-  # tokenizers. Very powerful tokenizers can be created using this class as
-  # can be seen from the StandardTokenizer class. Bellow is an example of a
-  # simple implementation of a LetterTokenizer using an RegExpTokenizer.
-  # Basically, a token is a sequence of alphabetic characters separated by
-  # one or more non-alphabetic characters.
-  #
-  #   class LetterTokenizer < RegExpTokenizer
-  #       def token_re()
-  #         /[[:alpha:]]+/
-  #       end
-  #   end
-  class RegExpTokenizer < Tokenizer
-
-    # Initialize with an IO implementing input such as a file.
-    #
-    # input:: must have a read(count) method which returns an array or string
-    #         of _count_ chars.
-    def initialize(input)
-      #@token_buffer = Token.new("", 0, 0)
-      if input.is_a? String
-        @ss = StringScanner.new(input)
-      else
-        @ss = StringScanner.new(input.read())
-      end
-    end
-
-    # Returns the next token in the stream, or null at EOS.
-    def next()
-      if @ss.scan_until(token_re)
-        term = @ss.matched
-        term_end = @ss.pos
-        term_start = term_end - term.size
-      else
-        return nil
-      end
-
-      #return @token_buffer.set!(normalize(term), term_start, term_end)
-      return Token.new(normalize(term), term_start, term_end)
-    end
-
-    def close()
-      @ss = nil
-    end
-
-    protected
-      # returns the regular expression used to find the next token
-      TOKEN_RE = /[[:alpha:]]+/
-      def token_re
-        TOKEN_RE
-      end
-
-      # Called on each token to normalize it before it is added to the
-      # token.  The default implementation does nothing.  Subclasses may
-      # use this to, e.g., lowercase tokens.
-      def normalize(str) return str end
-  end
-
-  
-  # A LetterTokenizer is a tokenizer that divides text at non-letters.
-  # That's to say, it defines tokens as maximal strings of adjacent letters,
-  # as defined by the regular expression _/[[:alpha:]]+/_.
-  class LetterTokenizer < RegExpTokenizer
-    protected
-      # Collects only characters which satisfy the regular expression
-      # _/[[:alpha:]]+/_.
-      TOKEN_RE = /[[:alpha:]]+/
-      def token_re
-        TOKEN_RE
-      end
-  end
-
-  # LowerCaseTokenizer performs the function of LetterTokenizer
-  # and LowerCaseFilter together.  It divides text at non-letters and converts
-  # them to lower case.
-  class LowerCaseTokenizer < LetterTokenizer
-    protected
-      def normalize(str)
-        return str.downcase
-      end
-  end
-
-  # A WhiteSpaceTokenizer is a tokenizer that divides text at whiteSpace.
-  # Adjacent sequences of non-WhiteSpace characters form tokens.
-  class WhiteSpaceTokenizer < RegExpTokenizer
-    protected
-      # Collects only characters which are not spaces tabs or carraige returns
-      TOKEN_RE = /\S+/
-      def token_re
-        TOKEN_RE
-      end
-  end
-end

data/lib/ferret/analysis/word_list_loader.rb

@@ -1,27 +0,0 @@
-require 'set'
-module Ferret::Analysis
-  # Loader for text files that represent a list of stopwords.
-  module WordListLoader
-    # Loads a text file and adds every line as an entry to a HashSet (omitting
-    # leading and trailing whitespace). Every line of the file should contain only 
-    # one word. The words need to be in lowercase if you make use of an
-    # Analyzer which uses LowerCaseFilter (like GermanAnalyzer).
-    # 
-    # path:: path to file containing the wordlist
-    # return:: A HashSet with the file's words
-    def WordListLoader.word_set_from_file(path)
-      result = Set.new()
-      File.open(path) do |word_file|
-        # we have to strip the end of line characters
-        word_file.each {|line| result << line[0..-2] }
-      end
-      return result
-    end
-
-    def WordListLoader.word_set_from_array(word_array)
-      result = Set.new()
-      word_array.each {|word| result << word }
-      return result
-    end
-  end
-end

data/lib/ferret/document/document.rb

@@ -1,152 +0,0 @@
-module Ferret::Document
-  # Documents are the unit of indexing and search.
-  #
-  # A Document is a set of fields.  Each field has a name and a textual
-  # value.  A field may be Field#stored?() with the document, in which case
-  # it is returned with search hits on the document.  Thus each document
-  # should typically contain one or more stored fields which uniquely
-  # identify it.
-  #
-  # Note that fields which are _not_ Field#stored?() are _not_ available in
-  # documents retrieved from the index, e.g. with Hits#doc, Searcher#doc or
-  # IndexReader#document.
-  #
-  # Several fields may be added with the same name.  In this case, if the
-  # fields are indexed, their text is treated as though appended for the
-  # purposes of search.
-  #
-  # Note that add like the remove_field(s) methods only makes sense prior to
-  # adding a document to an index. These methods cannot be used to change
-  # the content of an existing index! In order to achieve this, a document
-  # has to be deleted from an index and a new changed version of that
-  # document has to be added.
-  class Document
-    attr_accessor :boost
-
-    # Constructs a new document with no fields.
-    def initialize()
-      # Values are multiplied into the value of Field#boost of each field in
-      # this document.  Thus, this method in effect sets a default boost for
-      # the fields of this document.
-      #
-      # The default value is 1.0.
-      #
-      # Note: This value is not stored directly with the document in the
-      # index.  Documents returned from IndexReader#document and Hits#doc
-      # may thus not have the same value present as when this document was
-      # indexed.
-      @boost = 1.0
-      @fields = {}
-    end
-
-    # Returns an array of all fields. Note that it is possible for two
-    # fields to appear with the same field name. These will be concatenated
-    # in the index.
-    def all_fields
-      @fields.values.flatten
-    end
-
-    # Returns the number of distinct fields held within the document. This
-    # counts fields which have multiple entries as one.
-    def field_count()
-      return @fields.size
-    end
-
-    # Returns the number of entries held within the document. This counts
-    # all sections so for fields which have multiple entries, each entry
-    # is counted
-    def entry_count()
-      return @fields.values.flatten.size
-    end
-
-    # Adds a field to a document.  Several fields may be added with the same
-    # name.  In this case, if the fields are indexed, their text is treated
-    # as though appended for the purposes of search.
-    #
-    # Note that add like the remove_field(s) methods only makes sense prior
-    # to adding a document to an index. These methods cannot be used to
-    # change the content of an existing index! In order to achieve this, a
-    # document has to be deleted from an index and a new changed version of
-    # that document has to be added.
-    def add_field(field)
-      (@fields[field.name.to_s] ||= []) << field
-    end
-    alias :<< :add_field
-
-    # Removes the first field of this name if it exists.
-    def remove_field(name)
-      @fields[name.to_s].delete_at(0)
-    end
-
-    # Removes all fields with the given name from the document.
-    #
-    # If there is no field with the specified name, the document remains
-    # unchanged.
-    #
-    # Note that the remove_field(s) methods like the add method only make
-    # sense prior to adding a document to an index. These methods cannot be
-    # used to change the content of an existing index! In order to achieve
-    # this, a document has to be deleted from an index and a new changed
-    # version of that document has to be added.
-    def remove_fields(name)
-      @fields.delete(name.to_s)
-    end
-
-    # Returns the first field with the given name.
-    # This method can return _nil_.
-    #
-    # name:: the name of the field
-    # Return:: a _Field_ array
-    def field(name)
-      @fields[name.to_s] ? @fields[name.to_s][0] : nil
-    end
-
-    # Returns an array of all fields with the given name.
-    # This method can return _nil_.
-    #
-    # name:: the name of the field
-    # Return:: a _Field_ array
-    def fields(name)
-      @fields[name.to_s]
-    end
-
-    # Returns an array of values of the field specified as the method
-    # parameter.  This method can return _nil_.
-    #
-    # name:: the name of the field
-    # Return:: a _String_ of field values
-    def values(name)
-      return nil if @fields[name.to_s].nil?
-      @fields[name.to_s].map {|f| f.data if not f.binary? }.join(" ")
-    end
-    alias :[] :values
-
-    # Sets the data in field +field+ to +text+. If there is more than one
-    # field of that name then it will set the data in the first field of that
-    # name. If there is no field of that name, then a new one will be created
-    def []=(field_name, data)
-      field = field(field_name.to_s)
-      if field
-        field.data = data
-      else
-        add_field(Field.new(field_name.to_s, data))
-      end
-    end
-
-    # Returns an array of binaries of the field specified as the method
-    # parameter.  This method can return _nil_.
-    #
-    # name:: the name of the field
-    # Return:: a _String_ of field values
-    def binaries(name)
-      binaries = []
-      @fields[name.to_s].each {|f| binaries << f.data if f.binary? }
-      return binaries
-    end
-
-    # Prints the fields of a document for human consumption.#/
-    def to_s()
-      return "Document{\n  #{@fields.values.join("\n  ")}\n}"
-    end
-  end
-end

data/lib/ferret/document/field.rb

@@ -1,312 +0,0 @@
-module Ferret::Document
-  # A field is a section of a Document.  Each field has two parts, a name
-  # and a value.  Values may be free text, provided as a String or as a
-  # Reader, or they may be atomic keywords, which are not further processed.
-  # Such keywords may be used to represent dates, urls, etc.  Fields are
-  # optionally stored in the index, so that they may be returned with hits
-  # on the document.
-  class Field
-
-    # This value will be
-    # multiplied into the score of all hits on this field of this
-    # document.
-    #
-    # The boost is multiplied by Document#boost of the document
-    # containing this field.  If a document has multiple fields with the same
-    # name, all such values are multiplied together.  This product is then
-    # multipled by the value Similarity#length_norm(String,int), and
-    # rounded by Similarity#encode_norm(float) before it is stored in the
-    # index.  One should attempt to ensure that this product does not overflow
-    # the range of that encoding.
-    #
-    # See Document#set_boost(float)
-    # See Similarity#length_norm(String, int)
-    # See Similarity#encode_norm(float)
-    #
-    # Note: this value is not stored directly with the document in the index.
-    # Documents returned from IndexReader#document(int) and
-    # Hits#doc(int) may thus not have the same value present as when this field
-    # was indexed.
-    attr_accessor :boost, :data 
-    attr_reader :name
-
-    # True iff the value of the field is to be stored in the index for
-    # return with search hits.  It is an error for this to be true if a
-    # field is Reader-valued.
-    def stored?() return @stored end
-
-    # True iff the value of the field is to be indexed, so that it may be
-    # searched on.
-    def indexed?() return @indexed end
-
-    # True iff the value of the field should be tokenized as text prior to
-    # indexing.  Un-tokenized fields are indexed as a single word and may
-    # not be Reader-valued.
-    def tokenized?() return @tokenized end
-
-    # True if the field is to be stored as a binary value. This can be used
-    # to store images or other binary data in the index if you wish
-    def binary?() return @binary end
-
-    # True if you want to compress the data that you store. This is a good
-    # idea for really large text fields. The ruby Zlib library is used to do
-    # the compression
-    def compressed?() return @compressed end
-
-    # True iff the term or terms used to index this field are stored as a
-    # term vector, available from IndexReader#term_freq_vector(). These
-    # methods do not provide access to the original content of the field,
-    # only to terms used to index it. If the original content must be
-    # preserved, use the _stored_ attribute instead.
-    #
-    # See IndexReader#term_freq_vector()
-    def store_term_vector?() return @store_term_vector end
-
-    # True if the positions of the indexed terms in this field are stored.
-    def store_positions?() return @store_position end
-
-    # True if the offsets of this field are stored. The offsets are the
-    # positions of the start and end characters of the token in the whole
-    # field string
-    def store_offsets?() return @store_offset end
-
-    # True if the norms are not stored for this field. No norms means that
-    # index-time boosting and field length normalization will be disabled.
-    # The benefit is less memory usage as norms take up one byte per indexed
-    # field for every document in the index.
-    def omit_norms?() return @omit_norms end
-    
-    class Store < Ferret::Utils::Parameter
-      # Store the original field value in the index in a compressed form.
-      # This is useful for long documents and for binary valued fields.
-      COMPRESS = Store.new("COMPRESS")
-
-      # Store the original field value in the index. This is useful for
-      # short texts like a document's title which should be displayed with
-      # the results. The value is stored in its original form, i.e. no
-      # analyzer is used before it is stored. 
-      YES = Store.new("YES")
-
-      # Do not store the field value in the index.
-      NO = Store.new("NO")
-    end
-
-    class Index < Ferret::Utils::Parameter
-      # Do not index the field value. This field can thus not be searched,
-      # but one can still access its contents provided it is Field.Store
-      # stored
-      NO = Index.new("NO")
-
-      # Index the field's value so it can be searched. An Analyzer will be
-      # used to tokenize and possibly further normalize the text before its
-      # terms will be stored in the index. This is useful for common text.
-      TOKENIZED = Index.new("TOKENIZED")
-
-      # Index the field's value without using an Analyzer, so it can be
-      # searched.  As no analyzer is used the value will be stored as a
-      # single term. This is useful for unique Ids like product numbers.
-      UNTOKENIZED = Index.new("UNTOKENIZED")
-
-      # Index the field's value without an Analyzer, and disable the storing
-      # of norms.  No norms means that index-time boosting and field length
-      # normalization will be disabled.  The benefit is less memory usage as
-      # norms take up one byte per indexed field for every document in the
-      # index.
-      NO_NORMS = Index.new("NO_NORMS");
-    end
-
-    class TermVector < Ferret::Utils::Parameter
-      # Do not store term vectors. 
-      NO = TermVector.new("NO")
-
-      # Store the term vectors of each document. A term vector is a list of
-      # the document's terms and their number of occurences in that
-      # document.
-      YES = TermVector.new("YES")
-
-      # Store the term vector + token position information
-      # 
-      # See #YES
-      WITH_POSITIONS = TermVector.new("WITH_POSITIONS")
-
-      # Store the term vector + Token offset information
-      # 
-      # See #YES
-      WITH_OFFSETS = TermVector.new("WITH_OFFSETS")
-
-      # Store the term vector + Token position and offset information
-      # 
-      # See #YES See #WITH_POSITIONS See #WITH_OFFSETS
-      WITH_POSITIONS_OFFSETS = TermVector.new("WITH_POSITIONS_OFFSETS")
-    end
-
-    # Create a field by specifying its name, value and how it will
-    # be saved in the index.
-    # 
-    # name:: The name of the field
-    # value:: The string to process
-    # store:: Whether _value_ should be stored in the index
-    # index:: Whether the field should be indexed, and if so, if it should
-    #         be tokenized before indexing 
-    #
-    # store_term_vector:: Whether term vector should be stored
-    #  * the field is neither stored nor indexed
-    #  * the field is not indexed but term_vector is _TermVector::YES_
-    #
-    # binary:: Whether you want to store binary data in this field. Default is
-    #    false
-    # boost:: the boost for this field. Default is 1.0. A larger number makes
-    # this field more important.
-    def initialize(name,
-                   value,
-                   store = Store::YES,
-                   index = Index::UNTOKENIZED,
-                   term_vector = TermVector::NO,
-                   binary = false,
-                   boost = 1.0)
-      if (index == Index::NO and store == Store::NO)
-        raise ArgumentError, "it doesn't make sense to have a field that " +
-          "is neither indexed nor stored"
-      end
-      if (index == Index::NO && term_vector != TermVector::NO)
-        raise ArgumentError, "cannot store term vector information for a " +
-          "field that is not indexed"
-      end
-
-      # The name of the field (e.g., "date", "subject", "title", or "body")
-      @name = name.to_s
-
-      # the one and only data object for all different kind of field values
-      @data = value
-      self.store = store
-      self.index = index
-      self.term_vector = term_vector
-      @binary = binary
-      @boost = boost
-    end
-
-    def store=(store)
-      case store
-      when Store::YES
-        @stored = true
-        @compressed = false
-      when Store::COMPRESS
-        @stored = true
-        @compressed = true
-      when Store::NO
-        @stored = false
-        @compressed = false
-      else
-        raise "unknown stored parameter " + store.to_s
-      end
-    end
-
-    def index=(index)
-      @omit_norms = false
-      case index
-      when Index::NO
-        @indexed = false
-        @tokenized = false
-      when Index::TOKENIZED
-        @indexed = true
-        @tokenized = true
-      when Index::UNTOKENIZED
-        @indexed = true
-        @tokenized = false
-      when Index::NO_NORMS
-        @indexed = true
-        @tokenized = false
-        @omit_norms = true
-      else
-        raise "unknown stored parameter " + index.to_s
-      end
-    end
-
-    def term_vector=(term_vector)
-      case term_vector
-      when TermVector::NO
-        @store_term_vector = false
-        @store_position = false
-        @store_offset = false
-      when TermVector::YES
-        @store_term_vector = true
-        @store_position = false
-        @store_offset = false
-      when TermVector::WITH_POSITIONS
-        @store_term_vector = true
-        @store_position = true
-        @store_offset = false
-      when TermVector::WITH_OFFSETS
-        @store_term_vector = true
-        @store_position = false
-        @store_offset = true
-      when TermVector::WITH_POSITIONS_OFFSETS
-        @store_term_vector = true
-        @store_position = true
-        @store_offset = true
-      else
-        raise "unknown term_vector parameter " + store_term_vector.to_s
-      end
-    end
-
-    # Returns the string value of the data that is stored in this field
-    def string_value
-      if @data.instance_of? String
-        return @data
-      elsif @data.respond_to? :read
-        return @data.read()
-      else
-        # if it is binary object try to return a string representation
-        return @data.to_s
-      end
-    end
-
-    # if the data is stored as a binary, just return it.
-    def binary_value
-      return @data
-    end
-
-    # Returns the string value of the data that is stored in this field
-    def reader_value
-      if @data.respond_to? :read
-        return @data
-      elsif @data.instance_of? String
-        return Ferret::Utils::StringHelper::StringReader.new(@data)
-      else
-        # if it is binary object try to return a string representation
-        return Ferret::Utils::StringHelper::StringReader.new(@data.to_s)
-      end
-    end
-
-    # Create a stored field with binary value. Optionally the value
-    # may be compressed. But it obviously won't be tokenized or
-    # term vectored or anything like that.
-    # 
-    # name:: The name of the field
-    # value:: The binary value
-    # store:: How _value_ should be stored (compressed or not.)
-    def Field.new_binary_field(name, value, stored)
-      if (stored == Store::NO)
-        raise ArgumentError, "binary values can't be unstored"
-      end
-      Field.new(name, value, stored, Index::NO, TermVector::NO, true)
-    end
-
-    # Prints a Field for human consumption.
-    def to_s()
-      str = ""
-      if (@stored)
-        str << "stored"
-        str << (@compressed ? "/compressed," : "/uncompressed,")
-      end
-      str << "indexed," if (@indexed)
-      str << "tokenized," if (@tokenized)
-      str << "store_term_vector," if (@store_term_vector)
-      str << "store_offsets," if (@store_offset)
-      str << "store_positions," if (@store_position)
-      str << "omit_norms," if (@omit_norms)
-      str << "binary," if (@binary)
-      str << "<#{@name}:#{@binary ? '=bin_data=' : data}>"
-    end
-  end
-end