lucene 0.5.0.beta.1

data/lib/lucene/index.rb

@@ -0,0 +1,267 @@
+require 'monitor'
+require 'lucene/jars'
+require 'lucene/transaction'
+require 'lucene/index_searcher'
+require 'lucene/document'
+require 'lucene/field_info'
+require 'lucene/index_info'
+
+#
+# A wrapper for the Java lucene search library.
+#
+module Lucene
+
+  class DocumentDeletedException < StandardError;
+  end
+  class IdFieldMissingException < StandardError;
+  end
+
+  # 
+  # Represents a Lucene Index.
+  # The index is written/updated only when the commit method is called.
+  # This is done since writing to the index file should be done as a batch operation.
+  # (Performance will be bad otherwise).
+  #
+  # For each Thread there is zero or one Index instance. There are at most one Index instance per thread
+  # so there is no need for this class to use synchronization for Thread safety.
+  #  
+  class Index
+    attr_reader :path, :uncommited
+
+
+    # locks per index path, must not write to the same index from 2 threads 
+    @@locks = {}
+    @@locks.extend MonitorMixin
+
+    def initialize(path, index_info)
+      @path = path # a key (i.e. filepath) where the index is stored on disk/or RAM              
+      @index_info = index_info # the actual storage of the index
+      @uncommited = {}  # documents to be commited, a hash of Document
+      @deleted_ids = [] # documents to be deleted
+    end
+
+    def field_infos
+      IndexInfo.instance(@path)
+    end
+
+
+    # Returns an Index instance for the current running transaction.
+    #
+    # Tries to reuse an Index instance for the current running transaction.
+    # If a Lucene::Transaction is running it will register this index in that transaction if
+    # this has not already been done.
+    # When it has been registered in the transaction the transaction will commit the index
+    # when the transaction is finished.
+    # The configuration (kept in the #field_infos) for this index will be the same for all indexes with the same path/key.
+    #
+    # ==== Parameters
+    # path<String>:: The key or location where the index should be stored (relative Lucene::Config[:storage_path]
+    #
+    # ==== Examples
+    # Index.new 'foo/lucene-db'
+    #
+    # ==== Returns
+    # Returns a new or an already existing Index
+    #
+    def self.new(path)
+      # make sure no one modifies the index specified at given path
+      lock(path).synchronize do
+        # create a new transaction if needed
+        Transaction.new unless Transaction.running?
+
+        # create a new instance only if it does not already exist in the current transaction
+        unless Transaction.current.index?(path)
+          info = IndexInfo.instance(path)
+          index = super(path, info)
+          Transaction.current.register_index(path, index)
+        end
+      end
+      # return the index for the current transaction
+      Transaction.current.index(path)
+    end
+
+
+    #
+    # Delete all uncommited documents. Also deregister this index
+    # from the current transaction (if there is one transaction)
+    #
+    def clear
+      @uncommited.clear
+      Transaction.current.deregister_index self if Transaction.running?
+    end
+
+    #
+    # See instance method Index.clear
+    #
+    def self.clear(path)
+      return unless Transaction.running?
+      return unless Transaction.current.index?(path)
+      Transaction.current.index(path).clear
+    end
+
+    # Creates a new document from the given hash of values.
+    # This document will be stored in this instance till it is commited.
+    #
+    # ==== Parameters
+    # path<String>:: The key or location where the index should be stored (relative Lucene::Config[:storage_path]
+    #
+    # ==== Examples
+    # index = Index.new('name_or_path_to_index')
+    # index << {:id=>'1', :name=>'foo'} 
+    #
+    # ==== Returns
+    # Returns the index instance so that this method can be chained
+    #
+    def <<(key_values)
+      doc = Document.new(field_infos, key_values)
+      @uncommited[doc.id] = doc
+      self
+    end
+
+    def id_field
+      @index_info.id_field
+    end
+
+    #
+    # Updates the specified document.
+    # The index file will not be updated until the transaction commits.
+    # The doc is stored in memory till the transaction commits.
+    #
+    def update(doc)
+      @uncommited[doc.id] = doc
+    end
+
+    #
+    # Delete the specified document.
+    # The index file not be updated until the transaction commits.
+    # The id of the deleted document is stored in memory till the transaction commits.
+    #
+    def delete(id)
+      @deleted_ids << id.to_s
+    end
+
+
+    def deleted?(id)
+      @deleted_ids.include?(id.to_s)
+    end
+
+    def updated?(id)
+      @uncommited[id.to_s]
+    end
+
+    # Writes to the index files.
+    # Open and closes an lucene IndexWriter
+    # Close the IndexSearcher so that it will read the updated index next time.
+    # This method will automatically be called from a Lucene::Transaction if it was running when the index was created.
+    #
+    # This method is synchronized since it is not allowed to update a lucene index from several threads at the same time.
+    #
+    def commit
+      lock.synchronize do
+        delete_documents # deletes all documents given @deleted_ids
+
+        # are any updated document deleted ?
+        deleted_ids = @uncommited.keys & @deleted_ids
+        # make sure we don't index deleted document
+        deleted_ids.each {|id| @uncommited.delete(id)}
+
+        # update the remaining documents that has not been deleted
+
+        begin
+          index_writer = org.apache.lucene.index.IndexWriter.new(@index_info.storage, @index_info.analyzer, ! exist?)
+          # removes the document and adds it again
+          @uncommited.each_value { |doc| doc.update(index_writer) }
+        ensure
+          # TODO exception handling, what if ...
+          index_writer.close
+
+          @uncommited.clear
+          @deleted_ids.clear
+
+          # if we are running in a transaction remove this so it will not be committed twice
+          Transaction.current.deregister_index(self) if Transaction.running?
+        end
+      end
+    end
+
+
+    #
+    # Delegates to the IndexSearcher.find method
+    #
+    def find(*query, &block)
+      # new method is a factory method, does not create if it already exists
+      searcher = IndexSearcher.new(@index_info.storage)
+
+      if block.nil?
+        case query.first
+          when String
+            return searcher.find(@index_info, query)
+          when Hash, Array
+            return searcher.find(@index_info, query.first)
+        end
+      else
+        return searcher.find_dsl(@index_info, &block)
+      end
+    end
+
+
+    def to_s
+      "Index [path: '#@path', #{@uncommited.size} documents]"
+    end
+
+    #
+    # -------------------------------------------------------------------------
+    # Private methods
+    #
+
+    private
+
+    #
+    # There is one lock per index path.
+    #
+    def lock
+      @@locks.synchronize do
+        @@locks[@path] ||= Monitor.new
+        @@locks[@path]
+      end
+    end
+
+    def self.lock(path)
+      @@locks.synchronize do
+        @@locks[path] ||= Monitor.new
+        @@locks[path]
+      end
+    end
+
+    #
+    # Returns true if the index already exists.
+    #
+    def exist?
+      @index_info.index_exists?
+    end
+
+    #
+    # --------------------------------------------------------------------------
+    #
+    private
+
+    def delete_documents # :nodoc:
+      return unless exist? # if no index exists then there is nothing to do
+
+      writer = org.apache.lucene.index.IndexWriter.new(@index_info.storage, @index_info.analyzer, false)
+      id_field = @index_info.infos[@index_info.id_field]
+
+      @deleted_ids.each do |id|
+        converted_value = id_field.convert_to_lucene(id)
+        writer.deleteDocuments(org.apache.lucene.index.Term.new(@index_info.id_field.to_s, converted_value))
+      end
+    ensure
+      # TODO exception handling, what if ...
+      writer.close unless writer.nil?
+    end
+
+
+  end
+end
+
+

data/lib/lucene/index_info.rb

@@ -0,0 +1,146 @@
+module Lucene
+  
+  #
+  # Contains info for a specific Index identified by a path
+  # Contains a 
+  # * collection of FieldInfo objects. 
+  # * the name of the id field.
+  # * the index storage, either file based or RAM based.
+  # 
+  # Fields has default value IndexInfo::DEFAULTS.
+  # 
+  class IndexInfo #:nodoc: 
+    DEFAULTS = FieldInfo.new({}).freeze
+    
+    attr_reader :infos, :path
+    attr_accessor :id_field
+    attr_writer :store_on_file
+
+    # Initializes this object by setting values to default values specified in the Lucene::Config.
+    # The path/id to the index is specified by the the path parameter.
+    # If the index is Lucene::Config[:storage_path]
+    # ==== Block parameters
+    # path<String>:: The id or the (incomplete) path on the filesystem of the index
+    #
+    # :api: private
+    def initialize(path)
+      $LUCENE_LOGGER.debug{"IndexInfo#initialize(#{path})"}
+      @id_field = Lucene::Config[:id_field].to_sym
+      @path = path
+      @store_on_file = Lucene::Config[:store_on_file]
+      @infos = {}
+      # always store the id field
+      @infos[@id_field] = FieldInfo.new(:store => true)
+    end
+
+    def to_s
+      "IndexInfo [#{@id_field}, #{@infos.inspect}]"
+    end
+
+    def store_on_file?
+      @store_on_file
+    end
+    
+    def storage
+      @storage ||= create_storage
+    end
+
+    def create_storage
+      if store_on_file?
+        raise StandardError.new("Lucene::Config[:storage_path] is nil but index configured to be stored on filesystem") if Lucene::Config[:storage_path].nil?
+        Lucene::Config[:storage_path] + @path
+      else
+        org.apache.lucene.store.RAMDirectory.new
+      end
+    end
+
+    
+    def self.instance?(path)
+      return false if @instances.nil?
+      ! @instances[path].nil?
+    end
+
+    # Creates and initializes an IndexInfo object by setting values to default
+    # values specified in the Lucene::Config. Does not create new object if it has
+    # already been created before with the given path.
+    # 
+    # If the index is stored on the filesystem the complete path will be
+    # Lucene::Config[:storage_path] + /path
+    # 
+    # ==== Block parameters
+    # path<String>:: The id or the (incomplete) path on the filesystem of the index
+    #
+    # :api: public
+    def self.instance(path)
+      @instances ||= {}
+      $LUCENE_LOGGER.debug{"IndexInfos#instance(#{path}) : @instances[path]: #{@instances[path]}"}
+      @instances[path] ||= IndexInfo.new(path)
+    end
+    
+    def self.delete_all
+      $LUCENE_LOGGER.debug{"IndexInfos#delete_all"}
+      @instances = nil
+    end
+    
+    def self.index_exists(path)
+      return false if @instances[path].nil?
+      instance(path).index_exists?
+    end
+    
+    def index_exists?
+      org.apache.lucene.index.IndexReader.index_exists(storage)
+    end
+    
+    def each_pair
+      @infos.each_pair{|key,value| yield key,value}
+    end
+
+    def analyzer
+      # do all fields have the default value :standard analyzer ?
+      if @infos.values.find {|info| info[:analyzer] != :standard}
+        # no, one or more has set
+        wrapper = org.apache.lucene.analysis.PerFieldAnalyzerWrapper.new(org.apache.lucene.analysis.standard.StandardAnalyzer.new)
+        @infos.each_pair do |key,value|
+          case value[:analyzer]
+            when :keyword
+              wrapper.addAnalyzer(key.to_s, org.apache.lucene.analysis.KeywordAnalyzer.new)
+            when :standard
+              # default
+            when :simple
+              wrapper.addAnalyzer(key.to_s, org.apache.lucene.analysis.SimpleAnalyzer.new)
+            when :whitespace
+              wrapper.addAnalyzer(key.to_s, org.apache.lucene.analysis.WhitespaceAnalyzer.new)
+            when :stop
+              wrapper.addAnalyzer(key.to_s, org.apache.lucene.analysis.StopAnalyzer.new)
+            else
+              raise "Unknown analyzer, supports :keyword, :standard, :simple, :stop, :whitspace, got '#{value}' for field '#{key}'"
+          end
+        end
+        wrapper
+      else
+        # yes, all fields has standard analyzer
+        org.apache.lucene.analysis.standard.StandardAnalyzer.new
+      end
+    end
+
+    # Returns true if it has one or more tokenized fields
+    def tokenized?
+      @infos.values.find{|field_info| field_info.tokenized?}
+    end
+
+    def [](key)
+      k = key.to_sym
+      $LUCENE_LOGGER.debug{"FieldInfos create new FieldInfo key '#{k}'"} if @infos[k].nil?
+      @infos[k] ||= DEFAULTS.dup
+      @infos[k]
+    end
+    
+    def []=(key,value)
+      case value
+      when Hash then @infos[key] = FieldInfo.new(value)
+      when FieldInfo then @infos[key] = value
+      else raise ArgumentError.new("only accept Hash and FieldInfo, got #{value.class.to_s}")
+      end
+    end
+  end
+end

data/lib/lucene/index_searcher.rb

@@ -0,0 +1,157 @@
+module Lucene
+
+  class Asc
+    class << self
+
+      # Specifies which fields should be sorted in ascending order
+      #
+      # ==== Parameters
+      # fields:: One or more fields to sort in ascending order (Array)
+      #
+      # ==== Examples
+      #  Asc[:name, :age]
+      #
+      # ==== Returns
+      # An array of sort fields
+      #
+      def [](*fields)
+        fields.map{|x| org.apache.lucene.search.SortField.new(x.to_s)}
+      end
+    end
+  end
+
+  class Desc
+    class << self
+      # Specifies which fields should be sorted in descending order
+      #
+      # ==== Block parameters
+      # fields:: One or more fields to sort in descending order (Array)
+      #
+      # ==== Examples
+      #  Desc[:name, :age]
+      #
+      # ==== Returns
+      # An array of sort fields
+      #
+      def [](*fields)
+        fields.map{|x| org.apache.lucene.search.SortField.new(x.to_s, true)}
+        #org.apache.lucene.search.Sort.new(values.map{|x| org.apache.lucene.search.SortField.new(x.to_s, true)}.to_java(:'org.apache.lucene.search.SortField'))
+      end
+    end
+  end
+
+  #
+  # Does reuse Lucene Index Search for the same index.
+  # Reloads the index if the index has changed.
+  #
+  class IndexSearcher
+    
+    @@paths = {}
+    
+    def initialize(path)
+      @path = path
+    end
+
+    #
+    # Only create a new object if it does not already exist for this path    
+    #
+    def self.new(path)
+      @@paths[path] = super(path) if @@paths[path].nil?
+      @@paths[path]
+    end
+
+    def find_dsl(field_infos,&block)
+      exp = QueryDSL.parse(&block)  
+      query = exp.to_lucene(field_infos)
+      
+      Hits.new(field_infos, index_searcher.search(query))      
+    end
+    
+    
+    def find(field_info, query)
+      # are there any index for this node ?
+      # if not return an empty array
+      return [] unless exist?
+      
+      #puts "QUERY #{query.inspect}" # '#{query.first.class.to_s}' value #{query.first}"
+      sort_by ||= query[1].delete(:sort_by) if query[1].kind_of?(Hash)
+      sort_by ||= query.delete(:sort_by)
+      #puts "QUERY sort #{sort_by}"
+      # TODO Refactoring ! too long and complex method
+      lucene_query = case query
+      when Array
+        sort_by ||= query.last.delete(:sort_by) if query.last.kind_of?(Hash)
+        parser = org.apache.lucene.queryParser.QueryParser.new(field_info.id_field.to_s, field_info.analyzer)
+        parser.parse(query.first)
+      when Hash
+        bquery = org.apache.lucene.search.BooleanQuery.new
+        query.each_pair do |key,value|
+          field = field_info[key]
+          q = field.convert_to_query(key, value)
+          bquery.add(q, org.apache.lucene.search.BooleanClause::Occur::MUST)
+        end
+        bquery
+      else
+        raise StandardError.new("Unknown type #{query.class.to_s} for find #{query}")
+      end
+      
+      if sort_by.nil?
+        Hits.new(field_info, index_searcher.search(lucene_query))
+      else
+        sort = create_sort(sort_by) 
+        Hits.new(field_info, index_searcher.search(lucene_query, sort))
+      end
+      
+    end
+
+    def parse_field(field)
+      case field
+      when String,Symbol
+        [org.apache.lucene.search.SortField.new(field.to_s)]
+      when org.apache.lucene.search.SortField
+        [field]
+      when Array
+        raise StandardError.new("Unknown sort field '#{field}'") unless field.first.kind_of?(org.apache.lucene.search.SortField)
+        field
+      end
+    end
+
+
+    def create_sort(fields)
+      case fields
+      when String,Symbol
+        org.apache.lucene.search.Sort.new(fields.to_s)
+      when org.apache.lucene.search.SortField
+        org.apache.lucene.search.Sort.new(fields)
+      when Array
+        sorts = []
+        fields.each do |field|
+          sorts += parse_field(field)
+        end
+        org.apache.lucene.search.Sort.new(sorts.to_java(:'org.apache.lucene.search.SortField')) 
+      else
+        StandardError.new("Unknown type #{fields.class.to_s}")
+      end
+    end
+
+    #
+    # Checks if it needs to reload the index searcher
+    #
+    def index_searcher
+      if @index_reader.nil? || @index_reader.getVersion() != org.apache.lucene.index.IndexReader.getCurrentVersion(@path)
+        @index_reader = org.apache.lucene.index.IndexReader.open(@path)
+        @index_searcher = org.apache.lucene.search.IndexSearcher.new(@index_reader)
+        $LUCENE_LOGGER.debug("Opened new IndexSearcher for #{to_s}")
+      end
+      @index_searcher
+    end
+    
+    #
+    # Returns true if the index already exists.
+    #
+    def exist?
+      org.apache.lucene.index.IndexReader.index_exists(@path)
+    end
+
+  end
+end