xapian_db 1.2.5 → 1.2.5.1

data/CHANGELOG.md

@@ -1,3 +1,10 @@
+##1.2.5.1 (January 7th, 2013)
+
+Changes:
+
+  - New option to enable / disable phrase search support
+  - term splitter option to boost "live queries" (see README)
+
 ##1.2.5 (January 4th, 2013)
 
 Changes:

data/README.rdoc

@@ -83,6 +83,14 @@ You can override these defaults by placing a config file named 'xapian_db.yml' i
     database: db/xapian_db/production
     <<: *defaults
 
+==== Available options
+
+  - adapter: :active_record|:datamapper, default: :active_record
+  - language: any iso language code, default: :none (activates spelling corrections, stemmer and stop words if an iso language code ist set)
+  - term_min_length: <n>, default: 1 (do not index terms shorter than n)
+  - enable_phrase_search: true|false, default: false (see the xapian docs for an intro to phrase searching)
+  - term_splitter_count: <n>, default: 0 (see chapter Term Splitting)
+
 If you do not configure settings for an environment in this file, xapian_db applies the defaults.
 
 === Configure an index blueprint
@@ -370,6 +378,10 @@ XapianDb serializes objects to xapian documents using YAML by default. This way,
 
 However, dates need special handling to support date range queries. To support date range queries and allow the addition of other custom data types in the future, XapianDb uses a simple, extensible mechanism to serialize / deserialize your objects. An example on how to extend this mechanism is provided in examples/custom_serialization.rb.
 
+== Term Splitting
+
+If you want to build a realtime search showing results while the user types, you might experience very poor performance and a huge memory load for the first typed characters (1*, 12*...). XapianDb allows you to configure the term_splitter_count to avoid this. If you configure a term_splitter_count of e.g. 2, the term "test" will get indexed with "t", "te" and "test". Now you can apply the "*" only for search terms that are longer than the configured term_splitter_count resulting in a much better performance and lower memory footprint.
+
 == Production setup
 
 Since Xapian allows only one database instance to write to the index, the default setup of XapianDb will not work

data/lib/type_codec.rb

@@ -130,10 +130,13 @@ module XapianDb
       # @param [Integer, BigDecimal, Float] number a number object to encode
       # @return [String] the encoded number
       def self.encode(number)
-        begin
-          Xapian::sortable_serialise number
-        rescue TypeError
-          raise ArgumentError.new "#{number} was expected to be a number"
+        case number.class.name
+          when "Fixnum", "Float"
+            Xapian::sortable_serialise number
+          when "BigDecimal"
+            Xapian::sortable_serialise number.to_f
+          else
+            raise ArgumentError.new "#{number} was expected to be a number"
         end
       end
 

data/lib/xapian_db/config.rb

@@ -53,13 +53,21 @@ module XapianDb
         @config.instance_variable_get("@_term_min_length") || 1
       end
 
+      def phrase_search_enabled?
+        @config.instance_variable_get("@_phrase_search_enabled") || false
+      end
+
+      def term_splitter_count
+        @config.instance_variable_get("@_term_splitter_count") || 0
+      end
     end
 
     # ---------------------------------------------------------------------------------
     # DSL methods
     # ---------------------------------------------------------------------------------
 
-    attr_reader :_database, :_adapter, :_writer, :_beanstalk_daemon, :_resque_queue, :_stemmer, :_stopper, :_term_min_length
+    attr_reader :_database, :_adapter, :_writer, :_beanstalk_daemon, :_resque_queue, :_stemmer, :_stopper, :_term_min_length,
+                :_phrase_search_enabled, :_term_splitter_count
 
     # Set the global database to use
     # @param [String] path The path to the database. Either apply a file sytem path or :memory
@@ -144,6 +152,20 @@ module XapianDb
     def term_min_length(length)
       @_term_min_length = length
     end
+
+    # Enable phrase search support ("search this exact sentence")
+    def enable_phrase_search
+      @_phrase_search_enabled = true
+    end
+
+    # Disable phrase search support ("search this exact sentence")
+    def disable_phrase_search
+      @_phrase_search_enabled = false
+    end
+
+    def term_splitter_count(count)
+      @_term_splitter_count = count
+    end
   end
 
 end

data/lib/xapian_db/indexer.rb

@@ -86,6 +86,7 @@ module XapianDb
           values = get_values_to_index_from obj
           values.each do |value|
             terms = value.to_s.downcase
+            terms = split(terms) if XapianDb::Config.term_splitter_count > 0
             # Add value with field name
             term_generator.index_text(terms, options.weight, "X#{method.upcase}") if options.prefixed
             # Add value without field name
@@ -115,6 +116,16 @@ module XapianDb
       obj.to_s.nil? ? [] : [obj]
     end
 
-  end
+    private
+
+    def split(terms)
+      splitted_terms = []
+      terms.split(" ").each do |term|
+        (1..XapianDb::Config.term_splitter_count).each { |i| splitted_terms << term[0...i] }
+        splitted_terms << term
+      end
+      splitted_terms.join " "
+    end
 
+  end
 end

data/lib/xapian_db/query_parser.rb

@@ -21,7 +21,7 @@ module XapianDb
       @query_flags |= Xapian::QueryParser::FLAG_BOOLEAN             # enable boolean operators
       @query_flags |= Xapian::QueryParser::FLAG_BOOLEAN_ANY_CASE    # enable case insensitive boolean operators
       @query_flags |= Xapian::QueryParser::FLAG_SPELLING_CORRECTION # enable spelling corrections
-      @query_flags |= Xapian::QueryParser::FLAG_PHRASE              # enable phrase searches
+      @query_flags |= Xapian::QueryParser::FLAG_PHRASE if XapianDb::Config.phrase_search_enabled?
     end
 
     # Parse an expression

data/lib/xapian_db/railtie.rb

@@ -51,6 +51,12 @@ module XapianDb
         config.resque_queue @resque_queue
         config.language @language.try(:to_sym)
         config.term_min_length @term_min_length
+        if @enable_phrase_search
+          config.enable_phrase_search
+        else
+          config.disable_phrase_search
+        end
+        config.term_splitter_count @term_splitter_count
       end
 
     end
@@ -66,22 +72,26 @@ module XapianDb
 
     # use the config options from the config file
     def self.configure_from(env_config)
-      @database_path    = env_config["database"] || ":memory:"
-      @adapter          = env_config["adapter"]  || :active_record
-      @writer           = env_config["writer"]   || :direct
-      @beanstalk_daemon = env_config["beanstalk_daemon"]
-      @resque_queue     = env_config["resque_queue"]
-      @language         = env_config["language"]
-      @term_min_length  = env_config["term_min_length"]
+      @database_path        = env_config["database"] || ":memory:"
+      @adapter              = env_config["adapter"]  || :active_record
+      @writer               = env_config["writer"]   || :direct
+      @beanstalk_daemon_url = env_config["beanstalk_daemon"]
+      @resque_queue         = env_config["resque_queue"]
+      @language             = env_config["language"]
+      @term_min_length      = env_config["term_min_length"]
+      @enable_phrase_search = env_config["enable_phrase_search"] == true
+      @term_splitter_count  = env_config["term_splitter_count"] || 0
     end
 
     # set default config options
     def self.configure_defaults
       Rails.env == "test" ? @database_path = ":memory:" : @database_path = "db/xapian_db/#{Rails.env}"
-      @adapter          = :active_record
-      @writer           = :direct
-      @beanstalk_daemon = nil
-      @term_min_length  = 1
+      @adapter              = :active_record
+      @writer               = :direct
+      @beanstalk_daemon     = nil
+      @term_min_length      = 1
+      @enable_phrase_search = false
+      @term_splitter_count  = 0
     end
 
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: xapian_db
 version: !ruby/object:Gem::Version
-  version: 1.2.5
+  version: 1.2.5.1
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-01-05 00:00:00.000000000 Z
+date: 2013-01-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: daemons