ultrasphinx 1.8 → 1.9

data.tar.gz.sig

@@ -1,3 +1 @@
-U���%艠�}L~�@�0XU׸�
-�%�g($�.���#`������h�W����z\_{wo�N��@rf����4�EAʭ?��l��������П�;�g���͠v���
-�[�>A���"F������3d|�[4�Z������z�mD��E$`sˎ���b�+��*
Q�ߩp�R`��#~"�׏�B�T��
+(Qr��u'�9���_�U؇�4]�;��x�*����oA����[Y�?�.0�,7�˗�K�ƛ8@<�j�':
��4|�F����)�ɝ�}a�m��g8E#�D^+(aX]LM�k�
|��EL_P�b<�JU$����>��kr��>�Pp!�g	��m�Xy�T� �ZI��M�DB&�K�w����E���W�~�6��=���qO�	v$��1!��u�Qȷ�
n��5A�ٳ����=V��lz6Il

data/CHANGELOG

@@ -1,4 +1,8 @@
 
+v1.9. Delta indexing. ERb now supported in .base files. Allow setting the searched indexes at runtime.
+
+v1.8.1. Use multifind/multiget for record loading; avoid using HashWithIndifferentAccess internally for speed; other minor performance improvements.
+
 v1.8. Update client for compatibility with Sphinx 0.9.8 r1112. This is a breaking release! You need to update Sphinx along with Ultrasphinx. Float range bugfix; text faceting on association_include bugfix. Postgres users, please note that the return type of CRC32() has changed to bigint.
 
 v1.7. Deployment docs. Postgres fixes. Support association_name instead of class_name (Daniel Higginbotham). 

data/DEPLOYMENT_NOTES

@@ -15,12 +15,16 @@ You will <b>not</b> want to keep the generated <tt>development.conf</tt> in the
 
 It's easy to keep the search daemon and the indexer running in a production environment. Cronjobs are the best way:
 
-  0,30 * * * * bash -c 'cd /path/to/production/current/; RAILS_ENV=production \
-    rake ultrasphinx:index >> /log/ultrasphinx-index.log 2>&1'
+  */6 * * * * bash -c 'cd /path/to/production/current/; RAILS_ENV=production \
+    rake ultrasphinx:index:delta >> /log/ultrasphinx-index.log 2>&1'
+  1 4 * * * * bash -c 'cd /path/to/production/current/; RAILS_ENV=production \
+    rake ultrasphinx:index:main >> /log/ultrasphinx-index.log 2>&1'
   */3 * * * * bash -c 'cd /path/to/production/current/; RAILS_ENV=production \
     rake ultrasphinx:daemon:start >> /log/ultrasphinx-daemon.log 2>&1'
 
-The first line runs the indexer every thirty minutes. The second line will try to restart the search daemon every three minutes. If it's already running, nothing happens. 
+The first line reindexes the delta index every 10 minutes. The second line reindexes the main index once a day at 4am. The third line will try to restart the search daemon every three minutes. If it's already running, nothing happens. 
+
+Of course if you don't have any models with deltas, don't include the <tt>ultrasphinx:index:delta</tt> task.
 
 If you are under severe memory limitations you might want to manage the daemon with Monit instead, so you can keep a closer eye on it. The search daemon is extremely reliable, so don't bother with fancy monitoring infrastructure unless you're sure you need it.
 

data/Manifest

@@ -71,6 +71,7 @@ test/integration/app/config/environments/test.rb
 test/integration/app/config/locomotive.yml
 test/integration/app/config/routes.rb
 test/integration/app/config/ultrasphinx/default.base
+test/integration/app/config/ultrasphinx/development.conf
 test/integration/app/config/ultrasphinx/development.conf.canonical
 test/integration/app/db/migrate/001_create_users.rb
 test/integration/app/db/migrate/002_create_sellers.rb
@@ -81,6 +82,7 @@ test/integration/app/db/migrate/006_add_deleted_to_user.rb
 test/integration/app/db/migrate/007_add_lat_and_long_to_address.rb
 test/integration/app/db/migrate/008_add_mission_statement_to_seller.rb
 test/integration/app/db/migrate/009_create_countries.rb
+test/integration/app/db/schema.rb
 test/integration/app/doc/README_FOR_APP
 test/integration/app/public/404.html
 test/integration/app/public/500.html
@@ -123,13 +125,16 @@ test/integration/app/test/functional/states_controller_test.rb
 test/integration/app/test/functional/users_controller_test.rb
 test/integration/app/test/test_helper.rb
 test/integration/app/test/unit/address_test.rb
+test/integration/app/test/unit/country_test.rb
 test/integration/app/test/unit/seller_test.rb
 test/integration/app/test/unit/state_test.rb
 test/integration/app/test/unit/user_test.rb
 test/integration/configure_test.rb
+test/integration/delta_test.rb
 test/integration/search_test.rb
 test/integration/server_test.rb
 test/integration/spell_test.rb
+test/profile/benchmark.rb
 test/setup.rb
 test/teardown.rb
 test/test_all.rb
@@ -149,6 +154,7 @@ vendor/riddle/spec/fixtures/data/anchor.bin
 vendor/riddle/spec/fixtures/data/any.bin
 vendor/riddle/spec/fixtures/data/boolean.bin
 vendor/riddle/spec/fixtures/data/distinct.bin
+vendor/riddle/spec/fixtures/data/field_weights.bin
 vendor/riddle/spec/fixtures/data/filter.bin
 vendor/riddle/spec/fixtures/data/filter_array.bin
 vendor/riddle/spec/fixtures/data/filter_array_exclude.bin
@@ -159,12 +165,16 @@ vendor/riddle/spec/fixtures/data/filter_range.bin
 vendor/riddle/spec/fixtures/data/filter_range_exclude.bin
 vendor/riddle/spec/fixtures/data/group.bin
 vendor/riddle/spec/fixtures/data/index.bin
+vendor/riddle/spec/fixtures/data/index_weights.bin
 vendor/riddle/spec/fixtures/data/phrase.bin
+vendor/riddle/spec/fixtures/data/rank_mode.bin
 vendor/riddle/spec/fixtures/data/simple.bin
 vendor/riddle/spec/fixtures/data/sort.bin
 vendor/riddle/spec/fixtures/data/update_simple.bin
 vendor/riddle/spec/fixtures/data/weights.bin
+vendor/riddle/spec/fixtures/data_generator.php
 vendor/riddle/spec/fixtures/sphinx/configuration.erb
+vendor/riddle/spec/fixtures/sphinxapi.php
 vendor/riddle/spec/fixtures/sql/conf.example.yml
 vendor/riddle/spec/fixtures/sql/data.sql
 vendor/riddle/spec/fixtures/sql/structure.sql

data/RAKE_TASKS

@@ -4,7 +4,9 @@
 These Rake tasks are made available to your Rails app:
 
 <tt>ultrasphinx:configure</tt>:: Rebuild the configuration file for this particular environment.
-<tt>ultrasphinx:index</tt>:: Reindex the database and send an update signal to the search daemon.
+<tt>ultrasphinx:index</tt>:: Reindex and rotate all indexes.
+<tt>ultrasphinx:index:delta</tt>:: Reindex and rotate the delta index.
+<tt>ultrasphinx:index:main</tt>:: Reindex and rotate the main index.
 <tt>ultrasphinx:daemon:restart</tt>:: Restart the search daemon.
 <tt>ultrasphinx:daemon:start</tt>:: Start the search daemon.
 <tt>ultrasphinx:daemon:stop</tt>:: Stop the search daemon.
@@ -12,4 +14,4 @@ These Rake tasks are made available to your Rails app:
 <tt>ultrasphinx:spelling:build</tt>:: Rebuild the custom spelling dictionary. You may need to use <tt>sudo</tt> if your Aspell folder is not writable by the app user.
 <tt>ultrasphinx:bootstrap</tt>:: Bootstrap a full Sphinx environment by running configure, index, then daemon:start.
 
-All tasks have shortcuts. Use <tt>us:conf</tt>, <tt>us:in</tt>, <tt>us:restart</tt>, <tt>us:start</tt>, <tt>us:stop</tt>, <tt>us:stat</tt>, <tt>us:spell</tt>, and <tt>us:boot</tt>.
+All tasks have shortcuts. Use <tt>us:conf</tt>, <tt>us:index</tt>, <tt>us:main</tt>, <tt>us:delta</tt>, <tt>us:restart</tt>, <tt>us:start</tt>, <tt>us:stop</tt>, <tt>us:stat</tt>, <tt>us:spell</tt>, and <tt>us:boot</tt>.

data/README

@@ -5,7 +5,7 @@ Ruby on Rails configurator and client to the Sphinx full text search engine.
 
 == License
 
-Copyright 2008 Cloudburst, LLC. Licensed under the AFL 3. See the included LICENSE file. Some portions copyright Pat Allan, distributed under the MIT license, and used with permission. Some portions copyright PJ Hyett and Mislav Marohnić, distributed under the MIT license, and used with permission. 
+Copyright 2007-2008 Cloudburst, LLC. Licensed under the AFL 3. See the included LICENSE file. Some portions copyright Pat Allan, distributed under the MIT license, and used with permission. Some portions copyright PJ Hyett and Mislav Marohnić, distributed under the MIT license, and used with permission. 
 
 The public certificate for the gem is here[http://rubyforge.org/frs/download.php/25331/evan_weaver-original-public_cert.pem]. 
 
@@ -14,30 +14,27 @@ If you use this software, please {make a donation}[http://blog.evanweaver.com/do
 == Requirements
 
 * MySQL 5.0, or PostgreSQL 8.2
-* Sphinx 0.9.8-dev r1112
+* Sphinx 0.9.8-rc1
 * Rails 2.0.2
 
 More recent versions than listed are usually ok.
 
 == Features
 
-Advanced Sphinx usage:
+Sphinx/Ultrasphinx is the fastest and most stable Rails fulltext search solution.
+
+Features include:
+
 * searching and ranking across orthogonal models
+* delta index support
 * excerpt highlighting
-* field weighting
+* Google-style query parser
+* spellcheck
 * faceting on text, date, and numeric fields
-
-ActiveRecord-style SQL generation:
+* field weighting, merging, and aliases
 * <tt>belongs_to</tt> and <tt>has_many</tt> includes
-* field merging
-* field aliasing
-
-Good Rails integration:
-* automatic memcached loads via <tt>cache_fu</tt>
-* <tt>will_paginate</tt> compatibility
-* query spellcheck
-* Google-style query parser
-* error recovery
+* drop-in compatibility with will_paginate[http://err.lighthouseapp.com/projects/466/home]
+* drop-in compatibility with Interlock[http://blog.evanweaver.com/files/doc/fauna/interlock/]
 * multiple deployment environments
 * comprehensive Rake tasks
 
@@ -57,7 +54,7 @@ Then, install the plugin:
  
 Next, copy the <tt>examples/default.base</tt> file to <tt>RAILS_ROOT/config/ultrasphinx/default.base</tt>. This file sets up the  Sphinx daemon options such as port, host, and index location.
   
-If you need per-environment configuration, you can use <tt>RAILS_ROOT/config/ultrasphinx/development.base</tt>, etc. 
+If you need per-environment configuration, you can use <tt>RAILS_ROOT/config/ultrasphinx/development.base</tt>, etc. Note that ERb is also allowed within the <tt>.base</tt> files, and can be an alternative way to DRY up multiple configurations.
 
 Now, in your models, use the <tt>is_indexed</tt> method to configure a model as searchable. For example:
   
@@ -91,11 +88,29 @@ For more query options, including excerpt mode, see Ultrasphinx::Search.
 
 == Pagination
 
-Once the <tt>@search</tt> object has been <tt>run</tt>, it is directly compatible with the <tt>will_paginate</tt> view helper.
+Once the <tt>@search</tt> object has been <tt>run</tt>, it is directly compatible with the <tt>will_paginate</tt> view helper. In your view, just do:
+  <%= will_paginate(@search) %>
   
 == Spell checking
 
 See Ultrasphinx::Spell.
+
+== Delta indexing 
+
+Delta indexing speeds up your updates by not reindexing the entire dataset every time. 
+
+First, in your <tt>.base</tt> file, set the indexer option <tt>delta</tt> to your maximum interval between full reindexes. A day or a week is good, depending. Add a little bit to account for the time it takes the actual index to run:
+
+  delta = <%= 1.day + 30.minutes %> 
+
+Now, configure your models for delta indexing in the <tt>is_indexed</tt> call:
+
+  is_indexed :fields => ['created_at', 'title', 'body'],
+    :delta => true
+
+Now you can run <tt>rake ultrasphinx:index:delta</tt> frequently, and only records that were changed within 1 day will be reindexed. You will need to run <tt>rake ultrasphinx:index:main</tt> once a day to move the delta contents into the main index.
+
+See ActiveRecord::Base .is_indexed and DEPLOYMENT_NOTES[link:files/DEPLOYMENT_NOTES.html] for more.
   
 == Available Rake tasks
 
@@ -105,7 +120,9 @@ See RAKE_TASKS[link:files/RAKE_TASKS.html].
 
 See DEPLOYMENT_NOTES[link:files/DEPLOYMENT_NOTES.html].
 
-== A note about PostgreSQL
+== Gotchas
+
+Note that since Ultrasphinx preloads indexed models, you need to make sure those models have their own dependencies in place early in the boot process. This may require adjusting the general plugin load order or moving monkey-patches from <tt>lib/</tt> to <tt>vendor/plugins/</tt>.
 
 PostgreSQL 8.2 and higher are well supported. However, make sure you have executed <tt>CREATE LANGUAGE plpgsql;</tt> at least once. This step does not need to be repeated, so depending on your DB permissions, you might be able to put it in a migration.
 

data/TODO

@@ -8,6 +8,5 @@ Planned:
 
 Not sure:
 
-* Delta indexing
 * Use Pat Allan's association configurator, possibly with an API change to avoid the message-passing DSL
 * Use Treetop for the query parser instead of regexes

data/examples/default.base

@@ -3,8 +3,8 @@
 # Sphinx/Ultrasphinx user-configurable options.
 #
 # Copy this file to RAILS_ROOT/config/ultrasphinx. You can use individual 
-# namespaces if you want (e.g. development.base, production.base,
-# test.base).
+# namespaces if you want (e.g. development.base, production.base, 
+# test.base). Note that ERb is also allowed.
 #
 # This file should not be handed directly to Sphinx. Use the rake task 
 #
@@ -15,56 +15,61 @@
 # to Sphinx.
 # 
 # It is safe to edit .base files by hand. It is not safe to edit the generated 
-# .conf files. Do not symlink the .conf file to the .base file! I don't know why 
-# people think they need to do that. It's wrong.
+# .conf files. Do not symlink the .conf file to the .base file; it's wrong.
 #
 
+<% path = "/opt/local/var/db/sphinx/" %>
+
+# Indexing options
 indexer
-{
-  # Indexer running options
+{  
   mem_limit = 256M 
+
+  # Ultrasphinx-specific key
+  delta = <%= 1.day + 30.minutes %> 
 }
 
+# Daemon options
 searchd
-{
-  # Daemon options
+{  
   # What interface the search daemon should listen on and where to store its logs
   address = 0.0.0.0
   port = 3312
   seamless_rotate = 1
-  log = /opt/local/var/db/sphinx/log/searchd.log
-  query_log = /opt/local/var/db/sphinx/log/query.log
+  log = <%= path %>log/searchd.log
+  query_log = <%= path %>log/query.log
   read_timeout = 5
   max_children = 300
-  pid_file = /opt/local/var/db/sphinx/log/searchd.pid
+  pid_file = <%= path %>log/searchd.pid
   max_matches = 100000
 }
 
+# Client options
 client
 {
-  # Client options
   # Name of the Aspell dictionary (two letters max)
   dictionary_name = ap
+  
   # How your application connects to the search daemon (not necessarily the same as above)
   server_host = localhost
   server_port = 3312
 }
 
+# Individual SQL source options
 source
-{
-  # Individual SQL source options
+{  
   sql_ranged_throttle = 0  
   sql_range_step = 5000   
   sql_query_post =
 }
 
+# Index building options
 index
-{
-  # Index building options
-  path = /opt/local/var/db/sphinx/
-  docinfo = extern # just leave this alone
+{  
+  path = <%= path %>
+  docinfo = extern # Just leave this alone
   morphology = stem_en
-  stopwords = # /path/to/stopwords.txt
+  stopwords = # <%= path %>/ap-stopwords.txt
   min_word_len = 1
 
   # HTML-specific options

data/lib/ultrasphinx.rb

@@ -17,7 +17,7 @@ require 'ultrasphinx/associations'
 require 'ultrasphinx/core_extensions'
 require 'ultrasphinx/is_indexed'
 
-if (ActiveRecord::Base.connection rescue nil) # XXX Forget why this needed to be wrapped
+if (ActiveRecord::Base.connection rescue nil) # XXX Not sure why this needed to be wrapped.
   require 'ultrasphinx/configure'
   require 'ultrasphinx/autoload'
   require 'ultrasphinx/fields'

data/lib/ultrasphinx/configure.rb

@@ -12,7 +12,7 @@ module Ultrasphinx
           Dir["**/*.rb"].each do |filename|
             open(filename) do |file| 
               begin
-                if file.grep(/is_indexed/).any?
+                if file.grep(/^\s+is_indexed/).any?
                   filename = filename[0..-4]
                   begin                
                     File.basename(filename).camelize.constantize
@@ -41,21 +41,33 @@ module Ultrasphinx
               
         say "rebuilding configurations for #{RAILS_ENV} environment" 
         say "available models are #{MODEL_CONFIGURATION.keys.to_sentence}"
-        File.open(CONF_PATH, "w") do |conf|
-        
+        File.open(CONF_PATH, "w") do |conf|              
           conf.puts global_header            
-          sources = []
-          
-          say "generating SQL"
-          cached_groups = Fields.instance.groups.join("\n")
-          MODEL_CONFIGURATION.each_with_index do |model_options, class_id|
-            model, options = model_options
-            klass, source = model.constantize, model.tableize.gsub('/', '__')   
-            sources << source
-            conf.puts build_source(Fields.instance, model, options, class_id, klass, source, cached_groups)
+          say "generating SQL"    
+
+          INDEXES.each do |index|
+            sources = []
+            cached_groups = Fields.instance.groups.join("\n")
+
+            MODEL_CONFIGURATION.each_with_index do |model_and_options, class_id|              
+              # This relies on hash sort order being deterministic per-machine
+              model, options = model_and_options
+              klass = model.constantize
+              source = "#{model.tableize.gsub('/', '__')}_#{index}"
+ 
+              if index != DELTA_INDEX or options['delta']
+                # If we are building the delta, we only want to include the models that requested it
+                conf.puts build_source(index, Fields.instance, model, options, class_id, klass, source, cached_groups)
+                sources << source                
+              end
+            end
+            
+            if sources.any?
+              # Don't generate a delta index if there are no delta tables
+              conf.puts build_index(index, sources)
+            end
+            
           end
-          
-          conf.puts build_index(sources)
         end              
       end
       
@@ -68,7 +80,8 @@ module Ultrasphinx
         ["\n# Auto-generated at #{Time.now}.",
          "# Hand modifications will be overwritten.",
          "# #{BASE_PATH}\n",
-         INDEXER_SETTINGS._to_conf_string('indexer'),
+         INDEXER_SETTINGS.except('delta')._to_conf_string('indexer'),
+         "",
          DAEMON_SETTINGS._to_conf_string("searchd")]
       end      
       
@@ -86,9 +99,31 @@ module Ultrasphinx
         end                 
         conf.sort.join("\n")
       end
+
+      
+      def build_delta_condition(index, klass, options)
+        if index == DELTA_INDEX and options['delta']
+          # Add delta condition if necessary
+          table, field = klass.table_name, options['delta']['field']
+          source_string = "#{table}.#{field}"
+          delta_column = klass.columns_hash[field] 
+
+          if delta_column 
+            raise ConfigurationError, "#{source_string} is not a :datetime" unless delta_column.type == :datetime
+            if (options['fields'] + options['concatenate'] + options['include']).detect { |entry| entry['sortable'] }
+              # Warning about the sortable problem
+              # XXX Kind of in an odd place, but I want to happen at index time
+              Ultrasphinx.say "warning; text sortable columns on #{klass.name} will return wrong results with partial delta indexing"
+            end            
+            string = "#{source_string} > #{SQL_FUNCTIONS[ADAPTER]['delta']._interpolate(INDEXER_SETTINGS['delta'])}";
+          else
+            Ultrasphinx.say "warning; #{klass.name} will reindex the entire table during delta indexing"
+          end
+        end
+      end
       
       
-      def setup_source_arrays(klass, fields, class_id, conditions)        
+      def setup_source_arrays(index, klass, fields, class_id, conditions)        
         condition_strings = Array(conditions).map do |condition| 
           "(#{condition})"
         end
@@ -101,12 +136,13 @@ module Ultrasphinx
       end
       
       
-      def range_select_string(klass)
+      def range_select_string(klass, delta_condition)
         ["sql_query_range = SELECT",
           SQL_FUNCTIONS[ADAPTER]['range_cast']._interpolate("MIN(#{klass.primary_key})"),
-          ", ",
+          ",",
           SQL_FUNCTIONS[ADAPTER]['range_cast']._interpolate("MAX(#{klass.primary_key})"),
-          "FROM #{klass.table_name}"
+          "FROM #{klass.table_name}",
+          ("WHERE #{delta_condition}" if delta_condition),
         ].join(" ")
       end
       
@@ -116,11 +152,16 @@ module Ultrasphinx
       end      
       
             
-      def build_source(fields, model, options, class_id, klass, source, groups)
+      def build_source(index, fields, model, options, class_id, klass, source, groups)
                 
         column_strings, join_strings, condition_strings, group_bys, use_distinct, remaining_columns = 
           setup_source_arrays(
-            klass, fields, class_id, options['conditions'])
+            index, klass, fields, class_id, options['conditions'])
+            
+        delta_condition = 
+          build_delta_condition(
+            index, klass, options)             
+        condition_strings << delta_condition if delta_condition
 
         column_strings, join_strings, group_bys, remaining_columns = 
           build_regular_fields(
@@ -140,7 +181,7 @@ module Ultrasphinx
          "source #{source}\n{",
           SOURCE_SETTINGS._to_conf_string,
           setup_source_database(klass),
-          range_select_string(klass),
+          range_select_string(klass, delta_condition),
           build_query(klass, column_strings, join_strings, condition_strings, use_distinct, group_bys),
           "\n" + groups,
           query_info_string(klass, class_id),
@@ -271,13 +312,13 @@ module Ultrasphinx
       end
       
     
-      def build_index(sources)
+      def build_index(index, sources)
         ["\n# Index configuration\n\n",
-          "index #{UNIFIED_INDEX_NAME}\n{",
+          "index #{index}\n{",
           sources.sort.map do |source| 
             "  source = #{source}"
           end.join("\n"),          
-          INDEX_SETTINGS.merge('path' => INDEX_SETTINGS['path'] + "/sphinx_index_#{UNIFIED_INDEX_NAME}")._to_conf_string,
+          INDEX_SETTINGS.merge('path' => INDEX_SETTINGS['path'] + "/sphinx_index_#{index}")._to_conf_string,
          "}\n\n"]
       end