ultrasphinx 1 → 1.5

data.tar.gz.sig

@@ -0,0 +1,3 @@
+&��yKl���=
+��$RR�”vuւ�>Ҟ��
dӜp�Fz��[I��%]װ����snu?߫Z���;-�M��#�s:
+��| �6��������OW����-�8`u"y���Ԁ]!��j�̯�myng�Ǟ�������q�!�N�$@����lO���Ƭ�j���NRA��s�1G,JA��X9�#9I���UIX(�np�ٰ�y

data/CHANGELOG

@@ -1,3 +1,14 @@
 
-1. documentation, refactoring
-0. spike
+v1.5. API change. Change layout of base files to allow overriding of more options, see examples/default.base. Allow sorting on text fields (use the 'sortable' key).
+
+v1.4. New is_indexed 'fields' => {'function_sql'} key for custom field mangling; support setting textual keys in the 'filters'. 
+
+v1.3. Facets; configurable finder method array; support symbolic keys very reluctantly for legacy purposes.
+
+v1.2. API change. Support searching with empty query. Most keys are now singular strings (plural only if they *require* an Array value). Verbs have been un-nouned ('concatenate', not 'concats'; 'include', not 'includes'). 'filter', not 'raw_filters'; 'class_name', not 'model'. Finally, use a 'query' keypair instead of the first parameter to Ultrasphinx::Search.new for the query string (this simplifies form helpers). 
+
+v1.1. Parser rewrite and tests.
+
+v1. Documentation; refactoring.
+
+v0. Spike.

data/Manifest

@@ -1,21 +1,29 @@
-./CHANGELOG
-./LICENSE
-./Manifest
-./README
-./Rakefile
-./examples/app.multi
-./examples/default.base
-./init.rb
-./lib/ultrasphinx/autoload.rb
-./lib/ultrasphinx/core_extensions.rb
-./lib/ultrasphinx/fields.rb
-./lib/ultrasphinx/is_indexed.rb
-./lib/ultrasphinx/search.rb
-./lib/ultrasphinx/spell.rb
-./lib/ultrasphinx/ultrasphinx.rb
-./lib/ultrasphinx.rb
-./tasks/ultrasphinx.rake
-./vendor/sphinx/README
-./vendor/sphinx/Rakefile
-./vendor/sphinx/init.rb
-./vendor/sphinx/lib/client.rb
+vendor/will_paginate/LICENSE
+vendor/sphinx/README
+vendor/sphinx/Rakefile
+vendor/sphinx/LICENSE
+vendor/sphinx/lib/client.rb
+vendor/sphinx/init.rb
+TODO
+test/unit/parser_test.rb
+test/test_helper.rb
+test/config/ultrasphinx/test.base
+tasks/ultrasphinx.rake
+README
+Manifest
+LICENSE
+lib/ultrasphinx.rb
+lib/ultrasphinx/ultrasphinx.rb
+lib/ultrasphinx/spell.rb
+lib/ultrasphinx/search.rb
+lib/ultrasphinx/search/parser.rb
+lib/ultrasphinx/search/internals.rb
+lib/ultrasphinx/is_indexed.rb
+lib/ultrasphinx/fields.rb
+lib/ultrasphinx/core_extensions.rb
+lib/ultrasphinx/configure.rb
+lib/ultrasphinx/autoload.rb
+init.rb
+examples/default.base
+examples/app.multi
+CHANGELOG

data/README

@@ -5,46 +5,59 @@ Ruby on Rails configurator and client to the Sphinx full text search engine.
 
 == License
 
-Copyright 2007 Cloudburst, LLC. Licensed under the AFL 3. See included LICENSE file. Portions copyright Dmytro Shteflyuk and Alexey Kovyrin, distributed under the Ruby License, and used with permission.
+Copyright 2007 Cloudburst, LLC. Licensed under the AFL 3. See the included LICENSE file. Some portions copyright Dmytro Shteflyuk and Alexey Kovyrin, distributed under the Ruby License, and used with permission. Some portions copyright PJ Hyett and Mislav Marohnić, distributed under the MIT license, and used with permission.
 
 == Requirements
 
 * MySQL (or Postgres, experimental)
 * Sphinx 0.97
+* Rails 1.2.3 or greater
 
 == Features
 
 Advanced Sphinx usage:
-* multi-source merging
+* searching and ranking across orthogonal models
 * excerpt highlighting
 * field weighting
+* faceting on text and numeric fields
 
 ActiveRecord-style SQL generation:
-* belongs_to and has_many includes
+* <tt>belongs_to</tt> and <tt>has_many</tt> includes
 * field merging
 * field aliasing
 
 Good Rails integration:
-* automatic memcached loads via cache_fu
+* automatic memcached loads via <tt>cache_fu</tt>
+* <tt>will_paginate</tt> compatibility
 * query spellcheck
 * Google-style query parser
+* temporary error recovery
 * multiple deployment environments
-* comprehensive rake tasks
+* comprehensive Rake tasks
 
 And some other things.
 
-== Basic setup
+= Usage
+
+== Installation
   
-Install the plugin, and also Sphinx itself (http://www.sphinxsearch.com).
+First, compile and install Sphinx itself (http://www.sphinxsearch.com). 
+
+You also need the <tt>chronic</tt> gem:
+  sudo gem install chronic
+
+Then, install the plugin:
+  script/plugin install --svn svn://rubyforge.org/var/svn/fauna/ultrasphinx/trunk
+ 
 
-Next, copy the <tt>example/default.base</tt> file to <tt>RAILS_ROOT/config/ultrasphinx/default.base</tt>.
+Next, copy the <tt>example/default.base</tt> file to <tt>RAILS_ROOT/config/ultrasphinx/default.base</tt>. This file sets up the  Sphinx daemon options such as port, host, and index location.
   
-If you need per-environment configuration, you can use <tt>RAILS_ROOT/config/ultrasphinx/development.base</tt>, etc. This file sets up the basic Sphinx and Ultrasphinx options such as ports, hosts, and index locations.
+If you need per-environment configuration, you can use <tt>RAILS_ROOT/config/ultrasphinx/development.base</tt>, etc. 
 
-Now, in your models, use the <tt>is_indexed</tt> macro to configure a model as searchable. For example:
+Now, in your models, use the <tt>is_indexed</tt> method to configure a model as searchable. For example:
   
   class Post
-    is_indexed :fields => ["created_at", "title", "body"]
+    is_indexed :fields => ['created_at', 'title', 'body']
   end
   
 For more index options, see ActiveRecord::Base .is_indexed.
@@ -65,13 +78,19 @@ Make sure to manually stop and restart the daemon if you change the field config
     
 Query the daemon as so:
 
-  @search = Ultrasphinx::Search.new(@query)
+  @search = Ultrasphinx::Search.new(:query => @query)
   @search.run
-  @search.results             
+  @search.results
   
 For more query options, including excerpt mode, see Ultrasphinx::Search.
   
-== Spellcheck
+= Extras  
+
+== Pagination
+
+Once the <tt>@search</tt> object has been <tt>run</tt>, it is directly compatible with the <tt>will_paginate</tt> view helper.
+  
+== Spell checking
 
 See Ultrasphinx::Spell.
   
@@ -83,12 +102,19 @@ These rake tasks are made available to your Rails app:
 <tt>ultrasphinx:index</tt>:: Reindex the database and send an update signal to the search daemon.
 <tt>ultrasphinx:daemon:restart</tt>:: Restart the search daemon.
 <tt>ultrasphinx:daemon:start</tt>:: Start the search daemon.
-<tt>ultrasphinx:daemon:status</tt>:: Check if the search daemon is running.
 <tt>ultrasphinx:daemon:stop</tt>:: Stop the search daemon.
-<tt>ultrasphinx:spelling:build</tt>:: Rebuild the custom spelling dictionary.
+<tt>ultrasphinx:daemon:status</tt>:: Check if the search daemon is running.
+<tt>ultrasphinx:spelling:build</tt>:: Rebuild the custom spelling dictionary. You may need to use <tt>sudo</tt> if your Aspell folder is not writable by the app user.
 <tt>ultrasphinx:bootstrap</tt>:: Bootstrap a full Sphinx environment by running configure, index, then daemon:start.
 
+All tasks have shortcuts. Use <tt>us:conf</tt>, <tt>us:in</tt>, <tt>us:restart</tt>, <tt>us:start</tt>, <tt>us:stop</tt>, <tt>us:stat</tt>, <tt>us:spell</tt>, and <tt>us:boot</tt>.
+
+== Reporting problems
+
+* http://rubyforge.org/forum/forum.php?forum_id=14244
+
+Patches and contributions are very welcome. Please note that contributors are required to assign copyright for their additions to Cloudburst, LLC. 
+
 == Further resources
 
-http://blog.evanweaver.com/pages/code#ultrasphinx
-http://rubyforge.org/forum/forum.php?forum_id=14244
+* http://blog.evanweaver.com/articles/2007/07/09/ultrasphinx-searching-the-world-in-231-seconds

data/TODO

@@ -0,0 +1,6 @@
+
+* Support multiple spelling dictionaries per machine
+* Finish unifying filters and textfields
+* Support exclude filters
+* Make sure filters can be set to nil
+

data/examples/default.base

@@ -1,34 +1,56 @@
 
-# to put in RAILS_ROOT/config/ultrasphinx
-# can be namespaced individually (development.base)
+#
+# Sphinx/Ultrasphinx user-configurable options.
+#
+# Copy this file to RAILS_ROOT/config/ultrasphinx.
+# You can use individual namespaces if you want (e.g. "development.base").
+#
 
 indexer
 {
-  mem_limit = 256M
+  # Indexer running options
+  mem_limit = 256M 
 }
 
 searchd
 {
+  # Daemon options
+  # What interface the search daemon should listen on and where to store its logs
   address = 0.0.0.0
   port = 3312
   log = /opt/local/var/db/sphinx/log/searchd.log
   query_log = /opt/local/var/db/sphinx/log/query.log
   read_timeout = 5
-  max_children = 30
+  max_children = 300
   pid_file = /opt/local/var/db/sphinx/log/searchd.pid
   max_matches = 100000
 }
 
-ultrasphinx
+client
 {
-  # indexer options
+  # Client options
+  # How your application connects to the search daemon (not necessarily the same as above)
+  server_host = localhost
+  server_port = 3312
+}
+
+source
+{
+  # Individual SQL source options
+  sql_range_step = 20000   
+  strip_html = 0
+  index_html_attrs =
+  sql_query_post =
+}
+
+index
+{
+  # Index building options
   path = /opt/local/var/db/sphinx/
   docinfo = extern # just leave this alone
   morphology = stem_en
   stopwords = # /path/to/stopwords.txt
   min_word_len = 1
   charset_type = utf-8 # or sbcs (Single Byte Character Set)
-  # client options
-  server_host = localhost
-  server_port = 3312
+  charset_table = 0..9, A..Z->a..z, -, _, ., &, a..z, U+410..U+42F->U+430..U+44F, U+430..U+44F,U+C5->U+E5, U+E5, U+C4->U+E4, U+E4, U+D6->U+F6, U+F6, U+16B, U+0c1->a, U+0c4->a, U+0c9->e, U+0cd->i, U+0d3->o, U+0d4->o, U+0da->u, U+0dd->y, U+0e1->a, U+0e4->a, U+0e9->e, U+0ed->i, U+0f3->o, U+0f4->o, U+0fa->u, U+0fd->y, U+104->U+105, U+105, U+106->U+107, U+10c->c, U+10d->c, U+10e->d, U+10f->d, U+116->U+117, U+117, U+118->U+119, U+11a->e, U+11b->e, U+12E->U+12F, U+12F, U+139->l, U+13a->l, U+13d->l, U+13e->l, U+141->U+142, U+142, U+143->U+144, U+144,U+147->n, U+148->n, U+154->r, U+155->r, U+158->r, U+159->r, U+15A->U+15B, U+15B, U+160->s, U+160->U+161, U+161->s, U+164->t, U+165->t, U+16A->U+16B, U+16B, U+16e->u, U+16f->u, U+172->U+173, U+173, U+179->U+17A, U+17A, U+17B->U+17C, U+17C, U+17d->z, U+17e->z,
 }

data/lib/ultrasphinx.rb

@@ -1,11 +1,17 @@
 
+require 'fileutils'
+require 'chronic'
+
 require "#{File.dirname(__FILE__)}/../vendor/sphinx/lib/client"
 
 require 'ultrasphinx/core_extensions'
 require 'ultrasphinx/ultrasphinx'
+require 'ultrasphinx/configure'
 require 'ultrasphinx/autoload'
 require 'ultrasphinx/fields'
 require 'ultrasphinx/is_indexed'
+require 'ultrasphinx/search/internals'
+require 'ultrasphinx/search/parser'
 require 'ultrasphinx/search'
 
 Ultrasphinx.say(

data/lib/ultrasphinx/autoload.rb

@@ -5,7 +5,7 @@ class Rails::Initializer
 
   def after_initialize_with_ultrasphinx_configuration
     after_initialize_without_ultrasphinx_configuration
-    Ultrasphinx.load_constants
+    Ultrasphinx::Configure.load_constants
     Ultrasphinx.verify_database_name
   end     
   

data/lib/ultrasphinx/configure.rb

@@ -0,0 +1,266 @@
+
+module Ultrasphinx
+  class Configure  
+    class << self    
+  
+      # Force all the indexed models to load and fill the MODEL_CONFIGURATION hash.
+      def load_constants
+  
+        Dir["#{RAILS_ROOT}/app/models/**/*.rb"].each do |filename|
+          next if filename =~ /\/(\.svn|CVS|\.bzr)\//
+          begin
+            open(filename) {|file| load filename if file.grep(/is_indexed/).any?}
+          rescue Object => e
+            say "warning; possibly critical autoload error on #{filename}"
+            say e.inspect
+          end
+        end 
+  
+        # Build the field-to-type mappings.
+        Fields.instance.configure(MODEL_CONFIGURATION)
+      end
+      
+                    
+      # Main SQL builder.
+      def run       
+
+        load_constants
+              
+        puts "Rebuilding Ultrasphinx configurations for #{ENV['RAILS_ENV']} environment" 
+        puts "Available models are #{MODEL_CONFIGURATION.keys.to_sentence}"
+        File.open(CONF_PATH, "w") do |conf|
+        
+          conf.puts global_header            
+          sources = []
+          
+          puts "Generating SQL"
+          cached_groups = Fields.instance.groups.join("\n")
+          MODEL_CONFIGURATION.each_with_index do |model_options, class_id|
+            model, options = model_options
+            klass, source = model.constantize, model.tableize    
+            sources << source
+            conf.puts build_source(Fields.instance, model, options, class_id, klass, source, cached_groups)
+          end
+          
+          conf.puts build_index(sources)
+        end              
+      end
+      
+      
+      ######
+      
+      private
+      
+      def global_header
+        ["\n# Auto-generated at #{Time.now}.",
+         "# Hand modifications will be overwritten.",
+         "# #{BASE_PATH}",
+         INDEXER_SETTINGS._to_conf_string('indexer'),
+         DAEMON_SETTINGS._to_conf_string("searchd")]
+      end      
+      
+      
+      def setup_source_database(klass)
+        # Tentatively supporting Postgres now
+        connection_settings = klass.connection.instance_variable_get("@config")
+
+        adapter_defaults = ADAPTER_DEFAULTS[connection_settings[:adapter]]
+        raise ConfigurationError, "Unsupported database adapter" unless adapter_defaults
+
+        conf = [adapter_defaults]                  
+        connection_settings.reverse_merge(CONNECTION_DEFAULTS).each do |key, value|
+          conf << "#{CONFIG_MAP[key]} = #{value}" if CONFIG_MAP[key]          
+        end                 
+        conf.sort.join("\n")
+      end
+      
+      
+      def setup_source_arrays(klass, fields, class_id, conditions)        
+        condition_strings = Array(conditions).map do |condition| 
+          "(#{condition})"
+        end
+        
+        column_strings = [
+          "(#{klass.table_name}.#{klass.primary_key} * #{MODEL_CONFIGURATION.size} + #{class_id}) AS id", 
+          "#{class_id} AS class_id", "'#{klass.name}' AS class", 
+          "'#{EMPTY_SEARCHABLE}' AS empty_searchable"]
+        remaining_columns = fields.types.keys - ["class", "class_id", "empty_searchable"]        
+        [column_strings, [], condition_strings, remaining_columns]
+      end
+      
+      
+      def range_select_string(klass)
+        "sql_query_range = SELECT MIN(#{klass.primary_key}), MAX(#{klass.primary_key}) FROM #{klass.table_name}"
+      end
+      
+      
+      def query_info_string(klass, class_id)
+        "sql_query_info = SELECT * FROM #{klass.table_name} WHERE #{klass.table_name}.#{klass.primary_key} = (($id - #{class_id}) / #{MODEL_CONFIGURATION.size})"
+      end      
+      
+            
+      def build_source(fields, model, options, class_id, klass, source, groups)
+                
+        column_strings, join_strings, condition_strings, remaining_columns = 
+          setup_source_arrays(klass, fields, class_id, options['conditions'])
+
+        column_strings, join_strings, remaining_columns = 
+          build_regular_fields(klass, fields, options['fields'], column_strings, join_strings, remaining_columns)
+        column_strings, join_strings, remaining_columns = 
+          build_includes(klass, fields, options['include'], column_strings, join_strings, remaining_columns)
+        column_strings, join_strings, remaining_columns = 
+          build_concatenations(klass, fields, options['concatenate'], column_strings, join_strings, remaining_columns)
+        
+        column_strings = add_missing_columns(fields, remaining_columns, column_strings)
+       
+        ["\n# Source configuration\n\n",
+         "source #{source}\n{",
+          SOURCE_SETTINGS._to_conf_string,
+          setup_source_database(klass),
+          range_select_string(klass),
+          build_query(klass, column_strings, join_strings, condition_strings),
+          "\n" + groups,
+          query_info_string(klass, class_id),
+          "}\n\n"]
+      end
+      
+      
+      def build_query(klass, column_strings, join_strings, condition_strings)
+
+        connection_settings = klass.connection.instance_variable_get("@config")
+
+        ["sql_query =", 
+          "SELECT", 
+          column_strings.sort_by do |string| 
+            # sphinx wants them always in the same order, but "id" must be first
+            (field = string[/.*AS (.*)/, 1]) == "id" ? "*" : field
+          end.join(", "),
+          "FROM #{klass.table_name}",
+          join_strings.uniq,
+          "WHERE #{klass.table_name}.#{klass.primary_key} >= $start AND #{klass.table_name}.#{klass.primary_key} <= $end",
+          condition_strings.uniq.map do |condition| 
+            "AND #{condition}"
+          end,
+          ("GROUP BY id" if connection_settings[:adapter] == 'mysql') # XXX should be somewhere more obvious
+        ].flatten.join(" ")
+      end
+      
+      
+      def add_missing_columns(fields, remaining_columns, column_strings)
+        remaining_columns.each do |field|
+          column_strings << fields.null(field)
+        end
+        column_strings
+      end
+      
+
+      def build_regular_fields(klass, fields, entries, column_strings, join_strings, remaining_columns)          
+        entries.to_a.each do |entry|
+          source_string = "#{klass.table_name}.#{entry['field']}"
+          column_strings, remaining_columns = install_field(fields, source_string, entry['as'], entry['function_sql'], entry['facet'], column_strings, remaining_columns)
+        end
+        
+        [column_strings, join_strings, remaining_columns]
+      end
+      
+
+      def build_includes(klass, fields, entries, column_strings, join_strings, remaining_columns)                  
+        entries.to_a.each do |entry|
+          
+          join_klass = entry['class_name'].constantize
+          association = klass.reflect_on_association(entry['class_name'].underscore.to_sym)
+                        
+          raise ConfigurationError, "Unknown association from #{klass} to #{entry['class_name']}" if not association and not entry['association_sql']
+          
+          join_strings = install_join_unless_association_sql(entry['association_sql'], nil, join_strings) do 
+            "LEFT OUTER JOIN #{join_klass.table_name} ON " + 
+            if (macro = association.macro) == :belongs_to 
+              "#{join_klass.table_name}.#{join_klass.primary_key} = #{klass.table_name}.#{association.primary_key_name}" 
+            elsif macro == :has_one
+              "#{klass.table_name}.#{klass.primary_key} = #{join_klass.table_name}.#{association.instance_variable_get('@foreign_key_name')}" 
+            else
+              raise ConfigurationError, "Unidentified association macro #{macro.inspect}"
+            end
+          end
+          
+          source_string = "#{join_klass.table_name}.#{entry['field']}"
+          column_strings, remaining_columns = install_field(fields, source_string, entry['as'], entry['function_sql'], entry['facet'], column_strings, remaining_columns)                         
+        end
+        
+        [column_strings, join_strings, remaining_columns]
+      end
+      
+        
+      def build_concatenations(klass, fields, entries, column_strings, join_strings, remaining_columns)
+        entries.to_a.each do |entry|
+          if entry['class_name'] and entry['field']
+            # group concats
+            # only has_many's or explicit sql right now
+            join_klass = entry['class_name'].constantize
+        
+            join_strings = install_join_unless_association_sql(entry['association_sql'], nil, join_strings) do 
+              # XXX make sure foreign key is right for polymorphic relationships
+              association = klass.reflect_on_association(entry['association_name'] ? entry['association_name'].to_sym : entry['class_name'].underscore.pluralize.to_sym)
+              "LEFT OUTER JOIN #{join_klass.table_name} ON #{klass.table_name}.#{klass.primary_key} = #{join_klass.table_name}.#{association.primary_key_name}" + 
+                (entry['conditions'] ? " AND (#{entry['conditions']})" : "")
+            end
+            
+            source_string = "GROUP_CONCAT(#{join_klass.table_name}.#{entry['field']} SEPARATOR ' ')"
+            column_strings, remaining_columns = install_field(fields, source_string, entry['as'], entry['function_sql'], entry['facet'], column_strings, remaining_columns)
+            
+          elsif entry['fields']
+            # regular concats
+            source_string = "CONCAT_WS(' ', " + entry['fields'].map do |subfield| 
+              "#{klass.table_name}.#{subfield}"
+            end.join(', ') + ")"
+            
+            column_strings, remaining_columns = install_field(fields, source_string, entry['as'], entry['function_sql'], entry['facet'], column_strings, remaining_columns)              
+
+          else
+            raise ConfigurationError, "Invalid concatenate parameters for #{model}: #{entry.inspect}."
+          end
+        end
+        
+        [column_strings, join_strings, remaining_columns]
+      end
+      
+    
+      def build_index(sources)
+        ["\n# Index configuration\n\n",
+          "index #{UNIFIED_INDEX_NAME}\n{",
+          sources.sort.map do |source| 
+            "  source = #{source}"
+          end.join("\n"),          
+          INDEX_SETTINGS.merge('path' => INDEX_SETTINGS['path'] + "/sphinx_index_#{UNIFIED_INDEX_NAME}")._to_conf_string,
+         "}\n\n"]
+      end
+      
+    
+      def install_field(fields, source_string, as, function_sql, with_facet, column_strings, remaining_columns)
+        source_string = function_sql.sub('?', source_string) if function_sql
+
+        column_strings << fields.cast(source_string, as)
+        remaining_columns.delete(as)
+        
+        # Generate CRC integer fields for text grouping
+        if with_facet
+          # Postgres probably doesn't handle this
+          column_strings << "CRC32(#{source_string}) AS #{as}_facet"
+          remaining_columns.delete("#{as}_facet")
+        end
+        [column_strings, remaining_columns]
+      end
+      
+      
+      def install_join_unless_association_sql(association_sql, join_string, join_strings)
+        join_strings << (association_sql or join_string or yield)
+      end
+      
+      
+      def say(s)
+        Ultrasphinx.say s
+      end
+      
+    end 
+  end
+end