ultrasphinx 1.5.2 → 1.5.3

data.tar.gz.sig

@@ -1,3 +1,2 @@
-�v� #w��u����gxO��
�+sI)^�~�7ذ}P��]��c$���Rb�nb
-O���G��*������IG���
-T��8�?λ�p�	����PG&�9�b@
+��MÒ;�`鱤"��D8f���H^&�U(�
+ޣ���Œ8�P�qkz=������+i�l�g ��k�gZ��i��,���ޱ�����\Vl�Q��h�Q�����v�^~�3�W;�ef���lݟ��ؐ���K|�P���h�+vU�Ι턎W�p��1��U,�G���j�vo[�I!��u�5(
.I3.f��Bo� ]��T.l`�Х���u-�^�����&��B�ǭ��;b[8�9�c��PE�Y��G��O0\k�

data/CHANGELOG

@@ -1,4 +1,6 @@
 
+v1.5.3. 90% test coverage; support multiple spelling dictionaries per machine (configurable in the .base file); association_name key is right out; some bugfixes.
+
 v1.5.2. Fix association reloading issue; support float attributes on Sphinx 0.9.8; fix date range filters; import and update sample app (Mark Lane); start a comprehensive integration suite.
 
 v1.5.1. Various bugfixes.

data/Manifest

@@ -1,29 +1,135 @@
-vendor/will_paginate/LICENSE
-vendor/sphinx/README
-vendor/sphinx/Rakefile
-vendor/sphinx/LICENSE
-vendor/sphinx/lib/client.rb
-vendor/sphinx/init.rb
-TODO
-test/unit/parser_test.rb
-test/test_helper.rb
-test/config/ultrasphinx/test.base
-tasks/ultrasphinx.rake
-README
-Manifest
-LICENSE
-lib/ultrasphinx.rb
-lib/ultrasphinx/ultrasphinx.rb
-lib/ultrasphinx/spell.rb
-lib/ultrasphinx/search.rb
-lib/ultrasphinx/search/parser.rb
-lib/ultrasphinx/search/internals.rb
-lib/ultrasphinx/is_indexed.rb
-lib/ultrasphinx/fields.rb
-lib/ultrasphinx/core_extensions.rb
-lib/ultrasphinx/configure.rb
-lib/ultrasphinx/autoload.rb
-init.rb
-examples/default.base
-examples/app.multi
 CHANGELOG
+examples/ap.multi
+examples/default.base
+init.rb
+lib/ultrasphinx/autoload.rb
+lib/ultrasphinx/configure.rb
+lib/ultrasphinx/core_extensions.rb
+lib/ultrasphinx/fields.rb
+lib/ultrasphinx/hex_to_int.sql
+lib/ultrasphinx/is_indexed.rb
+lib/ultrasphinx/search/internals.rb
+lib/ultrasphinx/search/parser.rb
+lib/ultrasphinx/search.rb
+lib/ultrasphinx/spell.rb
+lib/ultrasphinx/ultrasphinx.rb
+lib/ultrasphinx.rb
+LICENSE
+Manifest
+README
+tasks/ultrasphinx.rake
+test/config/ultrasphinx/test.base
+test/integration/app/app/controllers/addresses_controller.rb
+test/integration/app/app/controllers/application.rb
+test/integration/app/app/controllers/sellers_controller.rb
+test/integration/app/app/controllers/states_controller.rb
+test/integration/app/app/controllers/users_controller.rb
+test/integration/app/app/helpers/addresses_helper.rb
+test/integration/app/app/helpers/application_helper.rb
+test/integration/app/app/helpers/sellers_helper.rb
+test/integration/app/app/helpers/states_helper.rb
+test/integration/app/app/helpers/users_helper.rb
+test/integration/app/app/models/geo/address.rb
+test/integration/app/app/models/geo/state.rb
+test/integration/app/app/models/person/user.rb
+test/integration/app/app/models/seller.rb
+test/integration/app/app/views/addresses/edit.html.erb
+test/integration/app/app/views/addresses/index.html.erb
+test/integration/app/app/views/addresses/new.html.erb
+test/integration/app/app/views/addresses/show.html.erb
+test/integration/app/app/views/layouts/addresses.html.erb
+test/integration/app/app/views/layouts/sellers.html.erb
+test/integration/app/app/views/layouts/states.html.erb
+test/integration/app/app/views/layouts/users.html.erb
+test/integration/app/app/views/sellers/edit.html.erb
+test/integration/app/app/views/sellers/index.html.erb
+test/integration/app/app/views/sellers/new.html.erb
+test/integration/app/app/views/sellers/show.html.erb
+test/integration/app/app/views/states/edit.html.erb
+test/integration/app/app/views/states/index.html.erb
+test/integration/app/app/views/states/new.html.erb
+test/integration/app/app/views/states/show.html.erb
+test/integration/app/app/views/users/edit.html.erb
+test/integration/app/app/views/users/index.html.erb
+test/integration/app/app/views/users/new.html.erb
+test/integration/app/app/views/users/show.html.erb
+test/integration/app/config/boot.rb
+test/integration/app/config/database.yml
+test/integration/app/config/environment.rb
+test/integration/app/config/environments/development.rb
+test/integration/app/config/environments/production.rb
+test/integration/app/config/environments/test.rb
+test/integration/app/config/locomotive.yml
+test/integration/app/config/routes.rb
+test/integration/app/config/ultrasphinx/default.base
+test/integration/app/config/ultrasphinx/development.conf
+test/integration/app/config/ultrasphinx/development.conf.canonical
+test/integration/app/db/migrate/001_create_users.rb
+test/integration/app/db/migrate/002_create_sellers.rb
+test/integration/app/db/migrate/003_create_addresses.rb
+test/integration/app/db/migrate/004_create_states.rb
+test/integration/app/db/migrate/005_add_capitalization_to_seller.rb
+test/integration/app/db/migrate/006_add_deleted_to_user.rb
+test/integration/app/db/migrate/007_add_lat_and_long_to_address.rb
+test/integration/app/db/migrate/008_add_mission_statement_to_seller.rb
+test/integration/app/db/schema.rb
+test/integration/app/doc/README_FOR_APP
+test/integration/app/public/404.html
+test/integration/app/public/500.html
+test/integration/app/public/dispatch.cgi
+test/integration/app/public/dispatch.fcgi
+test/integration/app/public/dispatch.rb
+test/integration/app/public/favicon.ico
+test/integration/app/public/images/rails.png
+test/integration/app/public/index.html
+test/integration/app/public/javascripts/application.js
+test/integration/app/public/javascripts/controls.js
+test/integration/app/public/javascripts/dragdrop.js
+test/integration/app/public/javascripts/effects.js
+test/integration/app/public/javascripts/prototype.js
+test/integration/app/public/robots.txt
+test/integration/app/public/stylesheets/scaffold.css
+test/integration/app/Rakefile
+test/integration/app/README
+test/integration/app/script/about
+test/integration/app/script/breakpointer
+test/integration/app/script/console
+test/integration/app/script/destroy
+test/integration/app/script/generate
+test/integration/app/script/performance/benchmarker
+test/integration/app/script/performance/profiler
+test/integration/app/script/plugin
+test/integration/app/script/process/inspector
+test/integration/app/script/process/reaper
+test/integration/app/script/process/spawner
+test/integration/app/script/runner
+test/integration/app/script/server
+test/integration/app/test/fixtures/addresses.yml
+test/integration/app/test/fixtures/sellers.yml
+test/integration/app/test/fixtures/states.yml
+test/integration/app/test/fixtures/users.yml
+test/integration/app/test/functional/addresses_controller_test.rb
+test/integration/app/test/functional/sellers_controller_test.rb
+test/integration/app/test/functional/states_controller_test.rb
+test/integration/app/test/functional/users_controller_test.rb
+test/integration/app/test/test_helper.rb
+test/integration/app/test/unit/address_test.rb
+test/integration/app/test/unit/seller_test.rb
+test/integration/app/test/unit/state_test.rb
+test/integration/app/test/unit/user_test.rb
+test/integration/configure_test.rb
+test/integration/search_test.rb
+test/integration/server_test.rb
+test/integration/spell_test.rb
+test/setup.rb
+test/test_all.rb
+test/test_helper.rb
+test/ts.multi
+test/unit/parser_test.rb
+TODO
+vendor/sphinx/init.rb
+vendor/sphinx/lib/client.rb
+vendor/sphinx/LICENSE
+vendor/sphinx/Rakefile
+vendor/sphinx/README
+vendor/will_paginate/LICENSE

data/TODO

@@ -1,5 +1,4 @@
 
-* Support multiple spelling dictionaries per machine
 * Finish unifying filters and textfields
 * Support exclude filters
 * Make sure filters can be set to nil

data/examples/{app.multi → ap.multi}

@@ -1,2 +1,2 @@
 add en_US-w_accents.multi
-add custom.rws
+add ap.rws

data/examples/default.base

@@ -29,6 +29,8 @@ searchd
 client
 {
   # Client options
+  # Name of the Aspell dictionary (two letters max)
+  dictionary_name = ap
   # How your application connects to the search daemon (not necessarily the same as above)
   server_host = localhost
   server_port = 3312

data/lib/ultrasphinx.rb

@@ -3,31 +3,33 @@ require 'fileutils'
 require 'chronic'
 require 'singleton'
 
+if defined? RAILS_ENV and RAILS_ENV == "development"
+  if ENV['USER'] == 'eweaver'
+    require 'ruby-debug'
+    Debugger.start
+  end
+end
+
 require "#{File.dirname(__FILE__)}/../vendor/sphinx/lib/client"
 
 require 'ultrasphinx/ultrasphinx'
 require 'ultrasphinx/core_extensions'
-require 'ultrasphinx/configure'
-require 'ultrasphinx/autoload'
-require 'ultrasphinx/fields'
 require 'ultrasphinx/is_indexed'
-require 'ultrasphinx/search/internals'
-require 'ultrasphinx/search/parser'
-require 'ultrasphinx/search'
 
-Ultrasphinx.say(
+if (ActiveRecord::Base.connection rescue nil)
+  require 'ultrasphinx/configure'
+  require 'ultrasphinx/autoload'
+  require 'ultrasphinx/fields'
+
+  require 'ultrasphinx/search/internals'
+  require 'ultrasphinx/search/parser'
+  require 'ultrasphinx/search'
+
   begin
     require 'raspell'
-    require 'ultrasphinx/spell'
-    "spelling support enabled"
   rescue Object => e
-    "spelling support not available (module load raised \"#{e}\")"
   end
-)
+  
+  require 'ultrasphinx/spell'
+end
 
-if defined? RAILS_ENV and RAILS_ENV == "development"
-  if ENV['USER'] == 'eweaver'
-    require 'ruby-debug'
-    Debugger.start
-  end
-end

data/lib/ultrasphinx/configure.rb

@@ -3,21 +3,28 @@ module Ultrasphinx
   class Configure  
     class << self    
   
-      # Force all the indexed models to load and fill the MODEL_CONFIGURATION hash.
+      # Force all the indexed models to load and register in the MODEL_CONFIGURATION hash.
       def load_constants
   
-        Dir["#{RAILS_ROOT}/app/models/**/*.rb"].each do |filename|
-          next if filename =~ /\/(\.svn|CVS|\.bzr)\//
-          open(filename) do |file| 
-            begin
-              if file.grep(/is_indexed/).any?
-                Dependencies.load_missing_constant(Kernel, File.basename(filename)[0..-4].classify) 
+        Dir.chdir "#{RAILS_ROOT}/app/models/" do
+          Dir["**/*.rb"].each do |filename|
+            open(filename) do |file| 
+              begin
+                if file.grep(/is_indexed/).any?
+                  filename = filename[0..-4]
+                  begin                
+                    File.basename(filename).camelize.constantize
+                  rescue NameError => e
+                    filename.camelize.constantize
+                  end
+                end
+              rescue Object => e
+                say "warning: possibly critical autoload error on #{filename}"
+                say e.inspect
+                #say e.backtrace.join("\n") if RAILS_ENV == "development"
               end
-            rescue Object => e
-              say "warning: possibly critical autoload error on #{filename}"
-              say e.inspect
-            end
-          end 
+            end 
+          end
         end
   
         # Build the field-to-type mappings.
@@ -41,7 +48,7 @@ module Ultrasphinx
           cached_groups = Fields.instance.groups.join("\n")
           MODEL_CONFIGURATION.each_with_index do |model_options, class_id|
             model, options = model_options
-            klass, source = model.constantize, model.tableize    
+            klass, source = model.constantize, model.tableize.gsub('/', '__')   
             sources << source
             conf.puts build_source(Fields.instance, model, options, class_id, klass, source, cached_groups)
           end
@@ -58,7 +65,7 @@ module Ultrasphinx
       def global_header
         ["\n# Auto-generated at #{Time.now}.",
          "# Hand modifications will be overwritten.",
-         "# #{BASE_PATH}",
+         "# #{BASE_PATH}\n",
          INDEXER_SETTINGS._to_conf_string('indexer'),
          DAEMON_SETTINGS._to_conf_string("searchd")]
       end      
@@ -130,7 +137,6 @@ module Ultrasphinx
       
       
       def build_query(klass, column_strings, join_strings, condition_strings)
-
         ["sql_query =", 
           "SELECT", 
           column_strings.sort_by do |string| 
@@ -140,9 +146,7 @@ module Ultrasphinx
           "FROM #{klass.table_name}",
           join_strings.uniq,
           "WHERE #{klass.table_name}.#{klass.primary_key} >= $start AND #{klass.table_name}.#{klass.primary_key} <= $end",
-          condition_strings.uniq.map do |condition| 
-            "AND #{condition}"
-          end,
+          condition_strings.uniq.map {|condition| "AND #{condition}" },
           ADAPTER_SQL_FUNCTIONS[ADAPTER]['group_by']
         ].flatten.join(" ")
       end
@@ -168,9 +172,9 @@ module Ultrasphinx
 
       def build_includes(klass, fields, entries, column_strings, join_strings, remaining_columns)                  
         entries.to_a.each do |entry|
-          
+        
           join_klass = entry['class_name'].constantize
-          association = klass.reflect_on_association(entry['class_name'].underscore.to_sym)
+          association = association_by_class_name(klass, entry['class_name'])
                         
           raise ConfigurationError, "Unknown association from #{klass} to #{entry['class_name']}" if not association and not entry['association_sql']
           
@@ -179,9 +183,9 @@ module Ultrasphinx
             if (macro = association.macro) == :belongs_to 
               "#{join_klass.table_name}.#{join_klass.primary_key} = #{klass.table_name}.#{association.primary_key_name}" 
             elsif macro == :has_one
-              "#{klass.table_name}.#{klass.primary_key} = #{join_klass.table_name}.#{association.instance_variable_get('@foreign_key_name')}" 
+              "#{klass.table_name}.#{klass.primary_key} = #{join_klass.table_name}.#{association.primary_key_name}" 
             else
-              raise ConfigurationError, "Unidentified association macro #{macro.inspect}"
+              raise ConfigurationError, "Unidentified association macro #{macro.inspect}. Please use the :association_sql key to manually specify the JOIN syntax."
             end
           end
           
@@ -202,7 +206,7 @@ module Ultrasphinx
         
             join_strings = install_join_unless_association_sql(entry['association_sql'], nil, join_strings) do 
               # XXX make sure foreign key is right for polymorphic relationships
-              association = klass.reflect_on_association(entry['association_name'] ? entry['association_name'].to_sym : entry['class_name'].underscore.pluralize.to_sym)
+              association = association_by_class_name(klass, entry['class_name'])
               "LEFT OUTER JOIN #{join_klass.table_name} ON #{klass.table_name}.#{klass.primary_key} = #{join_klass.table_name}.#{association.primary_key_name}" + 
                 (entry['conditions'] ? " AND (#{entry['conditions']})" : "")
             end
@@ -239,15 +243,14 @@ module Ultrasphinx
       
     
       def install_field(fields, source_string, as, function_sql, with_facet, column_strings, remaining_columns)
-        source_string = function_sql.sub('?', source_string) if function_sql
+        source_string = function_sql._interpolate(source_string) if function_sql
 
         column_strings << fields.cast(source_string, as)
         remaining_columns.delete(as)
         
-        # Generate CRC integer fields for text grouping
+        # Generate hashed integer fields for text grouping
         if with_facet
-          # Postgres probably doesn't handle this
-          column_strings << "CRC32(#{source_string}) AS #{as}_facet"
+          column_strings << "#{ADAPTER_SQL_FUNCTIONS[ADAPTER]['hash']._interpolate(source_string)} AS #{as}_facet"
           remaining_columns.delete("#{as}_facet")
         end
         [column_strings, remaining_columns]
@@ -257,7 +260,12 @@ module Ultrasphinx
       def install_join_unless_association_sql(association_sql, join_string, join_strings)
         join_strings << (association_sql or join_string or yield)
       end
-      
+
+      def association_by_class_name(klass, class_name)
+        klass.reflect_on_all_associations.detect do |assoc|
+          assoc.class_name == class_name
+        end
+      end
       
       def say(s)
         Ultrasphinx.say s

data/lib/ultrasphinx/core_extensions.rb

@@ -6,6 +6,12 @@ class Array
       set + Array(element)
     end
   end
+  
+  def _sum
+    self.inject(0) do |acc, element|
+      acc + element
+    end
+  end
 end
 
 class Object
@@ -66,9 +72,13 @@ class String
     elsif date = Chronic.parse(self)
       date.to_i
     else
-      self
+      raise Ultrasphinx::UsageError, "#{self.inspect} could not be coerced into a numeric value"
     end
   end
+  
+  def _interpolate(value)
+    self.sub('?', value)
+  end
 end
 
 module Ultrasphinx::NumericSelf

data/lib/ultrasphinx/fields.rb

@@ -11,11 +11,12 @@ This is a special singleton configuration class that stores the index field conf
     TYPE_MAP = {
       'string' => 'text', 
       'text' => 'text', 
-      'integer' => 'numeric', 
+      'integer' => 'integer', 
       'date' => 'date', 
       'datetime' => 'date',
       'timestamp' => 'date',
-      'float' => 'numeric'
+      'float' => 'float',
+      'boolean' => 'integer'
     }
     
     VERSIONS_REQUIRED = {'float' => '0.9.8'}
@@ -50,7 +51,7 @@ This is a special singleton configuration class that stores the index field conf
         classes[field] = [klass]
 
         @groups << case new_type
-          when 'numeric'
+          when 'float', 'integer'
             "sql_group_column = #{field}"
           when 'date'
             "sql_date_column = #{field}"
@@ -62,7 +63,9 @@ This is a special singleton configuration class that stores the index field conf
     
     def cast(source_string, field)
       if types[field] == "date"
-        "#{ADAPTER_SQL_FUNCTIONS[ADAPTER]['timestamp']}#{source_string})"
+        "#{ADAPTER_SQL_FUNCTIONS[ADAPTER]['timestamp']._interpolate(source_string)}"
+      elsif types[field] == "integer"
+        source_string # "CAST(#{source_string} AS UNSIGNED)"
       elsif source_string =~ /GROUP_CONCAT/
         "CAST(#{source_string} AS CHAR)"
       else
@@ -74,12 +77,14 @@ This is a special singleton configuration class that stores the index field conf
       case types[field]
         when 'text'
           "''"
-        when 'numeric'
+        when 'integer', 'float'
           "0"
         when 'date'
           "UNIX_TIMESTAMP('1970-01-01 00:00:00')"
+        when nil
+          raise "Field #{field} is missing"
         else
-          raise "Field #{field} does not have a valid type."
+          raise "Field #{field} does not have a valid type #{types[field]}."
       end + " AS #{field}"
     end
     
@@ -103,9 +108,9 @@ This is a special singleton configuration class that stores the index field conf
                 
         begin
         
-          # Fields are from the model. We destructively canonicize them back onto the configuration hash.
+          # Fields are from the model
+          # We destructively canonicize them back onto the configuration hash
           options['fields'] = options['fields'].to_a.map do |entry|
-            
             entry = {'field' => entry} unless entry.is_a? Hash
             entry['as'] = entry['field'] unless entry['as']
             
@@ -125,10 +130,12 @@ This is a special singleton configuration class that stores the index field conf
           
           # Joins are whatever they are in the target       
           options['include'].to_a.each do |entry|
+            entry['as'] = entry['field'] unless entry['as']
+            
             save_and_verify_type(entry['as'] || entry['field'], entry['class_name'].constantize.columns_hash[entry['field']].type, entry['sortable'], klass)
           end  
           
-          # Regular concats are CHAR (I think), group_concats are BLOB and need to be cast to CHAR, e.g. :text
+          # Regular concats are CHAR, group_concats are BLOB and need to be cast to CHAR
           options['concatenate'].to_a.each do |entry|
             save_and_verify_type(entry['as'], 'text', entry['sortable'], klass)
           end