ultrasphinx 1.5.1 → 1.5.2

data/CHANGELOG

@@ -1,4 +1,6 @@
 
+v1.5.2. Fix association reloading issue; support float attributes on Sphinx 0.9.8; fix date range filters; import and update sample app (Mark Lane); start a comprehensive integration suite.
+
 v1.5.1. Various bugfixes.
 
 v1.5. API change. Change layout of base files to allow overriding of more options, see examples/default.base. Allow sorting on text fields (use the 'sortable' key).

data/README

@@ -12,7 +12,7 @@ The public certificate for the gem is at http://rubyforge.org/frs/download.php/2
 == Requirements
 
 * MySQL (or Postgres, experimental)
-* Sphinx 0.97
+* Sphinx 0.97 or greater
 * Rails 1.2.3 or greater
 
 == Features
@@ -21,7 +21,7 @@ Advanced Sphinx usage:
 * searching and ranking across orthogonal models
 * excerpt highlighting
 * field weighting
-* faceting on text and numeric fields
+* faceting on text, date, and numeric fields
 
 ActiveRecord-style SQL generation:
 * <tt>belongs_to</tt> and <tt>has_many</tt> includes
@@ -49,9 +49,8 @@ You also need the <tt>chronic</tt> gem:
   sudo gem install chronic
 
 Then, install the plugin:
-  script/plugin install --svn svn://rubyforge.org/var/svn/fauna/ultrasphinx/trunk
+  script/plugin install -x svn://rubyforge.org/var/svn/fauna/ultrasphinx/trunk
  
-
 Next, copy the <tt>example/default.base</tt> file to <tt>RAILS_ROOT/config/ultrasphinx/default.base</tt>. This file sets up the  Sphinx daemon options such as port, host, and index location.
   
 If you need per-environment configuration, you can use <tt>RAILS_ROOT/config/ultrasphinx/development.base</tt>, etc. 

data/lib/ultrasphinx.rb

@@ -1,11 +1,12 @@
 
 require 'fileutils'
 require 'chronic'
+require 'singleton'
 
 require "#{File.dirname(__FILE__)}/../vendor/sphinx/lib/client"
 
-require 'ultrasphinx/core_extensions'
 require 'ultrasphinx/ultrasphinx'
+require 'ultrasphinx/core_extensions'
 require 'ultrasphinx/configure'
 require 'ultrasphinx/autoload'
 require 'ultrasphinx/fields'
@@ -24,3 +25,9 @@ Ultrasphinx.say(
   end
 )
 
+if defined? RAILS_ENV and RAILS_ENV == "development"
+  if ENV['USER'] == 'eweaver'
+    require 'ruby-debug'
+    Debugger.start
+  end
+end

data/lib/ultrasphinx/configure.rb

@@ -10,9 +10,11 @@ module Ultrasphinx
           next if filename =~ /\/(\.svn|CVS|\.bzr)\//
           open(filename) do |file| 
             begin
-              load filename if file.grep(/is_indexed/).any?
+              if file.grep(/is_indexed/).any?
+                Dependencies.load_missing_constant(Kernel, File.basename(filename)[0..-4].classify) 
+              end
             rescue Object => e
-              say "warning; possibly critical autoload error on #{filename}"
+              say "warning: possibly critical autoload error on #{filename}"
               say e.inspect
             end
           end 
@@ -28,14 +30,14 @@ module Ultrasphinx
 
         load_constants
               
-        puts "Rebuilding Ultrasphinx configurations for #{RAILS_ENV} environment" 
-        puts "Available models are #{MODEL_CONFIGURATION.keys.to_sentence}"
+        say "rebuilding configurations for #{RAILS_ENV} environment" 
+        say "available models are #{MODEL_CONFIGURATION.keys.to_sentence}"
         File.open(CONF_PATH, "w") do |conf|
         
           conf.puts global_header            
           sources = []
           
-          puts "Generating SQL"
+          say "generating SQL"
           cached_groups = Fields.instance.groups.join("\n")
           MODEL_CONFIGURATION.each_with_index do |model_options, class_id|
             model, options = model_options

data/lib/ultrasphinx/core_extensions.rb

@@ -1,5 +1,6 @@
 
 class Array
+  # Only flatten the first level of an array
   def _flatten_once
     self.inject([]) do |set, element| 
       set + Array(element)
@@ -15,29 +16,14 @@ class Object
   end
   
   def _deep_dup
-    # Cause Ruby's dup sucks.
+    # Cause Ruby's clone/dup sucks.
     Marshal.load(Marshal.dump(self))
   end
 end
 
-class String
-  def _to_numeric
-    zeroless = self.squeeze(" ").strip.sub(/^0+(\d)/, '\1')
-    zeroless.sub!(/(\...*?)0+$/, '\1')
-    if zeroless.to_i.to_s == zeroless
-      zeroless.to_i
-    elsif zeroless.to_f.to_s == zeroless
-      zeroless.to_f
-    elsif date = Chronic.parse(self)
-      date.to_i
-    else
-      self
-    end
-  end
-end
-
 class Hash
   def _coerce_basic_types
+    # XXX To remove
     Hash[*self.map do |key, value|
       [key.to_s,
         if value.respond_to?(:to_i) && value.to_i.to_s == value
@@ -50,12 +36,14 @@ class Hash
       end._flatten_once]
   end
   
+  # Delete by multiple keys
   def _delete(*args)
     args.map do |key|
       self.delete key
     end    
   end
   
+  # Convert a hash to a Sphinx-style conf string
   def _to_conf_string(section = nil)
     inner = self.map do |key, value|
       "  #{key} = #{value}"
@@ -64,3 +52,35 @@ class Hash
   end
 
 end
+
+### Filter type coercion methods
+
+class String
+  def _to_numeric
+    zeroless = self.squeeze(" ").strip.sub(/^0+(\d)/, '\1')
+    zeroless.sub!(/(\...*?)0+$/, '\1')
+    if zeroless.to_i.to_s == zeroless
+      zeroless.to_i
+    elsif zeroless.to_f.to_s == zeroless
+      zeroless.to_f
+    elsif date = Chronic.parse(self)
+      date.to_i
+    else
+      self
+    end
+  end
+end
+
+module Ultrasphinx::NumericSelf
+  def _to_numeric; self; end
+end
+
+module Ultrasphinx::DateSelf
+  def _to_numeric; self.to_i; end
+end
+
+class Fixnum; include Ultrasphinx::NumericSelf; end
+class Bignum; include Ultrasphinx::NumericSelf; end
+class Float; include Ultrasphinx::NumericSelf; end
+class Date; include Ultrasphinx::DateSelf; end
+class Time; include Ultrasphinx::DateSelf; end

data/lib/ultrasphinx/fields.rb

@@ -1,8 +1,10 @@
 
-require 'singleton'
-
 module Ultrasphinx
 
+=begin rdoc
+This is a special singleton configuration class that stores the index field configurations. Rather than using a magic hash and including relevant behavior in Ultrasphinx::Configure and Ultrasphinx::Search, we unify it here.
+=end
+
   class Fields
     include Singleton
     
@@ -11,8 +13,12 @@ module Ultrasphinx
       'text' => 'text', 
       'integer' => 'numeric', 
       'date' => 'date', 
-      'datetime' => 'date'
-    }    
+      'datetime' => 'date',
+      'timestamp' => 'date',
+      'float' => 'numeric'
+    }
+    
+    VERSIONS_REQUIRED = {'float' => '0.9.8'}
     
     attr_accessor :classes, :types
     
@@ -30,6 +36,7 @@ module Ultrasphinx
   
     def save_and_verify_type(field, new_type, string_sortable, klass)
       # Smoosh fields together based on their name in the Sphinx query schema
+      check_version(new_type.to_s)
       field, new_type = field.to_s, TYPE_MAP[new_type.to_s]
 
       if types[field]
@@ -63,7 +70,7 @@ module Ultrasphinx
       end + " AS #{field}"
     end    
       
-    def null(field)
+    def null(field)      
       case types[field]
         when 'text'
           "''"
@@ -76,6 +83,16 @@ module Ultrasphinx
       end + " AS #{field}"
     end
     
+    def check_version(field)
+      # XXX Awkward location for the compatibility check
+      if req = VERSIONS_REQUIRED[field]
+        unless SPHINX_VERSION.include? req
+          # Will we eventually need to check version ranges?
+          Ultrasphinx.say "warning: '#{field}' type requires Sphinx #{req}, but you have #{SPHINX_VERSION}"
+        end
+      end
+    end
+    
     def configure(configuration)
 
       configuration.each do |model, options|        
@@ -93,7 +110,7 @@ module Ultrasphinx
             entry['as'] = entry['field'] unless entry['as']
             
             unless klass.columns_hash[entry['field']]
-              Ultrasphinx.say "WARNING: field #{entry['field']} is not present in #{model}"
+              Ultrasphinx.say "warning: field #{entry['field']} is not present in #{model}"
             else
               save_and_verify_type(entry['as'], klass.columns_hash[entry['field']].type, entry['sortable'], klass)
             end
@@ -116,7 +133,7 @@ module Ultrasphinx
             save_and_verify_type(entry['as'], 'text', entry['sortable'], klass)
           end          
         rescue ActiveRecord::StatementInvalid
-          Ultrasphinx.say "WARNING: model #{model} does not exist in the database yet"
+          Ultrasphinx.say "warning: model #{model} does not exist in the database yet"
         end  
       end
       

data/lib/ultrasphinx/is_indexed.rb

@@ -30,6 +30,8 @@ To allow sorting by a text field, also pass a hash and set the <tt>:sortable</tt
 
 To apply an SQL function to a field before it is indexed, use the key <tt>:function_sql</tt>. Pass a string such as <tt>"REPLACE(?, '_', ' ')"</tt>. The table and column name for your field will be interpolated into the first <tt>?</tt> in the string.
 
+Note that <tt>float</tt> fields are supported, but require Sphinx 0.98.
+
 == Including a field from an association
 
 Use the <tt>:include</tt> key.

data/lib/ultrasphinx/search/internals.rb

@@ -37,7 +37,11 @@ module Ultrasphinx
       
         # Extract ranged raw filters 
         # Some of this mangling might not be necessary
-        opts['filters'].each do |field, value|
+        opts['filters'].each do |field, value|          
+          field = field.to_s
+          unless Fields.instance.types[field]
+            raise Sphinx::SphinxArgumentError, "field #{field.inspect} is invalid"
+          end
           begin
             case value
               when Fixnum, Float, BigDecimal, NilClass, Array
@@ -55,7 +59,7 @@ module Ultrasphinx
                 raise NoMethodError
             end
           rescue NoMethodError => e
-            raise Sphinx::SphinxArgumentError, "filter: #{field.inspect}:#{value.inspect} is invalid"
+            raise Sphinx::SphinxArgumentError, "filter value #{value.inspect} for field #{field.inspect} is invalid"
           end
         end
         

data/lib/ultrasphinx/ultrasphinx.rb

@@ -10,7 +10,7 @@ module Ultrasphinx
   class UsageError < Exception #:nodoc:
   end
 
-  # internal file paths
+  # Internal file paths
   
   SUBDIR = "config/ultrasphinx"
   
@@ -26,7 +26,7 @@ module Ultrasphinx
   
   raise ConfigurationError, "Please create a '#{SUBDIR}/#{RAILS_ENV}.base' or '#{SUBDIR}/default.base' file in order to use Ultrasphinx in your #{RAILS_ENV} environment." unless File.exist? BASE_PATH # XXX lame
 
-  # some miscellaneous constants
+  # Some miscellaneous constants
 
   MAX_INT = 2**32-1
 
@@ -36,6 +36,12 @@ module Ultrasphinx
   
   UNIFIED_INDEX_NAME = "complete"
 
+  SPHINX_VERSION = if `which indexer` =~ /\/indexer\n/m
+      `indexer`.split("\n").first[7..-1]
+    else
+      "unknown"
+    end 
+
   CONFIG_MAP = {
     # These must be symbols for key mapping against Rails itself
     :username => 'sql_user',
@@ -73,13 +79,19 @@ type = pgsql
   
   ADAPTER = ActiveRecord::Base.connection.instance_variable_get("@config")[:adapter]
      
+  mattr_accessor :with_rake
+     
   # Logger.
   def self.say msg
-    msg = "** ultrasphinx: #{msg}"
-    if defined? RAILS_DEFAULT_LOGGER
-      RAILS_DEFAULT_LOGGER.warn msg
+    if with_rake
+      puts msg[0..0].upcase + msg[1..-1]
     else
-      STDERR.puts msg
+      msg = "** ultrasphinx: #{msg}"
+      if defined? RAILS_DEFAULT_LOGGER
+        RAILS_DEFAULT_LOGGER.warn msg
+      else
+        STDERR.puts msg
+      end
     end
   end
   
@@ -100,8 +112,7 @@ type = pgsql
     
   end
 
-  # introspect on the existing generated conf files
-
+  # Introspect on the existing generated conf files
   INDEXER_SETTINGS = options_for('indexer', BASE_PATH)
   CLIENT_SETTINGS = options_for('client', BASE_PATH)
   DAEMON_SETTINGS = options_for('searchd', BASE_PATH)

data/tasks/ultrasphinx.rake

@@ -2,61 +2,68 @@
 ENV['RAILS_ENV'] ||= "development"
 
 namespace :ultrasphinx do  
+
+  task :_environment => [:environment] do
+    # We can't just chain :environment because we want to make 
+    # sure it's set only for known Sphinx tasks
+    Ultrasphinx.with_rake = true
+  end
   
   desc "Bootstrap a full Sphinx environment"
-  task :bootstrap => [:environment, :configure, :index, :"daemon:restart"] do
+  task :bootstrap => [:_environment, :configure, :index, :"daemon:restart"] do
   end
   
   desc "Rebuild the configuration file for this particular environment."
-  task :configure => :environment do
+  task :configure => [:_environment] do
     Ultrasphinx::Configure.run
   end
   
   desc "Reindex the database and send an update signal to the search daemon."
-  task :index => :environment do
+  task :index => [:_environment] do
+    mkdir_p Ultrasphinx::INDEX_SETTINGS['path']
     cmd = "indexer --config #{Ultrasphinx::CONF_PATH}"
     cmd << " #{ENV['OPTS']} " if ENV['OPTS']
     cmd << " --rotate" if ultrasphinx_daemon_running?
     cmd << " #{Ultrasphinx::UNIFIED_INDEX_NAME}"
-    puts cmd
+    say cmd
     system cmd
   end
   
   
   namespace :daemon do
     desc "Start the search daemon"
-    task :start => :environment do
+    task :start => [:_environment] do
       FileUtils.mkdir_p File.dirname(Ultrasphinx::DAEMON_SETTINGS["log"]) rescue nil
       raise Ultrasphinx::DaemonError, "Already running" if ultrasphinx_daemon_running?
       system "searchd --config #{Ultrasphinx::CONF_PATH}"
       sleep(2) # give daemon a chance to write the pid file
       if ultrasphinx_daemon_running?
-        puts "Started successfully"
+        say "started successfully"
       else
-        puts "Failed to start"
+        say "failed to start"
       end
     end
     
     desc "Stop the search daemon"
-    task :stop => [:environment] do
+    task :stop => [:_environment] do
       raise Ultrasphinx::DaemonError, "Doesn't seem to be running" unless ultrasphinx_daemon_running?
       system "kill #{pid = ultrasphinx_daemon_pid}"
-      puts "Stopped #{pid}."
+      say "stopped #{pid}."
     end
 
     desc "Restart the search daemon"
-    task :restart => [:environment] do
+    task :restart => [:_environment] do
       Rake::Task["ultrasphinx:daemon:stop"].invoke if ultrasphinx_daemon_running?
       sleep(3)
       Rake::Task["ultrasphinx:daemon:start"].invoke
     end
     
     desc "Check if the search daemon is running"
-    task :status => :environment do
+    task :status => [:_environment] do
       if ultrasphinx_daemon_running?
-        puts "Daemon is running."
+        say "daemon is running."
       else
-        puts "Daemon is stopped."
+        say "daemon is stopped."
       end
     end      
   end
@@ -64,12 +71,12 @@ namespace :ultrasphinx do
     
   namespace :spelling do
     desc "Rebuild the custom spelling dictionary. You may need to use 'sudo' if your Aspell folder is not writable by the app user."
-    task :build => :environment do    
+    task :build => [:_environment] do    
       ENV['OPTS'] = "--buildstops #{Ultrasphinx::STOPWORDS_PATH} #{Ultrasphinx::MAX_WORDS} --buildfreqs"
       Rake::Task["ultrasphinx:index"].invoke
       tmpfile = "/tmp/custom_words.txt"
       words = []
-      puts "Filtering"
+      say "filtering"
       File.open(Ultrasphinx::STOPWORDS_PATH).each do |line|
         if line =~ /^([^\s\d_]{4,}) (\d+)/
           # XXX should be configurable
@@ -78,9 +85,9 @@ namespace :ultrasphinx do
           # by aspell-en, in order to not add typos to the dictionary
         end
       end
-      puts "Writing #{words.size} words"
+      say "writing #{words.size} words"
       File.open(tmpfile, 'w').write(words.join("\n"))
-      puts "Loading into aspell"
+      say "loading into aspell"
       system("aspell --lang=en create master custom.rws < #{tmpfile}")
     end
   end
@@ -116,3 +123,8 @@ def ultrasphinx_daemon_running?
     false
   end  
 end
+
+def say msg
+  Ultrasphinx.say msg
+end
+  

data/test/test_all.rb

@@ -0,0 +1,7 @@
+
+Dir.chdir(File.dirname(__FILE__)) do
+  Dir["unit/*.rb", "integration/*.rb"].each do |file|
+    puts "\n*** #{file} ***"
+    system("ruby #{file}")
+  end
+end

data/test/test_helper.rb

@@ -25,8 +25,6 @@ ActiveRecord::Base.establish_connection(
   })
 ActiveRecord::Base.connection.instance_variable_set('@config', config)
 
-silently { require 'schema' }
-require 'models'
 require 'ultrasphinx'
 
 Debugger.start

data/test/unit/parser_test.rb

@@ -1,5 +1,5 @@
 
-require "#{File.dirname(__FILE__)}/../test_helper.rb"
+require "#{File.dirname(__FILE__)}/../test_helper"
 
 describe "parser" do
 

data/ultrasphinx.gemspec

@@ -1,11 +1,11 @@
 
-# Gem::Specification for Ultrasphinx-1.5.1
+# Gem::Specification for Ultrasphinx-1.5.2
 # Originally generated by Echoe
 
 Gem::Specification.new do |s|
   s.name = %q{ultrasphinx}
-  s.version = "1.5.1"
-  s.date = %q{2007-09-25}
+  s.version = "1.5.2"
+  s.date = %q{2007-09-27}
   s.summary = %q{Ruby on Rails configurator and client to the Sphinx fulltext search engine.}
   s.email = %q{}
   s.homepage = %q{http://blog.evanweaver.com/files/doc/fauna/ultrasphinx/}
@@ -13,8 +13,8 @@ Gem::Specification.new do |s|
   s.description = %q{Ruby on Rails configurator and client to the Sphinx fulltext search engine.}
   s.has_rdoc = true
   s.authors = [""]
-  s.files = ["vendor/will_paginate/LICENSE", "vendor/sphinx/README", "vendor/sphinx/Rakefile", "vendor/sphinx/LICENSE", "vendor/sphinx/lib/client.rb", "vendor/sphinx/init.rb", "TODO", "test/unit/parser_test.rb", "test/test_helper.rb", "test/config/ultrasphinx/test.base", "tasks/ultrasphinx.rake", "README", "Manifest", "LICENSE", "lib/ultrasphinx.rb", "lib/ultrasphinx/ultrasphinx.rb", "lib/ultrasphinx/spell.rb", "lib/ultrasphinx/search.rb", "lib/ultrasphinx/search/parser.rb", "lib/ultrasphinx/search/internals.rb", "lib/ultrasphinx/is_indexed.rb", "lib/ultrasphinx/fields.rb", "lib/ultrasphinx/core_extensions.rb", "lib/ultrasphinx/configure.rb", "lib/ultrasphinx/autoload.rb", "init.rb", "examples/default.base", "examples/app.multi", "CHANGELOG", "ultrasphinx.gemspec"]
-  s.test_files = ["test/test_helper.rb"]
+  s.files = ["vendor/will_paginate/LICENSE", "vendor/sphinx/README", "vendor/sphinx/Rakefile", "vendor/sphinx/LICENSE", "vendor/sphinx/lib/client.rb", "vendor/sphinx/init.rb", "TODO", "test/unit/parser_test.rb", "test/test_helper.rb", "test/config/ultrasphinx/test.base", "tasks/ultrasphinx.rake", "README", "Manifest", "LICENSE", "lib/ultrasphinx.rb", "lib/ultrasphinx/ultrasphinx.rb", "lib/ultrasphinx/spell.rb", "lib/ultrasphinx/search.rb", "lib/ultrasphinx/search/parser.rb", "lib/ultrasphinx/search/internals.rb", "lib/ultrasphinx/is_indexed.rb", "lib/ultrasphinx/fields.rb", "lib/ultrasphinx/core_extensions.rb", "lib/ultrasphinx/configure.rb", "lib/ultrasphinx/autoload.rb", "init.rb", "examples/default.base", "examples/app.multi", "CHANGELOG", "ultrasphinx.gemspec", "test/test_all.rb"]
+  s.test_files = ["test/test_all.rb"]
   s.add_dependency(%q<chronic>, ["> 0.0.0"])
 end
 

metadata

@@ -3,8 +3,8 @@ rubygems_version: 0.9.4
 specification_version: 1
 name: ultrasphinx
 version: !ruby/object:Gem::Version 
-  version: 1.5.1
-date: 2007-09-25 00:00:00 -04:00
+  version: 1.5.2
+date: 2007-09-27 00:00:00 -04:00
 summary: Ruby on Rails configurator and client to the Sphinx fulltext search engine.
 require_paths: 
 - lib
@@ -82,7 +82,7 @@ files:
 - CHANGELOG
 - ultrasphinx.gemspec
 test_files: 
-- test/test_helper.rb
+- test/test_all.rb
 rdoc_options: []
 
 extra_rdoc_files: []