ultrasphinx 1

data/lib/ultrasphinx/spell.rb

@@ -0,0 +1,41 @@
+
+
+module Ultrasphinx
+
+=begin rdoc
+== Spelling support
+
+In order to spellcheck your user's query, Ultrasphinx bundles a small spelling module. First, make sure Aspell 0.6, an appropriate Aspell dictionary, and the Rubygem 'raspell' are all installed.
+  
+Then, copy <tt>examples/app.multi</tt> into your Aspell dictionary folder. It allows you to use Sphinx to generate a custom wordlist for your app. Modify it if you don't want to also use the default American English dictionary.
+  
+Then, to build the custom wordlist, run:  
+  rake ultrasphinx:spelling:build    
+
+Now you can see if a query is correctly spelled as so:
+  @correction = Ultrasphinx::Spell.correct(@search.query)
+
+If @correction is not nil, go ahead and suggest it to the user. Otherwise, the query was already correct.
+
+=end
+
+  module Spell  
+    SP = Aspell.new("app")   
+    SP.suggestion_mode = Aspell::NORMAL
+    SP.set_option("ignore-case", "true")
+    
+    def self.correct string
+      correction = string.gsub(/[\w\']+/) do |word| 
+        unless SP.check(word)
+          SP.suggest(word).first
+        else
+          word
+        end
+      end
+      
+      correction if correction != string
+    end    
+    
+  end    
+end
+

data/lib/ultrasphinx/ultrasphinx.rb

@@ -0,0 +1,276 @@
+
+module Ultrasphinx
+
+  class Exception < ::Exception #:nodoc:
+  end
+  class ConfigurationError < Exception #:nodoc:
+  end  
+  class DaemonError < Exception #:nodoc:
+  end
+
+  # internal file paths
+  
+  SUBDIR = "config/ultrasphinx"
+  
+  DIR = "#{RAILS_ROOT}/#{SUBDIR}"
+
+  CONF_PATH = "#{DIR}/#{RAILS_ENV}.conf"
+  
+  ENV_BASE_PATH = "#{DIR}/#{RAILS_ENV}.base" 
+  
+  GENERIC_BASE_PATH = "#{DIR}/default.base"
+  
+  BASE_PATH = (File.exist?(ENV_BASE_PATH) ? ENV_BASE_PATH : GENERIC_BASE_PATH)
+  
+  raise ConfigurationError, "Please create a '#{SUBDIR}/#{RAILS_ENV}.base' or '#{SUBDIR}/default.base' file in order to use Ultrasphinx in your #{RAILS_ENV} environment." unless File.exist? BASE_PATH # XXX lame
+
+  # some miscellaneous constants
+
+  MAX_INT = 2**32-1
+
+  MAX_WORDS = 2**16 # maximum number of stopwords built  
+  
+  UNIFIED_INDEX_NAME = "complete"
+
+  COLUMN_TYPES = {:string => 'text', :text => 'text', :integer => 'numeric', :date => 'date', :datetime => 'date' }
+
+  CONFIG_MAP = {:username => 'sql_user',
+    :password => 'sql_pass',
+    :host => 'sql_host',
+    :database => 'sql_db',
+    :port => 'sql_port',
+    :socket => 'sql_sock'}
+
+  OPTIONAL_SPHINX_KEYS = ['morphology', 'stopwords', 'min_word_len', 'charset_type', 'charset_table', 'docinfo']
+  
+  # some default settings for the sphinx conf files
+  
+  SOURCE_DEFAULTS = %(
+strip_html = 0
+index_html_attrs =
+sql_query_post =
+sql_range_step = 20000
+  )
+  
+  ADAPTER_DEFAULTS = {
+    "mysql" => %(
+type = mysql
+sql_query_pre = SET SESSION group_concat_max_len = 65535
+sql_query_pre = SET NAMES utf8
+  ), 
+    "postgresql" => %(
+type = pgsql
+  )}
+ 
+  
+  # Configuration file parser.
+  def self.options_for(heading, path)
+    
+    section = open(path).read[/^#{heading}.*?\{(.*?)\}/m, 1]    
+    unless section
+      Ultrasphinx.say "#{path} appears to be corrupted; please delete file"
+      raise ConfigurationError, "Missing heading #{heading.inspect}" 
+    end
+    
+    options = section.split("\n").map do |line|
+      line =~ /\s*(.*?)\s*=\s*([^\#]*)/
+      $1 ? [$1, $2.strip] : []
+    end
+    
+    Hash[*options.flatten] 
+  end
+
+  # introspect on the existing generated conf files
+
+  PLUGIN_SETTINGS = options_for('ultrasphinx', BASE_PATH)
+
+  DAEMON_SETTINGS = options_for('searchd', BASE_PATH)
+
+  STOPWORDS_PATH = "#{Ultrasphinx::PLUGIN_SETTINGS['path']}/stopwords.txt}"
+
+  MODEL_CONFIGURATION = {}
+
+
+
+  class << self    
+  
+    # Logger.
+    def say msg
+      $stderr.puts "** ultrasphinx: #{msg}"
+    end
+
+    # Force all the indexed models to load and fill the MODEL_CONFIGURATION hash.
+    def load_constants
+
+      Dir["#{RAILS_ROOT}/app/models/**/*.rb"].each do |filename|
+        next if filename =~ /\/(\.svn|CVS|\.bzr)\//
+        begin
+          open(filename) {|file| load filename if file.grep(/is_indexed/).any?}
+        rescue Object => e
+          say "warning; possibly critical autoload error on #{filename}"
+          say e.inspect
+        end
+      end 
+
+      # build the field-to-type mappings
+      Fields.instance.configure(MODEL_CONFIGURATION)
+    end
+    
+    # Complain if the database names go out of sync.
+    def verify_database_name
+      if File.exist? CONF_PATH
+        if options_for("source", CONF_PATH)['sql_db'] != ActiveRecord::Base.connection.instance_variable_get("@config")[:database]
+           say "warning; configured database name is out-of-date"
+           say "please run 'rake ultrasphinx:configure'"
+        end rescue nil
+      end
+    end
+         
+                  
+    # Main SQL builder.
+    def configure       
+      load_constants
+            
+      puts "Rebuilding Ultrasphinx configurations for #{ENV['RAILS_ENV']} environment" 
+      puts "Available models are #{MODEL_CONFIGURATION.keys.to_sentence}"
+      File.open(CONF_PATH, "w") do |conf|
+        conf.puts "\n# Auto-generated at #{Time.now}.\n# Hand modifications will be overwritten.\n"
+        
+        conf.puts "\n# #{BASE_PATH}"
+        conf.puts open(BASE_PATH).read.sub(/^ultrasphinx.*?\{.*?\}/m, '') + "\n"
+        
+        sphinx_source_list = []
+        
+        conf.puts "\n# Source configuration\n\n"
+
+        puts "Generating SQL"
+        MODEL_CONFIGURATION.each_with_index do |model_options, class_id|
+          model, options = model_options
+          klass, source = model.constantize, model.tableize
+
+#          puts "SQL for #{model}"
+          
+          sphinx_source_list << source
+  
+          conf.puts "source #{source}\n{"
+          conf.puts SOURCE_DEFAULTS
+                    
+          # apparently we're supporting postgres now
+          connection_settings = klass.connection.instance_variable_get("@config")
+
+          adapter_defaults = ADAPTER_DEFAULTS[connection_settings[:adapter]]
+          raise ConfigurationError, "Unsupported database adapter" unless adapter_defaults
+          conf.puts adapter_defaults
+                    
+          connection_settings.each do |key, value|
+            conf.puts "#{CONFIG_MAP[key]} = #{value}" if CONFIG_MAP[key]          
+          end          
+          
+          table, pkey = klass.table_name, klass.primary_key
+          condition_strings, join_strings = Array(options[:conditions]).map{|condition| "(#{condition})"}, []
+          column_strings = ["(#{table}.#{pkey} * #{MODEL_CONFIGURATION.size} + #{class_id}) AS id", 
+                                       "#{class_id} AS class_id", "'#{klass.name}' AS class"]   
+          remaining_columns = Fields.instance.keys - ["class", "class_id"]
+          
+          conf.puts "\nsql_query_range = SELECT MIN(#{pkey}), MAX(#{pkey}) FROM #{table}"
+          
+          options[:fields].to_a.each do |f|
+            column, as = f.is_a?(Hash) ? [f[:field], f[:as]] : [f, f]
+            column_strings << Fields.instance.cast("#{table}.#{column}", as)
+            remaining_columns.delete(as)
+          end
+          
+          options[:includes].to_a.each do |join|
+            join_klass = join[:model].constantize
+            association = klass.reflect_on_association(join[:model].underscore.to_sym)
+            if not association 
+              if not join[:association_sql]
+                raise ConfigurationError, "Unknown association from #{klass} to #{join[:model]}"
+              else
+                join_strings << join[:association_sql]
+              end
+            else
+              join_strings << "LEFT OUTER JOIN #{join_klass.table_name} ON " + 
+                if (macro = association.macro) == :belongs_to 
+                  "#{join_klass.table_name}.#{join_klass.primary_key} = #{table}.#{association.primary_key_name}" 
+                elsif macro == :has_one
+                  "#{table}.#{klass.primary_key} = #{join_klass.table_name}.#{association.instance_variable_get('@foreign_key_name')}" 
+                else
+                  raise ConfigurationError, "Unidentified association macro #{macro.inspect}"
+                end
+            end
+            column_strings << "#{join_klass.table_name}.#{join[:field]} AS #{join[:as] or join[:field]}"
+            remaining_columns.delete(join[:as] || join[:field])
+          end
+          
+          options[:concats].to_a.select{|concat| concat[:model] and concat[:field]}.each do |group|
+            # only has_many's or explicit sql right now
+            join_klass = group[:model].constantize
+            if group[:association_sql]
+              join_strings << group[:association_sql]
+            else
+              association = klass.reflect_on_association(group[:association_name] ? group[:association_name].to_sym :  group[:model].underscore.pluralize.to_sym)
+              join_strings << "LEFT OUTER JOIN #{join_klass.table_name} ON #{table}.#{klass.primary_key} = #{join_klass.table_name}.#{association.primary_key_name}" + (" AND (#{group[:conditions]})" if group[:conditions]).to_s # XXX make sure foreign key is right for polymorphic relationships
+            end
+            column_strings << Fields.instance.cast("GROUP_CONCAT(#{join_klass.table_name}.#{group[:field]} SEPARATOR ' ')", group[:as])
+            remaining_columns.delete(group[:as])
+          end
+          
+          options[:concats].to_a.select{|concat| concat[:fields]}.each do |concat|
+            column_strings << Fields.instance.cast("CONCAT_WS(' ', #{concat[:fields].map{|field| "#{table}.#{field}"}.join(', ')})", concat[:as])
+            remaining_columns.delete(concat[:as])
+          end
+            
+#          puts "#{model} has #{remaining_columns.inspect} remaining"
+          remaining_columns.each do |field|
+            column_strings << Fields.instance.null(field)
+          end
+          
+          query_strings = ["SELECT", column_strings.sort_by do |string| 
+            # sphinx wants them always in the same order, but "id" must be first
+            (field = string[/.*AS (.*)/, 1]) == "id" ? "*" : field
+          end.join(", ")] 
+          query_strings << "FROM #{table}"                      
+          query_strings += join_strings.uniq
+          query_strings << "WHERE #{table}.#{pkey} >= $start AND #{table}.#{pkey} <= $end"
+          query_strings += condition_strings.uniq.map{|s| "AND #{s}"}
+          query_strings << "GROUP BY id"
+          
+          conf.puts "sql_query = #{query_strings.join(" ")}"
+          
+          groups = []
+          # group and date sorting params... this really only would have to be run once
+          Fields.instance.each do |field, type|
+            case type
+              when 'numeric'
+                groups << "sql_group_column = #{field}"
+              when 'date'
+                groups << "sql_date_column = #{field}"
+            end
+          end
+          conf.puts "\n" + groups.sort_by{|s| s[/= (.*)/, 1]}.join("\n")
+          conf.puts "\nsql_query_info = SELECT * FROM #{table} WHERE #{table}.#{pkey} = (($id - #{class_id}) / #{MODEL_CONFIGURATION.size})"           
+          conf.puts "}\n\n"                
+        end
+        
+        conf.puts "\n# Index configuration\n\n"
+        
+
+        # only output the unified index; no one uses the individual ones anyway        
+
+        conf.puts "index #{UNIFIED_INDEX_NAME}"
+        conf.puts "{"
+        conf.puts sphinx_source_list.map {|s| "source = #{s}" }
+
+        OPTIONAL_SPHINX_KEYS.each do |key|
+          conf.puts "#{key} = #{PLUGIN_SETTINGS[key]}" if PLUGIN_SETTINGS[key]
+        end
+        
+        conf.puts "path = #{PLUGIN_SETTINGS["path"]}/sphinx_index_#{UNIFIED_INDEX_NAME}"
+        conf.puts "}\n\n"
+      end
+            
+    end
+        
+  end
+end

data/tasks/ultrasphinx.rake

@@ -0,0 +1,125 @@
+
+ENV['RAILS_ENV'] ||= "development"
+
+namespace :ultrasphinx do  
+  
+  desc "Bootstrap a full Sphinx environment"
+  task :bootstrap => [:environment, :configure, :index, :start] do
+  end
+  
+  desc "Rebuild the configuration file for this particular environment."
+  task :configure => :environment do
+    Ultrasphinx::configure
+  end
+  
+  desc "Reindex the database and send an update signal to the search daemon."
+  task :index => :environment do
+    cmd = "indexer --config #{Ultrasphinx::CONF_PATH}"
+    cmd << " #{ENV['OPTS']} " if ENV['OPTS']
+    cmd << " --rotate" if ultrasphinx_daemon_running?
+    cmd << " #{Ultrasphinx::UNIFIED_INDEX_NAME}"
+    puts cmd
+    system cmd
+  end
+  
+  namespace :daemon do
+    desc "Start the search daemon"
+    task :start => :environment do
+      raise Ultrasphinx::DaemonError, "Already running" if ultrasphinx_daemon_running?
+      # remove lockfiles
+      Dir[Ultrasphinx::PLUGIN_SETTINGS["path"] + "*spl"].each {|file| File.delete(file)}
+      system "searchd --config #{Ultrasphinx::CONF_PATH}"
+      sleep(2) # give daemon a chance to write the pid file
+      if ultrasphinx_daemon_running?
+        puts "Started successfully"
+      else
+        puts "Failed to start"
+      end
+    end
+    
+    desc "Stop the search daemon"
+    task :stop => [:environment] do
+      raise Ultrasphinx::DaemonError, "Doesn't seem to be running" unless ultrasphinx_daemon_running?
+      system "kill #{pid = ultrasphinx_daemon_pid}"
+      puts "Stopped #{pid}."
+    end
+
+    desc "Restart the search daemon"
+    task :restart => [:environment, :stop, :start] do
+    end
+    
+    desc "Tail queries in the log"
+    task :tail => :environment do
+      require 'file/tail'
+      puts "Tailing #{filename = Ultrasphinx::DAEMON_SETTINGS['query_log']}"
+      File.open(filename) do |log|
+        log.extend(File::Tail)
+        log.interval = 1
+        log.backward(10)
+        last = nil
+        log.tail do |line| 
+          current = line[/\[\*\](.*)$/, 1]
+          last = current and puts current unless current == last
+        end
+      end 
+    end
+    
+    desc "Check if the search daemon is running"
+    task :status => :environment do
+      if ultrasphinx_daemon_running?
+        puts "Daemon is running."
+      else
+        puts "Daemon is stopped."
+      end
+    end      
+  end
+    
+    
+    
+  namespace :spelling do
+    desc "Rebuild custom spelling dictionary"
+    task :build => :environment do    
+      ENV['OPTS'] = "--buildstops #{Ultrasphinx::STOPWORDS_PATH} #{Ultrasphinx::MAX_WORDS} --buildfreqs"
+      Rake::Task["ultrasphinx:index"].invoke
+      tmpfile = "/tmp/custom_words.txt"
+      words = []
+      puts "Filtering"
+      File.open(Ultrasphinx::STOPWORDS_PATH).each do |line|
+        if line =~ /^([^\s\d_]{4,}) (\d+)/
+          # XXX should be configurable
+          words << $1 if $2.to_i > 40 
+          # ideally we would also skip words within X edit distance of a correction
+          # by aspell-en, in order to not add typos to the dictionary
+        end
+      end
+      puts "Writing #{words.size} words"
+      File.open(tmpfile, 'w').write(words.join("\n"))
+      puts "Loading into aspell"
+      system("aspell --lang=en create master custom.rws < #{tmpfile}")
+    end
+  end
+  
+end
+
+# task shortcuts
+namespace :us do
+  task :start => ["ultrasphinx:daemon:start"]
+  task :restart => ["ultrasphinx:daemon:restart"]
+  task :stop => ["ultrasphinx:daemon:stop"]
+  task :in => ["ultrasphinx:index"]
+  task :spell => ["ultrasphinx:spelling:build"]
+  task :conf => ["ultrasphinx:configure"]  
+  task :boot => ["ultrasphinx:bootstrap"]  
+end
+
+# support methods
+
+def ultrasphinx_daemon_pid
+  open(open(Ultrasphinx::BASE_PATH).readlines.map do |line| 
+    line[/^\s*pid_file\s*=\s*([^\s\#]*)/, 1]
+  end.compact.first).readline.chomp rescue nil # XXX ridiculous
+end
+
+def ultrasphinx_daemon_running?
+  ultrasphinx_daemon_pid and `ps #{ultrasphinx_daemon_pid} | wc`.to_i > 1 
+end

data/vendor/sphinx/README

@@ -0,0 +1,40 @@
+=Sphinx Client API 0.3.0
+
+Patched for Ultrasphinx.
+
+This document gives an overview of what is Sphinx itself and how to use in 
+within Ruby on Rails. For more information or documentation, 
+please go to http://www.sphinxsearch.com
+
+==Sphinx
+
+Sphinx is a standalone full-text search engine, meant to provide fast, 
+size-efficient and relevant fulltext search functions to other applications. 
+Sphinx was specially designed to integrate well with SQL databases and 
+scripting languages. Currently built-in data sources support fetching data 
+either via direct connection to MySQL, or from an XML pipe.
+ 
+Simplest way to communicate with Sphinx is to use <tt>searchd</tt> -
+a daemon to search through fulltext indices from external software.
+
+==Documentation
+
+You can create the documentation by running:
+
+	rake rdoc
+
+==Latest version
+
+You can always get latest version from
+http://kpumuk.info/projects/ror-plugins/sphinx
+
+==Credits
+
+Dmytro Shteflyuk <kpumuk@kpumuk.info> http://kpumuk.info
+
+Special thanks to Alexey Kovyrin <alexey@kovyrin.net> http://blog.kovyrin.net
+
+==License
+
+This library is distributed under the terms of the Ruby license.
+You can freely distribute/modify this library.