airblade-Sphincter 1.1.0

data/lib/sphincter/search.rb

@@ -0,0 +1,199 @@
+require 'date'
+require 'time'
+require 'sphincter'
+
+##
+# Search extension for ActiveRecord::Base that automatically extends
+# ActiveRecord::Base.
+
+module Sphincter::Search
+
+  ##
+  # Struct to hold search results.
+  #
+  # records:: ActiveRecord objects returned by sphinx
+  # total:: Total records searched
+  # per_page:: Size of each chunk of records
+
+  Results = Struct.new :records, :total, :per_page
+
+  ##
+  # Indexes registered with add_index.
+
+  @@indexes ||= Hash.new { |h, model| h[model] = [] }
+
+  ##
+  # Accessor for indexes registered with add_index.
+
+  def self.indexes
+    @@indexes
+  end
+
+  ##
+  # Adds an index with +options+.
+  #
+  # add_index will automatically add Sphincter::AssociationSearcher to any
+  # has_many associations referenced by this model's belongs_to associations.
+  # If this model belongs_to another model, and that model has_many of this
+  # model, then you will be able to <tt>other.models.search</tt> and recieve
+  # only records in the association.
+  #
+  # Currently, only has_many associations without conditions will have
+  # AssociationSearcher appended.
+  #
+  # Options are:
+  #
+  # :name:: Name of index.  Defaults to ActiveRecord::Base::table_name.
+  # :fields:: Array of fields to index.  Foreign key columns for belongs_to
+  #           associations are automatically added.  Fields from associations
+  #           may be included by using "association.field".
+  # :conditions:: Array of SQL conditions that will be ANDed together to
+  #               predicate inclusion in the search index.
+  #
+  # Example:
+  #
+  #   class Post < ActiveRecord::Base
+  #     belongs_to :user
+  #     belongs_to :blog
+  #     has_many :comments
+  #   
+  #     add_index :fields => %w[title body user.name, comments.body],
+  #               :conditions => ['published = 1']
+  #   end
+  #
+  # When including fields from associations, MySQL's GROUP_CONCAT() function
+  # is used.  By default this will create a string up to 1024 characters long.
+  # A larger string can be used by changing the value of MySQL's
+  # group_concat_max_len variable.  To do this, add the following to your
+  # sphincter.RAILS_ENV.yml files:
+  #
+  #   mysql:
+  #     sql_query_pre:
+  #       - SET NAMES utf8
+  #       - SET SESSION group_concat_max_len = VALUE
+
+  def add_index(options = {})
+    options[:fields] ||= []
+
+    reflect_on_all_associations.each do |my_assoc|
+      next unless my_assoc.macro == :belongs_to
+
+      options[:fields] << my_assoc.primary_key_name.to_s
+
+      has_many_klass = my_assoc.class_name.constantize
+
+      has_many_klass.reflect_on_all_associations.each do |opp_assoc|
+        next if opp_assoc.class_name != name or
+                opp_assoc.macro != :has_many or
+                opp_assoc.options[:conditions]
+
+        extends = Array(opp_assoc.options[:extend])
+        extends << Sphincter::AssociationSearcher
+        opp_assoc.options[:extend] = extends
+      end
+    end
+
+    options[:fields].uniq!
+
+    # Use class name rather than object identity to test for hash membership.
+    unless Sphincter::Search.indexes.keys.any? {|k| k.class_name == self.class_name}
+      Sphincter::Search.indexes[self] << options
+    end
+  end
+
+  ##
+  # Converts +values+ into an Array of values SetFilter can digest.
+  #
+  # true/false becomes 1/0, Time/Date/DateTime becomes a time in epoch
+  # seconds.  Everything else is passed straight through.
+
+  def sphincter_convert_values(values)
+    values.map do |value|
+      case value
+      when Date, DateTime then Time.parse(value.to_s).to_i
+      when FalseClass then 0
+      when Time then value.to_i
+      when TrueClass then 1
+      else value
+      end
+    end
+  end
+
+  ##
+  # Searches for +query+ with +options+.
+  #
+  # Allowed options are:
+  #
+  # :between:: Hash of Sphinx range filter conditions.  Hash keys are sphinx
+  #            group_column or date_column names.  Values can be
+  #            Date/Time/DateTime or Integers.
+  # :conditions:: Hash of Sphinx value filter conditions.  Hash keys are
+  #               sphinx group_column or date_column names.  Values can be a
+  #               single value or an Array of values.
+  # :index:: Name of Sphinx index to search.  Defaults to
+  #          ActiveRecord::Base::table_name.
+  # :page:: Page offset of records to return, for easy use with paginators.
+  # :per_page:: Size of a page.  Default page size is controlled by the
+  #             configuration.
+  #
+  # Returns a Sphincter::Search::Results object.
+
+  def search(query, options = {})
+    sphinx = Sphinx::Client.new
+
+    @host ||= Sphincter::Configure.get_conf['sphincter']['host']
+    @port ||= Sphincter::Configure.get_conf['sphincter']['port']
+
+    sphinx.SetServer @host, @port
+
+    options[:conditions] ||= {}
+    options[:conditions].each do |column, values|
+      values = sphincter_convert_values Array(values)
+      sphinx.SetFilter column.to_s, values
+    end
+
+    options[:between] ||= {}
+    options[:between].each do |column, between|
+      min, max = sphincter_convert_values between
+
+      sphinx.SetFilterRange column.to_s, min, max
+    end
+
+    @default_per_page ||= Sphincter::Configure.get_conf['sphincter']['per_page']
+
+    per_page = options[:per_page] || @default_per_page
+    page_offset = options.key?(:page) ? options[:page] - 1 : 0
+    offset = page_offset * per_page
+
+    sphinx.SetLimits offset, per_page
+
+    index_name = options[:index] || table_name
+
+    sphinx_result = sphinx.Query query, index_name
+
+    matches = sphinx_result['matches'].sort_by do |id, match|
+      -match['index'] # #find reverses, lame!
+    end
+
+    ids = matches.map do |id, match|
+      (id - match['attrs']['sphincter_index_id']) /
+        Sphincter::Configure.index_count
+    end
+
+    results = Results.new
+
+    results.records = find ids
+    results.total = sphinx_result['total_found']
+    results.per_page = per_page
+
+    results
+  end
+
+end
+
+# :stopdoc:
+module ActiveRecord; end
+class ActiveRecord::Base; end
+ActiveRecord::Base.extend Sphincter::Search
+# :startdoc:
+

data/lib/sphincter/search_stub.rb

@@ -0,0 +1,60 @@
+require 'sphincter'
+
+##
+# Stub for Sphincter searching.  Extend ActiveRecord::Base with this module in
+# your tests so you won't have to run a Sphinx searchd to test your searching.
+#
+# In test/testHelper.rb:
+#
+#   require 'sphincter/search_stub'
+#   ActiveRecord::Base.extend Sphincter::SearchStub
+#
+# Before running a search, you'll need to populate the stub's accessors:
+#
+#   def test_search
+#     Model.search_args = []
+#     Model.search_results = [Model.find(1, 2, 3)]
+#
+#     records = Model.search 'query'
+#
+#     assert_equal 1, Model.search_args.length
+#     assert_equal [...], Model.search_args
+#     assert_equal 0, Model.search_results.length
+#   end
+#
+# Since both search_args and search_results are an Array you can call #search
+# multiple times and get back different results per call.  #search will raise
+# an exception if you don't supply enough results.
+
+module Sphincter::SearchStub
+
+  ##
+  # An Array that records arguments #search was called with.  search_args
+  # isn't set to anything by default, so do that in your test setup.
+
+  attr_accessor :search_args
+
+  ##
+  # A pre-populated Array of search results for queries.  search_results isn't
+  # set to anything by default, so do that in your test setup.
+
+  attr_accessor :search_results
+
+  ##
+  # Overrides Sphincter::Search#search to use the search_args and
+  # search_results values instead of connecting to Sphinx.
+
+  def search(query, options = {})
+    unless @search_args and @search_results then
+      raise 'need to set up Sphincter::SearchStub#search_results and/or Sphincter::SearchStub#search_args in test setup'
+    end
+
+    @search_args << [query, options]
+
+    raise 'no more search results in search stub' if @search_results.empty?
+
+    @search_results.shift
+  end
+
+end
+

data/lib/sphincter/tasks.rb

@@ -0,0 +1,114 @@
+require 'rake'
+require 'sphincter/configure'
+
+namespace :sphincter do
+
+  desc 'Creates sphinx.conf if it doesn\'t exist'
+  task :configure => :environment do
+    sphinx_conf = Sphincter::Configure.sphinx_conf
+    Rake::Task['sphincter:reconfigure'].invoke unless File.exist? sphinx_conf
+  end
+
+  desc 'Creates sphinx.conf'
+  task :reconfigure => :environment do
+    Sphincter::Configure.configure
+  end
+
+  desc 'Runs the sphinx indexer if the indexes don\'t exist'
+  task :index => :configure do
+    indexes_defined = Sphincter::Configure.index_count
+    sphinx_dir = Sphincter::Configure.get_conf['sphincter']['path']
+
+    indexes_found = Dir[File.join(sphinx_dir, '*.spd')].length
+
+    Rake::Task['sphincter:reindex'].invoke if indexes_found > indexes_defined
+  end
+
+  desc 'Runs the sphinx indexer'
+  task :reindex => :configure do
+    sphinx_conf = Sphincter::Configure.sphinx_conf
+    cmd = %W[indexer --all --config #{sphinx_conf}]
+    cmd << "--quiet" unless Rake.application.options.trace
+    cmd << "--rotate" if Sphincter::Configure.searchd_running?
+    system(*cmd)
+  end
+
+  desc 'Stops searchd, reconfigures and reindexes'
+  task :reset => :configure do
+    Rake::Task['sphincter:stop_searchd'].invoke
+    FileUtils.rm_rf Sphincter::Configure.sphinx_dir,
+                    :verbose => Rake.application.options.trace
+    Rake::Task['sphincter:reconfigure'].execute # force reindex
+    Rake::Task['sphincter:reindex'].invoke
+  end
+
+  desc 'Restarts the searchd sphinx daemon'
+  task :restart_searchd => %w[sphincter:stop_searchd sphincter:start_searchd]
+
+  desc 'Sets up the sphinx client'
+  task :setup_sphinx do
+    require 'fileutils'
+    require 'tmpdir'
+    require 'open-uri'
+
+    verbose = Rake.application.options.trace
+
+    begin
+      tmpdir = File.join Dir.tmpdir, "Sphincter_setup_#{$$}"
+
+      mkdir tmpdir, :verbose => true
+
+      chdir tmpdir
+
+      src = open "http://rubyforge.org/frs/download.php/19571/sphinx-0.3.0.zip"
+      File.open('sphinx-0.3.0.zip', 'wb') { |dst| dst.write src.read }
+
+      quiet = verbose ? '' : ' -q'
+      sh "unzip#{quiet} sphinx-0.3.0.zip" or
+        raise "couldn't unzip sphinx-0.3.0.zip"
+
+      File.open 'sphinx.patch', 'wb' do |patch|
+        patch.puts <<-EOF
+--- sphinx/lib/client.rb.orig	2007-04-05 06:38:14.000000000 -0700
++++ sphinx/lib/client.rb	2007-07-29 20:23:18.000000000 -0700
+@@ -398,6 +398,7 @@
+   \r
+         result['matches'][doc] ||= {}\r
+         result['matches'][doc]['weight'] = weight\r
++        result['matches'][doc]['index'] = count\r
+         attrs_names_in_order.each do |attr|\r
+           val = response[p, 4].unpack('N*').first; p += 4\r
+           result['matches'][doc]['attrs'] ||= {}\r
+        EOF
+      end
+
+      quiet = verbose ? ' --verbose' : ''
+      sh "patch#{quiet} -p0 sphinx/lib/client.rb sphinx.patch" or
+        raise "couldn't patch sphinx"
+
+      sphinx_plugin_dir = File.join RAILS_ROOT, 'vendor', 'plugins', 'sphinx'
+      rm_rf sphinx_plugin_dir, :verbose => true
+
+      mv 'sphinx', sphinx_plugin_dir, :verbose => true
+    ensure
+      rm_rf tmpdir, :verbose => true
+    end
+  end
+
+  desc 'Starts the searchd sphinx daemon'
+  task :start_searchd => :index do
+    unless Sphincter::Configure.searchd_running? then
+      cmd = "searchd --config #{Sphincter::Configure.sphinx_conf}"
+      cmd << " > /dev/null" unless Rake.application.options.trace
+      system cmd
+    end
+  end
+
+  desc 'Stops the searchd daemon'
+  task :stop_searchd => :configure do
+    pid = Sphincter::Configure.searchd_running?
+    system 'kill', pid if pid
+  end
+
+end
+

data/test/sphincter_test_case.rb

@@ -0,0 +1,220 @@
+require 'test/unit'
+require 'fileutils'
+require 'tmpdir'
+
+$TESTING = true
+
+class String
+  def constantize
+    case self
+    when /belongs_to/i then SphincterTestCase::BelongsTo
+    when /many/i       then SphincterTestCase::HasMany
+    when /poly/i       then SphincterTestCase::Poly
+    else raise "missing klass for #{self} in #constantize"
+    end
+  end
+end
+
+require 'sphincter'
+
+class ActiveRecord::Base
+  def self.configurations
+    {
+      'development' => {
+        'adapter' => 'mysql',
+        'host' => 'host',
+        'username' => 'username',
+        'password' => 'password',
+        'database' => 'database',
+        'socket' => 'socket',
+      }
+    }
+  end
+end
+
+module Sphinx; end
+
+class Sphinx::Client
+
+  @@last_client = nil
+
+  attr_reader :host, :port, :query, :index, :filters, :offset, :limit
+
+  def self.last_client
+    @@last_client
+  end
+
+  def initialize
+    @filters = {}
+
+    @@last_client = self
+  end
+
+  def Query(query, index)
+    @query, @index = query, index
+
+    {
+      'matches' => {
+        12 => { 'attrs' => { 'sphincter_index_id' => 1 }, 'index' => 3 },
+        13 => { 'attrs' => { 'sphincter_index_id' => 1 }, 'index' => 1 },
+        14 => { 'attrs' => { 'sphincter_index_id' => 1 }, 'index' => 2 },
+      },
+      'total_found' => 3
+    }
+  end
+
+  def SetFilter(column, values)
+    @filters[column] = values
+  end
+
+  def SetFilterRange(column, min, max)
+    @filters[column] = [:range, min, max]
+  end
+
+  def SetLimits(offset, limit)
+    @offset, @limit = offset, limit
+  end
+
+  def SetServer(host, port)
+    @host, @port = host, port
+  end
+
+end
+
+class SphincterTestCase < Test::Unit::TestCase
+
+  undef_method :default_test
+
+  class Column
+    attr_accessor :type
+
+    def initialize(type)
+      @type = type
+    end
+  end
+
+  class Connection
+    def quote(name) "'#{name}'" end
+    def quote_column_name(name) "`#{name}`" end
+  end
+
+  class Reflection
+    attr_accessor :klass
+    attr_reader :macro, :options, :name
+
+    def initialize(macro, name, options = {})
+      @klass = Model
+      @macro = macro
+      @name = name.intern
+      @options = options
+    end
+
+    def class_name() @name.to_s.sub(/s$/, '').capitalize end
+    def primary_key_name() "#{@name}_id" end
+  end
+
+  class Model
+
+    @reflections = [Reflection.new(:belongs_to, 'belongs_to'),
+                    Reflection.new(:has_many, 'manys'),
+                    Reflection.new(:has_many, 'polys', :as => :polyable)]
+
+    class << self; attr_accessor :reflections; end
+
+    def self.connection() Connection.new end
+
+    def self.columns_hash
+      {
+        'boolean' => Column.new(:boolean),
+        'date' => Column.new(:date),
+        'datetime' => Column.new(:datetime),
+        'float' => Column.new(:float),
+        'integer' => Column.new(:integer),
+        'string' => Column.new(:string),
+        'text' => Column.new(:text),
+        'time' => Column.new(:time),
+        'timestamp' => Column.new(:timestamp),
+      }
+    end
+
+    def self.find(ids) ids end
+
+    def self.name() 'Model' end
+
+    def self.primary_key() 'id' end
+
+    def self.reflect_on_all_associations
+      @reflections
+    end
+
+    def self.table_name() 'models' end
+
+  end
+
+  class BelongsTo < Model
+    @reflections = [Reflection.new(:belongs_to, 'something'),
+                    Reflection.new(:has_many, 'models')]
+
+    def self.table_name() 'belongs_tos' end
+
+    def id() 42 end
+  end
+
+  class HasMany < Model
+    @reflections = [Reflection.new(:belongs_to, 'models')]
+
+    def self.table_name() 'has_manys' end
+
+    def id() 84 end
+  end
+
+  class Poly < Model
+    @reflections = [Reflection.new(:belongs_to, 'polyable',
+                                   :polymorphic => true)]
+
+    def self.table_name() 'polys' end
+
+    def id() 126 end
+  end
+
+  class Model
+    extend Sphincter::Search
+  end
+
+  def setup
+    @temp_dir = File.join Dir.tmpdir, "sphincter_test_case_#{$$}"
+    FileUtils.mkdir_p @temp_dir
+    @orig_dir = Dir.pwd
+    Dir.chdir @temp_dir
+
+    @old_RAILS_ROOT = (Object.const_get :RAILS_ROOT rescue nil)
+    Object.send :remove_const, :RAILS_ROOT rescue nil
+    Object.const_set :RAILS_ROOT, @temp_dir
+
+    @old_RAILS_ENV = (Object.const_get :RAILS_ENV rescue nil)
+    Object.send :remove_const, :RAILS_ENV rescue nil
+    Object.const_set :RAILS_ENV, 'development'
+
+    Sphincter::Search.indexes.replace Hash.new { |h,k| h[k] = [] }
+
+    Sphincter::Configure.instance_variable_set '@env_conf', nil if
+      Sphincter::Configure.instance_variables.include? '@env_conf'
+    Sphincter::Configure.instance_variable_set '@index_count', nil if
+      Sphincter::Configure.instance_variables.include? '@index_count'
+
+    BelongsTo.reflections.last.options.delete :extend
+  end
+
+  def teardown
+    FileUtils.rm_rf @temp_dir
+    Dir.chdir @orig_dir
+
+    Object.send :remove_const, :RAILS_ROOT
+    Object.const_set :RAILS_ROOT, @old_RAILS_ROOT
+
+    Object.send :remove_const, :RAILS_ENV
+    Object.const_set :RAILS_ENV, @old_RAILS_ENV
+  end
+
+end
+