Sphincter 1.0.0

data/History.txt

@@ -0,0 +1,5 @@
+== 1.0.0 / 2007-07-26
+
+* 1 major enhancement
+  * Birthday!
+

data/LICENSE.txt

@@ -0,0 +1,27 @@
+Copyright 2007 Eric Hodel.  All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+   notice, this list of conditions and the following disclaimer in the
+   documentation and/or other materials provided with the distribution.
+3. Neither the names of the authors nor the names of their contributors
+   may be used to endorse or promote products derived from this software
+   without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS
+OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
+OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
+OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+

data/Manifest.txt

@@ -0,0 +1,16 @@
+History.txt
+LICENSE.txt
+Manifest.txt
+README.txt
+Rakefile
+lib/sphincter.rb
+lib/sphincter/association_searcher.rb
+lib/sphincter/configure.rb
+lib/sphincter/search.rb
+lib/sphincter/search_stub.rb
+lib/sphincter/tasks.rb
+test/sphincter_test_case.rb
+test/test_sphincter_association_searcher.rb
+test/test_sphincter_configure.rb
+test/test_sphincter_search.rb
+test/test_sphincter_search_stub.rb

data/README.txt

@@ -0,0 +1,132 @@
+Sphincter
+
+Eric Hodel <drbrain@segment7.net>
+
+http://seattlerb.org/Sphincter
+
+Sphincter was named by David Yeu.
+
+== DESCRIPTION:
+
+Sphincter is an ActiveRecord extension for full-text searching with Sphinx.
+
+Sphincter uses Dmytro Shteflyuk's sphinx Ruby API and automatic
+configuration to make totally rad ActiveRecord searching.  Well, you
+still have to tell Sphincter what models you want to search.  It
+doesn't read your mind.
+
+For complete documentation:
+
+  ri Sphincter
+
+== FEATURES:
+
+* Automatically configures itself.
+* Handy set of rake tasks for easy, automatic management.
+* Automatically adds has_many metadata for searching across the
+  association.
+* Stub for testing without connecting to searchd, Sphincter::SearchStub.
+* Easy pagination support.
+* Filtering by index metadata and ranges, including dates.
+
+== PROBLEMS:
+
+* Setting match mode not supported.
+* Setting sort mode not supported.
+* Setting per-field weights not supported.
+* Setting id range not supported.
+* Setting group-by not supported.
+
+== QUICK-START:
+
+Download and install Sphinx from http://www.sphinxsearch.com/downloads.html
+
+Download Sphinx Ruby API from http://rubyforge.org/frs/?group_id=2604&release_id=11049
+
+Unpack Sphinx Ruby API into vendor/plugins/.
+
+Install Sphincter:
+
+  $ gem install Sphincter
+
+Load Sphincter in config/environment.rb:
+
+  require 'sphincter'
+
+By default, Sphincter will run searchd on the same port for all
+environments.  See Sphincter::Configure for how to configure different
+environments to use different ports.
+
+Add indexes to models:
+
+  class Post < ActiveRecord::Base
+    belongs_to :blog
+    add_index :fields => %w[title body published]
+  end
+
+Add searching UI:
+
+  class BlogController < ApplicationController
+    def search
+      @blog = Blog.find params[:id]
+
+      @results = @blog.posts.search params[:q]
+    end
+  end
+
+Start searchd:
+
+  $ rake sphincter:start_searchd
+
+Then test it out in your browser.
+
+== TESTING QUICK-START:
+
+See Sphinx::SearchStub.
+
+== EXAMPLES:
+
+See Sphincter::Search#search for full documentation.
+
+Example ActiveRecord model:
+
+  class Post < ActiveRecord::Base
+    belongs_to :blog
+  
+    # published is a boolean and title and body are string or text fields
+    add_index :fields => %w[title body published]
+  end
+  
+Simple search:
+
+  Post.search 'words'
+  
+Only search published posts:
+
+  Post.search 'words', :conditions => { :published => 1 }
+  
+Only search posts created in the last week:
+
+  now = Time.now
+  ago = now - 1.weeks
+  Post.search 'words', :between => { :created_on => [ago, now] }
+  
+Pagination (defaults to ten records/page):
+
+  Post.search 'words', :page => 2
+  
+Pagination with custom page size:
+
+  Post.search 'words', :page => 2, :per_page => 20
+  
+Pagination with custom page size (better):
+
+Add to config/sphincter.yml:
+
+  sphincter:
+    per_page: 20
+
+Then search:
+
+  Post.search 'words', :page => 2
+

data/Rakefile

@@ -0,0 +1,21 @@
+# -*- ruby -*-
+
+require 'rubygems'
+require 'hoe'
+$:.unshift 'lib'
+require 'sphincter'
+
+Hoe.new('Sphincter', Sphincter::VERSION) do |p|
+  p.rubyforge_name = 'seattlerb'
+  p.author = 'Eric Hodel'
+  p.email = 'drbrain@segment7.net'
+  p.summary = p.paragraphs_of('README.txt', 5).first
+  p.description = p.paragraphs_of('README.txt', 6).first
+  p.url = p.paragraphs_of('README.txt', 2).first
+  p.changes = p.paragraphs_of('History.txt', 0..1).join("\n\n")
+
+  p.extra_deps << ['rake', '>= 0.7.3']
+  p.extra_deps << ['rails', '>= 1.2.3']
+end
+
+# vim: syntax=Ruby

data/lib/sphincter.rb

@@ -0,0 +1,102 @@
+$TESTING = defined?($TESTING) && $TESTING
+
+##
+# Sphincter is a ActiveRecord extension for full-text searching using the
+# Sphinx library.
+#
+# For the quick-start guide and some examples, see README.txt.
+#
+# == Installing
+#
+# Download and install Sphinx from http://www.sphinxsearch.com/downloads.html
+#
+# Download Sphinx Ruby API from
+# http://rubyforge.org/frs/?group_id=2604&release_id=11049
+#
+# Unpack Sphinx Ruby API into vendor/plugins/.
+#
+# Install the gem:
+#
+#   gem install Sphincter
+#
+# Require Sphincter in config/environment.rb:
+#
+#   require 'sphincter'
+#
+# Require the Sphincter rake tasks in Rakefile:
+#
+#   require 'sphincter/tasks'
+#
+# == Setup
+#
+# At best, you don't do anything to setup Sphincter.  It has sensible built-in
+# defaults.
+#
+# If you're running Sphinx's searchd for multiple environments on the same
+# machine, you'll want to add a config file to change the port that searchd
+# and the RAILS_ENV will comminicate across.  Do that in a per-environment
+# configuration file.
+#
+# If you have multiple machines, you'll want to change which address searchd
+# will run on.  Do that in the global configuration file.
+#
+# See Sphincter::Configure for full information on how to setup these and
+# other options for Sphincter.
+#
+# When you're done, run:
+#
+#  $ rake sphincter:configure
+#
+# == Indexing
+#
+# Sphincter automatically extends ActiveRecord::Base with Sphincter::Search, so
+# you only have to call add_index in the models you want indexed:
+#
+#   class Model < ActiveRecord::Base
+#     belongs_to :other
+#   
+#     add_index :fields => %w[title body]
+#   end
+#   
+#   class Other < ActiveRecord::Base
+#     has_many :models
+#   end
+#
+# add_index automatically adds a #search method to has_many associations
+# referencing this model, so you could:
+#
+#   Other.find(id).models.search 'some query'
+#
+# See Sphincter::Search for details.
+#
+# When you're done, run:
+#
+#   rake sphincter:index
+#
+# == Tasks
+#
+# You can get a set of Sphincter tasks by requiring 'sphincter/tasks' in your
+# Rakefile.  These tasks are all in the 'sphincter' namespace:
+#
+# configure:: Creates sphinx.conf if it doesn't exist
+# reconfigure:: Creates sphinx.conf, replacing the existing one.
+# index:: Runs the sphinx indexer if the index doesn't exist.
+# reindex:: Runs the sphinx indexer.  Rotates the index if searchd is running.
+# reset:: Stops searchd, reconfigures and reindexes
+# restart_searchd:: Restarts the searchd sphinx daemon
+# start_searchd:: Starts the searchd sphinx daemon
+# stop_searchd:: Stops the searchd daemon
+
+module Sphincter
+
+  ##
+  # This is the version of Sphincter you are using.
+
+  VERSION = '1.0.0'
+
+end
+
+require 'sphincter/configure'
+require 'sphincter/association_searcher'
+require 'sphincter/search'
+

data/lib/sphincter/association_searcher.rb

@@ -0,0 +1,22 @@
+require 'sphincter'
+
+##
+# ActiveRecord::Associations::ClassMethods#has_many extension for searching
+# the items of an ActiveRecord::Associations::AssociationProxy.
+
+module Sphincter::AssociationSearcher
+
+  ##
+  # Searches for +query+ with +options+.  Adds a condition so only the
+  # proxy_owner's records are matched.
+
+  def search(query, options = {})
+    pkey = proxy_reflection.primary_key_name
+    options[:conditions] ||= {}
+    options[:conditions][pkey] = proxy_owner.id
+
+    proxy_reflection.klass.search query, options
+  end
+
+end
+

data/lib/sphincter/configure.rb

@@ -0,0 +1,380 @@
+require 'fileutils'
+require 'yaml'
+
+require 'sphincter'
+
+##
+# Configuration module for Sphincter.
+#
+# DEFAULT_CONF contains the default options.  They can be overridden in both a
+# global config/sphincter.yml and in a per-environment
+# config/environments/sphincter.RAILS_ENV.yml.
+#
+# The only option you should need to override is the port option of
+# sphincter, so a config file for separate test and development indexes would
+# look like:
+#
+# config/environments/sphincter.development.yml:
+#
+#   sphincter:
+#     port: 3313
+#
+# config/environments/sphincter.test.yml:
+#
+#   sphincter:
+#     port: 3314
+#
+# Configuration options:
+#
+# sphincter:: Options for serachd's and Sphinx's port and address, and
+#            paths for index files.
+# index:: Options for a sphinx index conf section
+# indexer:: Options for the sphinx indexer
+# mysql:: Options for the sphinx indexer's mysql database connection.  The
+#         important ones are filled from config/database.yml
+# searchd:: Options for a sphinx searchd conf section
+# source:: Options for a sphinx source conf section
+#
+# The sphincter entry contains:
+#
+# address:: Which host searchd will run on, and which host Sphincter will
+#           connect to.
+# port:: Which port searchd and Sphincter will connect to.
+# path:: Location of searchd indexes, relative to RAILS_ROOT.
+# per_page:: How many items to include in a search by default.
+#
+# All other entries are from Sphinx.
+#
+# See http://www.sphinxsearch.com/doc.html#reference for details on sphinx
+# conf file settings.
+
+module Sphincter::Configure
+
+  @env_conf = nil
+  @index_count = nil
+
+  rails_env = defined?(RAILS_ENV) ? RAILS_ENV : 'RAILS_ENV'
+
+  ##
+  # Default Sphincter configuration.
+
+  DEFAULT_CONF = {
+    'sphincter' => {
+      'address' => '127.0.0.1',
+      'path' => "sphinx/#{rails_env}",
+      'per_page' => 10,
+      'port' => 3312,
+    },
+
+    'index' => {
+      'charset_type' => 'utf-8',
+      'docinfo' => 'extern',
+      'min_word_len' => 1,
+      'morphology' => 'stem_en',
+      'stopwords' => '',
+    },
+
+    'indexer' => {
+      'mem_limit' => '32M',
+    },
+
+    'mysql' => {
+      'sql_query_pre' => [
+        'SET SESSION group_concat_max_len = 65535',
+        'SET NAMES utf8',
+      ],
+    },
+
+    'searchd' => {
+      'log' => "log/sphinx/searchd.#{rails_env}.log",
+      'max_children' => 30,
+      'max_matches' => 1000,
+      'query_log' => "log/sphinx/query.#{rails_env}.log",
+      'read_timeout' => 5,
+    },
+
+    'source' => {
+      'index_html_attrs' => '',
+      'sql_query_post' => '',
+      'sql_range_step' => 20000,
+      'strip_html' => 0,
+    },
+  }
+
+  ##
+  # Builds and writes out a sphinx.conf file.
+
+  def self.configure
+    conf = get_conf
+    db_conf = get_db_conf
+
+    db_conf = conf[db_conf['type']].merge db_conf
+
+    sources = get_sources
+
+    sources.each do |name, source_conf|
+      sources[name] = db_conf.merge source_conf
+    end
+
+    write_configuration conf, sources
+  end
+
+  ##
+  # Merges Hashes of Hashes +mergee+ and +hash+.
+
+  def self.deep_merge(mergee, hash)
+    mergee = mergee.dup
+    hash.keys.each do |key| mergee[key] ||= hash[key] end
+    mergee.each do |key, value|
+      next unless hash[key]
+      mergee[key] = value.merge hash[key]
+    end
+  end
+
+  ##
+  # Builds the Sphincter configuration.
+  #
+  # Automatically fills in searchd address, port and pid_file from 'sphincter'
+  # section.
+
+  def self.get_conf
+    return @env_conf unless @env_conf.nil?
+
+    base_file = File.expand_path File.join(RAILS_ROOT, 'config', 'sphincter.yml')
+    base_conf = deep_merge DEFAULT_CONF, get_conf_from(base_file)
+
+    env_file = File.expand_path File.join(RAILS_ROOT, 'config', 'environments',
+                                          "sphincter.#{RAILS_ENV}.yml")
+    env_conf = deep_merge base_conf, get_conf_from(env_file)
+
+    env_conf['searchd']['address'] = env_conf['sphincter']['address']
+    env_conf['searchd']['port'] = env_conf['sphincter']['port']
+    env_conf['searchd']['pid_file'] = File.join(env_conf['sphincter']['path'],
+                                                'searchd.pid')
+
+    @env_conf = env_conf
+  end
+
+  ##
+  # Reads configuration file +file+.  Returns {} if the file does not exist.
+
+  def self.get_conf_from(file)
+    if File.exist? file then
+      YAML.load File.read(file)
+    else
+      {}
+    end
+  end
+
+  ##
+  # Builds a sphinx.conf source configuration for each index.
+
+  def self.get_sources
+    load_models
+
+    indexes = Sphincter::Search.indexes
+    index_count # HACK necessary to set options[:index_id] per-index
+
+    sources = {}
+
+    indexes.each do |klass, model_indexes|
+      model_indexes.each do |options|
+        conn = klass.connection
+        table = klass.table_name
+        pk = conn.quote_column_name klass.primary_key
+        index_id = options[:index_id]
+
+        source_conf = {}
+        source_conf['sql_date_column'] = []
+        source_conf['sql_group_column'] = %w[sphincter_index_id]
+
+        fields = []
+        fields << "(#{table}.#{pk} * #{index_count} + #{index_id}) AS #{pk}"
+        fields << "#{index_id} AS sphincter_index_id"
+        fields << "#{conn.quote table} AS klass"
+
+        options[:fields].each do |field|
+          fields << get_sources_field(source_conf, klass, field)
+        end
+
+        fields = fields.join ', '
+
+        where = []
+        where << "#{table}.#{pk} >= $start AND #{table}.#{pk} <= $end"
+        where << options[:conditions]
+        where = where.compact.join ' AND '
+
+        source_conf['sql_query'] =
+          "SELECT #{fields} FROM #{table} WHERE #{where}"
+        source_conf['sql_query_info'] =
+          "SELECT * FROM #{table} " \
+            "WHERE #{table}.#{pk} = (($id - #{index_id}) / #{index_count})"
+        source_conf['sql_query_range'] =
+          "SELECT MIN(#{pk}), MAX(#{pk}) FROM #{table}"
+        source_conf['strip_html'] = options[:strip_html] ? 1 : 0
+
+        name = options[:name] || table
+        sources[name] = source_conf
+      end
+    end
+
+    sources
+  end
+
+  ##
+  # Builds a field for a source's sql_query sphinx.conf setting.
+  #
+  # get_sources_field only understands :datetime, :boolean, :integer, :string
+  # and :text column types.
+
+  def self.get_sources_field(source_conf, klass, field)
+    conn = klass.connection
+    table = klass.table_name
+
+    quoted_field = conn.quote_column_name field
+    case klass.columns_hash[field].type
+    when :date, :datetime, :time, :timestamp then
+      source_conf['sql_date_column'] << field
+      "UNIX_TIMESTAMP(#{table}.#{quoted_field}) AS #{quoted_field}"
+    when :boolean, :integer then
+      source_conf['sql_group_column'] << field
+      "#{table}.#{quoted_field} AS #{quoted_field}"
+    when :string, :text then
+      "#{table}.#{quoted_field} AS #{quoted_field}"
+    end
+  end
+
+  ##
+  # Retrieves the database configuration for ActiveRecord::Base and adapts it
+  # for a sphinx.conf file.
+
+  def self.get_db_conf
+    conf = {}
+    ar_conf = ActiveRecord::Base.configurations[::RAILS_ENV]
+
+    conf['type']     = ar_conf['adapter']
+    conf['sql_host'] = ar_conf['host']     if ar_conf.include? 'host'
+    conf['sql_user'] = ar_conf['username'] if ar_conf.include? 'username'
+    conf['sql_pass'] = ar_conf['password'] if ar_conf.include? 'password'
+    conf['sql_db']   = ar_conf['database'] if ar_conf.include? 'database'
+    conf['sql_sock'] = ar_conf['socket']   if ar_conf.include? 'socket'
+
+    conf
+  end
+
+  ##
+  # Iterates over the searchable ActiveRecord::Base classes and assigns an
+  # index to each one.  Returns the total number of indexes found.
+
+  def self.index_count
+    return @index_count unless @index_count.nil?
+
+    @index_count = 0
+
+    load_models
+
+    Sphincter::Search.indexes.each do |model, model_indexes|
+      model_indexes.each do |options|
+        options[:index_id] = @index_count
+        @index_count += 1
+      end
+    end
+    @index_count
+  end
+
+  ##
+  # Loads ActiveRecord::Base models from app/models.
+
+  def self.load_models
+    model_files = Dir[File.join(RAILS_ROOT, 'app', 'models', '*.rb')]
+    model_names = model_files.map { |name| File.basename name, '.rb' }
+    model_names.each { |name| name.camelize.constantize }
+  end
+
+  ##
+  # Returns the pid of searchd if searchd is running, otherwise false.
+
+  def self.searchd_running?
+    pid_file = Sphincter::Configure.get_conf['searchd']['pid_file']
+    return false unless File.exist? pid_file
+
+    pid = File.read pid_file
+    return false if pid.empty?
+
+    running = `ps -p #{pid}` =~ /#{pid}.*searchd/
+    running ? pid : false
+  end
+
+  ##
+  # Outputs a sphinx.conf configuration section titled +heading+ using the
+  # Hash +data+.  Values in +data+ may be a String or Array.  For an Array,
+  # the Hash key is printed multiple times.
+
+  def self.section(heading, data)
+    section = []
+    section << heading
+    section << '{'
+    data.sort_by { |k,| k }.each do |key, value|
+      case value
+      when Array then
+        next if value.empty?
+        value.each do |v|
+          section << "  #{key} = #{v}"
+        end
+      else
+        section << "  #{key} = #{value}"
+      end
+    end
+    section << '}'
+    section.join "\n"
+  end
+
+  ##
+  # The path to sphinx.conf.
+
+  def self.sphinx_conf
+    @sphinx_conf ||= File.join sphinx_dir, 'sphinx.conf'
+  end
+
+  ##
+  # The directory where sphinx's files live.
+
+  def self.sphinx_dir
+    @sphinx_dir ||= File.join(RAILS_ROOT,
+                              Sphincter::Configure.get_conf['sphincter']['path'])
+  end
+
+  ##
+  # Writes out a sphinx.conf configuration using +conf+ and +sources+.
+
+  def self.write_configuration(conf, sources)
+    FileUtils.mkdir_p sphinx_dir
+
+    out = []
+
+    out << section('indexer', conf['indexer'])
+    out << nil
+
+    out << section('searchd', conf['searchd'])
+    out << nil
+
+    sources.each do |index_name, values|
+      source_data = conf['source'].merge values
+      out << section("source #{index_name}", source_data)
+      out << nil
+
+      index_path = File.join sphinx_dir, index_name
+      index_data = conf['index'].merge 'source' => index_name,
+                                       'path' => index_path
+
+      out << section("index #{index_name}", index_data)
+      out << nil
+    end
+
+    File.open sphinx_conf, 'w' do |fp|
+      fp.write out.join("\n")
+    end
+  end
+
+end
+