mongoid-sphinx 0.0.4 → 0.0.5

data/README.markdown

@@ -23,49 +23,19 @@ and a MongoDB installation. Just add this to your Gemfile:
 No additional configuraton is needed for interfacing with MongoDB: Setup is
 done when Mongoid is able to talk to the MongoDB server.
 
-A proper "sphinx.conf" file and a script for retrieving index data have to
-be provided for interfacing with Sphinx: Sorry, no ThinkingSphinx like
-magic... :-) Depending on the amount of data, more than one index may be used
-and indexes may be consolidated from time to time.
-
-This is a sample configuration for a single "main" index:
-
-    searchd {
-      address = 0.0.0.0
-      port = 3312
-
-      log = ./sphinx/searchd.log
-      query_log = ./sphinx/query.log
-      pid_file = ./sphinx/searchd.pid
-    }
-
-    source mongoblog {
-      type = xmlpipe2
-  
-      xmlpipe_command = rake sphinx:genxml --silent
-    }
-
-    index mongoblog {
-      source = mongoblog
-
-      charset_type = utf-8
-      path = ./sphinx/sphinx_index_main
-    }
-
-Notice the line "xmlpipe_command =". This is what the indexer runs to generate 
-its input. You can change this to whatever works best for you, but I set it up as 
-a rake task, with the following in `lib/tasks/sphinx.rake` .
-
-Here :fields is a list of fields to export. Performance tends to suffer if you export
-everything, so you'll probably want to just list the fields you're indexing.
-
-    namespace :sphinx do
-      task :genxml => :environment do
-        MongoidSphinx::Indexer::XMLDocset.stream(Food)
-      end
-    end
+## Rake Tasks
+
+MongoidSphinx can now generate your configs and control indexer and searchd through rake 
+tasks (many thanks to Pat Allen for his work on Thinking Sphinx here. Much of the code to 
+make this happen is right out of TS). Here is a list of the available rake tasks:
 
-This uses MongoDB cursor to better stream collection. Instead of offset. See: http://groups.google.com/group/mongodb-user/browse_thread/thread/35f01db45ea3b0bd/96ebc49b511a6b41?lnk=gst&q=skip#96ebc49b511a6b41
+    mongoid_sphinx:configure # creates a configuration file in congif/{environment}.sphinx.conf
+    mongoid_sphinx:index     # indexes your data
+    mongoid_sphinx:start     # starts searchd
+    mongoid_sphinx:stop      # stops searchd
+    mongoid_sphinx:restart   # stops then start searchd
+    
+There are also some shortcuts you can use. See lib/mongoid_sphinx/tasks.rb for the full list.
 
 ## Models
 
@@ -85,8 +55,12 @@ Sample:
 
       field :title
       field :body
+      field :created_at, :type => 'DateTime'
+      field :comment_count, :type => 'Integer'
 
-      search_index :title, :body
+      search_index(:fields => [:title, :body], 
+                   :attributes => [:created_at, :comment_count],
+                   :options => {:stopwords => "#{Rails.root}/config/sphinx/stopwords.txt"})
     end
 
 You must also create a config/sphinx.yml file with the host and port of your sphinxd process like so:
@@ -114,7 +88,7 @@ Samples:
 
     Post.search('first')
     => [...]
-
+    
     post = Post.search('this is @title post').first
     post.title
     => "First Post"
@@ -129,6 +103,12 @@ document IDs but return the raw IDs instead.
 Sample:
 
     Post.search('my post', :limit => 100)
+    
+You can also specify filters based on attributes. Here is the format:
+
+    post = Post.search('first', :with => {:created_at => 1.day.ago..Time.now})
+    post = Post.search('first', :without => {:created_at => 1.day.ago..Time.now})
+    post = Post.search('first', :with => {:comment_count => [5,6]})
 
 ## Copyright
 

data/lib/mongoid_sphinx/configuration.rb

@@ -5,17 +5,156 @@ module MongoidSphinx
   class Configuration
     include Singleton
     
-    attr_accessor :address, :port, :configuration
+    SourceOptions = %w( mysql_connect_flags mysql_ssl_cert mysql_ssl_key
+      mysql_ssl_ca sql_range_step sql_query_pre sql_query_post
+      sql_query_killlist sql_ranged_throttle sql_query_post_index unpack_zlib
+      unpack_mysqlcompress unpack_mysqlcompress_maxsize )
+    
+    IndexOptions  = %w( blend_chars charset_table charset_type charset_dictpath
+      docinfo enable_star exceptions expand_keywords hitless_words
+      html_index_attrs html_remove_elements html_strip index_exact_words
+      ignore_chars inplace_docinfo_gap inplace_enable inplace_hit_gap
+      inplace_reloc_factor inplace_write_factor min_infix_len min_prefix_len
+      min_stemming_len min_word_len mlock morphology ngram_chars ngram_len
+      ondisk_dict overshort_step phrase_boundary phrase_boundary_step preopen
+      stopwords stopwords_step wordforms )
+    
+    attr_accessor :searchd_file_path, :model_directories, :indexed_models
+    attr_accessor :source_options, :index_options
+    attr_accessor :configuration, :controller
+    
+    def initialize
+      @configuration = Riddle::Configuration.new
+      @configuration.searchd.pid_file   = "#{Rails.root}/log/searchd.#{Rails.env}.pid"
+      @configuration.searchd.log        = "#{Rails.root}/log/searchd.log"
+      @configuration.searchd.query_log  = "#{Rails.root}/log/searchd.query.log"
+      
+      @controller = Riddle::Controller.new @configuration, "#{Rails.root}/config/#{Rails.env}.sphinx.conf"
+      
+      self.address              = "127.0.0.1"
+      self.port                 = 9312
+      self.searchd_file_path    = "#{Rails.root}/db/sphinx/#{Rails.env}"
+      self.model_directories    = ["#{Rails.root}/app/models/"] + Dir.glob("#{Rails.root}/vendor/plugins/*/app/models/")
+      self.indexed_models       = []
+      
+      self.source_options  = {
+        :type => "xmlpipe2"
+      }
+      self.index_options   = {
+        :charset_type => "utf-8",
+        :morphology => "stem_en"
+      }
+      
+      parse_config
+      
+      self
+    end
     
     def client
       @configuration ||= parse_config
       Riddle::Client.new address, port
     end
     
+    def build(file_path=nil)
+      file_path ||= "#{self.config_file}"
+      
+      @configuration.indexes.clear
+      
+      MongoidSphinx.context.indexed_models.each do |model|
+        model = model.constantize
+        @configuration.indexes.concat model.to_riddle
+      end
+      
+      open(file_path, "w") do |file|
+        file.write @configuration.render
+      end
+    end
+    
+    def address
+      @address
+    end
+
+    def address=(address)
+      @address = address
+      @configuration.searchd.address = address
+    end
+
+    def port
+      @port
+    end
+
+    def port=(port)
+      @port = port
+      @configuration.searchd.port = port
+    end
+    
+    def mem_limit
+      @mem_limit
+    end
+
+    def mem_limit=(mem_limit)
+      @mem_limit = mem_limit
+      @configuration.indexer.mem_limit = mem_limit
+    end
+
+    def pid_file
+      @configuration.searchd.pid_file
+    end
+
+    def pid_file=(pid_file)
+      @configuration.searchd.pid_file = pid_file
+    end
+
+    def searchd_log_file
+      @configuration.searchd.log
+    end
+
+    def searchd_log_file=(file)
+      @configuration.searchd.log = file
+    end
+
+    def query_log_file
+      @configuration.searchd.query_log
+    end
+
+    def query_log_file=(file)
+      @configuration.searchd.query_log = file
+    end
+
+    def config_file
+      @controller.path
+    end
+
+    def config_file=(file)
+      @controller.path = file
+    end
+
+    def bin_path
+      @controller.bin_path
+    end
+
+    def bin_path=(path)
+      @controller.bin_path = path
+    end
+
+    def searchd_binary_name
+      @controller.searchd_binary_name
+    end
+
+    def searchd_binary_name=(name)
+      @controller.searchd_binary_name = name
+    end
+
+    def indexer_binary_name
+      @controller.indexer_binary_name
+    end
+
+    def indexer_binary_name=(name)
+      @controller.indexer_binary_name = name
+    end
+    
     private
     
-    # Parse the config/sphinx.yml file - if it exists
-    #
     def parse_config
       path = "#{Rails.root}/config/sphinx.yml"
       return unless File.exists?(path)

data/lib/mongoid_sphinx/context.rb

@@ -0,0 +1,62 @@
+class MongoidSphinx::Context
+  attr_reader :indexed_models
+  
+  def initialize(*models)
+    @indexed_models = []
+  end
+  
+  def prepare
+    MongoidSphinx::Configuration.instance.indexed_models.each do |model|
+      add_indexed_model model
+    end
+    
+    return unless indexed_models.empty?
+    
+    load_models
+  end
+  
+  def define_indexes
+    indexed_models.each { |model|
+      model.constantize.define_indexes
+    }
+  end
+  
+  def add_indexed_model(model)
+    model = model.name if model.is_a?(Class)
+    
+    indexed_models << model
+    indexed_models.uniq!
+    indexed_models.sort!
+  end
+  
+  private
+  
+  # Make sure all models are loaded - without reloading any that
+  # ActiveRecord::Base is already aware of (otherwise we start to hit some
+  # messy dependencies issues).
+  #
+  def load_models
+    MongoidSphinx::Configuration.instance.model_directories.each do |base|
+      Dir["#{base}**/*.rb"].each do |file|
+        model_name = file.gsub(/^#{base}([\w_\/\\]+)\.rb/, '\1')
+        
+        next if model_name.nil?
+        next if ::ActiveRecord::Base.send(:descendants).detect { |model|
+          model.name == model_name.camelize
+        }
+        
+        begin
+          model_name.camelize.constantize
+        rescue LoadError
+          model_name.gsub!(/.*[\/\\]/, '').nil? ? next : retry
+        rescue NameError
+          next
+        rescue StandardError => err
+          STDERR.puts "Warning: Error loading #{file}:"
+          STDERR.puts err.message
+          STDERR.puts err.backtrace.join("\n"), ''
+        end
+      end
+    end
+  end
+end

data/lib/mongoid_sphinx/index.rb

@@ -0,0 +1,75 @@
+module MongoidSphinx
+  class Index
+    attr_accessor :name, :model, :source
+    
+    def initialize(model)
+      @name         = self.class.name_for model
+      @model        = model
+      @options      = model.index_options
+      @source       = Riddle::Configuration::XMLSource.new( "#{core_name}_0", config.source_options[:type])
+      @source.xmlpipe_command = "RAILS_ENV=#{Rails.env} script/rails runner '#{model.to_s}.sphinx_stream'"
+    end
+    
+    def core_name
+      "#{name}_core"
+    end
+    
+    def self.name_for(model)
+      model.name.underscore.tr(':/\\', '_')
+    end
+    
+    def local_options
+      @options
+    end
+    
+    def options
+      all_index_options = config.index_options.clone
+      @options.keys.select { |key|
+        MongoidSphinx::Configuration::IndexOptions.include?(key.to_s)
+      }.each { |key| all_index_options[key.to_sym] = @options[key] }
+      all_index_options
+    end
+    
+    def to_riddle
+      indexes = [to_riddle_for_core]
+      indexes << to_riddle_for_distributed
+    end
+    
+    private
+    
+    def utf8?
+      options[:charset_type] == "utf-8"
+    end
+    
+    def config
+      @config ||= MongoidSphinx::Configuration.instance
+    end
+    
+    def to_riddle_for_core
+      index = Riddle::Configuration::Index.new core_name
+      index.path = File.join config.searchd_file_path, index.name
+      
+      set_configuration_options_for_indexes index
+      index.sources << @source
+      
+      index
+    end
+    
+    def to_riddle_for_distributed
+      index = Riddle::Configuration::DistributedIndex.new name
+      index.local_indexes << core_name
+      index
+    end
+    
+    def set_configuration_options_for_indexes(index)
+      config.index_options.each do |key, value|
+        method = "#{key}=".to_sym
+        index.send(method, value) if index.respond_to?(method)
+      end
+      
+      options.each do |key, value|
+        index.send("#{key}=".to_sym, value) if MongoidSphinx::Configuration::IndexOptions.include?(key.to_s) && !value.nil?
+      end
+    end
+  end
+end

data/lib/mongoid_sphinx/mongoid/sphinx.rb

@@ -17,15 +17,32 @@ module Mongoid
       
       cattr_accessor :search_fields
       cattr_accessor :search_attributes
+      cattr_accessor :index_options
     end
     
     module ClassMethods
+      
       def search_index(options={})
         self.search_fields = options[:fields]
         self.search_attributes = {}
+        self.index_options = options[:options] || {}
         options[:attributes].each do |attrib|
           self.search_attributes[attrib] = SPHINX_TYPE_MAPPING[self.fields[attrib.to_s].type.to_s] || 'str2ordinal'
         end
+        
+        MongoidSphinx.context.add_indexed_model self
+      end
+      
+      def internal_sphinx_index
+        MongoidSphinx::Index.new(self)
+      end
+      
+      def has_sphinx_indexes?
+        self.search_fields && self.search_fields.length > 0
+      end
+      
+      def to_riddle
+        self.internal_sphinx_index.to_riddle
       end
       
       def sphinx_stream

data/lib/mongoid_sphinx/railtie.rb

@@ -0,0 +1,10 @@
+require 'mongoid_sphinx'
+require 'rails'
+
+module MongoidSphinx
+  class Railtie < Rails::Railtie
+    rake_tasks do
+      load File.expand_path('../tasks.rb', __FILE__)
+    end
+  end
+end

data/lib/mongoid_sphinx/tasks.rb

@@ -0,0 +1,113 @@
+require 'fileutils'
+
+namespace :mongoid_sphinx do
+  
+  desc "Output the current Mongoid Sphinx version"
+  task :version => :environment do
+    puts "Mongoid Sphinx v" + MongoidSphinx::VERSION
+  end
+  
+  desc "Stop if running, then start a Sphinx searchd daemon using Mongoid Sphinx's settings"
+  task :running_start => :environment do
+    Rake::Task["mongoid_sphinx:stop"].invoke if sphinx_running?
+    Rake::Task["mongoid_sphinx:start"].invoke
+  end
+  
+  desc "Start a Sphinx searchd daemon using Mongoid Sphinx's settings"
+  task :start => :environment do
+    config = MongoidSphinx::Configuration.instance
+    
+    FileUtils.mkdir_p config.searchd_file_path
+    raise RuntimeError, "searchd is already running." if sphinx_running?
+    
+    Dir["#{config.searchd_file_path}/*.spl"].each { |file| File.delete(file) }
+    
+    config.controller.start
+    
+    if sphinx_running?
+      puts "Started successfully (pid #{sphinx_pid})."
+    else
+      puts "Failed to start searchd daemon. Check #{config.searchd_log_file}"
+    end
+  end
+  
+  desc "Stop Sphinx using Mongoid Sphinx's settings"
+  task :stop => :environment do
+    unless sphinx_running?
+      puts "searchd is not running"
+    else
+      config = MongoidSphinx::Configuration.instance
+      pid    = sphinx_pid
+      config.controller.stop
+      puts "Stopped search daemon (pid #{pid})."
+    end
+  end
+  
+  desc "Restart Sphinx"
+  task :restart => [:environment, :stop, :start]
+  
+  desc "Generate the Sphinx configuration file using Mongoid Sphinx's settings"
+  task :configure => :environment do
+    config = MongoidSphinx::Configuration.instance
+    puts "Generating Configuration to #{config.config_file}"
+    config.build
+  end
+  
+  desc "Index data for Sphinx using Mongoid Sphinx's settings"
+  task :index => :environment do
+    config = MongoidSphinx::Configuration.instance
+    unless ENV["INDEX_ONLY"] == "true"
+      puts "Generating Configuration to #{config.config_file}"
+      config.build
+    end
+    
+    FileUtils.mkdir_p config.searchd_file_path
+    config.controller.index :verbose => true
+  end
+  
+  desc "Reindex Sphinx without regenerating the configuration file"
+  task :reindex => :environment do
+    config = MongoidSphinx::Configuration.instance
+    FileUtils.mkdir_p config.searchd_file_path
+    puts config.controller.index
+  end
+  
+  desc "Stop Sphinx (if it's running), rebuild the indexes, and start Sphinx"
+  task :rebuild => :environment do
+    Rake::Task["mongoid_sphinx:stop"].invoke if sphinx_running?
+    Rake::Task["mongoid_sphinx:index"].invoke
+    Rake::Task["mongoid_sphinx:start"].invoke
+  end
+end
+
+namespace :ms do
+  desc "Output the current Mongoid Sphinx version"
+  task :version => "mongoid_sphinx:version"
+  desc "Stop if running, then start a Sphinx searchd daemon using Mongoid Sphinx's settings"
+  task :run     => "mongoid_sphinx:running_start"
+  desc "Start a Sphinx searchd daemon using Mongoid Sphinx's settings"
+  task :start   => "mongoid_sphinx:start"
+  desc "Stop Sphinx using Mongoid Sphinx's settings"
+  task :stop    => "mongoid_sphinx:stop"
+  desc "Index data for Sphinx using Mongoid Sphinx's settings"
+  task :in      => "mongoid_sphinx:index"
+  task :index   => "mongoid_sphinx:index"
+  desc "Reindex Sphinx without regenerating the configuration file"
+  task :reindex => "mongoid_sphinx:reindex"
+  desc "Restart Sphinx"
+  task :restart => "mongoid_sphinx:restart"
+  desc "Generate the Sphinx configuration file using Mongoid Sphinx's settings"
+  task :conf    => "mongoid_sphinx:configure"
+  desc "Generate the Sphinx configuration file using Mongoid Sphinx's settings"
+  task :config  => "mongoid_sphinx:configure"
+  desc "Stop Sphinx (if it's running), rebuild the indexes, and start Sphinx"
+  task :rebuild => "mongoid_sphinx:rebuild"
+end
+
+def sphinx_pid
+  MongoidSphinx.sphinx_pid
+end
+
+def sphinx_running?
+  MongoidSphinx.sphinx_running?
+end

data/lib/mongoid_sphinx/version.rb

@@ -1,3 +1,3 @@
 module MongoidSphinx #:nodoc
-  VERSION = "0.0.4"
+  VERSION = "0.0.5"
 end

data/lib/mongoid_sphinx.rb

@@ -2,5 +2,54 @@ require "mongoid"
 require "riddle"
 
 require 'mongoid_sphinx/configuration'
+require 'mongoid_sphinx/context'
+require 'mongoid_sphinx/index'
 require 'mongoid_sphinx/mongoid/identity'
-require 'mongoid_sphinx/mongoid/sphinx'
+require 'mongoid_sphinx/mongoid/sphinx'
+require 'mongoid_sphinx/railtie'
+
+module MongoidSphinx
+  
+  @@sphinx_mutex = Mutex.new
+  @@context      = nil
+  
+  def self.context
+    if @@context.nil?
+      @@sphinx_mutex.synchronize do
+        if @@context.nil?
+          @@context = MongoidSphinx::Context.new
+          @@context.prepare
+        end
+      end
+    end
+    
+    @@context
+  end
+  
+  def self.reset_context!
+    @@sphinx_mutex.synchronize do
+      @@context = nil
+    end
+  end
+  
+  def self.pid_active?(pid)
+    !!Process.kill(0, pid.to_i)
+  rescue Errno::EPERM => e
+    true
+  rescue Exception => e
+    false
+  end
+  
+  def self.sphinx_running?
+    !!sphinx_pid && pid_active?(sphinx_pid)
+  end
+  
+  def self.sphinx_pid
+    if File.exists?(MongoidSphinx::Configuration.instance.pid_file)
+      File.read(MongoidSphinx::Configuration.instance.pid_file)[/\d+/]
+    else
+      nil
+    end
+  end
+  
+end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: mongoid-sphinx
 version: !ruby/object:Gem::Version 
-  hash: 23
+  hash: 21
   prerelease: false
   segments: 
   - 0
   - 0
-  - 4
-  version: 0.0.4
+  - 5
+  version: 0.0.5
 platform: ruby
 authors: 
 - Matt Hodgson
@@ -15,11 +15,11 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-01-04 00:00:00 -05:00
+date: 2011-01-12 00:00:00 -05:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
-  version_requirements: &id001 !ruby/object:Gem::Requirement 
+  requirement: &id001 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - "="
@@ -32,12 +32,12 @@ dependencies:
         - beta
         - 19
         version: 2.0.0.beta.19
-  requirement: *id001
   type: :runtime
   name: mongoid
   prerelease: false
+  version_requirements: *id001
 - !ruby/object:Gem::Dependency 
-  version_requirements: &id002 !ruby/object:Gem::Requirement 
+  requirement: &id002 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ~>
@@ -48,10 +48,10 @@ dependencies:
         - 1
         - 0
         version: 1.1.0
-  requirement: *id002
   type: :runtime
   name: riddle
   prerelease: false
+  version_requirements: *id002
 description: A full text indexing extension for MongoDB using Sphinx and Mongoid.
 email: 
 - mhodgson@redbeard-tech.com
@@ -63,8 +63,12 @@ extra_rdoc_files: []
 
 files: 
 - lib/mongoid_sphinx/configuration.rb
+- lib/mongoid_sphinx/context.rb
+- lib/mongoid_sphinx/index.rb
 - lib/mongoid_sphinx/mongoid/identity.rb
 - lib/mongoid_sphinx/mongoid/sphinx.rb
+- lib/mongoid_sphinx/railtie.rb
+- lib/mongoid_sphinx/tasks.rb
 - lib/mongoid_sphinx/version.rb
 - lib/mongoid_sphinx.rb
 - README.markdown