dm-sphinx-adapter 0.3

data/History.txt

@@ -0,0 +1,12 @@
+=== 0.3 / 2008-11-18
+
+  * Removed calls to indexer on create/update. See README.txt
+  * Made the client object available from the adapter.
+
+=== 0.2 / 2008-11-09
+
+  * Addributes.
+  * Self managed searchd daemon if you want it.
+
+=== 0.1 / 2008-10-24
+

data/LICENCE.txt

@@ -0,0 +1,20 @@
+Copyright (c) 2008 Shane Hanna
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Manifest.txt

@@ -0,0 +1,20 @@
+History.txt
+LICENCE.txt
+Manifest.txt
+README.txt
+Rakefile
+dm-sphinx-adapter.gemspec
+lib/dm-sphinx-adapter.rb
+lib/dm-sphinx-adapter/sphinx_adapter.rb
+lib/dm-sphinx-adapter/sphinx_attribute.rb
+lib/dm-sphinx-adapter/sphinx_client.rb
+lib/dm-sphinx-adapter/sphinx_config.rb
+lib/dm-sphinx-adapter/sphinx_index.rb
+lib/dm-sphinx-adapter/sphinx_resource.rb
+test/data/sphinx.conf
+test/fixtures/item.rb
+test/fixtures/item.sql
+test/helper.rb
+test/test_client.rb
+test/test_config.rb
+test/test_search.rb

data/README.txt

@@ -0,0 +1,192 @@
+= DataMapper Sphinx Adapter
+
+A Sphinx DataMapper adapter.
+
+== Synopsis
+
+DataMapper uses URIs or a connection has to connect to your data-stores. In this case the sphinx search daemon
+<tt>searchd</tt>.
+
+On its own this adapter will only return an array of document IDs when queried. The dm-more source (not the gem)
+however provides dm-is-searchable, a common interface to search one adapter and load documents from another. My
+suggestion is to use this adapter in tandem with dm-is-searchable.
+
+The dm-is-searchable plugin is part of dm-more though unfortunately isn't built and bundled with dm-more gem.
+You'll need to checkout the dm-more source with Git from git://github.com/sam/dm-more.git and build/install the
+gem yourself.
+
+  git clone git://github.com/sam/dm-more.git
+  cd dm-more/dm-is-searchable
+  sudo rake install_gem
+
+Like all DataMapper adapters you can connect with a Hash or URI.
+
+A URI:
+  DataMapper.setup(:search, 'sphinx://localhost')
+
+The breakdown is:
+  "#{adapter}://#{host}:#{port}/#{config}"
+  - adapter Must be :sphinx
+  - host    Hostname (default: localhost)
+  - port    Optional port number (default: 3312)
+  - config  Optional but strongly recommended path to sphinx config file.
+
+Alternatively supply a Hash:
+  DataMapper.setup(:search, {
+    :adapter  => 'sphinx',       # required
+    :config   => './sphinx.conf' # optional. Recommended though.
+    :host     => 'localhost',    # optional. Default: localhost
+    :port     => 3312            # optional. Default: 3312
+    :managed  => true            # optional. Self managed searchd server using daemon_controller.
+  }
+
+=== DataMapper
+
+  require 'rubygems'
+  require 'dm-sphinx-adapter'
+
+  DataMapper.setup(:default, 'sqlite3::memory:')
+  DataMapper.setup(:search, 'sphinx://localhost:3312')
+
+  class Item
+    include DataMapper::Resource
+    property :id, Serial
+    property :name, String
+  end
+
+  # Fire up your sphinx search daemon and start searching.
+  docs  = repository(:search){ Item.all(:name => 'barney') } # Search 'items' index for '@name barney'
+  ids   = docs.map{|doc| doc[:id]}
+  items = Item.all(:id => ids) # Search :default for all the document id's returned by sphinx.
+
+=== DataMapper and Is Searchable
+
+  require 'rubygems'
+  require 'dm-core'
+  require 'dm-is-searchable'
+  require 'dm-sphinx-adapter'
+
+  # Connections.
+  DataMapper.setup(:default, 'sqlite3::memory:')
+  DataMapper.setup(:search, 'sphinx://localhost:3312')
+
+  class Item
+    include DataMapper::Resource
+    property :id, Serial
+    property :name, String
+
+    is :searchable # defaults to :search repository though you can be explicit:
+    # is :searchable, :repository => :sphinx
+  end
+
+  # Fire up your sphinx search daemon and start searching.
+  items = Item.search(:name => 'barney') # Search 'items' index for '@name barney'
+
+=== Merb, DataMapper and Is Searchable
+
+  # config/init.rb
+  dependency 'dm-is-searchable'
+  dependency 'dm-sphinx-adapter'
+
+  # config/database.yml
+  ---
+  development: &defaults
+    repositories:
+      search:
+        adapter:  sphinx
+        host:     localhost
+        port:     3312
+
+  # app/models/item.rb
+  class Item
+    include DataMapper::Resource
+    property :id, Serial
+    property :name, String
+
+    is :searchable # defaults to :search repository though you can be explicit:
+    # is :searchable, :repository => :sphinx
+  end # Item
+
+  # Fire up your sphinx search daemon and start searching.
+  Item.search(:name => 'barney') # Search 'items' index for '@name barney'
+
+=== DataMapper::SphinxResource
+
+For finer grained control you can include DataMapper::SphinxResource. For instance you can search one or more indexes
+and sort, include or exclude by attributes defined in your sphinx configuration:
+
+  class Item
+    include DataMapper::Resource # Optional, included by SphinxResource if you leave it out yourself.
+    include DataMapper::SphinxResource
+    property :id, Serial
+    property :name, String
+
+    is :searchable
+    repository(:search) do
+      index :items
+      index :items_delta, :delta => true
+
+      # Sphinx attributes to sort include/exclude by.
+      attribute :updated_on, DateTime
+    end
+
+  end # Item
+
+  # Search 'items, items_delta' index for '@name barney' updated in the last 30 minutes.
+  Item.search(:name => 'barney', :updated => (Time.now - 1800 .. Time.now))
+
+== Sphinx Configuration.
+
+Though you don't have to supply the sphinx configuration file to dm-sphinx-adapter I'd recommend doing it anyway.
+It's more DRY since all searchd/indexer options can be read straight from the configuration.
+
+  DataMapper.setup(:search, :adapter => 'sphinx', :config => '/path/to/sphinx.conf')
+  DataMapper.setup(:search, 'sphinx://localhost/path/to/sphinx.conf')
+
+If your sphinx.conf lives in either of the default locations /usr/local/etc/sphinx.conf or ./sphinx.conf then you
+only need to supply:
+
+  DataMapper.setup(:search, :adapter => 'sphinx')
+
+== Searchd
+
+As of 0.2 I've added a managed searchd option using daemon_controller. It may come in handy if you only use God, Monit
+or whatever in production. Use the Hash form of DataMapper#setup and supply the option :managed with a true value and
+daemon_controller will start searchd on demand.
+
+It is already strongly encouraged but you will need to specify the path to your sphinx configuration file in order for
+searchd to run. See Sphinx Configuration, DataMapper::SphinxManagedClient.
+
+The daemon_controller library can be found only on github, not rubyforge.
+See http://github.com/FooBarWidget/daemon_controller/tree/master
+
+== Indexer and Live(ish) updates.
+
+As of 0.3 the indexer will no longer be fired on create/update even if you have delta indexes defined. Sphinx indexing
+is blazing fast but unless your resource sees very little activity you will run the risk of lock errors on
+the temporary delta index files (.tmpl.sp1) and your delta index won't be updated. Given this functionality is
+unreliable at best I've chosen to remove it.
+
+For reliable live(ish) updates in a main + delta scheme it's probably best you schedule them outside of your ORM.
+Andrew (Shodan) Aksyonoff of Sphinx suggests a cronjob or alternatively if you need even less lag to "run indexer in
+an endless loop, with a few seconds of sleep in between to allow searchd some headroom to pick up the changes".
+
+== Todo
+
+* Tests. Clearly I only test what I see as important or broken which drives TDD people crazy sometimes :)
+* Loads of documentation. Most of it is unchecked YARD at the moment.
+* Add DataMapper::SphinxClient#attribute_set to allow attribute modification on one or more indexes. It's the only
+  thing missing if you understand the pitfalls and still want to add thinking-sphinx like delta indexing to your
+  resource.
+
+== Dependencies
+
+dm-core and riddle are technically the only requirements though I'd recommend using the dm-more plugin dm-is-searchable
+instead of fetching the document id's yourself.
+
+Unfortunately dm-is-searchable isn't installed even when you build the dm-more gem from github master. You'll need to
+build and install the gem yourself from source.
+
+== Contributing
+
+Go nuts. Just send me a pull request (github or otherwise) when you are happy with your code.

data/Rakefile

@@ -0,0 +1,23 @@
+# -*- ruby -*-
+
+require 'rubygems'
+require 'hoe'
+
+Hoe.new('dm-sphinx-adapter', '0.3') do |p|
+  p.developer('Shane Hanna', 'shane.hanna@gmail.com')
+  p.extra_deps = [
+    ['dm-core', '~> 0.9.7'],
+    ['riddle',  '~> 0.9']
+  ]
+end
+
+# http://blog.behindlogic.com/2008/10/auto-generate-your-manifest-and-gemspec.html
+desc 'Rebuild manifest and gemspec.'
+task :cultivate do
+  Dir.chdir(File.dirname(__FILE__)) do #TODO: Is this required?
+    system %q{git ls-files | grep -v "\.gitignore" > Manifest.txt}
+    system %q{rake debug_gem | grep -v "(in " > `basename \`pwd\``.gemspec}
+  end
+end
+
+# vim: syntax=Ruby

data/dm-sphinx-adapter.gemspec

@@ -0,0 +1,39 @@
+Gem::Specification.new do |s|
+  s.name = %q{dm-sphinx-adapter}
+  s.version = "0.3"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Shane Hanna"]
+  s.date = %q{2008-11-18}
+  s.description = %q{}
+  s.email = ["shane.hanna@gmail.com"]
+  s.extra_rdoc_files = ["History.txt", "LICENCE.txt", "Manifest.txt", "README.txt"]
+  s.files = ["History.txt", "LICENCE.txt", "Manifest.txt", "README.txt", "Rakefile", "dm-sphinx-adapter.gemspec", "lib/dm-sphinx-adapter.rb", "lib/dm-sphinx-adapter/sphinx_adapter.rb", "lib/dm-sphinx-adapter/sphinx_attribute.rb", "lib/dm-sphinx-adapter/sphinx_client.rb", "lib/dm-sphinx-adapter/sphinx_config.rb", "lib/dm-sphinx-adapter/sphinx_index.rb", "lib/dm-sphinx-adapter/sphinx_resource.rb", "test/data/sphinx.conf", "test/fixtures/item.rb", "test/fixtures/item.sql", "test/helper.rb", "test/test_client.rb", "test/test_config.rb", "test/test_search.rb"]
+  s.has_rdoc = true
+  s.homepage = %q{A Sphinx DataMapper adapter.}
+  s.rdoc_options = ["--main", "README.txt"]
+  s.require_paths = ["lib"]
+  s.rubyforge_project = %q{dm-sphinx-adapter}
+  s.rubygems_version = %q{1.2.0}
+  s.summary = %q{}
+  s.test_files = ["test/test_client.rb", "test/test_config.rb", "test/test_search.rb"]
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 2
+
+    if current_version >= 3 then
+      s.add_runtime_dependency(%q<dm-core>, ["~> 0.9.7"])
+      s.add_runtime_dependency(%q<riddle>, ["~> 0.9"])
+      s.add_development_dependency(%q<hoe>, [">= 1.8.2"])
+    else
+      s.add_dependency(%q<dm-core>, ["~> 0.9.7"])
+      s.add_dependency(%q<riddle>, ["~> 0.9"])
+      s.add_dependency(%q<hoe>, [">= 1.8.2"])
+    end
+  else
+    s.add_dependency(%q<dm-core>, ["~> 0.9.7"])
+    s.add_dependency(%q<riddle>, ["~> 0.9"])
+    s.add_dependency(%q<hoe>, [">= 1.8.2"])
+  end
+end

data/lib/dm-sphinx-adapter/sphinx_adapter.rb

@@ -0,0 +1,220 @@
+require 'benchmark'
+
+# TODO: I think perhaps I should move all the query building code to a lib of its own.
+
+module DataMapper
+  module Adapters
+
+    # == Synopsis
+    # DataMapper uses URIs or a connection has to connect to your data-stores. In this case the sphinx search daemon
+    # <tt>searchd</tt>.
+    #
+    # On its own this adapter will only return an array of document IDs when queried. The dm-more source (not the gem)
+    # however provides dm-is-searchable, a common interface to search one adapter and load documents from another. My
+    # suggestion is to use this adapter in tandem with dm-is-searchable.
+    #
+    # The dm-is-searchable plugin is part of dm-more though unfortunately isn't built and bundled with dm-more gem.
+    # You'll need to checkout the dm-more source with Git from git://github.com/sam/dm-more.git and build/install the
+    # gem yourself.
+    #
+    #   git clone git://github.com/sam/dm-more.git
+    #   cd dm-more/dm-is-searchable
+    #   sudo rake install_gem
+    #
+    # Like all DataMapper adapters you can connect with a Hash or URI.
+    #
+    # A URI:
+    #   DataMapper.setup(:search, 'sphinx://localhost')
+    #
+    # The breakdown is:
+    #   "#{adapter}://#{host}:#{port}/#{config}"
+    #   - adapter Must be :sphinx
+    #   - host    Hostname (default: localhost)
+    #   - port    Optional port number (default: 3312)
+    #   - config  Optional but recommended path to sphinx config file.
+    #
+    # Alternatively supply a Hash:
+    #   DataMapper.setup(:search, {
+    #     :adapter  => 'sphinx',       # required
+    #     :config   => './sphinx.conf' # optional. Recommended though.
+    #     :host     => 'localhost',    # optional. Default: localhost
+    #     :port     => 3312            # optional. Default: 3312
+    #     :managed  => true            # optional. Self managed searchd server using daemon_controller.
+    #   })
+    class SphinxAdapter < AbstractAdapter
+      ##
+      # Initialize the sphinx adapter.
+      #
+      # @param [URI, DataObject::URI, Addressable::URI, String, Hash, Pathname] uri_or_options
+      # @see   DataMapper::SphinxConfig
+      # @see   DataMapper::SphinxClient
+      def initialize(name, uri_or_options)
+        super
+
+        managed = !!(uri_or_options.kind_of?(Hash) && uri_or_options[:managed])
+        @client  = managed ? SphinxManagedClient.new(uri_or_options) : SphinxClient.new(uri_or_options)
+      end
+
+      ##
+      # Interaction with searchd and indexer.
+      #
+      # @see DataMapper::SphinxClient
+      # @see DataMapper::SphinxManagedClient
+      attr_reader :client
+
+      def create(resources) #:nodoc:
+        true
+      end
+
+      def delete(query) #:nodoc:
+        true
+      end
+
+      def read_many(query)
+        read(query)
+      end
+
+      def read_one(query)
+        read(query).first
+      end
+
+      protected
+        ##
+        # List sphinx indexes to search.
+        # If no indexes are explicitly declared using DataMapper::SphinxResource then the tableized model name is used.
+        #
+        # @see DataMapper::SphinxResource#sphinx_indexes
+        def indexes(model)
+          indexes = model.sphinx_indexes(repository(self.name).name) if model.respond_to?(:sphinx_indexes)
+          if indexes.nil? or indexes.empty?
+            # TODO: Is it resource_naming_convention.call(model.name) ?
+            indexes = [SphinxIndex.new(model, Extlib::Inflection.tableize(model.name))]
+          end
+          indexes
+        end
+
+        ##
+        # List sphinx delta indexes to search.
+        #
+        # @see DataMapper::SphinxResource#sphinx_indexes
+        def delta_indexes(model)
+          indexes(model).find_all{|i| i.delta?}
+        end
+
+        ##
+        # Query sphinx for a list of document IDs.
+        #
+        # @param [DataMapper::Query]
+        def read(query)
+          from    = indexes(query.model).map{|index| index.name}.join(', ')
+          search  = search_query(query)
+          options = {
+            :match_mode => :extended, # TODO: Modes!
+            :limit      => (query.limit  ? query.limit.to_i : 0),
+            :offset     => (query.offset ? query.offset.to_i : 0),
+            :filters    => search_filters(query) # By attribute.
+          }
+          if order = search_order(query)
+            options.update(
+              :sort_mode => :extended,
+              :sort_by   => order
+            )
+          end
+
+          res = @client.search(search, from, options)
+          raise res[:error] unless res[:error].nil?
+
+          DataMapper.logger.info(
+            %q{Sphinx (%.3f): search '%s' in '%s' found %d documents} % [res[:time], search, from, res[:total]]
+          )
+          res[:matches].map{|doc| doc[:doc]}
+        end
+
+        ##
+        # Sphinx search query string from properties (fields).
+        #
+        # If the query has no conditions an '' empty string will be generated possibly triggering Sphinx's full scan
+        # mode.
+        #
+        # @see    http://www.sphinxsearch.com/doc.html#searching
+        # @see    http://www.sphinxsearch.com/doc.html#conf-docinfo
+        # @param  [DataMapper::Query]
+        # @return [String]
+        def search_query(query)
+          match  = []
+
+          if query.conditions.empty?
+            match << ''
+          else
+            # TODO: This needs to be altered by match mode since not everything is supported in different match modes.
+            query.conditions.each do |operator, property, value|
+              next if property.kind_of? SphinxAttribute # Filters are added elsewhere.
+              # TODO: Why does my gem riddle differ from the vendor riddle that comes with ts?
+              # escaped_value = Riddle.escape(value)
+              escaped_value = value.to_s.gsub(/[\(\)\|\-!@~"&\/]/){|char| "\\#{char}"}
+              match << case operator
+                when :eql, :like then "@#{property.field} #{escaped_value}"
+                when :not        then "@#{property.field} -#{escaped_value}"
+                when :lt, :gt, :lte, :gte
+                  DataMapper.logger.warn('Sphinx: Query properties with lt, gt, lte, gte are treated as .eql')
+                  "@#{name} #{escaped_value}"
+                when :raw
+                  "#{property}"
+              end
+            end
+          end
+          match.join(' ')
+        end
+
+        ##
+        # Sphinx search query filters from attributes.
+        # @param  [DataMapper::Query]
+        # @return [Array]
+        def search_filters(query)
+          filters = []
+          query.conditions.each do |operator, attribute, value|
+            next unless attribute.kind_of? SphinxAttribute
+            # TODO: Value cast to uint, bool, str2ordinal, float
+            filters << case operator
+              when :eql, :like then Riddle::Client::Filter.new(attribute.name.to_s, filter_value(value))
+              when :not        then Riddle::Client::Filter.new(attribute.name.to_s, filter_value(value), true)
+              else
+                error = "Sphinx: Query attributes do not support the #{operator} operator"
+                DataMapper.logger.error(error)
+                raise error # TODO: RuntimeError subclass and more information about the actual query.
+            end
+          end
+          filters
+        end
+
+        ##
+        # Order by attributes.
+        #
+        # @return [String or Symbol]
+        def search_order(query)
+          by = []
+          # TODO: How do you tell the difference between the default query order and someone explicitly asking for
+          # sorting by the primary key?
+          query.order.each do |order|
+            next unless order.property.kind_of? SphinxAttribute
+            by << [order.property.field, order.direction].join(' ')
+          end
+          by.empty? ? nil : by.join(', ')
+        end
+
+        # TODO: Move this to SphinxAttribute#something.
+        # This is ninja'd straight from TS just to get things going.
+        def filter_value(value)
+          case value
+            when Range
+              value.first.is_a?(Time) ? value.first.to_i..value.last.to_i : value
+            when Array
+              value.collect { |val| val.is_a?(Time) ? val.to_i : val }
+            else
+              Array(value)
+          end
+        end
+    end # SphinxAdapter
+  end # Adapters
+end # DataMapper
+

data/lib/dm-sphinx-adapter/sphinx_attribute.rb

@@ -0,0 +1,22 @@
+module DataMapper
+  class SphinxAttribute < Property
+
+    # DataMapper types supported as Sphinx attributes.
+    TYPES = [
+      TrueClass,                # sql_attr_bool
+      String,                   # sql_attr_str2ordinal
+      # DataMapper::Types::Text,
+      Float,                    # sql_attr_float
+      Integer,                  # sql_attr_uint
+      # BigDecimal,             # sql_attr_float?
+      DateTime,                 # sql_attr_timestamp
+      Date,                     # sql_attr_timestamp
+      Time,                     # sql_attr_timestamp
+      # Object,
+      # Class,
+      # DataMapper::Types::Discriminator,
+      DataMapper::Types::Serial # sql_attr_uint
+    ]
+
+  end # SphinxAttribute
+end # DataMapper

data/lib/dm-sphinx-adapter/sphinx_client.rb

@@ -0,0 +1,76 @@
+require 'rubygems'
+
+gem 'riddle', '~> 0.9'
+require 'riddle'
+
+module DataMapper
+  class SphinxClient
+    def initialize(uri_or_options)
+      @config = SphinxConfig.new(uri_or_options)
+    end
+
+    # TODO: What about filters?
+    def search(query, indexes = '*', options = {})
+      indexes = indexes.join(' ') if indexes.kind_of?(Array)
+
+      client = Riddle::Client.new(@config.address, @config.port)
+      options.each{|k, v| client.method("#{k}=".to_sym).call(v) if client.respond_to?("#{k}=".to_sym)}
+      client.query(query, indexes.to_s)
+    end
+
+    ##
+    # Index one or more indexes.
+    #
+    # @param [Array, String] indexes Defaults to --all if indexes is nil or '*'.
+    def index(indexes = nil, options = {})
+      indexes = indexes.join(' ') if indexes.kind_of?(Array)
+
+      command = @config.indexer_bin
+      command << " --rotate" if running?
+      command << ((indexes.nil? || indexes == '*') ? ' --all' : " #{indexes.to_s}")
+      warn "Sphinx: Indexer #{$1}" if `#{command}` =~ /(?:error|fatal|warning):?\s*([^\n]+)/i
+    end
+
+    protected
+
+      ##
+      # Is the client running.
+      #
+      # Tests the address and port set in the configuration file.
+      def running?
+        !!TCPSocket.new(@config.address, @config.port) rescue nil
+      end
+  end # SphinxClient
+
+  ##
+  # Managed searchd if you don't already have god/monit doing the job for you.
+  #
+  # Requires you have daemon_controller installed.
+  # @see http://github.com/FooBarWidget/daemon_controller/tree/master
+  class SphinxManagedClient < SphinxClient
+    def initialize(url_or_options)
+      super
+
+      # Fire up searchd.
+      require 'daemon_controller'
+      @client = DaemonController.new(
+        :identifier    => 'Sphinx searchd',
+        :start_command => @config.searchd_bin,
+        :stop_command  => "#{@config.searchd_bin} --stop",
+        :ping_command  => method(:running?),
+        :pid_file      => @config.pid_file,
+        :log_file      => @config.log
+      )
+    end
+
+    def search(*args)
+      @client.connect do
+        super *args
+      end
+    end
+
+    def stop
+      @client.stop
+    end
+  end # SphinxManagedClient
+end # DataMapper

data/lib/dm-sphinx-adapter/sphinx_config.rb

@@ -0,0 +1,160 @@
+require 'strscan'
+require 'pathname'
+
+# TODO: Error classes.
+# TODO: Just warn if a config file can't be found.
+
+module DataMapper
+  class SphinxConfig
+
+    ##
+    # Read a sphinx configuration file.
+    #
+    # This class just gives you access to handy searchd {} configuration options. It does not validate your
+    # configuration file beyond basic syntax checking.
+    def initialize(uri_or_options = {})
+      @config = []
+
+      path = case uri_or_options
+        when Addressable::URI, DataObjects::URI then uri_or_options.path
+        when Hash                               then uri_or_options[:config] || uri_or_options[:database]
+        when Pathname                           then uri_or_options
+        when String                             then DataObjects::URI.parse(uri_or_options).path
+      end
+      parse('' + path.to_s) # Force stringy since Pathname#to_s is broken IMO.
+    end
+
+    ##
+    # Configuration file full path name.
+    #
+    # @return [String]
+    def config
+      @config
+    end
+
+    ##
+    # Indexer binary full path name and config argument.
+    def indexer_bin(use_config = true)
+      path = 'indexer' # TODO: Real.
+      path << " --config #{config}" if config
+      path
+    end
+
+    ##
+    # Searchd binardy full path name and config argument.
+    def searchd_bin(use_config = true)
+      path = 'searchd' # TODO: Real.
+      path << " --config #{config}" if config
+      path
+    end
+
+    ##
+    # Searchd address.
+    def address
+      searchd['address']
+    end
+
+    ##
+    # Searchd port.
+    def port
+      searchd['port']
+    end
+
+    ##
+    # Searchd pid_file.
+    def pid_file
+      searchd['pid_file'] or raise "Mandatory pid_file option missing from searchd configuration."
+    end
+
+    ##
+    # Searchd log file.
+    def log
+      searchd['log']
+    end
+
+    ##
+    # Searchd configuration options.
+    #
+    # Defaults will be applied but no validation is done.
+    #
+    # @see http://www.sphinxsearch.com/doc.html#confgroup-searchd
+    def searchd
+      unless @searchd
+        searchd = @blocks.find{|c| c['type'] =~ /searchd/i} || {}
+        @searchd = {
+          'address'         => '0.0.0.0',
+          'log'             => 'searchd.log',
+          'max_children'    => 0,
+          'max_matches'     => 1000,
+          'pid_file'        => nil,
+          'port'            => 3312,
+          'preopen_indexes' => 0,
+          'query_log'       => '',
+          'read_timeout'    => 5,
+          'seamless_rotate' => 1,
+          'unlink_old'      => 1
+        }.update(searchd)
+      end
+      @searchd
+    end
+
+    protected
+
+      ##
+      # Parse a sphinx config file.
+      #
+      # @param [String] path Searches path, ./path, /path, /usr/local/etc/sphinx.conf, ./sphinx.conf in that order.
+      def parse(path = '')
+        paths = [
+          path,
+          path.gsub(%r{^/}, './'),
+          path.gsub(%r{^\./}, '/'),
+          '/usr/local/etc/sphinx.conf', # TODO: Does this one depend on where searchd/indexer is installed?
+          './sphinx.conf'
+        ]
+        paths.find do |path|
+          @config = Pathname.new(path).expand_path
+          @config.readable? && `#{indexer_bin}` !~ /fatal|error/i
+        end or raise IOError, %{No readable config file (looked in #{paths.join(', ')})}
+
+        source = config.read
+        source.gsub!(/\r\n|\r|\n/, "\n") # Everything in \n
+        source.gsub!(/\s*\\\n\s*/, ' ')  # Remove unixy line wraps.
+        @in = StringScanner.new(source)
+        blocks(@blocks = [])
+      end
+
+    private
+
+      def blocks(out = []) #:nodoc:
+        if @in.scan(/\#[^\n]*\n/) || @in.scan(/\s+/)
+          blocks(out)
+        elsif @in.scan(/indexer|searchd|source|index/i)
+          out << group = {'type' => @in.matched}
+          if @in.matched =~ /^(?:index|source)$/i
+            @in.scan(/\s* ([\w_\-]+) (?:\s*:\s*([\w_\-]+))? \s*/x) or raise "Expected #{group[:type]} name."
+            group['name']     = @in[1]
+            group['ancestor'] = @in[2]
+          end
+          @in.scan(/\s*\{/) or raise %q{Expected '\{'.}
+          pairs(kv = {})
+          group.merge!(kv)
+          @in.scan(/\s*\}/) or raise %q{Expected '\}'.}
+          blocks(out)
+        else
+          raise "Unknown near: #{@in.peek(30)}" unless @in.eos?
+        end
+      end
+
+      def pairs(out = {}) #:nodoc:
+        if @in.scan(/\#[^\n]*\n/) || @in.scan(/\s+/)
+          pairs(out)
+        elsif @in.scan(/[\w_-]+/)
+          key = @in.matched
+          @in.scan(/\s*=/) or raise %q{Expected '='.}
+          out[key] = @in.scan(/[^\n]*\n/).strip
+          pairs(out)
+        end
+      end
+  end # SphinxConfig
+end # DataMapper

data/lib/dm-sphinx-adapter/sphinx_index.rb

@@ -0,0 +1,21 @@
+module DataMapper
+  class SphinxIndex
+    include Assertions
+
+    attr_reader :model, :name, :options
+
+    def initialize(model, name, options = {})
+      assert_kind_of 'model',   model,   Model
+      assert_kind_of 'name',    name,    Symbol, String
+      assert_kind_of 'options', options, Hash
+
+      @model = model
+      @name  = name.to_sym
+      @delta = options.fetch(:delta, nil)
+    end
+
+    def delta?
+      !!@delta
+    end
+  end # SphinxIndex
+end # DataMapper

data/lib/dm-sphinx-adapter/sphinx_resource.rb

@@ -0,0 +1,88 @@
+module DataMapper
+  ##
+  # Declare Sphinx indexes in your resource.
+  #
+  #   model Items
+  #     include Sphinx::Resource
+  #
+  #     # .. normal properties and such for :default
+  #
+  #     repository(:search) do
+  #       # Query some_index, some_index_delta in that order.
+  #       index :some_index
+  #       index :some_index_delta, :delta => true
+  #
+  #       # Sortable by some attributes.
+  #       attribute :updated_at, DateTime  # sql_attr_timestamp
+  #       attribute :age, Integer          # sql_attr_uint
+  #       attribute :deleted, Boolean      # sql_attr_bool
+  #     end
+  #   end
+  module SphinxResource
+    def self.included(model) #:nodoc:
+      model.class_eval do
+        include DataMapper::Resource
+        extend ClassMethods
+      end
+    end
+
+    module ClassMethods
+      def self.extended(model) #:nodoc:
+        model.instance_variable_set(:@sphinx_indexes, {})
+        model.instance_variable_set(:@sphinx_attributes, {})
+      end
+
+      ##
+      # Defines a sphinx index on the resource.
+      #
+      # Indexes are naturally ordered, with delta indexes at the end of the list so that duplicate document IDs in
+      # delta indexes override your main indexes.
+      #
+      # @param [Symbol] name the name of a sphinx index to search for this resource
+      # @param [Hash(Symbol => String)] options a hash of available options
+      # @see   DataMapper::SphinxIndex
+      def index(name, options = {})
+        index   = SphinxIndex.new(self, name, options)
+        indexes = sphinx_indexes(repository_name)
+        indexes << index
+
+        # TODO: I'm such a Ruby nub. In the meantime I've gone back to my Perl roots.
+        # This is a Schwartzian transform to sort delta indexes to the bottom and natural sort by name.
+        mapped  = indexes.map{|i| [(i.delta? ? 1 : 0), i.name, i]}
+        sorted  = mapped.sort{|a, b| a[0] <=> b[0] || a[1] <=> b[1]}
+        indexes.replace(sorted.map{|i| i[2]})
+
+        index
+      end
+
+      ##
+      # List of declared sphinx indexes for this model.
+      def sphinx_indexes(repository_name = default_repository_name)
+        @sphinx_indexes[repository_name] ||= []
+      end
+
+      ##
+      # Defines a sphinx attribute on the resource.
+      #
+      # @param [Symbol] name the name of a sphinx attribute to order/restrict by for this resource
+      # @param [Class] type the type to define this attribute as
+      # @param [Hash(Symbol => String)] options a hash of available options
+      # @see   DataMapper::SphinxAttribute
+      def attribute(name, type, options = {})
+        # Attributes are just properties without a getter/setter in the model.
+        # This keeps DataMapper::Query happy when building queries.
+        attribute = SphinxAttribute.new(self, name, type, options)
+        properties(repository_name)[attribute.name] = attribute
+        attribute
+      end
+
+      ##
+      # List of declared sphinx attributes for this model.
+      def sphinx_attributes(repository_name = default_repository_name)
+        properties(repository_name).grep{|p| p.kind_of? SphinxAttribute}
+      end
+
+    end # ClassMethods
+  end # SphinxResource
+end # DataMapper
+

data/lib/dm-sphinx-adapter.rb

@@ -0,0 +1,17 @@
+require 'rubygems'
+
+# TODO: Hide the shitload of dm-core warnings or at least try to?
+$VERBOSE = nil
+gem 'dm-core', '~> 0.9.7'
+require 'dm-core'
+
+# TODO: I think I might move everything to DataMapper::Sphinx::* and ignore the default naming convention.
+require 'pathname'
+dir = Pathname(__FILE__).dirname.expand_path / 'dm-sphinx-adapter'
+require dir / 'sphinx_config'
+require dir / 'sphinx_client'
+require dir / 'sphinx_adapter'
+require dir / 'sphinx_index'
+require dir / 'sphinx_attribute'
+require dir / 'sphinx_resource'
+

data/test/data/sphinx.conf

@@ -0,0 +1,73 @@
+# searchd and indexer must be run from the root directory of this lib.
+
+indexer
+{
+  mem_limit = 64M
+}
+
+searchd
+{
+  address      = localhost
+  port         = 3312
+  log          = test/data/sphinx.log
+  query_log    = test/data/sphinx.query.log
+  read_timeout = 5
+  max_children = 30
+  pid_file     = test/data/sphinx.pid
+  max_matches  = 1000
+}
+
+source items
+{
+  type     = mysql
+  sql_host = localhost
+  sql_user = root
+  sql_pass =
+  sql_db   = dm_sphinx_adapter_test
+
+  sql_query_pre = set names utf8
+  sql_query_pre = \
+    replace into delta (name, updated_on) ( \
+      select 'items', updated_on \
+      from items \
+      order by updated_on desc \
+      limit 1\
+    )
+
+  sql_query = \
+    select id, name, likes, unix_timestamp(updated_on) as updated_on \
+    from items \
+    where updated_on <= ( \
+      select updated_on \
+      from delta \
+      where name = 'items'\
+    )
+
+  sql_query_info = select * from items where id = $id
+  sql_attr_timestamp = updated_on
+}
+
+source items_delta : items
+{
+  sql_query_pre = set names utf8
+  sql_query = \
+    select id, name, likes, unix_timestamp(updated_on) as updated_on \
+    from items \
+    where updated_on > ( \
+      select updated_on \
+      from delta \
+      where name = 'items'\
+    )
+}
+
+index items
+{
+  source = items
+  path   = test/data/sphinx/items
+}
+
+index items_delta : items
+{
+  source = items_delta
+  path   = test/data/sphinx/items_delta
+}

data/test/fixtures/item.rb

@@ -0,0 +1,30 @@
+require 'rubygems'
+require 'dm-is-searchable'
+require 'zlib'
+
+class Item
+  include DataMapper::Resource
+  include DataMapper::SphinxResource
+
+  property :id, Integer, :key => true, :writer => :private
+  property :name, String, :nullable => false, :length => 50
+  property :likes, Text
+  property :updated_on, DateTime
+
+  is :searchable
+  repository(:search) do
+    index :items
+    index :items_delta, :delta => true
+
+    # TODO: More attributes.
+    attribute :updated, DateTime
+  end
+
+  # I'm using my own (unreleased) Digest::CRC32 DataMapper::Type normally.
+  after :name, :set_id
+
+  protected
+    def set_id
+      attribute_set(:id, Zlib.crc32(name))
+    end
+end

data/test/fixtures/item.sql

@@ -0,0 +1,23 @@
+drop table if exists delta;
+
+create table delta (
+  name varchar(50) not null,
+  updated_on datetime,
+  primary key (name)
+) engine=innodb default charset=utf8;
+
+drop table if exists items;
+
+create table items (
+  id int(11) not null,
+  name varchar(50) not null,
+  likes text not null,
+  updated_on datetime,
+  primary key (id),
+  index (updated_on)
+) engine=innodb default charset=utf8;
+
+insert into items (id, name, likes, updated_on) values
+  (CRC32('foo'), 'foo', 'I really like foo!', now()),
+  (CRC32('bar'), 'bar', 'I really like bar!', now()),
+  (CRC32('baz'), 'baz', 'I really like baz!', now());

data/test/helper.rb

@@ -0,0 +1,4 @@
+require 'dm-sphinx-adapter'
+require 'test/fixtures/item'
+require 'test/unit'
+

data/test/test_client.rb

@@ -0,0 +1,43 @@
+require 'helper'
+
+class TestClient < Test::Unit::TestCase
+  def setup
+    @config = Pathname.new(__FILE__).dirname.expand_path / 'data' / 'sphinx.conf'
+
+    # TODO: A little too brutal for me.
+    Dir.chdir(File.join(File.dirname(__FILE__), 'fixtures')) do
+      system 'mysql -u root dm_sphinx_adapter_test < item.sql' \
+        or raise %q{Tests require the dm_sphinx_adapter_test.items table.}
+    end
+  end
+
+  def test_initialize
+    assert_nothing_raised do
+      DataMapper::SphinxClient.new(@config)
+    end
+  end
+
+  def test_index
+    client = DataMapper::SphinxClient.new(@config)
+    assert_nothing_raised{ client.index }
+    assert_nothing_raised{ client.index 'items' }
+    assert_nothing_raised{ client.index '*' }
+    assert_nothing_raised{ client.index ['items', 'items_delta'] }
+  end
+
+  def test_managed_initialize
+    assert_nothing_raised do
+      DataMapper::SphinxManagedClient.new(@config)
+    end
+  end
+
+  def test_search
+    begin
+      client = DataMapper::SphinxManagedClient.new(@config)
+      client.index
+      assert client.search('foo')
+    ensure
+      client.stop
+    end
+  end
+end # TestClient

data/test/test_config.rb

@@ -0,0 +1,41 @@
+require 'helper'
+
+class TestConfig < Test::Unit::TestCase
+  def setup
+    data    = Pathname(__FILE__).dirname.expand_path / 'data'
+    @config = data / 'sphinx.conf'
+    @log    = data / 'sphinx.log'
+  end
+
+  def test_initialize
+    assert_nothing_raised{ config_new }
+    assert_raise(IOError){ config_new(:config => nil) }
+    assert_raise(IOError){ config_new(:config => 'blah') }
+    assert_raise(IOError){ config_new(:config => @log) }
+  end
+
+  def test_initalize_forms
+    assert_nothing_raised{ config_new(:database => @config) }
+    # TODO: DataObjects::URI treats /test as the hostname.
+    # assert_nothing_raised{ config_new('file://test/data/sphinx.conf') }
+    assert_nothing_raised{ config_new('sphinx://localhost/test/data/sphinx.conf') }
+  end
+
+  def test_config
+    assert_equal @config, config_new.config
+  end
+
+  def test_searchd
+    assert_kind_of Hash,                 config_new.searchd
+    assert_equal 'localhost',            config_new.address
+    assert_equal '3312',                 config_new.port
+    assert_equal 'test/data/sphinx.pid', config_new.pid_file
+    assert_equal 'test/data/sphinx.log', config_new.log
+  end
+
+  protected
+    def config_new(options = {:config => @config})
+      DataMapper::SphinxConfig.new(options)
+    end
+
+end # TestConfig

data/test/test_search.rb

@@ -0,0 +1,37 @@
+require 'helper'
+
+class TestSearch < Test::Unit::TestCase
+  def setup
+    # TODO: A little too brutal for me.
+    Dir.chdir(File.join(File.dirname(__FILE__), 'fixtures')) do
+      system 'mysql -u root dm_sphinx_adapter_test < item.sql' \
+        or raise %q{Tests require the dm_sphinx_adapter_test.items table.}
+    end
+
+    @config = Pathname.new(__FILE__).dirname.expand_path / 'data' / 'sphinx.conf'
+    DataMapper.setup(:default, 'mysql://localhost/dm_sphinx_adapter_test')
+    DataMapper.setup(:search,
+      :adapter => 'sphinx',
+      :config  => @config,
+      :managed => true
+    )
+  end
+
+  def teardown
+    DataMapper.repository(:search).adapter.client.stop
+    # You can also build a new client with the same config and call stop on that.
+    # client = DataMapper::SphinxManagedClient.new(@config)
+    # client.stop
+  end
+
+  def test_search
+    assert_nothing_raised{ Item.search }
+    assert_nothing_raised{ Item.search(:name => 'foo') }
+  end
+
+  def test_search_attributes
+    assert_nothing_raised do
+      Item.search(:updated => (Time.now - 10 .. Time.now + 10))
+    end
+  end
+end # TestSearch

metadata

@@ -0,0 +1,108 @@
+--- !ruby/object:Gem::Specification 
+name: dm-sphinx-adapter
+version: !ruby/object:Gem::Version 
+  version: "0.3"
+platform: ruby
+authors: 
+- Shane Hanna
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2008-11-18 00:00:00 +11:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: dm-core
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        version: 0.9.7
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: riddle
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        version: "0.9"
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: hoe
+  type: :development
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 1.8.2
+    version: 
+description: ""
+email: 
+- shane.hanna@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- History.txt
+- LICENCE.txt
+- Manifest.txt
+- README.txt
+files: 
+- History.txt
+- LICENCE.txt
+- Manifest.txt
+- README.txt
+- Rakefile
+- dm-sphinx-adapter.gemspec
+- lib/dm-sphinx-adapter.rb
+- lib/dm-sphinx-adapter/sphinx_adapter.rb
+- lib/dm-sphinx-adapter/sphinx_attribute.rb
+- lib/dm-sphinx-adapter/sphinx_client.rb
+- lib/dm-sphinx-adapter/sphinx_config.rb
+- lib/dm-sphinx-adapter/sphinx_index.rb
+- lib/dm-sphinx-adapter/sphinx_resource.rb
+- test/data/sphinx.conf
+- test/fixtures/item.rb
+- test/fixtures/item.sql
+- test/helper.rb
+- test/test_client.rb
+- test/test_config.rb
+- test/test_search.rb
+has_rdoc: true
+homepage: A Sphinx DataMapper adapter.
+post_install_message: 
+rdoc_options: 
+- --main
+- README.txt
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: dm-sphinx-adapter
+rubygems_version: 1.3.0
+signing_key: 
+specification_version: 2
+summary: ""
+test_files: 
+- test/test_client.rb
+- test/test_config.rb
+- test/test_search.rb