shanna-dm-sphinx-adapter 0.4

data/lib/dm-sphinx-adapter/config.rb

@@ -0,0 +1,74 @@
+require 'rubygems'
+
+gem 'extlib', '~> 0.9.7'
+require 'extlib'
+
+module DataMapper
+  module Adapters
+    module Sphinx
+      class Config
+        include Extlib::Assertions
+        attr_reader :config, :address, :log, :port
+
+        ##
+        # Read a sphinx configuration file.
+        #
+        # This class just gives you access to handy searchd {} configuration options.
+        #
+        # @see http://www.sphinxsearch.com/doc.html#confgroup-searchd
+        def initialize(uri_or_options = {})
+          assert_kind_of 'uri_or_options', uri_or_options, Addressable::URI, DataObjects::URI, Hash, String, Pathname
+
+          options   = normalize_options(uri_or_options)
+          config    = parse_config("#{options[:path]}") # Pathname#to_s is broken?
+          @address  = options[:host]     || config['address']  || '0.0.0.0'
+          @port     = options[:port]     || config['port']     || 3312
+          @log      = options[:log]      || config['log']      || 'searchd.log'
+          @pid_file = options[:pid_file] || config['pid_file']
+        end
+
+        ##
+        # Indexer binary full path name and config argument.
+        def indexer_bin(use_config = true)
+          path = 'indexer' # TODO: Real.
+          path << " --config #{config}" if config
+          path
+        end
+
+        ##
+        # Searchd binary full path name and config argument.
+        def searchd_bin(use_config = true)
+          path = 'searchd' # TODO: Real.
+          path << " --config #{config}" if config
+          path
+        end
+
+        def pid_file
+          @pid_file or raise "Mandatory pid_file option missing from searchd configuration."
+        end
+
+        protected
+          def normalize_options(uri_or_options)
+            case uri_or_options
+              when String, Addressable::URI then DataObjects::URI.parse(uri_or_options).attributes
+              when DataObjects::URI         then uri_or_options.attributes
+              when Pathname                 then {:path => uri_or_options}
+              else
+                uri_or_options[:path] ||= uri_or_options.delete(:config) || uri_or_options.delete(:database)
+                uri_or_options
+            end
+          end
+
+          def parse_config(path)
+            paths = []
+            paths.push(path, path.gsub(%r{^/}, './'), path.gsub(%r{^\./}, '/')) unless path.blank?
+            paths.push('/usr/local/etc/sphinx.conf', './sphinx.conf')
+            paths.map!{|path| Pathname.new(path).expand_path}
+
+            @config = paths.find{|path| path.readable? && `#{indexer_bin} --config #{path}` !~ /fatal|error/i}
+            @config ? ConfigParser.parse(@config) : {}
+          end
+      end # Config
+    end # Sphinx
+  end # Adapters
+end # DataMapper

data/lib/dm-sphinx-adapter/config_parser.rb

@@ -0,0 +1,67 @@
+require 'rubygems'
+
+gem 'extlib', '~> 0.9.7'
+require 'extlib'
+
+require 'pathname'
+require 'strscan'
+
+module DataMapper
+  module Adapters
+    module Sphinx
+      module ConfigParser
+        extend Extlib::Assertions
+
+        ##
+        # Parse a sphinx config file and return searchd options as a hash.
+        #
+        # @param  [String] path Searches path, ./#{path}, /#{path}, /usr/local/etc/sphinx.conf, ./sphinx.conf in
+        # that order.
+        # @return [Hash]
+        def self.parse(path)
+          assert_kind_of 'path', path, Pathname, String
+
+          config = Pathname(path).read
+          config.gsub!(/\r\n|\r|\n/, "\n") # Everything in \n
+          config.gsub!(/\s*\\\n\s*/, ' ')  # Remove unixy line wraps.
+
+          blocks(StringScanner.new(config), out = [])
+          out.find{|c| c['type'] =~ /searchd/i} || {}
+        end
+
+        protected
+          def self.blocks(conf, out = []) #:nodoc:
+            if conf.scan(/\#[^\n]*\n/) || conf.scan(/\s+/)
+              blocks(conf, out)
+            elsif conf.scan(/indexer|searchd|source|index/i)
+              out << group = {'type' => conf.matched}
+              if conf.matched =~ /^(?:index|source)$/i
+                conf.scan(/\s* ([\w_\-]+) (?:\s*:\s*([\w_\-]+))? \s*/x) or raise "Expected #{group[:type]} name."
+                group['name']     = conf[1]
+                group['ancestor'] = conf[2]
+              end
+              conf.scan(/\s*\{/) or raise %q{Expected '\{'.}
+              pairs(conf, kv = {})
+              group.merge!(kv)
+              conf.scan(/\s*\}/) or raise %q{Expected '\}'.}
+              blocks(conf, out)
+            else
+              raise "Unknown near: #{conf.peek(30)}" unless conf.eos?
+            end
+          end
+
+          def self.pairs(conf, out = {}) #:nodoc:
+            if conf.scan(/\#[^\n]*\n/) || conf.scan(/\s+/)
+              pairs(conf, out)
+            elsif conf.scan(/[\w_-]+/)
+              key = conf.matched
+              conf.scan(/\s*=/) or raise %q{Expected '='.}
+              out[key] = conf.scan(/[^\n]*\n/).strip
+              pairs(conf, out)
+            end
+          end
+      end # ConfigParser
+    end # Sphinx
+  end # Adapters
+end # DataMapper
+

data/lib/dm-sphinx-adapter/index.rb

@@ -0,0 +1,25 @@
+module DataMapper
+  module Adapters
+    module Sphinx
+      class Index
+        include Assertions
+
+        attr_reader :model, :name, :options
+
+        def initialize(model, name, options = {})
+          assert_kind_of 'model',   model,   Model
+          assert_kind_of 'name',    name,    Symbol, String
+          assert_kind_of 'options', options, Hash
+
+          @model = model
+          @name  = name.to_sym
+          @delta = options.fetch(:delta, nil)
+        end
+
+        def delta?
+          !!@delta
+        end
+      end # Index
+    end # Sphinx
+  end # Adapters
+end # DataMapper

data/lib/dm-sphinx-adapter/resource.rb

@@ -0,0 +1,96 @@
+module DataMapper
+  module Adapters
+    module Sphinx
+
+      ##
+      # Declare Sphinx indexes in your resource.
+      #
+      #   model Items
+      #     include DataMapper::SphinxResource
+      #
+      #     # .. normal properties and such for :default
+      #
+      #     repository(:search) do
+      #       # Query some_index, some_index_delta in that order.
+      #       index :some_index
+      #       index :some_index_delta, :delta => true
+      #
+      #       # Sortable by some attributes.
+      #       attribute :updated_at, DateTime  # sql_attr_timestamp
+      #       attribute :age, Integer          # sql_attr_uint
+      #       attribute :deleted, Boolean      # sql_attr_bool
+      #     end
+      #   end
+      module Resource
+        def self.included(model) #:nodoc:
+          model.class_eval do
+            include DataMapper::Resource
+            extend ClassMethods
+          end
+        end
+
+        module ClassMethods
+          def self.extended(model) #:nodoc:
+            model.instance_variable_set(:@sphinx_indexes, {})
+            model.instance_variable_set(:@sphinx_attributes, {})
+          end
+
+          ##
+          # Defines a sphinx index on the resource.
+          #
+          # Indexes are naturally ordered, with delta indexes at the end of the list so that duplicate document IDs in
+          # delta indexes override your main indexes.
+          #
+          # @param [Symbol] name the name of a sphinx index to search for this resource
+          # @param [Hash(Symbol => String)] options a hash of available options
+          # @see   DataMapper::Adapters::Sphinx::Index
+          def index(name, options = {})
+            index   = Index.new(self, name, options)
+            indexes = sphinx_indexes(repository_name)
+            indexes << index
+
+            # TODO: I'm such a Ruby nub. In the meantime I've gone back to my Perl roots.
+            # This is a Schwartzian transform to sort delta indexes to the bottom and natural sort by name.
+            mapped  = indexes.map{|i| [(i.delta? ? 1 : 0), i.name, i]}
+            sorted  = mapped.sort{|a, b| a[0] <=> b[0] || a[1] <=> b[1]}
+            indexes.replace(sorted.map{|i| i[2]})
+
+            index
+          end
+
+          ##
+          # List of declared sphinx indexes for this model.
+          def sphinx_indexes(repository_name = default_repository_name)
+            @sphinx_indexes[repository_name] ||= []
+          end
+
+          ##
+          # Defines a sphinx attribute on the resource.
+          #
+          # @param [Symbol] name the name of a sphinx attribute to order/restrict by for this resource
+          # @param [Class] type the type to define this attribute as
+          # @param [Hash(Symbol => String)] options a hash of available options
+          # @see   DataMapper::Adapters::Sphinx::Attribute
+          def attribute(name, type, options = {})
+            # Attributes are just properties without a getter/setter in the model.
+            # This keeps DataMapper::Query happy when building queries.
+            attribute = Sphinx::Attribute.new(self, name, type, options)
+            properties(repository_name)[attribute.name] = attribute
+            attribute
+          end
+
+          ##
+          # List of declared sphinx attributes for this model.
+          def sphinx_attributes(repository_name = default_repository_name)
+            properties(repository_name).grep{|p| p.kind_of? Sphinx::Attribute}
+          end
+
+        end # ClassMethods
+      end # Resource
+    end # Sphinx
+  end # Adapters
+
+  # Follow DM naming convention.
+  SphinxResource = Adapters::Sphinx::Resource
+end # DataMapper
+

data/lib/dm-sphinx-adapter.rb

@@ -0,0 +1,18 @@
+require 'rubygems'
+
+# TODO: Hide the shitload of dm-core warnings or at least try to?
+$VERBOSE = nil
+gem 'dm-core', '~> 0.9.7'
+require 'dm-core'
+
+# TODO: I think I might move everything to DataMapper::Sphinx::* and ignore the default naming convention.
+require 'pathname'
+dir = Pathname(__FILE__).dirname.expand_path / 'dm-sphinx-adapter'
+require dir / 'config'
+require dir / 'config_parser'
+require dir / 'client'
+require dir / 'adapter'
+require dir / 'index'
+require dir / 'attribute'
+require dir / 'resource'
+

data/test/files/dm_sphinx_adapter_test.sql

@@ -0,0 +1,21 @@
+drop table if exists delta;
+create table delta (
+  name varchar(50) not null,
+  updated_on datetime,
+  primary key (name)
+) engine=innodb default charset=utf8;
+
+drop table if exists items;
+create table items (
+  id int(11) not null auto_increment,
+  name varchar(50) not null,
+  likes text not null,
+  updated_on datetime,
+  primary key (id),
+  index (updated_on)
+) engine=innodb default charset=utf8;
+
+insert into items (name, likes, updated_on) values
+  ('one', 'I really like one!', now()),
+  ('two', 'I really like two!', now()),
+  ('three', 'I really like three!', now());

data/test/files/resource_explicit.rb

@@ -0,0 +1,25 @@
+require 'rubygems'
+require 'dm-is-searchable'
+
+class Explicit
+  include DataMapper::Resource
+  include DataMapper::SphinxResource
+
+  property :id,         Serial
+  property :name,       String
+  property :likes,      Text, :lazy => false
+  property :updated_on, DateTime
+
+  is :searchable
+  repository(:search) do
+    properties(:search).clear
+    index     :items
+    index     :items_delta, :delta => true
+    property  :name,    String
+    attribute :updated_on, DateTime
+  end
+
+  def self.default_storage_name
+    'item'
+  end
+end # Explicit

data/test/files/resource_resource.rb

@@ -0,0 +1,19 @@
+require 'rubygems'
+require 'dm-is-searchable'
+
+class Resource
+  include DataMapper::Resource
+  include DataMapper::SphinxResource
+
+  property :id,         Serial
+  property :name,       String
+  property :likes,      Text, :lazy => false
+  property :updated_on, DateTime
+
+  is :searchable
+
+  def self.default_storage_name
+    'item'
+  end
+end # Resource
+

data/test/files/resource_searchable.rb

@@ -0,0 +1,16 @@
+require 'rubygems'
+require 'dm-is-searchable'
+
+class Searchable
+  include DataMapper::Resource
+  property :id,         Serial
+  property :name,       String
+  property :likes,      Text, :lazy => false
+  property :updated_on, DateTime
+
+  is :searchable
+
+  def self.default_storage_name
+    'item'
+  end
+end # Searchable

data/test/files/resource_storage_name.rb

@@ -0,0 +1,11 @@
+class StorageName
+  include DataMapper::Resource
+  property :id,         Serial
+  property :name,       String
+  property :likes,      Text, :lazy => false
+  property :updated_on, DateTime
+
+  def self.default_storage_name
+    'item'
+  end
+end # StorageName

data/test/files/resource_vanilla.rb

@@ -0,0 +1,7 @@
+class Vanilla
+  include DataMapper::Resource
+  property :id,         Serial
+  property :name,       String
+  property :likes,      Text, :lazy => false
+  property :updated_on, DateTime
+end # Vanilla

data/test/files/sphinx.conf

@@ -0,0 +1,78 @@
+# searchd and indexer must be run from the root directory of this lib.
+
+indexer
+{
+  mem_limit = 64M
+}
+
+searchd
+{
+  address      = localhost
+  port         = 3312
+  log          = test/var/sphinx.log
+  query_log    = test/var/sphinx.query.log
+  read_timeout = 5
+  pid_file     = test/var/sphinx.pid
+  max_matches  = 1000
+}
+
+source items
+{
+  type     = mysql
+  sql_host = localhost
+  sql_user = root
+  sql_pass =
+  sql_db   = dm_sphinx_adapter_test
+
+  sql_query_pre = set names utf8
+  sql_query_pre = \
+    replace into delta (name, updated_on) ( \
+      select 'items', updated_on \
+      from items \
+      order by updated_on desc \
+      limit 1\
+    )
+
+  sql_query = \
+    select id, name, likes, unix_timestamp(updated_on) as updated_on \
+    from items \
+    where updated_on <= ( \
+      select updated_on \
+      from delta \
+      where name = 'items'\
+    )
+
+  sql_query_info = select * from items where id = $id
+  sql_attr_timestamp = updated_on
+}
+
+source items_delta : items
+{
+  sql_query_pre = set names utf8
+  sql_query = \
+    select id, name, likes, unix_timestamp(updated_on) as updated_on \
+    from items \
+    where updated_on > ( \
+      select updated_on \
+      from delta \
+      where name = 'items'\
+    )
+}
+
+index items
+{
+  source = items
+  path   = test/var/sphinx/items
+}
+
+index items_delta : items
+{
+  source = items_delta
+  path   = test/var/sphinx/items_delta
+}
+
+index vanillas
+{
+  type  = distributed
+  local = items
+}

data/test/test_adapter.rb

@@ -0,0 +1,38 @@
+require 'dm-sphinx-adapter'
+require 'test/unit'
+
+# DataMapper::Logger.new(STDOUT, :debug)
+
+class TestAdapter < Test::Unit::TestCase
+  def setup
+    # TODO: A little too brutal even by my standards.
+    Dir.chdir(File.join(File.dirname(__FILE__), 'files')) do
+      system 'mysql -u root dm_sphinx_adapter_test < dm_sphinx_adapter_test.sql' \
+        or raise %q{Tests require the dm_sphinx_adapter_test database.}
+    end
+
+    DataMapper.setup(:default, 'mysql://localhost/dm_sphinx_adapter_test')
+
+    @config = Pathname.new(__FILE__).dirname.expand_path / 'files' / 'sphinx.conf'
+    @client = DataMapper::Adapters::Sphinx::ManagedClient.new(:config => @config)
+    @client.index
+    sleep 1
+  end
+
+  def test_unmanaged_setup
+    assert DataMapper.setup(:sphinx, :adapter => 'sphinx')
+    assert_kind_of DataMapper::Adapters::SphinxAdapter, repository(:sphinx).adapter
+    assert_kind_of DataMapper::Adapters::Sphinx::Client, repository(:sphinx).adapter.client
+  end
+
+  def test_managed_setup
+    assert DataMapper.setup(:sphinx, :adapter => 'sphinx', :config => @config, :managed => true)
+    assert_kind_of DataMapper::Adapters::SphinxAdapter, repository(:sphinx).adapter
+    assert_kind_of DataMapper::Adapters::Sphinx::ManagedClient, repository(:sphinx).adapter.client
+  end
+
+  def teardown
+    @client.stop
+    sleep 1
+  end
+end

data/test/test_adapter_explicit.rb

@@ -0,0 +1,48 @@
+require 'test_adapter'
+require 'files/resource_explicit'
+
+class TestAdapterExplicit < TestAdapter
+  def setup
+    super
+    DataMapper.setup(:search, :adapter => 'sphinx', :config => @config, :managed => true)
+  end
+
+  def teardown
+    DataMapper.repository(:search).adapter.client.stop
+    super
+  end
+
+  def test_initialize
+    assert_nothing_raised{ Explicit.new }
+  end
+
+  def test_search_properties
+    assert_equal Explicit.all, Explicit.search
+    assert_equal [Explicit.first(:id => 2)], Explicit.search(:name => 'two')
+  end
+
+  def test_search_delta
+    resource = Explicit.create(:name => 'four', :likes => 'chicken', :updated_on => Time.now)
+    DataMapper.repository(:search).adapter.client.index('items_delta')
+    assert_equal [resource], Explicit.search(:name => 'four')
+  end
+
+  def test_search_attribute_timestamp
+    time     = Time.now
+    resource = Explicit.create(:name => 'four', :likes => 'chicken', :updated_on => time)
+    DataMapper.repository(:search).adapter.client.index('items_delta')
+
+    assert_equal [resource], Explicit.search(:updated_on => time.to_i)
+    assert_equal [resource], Explicit.search(:updated_on => (time .. time + 1))
+    assert_equal [], Explicit.search(:updated_on => (time - 60 * 60))
+    assert_equal [], Explicit.search(:updated_on => (time + 60 * 60))
+  end
+
+  def test_search_attribute_boolean
+    # TODO:
+  end
+
+  def test_search_attribute_integer
+    # TODO
+  end
+end # TestAdapterExplicit

data/test/test_adapter_resource.rb

@@ -0,0 +1,25 @@
+require 'test_adapter'
+require 'files/resource_resource'
+
+class TestAdapterResource < TestAdapter
+  def setup
+    super
+    DataMapper.setup(:search, :adapter => 'sphinx', :config => @config, :managed => true)
+  end
+
+  def teardown
+    DataMapper.repository(:search).adapter.client.stop
+    super
+  end
+
+  def test_initialize
+    assert_nothing_raised{ Resource.new }
+  end
+
+  def test_search_properties
+    assert_equal Resource.all, Resource.search
+    assert_equal [Resource.first(:id => 2)], Resource.search(:name => 'two')
+    assert_equal [Resource.first(:id => 2)], Resource.search(:conditions => ['two'])
+  end
+end # TestAdapterResource
+

data/test/test_adapter_searchable.rb

@@ -0,0 +1,23 @@
+require 'test_adapter'
+require 'files/resource_searchable'
+
+class TestAdapterSearchable < TestAdapter
+  def setup
+    super
+    DataMapper.setup(:search, :adapter => 'sphinx', :config => @config, :managed => true)
+  end
+
+  def teardown
+    DataMapper.repository(:search).adapter.client.stop
+    super
+  end
+
+  def test_initialize
+    assert_nothing_raised{ Searchable.new }
+  end
+
+  def test_search
+    assert_equal Searchable.all, Searchable.search
+    assert_equal [Searchable.first(:id => 2)], Searchable.search(:name => 'two')
+  end
+end # TestAdapterSearchable

data/test/test_adapter_vanilla.rb

@@ -0,0 +1,46 @@
+require 'test_adapter'
+require 'files/resource_vanilla'
+require 'files/resource_storage_name'
+
+class TestAdapterVanilla < TestAdapter
+  def setup
+    super
+    DataMapper.setup(:default, :adapter => 'sphinx', :config => @config, :managed => true)
+  end
+
+  def teardown
+    DataMapper.repository(:default).adapter.client.stop
+    super
+  end
+
+  def test_initialize
+    assert_nothing_raised{ Vanilla.new }
+    assert_nothing_raised{ StorageName.new }
+  end
+
+  def test_all
+    assert_equal [{:id => 1}, {:id => 2}, {:id => 3}], Vanilla.all
+    assert_equal [{:id => 1}], Vanilla.all(:name => 'one')
+  end
+
+  def test_all_limit
+    assert_equal [{:id => 1}], Vanilla.all(:limit => 1)
+    assert_equal [{:id => 1}, {:id => 2}], Vanilla.all(:limit => 2)
+  end
+
+  def test_all_offset
+    assert_equal [{:id => 1}, {:id => 2}, {:id => 3}], Vanilla.all(:offset => 0)
+    assert_equal [{:id => 2}, {:id => 3}], Vanilla.all(:offset => 1)
+    assert_equal [], Vanilla.all(:offset => 3)
+  end
+
+  def test_first
+    assert_equal({:id => 1}, Vanilla.first(:name => 'one'))
+    assert_nil Vanilla.first(:name => 'missing')
+  end
+
+  def test_storage_name
+    assert_equal Vanilla.all, StorageName.all
+    assert_equal Vanilla.first, StorageName.first
+  end
+end

data/test/test_client.rb

@@ -0,0 +1,31 @@
+require 'test_adapter'
+
+class TestClient < TestAdapter
+  def test_initialize
+    assert_nothing_raised { DataMapper::Adapters::Sphinx::Client.new(@config) }
+  end
+
+  def test_index
+    client = DataMapper::Adapters::Sphinx::Client.new(@config)
+    assert_nothing_raised{ client.index }
+    assert_nothing_raised{ client.index 'items' }
+    assert_nothing_raised{ client.index '*' }
+    assert_nothing_raised{ client.index ['items', 'items_delta'] }
+  end
+
+  def test_managed_initialize
+    assert_nothing_raised { DataMapper::Adapters::Sphinx::ManagedClient.new(@config) }
+  end
+
+  def test_search
+    begin
+      client = DataMapper::Adapters::Sphinx::ManagedClient.new(@config)
+      client.index
+      assert match = client.search('two')
+      assert_equal 1, match[:total]
+      assert_equal 2, match[:matches][0][:doc]
+    ensure
+      client.stop
+    end
+  end
+end # TestClient