dm-ferret-adapter 0.9.7

data/.gitignore

@@ -0,0 +1 @@
+index

data/History.txt

@@ -0,0 +1 @@
+

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2008 Bernerd Schaefer
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Manifest.txt

@@ -0,0 +1,16 @@
+.gitignore
+History.txt
+LICENSE
+Manifest.txt
+README.txt
+Rakefile
+TODO
+bin/ferret
+lib/ferret_adapter.rb
+lib/ferret_adapter/local_index.rb
+lib/ferret_adapter/remote_index.rb
+lib/ferret_adapter/repository_ext.rb
+lib/ferret_adapter/version.rb
+spec/adapter_spec.rb
+spec/helper.rb
+spec/spec.opts

data/README.txt

@@ -0,0 +1,62 @@
+This is a DataMapper plugin for Ferret.
+
+= Setup code
+
+For a single process site, use the ferret index directly:
+
+  DataMapper.setup :search, "ferret:///path/to/index"
+
+For a multi-process site, use the distributed index by running `ferret start`
+inside your project's directory and then setting up the :search repository:
+
+  DataMapper.setup :search, "ferret:///tmp/ferret_index.sock"
+
+= Sample Code
+
+require "rubygems"
+require "dm-core"
+require "dm-is-searchable"
+
+DataMapper.setup(:default, "sqlite3::memory:")
+DataMapper.setup(:search, "ferret://#{Pathname(__FILE__).dirname.expand_path.parent + "index"}")
+
+class Image
+  include DataMapper::Resource
+  property :id, Serial
+  property :title, String
+
+  is :searchable # this defaults to :search repository, you could also do
+  # is :searchable, :repository => :ferret
+
+end
+
+class Story
+  include DataMapper::Resource
+  property :id, Serial
+  property :title, String
+  property :author, String
+
+  repository(:search) do
+    # We only want to search on id and title.
+    properties(:search).clear
+    property :id, Serial
+    property :title, String
+  end
+
+  is :searchable
+
+end
+
+Image.auto_migrate!
+Story.auto_migrate!
+image = Image.create(:title => "Oil Rig");
+story = Story.create(:title => "Big Oil", :author => "John Doe") }
+
+puts Image.search(:title => "Oil Rig").inspect # => [<Image title="Oil Rig">]
+
+# For info on this, see DM::Repository#search and DM::Adapters::FerretAdapter#search.
+puts repository(:search).search('title:"Oil"').inspect # => { Image => ["1"], Story => ["1"] }
+
+image.destroy
+
+puts Image.search(:title => "Oil Rig").inspect # => []

data/Rakefile

@@ -0,0 +1,53 @@
+require 'rubygems'
+require 'spec'
+require 'spec/rake/spectask'
+require 'pathname'
+
+ROOT = Pathname(__FILE__).dirname.expand_path
+require ROOT + 'lib/ferret_adapter/version'
+
+AUTHOR = "Bernerd Schaefer"
+EMAIL  = "bernerd@wieck.com"
+GEM_NAME = "dm-ferret-adapter"
+GEM_VERSION = DataMapper::More::FerretAdapter::VERSION
+GEM_DEPENDENCIES = [["dm-core", ">=#{GEM_VERSION}"]]
+GEM_CLEAN = ["log", "pkg"]
+GEM_EXTRAS = { :has_rdoc => true, :extra_rdoc_files => %w[ README.txt LICENSE TODO ] }
+
+PROJECT_NAME = "datamapper"
+PROJECT_URL  = "http://github.com/sam/dm-more/tree/master/adapters/dm-ferret-adapter"
+PROJECT_DESCRIPTION = PROJECT_SUMMARY = "Ferret Adapter for DataMapper"
+
+require ROOT.parent.parent + 'tasks/hoe'
+
+task :default => [ :spec ]
+
+WIN32 = (RUBY_PLATFORM =~ /win32|mingw|cygwin/) rescue nil
+SUDO  = WIN32 ? '' : ('sudo' unless ENV['SUDOLESS'])
+
+desc "Install #{GEM_NAME} #{GEM_VERSION}"
+task :install => [ :package ] do
+  sh "#{SUDO} gem install --local pkg/#{GEM_NAME}-#{GEM_VERSION} --no-update-sources", :verbose => false
+end
+
+desc "Uninstall #{GEM_NAME} #{GEM_VERSION} (default ruby)"
+task :uninstall => [ :clobber ] do
+  sh "#{SUDO} gem uninstall #{GEM_NAME} -v#{GEM_VERSION} -I -x", :verbose => false
+end
+
+desc 'Run specifications'
+Spec::Rake::SpecTask.new(:spec) do |t|
+  if File.exists?('spec/spec.opts')
+    t.spec_opts << '--options' << 'spec/spec.opts'
+  end
+  t.spec_files = Pathname.glob((ROOT + 'spec/**/*_spec.rb').to_s)
+
+  begin
+    t.rcov = ENV.has_key?('NO_RCOV') ? ENV['NO_RCOV'] != 'true' : true
+    t.rcov_opts << '--exclude' << 'spec'
+    t.rcov_opts << '--text-summary'
+    t.rcov_opts << '--sort' << 'coverage' << '--sort-reverse'
+  rescue Exception
+    # rcov not installed
+  end
+end

data/TODO

@@ -0,0 +1,8 @@
+TODO
+====
+
+
+
+---
+TODO tickets may also be found in the DataMapper Issue Tracker:
+http://wm.lighthouseapp.com/projects/4819-datamapper/overview

data/bin/ferret

@@ -0,0 +1,115 @@
+#!/usr/bin/env ruby
+require "rubygems"
+require "dm-core"
+require "ferret_adapter"
+
+require "fileutils"
+require "drb"
+require "drb/unix"
+require "rinda/tuplespace"
+require "optparse"
+
+options = {
+  :name => "ferret_index",
+  :index => Dir.pwd + "/index",
+  :pid_file => "tmp/ferret.pid",
+  :log_file => "log/ferret.log",
+  :log_level => :error
+}
+
+option_parser = OptionParser.new do |opts|
+  opts.banner = "Usage: ferret.rb [options] start|stop"
+  opts.on("-n", "--name NAME", "The name to use for the Rinda Ring service. Defaults to 'ferret_index'.") { |name| options[:name] = name }
+  opts.on("-P", "--pid PIDFILE", "PID file, defaults to tmp/ferret.pid") { |pid| options[:pid_file] = pid }
+  opts.on("-L", "--log LOGFILE", "Log file, defaults to log/ferret.log") { |log| options[:log_file] = log }
+  opts.on("-l", "--log-level LEVEL", [:debug, :error], "Log levels can be: debug, error. Default is error.") { |level| options[:log_level] = level }
+  opts.on("-i", "--index INDEX", "Index path. Defaults to Dir.pwd + '/index'") { |index| options[:index] = index }
+end
+
+unless %w(start stop).include? ARGV.last
+  puts option_parser.help
+  exit
+end
+
+option_parser.parse!
+
+command = ARGV.shift
+
+if command == "stop"
+  Process.kill("INT", File.read(options[:pid_file]).to_i) if File.exists?(options[:pid_file])
+  exit
+else
+  fork do
+    # Promote this process.
+    Process.setsid
+
+    FileUtils.mkdir_p(Pathname(options[:pid_file]).dirname)
+    FileUtils.mkdir_p(Pathname(options[:log_file]).dirname)
+
+    # We redirect STDOUT to the :log_file only in debug mode.
+    if options[:log_level] == :debug
+      STDOUT.reopen options[:log_file], "a"
+    else
+      STDOUT.reopen "/dev/null", "a"
+    end
+    STDERR.reopen options[:log_file], "a"
+
+    tuple_space = Rinda::TupleSpace.new
+
+    DRb.start_service "drbunix:///tmp/#{options[:name]}.sock", tuple_space
+
+    STDOUT.puts "Server started at #{DRb.uri}"
+    STDOUT.flush
+
+    # Write the process id to the specified :pid_file
+    File.open(options[:pid_file], "w") { |f| f.write(Process.pid) }
+
+    # Cleanup the pid.
+    at_exit do
+      File.unlink(options[:pid_file]) if options[:pid_file]
+    end
+
+    uri = Addressable::URI.parse(options[:index])
+    @index = DataMapper::Adapters::FerretAdapter::LocalIndex.new(uri)
+
+    loop do
+      begin
+        command, uri, value = tuple_space.take([nil, nil, nil])
+        case command
+        when :search
+          puts "Search"
+          puts " - #{value[0].inspect}"
+          puts " - #{value[1].inspect}"
+          STDOUT.flush
+          begin
+            result = @index.search(*value)
+            tuple_space.write [:search_result, uri, value, result]
+          rescue
+            tuple_space.write [:search_result, uri, value, nil]
+            raise $!
+          end
+        when :add
+          puts "Insert"
+          puts " - #{value.inspect}"
+          STDOUT.flush
+          @index.add value
+        when :delete
+          puts "Delete"
+          puts " - #{value.inspect}"
+          STDOUT.flush
+          @index.delete value
+        end
+      rescue Interrupt
+        STDOUT.puts "Shutting down server"
+        STDOUT.flush
+        break
+      rescue
+        STDERR.puts "=== #{Time.now} ==="
+        STDERR.puts $!
+        STDERR.puts $!.backtrace
+        STDERR.puts ""
+        STDERR.flush
+      end
+    end
+  end
+end

data/lib/ferret_adapter/local_index.rb

@@ -0,0 +1,43 @@
+module DataMapper
+  module Adapters
+    class FerretAdapter::LocalIndex
+
+      attr_accessor :uri
+
+      def initialize(uri)
+        @uri = uri
+        @options = { :path => @uri.path, :key => [:id, :_type] }
+        create_or_initialize_index
+      end
+
+      def add(doc)
+        @index << doc
+      end
+
+      def delete(query)
+        @index.query_delete(query)
+      end
+
+      def search(query, options = {})
+        @index.search(query, options).hits.collect { |hit, score| @index[hit.doc] }
+      end
+
+      def [](id)
+        @index[id]
+      end
+
+      private
+
+      def create_or_initialize_index
+        unless File.exists?(@uri.path + "segments")
+          field_infos = ::Ferret::Index::FieldInfos.new(:store => :no)
+          field_infos.add_field(:id, :index => :untokenized, :term_vector => :no, :store => :yes)
+          field_infos.add_field(:_type, :index => :untokenized, :term_vector => :no, :store => :yes)
+          @index = ::Ferret::Index::Index.new( @options.merge(:field_infos => field_infos) )
+        else
+          @index = ::Ferret::Index::Index.new( @options )
+        end
+      end
+    end
+  end
+end

data/lib/ferret_adapter/remote_index.rb

@@ -0,0 +1,55 @@
+module DataMapper
+  module Adapters
+    class FerretAdapter::RemoteIndex
+
+      class IndexNotFound < Exception; end
+      class SearchError < Exception; end
+
+      attr_accessor :uri
+
+      def initialize(uri)
+        @uri = uri
+
+        connect_to_remote_index
+      end
+
+      def add(doc)
+        @index.write [:add, DRb.uri, doc]
+      end
+
+      def delete(query)
+        @index.write [:delete, DRb.uri, query]
+      end
+
+      def search(query, options)
+        tuple = [query, options]
+        @index.write [:search, DRb.uri, tuple]
+        result = @index.take([:search_result, DRb.uri, tuple, nil]).last
+        if result == nil
+          raise SearchError.new("An error occurred performing this search. Check the Ferret logs.")
+        end
+        result
+      end
+
+      private
+
+      def connect_to_remote_index
+        require "drb"
+        require "drb/unix"
+        require "rinda/tuplespace"
+
+        DRb.start_service
+        tuple_space = DRb::DRbObject.new(nil, "drbunix://#{@uri.path}")
+
+        # This will throw Errno::ENOENT if the socket does not exist.
+        tuple_space.respond_to?(:write)
+
+        @index = Rinda::TupleSpaceProxy.new(tuple_space)
+
+      rescue Errno::ENOENT
+        raise IndexNotFound.new("Your remote index server is not running.")
+      end
+
+    end
+  end
+end

data/lib/ferret_adapter/repository_ext.rb

@@ -0,0 +1,13 @@
+module DataMapper
+  class Repository
+    # This accepts a ferret query string and an optional limit argument
+    # which defaults to all. This is the proper way to perform searches more
+    # complicated than DM's query syntax can handle (such as OR searches).
+    #
+    # See DataMapper::Adapters::FerretAdapter#search for information on
+    # the return value.
+    def search(query, limit = :all)
+      adapter.search(query, limit)
+    end
+  end
+end

data/lib/ferret_adapter/version.rb

@@ -0,0 +1,7 @@
+module DataMapper
+  module More
+    class FerretAdapter
+      VERSION = "0.9.7"
+    end
+  end
+end

data/lib/ferret_adapter.rb

@@ -0,0 +1,111 @@
+require 'rubygems'
+require 'pathname'
+require Pathname(__FILE__).dirname + 'ferret_adapter/version'
+
+gem 'dm-core', ">=#{DataMapper::More::FerretAdapter::VERSION}"
+require 'dm-core'
+
+gem "ferret"
+require "ferret"
+
+module DataMapper
+  module Adapters
+    class FerretAdapter < AbstractAdapter
+      def initialize(name, uri_or_options)
+        super
+        unless File.extname(@uri.path) == ".sock"
+          @index = LocalIndex.new(@uri)
+        else
+          @index = RemoteIndex.new(@uri)
+        end
+      end
+
+      def create(resources)
+        resources.each do |resource|
+          attributes = repository(self.name) do
+            attrs = resource.attributes
+            attrs.delete_if { |name, value| !resource.class.properties(self.name).has_property?(name) }
+            resource.class.new(attrs).attributes
+          end
+
+          # Since we don't inspect the models before generating the indices,
+          # we'll map the resource's key to the :id column.
+          key = resource.class.key.first
+          attributes[:id] = attributes.delete(key.name) unless key.name == :id
+          attributes[:_type] = resource.class.name
+
+          @index.add attributes
+        end
+        1
+      end
+
+      def delete(query)
+        ferret_query = dm_query_to_ferret_query(query)
+        @index.delete ferret_query
+        1
+      end
+
+      # This returns an array of Ferret docs (glorified hashes) which can
+      # be used to instantiate objects by doc[:_type] and doc[:_id]
+      def read_many(query, limit = query.limit)
+        ferret_query = dm_query_to_ferret_query(query)
+        @index.search(ferret_query, :limit => (limit || :all))
+      end
+
+      def read_one(query)
+        read_many(query).first
+      end
+
+      # This returns a hash of the resource constant and the ids returned for it
+      # from the search.
+      #   { Story => ["1", "2"], Image => ["2"] }
+      def search(query, limit)
+        results = Hash.new { |h, k| h[k] = [] }
+        read_many(query, limit).each do |doc|
+          results[Object.const_get(doc[:_type])] << doc[:id]
+        end
+        results
+      end
+
+      private
+
+      def dm_query_to_ferret_query(query)
+        # If we already have a ferret query, do nothing
+        return query if query.is_a?(String)
+
+        ferret = []
+
+        # We scope the query by the _type field to the query's model.
+        ferret << "+_type:\"#{query.model.name}\""
+
+        if query.conditions.empty?
+          ferret << "*"
+        else
+          query.conditions.each do |operator, property, value|
+            # We use property.field here, so that you can declare composite
+            # fields:
+            #     property :content, String, :field => "title|description"
+            name = property.field
+
+            # Since DM's query syntax does not support OR's, we prefix
+            # each condition with ferret's operator of +.
+            ferret << case operator
+            when :eql, :like  then "+#{name}:\"#{value}\""
+            when :not         then "-#{name}:\"#{value}\""
+            when :lt          then "+#{name}: < #{value}"
+            when :gt          then "+#{name}: > #{value}"
+            when :lte         then "+#{name}: <= #{value}"
+            when :gte         then "+#{name}: >= #{value}"
+            end
+          end
+        end
+        ferret.join(" ")
+      end
+
+    end
+  end
+end
+
+require Pathname(__FILE__).dirname + "ferret_adapter/local_index"
+require Pathname(__FILE__).dirname + "ferret_adapter/remote_index"
+require Pathname(__FILE__).dirname + "ferret_adapter/repository_ext"

data/spec/adapter_spec.rb

@@ -0,0 +1,35 @@
+require "pathname"
+require Pathname(__FILE__).dirname + "helper"
+
+class User
+  include DataMapper::Resource
+  property :id, Serial
+end
+
+class Photo
+  include DataMapper::Resource
+  property :uuid, String, :default => lambda { `uuidgen`.chomp }, :key => true
+end
+
+describe "FerretAdapter" do
+  before :each do
+    @index = Pathname(__FILE__).dirname.expand_path + "index"
+    DataMapper.setup :search, "ferret://#{@index}"
+  end
+
+  after :each do
+    FileUtils.rm_r(@index)
+  end
+
+  it "should work with a model using id" do
+    u = User.new(:id => 2)
+    repository(:search).create([u])
+    repository(:search).search("*").should == { User => ["2"] }
+  end
+
+  it "should work with a model using another key than id" do
+    p = Photo.new
+    repository(:search).create([p])
+    repository(:search).search("*").should == { Photo => [p.uuid] }
+  end
+end

data/spec/helper.rb

@@ -0,0 +1,4 @@
+require "pathname"
+require Pathname(__FILE__).dirname.parent + "lib/ferret_adapter"
+
+require "spec"

data/spec/spec.opts

@@ -0,0 +1,2 @@
+--format specdoc
+--colour

metadata

@@ -0,0 +1,91 @@
+--- !ruby/object:Gem::Specification 
+name: dm-ferret-adapter
+version: !ruby/object:Gem::Version 
+  version: 0.9.7
+platform: ruby
+authors: 
+- Bernerd Schaefer
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2008-11-18 00:00:00 -08:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: dm-core
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 0.9.7
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: hoe
+  type: :development
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 1.8.2
+    version: 
+description: Ferret Adapter for DataMapper
+email: 
+- bernerd@wieck.com
+executables: 
+- ferret
+extensions: []
+
+extra_rdoc_files: 
+- README.txt
+- LICENSE
+- TODO
+files: 
+- .gitignore
+- History.txt
+- LICENSE
+- Manifest.txt
+- README.txt
+- Rakefile
+- TODO
+- bin/ferret
+- lib/ferret_adapter.rb
+- lib/ferret_adapter/local_index.rb
+- lib/ferret_adapter/remote_index.rb
+- lib/ferret_adapter/repository_ext.rb
+- lib/ferret_adapter/version.rb
+- spec/adapter_spec.rb
+- spec/helper.rb
+- spec/spec.opts
+has_rdoc: true
+homepage: http://github.com/sam/dm-more/tree/master/adapters/dm-ferret-adapter
+post_install_message: 
+rdoc_options: 
+- --main
+- README.txt
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: datamapper
+rubygems_version: 1.3.1
+signing_key: 
+specification_version: 2
+summary: Ferret Adapter for DataMapper
+test_files: []
+