dm-groonga-adapter 0.1.0.pre

data/.document

@@ -0,0 +1,5 @@
+README.rdoc
+lib/**/*.rb
+bin/*
+features/**/*.feature
+LICENSE

data/.gitignore

@@ -0,0 +1,26 @@
+## Github Page
+_site
+
+## MAC OS
+.DS_Store
+
+## TEXTMATE
+*.tmproj
+tmtags
+
+## EMACS
+*~
+\#*
+.\#*
+
+## VIM
+*.swp
+
+## PROJECT::GENERAL
+coverage
+rdoc
+pkg
+
+## PROJECT::SPECIFIC
+#
+spec/test

data/LICENSE

@@ -0,0 +1,14 @@
+Copyright (c) 2009 hiroyuki
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the Gnu Lesser General Public License as 
+published by the Free Software Foundation, either version 2.1 of the
+License, or any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+Gnu Lesser General Public License for more details.
+
+You should have received a copy of the Gnu Lesser General Public
+License along with this program. If not, see <http://www.gnu.org/licenses/>.

data/README.rdoc

@@ -0,0 +1,43 @@
+= dm-groonga-adapter
+
+dm-groonga-adapter provides is-search adapter for groonga (http://groonga.org).
+With dm-is-search, you can use groonga as search repository. Currently, This 
+module supports groonga 0.1.7 or later.
+
+== Install
+
+gem install dm-groonga-adapter
+
+== Dependencies
+
+* groonga (ruby binding) >= 0.9.1
+* dm-core >= 0.10.0
+* dm-more >= 0.10.0
+
+== Setup Repository
+
+For a single process site, use groonga dataase files directory.
+
+  DataMapper.setup :search, "groonga:///path/to/database"
+
+For a multi-process site, use url for a groonga server process.
+
+  DataMapper.setup :search, "groonga://127.0.0.1:10041"
+
+== Sample Code
+
+See examples/base.rb and spec/shared/search_spec.rb
+
+== Note on Patches/Pull Requests
+ 
+* Fork the project.
+* Make your feature addition or bug fix.
+* Add tests for it. This is important so I don't break it in a
+  future version unintentionally.
+* Commit, do not mess with rakefile, version, or history.
+  (if you want to have your own version, that is fine but bump version in a commit by itself I can ignore when I pull)
+* Send me a pull request. Bonus points for topic branches.
+
+== Copyright
+
+Copyright (c) 2010 hiroyuki. See LICENSE for details.

data/Rakefile

@@ -0,0 +1,52 @@
+require 'rubygems'
+require 'rake'
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gem|
+    gem.name = "dm-groonga-adapter"
+    gem.summary = %Q{datamapper adapter for groonga search engine}
+    gem.description = gem.summary
+    gem.email = "hello@hryk.info"
+    gem.homepage = "http://github.com/hryk/dm-groonga-adapter"
+    gem.authors = ["hiroyuki"]
+
+    gem.add_dependency "groonga", ">= 0.9"
+    gem.add_dependency "dm-core", "~> 0.10.2"
+    gem.add_dependency "dm-more", "~> 0.10.2"
+
+    gem.add_development_dependency "rspec", ">= 1.2.9"
+    gem.add_development_dependency "rcov", ">= 0"
+    # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
+  end
+  Jeweler::GemcutterTasks.new
+rescue LoadError
+  puts "Jeweler (or a dependency) not available. Install it with: gem install jeweler"
+end
+
+require 'spec/rake/spectask'
+Spec::Rake::SpecTask.new(:spec) do |spec|
+  spec.libs << 'lib' << 'spec'
+  spec.spec_files = FileList['spec/**/*_spec.rb']
+end
+
+Spec::Rake::SpecTask.new(:rcov) do |spec|
+  spec.libs << 'lib' << 'spec'
+  spec.pattern = 'spec/**/*_spec.rb'
+  spec.rcov = true
+end
+
+task :spec => :check_dependencies
+
+task :default => :spec
+
+require 'rake/rdoctask'
+Rake::RDocTask.new do |rdoc|
+  version = File.exist?('VERSION') ? File.read('VERSION') : ""
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "dm-groonga-adapter #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end
+

data/VERSION.yml

@@ -0,0 +1,5 @@
+--- 
+:minor: 1
+:patch: 0
+:build: pre
+:major: 0

data/dm-groonga-adapter.gemspec

@@ -0,0 +1,85 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run the gemspec command
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{dm-groonga-adapter}
+  s.version = "0.1.0.pre"
+
+  s.required_rubygems_version = Gem::Requirement.new("> 1.3.1") if s.respond_to? :required_rubygems_version=
+  s.authors = ["hiroyuki"]
+  s.date = %q{2010-04-08}
+  s.description = %q{datamapper adapter for groonga search engine}
+  s.email = %q{hello@hryk.info}
+  s.extra_rdoc_files = [
+    "LICENSE",
+     "README.rdoc"
+  ]
+  s.files = [
+    ".document",
+     ".gitignore",
+     "LICENSE",
+     "README.rdoc",
+     "Rakefile",
+     "VERSION.yml",
+     "dm-groonga-adapter.gemspec",
+     "examples/basic.rb",
+     "lib/groonga_adapter.rb",
+     "lib/groonga_adapter/adapter.rb",
+     "lib/groonga_adapter/local_index.rb",
+     "lib/groonga_adapter/model_ext.rb",
+     "lib/groonga_adapter/remote_index.rb",
+     "lib/groonga_adapter/remote_result.rb",
+     "lib/groonga_adapter/repository_ext.rb",
+     "lib/groonga_adapter/unicode_ext.rb",
+     "spec/rcov.opts",
+     "spec/shared/adapter_example.rb",
+     "spec/shared/search_example.rb",
+     "spec/spec.opts",
+     "spec/spec_helper.rb",
+     "spec/specs/adapter_spec.rb",
+     "spec/specs/remote_result_spec.rb",
+     "spec/specs/search_spec.rb"
+  ]
+  s.homepage = %q{http://github.com/hryk/dm-groonga-adapter}
+  s.rdoc_options = ["--charset=UTF-8"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = %q{1.3.6}
+  s.summary = %q{datamapper adapter for groonga search engine}
+  s.test_files = [
+    "spec/shared/adapter_example.rb",
+     "spec/shared/search_example.rb",
+     "spec/spec_helper.rb",
+     "spec/specs/adapter_spec.rb",
+     "spec/specs/remote_result_spec.rb",
+     "spec/specs/search_spec.rb",
+     "examples/basic.rb"
+  ]
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<groonga>, [">= 0.9"])
+      s.add_runtime_dependency(%q<dm-core>, ["~> 0.10.2"])
+      s.add_runtime_dependency(%q<dm-more>, ["~> 0.10.2"])
+      s.add_development_dependency(%q<rspec>, [">= 1.2.9"])
+      s.add_development_dependency(%q<rcov>, [">= 0"])
+    else
+      s.add_dependency(%q<groonga>, [">= 0.9"])
+      s.add_dependency(%q<dm-core>, ["~> 0.10.2"])
+      s.add_dependency(%q<dm-more>, ["~> 0.10.2"])
+      s.add_dependency(%q<rspec>, [">= 1.2.9"])
+      s.add_dependency(%q<rcov>, [">= 0"])
+    end
+  else
+    s.add_dependency(%q<groonga>, [">= 0.9"])
+    s.add_dependency(%q<dm-core>, ["~> 0.10.2"])
+    s.add_dependency(%q<dm-more>, ["~> 0.10.2"])
+    s.add_dependency(%q<rspec>, [">= 1.2.9"])
+    s.add_dependency(%q<rcov>, [">= 0"])
+  end
+end
+

data/examples/basic.rb

@@ -0,0 +1,45 @@
+require 'rubygems'
+require 'dm-core'
+require 'dm-is-searchable'
+
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+
+require 'groonga_adapter'
+
+DataMapper.setup(:default, "sqlite3::memory:")
+DataMapper.setup(:search, "groonga://#{Pathname(__FILE__).dirname.expand_path + "test/db"}")
+
+class Image
+  include DataMapper::Resource
+  property :id, Serial
+  property :title, String
+
+  is :searchable # this defaults to :search repository, you could also do
+  # is :searchable, :repository => :ferret
+
+end
+
+class Story
+  include DataMapper::Resource
+  property :id, Serial
+  property :title, String
+  property :author, String
+
+#  repository(:search) do
+#    # We only want to search on id and title.
+#    #properties(:search).clear
+#    property :id, Serial
+#    property :title, String
+#  end
+
+  is :searchable
+end
+
+Image.auto_migrate!
+Story.auto_migrate!
+
+image = Image.create(:title => "Oil Rig");
+story = Story.create(:title => "Oil Rig", :author => "John Doe");
+
+puts Image.search(:title => "Oil Rig").inspect # => [<Image title="Oil Rig">]

data/lib/groonga_adapter/adapter.rb

@@ -0,0 +1,207 @@
+$KCODE = 'UTF-8'
+module DataMapper
+  module Adapters
+    class GroongaAdapter < AbstractAdapter
+
+      def initialize(name, options)
+        super
+        Groonga::Context.default = nil # Reset Groonga::Context
+        @database = if @options[:port].nil? #unless File.extname(@options[:path]) == '.sock'
+                   LocalIndex.new(@options)
+                 else
+                   RemoteIndex.new(@options)
+                 end
+        @database.logger = ::DataMapper.logger
+      end
+
+      def create(resources)
+        name = self.name
+
+        resources.each do |resource|
+          model = resource.model
+          attributes = resource.attributes(:field).to_mash
+
+          # Since we don't inspect the models before generating the indices,
+          # we'll map the resource's key to the :id column.
+          attributes[:id]    ||= resource.key.first
+
+          unless @database.exist_table resource.model.name
+            @database.create_table(model.name,
+                                   model.properties(name),
+                                   model.key.first # <- key attribute.
+                                  )
+          end
+          @database.add model.name, attributes
+        end
+      end
+
+      # This returns an array of Groonga docs (array of Groonga::Record) which can
+      # be used to instantiate objects by doc[:_type] and doc[:_id]
+      def read(query) # query is DataMapper::Query
+        table_name = query.model.name
+        grn_query = unless query.conditions.operands.empty?
+                      create_grn_query(query)
+                    else
+                      ""
+                    end
+        grn_sort = create_grn_sort(query)
+        fields = query.fields
+        key    = query.model.key(name).first
+        @database.search(table_name, grn_query, grn_sort).map do |lazy_doc|
+          fmap = fields.map { |p|
+            p_field = (p.field == "id") ? "_key" : p.field
+            [ p, p.typecast(lazy_doc[p_field]) ]
+          }.to_hash
+          fmap.update(
+            key.field => key.typecast(lazy_doc['_key'])
+          )
+        end
+      end
+
+      def read_many(query)
+        read(query)
+      end
+
+      def read_one(query)
+        read(query).first
+      end
+
+      # TODO : implement #update
+      #      def update(attributes, collection)
+      #        query = collection.query
+      #        1
+      #      end
+
+      def delete(collection)
+        query      = collection.query
+        table_name = query.model.name
+
+        @database.delete(table_name, create_grn_query(query))
+        1
+      end
+
+      # This returns a hash of the resource constant and the ids returned for it
+      # from the search.
+      #   { Story => ["1", "2"], Image => ["2"] }
+      # query is groonga query.
+      #   options are;
+      #     :operator
+      #     :exact
+      #     :longest_common_prefix
+      #     :suffix
+      #     :prefix
+      #     :near
+      def search(model, groonga_query, groonga_sort=[], query_option={})
+        results = {}
+        groonga_sort = unless groonga_sort.empty?
+                         groonga_sort
+                       else
+                         default_groonga_sort
+                       end
+        @database.search(model.to_s, groonga_query, groonga_sort, query_option).each do |doc| 
+          resources = results[Object.const_get(model.to_s)] ||= []
+          resources << doc[:_key]
+        end
+        results
+      end
+
+      private
+
+      def default_groonga_sort
+        [[{:key => '_key', :order => :asc}], { :limit => -1, :offset => 0}]
+      end
+
+      def create_grn_query(query)
+        conditions_statement(query.conditions)
+      end
+
+      ## from dm-ferret-adapter ##
+
+      def conditions_statement(conditions)
+        case conditions
+          when Query::Conditions::NotOperation       then negate_operation(conditions)
+          when Query::Conditions::AbstractOperation  then operation_statement(conditions)
+          when Query::Conditions::AbstractComparison then comparison_statement(conditions)
+        end
+      end
+
+      def negate_operation(operation)
+        "- (#{conditions_statement(operation.operands.first)})"
+      end
+
+      def operation_statement(operation)
+        statements  = []
+
+        operation.each do |operand|
+          statement = conditions_statement(operand)
+
+          if operand.respond_to?(:operands) && operand.operands.size > 1
+            statement = "(#{statement})"
+          end
+
+          statements << statement
+        end
+
+        join_with = operation.kind_of?(Query::Conditions::AndOperation) ? '+' : 'OR'
+        statements.join(" #{join_with} ")
+      end
+
+      def comparison_statement(comparison)
+        value = comparison.value
+
+        # TODO: move exclusive Range handling into another method, and
+        # update conditions_statement to use it
+
+        # break exclusive Range queries up into two comparisons ANDed together
+        if value.kind_of?(Range) && value.exclude_end?
+          operation = Query::Conditions::BooleanOperation.new(:and,
+            Query::Conditions::Comparison.new(:gte, comparison.subject, value.first),
+            Query::Conditions::Comparison.new(:lt,  comparison.subject, value.last)
+          )
+
+          return "(#{operation_statement(operation)})"
+        end
+
+        operator = case comparison
+          when Query::Conditions::EqualToComparison              then ''
+          when Query::Conditions::InclusionComparison            then '@'
+          when Query::Conditions::RegexpComparison               then raise NotImplementedError, 'no support for regexp match yet'
+          when Query::Conditions::LikeComparison                 then '@'
+          when Query::Conditions::GreaterThanComparison          then '>'
+          when Query::Conditions::LessThanComparison             then '<'
+          when Query::Conditions::GreaterThanOrEqualToComparison then '>='
+          when Query::Conditions::LessThanOrEqualToComparison    then '<='
+        end
+
+        # We use property.field here, so that you can declare composite
+        # fields:
+        #     property :content, String, :field => "title|description"
+        grn_field = (comparison.subject.field.to_s == 'id') ? '_key' : comparison.subject.field
+        [ "#{grn_field}:", ((value.is_a? String) ? quote_value(value) : value) ].join(operator)
+      end
+
+      ## from dm-ferret-adapter ##
+
+      def create_grn_sort(query)
+        keys = []
+        options = { :limit => -1, :offset => 0}
+        options[:limit]  = query.limit unless query.limit.nil?
+        options[:offset] = query.offset
+        if query.order.empty?
+          keys << {:key => '_key', :order => :asc}
+        else
+          query.order.each do |direction|
+            grn_field = (direction.target.name == :id) ? '_key' : direction.target.name 
+            keys << { :key => grn_field.to_s, :order => direction.operator }
+          end
+        end
+        [ keys, options ]
+      end
+
+      def quote_value(value)
+        return value.gsub(/"/, '\"').gsub(/\s/, '\ ')
+      end
+
+    end # DataMapper::Adapters::GroongaAdapter
+  end
+end

data/lib/groonga_adapter/local_index.rb

@@ -0,0 +1,191 @@
+$KCODE = "UTF-8"
+module DataMapper
+  module Adapters
+    class GroongaAdapter::LocalIndex
+      attr_accessor :logger
+
+      def initialize(options)
+        @options = options
+        @context = Groonga::Context.default
+        create_or_init_database
+        @tables = Mash.new
+        create_or_init_term_table
+      end
+
+      def add(table_name, doc)
+        return unless exist_table(table_name)
+        table = table(table_name)
+        doc_id = doc.delete(:id)
+        record = table.add(doc_id)
+
+        doc.each do |k, v|
+          begin
+            if record.have_column? k
+              record[k] = v
+            else
+              puts "column #{k} is not defined."
+            end
+          rescue => e
+            puts record.inspect
+            puts record.columns.inspect
+            puts k
+            puts v
+            raise e
+          end
+        end
+        doc
+      end
+
+      # FIXME : WTF.
+      def delete(table_name, grn_query)
+        unless grn_query.empty?
+          # table = @tables[table_name]
+          ids = {}
+          # WTF start
+          @tables[table_name].select(grn_query, {}).records.each {|r|
+            # r.delete <-- Not work.
+            # ids[r[:dmid]] == true
+            ids[r['_key']] = true
+          }
+          @tables[table_name].records.each {|r|
+            # if ids[r[:dmid]] == true
+            if ids[r['_key']] == true
+              r.delete
+            end
+          }
+          # WTF end
+          #ids.each { |id| @tables[table_name].delete id }
+        end
+        1
+      end
+
+      # table_name : String
+      # grn_query  : String (e.g., "title:@foovar"
+      # grn_sort   : [{:key => "_id", :order => :asc }]
+      def search(table_name, grn_query, grn_sort=[], options={})
+        table = @tables[table_name]
+        table = @tables[table_name].select(grn_query, options) unless grn_query.empty?
+
+        if grn_sort.empty?
+          grn_sort << {:key => "_key", :order => :asc }
+        end
+        table.sort(*grn_sort)
+      end
+
+      def exist_table(table_name)
+        begin
+          Groonga::Hash.open(:name => table_name)
+        rescue Groonga::InvalidArgument
+          return false
+        rescue => e
+          raise e
+        else
+          return true
+        end
+      end
+
+      def open_table(table_name)
+        @tables[table_name] = Groonga::Hash.open(:name => table_name)
+      end
+
+      def create_table(table_name, properties, key_prop=nil)
+        key_type = (key_prop.nil?) ? Groonga::Type::UINT64 : trans_type(key_prop.type)
+        @tables[table_name] = Groonga::Hash.create(
+          :name       => table_name,
+          :persistent => true,
+          :key_type   => key_type
+        )
+
+        # add columns
+        properties.each do |prop|
+          type = trans_type(prop.type)
+          propname = prop.name.to_s
+          @tables[table_name].define_column(propname, type, {:persistent => true})
+          if type == "ShortText" || type == "Text" || type == "LongText"
+            index_column = add_term(table_name, propname)
+          end
+        end
+      end
+
+      private
+
+      def add_term(table, prop)
+        @tables['DMGterms'].define_index_column(
+          "#{table}_#{prop}", @tables[table],
+          :source => "#{table}.#{prop}"
+        )
+      end
+
+      # translate DataMapper::Property::TYPES to Groonga::Type
+      def trans_type(dmtype)
+        case dmtype.to_s
+        when 'String'
+          return Groonga::Type::SHORT_TEXT
+        when 'Text'
+          return Groonga::Type::TEXT
+        when 'Float'
+          return Groonga::Type::FLOAT
+        when 'Bool'
+          return Groonga::Type::BOOL
+        when 'Boolean'
+          return Groonga::Type::BOOLEAN
+        when 'Integer'
+          return Groonga::Type::INT32
+        when 'BigDecimal'
+          return Groonga::Type::INT64
+        when 'Time'
+          return Groonga::Type::TIME
+        when /^DataMapper::Types::(.+)$/
+          case $1
+          when "Boolean"
+            return Groonga::Type::BOOL
+          when "Serial"
+            return Groonga::Type::UINT32
+          end
+        else
+          return Groonga::Type::SHORT_TEXT
+        end
+      end
+
+      def table(table_name)
+        unless @tables.key? table_name
+          if exist_table(table_name)
+            open_table(table_name)
+          else
+            false # no such table.
+          end
+        end
+        return @tables[table_name]
+      end
+
+      def create_or_init_term_table
+        unless exist_table('DMGterms')
+          @tables['DMGterms'] = Groonga::Hash.create(:name       => "DMGterms",
+                                                   :persistent => true,
+                                                   :key_type   => Groonga::Type::UINT64,
+                                                   :default_tokenizer => "TokenBigram")
+        else
+          open_table('DMGterms')
+        end
+      end
+
+      def create_or_init_database
+        # try to open database.
+        path = Pathname(@options[:path])
+
+        if path.exist? && path.file?
+          # open database
+          @database = Groonga::Database.open(path.to_s)
+        else
+          # check directory.
+          unless path.dirname.directory?
+            path.dirname.mkpath
+          end
+          # create database.
+          @database = Groonga::Database.create(:path => path.to_s)
+        end
+      end
+
+    end
+  end
+end

data/lib/groonga_adapter/model_ext.rb

@@ -0,0 +1,12 @@
+module DataMapper
+  module Is
+    module Searchable
+      module ClassMethods
+        def fulltext_search(query, options={})
+          docs = repository(@search_repository).adapter.search(self.name, query, [], options)
+          self.all(options.merge(key.first => docs.values.flatten!))
+        end
+      end
+    end
+  end
+end