dm-groonga-adapter 0.1.0.pre

data/lib/groonga_adapter/remote_index.rb

@@ -0,0 +1,250 @@
+require 'json'
+
+module DataMapper
+  module Adapters
+    class GroongaAdapter::RemoteIndex
+      attr_accessor :logger
+      attr_accessor :context
+
+      def initialize(options)
+        @context = Groonga::Context.default
+        @context.connect(:host => options[:host], :port => options[:port])
+        # request "status" # <- TODO check connection with status command
+      end
+
+      def add(table_name, doc)
+        return unless exist_table(table_name)
+        doc_id = doc[:id] #doc.delete(:id)
+        record = []
+        record << doc.update("_key" => doc_id)
+        json = JSON.generate record
+        res = request "load --table #{table_name} --values #{Unicode.unescape(json.gsub(/"/, '\"').gsub(/\s/, '\ '))}"
+        result = GroongaResult::Count.new res
+        if result.success? && result.count > 0
+          return doc
+        else
+          throw "failed to load record. : #{result.err_code}"
+        end
+      end
+
+      def delete(table_name, grn_query)
+        self.search(table_name, grn_query).each do |i|
+          request "delete #{table_name} --id #{i['_id']}"
+        end
+      end
+
+      # select table [match_columns [query [filter [scorer [sortby [output_columns
+      #              [offset [limit [drilldown [drilldown_sortby [drilldown_output_columns
+      #                           [drilldown_offset [drilldown_limit [output_type]]]]]]]]]]]]]]
+      def search(table_name, grn_query, grn_sort=[], options={})
+        sort_by, offset, limit = parse_grn_sort grn_sort
+        remote_query = (grn_query.empty?) ? "" : "--query #{grn_query}"
+        remote_sort_by   = (sort_by.empty?) ? "" : "--sort-by #{sort_by}"
+        res = request "select #{table_name} #{remote_query} #{remote_sort_by} --offset #{offset} --limit #{limit}"
+        list = GroongaResult::List.new res
+        if list.success?
+          return list.to_a
+        else
+          throw list.err_msg
+        end
+      end
+
+      def exist_table(table_name)
+        res = request "table_list"
+        table_list = GroongaResult::List.new res
+        if table_list.success?
+          existence = false
+          table_list.each do |row|
+            existence = true if row[:name] == table_name
+          end
+          return existence
+        else
+          throw table_list.err_msg
+        end
+      end
+
+      def exist_column(table_name, column_name)
+        # groonga 1.4
+        #   [["id","name","path","type","flags","domain"],[260,"title","test.0000104","var",49152,259]]
+        # groonga 1.7
+        #   [
+        #    [0,1269972586.4569,1.4e-05],
+        #    [[["id", "UInt32"],["name","ShortText"],["path","ShortText"],["type","ShortText"],["flags","ShortText"],["domain", "ShortText"],["range", "ShortText"],["source","ShortText"]]]
+        #    ]
+        res = request "column_list #{table_name}"
+        list = GroongaResult::List.new res
+        if list.success?
+          existence = false
+          list.each do |row|
+            existence = true if row[:name] == column_name
+          end
+          existence
+        else
+          throw list.err_msg
+        end
+      end
+
+      def create_table(table_name, properties, key_prop=nil)
+        key_type = (key_prop.nil?) ? "UInt64" : trans_type(key_prop.type)
+        # create table
+        res = request "table_create #{table_name} 0 #{key_type}";
+        result = GroongaResult::Base.new res
+        throw result.err_msg unless result.err_code == 0 || result.err_code == -22
+        properties.each do |prop|
+          type = trans_type(prop.type)
+          propname = prop.name.to_s
+          query = "column_create #{table_name} #{propname} 0 #{type}"
+          res = GroongaResult::Base.new(request query)
+          err = res.err_code
+
+          unless err == 0 || err == -22
+            throw "Create Column Failed : #{res.inspect} : #{query}"
+          end
+
+          if type == "ShortText" || type == "Text" || type == "LongText"
+            add_term(table_name, propname)
+          end
+        end
+      end
+
+      protected
+
+      def err_code(res)
+        return if res.nil?
+        code = res[0][0]
+        code
+      end
+
+      def create_term_table(table_name, key_prop="ShortText", tokenizer="TokenBigram")
+        res = request "table_create #{table_name} TABLE_PAT_KEY|KEY_NORMALIZE #{key_prop} Void #{tokenizer}"
+        throw "Fale to create term table." unless err_code(res) == 0 || err_code(res) == -22
+        true
+      end
+
+      def add_term(table_name, propname)
+        term_table_name = 'DMGTerms'
+        term_column_name = "#{table_name.downcase}_#{propname.downcase}"
+        # check existence of term table
+        unless exist_table term_table_name
+          create_term_table term_table_name
+        end
+        # check existence of column in term table
+        unless exist_column(term_table_name, term_column_name)
+          request "column_create DMGTerms #{term_column_name} COLUMN_INDEX|WITH_POSITION #{table_name} #{propname}"
+        end
+      end
+
+      def request(message)
+        @context.send message
+        self.logger.debug "Query: " + message
+        id, result = @context.receive
+        self.logger.debug "Result: " + result
+        if result == 'true'
+          true
+        elsif result == 'false'
+          false
+        else
+          JSON.parse(result)
+        end
+      end
+
+      def parse_grn_sort(grn_sort=[])
+        return "" if grn_sort == []
+        sort = grn_sort[0]
+        options = grn_sort[1]
+        sort_str = sort.map {|i|
+          desc = (i[:order] == :desc) ? '-' : ''
+          "#{desc}#{i[:key]}"
+        }.join(',')
+        [ sort_str, options[:offset], options[:limit] ]
+      end
+
+      def trans_type(dmtype)
+        case dmtype.to_s
+        when 'String'
+          return 'ShortText'
+        when 'Text'
+          return 'LongText'
+        when 'Float'
+          return 'Float'
+        when 'Bool'
+          return 'Bool'
+        when 'Boolean'
+          return 'Bool'
+        when 'Integer'
+          return 'Int32'
+        when 'BigDecimal'
+          return 'Int64'
+        when 'Time'
+          return 'Time'
+        when /^DataMapper::Types::(.+)$/
+          case $1
+          when "Boolean"
+            return 'Bool'
+          when "Serial"
+            return 'Int32'
+          end
+        else
+          return 'ShortText'
+        end
+      end
+    end # class GroongaAdapter::RemoteIndex
+  end # module Adapters
+end # module DataMapper
+
+
+__END__
+
+def test_send
+  _context = Groonga::Context.new
+  _context.connect(:host => @host, :port => @port)
+  assert_equal(0, _context.send("status"))
+  id, result = _context.receive
+  assert_equal(0, id)
+  status, values = JSON.load(result)
+  return_code, start_time, elapsed,  = status
+  assert_equal([0, ["alloc_count", "starttime", "uptime"]],
+               [return_code, values.keys.sort])
+end
+
+Commands
+
+add
+column_create
+column_list
+define_selector
+delete
+get
+load
+log_level
+log_put
+log_put
+quit
+select
+set
+shutdown
+status
+table_create
+table_list
+view_add
+
+Types
+
+Object        任意のテーブルに属する全てのレコード [1]
+Bool          bool型。trueとfalse。
+Int8          8bit符号付き整数。
+UInt8         8bit符号なし整数。
+Int16         16bit符号付き整数。
+UInt16        16bit符号なし整数。
+Int32         32bit符号付き整数。
+UInt32        32bit符号なし整数。
+Int64         64bit符号付き整数。
+UInt64        64bit符号なし整数。
+Float         ieee754形式の64bit浮動小数点数。
+Time          1970年1月1日0時0分0秒からの経過マイクロ秒数を
+64bit符号付き整数で表現した値。
+ShortText     4Kbyte以下の文字列。
+Text          64Kbyte以下の文字列。
+LongText      2Gbyte以下の文字列。
+TokyoGeoPoint 日本測地系緯度経度座標。
+WGS84GeoPoint 世界測地系緯度経度座標。

data/lib/groonga_adapter/remote_result.rb

@@ -0,0 +1,101 @@
+module DataMapper
+  module Adapters
+    module GroongaResult
+      class Base
+        attr_accessor :err_code
+        attr_accessor :err_msg
+        attr_accessor :start_time
+        attr_accessor :elapsed_time
+
+        def initialize(raw_result)
+          @err_code, @start_time, @elased_time, @err_msg = raw_result[0]
+        end
+
+        def success?
+          if @err_code == 0
+            true
+          else
+            false
+          end
+        end
+      end # class Result::Base
+
+      class Count < Base
+        attr_accessor :count
+        def initialize(raw_result)
+          super raw_result
+          # [[0,1270199923.22467,5.2e-05],1]
+          @count = raw_result[1]
+        end
+      end
+
+      class Status < Base
+        attr_accessor :alloc_count
+        attr_accessor :process_starttime
+        attr_accessor :uptime
+        attr_accessor :version
+
+        def initialize(raw_result)
+          super(raw_result)
+          if success?
+            @alloc_count = raw_result[1]['alloc_count']
+            @process_starttime = raw_result[1]['starttime']
+            @uptime  = raw_result[1]['uptime']
+            @version = raw_result[1]['version']
+          end
+        end
+      end
+
+      class List < Base
+        include Enumerable
+        attr_accessor :columns
+        attr_accessor :rows
+#        attr_accessor :raw_rows
+#        attr_accessor :raw_columns
+        attr_accessor :size
+
+        def initialize(raw_result)
+          super(raw_result)
+          if success?
+            @raw_columns, @rows, @size = if raw_result[1].size > 1
+                                               # no count
+                                               raws = raw_result[1].dup
+                                               [raws.shift,raws,nil]
+                                             else
+                                               # with count
+                                               raws = raw_result[1].dup.shift
+                                               size = raws.shift.shift
+                                               rawcols = raws.shift
+                                               [rawcols, raws, size]
+                                             end
+            # columns
+            @columns = @raw_columns.map {|item| item[0] }
+            parse_rows
+            self
+          end
+        end
+
+        def each
+          @mash_rows.each do |m|
+            yield m
+          end
+        end
+
+        def to_a
+          return @mash_rows unless @mash_rows.nil?
+          []
+        end
+
+        def parse_rows
+          @mash_rows = @rows.map {|row|
+            m = Mash.new
+            @columns.each_with_index {|item, idx|
+              m[@columns[idx]] = row[idx]
+            }
+            m
+          }
+        end
+      end
+    end # now # module Result
+  end 
+end

data/lib/groonga_adapter/repository_ext.rb

@@ -0,0 +1,13 @@
+module DataMapper
+  class Repository
+    # This accepts a ferret query string and an optional limit argument
+    # which defaults to all. This is the proper way to perform searches more
+    # complicated than DM's query syntax can handle (such as OR searches).
+    #
+    # See DataMapper::Adapters::GroongaAdapter#search for information on
+    # the return value.
+    def search(query, limit = :all)
+      adapter.search(query, limit)
+    end
+  end
+end

data/lib/groonga_adapter/unicode_ext.rb

@@ -0,0 +1,17 @@
+# http://d.hatena.ne.jp/cesar/20070401/p1
+module Unicode
+  def escape(str)
+    ary = str.unpack("U*").map!{|i| "\\u#{i.to_s(16)}"}
+    ary.join
+  end
+
+  UNESCAPE_WORKER_ARRAY = []
+  def unescape(str)
+    str.gsub(/\\u([0-9a-f]{4})/) {
+      UNESCAPE_WORKER_ARRAY[0] = $1.hex
+      UNESCAPE_WORKER_ARRAY.pack("U")
+    }
+  end
+
+  module_function :escape, :unescape
+end

data/lib/groonga_adapter.rb

@@ -0,0 +1,9 @@
+require 'groonga'
+
+require 'groonga_adapter/adapter'
+require 'groonga_adapter/local_index'
+require 'groonga_adapter/remote_index'
+require 'groonga_adapter/remote_result'
+require 'groonga_adapter/repository_ext'
+require 'groonga_adapter/model_ext'
+require 'groonga_adapter/unicode_ext'

data/spec/rcov.opts

@@ -0,0 +1,6 @@
+--exclude "spec"
+--sort coverage
+--callsites
+--xrefs
+--profile
+--text-summary

data/spec/shared/adapter_example.rb

@@ -0,0 +1,53 @@
+shared_examples_for "as adapter" do
+  before(:each) do
+    @adapter = DataMapper.setup(:default, "groonga://#{index_path}")
+
+    Object.send(:remove_const, :User) if defined?(User)
+    class ::User
+      include DataMapper::Resource
+
+      property :id, Serial
+      property :name, String
+    end
+
+    Object.send(:remove_const, :Photo) if defined?(Photo)
+    class ::Photo
+      include DataMapper::Resource
+
+      property :uuid, String, :default => proc { UUIDTools::UUID.random_create }, :key => true
+      property :happy, Boolean, :default => true
+      property :description, String
+    end
+
+    User.auto_migrate!
+    Photo.auto_migrate!
+  end
+
+  it 'should work with a model using id' do
+    u = User.create(:id => 2)
+    repository.search(User, '').should == { User => [ 2 ] }
+  end
+
+  it 'should work with a model using another key than id' do
+    p = Photo.create
+    repository.search(Photo, '').should == { Photo => [p.uuid] }
+    p.destroy!
+  end
+
+  it 'should allow lookups using Model#get' do
+    u = User.create(:id => 2, :name => "foovarbuz")
+    User.get(2).should == u
+  end
+
+  it 'should allow delete rows using Model#destroy' do
+    u  = User.create(:id => 2, :name => "Alice")
+    u2 = User.create(:id => 3, :name => "Bob")
+    User.get(2).should == u
+    bob = User.get(3)
+    repository.search(User,'name:Bob').should == { User => [ 3 ] } #[User].size.should == 1
+    bob.destroy!.should == true
+    repository.search(User,'name:Bob').should == {}
+    repository.search(User,'name:Alice').should == {User => [ 2 ]}
+  end
+
+end

data/spec/shared/search_example.rb

@@ -0,0 +1,64 @@
+shared_examples_for 'as is_search plugin' do
+
+  before(:each) do
+    DataMapper.setup(:default, "sqlite3::memory:")
+    DataMapper.setup(:search, "groonga://#{index_path}")
+    DataMapper::Logger.new($stderr, :debug)
+    Object.send(:remove_const, :Image) if defined?(Image)
+    class ::Image
+      include DataMapper::Resource
+      property :id, Serial
+      property :title, String
+
+      is :searchable # this defaults to :search repository, you could also do
+    end
+
+    Object.send(:remove_const, :Story) if defined?(Story)
+    class ::Story
+      include DataMapper::Resource
+      property :id, Serial
+      property :title, String
+      property :author, String
+
+      is :searchable
+    end
+
+    Story.auto_migrate!
+    Image.auto_migrate!
+  end
+
+  it 'should allow search with no operator' do
+
+    pending "grn expression may have bug." # FIXME
+
+    image = Image.create(:title => "Oil Rig");
+    story = Story.create(:title => "Oil Rig",
+                         :author => "John Doe");
+    Image.search(:title => "Oil Rig").should == [image]
+  end
+
+  it 'should allow search with :like operator' do
+    image = Image.create(:title => "Oil Rig");
+    Image.search(:title.like => "Oil").should == [image]
+    image.title = "Owl Owl"
+    image.save
+    Image.search(:title.like => "Owl").should == [image]
+  end
+
+  it "should allow search with japanese" do
+    image = Image.create(:title => "お腹すいた");
+    Image.search(:title.like => "お腹").should == [image]
+    image.title = "すいてない"
+    image.save
+    Image.search(:title.like => "すいてない").should == [image]
+  end
+
+  it 'should allow search with all columns' do
+    story = Story.create(:title  => "Oil Rig",
+                         :author => "John Doe");
+    story2 = Story.create(:title => "Lolem ipsum",
+                         :author => "John Doe");
+    # Story.fulltext_search("John").should == [story, story2] # <--- Crash on local index.
+    Story.fulltext_search("author:@John").should == [story, story2]
+  end
+end

data/spec/spec.opts

@@ -0,0 +1 @@
+--color

data/spec/spec_helper.rb

@@ -0,0 +1,46 @@
+require 'rubygems'
+require 'uuidtools'
+require 'dm-core'
+require 'dm-is-searchable'
+
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+SPEC_ROOT = Pathname(__FILE__).dirname.expand_path
+
+require 'groonga_adapter'
+require 'spec'
+require 'spec/autorun'
+
+(Pathname.new(__FILE__).parent + "shared").children.grep(/\.rb$/).each do |example|
+  puts example
+  require example
+end
+
+def load_driver(name, default_uri)
+  return false if ENV['ADAPTER'] != name.to_s
+
+  begin
+    DataMapper.setup(name, ENV["#{name.to_s.upcase}_SPEC_URI"] || default_uri)
+    DataMapper::Repository.adapters[:default] =  DataMapper::Repository.adapters[name]
+    true
+  rescue LoadError => e
+    warn "Could not load do_#{name}: #{e}"
+    false
+  end
+end
+
+ENV['ADAPTER'] ||= 'sqlite3'
+
+HAS_SQLITE3  = load_driver(:sqlite3,  'sqlite3::memory:')
+HAS_MYSQL    = load_driver(:mysql,    'mysql://localhost/dm_core_test')
+HAS_POSTGRES = load_driver(:postgres, 'postgres://postgres@localhost/dm_core_test')
+
+def local_groonga_path
+  Pathname(SPEC_ROOT) + 'test/index'
+end
+def remote_groonga_path
+  ENV["DM_GRN_URL"] || "127.0.0.1:10041" # "192.168.81.132:8888" <- 1.4.0
+end
+
+# Spec::Runner.configure do |config|
+# end

data/spec/specs/adapter_spec.rb

@@ -0,0 +1,25 @@
+require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
+
+describe "local_index adapter" do
+  def index_path;local_groonga_path;end
+
+  after(:all) do
+    Pathname.new(index_path).parent.children.each do |f|
+      f.delete
+    end
+  end
+
+  before(:each) do
+    # remove indeces before running spec.
+    Pathname.new(index_path).parent.children.each do |f|
+      f.delete
+    end
+  end
+
+  it_should_behave_like "as adapter"
+end
+
+describe "remote_index adapter" do
+  def index_path;remote_groonga_path;end
+  it_should_behave_like "as adapter"
+end