dm-groonga-adapter 0.1.0.pre → 0.1.0.pre2

data/Rakefile

@@ -8,7 +8,7 @@ begin
     gem.summary = %Q{datamapper adapter for groonga search engine}
     gem.description = gem.summary
     gem.email = "hello@hryk.info"
-    gem.homepage = "http://github.com/hryk/dm-groonga-adapter"
+    gem.homepage = "http://hryk.github.com/dm-groonga-adapter"
     gem.authors = ["hiroyuki"]
 
     gem.add_dependency "groonga", ">= 0.9"

data/VERSION.yml

@@ -1,5 +1,5 @@
 --- 
-:minor: 1
 :patch: 0
-:build: pre
 :major: 0
+:build: pre2
+:minor: 1

data/dm-groonga-adapter.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{dm-groonga-adapter}
-  s.version = "0.1.0.pre"
+  s.version = "0.1.0.pre2"
 
   s.required_rubygems_version = Gem::Requirement.new("> 1.3.1") if s.respond_to? :required_rubygems_version=
   s.authors = ["hiroyuki"]
-  s.date = %q{2010-04-08}
+  s.date = %q{2010-04-15}
   s.description = %q{datamapper adapter for groonga search engine}
   s.email = %q{hello@hryk.info}
   s.extra_rdoc_files = [
@@ -27,6 +27,7 @@ Gem::Specification.new do |s|
      "examples/basic.rb",
      "lib/groonga_adapter.rb",
      "lib/groonga_adapter/adapter.rb",
+     "lib/groonga_adapter/jsonparser.rb",
      "lib/groonga_adapter/local_index.rb",
      "lib/groonga_adapter/model_ext.rb",
      "lib/groonga_adapter/remote_index.rb",
@@ -42,7 +43,7 @@ Gem::Specification.new do |s|
      "spec/specs/remote_result_spec.rb",
      "spec/specs/search_spec.rb"
   ]
-  s.homepage = %q{http://github.com/hryk/dm-groonga-adapter}
+  s.homepage = %q{http://hryk.github.com/dm-groonga-adapter}
   s.rdoc_options = ["--charset=UTF-8"]
   s.require_paths = ["lib"]
   s.rubygems_version = %q{1.3.6}

data/lib/groonga_adapter/adapter.rb

@@ -3,6 +3,10 @@ module DataMapper
   module Adapters
     class GroongaAdapter < AbstractAdapter
 
+      # Initialize adapter
+      #   
+      # @param [String] name
+      # @param [Hash] options the options passed to RemoteIndex or LocalIndex.
       def initialize(name, options)
         super
         Groonga::Context.default = nil # Reset Groonga::Context

data/lib/groonga_adapter/jsonparser.rb

@@ -0,0 +1,336 @@
+# = Simple JSON parser & builder
+#
+# Author::  Chihiro Ito
+# License:: Public domain (unlike other files)
+# Support:: http://groups.google.com/group/webos-goodies/
+#
+# This file contains two simple JSON processing classes. JsonParser
+# converts a JSON string to an array or a hash. JsonBuilder performs
+# vice versa. These classes are standard compliant and are designed for
+# stability and reliability. Especially JsonParser has UTF-8 validation
+# functionality so you can avoid some kind of security attack.
+
+require 'strscan'
+require 'json' if RUBY_VERSION >= '1.9.0'
+
+
+# = Simple JSON parser
+#
+# This class converts a JSON string to an array or a hash. If *json_str*
+# contains a JSON form string, you can convert it like below.
+#
+#   ruby_obj = JsonParser.new.parse(json_str)
+#
+# If *json_str* has one or more invalid UTF-8 sequence, JsonParser throws
+# exception by default. You can change this behavior to replacing with an
+# arbitrary unicode character. See below for details.
+class JsonParser
+
+  #:stopdoc:
+  RUBY19             = RUBY_VERSION >= '1.9.0'
+  Debug              = false
+  Name               = 'JsonParser'
+  ERR_IllegalSyntax  = "[#{Name}] Syntax error"
+  ERR_IllegalUnicode = "[#{Name}] Illegal unicode sequence"
+
+  StringRegex = /\s*"((?:\\.|[^"\\])*)"/n
+  ValueRegex  = /\s*(?:
+		(true)|(false)|(null)|            # 1:true, 2:false, 3:null
+		(?:\"((?:\\.|[^\"\\])*)\")|       # 4:String
+		([-+]?\d+\.\d+(?:[eE][-+]?\d+)?)| # 5:Float
+		([-+]?\d+)|                       # 6:Integer
+		(\{)|(\[))/xn                     # 7:Hash, 8:Array
+  #:startdoc:
+
+  # Create a new instance of JsonParser. *options* can contain these values.
+  # [:validation]
+  #     If set to false, UTF-8 validation is disabled. true by default.
+  # [:surrogate]
+  #     If set to false, surrogate pair support is disabled. true by default.
+  # [:malformed_chr]
+  #     An invalid sequence in JSON string will be replaced with this value.
+  #     If set to nil, An exception will be thrown in this situation.
+  #     nil by default.
+  # [:compatible]
+  #     If set to true, Ruby1.9's JSON module is never used. false by default.
+  def initialize(options = {})
+    @default_validation    = options.has_key?(:validation)    ? options[:validation]    : true
+    @default_surrogate     = options.has_key?(:surrogate)     ? options[:surrogate]     : true
+    @default_malformed_chr = options.has_key?(:malformed_chr) ? options[:malformed_chr] : nil
+    @default_compatible    = options.has_key?(:compatible)    ? options[:compatible]    : false
+  end
+
+  # Convert *str* to an array or hash.
+  # [str]
+  #     A JSON form string. This must be encoded using UTF-8.
+  # [options]
+  #     Same as new.
+  def parse(str, options = {})
+    @enable_validation = options.has_key?(:validation)    ? options[:validation]    : @default_validation
+    @enable_surrogate  = options.has_key?(:surrogate)     ? options[:surrogate]     : @default_surrogate
+    @malformed_chr     = options.has_key?(:malformed_chr) ? options[:malformed_chr] : @default_malformed_chr
+    @compatible        = options.has_key?(:compatible)    ? options[:compatible]    : @default_compatible
+    @malformed_chr = @malformed_chr[0].ord if String === @malformed_chr
+    if RUBY19
+      str = (str.encode('UTF-8') rescue str.dup)
+      if @enable_validation && !@malformed_chr
+        raise err_msg(ERR_IllegalUnicode) unless str.valid_encoding?
+        @enable_validation = false
+      end
+      if !@enable_validation && @enable_surrogate && !@malformed_chr && !@compatible
+        begin
+          return JSON.parse(str, :max_nesting => false)
+        rescue JSON::JSONError => e
+          exception = RuntimeError.new(e.message)
+          exception.set_backtrace(e.backtrace)
+          raise exception
+        end
+      end
+      str.force_encoding('ASCII-8BIT')
+    end
+    @scanner = StringScanner.new(str)
+    obj = case get_symbol[0]
+          when ?{ then parse_hash
+          when ?[ then parse_array
+          else         raise err_msg(ERR_IllegalSyntax)
+          end
+    @scanner = nil
+    obj
+  end
+
+  private #---------------------------------------------------------
+
+  def validate_string(str, malformed_chr = nil)
+    code  = 0
+    rest  = 0
+    range = nil
+    ucs   = []
+    str.each_byte do |c|
+      if rest <= 0
+        case c
+        when 0x01..0x7f then rest = 0 ; ucs << c
+        when 0xc0..0xdf then rest = 1 ; code = c & 0x1f ; range = 0x00080..0x0007ff
+        when 0xe0..0xef then rest = 2 ; code = c & 0x0f ; range = 0x00800..0x00ffff
+        when 0xf0..0xf7 then rest = 3 ; code = c & 0x07 ; range = 0x10000..0x10ffff
+        else                 ucs << handle_malformed_chr(malformed_chr)
+        end
+      elsif (0x80..0xbf) === c
+        code = (code << 6) | (c & 0x3f)
+        if (rest -= 1) <= 0
+          if !(range === code) || (0xd800..0xdfff) === code
+            code = handle_malformed_chr(malformed_chr)
+          end
+          ucs << code
+        end
+      else
+        ucs << handle_malformed_chr(malformed_chr)
+        rest = 0
+      end
+    end
+    ucs << handle_malformed_chr(malformed_chr) if rest > 0
+    ucs.pack('U*')
+  end
+
+  def handle_malformed_chr(chr)
+    raise err_msg(ERR_IllegalUnicode) unless chr
+    chr
+  end
+
+  def err_msg(err)
+    err + (Debug ? " #{@scanner.string[[0, @scanner.pos - 8].max,16].inspect}" : "")
+  end
+
+  def unescape_string(str)
+    str = str.gsub(/\\(["\\\/bfnrt])/n) do
+      $1.tr('"\\/bfnrt', "\"\\/\b\f\n\r\t")
+    end.gsub(/(\\u[0-9a-fA-F]{4})+/n) do |matched|
+      seq = matched.scan(/\\u([0-9a-fA-F]{4})/n).flatten.map { |c| c.hex }
+      if @enable_surrogate
+        seq.each_index do |index|
+          if seq[index] && (0xd800..0xdbff) === seq[index]
+            n = index + 1
+            raise err_msg(ERR_IllegalUnicode) unless seq[n] && 0xdc00..0xdfff === seq[n]
+            seq[index] = 0x10000 + ((seq[index] & 0x03ff) << 10) + (seq[n] & 0x03ff)
+            seq[n] = nil
+          end
+        end.compact!
+      end
+      seq.pack('U*')
+    end
+    str = validate_string(str, @malformed_chr) if @enable_validation
+    RUBY19 ? str.force_encoding('UTF-8') : str
+  end
+
+  def get_symbol
+    raise err_msg(ERR_IllegalSyntax) unless @scanner.scan(/\s*(.)/n)
+    @scanner[1]
+  end
+
+  def peek_symbol
+    @scanner.match?(/\s*(.)/n) ? @scanner[1] : nil
+  end
+
+  def parse_string
+    raise err_msg(ERR_IllegalSyntax) unless @scanner.scan(StringRegex)
+    unescape_string(@scanner[1])
+  end
+
+  def parse_value
+    raise err_msg(ERR_IllegalSyntax) unless @scanner.scan(ValueRegex)
+    case
+    when @scanner[1] then true
+    when @scanner[2] then false
+    when @scanner[3] then nil
+    when @scanner[4] then unescape_string(@scanner[4])
+    when @scanner[5] then @scanner[5].to_f
+    when @scanner[6] then @scanner[6].to_i
+    when @scanner[7] then parse_hash
+    when @scanner[8] then parse_array
+    else                  raise err_msg(ERR_IllegalSyntax)
+    end
+  end
+
+  def parse_hash
+    obj = {}
+    if peek_symbol[0] == ?} then get_symbol ; return obj ; end
+    while true
+      index = parse_string
+      raise err_msg(ERR_IllegalSyntax) unless get_symbol[0] == ?:
+      value = parse_value
+      obj[index] = value
+      case get_symbol[0]
+      when ?} then return obj
+      when ?, then next
+      else         raise err_msg(ERR_IllegalSyntax)
+      end
+    end
+  end
+
+  def parse_array
+    obj = []
+    if peek_symbol[0] == ?] then get_symbol ; return obj ; end
+    while true
+      obj << parse_value
+      case get_symbol[0]
+      when ?] then return obj
+      when ?, then next
+      else         raise err_msg(ERR_IllegalSyntax)
+      end
+    end
+  end
+
+end
+
+# = Simple JSON builder
+#
+# This class converts an Ruby object to a JSON string. you can convert
+# *ruby_obj* like below.
+#
+#   json_str = JsonBuilder.new.build(ruby_obj)
+#
+# *ruby_obj* must satisfy these conditions.
+# - It must support to_s method, otherwise must be an array, a hash or nil.
+# - All keys of a hash must support to_s method.
+# - All values of an array or a hash must satisfy all conditions mentioned above.
+#
+# If the *ruby_obj* is not an array or a hash, it will be converted to an array
+# with a single element.
+class JsonBuilder
+
+  #:stopdoc:
+  RUBY19             = RUBY_VERSION >= '1.9.0'
+  Name               = 'JsonBuilder'
+  ERR_NestIsTooDeep  = "[#{Name}] Array / Hash nested too deep."
+  ERR_NaN            = "[#{Name}] NaN and Infinite are not permitted in JSON."
+  #:startdoc:
+
+  # Create a new instance of JsonBuilder. *options* can contain these values.
+  # [:max_nest]
+  #     If Array / Hash is nested more than this value, an exception would be thrown.
+  #     19 by default.
+  # [:nan]
+  #     NaN is replaced with this value. If nil or false, an exception will be thrown.
+  #     nil by default.
+  def initialize(options = {})
+    @default_max_nest = options.has_key?(:max_nest) ? options[:max_nest] : 19
+    @default_nan      = options.has_key?(:nan)      ? options[:nan]      : nil
+  end
+
+  # Convert *obj* to a JSON form string.
+  # [obj]
+  #     A ruby object. this object must satisfy all conditions mentioned above.
+  # [options]
+  #     Same as new.
+  def build(obj, options = {})
+    @max_nest = options.has_key?(:max_nest) ? options[:max_nest] : @default_max_nest
+    @nan      = options.has_key?(:nan)      ? options[:nan]      : @default_nan
+    if RUBY19 && !@nan
+      begin
+        JSON.generate(obj, :max_nesting => @max_nest, :check_circular => false)
+      rescue JSON::JSONError => e
+        exception = RuntimeError.new(e.message)
+        exception.set_backtrace(e.backtrace)
+        raise exception
+      end
+    else
+      case obj
+      when Array then build_array(obj, 0)
+      when Hash  then build_object(obj, 0)
+      else            build_array([obj], 0)
+      end
+    end
+  end
+
+  private #---------------------------------------------------------
+
+  ESCAPE_CONVERSION = {
+    '"'    => '\"', '\\'   => '\\\\', '/'    => '/', "\x08" => '\b',
+    "\x0c" => '\f', "\x0a" => '\n',   "\x0d" => '\r', "\x09" => '\t'
+  }
+  if RUBY19
+    def escape(str)
+      str = str.to_s.encode('UTF-8')
+      str.force_encoding('ASCII-8BIT')
+      str = str.gsub(/[^\x20-\x21\x23-\x2e\x30-\x5b\x5d-\xff]/n) do |chr|
+        escaped = ESCAPE_CONVERSION[chr]
+        escaped = sprintf("\\u%04X", chr[0].ord) unless escaped
+        escaped
+      end
+      str.force_encoding('UTF-8')
+      "\"#{str}\""
+    end
+  else
+    def escape(str)
+      str = str.gsub(/[^\x20-\x21\x23-\x2e\x30-\x5b\x5d-\xff]/n) do |chr|
+        escaped = ESCAPE_CONVERSION[chr]
+        escaped = sprintf("\\u%04x", chr[0]) unless escaped
+        escaped
+      end
+      "\\\"#{str}\\\""
+    end
+  end
+
+  def build_value(obj, level)
+    case obj
+    when Integer, TrueClass, FalseClass then obj.to_s
+    when Float    then raise ERR_NaN unless obj.finite? || (obj = @nan) ; obj.to_s
+    when NilClass then 'null'
+    when Array    then build_array(obj, level + 1)
+    when Hash     then build_object(obj, level + 1)
+    else               escape(obj)
+    end
+  end
+
+  def build_array(obj, level)
+    raise ERR_NestIsTooDeep if level >= @max_nest
+    '[' + obj.map { |item| build_value(item, level) }.join(',') + ']'
+  end
+
+  def build_object(obj, level)
+    raise ERR_NestIsTooDeep if level >= @max_nest
+    '{' + obj.map do |item|
+      "#{build_value(item[0].to_s,level)}:#{build_value(item[1],level)}"
+    end.join(',') + '}'
+  end
+
+end

data/lib/groonga_adapter/local_index.rb

@@ -141,6 +141,8 @@ module DataMapper
             return Groonga::Type::BOOL
           when "Serial"
             return Groonga::Type::UINT32
+          when "Text"
+            return Groonga::Type::TEXT
           end
         else
           return Groonga::Type::SHORT_TEXT

data/lib/groonga_adapter/remote_index.rb

@@ -1,4 +1,5 @@
 require 'json'
+require 'nkf'
 
 module DataMapper
   module Adapters
@@ -9,6 +10,7 @@ module DataMapper
       def initialize(options)
         @context = Groonga::Context.default
         @context.connect(:host => options[:host], :port => options[:port])
+        @jsonbuilder = JsonBuilder.new
         # request "status" # <- TODO check connection with status command
       end
 
@@ -17,8 +19,8 @@ module DataMapper
         doc_id = doc[:id] #doc.delete(:id)
         record = []
         record << doc.update("_key" => doc_id)
-        json = JSON.generate record
-        res = request "load --table #{table_name} --values #{Unicode.unescape(json.gsub(/"/, '\"').gsub(/\s/, '\ '))}"
+        values = escape_value record
+        res = request "load --table #{table_name} --values #{values}"
         result = GroongaResult::Count.new res
         if result.success? && result.count > 0
           return doc
@@ -37,9 +39,20 @@ module DataMapper
       #              [offset [limit [drilldown [drilldown_sortby [drilldown_output_columns
       #                           [drilldown_offset [drilldown_limit [output_type]]]]]]]]]]]]]]
       def search(table_name, grn_query, grn_sort=[], options={})
+        # In case of columns contain long string, groonga return 10 results with no-limit..
+        # --limit #{num} option is needed to get all result.
+        # because of this, current remote_index request at twice.
         sort_by, offset, limit = parse_grn_sort grn_sort
-        remote_query = (grn_query.empty?) ? "" : "--query #{grn_query}"
-        remote_sort_by   = (sort_by.empty?) ? "" : "--sort-by #{sort_by}"
+        remote_query     = (grn_query.empty?) ? "" : "--query #{grn_query}"
+        remote_sort_by   = (sort_by.empty?)   ? "" : "--sortby #{sort_by}"
+        # 1st request for get count.
+
+        if limit.nil? or limit == -1
+          count = GroongaResult::List.new(request "select #{table_name} #{remote_query} #{remote_sort_by} --output_columns _id")
+          limit = count.size
+        end
+
+        # 2nd request for get ids.
         res = request "select #{table_name} #{remote_query} #{remote_sort_by} --offset #{offset} --limit #{limit}"
         list = GroongaResult::List.new res
         if list.success?
@@ -109,12 +122,66 @@ module DataMapper
 
       protected
 
+      def escape_value record
+        record.map! {|r|
+          tmp = {}
+          r.each {|k,v|
+            if !v.nil?
+              if v.is_a? String
+                tmp[k] = NKF.nkf('-wZ1',v)
+                tmp[k].gsub!(/\\/, '\\\\\\')
+                tmp[k].gsub!(/"/) { |c| "\\\\\\#{c}" }
+              else
+                tmp[k] = v.to_s
+              end
+            end
+          }
+          tmp
+        }
+        json = @jsonbuilder.build record
+        json.gsub!(/'|\s|\(|\)|\t/) { |c| "\\#{c}" }
+        json
+      end
+
       def err_code(res)
         return if res.nil?
         code = res[0][0]
         code
       end
 
+      def trans_type(dmtype)
+        case dmtype.to_s
+        when 'String'
+          return 'ShortText'
+        when 'Text'
+          return 'Text'
+        when 'Float'
+          return 'Float'
+        when 'Bool'
+          return 'Bool'
+        when 'Boolean'
+          return 'Bool'
+        when 'Integer'
+          return 'Int32'
+        when 'BigDecimal'
+          return 'Int64'
+        when 'Time'
+          return 'Time'
+        when /^DataMapper::Types::(.+)$/
+          case $1
+          when "Boolean"
+            return 'Bool'
+          when "Serial"
+            return 'Int32'
+          when "Text"
+            return "Text"
+          end
+        else
+          return 'ShortText'
+        end
+      end
+
+
       def create_term_table(table_name, key_prop="ShortText", tokenizer="TokenBigram")
         res = request "table_create #{table_name} TABLE_PAT_KEY|KEY_NORMALIZE #{key_prop} Void #{tokenizer}"
         throw "Fale to create term table." unless err_code(res) == 0 || err_code(res) == -22
@@ -159,35 +226,6 @@ module DataMapper
         [ sort_str, options[:offset], options[:limit] ]
       end
 
-      def trans_type(dmtype)
-        case dmtype.to_s
-        when 'String'
-          return 'ShortText'
-        when 'Text'
-          return 'LongText'
-        when 'Float'
-          return 'Float'
-        when 'Bool'
-          return 'Bool'
-        when 'Boolean'
-          return 'Bool'
-        when 'Integer'
-          return 'Int32'
-        when 'BigDecimal'
-          return 'Int64'
-        when 'Time'
-          return 'Time'
-        when /^DataMapper::Types::(.+)$/
-          case $1
-          when "Boolean"
-            return 'Bool'
-          when "Serial"
-            return 'Int32'
-          end
-        else
-          return 'ShortText'
-        end
-      end
     end # class GroongaAdapter::RemoteIndex
   end # module Adapters
 end # module DataMapper

data/lib/groonga_adapter.rb

@@ -7,3 +7,4 @@ require 'groonga_adapter/remote_result'
 require 'groonga_adapter/repository_ext'
 require 'groonga_adapter/model_ext'
 require 'groonga_adapter/unicode_ext'
+require 'groonga_adapter/jsonparser'

data/spec/shared/search_example.rb

@@ -3,13 +3,13 @@ shared_examples_for 'as is_search plugin' do
   before(:each) do
     DataMapper.setup(:default, "sqlite3::memory:")
     DataMapper.setup(:search, "groonga://#{index_path}")
-    DataMapper::Logger.new($stderr, :debug)
+#    DataMapper::Logger.new($stderr, :debug)
     Object.send(:remove_const, :Image) if defined?(Image)
     class ::Image
       include DataMapper::Resource
       property :id, Serial
       property :title, String
-
+      property :description, Text
       is :searchable # this defaults to :search repository, you could also do
     end
 
@@ -61,4 +61,15 @@ shared_examples_for 'as is_search plugin' do
     # Story.fulltext_search("John").should == [story, story2] # <--- Crash on local index.
     Story.fulltext_search("author:@John").should == [story, story2]
   end
+
+  it 'should return all result when there is more than 10 result' do
+    suffix = (0..10).map {|i| "long description." }
+    (0..157).each do |num|
+      img = Image.create(:title => "Picture_#{num}", :description => "#{suffix}")
+      img.save
+    end
+    results = Image.search(:title.like => "Picture")
+    results.size.should == 158
+  end
 end
+

metadata

@@ -6,8 +6,8 @@ version: !ruby/object:Gem::Version
   - 0
   - 1
   - 0
-  - pre
-  version: 0.1.0.pre
+  - pre2
+  version: 0.1.0.pre2
 platform: ruby
 authors: 
 - hiroyuki
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-04-08 00:00:00 +09:00
+date: 2010-04-15 00:00:00 +09:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -105,6 +105,7 @@ files:
 - examples/basic.rb
 - lib/groonga_adapter.rb
 - lib/groonga_adapter/adapter.rb
+- lib/groonga_adapter/jsonparser.rb
 - lib/groonga_adapter/local_index.rb
 - lib/groonga_adapter/model_ext.rb
 - lib/groonga_adapter/remote_index.rb
@@ -120,7 +121,7 @@ files:
 - spec/specs/remote_result_spec.rb
 - spec/specs/search_spec.rb
 has_rdoc: true
-homepage: http://github.com/hryk/dm-groonga-adapter
+homepage: http://hryk.github.com/dm-groonga-adapter
 licenses: []
 
 post_install_message: