groonga-command 1.4.5 → 1.4.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1e8e7e4033e77829c575d2ef6832f8ad5c8d8883c0279d5fa56b8f368f66d671
-  data.tar.gz: ba868ba8b7f5a04b17ace4c18fe438cd7ecd2e3581d072f8cda4431d09f2ca84
+  metadata.gz: 04a4658a7058ae319eef6558e4b0890cf63f1374cbc18ca6620f89c1713d1079
+  data.tar.gz: 585f68bee46bc0455b5b0be0056636b9b61518bf53354513081191ceb7156996
 SHA512:
-  metadata.gz: fcde76968b9b36f013419f3a38ece4caf3e81550b272a3bcf906b4c9e011be6a4d008a91ef79f7578ac0fd9365233994c1cbbf08c0ac0c25c9babfab75887193
-  data.tar.gz: 6afc4ddf0367b6de2e38de283176f92f82bd95c03d18ed25113983f16b368b207b1728db10d7fb98699766ba1d63c4f8022eb2bbd86f1c95dfa309701cda6f40
+  metadata.gz: 6141e8378ce9f64024c481af745a519e598c40b15b313447537d78030f24ce8d7a84ca1f2df52ad06176a1153792f2edbde20cf32d4dce40ff1b043a53780ba3
+  data.tar.gz: eee5697534989943e5d0b038b14af59eea0a811c9aab09e2bf25d4251c7783efac899846c6a631e3de291b16d95fc33f9f88fe5e318be87eba61c7331870deba

data/doc/text/news.md

@@ -1,5 +1,11 @@
 # News
 
+## 1.4.6: 2020-01-10
+
+### Improvements
+
+  * {Groonga::Command::Load#build_arrow_table}: Added.
+
 ## 1.4.5: 2019-09-02
 
 ### Improvements

data/groonga-command.gemspec

@@ -1,6 +1,6 @@
-# -*- mode: ruby; coding: utf-8 -*-
+# -*- ruby -*-
 #
-# Copyright (C) 2012-2013  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2012-2020  Sutou Kouhei <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -54,12 +54,13 @@ Gem::Specification.new do |spec|
 
   spec.add_runtime_dependency("json")
 
-  spec.add_development_dependency("test-unit")
-  spec.add_development_dependency("test-unit-notify")
-  spec.add_development_dependency("rake")
   spec.add_development_dependency("bundler")
   spec.add_development_dependency("packnga")
-  spec.add_development_dependency("yard")
+  spec.add_development_dependency("rake")
+  spec.add_development_dependency("red-arrow")
   spec.add_development_dependency("redcarpet")
+  spec.add_development_dependency("test-unit")
+  spec.add_development_dependency("test-unit-notify")
+  spec.add_development_dependency("yard")
 end
 

data/lib/groonga/command/load.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2012-2017  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2012-2020  Sutou Kouhei <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -16,6 +16,11 @@
 
 require "json"
 
+begin
+  require "arrow"
+rescue LoadError
+end
+
 require "groonga/command/base"
 
 module Groonga
@@ -66,6 +71,23 @@ module Groonga
         @columns ||= parse_columns(self[:columns])
       end
 
+      # Builds `Arrow::Table` for data of this `load` command.
+      #
+      # This requires red-arrow gem. If red-arrow gem isn't available,
+      # `NotImplementedError` is raised.
+      #
+      # @return [Arrow::Table, nil] `Arrow::Table` if there is one or more
+      #    records, `nil` otherwise.
+      #
+      # @since 1.4.6
+      def build_arrow_table
+        unless defined?(::Arrow)
+          raise NotImplementedError, "red-arrow is required"
+        end
+        builder = ArrowTableBuilder.new(columns, values)
+        builder.build
+      end
+
       # @return [Boolean] `true` if `output_ids` value is `"yes"`.
       #
       # @since 1.3.0
@@ -85,6 +107,147 @@ module Groonga
         return columns if columns.nil?
         columns.split(/\s*,\s*/)
       end
+
+      class ArrowTableBuilder
+        def initialize(columns, values)
+          @columns = columns
+          @values = values
+        end
+
+        def build
+          raw_table = build_raw_table
+          return nil if raw_table.empty?
+          build_arrow_table(raw_table)
+        end
+
+        private
+        def build_raw_table
+          raw_table = {}
+          if @values.first.is_a?(Array)
+            columns = @columns
+            if columns
+              records = @values
+            else
+              columns = @values.first
+              records = @values[1..-1]
+            end
+            records.each_with_index do |record, i|
+              columns.zip(record).each do |name, value|
+                raw_table[name] ||= []
+                raw_table[name][i] = value
+              end
+            end
+          else
+            @values.each_with_index do |record, i|
+              record.each do |name, value|
+                raw_table[name] ||= []
+                raw_table[name][i] = value
+              end
+            end
+            raw_table.each_key do |key|
+              if @values.size > raw_table[key].size
+                raw_table[key][@values.size - 1] = nil
+              end
+            end
+          end
+          raw_table
+        end
+
+        def build_arrow_table(raw_table)
+          arrow_fields = []
+          arrow_arrays = []
+          raw_table.each do |name, raw_array|
+            sample = raw_array.find {|element| not element.nil?}
+            case sample
+            when Array
+              data_type = nil
+              raw_array.each do |sub_raw_array|
+                next if sub_raw_array.nil?
+                data_type = detect_arrow_data_type(sub_raw_array)
+                break if data_type
+              end
+              data_type ||= :string
+              arrow_array = build_arrow_array(data_type, raw_array)
+            when Hash
+              arrow_array = build_arrow_array(arrow_weight_vector_data_type,
+                                              raw_array)
+            else
+              data_type = detect_arrow_data_type(raw_array) || :string
+              if data_type == :string
+                raw_array = raw_array.collect do |element|
+                  element&.to_s
+                end
+              end
+              data_type = Arrow::DataType.resolve(data_type)
+              arrow_array = data_type.build_array(raw_array)
+            end
+            arrow_fields << Arrow::Field.new(name,
+                                             arrow_array.value_data_type)
+            arrow_arrays << arrow_array
+          end
+          arrow_schema = Arrow::Schema.new(arrow_fields)
+          Arrow::Table.new(arrow_schema, arrow_arrays)
+        end
+
+        def prepare_raw_array(raw_array)
+          raw_array.collect do |element|
+            case element
+            when Array
+              prepare_raw_array(element)
+            when Hash
+              element.collect do |value, weight|
+                {
+                  "value" => value,
+                  "weight" => weight,
+                }
+              end
+            else
+              element
+            end
+          end
+        end
+
+        def build_arrow_array(data_type, raw_array)
+          arrow_list_field = Arrow::Field.new("item", data_type)
+          arrow_list_data_type = Arrow::ListDataType.new(arrow_list_field)
+          raw_array = prepare_raw_array(raw_array)
+          Arrow::ListArrayBuilder.build(arrow_list_data_type, raw_array)
+        end
+
+        def arrow_weight_vector_data_type
+          Arrow::StructDataType.new("value" => :string,
+                                    "weight" => :int32)
+        end
+
+        def detect_arrow_data_type(raw_array)
+          type = nil
+          raw_array.each do |element|
+            case element
+            when nil
+            when true, false
+              type ||= :boolean
+            when Integer
+              if element >= (2 ** 63)
+                type = nil if type == :int64
+                type ||= :uint64
+              else
+                type ||= :int64
+              end
+            when Float
+              type = nil if type == :int64
+              type ||= :double
+            when Hash
+              arrow_list_field =
+                Arrow::Field.new("item", arrow_weight_vector_data_type)
+              arrow_list_data_type = Arrow::ListDataType.new(arrow_list_field)
+              return arrow_list_data_type
+            else
+              return :string
+            end
+          end
+          type
+        end
+      end
     end
   end
 end

data/lib/groonga/command/version.rb

@@ -16,6 +16,6 @@
 
 module Groonga
   module Command
-    VERSION = "1.4.5"
+    VERSION = "1.4.6"
   end
 end

data/test/command/test-load.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2012-2017  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2012-2020  Sutou Kouhei <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -116,4 +116,57 @@ class LoadCommandTest < Test::Unit::TestCase
       end
     end
   end
+
+  sub_test_case("#build_arrow_table") do
+    def setup
+      omit("red-arrow is needed") unless defined?(Arrow)
+    end
+
+    sub_test_case("Array") do
+      def test_no_columns_argument
+        command = load_command({"values" => [
+                                  ["column1", "column2"],
+                                  ["value1", "value2"],
+                                ].to_json})
+        assert_equal(Arrow::Table.new({
+                                        "column1" => ["value1"],
+                                        "column2" => ["value2"],
+                                      }),
+                     command.build_arrow_table)
+      end
+
+      def test_columns_argument
+        command = load_command({"columns" => ["column1", "column2"].join(", "),
+                                "values" => [
+                                  ["value1", "value2"],
+                                ].to_json})
+        assert_equal(Arrow::Table.new({
+                                        "column1" => ["value1"],
+                                        "column2" => ["value2"],
+                                      }),
+                     command.build_arrow_table)
+      end
+    end
+
+    sub_test_case("Hash") do
+      def test_integer
+        command = load_command({"values" => [
+                                  {
+                                    "column1" => 1,
+                                    "column2" => 2,
+                                  },
+                                  {
+                                    "column1" => 10,
+                                    "column2" => 20,
+                                  },
+                                ].to_json})
+        columns = {
+          "column1" => Arrow::Int64Array.new([1, 10]),
+          "column2" => Arrow::Int64Array.new([2, 20]),
+        }
+        assert_equal(Arrow::Table.new(columns),
+                     command.build_arrow_table)
+      end
+    end
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: groonga-command
 version: !ruby/object:Gem::Version
-  version: 1.4.5
+  version: 1.4.6
 platform: ruby
 authors:
 - Kouhei Sutou
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-09-02 00:00:00.000000000 Z
+date: 2020-01-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: json
@@ -25,7 +25,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: test-unit
+  name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -39,7 +39,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: test-unit-notify
+  name: packnga
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -67,7 +67,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: bundler
+  name: red-arrow
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -81,7 +81,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: packnga
+  name: redcarpet
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -95,7 +95,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: yard
+  name: test-unit
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -109,7 +109,21 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: redcarpet
+  name: test-unit-notify
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: yard
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="