embulk 0.8.35 → 0.8.36

data/lib/embulk/guess/schema_guess.rb

@@ -105,7 +105,7 @@ module Embulk::Guess
       FALSE_STRINGS = Hash[%w[
         false False FALSE
         no No NO
-        f N n N
+        f F n N
         off Off OFF
       ].map {|k| [k, true] }]
 

data/lib/embulk/input_plugin.rb

@@ -34,6 +34,10 @@ module Embulk
     def init
     end
 
+    def self.create_page_builder(task_source: nil, schema: nil, processor_index: nil, java_page_output: nil)
+      return PageBuilder.new(schema, java_page_output)
+    end
+
     def run
       raise NotImplementedError, "InputPlugin#run must be implemented"
     end
@@ -95,7 +99,10 @@ module Embulk
       def run(java_task_source, java_schema, processor_index, java_output)
         task_source = DataSource.from_java(java_task_source)
         schema = Schema.from_java(java_schema)
-        page_builder = PageBuilder.new(schema, java_output)
+        page_builder = @ruby_class.create_page_builder(task_source: task_source,
+                                                       schema: schema,
+                                                       processor_index: processor_index,
+                                                       java_page_output: java_output)
         begin
           task_report_hash = @ruby_class.new(task_source, schema, processor_index, page_builder).run
           return DataSource.from_ruby_hash(task_report_hash).to_java

data/lib/embulk/page_builder.rb

@@ -1,15 +1,48 @@
 module Embulk
 
   org.embulk.spi.util.dynamic.AbstractDynamicColumnSetter.module_eval do
-    alias_method(:set, :setRubyObject)
+    def set(ruby_object)
+      # Using |java_send| so that it calls Java's correct overloaded |set| method, not Ruby's itself.
+      if ruby_object.nil?
+        self.java_send(:setNull, [])
+      elsif ruby_object == true || ruby_object == false
+        self.java_send(:set, [::Java::boolean], ruby_object.to_java(:boolean))
+      elsif ruby_object.kind_of?(Integer)
+        self.java_send(:set, [::Java::long], ruby_object.to_java(:long))
+      elsif ruby_object.kind_of?(Float)
+        self.java_send(:set, [::Java::double], ruby_object.to_java(:double))
+      elsif ruby_object.kind_of?(String)
+        self.java_send(:set, [::Java::java.lang.String], ruby_object.to_java(::Java::java.lang.String))
+      elsif ruby_object.kind_of?(Time)
+        self.java_send(:set, [::Java::org.embulk.spi.time.Timestamp],
+                       ::Java::org.embulk.spi.time.Timestamp.ofEpochSecond(ruby_object.to_i, ruby_object.nsec))
+      else
+        self.java_send(:set, [::Java::org.msgpack.value.Value],
+                       ::Java::org.msgpack.core.MessagePack.newDefaultUnpacker(ruby_object.to_msgpack.to_java_bytes).unpackValue())
+      end
+    end
   end
 
   class PageBuilder
-    def initialize(schema, java_page_output)
+    def initialize(schema, java_page_output=nil, java_dynamic_page_builder: nil)
+      if java_page_output.nil? && java_dynamic_page_builder.nil?
+        raise ArgumentError, 'PageBuilder.new must take either of java_page_output or java_dynamic_page_builder'
+      end
+      if java_page_output && java_dynamic_page_builder
+        raise ArgumentError, 'PageBuilder.new must not take both java_page_output and java_dynamic_page_builder'
+      end
+
       require 'msgpack'  # used at DynamicPageBuilder.set(Value)
-      # TODO get task as an argument
-      task = Java::SPI::Exec.newConfigSource.load_config(Java::DynamicPageBuilder::BuilderTask.java_class)
-      @page_builder = Java::DynamicPageBuilder.new(task, Java::Injected::BufferAllocator, schema.to_java, java_page_output)
+
+      if java_page_output
+        # TODO get task as an argument
+        task = Java::SPI::Exec.newConfigSource.load_config(Java::DynamicPageBuilder::BuilderTask.java_class)
+        @page_builder = Java::DynamicPageBuilder.createWithTimestampMetadataFromBuilderTask(
+            task, Java::Injected::BufferAllocator, schema.to_java, java_page_output)
+      else
+        @page_builder = java_dynamic_page_builder
+      end
+
       @schema = schema
     end
 

data/lib/embulk/schema.rb

@@ -20,7 +20,6 @@ module Embulk
       each do |column|
         idx = column.index
         column_script =
-          "value_api = ::Embulk::Java::SPI::Json::RubyValueApi\n" <<
           "if reader.isNull(#{idx})\n" <<
           "record << nil\n" <<
           "else\n" <<
@@ -34,9 +33,9 @@ module Embulk
           when :string
             "record << reader.getString(#{idx})"
           when :timestamp
-            "record << reader.getTimestamp(#{idx}).getRubyTime(JRuby.runtime)"
+            "record << (java_timestamp = reader.getTimestamp(#{idx}); ruby_time = Java::org.jruby.RubyTime.new(JRuby.runtime, JRuby.runtime.getClass('Time'), Java::org.joda.time.DateTime.new(java_timestamp.toEpochMilli())).gmtime().to_java(Java::org.jruby.RubyTime); ruby_time.setNSec(java_timestamp.getNano()); ruby_time)"
           when :json
-            "record << MessagePack.unpack(value_api.toMessagePack(JRuby.runtime, reader.getJson(#{idx})))"
+            "record << MessagePack.unpack(String.from_java_bytes((::Java::org.msgpack.core.MessagePack.newDefaultBufferPacker()).packValue(reader.getJson(#{idx})).toMessageBuffer().toByteArray()))"
           else
             raise "Unknown type #{column.type.inspect}"
           end <<
@@ -49,7 +48,6 @@ module Embulk
 
       record_writer_script = "lambda do |builder,record|\n"
       record_writer_script << "java_timestamp_class = ::Embulk::Java::Timestamp\n"
-      record_writer_script << "value_api = ::Embulk::Java::SPI::Json::RubyValueApi\n"
       each do |column|
         idx = column.index
         column_script =
@@ -66,9 +64,10 @@ module Embulk
           when :string
             "builder.setString(#{idx}, record[#{idx}])"
           when :timestamp
-            "builder.setTimestamp(#{idx}, java_timestamp_class.fromRubyTime(record[#{idx}]))"
+            # It was originally expecting that `record[#{idx}]` was a Ruby Time object. Does it really happen?
+            "(ruby_time = record[#{idx}].to_java(Java::org.jruby.RubyTime); msec = ruby_time.getDateTime().getMillis(); builder.setTimestamp(#{idx}, java_timestamp_class.ofEpochSecond(msec / 1000, ruby_time.getNSec() + (msec % 1000) * 1000000)))"
           when :json
-            "builder.setJson(#{idx}, value_api.fromMessagePack(MessagePack.pack(record[#{idx}])))"
+            "builder.setJson(#{idx}, ::Java::org.msgpack.core.MessagePack.newDefaultUnpacker(MessagePack.pack(record[#{idx}]).to_java_bytes).unpackValue())"
           else
             raise "Unknown type #{column.type.inspect}"
           end <<

data/lib/embulk/version.rb

@@ -3,7 +3,7 @@
 module Embulk
   @@warned = false
 
-  VERSION_INTERNAL = '0.8.35'
+  VERSION_INTERNAL = '0.8.36'
 
   DEPRECATED_MESSAGE = 'Embulk::VERSION in (J)Ruby is deprecated. Use org.embulk.EmbulkVersion::VERSION instead. If this message is from a plugin, please tell this to the author of the plugin!'
   def self.const_missing(name)

data/test/guess/test_schema_guess.rb

@@ -26,4 +26,22 @@ class SchemaGuessTest < ::Test::Unit::TestCase
         {"a" => "12345678"},
       ]))
   end
+
+  def test_boolean
+    %w[
+      true false t f
+      yes no y n
+      on off
+    ].each do |str|
+      # If at least one of three kinds of boolean strings (i.e., downcase, upcase, capitalize) is
+      # mistakenly detected as "string," the guesser concludes the column type is "string."
+      assert_equal(
+        [C.new(0, "a", :boolean)],
+        G.from_hash_records([
+          {"a" => str.downcase},
+          {"a" => str.upcase},
+          {"a" => str.capitalize},
+        ]))
+    end
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk
 version: !ruby/object:Gem::Version
-  version: 0.8.35
+  version: 0.8.36
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-09-26 00:00:00.000000000 Z
+date: 2017-10-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: jruby-jars
@@ -115,10 +115,10 @@ files:
 - classpath/commons-compress-1.10.jar
 - classpath/commons-lang-2.4.jar
 - classpath/commons-lang3-3.4.jar
-- classpath/embulk-cli-0.8.35.jar
-- classpath/embulk-core-0.8.35.jar
-- classpath/embulk-jruby-strptime-0.8.35.jar
-- classpath/embulk-standards-0.8.35.jar
+- classpath/embulk-cli-0.8.36.jar
+- classpath/embulk-core-0.8.36.jar
+- classpath/embulk-jruby-strptime-0.8.36.jar
+- classpath/embulk-standards-0.8.36.jar
 - classpath/guava-18.0.jar
 - classpath/guice-4.0.jar
 - classpath/guice-bootstrap-0.1.1.jar
@@ -525,6 +525,7 @@ files:
 - embulk-docs/src/release/release-0.8.33.rst
 - embulk-docs/src/release/release-0.8.34.rst
 - embulk-docs/src/release/release-0.8.35.rst
+- embulk-docs/src/release/release-0.8.36.rst
 - embulk-docs/src/release/release-0.8.4.rst
 - embulk-docs/src/release/release-0.8.5.rst
 - embulk-docs/src/release/release-0.8.6.rst