embulk 0.8.0 → 0.8.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0b3174c7826cebe5dd528fb3aa5f577367f30320
-  data.tar.gz: a9d08926396181e890031f8e57cabf9a2db98e18
+  metadata.gz: a189a85d312e0621daf89d790baf4fa45d0e0125
+  data.tar.gz: d7355b10940135fce461332f40d6f8180b3c41c2
 SHA512:
-  metadata.gz: 3b25dbabad3667aa97ec211c65289f0668d4e9c38ca069012436748be8415368078051b0fffb0bf45b3c40e71473e75136bf1b2fb7b79ca05fb67440aa5ba12a
-  data.tar.gz: 5842d8686dcbf6e2e52a03a2f56d32a2830657d8b70f4101cb2294281db46b9bfc88bb6ef86fec9094e7edef1c2c3287fd240613d3cf887607049c85533ce4e3
+  metadata.gz: 09609e12b41ce696b190be750cee3687fcd0f9f04d24597345415080c8ffdd7319ed33129584f8c500a6893c2cdb4cba86d7ddeb34bab0cf4727c2beb3388336
+  data.tar.gz: b46a461b1f845e7020d6ac5c3553ed4ca1a764ae0ae80fccc52d4bd9a35db11fe24ec6ab721fd3ae841f365d6d92fc0f40213d2053d5c9a1c9d74ae6bb8e62f6

data/build.gradle

@@ -16,7 +16,7 @@ def release_projects = [project(":embulk-core"), project(":embulk-standards")]
 
 allprojects {
     group = 'org.embulk'
-    version = '0.8.0'
+    version = '0.8.1'
 
     ext {
         jrubyVersion = '9.0.4.0'

data/embulk-core/build.gradle

@@ -44,6 +44,7 @@ dependencies {
     compile 'com.ibm.icu:icu4j:54.1.1'
 
     gems 'rubygems:bundler:1.10.6'
+    gems 'rubygems:msgpack:0.7.4'
     gems 'rubygems:liquid:3.0.6'
 }
 

data/embulk-core/src/main/java/org/embulk/command/PreviewPrinter.java

@@ -11,6 +11,7 @@ import org.embulk.spi.Schema;
 import org.embulk.spi.time.Timestamp;
 import org.embulk.spi.Page;
 import org.embulk.spi.util.Pages;
+import org.msgpack.value.Value;
 
 public abstract class PreviewPrinter
         implements Closeable
@@ -68,6 +69,8 @@ public abstract class PreviewPrinter
             return obj.toString();
         } else if (obj instanceof Timestamp) {
             return obj.toString();
+        } else if (obj instanceof Value) {
+            return obj.toString();
         } else {
             return modelManager.writeObject(obj);
         }

data/embulk-core/src/main/java/org/embulk/spi/json/RubyValueApi.java

@@ -9,6 +9,7 @@ import org.msgpack.core.buffer.MessageBuffer;
 import org.msgpack.value.Value;
 import org.jruby.Ruby;
 import org.jruby.RubyString;
+import org.jruby.runtime.builtin.IRubyObject;
 import org.jruby.util.ByteList;
 import org.jcodings.specific.ASCIIEncoding;
 
@@ -52,4 +53,10 @@ public class RubyValueApi
             throw Throwables.propagate(ex);
         }
     }
+
+    public static Value toValue(Ruby runtime, IRubyObject object)
+    {
+        RubyString string = (RubyString) object.callMethod(runtime.getCurrentContext(), "to_msgpack");
+        return fromMessagePack(string);
+    }
 }

data/embulk-core/src/main/java/org/embulk/spi/util/DynamicColumnSetter.java

@@ -1,6 +1,7 @@
 package org.embulk.spi.util;
 
 import org.embulk.spi.time.Timestamp;
+import org.msgpack.value.Value;
 
 public interface DynamicColumnSetter
 {
@@ -15,4 +16,6 @@ public interface DynamicColumnSetter
     void set(String value);
 
     void set(Timestamp value);
+
+    void set(Value value);
 }

data/embulk-core/src/main/java/org/embulk/spi/util/dynamic/AbstractDynamicColumnSetter.java

@@ -12,6 +12,7 @@ import org.embulk.spi.PageBuilder;
 import org.embulk.spi.Column;
 import org.embulk.spi.util.DynamicColumnSetter;
 import org.embulk.spi.time.Timestamp;
+import org.embulk.spi.json.RubyValueApi;
 import org.msgpack.value.Value;
 
 public abstract class AbstractDynamicColumnSetter
@@ -76,7 +77,7 @@ public abstract class AbstractDynamicColumnSetter
             int nano = (int) ((msec % 1000) * 1000000 + nsec % 1000000000);
             set(Timestamp.ofEpochSecond(sec, nano));
         } else {
-            throw rubyObject.getRuntime().newTypeError("cannot convert instance of " + rubyObject.getMetaClass() + " to nil, true, false, Integer, Float, String, or Time");
+            set(RubyValueApi.toValue(rubyObject.getRuntime(), rubyObject));
         }
     }
 }

data/embulk-core/src/main/java/org/embulk/spi/util/dynamic/BooleanColumnSetter.java

@@ -53,7 +53,7 @@ public class BooleanColumnSetter
         if (TRUE_STRINGS.contains(v)) {
             pageBuilder.setBoolean(column, true);
         } else {
-            defaultValue.setDouble(pageBuilder, column);
+            defaultValue.setBoolean(pageBuilder, column);
         }
     }
 
@@ -66,6 +66,6 @@ public class BooleanColumnSetter
     @Override
     public void set(Value v)
     {
-        pageBuilder.setJson(column, v);
+        defaultValue.setBoolean(pageBuilder, column);
     }
 }

data/embulk-core/src/main/java/org/embulk/spi/util/dynamic/DoubleColumnSetter.java

@@ -63,6 +63,6 @@ public class DoubleColumnSetter
     @Override
     public void set(Value v)
     {
-        pageBuilder.setJson(column, v);
+        defaultValue.setDouble(pageBuilder, column);
     }
 }

data/embulk-core/src/main/java/org/embulk/spi/util/dynamic/LongColumnSetter.java

@@ -67,12 +67,12 @@ public class LongColumnSetter
     @Override
     public void set(Timestamp v)
     {
-        pageBuilder.setDouble(column, v.getEpochSecond());
+        pageBuilder.setLong(column, v.getEpochSecond());
     }
 
     @Override
     public void set(Value v)
     {
-        pageBuilder.setJson(column, v);
+        defaultValue.setLong(pageBuilder, column);
     }
 }

data/embulk-core/src/main/java/org/embulk/spi/util/dynamic/StringColumnSetter.java

@@ -58,6 +58,6 @@ public class StringColumnSetter
     @Override
     public void set(Value v)
     {
-        pageBuilder.setJson(column, v);
+        pageBuilder.setString(column, v.toJson());
     }
 }

data/embulk-core/src/main/java/org/embulk/spi/util/dynamic/TimestampColumnSetter.java

@@ -67,6 +67,6 @@ public class TimestampColumnSetter
     @Override
     public void set(Value v)
     {
-        pageBuilder.setJson(column, v);
+        defaultValue.setTimestamp(pageBuilder, column);
     }
 }

data/embulk-docs/src/release.rst

@@ -4,6 +4,7 @@ Release Notes
 .. toctree::
     :maxdepth: 1
 
+    release/release-0.8.1
     release/release-0.8.0
     release/release-0.7.10
     release/release-0.7.9

data/embulk-docs/src/release/release-0.8.0.rst

@@ -1,28 +1,32 @@
 Release 0.8.0
 ==================================
 
-General Changes
+JSON type
 ------------------
 
-* Added JSON type support.
+Embulk v0.8.0 added JSON type support.
+
+* A column with ``json`` type can represent nested values such as maps or arrays. This is useful when both input and output support dynamically-typed values.
 
-  * A column with ``json`` type can represent nested values such as maps or arrays. This is useful when both input and output support dynamically-typed values.
+* **IMPORTANT**: If input plugin uses JSON type but output plugin is compiled with an old embulk (< 0.8.0), a bulk load transaction fails with a confusing error message. To avoid this issue, please run ``embulk migrate /path/to/embulk-plugin-directory`` to upgrade plugin code, and use the latest plugin. This problem doesn't happen if input plugin doesn't use json type.
 
-  * **IMPORTANT**: If input plugin uses JSON type but output plugin is compiled with an old embulk (< 0.8.0), a bulk load transaction fails with a confusing error message. To avoid this issue, please run ``embulk migrate /path/to/embulk-plugin-directory`` to upgrade plugin code, and use the latest plugin. This problem doesn't happen if input plugin doesn't use json type.
+* Filter plugins to process JSON types are not ready yet. Expected plugins are for example, flatten a json column into statically-typed columns with guess plugin, extracting a value from a json column using an expression (such as JSONPath) and set it to another column, or building a json column by copying values from other columns.
 
-  * Filter plugins to process JSON types are not ready yet. Expected plugins are for example, flatten a json column into statically-typed columns with guess plugin, extracting a value from a json column using an expression (such as JSONPath) and set it to another column, or building a json column by copying values from other columns.
+Page scattering
+------------------
 
-* Local executor plugin (the default executor) runs multiple tasks even if there is only 1 input task. This improves performance a lot especially if input is a single huge file.
+Local executor plugin (the default executor) runs multiple tasks even if there is only 1 input task. This improves performance a lot especially if input is a single huge file.
 
-  * Its mechanism is that the executor creates 2, 3, 4, or more number of output tasks for each input task. Page chunks from a input task is scattered to output tasks. All of the tasks run in parallel using threads. This feature is called "page scattering".
+* Its mechanism is that the executor creates 2, 3, 4, or more number of output tasks for each input task. Page chunks from a input task is scattered to output tasks. All of the tasks run in parallel using threads. This feature is called "page scattering".
 
-  * Added ``min_output_tasks`` option at ``exec:`` section. Default is 1x of available CPU cores. Page scattering is enabled if number of input tasks is less than this ``min_output_tasks`` option. Setting larger number here is useful if embulk doesn't use multi-threading with enough concurrency due to too few number of input tasks.
+* Added ``min_output_tasks`` option at ``exec:`` section. Default is 1x of available CPU cores. Page scattering is enabled if number of input tasks is less than this ``min_output_tasks`` option. Setting larger number here is useful if embulk doesn't use multi-threading with enough concurrency due to too few number of input tasks.
 
-  * Added ``max_threads`` option at ``exec:`` section. Default is 2x of availalbe CPU cores. This option controls maximum concurrency. Setting smaller number here is useful if too many threads make the destination or source storage overloaded. Setting larger number here is useful if CPU utilization is too low due to high latency.
+* Added ``max_threads`` option at ``exec:`` section. Default is 2x of availalbe CPU cores. This option controls maximum concurrency. Setting smaller number here is useful if too many threads make the destination or source storage overloaded. Setting larger number here is useful if CPU utilization is too low due to high latency.
 
-  * The results of output transaction will be deterministic. There're no randomness that depends on timing. However, task assignment changes if ``min_output_tasks`` changes. If you need deterministic results regardless of machines that may have different number of CPU cores, please add ``min_output_tasks`` option to ``exec:`` section. Setting 1 there will disable page scattering completely.
+* The results of output transaction will be deterministic. There're no randomness that depends on timing. However, task assignment changes if ``min_output_tasks`` changes. If you need deterministic results regardless of machines that may have different number of CPU cores, please add ``min_output_tasks`` option to ``exec:`` section. Setting 1 there will disable page scattering completely.
 
-* Upraded JRuby version to 9.0.4.0.
+General Changes
+------------------
 
 * YAML configuration parser uses stricter rules when it converts type of a non-quoted strings.
 
@@ -32,6 +36,8 @@ General Changes
 
   * On, Off, Yes, and No (case-insensitive) will be a string instead of boolean. Only true, True, false, False are recognized as a boolean.
 
+* Upraded JRuby version to 9.0.4.0.
+
 Java Plugin API
 ------------------
 

data/embulk-docs/src/release/release-0.8.1.rst

@@ -0,0 +1,18 @@
+Release 0.8.1
+==================================
+
+General Changes
+------------------
+
+* Ruby-based plugins use ``>=`` instead of ``~>`` to depend on bundler and embulk so that users can use both older plugins and newer plugins together. This assumes major versions of those dependencies most likely don't break backward compatibility.
+
+* Added bulit-in dependency to msgpack.gem v0.7.4 which is used to support json types at ruby plugins.
+
+* Fixed json value creation using ruby-based parser and input plugins.
+
+* Fixed preview of json types.
+
+
+Release Date
+------------------
+2016-01-13

data/embulk.gemspec

@@ -20,7 +20,8 @@ Gem::Specification.new do |gem|
   gem.has_rdoc      = false
 
   if RUBY_PLATFORM =~ /java/i
-    gem.add_dependency "bundler", '~> 1.10.6'
+    gem.add_dependency "bundler", '>= 1.10.6'
+    gem.add_dependency "msgpack", '~> 0.7.3'
     gem.add_dependency "liquid", '~> 3.0.6'
 
     # For embulk/guess/charset.rb. See also embulk-core/build.gradle

data/lib/embulk/command/embulk_migrate_plugin.rb

@@ -128,10 +128,12 @@ EOF
     if from_ver <= version("0.1.0")
       # add add_development_dependency
       migrator.insert_line("**/*.gemspec", /([ \t]*\w+)\.add_development_dependency/) {|m|
-        "#{m[1]}.add_development_dependency 'embulk', ['~> #{Embulk::VERSION}']"
+        "#{m[1]}.add_development_dependency 'embulk', ['>= #{Embulk::VERSION}']"
       }
     else
-      migrator.replace("**/*.gemspec", /add_(?:development_)?dependency\s+\W+embulk\W+\s+([\d\.]+)\W+/, Embulk::VERSION)
+      unless migrator.replace("**/*.gemspec", /add_(?:development_)?dependency\s+\W+embulk\W+\s*(\~\>\s*[\d\.]+)\W+/, ">= #{Embulk::VERSION}")
+        migrator.replace("**/*.gemspec", /add_(?:development_)?dependency\s+\W+embulk\W+\s*([\d\.]+)\W+/, Embulk::VERSION)
+      end
     end
   end
 

data/lib/embulk/data/new/ruby/gemspec.erb

@@ -14,7 +14,7 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   #spec.add_dependency 'YOUR_GEM_DEPENDENCY', ['~> YOUR_GEM_DEPENDENCY_VERSION']
-  spec.add_development_dependency 'embulk', ['~> <%= Embulk::VERSION %>']
-  spec.add_development_dependency 'bundler', ['~> 1.0']
+  spec.add_development_dependency 'embulk', ['>= <%= Embulk::VERSION %>']
+  spec.add_development_dependency 'bundler', ['>= 1.10.6']
   spec.add_development_dependency 'rake', ['>= 10.0']
 end

data/lib/embulk/java/imports.rb

@@ -19,23 +19,23 @@ module Embulk::Java
 
   module SPI
     module Time
-      include_package 'org.embulk.time'
+      include_package 'org.embulk.spi.time'
     end
 
     module Json
-      include_package 'org.embulk.json'
+      include_package 'org.embulk.spi.json'
     end
 
     module Type
-      include_package 'org.embulk.type'
+      include_package 'org.embulk.spi.type'
     end
 
     module Unit
-      include_package 'org.embulk.unit'
+      include_package 'org.embulk.spi.unit'
     end
 
     module Util
-      include_package 'org.embulk.util'
+      include_package 'org.embulk.spi.util'
     end
 
     include_package 'org.embulk.spi'

data/lib/embulk/page_builder.rb

@@ -6,6 +6,7 @@ module Embulk
 
   class PageBuilder
     def initialize(schema, java_page_output)
+      require 'msgpack'  # used at DynamicPageBuilder.set(Value)
       # TODO get task as an argument
       task = Java::SPI::Exec.newConfigSource.load_config(Java::DynamicPageBuilder::BuilderTask.java_class)
       @page_builder = Java::DynamicPageBuilder.new(task, Java::Injected::BufferAllocator, schema.to_java, java_page_output)

data/lib/embulk/version.rb

@@ -1,3 +1,3 @@
 module Embulk
-  VERSION = '0.8.0'
+  VERSION = '0.8.1'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk
 version: !ruby/object:Gem::Version
-  version: 0.8.0
+  version: 0.8.1
 platform: ruby
 authors:
 - Sadayuki Furuhashi
@@ -107,8 +107,8 @@ files:
 - classpath/bval-jsr303-0.5.jar
 - classpath/commons-beanutils-core-1.8.3.jar
 - classpath/commons-lang3-3.1.jar
-- classpath/embulk-core-0.8.0.jar
-- classpath/embulk-standards-0.8.0.jar
+- classpath/embulk-core-0.8.1.jar
+- classpath/embulk-standards-0.8.1.jar
 - classpath/guava-18.0.jar
 - classpath/guice-4.0.jar
 - classpath/guice-bootstrap-0.1.1.jar
@@ -418,6 +418,7 @@ files:
 - embulk-docs/src/release/release-0.7.8.rst
 - embulk-docs/src/release/release-0.7.9.rst
 - embulk-docs/src/release/release-0.8.0.rst
+- embulk-docs/src/release/release-0.8.1.rst
 - embulk-standards/build.gradle
 - embulk-standards/src/main/java/org/embulk/standards/CsvFormatterPlugin.java
 - embulk-standards/src/main/java/org/embulk/standards/CsvParserPlugin.java