embulk 0.8.13 → 0.8.14

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a33de4d667fc4550e065bcdf0017ffda402ab93c
-  data.tar.gz: 86177d32e0887e0bade98b7778e6d59c9843bfb2
+  metadata.gz: 20a4f8e68f013bad287fa60a52982eacdad43667
+  data.tar.gz: 656680f1b59861633ae2f5575b46359de58c44b1
 SHA512:
-  metadata.gz: 93cb5c92f584ff98e2e58f59504fa2fc32b6478d575e72918e6b37801fc2147c3bc455f332a9d406b76a0f9ff528009313138362321bf2d33fb009a6d9bfd6df
-  data.tar.gz: 374201ee5dfad522c47079e7f95250f95ed64d8f8e2f6f16eacf2e18b588f24ec115e7555a431a9734ff55b8b64c7f360a1c204e5495aeda51e41c01875151d4
+  metadata.gz: 1f70f2fdcc4856b718fc4825bede8aa8b9e28022aed398df5e5a82b86754dfd9ef29e1b9ea3db92d2c9f0f1094cf0b9110550e9b2c4edfeaee67d0e18a2fe6a4
+  data.tar.gz: 678b00d7dc6cfa0703659c7fdf028aed9f7100246dbfbc186d5c4309f72f85a87e4ae4e0a45074996d2179e360a7169f57c3997bdc3348922518a14c64996f39

data/Gemfile.lock

@@ -1,13 +1,13 @@
 PATH
   remote: .
   specs:
-    embulk (0.8.9)
-      jruby-jars (= 9.1.2.0)
+    embulk (0.8.13)
+      jruby-jars (= 9.1.5.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    jruby-jars (9.1.2.0)
+    jruby-jars (9.1.5.0)
     kramdown (1.5.0)
     power_assert (0.2.2)
     rake (10.4.2)
@@ -27,4 +27,4 @@ DEPENDENCIES
   yard (~> 0.8.7)
 
 BUNDLED WITH
-   1.12.4
+   1.13.2

data/build.gradle

@@ -16,10 +16,10 @@ def release_projects = [project(":embulk-core"), project(":embulk-standards")]
 
 allprojects {
     group = 'org.embulk'
-    version = '0.8.13'
+    version = '0.8.14'
 
     ext {
-        jrubyVersion = '9.1.2.0'
+        jrubyVersion = '9.1.5.0'
     }
 
     apply plugin: 'java'

data/embulk-core/build.gradle

@@ -38,7 +38,7 @@ dependencies {
     compile 'joda-time:joda-time:2.9.2'
     compile 'io.netty:netty-buffer:5.0.0.Alpha1'
     compile 'org.fusesource.jansi:jansi:1.11'
-    compile 'org.msgpack:msgpack-core:0.8.8'
+    compile 'org.msgpack:msgpack-core:0.8.11'
 
     // For embulk/guess/charset.rb. See also embulk.gemspec
     compile 'com.ibm.icu:icu4j:54.1.1'

data/embulk-core/src/main/java/org/embulk/spi/Schema.java

@@ -31,14 +31,19 @@ public class Schema
         return new Builder();
     }
 
-    private final List<Column> columns;
+    private final ImmutableList<Column> columns;
 
     @JsonCreator
     public Schema(List<Column> columns)
     {
-        this.columns = columns;
+        this.columns = ImmutableList.copyOf(columns);
     }
 
+    /**
+     * Returns the list of Column objects.
+     *
+     * It always returns an immutable list.
+     */
     @JsonValue
     public List<Column> getColumns()
     {

data/embulk-core/src/main/java/org/embulk/spi/util/ResumableInputStream.java

@@ -32,7 +32,10 @@ public class ResumableInputStream
     private void reopen(Exception closedCause) throws IOException
     {
         if (in != null) {
-            in.close();
+            try {
+                in.close();
+            } catch (IOException ignored) {
+            }
             in = null;
         }
         in = reopener.reopen(offset, closedCause);

data/embulk-docs/src/built-in.rst

@@ -633,3 +633,40 @@ Example
     out:
       type: ...
       ...
+
+Guess executor
+------------------
+
+The guess executor is called by ``guess`` command. It executes default guess plugins in a sequential order and suggests Embulk config by appropriate guess plugin. The default guess plugins and the order are ``gzip``, ``'bzip2``, ``json`` and ``csv``.
+
+Options
+~~~~~~~~~~~~~~~~~~
+
++-----------------------+----------+----------------------------------------------------------------------+--------------------------------------+
+| name                  | type     | description                                                          | required?                            |
++=======================+==========+======================================================================+======================================+
+| guess_plugins         | array    | ``guess`` command uses specified guess plugins.                      | ``[]`` by default                    |
++-----------------------+----------+----------------------------------------------------------------------+--------------------------------------+
+| exclude_guess_plugins | array    | ``guess`` command doesn't use specified plugins.                     | ``[]`` by default                    |
++-----------------------+----------+----------------------------------------------------------------------+--------------------------------------+
+
+The ``guess_plugins`` option includes specified guess plugin in the bottom of the list of default guess plugins.
+
+The ``exclude_guess_plugins`` option exclude specified guess plugins from the list of default guess plugins that the guess executor uses.
+
+This example shows how to use ``csv_all_strings`` guess plugin, which suggests column types within CSV files as string types. It needs to be explicitly specified by users when it's used instead of ``csv`` guess plugin because the plugin is not included in default guess plugins. We also can exclude default ``csv`` guess plugin.
+
+Example
+~~~~~~~~~~~~~~~~~~
+
+.. code-block:: yaml
+
+    exec:
+      guess_plugins: ['csv_all_strings']
+      exclude_guess_plugins: ['csv']
+    in:
+      type: ...
+      ...
+    out:
+      type: ...
+      ...

data/embulk-docs/src/release.rst

@@ -4,6 +4,7 @@ Release Notes
 .. toctree::
     :maxdepth: 1
 
+    release/release-0.8.14
     release/release-0.8.13
     release/release-0.8.12
     release/release-0.8.11

data/embulk-docs/src/release/release-0.8.14.rst

@@ -0,0 +1,31 @@
+Release 0.8.14
+==================================
+
+General Changes
+------------------
+
+* Improved IOException handling within ResumableInputStream#reopen method. [#470]
+
+* Added "rules" config option to rename filter plugin. It allows users to specify rule based policy to rename columns. [#483]
+
+* Added "csv_all_strings" guess plugin, which suggests columns in csv files as string typed columns. [#489]
+
+* Updated JRuby from 9.1.2.0 to 9.1.5.0. Release notes:
+
+  * http://jruby.org/2016/08/30/jruby-9-1-3-0.html
+
+  * http://jruby.org/2016/09/01/jruby-9-1-4-0.html
+
+  * http://jruby.org/2016/09/06/jruby-9-1-5-0.html
+
+* Updated msgpack-java from 0.8.8 to 0.8.11. Release notes
+
+  * https://github.com/msgpack/msgpack-java/blob/0.8.9/RELEASE_NOTES.md
+
+  * https://github.com/msgpack/msgpack-java/blob/0.8.10/RELEASE_NOTES.md
+
+  * https://github.com/msgpack/msgpack-java/blob/0.8.11/RELEASE_NOTES.md
+
+Release Date
+------------------
+2016-10-21

data/embulk-standards/src/main/java/org/embulk/standards/RenameFilterPlugin.java

@@ -2,6 +2,7 @@ package org.embulk.standards;
 
 import org.embulk.config.Config;
 import org.embulk.config.ConfigDefault;
+import org.embulk.config.ConfigException;
 import org.embulk.config.ConfigSource;
 import org.embulk.config.Task;
 import org.embulk.config.TaskSource;
@@ -10,8 +11,10 @@ import org.embulk.spi.FilterPlugin;
 import org.embulk.spi.PageOutput;
 import org.embulk.spi.Schema;
 
+import java.util.List;
 import java.util.Map;
 
+
 public class RenameFilterPlugin
         implements FilterPlugin
 {
@@ -21,6 +24,10 @@ public class RenameFilterPlugin
         @Config("columns")
         @ConfigDefault("{}")
         Map<String, String> getRenameMap();
+
+        @Config("rules")
+        @ConfigDefault("[]")
+        List<ConfigSource> getRulesList();
     }
 
     @Override
@@ -29,12 +36,14 @@ public class RenameFilterPlugin
     {
         PluginTask task = config.loadConfig(PluginTask.class);
         Map<String, String> renameMap = task.getRenameMap();
+        List<ConfigSource> rulesList = task.getRulesList();
 
-        // check column_options is valid or not
+        // Check if the given column in "columns" exists or not.
         for (String columnName : renameMap.keySet()) {
             inputSchema.lookupColumn(columnName); // throws SchemaConfigException
         }
 
+        // Rename by "columns": to be applied before "rules".
         Schema.Builder builder = Schema.builder();
         for (Column column : inputSchema.getColumns()) {
             String name = column.getName();
@@ -43,8 +52,16 @@ public class RenameFilterPlugin
             }
             builder.add(name, column.getType());
         }
+        Schema intermediateSchema = builder.build();
+
+        // Rename by "rules".
+        Schema outputSchema = intermediateSchema;
+        for (ConfigSource rule : rulesList) {
+            outputSchema = applyRule(rule, intermediateSchema);
+            intermediateSchema = outputSchema;
+        }
 
-        control.run(task.dump(), builder.build());
+        control.run(task.dump(), outputSchema);
     }
 
     @Override
@@ -53,4 +70,24 @@ public class RenameFilterPlugin
     {
         return output;
     }
-}
+
+
+    // Extending Task is required to be deserialized with ConfigSource.loadConfig()
+    // although this Rule is not really a Task.
+    // TODO(dmikurube): Revisit this to consider how not to extend Task for this.
+    private interface Rule
+            extends Task
+    {
+        @Config("rule")
+        String getRule();
+    }
+
+    private Schema applyRule(ConfigSource ruleConfig, Schema inputSchema) throws ConfigException
+    {
+        Rule rule = ruleConfig.loadConfig(Rule.class);
+        switch (rule.getRule()) {
+        default:
+            throw new ConfigException("Renaming rule \"" +rule+ "\" is unknown");
+        }
+    }
+}

data/embulk-standards/src/test/java/org/embulk/standards/TestRenameFilterPlugin.java

@@ -1,7 +1,9 @@
 package org.embulk.standards;
 
+import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import org.embulk.EmbulkTestRuntime;
+import org.embulk.config.ConfigException;
 import org.embulk.config.ConfigSource;
 import org.embulk.config.TaskSource;
 import org.embulk.spi.Column;
@@ -85,4 +87,54 @@ public class TestRenameFilterPlugin
             }
         });
     }
+
+    @Test
+    public void checkConfigExceptionIfUnknownStringTypeOfRenamingOperator()
+    {
+        // A simple string shouldn't come as a renaming rule.
+        ConfigSource pluginConfig = Exec.newConfigSource()
+                .set("rules", ImmutableList.of("string_rule"));
+
+        try {
+            filter.transaction(pluginConfig, SCHEMA, new FilterPlugin.Control() {
+                public void run(TaskSource task, Schema schema) { }
+            });
+            fail();
+        } catch (Throwable t) {
+            assertTrue(t instanceof ConfigException);
+        }
+    }
+
+    @Test
+    public void checkConfigExceptionIfUnknownListTypeOfRenamingOperator()
+    {
+        // A list [] shouldn't come as a renaming rule.
+        ConfigSource pluginConfig = Exec.newConfigSource()
+                .set("rules", ImmutableList.of(ImmutableList.of("listed_operator1", "listed_operator2")));
+
+        try {
+            filter.transaction(pluginConfig, SCHEMA, new FilterPlugin.Control() {
+                public void run(TaskSource task, Schema schema) { }
+            });
+            fail();
+        } catch (Throwable t) {
+            assertTrue(t instanceof ConfigException);
+        }
+    }
+
+    @Test
+    public void checkConfigExceptionIfUnknownRenamingOperatorName()
+    {
+        ConfigSource pluginConfig = Exec.newConfigSource()
+                .set("rules", ImmutableList.of(ImmutableMap.of("rule", "some_unknown_renaming_operator")));
+
+        try {
+            filter.transaction(pluginConfig, SCHEMA, new FilterPlugin.Control() {
+                public void run(TaskSource task, Schema schema) { }
+            });
+            fail();
+        } catch (Throwable t) {
+            assertTrue(t instanceof ConfigException);
+        }
+    }
 }

data/embulk.gemspec

@@ -30,7 +30,7 @@ Gem::Specification.new do |gem|
     gem.platform = 'java'
 
   else
-    gem.add_dependency "jruby-jars", '= 9.1.2.0'
+    gem.add_dependency "jruby-jars", '= 9.1.5.0'
   end
 
   gem.add_development_dependency "rake", [">= 0.10.0"]

data/lib/embulk/command/embulk_migrate_plugin.rb

@@ -122,7 +122,7 @@ EOF
     # add rules...
     ##
 
-    migrator.write(".ruby-version", "jruby-9.1.2.0")
+    migrator.write(".ruby-version", "jruby-9.1.5.0")
 
     # update version at the end
     if from_ver <= version("0.1.0")

data/lib/embulk/data/bundle/.ruby-version

@@ -1 +1 @@
-jruby-9.1.2.0
+jruby-9.1.5.0

data/lib/embulk/data/new/ruby/.ruby-version

@@ -1 +1 @@
-jruby-9.1.2.0
+jruby-9.1.5.0

data/lib/embulk/file_input_plugin.rb

@@ -13,7 +13,7 @@ module Embulk
     module RubyAdapter
       module ClassMethods
         def new_java
-          Java::FileInputRunner.new(Java.injector.getInstance(java_class))
+          Java::FileInputRunner.new(Java.injector.getInstance(plugin_java_class))
         end
         # TODO transaction, resume, cleanup
       end

data/lib/embulk/file_output_plugin.rb

@@ -13,7 +13,7 @@ module Embulk
     module RubyAdapter
       module ClassMethods
         def new_java
-          Java::FileOutputRunner.new(Java.injector.getInstance(java_class))
+          Java::FileOutputRunner.new(Java.injector.getInstance(plugin_java_class))
         end
         # TODO transaction, resume, cleanup
       end

data/lib/embulk/guess/csv.rb

@@ -142,11 +142,7 @@ module Embulk
         schema = []
         column_names.zip(other_types).each do |name,type|
           if name && type
-            if type.is_a?(SchemaGuess::TimestampTypeMatch)
-              schema << {"name" => name, "type" => type, "format" => type.format}
-            else
-              schema << {"name" => name, "type" => type}
-            end
+            schema << new_column(name, type)
           end
         end
         parser_guessed["columns"] = schema
@@ -154,6 +150,14 @@ module Embulk
         return {"parser" => parser_guessed}
       end
 
+      def new_column(name, type)
+        if type.is_a?(SchemaGuess::TimestampTypeMatch)
+          {"name" => name, "type" => type, "format" => type.format}
+        else
+          {"name" => name, "type" => type}
+        end
+      end
+
       private
 
       def split_lines(parser_config, skip_empty_lines, sample_lines, delim, extra_config)

data/lib/embulk/guess/csv_all_strings.rb

@@ -0,0 +1,13 @@
+module Embulk
+  module Guess
+    require 'embulk/guess/csv'
+
+    class CsvAllStringsGuessPlugin < CsvGuessPlugin
+      Plugin.register_guess("csv_all_strings", self)
+
+      def new_column(name, type)
+        {"name" => name, "type" => "string"}
+      end
+    end
+  end
+end

data/lib/embulk/guess_plugin.rb

@@ -37,7 +37,7 @@ module Embulk
       def guess(config, sample)
         java_config = config.to_java
         java_sample = sample.to_java
-        java_config_diff = java_object.guess(java_config, java_sample)
+        java_config_diff = plugin_java_object.guess(java_config, java_sample)
         return DataSource.from_java(java_config_diff)
       end
     end

data/lib/embulk/java_plugin.rb

@@ -57,20 +57,20 @@ module Embulk
 
     def self.ruby_adapter_class(java_class, ruby_base_class, ruby_module)
       Class.new(ruby_base_class) do
-        const_set(:JAVA_CLASS, java_class)
+        const_set(:PLUGIN_JAVA_CLASS, java_class)
 
         include ruby_module
         extend ruby_module::ClassMethods
 
-        unless method_defined?(:java_object)
-          def java_object
-            @java_object ||= self.class.new_java
+        unless method_defined?(:plugin_java_object)
+          def plugin_java_object
+            @plugin_java_object ||= self.class.new_java
           end
         end
 
-        unless (class<<self;self;end).method_defined?(:java_class)
-          def self.java_class
-            self::JAVA_CLASS
+        unless (class<<self;self;end).method_defined?(:plugin_java_class)
+          def self.plugin_java_class
+            self::PLUGIN_JAVA_CLASS
           end
         end
 
@@ -81,7 +81,7 @@ module Embulk
         #      ruby_module::ClassMethods includes other modules.
         unless ruby_module::ClassMethods.method_defined?(:new_java)
           def self.new_java
-            Java.injector.getInstance(java_class)
+            Java.injector.getInstance(plugin_java_class)
           end
         end
       end

data/lib/embulk/version.rb

@@ -1,3 +1,3 @@
 module Embulk
-  VERSION = '0.8.13'
+  VERSION = '0.8.14'
 end

data/test/guess/test_csv_all_strings.rb

@@ -0,0 +1,43 @@
+require 'helper'
+require 'time'
+require 'embulk/guess/csv_all_strings'
+
+class CsvAllStringsGuessTest < ::Test::Unit::TestCase
+  class TestAllStrings < self
+    def test_columns_without_header
+      actual = guess([
+        "1\tfoo\t2000-01-01T00:00:00+0900",
+        "2\tbar\t2000-01-01T00:00:00+0900",
+      ])
+      expected = [
+        {"name" => "c0", "type" => "string"},
+        {"name" => "c1", "type" => "string"},
+        {"name" => "c2", "type" => "string"},
+      ]
+      assert_equal expected, actual["parser"]["columns"]
+    end
+
+    def test_columns_with_header
+      actual = guess([
+        "num\tstr\ttime",
+        "1\tfoo\t2000-01-01T00:00:00+0900",
+        "2\tbar\t2000-01-01T00:00:00+0900",
+      ])
+      expected = [
+        {"name" => "num", "type" => "string"},
+        {"name" => "str", "type" => "string"},
+        {"name" => "time", "type" => "string"},
+      ]
+      assert_equal expected, actual["parser"]["columns"]
+    end
+  end
+
+  def guess(texts)
+    conf = Embulk::DataSource.new({
+      parser: {
+        type: "csv"
+      }
+    })
+    Embulk::Guess::CsvAllStringsGuessPlugin.new.guess_lines(conf, Array(texts))
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk
 version: !ruby/object:Gem::Version
-  version: 0.8.13
+  version: 0.8.14
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-08-12 00:00:00.000000000 Z
+date: 2016-10-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: jruby-jars
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 9.1.2.0
+        version: 9.1.5.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 9.1.2.0
+        version: 9.1.5.0
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -108,9 +108,9 @@ files:
 - classpath/commons-beanutils-core-1.8.3.jar
 - classpath/commons-compress-1.10.jar
 - classpath/commons-lang3-3.1.jar
-- classpath/embulk-cli-0.8.13.jar
-- classpath/embulk-core-0.8.13.jar
-- classpath/embulk-standards-0.8.13.jar
+- classpath/embulk-cli-0.8.14.jar
+- classpath/embulk-core-0.8.14.jar
+- classpath/embulk-standards-0.8.14.jar
 - classpath/guava-18.0.jar
 - classpath/guice-4.0.jar
 - classpath/guice-bootstrap-0.1.1.jar
@@ -126,7 +126,7 @@ files:
 - classpath/joda-time-2.9.2.jar
 - classpath/logback-classic-1.1.3.jar
 - classpath/logback-core-1.1.3.jar
-- classpath/msgpack-core-0.8.8.jar
+- classpath/msgpack-core-0.8.11.jar
 - classpath/netty-buffer-5.0.0.Alpha1.jar
 - classpath/netty-common-5.0.0.Alpha1.jar
 - classpath/slf4j-api-1.7.12.jar
@@ -427,6 +427,7 @@ files:
 - embulk-docs/src/release/release-0.8.11.rst
 - embulk-docs/src/release/release-0.8.12.rst
 - embulk-docs/src/release/release-0.8.13.rst
+- embulk-docs/src/release/release-0.8.14.rst
 - embulk-docs/src/release/release-0.8.2.rst
 - embulk-docs/src/release/release-0.8.3.rst
 - embulk-docs/src/release/release-0.8.4.rst
@@ -527,6 +528,7 @@ files:
 - lib/embulk/guess/bzip2.rb
 - lib/embulk/guess/charset.rb
 - lib/embulk/guess/csv.rb
+- lib/embulk/guess/csv_all_strings.rb
 - lib/embulk/guess/gzip.rb
 - lib/embulk/guess/json.rb
 - lib/embulk/guess/newline.rb
@@ -549,6 +551,7 @@ files:
 - lib/embulk/schema.rb
 - lib/embulk/version.rb
 - settings.gradle
+- test/guess/test_csv_all_strings.rb
 - test/guess/test_csv_guess.rb
 - test/guess/test_schema_guess.rb
 - test/guess/test_time_format_guess.rb
@@ -579,9 +582,9 @@ signing_key:
 specification_version: 4
 summary: Embulk, a plugin-based parallel bulk data loader
 test_files:
+- test/guess/test_csv_all_strings.rb
 - test/guess/test_csv_guess.rb
 - test/guess/test_schema_guess.rb
 - test/guess/test_time_format_guess.rb
 - test/helper.rb
 - test/run-test.rb
-has_rdoc: false