embulk 0.7.6 → 0.7.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 29b4fbf4ef36c7354cf89e39b25212d5dbf6a0fa
-  data.tar.gz: 8e2387077d348272bf6e09736f51b48722b9287a
+  metadata.gz: 98c34f07d3e87eda540f68191fea784dd4e32aa9
+  data.tar.gz: 21df9853971aaa0a124021b7e2e6d0ad38961668
 SHA512:
-  metadata.gz: c39a0c665331889057cd4c2c74cfb7610f56dc1665c3e8e4fae8513f8a37f7dab7f463d68d814794b4a2b5a8401ca6ddbb366ade4ab0d2ef85a82d6f895776d6
-  data.tar.gz: 491c5c617a8868ac232fca6a7034263ef338e6213bca61806223acd10dd5a59a61e5fed8f09e86809bdc5c880906baf4122563d4c7187413a660a3d55eebf127
+  metadata.gz: 4f2f4c9c9e4dab482962d59f9bea20ca12300b7e6cfecc6af5f3e472ffe9cb0bf26276073ff0de62871281462616249b7858ac6f03149b036a6356d2b0054262
+  data.tar.gz: a8182664cd36f762b840de0777e084064bf930c0162d797600fc3efc8241195a3f7fb51dd2427ea0f9d9a79de1ef44e3913b02af480af96ad14c1cc6f08b3405

data/build.gradle

@@ -16,7 +16,7 @@ def release_projects = [project(":embulk-core"), project(":embulk-standards")]
 
 allprojects {
     group = 'org.embulk'
-    version = '0.7.6'
+    version = '0.7.7'
 
     ext {
         jrubyVersion = '9.0.0.0'

data/embulk-core/src/main/java/org/embulk/exec/BulkLoader.java

@@ -163,10 +163,20 @@ public class BulkLoader
 
         public boolean isAllTasksCommitted()
         {
-            if (outputTaskStates == null) {
+            // here can't assume that input tasks are committed when output tasks are
+            // committed because that's controlled by executor plugins. some executor
+            // plugins (especially mapreduce executor) may commit output tasks even
+            // when some input tasks failed. This is asemantically allowed behavior for
+            // executor plugins (as long as output plugin is atomic and idempotent).
+            if (inputTaskStates == null || outputTaskStates == null) {
                 // not initialized
                 return false;
             }
+            for (TaskState inputTaskState : inputTaskStates) {
+                if (!inputTaskState.isCommitted()) {
+                    return false;
+                }
+            }
             for (TaskState outputTaskState : outputTaskStates) {
                 if (!outputTaskState.isCommitted()) {
                     return false;
@@ -175,6 +185,36 @@ public class BulkLoader
             return true;
         }
 
+        public int countUncommittedInputTasks()
+        {
+            if (inputTaskStates == null) {
+                // not initialized
+                return 0;
+            }
+            int count = 0;
+            for (TaskState inputTaskState : inputTaskStates) {
+                if (!inputTaskState.isCommitted()) {
+                    count++;
+                }
+            }
+            return count;
+        }
+
+        public int countUncommittedOutputTasks()
+        {
+            if (outputTaskStates == null) {
+                // not initialized
+                return 0;
+            }
+            int count = 0;
+            for (TaskState outputTaskState : outputTaskStates) {
+                if (!outputTaskState.isCommitted()) {
+                    count++;
+                }
+            }
+            return count;
+        }
+
         public boolean isAllTransactionsCommitted()
         {
             return inputConfigDiff != null && outputConfigDiff != null;
@@ -498,6 +538,11 @@ public class BulkLoader
                                                 execute(task, executor, state);
                                             }
 
+                                            if (!state.isAllTasksCommitted()) {
+                                                throw new RuntimeException(String.format("%d input tasks and %d output tasks failed",
+                                                            state.countUncommittedInputTasks(), state.countUncommittedOutputTasks()));
+                                            }
+
                                             return state.getAllOutputTaskReports();
                                         }
                                     });
@@ -563,6 +608,11 @@ public class BulkLoader
                                                 execute(task, executor, state);
                                             }
 
+                                            if (!state.isAllTasksCommitted()) {
+                                                throw new RuntimeException(String.format("%d input tasks and %d output tasks failed",
+                                                            state.countUncommittedInputTasks(), state.countUncommittedOutputTasks()));
+                                            }
+
                                             return state.getAllOutputTaskReports();
                                         }
                                     });

data/embulk-docs/src/built-in.rst

@@ -88,6 +88,38 @@ Environment variables are set to ``env`` variable.
       table: {{ env.pg_table }}
 
 
+Including files
+~~~~~~~~~~~~~~~~~~
+
+Configuration file can include another configuration file. To use it, configuration file name must end with ``.yml.liquid``.
+
+File will be searched from the relative path of the input configuration file. And file name will be ``_<name>.yml.liquid``. For example, if you add ``{% include 'subdir/inc' %}`` tag to ``myconfig/config.yml.liquid`` file, it includes ``myconfig/subdir/_inc.yml.liquid`` file.
+
+.. code-block:: yaml
+
+    # config.yml.liquid
+    {% include 'in_mysql' %}
+    out:
+      type: stdout
+
+.. code-block:: yaml
+
+    # _in_mysql.yml.liquid
+    in:
+      type: mysql
+
+With above 2 files, actual configuration file will be:
+
+.. code-block:: yaml
+
+    # $ embulk run config.yml.liquid
+    in:
+      type: mysql
+    out:
+      type: stdout
+
+
+
 Local file input plugin
 ------------------
 

data/embulk-docs/src/release.rst

@@ -4,6 +4,7 @@ Release Notes
 .. toctree::
     :maxdepth: 1
 
+    release/release-0.7.7
     release/release-0.7.6
     release/release-0.7.5
     release/release-0.7.4

data/embulk-docs/src/release/release-0.7.7.rst

@@ -0,0 +1,13 @@
+Release 0.7.7
+==================================
+
+General Changes
+------------------
+
+* Configuration file can use ``{% include 'path' %}`` tag to include another file. This is very useful to organize a complex configuration from multiple files. See documents for details.
+* Fixed compatibility with mapreduce executor. Failure message includes appropriate cause instead of IllegalStateException.
+
+
+Release Date
+------------------
+2015-10-28

data/lib/embulk/runner.rb

@@ -113,7 +113,8 @@ module Embulk
         when /\.yml\.liquid$/
           require 'liquid'
           template_params = options[:template_params] || {}
-          @embed.newConfigLoader.fromYamlString run_liquid(File.read(config), template_params)
+          template_include_path = File.expand_path(options[:template_include_path] || File.dirname(config)) unless options[:template_include_path] == false
+          @embed.newConfigLoader.fromYamlString run_liquid(File.read(config), template_params, template_include_path)
         when /\.yml$/
           @embed.newConfigLoader.fromYamlString File.read(config)
         else
@@ -129,9 +130,10 @@ module Embulk
       @embed.newConfigLoader.fromYamlString File.read(config)
     end
 
-    def run_liquid(source, params)
+    def run_liquid(source, params, template_include_path)
       require 'liquid'
       template = Liquid::Template.parse(source)
+      template.registers[:file_system] = Liquid::LocalFileSystem.new(template_include_path, "_%s.yml.liquid") if template_include_path
 
       data = {
         "env" => ENV.to_h,

data/lib/embulk/version.rb

@@ -1,3 +1,3 @@
 module Embulk
-  VERSION = '0.7.6'
+  VERSION = '0.7.7'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk
 version: !ruby/object:Gem::Version
-  version: 0.7.6
+  version: 0.7.7
 platform: ruby
 authors:
 - Sadayuki Furuhashi
@@ -121,8 +121,8 @@ files:
 - classpath/bval-jsr303-0.5.jar
 - classpath/commons-beanutils-core-1.8.3.jar
 - classpath/commons-lang3-3.1.jar
-- classpath/embulk-core-0.7.6.jar
-- classpath/embulk-standards-0.7.6.jar
+- classpath/embulk-core-0.7.7.jar
+- classpath/embulk-standards-0.7.7.jar
 - classpath/guava-18.0.jar
 - classpath/guice-4.0.jar
 - classpath/guice-multibindings-4.0.jar
@@ -426,6 +426,7 @@ files:
 - embulk-docs/src/release/release-0.7.4.rst
 - embulk-docs/src/release/release-0.7.5.rst
 - embulk-docs/src/release/release-0.7.6.rst
+- embulk-docs/src/release/release-0.7.7.rst
 - embulk-standards/build.gradle
 - embulk-standards/src/main/java/org/embulk/standards/CsvFormatterPlugin.java
 - embulk-standards/src/main/java/org/embulk/standards/CsvParserPlugin.java