embulk 0.7.6-java → 0.7.7-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6e299148e196ac4521c81da16fe9073af066027e
-  data.tar.gz: 968cca1d984cfbf079a26cee217656f56b69ab53
+  metadata.gz: 885a6d1fb403c2485fc750a61510292d09d608ef
+  data.tar.gz: 292732f9ef40bd7c9e447eb6e7f8d0808ecef751
 SHA512:
-  metadata.gz: 28b9b3db6b11bedf305be41986d89941a404743a1d463a0f573e1bfc808ccc6503ebb8d05f7c83dbdb3403e9d8985e88194cd6e9ac1be65aa5507e89db2dd951
-  data.tar.gz: 63e207246f689796fc132b43d4398fcd2725fc31d4a76e050e8b868de1572543add56ed15dc31b93c4db4164693dc100da448335db8e058ecbbe10046aa10156
+  metadata.gz: 01c5228b4b4ff13849d0221f5855523418e262b803088250c0eb951f2b4bf7cf6b17024723b83cfeb78f7b30f92bc6b6ce3b18567ed1f4d4999f2b1d498938cb
+  data.tar.gz: ed79c753d4484f0c62d19529f2215d746102f72892ffeb55c36cb1a14792c12964c354121e0c02c17a0c1b81d392479d131c9f206bfa23f0e4fd19e0be090a4c

data/build.gradle

@@ -16,7 +16,7 @@ def release_projects = [project(":embulk-core"), project(":embulk-standards")]
 
 allprojects {
     group = 'org.embulk'
-    version = '0.7.6'
+    version = '0.7.7'
 
     ext {
         jrubyVersion = '9.0.0.0'

data/embulk-core/src/main/java/org/embulk/exec/BulkLoader.java

@@ -163,10 +163,20 @@ public class BulkLoader
 
         public boolean isAllTasksCommitted()
         {
-            if (outputTaskStates == null) {
+            // here can't assume that input tasks are committed when output tasks are
+            // committed because that's controlled by executor plugins. some executor
+            // plugins (especially mapreduce executor) may commit output tasks even
+            // when some input tasks failed. This is asemantically allowed behavior for
+            // executor plugins (as long as output plugin is atomic and idempotent).
+            if (inputTaskStates == null || outputTaskStates == null) {
                 // not initialized
                 return false;
             }
+            for (TaskState inputTaskState : inputTaskStates) {
+                if (!inputTaskState.isCommitted()) {
+                    return false;
+                }
+            }
             for (TaskState outputTaskState : outputTaskStates) {
                 if (!outputTaskState.isCommitted()) {
                     return false;
@@ -175,6 +185,36 @@ public class BulkLoader
             return true;
         }
 
+        public int countUncommittedInputTasks()
+        {
+            if (inputTaskStates == null) {
+                // not initialized
+                return 0;
+            }
+            int count = 0;
+            for (TaskState inputTaskState : inputTaskStates) {
+                if (!inputTaskState.isCommitted()) {
+                    count++;
+                }
+            }
+            return count;
+        }
+
+        public int countUncommittedOutputTasks()
+        {
+            if (outputTaskStates == null) {
+                // not initialized
+                return 0;
+            }
+            int count = 0;
+            for (TaskState outputTaskState : outputTaskStates) {
+                if (!outputTaskState.isCommitted()) {
+                    count++;
+                }
+            }
+            return count;
+        }
+
         public boolean isAllTransactionsCommitted()
         {
             return inputConfigDiff != null && outputConfigDiff != null;
@@ -498,6 +538,11 @@ public class BulkLoader
                                                 execute(task, executor, state);
                                             }
 
+                                            if (!state.isAllTasksCommitted()) {
+                                                throw new RuntimeException(String.format("%d input tasks and %d output tasks failed",
+                                                            state.countUncommittedInputTasks(), state.countUncommittedOutputTasks()));
+                                            }
+
                                             return state.getAllOutputTaskReports();
                                         }
                                     });
@@ -563,6 +608,11 @@ public class BulkLoader
                                                 execute(task, executor, state);
                                             }
 
+                                            if (!state.isAllTasksCommitted()) {
+                                                throw new RuntimeException(String.format("%d input tasks and %d output tasks failed",
+                                                            state.countUncommittedInputTasks(), state.countUncommittedOutputTasks()));
+                                            }
+
                                             return state.getAllOutputTaskReports();
                                         }
                                     });

data/embulk-docs/src/built-in.rst

@@ -88,6 +88,38 @@ Environment variables are set to ``env`` variable.
       table: {{ env.pg_table }}
 
 
+Including files
+~~~~~~~~~~~~~~~~~~
+
+Configuration file can include another configuration file. To use it, configuration file name must end with ``.yml.liquid``.
+
+File will be searched from the relative path of the input configuration file. And file name will be ``_<name>.yml.liquid``. For example, if you add ``{% include 'subdir/inc' %}`` tag to ``myconfig/config.yml.liquid`` file, it includes ``myconfig/subdir/_inc.yml.liquid`` file.
+
+.. code-block:: yaml
+
+    # config.yml.liquid
+    {% include 'in_mysql' %}
+    out:
+      type: stdout
+
+.. code-block:: yaml
+
+    # _in_mysql.yml.liquid
+    in:
+      type: mysql
+
+With above 2 files, actual configuration file will be:
+
+.. code-block:: yaml
+
+    # $ embulk run config.yml.liquid
+    in:
+      type: mysql
+    out:
+      type: stdout
+
+
+
 Local file input plugin
 ------------------
 

data/embulk-docs/src/release.rst

@@ -4,6 +4,7 @@ Release Notes
 .. toctree::
     :maxdepth: 1
 
+    release/release-0.7.7
     release/release-0.7.6
     release/release-0.7.5
     release/release-0.7.4

data/embulk-docs/src/release/release-0.7.7.rst

@@ -0,0 +1,13 @@
+Release 0.7.7
+==================================
+
+General Changes
+------------------
+
+* Configuration file can use ``{% include 'path' %}`` tag to include another file. This is very useful to organize a complex configuration from multiple files. See documents for details.
+* Fixed compatibility with mapreduce executor. Failure message includes appropriate cause instead of IllegalStateException.
+
+
+Release Date
+------------------
+2015-10-28

data/lib/embulk/runner.rb

@@ -113,7 +113,8 @@ module Embulk
         when /\.yml\.liquid$/
           require 'liquid'
           template_params = options[:template_params] || {}
-          @embed.newConfigLoader.fromYamlString run_liquid(File.read(config), template_params)
+          template_include_path = File.expand_path(options[:template_include_path] || File.dirname(config)) unless options[:template_include_path] == false
+          @embed.newConfigLoader.fromYamlString run_liquid(File.read(config), template_params, template_include_path)
         when /\.yml$/
           @embed.newConfigLoader.fromYamlString File.read(config)
         else
@@ -129,9 +130,10 @@ module Embulk
       @embed.newConfigLoader.fromYamlString File.read(config)
     end
 
-    def run_liquid(source, params)
+    def run_liquid(source, params, template_include_path)
       require 'liquid'
       template = Liquid::Template.parse(source)
+      template.registers[:file_system] = Liquid::LocalFileSystem.new(template_include_path, "_%s.yml.liquid") if template_include_path
 
       data = {
         "env" => ENV.to_h,

data/lib/embulk/version.rb

@@ -1,3 +1,3 @@
 module Embulk
-  VERSION = '0.7.6'
+  VERSION = '0.7.7'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk
 version: !ruby/object:Gem::Version
-  version: 0.7.6
+  version: 0.7.7
 platform: java
 authors:
 - Sadayuki Furuhashi
@@ -133,8 +133,8 @@ files:
 - classpath/bval-jsr303-0.5.jar
 - classpath/commons-beanutils-core-1.8.3.jar
 - classpath/commons-lang3-3.1.jar
-- classpath/embulk-core-0.7.6.jar
-- classpath/embulk-standards-0.7.6.jar
+- classpath/embulk-core-0.7.7.jar
+- classpath/embulk-standards-0.7.7.jar
 - classpath/guava-18.0.jar
 - classpath/guice-4.0.jar
 - classpath/guice-multibindings-4.0.jar
@@ -438,6 +438,7 @@ files:
 - embulk-docs/src/release/release-0.7.4.rst
 - embulk-docs/src/release/release-0.7.5.rst
 - embulk-docs/src/release/release-0.7.6.rst
+- embulk-docs/src/release/release-0.7.7.rst
 - embulk-standards/build.gradle
 - embulk-standards/src/main/java/org/embulk/standards/CsvFormatterPlugin.java
 - embulk-standards/src/main/java/org/embulk/standards/CsvParserPlugin.java