embulk 0.8.1 → 0.8.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a189a85d312e0621daf89d790baf4fa45d0e0125
-  data.tar.gz: d7355b10940135fce461332f40d6f8180b3c41c2
+  metadata.gz: e49862e13c81831477cfed94aea2db310b8bc51c
+  data.tar.gz: 9b59130a3b0ceca8d4980a7c32e979d84f7b5ab1
 SHA512:
-  metadata.gz: 09609e12b41ce696b190be750cee3687fcd0f9f04d24597345415080c8ffdd7319ed33129584f8c500a6893c2cdb4cba86d7ddeb34bab0cf4727c2beb3388336
-  data.tar.gz: b46a461b1f845e7020d6ac5c3553ed4ca1a764ae0ae80fccc52d4bd9a35db11fe24ec6ab721fd3ae841f365d6d92fc0f40213d2053d5c9a1c9d74ae6bb8e62f6
+  metadata.gz: e8d58a7659d4445c83534859d5cf8043099877b03e1544be8f531b6a644ebfd116de970ac38d92558310ed19991aa476e1a4f96a829382a0692ac57b506b1fb1
+  data.tar.gz: 52e734f97666ea28f4a3f6564f7f74001d35f85e59a0b4cf3c6f9aa3c77902ac06438184b13ee44b906ecd5760b01361f799d967167617d2a7c19f1020ce7cb2

data/README.md

@@ -46,7 +46,7 @@ Next step: [Running example in 4 commands](#running-example)
 
 ```
 embulk example ./try1
-embulk guess   ./try1/example.yml -o config.yml
+embulk guess   ./try1/seed.yml -o config.yml
 embulk preview config.yml
 embulk run     config.yml
 ```

data/build.gradle

@@ -16,7 +16,7 @@ def release_projects = [project(":embulk-core"), project(":embulk-standards")]
 
 allprojects {
     group = 'org.embulk'
-    version = '0.8.1'
+    version = '0.8.2'
 
     ext {
         jrubyVersion = '9.0.4.0'

data/embulk-core/src/main/java/org/embulk/exec/LocalExecutorPlugin.java

@@ -60,7 +60,7 @@ public class LocalExecutorPlugin
         Logger log = Exec.getLogger(LocalExecutorPlugin.class);
         int maxThreads = config.get(Integer.class, "max_threads", defaultMaxThreads);
         int minThreads = config.get(Integer.class, "min_output_tasks", defaultMinThreads);
-        if (inputTaskCount < minThreads) {
+        if (inputTaskCount > 0 && inputTaskCount < minThreads) {
             int scatterCount = (minThreads + inputTaskCount - 1) / inputTaskCount;
             log.info("Using local thread executor with max_threads={} / output tasks {} = input tasks {} * {}",
                     maxThreads, inputTaskCount * scatterCount, inputTaskCount, scatterCount);
@@ -320,33 +320,55 @@ public class LocalExecutorPlugin
         {
             private final PageOutput output;
             private final Future<Throwable> future;
-            private boolean done;
-            private Page queued;
+            private volatile int addWaiting;
+            private volatile Page queued;
 
             public OutputWorker(PageOutput output, ExecutorService executor)
             {
                 this.output = output;
-                this.done = done;
+                this.addWaiting = 0;
                 this.future = executor.submit(this);
             }
 
-            public synchronized void add(Page page)
+            public synchronized void done()
                     throws InterruptedException
             {
                 while (true) {
-                    if (queued == null) {
-                        queued = page;
+                    if (queued == null && addWaiting == 0) {
+                        queued = DONE_PAGE;
                         notifyAll();
                         return;
                     }
                     else if (queued == DONE_PAGE) {
-                        page.release();
                         return;
                     }
                     wait();
                 }
             }
 
+            public synchronized void add(Page page)
+                    throws InterruptedException
+            {
+                addWaiting++;
+                try {
+                    while (true) {
+                        if (queued == null) {
+                            queued = page;
+                            notifyAll();
+                            return;
+                        }
+                        else if (queued == DONE_PAGE) {
+                            page.release();
+                            return;
+                        }
+                        wait();
+                    }
+                }
+                finally {
+                    addWaiting--;
+                }
+            }
+
             public Throwable join()
                     throws InterruptedException
             {
@@ -470,8 +492,8 @@ public class LocalExecutorPlugin
         {
             completeWorkers();
             for (int i = 0; i < scatterCount; i++) {
-                if (trans[i] != null) {
-                    trans[i].finish();
+                if (filtereds[i] != null) {
+                    filtereds[i].finish();
                 }
             }
         }
@@ -517,7 +539,7 @@ public class LocalExecutorPlugin
                 OutputWorker worker = outputWorkers[i];
                 if (worker != null) {
                     try {
-                        worker.add(DONE_PAGE);
+                        worker.done();
                     }
                     catch (InterruptedException ex) {
                         throw Throwables.propagate(ex);

data/embulk-core/src/main/java/org/embulk/jruby/JRubyScriptingModule.java

@@ -14,7 +14,6 @@ import com.google.inject.Injector;
 import com.google.inject.Key;
 import com.google.inject.spi.Dependency;
 import com.google.inject.spi.ProviderWithDependencies;
-import org.jruby.CompatVersion;
 import org.jruby.embed.LocalContextScope;
 import org.jruby.embed.ScriptingContainer;
 import org.embulk.plugin.PluginSource;
@@ -44,6 +43,7 @@ public class JRubyScriptingModule
     {
         private final Injector injector;
         private final boolean useGlobalRubyRuntime;
+        private final String gemHome;
 
         @Inject
         public ScriptingContainerProvider(Injector injector, @ForSystemConfig ConfigSource systemConfig)
@@ -54,6 +54,8 @@ public class JRubyScriptingModule
             // instantiated in this JVM.
             this.useGlobalRubyRuntime = systemConfig.get(boolean.class, "use_global_ruby_runtime", false);
 
+            this.gemHome = systemConfig.get(String.class, "gem_home", null);
+
             // TODO get jruby-home from systemConfig to call jruby.container.setHomeDirectory
             // TODO get jruby-load-paths from systemConfig to call jruby.container.setLoadPaths
         }
@@ -75,6 +77,18 @@ public class JRubyScriptingModule
 //            }
 //            jruby.setLoadPaths(loadPaths);
 
+            if (gemHome != null) {
+                // Overwrites GEM_HOME and GEM_PATH. GEM_PATH becomes same with GEM_HOME. Therefore
+                // with this code, there're no ways to set extra GEM_PATHs in addition to GEM_HOME.
+                // Here doesn't modify ENV['GEM_HOME'] so that a JVM process can create multiple
+                // JRubyScriptingModule instances. However, because Gem loads ENV['GEM_HOME'] when
+                // Gem.clear_paths is called, applications may use unexpected GEM_HOME if clear_path
+                // is used.
+                jruby.callMethod(
+                        jruby.runScriptlet("Gem"),
+                        "use_paths", gemHome, gemHome);
+            }
+
             // load embulk.rb
             jruby.runScriptlet("require 'embulk'");
 

data/embulk-core/src/main/java/org/embulk/spi/util/dynamic/JsonColumnSetter.java

@@ -4,8 +4,6 @@ import org.embulk.spi.Column;
 import org.embulk.spi.PageBuilder;
 import org.embulk.spi.time.Timestamp;
 import org.embulk.spi.time.TimestampFormatter;
-import org.embulk.spi.json.JsonParser;
-import org.embulk.spi.json.JsonParseException;
 import org.msgpack.value.Value;
 import org.msgpack.value.ValueFactory;
 
@@ -13,7 +11,6 @@ public class JsonColumnSetter
         extends AbstractDynamicColumnSetter
 {
     private final TimestampFormatter timestampFormatter;
-    private final JsonParser jsonParser;
 
     public JsonColumnSetter(PageBuilder pageBuilder, Column column,
                             DefaultValueSetter defaultValue,
@@ -21,7 +18,6 @@ public class JsonColumnSetter
     {
         super(pageBuilder, column, defaultValue);
         this.timestampFormatter = timestampFormatter;
-        this.jsonParser = new JsonParser();
     }
 
     @Override
@@ -51,12 +47,7 @@ public class JsonColumnSetter
     @Override
     public void set(String v)
     {
-        try {
-            pageBuilder.setJson(column, jsonParser.parse(v));
-        }
-        catch (JsonParseException ex) {
-            defaultValue.setTimestamp(pageBuilder, column);
-        }
+        pageBuilder.setJson(column, ValueFactory.newString(v));
     }
 
     @Override

data/embulk-core/src/test/java/org/embulk/spi/PageTestUtils.java

@@ -11,7 +11,7 @@ import org.embulk.spi.Schema;
 import org.embulk.spi.SchemaConfig;
 import org.embulk.spi.ColumnVisitor;
 import org.embulk.spi.type.Type;
-
+import org.msgpack.value.Value;
 import com.google.common.collect.ImmutableList;
 
 public class PageTestUtils
@@ -42,6 +42,8 @@ public class PageTestUtils
                         builder.setString(column, (String) value);
                     } else if (value instanceof Timestamp) {
                         builder.setTimestamp(column, (Timestamp) value);
+                    } else if (value instanceof Value) {
+                        builder.setJson(column, (Value) value);
                     } else {
                         throw new IllegalStateException(
                                 "Unsupported type in test utils: "

data/embulk-docs/src/release.rst

@@ -4,6 +4,7 @@ Release Notes
 .. toctree::
     :maxdepth: 1
 
+    release/release-0.8.2
     release/release-0.8.1
     release/release-0.8.0
     release/release-0.7.10

data/embulk-docs/src/release/release-0.8.2.rst

@@ -0,0 +1,19 @@
+Release 0.8.2
+==================================
+
+General Changes
+------------------
+
+* Fixed race condition around transaction completion when page scattering is enabled.
+* Fixed unexpected error that happens when number of input tasks is 0 and page scattering is enabled.
+* Fixed preview output when a json column is included.
+
+CLI
+------------------
+
+* ``-c, --config-diff`` option replaces ``-o, --output`` option.
+
+
+Release Date
+------------------
+2016-01-31

data/lib/embulk/command/embulk_example.rb

@@ -18,8 +18,8 @@ id,account,time,purchase,comment
 EOF
     end
 
-    puts "  Creating #{path}/example.yml"
-    File.open(File.join(path, 'example.yml'), 'w') do |f|
+    puts "  Creating #{path}/seed.yml"
+    File.open(File.join(path, 'seed.yml'), 'w') do |f|
       f.write <<EOF
 in:
   type: file

data/lib/embulk/command/embulk_run.rb

@@ -78,9 +78,19 @@ module Embulk
       op.on('-r', '--resume-state PATH', 'Path to a file to write or read resume state') do |path|
         options[:resume_state_path] = path
       end
-      op.on('-o', '--output PATH', 'Path to a file to write the next configuration') do |path|
+      op.on('-o', '--output PATH', '(deprecated)') do |path|
+        STDERR.puts "#{Time.now.strftime("%Y-%m-%d %H:%M:%S.%3N %z")}: Run with -o option is deprecated. Please use -c option instead. For example,"
+        STDERR.puts "#{Time.now.strftime("%Y-%m-%d %H:%M:%S.%3N %z")}: "
+        STDERR.puts "#{Time.now.strftime("%Y-%m-%d %H:%M:%S.%3N %z")}:   $ embulk run config.yml -c diff.yml"
+        STDERR.puts "#{Time.now.strftime("%Y-%m-%d %H:%M:%S.%3N %z")}: "
+        STDERR.puts "#{Time.now.strftime("%Y-%m-%d %H:%M:%S.%3N %z")}: This -c option stores only diff of the next configuration."
+        STDERR.puts "#{Time.now.strftime("%Y-%m-%d %H:%M:%S.%3N %z")}: The diff will be merged to the original config.yml file."
+        STDERR.puts "#{Time.now.strftime("%Y-%m-%d %H:%M:%S.%3N %z")}: "
         options[:next_config_output_path] = path
       end
+      op.on('-c', '--config-diff PATH', 'Path to a file to read & write the next configuration diff') do |path|
+        options[:next_config_diff_path] = path
+      end
       plugin_load_ops.call
       java_embed_ops.call
       args = 1..1
@@ -227,7 +237,7 @@ examples:
       puts ""
       puts "Run following subcommands to try embulk:"
       puts ""
-      puts "   1. embulk guess #{File.join(path, 'example.yml')} -o config.yml"
+      puts "   1. embulk guess #{File.join(path, 'seed.yml')} -o config.yml"
       puts "   2. embulk preview config.yml"
       puts "   3. embulk run config.yml"
       puts ""

data/lib/embulk/runner.rb

@@ -53,12 +53,19 @@ module Embulk
 
     def run(config, options={})
       configSource = read_config(config, options)
-      output_path = options[:next_config_output_path]
+      config_diff_path = options[:next_config_diff_path]
+      output_path = options[:next_config_output_path]  # deprecated
       resume_state_path = options[:resume_state_path]
 
-      check_file_writable(output_path)
+      check_file_writable(output_path)  # deprecated
+      check_file_writable(config_diff_path)
       check_file_writable(resume_state_path)
 
+      if config_diff_path
+        lastConfigDiff = read_config(config_diff_path, options)
+        configSource = configSource.merge(lastConfigDiff)
+      end
+
       if resume_state_path
         begin
           resumeConfig = read_yaml_config_file(resume_state_path)
@@ -100,7 +107,8 @@ module Embulk
       Embulk.logger.info("Committed.")
       Embulk.logger.info("Next config diff: #{configDiff.toString}")
 
-      write_config(output_path, configSource.merge(configDiff))
+      write_config(config_diff_path, configDiff)
+      write_config(output_path, configSource.merge(configDiff))  # deprecated
     end
 
     #def resume_state(config, options={})

data/lib/embulk/version.rb

@@ -1,3 +1,3 @@
 module Embulk
-  VERSION = '0.8.1'
+  VERSION = '0.8.2'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk
 version: !ruby/object:Gem::Version
-  version: 0.8.1
+  version: 0.8.2
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-01-14 00:00:00.000000000 Z
+date: 2016-02-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: jruby-jars
@@ -107,8 +107,8 @@ files:
 - classpath/bval-jsr303-0.5.jar
 - classpath/commons-beanutils-core-1.8.3.jar
 - classpath/commons-lang3-3.1.jar
-- classpath/embulk-core-0.8.1.jar
-- classpath/embulk-standards-0.8.1.jar
+- classpath/embulk-core-0.8.2.jar
+- classpath/embulk-standards-0.8.2.jar
 - classpath/guava-18.0.jar
 - classpath/guice-4.0.jar
 - classpath/guice-bootstrap-0.1.1.jar
@@ -419,6 +419,7 @@ files:
 - embulk-docs/src/release/release-0.7.9.rst
 - embulk-docs/src/release/release-0.8.0.rst
 - embulk-docs/src/release/release-0.8.1.rst
+- embulk-docs/src/release/release-0.8.2.rst
 - embulk-standards/build.gradle
 - embulk-standards/src/main/java/org/embulk/standards/CsvFormatterPlugin.java
 - embulk-standards/src/main/java/org/embulk/standards/CsvParserPlugin.java