embulk 0.6.27 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9d6457e1454597d2e0fac7b4411651cdab29de99
-  data.tar.gz: 2a75ba2f38ffee891935b9072a04f14f67f8b651
+  metadata.gz: c6b45b793c322d59468c3d388e09313718a55de7
+  data.tar.gz: d34278276b5b486c18fd365382a817b1227772b9
 SHA512:
-  metadata.gz: 20cfa0be896376240e28935b95f31c270c32528d8a60b8b96aaf8906369bf6783120bb16375fc4fe37e447e3895aaa6000c87c6ca3ff99cb3ed43b55b5e7b7f9
-  data.tar.gz: 667309b4c8bca0542e43ead5d75b762ef3d552ddcb5a83b8ced5368f9913cfbae51f5e5c72a0f8426fc5662a5a27e5448ecfe200a64402fb545a7a2acedc1cd0
+  metadata.gz: 103dd7e4b4f913ea106fa0d9161e0876234c6f726e21eaca87994094006876f75990fc84d0350dcf2cf1c73570094e8208ce6db56bb6e249ce341ca4e447e5fd
+  data.tar.gz: 305595f0a9bf0a887b6f7eba813df88f6950f011afc82915fe9e88e9eb1bac83adf11bfde0dffa054dd59411eaf2c66149fd1eab8c0b8256f9dc22c08b23d786

data/.ruby-version

@@ -0,0 +1 @@
+2.2.2

data/Gemfile.lock

@@ -1,11 +1,13 @@
 PATH
   remote: .
   specs:
-    embulk (0.6.12)
+    embulk (0.7.0)
+      jruby-jars (= 9.0.0.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
+    jruby-jars (9.0.0.0)
     kramdown (1.5.0)
     power_assert (0.2.2)
     rake (10.4.2)
@@ -24,3 +26,6 @@ DEPENDENCIES
   rake (>= 0.10.0)
   test-unit (~> 3.0.9)
   yard (~> 0.8.7)
+
+BUNDLED WITH
+   1.10.6

data/README.md

@@ -104,13 +104,13 @@ embulk cleanup config.yml -r resume-state.yml
 
 ### Using plugin bundle
 
-`embulk bundle` subcommand creates (or updates if already exists) a private (isolated) bundle of a plugins.
+`embulk bundle new` subcommand creates (or updates if already exists) a private (isolated) bundle of a plugins.
 You can use the bundle using `-b <bundle_dir>` option. `embulk bundle` also generates some example plugins to \<bundle_dir>/embulk/\*.rb directory.
 
 See the generated \<bundle_dir>/Gemfile file how to plugin bundles work.
 
 ```
-embulk bundle ./embulk_bundle
+embulk bundle new ./embulk_bundle
 embulk guess  -b ./embulk_bundle ...
 embulk run    -b ./embulk_bundle ...
 ```
@@ -141,10 +141,10 @@ embulk selfupdate
 You can see JaCoCo's test coverage report at `${project}/build/reports/tests/index.html`
 You can see Findbug's report at `${project}/build/reports/findbug/main.html`  # FIXME coverage information is not included somehow
 
-You can use `classpath` task to use `./bin/embulk` for development:
+You can use `classpath` task to use `bundle exec ./bin/embulk` for development:
 
 ```
-./gradlew classpath  # -x test: skip test
+./gradlew -t classpath  # -x test: skip test
 ./bin/embulk
 ```
 

data/bin/embulk

@@ -1,5 +1,18 @@
 #!/usr/bin/env ruby
 
+if RUBY_PLATFORM =~ /java/i
+  # Enable embulk_bundle if run by CRuby.
+  # Disable embulk_bundle if run by JRuby.
+  if ENV['EMBULK_BIN_ENABLE_BUNDLE'] == File.expand_path(__FILE__)
+    ENV.delete('EMBULK_BIN_ENABLE_BUNDLE')
+    require_relative '../lib/embulk/command/embulk_bundle'
+  else
+    $LOAD_PATH << File.expand_path('../lib', File.dirname(__FILE__))
+    require 'embulk/command/embulk_main'
+  end
+  exit 0
+end
+
 java_args = []
 jruby_args = []
 default_optimize = false
@@ -55,30 +68,39 @@ unless java_cmd
   end
 end
 
-embulk_home = File.dirname(File.dirname(__FILE__))
-classpath_dir = File.join(embulk_home, 'classpath')
-lib_dir = File.join(embulk_home, 'lib')
-
-jruby_complete = Dir.entries(classpath_dir).find {|jar| jar =~ /jruby-complete-[\d\.]+\.jar/ } rescue nil
-unless jruby_complete
-  STDERR.puts "Could not find jruby-complete at #{embulk_home}/classpath directory."
-  if embulk_home == '.'
-    STDERR.puts "Did you run ./gradlew classpath? You need to build java code and create ./classpath directory first."
-  end
-  raise SystemExit.new(1)
-end
-
 if overwrite_optimize == true || (default_optimize == true && overwrite_optimize != false)
   java_args = %w[-XX:+AggressiveOpts -XX:+UseConcMarkSweepGC] + java_args
 else
   java_args = %w[-XX:+AggressiveOpts -XX:+TieredCompilation -XX:TieredStopAtLevel=1 -Xverify:none] + java_args
 end
 
-# java ... -jar ruby-complete.jar $EMBULK_HOME/lib/embulk/command/embulk.rb "$@"
+# Disable bundler options of CRuby before starting JRuby
+ENV.delete('GEM_HOME')
+ENV.delete('GEM_PATH')
+ENV.delete('BUNDLE_GEMFILE')
+ENV.delete('BUNDLE_BIN_PATH')
+ENV.delete('RUBYLIB')
+ENV.delete('RUBYOPT')
+
+env = {
+  'EMBULK_BIN_ENABLE_BUNDLE' => File.expand_path(__FILE__)
+}
+
+begin
+  require 'jruby-jars'
+rescue LoadError => e
+  STDERR.puts  "Could not load jruby-jar.gem. Did you run with \"bundle exec\"?"
+  raise e
+end
+jruby_cp = "#{File.dirname(JRubyJars.core_jar_path)}/*"
+
+# java ... -jar ruby-complete.jar bin/embulk "$@"
 cmdline = [java_cmd]
 cmdline.concat java_args
-cmdline << '-jar' << File.join(classpath_dir, jruby_complete)
+cmdline << '-cp' << jruby_cp << 'org.jruby.Main'
 cmdline.concat jruby_args
-cmdline << File.join(lib_dir, 'embulk/command/embulk.rb')
+cmdline << __FILE__
 cmdline.concat ARGV
-exec *cmdline
+exec env, *cmdline
+exit 127
+

data/build.gradle

@@ -16,10 +16,10 @@ def release_projects = [project(":embulk-core"), project(":embulk-standards")]
 
 allprojects {
     group = 'org.embulk'
-    version = '0.6.27'
+    version = '0.7.0'
 
     ext {
-        jrubyVersion = '1.7.21'
+        jrubyVersion = '9.0.0.0'
     }
 
     apply plugin: 'java'
@@ -247,21 +247,35 @@ rubyTest.dependsOn('classpath')
 //
 // gem task
 //
-task gem(type: JRubyExec) {
+task gemJRuby(type: JRubyExec, dependsOn: ["classpath"]) {
     jrubyArgs '-rrubygems/gem_runner', '-eGem::GemRunner.new.run(ARGV)'
     script './lib/embulk/version.rb'  // dummy
     scriptArgs 'build', 'embulk.gemspec'
+    doLast { file("${project.name}-${project.version}-java.gem").renameTo("pkg/${project.name}-${project.version}-java.gem") }
+}
+
+task gem(type: Exec, dependsOn: ["classpath", "gemJRuby"]) {
+    executable "gem"
+    args "build", "embulk.gemspec"
+    doFirst {
+        def rubyVersion = file(".ruby-version").getText().trim()
+        if (!"ruby --version".execute().text.contains(rubyVersion)) {
+            throw new GradleException("Version of 'ruby' command in PATH must be ${rubyVersion}. Use rbenv and install ${rubyVersion}")
+        }
+    }
     doLast { file("${project.name}-${project.version}.gem").renameTo("pkg/${project.name}-${project.version}.gem") }
 }
-gem.dependsOn('classpath')
 
 //
 // rubyGemsUpload task
 //
-task rubyGemsUpload(type: JRubyExec, dependsOn: ["gem"]) {
-    jrubyArgs '-rrubygems/gem_runner', '-eGem::GemRunner.new.run(ARGV)'
-    script './lib/embulk/version.rb'  // dummy
-    scriptArgs 'push', "pkg/embulk-${project.version}.gem"
+task rubyGemsUploadJRuby(type: Exec, dependsOn: ["gem"]) {
+    executable "gem"
+    args "push", "pkg/embulk-${project.version}-java.gem"
+}
+task rubyGemsUpload(type: Exec, dependsOn: ["gem", "rubyGemsUploadJRuby"]) {
+    executable "gem"
+    args "push", "pkg/embulk-${project.version}.gem"
 }
 
 //
@@ -296,6 +310,7 @@ Manual operations:
 }
 bintrayUpload.mustRunAfter('releaseCheck')
 rubyGemsUpload.mustRunAfter('releaseCheck')
+rubyGemsUploadJRuby.mustRunAfter('releaseCheck')
 
 task setVersion << {
     if (!project.hasProperty("to")) {

data/embulk-cli/src/main/java/org/embulk/cli/Main.java

@@ -1,12 +1,21 @@
 package org.embulk.cli;
 
+import java.net.URISyntaxException;
+
 public class Main
 {
     public static void main(String[] args)
     {
-        // $ java -jar jruby-complete.jar classpath:embulk/command/embulk.rb "$@"
+        // $ java -jar jruby-complete.jar embulk-core.jar!/embulk/command/embulk_bundle.rb "$@"
         String[] jrubyArgs = new String[args.length + 1];
-        jrubyArgs[0] = "classpath:embulk/command/embulk.rb";
+        String resourcePath;
+        try {
+            resourcePath = Main.class.getProtectionDomain().getCodeSource().getLocation().toURI().toString() + "!/";
+        }
+        catch (URISyntaxException ex) {
+            resourcePath = "classpath:";
+        }
+        jrubyArgs[0] = resourcePath + "embulk/command/embulk_bundle.rb";
         System.arraycopy(args, 0, jrubyArgs, 1, args.length);
         org.jruby.Main.main(jrubyArgs);
     }

data/embulk-core/build.gradle

@@ -2,7 +2,7 @@ apply plugin: "com.github.jruby-gradle.jar"
 
 // include ruby scripts to jar. don't use sourceSets.main.resources.srcDirs
 // because IntelliJ causes error if srcDirs includes files out of projectDir.
-processResources.from("${rootProject.projectDir}/lib", "${buildDir}/jruby")
+processResources.from("${rootProject.projectDir}/lib/", "${buildDir}/gemlib")
 
 configurations {
     // com.google.inject:guice depends on asm and cglib but version of the libraries conflict
@@ -38,17 +38,22 @@ dependencies {
     compile 'io.netty:netty-buffer:5.0.0.Alpha1'
     compile 'org.fusesource.jansi:jansi:1.11'
 
-    // for embulk/guess/charset.rb
+    // For embulk/guess/charset.rb. See also embulk.gemspec
     compile 'com.ibm.icu:icu4j:54.1.1'
 
     gems 'rubygems:liquid:3.0.6'
 }
 
 task unpackGems(type: JRubyPrepare) {
-    outputDir file("${buildDir}/jruby/embulk/gems")
+    outputDir file("${buildDir}/gem")
     dependencies configurations.gems
     doLast {
-        fileTree(dir: "${buildDir}/jruby/embulk/gems/cache", include: "*.gem").each { f -> f.delete() }
+        // move to build/gem/*/lib/* to build/gemlib/
+        file("${buildDir}/gemlib").mkdirs()
+        file("${buildDir}/gem/gems").eachDir { gemDir ->
+            file("${gemDir}/lib").renameTo("${buildDir}/gemlib/")
+        }
+        fileTree(dir: "${buildDir}/gemlib", include: "**/.jrubydir").each { f -> f.delete() }
     }
 }
 processResources.dependsOn("unpackGems")

data/embulk-core/src/main/java/org/embulk/EmbulkEmbed.java

@@ -1,12 +1,12 @@
 package org.embulk;
 
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 import com.google.common.base.Function;
 import com.google.common.base.Throwables;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Iterables;
-import com.google.common.annotations.Beta;
 import com.google.inject.Injector;
 import com.google.inject.Module;
 import com.fasterxml.jackson.databind.ObjectMapper;
@@ -14,7 +14,6 @@ import org.embulk.config.ModelManager;
 import org.embulk.config.ConfigSource;
 import org.embulk.config.ConfigDiff;
 import org.embulk.config.ConfigLoader;
-import org.embulk.EmbulkService;
 import org.embulk.exec.BulkLoader;
 import org.embulk.exec.GuessExecutor;
 import org.embulk.exec.PreviewExecutor;
@@ -22,69 +21,115 @@ import org.embulk.exec.PreviewResult;
 import org.embulk.exec.ExecutionResult;
 import org.embulk.exec.PartialExecutionException;
 import org.embulk.exec.ResumeState;
+import org.embulk.spi.BufferAllocator;
 import org.embulk.spi.ExecSession;
 import org.embulk.guice.Bootstrap;
-import org.embulk.guice.CloseableInjector;
+import org.embulk.guice.LifeCycleInjector;
 import static com.google.common.base.Preconditions.checkState;
 import static com.google.common.base.Preconditions.checkNotNull;
 
-@Beta
 public class EmbulkEmbed
-        implements AutoCloseable
 {
     public static ConfigLoader newSystemConfigLoader()
     {
         return new ConfigLoader(new ModelManager(null, new ObjectMapper()));
     }
 
-    private final CloseableInjector injector;
-    private final BulkLoader bulkLoader;
-    private final GuessExecutor guessExecutor;
-    private final PreviewExecutor previewExecutor;
-
-    public EmbulkEmbed(ConfigSource systemConfig, Module... additionalModules)
+    public static class Bootstrap
     {
-        this(systemConfig, Arrays.asList(additionalModules));
-    }
+        private final ConfigLoader systemConfigLoader;
 
-    public EmbulkEmbed(ConfigSource systemConfig,
-            final Iterable<? extends Module> additionalModules)
-    {
-        this(systemConfig,
-                new Function<List<Module>, Iterable<Module>>()
-                {
-                    public Iterable<Module> apply(List<Module> source)
+        private ConfigSource systemConfig;
+
+        private final List<Function<? super List<Module>, ? extends Iterable<? extends Module>>> moduleOverrides;
+
+        public Bootstrap()
+        {
+            this.systemConfigLoader = newSystemConfigLoader();
+            this.systemConfig = systemConfigLoader.newConfigSource();
+            this.moduleOverrides = new ArrayList<>();
+        }
+
+        public ConfigLoader getSystemConfigLoader()
+        {
+            return systemConfigLoader;
+        }
+
+        public Bootstrap setSystemConfig(ConfigSource systemConfig)
+        {
+            this.systemConfig = systemConfig.deepCopy();
+            return this;
+        }
+
+        public Bootstrap addModules(Module... additionalModules)
+        {
+            return addModules(Arrays.asList(additionalModules));
+        }
+
+        public Bootstrap addModules(Iterable<? extends Module> additionalModules)
+        {
+            final List<Module> copy = ImmutableList.copyOf(additionalModules);
+            return overrideModules(
+                    new Function<List<Module>, Iterable<Module>>()
                     {
-                        return Iterables.concat(source, additionalModules);
-                    }
-                });
+                        public Iterable<Module> apply(List<Module> modules)
+                        {
+                            return Iterables.concat(modules, copy);
+                        }
+                    });
+        }
+
+        public Bootstrap overrideModules(Function<? super List<Module>, ? extends Iterable<? extends Module>> function)
+        {
+            moduleOverrides.add(function);
+            return this;
+        }
+
+        public EmbulkEmbed initialize()
+        {
+            return build(true);
+        }
+
+        public EmbulkEmbed initializeCloseable()
+        {
+            return build(false);
+        }
+
+        private EmbulkEmbed build(boolean destroyOnShutdownHook)
+        {
+            org.embulk.guice.Bootstrap bootstrap = new org.embulk.guice.Bootstrap()
+                .requireExplicitBindings(false)
+                .addModules(EmbulkService.standardModuleList(systemConfig));
+
+            for (Function<? super List<Module>, ? extends Iterable<? extends Module>> override : moduleOverrides) {
+                bootstrap = bootstrap.overrideModules(override);
+            }
+
+            LifeCycleInjector injector;
+            if (destroyOnShutdownHook) {
+                injector = bootstrap.initialize();
+            } else {
+                injector = bootstrap.initializeCloseable();
+            }
+
+            return new EmbulkEmbed(systemConfig, injector);
+        }
     }
 
-    public EmbulkEmbed(ConfigSource systemConfig,
-            Function<? super List<Module>, ? extends Iterable<? extends Module>> overrideModules)
+    private final LifeCycleInjector injector;
+    private final BulkLoader bulkLoader;
+    private final GuessExecutor guessExecutor;
+    private final PreviewExecutor previewExecutor;
+
+    EmbulkEmbed(ConfigSource systemConfig, LifeCycleInjector injector)
     {
-        this.injector = new Bootstrap()
-            .requireExplicitBindings(false)
-            .addModules(EmbulkService.standardModuleList(systemConfig))
-            .overrideModules(overrideModules)
-            .initializeCloseable();
+        this.injector = injector;
         injector.getInstance(org.slf4j.ILoggerFactory.class);
         this.bulkLoader = new BulkLoader(injector, systemConfig);
         this.guessExecutor = injector.getInstance(GuessExecutor.class);
         this.previewExecutor = injector.getInstance(PreviewExecutor.class);
     }
 
-    @Override
-    public void close()
-    {
-        try {
-            injector.close();
-        }
-        catch (Exception ex) {
-            throw Throwables.propagate(ex);
-        }
-    }
-
     public Injector getInjector()
     {
         return injector;
@@ -95,31 +140,45 @@ public class EmbulkEmbed
         return injector.getInstance(ModelManager.class);
     }
 
-    public ConfigLoader newConfigLoader()
+    public BufferAllocator getBufferAllocator()
     {
-        return injector.getInstance(ConfigLoader.class);
+        return injector.getInstance(BufferAllocator.class);
     }
 
-    public ExecSession.Builder sessionBuilder(ConfigSource execConfig)
+    public ConfigLoader newConfigLoader()
     {
-        return ExecSession.builder(injector).fromExecConfig(execConfig);
+        return injector.getInstance(ConfigLoader.class);
     }
 
-    public ConfigDiff guess(ExecSession exec, ConfigSource config)
+    public ConfigDiff guess(ConfigSource config)
     {
-        return guessExecutor.guess(exec, config);
+        ExecSession exec = newExecSession(config);
+        try {
+            return guessExecutor.guess(exec, config);
+        }
+        finally {
+            exec.cleanup();
+        }
     }
 
-    public PreviewResult preview(ExecSession exec, ConfigSource config)
+    public PreviewResult preview(ConfigSource config)
     {
-        return previewExecutor.preview(exec, config);
+        ExecSession exec = newExecSession(config);
+        try {
+            return previewExecutor.preview(exec, config);
+        }
+        finally {
+            exec.cleanup();
+        }
     }
 
-    public ExecutionResult run(ExecSession exec, ConfigSource config)
+    public ExecutionResult run(ConfigSource config)
     {
+        ExecSession exec = newExecSession(config);
         try {
             return bulkLoader.run(exec, config);
-        } catch (PartialExecutionException partial) {
+        }
+        catch (PartialExecutionException partial) {
             try {
                 bulkLoader.cleanup(config, partial.getResumeState());
             } catch (Throwable ex) {
@@ -127,22 +186,52 @@ public class EmbulkEmbed
             }
             throw partial;
         }
+        finally {
+            try {
+                exec.cleanup();
+            }
+            catch (Exception ex) {
+                // TODO add this exception to ExecutionResult.getIgnoredExceptions
+                // or partial.addSuppressed
+                ex.printStackTrace(System.err);
+            }
+        }
     }
 
-    public ResumableResult runResumable(ExecSession exec, ConfigSource config)
+    public ResumableResult runResumable(ConfigSource config)
     {
-        ExecutionResult result;
+        ExecSession exec = newExecSession(config);
         try {
-            result = bulkLoader.run(exec, config);
-        } catch (PartialExecutionException partial) {
-            return new ResumableResult(partial);
+            ExecutionResult result;
+            try {
+                result = bulkLoader.run(exec, config);
+            } catch (PartialExecutionException partial) {
+                return new ResumableResult(partial);
+            }
+            return new ResumableResult(result);
+        }
+        finally {
+            try {
+                exec.cleanup();
+            }
+            catch (Exception ex) {
+                // TODO add this exception to ExecutionResult.getIgnoredExceptions
+                // or partial.addSuppressed
+                ex.printStackTrace(System.err);
+            }
         }
-        return new ResumableResult(result);
     }
 
-    public ResumeAction resumeAction(ConfigSource config, ResumeState resumeState)
+    private ExecSession newExecSession(ConfigSource config)
     {
-        return new ResumeAction(config, resumeState);
+        ConfigSource execConfig = config.deepCopy().getNestedOrSetEmpty("exec");
+        return ExecSession.builder(injector).fromExecConfig(execConfig).build();
+    }
+
+    public ResumeStateAction resumeState(ConfigSource config, ConfigSource resumeStateConfig)
+    {
+        ResumeState resumeState = resumeStateConfig.loadConfig(ResumeState.class);
+        return new ResumeStateAction(config, resumeState);
     }
 
     public static class ResumableResult
@@ -186,12 +275,12 @@ public class EmbulkEmbed
         }
     }
 
-    public class ResumeAction
+    public class ResumeStateAction
     {
         private final ConfigSource config;
         private final ResumeState resumeState;
 
-        public ResumeAction(ConfigSource config, ResumeState resumeState)
+        public ResumeStateAction(ConfigSource config, ResumeState resumeState)
         {
             this.config = config;
             this.resumeState = resumeState;
@@ -213,4 +302,14 @@ public class EmbulkEmbed
             bulkLoader.cleanup(config, resumeState);
         }
     }
+
+    public void destroy()
+    {
+        try {
+            injector.destroy();
+        }
+        catch (Exception ex) {
+            throw Throwables.propagate(ex);
+        }
+    }
 }