embulk 0.6.21 → 0.6.22

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 90d6280d5bdcffb92922bfd929eae4bcc6d83b92
-  data.tar.gz: 0e2c0d5e8a9f990cdfbb7006be196f9bfef19030
+  metadata.gz: 9517ba89a58e4c0980976822f7508f0b1040ea6c
+  data.tar.gz: dc378ef610b2e1cab1d4b66eb3ff46b64e0817bf
 SHA512:
-  metadata.gz: e2ccbee9c830dd29e86734c1615354083fca35068476a30d0a06944926c3eb63b0386ab5db39d0ad7ed660a2c395da84e68d8433ee222274102dc01b30312de6
-  data.tar.gz: 27174e4750c66516ead7b0ff90df6dc804e8d45694354cd03602654680b30dd0464fef0221aa18a9c28e6495ad6709e44e0c4469b12e55f3ca81d8b1bdb06fdc
+  metadata.gz: 07bd4d4bd1688835f9c830dab95206d7343c0c183427472842106f341d26feea84189f032499dbb1d5c955ae88a348fd806a09003b8fc0b46cd14e41fadf94de
+  data.tar.gz: 976b17fa42e6ba9dd0c02506097af5a0ac32b50155024a70cd9fa831f905ab5909caa9b0fec62ce96f86fd347271d8d39587e99d2a39805287280b42d4480695

data/.gitignore

@@ -12,3 +12,4 @@ vendor/
 .yardoc
 /embulk-*.jar
 /.gradle
+/coverage

data/build.gradle

@@ -1,20 +1,25 @@
-
-plugins {
-    id 'com.jfrog.bintray' version '1.1'
-    id 'com.github.ben-manes.versions' version '0.7'
-    id 'com.github.jruby-gradle.base' version '0.1.5'
-    id 'com.github.johnrengelman.shadow' version '1.2.0'
+buildscript {
+    repositories { jcenter() }
+    dependencies {
+        classpath 'com.jfrog.bintray.gradle:gradle-bintray-plugin:1.3.1'
+        classpath "com.github.jruby-gradle:jruby-gradle-jar-plugin:1.0.1"
+        classpath 'com.github.ben-manes:gradle-versions-plugin:0.11.3'
+        classpath 'com.github.jengelman.gradle.plugins:shadow:1.2.2'
+    }
 }
+apply plugin: "com.github.jruby-gradle.jar"
+apply plugin: 'com.jfrog.bintray'
+apply plugin: 'com.github.johnrengelman.shadow'
 
 def java_projects = [project(":embulk-core"), project(":embulk-standards"), project(":embulk-cli")]
 def release_projects = [project(":embulk-core"), project(":embulk-standards")]
 
 allprojects {
     group = 'org.embulk'
-    version = '0.6.21'
+    version = '0.6.22'
 
     ext {
-        jrubyVersion = '1.7.19'
+        jrubyVersion = '1.7.21'
     }
 
     apply plugin: 'java'
@@ -140,12 +145,17 @@ subprojects {
     }
 }
 
+task wrapper(type: Wrapper) {
+    gradleVersion = '2.6'
+}
+
 import com.github.jrubygradle.JRubyExec
 jruby {
     execVersion = project.jrubyVersion
 }
 
 dependencies {
+    jrubyExec 'rubygems:simplecov:0.10.+'
     jrubyExec 'rubygems:test-unit:3.0.+'
 }
 
@@ -226,8 +236,8 @@ project(':embulk-cli') {
 }
 
 task rubyTest(type: JRubyExec) {
-    jrubyArgs '-Ilib', '-Itest', '-rtest/unit', '-eTest::Unit::AutoRunner.run(true, *ARGV)'
-    script 'test'
+    jrubyArgs '-Ilib', '-Itest', '-rtest/unit', '--debug', '-eARGV.replace([]); Test::Unit::AutoRunner.run(true, "test")'
+    script './lib/embulk/version.rb'  // dummy
 }
 rubyTest.dependsOn('classpath')
 
@@ -235,9 +245,10 @@ rubyTest.dependsOn('classpath')
 // gem task
 //
 task gem(type: JRubyExec) {
-    jrubyArgs '-rrubygems/gem_runner', '-eGem::GemRunner.new.run(ARGV)', 'build'
-    script 'embulk.gemspec'
-    doLast { ant.move(file: "${project.name}-${project.version}.gem", todir: "pkg") }
+    jrubyArgs '-rrubygems/gem_runner', '-eGem::GemRunner.new.run(ARGV)'
+    script './lib/embulk/version.rb'  // dummy
+    scriptArgs 'build', 'embulk.gemspec'
+    doLast { file("${project.name}-${project.version}.gem").renameTo("pkg/${project.name}-${project.version}.gem") }
 }
 gem.dependsOn('classpath')
 
@@ -245,8 +256,9 @@ gem.dependsOn('classpath')
 // rubyGemsUpload task
 //
 task rubyGemsUpload(type: JRubyExec, dependsOn: ["gem"]) {
-    jrubyArgs '-rrubygems/gem_runner', '-eGem::GemRunner.new.run(ARGV)', 'push'
-    script "pkg/embulk-${project.version}.gem"
+    jrubyArgs '-rrubygems/gem_runner', '-eGem::GemRunner.new.run(ARGV)'
+    script './lib/embulk/version.rb'  // dummy
+    scriptArgs 'push', "pkg/embulk-${project.version}.gem"
 }
 
 //

data/embulk-core/build.gradle

@@ -1,6 +1,8 @@
+apply plugin: "com.github.jruby-gradle.jar"
+
 // include ruby scripts to jar. don't use sourceSets.main.resources.srcDirs
 // because IntelliJ causes error if srcDirs includes files out of projectDir.
-processResources.from "${rootProject.projectDir}/lib"
+processResources.from("${rootProject.projectDir}/lib", "${buildDir}/jruby")
 
 configurations {
     // com.google.inject:guice depends on asm and cglib but version of the libraries conflict
@@ -9,6 +11,9 @@ configurations {
     compile.exclude group: 'org.sonatype.sisu.inject', module: 'cglib'
 }
 
+import com.github.jrubygradle.JRubyExec
+import com.github.jrubygradle.JRubyPrepare
+
 // determine which dependencies have updates: $ gradle dependencyUpdates
 dependencies {
     compile 'com.google.guava:guava:18.0'
@@ -35,4 +40,15 @@ dependencies {
 
     // for embulk/guess/charset.rb
     compile 'com.ibm.icu:icu4j:54.1.1'
+
+    gems 'rubygems:liquid:3.0.6'
+}
+
+task unpackGems(type: JRubyPrepare) {
+    outputDir file("${buildDir}/jruby/embulk/gems")
+    dependencies configurations.gems
+    doLast {
+        fileTree(dir: "${buildDir}/jruby/embulk/gems/cache", include: "*.gem").each { f -> f.delete() }
+    }
 }
+processResources.dependsOn("unpackGems")

data/embulk-core/src/main/java/org/embulk/EmbulkEmbed.java

@@ -0,0 +1,216 @@
+package org.embulk;
+
+import java.util.Arrays;
+import java.util.List;
+import com.google.common.base.Function;
+import com.google.common.base.Throwables;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.common.annotations.Beta;
+import com.google.inject.Injector;
+import com.google.inject.Module;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.embulk.config.ModelManager;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.ConfigDiff;
+import org.embulk.config.ConfigLoader;
+import org.embulk.EmbulkService;
+import org.embulk.exec.BulkLoader;
+import org.embulk.exec.GuessExecutor;
+import org.embulk.exec.PreviewExecutor;
+import org.embulk.exec.PreviewResult;
+import org.embulk.exec.ExecutionResult;
+import org.embulk.exec.PartialExecutionException;
+import org.embulk.exec.ResumeState;
+import org.embulk.spi.ExecSession;
+import org.embulk.guice.Bootstrap;
+import org.embulk.guice.CloseableInjector;
+import static com.google.common.base.Preconditions.checkState;
+import static com.google.common.base.Preconditions.checkNotNull;
+
+@Beta
+public class EmbulkEmbed
+        implements AutoCloseable
+{
+    public static ConfigLoader newSystemConfigLoader()
+    {
+        return new ConfigLoader(new ModelManager(null, new ObjectMapper()));
+    }
+
+    private final CloseableInjector injector;
+    private final BulkLoader bulkLoader;
+    private final GuessExecutor guessExecutor;
+    private final PreviewExecutor previewExecutor;
+
+    public EmbulkEmbed(ConfigSource systemConfig, Module... additionalModules)
+    {
+        this(systemConfig, Arrays.asList(additionalModules));
+    }
+
+    public EmbulkEmbed(ConfigSource systemConfig,
+            final Iterable<? extends Module> additionalModules)
+    {
+        this(systemConfig,
+                new Function<List<Module>, Iterable<Module>>()
+                {
+                    public Iterable<Module> apply(List<Module> source)
+                    {
+                        return Iterables.concat(source, additionalModules);
+                    }
+                });
+    }
+
+    public EmbulkEmbed(ConfigSource systemConfig,
+            Function<? super List<Module>, ? extends Iterable<? extends Module>> overrideModules)
+    {
+        this.injector = new Bootstrap()
+            .requireExplicitBindings(false)
+            .addModules(EmbulkService.standardModuleList(systemConfig))
+            .overrideModules(overrideModules)
+            .initializeCloseable();
+        injector.getInstance(org.slf4j.ILoggerFactory.class);
+        this.bulkLoader = new BulkLoader(injector, systemConfig);
+        this.guessExecutor = injector.getInstance(GuessExecutor.class);
+        this.previewExecutor = injector.getInstance(PreviewExecutor.class);
+    }
+
+    @Override
+    public void close()
+    {
+        try {
+            injector.close();
+        }
+        catch (Exception ex) {
+            throw Throwables.propagate(ex);
+        }
+    }
+
+    public Injector getInjector()
+    {
+        return injector;
+    }
+
+    public ModelManager getModelManager()
+    {
+        return injector.getInstance(ModelManager.class);
+    }
+
+    public ConfigLoader newConfigLoader()
+    {
+        return injector.getInstance(ConfigLoader.class);
+    }
+
+    public ExecSession.Builder sessionBuilder(ConfigSource execConfig)
+    {
+        return ExecSession.builder(injector).fromExecConfig(execConfig);
+    }
+
+    public ConfigDiff guess(ExecSession exec, ConfigSource config)
+    {
+        return guessExecutor.guess(exec, config);
+    }
+
+    public PreviewResult preview(ExecSession exec, ConfigSource config)
+    {
+        return previewExecutor.preview(exec, config);
+    }
+
+    public ExecutionResult run(ExecSession exec, ConfigSource config)
+    {
+        try {
+            return bulkLoader.run(exec, config);
+        } catch (PartialExecutionException partial) {
+            try {
+                bulkLoader.cleanup(config, partial.getResumeState());
+            } catch (Throwable ex) {
+                partial.addSuppressed(ex);
+            }
+            throw partial;
+        }
+    }
+
+    public ResumableResult runResumable(ExecSession exec, ConfigSource config)
+    {
+        ExecutionResult result;
+        try {
+            result = bulkLoader.run(exec, config);
+        } catch (PartialExecutionException partial) {
+            return new ResumableResult(partial);
+        }
+        return new ResumableResult(result);
+    }
+
+    public ResumeAction resumeAction(ConfigSource config, ResumeState resumeState)
+    {
+        return new ResumeAction(config, resumeState);
+    }
+
+    public static class ResumableResult
+    {
+        private final ExecutionResult successfulResult;
+        private final PartialExecutionException partialExecutionException;
+
+        public ResumableResult(PartialExecutionException partialExecutionException)
+        {
+            this.successfulResult = null;
+            this.partialExecutionException = checkNotNull(partialExecutionException);
+        }
+
+        public ResumableResult(ExecutionResult successfulResult)
+        {
+            this.successfulResult = checkNotNull(successfulResult);
+            this.partialExecutionException = null;
+        }
+
+        public boolean isSuccessful()
+        {
+            return successfulResult != null;
+        }
+
+        public ExecutionResult getSuccessfulResult()
+        {
+            checkState(successfulResult != null);
+            return successfulResult;
+        }
+
+        public Throwable getCause()
+        {
+            checkState(partialExecutionException != null);
+            return partialExecutionException.getCause();
+        }
+
+        public ResumeState getResumeState()
+        {
+            checkState(partialExecutionException != null);
+            return partialExecutionException.getResumeState();
+        }
+    }
+
+    public class ResumeAction
+    {
+        private final ConfigSource config;
+        private final ResumeState resumeState;
+
+        public ResumeAction(ConfigSource config, ResumeState resumeState)
+        {
+            this.config = config;
+            this.resumeState = resumeState;
+        }
+
+        public ResumableResult resume()
+        {
+            ExecutionResult result;
+            try {
+                result = bulkLoader.resume(config, resumeState);
+            } catch (PartialExecutionException partial) {
+                return new ResumableResult(partial);
+            }
+            return new ResumableResult(result);
+        }
+
+        public void cleanup()
+        {
+            bulkLoader.cleanup(config, resumeState);
+        }
+    }
+}

data/embulk-core/src/main/java/org/embulk/EmbulkService.java

@@ -1,5 +1,6 @@
 package org.embulk;
 
+import java.util.List;
 import com.google.common.collect.ImmutableList;
 import com.google.inject.Guice;
 import com.google.inject.Injector;
@@ -35,18 +36,23 @@ public class EmbulkService
         return modules;
     }
 
+    static List<Module> standardModuleList(ConfigSource systemConfig)
+    {
+        return ImmutableList.of(
+                new SystemConfigModule(systemConfig),
+                new ExecModule(),
+                new ExtensionServiceLoaderModule(systemConfig),
+                new PluginClassLoaderModule(systemConfig),
+                new BuiltinPluginSourceModule(),
+                new JRubyScriptingModule(systemConfig));
+    }
+
     public Injector initialize()
     {
         checkState(!initialized, "Already initialized");
 
         ImmutableList.Builder<Module> builder = ImmutableList.builder();
-        builder.add(new SystemConfigModule(systemConfig));
-        builder.add(new ExecModule());
-        builder.add(new ExtensionServiceLoaderModule(systemConfig));
-        builder.add(new PluginClassLoaderModule(systemConfig));
-        builder.add(new BuiltinPluginSourceModule());
-        builder.add(new JRubyScriptingModule(systemConfig));
-
+        builder.addAll(standardModuleList(systemConfig));
         builder.addAll(getAdditionalModules(systemConfig));
 
         Iterable<? extends Module> modules = builder.build();

data/embulk-core/src/main/java/org/embulk/command/LiquidTemplate.java

@@ -0,0 +1,8 @@
+package org.embulk.command;
+
+import java.util.Map;
+
+public interface LiquidTemplate
+{
+    public String render(String source, Map<String, String> params);
+}

data/embulk-core/src/main/java/org/embulk/command/Runner.java

@@ -10,10 +10,15 @@ import java.io.FileOutputStream;
 import java.io.BufferedWriter;
 import java.io.OutputStreamWriter;
 import java.io.Writer;
+import java.io.ByteArrayInputStream;
+import java.nio.charset.StandardCharsets;
 import org.yaml.snakeyaml.Yaml;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.base.Function;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.io.Files;
 import com.google.inject.Injector;
+import org.jruby.embed.ScriptingContainer;
 import org.embulk.config.ConfigSource;
 import org.embulk.config.DataSource;
 import org.embulk.config.ConfigLoader;
@@ -30,7 +35,7 @@ import org.embulk.exec.ResumeState;
 import org.embulk.exec.PartialExecutionException;
 import org.embulk.spi.time.Timestamp;
 import org.embulk.spi.ExecSession;
-import org.embulk.EmbulkService;
+import org.embulk.EmbulkEmbed;
 
 public class Runner
 {
@@ -60,8 +65,9 @@ public class Runner
 
     private final Options options;
     private final ConfigSource systemConfig;
-    private final EmbulkService service;
-    private final Injector injector;
+
+    private EmbulkEmbed embed;
+    private Injector injector;
 
     public Runner(String optionJson)
     {
@@ -73,8 +79,6 @@ public class Runner
         mergeOptionsToSystemConfig(options, configLoader, systemConfig);
 
         this.systemConfig = systemConfig;
-        this.service = new EmbulkService(systemConfig);
-        this.injector = service.initialize();
     }
 
     @SuppressWarnings("unchecked")
@@ -105,27 +109,31 @@ public class Runner
 
     public void main(String command, String[] args)
     {
-        switch (command) {
-        case "run":
-            run(args[0]);
-            break;
-        case "cleanup":
-            cleanup(args[0]);
-            break;
-        case "guess":
-            guess(args[0]);
-            break;
-        case "preview":
-            preview(args[0]);
-            break;
-        default:
-            throw new RuntimeException("Unsupported command: "+command);
+        try (EmbulkEmbed embed = new EmbulkEmbed(systemConfig)) {
+            this.injector = embed.getInjector();
+
+            switch (command) {
+            case "run":
+                run(args[0]);
+                break;
+            case "cleanup":
+                cleanup(args[0]);
+                break;
+            case "guess":
+                guess(args[0]);
+                break;
+            case "preview":
+                preview(args[0]);
+                break;
+            default:
+                throw new RuntimeException("Unsupported command: "+command);
+            }
         }
     }
 
     public void run(String configPath)
     {
-        ConfigSource config = loadYamlConfig(configPath);
+        ConfigSource config = loadConfig(configPath);
         checkFileWritable(options.getNextConfigOutputPath());
         checkFileWritable(options.getResumeStatePath());
 
@@ -199,7 +207,7 @@ public class Runner
         if (resumePath == null) {
             throw new IllegalArgumentException("Resume path is required for cleanup");
         }
-        ConfigSource config = loadYamlConfig(configPath);
+        ConfigSource config = loadConfig(configPath);
         ConfigSource resumeConfig = loadYamlConfig(resumePath);
         ResumeState resume = resumeConfig.loadConfig(ResumeState.class);
 
@@ -212,7 +220,7 @@ public class Runner
 
     public void guess(String partialConfigPath)
     {
-        ConfigSource config = loadYamlConfig(partialConfigPath);
+        ConfigSource config = loadConfig(partialConfigPath);
         checkFileWritable(options.getNextConfigOutputPath());
 
         ConfigDiff configDiff;
@@ -268,7 +276,7 @@ public class Runner
     {
         PreviewResult result;
         {
-            ConfigSource config = loadYamlConfig(partialConfigPath);
+            ConfigSource config = loadConfig(partialConfigPath);
             ExecSession exec = newExecSession(config);
             try {
                 PreviewExecutor preview = injector.getInstance(PreviewExecutor.class);
@@ -304,12 +312,40 @@ public class Runner
         }
     }
 
-    private ConfigSource loadYamlConfig(String yamlPath)
+    private ConfigSource loadConfig(String path)
+    {
+        if (path.endsWith(".yml") || path.endsWith(".yaml")) {
+            return loadYamlConfig(path);
+        }
+        else if (path.endsWith(".yml.liquid") || path.endsWith(".yaml.liquid")) {
+            return loadLiquidYamlConfig(path);
+        }
+        else {
+            throw new ConfigException("Unknown file extension. Supported file extentions are .yml and .yml.liquid: "+path);
+        }
+    }
+
+    private ConfigSource loadYamlConfig(String path)
     {
         try {
-            return injector.getInstance(ConfigLoader.class).fromYamlFile(new File(yamlPath));
+            return injector.getInstance(ConfigLoader.class).fromYamlFile(new File(path));
+        }
+        catch (IOException ex) {
+            throw new ConfigException(ex);
+        }
+    }
 
-        } catch (IOException ex) {
+    private ConfigSource loadLiquidYamlConfig(String path)
+    {
+        LiquidTemplate helper = (LiquidTemplate) injector.getInstance(ScriptingContainer.class).runScriptlet("Embulk::Java::LiquidTemplateHelper.new");
+        try {
+            String source = Files.toString(new File(path), StandardCharsets.UTF_8);
+            String data = helper.render(source, ImmutableMap.<String,String>of());
+            try (ByteArrayInputStream in = new ByteArrayInputStream(data.getBytes(StandardCharsets.UTF_8))) {
+                return injector.getInstance(ConfigLoader.class).fromYaml(in);
+            }
+        }
+        catch (IOException ex) {
             throw new ConfigException(ex);
         }
     }