embulk 0.7.0-java

data/embulk-cli/src/test/java/org/embulk/cli/DummyMain.java

@@ -0,0 +1,23 @@
+package org.embulk.cli;
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.OutputStreamWriter;
+import java.nio.charset.Charset;
+import java.util.Arrays;
+
+public class DummyMain {
+
+    public static void main(String[] args) throws Exception {
+        System.out.println(Arrays.asList(args));
+        File thisFolder = new File(SelfrunTest.class.getResource("/org/embulk/cli/DummyMain.class").toURI()).getParentFile();
+        try (BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(new File(thisFolder, "args.txt")), Charset.defaultCharset()))) {
+            for (String arg : args) {
+                writer.write(arg);
+                writer.newLine();
+            }
+        }
+    }
+
+}

data/embulk-cli/src/test/java/org/embulk/cli/SelfrunTest.java

@@ -0,0 +1,281 @@
+package org.embulk.cli;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
+import java.nio.charset.Charset;
+import java.nio.file.FileSystem;
+import java.nio.file.FileSystems;
+import java.nio.file.Files;
+import java.nio.file.StandardOpenOption;
+import java.util.Arrays;
+import java.util.List;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+
+public class SelfrunTest {
+
+    private static File testSelfrun;
+
+    @BeforeClass
+    public static void prepare() throws Exception {
+        File selfrun = findSelfrun();
+        FileSystem fs = FileSystems.getDefault();
+        String line = new String(Files.readAllBytes(fs.getPath(selfrun.getAbsolutePath())), Charset.defaultCharset());
+
+        File thisFolder = new File(SelfrunTest.class.getResource("/org/embulk/cli/SelfrunTest.class").toURI()).getParentFile();
+        testSelfrun = new File(thisFolder, System.getProperty("file.separator").equals("\\") ? "selfrun.bat" : "selfrun.sh");
+
+        File classpath = thisFolder.getParentFile().getParentFile().getParentFile();
+        line = line.replaceAll("java ", "java -classpath " + classpath.getAbsolutePath().replaceAll("\\\\", "\\\\\\\\") + " org.embulk.cli.DummyMain ");
+
+        // Modify selfrun so that arguments are written in 'args.txt' .
+        Files.write(fs.getPath(testSelfrun.getAbsolutePath()), line.getBytes(Charset.defaultCharset()), StandardOpenOption.CREATE);
+        if (!testSelfrun.setExecutable(true)) {
+        	throw new Exception("Cannot se executable.");
+        }
+    }
+
+
+    @Test
+    public void testNoArgument() throws Exception {
+        List<String> args = execute();
+        assertEquals(Arrays.asList(
+                "-XX:+AggressiveOpts",
+                "-XX:+TieredCompilation",
+                "-XX:TieredStopAtLevel=1",
+                "-Xverify:none",
+                "-jar",
+                testSelfrun.getAbsolutePath()),
+                args);
+    }
+
+    @Test
+    public void testArguments() throws Exception {
+        List<String> args = execute("a1", "a2", "\"a3=v3\"");
+        assertEquals(Arrays.asList(
+                "-XX:+AggressiveOpts",
+                "-XX:+TieredCompilation",
+                "-XX:TieredStopAtLevel=1",
+                "-Xverify:none",
+                "-jar",
+                testSelfrun.getAbsolutePath(),
+                "a1",
+                "a2",
+                "a3=v3"),
+                args);
+    }
+
+    @Test
+    public void testRun() throws Exception {
+        List<String> args = execute("run", "a1");
+        assertEquals(Arrays.asList(
+                "-XX:+AggressiveOpts",
+                "-XX:+UseConcMarkSweepGC",
+                "-jar",
+                testSelfrun.getAbsolutePath(),
+                "run",
+                "a1"),
+                args);
+    }
+
+    @Test
+    public void testJpO() throws Exception {
+        List<String> args = execute("-J+O", "a1", "a2");
+        assertEquals(Arrays.asList(
+                "-XX:+AggressiveOpts",
+                "-XX:+UseConcMarkSweepGC",
+                "-jar",
+                testSelfrun.getAbsolutePath(),
+                "a1",
+                "a2"),
+                args);
+    }
+
+    @Test
+    public void testJmO() throws Exception {
+        List<String> args = execute("-J-O", "a1", "a2");
+        assertEquals(Arrays.asList(
+                "-XX:+AggressiveOpts",
+                "-XX:+TieredCompilation",
+                "-XX:TieredStopAtLevel=1",
+                "-Xverify:none",
+                "-jar",
+                testSelfrun.getAbsolutePath(),
+                "a1",
+                "a2"),
+                args);
+    }
+
+    @Test
+    public void testR1() throws Exception {
+        List<String> args = execute("-Rr1", "a1", "a2");
+        assertEquals(Arrays.asList(
+                "-XX:+AggressiveOpts",
+                "-XX:+TieredCompilation",
+                "-XX:TieredStopAtLevel=1",
+                "-Xverify:none",
+                "-jar",
+                testSelfrun.getAbsolutePath(),
+                "r1",
+                "a1",
+                "a2"),
+                args);
+    }
+
+    @Test
+    public void testR2() throws Exception {
+        List<String> args = execute("\"-Rr1=v1\"", "\"-Rr2=v2\"", "a1", "a2");
+        assertEquals(Arrays.asList(
+                "-XX:+AggressiveOpts",
+                "-XX:+TieredCompilation",
+                "-XX:TieredStopAtLevel=1",
+                "-Xverify:none",
+                "-jar",
+                testSelfrun.getAbsolutePath(),
+                "r1=v1",
+                "r2=v2",
+                "a1",
+                "a2"),
+                args);
+    }
+
+    @Test
+    public void testRRun() throws Exception {
+        List<String> args = execute("-Rr1", "run", "a1");
+        assertEquals(Arrays.asList(
+                "-XX:+AggressiveOpts",
+                "-XX:+UseConcMarkSweepGC",
+                "-jar",
+                testSelfrun.getAbsolutePath(),
+                "r1",
+                "run",
+                "a1"),
+                args);
+    }
+
+    @Test
+    public void testJ1() throws Exception {
+        List<String> args = execute("-J-Dj1", "a1", "a2");
+        assertEquals(Arrays.asList(
+                "-XX:+AggressiveOpts",
+                "-XX:+TieredCompilation",
+                "-XX:TieredStopAtLevel=1",
+                "-Xverify:none",
+                "-Dj1",
+                "-jar",
+                testSelfrun.getAbsolutePath(),
+                "a1",
+                "a2"),
+                args);
+    }
+
+    @Test
+    public void testJ2() throws Exception {
+        List<String> args = execute("\"-J-Dj1=v1\"", "\"-J-Dj2=v2\"", "a1", "a2");
+        assertEquals(Arrays.asList(
+                "-XX:+AggressiveOpts",
+                "-XX:+TieredCompilation",
+                "-XX:TieredStopAtLevel=1",
+                "-Xverify:none",
+                "-Dj1=v1",
+                "-Dj2=v2",
+                "-jar",
+                testSelfrun.getAbsolutePath(),
+                "a1",
+                "a2"),
+                args);
+    }
+
+    @Test
+    public void testJR() throws Exception {
+        List<String> args = execute("-Jj1", "-Rr1", "a1", "a2");
+        assertEquals(Arrays.asList(
+                "-XX:+AggressiveOpts",
+                "-XX:+TieredCompilation",
+                "-XX:TieredStopAtLevel=1",
+                "-Xverify:none",
+                "j1",
+                "-jar",
+                testSelfrun.getAbsolutePath(),
+                "r1",
+                "a1",
+                "a2"),
+                args);
+    }
+
+    @Test
+    public void testJFile() throws Exception {
+        File javaArgsFile = new File(testSelfrun.getParentFile(), "java_args.txt");
+        FileSystem fs = FileSystems.getDefault();
+        Files.write(fs.getPath(javaArgsFile.getAbsolutePath()), "j1 j2 j3".getBytes(Charset.defaultCharset()), StandardOpenOption.CREATE);
+
+        List<String> args = execute("-J", javaArgsFile.getAbsolutePath(), "a1", "a2");
+        assertEquals(Arrays.asList(
+                "-XX:+AggressiveOpts",
+                "-XX:+TieredCompilation",
+                "-XX:TieredStopAtLevel=1",
+                "-Xverify:none",
+                "j1",
+                "j2",
+                "j3",
+                "-jar",
+                testSelfrun.getAbsolutePath(),
+                "a1",
+                "a2"),
+                args);
+    }
+
+    private List<String> execute(String... arguments) throws Exception {
+        File temp = new File(testSelfrun.getParentFile(), "call-" + testSelfrun.getName());
+        try (BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(temp), Charset.defaultCharset()))) {
+            writer.write(testSelfrun.getAbsolutePath());
+            for (String argument : arguments) {
+                writer.write(" ");
+                writer.write(argument);
+            }
+        }
+        if (!temp.setExecutable(true)) {
+        	throw new Exception("Cannot se executable.");
+        }
+
+        File argsFile = new File(testSelfrun.getParentFile(), "args.txt");
+        if (argsFile.exists()) {
+            if (!argsFile.delete()) {
+            	throw new IOException("Cannot delete " + argsFile);
+            }
+        }
+
+        Process process = Runtime.getRuntime().exec(temp.getAbsolutePath());
+        int exitCode = process.waitFor();
+        if (exitCode != 0 || !argsFile.exists()) {
+            StringBuilder builder = new StringBuilder();
+            try (BufferedReader reader = new BufferedReader(new InputStreamReader(process.getErrorStream(), Charset.defaultCharset()))) {
+                builder.append(reader.readLine());
+                builder.append(System.getProperty("line.separator"));
+            }
+            throw new Exception(builder.toString());
+        }
+
+        FileSystem fs = FileSystems.getDefault();
+        List<String> args = Files.readAllLines(fs.getPath(argsFile.getAbsolutePath()), Charset.defaultCharset());
+        return args;
+    }
+
+    private static File findSelfrun() {
+        File folder = new File(".");
+        if (new File(folder, "embulk-cli").exists()) {
+            folder = new File(folder, "embulk-cli");
+        }
+        return new File(new File(new File(new File(folder, "src"), "main"), "sh"), "selfrun.sh");
+    }
+
+}

data/embulk-core/build.gradle

@@ -0,0 +1,59 @@
+apply plugin: "com.github.jruby-gradle.jar"
+
+// include ruby scripts to jar. don't use sourceSets.main.resources.srcDirs
+// because IntelliJ causes error if srcDirs includes files out of projectDir.
+processResources.from("${rootProject.projectDir}/lib/", "${buildDir}/gemlib")
+
+configurations {
+    // com.google.inject:guice depends on asm and cglib but version of the libraries conflict
+    // with ones bundled in jruby-complete and cause bytecode compatibility error
+    compile.exclude group: 'asm', module: 'asm'
+    compile.exclude group: 'org.sonatype.sisu.inject', module: 'cglib'
+}
+
+import com.github.jrubygradle.JRubyExec
+import com.github.jrubygradle.JRubyPrepare
+
+// determine which dependencies have updates: $ gradle dependencyUpdates
+dependencies {
+    compile 'com.google.guava:guava:18.0'
+    compile 'com.google.inject:guice:4.0'
+    compile 'com.google.inject.extensions:guice-multibindings:4.0'
+    compile 'javax.inject:javax.inject:1'
+    compile 'com.fasterxml.jackson.core:jackson-annotations:2.5.3'
+    compile 'com.fasterxml.jackson.core:jackson-core:2.5.3'
+    compile 'com.fasterxml.jackson.core:jackson-databind:2.5.3'
+    compile 'com.fasterxml.jackson.datatype:jackson-datatype-guava:2.5.3'
+    compile 'com.fasterxml.jackson.datatype:jackson-datatype-joda:2.5.3'
+    compile 'com.fasterxml.jackson.module:jackson-module-guice:2.5.3'
+    compile 'ch.qos.logback:logback-classic:1.1.3'
+    compile 'org.slf4j:slf4j-api:1.7.12'
+    compile 'org.jruby:jruby-complete:' + project.jrubyVersion
+    compile 'com.google.code.findbugs:annotations:3.0.0'
+    compile 'org.yaml:snakeyaml:1.14'
+    compile 'javax.validation:validation-api:1.1.0.Final'
+    compile 'org.apache.bval:bval-jsr303:0.5'
+    compile 'io.airlift:slice:0.9'
+    compile 'joda-time:joda-time:2.8.1'
+    compile 'io.netty:netty-buffer:5.0.0.Alpha1'
+    compile 'org.fusesource.jansi:jansi:1.11'
+
+    // For embulk/guess/charset.rb. See also embulk.gemspec
+    compile 'com.ibm.icu:icu4j:54.1.1'
+
+    gems 'rubygems:liquid:3.0.6'
+}
+
+task unpackGems(type: JRubyPrepare) {
+    outputDir file("${buildDir}/gem")
+    dependencies configurations.gems
+    doLast {
+        // move to build/gem/*/lib/* to build/gemlib/
+        file("${buildDir}/gemlib").mkdirs()
+        file("${buildDir}/gem/gems").eachDir { gemDir ->
+            file("${gemDir}/lib").renameTo("${buildDir}/gemlib/")
+        }
+        fileTree(dir: "${buildDir}/gemlib", include: "**/.jrubydir").each { f -> f.delete() }
+    }
+}
+processResources.dependsOn("unpackGems")

data/embulk-core/src/main/java/org/embulk/EmbulkEmbed.java

@@ -0,0 +1,315 @@
+package org.embulk;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import com.google.common.base.Function;
+import com.google.common.base.Throwables;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.inject.Injector;
+import com.google.inject.Module;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.embulk.config.ModelManager;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.ConfigDiff;
+import org.embulk.config.ConfigLoader;
+import org.embulk.exec.BulkLoader;
+import org.embulk.exec.GuessExecutor;
+import org.embulk.exec.PreviewExecutor;
+import org.embulk.exec.PreviewResult;
+import org.embulk.exec.ExecutionResult;
+import org.embulk.exec.PartialExecutionException;
+import org.embulk.exec.ResumeState;
+import org.embulk.spi.BufferAllocator;
+import org.embulk.spi.ExecSession;
+import org.embulk.guice.Bootstrap;
+import org.embulk.guice.LifeCycleInjector;
+import static com.google.common.base.Preconditions.checkState;
+import static com.google.common.base.Preconditions.checkNotNull;
+
+public class EmbulkEmbed
+{
+    public static ConfigLoader newSystemConfigLoader()
+    {
+        return new ConfigLoader(new ModelManager(null, new ObjectMapper()));
+    }
+
+    public static class Bootstrap
+    {
+        private final ConfigLoader systemConfigLoader;
+
+        private ConfigSource systemConfig;
+
+        private final List<Function<? super List<Module>, ? extends Iterable<? extends Module>>> moduleOverrides;
+
+        public Bootstrap()
+        {
+            this.systemConfigLoader = newSystemConfigLoader();
+            this.systemConfig = systemConfigLoader.newConfigSource();
+            this.moduleOverrides = new ArrayList<>();
+        }
+
+        public ConfigLoader getSystemConfigLoader()
+        {
+            return systemConfigLoader;
+        }
+
+        public Bootstrap setSystemConfig(ConfigSource systemConfig)
+        {
+            this.systemConfig = systemConfig.deepCopy();
+            return this;
+        }
+
+        public Bootstrap addModules(Module... additionalModules)
+        {
+            return addModules(Arrays.asList(additionalModules));
+        }
+
+        public Bootstrap addModules(Iterable<? extends Module> additionalModules)
+        {
+            final List<Module> copy = ImmutableList.copyOf(additionalModules);
+            return overrideModules(
+                    new Function<List<Module>, Iterable<Module>>()
+                    {
+                        public Iterable<Module> apply(List<Module> modules)
+                        {
+                            return Iterables.concat(modules, copy);
+                        }
+                    });
+        }
+
+        public Bootstrap overrideModules(Function<? super List<Module>, ? extends Iterable<? extends Module>> function)
+        {
+            moduleOverrides.add(function);
+            return this;
+        }
+
+        public EmbulkEmbed initialize()
+        {
+            return build(true);
+        }
+
+        public EmbulkEmbed initializeCloseable()
+        {
+            return build(false);
+        }
+
+        private EmbulkEmbed build(boolean destroyOnShutdownHook)
+        {
+            org.embulk.guice.Bootstrap bootstrap = new org.embulk.guice.Bootstrap()
+                .requireExplicitBindings(false)
+                .addModules(EmbulkService.standardModuleList(systemConfig));
+
+            for (Function<? super List<Module>, ? extends Iterable<? extends Module>> override : moduleOverrides) {
+                bootstrap = bootstrap.overrideModules(override);
+            }
+
+            LifeCycleInjector injector;
+            if (destroyOnShutdownHook) {
+                injector = bootstrap.initialize();
+            } else {
+                injector = bootstrap.initializeCloseable();
+            }
+
+            return new EmbulkEmbed(systemConfig, injector);
+        }
+    }
+
+    private final LifeCycleInjector injector;
+    private final BulkLoader bulkLoader;
+    private final GuessExecutor guessExecutor;
+    private final PreviewExecutor previewExecutor;
+
+    EmbulkEmbed(ConfigSource systemConfig, LifeCycleInjector injector)
+    {
+        this.injector = injector;
+        injector.getInstance(org.slf4j.ILoggerFactory.class);
+        this.bulkLoader = new BulkLoader(injector, systemConfig);
+        this.guessExecutor = injector.getInstance(GuessExecutor.class);
+        this.previewExecutor = injector.getInstance(PreviewExecutor.class);
+    }
+
+    public Injector getInjector()
+    {
+        return injector;
+    }
+
+    public ModelManager getModelManager()
+    {
+        return injector.getInstance(ModelManager.class);
+    }
+
+    public BufferAllocator getBufferAllocator()
+    {
+        return injector.getInstance(BufferAllocator.class);
+    }
+
+    public ConfigLoader newConfigLoader()
+    {
+        return injector.getInstance(ConfigLoader.class);
+    }
+
+    public ConfigDiff guess(ConfigSource config)
+    {
+        ExecSession exec = newExecSession(config);
+        try {
+            return guessExecutor.guess(exec, config);
+        }
+        finally {
+            exec.cleanup();
+        }
+    }
+
+    public PreviewResult preview(ConfigSource config)
+    {
+        ExecSession exec = newExecSession(config);
+        try {
+            return previewExecutor.preview(exec, config);
+        }
+        finally {
+            exec.cleanup();
+        }
+    }
+
+    public ExecutionResult run(ConfigSource config)
+    {
+        ExecSession exec = newExecSession(config);
+        try {
+            return bulkLoader.run(exec, config);
+        }
+        catch (PartialExecutionException partial) {
+            try {
+                bulkLoader.cleanup(config, partial.getResumeState());
+            } catch (Throwable ex) {
+                partial.addSuppressed(ex);
+            }
+            throw partial;
+        }
+        finally {
+            try {
+                exec.cleanup();
+            }
+            catch (Exception ex) {
+                // TODO add this exception to ExecutionResult.getIgnoredExceptions
+                // or partial.addSuppressed
+                ex.printStackTrace(System.err);
+            }
+        }
+    }
+
+    public ResumableResult runResumable(ConfigSource config)
+    {
+        ExecSession exec = newExecSession(config);
+        try {
+            ExecutionResult result;
+            try {
+                result = bulkLoader.run(exec, config);
+            } catch (PartialExecutionException partial) {
+                return new ResumableResult(partial);
+            }
+            return new ResumableResult(result);
+        }
+        finally {
+            try {
+                exec.cleanup();
+            }
+            catch (Exception ex) {
+                // TODO add this exception to ExecutionResult.getIgnoredExceptions
+                // or partial.addSuppressed
+                ex.printStackTrace(System.err);
+            }
+        }
+    }
+
+    private ExecSession newExecSession(ConfigSource config)
+    {
+        ConfigSource execConfig = config.deepCopy().getNestedOrSetEmpty("exec");
+        return ExecSession.builder(injector).fromExecConfig(execConfig).build();
+    }
+
+    public ResumeStateAction resumeState(ConfigSource config, ConfigSource resumeStateConfig)
+    {
+        ResumeState resumeState = resumeStateConfig.loadConfig(ResumeState.class);
+        return new ResumeStateAction(config, resumeState);
+    }
+
+    public static class ResumableResult
+    {
+        private final ExecutionResult successfulResult;
+        private final PartialExecutionException partialExecutionException;
+
+        public ResumableResult(PartialExecutionException partialExecutionException)
+        {
+            this.successfulResult = null;
+            this.partialExecutionException = checkNotNull(partialExecutionException);
+        }
+
+        public ResumableResult(ExecutionResult successfulResult)
+        {
+            this.successfulResult = checkNotNull(successfulResult);
+            this.partialExecutionException = null;
+        }
+
+        public boolean isSuccessful()
+        {
+            return successfulResult != null;
+        }
+
+        public ExecutionResult getSuccessfulResult()
+        {
+            checkState(successfulResult != null);
+            return successfulResult;
+        }
+
+        public Throwable getCause()
+        {
+            checkState(partialExecutionException != null);
+            return partialExecutionException.getCause();
+        }
+
+        public ResumeState getResumeState()
+        {
+            checkState(partialExecutionException != null);
+            return partialExecutionException.getResumeState();
+        }
+    }
+
+    public class ResumeStateAction
+    {
+        private final ConfigSource config;
+        private final ResumeState resumeState;
+
+        public ResumeStateAction(ConfigSource config, ResumeState resumeState)
+        {
+            this.config = config;
+            this.resumeState = resumeState;
+        }
+
+        public ResumableResult resume()
+        {
+            ExecutionResult result;
+            try {
+                result = bulkLoader.resume(config, resumeState);
+            } catch (PartialExecutionException partial) {
+                return new ResumableResult(partial);
+            }
+            return new ResumableResult(result);
+        }
+
+        public void cleanup()
+        {
+            bulkLoader.cleanup(config, resumeState);
+        }
+    }
+
+    public void destroy()
+    {
+        try {
+            injector.destroy();
+        }
+        catch (Exception ex) {
+            throw Throwables.propagate(ex);
+        }
+    }
+}