embulk 0.8.14 → 0.8.15

data/embulk-test/build.gradle

@@ -0,0 +1,6 @@
+dependencies {
+    compile 'junit:junit:4.12'
+    compile 'org.hamcrest:hamcrest-library:1.3'
+
+    compile project(':embulk-core')
+}

data/embulk-test/src/main/java/org/embulk/test/EmbulkTests.java

@@ -0,0 +1,75 @@
+package org.embulk.test;
+
+import com.google.common.base.Throwables;
+import com.google.common.io.CharStreams;
+import com.google.common.io.Resources;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Collections;
+import java.util.List;
+import org.embulk.EmbulkEmbed;
+import org.embulk.config.ConfigSource;
+import static com.google.common.base.Strings.isNullOrEmpty;
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static java.nio.file.StandardCopyOption.REPLACE_EXISTING;
+import static org.hamcrest.Matchers.is;
+import static org.junit.Assume.assumeThat;
+
+public class EmbulkTests
+{
+    private EmbulkTests()
+    { }
+
+    public static ConfigSource config(String envName)
+    {
+        String path = System.getenv(envName);
+        assumeThat(isNullOrEmpty(path), is(false));
+        try {
+            return EmbulkEmbed.newSystemConfigLoader().fromYamlFile(new File(path));
+        } catch (IOException ex) {
+            throw Throwables.propagate(ex);
+        }
+    }
+
+    public static String readResource(String name)
+    {
+        try (InputStream in = Resources.getResource(name).openStream()) {
+            return CharStreams.toString(new InputStreamReader(in, UTF_8));
+        }
+        catch (IOException ex) {
+            throw new RuntimeException(ex);
+        }
+    }
+
+    public static void copyResource(String resource, Path dest)
+        throws IOException
+    {
+        Files.createDirectories(dest.getParent());
+        try (InputStream input = Resources.getResource(resource).openStream()) {
+            Files.copy(input, dest, REPLACE_EXISTING);
+        }
+    }
+
+    public static String readFile(Path path) throws IOException
+    {
+        try (InputStream in = Files.newInputStream(path)) {
+            return CharStreams.toString(new InputStreamReader(in, UTF_8));
+        }
+    }
+
+    public static String readSortedFile(Path path) throws IOException
+    {
+        List<String> lines = Files.readAllLines(path, UTF_8);
+        Collections.sort(lines);
+        StringBuilder sb = new StringBuilder();
+        for (String line : lines) {
+            sb.append(line);
+            sb.append("\n");
+        }
+        return sb.toString();
+    }
+}

data/embulk-test/src/main/java/org/embulk/test/TestingBulkLoader.java

@@ -0,0 +1,124 @@
+package org.embulk.test;
+
+import com.google.common.base.Function;
+import com.google.common.base.Optional;
+import com.google.common.collect.ImmutableList;
+import com.google.inject.Binder;
+import com.google.inject.Inject;
+import com.google.inject.Injector;
+import com.google.inject.Module;
+import com.google.inject.util.Modules;
+import java.util.List;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.TaskReport;
+import org.embulk.exec.BulkLoader;
+import org.embulk.exec.ExecutionResult;
+import org.embulk.exec.ForSystemConfig;
+import org.embulk.exec.ResumeState;
+import org.embulk.spi.Exec;
+import org.embulk.spi.ExecSession;
+import org.embulk.spi.Schema;
+import org.slf4j.Logger;
+
+class TestingBulkLoader
+        extends BulkLoader
+{
+    static Function<List<Module>, List<Module>> override()
+    {
+        return new Function<List<Module>, List<Module>>() {
+            @Override
+            public List<Module> apply(List<Module> modules)
+            {
+                Module override = new Module() {
+                    public void configure(Binder binder)
+                    {
+                        binder.bind(BulkLoader.class).to(TestingBulkLoader.class);
+                    }
+                };
+                return ImmutableList.of(Modules.override(modules).with(ImmutableList.of(override)));
+            }
+        };
+    }
+
+    @Inject
+    public TestingBulkLoader(Injector injector,
+            @ForSystemConfig ConfigSource systemConfig)
+    {
+        super(injector, systemConfig);
+    }
+
+    @Override
+    protected LoaderState newLoaderState(Logger logger, ProcessPluginSet plugins)
+    {
+        return new TestingLoaderState(logger, plugins);
+    }
+
+    protected static class TestingLoaderState
+            extends LoaderState
+    {
+        public TestingLoaderState(Logger logger, ProcessPluginSet plugins)
+        {
+            super(logger, plugins);
+        }
+
+        @Override
+        public ExecutionResult buildExecuteResultWithWarningException(Throwable ex)
+        {
+            ExecutionResult result = super.buildExecuteResultWithWarningException(ex);
+            return new TestingExecutionResult(result, buildResumeState(Exec.session()), Exec.session());
+        }
+    }
+
+    static class TestingExecutionResult
+            extends ExecutionResult
+            implements TestingEmbulk.RunResult
+    {
+        private final Schema inputSchema;
+        private final Schema outputSchema;
+        private final List<TaskReport> inputTaskReports;
+        private final List<TaskReport> outputTaskReports;
+
+        public TestingExecutionResult(ExecutionResult orig,
+                ResumeState resumeState, ExecSession session)
+        {
+            super(orig.getConfigDiff(), orig.isSkipped(), orig.getIgnoredExceptions());
+            this.inputSchema = resumeState.getInputSchema();
+            this.outputSchema = resumeState.getOutputSchema();
+            this.inputTaskReports = buildReports(resumeState.getInputTaskReports(), session);
+            this.outputTaskReports = buildReports(resumeState.getOutputTaskReports(), session);
+        }
+
+        private static List<TaskReport> buildReports(List<Optional<TaskReport>> optionalReports, ExecSession session)
+        {
+            ImmutableList.Builder<TaskReport> reports = ImmutableList.builder();
+            for (Optional<TaskReport> report : optionalReports) {
+                reports.add(report.or(session.newTaskReport()));
+            }
+            return reports.build();
+        }
+
+        @Override
+        public Schema getInputSchema()
+        {
+            return inputSchema;
+        }
+
+        @Override
+        public Schema getOutputSchema()
+        {
+            return outputSchema;
+        }
+
+        @Override
+        public List<TaskReport> getInputTaskReports()
+        {
+            return inputTaskReports;
+        }
+
+        @Override
+        public List<TaskReport> getOutputTaskReports()
+        {
+            return outputTaskReports;
+        }
+    }
+}

data/embulk-test/src/main/java/org/embulk/test/TestingEmbulk.java

@@ -0,0 +1,223 @@
+package org.embulk.test;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.base.Optional;
+import com.google.common.io.ByteStreams;
+import com.google.inject.Binder;
+import com.google.inject.Injector;
+import com.google.inject.Module;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.file.DirectoryStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import org.embulk.EmbulkEmbed;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDiff;
+import org.embulk.config.ConfigLoader;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.TaskReport;
+import org.embulk.exec.ExecutionResult;
+import org.embulk.spi.Schema;
+import org.embulk.spi.TempFileException;
+import org.embulk.spi.TempFileSpace;
+import org.junit.rules.TestRule;
+import org.junit.rules.TestWatcher;
+import org.junit.runner.Description;
+import org.junit.runners.model.Statement;
+import static com.google.common.base.Preconditions.checkArgument;
+import static org.embulk.plugin.InjectedPluginSource.registerPluginTo;
+
+public class TestingEmbulk
+        implements TestRule
+{
+    public static class Builder
+    {
+        private List<Module> modules = new ArrayList<>();
+
+        Builder()
+        { }
+
+        public <T> Builder registerPlugin(final Class<T> iface, final String name, final Class<?> impl)
+        {
+            modules.add(new Module() {
+                public void configure(Binder binder)
+                {
+                    registerPluginTo(binder, iface, name, impl);
+                }
+            });
+            return this;
+        }
+
+        public TestingEmbulk build()
+        {
+            return new TestingEmbulk(this);
+        }
+    }
+
+    public static Builder builder()
+    {
+        return new Builder();
+    }
+
+    private final List<Module> modules;
+
+    private EmbulkEmbed embed;
+    private TempFileSpace tempFiles;
+
+    TestingEmbulk(Builder builder)
+    {
+        this.modules = ImmutableList.copyOf(builder.modules);
+        reset();
+    }
+
+    public void reset()
+    {
+        destroy();
+
+        this.embed = new EmbulkEmbed.Bootstrap()
+            .addModules(modules)
+            .overrideModules(TestingBulkLoader.override())
+            .initializeCloseable();
+
+        try {
+            this.tempFiles = new TempFileSpace(Files.createTempDirectory("embulk-test-temp-").toFile());
+        }
+        catch (IOException ex) {
+            throw new TempFileException(ex);
+        }
+    }
+
+    public void destroy()
+    {
+        if (embed != null) {
+            embed.destroy();
+            embed = null;
+        }
+        if (tempFiles != null) {
+            tempFiles.cleanup();
+            tempFiles = null;
+        }
+    }
+
+    @Override
+    public Statement apply(Statement base, Description description)
+    {
+        return new EmbulkTestingEmbedWatcher().apply(base, description);
+    }
+
+    private class EmbulkTestingEmbedWatcher
+            extends TestWatcher
+    {
+        @Override
+        protected void starting(Description description)
+        {
+            reset();
+        }
+
+        @Override
+        protected void finished(Description description)
+        {
+            destroy();
+        }
+    }
+
+    public Path createTempFile(String suffix)
+    {
+        return tempFiles.createTempFile(suffix).toPath();
+    }
+
+    public Injector injector()
+    {
+        return embed.getInjector();
+    }
+
+    public ConfigLoader configLoader()
+    {
+        return embed.newConfigLoader();
+    }
+
+    public ConfigSource newConfig()
+    {
+        return configLoader().newConfigSource();
+    }
+
+    public ConfigSource loadYamlResource(String name)
+    {
+        return configLoader()
+            .fromYamlString(EmbulkTests.readResource(name));
+    }
+
+    public static interface RunResult
+    {
+        ConfigDiff getConfigDiff();
+
+        List<Throwable> getIgnoredExceptions();
+
+        Schema getInputSchema();
+
+        Schema getOutputSchema();
+
+        List<TaskReport> getInputTaskReports();
+
+        List<TaskReport> getOutputTaskReports();
+    }
+
+    public RunResult runInput(ConfigSource inConfig, Path outputPath)
+        throws IOException
+    {
+        String fileName = outputPath.getFileName().toString();
+        checkArgument(fileName.endsWith(".csv"), "outputPath must end with .csv");
+        Path dir = outputPath.getParent().resolve(fileName.substring(0, fileName.length() - 4));
+
+        Files.createDirectories(dir);
+
+        ConfigSource execConfig = newConfig()
+            .set("min_output_tasks", 1);
+
+        ConfigSource outConfig = newConfig()
+            .set("type", "file")
+            .set("path_prefix", dir.resolve("fragments_").toString())
+            .set("file_ext", "csv")
+            .set("formatter", newConfig()
+                    .set("type", "csv")
+                    .set("header_line", false)
+                    .set("newline", "LF"));
+
+        ConfigSource config = newConfig()
+            .set("exec", execConfig)
+            .set("in", inConfig)
+            .set("out", outConfig);
+
+        // embed.run returns TestingBulkLoader.TestingExecutionResult because
+        RunResult result = (RunResult) embed.run(config);
+
+        try (OutputStream out = Files.newOutputStream(outputPath)) {
+            List<Path> fragments = new ArrayList<Path>();
+            try (DirectoryStream<Path> stream = Files.newDirectoryStream(dir, "fragments_*.csv")) {
+                for (Path fragment : stream) {
+                    fragments.add(fragment);
+                }
+            }
+            Collections.sort(fragments);
+            for (Path fragment : fragments) {
+                try (InputStream in = Files.newInputStream(fragment)) {
+                    ByteStreams.copy(in, out);
+                }
+            }
+        }
+
+        return result;
+    }
+
+    // TODO add runOutput(ConfigSource outConfig, Path inputPath) where inputPath is a path to a CSV file
+    // whose column types can be naturally guessed using csv guess plugin. Callers use EmbulkTests.copyResource
+    // to copy a resource file to a temp file before calling it.
+
+    // TODO add runFilter(ConfigSource filterConfig, Path inputPath, Path outputPath) where inputPath is a path to
+    // a CSV file whose column types can be naturally guessed using csv guess plugin.
+}

data/lib/embulk/version.rb

@@ -1,3 +1,3 @@
 module Embulk
-  VERSION = '0.8.14'
+  VERSION = '0.8.15'
 end

data/settings.gradle

@@ -3,4 +3,5 @@ rootProject.name = 'embulk'
 include 'embulk-core'
 include 'embulk-standards'
 include 'embulk-cli'
+include 'embulk-test'
 include 'embulk-docs'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk
 version: !ruby/object:Gem::Version
-  version: 0.8.14
+  version: 0.8.15
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-10-22 00:00:00.000000000 Z
+date: 2016-11-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: jruby-jars
@@ -108,9 +108,9 @@ files:
 - classpath/commons-beanutils-core-1.8.3.jar
 - classpath/commons-compress-1.10.jar
 - classpath/commons-lang3-3.1.jar
-- classpath/embulk-cli-0.8.14.jar
-- classpath/embulk-core-0.8.14.jar
-- classpath/embulk-standards-0.8.14.jar
+- classpath/embulk-cli-0.8.15.jar
+- classpath/embulk-core-0.8.15.jar
+- classpath/embulk-standards-0.8.15.jar
 - classpath/guava-18.0.jar
 - classpath/guice-4.0.jar
 - classpath/guice-bootstrap-0.1.1.jar
@@ -188,6 +188,7 @@ files:
 - embulk-core/src/main/java/org/embulk/exec/ResumeState.java
 - embulk-core/src/main/java/org/embulk/exec/SamplingParserPlugin.java
 - embulk-core/src/main/java/org/embulk/exec/SetCurrentThreadName.java
+- embulk-core/src/main/java/org/embulk/exec/SkipTransactionException.java
 - embulk-core/src/main/java/org/embulk/exec/SystemConfigModule.java
 - embulk-core/src/main/java/org/embulk/exec/TempFileAllocator.java
 - embulk-core/src/main/java/org/embulk/exec/TransactionStage.java
@@ -428,6 +429,7 @@ files:
 - embulk-docs/src/release/release-0.8.12.rst
 - embulk-docs/src/release/release-0.8.13.rst
 - embulk-docs/src/release/release-0.8.14.rst
+- embulk-docs/src/release/release-0.8.15.rst
 - embulk-docs/src/release/release-0.8.2.rst
 - embulk-docs/src/release/release-0.8.3.rst
 - embulk-docs/src/release/release-0.8.4.rst
@@ -458,6 +460,10 @@ files:
 - embulk-standards/src/test/java/org/embulk/standards/TestCsvTokenizer.java
 - embulk-standards/src/test/java/org/embulk/standards/TestJsonParserPlugin.java
 - embulk-standards/src/test/java/org/embulk/standards/TestRenameFilterPlugin.java
+- embulk-test/build.gradle
+- embulk-test/src/main/java/org/embulk/test/EmbulkTests.java
+- embulk-test/src/main/java/org/embulk/test/TestingBulkLoader.java
+- embulk-test/src/main/java/org/embulk/test/TestingEmbulk.java
 - embulk.gemspec
 - gradle/wrapper/gradle-wrapper.jar
 - gradle/wrapper/gradle-wrapper.properties