embulk 0.2.1 → 0.3.0

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    NGUwOTc0ZDE1MWZlZjJhYjdhNmJmMjQwZjliOWU3MmEyYmM5ZTczNQ==
+    N2ZlNTcxZjM0ZjA4MDE5YmM0NmU2ZTQ3ZmZkM2ZkZDM5OWFkNjRiNA==
   data.tar.gz: !binary |-
-    ZjA1YTE5NDlhZGViMTU1NjVmOTBhZDVlZDY5NGZjODI0NGU5OGViZA==
+    NzU2NjYzOTQzM2ExYzdiNzc0YzUzYWZiNzcwZTU2ZmFkZmUxYjMzOQ==
 SHA512:
   metadata.gz: !binary |-
-    NDY3NzQ1NTkxNTk5MzAzMGQ2ZmIzYjM0YjMyMTczOGM1YjhmYzFkYTg0YTY3
-    ZDBlNzdiYWIwZmVkMWU5YzA3NTEyYzA2ZGI3YjQyMTQ5ZDI2MWI3ZWEwZTM0
-    YjQ3MTllMTZkYzdlYTM2YjNlZDVjNGIwYjEwNDVhNjlmN2IxYTk=
+    NWM5ZjJiNjhkNDQwZDgyZGU4MzhiMmNmYjI5OGZhMjk2OTAyM2FlOWM4YzYy
+    ZTczZTgzMDBkYjY3ZmZmODM0NGU0NmM2MTBjYmRiNDY1ZDliY2QzZmM2OTZi
+    ZmIzZjZmYzc4MTJkOWE2ODM5ODAxYTZhMzgzNzE0NzM5YzgyODU=
   data.tar.gz: !binary |-
-    MGZjMzM1NmVhNzdhZDhjODg3ZWZiNGRmOWQwMTU5MzUwZmEwYTBkMDY1MTgz
-    NDBhOTAwM2Y3NDNjM2VlZTE1YjRkZjA4MWNiZjZjN2QzOTBjYTliMzJlYTgw
-    OGY2ZGZmMDJmMTI4ZWU1YjNmMTMxNTc5NDdjN2NiODkxYzQ4MmI=
+    NTg3MjdiZWNjNGYzN2MzZTM0YzY0OTRmNGJlYmI5ODY3ZmY5ZWFiY2RkN2Yy
+    NjI0ZmIzOWJjNDU0NzM4ZDhlYTJjZTBkODc3MmRjYjI5Mzc5OWUyZDQwYzA5
+    MTRiMGE1NmU4YmI1MWU1Yjk1ZmJlZTU3Y2Y0NDY5YTFjN2EzOTQ=

data/ChangeLog

@@ -1,4 +1,16 @@
 
+2015-02-03 version 0.3.0:
+
+* Added resume functionality. InputPlugin and OutputPlugin needs to implement
+  resume and cleanup methods.
+* cli: embulk-run supports -r, --resume-state PATH option.
+* Added FilterInputPlugin Java API.
+* Added FilterInputPlugin JRuby API.
+* Configuration file accepts filters: array entry.
+* Added gradle-versions-plugin to build.gradle (@seratch++)
+* Fixed broken dependencies at build.gradle (@thagikura++)
+
+
 2015-01-29 version 0.2.1:
 
 * Fixed LineEncoder#finish to flush all remaining buffer (reported by @aibou)

data/README.md

@@ -1,27 +1,30 @@
-# Embulk
-
-A plugin-based parallel bulk data loader that makes painful data integration works relaxed.
-
-## What's Embulk?
+# What's Embulk?
 
 Embulk is a plugin-based parallel bulk data loader that helps **data transfer** between various **storages**, **databases**, **NoSQL** and **cloud services**.
 
-You can install input and output plugins to integrate many other file formats and storages.
-
-You also can release plugins to share your efforts of data cleaning, error handling, transaction control, and retrying. 
-Packaging effrots into plugins **brings OSS-style development to the data scripts** which **was tend to be one-time adhoc scripts**.
+You can release plugins to share your efforts of data cleaning, error handling, transaction control, and retrying. Packaging effrots into plugins **brings OSS-style development to the data scripts** which **was tend to be one-time adhoc scripts**.
 
-[Embuk, an open-source plugin-based parallel bulk data loader](http://www.slideshare.net/frsyuki/embuk-making-data-integration-works-relaxed) at Slideshare
+[Embulk, an open-source plugin-based parallel bulk data loader](http://www.slideshare.net/frsyuki/embuk-making-data-integration-works-relaxed) at Slideshare
 
 [![Embulk](https://gist.githubusercontent.com/frsyuki/f322a77ee2766a508ba9/raw/e8539b6b4fda1b3357e8c79d3966aa8148dbdbd3/embulk-overview.png)](http://www.slideshare.net/frsyuki/embuk-making-data-integration-works-relaxed/12)
 
+# Document
+
+* [Quick Start](#quick-start)
+  * [Using plugins](#using-plugins)
+  * [Using plugin bundle](#using-plugin-bundle)
+  * [Releasing plugins to RubyGems](#releasing-plugins-to-rubygems)
+  * [Resuming a failed transaction](#resuming-a-failed-transaction)
+* [Embulk Development](#embulk-development)
+  * [Build](#build)
+  * [Release](#release)
 
 ## Quick Start
 
 The single-file package is the simplest way to try Embulk. You can download the latest embulk-VERSION.jar from [the releases page](https://bintray.com/embulk/maven/embulk/view#files) and run it with java:
 
 ```
-wget https://bintray.com/artifact/download/embulk/maven/embulk-0.2.1.jar -O embulk.jar
+wget https://bintray.com/artifact/download/embulk/maven/embulk-0.3.0.jar -O embulk.jar
 java -jar embulk.jar --help
 ```
 
@@ -37,10 +40,10 @@ java -jar embulk.jar run     config.yml
 ### Using plugins
 
 You can use plugins to load data from/to various systems and file formats.
-An example is [embulk-output-postgres-json](https://github.com/frsyuki/embulk-plugin-postgres-json) plugin. It outputs data into PostgreSQL server using "json" column type.
+An example is [embulk-plugin-postgres-json](https://github.com/frsyuki/embulk-plugin-postgres-json) plugin. It outputs data into PostgreSQL server using "json" column type.
 
 ```
-java -jar embulk.jar gem install embulk-output-postgres-json
+java -jar embulk.jar gem install embulk-plugin-postgres-json
 java -jar embulk.jar gem list
 ```
 
@@ -67,6 +70,28 @@ TODO: documents
 embulk-plugin-xyz
 ```
 
+### Resuming a failed transaction
+
+Embulk supports resuming failed transactions.
+To enable resuming, you need to start transaction with `-r PATH` option:
+
+```
+java -jar embulk.jar run config.yml -r resume-state.yml
+```
+
+If the transaction fails, embulk stores state some states to the yaml file. You can retry the transaction using exactly same command:
+
+```
+java -jar embulk.jar run config.yml -r resume-state.yml
+```
+
+If you giveup to resume the transaction, you can use `embulk cleanup` subcommand to delete intermediate data:
+
+```
+java -jar embulk.jar cleanup config.yml -r resume-state.yml
+```
+
+
 ## Embulk Development
 
 ### Build

data/build.gradle

@@ -9,6 +9,7 @@ buildscript {
     }
     dependencies {
         classpath 'com.jfrog.bintray.gradle:gradle-bintray-plugin:1.0'
+        classpath 'com.github.ben-manes:gradle-versions-plugin:0.7'
     }
 }
 
@@ -22,8 +23,12 @@ allprojects {
     apply plugin: 'maven-publish'
     apply plugin: 'com.jfrog.bintray'
 
+    // determine which dependencies have updates
+    // $ gradle dependencyUpdates
+    apply plugin: 'com.github.ben-manes.versions'
+
     group = 'org.embulk'
-    version = '0.2.1'
+    version = '0.3.0'
 
     // to upload artifacts to Bintray by gradle-bintray-plugin
     // $ gradle bintrayUpload

data/embulk-cli/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>org.embulk</groupId>
         <artifactId>embulk-parent</artifactId>
-        <version>0.2.1-SNAPSHOT</version>
+        <version>0.3.0-SNAPSHOT</version>
     </parent>
 
     <artifactId>embulk-cli</artifactId>

data/embulk-core/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>org.embulk</groupId>
         <artifactId>embulk-parent</artifactId>
-        <version>0.2.1-SNAPSHOT</version>
+        <version>0.3.0-SNAPSHOT</version>
     </parent>
 
     <artifactId>embulk-core</artifactId>

data/embulk-core/src/main/java/org/embulk/command/Runner.java

@@ -21,10 +21,12 @@ import org.embulk.config.NextConfig;
 import org.embulk.config.ModelManager;
 import org.embulk.config.ConfigException;
 import org.embulk.exec.LocalExecutor;
-import org.embulk.exec.ExecuteResult;
+import org.embulk.exec.ExecutionResult;
 import org.embulk.exec.GuessExecutor;
 import org.embulk.exec.PreviewExecutor;
 import org.embulk.exec.PreviewResult;
+import org.embulk.exec.ResumeState;
+import org.embulk.exec.PartialExecutionException;
 import org.embulk.spi.time.Timestamp;
 import org.embulk.spi.ExecSession;
 import org.embulk.spi.util.Pages;
@@ -36,6 +38,9 @@ public class Runner
     {
         private String nextConfigOutputPath;
         public String getNextConfigOutputPath() { return nextConfigOutputPath; }
+
+        private String resumeStatePath;
+        public String getResumeStatePath() { return resumeStatePath; }
     }
 
     private final Options options;
@@ -58,6 +63,9 @@ public class Runner
         case "run":
             run(args[0]);
             break;
+        case "cleanup":
+            cleanup(args[0]);
+            break;
         case "guess":
             guess(args[0]);
             break;
@@ -72,21 +80,87 @@ public class Runner
     public void run(String configPath)
     {
         ConfigSource config = loadYamlConfig(configPath);
-        checkNextConfigOutputPath(options.getNextConfigOutputPath());
+        checkFileWritable(options.getNextConfigOutputPath());
+        checkFileWritable(options.getResumeStatePath());
+
+        // load resume state file
+        ResumeState resume = null;
+        String resumePath = options.getResumeStatePath();
+        if (resumePath != null) {
+            ConfigSource resumeConfig = null;
+            try {
+                resumeConfig = loadYamlConfig(resumePath);
+                if (resumeConfig.isEmpty()) {
+                    resumeConfig = null;
+                }
+            } catch (RuntimeException ex) {
+                // leave resumeConfig == null
+            }
+            if (resumeConfig != null) {
+                resume = resumeConfig.loadConfig(ResumeState.class);
+            }
+        }
 
         ExecSession exec = newExecSession(config);
         LocalExecutor local = injector.getInstance(LocalExecutor.class);
-        ExecuteResult result = local.run(exec, config);
-        NextConfig nextConfig = result.getNextConfig();
+        ExecutionResult result;
+        try {
+            if (resume != null) {
+                result = local.resume(config, resume);
+            } else {
+                result = local.run(exec, config);
+            }
+        } catch (PartialExecutionException partial) {
+            if (options.getResumeStatePath() == null) {
+                // resume state path is not set. cleanup the transaction
+                exec.getLogger(Runner.class).info("Transaction partially failed. Cleaning up the intermediate data. Use -r option to make it resumable.");
+                try {
+                    local.cleanup(config, partial.getResumeState());
+                } catch (Throwable ex) {
+                    partial.addSuppressed(ex);
+                }
+                throw partial;
+            }
+            // save the resume state
+            exec.getLogger(Runner.class).info("Writing resume state to '{}'", options.getResumeStatePath());
+            writeYaml(options.getResumeStatePath(), partial.getResumeState());
+            exec.getLogger(Runner.class).info("Resume state is written. Run the transaction again with -r option to resume or use \"cleanup\" subcommand to delete intermediate data.");
+            throw partial;
+        }
 
+        // delete resume file
+        if (options.getResumeStatePath() != null) {
+            new File(options.getResumeStatePath()).delete();
+        }
+
+        // write next config
+        NextConfig nextConfig = result.getNextConfig();
         exec.getLogger(Runner.class).info("next config: {}", nextConfig.toString());
         writeNextConfig(options.getNextConfigOutputPath(), config, nextConfig);
     }
 
+    public void cleanup(String configPath)
+    {
+        String resumePath = options.getResumeStatePath();
+        if (resumePath == null) {
+            throw new IllegalArgumentException("Resume path is required for cleanup");
+        }
+        ConfigSource config = loadYamlConfig(configPath);
+        ConfigSource resumeConfig = loadYamlConfig(resumePath);
+        ResumeState resume = resumeConfig.loadConfig(ResumeState.class);
+
+        ExecSession exec = newExecSession(config);
+        LocalExecutor local = injector.getInstance(LocalExecutor.class);
+        local.cleanup(config, resume);
+
+        // delete resume file
+        new File(options.getResumeStatePath()).delete();
+    }
+
     public void guess(String partialConfigPath)
     {
         ConfigSource config = loadYamlConfig(partialConfigPath);
-        checkNextConfigOutputPath(options.getNextConfigOutputPath());
+        checkFileWritable(options.getNextConfigOutputPath());
 
         ExecSession exec = newExecSession(config);
         GuessExecutor guess = injector.getInstance(GuessExecutor.class);
@@ -96,7 +170,7 @@ public class Runner
         System.err.println(yml);
     }
 
-    private void checkNextConfigOutputPath(String path)
+    private void checkFileWritable(String path)
     {
         if (path != null) {
             try (FileOutputStream in = new FileOutputStream(path, true)) {
@@ -109,7 +183,12 @@ public class Runner
 
     private String writeNextConfig(String path, ConfigSource originalConfig, NextConfig nextConfigDiff)
     {
-        String yml = dumpConfigInYaml(originalConfig.merge(nextConfigDiff));
+        return writeYaml(path, originalConfig.merge(nextConfigDiff));
+    }
+
+    private String writeYaml(String path, Object obj)
+    {
+        String yml = dumpYaml(obj);
         if (path != null) {
             if (path.equals("-")) {
                 System.out.print(yml);
@@ -181,7 +260,7 @@ public class Runner
         }
     }
 
-    private String dumpConfigInYaml(DataSource config)
+    private String dumpYaml(Object config)
     {
         ModelManager model = injector.getInstance(ModelManager.class);
         Map<String, Object> map = model.readObject(MapType.class, model.writeObject(config));

data/embulk-core/src/main/java/org/embulk/config/ConfigSource.java

@@ -3,7 +3,7 @@ package org.embulk.config;
 public interface ConfigSource
         extends DataSource
 {
-    public <T extends Task> T loadConfig(Class<T> taskType);
+    public <T> T loadConfig(Class<T> taskType);
 
     @Override
     public ConfigSource getNested(String attrName);

data/embulk-core/src/main/java/org/embulk/config/DataSourceImpl.java

@@ -170,17 +170,30 @@ public class DataSourceImpl
 
     private static void mergeJsonArray(ArrayNode src, ArrayNode other)
     {
-        src.addAll(other);
+        for (int i=0; i < other.size(); i++) {
+            JsonNode s = src.get(i);
+            JsonNode v = other.get(i);
+            if (s == null) {
+                src.add(v);
+            } else if (v.isObject() && s.isObject()) {
+                mergeJsonObject((ObjectNode) s, (ObjectNode) v);
+            } else if (v.isArray() && s.isArray()) {
+                mergeJsonArray((ArrayNode) s, (ArrayNode) v);
+            } else {
+                src.remove(i);
+                src.insert(i, v);
+            }
+        }
     }
 
     @Override
-    public <T extends Task> T loadTask(Class<T> taskType)
+    public <T> T loadTask(Class<T> taskType)
     {
         return model.readObject(taskType, data.traverse());
     }
 
     @Override
-    public <T extends Task> T loadConfig(Class<T> taskType)
+    public <T> T loadConfig(Class<T> taskType)
     {
         return model.readObjectWithConfigSerDe(taskType, data.traverse());
     }

data/embulk-core/src/main/java/org/embulk/config/TaskSource.java

@@ -3,7 +3,7 @@ package org.embulk.config;
 public interface TaskSource
         extends DataSource
 {
-    public <T extends Task> T loadTask(Class<T> taskType);
+    public <T> T loadTask(Class<T> taskType);
 
     @Override
     public TaskSource getNested(String attrName);

data/embulk-core/src/main/java/org/embulk/exec/ExecutionInterruptedException.java

@@ -0,0 +1,10 @@
+package org.embulk.exec;
+
+public class ExecutionInterruptedException
+        extends RuntimeException
+{
+    public ExecutionInterruptedException(Exception cause)
+    {
+        super(cause);
+    }
+}

data/embulk-core/src/main/java/org/embulk/exec/ExecutionResult.java

@@ -0,0 +1,26 @@
+package org.embulk.exec;
+
+import java.util.List;
+import org.embulk.config.NextConfig;
+
+public class ExecutionResult
+{
+    private final NextConfig nextConfig;
+    private final List<Throwable> ignoredExceptions;
+
+    public ExecutionResult(NextConfig nextConfig, List<Throwable> ignoredExceptions)
+    {
+        this.nextConfig = nextConfig;
+        this.ignoredExceptions = ignoredExceptions;
+    }
+
+    public NextConfig getNextConfig()
+    {
+        return nextConfig;
+    }
+
+    public List<Throwable> getIgnoredExceptions()
+    {
+        return ignoredExceptions;
+    }
+}

data/embulk-core/src/main/java/org/embulk/exec/GuessExecutor.java

@@ -158,7 +158,7 @@ public class GuessExecutor
     private static class BufferFileInputPlugin
             implements FileInputPlugin
     {
-        private final Buffer buffer;
+        private Buffer buffer;
 
         public BufferFileInputPlugin(Buffer buffer)
         {
@@ -171,6 +171,23 @@ public class GuessExecutor
             return Exec.newNextConfig();
         }
 
+        public NextConfig resume(TaskSource taskSource,
+                int processorCount,
+                FileInputPlugin.Control control)
+        {
+            throw new UnsupportedOperationException();
+        }
+
+        public void cleanup(TaskSource taskSource,
+                int processorCount,
+                List<CommitReport> successCommitReports)
+        {
+            if (buffer != null) {
+                buffer.release();
+                buffer = null;
+            }
+        }
+
         public TransactionalFileInput open(TaskSource taskSource, int processorIndex)
         {
             return new BufferTransactionalFileInput(buffer);
@@ -257,6 +274,7 @@ public class GuessExecutor
             NextConfig mergedGuessed = Exec.newNextConfig();
             for (int i=0; i < guesses.size(); i++) {
                 NextConfig guessed = guesses.get(i).guess(originalConfig, sample);
+                guessed = addAssumedDecoderConfigs(originalConfig, guessed);
                 mergedGuessed.merge(guessed);
                 mergedConfig.merge(mergedGuessed);
                 if (!mergedConfig.equals(originalConfig)) {
@@ -287,6 +305,24 @@ public class GuessExecutor
             }
             throw new NoSampleException("No input buffer to guess");
         }
+
+        private static class ConfigSourceList extends ArrayList<ConfigSource> { };
+
+        private static NextConfig addAssumedDecoderConfigs(ConfigSource originalConfig, NextConfig guessed)
+        {
+            List<ConfigSource> guessedDecoders = guessed.get(ConfigSourceList.class, "decoders", null);
+            if (guessedDecoders == null) {
+                return guessed;
+            } else {
+                List<ConfigSource> assumedDecoders = originalConfig.get(ConfigSourceList.class, "decoders", new ConfigSourceList());
+                ImmutableList.Builder<ConfigSource> added = ImmutableList.builder();
+                for (ConfigSource assuemed : assumedDecoders) {
+                    added.add(Exec.newConfigSource());
+                }
+                added.addAll(guessedDecoders);
+                return guessed.set("decoders", added.build());
+            }
+        }
     }
 
     public static class GuessedNoticeError