embulk 0.3.2 → 0.4.0

data/embulk-core/build.gradle

@@ -1,6 +1,6 @@
 sourceSets {
     main.resources {
         srcDirs 'src/main/resources'
-        srcDirs '../lib'
+        srcDirs "${parent.projectDir}/lib"
     }
 }

data/embulk-core/src/main/java/org/embulk/command/Runner.java

@@ -17,7 +17,7 @@ import com.google.inject.Injector;
 import org.embulk.config.ConfigSource;
 import org.embulk.config.DataSource;
 import org.embulk.config.ConfigLoader;
-import org.embulk.config.NextConfig;
+import org.embulk.config.ConfigDiff;
 import org.embulk.config.ModelManager;
 import org.embulk.config.ConfigException;
 import org.embulk.exec.LocalExecutor;
@@ -130,13 +130,14 @@ public class Runner
 
         // delete resume file
         if (options.getResumeStatePath() != null) {
-            new File(options.getResumeStatePath()).delete();
+            boolean dontCare = new File(options.getResumeStatePath()).delete();
         }
 
         // write next config
-        NextConfig nextConfig = result.getNextConfig();
-        exec.getLogger(Runner.class).info("next config: {}", nextConfig.toString());
-        writeNextConfig(options.getNextConfigOutputPath(), config, nextConfig);
+        ConfigDiff configDiff = result.getConfigDiff();
+        exec.getLogger(Runner.class).info("Committed.");
+        exec.getLogger(Runner.class).info("Next config diff: {}", configDiff.toString());
+        writeNextConfig(options.getNextConfigOutputPath(), config, configDiff);
     }
 
     public void cleanup(String configPath)
@@ -154,7 +155,7 @@ public class Runner
         local.cleanup(config, resume);
 
         // delete resume file
-        new File(options.getResumeStatePath()).delete();
+        boolean dontCare = new File(options.getResumeStatePath()).delete();
     }
 
     public void guess(String partialConfigPath)
@@ -164,9 +165,9 @@ public class Runner
 
         ExecSession exec = newExecSession(config);
         GuessExecutor guess = injector.getInstance(GuessExecutor.class);
-        NextConfig nextConfig = guess.guess(exec, config);
+        ConfigDiff configDiff = guess.guess(exec, config);
 
-        String yml = writeNextConfig(options.getNextConfigOutputPath(), config, nextConfig);
+        String yml = writeNextConfig(options.getNextConfigOutputPath(), config, configDiff);
         System.err.println(yml);
     }
 
@@ -181,9 +182,9 @@ public class Runner
         }
     }
 
-    private String writeNextConfig(String path, ConfigSource originalConfig, NextConfig nextConfigDiff)
+    private String writeNextConfig(String path, ConfigSource originalConfig, ConfigDiff configDiff)
     {
-        return writeYaml(path, originalConfig.merge(nextConfigDiff));
+        return writeYaml(path, originalConfig.merge(configDiff));
     }
 
     private String writeYaml(String path, Object obj)

data/embulk-core/src/main/java/org/embulk/config/ConfigDiff.java

@@ -0,0 +1,26 @@
+package org.embulk.config;
+
+public interface ConfigDiff
+        extends DataSource
+{
+    @Override
+    public ConfigDiff getNested(String attrName);
+
+    @Override
+    public ConfigDiff getNestedOrSetEmpty(String attrName);
+
+    @Override
+    public ConfigDiff set(String attrName, Object v);
+
+    @Override
+    public ConfigDiff setNested(String attrName, DataSource v);
+
+    @Override
+    public ConfigDiff setAll(DataSource other);
+
+    @Override
+    public ConfigDiff deepCopy();
+
+    @Override
+    public ConfigDiff merge(DataSource other);
+}

data/embulk-core/src/main/java/org/embulk/config/ConfigInject.java

@@ -0,0 +1,14 @@
+package org.embulk.config;
+
+import java.lang.annotation.Documented;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+@Documented
+@Retention(RetentionPolicy.RUNTIME)
+@Target(ElementType.METHOD)
+public @interface ConfigInject
+{
+}

data/embulk-core/src/main/java/org/embulk/config/DataSourceImpl.java

@@ -13,7 +13,7 @@ import com.fasterxml.jackson.databind.node.ArrayNode;
 import com.fasterxml.jackson.databind.node.JsonNodeFactory;
 
 public class DataSourceImpl
-        implements ConfigSource, TaskSource, CommitReport, NextConfig
+        implements ConfigSource, TaskSource, CommitReport, ConfigDiff
 {
     protected final ObjectNode data;
     protected final ModelManager model;
@@ -51,7 +51,7 @@ public class DataSourceImpl
     @Override
     public Iterable<Map.Entry<String, JsonNode>> getAttributes()
     {
-        return new Iterable() {
+        return new Iterable<Map.Entry<String,JsonNode>>() {
             public Iterator<Map.Entry<String, JsonNode>> iterator()
             {
                 return data.fields();

data/embulk-core/src/main/java/org/embulk/config/DataSourceSerDe.java

@@ -38,9 +38,9 @@ public class DataSourceSerDe
             addSerializer(CommitReport.class, new DataSourceSerializer<CommitReport>());
             addDeserializer(CommitReport.class, new DataSourceDeserializer<CommitReport>(model));
 
-            // NextConfig
-            addSerializer(NextConfig.class, new DataSourceSerializer<NextConfig>());
-            addDeserializer(NextConfig.class, new DataSourceDeserializer<NextConfig>(model));
+            // ConfigDiff
+            addSerializer(ConfigDiff.class, new DataSourceSerializer<ConfigDiff>());
+            addDeserializer(ConfigDiff.class, new DataSourceDeserializer<ConfigDiff>(model));
         }
     }
 
@@ -57,6 +57,7 @@ public class DataSourceSerDe
         }
 
         @Override
+        @SuppressWarnings("unchecked")
         public T deserialize(JsonParser jp, DeserializationContext ctxt) throws IOException
         {
             JsonNode json = treeObjectMapper.readTree(jp);

data/embulk-core/src/main/java/org/embulk/config/TaskSerDe.java

@@ -28,7 +28,6 @@ import com.fasterxml.jackson.databind.SerializerProvider;
 import com.fasterxml.jackson.databind.DeserializationContext;
 import com.fasterxml.jackson.databind.DeserializationConfig;
 import com.fasterxml.jackson.databind.JsonMappingException;
-import com.fasterxml.jackson.annotation.JacksonInject;
 
 class TaskSerDe
 {
@@ -93,7 +92,7 @@ class TaskSerDe
                 Method getterMethod = getter.getValue();
                 String fieldName = getter.getKey();
 
-                if (getterMethod.getAnnotation(JacksonInject.class) != null) {
+                if (getterMethod.getAnnotation(ConfigInject.class) != null) {
                     // InjectEntry
                     continue;
                 }
@@ -117,7 +116,7 @@ class TaskSerDe
             for (Map.Entry<String, Method> getter : TaskInvocationHandler.fieldGetters(iface).entrySet()) {
                 Method getterMethod = getter.getValue();
                 String fieldName = getter.getKey();
-                JacksonInject inject = getterMethod.getAnnotation(JacksonInject.class);
+                ConfigInject inject = getterMethod.getAnnotation(ConfigInject.class);
                 if (inject != null) {
                     // InjectEntry
                     builder.add(new InjectEntry(fieldName, getterMethod.getReturnType()));
@@ -137,6 +136,7 @@ class TaskSerDe
         }
 
         @Override
+        @SuppressWarnings("unchecked")
         public T deserialize(JsonParser jp, DeserializationContext ctxt) throws IOException
         {
             Map<String, Object> objects = new ConcurrentHashMap<String, Object>();
@@ -317,6 +317,7 @@ class TaskSerDe
             });
         }
 
+        @SuppressWarnings("unchecked")
         protected JsonDeserializer<?> newTaskDeserializer(Class<?> raw)
         {
             return new TaskDeserializer(nestedObjectMapper, model, raw);
@@ -335,6 +336,7 @@ class TaskSerDe
         public String getModuleName() { return "embulk.config.ConfigTaskSerDe"; }
 
         @Override
+        @SuppressWarnings("unchecked")
         protected JsonDeserializer<?> newTaskDeserializer(Class<?> raw)
         {
             return new ConfigTaskDeserializer(nestedObjectMapper, model, raw);

data/embulk-core/src/main/java/org/embulk/config/TaskValidationException.java

@@ -15,6 +15,7 @@ public class TaskValidationException
         this.violations = violations;
     }
 
+    @SuppressWarnings("unchecked")
     public Set<ConstraintViolation<?>> getViolations()
     {
         return violations;

data/embulk-core/src/main/java/org/embulk/exec/ExecutionResult.java

@@ -1,22 +1,22 @@
 package org.embulk.exec;
 
 import java.util.List;
-import org.embulk.config.NextConfig;
+import org.embulk.config.ConfigDiff;
 
 public class ExecutionResult
 {
-    private final NextConfig nextConfig;
+    private final ConfigDiff configDiff;
     private final List<Throwable> ignoredExceptions;
 
-    public ExecutionResult(NextConfig nextConfig, List<Throwable> ignoredExceptions)
+    public ExecutionResult(ConfigDiff configDiff, List<Throwable> ignoredExceptions)
     {
-        this.nextConfig = nextConfig;
+        this.configDiff = configDiff;
         this.ignoredExceptions = ignoredExceptions;
     }
 
-    public NextConfig getNextConfig()
+    public ConfigDiff getConfigDiff()
     {
-        return nextConfig;
+        return configDiff;
     }
 
     public List<Throwable> getIgnoredExceptions()

data/embulk-core/src/main/java/org/embulk/exec/GuessExecutor.java

@@ -8,7 +8,7 @@ import com.google.inject.Injector;
 import com.google.common.base.Throwables;
 import org.embulk.config.Config;
 import org.embulk.config.ConfigDefault;
-import org.embulk.config.NextConfig;
+import org.embulk.config.ConfigDiff;
 import org.embulk.config.DataSource;
 import org.embulk.config.Task;
 import org.embulk.config.TaskSource;
@@ -64,11 +64,11 @@ public class GuessExecutor
                 new PluginType("csv"));
     }
 
-    public NextConfig guess(ExecSession exec, final ConfigSource config)
+    public ConfigDiff guess(ExecSession exec, final ConfigSource config)
     {
         try {
-            return Exec.doWith(exec, new ExecAction<NextConfig>() {
-                public NextConfig run()
+            return Exec.doWith(exec, new ExecAction<ConfigDiff>() {
+                public ConfigDiff run()
                 {
                     return doGuess(config);
                 }
@@ -78,7 +78,7 @@ public class GuessExecutor
         }
     }
 
-    private NextConfig doGuess(ConfigSource config)
+    private ConfigDiff doGuess(ConfigSource config)
     {
         Buffer sample = SamplingParserPlugin.runFileInputSampling(config);
         if (sample.limit() == 0) {
@@ -93,11 +93,11 @@ public class GuessExecutor
         return runGuessInput(sample, config, guessPlugins);
     }
 
-    private NextConfig runGuessInput(Buffer sample,
+    private ConfigDiff runGuessInput(Buffer sample,
             ConfigSource config, List<PluginType> guessPlugins)
     {
         // repeat guessing upto 10 times
-        NextConfig lastGuessed = Exec.newNextConfig();
+        ConfigDiff lastGuessed = Exec.newConfigDiff();
         for (int i=0; i < 10; i++) {
             // include last-guessed config to run guess input
             ConfigSource originalConfig = config.getNested("in").deepCopy().merge(lastGuessed);
@@ -109,7 +109,7 @@ public class GuessExecutor
 
             // run FileInputPlugin
             final FileInputRunner input = new FileInputRunner(new BufferFileInputPlugin(sample));
-            NextConfig guessed;
+            ConfigDiff guessed;
             try {
                 input.transaction(guessInputConfig, new InputPlugin.Control() {
                     public List<CommitReport> run(TaskSource inputTaskSource, Schema schema, int processorCount)
@@ -148,9 +148,9 @@ public class GuessExecutor
         return wrapInIn(lastGuessed);
     }
 
-    private static NextConfig wrapInIn(NextConfig lastGuessed)
+    private static ConfigDiff wrapInIn(ConfigDiff lastGuessed)
     {
-        NextConfig wrapped = Exec.newNextConfig();
+        ConfigDiff wrapped = Exec.newConfigDiff();
         wrapped.getNestedOrSetEmpty("in").merge(lastGuessed);
         return wrapped;
     }
@@ -165,13 +165,13 @@ public class GuessExecutor
             this.buffer = buffer;
         }
 
-        public NextConfig transaction(ConfigSource config, FileInputPlugin.Control control)
+        public ConfigDiff transaction(ConfigSource config, FileInputPlugin.Control control)
         {
             control.run(Exec.newTaskSource(), 1);
-            return Exec.newNextConfig();
+            return Exec.newConfigDiff();
         }
 
-        public NextConfig resume(TaskSource taskSource,
+        public ConfigDiff resume(TaskSource taskSource,
                 int processorCount,
                 FileInputPlugin.Control control)
         {
@@ -271,9 +271,9 @@ public class GuessExecutor
 
             // run guess plugins
             ConfigSource mergedConfig = originalConfig.deepCopy();
-            NextConfig mergedGuessed = Exec.newNextConfig();
+            ConfigDiff mergedGuessed = Exec.newConfigDiff();
             for (int i=0; i < guesses.size(); i++) {
-                NextConfig guessed = guesses.get(i).guess(originalConfig, sample);
+                ConfigDiff guessed = guesses.get(i).guess(originalConfig, sample);
                 guessed = addAssumedDecoderConfigs(originalConfig, guessed);
                 mergedGuessed.merge(guessed);
                 mergedConfig.merge(mergedGuessed);
@@ -308,7 +308,7 @@ public class GuessExecutor
 
         private static class ConfigSourceList extends ArrayList<ConfigSource> { };
 
-        private static NextConfig addAssumedDecoderConfigs(ConfigSource originalConfig, NextConfig guessed)
+        private static ConfigDiff addAssumedDecoderConfigs(ConfigSource originalConfig, ConfigDiff guessed)
         {
             List<ConfigSource> guessedDecoders = guessed.get(ConfigSourceList.class, "decoders", null);
             if (guessedDecoders == null) {
@@ -328,14 +328,14 @@ public class GuessExecutor
     public static class GuessedNoticeError
             extends Error
     {
-        private final NextConfig guessedConfig;
+        private final ConfigDiff guessedConfig;
 
-        public GuessedNoticeError(NextConfig guessedConfig)
+        public GuessedNoticeError(ConfigDiff guessedConfig)
         {
             this.guessedConfig = guessedConfig;
         }
 
-        public NextConfig getGuessedConfig()
+        public ConfigDiff getGuessedConfig()
         {
             return guessedConfig;
         }

data/embulk-core/src/main/java/org/embulk/exec/LocalExecutor.java

@@ -22,7 +22,7 @@ import org.embulk.config.Config;
 import org.embulk.config.ConfigDefault;
 import org.embulk.config.ConfigSource;
 import org.embulk.config.TaskSource;
-import org.embulk.config.NextConfig;
+import org.embulk.config.ConfigDiff;
 import org.embulk.config.CommitReport;
 import org.embulk.plugin.PluginType;
 import org.embulk.spi.Schema;
@@ -93,8 +93,8 @@ public class LocalExecutor
         private volatile Throwable[] exceptions;
         private volatile CommitReport[] inputCommitReports;
         private volatile CommitReport[] outputCommitReports;
-        private volatile NextConfig inputNextConfig;
-        private volatile NextConfig outputNextConfig;
+        private volatile ConfigDiff inputConfigDiff;
+        private volatile ConfigDiff outputConfigDiff;
         private int processorCount;
 
         public ProcessState(Logger logger)
@@ -231,20 +231,20 @@ public class LocalExecutor
             return false;
         }
 
-        public void setOutputNextConfig(NextConfig outputNextConfig)
+        public void setOutputConfigDiff(ConfigDiff outputConfigDiff)
         {
-            if (outputNextConfig == null) {
-                outputNextConfig = Exec.newNextConfig();
+            if (outputConfigDiff == null) {
+                outputConfigDiff = Exec.newConfigDiff();
             }
-            this.outputNextConfig = outputNextConfig;
+            this.outputConfigDiff = outputConfigDiff;
         }
 
-        public void setInputNextConfig(NextConfig inputNextConfig)
+        public void setInputConfigDiff(ConfigDiff inputConfigDiff)
         {
-            if (inputNextConfig == null) {
-                inputNextConfig = Exec.newNextConfig();
+            if (inputConfigDiff == null) {
+                inputConfigDiff = Exec.newConfigDiff();
             }
-            this.inputNextConfig = inputNextConfig;
+            this.inputConfigDiff = inputConfigDiff;
         }
 
         public List<CommitReport> getInputCommitReports()
@@ -297,12 +297,12 @@ public class LocalExecutor
 
         public ExecutionResult buildExecuteResultWithWarningException(Throwable ex)
         {
-            NextConfig nextConfig = Exec.newNextConfig();
-            if (inputNextConfig != null) {
-                nextConfig.getNestedOrSetEmpty("in").merge(inputNextConfig);
+            ConfigDiff configDiff = Exec.newConfigDiff();
+            if (inputConfigDiff != null) {
+                configDiff.getNestedOrSetEmpty("in").merge(inputConfigDiff);
             }
-            if (outputNextConfig != null) {
-                nextConfig.getNestedOrSetEmpty("out").merge(outputNextConfig);
+            if (outputConfigDiff != null) {
+                configDiff.getNestedOrSetEmpty("out").merge(outputConfigDiff);
             }
 
             ImmutableList.Builder<Throwable> ignoredExceptions = ImmutableList.builder();
@@ -315,17 +315,22 @@ public class LocalExecutor
                 ignoredExceptions.add(ex);
             }
 
-            return new ExecutionResult(nextConfig, ignoredExceptions.build());
+            return new ExecutionResult(configDiff, ignoredExceptions.build());
+        }
+
+        public ResumeState buildResumeState(ExecutorTask task, ExecSession exec)
+        {
+            return new ResumeState(
+                    exec.getSessionConfigSource(),
+                    task.getInputTask(), task.getOutputTask(),
+                    inputSchema, outputSchema,
+                    Arrays.asList(inputCommitReports), Arrays.asList(outputCommitReports));
         }
 
         public PartialExecutionException buildPartialExecuteException(Throwable cause,
                 ExecutorTask task, ExecSession exec)
         {
-            return new PartialExecutionException(cause, new ResumeState(
-                        exec.getSessionConfigSource(),
-                        task.getInputTask(), task.getOutputTask(),
-                        inputSchema, outputSchema,
-                        Arrays.asList(inputCommitReports), Arrays.asList(outputCommitReports)));
+            return new PartialExecutionException(cause, buildResumeState(task, exec));
         }
     }
 
@@ -417,7 +422,7 @@ public class LocalExecutor
 
         final ProcessState state = new ProcessState(Exec.getLogger(LocalExecutor.class));
         try {
-            NextConfig inputNextConfig = in.transaction(task.getInputConfig(), new InputPlugin.Control() {
+            ConfigDiff inputConfigDiff = in.transaction(task.getInputConfig(), new InputPlugin.Control() {
                 public List<CommitReport> run(final TaskSource inputTask, final Schema inputSchema, final int processorCount)
                 {
                     state.initialize(processorCount);
@@ -427,27 +432,37 @@ public class LocalExecutor
                         {
                             Schema outputSchema = last(filterSchemas);
                             state.setOutputSchema(outputSchema);
-                            NextConfig outputNextConfig = out.transaction(task.getOutputConfig(), outputSchema, processorCount, new OutputPlugin.Control() {
+                            ConfigDiff outputConfigDiff = out.transaction(task.getOutputConfig(), outputSchema, processorCount, new OutputPlugin.Control() {
                                 public List<CommitReport> run(final TaskSource outputTask)
                                 {
                                     task.setInputTask(inputTask);
                                     task.setFilterTasks(filterTasks);
                                     task.setOutputTask(outputTask);
 
-                                    process(task.dump(), filterSchemas, processorCount, state);
-                                    if (!state.isAllCommitted()) {
-                                        throw state.getRepresentativeException();
+                                    if (processorCount > 0) {
+                                        process(task.dump(), filterSchemas, processorCount, state);
+                                        if (!state.isAllCommitted()) {
+                                            throw state.getRepresentativeException();
+                                        }
+                                    } else {
+                                        // TODO warning?
                                     }
                                     return state.getOutputCommitReports();
                                 }
                             });
-                            state.setOutputNextConfig(outputNextConfig);
+                            state.setOutputConfigDiff(outputConfigDiff);
                         }
                     });
                     return state.getInputCommitReports();
                 }
             });
-            state.setInputNextConfig(inputNextConfig);
+            state.setInputConfigDiff(inputConfigDiff);
+
+            try {
+                doCleanup(config, state.buildResumeState(task, Exec.session()));
+            } catch (Exception ex) {
+                state.logger.warn("Commit succeeded but cleanup failed. Ignoring this exception.", ex);  // TODO
+            }
 
             return state.buildExecuteResult();
 
@@ -473,7 +488,7 @@ public class LocalExecutor
 
         final ProcessState state = new ProcessState(Exec.getLogger(LocalExecutor.class));
         try {
-            NextConfig inputNextConfig = in.resume(resume.getInputTaskSource(), resume.getInputSchema(), resume.getInputCommitReports().size(), new InputPlugin.Control() {
+            ConfigDiff inputConfigDiff = in.resume(resume.getInputTaskSource(), resume.getInputSchema(), resume.getInputCommitReports().size(), new InputPlugin.Control() {
                 public List<CommitReport> run(final TaskSource inputTask, final Schema inputSchema, final int processorCount)
                 {
                     // TODO validate inputTask?
@@ -485,7 +500,7 @@ public class LocalExecutor
                         {
                             Schema outputSchema = last(filterSchemas);
                             state.setOutputSchema(outputSchema);
-                            NextConfig outputNextConfig = out.resume(resume.getOutputTaskSource(), outputSchema, processorCount, new OutputPlugin.Control() {
+                            ConfigDiff outputConfigDiff = out.resume(resume.getOutputTaskSource(), outputSchema, processorCount, new OutputPlugin.Control() {
                                 public List<CommitReport> run(final TaskSource outputTask)
                                 {
                                     // TODO validate outputTask?
@@ -502,20 +517,30 @@ public class LocalExecutor
                                         }
                                     }
 
-                                    process(task.dump(), filterSchemas, processorCount, state);
-                                    if (!state.isAllCommitted()) {
-                                        throw state.getRepresentativeException();
+                                    if (processorCount > 0) {
+                                        process(task.dump(), filterSchemas, processorCount, state);
+                                        if (!state.isAllCommitted()) {
+                                            throw state.getRepresentativeException();
+                                        }
+                                    } else {
+                                        // TODO warning?
                                     }
                                     return state.getOutputCommitReports();
                                 }
                             });
-                            state.setOutputNextConfig(outputNextConfig);
+                            state.setOutputConfigDiff(outputConfigDiff);
                         }
                     });
                     return state.getInputCommitReports();
                 }
             });
-            state.setInputNextConfig(inputNextConfig);
+            state.setInputConfigDiff(inputConfigDiff);
+
+            try {
+                doCleanup(config, state.buildResumeState(task, Exec.session()));
+            } catch (Exception ex) {
+                state.logger.warn("Commit succeeded but cleanup failed. Ignoring this exception.", ex);  // TODO
+            }
 
             return state.buildExecuteResult();