embulk 0.6.27 → 0.7.0

data/embulk-core/src/main/java/org/embulk/EmbulkService.java

@@ -14,6 +14,7 @@ import org.embulk.plugin.BuiltinPluginSourceModule;
 import org.embulk.jruby.JRubyScriptingModule;
 import static com.google.common.base.Preconditions.checkState;
 
+@Deprecated
 public class EmbulkService
 {
     private final ConfigSource systemConfig;

data/embulk-core/src/main/java/org/embulk/command/TablePreviewPrinter.java

@@ -5,7 +5,6 @@ import java.util.Arrays;
 import java.util.ArrayList;
 import java.io.PrintStream;
 import java.io.IOException;
-import com.google.common.base.Function;
 import org.embulk.config.ModelManager;
 import org.embulk.spi.Schema;
 

data/embulk-core/src/main/java/org/embulk/config/CommitReport.java

@@ -1,7 +1,11 @@
 package org.embulk.config;
 
+/**
+ * Simply replaced by TaskReport.
+ */
+@Deprecated
 public interface CommitReport
-        extends DataSource
+        extends TaskReport
 {
     @Override
     CommitReport getNested(String attrName);

data/embulk-core/src/main/java/org/embulk/config/ConfigLoader.java

@@ -31,6 +31,19 @@ public class ConfigLoader
         return new DataSourceImpl(model);
     }
 
+    public ConfigSource fromJsonString(String string)
+    {
+        JsonNode node;
+        try {
+            node = new ObjectMapper().readTree(string);
+        }
+        catch (IOException ex) {
+            throw new RuntimeException(ex);
+        }
+        validateJsonNode(node);
+        return new DataSourceImpl(model, (ObjectNode) node);
+    }
+
     public ConfigSource fromJsonFile(File file) throws IOException
     {
         try (FileInputStream is = new FileInputStream(file)) {
@@ -41,9 +54,14 @@ public class ConfigLoader
     public ConfigSource fromJson(InputStream stream) throws IOException
     {
         JsonNode node = new ObjectMapper().readTree(stream);
-        if (!node.isObject()) {
-            throw new RuntimeJsonMappingException("Expected object to load ConfigSource but got: "+node.getNodeType());
-        }
+        validateJsonNode(node);
+        return new DataSourceImpl(model, (ObjectNode) node);
+    }
+
+    public ConfigSource fromYamlString(String string)
+    {
+        JsonNode node = objectToJson(new Yaml().load(string));
+        validateJsonNode(node);
         return new DataSourceImpl(model, (ObjectNode) node);
     }
 
@@ -56,13 +74,16 @@ public class ConfigLoader
 
     public ConfigSource fromYaml(InputStream stream) throws IOException
     {
-        Yaml yaml = new Yaml();
-        Object object = yaml.load(stream);
-        JsonNode node = objectToJson(object);
+        JsonNode node = objectToJson(new Yaml().load(stream));
+        validateJsonNode(node);
+        return new DataSourceImpl(model, (ObjectNode) node);
+    }
+
+    private static void validateJsonNode(JsonNode node)
+    {
         if (!node.isObject()) {
             throw new RuntimeJsonMappingException("Expected object to load ConfigSource but got "+node);
         }
-        return new DataSourceImpl(model, (ObjectNode) node);
     }
 
     @Deprecated

data/embulk-core/src/main/java/org/embulk/config/DataSourceImpl.java

@@ -4,16 +4,13 @@ import java.util.List;
 import java.util.Map;
 import java.util.Iterator;
 import com.google.common.collect.ImmutableList;
-import com.fasterxml.jackson.core.JsonParser;
-import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.JsonMappingException;
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.fasterxml.jackson.databind.node.ArrayNode;
 import com.fasterxml.jackson.databind.node.JsonNodeFactory;
 
 public class DataSourceImpl
-        implements ConfigSource, TaskSource, CommitReport, ConfigDiff
+        implements ConfigSource, TaskSource, TaskReport, /* Deprecated */ CommitReport, ConfigDiff
 {
     protected final ObjectNode data;
     protected final ModelManager model;

data/embulk-core/src/main/java/org/embulk/config/DataSourceSerDe.java

@@ -1,7 +1,6 @@
 package org.embulk.config;
 
 import java.io.IOException;
-import com.google.inject.Inject;
 import com.fasterxml.jackson.core.JsonGenerator;
 import com.fasterxml.jackson.core.JsonParser;
 import com.fasterxml.jackson.databind.module.SimpleModule;
@@ -13,13 +12,13 @@ import com.fasterxml.jackson.databind.DeserializationContext;
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.JsonMappingException;
 import com.fasterxml.jackson.databind.node.ObjectNode;
-import com.fasterxml.jackson.module.guice.ObjectMapperModule;
 
 public class DataSourceSerDe
 {
     public static class SerDeModule
             extends SimpleModule
     {
+        @SuppressWarnings("deprecation")
         public SerDeModule(final ModelManager model)
         {
             // DataSourceImpl
@@ -34,7 +33,11 @@ public class DataSourceSerDe
             addSerializer(TaskSource.class, new DataSourceSerializer<TaskSource>());
             addDeserializer(TaskSource.class, new DataSourceDeserializer<TaskSource>(model));
 
-            // CommitReport
+            // TaskReport
+            addSerializer(TaskReport.class, new DataSourceSerializer<TaskReport>());
+            addDeserializer(TaskReport.class, new DataSourceDeserializer<TaskReport>(model));
+
+            // CommitReport (Deprecated)
             addSerializer(CommitReport.class, new DataSourceSerializer<CommitReport>());
             addDeserializer(CommitReport.class, new DataSourceDeserializer<CommitReport>(model));
 

data/embulk-core/src/main/java/org/embulk/config/ModelManager.java

@@ -6,11 +6,9 @@ import com.google.inject.Inject;
 import com.google.inject.Injector;
 import com.google.common.base.Throwables;
 import com.fasterxml.jackson.core.JsonParser;
-import com.fasterxml.jackson.databind.Module;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.node.ObjectNode;
-import com.fasterxml.jackson.databind.JsonMappingException;
 
 public class ModelManager
 {

data/embulk-core/src/main/java/org/embulk/config/TaskReport.java

@@ -0,0 +1,29 @@
+package org.embulk.config;
+
+public interface TaskReport
+        extends DataSource
+{
+    @Override
+    TaskReport getNested(String attrName);
+
+    @Override
+    TaskReport getNestedOrSetEmpty(String attrName);
+
+    @Override
+    TaskReport set(String attrName, Object v);
+
+    @Override
+    TaskReport setNested(String attrName, DataSource v);
+
+    @Override
+    TaskReport setAll(DataSource other);
+
+    @Override
+    TaskReport remove(String attrName);
+
+    @Override
+    TaskReport deepCopy();
+
+    @Override
+    TaskReport merge(DataSource other);
+}

data/embulk-core/src/main/java/org/embulk/exec/BulkLoader.java

@@ -1,15 +1,12 @@
 package org.embulk.exec;
 
 import java.util.List;
-import java.util.Arrays;
 import java.util.concurrent.ExecutionException;
 import com.google.common.base.Optional;
 import com.google.common.collect.ImmutableList;
 import com.google.inject.Inject;
 import com.google.inject.Injector;
 import com.google.common.base.Throwables;
-import com.google.common.base.Predicates;
-import com.google.common.collect.Iterables;
 import org.embulk.config.Task;
 import org.embulk.config.Config;
 import org.embulk.config.ConfigDefault;
@@ -17,7 +14,7 @@ import org.embulk.config.ConfigSource;
 import org.embulk.config.ConfigException;
 import org.embulk.config.TaskSource;
 import org.embulk.config.ConfigDiff;
-import org.embulk.config.CommitReport;
+import org.embulk.config.TaskReport;
 import org.embulk.plugin.PluginType;
 import org.embulk.spi.Schema;
 import org.embulk.spi.Exec;
@@ -132,7 +129,7 @@ public class BulkLoader
         public void initialize(int inputTaskCount, int outputTaskCount)
         {
             if (inputTaskStates != null || outputTaskStates != null) {
-                // initialize is called twice if resume (by restoreResumedCommitReports and ExecutorPlugin.execute)
+                // initialize is called twice if resume (by restoreResumedTaskReports and ExecutorPlugin.execute)
                 if (inputTaskStates.size() != inputTaskCount || outputTaskStates.size() != outputTaskCount) {
                     throw new ConfigException(String.format(
                                 "input task count and output task (%d and %d) must be same with the first execution (%d and %d) whenre resumed",
@@ -213,38 +210,38 @@ public class BulkLoader
             this.inputConfigDiff = inputConfigDiff;
         }
 
-        private List<Optional<CommitReport>> getInputCommitReports()
+        private List<Optional<TaskReport>> getInputTaskReports()
         {
-            ImmutableList.Builder<Optional<CommitReport>> builder = ImmutableList.builder();
+            ImmutableList.Builder<Optional<TaskReport>> builder = ImmutableList.builder();
             for (TaskState inputTaskState : inputTaskStates) {
-                builder.add(inputTaskState.getCommitReport());
+                builder.add(inputTaskState.getTaskReport());
             }
             return builder.build();
         }
 
-        private List<Optional<CommitReport>> getOutputCommitReports()
+        private List<Optional<TaskReport>> getOutputTaskReports()
         {
-            ImmutableList.Builder<Optional<CommitReport>> builder = ImmutableList.builder();
+            ImmutableList.Builder<Optional<TaskReport>> builder = ImmutableList.builder();
             for (TaskState outputTaskState : outputTaskStates) {
-                builder.add(outputTaskState.getCommitReport());
+                builder.add(outputTaskState.getTaskReport());
             }
             return builder.build();
         }
 
-        public List<CommitReport> getAllInputCommitReports()
+        public List<TaskReport> getAllInputTaskReports()
         {
-            ImmutableList.Builder<CommitReport> builder = ImmutableList.builder();
+            ImmutableList.Builder<TaskReport> builder = ImmutableList.builder();
             for (TaskState inputTaskState : inputTaskStates) {
-                builder.add(inputTaskState.getCommitReport().get());
+                builder.add(inputTaskState.getTaskReport().get());
             }
             return builder.build();
         }
 
-        public List<CommitReport> getAllOutputCommitReports()
+        public List<TaskReport> getAllOutputTaskReports()
         {
-            ImmutableList.Builder<CommitReport> builder = ImmutableList.builder();
+            ImmutableList.Builder<TaskReport> builder = ImmutableList.builder();
             for (TaskState outputTaskState : outputTaskStates) {
-                builder.add(outputTaskState.getCommitReport().get());
+                builder.add(outputTaskState.getTaskReport().get());
             }
             return builder.build();
         }
@@ -323,7 +320,7 @@ public class BulkLoader
                     exec.getSessionExecConfig(),
                     inputTaskSource, outputTaskSource,
                     first(schemas), executorSchema,
-                    getInputCommitReports(), getOutputCommitReports());
+                    getInputTaskReports(), getOutputTaskReports());
         }
 
         public PartialExecutionException buildPartialExecuteException(Throwable cause, ExecSession exec)
@@ -442,24 +439,24 @@ public class BulkLoader
         BulkLoaderTask task = config.loadConfig(BulkLoaderTask.class);
         ProcessPluginSet plugins = new ProcessPluginSet(task);  // TODO don't create filter plugins
 
-        ImmutableList.Builder<CommitReport> successfulInputCommitReports = ImmutableList.builder();
-        ImmutableList.Builder<CommitReport> successfulOutputCommitReports = ImmutableList.builder();
-        for (Optional<CommitReport> inputCommitReport : resume.getInputCommitReports()) {
-            if (inputCommitReport.isPresent()) {
-                successfulInputCommitReports.add(inputCommitReport.get());
+        ImmutableList.Builder<TaskReport> successfulInputTaskReports = ImmutableList.builder();
+        ImmutableList.Builder<TaskReport> successfulOutputTaskReports = ImmutableList.builder();
+        for (Optional<TaskReport> inputTaskReport : resume.getInputTaskReports()) {
+            if (inputTaskReport.isPresent()) {
+                successfulInputTaskReports.add(inputTaskReport.get());
             }
         }
-        for (Optional<CommitReport> outputCommitReport : resume.getOutputCommitReports()) {
-            if (outputCommitReport.isPresent()) {
-                successfulOutputCommitReports.add(outputCommitReport.get());
+        for (Optional<TaskReport> outputTaskReport : resume.getOutputTaskReports()) {
+            if (outputTaskReport.isPresent()) {
+                successfulOutputTaskReports.add(outputTaskReport.get());
             }
         }
 
         plugins.getInputPlugin().cleanup(resume.getInputTaskSource(), resume.getInputSchema(),
-                resume.getInputCommitReports().size(), successfulInputCommitReports.build());
+                resume.getInputTaskReports().size(), successfulInputTaskReports.build());
 
         plugins.getOutputPlugin().cleanup(resume.getOutputTaskSource(), resume.getOutputSchema(),
-                resume.getOutputCommitReports().size(), successfulOutputCommitReports.build());
+                resume.getOutputTaskReports().size(), successfulOutputTaskReports.build());
     }
 
     private ExecutorPlugin newExecutorPlugin(BulkLoaderTask task)
@@ -478,7 +475,7 @@ public class BulkLoader
         final LoaderState state = new LoaderState(Exec.getLogger(BulkLoader.class), plugins);
         try {
             ConfigDiff inputConfigDiff = plugins.getInputPlugin().transaction(task.getInputConfig(), new InputPlugin.Control() {
-                public List<CommitReport> run(final TaskSource inputTask, final Schema inputSchema, final int inputTaskCount)
+                public List<TaskReport> run(final TaskSource inputTask, final Schema inputSchema, final int inputTaskCount)
                 {
                     state.setInputTaskSource(inputTask);
                     Filters.transaction(plugins.getFilterPlugins(), task.getFilterConfigs(), inputSchema, new Filters.Control() {
@@ -491,7 +488,7 @@ public class BulkLoader
                                 {
                                     state.setExecutorSchema(executorSchema);
                                     ConfigDiff outputConfigDiff = plugins.getOutputPlugin().transaction(task.getOutputConfig(), executorSchema, outputTaskCount, new OutputPlugin.Control() {
-                                        public List<CommitReport> run(final TaskSource outputTask)
+                                        public List<TaskReport> run(final TaskSource outputTask)
                                         {
                                             state.setOutputTaskSource(outputTask);
 
@@ -501,7 +498,7 @@ public class BulkLoader
                                                 execute(task, executor, state);
                                             }
 
-                                            return state.getAllOutputCommitReports();
+                                            return state.getAllOutputTaskReports();
                                         }
                                     });
                                     state.setOutputConfigDiff(outputConfigDiff);
@@ -509,7 +506,7 @@ public class BulkLoader
                             });
                         }
                     });
-                    return state.getAllInputCommitReports();
+                    return state.getAllInputTaskReports();
                 }
             });
             state.setInputConfigDiff(inputConfigDiff);
@@ -539,8 +536,8 @@ public class BulkLoader
 
         final LoaderState state = new LoaderState(Exec.getLogger(BulkLoader.class), plugins);
         try {
-            ConfigDiff inputConfigDiff = plugins.getInputPlugin().resume(resume.getInputTaskSource(), resume.getInputSchema(), resume.getInputCommitReports().size(), new InputPlugin.Control() {
-                public List<CommitReport> run(final TaskSource inputTask, final Schema inputSchema, final int inputTaskCount)
+            ConfigDiff inputConfigDiff = plugins.getInputPlugin().resume(resume.getInputTaskSource(), resume.getInputSchema(), resume.getInputTaskReports().size(), new InputPlugin.Control() {
+                public List<TaskReport> run(final TaskSource inputTask, final Schema inputSchema, final int inputTaskCount)
                 {
                     // TODO validate inputTask?
                     // TODO validate inputSchema
@@ -556,17 +553,17 @@ public class BulkLoader
                                     // TODO validate executorSchema
                                     state.setExecutorSchema(executorSchema);
                                     ConfigDiff outputConfigDiff = plugins.getOutputPlugin().resume(resume.getOutputTaskSource(), executorSchema, outputTaskCount, new OutputPlugin.Control() {
-                                        public List<CommitReport> run(final TaskSource outputTask)
+                                        public List<TaskReport> run(final TaskSource outputTask)
                                         {
                                             // TODO validate outputTask?
                                             state.setOutputTaskSource(outputTask);
 
-                                            restoreResumedCommitReports(resume, state);
+                                            restoreResumedTaskReports(resume, state);
                                             if (!state.isAllTasksCommitted()) {
                                                 execute(task, executor, state);
                                             }
 
-                                            return state.getAllOutputCommitReports();
+                                            return state.getAllOutputTaskReports();
                                         }
                                     });
                                     state.setOutputConfigDiff(outputConfigDiff);
@@ -574,7 +571,7 @@ public class BulkLoader
                             });
                         }
                     });
-                    return state.getAllInputCommitReports();
+                    return state.getAllInputTaskReports();
                 }
             });
             state.setInputConfigDiff(inputConfigDiff);
@@ -595,29 +592,29 @@ public class BulkLoader
         }
     }
 
-    private static void restoreResumedCommitReports(ResumeState resume, LoaderState state)
+    private static void restoreResumedTaskReports(ResumeState resume, LoaderState state)
     {
-        int inputTaskCount = resume.getInputCommitReports().size();
-        int outputTaskCount = resume.getOutputCommitReports().size();
+        int inputTaskCount = resume.getInputTaskReports().size();
+        int outputTaskCount = resume.getOutputTaskReports().size();
 
         state.initialize(inputTaskCount, outputTaskCount);
 
         for (int i=0; i < inputTaskCount; i++) {
-            Optional<CommitReport> report = resume.getInputCommitReports().get(i);
+            Optional<TaskReport> report = resume.getInputTaskReports().get(i);
             if (report.isPresent()) {
                 TaskState task = state.getInputTaskState(i);
                 task.start();
-                task.setCommitReport(report.get());
+                task.setTaskReport(report.get());
                 task.finish();
             }
         }
 
         for (int i=0; i < outputTaskCount; i++) {
-            Optional<CommitReport> report = resume.getOutputCommitReports().get(i);
+            Optional<TaskReport> report = resume.getOutputTaskReports().get(i);
             if (report.isPresent()) {
                 TaskState task = state.getOutputTaskState(i);
                 task.start();
-                task.setCommitReport(report.get());
+                task.setTaskReport(report.get());
                 task.finish();
             }
         }

data/embulk-core/src/main/java/org/embulk/exec/ExecModule.java

@@ -3,7 +3,6 @@ package org.embulk.exec;
 import org.slf4j.ILoggerFactory;
 import com.google.common.base.Preconditions;
 import com.google.inject.Module;
-import com.google.inject.name.Names;
 import com.google.inject.Binder;
 import com.google.inject.Scopes;
 import com.fasterxml.jackson.module.guice.ObjectMapperModule;

data/embulk-core/src/main/java/org/embulk/exec/GuessExecutor.java

@@ -7,20 +7,17 @@ import java.util.concurrent.ExecutionException;
 import com.google.common.collect.ImmutableList;
 import com.google.common.base.Throwables;
 import com.google.inject.Inject;
-import com.google.inject.Injector;
 import com.google.inject.Binder;
 import com.google.inject.multibindings.Multibinder;
 import org.embulk.plugin.PluginType;
 import org.embulk.config.Config;
 import org.embulk.config.ConfigDefault;
 import org.embulk.config.ConfigDiff;
-import org.embulk.config.DataSource;
 import org.embulk.config.Task;
 import org.embulk.config.TaskSource;
 import org.embulk.config.ConfigSource;
-import org.embulk.config.CommitReport;
+import org.embulk.config.TaskReport;
 import org.embulk.spi.Schema;
-import org.embulk.spi.Column;
 import org.embulk.spi.Page;
 import org.embulk.spi.Buffer;
 import org.embulk.spi.InputPlugin;
@@ -148,7 +145,7 @@ public class GuessExecutor
             ConfigDiff guessed;
             try {
                 input.transaction(guessInputConfig, new InputPlugin.Control() {
-                    public List<CommitReport> run(TaskSource inputTaskSource, Schema schema, int taskCount)
+                    public List<TaskReport> run(TaskSource inputTaskSource, Schema schema, int taskCount)
                     {
                         if (taskCount == 0) {
                             throw new NoSampleException("No input files to guess");
@@ -212,7 +209,7 @@ public class GuessExecutor
 
         public void cleanup(TaskSource taskSource,
                 int taskCount,
-                List<CommitReport> successCommitReports)
+                List<TaskReport> successTaskReports)
         {
             if (buffer != null) {
                 buffer.release();
@@ -257,7 +254,7 @@ public class GuessExecutor
         public void abort() { }
 
         @Override
-        public CommitReport commit()
+        public TaskReport commit()
         {
             return null;
         }