embulk 0.4.3 → 0.4.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b24d1b7fd55459a318f9dbce2fff9b976f40738b
-  data.tar.gz: 88573c322b4d3e32f4fb11775122db1c1b28f3d2
+  metadata.gz: 157c521d28f82705b9162cca7ca03e052d5e478f
+  data.tar.gz: 5063ed2e3544fede216449ea7cebc11c427faf9c
 SHA512:
-  metadata.gz: 3179a8555d542ae38a785ad308a9b44560ae66889f263c932675e4ef5050b719636126ec0a960c6dba098030e269c3b0dd7b34f841d278857464c143ea0a2940
-  data.tar.gz: 20772970a71e622473f119ebc4ee9316fdbfff48e96948345fb6a001342d727a70ecd525e217fac98eb0838f2e20ccebcfd6d48603c945a0d9b096f7070ea92a
+  metadata.gz: 694c3da663fbb60de1ac1529e9d17c561785bc0637713d3d6d325e11fdd69204e200bb359d2592b177cf5a05fb743b53d80083dfb9216807e3f1cc65a54de47b
+  data.tar.gz: 3b4dcaf0704cce4a6bf6af89f8e1929412b7d85c54b59d98da84efdc24ef2d55dc28f4e347fe8b15f7f887c49c77a70a405ea3777723c5b2aed7f0c0a89fb1d2

data/README.md

@@ -24,7 +24,7 @@ You can release plugins to share your efforts of data cleaning, error handling,
 The single-file package is the simplest way to try Embulk. You can download the latest embulk-VERSION.jar from [the releases page](https://bintray.com/embulk/maven/embulk/view#files) and run it with java:
 
 ```
-wget https://bintray.com/artifact/download/embulk/maven/embulk-0.4.3.jar -O embulk.jar
+wget https://bintray.com/artifact/download/embulk/maven/embulk-0.4.4.jar -O embulk.jar
 java -jar embulk.jar --help
 ```
 
@@ -150,20 +150,17 @@ bintray_user=(bintray user name)
 bintray_api_key=(bintray api key)
 ```
 
-Update following files:
+Run following commands and follow its instruction:
 
-* embulk-docs/src/release/release-VERSION.rst (release note)
-* build.gradle (version number)
-* lib/embulk/version.rb (version number)
-
-Then, build and upload using gradle:
+```
+./gradlew set_version -Pto=$VERSION
+```
 
 ```
 ./gradlew releaseCheck
-./gradlew cli gem
-./gradlew bintrayUpload
-gem push pkg/embulk-....gem
-open "https://bintray.com/embulk/maven/embulk"  # and upload pkg/embulk-....jar
+./gradlew release
+git commit -am v$VERSION
+git tag v$VERSION
 ```
 
 See also:

data/build.gradle

@@ -11,7 +11,7 @@ def release_projects = [project(":embulk-core"), project(":embulk-standards")]
 
 allprojects {
     group = 'org.embulk'
-    version = '0.4.3'
+    version = '0.4.4'
 
     apply plugin: 'java'
     apply plugin: 'maven-publish'
@@ -32,7 +32,7 @@ allprojects {
         filesSpec {
             // include embulk-<version>.jar built by 'cli' task
             from "pkg/embulk-${project.version}.jar"
-            into "embulk-${project.version}.jar"
+            into "."
         }
 
         dryRun = false
@@ -229,6 +229,26 @@ Manual operations:
 bintrayUpload.mustRunAfter('releaseCheck')
 rubyGemsUpload.mustRunAfter('releaseCheck')
 
+task set_version << {
+    if (!project.hasProperty("to")) {
+        throw new GradleException("Usage: ./gradlew -Pto=VERSION")
+    }
+
+    File gradle_ver = file('build.gradle')
+    gradle_ver.write(gradle_ver.getText().replaceFirst("version = '(\\d+)(\\.\\d+){2}'", "version = '${to}'"))
+
+    File ruby_ver = file('lib/embulk/version.rb')
+    ruby_ver.write(ruby_ver.getText().replaceFirst("VERSION = '(\\d+)(\\.\\d+){2}'", "VERSION = '${to}'"))
+
+    File readme_ver = file('README.md')
+    readme_ver.write(readme_ver.getText().replaceFirst('embulk-(\\d+)(\\.\\d+){2}', "embulk-${to}"))
+
+    file("embulk-docs/src/release/release-${to}.rst").append("")
+    "git add embulk-docs/src/release/release-${to}.rst".execute().waitFor()
+
+    println "add 'release/release-${to}' line to embulk-docs/src/release.rst"
+}
+
 task gemspec << {
     file('build').mkdirs()
     file('build/gemspec').write($/

data/embulk-core/src/main/java/org/embulk/command/Runner.java

@@ -41,6 +41,9 @@ public class Runner
 
         private String resumeStatePath;
         public String getResumeStatePath() { return resumeStatePath; }
+
+        private String logLevel;
+        public String getLogLevel() { return logLevel; }
     }
 
     private final Options options;
@@ -53,6 +56,10 @@ public class Runner
         ModelManager bootstrapModelManager = new ModelManager(null, new ObjectMapper());
         this.options = bootstrapModelManager.readObject(Options.class, optionJson);
         this.systemConfig = new ConfigLoader(bootstrapModelManager).fromPropertiesYamlLiteral(System.getProperties(), "embulk.");
+        String logLevel = options.getLogLevel();
+        if (logLevel != null) {
+            systemConfig.set("logLevel", logLevel);
+        }
         this.service = new EmbulkService(systemConfig);
         this.injector = service.getInjector();
     }

data/embulk-core/src/main/java/org/embulk/exec/GuessExecutor.java

@@ -70,7 +70,9 @@ public class GuessExecutor
             return Exec.doWith(exec, new ExecAction<ConfigDiff>() {
                 public ConfigDiff run()
                 {
-                    return doGuess(config);
+                    try (SetCurrentThreadName dontCare = new SetCurrentThreadName("guess")) {
+                        return doGuess(config);
+                    }
                 }
             });
         } catch (Exception ex) {
@@ -112,9 +114,9 @@ public class GuessExecutor
             ConfigDiff guessed;
             try {
                 input.transaction(guessInputConfig, new InputPlugin.Control() {
-                    public List<CommitReport> run(TaskSource inputTaskSource, Schema schema, int processorCount)
+                    public List<CommitReport> run(TaskSource inputTaskSource, Schema schema, int taskCount)
                     {
-                        // TODO repeat runwith processorIndex++ if NoSampleException happens
+                        // TODO repeat runwith taskIndex++ if NoSampleException happens
                         input.run(inputTaskSource, null, 0, new PageOutput() {
                             @Override
                             public void add(Page page)
@@ -172,14 +174,14 @@ public class GuessExecutor
         }
 
         public ConfigDiff resume(TaskSource taskSource,
-                int processorCount,
+                int taskCount,
                 FileInputPlugin.Control control)
         {
             throw new UnsupportedOperationException();
         }
 
         public void cleanup(TaskSource taskSource,
-                int processorCount,
+                int taskCount,
                 List<CommitReport> successCommitReports)
         {
             if (buffer != null) {
@@ -188,7 +190,7 @@ public class GuessExecutor
             }
         }
 
-        public TransactionalFileInput open(TaskSource taskSource, int processorIndex)
+        public TransactionalFileInput open(TaskSource taskSource, int taskIndex)
         {
             return new BufferTransactionalFileInput(buffer);
         }

data/embulk-core/src/main/java/org/embulk/exec/LocalExecutor.java

@@ -95,7 +95,7 @@ public class LocalExecutor
         private volatile CommitReport[] outputCommitReports;
         private volatile ConfigDiff inputConfigDiff;
         private volatile ConfigDiff outputConfigDiff;
-        private int processorCount;
+        private int taskCount;
 
         public ProcessState(Logger logger)
         {
@@ -114,7 +114,7 @@ public class LocalExecutor
             this.exceptions = new Throwable[count];
             this.inputCommitReports = new CommitReport[count];
             this.outputCommitReports = new CommitReport[count];
-            this.processorCount = count;
+            this.taskCount = count;
         }
 
         public void setInputSchema(Schema inputSchema)
@@ -158,9 +158,9 @@ public class LocalExecutor
             finished[i] = true;
         }
 
-        public int getProcessorCount()
+        public int getTaskCount()
         {
-            return processorCount;
+            return taskCount;
         }
 
         public int getStartedCount()
@@ -209,11 +209,11 @@ public class LocalExecutor
 
         public boolean isAllCommitted()
         {
-            if (processorCount <= 0) {
+            if (taskCount <= 0) {
                 // not initialized
                 return false;
             }
-            for (int i=0; i < processorCount; i++) {
+            for (int i=0; i < taskCount; i++) {
                 if (!isOutputCommitted(i)) {
                     return false;
                 }
@@ -223,7 +223,7 @@ public class LocalExecutor
 
         public boolean isAnyCommitted()
         {
-            for (int i=0; i < processorCount; i++) {
+            for (int i=0; i < taskCount; i++) {
                 if (isOutputCommitted(i)) {
                     return true;
                 }
@@ -355,7 +355,9 @@ public class LocalExecutor
             return Exec.doWith(exec, new ExecAction<ExecutionResult>() {
                 public ExecutionResult run()
                 {
-                    return doRun(config);
+                    try (SetCurrentThreadName dontCare = new SetCurrentThreadName("transaction")) {
+                        return doRun(config);
+                    }
                 }
             });
         } catch (Exception ex) {
@@ -370,7 +372,9 @@ public class LocalExecutor
             return Exec.doWith(exec, new ExecAction<ExecutionResult>() {
                 public ExecutionResult run()
                 {
-                    return doResume(config, resume);
+                    try (SetCurrentThreadName dontCare = new SetCurrentThreadName("resume")) {
+                        return doResume(config, resume);
+                    }
                 }
             });
         } catch (Exception ex) {
@@ -385,8 +389,10 @@ public class LocalExecutor
             Exec.doWith(exec, new ExecAction<Void>() {
                 public Void run()
                 {
-                    doCleanup(config, resume);
-                    return null;
+                    try (SetCurrentThreadName dontCare = new SetCurrentThreadName("cleanup")) {
+                        doCleanup(config, resume);
+                        return null;
+                    }
                 }
             });
         } catch (Exception ex) {
@@ -423,24 +429,24 @@ public class LocalExecutor
         final ProcessState state = new ProcessState(Exec.getLogger(LocalExecutor.class));
         try {
             ConfigDiff inputConfigDiff = in.transaction(task.getInputConfig(), new InputPlugin.Control() {
-                public List<CommitReport> run(final TaskSource inputTask, final Schema inputSchema, final int processorCount)
+                public List<CommitReport> run(final TaskSource inputTask, final Schema inputSchema, final int taskCount)
                 {
-                    state.initialize(processorCount);
+                    state.initialize(taskCount);
                     state.setInputSchema(inputSchema);
                     Filters.transaction(filterPlugins, task.getFilterConfigs(), inputSchema, new Filters.Control() {
                         public void run(final List<TaskSource> filterTasks, final List<Schema> filterSchemas)
                         {
                             Schema outputSchema = last(filterSchemas);
                             state.setOutputSchema(outputSchema);
-                            ConfigDiff outputConfigDiff = out.transaction(task.getOutputConfig(), outputSchema, processorCount, new OutputPlugin.Control() {
+                            ConfigDiff outputConfigDiff = out.transaction(task.getOutputConfig(), outputSchema, taskCount, new OutputPlugin.Control() {
                                 public List<CommitReport> run(final TaskSource outputTask)
                                 {
                                     task.setInputTask(inputTask);
                                     task.setFilterTasks(filterTasks);
                                     task.setOutputTask(outputTask);
 
-                                    if (processorCount > 0) {
-                                        process(task.dump(), filterSchemas, processorCount, state);
+                                    if (taskCount > 0) {
+                                        process(task.dump(), filterSchemas, taskCount, state);
                                         if (!state.isAllCommitted()) {
                                             throw state.getRepresentativeException();
                                         }
@@ -489,18 +495,18 @@ public class LocalExecutor
         final ProcessState state = new ProcessState(Exec.getLogger(LocalExecutor.class));
         try {
             ConfigDiff inputConfigDiff = in.resume(resume.getInputTaskSource(), resume.getInputSchema(), resume.getInputCommitReports().size(), new InputPlugin.Control() {
-                public List<CommitReport> run(final TaskSource inputTask, final Schema inputSchema, final int processorCount)
+                public List<CommitReport> run(final TaskSource inputTask, final Schema inputSchema, final int taskCount)
                 {
                     // TODO validate inputTask?
                     // TODO validate inputSchema
-                    // TODO validate processorCount
-                    state.initialize(processorCount);
+                    // TODO validate taskCount
+                    state.initialize(taskCount);
                     Filters.transaction(filterPlugins, task.getFilterConfigs(), inputSchema, new Filters.Control() {
                         public void run(final List<TaskSource> filterTasks, final List<Schema> filterSchemas)
                         {
                             Schema outputSchema = last(filterSchemas);
                             state.setOutputSchema(outputSchema);
-                            ConfigDiff outputConfigDiff = out.resume(resume.getOutputTaskSource(), outputSchema, processorCount, new OutputPlugin.Control() {
+                            ConfigDiff outputConfigDiff = out.resume(resume.getOutputTaskSource(), outputSchema, taskCount, new OutputPlugin.Control() {
                                 public List<CommitReport> run(final TaskSource outputTask)
                                 {
                                     // TODO validate outputTask?
@@ -517,8 +523,8 @@ public class LocalExecutor
                                         }
                                     }
 
-                                    if (processorCount > 0) {
-                                        process(task.dump(), filterSchemas, processorCount, state);
+                                    if (taskCount > 0) {
+                                        process(task.dump(), filterSchemas, taskCount, state);
                                         if (!state.isAllCommitted()) {
                                             throw state.getRepresentativeException();
                                         }
@@ -556,12 +562,12 @@ public class LocalExecutor
         }
     }
 
-    private void process(TaskSource taskSource, List<Schema> filterSchemas, int processorCount,
+    private void process(TaskSource taskSource, List<Schema> filterSchemas, int taskCount,
             ProcessState state)
     {
-        List<Future<Throwable>> futures = new ArrayList<>(processorCount);
+        List<Future<Throwable>> futures = new ArrayList<>(taskCount);
         try {
-            for (int i=0; i < processorCount; i++) {
+            for (int i=0; i < taskCount; i++) {
                 if (state.isOutputCommitted(i)) {
                     state.getLogger().warn("Skipped resumed task {}", i);
                     futures.add(null);  // resumed
@@ -571,7 +577,7 @@ public class LocalExecutor
             }
             showProgress(state);
 
-            for (int i=0; i < processorCount; i++) {
+            for (int i=0; i < taskCount; i++) {
                 if (futures.get(i) == null) {
                     continue;
                 }
@@ -597,7 +603,7 @@ public class LocalExecutor
 
     private void showProgress(ProcessState state)
     {
-        int total = state.getProcessorCount();
+        int total = state.getTaskCount();
         int finished = state.getFinishedCount();
         int started = state.getStartedCount();
         state.getLogger().info(String.format("{done:%3d / %d, running: %d}", finished, total, started - finished));
@@ -610,30 +616,32 @@ public class LocalExecutor
         return executor.submit(new Callable<Throwable>() {
             public Throwable call()
             {
-                final ExecutorTask task = taskSource.loadTask(ExecutorTask.class);
-                final InputPlugin in = newInputPlugin(task);
-                final List<FilterPlugin> filterPlugins = newFilterPlugins(task);
-                final OutputPlugin out = newOutputPlugin(task);
-
-                TransactionalPageOutput tran = out.open(task.getOutputTask(), last(filterSchemas), index);
-                PageOutput closeThis = tran;
-                state.start(index);
-                try {
-                    PageOutput filtered = closeThis = Filters.open(filterPlugins, task.getFilterTasks(), filterSchemas, tran);
-                    state.setInputCommitReport(index, in.run(task.getInputTask(), first(filterSchemas), index, filtered));
-                    state.setOutputCommitReport(index, tran.commit());  // TODO check output.finish() is called. wrap or abstract
-                    return null;
-                } finally {
+                try (SetCurrentThreadName dontCare = new SetCurrentThreadName(String.format("task-%04d", index))) {
+                    final ExecutorTask task = taskSource.loadTask(ExecutorTask.class);
+                    final InputPlugin in = newInputPlugin(task);
+                    final List<FilterPlugin> filterPlugins = newFilterPlugins(task);
+                    final OutputPlugin out = newOutputPlugin(task);
+
+                    TransactionalPageOutput tran = out.open(task.getOutputTask(), last(filterSchemas), index);
+                    PageOutput closeThis = tran;
+                    state.start(index);
                     try {
+                        PageOutput filtered = closeThis = Filters.open(filterPlugins, task.getFilterTasks(), filterSchemas, tran);
+                        state.setInputCommitReport(index, in.run(task.getInputTask(), first(filterSchemas), index, filtered));
+                        state.setOutputCommitReport(index, tran.commit());  // TODO check output.finish() is called. wrap or abstract
+                        return null;
+                    } finally {
                         try {
-                            if (!state.isOutputCommitted(index)) {
-                                tran.abort();
+                            try {
+                                if (!state.isOutputCommitted(index)) {
+                                    tran.abort();
+                                }
+                            } finally {
+                                closeThis.close();
                             }
                         } finally {
-                            closeThis.close();
+                            state.finish(index);
                         }
-                    } finally {
-                        state.finish(index);
                     }
                 }
             }

data/embulk-core/src/main/java/org/embulk/exec/LoggerProvider.java

@@ -4,20 +4,37 @@ import java.util.Properties;
 import org.slf4j.ILoggerFactory;
 import org.slf4j.LoggerFactory;
 import org.apache.log4j.PropertyConfigurator;
+import com.google.inject.Inject;
 import com.google.inject.Provider;
+import org.embulk.config.ConfigSource;
 
 public class LoggerProvider
         implements Provider<ILoggerFactory>
 {
-    public LoggerProvider()
+    @Inject
+    public LoggerProvider(@ForSystemConfig ConfigSource systemConfig)
     {
         // TODO system config
         Properties prop = new Properties();
 
-        prop.setProperty("log4j.rootLogger", "INFO,root");
+        final String level;
+        String logLevel = systemConfig.get(String.class, "logLevel", "info");
+        switch (logLevel) {
+        case "fatal": level = "FATAL"; break;
+        case "error": level = "ERROR"; break;
+        case "warn":  level = "WARN";  break;
+        case "info":  level = "INFO";  break;
+        case "debug": level = "DEBUG"; break;
+        case "trace": level = "TRACE"; break;
+        default:
+            throw new IllegalArgumentException(String.format(
+                        "System property embulk.logLevel=%s is invalid. Available levels are fatal, error, warn, info, debug and trace.", logLevel));
+        }
+
+        prop.setProperty("log4j.rootLogger", level+",root");
         prop.setProperty("log4j.appender.root", "org.apache.log4j.ConsoleAppender");
         prop.setProperty("log4j.appender.root.layout", "org.apache.log4j.PatternLayout");
-        prop.setProperty("log4j.appender.root.layout.ConversionPattern", "%d [%p]: %t:%c: %m%n");
+        prop.setProperty("log4j.appender.root.layout.ConversionPattern", "%d{yyyy-MM-dd HH:mm:ss.SSS Z} [%p] (%t): %m%n");
 
         // TODO
         PropertyConfigurator.configure(prop);

data/embulk-core/src/main/java/org/embulk/exec/PreviewExecutor.java

@@ -18,9 +18,11 @@ import org.embulk.spi.Page;
 import org.embulk.spi.PageOutput;
 import org.embulk.spi.PageReader;
 import org.embulk.spi.InputPlugin;
+import org.embulk.spi.FilterPlugin;
 import org.embulk.spi.Exec;
 import org.embulk.spi.ExecSession;
 import org.embulk.spi.ExecAction;
+import org.embulk.spi.util.Filters;
 
 public class PreviewExecutor
 {
@@ -34,6 +36,11 @@ public class PreviewExecutor
         @NotNull
         public ConfigSource getInputConfig();
 
+        @Config("filters")
+        @ConfigDefault("[]")
+        public List<ConfigSource> getFilterConfigs();
+
+        // TODO take preview_sample_rows from exec: config
         @Config("preview_sample_rows")
         @ConfigDefault("15")
         public int getSampleRows();
@@ -53,10 +60,12 @@ public class PreviewExecutor
     public PreviewResult preview(ExecSession exec, final ConfigSource config)
     {
         try {
-            return Exec.doWith(exec, new ExecAction<PreviewResult>() {
+            return Exec.doWith(exec.copyForPreview(), new ExecAction<PreviewResult>() {
                 public PreviewResult run()
                 {
-                    return doPreview(config);
+                    try (SetCurrentThreadName dontCare = new SetCurrentThreadName("preview")) {
+                        return doPreview(config);
+                    }
                 }
             });
         } catch (Exception ex) {
@@ -69,19 +78,38 @@ public class PreviewExecutor
         return Exec.newPlugin(InputPlugin.class, task.getInputConfig().get(PluginType.class, "type"));
     }
 
+    protected List<FilterPlugin> newFilterPlugins(PreviewTask task)
+    {
+        return Filters.newFilterPlugins(Exec.session(), task.getFilterConfigs());
+    }
+
     private PreviewResult doPreview(ConfigSource config)
     {
         final PreviewTask task = config.loadConfig(PreviewTask.class);
-        InputPlugin input = newInputPlugin(task);
+        final InputPlugin input = newInputPlugin(task);
+        final List<FilterPlugin> filterPlugins = newFilterPlugins(task);
 
         try {
             input.transaction(task.getInputConfig(), new InputPlugin.Control() {
-                public List<CommitReport> run(TaskSource taskSource, Schema schema, int processorCount)
+                public List<CommitReport> run(final TaskSource inputTask, Schema inputSchema, int taskCount)
                 {
-                    InputPlugin input = newInputPlugin(task);
-                    try (SamplingPageOutput out = new SamplingPageOutput(task.getSampleRows(), schema)) {
-                        input.run(taskSource, schema, 0, out);
-                    }
+                    Filters.transaction(filterPlugins, task.getFilterConfigs(), inputSchema, new Filters.Control() {
+                        public void run(final List<TaskSource> filterTasks, final List<Schema> filterSchemas)
+                        {
+                            InputPlugin input = newInputPlugin(task);
+                            List<FilterPlugin> filterPlugins = newFilterPlugins(task);
+                            Schema filteredSchema = filterSchemas.get(filterSchemas.size() - 1);
+
+                            PageOutput out = new SamplingPageOutput(task.getSampleRows(), filteredSchema);
+                            try {
+                                out = Filters.open(filterPlugins, filterTasks, filterSchemas, out);
+                                input.run(inputTask, filteredSchema, 0, out);
+                            } finally {
+                                out.close();
+                            }
+                        }
+                    });
+                    // program never reaches here because SamplingPageOutput.finish throws an error.
                     throw new NoSampleException("No input records to preview");
                 }
             });
@@ -130,8 +158,11 @@ public class PreviewExecutor
         @Override
         public void close()
         {
-            for (Page page : pages) {
-                page.release();
+            if (pages != null) {
+                for (Page page : pages) {
+                    page.release();
+                }
+                pages = null;
             }
         }
     }