embulk 0.4.3 → 0.4.4

data/embulk-core/src/test/java/org/embulk/spi/TestFileInputInputStream.java

@@ -25,7 +25,7 @@ public class TestFileInputInputStream
     private void newOutputStream()
     {
         fileOutput = new MockFileOutput();
-        out = new FileOutputOutputStream(fileOutput, runtime.getBufferAllocator());
+        out = new FileOutputOutputStream(fileOutput, runtime.getBufferAllocator(), FileOutputOutputStream.CloseMode.CLOSE);
     }
 
     private void newInputStream()

data/embulk-core/src/test/java/org/embulk/spi/TestFileInputRunner.java

@@ -53,7 +53,7 @@ public class TestFileInputRunner
 
         @Override
         public ConfigDiff resume(TaskSource taskSource,
-                int processorCount,
+                int taskCount,
                 FileInputPlugin.Control control)
         {
             throw new UnsupportedOperationException();
@@ -61,13 +61,13 @@ public class TestFileInputRunner
 
         @Override
         public void cleanup(TaskSource taskSource,
-                int processorCount,
+                int taskCount,
                 List<CommitReport> successCommitReports)
         {
         }
 
         public TransactionalFileInput open(TaskSource taskSource,
-                int processorIndex)
+                int taskIndex)
         {
             return new TransactionalFileInput()
             {
@@ -127,7 +127,7 @@ public class TestFileInputRunner
         runner.transaction(config, new InputPlugin.Control()
         {
             public List<CommitReport> run(TaskSource inputTaskSource,
-                    Schema schema, int processorCount)
+                    Schema schema, int taskCount)
             {
                 List<CommitReport> reports = new ArrayList<>();
                 reports.add(runner.run(inputTaskSource, schema, 0, output));
@@ -181,7 +181,7 @@ public class TestFileInputRunner
             runner.transaction(config, new InputPlugin.Control()
             {
                 public List<CommitReport> run(TaskSource inputTaskSource,
-                        Schema schema, int processorCount)
+                        Schema schema, int taskCount)
                 {
                     List<CommitReport> reports = new ArrayList<>();
                     reports.add(runner.run(inputTaskSource, schema, 0, output));

data/embulk-core/src/test/java/org/embulk/spi/TestFileOutputRunner.java

@@ -33,7 +33,7 @@ public class TestFileOutputRunner
         Boolean transactionCompleted = null;
 
         @Override
-        public ConfigDiff transaction(ConfigSource config, int processorCount,
+        public ConfigDiff transaction(ConfigSource config, int taskCount,
                 FileOutputPlugin.Control control)
         {
             PluginTask task = config.loadConfig(PluginTask.class);
@@ -43,7 +43,7 @@ public class TestFileOutputRunner
 
         @Override
         public ConfigDiff resume(TaskSource taskSource,
-                int processorCount,
+                int taskCount,
                 FileOutputPlugin.Control control)
         {
             throw new UnsupportedOperationException();
@@ -51,14 +51,14 @@ public class TestFileOutputRunner
 
         @Override
         public void cleanup(TaskSource taskSource,
-                int processorCount,
+                int taskCount,
                 List<CommitReport> successCommitReports)
         {
         }
 
         @Override
         public TransactionalFileOutput open(TaskSource taskSource,
-                final int processorIndex)
+                final int taskIndex)
         {
             return new TransactionalFileOutput()
             {

data/embulk-docs/src/release.rst

@@ -14,4 +14,5 @@ Release Notes
     release/release-0.4.1
     release/release-0.4.2
     release/release-0.4.3
+    release/release-0.4.4
 

data/embulk-docs/src/release/release-0.4.4.rst

@@ -0,0 +1,39 @@
+Release 0.4.4
+==================================
+
+CLI
+------------------
+
+* Supports ``-l, --log-level LEVEL`` option to specify log level. (@ykubota++)
+
+Plugin API Changes
+------------------
+
+* Added ``Exec.isPreview()`` method which returns true if the transaction is running in in preview mode.
+* Added ``util.OutputStreamFileOutput`` class.
+* ``util.FileOutputOutputStream`` requires ``CloseMode`` at the constructor.
+* When ``Embulk::DataSource#prop`` returns a String, Array, or Hash value, it returns a copy of the value.
+* When ``Embulk::DataSource#prop`` returns a Hash value, it returns ``DataSource`` instance.
+
+Plugin SPI
+------------------
+
+* Changed local variable name ``processorIndex`` to ``taskIndex``.
+* Changed local variable name ``processorCount`` to ``taskCount``.
+
+Built-in plugins
+------------------
+
+* Added ``gzip`` encoder plugin.
+
+General Changes
+------------------
+
+* ``preview`` runs filter plugins in addition to input plugins.
+* Fixed a problem of ``guess`` where guess plugins ignore formatter properties (such as ``charset``) guessed by other guess plugins (@hiroyuki-sato++).
+* Log messages include "transaction", "resume", "cleanup", or task index at the header.
+* Timestamp in log messages include time zone.
+
+Release Date
+------------------
+2015-02-18

data/embulk-standards/src/main/java/org/embulk/standards/GzipFileEncoderPlugin.java

@@ -5,12 +5,15 @@ import java.io.IOException;
 import java.util.zip.GZIPOutputStream;
 import org.embulk.config.Task;
 import org.embulk.config.Config;
+import org.embulk.config.ConfigInject;
 import org.embulk.config.ConfigDefault;
 import org.embulk.config.TaskSource;
 import org.embulk.config.ConfigSource;
 import org.embulk.spi.EncoderPlugin;
 import org.embulk.spi.FileOutput;
+import org.embulk.spi.BufferAllocator;
 import org.embulk.spi.util.FileOutputOutputStream;
+import org.embulk.spi.util.OutputStreamFileOutput;
 
 public class GzipFileEncoderPlugin
         implements EncoderPlugin
@@ -18,9 +21,13 @@ public class GzipFileEncoderPlugin
     public interface PluginTask
             extends Task
     {
-        @Config("level")
-        @ConfigDefault("6")
-        public int getLevel();
+        // TODO java.util.zip.GZIPOutputStream doesn't support compression level
+        //@Config("level")
+        //@ConfigDefault("6")
+        //public int getLevel();
+
+        @ConfigInject
+        public BufferAllocator getBufferAllocator();
     }
 
     public void transaction(ConfigSource config, EncoderPlugin.Control control)
@@ -30,10 +37,28 @@ public class GzipFileEncoderPlugin
     }
 
     @Override
-    public FileOutput open(TaskSource taskSource, FileOutput fileOutput)
+    public FileOutput open(TaskSource taskSource, final FileOutput fileOutput)
     {
-        throw new AssertionError("OutputStreamFileOutput is not implemented yet");
-        // TODO GZIPOutputStream doesn't support level option?
-        //return new OutputStreamFileOutput(new GZIPOutputStream(new FileOutputOutputStream(fileOutput)));
+        PluginTask task = taskSource.loadTask(PluginTask.class);
+
+        final FileOutputOutputStream output = new FileOutputOutputStream(fileOutput, task.getBufferAllocator(), FileOutputOutputStream.CloseMode.CLOSE);
+
+        return new OutputStreamFileOutput(new OutputStreamFileOutput.Provider() {
+            public OutputStream openNext() throws IOException
+            {
+                output.nextFile();
+                return new GZIPOutputStream(output);
+            }
+
+            public void finish() throws IOException
+            {
+                fileOutput.finish();
+            }
+
+            public void close() throws IOException
+            {
+                fileOutput.close();
+            }
+        });
     }
 }

data/embulk-standards/src/main/java/org/embulk/standards/LocalFileInputPlugin.java

@@ -64,22 +64,22 @@ public class LocalFileInputPlugin
         task.setFiles(files);
 
         // number of processors is same with number of files
-        int processorCount = task.getFiles().size();
-        return resume(task.dump(), processorCount, control);
+        int taskCount = task.getFiles().size();
+        return resume(task.dump(), taskCount, control);
     }
 
     @Override
     public ConfigDiff resume(TaskSource taskSource,
-            int processorCount,
+            int taskCount,
             FileInputPlugin.Control control)
     {
-        control.run(taskSource, processorCount);
+        control.run(taskSource, taskCount);
         return Exec.newConfigDiff();
     }
 
     @Override
     public void cleanup(TaskSource taskSource,
-            int processorCount,
+            int taskCount,
             List<CommitReport> successCommitReports)
     { }
 
@@ -155,10 +155,10 @@ public class LocalFileInputPlugin
     }
 
     @Override
-    public TransactionalFileInput open(TaskSource taskSource, int processorIndex)
+    public TransactionalFileInput open(TaskSource taskSource, int taskIndex)
     {
         PluginTask task = taskSource.loadTask(PluginTask.class);
-        return new LocalFileInput(task, processorIndex);
+        return new LocalFileInput(task, taskIndex);
     }
 
     public static class LocalFileInput
@@ -191,9 +191,9 @@ public class LocalFileInputPlugin
             public void close() { }
         }
 
-        public LocalFileInput(PluginTask task, int processorIndex)
+        public LocalFileInput(PluginTask task, int taskIndex)
         {
-            super(task.getBufferAllocator(), new SingleFileProvider(new File(task.getFiles().get(processorIndex))));
+            super(task.getBufferAllocator(), new SingleFileProvider(new File(task.getFiles().get(taskIndex))));
         }
 
         @Override

data/embulk-standards/src/main/java/org/embulk/standards/LocalFileOutputPlugin.java

@@ -40,7 +40,7 @@ public class LocalFileOutputPlugin
     private final Logger log = Exec.getLogger(getClass());
 
     @Override
-    public ConfigDiff transaction(ConfigSource config, int processorCount,
+    public ConfigDiff transaction(ConfigSource config, int taskCount,
             FileOutputPlugin.Control control)
     {
         PluginTask task = config.loadConfig(PluginTask.class);
@@ -48,12 +48,12 @@ public class LocalFileOutputPlugin
         // validate sequence_format
         String.format(task.getSequenceFormat(), 0, 0);
 
-        return resume(task.dump(), processorCount, control);
+        return resume(task.dump(), taskCount, control);
     }
 
     @Override
     public ConfigDiff resume(TaskSource taskSource,
-            int processorCount,
+            int taskCount,
             FileOutputPlugin.Control control)
     {
         control.run(taskSource);
@@ -62,12 +62,12 @@ public class LocalFileOutputPlugin
 
     @Override
     public void cleanup(TaskSource taskSource,
-            int processorCount,
+            int taskCount,
             List<CommitReport> successCommitReports)
     { }
 
     @Override
-    public TransactionalFileOutput open(TaskSource taskSource, final int processorIndex)
+    public TransactionalFileOutput open(TaskSource taskSource, final int taskIndex)
     {
         PluginTask task = taskSource.loadTask(PluginTask.class);
 
@@ -84,7 +84,7 @@ public class LocalFileOutputPlugin
             public void nextFile()
             {
                 closeFile();
-                String path = pathPrefix + String.format(sequenceFormat, processorIndex, fileIndex) + pathSuffix;
+                String path = pathPrefix + String.format(sequenceFormat, taskIndex, fileIndex) + pathSuffix;
                 log.info("Writing local file '{}'", path);
                 fileNames.add(path);
                 try {

data/embulk-standards/src/main/java/org/embulk/standards/NullOutputPlugin.java

@@ -16,14 +16,14 @@ public class NullOutputPlugin
 {
     @Override
     public ConfigDiff transaction(ConfigSource config,
-            Schema schema, int processorCount,
+            Schema schema, int taskCount,
             OutputPlugin.Control control)
     {
-        return resume(Exec.newTaskSource(), schema, processorCount, control);
+        return resume(Exec.newTaskSource(), schema, taskCount, control);
     }
 
     public ConfigDiff resume(TaskSource taskSource,
-            Schema schema, int processorCount,
+            Schema schema, int taskCount,
             OutputPlugin.Control control)
     {
         control.run(taskSource);
@@ -31,12 +31,12 @@ public class NullOutputPlugin
     }
 
     public void cleanup(TaskSource taskSource,
-            Schema schema, int processorCount,
+            Schema schema, int taskCount,
             List<CommitReport> successCommitReports)
     { }
 
     @Override
-    public TransactionalPageOutput open(TaskSource taskSource, Schema schema, int processorIndex)
+    public TransactionalPageOutput open(TaskSource taskSource, Schema schema, int taskIndex)
     {
         return new TransactionalPageOutput() {
             public void add(Page page)

data/embulk-standards/src/main/java/org/embulk/standards/StdoutOutputPlugin.java

@@ -27,16 +27,16 @@ public class StdoutOutputPlugin
 
     @Override
     public ConfigDiff transaction(ConfigSource config,
-            Schema schema, int processorCount,
+            Schema schema, int taskCount,
             OutputPlugin.Control control)
     {
         final PluginTask task = config.loadConfig(PluginTask.class);
-        return resume(task.dump(), schema, processorCount, control);
+        return resume(task.dump(), schema, taskCount, control);
     }
 
     @Override
     public ConfigDiff resume(TaskSource taskSource,
-            Schema schema, int processorCount,
+            Schema schema, int taskCount,
             OutputPlugin.Control control)
     {
         control.run(taskSource);
@@ -44,13 +44,13 @@ public class StdoutOutputPlugin
     }
 
     public void cleanup(TaskSource taskSource,
-            Schema schema, int processorCount,
+            Schema schema, int taskCount,
             List<CommitReport> successCommitReports)
     { }
 
     @Override
     public TransactionalPageOutput open(TaskSource taskSource, final Schema schema,
-            int processorIndex)
+            int taskIndex)
     {
         final PluginTask task = taskSource.loadTask(PluginTask.class);
 

data/lib/embulk/command/embulk_run.rb

@@ -32,7 +32,7 @@ module Embulk
     require 'optparse'
     op = OptionParser.new
 
-    puts "#{Time.now.strftime("%Y-%m-%d %H:%M:%S %Z")}: Embulk v#{Embulk::VERSION}"
+    puts "#{Time.now.strftime("%Y-%m-%d %H:%M:%S,%3N %z")}: Embulk v#{Embulk::VERSION}"
 
     load_paths = []
     classpaths = []
@@ -56,6 +56,9 @@ module Embulk
 
     when :run
       op.banner = "Usage: run <config.yml>"
+      op.on('-l', '--log-level LEVEL', 'Log level (fatal, error, warn, info, or trace)') do |level|
+        options[:logLevel] = level
+      end
       op.on('-I', '--load-path PATH', 'Add ruby script directory path ($LOAD_PATH)') do |load_path|
         load_paths << load_path
       end
@@ -71,7 +74,10 @@ module Embulk
       args = 1..1
 
     when :cleanup
-      op.banner = "Usage: run <config.yml>"
+      op.banner = "Usage: cleanup <config.yml>"
+      op.on('-l', '--log-level LEVEL', 'Log level (fatal, error, warn, info, or trace)') do |level|
+        options[:logLevel] = level
+      end
       op.on('-I', '--load-path PATH', 'Add ruby script directory path ($LOAD_PATH)') do |load_path|
         load_paths << load_path
       end
@@ -85,6 +91,9 @@ module Embulk
 
     when :preview
       op.banner = "Usage: preview <config.yml>"
+      op.on('-l', '--log-level LEVEL', 'Log level (fatal, error, warn, info, or trace)') do |level|
+        options[:logLevel] = level
+      end
       op.on('-I', '--load-path PATH', 'Add ruby script directory path ($LOAD_PATH)') do |load_path|
         load_paths << load_path
       end
@@ -95,6 +104,9 @@ module Embulk
 
     when :guess
       op.banner = "Usage: guess <partial-config.yml>"
+      op.on('-l', '--log-level LEVEL', 'Log level (fatal, error, warn, info, or trace)') do |level|
+        options[:logLevel] = level
+      end
       op.on('-o', '--output PATH', 'Path to a file to write the guessed configuration') do |path|
         options[:nextConfigOutputPath] = path
       end

data/lib/embulk/data/new/java/file_input.java.erb

@@ -31,30 +31,30 @@ public class <%= java_class_name %>
     {
         PluginTask task = config.loadConfig(PluginTask.class);
 
-        // processorCount is usually number of input files
-        int processorCount = 1;  // number of run() method calls
+        // taskCount is usually number of input files
+        int taskCount = 1;  // number of run() method calls
 
-        return resume(task.dump(), processorCount, control);
+        return resume(task.dump(), taskCount, control);
     }
 
     @Override
     public ConfigDiff resume(TaskSource taskSource,
-            int processorCount,
+            int taskCount,
             FileInputPlugin.Control control)
     {
-        control.run(taskSource, processorCount);
+        control.run(taskSource, taskCount);
         return Exec.newConfigDiff();
     }
 
     @Override
     public void cleanup(TaskSource taskSource,
-            int processorCount,
+            int taskCount,
             List<CommitReport> successCommitReports)
     {
     }
 
     @Override
-    public TransactionalFileInput open(TaskSource taskSource, int processorIndex)
+    public TransactionalFileInput open(TaskSource taskSource, int taskIndex)
     {
         PluginTask task = taskSource.loadTask(PluginTask.class);