embulk 0.4.3 → 0.4.4

data/embulk-core/src/main/java/org/embulk/exec/SamplingParserPlugin.java

@@ -53,7 +53,7 @@ class SamplingParserPlugin
         final InputPlugin input = Exec.newPlugin(InputPlugin.class, samplingInputConfig.get(PluginType.class, "type"));
         try {
             input.transaction(samplingInputConfig, new InputPlugin.Control() {
-                public List<CommitReport> run(TaskSource taskSource, Schema schema, int processorCount)
+                public List<CommitReport> run(TaskSource taskSource, Schema schema, int taskCount)
                 {
                     input.run(taskSource, schema, 0, new PageOutput() {
                         @Override

data/embulk-core/src/main/java/org/embulk/exec/SetCurrentThreadName.java

@@ -0,0 +1,19 @@
+package org.embulk.exec;
+
+public class SetCurrentThreadName
+        implements AutoCloseable
+{
+    private final String original;
+
+    public SetCurrentThreadName(String name)
+    {
+        this.original = Thread.currentThread().getName();
+        Thread.currentThread().setName(name);
+    }
+
+    @Override
+    public void close()
+    {
+        Thread.currentThread().setName(original);
+    }
+}

data/embulk-core/src/main/java/org/embulk/spi/Exec.java

@@ -73,4 +73,9 @@ public class Exec
     {
         return session().newTaskSource();
     }
+
+    public boolean isPreview()
+    {
+        return session().isPreview();
+    }
 }

data/embulk-core/src/main/java/org/embulk/spi/ExecSession.java

@@ -29,6 +29,7 @@ public class ExecSession
     private final BufferAllocator bufferAllocator;
     private final Timestamp transactionTime;
     private final DateTimeZone transactionTimeZone;
+    private final boolean preview;
 
     public interface SessionTask
             extends Task
@@ -62,6 +63,24 @@ public class ExecSession
 
         this.transactionTime = task.getTransactionTime().or(Timestamp.ofEpochMilli(System.currentTimeMillis()));  // TODO get nanoseconds for default
         this.transactionTimeZone = task.getTransactionTimeZone();
+        this.preview = false;
+    }
+
+    private ExecSession(ExecSession copy, boolean preview)
+    {
+        this.injector = copy.injector;
+        this.loggerFactory = copy.loggerFactory;
+        this.modelManager = copy.modelManager;
+        this.pluginManager = copy.pluginManager;
+        this.bufferAllocator = copy.bufferAllocator;
+        this.transactionTime = copy.transactionTime;
+        this.transactionTimeZone = copy.transactionTimeZone;
+        this.preview = preview;
+    }
+
+    public ExecSession copyForPreview()
+    {
+        return new ExecSession(this, true);
     }
 
     public ConfigSource getSessionConfigSource()
@@ -133,4 +152,9 @@ public class ExecSession
         FormatterTask formatterTask = config.loadConfig(FormatterTask.class);
         return new TimestampFormatter(format, formatterTask);
     }
+
+    public boolean isPreview()
+    {
+        return preview;
+    }
 }

data/embulk-core/src/main/java/org/embulk/spi/FileInputPlugin.java

@@ -11,20 +11,20 @@ public interface FileInputPlugin
     public interface Control
     {
         public List<CommitReport> run(TaskSource taskSource,
-                int processorCount);
+                int taskCount);
     }
 
     public ConfigDiff transaction(ConfigSource config,
             FileInputPlugin.Control control);
 
     public ConfigDiff resume(TaskSource taskSource,
-            int processorCount,
+            int taskCount,
             FileInputPlugin.Control control);
 
     public void cleanup(TaskSource taskSource,
-            int processorCount,
+            int taskCount,
             List<CommitReport> successCommitReports);
 
     public TransactionalFileInput open(TaskSource taskSource,
-            int processorIndex);
+            int taskIndex);
 }

data/embulk-core/src/main/java/org/embulk/spi/FileInputRunner.java

@@ -61,11 +61,11 @@ public class FileInputRunner
     }
 
     public ConfigDiff resume(TaskSource taskSource,
-            Schema schema, int processorCount,
+            Schema schema, int taskCount,
             InputPlugin.Control control)
     {
         final RunnerTask task = taskSource.loadTask(RunnerTask.class);
-        return fileInputPlugin.resume(task.getFileInputTaskSource(), processorCount, new RunnerControl(task, control));
+        return fileInputPlugin.resume(task.getFileInputTaskSource(), taskCount, new RunnerControl(task, control));
     }
 
     private class RunnerControl
@@ -86,7 +86,7 @@ public class FileInputRunner
         }
 
         @Override
-        public List<CommitReport> run(final TaskSource fileInputTaskSource, final int processorCount)
+        public List<CommitReport> run(final TaskSource fileInputTaskSource, final int taskCount)
         {
             final List<CommitReport> commitReports = new ArrayList<CommitReport>();
             Decoders.transaction(decoderPlugins, task.getDecoderConfigs(), new Decoders.Control() {
@@ -98,7 +98,7 @@ public class FileInputRunner
                             task.setFileInputTaskSource(fileInputTaskSource);
                             task.setDecoderTaskSources(decoderTaskSources);
                             task.setParserTaskSource(parserTaskSource);
-                            commitReports.addAll(nextControl.run(task.dump(), schema, processorCount));
+                            commitReports.addAll(nextControl.run(task.dump(), schema, taskCount));
                         }
                     });
                 }
@@ -108,21 +108,21 @@ public class FileInputRunner
     }
 
     public void cleanup(TaskSource taskSource,
-            Schema schema, int processorCount,
+            Schema schema, int taskCount,
             List<CommitReport> successCommitReports)
     {
-        fileInputPlugin.cleanup(taskSource, processorCount, successCommitReports);
+        fileInputPlugin.cleanup(taskSource, taskCount, successCommitReports);
     }
 
     @Override
-    public CommitReport run(TaskSource taskSource, Schema schema, int processorIndex,
+    public CommitReport run(TaskSource taskSource, Schema schema, int taskIndex,
             PageOutput output)
     {
         final RunnerTask task = taskSource.loadTask(RunnerTask.class);
         List<DecoderPlugin> decoderPlugins = newDecoderPlugins(task);
         ParserPlugin parserPlugin = newParserPlugin(task);
 
-        TransactionalFileInput tran = fileInputPlugin.open(task.getFileInputTaskSource(), processorIndex);
+        TransactionalFileInput tran = fileInputPlugin.open(task.getFileInputTaskSource(), taskIndex);
         FileInput fileInput = tran;
         try {
             fileInput = Decoders.open(decoderPlugins, task.getDecoderTaskSources(), fileInput);

data/embulk-core/src/main/java/org/embulk/spi/FileOutputPlugin.java

@@ -13,16 +13,16 @@ public interface FileOutputPlugin
         public List<CommitReport> run(TaskSource taskSource);
     }
 
-    public ConfigDiff transaction(ConfigSource config, int processorCount,
+    public ConfigDiff transaction(ConfigSource config, int taskCount,
             FileOutputPlugin.Control control);
 
     public ConfigDiff resume(TaskSource taskSource,
-            int processorCount,
+            int taskCount,
             FileOutputPlugin.Control control);
 
     public void cleanup(TaskSource taskSource,
-            int processorCount,
+            int taskCount,
             List<CommitReport> successCommitReports);
 
-    public TransactionalFileOutput open(TaskSource taskSource, int processorIndex);
+    public TransactionalFileOutput open(TaskSource taskSource, int taskIndex);
 }

data/embulk-core/src/main/java/org/embulk/spi/FileOutputRunner.java

@@ -56,19 +56,19 @@ public class FileOutputRunner
 
     @Override
     public ConfigDiff transaction(ConfigSource config,
-            final Schema schema, final int processorCount,
+            final Schema schema, final int taskCount,
             final OutputPlugin.Control control)
     {
         final RunnerTask task = config.loadConfig(RunnerTask.class);
-        return fileOutputPlugin.transaction(config, processorCount, new RunnerControl(schema, task, control));
+        return fileOutputPlugin.transaction(config, taskCount, new RunnerControl(schema, task, control));
     }
 
     public ConfigDiff resume(TaskSource taskSource,
-            Schema schema, int processorCount,
+            Schema schema, int taskCount,
             final OutputPlugin.Control control)
     {
         final RunnerTask task = taskSource.loadTask(RunnerTask.class);
-        return fileOutputPlugin.resume(task.getFileOutputTaskSource(), processorCount, new RunnerControl(schema, task, control));
+        return fileOutputPlugin.resume(task.getFileOutputTaskSource(), taskCount, new RunnerControl(schema, task, control));
     }
 
     private class RunnerControl
@@ -113,14 +113,14 @@ public class FileOutputRunner
     }
 
     public void cleanup(TaskSource taskSource,
-            Schema schema, int processorCount,
+            Schema schema, int taskCount,
             List<CommitReport> successCommitReports)
     {
-        fileOutputPlugin.cleanup(taskSource, processorCount, successCommitReports);
+        fileOutputPlugin.cleanup(taskSource, taskCount, successCommitReports);
     }
 
     @Override
-    public TransactionalPageOutput open(TaskSource taskSource, Schema schema, int processorIndex)
+    public TransactionalPageOutput open(TaskSource taskSource, Schema schema, int taskIndex)
     {
         final RunnerTask task = taskSource.loadTask(RunnerTask.class);
         List<EncoderPlugin> encoderPlugins = newEncoderPlugins(task);
@@ -130,7 +130,7 @@ public class FileOutputRunner
         FileOutput fileOutput = null;
         PageOutput output = null;
         try {
-            fileOutput = tran = fileOutputPlugin.open(task.getFileOutputTaskSource(), processorIndex);
+            fileOutput = tran = fileOutputPlugin.open(task.getFileOutputTaskSource(), taskIndex);
 
             fileOutput = Encoders.open(encoderPlugins, task.getEncoderTaskSources(), fileOutput);
             output = formatterPlugin.open(task.getFormatterTaskSource(), schema, fileOutput);

data/embulk-core/src/main/java/org/embulk/spi/InputPlugin.java

@@ -11,21 +11,21 @@ public interface InputPlugin
     public interface Control
     {
         public List<CommitReport> run(TaskSource taskSource,
-                Schema schema, int processorCount);
+                Schema schema, int taskCount);
     }
 
     public ConfigDiff transaction(ConfigSource config,
             InputPlugin.Control control);
 
     public ConfigDiff resume(TaskSource taskSource,
-            Schema schema, int processorCount,
+            Schema schema, int taskCount,
             InputPlugin.Control control);
 
     public void cleanup(TaskSource taskSource,
-            Schema schema, int processorCount,
+            Schema schema, int taskCount,
             List<CommitReport> successCommitReports);
 
     public CommitReport run(TaskSource taskSource,
-            Schema schema, int processorIndex,
+            Schema schema, int taskIndex,
             PageOutput output);
 }

data/embulk-core/src/main/java/org/embulk/spi/OutputPlugin.java

@@ -14,16 +14,16 @@ public interface OutputPlugin
     }
 
     public ConfigDiff transaction(ConfigSource config,
-            Schema schema, int processorCount,
+            Schema schema, int taskCount,
             OutputPlugin.Control control);
 
     public ConfigDiff resume(TaskSource taskSource,
-            Schema schema, int processorCount,
+            Schema schema, int taskCount,
             OutputPlugin.Control control);
 
     public void cleanup(TaskSource taskSource,
-            Schema schema, int processorCount,
+            Schema schema, int taskCount,
             List<CommitReport> successCommitReports);
 
-    public TransactionalPageOutput open(TaskSource taskSource, Schema schema, int processorIndex);
+    public TransactionalPageOutput open(TaskSource taskSource, Schema schema, int taskIndex);
 }

data/embulk-core/src/main/java/org/embulk/spi/time/TimestampFormat.java

@@ -41,7 +41,7 @@ public class TimestampFormat
     public static DateTimeZone parseDateTimeZone(String s)
     {
         if(s.startsWith("+") || s.startsWith("-")) {
-            return DateTimeZone.forID("GMT"+s);
+            return DateTimeZone.forID(s);
 
         } else {
             try {

data/embulk-core/src/main/java/org/embulk/spi/util/FileOutputOutputStream.java

@@ -10,14 +10,23 @@ public class FileOutputOutputStream
 {
     private final FileOutput out;
     private final BufferAllocator allocator;
+    private final CloseMode closeMode;
     private int pos;
     private Buffer buffer;
 
-    public FileOutputOutputStream(FileOutput out, BufferAllocator allocator)
+    public static enum CloseMode {
+        FLUSH,
+        FLUSH_FINISH,
+        FLUSH_FINISH_CLOSE,
+        CLOSE;
+    }
+
+    public FileOutputOutputStream(FileOutput out, BufferAllocator allocator, CloseMode closeMode)
     {
         this.out = out;
         this.allocator = allocator;
         this.buffer = allocator.allocate();
+        this.closeMode = closeMode;
     }
 
     public void nextFile()
@@ -25,6 +34,11 @@ public class FileOutputOutputStream
         out.nextFile();
     }
 
+    public void finish()
+    {
+        out.finish();
+    }
+
     @Override
     public void write(int b)
     {
@@ -77,16 +91,26 @@ public class FileOutputOutputStream
         }
     }
 
-    public void finish()
-    {
-        doFlush();
-        out.finish();
-    }
-
     @Override
     public void close()
     {
-        out.close();
+        switch (closeMode) {
+        case FLUSH:
+            doFlush();
+            break;
+        case FLUSH_FINISH:
+            doFlush();
+            out.finish();
+            break;
+        case FLUSH_FINISH_CLOSE:
+            doFlush();
+            out.finish();
+            out.close();
+            break;
+        case CLOSE:
+            out.close();
+            break;
+        }
         buffer.release();
         buffer = Buffer.EMPTY;
         pos = 0;

data/embulk-core/src/main/java/org/embulk/spi/util/InputStreamFileInput.java

@@ -64,7 +64,7 @@ public class InputStreamFileInput
     {
         // TODO check current != null and throw Illegal State - file is not opened
         if (current == null) {
-            throw new IllegalStateException("openNext must be called before poll()");
+            throw new IllegalStateException("nextFile() must be called before poll()");
         }
         Buffer buffer = allocator.allocate();
         try {

data/embulk-core/src/main/java/org/embulk/spi/util/LineEncoder.java

@@ -44,7 +44,7 @@ public class LineEncoder
             .onMalformedInput(CodingErrorAction.REPLACE)  // TODO configurable?
             .onUnmappableCharacter(CodingErrorAction.REPLACE);  // TODO configurable?
         this.newline = task.getNewline().getString();
-        this.outputStream = new FileOutputOutputStream(out, task.getBufferAllocator());
+        this.outputStream = new FileOutputOutputStream(out, task.getBufferAllocator(), FileOutputOutputStream.CloseMode.CLOSE);
         this.writer = new OutputStreamWriter(outputStream, encoder);
     }
 
@@ -93,12 +93,11 @@ public class LineEncoder
     public void finish()
     {
         try {
-            writer.flush();   // flush all remaining buffer in writer because FileOutputOutputStream.close() doesn't flush buffer
-            outputStream.finish();
-            writer.close();
+            writer.flush();
         } catch (IOException ex) {
             throw new RuntimeException(ex);
         }
+        outputStream.finish();
     }
 
     @Override

data/embulk-core/src/main/java/org/embulk/spi/util/OutputStreamFileOutput.java

@@ -0,0 +1,88 @@
+package org.embulk.spi.util;
+
+import java.io.OutputStream;
+import java.io.Closeable;
+import java.io.IOException;
+import org.embulk.spi.Buffer;
+import org.embulk.spi.FileOutput;
+
+public class OutputStreamFileOutput
+        implements FileOutput
+{
+    public interface Provider extends Closeable
+    {
+        public OutputStream openNext() throws IOException;
+
+        public void finish() throws IOException;
+
+        public void close() throws IOException;
+    }
+
+    private final Provider provider;
+    private OutputStream current;
+
+    public OutputStreamFileOutput(Provider provider)
+    {
+        this.provider = provider;
+        this.current = null;
+    }
+
+    public void nextFile()
+    {
+        closeCurrent();
+        try {
+            current = provider.openNext();
+        } catch (IOException ex) {
+            throw new RuntimeException(ex);
+        }
+    }
+
+    public void add(Buffer buffer)
+    {
+        if (current == null) {
+            throw new IllegalStateException("nextFile() must be called before poll()");
+        }
+        try {
+            current.write(buffer.array(), buffer.offset(), buffer.limit());
+        } catch (IOException ex) {
+            throw new RuntimeException(ex);
+        } finally {
+            buffer.release();
+        }
+    }
+
+    public void finish()
+    {
+        closeCurrent();
+        try {
+            provider.finish();
+        } catch (IOException ex) {
+            throw new RuntimeException(ex);
+        }
+    }
+
+    public void close()
+    {
+        try {
+            closeCurrent();
+        } finally {
+            try {
+                provider.close();
+            } catch (IOException ex) {
+                throw new RuntimeException(ex);
+            }
+        }
+    }
+
+    private void closeCurrent()
+    {
+        try {
+            if (current != null) {
+                current.close();
+                current = null;
+            }
+        } catch (IOException ex) {
+            throw new RuntimeException(ex);
+        }
+    }
+}