embulk 0.7.9 → 0.7.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2a344520dc3aded72b2bf53a735a4a46f6d8668a
-  data.tar.gz: 6211c301cce4f7d7a4beb9e7a011d6b7971965e2
+  metadata.gz: f28491871fb762807b53e4eb27d97b15800be9f9
+  data.tar.gz: 6e38eb058c63710bd9e43a5bd2c0218a8b797bb1
 SHA512:
-  metadata.gz: 12dca07d9d57ae0f6d36466c0675a66e2755f09cc25d252eb5bcb14a86cfa6fd27110782f9ce1af0b0b31620da45a3666aae90c5e14d3d16f91ddab5227f805b
-  data.tar.gz: 022627716d3db31dee527a5848f0d66766593e9c4a65dadd7c093cbc50bbbfdc81a34aa5aef6b3ca7a829db1f0a4c817c71b8d0beb26f028f0e7efdd65318fcc
+  metadata.gz: 5e2d90085bb8fddf565f4a936ec6ddced865fd3f71ea566bcc347904962564a67af243a41dff8d5875447a329f9f99a8e64f771e3dd7dc5d68ee86a7798fed9a
+  data.tar.gz: 2b6b729becf7333c1af3ae1fc44cfbb643709382aaf59369c53f37a2f2326f99fae17a48fc9e5b23fb5545f101e38f8e3b17e3aa0eaecd703f63f167a1e8536b

data/README.md

@@ -135,6 +135,8 @@ Following command updates embulk itself to the specific released version.
 embulk selfupdate x.y.z
 ```
 
+Older versions are available at [dl.embulk.org](http://dl.embulk.org).
+
 
 ## Embulk Development
 

data/build.gradle

@@ -16,7 +16,7 @@ def release_projects = [project(":embulk-core"), project(":embulk-standards")]
 
 allprojects {
     group = 'org.embulk'
-    version = '0.7.9'
+    version = '0.7.10'
 
     ext {
         jrubyVersion = '9.0.0.0'

data/embulk-core/src/main/java/org/embulk/EmbulkEmbed.java

@@ -21,6 +21,7 @@ import org.embulk.exec.PreviewResult;
 import org.embulk.exec.ExecutionResult;
 import org.embulk.exec.PartialExecutionException;
 import org.embulk.exec.ResumeState;
+import org.embulk.exec.TransactionStage;
 import org.embulk.spi.BufferAllocator;
 import org.embulk.spi.ExecSession;
 import org.embulk.guice.Bootstrap;
@@ -273,6 +274,11 @@ public class EmbulkEmbed
             checkState(partialExecutionException != null);
             return partialExecutionException.getResumeState();
         }
+
+        public TransactionStage getTransactionStage()
+        {
+            return partialExecutionException.getTransactionStage();
+        }
     }
 
     public class ResumeStateAction

data/embulk-core/src/main/java/org/embulk/exec/BulkLoader.java

@@ -74,6 +74,7 @@ public class BulkLoader
         private volatile List<TaskSource> filterTaskSources;
         private volatile List<Schema> schemas;
         private volatile Schema executorSchema;
+        private volatile TransactionStage transactionStage;
 
         private volatile ConfigDiff inputConfigDiff;
         private volatile ConfigDiff outputConfigDiff;
@@ -102,6 +103,11 @@ public class BulkLoader
             this.executorSchema = executorSchema;
         }
 
+        public void setTransactionStage(TransactionStage transactionStage)
+        {
+            this.transactionStage = transactionStage;
+        }
+
         public void setInputTaskSource(TaskSource inputTaskSource)
         {
             this.inputTaskSource = inputTaskSource;
@@ -220,20 +226,6 @@ public class BulkLoader
             return inputConfigDiff != null && outputConfigDiff != null;
         }
 
-        public boolean isAnyStarted()
-        {
-            if (inputTaskStates == null) {
-                // not initialized
-                return false;
-            }
-            for (TaskState inputTaskState : inputTaskStates) {
-                if (inputTaskState.isStarted()) {
-                    return true;
-                }
-            }
-            return false;
-        }
-
         public void setOutputConfigDiff(ConfigDiff outputConfigDiff)
         {
             if (outputConfigDiff == null) {
@@ -356,16 +348,19 @@ public class BulkLoader
 
         public ResumeState buildResumeState(ExecSession exec)
         {
+            Schema inputSchema = (schemas == null) ? null : schemas.get(0);
+            List<Optional<TaskReport>> inputTaskReports = (inputTaskStates == null) ? null : getInputTaskReports();
+            List<Optional<TaskReport>> outputTaskReports = (outputTaskStates == null) ? null : getOutputTaskReports();
             return new ResumeState(
                     exec.getSessionExecConfig(),
                     inputTaskSource, outputTaskSource,
-                    first(schemas), executorSchema,
-                    getInputTaskReports(), getOutputTaskReports());
+                    inputSchema, executorSchema,
+                    inputTaskReports, outputTaskReports);
         }
 
         public PartialExecutionException buildPartialExecuteException(Throwable cause, ExecSession exec)
         {
-            return new PartialExecutionException(cause, buildResumeState(exec));
+            return new PartialExecutionException(cause, buildResumeState(exec), transactionStage);
         }
     }
 
@@ -513,26 +508,30 @@ public class BulkLoader
         final ProcessPluginSet plugins = new ProcessPluginSet(task);
 
         final LoaderState state = new LoaderState(Exec.getLogger(BulkLoader.class), plugins);
+        state.setTransactionStage(TransactionStage.INPUT_BEGIN);
         try {
             ConfigDiff inputConfigDiff = plugins.getInputPlugin().transaction(task.getInputConfig(), new InputPlugin.Control() {
                 public List<TaskReport> run(final TaskSource inputTask, final Schema inputSchema, final int inputTaskCount)
                 {
                     state.setInputTaskSource(inputTask);
+                    state.setTransactionStage(TransactionStage.FILTER_BEGIN);
                     Filters.transaction(plugins.getFilterPlugins(), task.getFilterConfigs(), inputSchema, new Filters.Control() {
                         public void run(final List<TaskSource> filterTasks, final List<Schema> schemas)
                         {
                             state.setSchemas(schemas);
                             state.setFilterTaskSources(filterTasks);
+                            state.setTransactionStage(TransactionStage.EXECUTOR_BEGIN);
                             exec.transaction(task.getExecConfig(), last(schemas), inputTaskCount, new ExecutorPlugin.Control() {
                                 public void transaction(final Schema executorSchema, final int outputTaskCount, final ExecutorPlugin.Executor executor)
                                 {
                                     state.setExecutorSchema(executorSchema);
+                                    state.setTransactionStage(TransactionStage.OUTPUT_BEGIN);
                                     ConfigDiff outputConfigDiff = plugins.getOutputPlugin().transaction(task.getOutputConfig(), executorSchema, outputTaskCount, new OutputPlugin.Control() {
                                         public List<TaskReport> run(final TaskSource outputTask)
                                         {
                                             state.setOutputTaskSource(outputTask);
-
                                             state.initialize(inputTaskCount, outputTaskCount);
+                                            state.setTransactionStage(TransactionStage.RUN);
 
                                             if (!state.isAllTasksCommitted()) {  // inputTaskCount == 0
                                                 execute(task, executor, state);
@@ -543,18 +542,23 @@ public class BulkLoader
                                                             state.countUncommittedInputTasks(), state.countUncommittedOutputTasks()));
                                             }
 
+                                            state.setTransactionStage(TransactionStage.OUTPUT_COMMIT);
                                             return state.getAllOutputTaskReports();
                                         }
                                     });
                                     state.setOutputConfigDiff(outputConfigDiff);
+                                    state.setTransactionStage(TransactionStage.EXECUTOR_COMMIT);
                                 }
                             });
+                            state.setTransactionStage(TransactionStage.FILTER_COMMIT);
                         }
                     });
+                    state.setTransactionStage(TransactionStage.INPUT_COMMIT);
                     return state.getAllInputTaskReports();
                 }
             });
             state.setInputConfigDiff(inputConfigDiff);
+            state.setTransactionStage(TransactionStage.CLEANUP);
 
             cleanupCommittedTransaction(config, state);
 
@@ -565,9 +569,6 @@ public class BulkLoader
                 // ignore the exception
                 return state.buildExecuteResultWithWarningException(ex);
             }
-            if (!state.isAnyStarted()) {
-                throw ex;
-            }
             throw state.buildPartialExecuteException(ex, Exec.session());
         }
     }
@@ -580,6 +581,7 @@ public class BulkLoader
         final ProcessPluginSet plugins = new ProcessPluginSet(task);
 
         final LoaderState state = new LoaderState(Exec.getLogger(BulkLoader.class), plugins);
+        state.setTransactionStage(TransactionStage.INPUT_BEGIN);
         try {
             ConfigDiff inputConfigDiff = plugins.getInputPlugin().resume(resume.getInputTaskSource(), resume.getInputSchema(), resume.getInputTaskReports().size(), new InputPlugin.Control() {
                 public List<TaskReport> run(final TaskSource inputTask, final Schema inputSchema, final int inputTaskCount)
@@ -587,23 +589,27 @@ public class BulkLoader
                     // TODO validate inputTask?
                     // TODO validate inputSchema
                     state.setInputTaskSource(inputTask);
+                    state.setTransactionStage(TransactionStage.FILTER_BEGIN);
                     Filters.transaction(plugins.getFilterPlugins(), task.getFilterConfigs(), inputSchema, new Filters.Control() {
                         public void run(final List<TaskSource> filterTasks, final List<Schema> schemas)
                         {
                             state.setSchemas(schemas);
                             state.setFilterTaskSources(filterTasks);
+                            state.setTransactionStage(TransactionStage.EXECUTOR_BEGIN);
                             exec.transaction(task.getExecConfig(), last(schemas), inputTaskCount, new ExecutorPlugin.Control() {
                                 public void transaction(final Schema executorSchema, final int outputTaskCount, final ExecutorPlugin.Executor executor)
                                 {
                                     // TODO validate executorSchema
                                     state.setExecutorSchema(executorSchema);
+                                    state.setTransactionStage(TransactionStage.OUTPUT_BEGIN);
                                     ConfigDiff outputConfigDiff = plugins.getOutputPlugin().resume(resume.getOutputTaskSource(), executorSchema, outputTaskCount, new OutputPlugin.Control() {
                                         public List<TaskReport> run(final TaskSource outputTask)
                                         {
                                             // TODO validate outputTask?
                                             state.setOutputTaskSource(outputTask);
-
                                             restoreResumedTaskReports(resume, state);
+                                            state.setTransactionStage(TransactionStage.RUN);
+
                                             if (!state.isAllTasksCommitted()) {
                                                 execute(task, executor, state);
                                             }
@@ -613,18 +619,23 @@ public class BulkLoader
                                                             state.countUncommittedInputTasks(), state.countUncommittedOutputTasks()));
                                             }
 
+                                            state.setTransactionStage(TransactionStage.OUTPUT_COMMIT);
                                             return state.getAllOutputTaskReports();
                                         }
                                     });
                                     state.setOutputConfigDiff(outputConfigDiff);
+                                    state.setTransactionStage(TransactionStage.EXECUTOR_COMMIT);
                                 }
                             });
+                            state.setTransactionStage(TransactionStage.FILTER_COMMIT);
                         }
                     });
+                    state.setTransactionStage(TransactionStage.INPUT_COMMIT);
                     return state.getAllInputTaskReports();
                 }
             });
             state.setInputConfigDiff(inputConfigDiff);
+            state.setTransactionStage(TransactionStage.CLEANUP);
 
             cleanupCommittedTransaction(config, state);
 
@@ -635,9 +646,6 @@ public class BulkLoader
                 // ignore the exception
                 return state.buildExecuteResultWithWarningException(ex);
             }
-            if (!state.isAnyStarted()) {
-                throw ex;
-            }
             throw state.buildPartialExecuteException(ex, Exec.session());
         }
     }

data/embulk-core/src/main/java/org/embulk/exec/GuessExecutor.java

@@ -31,6 +31,7 @@ import org.embulk.spi.FileInput;
 import org.embulk.spi.PageOutput;
 import org.embulk.spi.TransactionalFileInput;
 import org.embulk.spi.FileInputRunner;
+import static org.embulk.spi.util.Inputs.each;
 
 public class GuessExecutor
 {
@@ -294,7 +295,7 @@ public class GuessExecutor
             final ConfigSource originalConfig = task.getOriginalConfig();
 
             // get sample buffer
-            Buffer sample = getFirstBuffer(input);
+            Buffer sample = readSample(input, 32*1024);  // TODO get sample size from system config. See also SamplingParserPlugin().
 
             // load guess plugins
             ImmutableList.Builder<GuessPlugin> builder = ImmutableList.builder();
@@ -320,24 +321,18 @@ public class GuessExecutor
             throw new GuessedNoticeError(mergedGuessed);
         }
 
-        private static Buffer getFirstBuffer(FileInput input)
+        private static Buffer readSample(FileInput fileInput, int sampleSize)
         {
-            // The first buffer is created by SamplingParserPlugin. See FileInputRunner.guess.
-            RuntimeException decodeException = null;
+            Buffer sample = Buffer.allocate(sampleSize);
             try {
-                while (input.nextFile()) {
-                    Buffer sample = input.poll();
-                    if (sample != null) {
-                        return sample;
-                    }
-                }
+                SamplingParserPlugin.readSample(fileInput, sample, 0, sampleSize);
             } catch (RuntimeException ex) {
                 // ignores exceptions because FileDecoderPlugin can throw exceptions
-                // such as "Unexpected end of ZLIB input stream"
-                decodeException = ex;
+                // such as "Unexpected end of ZLIB input stream" if decoder plugin
+                // is wrongly guessed.
             }
-            if (decodeException != null) {
-                throw decodeException;
+            if (sample.limit() > 0) {
+                return sample;
             }
             throw new NoSampleException("No input buffer to guess");
         }

data/embulk-core/src/main/java/org/embulk/exec/PartialExecutionException.java

@@ -4,15 +4,23 @@ public class PartialExecutionException
         extends RuntimeException
 {
     private final ResumeState resumeState;
+    private final TransactionStage transactionStage;
 
-    public PartialExecutionException(Throwable cause, ResumeState resumeState)
+    public PartialExecutionException(Throwable cause, ResumeState resumeState,
+            TransactionStage transactionStage)
     {
         super(cause);
         this.resumeState = resumeState;
+        this.transactionStage = transactionStage;
     }
 
     public ResumeState getResumeState()
     {
         return resumeState;
     }
+
+    public TransactionStage getTransactionStage()
+    {
+        return transactionStage;
+    }
 }

data/embulk-core/src/main/java/org/embulk/exec/SamplingParserPlugin.java

@@ -90,7 +90,7 @@ public class SamplingParserPlugin
     public SamplingParserPlugin(@ForSystemConfig ConfigSource systemConfig)
     {
         this.minSampleSize = 40;  // empty gzip file is 33 bytes. // TODO get sample size from system config
-        this.sampleSize = 32*1024;  // TODO get sample size from system config
+        this.sampleSize = 32*1024;  // TODO get sample size from system config. See also GuessExecutor.run.
         Preconditions.checkArgument(minSampleSize < sampleSize, "minSampleSize must be smaller than sampleSize");
     }
 
@@ -111,26 +111,32 @@ public class SamplingParserPlugin
         throw new SampledNoticeError(buffer);
     }
 
-    private static Buffer readSample(FileInput fileInput, int sampleSize)
+    public static Buffer readSample(FileInput fileInput, int sampleSize)
+    {
+        return readSample(fileInput, Buffer.allocate(sampleSize), 0, sampleSize);
+    }
+
+    public static Buffer readSample(FileInput fileInput, Buffer sample, int offset, int sampleSize)
     {
         if (!fileInput.nextFile()) {
             // no input files
-            return Buffer.EMPTY;
+            return sample;
         }
 
-        Buffer sample = Buffer.allocate(sampleSize);
-        int offset = 0;
-
-        for (Buffer buffer : each(fileInput)) {
-            int size = Math.min(buffer.limit(), sample.capacity() - offset);
-            sample.setBytes(offset, buffer, 0, size);
-            offset += size;
-            buffer.release();
-            if (offset >= sampleSize) {
-                break;
+        try {
+            for (Buffer buffer : each(fileInput)) {
+                int size = Math.min(buffer.limit(), sample.capacity() - offset);
+                sample.setBytes(offset, buffer, 0, size);
+                offset += size;
+                buffer.release();
+                if (offset >= sampleSize) {
+                    break;
+                }
             }
         }
-        sample.limit(offset);
+        finally {
+            sample.limit(offset);
+        }
         return sample;
     }
 }

data/embulk-core/src/main/java/org/embulk/exec/TransactionStage.java

@@ -0,0 +1,27 @@
+package org.embulk.exec;
+
+public enum TransactionStage
+{
+    INPUT_BEGIN(1),
+    FILTER_BEGIN(2),
+    EXECUTOR_BEGIN(3),
+    OUTPUT_BEGIN(4),
+    RUN(5),
+    OUTPUT_COMMIT(6),
+    EXECUTOR_COMMIT(7),
+    FILTER_COMMIT(8),
+    INPUT_COMMIT(9),
+    CLEANUP(10);
+
+    private final int index;
+
+    private TransactionStage(int index)
+    {
+        this.index = index;
+    }
+
+    public boolean isBefore(TransactionStage another)
+    {
+        return index < another.index;
+    }
+}

data/embulk-docs/src/release.rst

@@ -4,6 +4,7 @@ Release Notes
 .. toctree::
     :maxdepth: 1
 
+    release/release-0.7.10
     release/release-0.7.9
     release/release-0.7.8
     release/release-0.7.7

data/embulk-docs/src/release/release-0.7.10.rst

@@ -0,0 +1,13 @@
+Release 0.7.10
+==================================
+
+General Changes
+------------------
+
+* Fixed a problem where guessing reads only 512 bytes when input is gzip-compressed and gzip decoder is guseed during the guessing.
+* Added ``PartialExecutionException.getTransactionStage()`` method that tells in which stage a transaction failed. It will be either of beginning of (input, filter, executor, output), run, end of (output, executor, filter, input), or cleanup.
+
+
+Release Date
+------------------
+2015-12-01

data/embulk-standards/src/main/java/org/embulk/standards/GzipFileDecoderPlugin.java

@@ -43,7 +43,7 @@ public class GzipFileDecoderPlugin
                         if (!files.nextFile()) {
                             return null;
                         }
-                        return new GZIPInputStream(files);
+                        return new GZIPInputStream(files, 8*1024);
                     }
 
                     public void close() throws IOException

data/lib/embulk/runner.rb

@@ -79,18 +79,22 @@ module Embulk
 
       unless executionResult
         unless resumableResult.isSuccessful
-          Embulk.logger.info "Writing resume state to '#{resume_state_path}'"
-          write_config(resume_state_path, resumableResult.getResumeState)
-          Embulk.logger.info "Resume state is written. Run the transaction again with -r option to resume or use \"cleanup\" subcommand to delete intermediate data."
+          if resumableResult.getTransactionStage.isBefore(org.embulk.exec.TransactionStage::RUN)
+            # retry without resume state file if no tasks started yet
+            # delete resume file
+            File.delete(resume_state_path) rescue nil if resume_state_path
+          else
+            Embulk.logger.info "Writing resume state to '#{resume_state_path}'"
+            write_config(resume_state_path, resumableResult.getResumeState)
+            Embulk.logger.info "Resume state is written. Run the transaction again with -r option to resume or use \"cleanup\" subcommand to delete intermediate data."
+          end
           raise resumableResult.getCause
         end
         executionResult = resumableResult.getSuccessfulResult
       end
 
       # delete resume file
-      if resume_state_path
-        File.delete(resume_state_path) rescue nil
-      end
+      File.delete(resume_state_path) rescue nil if resume_state_path
 
       configDiff = executionResult.getConfigDiff
       Embulk.logger.info("Committed.")

data/lib/embulk/version.rb

@@ -1,3 +1,3 @@
 module Embulk
-  VERSION = '0.7.9'
+  VERSION = '0.7.10'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk
 version: !ruby/object:Gem::Version
-  version: 0.7.9
+  version: 0.7.10
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-11-11 00:00:00.000000000 Z
+date: 2015-12-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: jruby-jars
@@ -107,8 +107,8 @@ files:
 - classpath/bval-jsr303-0.5.jar
 - classpath/commons-beanutils-core-1.8.3.jar
 - classpath/commons-lang3-3.1.jar
-- classpath/embulk-core-0.7.9.jar
-- classpath/embulk-standards-0.7.9.jar
+- classpath/embulk-core-0.7.10.jar
+- classpath/embulk-standards-0.7.10.jar
 - classpath/guava-18.0.jar
 - classpath/guice-4.0.jar
 - classpath/guice-multibindings-4.0.jar
@@ -185,6 +185,7 @@ files:
 - embulk-core/src/main/java/org/embulk/exec/SetCurrentThreadName.java
 - embulk-core/src/main/java/org/embulk/exec/SystemConfigModule.java
 - embulk-core/src/main/java/org/embulk/exec/TempFileAllocator.java
+- embulk-core/src/main/java/org/embulk/exec/TransactionStage.java
 - embulk-core/src/main/java/org/embulk/guice/Bootstrap.java
 - embulk-core/src/main/java/org/embulk/guice/CloseableInjector.java
 - embulk-core/src/main/java/org/embulk/guice/InjectorProxy.java
@@ -409,6 +410,7 @@ files:
 - embulk-docs/src/release/release-0.6.9.rst
 - embulk-docs/src/release/release-0.7.0.rst
 - embulk-docs/src/release/release-0.7.1.rst
+- embulk-docs/src/release/release-0.7.10.rst
 - embulk-docs/src/release/release-0.7.2.rst
 - embulk-docs/src/release/release-0.7.3.rst
 - embulk-docs/src/release/release-0.7.4.rst