embulk 0.6.7 → 0.6.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c812083e751a2593484a749af8ac7f0519cbe5f7
-  data.tar.gz: d7d74ec957f1cbb9951850199524a3e039be4640
+  metadata.gz: a9e8a1f4e6ada76c6388248ae5ac9a2cdc6876ea
+  data.tar.gz: b21c680b792319509aa443b4e01809af8d3dcb56
 SHA512:
-  metadata.gz: 9f727dfbf18ab2dc4e768b7ea132413c30fc63330a6aa3c0d5ffc4285623c7f03eaa66c177ed9c4c57d1232f59e582583cb2255c73d8f0a4045831f126f040de
-  data.tar.gz: a0218b10b993ad8cb09afc3fc7f94454da50410979e70a74375d2c3d739077e20ae16b27e0fb30d369137cc8c29b6bddaace9955746f66fcb3ae8c72c7a39e82
+  metadata.gz: 7990996b97d6ae238dd20c8f997af9cb68f2d557b94b3d27388b8ede4e675c1c0f044072310d76ce37c4e4625497510ddecc9dee594a783ac91875f398c0137b
+  data.tar.gz: c2421ec9edd3f3e404302699526b1e226af19755186962b3b27074e53aebbfdb45dabf9e832c0b5a285bb40aad3e5a7aac7000724d735f43cae009b53150e2c9

data/build.gradle

@@ -11,7 +11,7 @@ def release_projects = [project(":embulk-core"), project(":embulk-standards")]
 
 allprojects {
     group = 'org.embulk'
-    version = '0.6.7'
+    version = '0.6.8'
 
     ext {
         jrubyVersion = '1.7.19'

data/embulk-core/build.gradle

@@ -1,9 +1,6 @@
-// include ruby scripts
-sourceSets {
-    main.resources {
-        srcDirs "${rootProject.projectDir}/lib"
-    }
-}
+// include ruby scripts to jar. don't use sourceSets.main.resources.srcDirs
+// because IntelliJ causes error if srcDirs includes files out of projectDir.
+processResources.from "${rootProject.projectDir}/lib"
 
 configurations {
     // com.google.inject:guice depends on asm and cglib but version of the libraries conflict

data/embulk-core/src/main/java/org/embulk/exec/BulkLoader.java

@@ -164,7 +164,7 @@ public class BulkLoader
             return outputTaskStates.get(outputTaskIndex);
         }
 
-        public boolean isAllCommitted()
+        public boolean isAllTasksCommitted()
         {
             if (outputTaskStates == null) {
                 // not initialized
@@ -178,6 +178,11 @@ public class BulkLoader
             return true;
         }
 
+        public boolean isAllTransactionsCommitted()
+        {
+            return inputConfigDiff != null && outputConfigDiff != null;
+        }
+
         public boolean isAnyStarted()
         {
             if (inputTaskStates == null) {
@@ -489,7 +494,7 @@ public class BulkLoader
 
                                             state.initialize(inputTaskCount, outputTaskCount);
 
-                                            if (!state.isAllCommitted()) {  // inputTaskCount == 0
+                                            if (!state.isAllTasksCommitted()) {  // inputTaskCount == 0
                                                 execute(task, executor, state);
                                             }
 
@@ -511,7 +516,7 @@ public class BulkLoader
             return state.buildExecuteResult();
 
         } catch (Throwable ex) {
-            if (state.isAllCommitted()) {
+            if (state.isAllTasksCommitted() && state.isAllTransactionsCommitted()) {
                 // ignore the exception
                 return state.buildExecuteResultWithWarningException(ex);
             }
@@ -554,7 +559,7 @@ public class BulkLoader
                                             state.setOutputTaskSource(outputTask);
 
                                             restoreResumedCommitReports(resume, state);
-                                            if (!state.isAllCommitted()) {
+                                            if (!state.isAllTasksCommitted()) {
                                                 execute(task, executor, state);
                                             }
 
@@ -576,7 +581,7 @@ public class BulkLoader
             return state.buildExecuteResult();
 
         } catch (Throwable ex) {
-            if (state.isAllCommitted()) {
+            if (state.isAllTasksCommitted() && state.isAllTransactionsCommitted()) {
                 // ignore the exception
                 return state.buildExecuteResultWithWarningException(ex);
             }
@@ -621,7 +626,7 @@ public class BulkLoader
 
         executor.execute(procTask, state);
 
-        if (!state.isAllCommitted()) {
+        if (!state.isAllTasksCommitted()) {
             throw state.getRepresentativeException();
         }
     }

data/embulk-core/src/main/java/org/embulk/spi/time/TimestampFormatter.java

@@ -39,6 +39,11 @@ public class TimestampFormatter
         this.dateFormat = new RubyDateFormat(format, Locale.ENGLISH, true);
     }
 
+    public DateTimeZone getTimeZone()
+    {
+        return timeZone;
+    }
+
     public void format(Timestamp value, LineEncoder encoder)
     {
         // TODO optimize by directly appending to internal buffer

data/embulk-core/src/main/java/org/embulk/spi/time/TimestampParser.java

@@ -30,6 +30,11 @@ public class TimestampParser
         this(task.getJRuby(), format, task.getDefaultTimeZone());
     }
 
+    public DateTimeZone getDefaultTimeZone()
+    {
+        return defaultTimeZone;
+    }
+
     // TODO this is still private because this might need current time
     private TimestampParser(ScriptingContainer jruby, String format, DateTimeZone defaultTimeZone)
     {

data/embulk-core/src/main/java/org/embulk/spi/util/ResumableInputStream.java

@@ -0,0 +1,128 @@
+package org.embulk.spi.util;
+
+import java.io.InputStream;
+import java.io.IOException;
+
+public class ResumableInputStream
+        extends InputStream
+{
+    public interface Reopener
+    {
+        public InputStream reopen(long offset, Exception closedCause) throws IOException;
+    }
+
+    private final Reopener reopener;
+    protected InputStream in;
+    private long offset;
+    private long markedOffset;
+
+    public ResumableInputStream(InputStream initialInputStream, Reopener reopener)
+    {
+        this.reopener = reopener;
+        this.in = initialInputStream;
+        this.offset = 0L;
+        this.markedOffset = 0L;
+    }
+
+    public ResumableInputStream(Reopener reopener) throws IOException
+    {
+        this(reopener.reopen(0, null), reopener);
+    }
+
+    private void reopen(Exception closedCause) throws IOException
+    {
+        if (in != null) {
+            in.close();
+            in = null;
+        }
+        in = reopener.reopen(offset, closedCause);
+    }
+
+    @Override
+    public int read() throws IOException
+    {
+        while (true) {
+            try {
+                int v = in.read();
+                offset += 1;
+                return v;
+            } catch (IOException | RuntimeException ex) {
+                reopen(ex);
+            }
+        }
+    }
+
+    @Override
+    public int read(byte[] b) throws IOException
+    {
+        while (true) {
+            try {
+                int r = in.read(b);
+                offset += r;
+                return r;
+            } catch (IOException | RuntimeException ex) {
+                reopen(ex);
+            }
+        }
+    }
+
+    @Override
+    public int read(byte[] b, int off, int len) throws IOException
+    {
+        while (true) {
+            try {
+                int r = in.read(b, off, len);
+                offset += r;
+                return r;
+            } catch (IOException | RuntimeException ex) {
+                reopen(ex);
+            }
+        }
+    }
+
+    @Override
+    public long skip(long n) throws IOException
+    {
+        while (true) {
+            try {
+                long r = in.skip(n);
+                offset += r;
+                return r;
+            } catch (IOException | RuntimeException ex) {
+                reopen(ex);
+            }
+        }
+    }
+
+    @Override
+    public int available() throws IOException
+    {
+        return in.available();
+    }
+
+    @Override
+    public void close() throws IOException
+    {
+        in.close();
+    }
+
+    @Override
+    public void mark(int readlimit)
+    {
+        in.mark(readlimit);
+        markedOffset = offset;
+    }
+
+    @Override
+    public void reset() throws IOException
+    {
+        in.reset();
+        offset = markedOffset;
+    }
+
+    @Override
+    public boolean markSupported()
+    {
+        return in.markSupported();
+    }
+}

data/embulk-core/src/main/java/org/embulk/spi/util/RetryExecutor.java

@@ -0,0 +1,130 @@
+package org.embulk.spi.util;
+
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+
+public class RetryExecutor
+{
+    public static RetryExecutor retryExecutor()
+    {
+        // TODO default configuration
+        return new RetryExecutor(3, 500, 30*60*1000);
+    }
+
+    public static class RetryGiveupException
+            extends ExecutionException
+    {
+        public RetryGiveupException(String message, Exception cause)
+        {
+            super(cause);
+        }
+
+        public RetryGiveupException(Exception cause)
+        {
+            super(cause);
+        }
+
+        public Exception getCause()
+        {
+            return (Exception) super.getCause();
+        }
+    }
+
+    public static interface Retryable<T>
+            extends Callable<T>
+    {
+        public T call()
+            throws Exception;
+
+        public boolean isRetryableException(Exception exception);
+
+        public void onRetry(Exception exception, int retryCount, int retryLimit, int retryWait)
+            throws RetryGiveupException;
+
+        public void onGiveup(Exception firstException, Exception lastException)
+            throws RetryGiveupException;
+    }
+
+    private final int retryLimit;
+    private final int initialRetryWait;
+    private final int maxRetryWait;
+
+    private RetryExecutor(int retryLimit, int initialRetryWait, int maxRetryWait)
+    {
+        this.retryLimit = retryLimit;
+        this.initialRetryWait = initialRetryWait;
+        this.maxRetryWait = maxRetryWait;
+    }
+
+    public RetryExecutor withRetryLimit(int count)
+    {
+        return new RetryExecutor(count, initialRetryWait, maxRetryWait);
+    }
+
+    public RetryExecutor withInitialRetryWait(int msec)
+    {
+        return new RetryExecutor(retryLimit, msec, maxRetryWait);
+    }
+
+    public RetryExecutor withMaxRetryWait(int msec)
+    {
+        return new RetryExecutor(retryLimit, initialRetryWait, msec);
+    }
+
+    public <T> T runInterruptible(Retryable<T> op)
+            throws InterruptedException, RetryGiveupException
+    {
+        return run(op, true);
+    }
+
+    public <T> T run(Retryable<T> op)
+            throws RetryGiveupException
+    {
+        try {
+            return run(op, false);
+        } catch (InterruptedException ex) {
+            throw new RetryGiveupException("Unexpected interruption", ex);
+        }
+    }
+
+    private <T> T run(Retryable<T> op, boolean interruptible)
+            throws InterruptedException, RetryGiveupException
+    {
+        int retryWait = initialRetryWait;
+        int retryCount = 0;
+
+        Exception firstException = null;
+
+        while(true) {
+            try {
+                return op.call();
+            } catch (Exception exception) {
+                if (firstException == null) {
+                    firstException = exception;
+                }
+                if (!op.isRetryableException(exception) || retryCount >= retryLimit) {
+                    op.onGiveup(firstException, exception);
+                    throw new RetryGiveupException(firstException);
+                }
+
+                retryCount++;
+                op.onRetry(exception, retryCount, retryLimit, retryWait);
+
+                try {
+                    Thread.sleep(retryWait);
+                } catch (InterruptedException ex) {
+                    if (interruptible) {
+                        throw ex;
+                    }
+                }
+
+                // exponential back-off with hard limit
+                retryWait *= 2;
+                if (retryWait > maxRetryWait) {
+                    retryWait = maxRetryWait;
+                }
+            }
+        }
+    }
+}
+

data/embulk-docs/src/built-in.rst

@@ -145,6 +145,8 @@ Options
 +----------------------------+----------+----------------------------------------------------------------------------------------------------------------+------------------------+
 | allow\_optional\_columns   | boolean  | If true, set null to insufficient columns. Otherwise, skip the row in case of insufficient number of columns   | ``false`` by default   |
 +----------------------------+----------+----------------------------------------------------------------------------------------------------------------+------------------------+
+| allow\_extra\_columns      | boolean  | If true, ignore too many columns. Otherwise, skip the row in case of too many columns                          | ``false`` by default   |
++----------------------------+----------+----------------------------------------------------------------------------------------------------------------+------------------------+
 | max\_quoted\_size\_limit   | integer  | Maximum number of bytes of a quoted value. If a value exceeds the limit, the row will be skipped               | ``131072`` by default  |
 +----------------------------+----------+----------------------------------------------------------------------------------------------------------------+------------------------+
 | default\_timezone          | string   | Time zone of timestamp columns if the value itself doesn't include time zone description (eg. Asia/Tokyo)      | ``UTC`` by default     |

data/embulk-docs/src/release.rst

@@ -4,6 +4,7 @@ Release Notes
 .. toctree::
     :maxdepth: 1
 
+    release/release-0.6.8
     release/release-0.6.7
     release/release-0.6.6
     release/release-0.6.5

data/embulk-docs/src/release/release-0.6.8.rst

@@ -0,0 +1,24 @@
+Release 0.6.8
+==================================
+
+Plugin API
+------------------
+
+* Added utility class ``spi.util.ResumableInputStream``
+* Added utility class ``spi.util.RetryExecutor``
+
+Built-in plugins
+------------------
+
+* ``parser-csv`` rejects rows if one includes too many columns by default. Setting ``allow extra_columns`` option to ``true`` will make the behavior same with before.
+* ``guess-csv`` guesses ``columns`` option every time.
+
+General Changes
+------------------
+
+* Fixed a problem that IntelliJ IDEA causes problem when it imports embulk source code.
+* Fixed a problem that transaction silently succeeds when an exception happens after all taks succeeded.
+
+Release Date
+------------------
+2015-05-12

data/embulk-standards/src/main/java/org/embulk/standards/CsvParserPlugin.java

@@ -78,6 +78,10 @@ public class CsvParserPlugin
         @Config("allow_optional_columns")
         @ConfigDefault("false")
         public boolean getAllowOptionalColumns();
+
+        @Config("allow_extra_columns")
+        @ConfigDefault("false")
+        public boolean getAllowExtraColumns();
     }
 
     private final Logger log;
@@ -130,6 +134,7 @@ public class CsvParserPlugin
         final CsvTokenizer tokenizer = new CsvTokenizer(lineDecoder, task);
         final String nullStringOrNull = task.getNullString().orNull();
         final boolean allowOptionalColumns = task.getAllowOptionalColumns();
+        final boolean allowExtraColumns = task.getAllowExtraColumns();
         int skipHeaderLines = task.getSkipHeaderLines();
 
         try (final PageBuilder pageBuilder = new PageBuilder(Exec.getBufferAllocator(), schema, output)) {
@@ -141,12 +146,15 @@ public class CsvParserPlugin
                     }
                 }
 
+                if (!tokenizer.nextRecord()) {
+                    // empty file
+                    continue;
+                }
+
                 while (true) {
-                    try {
-                        if (!tokenizer.nextRecord()) {
-                            break;
-                        }
+                    boolean hasNextRecord;
 
+                    try {
                         schema.visitColumns(new ColumnVisitor() {
                             public void booleanColumn(Column column)
                             {
@@ -216,6 +224,7 @@ public class CsvParserPlugin
                             private String nextColumn()
                             {
                                 if (allowOptionalColumns && !tokenizer.hasNextColumn()) {
+                                    //TODO warning
                                     return null;
                                 }
                                 String v = tokenizer.nextColumn();
@@ -231,13 +240,32 @@ public class CsvParserPlugin
                                 }
                             }
                         });
+
+                        try {
+                            hasNextRecord = tokenizer.nextRecord();
+                        } catch (CsvTokenizer.TooManyColumnsException ex) {
+                            if (allowExtraColumns) {
+                                String tooManyColumnsLine = tokenizer.skipCurrentLine();
+                                // TODO warning
+                                hasNextRecord = tokenizer.nextRecord();
+                            } else {
+                                // this line will be skipped at the following catch section
+                                throw ex;
+                            }
+                        }
                         pageBuilder.addRecord();
 
                     } catch (CsvTokenizer.InvalidFormatException | CsvRecordValidateException e) {
                         long lineNumber = tokenizer.getCurrentLineNumber();
                         String skippedLine = tokenizer.skipCurrentLine();
-                        log.warn(String.format("Skipped (line %d): %s", lineNumber, skippedLine), e);
+                        log.warn(String.format("Skipped line %d (%s): %s", lineNumber, e.getMessage(), skippedLine));
                         //exec.notice().skippedLine(skippedLine);
+
+                        hasNextRecord = tokenizer.nextRecord();
+                    }
+
+                    if (!hasNextRecord) {
+                        break;
                     }
                 }
             }

data/lib/embulk/guess/csv.rb

@@ -75,24 +75,25 @@ module Embulk
           parser_guessed["skip_header_lines"] = skip_header_lines
         end
 
-        unless parser_config.has_key?("columns")
-          if header_line
-            column_names = sample_records.first
-          else
-            column_names = (0..other_types.size).to_a.map {|i| "c#{i}" }
-          end
-          schema = []
-          column_names.zip(other_types).each do |name,type|
-            if name && type
-              if type.is_a?(SchemaGuess::TimestampTypeMatch)
-                schema << {"name" => name, "type" => type, "format" => type.format}
-              else
-                schema << {"name" => name, "type" => type}
-              end
+        parser_guessed["allow_extra_columns"] = false
+        parser_guessed["allow_optional_columns"] = false
+
+        if header_line
+          column_names = sample_records.first
+        else
+          column_names = (0..other_types.size).to_a.map {|i| "c#{i}" }
+        end
+        schema = []
+        column_names.zip(other_types).each do |name,type|
+          if name && type
+            if type.is_a?(SchemaGuess::TimestampTypeMatch)
+              schema << {"name" => name, "type" => type, "format" => type.format}
+            else
+              schema << {"name" => name, "type" => type}
             end
           end
-          parser_guessed["columns"] = schema
         end
+        parser_guessed["columns"] = schema
 
         return {"parser" => parser_guessed}
       end

data/lib/embulk/version.rb

@@ -1,3 +1,3 @@
 module Embulk
-  VERSION = '0.6.7'
+  VERSION = '0.6.8'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk
 version: !ruby/object:Gem::Version
-  version: 0.6.7
+  version: 0.6.8
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-05-07 00:00:00.000000000 Z
+date: 2015-05-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -228,6 +228,8 @@ files:
 - embulk-core/src/main/java/org/embulk/spi/util/OutputStreamFileOutput.java
 - embulk-core/src/main/java/org/embulk/spi/util/PagePrinter.java
 - embulk-core/src/main/java/org/embulk/spi/util/Pages.java
+- embulk-core/src/main/java/org/embulk/spi/util/ResumableInputStream.java
+- embulk-core/src/main/java/org/embulk/spi/util/RetryExecutor.java
 - embulk-core/src/test/java/org/embulk/EmbulkTestRuntime.java
 - embulk-core/src/test/java/org/embulk/GuiceBinder.java
 - embulk-core/src/test/java/org/embulk/RandomManager.java
@@ -297,6 +299,7 @@ files:
 - embulk-docs/src/release/release-0.6.5.rst
 - embulk-docs/src/release/release-0.6.6.rst
 - embulk-docs/src/release/release-0.6.7.rst
+- embulk-docs/src/release/release-0.6.8.rst
 - embulk-standards/build.gradle
 - embulk-standards/src/main/java/org/embulk/standards/CsvFormatterPlugin.java
 - embulk-standards/src/main/java/org/embulk/standards/CsvParserPlugin.java
@@ -403,8 +406,8 @@ files:
 - classpath/bval-jsr303-0.5.jar
 - classpath/commons-beanutils-core-1.8.3.jar
 - classpath/commons-lang3-3.1.jar
-- classpath/embulk-core-0.6.7.jar
-- classpath/embulk-standards-0.6.7.jar
+- classpath/embulk-core-0.6.8.jar
+- classpath/embulk-standards-0.6.8.jar
 - classpath/guava-18.0.jar
 - classpath/guice-4.0.jar
 - classpath/guice-multibindings-4.0.jar