embulk 0.5.3 → 0.5.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0cd34264ca6a673f948b77e42cb3049e184a4f40
-  data.tar.gz: 8d56e898df4411110bc5c42c6817803d93a732fe
+  metadata.gz: e0be98e5dbe81e40c6562142d2cdf44cc4f8cf34
+  data.tar.gz: f907d431af0add753761547f09dec47113b3b236
 SHA512:
-  metadata.gz: 13ad1c043fff795f33199985f955d67238d4d54297516cd3d3ac8abcd1ea90e9cc43569e2745918c6090ce4373ae01a92f2cd26f35fddd37a93aa5d70c8be273
-  data.tar.gz: f02d113e3eddb344f9080d9dc5163c61e6718455b4d404607eb0a1bd62ac94da547697dd2be5796c6a44404a92ef01c845553f7da7a672b005dc434c5b3c5c13
+  metadata.gz: 2a1690e94a7622db588cc6511f1dec583320192960ecdf9851c74ac1f5feaf7bda478b8e6dbf16a7b58f870e38dd464ee949cc20c6f5cadb3473b04bf3cf23db
+  data.tar.gz: 0eb2a31661f7772cadee71642781d2ef30cfc5015ce5bf549a3cc36310fbe29283f80d272b91ee788a4513c52ae8c09ecad8e4dce28e459ad11c5370f68a6e22

data/README.md

@@ -30,7 +30,7 @@ Embulk is a Java application. Please make sure that you installed [Java](http://
 Following 4 commands install embulk to your home directory:
 
 ```
-curl --create-dirs -o ~/.embulk/bin/embulk -L https://bintray.com/artifact/download/embulk/maven/embulk-0.5.3.jar
+curl --create-dirs -o ~/.embulk/bin/embulk -L https://bintray.com/artifact/download/embulk/maven/embulk-0.5.4.jar
 chmod +x ~/.embulk/bin/embulk
 echo 'export PATH="$HOME/.embulk/bin:$PATH"' >> ~/.bashrc
 source ~/.bashrc
@@ -45,7 +45,7 @@ Embulk is a Java application. Please make sure that you installed [Java](http://
 You can assume the jar file is a .bat file.
 
 ```
-PowerShell -Command "& {Invoke-WebRequest https://bintray.com/artifact/download/embulk/maven/embulk-0.5.3.jar -OutFile embulk.bat}"
+PowerShell -Command "& {Invoke-WebRequest https://bintray.com/artifact/download/embulk/maven/embulk-0.5.4.jar -OutFile embulk.bat}"
 ```
 
 Next step: [Trying examples](#trying-examples)

data/build.gradle

@@ -12,7 +12,7 @@ def release_projects = [project(":embulk-core"), project(":embulk-standards")]
 
 allprojects {
     group = 'org.embulk'
-    version = '0.5.3'
+    version = '0.5.4'
 
     apply plugin: 'java'
     apply plugin: 'maven-publish'

data/embulk-core/src/main/java/org/embulk/exec/PooledBufferAllocator.java

@@ -11,6 +11,9 @@ public class PooledBufferAllocator
 {
     private PooledByteBufAllocator nettyBuffer;
 
+    private int DEFAULT_BUFFER_SIZE = 32*1024;
+    private int MINIMUM_BUFFER_SIZE = 8*1024;
+
     public PooledBufferAllocator()
     {
         // TODO configure parameters
@@ -19,12 +22,12 @@ public class PooledBufferAllocator
 
     public Buffer allocate()
     {
-        return new NettyByteBufBuffer(nettyBuffer.buffer());
+        return allocate(DEFAULT_BUFFER_SIZE);
     }
 
     public Buffer allocate(int minimumCapacity)
     {
-        int size = 32*1024;
+        int size = MINIMUM_BUFFER_SIZE;
         while (size < minimumCapacity) {
             size *= 2;
         }

data/embulk-core/src/main/java/org/embulk/plugin/PluginManager.java

@@ -28,6 +28,10 @@ public class PluginManager
             throw new ConfigException("No PluginSource is installed");
         }
 
+        if (type == null) {
+            throw new ConfigException(String.format("%s type is not set (if you intend to use NullOutputPlugin, you should enclose null in quotes such as {type: \"null\"}.", iface.getSimpleName()));
+        }
+
         List<Throwable> causes = new ArrayList<Throwable>();
         for (PluginSource source : sources) {
             try {

data/embulk-core/src/main/java/org/embulk/spi/util/FileOutputOutputStream.java

@@ -36,6 +36,7 @@ public class FileOutputOutputStream
 
     public void finish()
     {
+        doFlush();
         out.finish();
     }
 

data/embulk-docs/src/recipe/scheduled-csv-load-to-elasticsearch-kibana4.rst

@@ -56,7 +56,7 @@ You can find the latest embulk binary from the `releases <https://bintray.com/em
 
 .. code-block:: console
 
-    $ sudo wget https://bintray.com/artifact/download/embulk/maven/embulk-0.5.3.jar -O /usr/local/bin/embulk
+    $ sudo wget https://bintray.com/artifact/download/embulk/maven/embulk-0.5.4.jar -O /usr/local/bin/embulk
     $ sudo chmod +x /usr/local/bin/embulk
 
 Step 2. Install Elasticsearch plugin

data/embulk-docs/src/release.rst

@@ -25,4 +25,5 @@ Release Notes
     release/release-0.5.1
     release/release-0.5.2
     release/release-0.5.3
+    release/release-0.5.4
 

data/embulk-docs/src/release/release-0.5.4.rst

@@ -0,0 +1,24 @@
+Release 0.5.4
+==================================
+
+Built-in plugins
+------------------
+
+* ``parser-csv`` supports ``allow_optional_columns`` option. With this option set to ``true``, the parser sets null to insufficient columns rather than skipping the entire row (@kamatama41++)
+
+* Fixed exception handling of ``parser-csv`` so that the transaction properly fails with underlaying exceptions such as IOException
+
+
+General Changes
+------------------
+
+* Increased buffer size from 256 bytes to 32 KB. This improves performance significantly. (@hito4t++)
+
+* If plugin type is null, suggest to use ``{type: "null"}`` (@hito4t++)
+
+* Embulk logo is available! See the orca: https://github.com/embulk/embulk/issues/12
+
+
+Release Date
+------------------
+2015-03-23

data/embulk-standards/src/main/java/org/embulk/standards/CsvParserPlugin.java

@@ -1,6 +1,5 @@
 package org.embulk.standards;
 
-import com.google.common.base.Preconditions;
 import com.google.common.base.Optional;
 import com.google.common.collect.ImmutableSet;
 import org.embulk.config.Task;
@@ -21,7 +20,6 @@ import org.embulk.spi.ParserPlugin;
 import org.embulk.spi.Exec;
 import org.embulk.spi.FileInput;
 import org.embulk.spi.PageOutput;
-import org.embulk.spi.BufferAllocator;
 import org.embulk.spi.util.LineDecoder;
 import org.slf4j.Logger;
 
@@ -76,6 +74,10 @@ public class CsvParserPlugin
         @Config("max_quoted_size_limit")
         @ConfigDefault("131072") //128kB
         public long getMaxQuotedSizeLimit();
+
+        @Config("allow_optional_columns")
+        @ConfigDefault("false")
+        public boolean getAllowOptionalColumns();
     }
 
     private final Logger log;
@@ -127,6 +129,7 @@ public class CsvParserPlugin
         LineDecoder lineDecoder = new LineDecoder(input, task);
         final CsvTokenizer tokenizer = new CsvTokenizer(lineDecoder, task);
         final String nullStringOrNull = task.getNullString().orNull();
+        final boolean allowOptionalColumns = task.getAllowOptionalColumns();
         int skipHeaderLines = task.getSkipHeaderLines();
 
         try (final PageBuilder pageBuilder = new PageBuilder(Exec.getBufferAllocator(), schema, output)) {
@@ -147,7 +150,7 @@ public class CsvParserPlugin
                         schema.visitColumns(new ColumnVisitor() {
                             public void booleanColumn(Column column)
                             {
-                                String v = nextColumn(schema, tokenizer, nullStringOrNull);
+                                String v = nextColumn(schema, tokenizer, nullStringOrNull, allowOptionalColumns);
                                 if (v == null) {
                                     pageBuilder.setNull(column);
                                 } else {
@@ -157,7 +160,7 @@ public class CsvParserPlugin
 
                             public void longColumn(Column column)
                             {
-                                String v = nextColumn(schema, tokenizer, nullStringOrNull);
+                                String v = nextColumn(schema, tokenizer, nullStringOrNull, allowOptionalColumns);
                                 if (v == null) {
                                     pageBuilder.setNull(column);
                                 } else {
@@ -172,7 +175,7 @@ public class CsvParserPlugin
 
                             public void doubleColumn(Column column)
                             {
-                                String v = nextColumn(schema, tokenizer, nullStringOrNull);
+                                String v = nextColumn(schema, tokenizer, nullStringOrNull, allowOptionalColumns);
                                 if (v == null) {
                                     pageBuilder.setNull(column);
                                 } else {
@@ -187,7 +190,7 @@ public class CsvParserPlugin
 
                             public void stringColumn(Column column)
                             {
-                                String v = nextColumn(schema, tokenizer, nullStringOrNull);
+                                String v = nextColumn(schema, tokenizer, nullStringOrNull, allowOptionalColumns);
                                 if (v == null) {
                                     pageBuilder.setNull(column);
                                 } else {
@@ -197,7 +200,7 @@ public class CsvParserPlugin
 
                             public void timestampColumn(Column column)
                             {
-                                String v = nextColumn(schema, tokenizer, nullStringOrNull);
+                                String v = nextColumn(schema, tokenizer, nullStringOrNull, allowOptionalColumns);
                                 if (v == null) {
                                     pageBuilder.setNull(column);
                                 } else {
@@ -212,8 +215,7 @@ public class CsvParserPlugin
                         });
                         pageBuilder.addRecord();
 
-                    } catch (Exception e) {
-                        // TODO logging
+                    } catch (CsvTokenizer.InvalidFormatException e) {
                         long lineNumber = tokenizer.getCurrentLineNumber();
                         String skippedLine = tokenizer.skipCurrentLine();
                         log.warn(String.format("Skipped (line %d): %s", lineNumber, skippedLine), e);
@@ -226,8 +228,11 @@ public class CsvParserPlugin
         }
     }
 
-    private static String nextColumn(Schema schema, CsvTokenizer tokenizer, String nullStringOrNull)
+    private static String nextColumn(Schema schema, CsvTokenizer tokenizer, String nullStringOrNull, boolean allowOptionalColumns)
     {
+        if(allowOptionalColumns && !tokenizer.hasNextColumn()) {
+            return null;
+        }
         String v = tokenizer.nextColumn();
         if (!v.isEmpty()) {
             if (v.equals(nullStringOrNull)) {

data/embulk-standards/src/main/java/org/embulk/standards/CsvTokenizer.java

@@ -20,7 +20,6 @@ public class CsvTokenizer
     }
 
     private static final char END_OF_LINE = '\0';
-    private static final boolean TRACE = false;
 
     private final char delimiter;
     private final char quote;
@@ -81,7 +80,10 @@ public class CsvTokenizer
     public boolean nextRecord()
     {
         // If at the end of record, read the next line and initialize the state
-        Preconditions.checkState(recordState == RecordState.END, "too many columns");  // TODO exception class
+        if (recordState != RecordState.END) {
+            throw new TooManyColumnsException("Too many columns");
+        }
+
         boolean hasNext = nextLine(true);
         if (hasNext) {
             recordState = RecordState.NOT_END;
@@ -105,10 +107,6 @@ public class CsvTokenizer
             linePos = 0;
             lineNumber++;
 
-            if (TRACE) {
-                System.out.println("#MN line: " + line + " (" + lineNumber + ")");
-            }
-
             if (!line.isEmpty() || !ignoreEmptyLine) {
                 return true;
             }
@@ -122,7 +120,9 @@ public class CsvTokenizer
 
     public String nextColumn()
     {
-        Preconditions.checkState(hasNextColumn(), "doesn't have enough columns");  // TODO exception class
+        if (!hasNextColumn()) {
+            throw new TooFewColumnsException("Too few columns");
+        }
 
         // reset last state
         wasQuotedColumn = false;
@@ -136,10 +136,6 @@ public class CsvTokenizer
 
         while (true) {
             final char c = nextChar();
-            if (TRACE) {
-                System.out.println("#MN c: " + c + " (" + columnState + "," + recordState + ")");
-                try { Thread.sleep(100); } catch (Exception e) {}
-            }
 
             switch (columnState) {
                 case BEGIN:
@@ -241,15 +237,12 @@ public class CsvTokenizer
                         quotedValue.append(newline);
                         quotedValueLines.add(line);
                         if (!nextLine(false)) {
-                            throw new RuntimeException("Unexpected end of line during parsing a quoted value");  // TODO exception class
+                            throw new InvalidValueException("Unexpected end of line during parsing a quoted value");
                         }
                         valueStartPos = 0;
 
                     } else if (isQuote(c)) {
                         char next = peekNextChar();
-                        if (TRACE) {
-                            System.out.println("#MN peeked c: " + next + " (" + columnState + "," + recordState + ")");
-                        }
                         if (isQuote(next)) { // escaped quote
                             quotedValue.append(line.substring(valueStartPos, linePos));
                             valueStartPos = ++linePos;
@@ -261,15 +254,12 @@ public class CsvTokenizer
                     } else if (isEscape(c)) {  // isQuote must be checked first in case of quote == escape
                         // In RFC 4180, CSV's escape char is '\"'. But '\\' is often used.
                         char next = peekNextChar();
-                        if (TRACE) {
-                            System.out.println("#MN peeked c: " + next + " (" + columnState + "," + recordState + ")");
-                        }
                         if (isEndOfLine(c)) {
                             // escape end of line. TODO assuming multi-line quoted value without newline?
                             quotedValue.append(line.substring(valueStartPos, linePos));
                             quotedValueLines.add(line);
                             if (!nextLine(false)) {
-                                throw new RuntimeException("Unexpected end of line during parsing a quoted value");  // TODO exception class
+                                throw new InvalidValueException("Unexpected end of line during parsing a quoted value");
                             }
                             valueStartPos = 0;
                         } else if (isQuote(next) || isEscape(next)) { // escaped quote
@@ -298,7 +288,7 @@ public class CsvTokenizer
                         // column has trailing spaces and quoted. TODO should this be rejected?
 
                     } else {
-                        throw new RuntimeException("Unexpected extra character after quoted value");  // TODO exception class
+                        throw new InvalidValueException("Unexpected extra character after quoted value");
                     }
                     break;
 
@@ -360,10 +350,46 @@ public class CsvTokenizer
         return c == escape;
     }
 
-    static class QuotedSizeLimitExceededException
+    public static class InvalidFormatException
             extends RuntimeException
     {
-        QuotedSizeLimitExceededException(String message)
+        public InvalidFormatException(String message)
+        {
+            super(message);
+        }
+    }
+
+    public static class InvalidValueException
+            extends RuntimeException
+    {
+        public InvalidValueException(String message)
+        {
+            super(message);
+        }
+    }
+
+    public static class QuotedSizeLimitExceededException
+            extends InvalidValueException
+    {
+        public QuotedSizeLimitExceededException(String message)
+        {
+            super(message);
+        }
+    }
+
+    public class TooManyColumnsException
+            extends InvalidFormatException
+    {
+        public TooManyColumnsException(String message)
+        {
+            super(message);
+        }
+    }
+
+    public class TooFewColumnsException
+            extends InvalidFormatException
+    {
+        public TooFewColumnsException(String message)
         {
             super(message);
         }

data/embulk-standards/src/test/java/org/embulk/standards/TestCsvParserPlugin.java

@@ -1,7 +1,6 @@
 package org.embulk.standards;
 
 import org.junit.Rule;
-import org.junit.Before;
 import org.junit.Test;
 import static org.junit.Assert.assertEquals;
 import java.nio.charset.Charset;
@@ -34,6 +33,7 @@ public class TestCsvParserPlugin
         assertEquals(false, task.getHeaderLine().or(false));
         assertEquals(',', task.getDelimiterChar());
         assertEquals('\"', task.getQuoteChar());
+        assertEquals(false, task.getAllowOptionalColumns());
     }
 
     @Test(expected = ConfigException.class)
@@ -53,6 +53,7 @@ public class TestCsvParserPlugin
                 .set("header_line", true)
                 .set("delimiter", "\t")
                 .set("quote", "\\")
+                .set("allow_optional_columns", true)
                 .set("columns", ImmutableList.of(
                             ImmutableMap.of(
                                 "name", "date_code",
@@ -65,5 +66,6 @@ public class TestCsvParserPlugin
         assertEquals(true, task.getHeaderLine().or(false));
         assertEquals('\t', task.getDelimiterChar());
         assertEquals('\\', task.getQuoteChar());
+        assertEquals(true, task.getAllowOptionalColumns());
     }
 }

data/lib/embulk/version.rb

@@ -1,3 +1,3 @@
 module Embulk
-  VERSION = '0.5.3'
+  VERSION = '0.5.4'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk
 version: !ruby/object:Gem::Version
-  version: 0.5.3
+  version: 0.5.4
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-03-17 00:00:00.000000000 Z
+date: 2015-03-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -275,6 +275,7 @@ files:
 - embulk-docs/src/release/release-0.5.1.rst
 - embulk-docs/src/release/release-0.5.2.rst
 - embulk-docs/src/release/release-0.5.3.rst
+- embulk-docs/src/release/release-0.5.4.rst
 - embulk-standards/build.gradle
 - embulk-standards/src/main/java/org/embulk/standards/CsvFormatterPlugin.java
 - embulk-standards/src/main/java/org/embulk/standards/CsvParserPlugin.java
@@ -379,8 +380,8 @@ files:
 - classpath/bval-jsr303-0.5.jar
 - classpath/commons-beanutils-core-1.8.3.jar
 - classpath/commons-lang3-3.1.jar
-- classpath/embulk-core-0.5.3.jar
-- classpath/embulk-standards-0.5.3.jar
+- classpath/embulk-core-0.5.4.jar
+- classpath/embulk-standards-0.5.4.jar
 - classpath/guava-18.0.jar
 - classpath/guice-3.0.jar
 - classpath/guice-multibindings-3.0.jar