embulk 0.6.14 → 0.6.15

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: abf8ae432c5b7f3817ac7acd72aee485a9b4f616
-  data.tar.gz: 8454af492f563890e26f11cc9f6fbf7fe8a7b66a
+  metadata.gz: 48e00af06c823ff60ec39a9d020896c6b16442a5
+  data.tar.gz: 711922cebca874837dce9d100cbee5d52e5c833f
 SHA512:
-  metadata.gz: 49bb503770d751d426dcb2f4331faabff654bc906cd91610980bd435fde69c78e85043aea0a2f34f9906a293d3987d725493def95a769132993849cfcc14fafb
-  data.tar.gz: 50b97e0ffc2d76581eabcb174cbebf638cc1c8eae34fbaf139e711a3856a4eb6f4b197637213d13cbc17a9d36f9256f77f65682413e0727cae39c2c0c392882b
+  metadata.gz: 714466b1fbd8794ace91fcc3b074dac84b2c72d5504e9751da9f04045f86861c653b0cd9088e443b3b53fda56b400ba7bb8e1d1a75ca8e0c2e2ba2dcbd66e745
+  data.tar.gz: 7ffa66ba254cd40e2cb4427d356648eec8e61e194c00d94b2f93817f22bb70d182154fe88ce0d38c7088507b8a4c0c445e45f6ab8da37627a8158113824921a5

data/build.gradle

@@ -11,7 +11,7 @@ def release_projects = [project(":embulk-core"), project(":embulk-standards")]
 
 allprojects {
     group = 'org.embulk'
-    version = '0.6.14'
+    version = '0.6.15'
 
     ext {
         jrubyVersion = '1.7.19'

data/embulk-core/src/main/java/org/embulk/exec/PreviewExecutor.java

@@ -91,7 +91,7 @@ public class PreviewExecutor
 
         try {
             input.transaction(task.getInputConfig(), new InputPlugin.Control() {
-                public List<CommitReport> run(final TaskSource inputTask, Schema inputSchema, int taskCount)
+                public List<CommitReport> run(final TaskSource inputTask, Schema inputSchema, final int taskCount)
                 {
                     Filters.transaction(filterPlugins, task.getFilterConfigs(), inputSchema, new Filters.Control() {
                         public void run(final List<TaskSource> filterTasks, final List<Schema> filterSchemas)
@@ -103,8 +103,16 @@ public class PreviewExecutor
 
                             PageOutput out = new SamplingPageOutput(task.getSampleRows(), outputSchema);
                             try {
-                                out = Filters.open(filterPlugins, filterTasks, filterSchemas, out);
-                                input.run(inputTask, inputSchema, 0, out);
+                                for (int taskIndex=0; taskIndex < taskCount; taskIndex++) {
+                                    try {
+                                        out = Filters.open(filterPlugins, filterTasks, filterSchemas, out);
+                                        input.run(inputTask, inputSchema, taskIndex, out);
+                                    } catch (NoSampleException ex) {
+                                        if (taskIndex == taskCount - 1) {
+                                            throw ex;
+                                        }
+                                    }
+                                }
                             } finally {
                                 out.close();
                             }
@@ -135,6 +143,11 @@ public class PreviewExecutor
             this.pages = new ArrayList<Page>();
         }
 
+        public int getRecordCount()
+        {
+            return recordCount;
+        }
+
         @Override
         public void add(Page page)
         {

data/embulk-core/src/main/java/org/embulk/spi/ColumnConfig.java

@@ -3,6 +3,7 @@ package org.embulk.spi;
 import java.util.Objects;
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonValue;
 import org.embulk.config.ConfigSource;
 import org.embulk.spi.type.Type;
 import org.embulk.spi.type.TimestampType;
@@ -25,35 +26,47 @@ public class ColumnConfig
         }
     }
 
-    @JsonCreator
-    public ColumnConfig(
-            @JsonProperty("name") String name,
-            @JsonProperty("type") Type type,
-            @JsonProperty("option") ConfigSource option)
+    public ColumnConfig(String name, Type type, ConfigSource option)
     {
         this.name = name;
         this.type = type;
         this.option = option;
     }
 
-    @JsonProperty("name")
+    @JsonCreator
+    public ColumnConfig(ConfigSource conf)
+    {
+        this.name = conf.get(String.class, "name");
+        this.type = conf.get(Type.class, "type");
+        this.option = conf.deepCopy();
+        option.remove("name");
+        option.remove("type");
+    }
+
     public String getName()
     {
         return name;
     }
 
-    @JsonProperty("type")
     public Type getType()
     {
         return type;
     }
 
-    @JsonProperty("option")
     public ConfigSource getOption()
     {
         return option;
     }
 
+    @JsonValue
+    public ConfigSource getConfigSource()
+    {
+        ConfigSource conf = option.deepCopy();
+        conf.set("name", name);
+        conf.set("type", type);
+        return conf;
+    }
+
     @Deprecated
     public String getFormat()
     {

data/embulk-core/src/main/java/org/embulk/spi/PageBuilder.java

@@ -39,6 +39,7 @@ public class PageBuilder
         this.schema = schema;
         this.columnOffsets = PageFormat.columnOffsets(schema);
         this.nullBitSet = new byte[PageFormat.nullBitSetSize(schema)];
+        Arrays.fill(nullBitSet, (byte) -1);
         this.fixedRecordSize = PageFormat.recordHeaderSize(schema) + PageFormat.totalColumnSize(schema);
         this.nextVariableLengthDataOffset = fixedRecordSize;
         newBuffer();
@@ -69,6 +70,11 @@ public class PageBuilder
         nullBitSet[columnIndex >>> 3] |= (1 << (columnIndex & 7));
     }
 
+    private void clearNull(int columnIndex)
+    {
+        nullBitSet[columnIndex >>> 3] &= ~(1 << (columnIndex & 7));
+    }
+
     public void setBoolean(Column column, boolean value)
     {
         // TODO check type?
@@ -78,6 +84,7 @@ public class PageBuilder
     public void setBoolean(int columnIndex, boolean value)
     {
         bufferSlice.setByte(getOffset(columnIndex), value ? (byte) 1 : (byte) 0);
+        clearNull(columnIndex);
     }
 
     public void setLong(Column column, long value)
@@ -89,6 +96,7 @@ public class PageBuilder
     public void setLong(int columnIndex, long value)
     {
         bufferSlice.setLong(getOffset(columnIndex), value);
+        clearNull(columnIndex);
     }
 
     public void setDouble(Column column, double value)
@@ -100,6 +108,7 @@ public class PageBuilder
     public void setDouble(int columnIndex, double value)
     {
         bufferSlice.setDouble(getOffset(columnIndex), value);
+        clearNull(columnIndex);
     }
 
     public void setString(Column column, String value)
@@ -119,6 +128,7 @@ public class PageBuilder
             bufferSlice.setInt(getOffset(columnIndex), index);
             stringReferenceSize += value.length() * 2 + 4;  // assuming size of char = size of byte * 2 + length
         }
+        clearNull(columnIndex);
     }
 
     public void setTimestamp(Column column, Timestamp value)
@@ -132,6 +142,7 @@ public class PageBuilder
         int offset = getOffset(columnIndex);
         bufferSlice.setLong(offset, value.getEpochSecond());
         bufferSlice.setInt(offset + 8, value.getNano());
+        clearNull(columnIndex);
     }
 
     private int getOffset(int columnIndex)
@@ -175,7 +186,7 @@ public class PageBuilder
 
         this.position += nextVariableLengthDataOffset;
         this.nextVariableLengthDataOffset = fixedRecordSize;
-        Arrays.fill(nullBitSet, (byte) 0);
+        Arrays.fill(nullBitSet, (byte) -1);
 
         // flush if next record will not fit in this buffer
         if (buffer.capacity() < position + nextVariableLengthDataOffset + stringReferenceSize) {

data/embulk-core/src/main/java/org/embulk/spi/util/DynamicPageBuilder.java

@@ -27,7 +27,7 @@ public class DynamicPageBuilder
 {
     private final PageBuilder pageBuilder;
     private final Schema schema;
-    private final List<DynamicColumnSetter> setters;
+    private final DynamicColumnSetter[] setters;
     private final Map<String, DynamicColumnSetter> columnLookup;
 
     public static interface BuilderTask
@@ -75,7 +75,7 @@ public class DynamicPageBuilder
             setters.add(setter);
             lookup.put(c.getName(), setter);
         }
-        this.setters = setters.build();
+        this.setters = setters.build().toArray(new DynamicColumnSetter[0]);
         this.columnLookup = lookup.build();
     }
 
@@ -86,15 +86,15 @@ public class DynamicPageBuilder
 
     public DynamicColumnSetter column(Column c)
     {
-        return setters.get(c.getIndex());
+        return setters[c.getIndex()];
     }
 
     public DynamicColumnSetter column(int index)
     {
-        if (index < 0 || setters.size() <= index) {
+        if (index < 0 || setters.length <= index) {
             throw new DynamicColumnNotFoundException("Column index '"+index+"' is not exist");
         }
-        return setters.get(index);
+        return setters[index];
     }
 
     public DynamicColumnSetter lookupColumn(String columnName)
@@ -108,10 +108,10 @@ public class DynamicPageBuilder
 
     public DynamicColumnSetter columnOrSkip(int index)
     {
-        if (index < 0 || setters.size() <= index) {
+        if (index < 0 || setters.length <= index) {
             return SkipColumnSetter.get();
         }
-        return setters.get(index);
+        return setters[index];
     }
 
     public DynamicColumnSetter columnOrSkip(String columnName)
@@ -126,10 +126,10 @@ public class DynamicPageBuilder
     // for jruby
     protected DynamicColumnSetter columnOrNull(int index)
     {
-        if (index < 0 || setters.size() <= index) {
+        if (index < 0 || setters.length <= index) {
             return null;
         }
-        return setters.get(index);
+        return setters[index];
     }
 
     // for jruby

data/embulk-core/src/main/java/org/embulk/spi/util/dynamic/AbstractDynamicColumnSetter.java

@@ -40,9 +40,9 @@ public abstract class AbstractDynamicColumnSetter
 
     public abstract void set(Timestamp value);
 
-    public IRubyObject setRubyObject(IRubyObject rubyObject)
+    public void setRubyObject(IRubyObject rubyObject)
     {
-        if (rubyObject instanceof RubyNil) {
+        if (rubyObject == null || rubyObject instanceof RubyNil) {
             setNull();
         } else if (rubyObject instanceof RubyBoolean) {
             RubyBoolean b = (RubyBoolean) rubyObject;
@@ -75,6 +75,5 @@ public abstract class AbstractDynamicColumnSetter
         } else {
             throw rubyObject.getRuntime().newTypeError("cannot convert instance of " + rubyObject.getMetaClass() + " to nil, true, false, Integer, Float, String, or Time");
         }
-        return rubyObject.getRuntime().getNil();
     }
 }

data/embulk-core/src/main/java/org/embulk/spi/util/dynamic/SkipColumnSetter.java

@@ -47,8 +47,6 @@ public class SkipColumnSetter
     { }
 
     @Override
-    public IRubyObject setRubyObject(IRubyObject rubyObject)
-    {
-        return rubyObject.getRuntime().getNil();
-    }
+    public void setRubyObject(IRubyObject rubyObject)
+    { }
 }

data/embulk-docs/src/built-in.rst

@@ -248,12 +248,12 @@ Options
 +====================+==========+===================================================+============================+
 | path\_prefix       | string   | Path prefix of the output files                   | required                   |
 +--------------------+----------+---------------------------------------------------+----------------------------+
-| sequence\_format   | string   | Format of the sequence number of the output files | ``.%03d.%02d`` by default  |
+| sequence\_format   | string   | Format of the sequence number of the output files | ``%03d.%02d.`` by default |
 +--------------------+----------+---------------------------------------------------+----------------------------+
-| file\_ext          | string   | Path suffix of the output files                   | required                   |
+| file\_ext          | string   | Path suffix of the output files (e.g. ``"csv"``)  | required                   |
 +--------------------+----------+---------------------------------------------------+----------------------------+
 
-For example, if you set ``path_prefix: /path/to/output``, ``sequence_format: ".%03d.%02d"``, and ``file_ext: .csv``, name of the output files will be as following:
+For example, if you set ``path_prefix: /path/to/output/sample_``, ``sequence_format: "%03d.%02d."``, and ``file_ext: csv``, name of the output files will be as following:
 
 ::
 
@@ -261,10 +261,10 @@ For example, if you set ``path_prefix: /path/to/output``, ``sequence_format: ".%
     `-- path
         `-- to
             `-- output
-                |-- sample.01.000.csv
-                |-- sample.02.000.csv
-                |-- sample.03.000.csv
-                |-- sample.04.000.csv
+                |-- sample_01.000.csv
+                |-- sample_02.000.csv
+                |-- sample_03.000.csv
+                |-- sample_04.000.csv
 
 ``sequence_format`` formats task index and sequence number in a task.
 
@@ -275,8 +275,8 @@ Example
 
     out:
       type: file
-      path_prefix: /path/to/output/sample
-      file_ext: .csv
+      path_prefix: /path/to/output/sample_
+      file_ext: csv
       formatter:
         ...
 

data/embulk-docs/src/release.rst

@@ -4,6 +4,7 @@ Release Notes
 .. toctree::
     :maxdepth: 1
 
+    release/release-0.6.15
     release/release-0.6.14
     release/release-0.6.13
     release/release-0.6.12

data/embulk-docs/src/release/release-0.6.15.rst

@@ -0,0 +1,26 @@
+Release 0.6.15
+==================================
+
+Plugin API
+------------------
+
+* PageBuilder sets null to a column if setXxx method is not called to the column.
+
+
+Built-in plugins
+------------------
+
+* **IMPORTANT**: Because of default value of ``sequence_format`` option, ``formatter-csv`` adds ``.`` (dot) before ``file_ext`` rather than after ``path_prefix``. If you're using ``out: type: file``, see the document of file output plugin at :doc:`../built-in`.
+
+
+General Changes
+------------------
+
+* ``preview`` reads next file (task) when the first file (task) is empty.
+* Fixed serialize/deserialization bug of ColumnConfig.
+* Fixed NullPointerException caused when ``PageBuilder#setRubyObject(IRubyObject)`` is called with ``nil``.
+
+
+Release Date
+------------------
+2015-06-29

data/embulk-standards/src/main/java/org/embulk/standards/CsvFormatterPlugin.java

@@ -112,6 +112,7 @@ public class CsvFormatterPlugin
                 Optional<TimestampColumnOption> option = Optional.fromNullable(columnOptions.get(column.getName()));
                 formatters[i] = new TimestampFormatter(formatterTask, option);
             }
+            i++;
         }
         return formatters;
     }

data/embulk-standards/src/main/java/org/embulk/standards/CsvParserPlugin.java

@@ -88,13 +88,6 @@ public class CsvParserPlugin
         @Config("allow_extra_columns")
         @ConfigDefault("false")
         public boolean getAllowExtraColumns();
-
-        @Config("error")
-        @ConfigDefault("{\"type\": \"warning\"}")
-        public ConfigSource getErrorConfig();
-
-        public TaskSource getErorrTaskSource();
-        public void setErrorTaskSource(TaskSource errorTask);
     }
 
     public interface TimestampColumnOption

data/embulk-standards/src/main/java/org/embulk/standards/LocalFileOutputPlugin.java

@@ -36,7 +36,7 @@ public class LocalFileOutputPlugin
         public String getFileNameExtension();
 
         @Config("sequence_format")
-        @ConfigDefault("\".%03d.%02d\"")
+        @ConfigDefault("\"%03d.%02d.\"")
         public String getSequenceFormat();
     }
 

data/lib/embulk/version.rb

@@ -1,3 +1,3 @@
 module Embulk
-  VERSION = '0.6.14'
+  VERSION = '0.6.15'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk
 version: !ruby/object:Gem::Version
-  version: 0.6.14
+  version: 0.6.15
 platform: ruby
 authors:
 - Sadayuki Furuhashi
@@ -318,6 +318,7 @@ files:
 - embulk-docs/src/release/release-0.6.12.rst
 - embulk-docs/src/release/release-0.6.13.rst
 - embulk-docs/src/release/release-0.6.14.rst
+- embulk-docs/src/release/release-0.6.15.rst
 - embulk-docs/src/release/release-0.6.2.rst
 - embulk-docs/src/release/release-0.6.3.rst
 - embulk-docs/src/release/release-0.6.4.rst
@@ -434,8 +435,8 @@ files:
 - classpath/bval-jsr303-0.5.jar
 - classpath/commons-beanutils-core-1.8.3.jar
 - classpath/commons-lang3-3.1.jar
-- classpath/embulk-core-0.6.14.jar
-- classpath/embulk-standards-0.6.14.jar
+- classpath/embulk-core-0.6.15.jar
+- classpath/embulk-standards-0.6.15.jar
 - classpath/guava-18.0.jar
 - classpath/guice-4.0.jar
 - classpath/guice-multibindings-4.0.jar