embulk 0.6.12 → 0.6.13

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c73174238cf469f8b78f7e7225d35e2ae9c5d0c0
-  data.tar.gz: c3cb6c769f05c1ed0d636b84232018e97a0a955b
+  metadata.gz: 98a09da0f4c6425b69f3094d7c5b105eea0e7d36
+  data.tar.gz: 26383243a036baa447c9bf3ab28d554e2a5fc9e5
 SHA512:
-  metadata.gz: 1c17333b98eb2d249a909d4b7603718090a8d0cd4fc8bb087b7c4c3ad712cf05c7e2db4c51bd0947004fc40d0f0052e3eeac33bb569116e16031d5a0aa256809
-  data.tar.gz: c439b2981678bcb6698173e535f63baa72411f78e1099b89888a25bd96c758f8d9947622b78cc2506b9369d1086da9598a59a462fa5d15ce6c5569fd06602dca
+  metadata.gz: acc2663c2fc200aeb2aeffc5e3156cdffbfbd8321a87bfbac1fa125e46f5c2006bdc9cdf63484b0157c622c32cf8be81bec336a029bd6c655e866537bc689578
+  data.tar.gz: 41696a15d2d1cd12efcd1d35574b19ddeeaa931f386f8c7093ab123328d460d5b1d82a94880f38c679b8f690edd43c7d5fcf679cee08b9ba019236b36460a9c7

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    embulk (0.4.5)
+    embulk (0.6.12)
 
 GEM
   remote: https://rubygems.org/

data/build.gradle

@@ -11,7 +11,7 @@ def release_projects = [project(":embulk-core"), project(":embulk-standards")]
 
 allprojects {
     group = 'org.embulk'
-    version = '0.6.12'
+    version = '0.6.13'
 
     ext {
         jrubyVersion = '1.7.19'

data/embulk-core/src/main/java/org/embulk/spi/Exec.java

@@ -33,7 +33,6 @@ public class Exec
     {
         ExecSession session = Exec.session.get();
         if (session == null) {
-            new NullPointerException().printStackTrace();
             throw new NullPointerException("Exec is used outside of Exec.doWith");
         }
         return session;

data/embulk-core/src/main/java/org/embulk/spi/time/TimestampParser.java

@@ -35,8 +35,7 @@ public class TimestampParser
         return defaultTimeZone;
     }
 
-    // TODO this is still private because this might need current time
-    private TimestampParser(ScriptingContainer jruby, String format, DateTimeZone defaultTimeZone)
+    public TimestampParser(ScriptingContainer jruby, String format, DateTimeZone defaultTimeZone)
     {
         JRubyTimeParserHelperFactory helperFactory = (JRubyTimeParserHelperFactory) jruby.runScriptlet("Embulk::Java::TimeParserHelper::Factory.new");
         // TODO get default current time from ExecTask.getExecTimestamp

data/embulk-core/src/main/java/org/embulk/spi/util/DynamicColumnNotFoundException.java

@@ -0,0 +1,10 @@
+package org.embulk.spi.util;
+
+public class DynamicColumnNotFoundException
+        extends RuntimeException
+{
+    public DynamicColumnNotFoundException(String message)
+    {
+        super(message);
+    }
+}

data/embulk-core/src/main/java/org/embulk/spi/util/DynamicColumnSetter.java

@@ -0,0 +1,18 @@
+package org.embulk.spi.util;
+
+import org.embulk.spi.time.Timestamp;
+
+public interface DynamicColumnSetter
+{
+    public void setNull();
+
+    public void set(boolean value);
+
+    public void set(long value);
+
+    public void set(double value);
+
+    public void set(String value);
+
+    public void set(Timestamp value);
+}

data/embulk-core/src/main/java/org/embulk/spi/util/DynamicColumnSetterFactory.java

@@ -0,0 +1,94 @@
+package org.embulk.spi.util;
+
+import org.joda.time.DateTimeZone;
+import org.embulk.config.ConfigSource;
+import org.embulk.spi.type.Type;
+import org.embulk.spi.type.BooleanType;
+import org.embulk.spi.type.LongType;
+import org.embulk.spi.type.DoubleType;
+import org.embulk.spi.type.StringType;
+import org.embulk.spi.type.TimestampType;
+import org.embulk.spi.util.dynamic.BooleanColumnSetter;
+import org.embulk.spi.util.dynamic.LongColumnSetter;
+import org.embulk.spi.util.dynamic.DoubleColumnSetter;
+import org.embulk.spi.util.dynamic.StringColumnSetter;
+import org.embulk.spi.util.dynamic.TimestampColumnSetter;
+import org.embulk.spi.util.dynamic.DefaultValueSetter;
+import org.embulk.spi.util.dynamic.NullDefaultValueSetter;
+import org.embulk.spi.time.TimestampFormatter;
+import org.embulk.spi.time.TimestampParser;
+import org.embulk.spi.time.TimestampFormat;
+import org.embulk.spi.Column;
+import org.embulk.spi.PageBuilder;
+import org.embulk.config.ConfigException;
+
+public class DynamicColumnSetterFactory
+{
+    private final DefaultValueSetter defaultValue;
+    private final DynamicPageBuilder.BuilderTask task;
+
+    DynamicColumnSetterFactory(
+            DynamicPageBuilder.BuilderTask task,
+            DefaultValueSetter defaultValue)
+    {
+        this.defaultValue = defaultValue;
+        this.task = task;
+    }
+
+    public static DefaultValueSetter nullDefaultValue()
+    {
+        return new NullDefaultValueSetter();
+    }
+
+    public DynamicColumnSetter newColumnSetter(PageBuilder pageBuilder, Column column)
+    {
+        Type type = column.getType();
+        if (type instanceof BooleanType) {
+            return new BooleanColumnSetter(pageBuilder, column, defaultValue);
+        } else if (type instanceof LongType) {
+            return new LongColumnSetter(pageBuilder, column, defaultValue);
+        } else if (type instanceof DoubleType) {
+            return new DoubleColumnSetter(pageBuilder, column, defaultValue);
+        } else if (type instanceof StringType) {
+            TimestampFormatter formatter = new TimestampFormatter(task.getJRuby(),
+                    getTimestampFormat(column).getFormat(), getTimeZone(column));
+            return new StringColumnSetter(pageBuilder, column, defaultValue, formatter);
+        } else if (type instanceof TimestampType) {
+            // TODO use flexible time format like Ruby's Time.parse
+            TimestampParser parser = new TimestampParser(task.getJRuby(),
+                    getTimestampFormat(column).getFormat(), getTimeZone(column));
+            return new TimestampColumnSetter(pageBuilder, column, defaultValue, parser);
+        }
+        throw new ConfigException("Unknown column type: "+type);
+    }
+
+    private TimestampFormat getTimestampFormat(Column column)
+    {
+        DynamicPageBuilder.ColumnOption option = getColumnOption(column);
+        if (option != null) {
+            return option.getTimestampFormat();
+        } else {
+            return new TimestampFormat("%Y-%m-%d %H:%M:%S.%6N");
+        }
+    }
+
+    private DateTimeZone getTimeZone(Column column)
+    {
+        DynamicPageBuilder.ColumnOption option = getColumnOption(column);
+        if (option != null) {
+            return option.getTimeZone().or(task.getDefaultTimeZone());
+        } else {
+            return task.getDefaultTimeZone();
+        }
+    }
+
+    private DynamicPageBuilder.ColumnOption getColumnOption(Column column)
+    {
+        ConfigSource option = task.getColumnOptions().get(column.getName());
+        if (option != null) {
+            return option.loadConfig(DynamicPageBuilder.ColumnOption.class);
+        } else {
+            return null;
+        }
+    }
+}

data/embulk-core/src/main/java/org/embulk/spi/util/DynamicPageBuilder.java

@@ -0,0 +1,161 @@
+package org.embulk.spi.util;
+
+import java.util.List;
+import java.util.Map;
+import com.google.common.base.Optional;
+import org.joda.time.DateTimeZone;
+import org.jruby.embed.ScriptingContainer;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.config.ConfigInject;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.Task;
+import org.embulk.spi.Schema;
+import org.embulk.spi.Column;
+import org.embulk.spi.BufferAllocator;
+import org.embulk.spi.PageBuilder;
+import org.embulk.spi.PageOutput;
+import org.embulk.spi.time.TimestampFormatter;
+import org.embulk.spi.time.TimestampParser;
+import org.embulk.spi.time.TimestampFormat;
+import org.embulk.spi.util.dynamic.SkipColumnSetter;
+
+public class DynamicPageBuilder
+        implements AutoCloseable
+{
+    private final PageBuilder pageBuilder;
+    private final Schema schema;
+    private final List<DynamicColumnSetter> setters;
+    private final Map<String, DynamicColumnSetter> columnLookup;
+
+    public static interface BuilderTask
+            extends Task
+    {
+        @Config("default_timezone")
+        @ConfigDefault("\"UTC\"")
+        public DateTimeZone getDefaultTimeZone();
+
+        @Config("column_options")
+        @ConfigDefault("{}")
+        public Map<String, ConfigSource> getColumnOptions();
+
+        // required by TimestampFormatter
+        @ConfigInject
+        public ScriptingContainer getJRuby();
+    }
+
+    public static interface ColumnOption
+            extends Task
+    {
+        @Config("timestamp_format")
+        @ConfigDefault("\"%Y-%m-%d %H:%M:%S.%6N\"")
+        public TimestampFormat getTimestampFormat();
+
+        @Config("timezone")
+        @ConfigDefault("null")
+        public Optional<DateTimeZone> getTimeZone();
+    }
+
+    public DynamicPageBuilder(BuilderTask task,
+            BufferAllocator allocator, Schema schema, PageOutput output)
+    {
+        this.pageBuilder = new PageBuilder(allocator, schema, output);
+        this.schema = schema;
+
+        // TODO configurable default value
+        DynamicColumnSetterFactory factory = new DynamicColumnSetterFactory(task,
+                DynamicColumnSetterFactory.nullDefaultValue());
+
+        ImmutableList.Builder<DynamicColumnSetter> setters = ImmutableList.builder();
+        ImmutableMap.Builder<String, DynamicColumnSetter> lookup = ImmutableMap.builder();
+        for (Column c : schema.getColumns()) {
+            DynamicColumnSetter setter = factory.newColumnSetter(pageBuilder, c);
+            setters.add(setter);
+            lookup.put(c.getName(), setter);
+        }
+        this.setters = setters.build();
+        this.columnLookup = lookup.build();
+    }
+
+    public List<Column> getColumns()
+    {
+        return schema.getColumns();
+    }
+
+    public DynamicColumnSetter column(Column c)
+    {
+        return setters.get(c.getIndex());
+    }
+
+    public DynamicColumnSetter column(int index)
+    {
+        if (index < 0 || setters.size() <= index) {
+            throw new DynamicColumnNotFoundException("Column index '"+index+"' is not exist");
+        }
+        return setters.get(index);
+    }
+
+    public DynamicColumnSetter lookupColumn(String columnName)
+    {
+        DynamicColumnSetter setter = columnLookup.get(columnName);
+        if (setter == null) {
+            throw new DynamicColumnNotFoundException("Column '"+columnName+"' is not exist");
+        }
+        return setter;
+    }
+
+    public DynamicColumnSetter columnOrSkip(int index)
+    {
+        if (index < 0 || setters.size() <= index) {
+            return SkipColumnSetter.get();
+        }
+        return setters.get(index);
+    }
+
+    public DynamicColumnSetter columnOrSkip(String columnName)
+    {
+        DynamicColumnSetter setter = columnLookup.get(columnName);
+        if (setter == null) {
+            return SkipColumnSetter.get();
+        }
+        return setter;
+    }
+
+    // for jruby
+    protected DynamicColumnSetter columnOrNull(int index)
+    {
+        if (index < 0 || setters.size() <= index) {
+            return null;
+        }
+        return setters.get(index);
+    }
+
+    // for jruby
+    protected DynamicColumnSetter columnOrNull(String columnName)
+    {
+        return columnLookup.get(columnName);
+    }
+
+    public void addRecord()
+    {
+        pageBuilder.addRecord();
+    }
+
+    public void flush()
+    {
+        pageBuilder.flush();
+    }
+
+    public void finish()
+    {
+        pageBuilder.finish();
+    }
+
+    @Override
+    public void close()
+    {
+        pageBuilder.close();
+    }
+}

data/embulk-core/src/main/java/org/embulk/spi/util/dynamic/AbstractDynamicColumnSetter.java

@@ -0,0 +1,80 @@
+package org.embulk.spi.util.dynamic;
+
+import org.jruby.runtime.builtin.IRubyObject;
+import org.jruby.RubyNil;
+import org.jruby.RubyBoolean;
+import org.jruby.RubyInteger;
+import org.jruby.RubyFloat;
+import org.jruby.RubyString;
+import org.jruby.RubyTime;
+import org.jruby.exceptions.RaiseException;
+import org.embulk.spi.PageBuilder;
+import org.embulk.spi.Column;
+import org.embulk.spi.util.DynamicColumnSetter;
+import org.embulk.spi.time.Timestamp;
+
+public abstract class AbstractDynamicColumnSetter
+        implements DynamicColumnSetter
+{
+    protected final PageBuilder pageBuilder;
+    protected final Column column;
+    protected final DefaultValueSetter defaultValue;
+
+    protected AbstractDynamicColumnSetter(PageBuilder pageBuilder, Column column,
+            DefaultValueSetter defaultValue)
+    {
+        this.pageBuilder = pageBuilder;
+        this.column = column;
+        this.defaultValue = defaultValue;
+    }
+
+    public abstract void setNull();
+
+    public abstract void set(boolean value);
+
+    public abstract void set(long value);
+
+    public abstract void set(double value);
+
+    public abstract void set(String value);
+
+    public abstract void set(Timestamp value);
+
+    public IRubyObject setRubyObject(IRubyObject rubyObject)
+    {
+        if (rubyObject instanceof RubyNil) {
+            setNull();
+        } else if (rubyObject instanceof RubyBoolean) {
+            RubyBoolean b = (RubyBoolean) rubyObject;
+            set(b.isTrue());
+        } else if (rubyObject instanceof RubyInteger) {
+            RubyInteger i = (RubyInteger) rubyObject;
+            try {
+                set(i.getLongValue());
+            } catch (RaiseException ex) {
+                if ("RangeError".equals(ex.getException().getMetaClass().getBaseName())) {
+                    // integer is too large
+                    throw ex;  //TODO setDefaultValue();
+                } else {
+                    throw ex;
+                }
+            }
+        } else if (rubyObject instanceof RubyFloat) {
+            RubyFloat f = (RubyFloat) rubyObject;
+            set(f.getDoubleValue());
+        } else if (rubyObject instanceof RubyString) {
+            RubyString s = (RubyString) rubyObject;
+            set(s.asJavaString());
+        } else if (rubyObject instanceof RubyTime) {
+            RubyTime time = (RubyTime) rubyObject;
+            long msec = time.getDateTime().getMillis();
+            long nsec = time.getNSec();
+            long sec = msec / 1000 + nsec / 1000000000;
+            int nano = (int) ((msec % 1000) * 1000000 + nsec % 1000000000);
+            set(Timestamp.ofEpochSecond(sec, nano));
+        } else {
+            throw rubyObject.getRuntime().newTypeError("cannot convert instance of " + rubyObject.getMetaClass() + " to nil, true, false, Integer, Float, String, or Time");
+        }
+        return rubyObject.getRuntime().getNil();
+    }
+}

data/embulk-core/src/main/java/org/embulk/spi/util/dynamic/BooleanColumnSetter.java

@@ -0,0 +1,64 @@
+package org.embulk.spi.util.dynamic;
+
+import com.google.common.collect.ImmutableSet;
+import org.embulk.spi.Column;
+import org.embulk.spi.PageBuilder;
+import org.embulk.spi.time.Timestamp;
+
+public class BooleanColumnSetter
+        extends AbstractDynamicColumnSetter
+{
+    private static final ImmutableSet<String> TRUE_STRINGS =
+        ImmutableSet.of(
+                "true", "True", "TRUE",
+                "yes", "Yes", "YES",
+                "t", "T", "y", "Y",
+                "on", "On", "ON",
+                "1");
+
+    public BooleanColumnSetter(PageBuilder pageBuilder, Column column,
+            DefaultValueSetter defaultValue)
+    {
+        super(pageBuilder, column, defaultValue);
+    }
+
+    @Override
+    public void setNull()
+    {
+        pageBuilder.setNull(column);
+    }
+
+    @Override
+    public void set(boolean v)
+    {
+        pageBuilder.setBoolean(column, v);
+    }
+
+    @Override
+    public void set(long v)
+    {
+        pageBuilder.setBoolean(column, v > 0);
+    }
+
+    @Override
+    public void set(double v)
+    {
+        pageBuilder.setBoolean(column, v > 0.0);
+    }
+
+    @Override
+    public void set(String v)
+    {
+        if (TRUE_STRINGS.contains(v)) {
+            pageBuilder.setBoolean(column, true);
+        } else {
+            defaultValue.setDouble(pageBuilder, column);
+        }
+    }
+
+    @Override
+    public void set(Timestamp v)
+    {
+        defaultValue.setBoolean(pageBuilder, column);
+    }
+}

data/embulk-core/src/main/java/org/embulk/spi/util/dynamic/DefaultValueSetter.java

@@ -0,0 +1,18 @@
+package org.embulk.spi.util.dynamic;
+
+import org.embulk.spi.PageBuilder;
+import org.embulk.spi.Column;
+import org.embulk.spi.time.Timestamp;
+
+public interface DefaultValueSetter
+{
+    public void setBoolean(PageBuilder pageBuilder, Column c);
+
+    public void setLong(PageBuilder pageBuilder, Column c);
+
+    public void setDouble(PageBuilder pageBuilder, Column c);
+
+    public void setString(PageBuilder pageBuilder, Column c);
+
+    public void setTimestamp(PageBuilder pageBuilder, Column c);
+}

data/embulk-core/src/main/java/org/embulk/spi/util/dynamic/DoubleColumnSetter.java

@@ -0,0 +1,61 @@
+package org.embulk.spi.util.dynamic;
+
+import java.math.RoundingMode;
+import org.embulk.spi.Column;
+import org.embulk.spi.PageBuilder;
+import org.embulk.spi.time.Timestamp;
+
+public class DoubleColumnSetter
+        extends AbstractDynamicColumnSetter
+{
+    public DoubleColumnSetter(PageBuilder pageBuilder, Column column,
+            DefaultValueSetter defaultValue)
+    {
+        super(pageBuilder, column, defaultValue);
+    }
+
+    @Override
+    public void setNull()
+    {
+        pageBuilder.setNull(column);
+    }
+
+    @Override
+    public void set(boolean v)
+    {
+        pageBuilder.setDouble(column, v ? 1.0 : 0.0);
+    }
+
+    @Override
+    public void set(long v)
+    {
+        pageBuilder.setDouble(column, (double) v);
+    }
+
+    @Override
+    public void set(double v)
+    {
+        pageBuilder.setDouble(column, v);
+    }
+
+    @Override
+    public void set(String v)
+    {
+        double dv;
+        try {
+            dv = Double.parseDouble(v);
+        } catch (NumberFormatException e) {
+            defaultValue.setDouble(pageBuilder, column);
+            return;
+        }
+        pageBuilder.setDouble(column, dv);
+    }
+
+    @Override
+    public void set(Timestamp v)
+    {
+        double sec = (double) v.getEpochSecond();
+        double frac = v.getNano() / 1000000000.0;
+        pageBuilder.setDouble(column, sec + frac);
+    }
+}

data/embulk-core/src/main/java/org/embulk/spi/util/dynamic/LongColumnSetter.java

@@ -0,0 +1,69 @@
+package org.embulk.spi.util.dynamic;
+
+import java.math.RoundingMode;
+import com.google.common.math.DoubleMath;
+import org.embulk.spi.Column;
+import org.embulk.spi.PageBuilder;
+import org.embulk.spi.time.Timestamp;
+
+public class LongColumnSetter
+        extends AbstractDynamicColumnSetter
+{
+    public LongColumnSetter(PageBuilder pageBuilder, Column column,
+            DefaultValueSetter defaultValue)
+    {
+        super(pageBuilder, column, defaultValue);
+    }
+
+    @Override
+    public void setNull()
+    {
+        pageBuilder.setNull(column);
+    }
+
+    @Override
+    public void set(boolean v)
+    {
+        pageBuilder.setLong(column, v ? 1L : 0L);
+    }
+
+    @Override
+    public void set(long v)
+    {
+        pageBuilder.setLong(column, v);
+    }
+
+    @Override
+    public void set(double v)
+    {
+        long lv;
+        try {
+            // TODO configurable rounding mode
+            lv = DoubleMath.roundToLong(v, RoundingMode.HALF_UP);
+        } catch (ArithmeticException ex) {
+            // NaN / Infinite / -Infinite
+            defaultValue.setLong(pageBuilder, column);
+            return;
+        }
+        pageBuilder.setLong(column, lv);
+    }
+
+    @Override
+    public void set(String v)
+    {
+        long lv;
+        try {
+            lv = Long.parseLong(v);
+        } catch (NumberFormatException e) {
+            defaultValue.setLong(pageBuilder, column);
+            return;
+        }
+        pageBuilder.setLong(column, lv);
+    }
+
+    @Override
+    public void set(Timestamp v)
+    {
+        pageBuilder.setDouble(column, v.getEpochSecond());
+    }
+}

data/embulk-core/src/main/java/org/embulk/spi/util/dynamic/NullDefaultValueSetter.java

@@ -0,0 +1,34 @@
+package org.embulk.spi.util.dynamic;
+
+import org.embulk.spi.PageBuilder;
+import org.embulk.spi.Column;
+import org.embulk.spi.time.Timestamp;
+
+public class NullDefaultValueSetter
+        implements DefaultValueSetter
+{
+    public void setBoolean(PageBuilder pageBuilder, Column c)
+    {
+        pageBuilder.setNull(c);
+    }
+
+    public void setLong(PageBuilder pageBuilder, Column c)
+    {
+        pageBuilder.setNull(c);
+    }
+
+    public void setDouble(PageBuilder pageBuilder, Column c)
+    {
+        pageBuilder.setNull(c);
+    }
+
+    public void setString(PageBuilder pageBuilder, Column c)
+    {
+        pageBuilder.setNull(c);
+    }
+
+    public void setTimestamp(PageBuilder pageBuilder, Column c)
+    {
+        pageBuilder.setNull(c);
+    }
+}

data/embulk-core/src/main/java/org/embulk/spi/util/dynamic/SkipColumnSetter.java

@@ -0,0 +1,54 @@
+package org.embulk.spi.util.dynamic;
+
+import org.jruby.runtime.builtin.IRubyObject;
+import org.embulk.spi.PageBuilder;
+import org.embulk.spi.Column;
+import org.embulk.spi.time.Timestamp;
+import org.embulk.spi.time.TimestampParser;
+import org.embulk.spi.time.TimestampParseException;
+
+public class SkipColumnSetter
+        extends AbstractDynamicColumnSetter
+{
+    private static final SkipColumnSetter instance = new SkipColumnSetter();
+
+    public static SkipColumnSetter get()
+    {
+        return instance;
+    }
+
+    private SkipColumnSetter()
+    {
+        super(null, null, null);
+    }
+
+    @Override
+    public void setNull()
+    { }
+
+    @Override
+    public void set(boolean v)
+    { }
+
+    @Override
+    public void set(long v)
+    { }
+
+    @Override
+    public void set(double v)
+    { }
+
+    @Override
+    public void set(String v)
+    { }
+
+    @Override
+    public void set(Timestamp v)
+    { }
+
+    @Override
+    public IRubyObject setRubyObject(IRubyObject rubyObject)
+    {
+        return rubyObject.getRuntime().getNil();
+    }
+}

data/embulk-core/src/main/java/org/embulk/spi/util/dynamic/StringColumnSetter.java

@@ -0,0 +1,56 @@
+package org.embulk.spi.util.dynamic;
+
+import org.embulk.spi.PageBuilder;
+import org.embulk.spi.Column;
+import org.embulk.spi.time.Timestamp;
+import org.embulk.spi.time.TimestampFormatter;
+
+public class StringColumnSetter
+        extends AbstractDynamicColumnSetter
+{
+    private final TimestampFormatter timestampFormatter;
+
+    public StringColumnSetter(PageBuilder pageBuilder, Column column,
+            DefaultValueSetter defaultValue,
+            TimestampFormatter timestampFormatter)
+    {
+        super(pageBuilder, column, defaultValue);
+        this.timestampFormatter = timestampFormatter;
+    }
+
+    @Override
+    public void setNull()
+    {
+        pageBuilder.setNull(column);
+    }
+
+    @Override
+    public void set(boolean v)
+    {
+        pageBuilder.setString(column, Boolean.toString(v));
+    }
+
+    @Override
+    public void set(long v)
+    {
+        pageBuilder.setString(column, Long.toString(v));
+    }
+
+    @Override
+    public void set(double v)
+    {
+        pageBuilder.setString(column, Double.toString(v));
+    }
+
+    @Override
+    public void set(String v)
+    {
+        pageBuilder.setString(column, v);
+    }
+
+    @Override
+    public void set(Timestamp v)
+    {
+        pageBuilder.setString(column, timestampFormatter.format(v));
+    }
+}

data/embulk-core/src/main/java/org/embulk/spi/util/dynamic/TimestampColumnSetter.java

@@ -0,0 +1,64 @@
+package org.embulk.spi.util.dynamic;
+
+import org.embulk.spi.PageBuilder;
+import org.embulk.spi.Column;
+import org.embulk.spi.time.Timestamp;
+import org.embulk.spi.time.TimestampParser;
+import org.embulk.spi.time.TimestampParseException;
+
+public class TimestampColumnSetter
+        extends AbstractDynamicColumnSetter
+{
+    private final TimestampParser timestampParser;
+
+    public TimestampColumnSetter(PageBuilder pageBuilder, Column column,
+            DefaultValueSetter defaultValue,
+            TimestampParser timestampParser)
+    {
+        super(pageBuilder, column, defaultValue);
+        this.timestampParser = timestampParser;
+    }
+
+    @Override
+    public void setNull()
+    {
+        pageBuilder.setNull(column);
+    }
+
+    @Override
+    public void set(boolean v)
+    {
+        defaultValue.setTimestamp(pageBuilder, column);
+    }
+
+    @Override
+    public void set(long v)
+    {
+        pageBuilder.setTimestamp(column, Timestamp.ofEpochSecond(v));
+    }
+
+    @Override
+    public void set(double v)
+    {
+        long sec = (long) v;
+        int nsec = (int) ((v - (double) sec) * 1000000000);
+        pageBuilder.setTimestamp(column, Timestamp.ofEpochSecond(sec, nsec));
+        defaultValue.setTimestamp(pageBuilder, column);
+    }
+
+    @Override
+    public void set(String v)
+    {
+        try {
+            pageBuilder.setTimestamp(column, timestampParser.parse(v));
+        } catch (TimestampParseException e) {
+            defaultValue.setTimestamp(pageBuilder, column);
+        }
+    }
+
+    @Override
+    public void set(Timestamp v)
+    {
+        pageBuilder.setTimestamp(column, v);
+    }
+}

data/embulk-docs/src/release.rst

@@ -4,6 +4,7 @@ Release Notes
 .. toctree::
     :maxdepth: 1
 
+    release/release-0.6.13
     release/release-0.6.12
     release/release-0.6.11
     release/release-0.6.10

data/embulk-docs/src/release/release-0.6.13.rst

@@ -0,0 +1,23 @@
+Release 0.6.13
+==================================
+
+Plugin API
+------------------
+
+* Aded spi.util.DynamicPageBuilder utility class. This is a wrapper of PageBuilder which converts types automatically.
+* ``page_builder`` used by Ruby plugins are now backed by DynamicPageBuilder. This means that Ruby plugins don't need type conversion code such as to_i or to_s any more.
+
+Built-in plugins
+------------------
+
+* ``guess-csv`` plugin guesses ``trim_if_not_quoted`` option. This works well if a CSV file includes integers with extra spaces (such as CSV files exported by Excel).
+* Fixed a problem where ``guess-csv`` plugin guessed type of a column as "string" if it includes non-empty null_string.
+
+General Changes
+------------------
+
+* "-b" option installs bundler automatically if it is not installed rather shows "no such file to load -- bundler" error message.
+
+Release Date
+------------------
+2015-06-25

data/embulk-standards/src/main/java/org/embulk/standards/CsvTokenizer.java

@@ -80,14 +80,20 @@ public class CsvTokenizer
         return input.nextFile();
     }
 
+    // used by guess-csv
     public boolean nextRecord()
+    {
+        return nextRecord(true);
+    }
+
+    public boolean nextRecord(boolean skipEmptyLine)
     {
         // If at the end of record, read the next line and initialize the state
         if (recordState != RecordState.END) {
             throw new TooManyColumnsException("Too many columns");
         }
 
-        boolean hasNext = nextLine(true);
+        boolean hasNext = nextLine(skipEmptyLine);
         if (hasNext) {
             recordState = RecordState.NOT_END;
             return true;

data/lib/embulk/command/embulk.rb

@@ -18,7 +18,11 @@ if bundle_path
   Gem.clear_paths  # force rubygems to reload GEM_HOME
 
   ENV['BUNDLE_GEMFILE'] = File.expand_path File.join(bundle_path, "Gemfile")
-  require 'bundler'
+  begin
+    require 'bundler'
+  rescue LoadError => e
+    raise "#{e}\nBundler is not installed. Did you run \`$ embulk bundle #{bundle_path}\` ?"
+  end
   Bundler.load.setup_environment
   require 'bundler/setup'
   # since here, `require` may load files of different (newer) embulk versions

data/lib/embulk/command/embulk_run.rb

@@ -261,7 +261,19 @@ examples:
 
       ENV['BUNDLE_GEMFILE'] = File.expand_path File.join(path, "Gemfile")
       Dir.chdir(path) do
-        require 'bundler'
+        begin
+          require 'bundler'
+        rescue LoadError
+          require 'rubygems/gem_runner'
+          begin
+            puts "#{Time.now.strftime("%Y-%m-%d %H:%M:%S.%3N %z")}: installing bundler..."
+            Gem::GemRunner.new.run %w[install bundler]
+          rescue Gem::SystemExitException => e
+            raise e if e.exit_code != 0
+          end
+          Gem.clear_paths
+          require 'bundler'
+        end
         require 'bundler/friendly_errors'
         require 'bundler/cli'
         Bundler.with_friendly_errors do

data/lib/embulk/guess/csv.rb

@@ -60,12 +60,24 @@ module Embulk
           # don't even set null_string to avoid confusion of null and 'null' in YAML format
         end
 
-        sample_records = split_lines(parser_guessed, sample_lines, delim)
-
+        # guessing skip_header_lines should be before guessing guess_comment_line_marker
+        # because lines supplied to CsvTokenizer already don't include skipped header lines.
+        # skipping empty lines is also disabled here because skipping header lines is done by
+        # CsvParser which doesn't skip empty lines automatically
+        sample_records = split_lines(parser_guessed, false, sample_lines, delim, {})
         skip_header_lines = guess_skip_header_lines(sample_records)
+        sample_lines = sample_lines[skip_header_lines..-1]
         sample_records = sample_records[skip_header_lines..-1]
 
-        comment_line_marker, sample_records = guess_comment_line_marker(sample_records)
+        unless parser_guessed.has_key?("comment_line_marker")
+          comment_line_marker, sample_lines =
+            guess_comment_line_marker(sample_lines, delim, parser_guessed["quote"], parser_guessed["null_string"])
+          if comment_line_marker
+            parser_guessed["comment_line_marker"] = comment_line_marker
+          end
+        end
+
+        sample_records = split_lines(parser_guessed, true, sample_lines, delim, {})
 
         first_types = SchemaGuess.types_from_array_records(sample_records[0, 1])
         other_types = SchemaGuess.types_from_array_records(sample_records[1..-1] || [])
@@ -75,6 +87,17 @@ module Embulk
           return {}
         end
 
+        unless parser_guessed.has_key?("trim_if_not_quoted")
+          sample_records_trimmed = split_lines(parser_guessed, true, sample_lines, delim, {"trim_if_not_quoted" => true})
+          other_types_trimmed = SchemaGuess.types_from_array_records(sample_records_trimmed[1..-1] || [])
+          if other_types != other_types_trimmed
+            parser_guessed["trim_if_not_quoted"] = true
+            other_types = other_types_trimmed
+          else
+            parser_guessed["trim_if_not_quoted"] = false
+          end
+        end
+
         header_line = (first_types != other_types && first_types.all? {|t| ["string", "boolean"].include?(t) })
 
         if header_line
@@ -83,10 +106,8 @@ module Embulk
           parser_guessed["skip_header_lines"] = skip_header_lines
         end
 
-        parser_guessed["comment_line_marker"] = comment_line_marker  # always set comment_line_marker even if it's null
-
-        parser_guessed["allow_extra_columns"] = false
-        parser_guessed["allow_optional_columns"] = false
+        parser_guessed["allow_extra_columns"] = false unless parser_guessed.has_key?("allow_extra_columns")
+        parser_guessed["allow_optional_columns"] = false unless parser_guessed.has_key?("allow_optional_columns")
 
         if header_line
           column_names = sample_records.first
@@ -110,22 +131,26 @@ module Embulk
 
       private
 
-      def split_lines(parser_config, sample_lines, delim)
-        parser_task = parser_config.merge({"charset" => "UTF-8", "columns" => []}).load_config(org.embulk.standards.CsvParserPlugin::PluginTask)
-        data = sample_lines.map {|x| x.force_encoding('UTF-8') }.join(parser_task.getNewline.getString.encode('UTF-8'))
+      def split_lines(parser_config, skip_empty_lines, sample_lines, delim, extra_config)
+        null_string = parser_config["null_string"]
+        config = parser_config.merge(extra_config).merge({"charset" => "UTF-8", "columns" => []})
+        parser_task = config.load_config(org.embulk.standards.CsvParserPlugin::PluginTask)
+        data = sample_lines.map {|line| line.force_encoding('UTF-8') }.join(parser_task.getNewline.getString.encode('UTF-8'))
         sample = Buffer.from_ruby_string(data)
         decoder = Java::LineDecoder.new(Java::ListFileInput.new([[sample.to_java]]), parser_task)
         tokenizer = org.embulk.standards.CsvTokenizer.new(decoder, parser_task)
         rows = []
         while tokenizer.nextFile
-          while tokenizer.nextRecord
+          while tokenizer.nextRecord(skip_empty_lines)
             begin
               columns = []
               while true
                 begin
                   column = tokenizer.nextColumn
                   quoted = tokenizer.wasQuotedColumn
-                  column.define_singleton_method(:quoted?) { quoted }
+                  if null_string && !quoted && column == null_string
+                    column = nil
+                  end
                   columns << column
                 rescue org.embulk.standards.CsvTokenizer::TooFewColumnsException
                   rows << columns
@@ -228,20 +253,25 @@ module Embulk
         return 0
       end
 
-      def guess_comment_line_marker(sample_records)
+      def guess_comment_line_marker(sample_lines, delim, quote, null_string)
+        exclude = []
+        exclude << /^#{Regexp.escape(quote)}/ if quote && !quote.empty?
+        exclude << /^#{Regexp.escape(null_string)}(?:#{Regexp.escape(delim)}|$)/ if null_string
+
         guessed = COMMENT_LINE_MARKER_CANDIDATES.map do |str|
           regexp = /^#{Regexp.quote(str)}/
-          records = sample_records.reject do |records|
-            !records[0].quoted? && !NULL_STRING_CANDIDATES.include?(records[0]) && records[0] =~ regexp
+          unmatch_lines = sample_lines.reject do |line|
+            exclude.all? {|ex| line !~ ex } && line =~ regexp
           end
-          count = sample_records.size - records.size
-          [str, count, records]
-        end.select {|str,count,records| count > 0 }.sort_by {|str,count,records| -count }
-        found_str, found_count, found_records = guessed.first
-        if found_str
-          return found_str, found_records
+          match_count = sample_lines.size - unmatch_lines.size
+          [str, match_count, unmatch_lines]
+        end.select {|str,match_count,unmatch_lines| match_count > 0 }.sort_by {|str,match_count,unmatch_lines| -match_count }
+
+        str, match_count, unmatch_lines = guessed.first
+        if str
+          return str, unmatch_lines
         else
-          return nil, sample_records
+          return nil, sample_lines
         end
       end
 

data/lib/embulk/java/imports.rb

@@ -21,6 +21,7 @@ module Embulk::Java
   java_import 'org.embulk.spi.TransactionalPageOutput'
   java_import 'org.embulk.spi.PageReader'
   java_import 'org.embulk.spi.PageBuilder'
+  java_import 'org.embulk.spi.util.DynamicPageBuilder'
   java_import 'org.embulk.spi.util.LineDecoder'
   java_import 'org.embulk.spi.util.ListFileInput'
   java_import 'org.embulk.spi.Schema'
@@ -30,6 +31,7 @@ module Embulk::Java
   java_import 'org.embulk.spi.PluginClassLoader'
   java_import 'org.embulk.spi.FileInputRunner'
   java_import 'org.embulk.spi.FileOutputRunner'
+  java_import 'org.embulk.spi.Exec'
 
   # TODO
 end

data/lib/embulk/page_builder.rb

@@ -1,13 +1,67 @@
 module Embulk
 
+  org.embulk.spi.util.dynamic.AbstractDynamicColumnSetter.module_eval do
+    alias_method(:set, :setRubyObject)
+  end
+
   class PageBuilder
     def initialize(schema, java_page_output)
-      @page_builder = Java::PageBuilder.new(Java::Injected::BufferAllocator, schema.to_java, java_page_output)
+      # TODO get task as an argument
+      task = Java::Exec.newConfigSource.load_config(Java::DynamicPageBuilder::BuilderTask.java_class)
+      @page_builder = Java::DynamicPageBuilder.new(task, Java::Injected::BufferAllocator, schema.to_java, java_page_output)
       @schema = schema
     end
 
     def add(record)
-      @schema.write_record(@page_builder, record)
+      i = 0
+      m = record.size
+      while i < m
+        @page_builder.column(i).set(record[i])
+        i += 1
+      end
+      @page_builder.addRecord
+      nil
+    end
+
+    def [](index_or_column)
+      case index_or_column
+      when Integer
+        @page_builder.column_or_null(index_or_column)
+      when Column
+        @page_builder.column_or_null(index_or_column.index)
+      else
+        @page_builder.column_or_null(index_or_column)
+      end
+    end
+
+    def column(index_or_column)
+      case index_or_column
+      when Integer
+        @page_builder.column(index_or_column)
+      when Column
+        @page_builder.column(index_or_column.index)
+      else
+        @page_builder.lookupColumn(index_or_column)
+      end
+    end
+
+    def column_or_skip(index_or_column)
+      case index_or_column
+      when Integer
+        @page_builder.column_or_skip(index_or_column)
+      when Column
+        @page_builder.column_or_skip(index_or_column.index)
+      else
+        @page_builder.column_or_skip(index_or_column)
+      end
+    end
+
+    def add!
+      @page_builder.add_record
+    end
+
+    def flush
+      @page_builder.flush
     end
 
     def finish

data/lib/embulk/version.rb

@@ -1,3 +1,3 @@
 module Embulk
-  VERSION = '0.6.12'
+  VERSION = '0.6.13'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk
 version: !ruby/object:Gem::Version
-  version: 0.6.12
+  version: 0.6.13
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-06-22 00:00:00.000000000 Z
+date: 2015-06-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -216,6 +216,10 @@ files:
 - embulk-core/src/main/java/org/embulk/spi/unit/ByteSize.java
 - embulk-core/src/main/java/org/embulk/spi/util/CharsetSerDe.java
 - embulk-core/src/main/java/org/embulk/spi/util/Decoders.java
+- embulk-core/src/main/java/org/embulk/spi/util/DynamicColumnNotFoundException.java
+- embulk-core/src/main/java/org/embulk/spi/util/DynamicColumnSetter.java
+- embulk-core/src/main/java/org/embulk/spi/util/DynamicColumnSetterFactory.java
+- embulk-core/src/main/java/org/embulk/spi/util/DynamicPageBuilder.java
 - embulk-core/src/main/java/org/embulk/spi/util/Encoders.java
 - embulk-core/src/main/java/org/embulk/spi/util/Executors.java
 - embulk-core/src/main/java/org/embulk/spi/util/FileInputInputStream.java
@@ -232,6 +236,15 @@ files:
 - embulk-core/src/main/java/org/embulk/spi/util/Pages.java
 - embulk-core/src/main/java/org/embulk/spi/util/ResumableInputStream.java
 - embulk-core/src/main/java/org/embulk/spi/util/RetryExecutor.java
+- embulk-core/src/main/java/org/embulk/spi/util/dynamic/AbstractDynamicColumnSetter.java
+- embulk-core/src/main/java/org/embulk/spi/util/dynamic/BooleanColumnSetter.java
+- embulk-core/src/main/java/org/embulk/spi/util/dynamic/DefaultValueSetter.java
+- embulk-core/src/main/java/org/embulk/spi/util/dynamic/DoubleColumnSetter.java
+- embulk-core/src/main/java/org/embulk/spi/util/dynamic/LongColumnSetter.java
+- embulk-core/src/main/java/org/embulk/spi/util/dynamic/NullDefaultValueSetter.java
+- embulk-core/src/main/java/org/embulk/spi/util/dynamic/SkipColumnSetter.java
+- embulk-core/src/main/java/org/embulk/spi/util/dynamic/StringColumnSetter.java
+- embulk-core/src/main/java/org/embulk/spi/util/dynamic/TimestampColumnSetter.java
 - embulk-core/src/main/resources/embulk/logback-color.xml
 - embulk-core/src/main/resources/embulk/logback-console.xml
 - embulk-core/src/main/resources/embulk/logback-file.xml
@@ -301,6 +314,7 @@ files:
 - embulk-docs/src/release/release-0.6.10.rst
 - embulk-docs/src/release/release-0.6.11.rst
 - embulk-docs/src/release/release-0.6.12.rst
+- embulk-docs/src/release/release-0.6.13.rst
 - embulk-docs/src/release/release-0.6.2.rst
 - embulk-docs/src/release/release-0.6.3.rst
 - embulk-docs/src/release/release-0.6.4.rst
@@ -417,8 +431,8 @@ files:
 - classpath/bval-jsr303-0.5.jar
 - classpath/commons-beanutils-core-1.8.3.jar
 - classpath/commons-lang3-3.1.jar
-- classpath/embulk-core-0.6.12.jar
-- classpath/embulk-standards-0.6.12.jar
+- classpath/embulk-core-0.6.13.jar
+- classpath/embulk-standards-0.6.13.jar
 - classpath/guava-18.0.jar
 - classpath/guice-4.0.jar
 - classpath/guice-multibindings-4.0.jar