embulk-input-athena 0.1.0

data/src/main/java/org/embulk/input/jdbc/getter/BooleanColumnGetter.java

@@ -0,0 +1,56 @@
+package org.embulk.input.jdbc.getter;
+
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import org.embulk.spi.Column;
+import org.embulk.spi.PageBuilder;
+import org.embulk.spi.type.Type;
+import org.embulk.spi.type.Types;
+
+public class BooleanColumnGetter
+        extends AbstractColumnGetter
+{
+    protected boolean value;
+
+    public BooleanColumnGetter(PageBuilder to, Type toType)
+    {
+        super(to, toType);
+    }
+
+    @Override
+    protected void fetch(ResultSet from, int fromIndex) throws SQLException
+    {
+        value = from.getBoolean(fromIndex);
+    }
+
+    @Override
+    protected Type getDefaultToType()
+    {
+        return Types.BOOLEAN;
+    }
+
+    @Override
+    public void booleanColumn(Column column)
+    {
+        to.setBoolean(column, value);
+    }
+
+    @Override
+    public void longColumn(Column column)
+    {
+        to.setLong(column, value ? 1L : 0L);
+    }
+
+    @Override
+    public void doubleColumn(Column column)
+    {
+        to.setDouble(column, value ? 1.0 : 0.0);
+    }
+
+    @Override
+    public void stringColumn(Column column)
+    {
+        to.setString(column, Boolean.toString(value));
+    }
+
+}

data/src/main/java/org/embulk/input/jdbc/getter/ColumnGetter.java

@@ -0,0 +1,21 @@
+package org.embulk.input.jdbc.getter;
+
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.PreparedStatement;
+import com.fasterxml.jackson.databind.JsonNode;
+import org.embulk.spi.Column;
+import org.embulk.spi.type.Type;
+
+public interface ColumnGetter
+{
+    public void getAndSet(ResultSet from, int fromIndex,
+            Column toColumn) throws SQLException;
+
+    public Type getToType();
+
+    public JsonNode encodeToJson();
+
+    public void decodeFromJsonTo(PreparedStatement toStatement, int toIndex, JsonNode fromValue)
+        throws SQLException;
+}

data/src/main/java/org/embulk/input/jdbc/getter/ColumnGetterFactory.java

@@ -0,0 +1,207 @@
+package org.embulk.input.jdbc.getter;
+
+import java.lang.reflect.Field;
+import java.sql.Types;
+import java.util.HashMap;
+import java.util.Map;
+
+import com.google.common.base.Optional;
+import org.embulk.config.ConfigException;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.Task;
+import org.embulk.input.jdbc.AbstractJdbcInputPlugin.PluginTask;
+import org.embulk.input.jdbc.JdbcColumn;
+import org.embulk.input.jdbc.JdbcColumnOption;
+import org.embulk.input.jdbc.JdbcInputConnection;
+import org.embulk.spi.Exec;
+import org.embulk.spi.PageBuilder;
+import org.embulk.spi.time.TimestampFormatter;
+import org.embulk.spi.type.TimestampType;
+import org.embulk.spi.type.Type;
+import org.joda.time.DateTimeZone;
+
+import static java.util.Locale.ENGLISH;
+
+public class ColumnGetterFactory
+{
+    protected final PageBuilder to;
+    private final DateTimeZone defaultTimeZone;
+    private final Map<Integer, String> jdbcTypes = getAllJDBCTypes();
+
+    public ColumnGetterFactory(PageBuilder to, DateTimeZone defaultTimeZone)
+    {
+        this.to = to;
+        this.defaultTimeZone = defaultTimeZone;
+    }
+
+    public ColumnGetter newColumnGetter(JdbcInputConnection con, PluginTask task, JdbcColumn column, JdbcColumnOption option)
+    {
+        return newColumnGetter(con, task, column, option, option.getValueType());
+    }
+
+    private ColumnGetter newColumnGetter(JdbcInputConnection con, PluginTask task, JdbcColumn column, JdbcColumnOption option, String valueType)
+    {
+        Type toType = getToType(option);
+        switch(valueType) {
+        case "coalesce":
+            // resolve actual valueType using sqlTypeToValueType() method and retry.
+            return newColumnGetter(con, task, column, option, sqlTypeToValueType(column, column.getSqlType()));
+        case "long":
+            return new LongColumnGetter(to, toType);
+        case "float":
+            return new FloatColumnGetter(to, toType);
+        case "double":
+            return new DoubleColumnGetter(to, toType);
+        case "boolean":
+            return new BooleanColumnGetter(to, toType);
+        case "string":
+            return new StringColumnGetter(to, toType);
+        case "json":
+            return new JsonColumnGetter(to, toType);
+        case "date":
+            return new DateColumnGetter(to, toType, newTimestampFormatter(option, DateColumnGetter.DEFAULT_FORMAT));
+        case "time":
+            return new TimeColumnGetter(to, toType, newTimestampFormatter(option, DateColumnGetter.DEFAULT_FORMAT));
+        case "timestamp":
+            return new TimestampColumnGetter(to, toType, newTimestampFormatter(option, DateColumnGetter.DEFAULT_FORMAT));
+        case "decimal":
+            return new BigDecimalColumnGetter(to, toType);
+        default:
+            throw new ConfigException(String.format(ENGLISH,
+                        "Unknown value_type '%s' for column '%s'", option.getValueType(), column.getName()));
+        }
+    }
+
+    protected Map<Integer,String> getAllJDBCTypes() {
+        Map<Integer,String> map = new HashMap<Integer, String>();
+        for(Field f: Types.class.getFields()){
+            try {
+                map.put((Integer) f.get(null), f.getName());
+            } catch(IllegalAccessException iea){
+            }
+        }
+        return map;
+    }
+
+    public String getJdbcType(int sqlType)
+    {
+        return jdbcTypes.get(sqlType);
+    }
+
+
+    protected String sqlTypeToValueType(JdbcColumn column, int sqlType)
+    {
+        switch(sqlType) {
+        // getLong
+        case Types.TINYINT:
+        case Types.SMALLINT:
+        case Types.INTEGER:
+        case Types.BIGINT:
+            return "long";
+
+        // getFloat
+        case Types.FLOAT:
+        case Types.REAL:
+            return "float";
+
+        // getDouble
+        case Types.DOUBLE:
+            return "double";
+
+        // getBool
+        case Types.BOOLEAN:
+        case Types.BIT:  // JDBC BIT is boolean, unlike SQL-92
+            return "boolean";
+
+        // getString, Clob
+        case Types.CHAR:
+        case Types.VARCHAR:
+        case Types.LONGVARCHAR:
+        case Types.CLOB:
+        case Types.NCHAR:
+        case Types.NVARCHAR:
+        case Types.LONGNVARCHAR:
+            return "string";
+
+        // TODO
+        //// getBytes Blob
+        //case Types.BINARY:
+        //case Types.VARBINARY:
+        //case Types.LONGVARBINARY:
+        //case Types.BLOB:
+        //    return new BytesColumnGetter();
+
+        // getDate
+        case Types.DATE:
+            return "date";
+
+        // getTime
+        case Types.TIME:
+            return "time";
+
+        // getTimestamp
+        case Types.TIMESTAMP:
+            return "timestamp";
+
+        // TODO
+        //// Null
+        //case Types.NULL:
+        //    return new NullColumnGetter();
+
+        // getBigDecimal
+        case Types.NUMERIC:
+        case Types.DECIMAL:
+            return "decimal";
+
+        // others
+        case Types.ARRAY:  // array
+        case Types.STRUCT: // map
+        case Types.REF:
+        case Types.DATALINK:
+        case Types.SQLXML: // XML
+        case Types.ROWID:
+        case Types.DISTINCT:
+        case Types.JAVA_OBJECT:
+        case Types.OTHER:
+        default:
+            throw unsupportedOperationException(column);
+        }
+    }
+
+    protected Type getToType(JdbcColumnOption option)
+    {
+        if (!option.getType().isPresent()) {
+            return null;
+        }
+        Type toType = option.getType().get();
+        if (toType instanceof TimestampType && option.getTimestampFormat().isPresent()) {
+            toType = ((TimestampType)toType).withFormat(option.getTimestampFormat().get().getFormat());
+        }
+        return toType;
+    }
+
+    private static interface FormatterIntlTask extends Task, TimestampFormatter.Task {}
+    private static interface FormatterIntlColumnOption extends Task, TimestampFormatter.TimestampColumnOption {}
+
+    private TimestampFormatter newTimestampFormatter(JdbcColumnOption option, String defaultTimestampFormat)
+    {
+        // TODO: Switch to a newer TimestampFormatter constructor after a reasonable interval.
+        // Traditional constructor is used here for compatibility.
+        final ConfigSource configSource = Exec.newConfigSource();
+        configSource.set("format", option.getTimestampFormat().isPresent()
+                                   ? option.getTimestampFormat().get().getFormat()
+                                   : defaultTimestampFormat);
+        configSource.set("timezone", option.getTimeZone().or(this.defaultTimeZone));
+        return new TimestampFormatter(
+            Exec.newConfigSource().loadConfig(FormatterIntlTask.class),
+            Optional.fromNullable(configSource.loadConfig(FormatterIntlColumnOption.class)));
+    }
+
+    private static UnsupportedOperationException unsupportedOperationException(JdbcColumn column)
+    {
+        throw new UnsupportedOperationException(
+                String.format(ENGLISH,
+                    "Unsupported type %s (sqlType=%d) of '%s' column. Please add '%s: {value_type: string}' to 'column_options: {...}' option to convert the values to strings, or exclude the column from 'select:' option",
+                    column.getTypeName(), column.getSqlType(), column.getName(), column.getName()));
+    }
+}

data/src/main/java/org/embulk/input/jdbc/getter/DateColumnGetter.java

@@ -0,0 +1,37 @@
+package org.embulk.input.jdbc.getter;
+
+import java.sql.Date;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import org.embulk.spi.PageBuilder;
+import org.embulk.spi.time.Timestamp;
+import org.embulk.spi.time.TimestampFormatter;
+import org.embulk.spi.type.Type;
+import org.embulk.spi.type.Types;
+
+public class DateColumnGetter
+        extends AbstractTimestampColumnGetter
+{
+    static final String DEFAULT_FORMAT = "%Y-%m-%d";
+
+    public DateColumnGetter(PageBuilder to, Type toType, TimestampFormatter timestampFormatter)
+    {
+        super(to, toType, timestampFormatter);
+    }
+
+    @Override
+    protected void fetch(ResultSet from, int fromIndex) throws SQLException
+    {
+        Date date = from.getDate(fromIndex);
+        if (date != null) {
+            value = Timestamp.ofEpochMilli(date.getTime());
+        }
+    }
+
+    @Override
+    protected Type getDefaultToType()
+    {
+        return Types.TIMESTAMP.withFormat(DEFAULT_FORMAT);
+    }
+
+}

data/src/main/java/org/embulk/input/jdbc/getter/DoubleColumnGetter.java

@@ -0,0 +1,66 @@
+package org.embulk.input.jdbc.getter;
+
+import java.math.RoundingMode;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import org.embulk.spi.Column;
+import org.embulk.spi.PageBuilder;
+import org.embulk.spi.type.Type;
+import org.embulk.spi.type.Types;
+import com.google.common.math.DoubleMath;
+
+public class DoubleColumnGetter
+        extends AbstractColumnGetter
+{
+    protected double value;
+
+    public DoubleColumnGetter(PageBuilder to, Type toType)
+    {
+        super(to, toType);
+    }
+
+    @Override
+    protected void fetch(ResultSet from, int fromIndex) throws SQLException
+    {
+        value = from.getDouble(fromIndex);
+    }
+
+    @Override
+    protected Type getDefaultToType()
+    {
+        return Types.DOUBLE;
+    }
+
+    @Override
+    public void booleanColumn(Column column)
+    {
+        to.setBoolean(column, value > 0.0);
+    }
+
+    @Override
+    public void longColumn(Column column)
+    {
+        long l;
+        try {
+            // TODO configurable rounding mode
+            l = DoubleMath.roundToLong(value, RoundingMode.HALF_UP);
+        } catch (ArithmeticException e) {
+            // NaN / Infinite / -Infinite
+            super.longColumn(column);
+            return;
+        }
+        to.setLong(column, l);
+    }
+
+    @Override
+    public void doubleColumn(Column column)
+    {
+        to.setDouble(column, value);
+    }
+
+    @Override
+    public void stringColumn(Column column)
+    {
+        to.setString(column, Double.toString(value));
+    }
+}

data/src/main/java/org/embulk/input/jdbc/getter/FloatColumnGetter.java

@@ -0,0 +1,66 @@
+package org.embulk.input.jdbc.getter;
+
+import java.math.RoundingMode;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import org.embulk.spi.Column;
+import org.embulk.spi.PageBuilder;
+import org.embulk.spi.type.Type;
+import org.embulk.spi.type.Types;
+import com.google.common.math.DoubleMath;
+
+public class FloatColumnGetter
+        extends AbstractColumnGetter
+{
+    protected float value;
+
+    public FloatColumnGetter(PageBuilder to, Type toType)
+    {
+        super(to, toType);
+    }
+
+    @Override
+    protected void fetch(ResultSet from, int fromIndex) throws SQLException
+    {
+        value = from.getFloat(fromIndex);
+    }
+
+    @Override
+    protected Type getDefaultToType()
+    {
+        return Types.DOUBLE;
+    }
+
+    @Override
+    public void booleanColumn(Column column)
+    {
+        to.setBoolean(column, value > 0.0);
+    }
+
+    @Override
+    public void longColumn(Column column)
+    {
+        long l;
+        try {
+            // TODO configurable rounding mode
+            l = DoubleMath.roundToLong(value, RoundingMode.HALF_UP);
+        } catch (ArithmeticException e) {
+            // NaN / Infinite / -Infinite
+            super.longColumn(column);
+            return;
+        }
+        to.setLong(column, l);
+    }
+
+    @Override
+    public void doubleColumn(Column column)
+    {
+        to.setDouble(column, value);
+    }
+
+    @Override
+    public void stringColumn(Column column)
+    {
+        to.setString(column, Float.toString(value));
+    }
+}