embulk-input-jdbc 0.6.4 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d64ee420864ebfe1de416471638a0df1348f9338
-  data.tar.gz: 79cfb08db0a917123d6964e2670b7df9d17b060a
+  metadata.gz: 5f5b564e877d141a046fdb04fc281bc9730af498
+  data.tar.gz: aab75d9db9406b59446617ce03240b615db1f090
 SHA512:
-  metadata.gz: 625c73db1ce05cd04aad35bf9a2da954f42a4042aab27009ae85029b84d845480413e6b6fad547474ac39bd495654d3e90ffb57ddda12ec00fe38144272fa008
-  data.tar.gz: dff6a93e278ad1a3803c142135164260d451c3c77de796f901a22d26a39b552f3f0247a42e1aa8193c01177e0c832ff2c58542da94e5b955a946de054c0c1039
+  metadata.gz: 3924ef7b5698e9e9aaea6ed11763d3ff38acdb6bd963e352adca85912c6efc8ca12d0d159bf92d2b8e340e93e5585bfee3f668ef895bf7995c7eaf3d29155596
+  data.tar.gz: cadb17a65f0b6a9e658f4d4e830841e7d44662bc668c4964a9c0f67918d836c8f88b9260bfe21ca4991c59a96cc002ba7b51260335a8d28dcd9f9478a0a43f59

data/src/main/java/org/embulk/input/jdbc/AbstractJdbcInputPlugin.java

@@ -120,6 +120,14 @@ public abstract class AbstractJdbcInputPlugin
         @ConfigDefault("\"UTC\"")
         public DateTimeZone getDefaultTimeZone();
 
+        @Config("default_column_options")
+        @ConfigDefault("{}")
+        public Map<String, JdbcColumnOption> getDefaultColumnOptions();
+
+        @Config("after_select")
+        @ConfigDefault("null")
+        public Optional<String> getAfterSelect();
+
         public JdbcSchema getQuerySchema();
         public void setQuerySchema(JdbcSchema schema);
 
@@ -158,7 +166,7 @@ public abstract class AbstractJdbcInputPlugin
     private Schema setupTask(JdbcInputConnection con, PluginTask task) throws SQLException
     {
         // build SELECT query and gets schema of its result
-    	String query = getQuery(task, con);
+        String query = getQuery(task, con);
         logger.info("SQL: " + query);
         JdbcSchema querySchema = con.getSchemaOfQuery(query);
         task.setQuerySchema(querySchema);
@@ -166,11 +174,11 @@ public abstract class AbstractJdbcInputPlugin
         // validate column_options
         newColumnGetters(task, querySchema, null);
 
-        ColumnGetterFactory factory = new ColumnGetterFactory(null, task.getDefaultTimeZone());
+        ColumnGetterFactory factory = newColumnGetterFactory(null, task.getDefaultTimeZone());
         ImmutableList.Builder<Column> columns = ImmutableList.builder();
         for (int i = 0; i < querySchema.getCount(); i++) {
             JdbcColumn column = querySchema.getColumn(i);
-            JdbcColumnOption columnOption = columnOptionOf(task.getColumnOptions(), column);
+            JdbcColumnOption columnOption = columnOptionOf(task.getColumnOptions(), task.getDefaultColumnOptions(), column, factory.getJdbcType(column.getSqlType()));
             columns.add(new Column(i,
                     column.getName(),
                     factory.newColumnGetter(column, columnOption).getToType()));
@@ -258,6 +266,11 @@ public abstract class AbstractJdbcInputPlugin
                         }
                     }
                 }
+
+                if (task.getAfterSelect().isPresent()) {
+                    con.executeUpdate(task.getAfterSelect().get());
+                    con.connection.commit();
+                }
             }
 
         } catch (SQLException ex) {
@@ -273,21 +286,29 @@ public abstract class AbstractJdbcInputPlugin
         return report;
     }
 
+    protected ColumnGetterFactory newColumnGetterFactory(PageBuilder pageBuilder, DateTimeZone dateTimeZone)
+    {
+        return new ColumnGetterFactory(pageBuilder, dateTimeZone);
+    }
+
     private List<ColumnGetter> newColumnGetters(PluginTask task, JdbcSchema querySchema, PageBuilder pageBuilder)
             throws SQLException
     {
-        ColumnGetterFactory factory = new ColumnGetterFactory(pageBuilder, task.getDefaultTimeZone());
+        ColumnGetterFactory factory = newColumnGetterFactory(pageBuilder, task.getDefaultTimeZone());
         ImmutableList.Builder<ColumnGetter> getters = ImmutableList.builder();
         for (JdbcColumn c : querySchema.getColumns()) {
-            JdbcColumnOption columnOption = columnOptionOf(task.getColumnOptions(), c);
+            JdbcColumnOption columnOption = columnOptionOf(task.getColumnOptions(), task.getDefaultColumnOptions(), c, factory.getJdbcType(c.getSqlType()));
             getters.add(factory.newColumnGetter(c, columnOption));
         }
         return getters.build();
     }
 
-    private static JdbcColumnOption columnOptionOf(Map<String, JdbcColumnOption> columnOptions, JdbcColumn targetColumn)
+    private static JdbcColumnOption columnOptionOf(Map<String, JdbcColumnOption> columnOptions, Map<String, JdbcColumnOption> defaultColumnOptions, JdbcColumn targetColumn, String targetColumnSQLType)
     {
-        return Optional.fromNullable(columnOptions.get(targetColumn.getName())).or(
+        return Optional
+                .fromNullable(columnOptions.get(targetColumn.getName()))
+                .or(Optional.fromNullable(defaultColumnOptions.get(targetColumnSQLType)))
+                .or(
                     // default column option
                     new Supplier<JdbcColumnOption>()
                     {

data/src/main/java/org/embulk/input/jdbc/JdbcInputConnection.java

@@ -94,7 +94,7 @@ public class JdbcInputConnection
         private final PreparedStatement fetchStatement;
         private boolean fetched = false;
 
-        public SingleSelect(PreparedStatement fetchStatement) throws SQLException
+        public SingleSelect(PreparedStatement fetchStatement)
         {
             this.fetchStatement = fetchStatement;
         }

data/src/main/java/org/embulk/input/jdbc/getter/AbstractColumnGetter.java

@@ -57,9 +57,7 @@ public abstract class AbstractColumnGetter implements ColumnGetter, ColumnVisito
     }
 
     @Override
-    public void jsonColumn(Column column) {
-        throw new UnsupportedOperationException("This plugin doesn't support json type. Please try to upgrade version of the plugin using 'embulk gem update' command. If the latest version still doesn't support json type, please contact plugin developers, or change configuration of input plugin not to use json type.");
-    }
+    public void jsonColumn(Column column) { to.setNull(column); }
 
     @Override
     public void timestampColumn(Column column)

data/src/main/java/org/embulk/input/jdbc/getter/ColumnGetterFactory.java

@@ -1,6 +1,9 @@
 package org.embulk.input.jdbc.getter;
 
+import java.lang.reflect.Field;
 import java.sql.Types;
+import java.util.HashMap;
+import java.util.Map;
 
 import org.embulk.config.ConfigException;
 import org.embulk.input.jdbc.JdbcColumn;
@@ -13,8 +16,9 @@ import org.joda.time.DateTimeZone;
 
 public class ColumnGetterFactory
 {
-    private final PageBuilder to;
+    protected final PageBuilder to;
     private final DateTimeZone defaultTimeZone;
+    private final Map<Integer, String> jdbcTypes = getAllJDBCTypes();
 
     public ColumnGetterFactory(PageBuilder to, DateTimeZone defaultTimeZone)
     {
@@ -43,6 +47,8 @@ public class ColumnGetterFactory
             return new BooleanColumnGetter(to, toType);
         case "string":
             return new StringColumnGetter(to, toType);
+        case "json":
+            return new JsonColumnGetter(to, toType);
         case "date":
             return new DateColumnGetter(to, toType, newTimestampFormatter(option, DateColumnGetter.DEFAULT_FORMAT));
         case "time":
@@ -56,7 +62,28 @@ public class ColumnGetterFactory
         }
     }
 
-    private String sqlTypeToValueType(JdbcColumn column, int sqlType)
+    protected Map<Integer,String> getAllJDBCTypes() {
+        Map<Integer,String> map = new HashMap<Integer, String>();
+        for(Field f: Types.class.getFields()){
+            try {
+                map.put((Integer) f.get(null), f.getName());
+            } catch(IllegalAccessException iea){
+            }
+        }
+        return map;
+    }
+
+    public String getJdbcType(int sqlType)
+    {
+        String sqlTypeName = jdbcTypes.get(sqlType);
+        if(sqlTypeName == null){
+            throw new UnsupportedOperationException(String.format("Unknown SQL type value '%d'", sqlType));
+        }
+        return sqlTypeName;
+    }
+
+
+    protected String sqlTypeToValueType(JdbcColumn column, int sqlType)
     {
         switch(sqlType) {
         // getLong
@@ -135,7 +162,7 @@ public class ColumnGetterFactory
         }
     }
 
-    private Type getToType(JdbcColumnOption option)
+    protected Type getToType(JdbcColumnOption option)
     {
         if (!option.getType().isPresent()) {
             return null;

data/src/main/java/org/embulk/input/jdbc/getter/JsonColumnGetter.java

@@ -0,0 +1,57 @@
+package org.embulk.input.jdbc.getter;
+
+import java.sql.ResultSet;
+import java.sql.SQLException;
+
+import org.embulk.input.jdbc.getter.AbstractColumnGetter;
+import org.embulk.spi.Column;
+import org.embulk.spi.PageBuilder;
+import org.embulk.spi.json.JsonParseException;
+import org.embulk.spi.json.JsonParser;
+import org.embulk.spi.type.Type;
+import org.embulk.spi.type.Types;
+import org.msgpack.value.Value;
+
+public class JsonColumnGetter
+        extends AbstractColumnGetter
+{
+    final JsonParser jsonParser = new JsonParser();
+
+    private String value;
+
+    public JsonColumnGetter(PageBuilder to, Type toType)
+    {
+        super(to, toType);
+    }
+
+    @Override
+    protected void fetch(ResultSet from, int fromIndex) throws SQLException
+    {
+        value = from.getString(fromIndex);
+    }
+
+    @Override
+    protected Type getDefaultToType()
+    {
+        return Types.JSON;
+    }
+
+    @Override
+    public void jsonColumn(Column column)
+    {
+        Value v;
+        try {
+            v = jsonParser.parse(value);
+        } catch (JsonParseException e) {
+            super.jsonColumn(column);
+            return;
+        }
+        to.setJson(column, v);
+    }
+
+    @Override
+    public void stringColumn(Column column)
+    {
+        to.setString(column, value);
+    }
+}

data/src/main/java/org/embulk/input/jdbc/getter/StringColumnGetter.java

@@ -4,12 +4,17 @@ import java.sql.ResultSet;
 import java.sql.SQLException;
 import org.embulk.spi.Column;
 import org.embulk.spi.PageBuilder;
+import org.embulk.spi.json.JsonParseException;
+import org.embulk.spi.json.JsonParser;
 import org.embulk.spi.type.Type;
 import org.embulk.spi.type.Types;
+import org.msgpack.value.Value;
 
 public class StringColumnGetter
         extends AbstractColumnGetter
 {
+    final JsonParser jsonParser = new JsonParser();
+
     private String value;
 
     public StringColumnGetter(PageBuilder to, Type toType)
@@ -55,6 +60,19 @@ public class StringColumnGetter
         to.setDouble(column, d);
     }
 
+    @Override
+    public void jsonColumn(Column column)
+    {
+        Value v;
+        try {
+            v = jsonParser.parse(value);
+        } catch (JsonParseException e) {
+            super.jsonColumn(column);
+            return;
+        }
+        to.setJson(column, v);
+    }
+
     @Override
     public void stringColumn(Column column)
     {

data/src/test/java/org/embulk/input/EmbulkPluginTester.java

@@ -0,0 +1,79 @@
+package org.embulk.input;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.embulk.EmbulkEmbed;
+import org.embulk.EmbulkEmbed.Bootstrap;
+import org.embulk.config.ConfigSource;
+import org.embulk.plugin.InjectedPluginSource;
+
+import com.google.inject.Binder;
+import com.google.inject.Module;
+
+public class EmbulkPluginTester
+{
+    private static class PluginDefinition
+    {
+        public final Class<?> iface;
+        public final String name;
+        public final Class<?> impl;
+
+
+        public PluginDefinition(Class<?> iface, String name, Class<?> impl)
+        {
+            this.iface = iface;
+            this.name = name;
+            this.impl = impl;
+        }
+
+    }
+
+    private final List<PluginDefinition> plugins = new ArrayList<PluginDefinition>();
+
+    private EmbulkEmbed embulk;
+
+    public EmbulkPluginTester()
+    {
+    }
+
+    public EmbulkPluginTester(Class<?> iface, String name, Class<?> impl)
+    {
+        addPlugin(iface, name, impl);
+    }
+
+    public void addPlugin(Class<?> iface, String name, Class<?> impl)
+    {
+        plugins.add(new PluginDefinition(iface, name, impl));
+    }
+
+    public void run(String ymlPath) throws Exception
+    {
+        if (embulk == null) {
+            Bootstrap bootstrap = new EmbulkEmbed.Bootstrap();
+            bootstrap.addModules(new Module()
+            {
+                @Override
+                public void configure(Binder binder)
+                {
+                    for (PluginDefinition plugin : plugins) {
+                        InjectedPluginSource.registerPluginTo(binder, plugin.iface, plugin.name, plugin.impl);
+                    }
+                }
+            });
+            embulk = bootstrap.initializeCloseable();
+        }
+
+        ConfigSource config = embulk.newConfigLoader().fromYamlFile(new File(ymlPath));
+        embulk.run(config);
+    }
+
+    public void destroy() {
+        if (embulk != null) {
+            embulk.destroy();
+            embulk = null;
+        }
+    }
+
+}

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-jdbc
 version: !ruby/object:Gem::Version
-  version: 0.6.4
+  version: 0.7.0
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-01-15 00:00:00.000000000 Z
+date: 2016-03-29 00:00:00.000000000 Z
 dependencies: []
 description: Selects records from a table.
 email:
@@ -36,11 +36,13 @@ files:
 - src/main/java/org/embulk/input/jdbc/getter/DateColumnGetter.java
 - src/main/java/org/embulk/input/jdbc/getter/DoubleColumnGetter.java
 - src/main/java/org/embulk/input/jdbc/getter/FloatColumnGetter.java
+- src/main/java/org/embulk/input/jdbc/getter/JsonColumnGetter.java
 - src/main/java/org/embulk/input/jdbc/getter/LongColumnGetter.java
 - src/main/java/org/embulk/input/jdbc/getter/StringColumnGetter.java
 - src/main/java/org/embulk/input/jdbc/getter/TimeColumnGetter.java
 - src/main/java/org/embulk/input/jdbc/getter/TimestampColumnGetter.java
-- classpath/embulk-input-jdbc-0.6.4.jar
+- src/test/java/org/embulk/input/EmbulkPluginTester.java
+- classpath/embulk-input-jdbc-0.7.0.jar
 homepage: https://github.com/embulk/embulk-input-jdbc
 licenses:
 - Apache 2.0