embulk-input-clickhouse 0.1.0

data/src/main/java/org/embulk/input/clickhouse/getter/ArrayColumnGetter.java

@@ -0,0 +1,59 @@
+package org.embulk.input.clickhouse.getter;
+
+import org.embulk.input.jdbc.getter.AbstractColumnGetter;
+import org.embulk.spi.Column;
+import org.embulk.spi.PageBuilder;
+import org.embulk.spi.json.JsonParseException;
+import org.embulk.spi.type.Type;
+
+import java.sql.Array;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+
+/**
+ * Supported JSON or String
+ */
+public class ArrayColumnGetter extends AbstractColumnGetter
+{
+    protected Array value;
+    Column2JsonUtil c2j = new Column2JsonUtil();
+
+    public ArrayColumnGetter(PageBuilder to, Type toType)
+    {
+        super(to, toType);
+    }
+
+    @Override
+    protected void fetch(ResultSet from, int fromIndex) throws SQLException
+    {
+        value = from.getArray(fromIndex);
+    }
+
+    @Override
+    protected Type getDefaultToType()
+    {
+        return org.embulk.spi.type.Types.JSON;
+    }
+
+    @Override
+    public void jsonColumn(Column column)
+    {
+        try {
+            c2j.jsonColumn(column, to, value.getArray());
+        }
+        catch (JsonParseException | SQLException | ClassCastException e) {
+            super.jsonColumn(column);
+        }
+    }
+
+    @Override
+    public void stringColumn(Column column)
+    {
+        try {
+            c2j.stringColumn(column, to, value.getArray());
+        }
+        catch (SQLException e) {
+            to.setString(column, value.toString());
+        }
+    }
+}

data/src/main/java/org/embulk/input/clickhouse/getter/BigIntegerColumnGetter.java

@@ -0,0 +1,96 @@
+package org.embulk.input.clickhouse.getter;
+
+import com.google.gson.Gson;
+
+import org.embulk.input.jdbc.getter.AbstractColumnGetter;
+import org.embulk.spi.Column;
+import org.embulk.spi.PageBuilder;
+import org.embulk.spi.json.JsonParseException;
+import org.embulk.spi.json.JsonParser;
+import org.embulk.spi.type.Type;
+import org.embulk.spi.type.Types;
+import org.msgpack.value.Value;
+
+import java.math.BigInteger;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+
+import ru.yandex.clickhouse.response.ClickHouseResultSet;
+
+public class BigIntegerColumnGetter extends AbstractColumnGetter
+{
+
+    protected BigInteger value;
+
+    private final Gson gson = new Gson();
+    private final JsonParser jsonParser = new JsonParser();
+
+    private String originalColumnTypeName;
+
+    public BigIntegerColumnGetter(PageBuilder to, Type toType, String originalColumnTypeName)
+    {
+        super(to, toType);
+        this.originalColumnTypeName = originalColumnTypeName;
+    }
+
+    @Override
+    protected void fetch(ResultSet from, int fromIndex) throws SQLException
+    {
+        ClickHouseResultSet chFrom = (ClickHouseResultSet) from;
+        value = (BigInteger) chFrom.getObject(fromIndex);
+    }
+
+    @Override
+    protected Type getDefaultToType()
+    {
+        // In default, this ColumnGetter try to convert BigInteger value to Long value.
+        // If value is too big for Long type, please choose Type.String or Type.JSON
+        return Types.LONG;
+    }
+
+    @Override
+    public void longColumn(Column column)
+    {
+        try {
+
+            to.setLong(column, Long.parseLong(value.toString()));
+
+        }
+        catch (NumberFormatException e) {
+            throw new NumberFormatException(
+                    String.format(
+                            "%s In '%s %s' is too large for Long type. \n" +
+                            "Please set other type (string or json) in your config file : \n" +
+                            "\n" +
+                            "in:\n" +
+                            "  type: clickhouse\n" +
+                            "  ...\n" +
+                            "  column_option:\n" +
+                            "    %s: {type: string}\n",
+                            value.toString(),
+                            column.getName(),
+                            originalColumnTypeName,
+                            column.getName()
+                    ));
+        }
+    }
+
+    @Override
+    public void stringColumn(Column column)
+    {
+        to.setString(column, value.toString());
+    }
+
+    @Override
+    public void jsonColumn(Column column)
+    {
+        try {
+            String jsonString = gson.toJson(value.toString());
+            Value v = jsonParser.parse(jsonString);
+            to.setJson(column, v);
+        }
+        catch (JsonParseException e) {
+            super.jsonColumn(column);
+        }
+    }
+}

data/src/main/java/org/embulk/input/clickhouse/getter/Column2JsonUtil.java

@@ -0,0 +1,30 @@
+package org.embulk.input.clickhouse.getter;
+
+import com.google.gson.Gson;
+import com.google.gson.JsonParseException;
+
+import org.embulk.spi.Column;
+import org.embulk.spi.PageBuilder;
+import org.embulk.spi.json.JsonParser;
+import org.msgpack.value.Value;
+
+import java.sql.SQLException;
+
+public class Column2JsonUtil
+{
+    private final Gson gson = new Gson();
+    private final JsonParser jsonParser = new JsonParser();
+
+    public void jsonColumn(Column column, PageBuilder to, Object value) throws JsonParseException, SQLException, ClassCastException
+    {
+        String jsonString = gson.toJson(value);
+        Value v = jsonParser.parse(jsonString);
+        to.setJson(column, v);
+    }
+
+    public void stringColumn(Column column, PageBuilder to, Object value)
+    {
+        String jsonString = gson.toJson(value);
+        to.setString(column, jsonString);
+    }
+}

data/src/main/java/org/embulk/input/clickhouse/getter/TupleColumnGetter.java

@@ -0,0 +1,55 @@
+package org.embulk.input.clickhouse.getter;
+
+import org.embulk.input.jdbc.getter.AbstractColumnGetter;
+import org.embulk.spi.Column;
+import org.embulk.spi.PageBuilder;
+import org.embulk.spi.json.JsonParseException;
+import org.embulk.spi.type.Type;
+import org.embulk.spi.type.Types;
+
+import java.sql.ResultSet;
+import java.sql.SQLException;
+
+/**
+ * Tuple is provided as String value.
+ */
+public class TupleColumnGetter extends AbstractColumnGetter
+{
+    protected String value;
+    Column2JsonUtil c2j = new Column2JsonUtil();
+
+    public TupleColumnGetter(PageBuilder to, Type toType)
+    {
+        super(to, toType);
+    }
+
+    @Override
+    protected void fetch(ResultSet from, int fromIndex) throws SQLException
+    {
+        value = from.getString(fromIndex);
+    }
+
+    @Override
+    protected Type getDefaultToType()
+    {
+        return Types.STRING;
+    }
+
+    @Override
+    public void jsonColumn(Column column)
+    {
+        //FIXME: convert to flat array as JSON
+        try {
+            c2j.jsonColumn(column, to, value);
+        }
+        catch (JsonParseException | SQLException | ClassCastException e) {
+            super.jsonColumn(column);
+        }
+    }
+
+    @Override
+    public void stringColumn(Column column)
+    {
+        to.setString(column, value);
+    }
+}

data/src/test/java/org/embulk/input/clickhouse/TestClickhouseInputPlugin.java

@@ -0,0 +1,5 @@
+package org.embulk.input.clickhouse;
+
+public class TestClickhouseInputPlugin
+{
+}

metadata

@@ -0,0 +1,107 @@
+--- !ruby/object:Gem::Specification
+name: embulk-input-clickhouse
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- TAC
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2018-10-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  name: bundler
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  name: rake
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '10.0'
+description: Loads records from Clickhouse.
+email:
+- tac@tac42.net
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- LICENSE.txt
+- README.md
+- build.gradle
+- classpath/clickhouse-jdbc-0.1.41.jar
+- classpath/commons-codec-1.9.jar
+- classpath/commons-logging-1.2.jar
+- classpath/embulk-input-clickhouse-0.1.0.jar
+- classpath/embulk-input-jdbc-0.9.3.jar
+- classpath/gson-2.8.5.jar
+- classpath/guava-19.0.jar
+- classpath/httpclient-4.5.2.jar
+- classpath/httpcore-4.4.4.jar
+- classpath/httpmime-4.5.2.jar
+- classpath/jackson-annotations-2.7.0.jar
+- classpath/jackson-core-2.7.3.jar
+- classpath/jackson-databind-2.7.3.jar
+- classpath/joda-time-2.9.9.jar
+- classpath/lz4-1.3.0.jar
+- classpath/slf4j-api-1.7.21.jar
+- config/checkstyle/checkstyle.xml
+- config/checkstyle/default.xml
+- gradle/wrapper/gradle-wrapper.jar
+- gradle/wrapper/gradle-wrapper.properties
+- gradlew
+- gradlew.bat
+- lib/embulk/input/clickhouse.rb
+- src/main/java/org/embulk/input/clickhouse/ClickHouseColumnGetterFactory.java
+- src/main/java/org/embulk/input/clickhouse/ClickHouseInputConnection.java
+- src/main/java/org/embulk/input/clickhouse/ClickhouseInputPlugin.java
+- src/main/java/org/embulk/input/clickhouse/getter/ArrayColumnGetter.java
+- src/main/java/org/embulk/input/clickhouse/getter/BigIntegerColumnGetter.java
+- src/main/java/org/embulk/input/clickhouse/getter/Column2JsonUtil.java
+- src/main/java/org/embulk/input/clickhouse/getter/TupleColumnGetter.java
+- src/test/java/org/embulk/input/clickhouse/TestClickhouseInputPlugin.java
+homepage: https://github.com/tac0x2a/embulk-input-clickhouse
+licenses:
+- MIT
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.6.8
+signing_key:
+specification_version: 4
+summary: ClickHouse input plugin for Embulk
+test_files: []