embulk-output-clickhouse 0.1.4

data/src/main/java/org/embulk/output/clickhouse/ClickhouseBatchInsert.java

@@ -0,0 +1,223 @@
+package org.embulk.output.clickhouse;
+
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.sql.*;
+import java.util.Calendar;
+import java.util.Locale;
+
+import org.embulk.output.jdbc.*;
+
+import com.google.common.base.Optional;
+import org.embulk.spi.Exec;
+import org.embulk.spi.time.Timestamp;
+import org.slf4j.Logger;
+
+public class ClickhouseBatchInsert
+        implements BatchInsert {
+
+    private final Logger logger = Exec.getLogger(StandardBatchInsert.class);
+
+    private final JdbcOutputConnector connector;
+    private final Optional<MergeConfig> mergeConfig;
+
+    private JdbcOutputConnection connection;
+    private PreparedStatement batch;
+    private int index;
+    private int batchWeight;
+    private int batchRows;
+    private long totalRows;
+    private int[] lastUpdateCounts;
+
+    public ClickhouseBatchInsert(JdbcOutputConnector connector, Optional <MergeConfig> mergeConfig) throws
+        IOException, SQLException
+    {
+        this.connector = connector;
+        this.mergeConfig = mergeConfig;
+    }
+
+    public void prepare(TableIdentifier loadTable, JdbcSchema insertSchema) throws SQLException
+    {
+        this.connection = connector.connect(true);
+        this.index = 1;  // PreparedStatement index begings from 1
+        this.batchRows = 0;
+        this.totalRows = 0;
+        this.batch = prepareStatement(loadTable, insertSchema);
+        batch.clearBatch();
+    }
+
+    protected PreparedStatement prepareStatement(TableIdentifier loadTable, JdbcSchema insertSchema) throws
+    SQLException
+    {
+        return connection.prepareBatchInsertStatement(loadTable, insertSchema, mergeConfig);
+    }
+
+    public int getBatchWeight()
+    {
+        return batchWeight;
+    }
+
+    public void add() throws IOException, SQLException
+    {
+        batch.addBatch();
+        index = 1;  // PreparedStatement index begins from 1
+        batchRows++;
+        batchWeight += 32;  // add weight as overhead of each rows
+    }
+
+    public void close() throws IOException, SQLException
+    {
+        if (connection != null) {
+            connection.close();
+        }
+    }
+
+    public void flush() throws IOException, SQLException
+    {
+        lastUpdateCounts = new int[]{};
+
+        if (batchWeight == 0) return;
+
+        logger.info(String.format("Loading %,d rows", batchRows));
+        long startTime = System.currentTimeMillis();
+        try {
+            lastUpdateCounts = batch.executeBatch();  // here can't use returned value because MySQL Connector/J returns SUCCESS_NO_INFO as a batch result
+            double seconds = (System.currentTimeMillis() - startTime) / 1000.0;
+
+            totalRows += batchRows;
+            logger.info(String.format("> %.2f seconds (loaded %,d rows in total)", seconds, totalRows));
+
+        } catch (BatchUpdateException e) {
+            // will be used for retry
+            lastUpdateCounts = e.getUpdateCounts();
+            throw e;
+
+        } finally {
+            // clear for retry
+            batch.clearBatch();
+            batchRows = 0;
+            batchWeight = 0;
+        }
+    }
+
+    @Override
+    public int[] getLastUpdateCounts()
+    {
+        return lastUpdateCounts;
+    }
+
+    public void finish() throws IOException, SQLException
+    {
+    }
+
+    public void setNull(int sqlType) throws IOException, SQLException
+    {
+        batch.setNull(index, sqlType);
+        nextColumn(0);
+    }
+
+    public void setBoolean(boolean v) throws IOException, SQLException
+    {
+        batch.setBoolean(index, v);
+        nextColumn(1);
+    }
+
+    public void setByte(byte v) throws IOException, SQLException
+    {
+        batch.setByte(index, v);
+        nextColumn(1);
+    }
+
+    public void setShort(short v) throws IOException, SQLException
+    {
+        batch.setShort(index, v);
+        nextColumn(2);
+    }
+
+    public void setInt(int v) throws IOException, SQLException
+    {
+        batch.setInt(index, v);
+        nextColumn(4);
+    }
+
+    public void setLong(long v) throws IOException, SQLException
+    {
+        batch.setLong(index, v);
+        nextColumn(8);
+    }
+
+    public void setFloat(float v) throws IOException, SQLException
+    {
+        batch.setFloat(index, v);
+        nextColumn(4);
+    }
+
+    public void setDouble(double v) throws IOException, SQLException
+    {
+        batch.setDouble(index, v);
+        nextColumn(8);
+    }
+
+    public void setBigDecimal(BigDecimal v) throws IOException, SQLException
+    {
+        // use estimated number of necessary bytes + 8 byte for the weight
+        // assuming one place needs 4 bits. ceil(v.precision() / 2.0) + 8
+        batch.setBigDecimal(index, v);
+        nextColumn((v.precision() & ~2) / 2 + 8);
+    }
+
+    public void setString(String v) throws IOException, SQLException
+    {
+        batch.setString(index, v);
+        // estimate all chracters use 2 bytes; almost enough for the worst case
+        nextColumn(v.length() * 2 + 4);
+    }
+
+    public void setNString(String v) throws IOException, SQLException
+    {
+        batch.setNString(index, v);
+        // estimate all chracters use 2 bytes; almost enough for the worst case
+        nextColumn(v.length() * 2 + 4);
+    }
+
+    public void setBytes(byte[] v) throws IOException, SQLException
+    {
+        batch.setBytes(index, v);
+        nextColumn(v.length + 4);
+    }
+
+    public void setSqlDate(Timestamp v, Calendar cal) throws IOException, SQLException
+    {
+        // JavaDoc of java.sql.Time says:
+        // >> To conform with the definition of SQL DATE, the millisecond values wrapped by a java.sql.Date instance must be 'normalized' by setting the hours, minutes, seconds, and milliseconds to zero in the particular time zone with which the instance is associated.
+        cal.setTimeInMillis(v.getEpochSecond() * 1000);
+        cal.set(Calendar.SECOND, 0);
+        cal.set(Calendar.MINUTE, 0);
+        cal.set(Calendar.HOUR_OF_DAY, 0);
+        Date normalized = new Date(cal.getTimeInMillis());
+        batch.setDate(index, normalized, cal);
+        nextColumn(32);
+    }
+
+    public void setSqlTime(Timestamp v, Calendar cal) throws IOException, SQLException
+    {
+        Time t = new Time(v.toEpochMilli());
+        batch.setTime(index, t, cal);
+        nextColumn(32);
+    }
+
+    public void setSqlTimestamp(Timestamp v, Calendar cal) throws IOException, SQLException
+    {
+        java.sql.Timestamp t = new java.sql.Timestamp(v.getEpochSecond() * 1000);
+        t.setNanos(v.getNano());
+        batch.setTimestamp(index, t);
+        nextColumn(32);
+    }
+
+    private void nextColumn(int weight)
+    {
+        index++;
+        batchWeight += weight + 4;  // add weight as overhead of each columns
+
+    }
+}

data/src/main/java/org/embulk/output/clickhouse/ClickhouseOutputConnection.java

@@ -0,0 +1,40 @@
+package org.embulk.output.clickhouse;
+
+import java.util.List;
+import java.sql.Connection;
+import java.sql.SQLException;
+
+import com.google.common.base.Optional;
+import org.embulk.output.jdbc.JdbcColumn;
+import org.embulk.output.jdbc.JdbcSchema;
+import org.embulk.output.jdbc.JdbcOutputConnection;
+import org.embulk.output.jdbc.MergeConfig;
+import org.embulk.output.jdbc.TableIdentifier;
+import ru.yandex.clickhouse.domain.ClickHouseDataType;
+
+public class ClickhouseOutputConnection
+        extends JdbcOutputConnection
+{
+    public ClickhouseOutputConnection(Connection connection)
+            throws SQLException
+    {
+        super(connection, null);
+    }
+
+    @Override
+    protected String buildColumnTypeName(JdbcColumn c)
+    {
+        switch(c.getSimpleTypeName()) {
+            case "CLOB":
+                return "String";
+            case "DOUBLE PRECISION":
+                return "Float64";
+            case "DATETIME64(3)":
+                return "DateTime64";
+
+            default:
+                return ClickHouseDataType.fromTypeString(c.getSimpleTypeName()).toString();
+        }
+    }
+
+}

data/src/main/java/org/embulk/output/clickhouse/ClickhouseOutputConnector.java

@@ -0,0 +1,44 @@
+package org.embulk.output.clickhouse;
+
+import java.util.Properties;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+
+import org.embulk.output.jdbc.AbstractJdbcOutputConnector;
+import org.embulk.output.jdbc.JdbcOutputConnection;
+import org.embulk.output.jdbc.TransactionIsolation;
+
+import com.google.common.base.Optional;
+
+public class ClickhouseOutputConnector
+        extends AbstractJdbcOutputConnector
+{
+
+    private final String url;
+    private final Properties properties;
+
+    public ClickhouseOutputConnector(String url, Properties properties,
+            Optional<TransactionIsolation> transactionIsolation)
+    {
+        super(transactionIsolation);
+
+        this.url = url;
+        this.properties = properties;
+    }
+
+    @Override
+    protected JdbcOutputConnection connect() throws SQLException
+    {
+        Connection c = DriverManager.getConnection(url, properties);
+        try {
+          ClickhouseOutputConnection con = new ClickhouseOutputConnection(c);
+            c = null;
+            return con;
+        } finally {
+            if (c != null) {
+                c.close();
+            }
+        }
+    }
+}

data/src/main/java/org/embulk/output/clickhouse/setter/ClickhouseColumnSetterFactory.java

@@ -0,0 +1,31 @@
+package org.embulk.output.clickhouse.setter;
+
+import org.embulk.output.jdbc.BatchInsert;
+import org.embulk.output.jdbc.JdbcColumn;
+import org.embulk.output.jdbc.JdbcColumnOption;
+import org.embulk.output.jdbc.setter.ColumnSetter;
+import org.embulk.output.jdbc.setter.ColumnSetterFactory;
+import org.embulk.output.jdbc.setter.SqlTimestampColumnSetter;
+import org.embulk.output.jdbc.setter.StringColumnSetter;
+import org.joda.time.DateTimeZone;
+
+public class ClickhouseColumnSetterFactory
+        extends ColumnSetterFactory
+{
+    public ClickhouseColumnSetterFactory(BatchInsert batch, DateTimeZone defaultTimeZone)
+    {
+        super(batch, defaultTimeZone);
+    }
+
+    @Override
+    public ColumnSetter newCoalesceColumnSetter(JdbcColumn column, JdbcColumnOption option)
+    {
+        if (column.getSimpleTypeName().equalsIgnoreCase("datetime64(3)")) {
+            // actually "timestamp"
+            return new ClickhouseSqlTimeColumnSetter(batch, column, newDefaultValueSetter(column, option), newCalendar(option));
+        } else {
+            return super.newCoalesceColumnSetter(column, option);
+        }
+    }
+
+}

data/src/main/java/org/embulk/output/clickhouse/setter/ClickhouseSqlTimeColumnSetter.java

@@ -0,0 +1,30 @@
+package org.embulk.output.clickhouse.setter;
+
+import java.io.IOException;
+import java.sql.SQLException;
+import java.util.Calendar;
+
+import org.embulk.output.jdbc.BatchInsert;
+import org.embulk.output.jdbc.JdbcColumn;
+import org.embulk.output.jdbc.setter.DefaultValueSetter;
+import org.embulk.output.jdbc.setter.SqlTimeColumnSetter;
+import org.embulk.spi.time.Timestamp;
+
+public class ClickhouseSqlTimeColumnSetter
+        extends SqlTimeColumnSetter
+{
+
+    public ClickhouseSqlTimeColumnSetter(BatchInsert batch, JdbcColumn column,
+                                         DefaultValueSetter defaultValue,
+                                         Calendar calendar)
+    {
+        super(batch, column, defaultValue, calendar);
+    }
+
+    @Override
+    public void timestampValue(Timestamp v) throws IOException, SQLException
+    {
+        batch.setSqlTimestamp(v, calendar);
+    }
+
+}

metadata

@@ -0,0 +1,77 @@
+--- !ruby/object:Gem::Specification
+name: embulk-output-clickhouse
+version: !ruby/object:Gem::Version
+  version: 0.1.4
+platform: ruby
+authors:
+- Karri Niemelä
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2020-02-10 00:00:00.000000000 Z
+dependencies: []
+description: Dumps records to Clickhouse.
+email:
+- karri.niemela@beans.fi
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- LICENSE
+- LICENSE.txt
+- README.md
+- build.gradle
+- classpath/embulk-output-clickhouse-0.1.4.jar
+- classpath/embulk-output-jdbc-0.8.7.jar
+- config/checkstyle/checkstyle.xml
+- config/checkstyle/default.xml
+- default_jdbc_driver/clickhouse-jdbc-0.2.4.jar
+- default_jdbc_driver/commons-codec-1.9.jar
+- default_jdbc_driver/commons-logging-1.2.jar
+- default_jdbc_driver/guava-19.0.jar
+- default_jdbc_driver/httpclient-4.5.2.jar
+- default_jdbc_driver/httpcore-4.4.4.jar
+- default_jdbc_driver/httpmime-4.5.2.jar
+- default_jdbc_driver/jackson-annotations-2.7.0.jar
+- default_jdbc_driver/jackson-core-2.7.3.jar
+- default_jdbc_driver/jackson-databind-2.7.3.jar
+- default_jdbc_driver/jaxb-api-2.3.0.jar
+- default_jdbc_driver/lz4-1.3.0.jar
+- default_jdbc_driver/slf4j-api-1.7.21.jar
+- gradle/wrapper/gradle-wrapper.jar
+- gradle/wrapper/gradle-wrapper.properties
+- gradlew
+- gradlew.bat
+- lib/embulk/output/clickhouse.rb
+- src/main/java/org/embulk/output/ClickhouseOutputPlugin.java
+- src/main/java/org/embulk/output/clickhouse/ClickhouseBatchInsert.java
+- src/main/java/org/embulk/output/clickhouse/ClickhouseOutputConnection.java
+- src/main/java/org/embulk/output/clickhouse/ClickhouseOutputConnector.java
+- src/main/java/org/embulk/output/clickhouse/setter/ClickhouseColumnSetterFactory.java
+- src/main/java/org/embulk/output/clickhouse/setter/ClickhouseSqlTimeColumnSetter.java
+homepage: https://github.com/kakoni/embulk-output-clickhouse
+licenses:
+- MIT
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.4.8
+signing_key:
+specification_version: 4
+summary: Clickhouse output plugin for Embulk
+test_files: []