embulk-output-postgresql 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: d0c60c7936524e61c263938180308a38347e99ff
+  data.tar.gz: b9e0957dabaf0373af15303df9f20000185533af
+SHA512:
+  metadata.gz: 6325e1a100edae2e0ff2aa6123e01ac658d79f654b4aea880bde69a01602250808971d95621d7a79e9b4a70ffe7b2f7f3aeeef57cf7ea4c9209ad6b2215903d0
+  data.tar.gz: 282c82aac27df9ffaf9298f4f4fac3f31c2a6c67bbb3eb27a2d7bfede1f87613472348dfdbf46cf20241c7048d5617052d8b24f3ff0aa518996a4944ed1a18b2

data/README.md

@@ -0,0 +1,43 @@
+# PostgreSQL output plugins for Embulk
+
+PostgreSQL output plugins for Embulk loads records to PostgreSQL.
+
+## Overview
+
+* **Plugin type**: output
+* **Load all or nothing**: depnds on the mode:
+  * **insert**: no
+  * **replace**: yes
+* **Resume supported**: no
+
+## Configuration
+
+- **host**: database host name (string, required)
+- **port**: database port number (integer, default: 5432)
+- **user**: database login user name (string, required)
+- **password**: database login password (string, default: "")
+- **database**: destination database name (string, required)
+- **schema**: destination name (string, default: "public")
+- **table**: destination name (string, required)
+- **mode**: "replace" or "insert" (string, required)
+- **batch_size**: size of a single batch insert (integer, default: 16777216)
+- **options**: extra connection properties (hash, default: {})
+
+### Example
+
+```yaml
+out:
+  type: postgresql
+  host: localhost
+  user: pg
+  password: ""
+  database: my_database
+  table: my_table
+  mode: insert
+```
+
+### Build
+
+```
+$ ./gradlew gem
+```

data/build.gradle

@@ -0,0 +1,7 @@
+dependencies {
+    compile project(':embulk-output-jdbc')
+
+    compile 'org.postgresql:postgresql:9.4-1200-jdbc41'
+
+    testCompile project(':embulk-output-jdbc').sourceSets.test.output
+}

data/lib/embulk/output/postgresql.rb

@@ -0,0 +1,3 @@
+Embulk::JavaPlugin.register_output(
+  :postgresql, "org.embulk.output.PostgreSQLOutputPlugin",
+  File.expand_path('../../../../classpath', __FILE__))

data/src/main/java/org/embulk/output/PostgreSQLOutputPlugin.java

@@ -0,0 +1,61 @@
+package org.embulk.output;
+
+import java.util.Properties;
+import java.io.IOException;
+import java.sql.SQLException;
+import org.embulk.spi.Exec;
+import org.embulk.output.jdbc.AbstractJdbcOutputPlugin;
+import org.embulk.output.jdbc.BatchInsert;
+import org.embulk.output.postgresql.PostgreSQLOutputConnector;
+import org.embulk.output.postgresql.PostgreSQLCopyBatchInsert;
+
+public class PostgreSQLOutputPlugin
+        extends AbstractJdbcOutputPlugin
+{
+    private static final String DEFAULT_SCHEMA = "public";
+    private static final int DEFAULT_PORT = 5432;
+
+    @Override
+    protected PostgreSQLOutputConnector getConnector(PluginTask task, boolean retryableMetadataOperation)
+    {
+        String url = String.format("jdbc:postgresql://%s:%d/%s",
+                task.getHost(), task.getPort().or(DEFAULT_PORT), task.getDatabase());
+
+        Properties props = new Properties();
+        props.setProperty("user", task.getUser());
+        props.setProperty("password", task.getPassword());
+        props.setProperty("loginTimeout",   "300"); // seconds
+        props.setProperty("socketTimeout", "1800"); // seconds
+
+        // Enable keepalive based on tcp_keepalive_time, tcp_keepalive_intvl and tcp_keepalive_probes kernel parameters.
+        // Socket options TCP_KEEPCNT, TCP_KEEPIDLE, and TCP_KEEPINTVL are not configurable.
+        props.setProperty("tcpKeepAlive", "true");
+
+        // TODO
+        //switch task.getSssl() {
+        //when "disable":
+        //    break;
+        //when "enable":
+        //    props.setProperty("sslfactory", "org.postgresql.ssl.NonValidatingFactory");  // disable server-side validation
+        //when "verify":
+        //    props.setProperty("ssl", "true");
+        //    break;
+        //}
+
+        if (!retryableMetadataOperation) {
+            // non-retryable batch operation uses longer timeout
+            props.setProperty("loginTimeout",    "300");  // seconds
+            props.setProperty("socketTimeout", "28800");  // seconds
+        }
+
+        props.putAll(task.getOptions());
+
+        return new PostgreSQLOutputConnector(url, props, task.getSchema().or(DEFAULT_SCHEMA));
+    }
+
+    @Override
+    protected BatchInsert newBatchInsert(PluginTask task) throws IOException, SQLException
+    {
+        return new PostgreSQLCopyBatchInsert(getConnector(task, true));
+    }
+}

data/src/main/java/org/embulk/output/postgresql/AbstractPostgreSQLCopyBatchInsert.java

@@ -0,0 +1,217 @@
+package org.embulk.output.postgresql;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.Writer;
+import java.io.BufferedWriter;
+import java.io.OutputStreamWriter;
+import java.io.IOException;
+import java.nio.charset.Charset;
+import java.math.BigDecimal;
+import java.sql.Date;
+import java.sql.Time;
+import java.sql.Timestamp;
+import java.sql.SQLException;
+import org.embulk.spi.Exec;
+import org.embulk.output.jdbc.JdbcSchema;
+import org.embulk.output.jdbc.BatchInsert;
+
+public abstract class AbstractPostgreSQLCopyBatchInsert
+        implements BatchInsert
+{
+    protected static final Charset FILE_CHARSET = Charset.forName("UTF-8");
+
+    protected static final String nullString = "\\N";
+    protected static final String newLineString = "\n";
+    protected static final String delimiterString = "\t";
+
+    protected File currentFile;
+    protected BufferedWriter writer;
+    protected int index;
+    protected int batchRows;
+
+    protected AbstractPostgreSQLCopyBatchInsert() throws IOException
+    {
+        this.index = 0;
+        openNewFile();
+    }
+
+    private File createTempFile() throws IOException
+    {
+        return File.createTempFile("embulk-output-postgres-copy-", ".tsv.tmp");  // TODO configurable temporary file path
+    }
+
+    protected File openNewFile() throws IOException
+    {
+        File newFile = createTempFile();
+        File oldFile = closeCurrentFile();
+        this.writer = openWriter(newFile);
+        currentFile = newFile;
+        return oldFile;
+    }
+
+    protected File closeCurrentFile() throws IOException
+    {
+        if(writer != null) {
+            writer.close();
+            writer = null;
+        }
+        return currentFile;
+    }
+
+    protected BufferedWriter openWriter(File newFile) throws IOException
+    {
+        return new BufferedWriter(new OutputStreamWriter(new FileOutputStream(newFile), FILE_CHARSET));
+    }
+
+    public int getBatchWeight()
+    {
+        long fsize = currentFile.length();
+        if (fsize > Integer.MAX_VALUE) {
+            return Integer.MAX_VALUE;
+        } else {
+            return (int) fsize;
+        }
+    }
+
+    public void finish() throws IOException, SQLException
+    {
+        closeCurrentFile();  // this is necessary to make getBatchWeight() work
+        if (getBatchWeight() != 0) {
+            flush();
+        }
+    }
+
+    public void add() throws IOException
+    {
+        writer.write(newLineString);
+        batchRows++;
+        index = 0;
+    }
+
+    private void appendDelimiter() throws IOException
+    {
+        if(index != 0) {
+            writer.write(delimiterString);
+        }
+        index++;
+    }
+
+    public void setNull(int sqlType) throws IOException
+    {
+        appendDelimiter();
+        writer.write(nullString);
+    }
+
+    public void setBoolean(boolean v) throws IOException
+    {
+        appendDelimiter();
+        writer.write(String.valueOf(v));
+    }
+
+    public void setByte(byte v) throws IOException
+    {
+        appendDelimiter();
+        setEscapedString(String.valueOf(v));
+    }
+
+    public void setShort(short v) throws IOException
+    {
+        appendDelimiter();
+        writer.write(String.valueOf(v));
+    }
+
+    public void setInt(int v) throws IOException
+    {
+        appendDelimiter();
+        writer.write(String.valueOf(v));
+    }
+
+    public void setLong(long v) throws IOException
+    {
+        appendDelimiter();
+        writer.write(String.valueOf(v));
+    }
+
+    public void setFloat(float v) throws IOException
+    {
+        appendDelimiter();
+        writer.write(String.valueOf(v));
+    }
+
+    public void setDouble(double v) throws IOException
+    {
+        appendDelimiter();
+        writer.write(String.valueOf(v));
+    }
+
+    public void setBigDecimal(BigDecimal v) throws IOException
+    {
+        appendDelimiter();
+        writer.write(String.valueOf(v));
+    }
+
+    public void setString(String v) throws IOException
+    {
+        appendDelimiter();
+        setEscapedString(v);
+    }
+
+    public void setNString(String v) throws IOException
+    {
+        appendDelimiter();
+        setEscapedString(v);
+    }
+
+    public void setBytes(byte[] v) throws IOException
+    {
+        appendDelimiter();
+        setEscapedString(String.valueOf(v));
+    }
+
+    public void setSqlDate(Date v, int sqlType) throws IOException
+    {
+        appendDelimiter();
+        writer.write(v.toString());
+    }
+
+    public void setSqlTime(Time v, int sqlType) throws IOException
+    {
+        appendDelimiter();
+        writer.write(v.toString());
+    }
+
+    public void setSqlTimestamp(Timestamp v, int sqlType) throws IOException
+    {
+        appendDelimiter();
+        writer.write(v.toString());
+    }
+
+    // Escape \, \n, \t, \r
+    // Remove \0
+    private void setEscapedString(String v) throws IOException{
+        for (char c : v.toCharArray()) {
+            String s;
+            switch (c) {
+            case '\\':
+                s = "\\\\";
+                break;
+            case '\n':
+                s = "\\n";
+                break;
+            case '\t':
+                s = "\\t";
+                break;
+            case '\r':
+                s = "\\r";
+                break;
+            case 0:
+                s = "";
+                break;
+            default:
+                s = String.valueOf(c);
+            }
+            writer.write(s);
+        }
+    }
+}

data/src/main/java/org/embulk/output/postgresql/PostgreSQLCopyBatchInsert.java

@@ -0,0 +1,73 @@
+package org.embulk.output.postgresql;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.FileInputStream;
+import java.sql.Connection;
+import java.sql.SQLException;
+import org.slf4j.Logger;
+import org.postgresql.copy.CopyManager;
+import org.postgresql.core.BaseConnection;
+import org.embulk.spi.Exec;
+import org.embulk.output.jdbc.JdbcSchema;
+
+public class PostgreSQLCopyBatchInsert
+        extends AbstractPostgreSQLCopyBatchInsert
+{
+    private final Logger logger = Exec.getLogger(PostgreSQLCopyBatchInsert.class);
+    private final PostgreSQLOutputConnector connector;
+
+    private PostgreSQLOutputConnection connection = null;
+    private CopyManager copyManager = null;
+    private String copySql = null;
+    private long totalRows;
+
+    public PostgreSQLCopyBatchInsert(PostgreSQLOutputConnector connector) throws IOException, SQLException
+    {
+        super();
+        this.connector = connector;
+    }
+
+    @Override
+    public void prepare(String loadTable, JdbcSchema insertSchema) throws SQLException
+    {
+        this.connection = connector.connect(true);
+        this.copySql = connection.buildCopySql(loadTable, insertSchema);
+        this.copyManager = connection.newCopyManager();
+        logger.info("Copy SQL: "+copySql);
+    }
+
+    @Override
+    public void flush() throws IOException, SQLException
+    {
+        File file = closeCurrentFile();  // flush buffered data in writer
+
+        logger.info(String.format("Loading %,d rows (%,d bytes)", batchRows, file.length()));
+        long startTime = System.currentTimeMillis();
+        FileInputStream in = new FileInputStream(file);
+        try {
+            // TODO check age of connection and call isValid if it's old and reconnect if it's invalid
+            copyManager.copyIn(copySql, in);
+        } finally {
+            in.close();
+        }
+        double seconds = (System.currentTimeMillis() - startTime) / 1000.0;
+
+        totalRows += batchRows;
+        batchRows = 0;
+        logger.info(String.format("> %.2f seconds (loaded %,d rows in total)", seconds, totalRows));
+
+        openNewFile();
+        file.delete();
+    }
+
+    @Override
+    public void close() throws IOException, SQLException
+    {
+        closeCurrentFile().delete();
+        if (connection != null) {
+            connection.close();
+            connection = null;
+        }
+    }
+}

data/src/main/java/org/embulk/output/postgresql/PostgreSQLOutputConnection.java

@@ -0,0 +1,57 @@
+package org.embulk.output.postgresql;
+
+import java.sql.Connection;
+import java.sql.SQLException;
+import java.sql.Statement;
+import org.postgresql.copy.CopyManager;
+import org.postgresql.core.BaseConnection;
+import org.embulk.spi.Exec;
+import org.embulk.output.jdbc.JdbcOutputConnection;
+import org.embulk.output.jdbc.JdbcColumn;
+import org.embulk.output.jdbc.JdbcSchema;
+
+public class PostgreSQLOutputConnection
+        extends JdbcOutputConnection
+{
+    public PostgreSQLOutputConnection(Connection connection, String schemaName, boolean autoCommit)
+            throws SQLException
+    {
+        super(connection, schemaName);
+        connection.setAutoCommit(autoCommit);
+    }
+
+    public String buildCopySql(String toTable, JdbcSchema toTableSchema)
+    {
+        StringBuilder sb = new StringBuilder();
+
+        sb.append("COPY ");
+        quoteIdentifierString(sb, toTable);
+        sb.append(" (");
+        for (int i=0; i < toTableSchema.getCount(); i++) {
+            if(i != 0) { sb.append(", "); }
+            quoteIdentifierString(sb, toTableSchema.getColumnName(i));
+        }
+        sb.append(") ");
+        sb.append("FROM STDIN");
+
+        return sb.toString();
+    }
+
+    public CopyManager newCopyManager() throws SQLException
+    {
+        return new CopyManager((BaseConnection) connection);
+    }
+
+    @Override
+    protected String convertTypeName(String typeName)
+    {
+        switch(typeName) {
+        case "CLOB":
+            return "TEXT";
+        case "BLOB":
+            return "BYTEA";
+        default:
+            return typeName;
+        }
+    }
+}

data/src/main/java/org/embulk/output/postgresql/PostgreSQLOutputConnector.java

@@ -0,0 +1,40 @@
+package org.embulk.output.postgresql;
+
+import java.util.Properties;
+import java.sql.Driver;
+import java.sql.Connection;
+import java.sql.SQLException;
+import org.embulk.output.jdbc.JdbcOutputConnector;
+import org.embulk.output.jdbc.JdbcOutputConnection;
+
+public class PostgreSQLOutputConnector
+        implements JdbcOutputConnector
+{
+    private static final Driver driver = new org.postgresql.Driver();
+
+    private final String url;
+    private final Properties properties;
+    private final String schemaName;
+
+    public PostgreSQLOutputConnector(String url, Properties properties, String schemaName)
+    {
+        this.url = url;
+        this.properties = properties;
+        this.schemaName = schemaName;
+    }
+
+    @Override
+    public PostgreSQLOutputConnection connect(boolean autoCommit) throws SQLException
+    {
+        Connection c = driver.connect(url, properties);
+        try {
+            PostgreSQLOutputConnection con = new PostgreSQLOutputConnection(c, schemaName, autoCommit);
+            c = null;
+            return con;
+        } finally {
+            if (c != null) {
+                c.close();
+            }
+        }
+    }
+}

metadata

@@ -0,0 +1,59 @@
+--- !ruby/object:Gem::Specification
+name: embulk-output-postgresql
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- FURUHASHI Sadayuki
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2015-02-16 00:00:00.000000000 Z
+dependencies: []
+description: JDBC output plugin is an Embulk plugin that loads records to JDBC read by any input plugins. Search the input plugins by "embulk-input" keyword.
+email:
+- frsyuki@users.sourceforge.jp
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- README.md
+- build.gradle
+- lib/embulk/output/postgresql.rb
+- src/main/java/org/embulk/output/PostgreSQLOutputPlugin.java
+- src/main/java/org/embulk/output/postgresql/AbstractPostgreSQLCopyBatchInsert.java
+- src/main/java/org/embulk/output/postgresql/PostgreSQLCopyBatchInsert.java
+- src/main/java/org/embulk/output/postgresql/PostgreSQLOutputConnection.java
+- src/main/java/org/embulk/output/postgresql/PostgreSQLOutputConnector.java
+- classpath/embulk-output-jdbc-0.1.0.jar
+- classpath/embulk-output-postgresql-0.1.0.jar
+- classpath/jna-4.1.0.jar
+- classpath/jna-platform-4.1.0.jar
+- classpath/postgresql-9.4-1200-jdbc41.jar
+- classpath/slf4j-simple-1.7.7.jar
+- classpath/waffle-jna-1.7.jar
+homepage: https://github.com/embulk/embulk-output-jdbc
+licenses:
+- Apache 2.0
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.1.9
+signing_key:
+specification_version: 4
+summary: JDBC output plugin for Embulk
+test_files: []