embulk-input-postgresql 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 2a8135b43af2ab08ff290956926610cab241f9da
+  data.tar.gz: 8c6ad6f3da2c5a86a69225fbc0c18fc82f2bfcfc
+SHA512:
+  metadata.gz: 4263cbf52d0f00aa26d19d621baf2a3c14a31bd90f5e2505a4e891fdaee9118d2c87977ac48dd84d65c1051f224a7c74b50fe4be75aa485ac0c05d2c539ef544
+  data.tar.gz: 0f81e7b36f57a492cb67508d6a813807351d91b0ce9ce30a0fdb9c56fa0b45a665c78abc077d0e85f91732d5ec8b89d55ae83c50144ee7d67dc3c6044022d5a0

data/README.md

@@ -0,0 +1,42 @@
+# PostgreSQL input plugins for Embulk
+
+PostgreSQL input plugins for Embulk loads records from PostgreSQL.
+
+## Overview
+
+* **Plugin type**: input
+* **Resume supported**: yes
+
+## Configuration
+
+- **host**: database host name (string, required)
+- **port**: database port number (integer, 5432)
+- **user**: database login user name (string, required)
+- **password**: database login password (string, default: "")
+- **database**: destination database name (string, required)
+- **schema**: destination name (string, default: "public")
+- **table**: destination name (string, required)
+- **select**: comma-separated list of columns to select (string, default: "*")
+- **where**: WHERE condition to filter the rows (string, default: no-condition)
+- **fetch_rows**: number of rows to fetch one time (used for java.sql.Statement#setFetchSize) (integer, default: 10000)
+- **options**: extra JDBC properties (hash, default: {})
+
+## Example
+
+```yaml
+in:
+  type: postgresql
+  host: localhost
+  user: myuser
+  password: ""
+  database: my_database
+  table: my_table
+  select: "col1, col2, col3"
+  where: "col4 != 'a'"
+```
+
+## Build
+
+```
+$ ./gradlew gem
+```

data/build.gradle

@@ -0,0 +1,7 @@
+dependencies {
+    compile project(':embulk-input-jdbc')
+
+    compile 'org.postgresql:postgresql:9.4-1200-jdbc41'
+
+    testCompile project(':embulk-input-jdbc').sourceSets.test.output
+}

data/lib/embulk/input/postgresql.rb

@@ -0,0 +1,3 @@
+Embulk::JavaPlugin.register_input(
+  :postgresql, "org.embulk.input.PostgreSQLInputPlugin",
+  File.expand_path('../../../../classpath', __FILE__))

data/src/main/java/org/embulk/input/PostgreSQLInputPlugin.java

@@ -0,0 +1,60 @@
+package org.embulk.input;
+
+import java.util.Properties;
+import java.sql.Connection;
+import java.sql.Driver;
+import java.sql.SQLException;
+import com.google.common.base.Throwables;
+import org.embulk.config.Config;
+import org.embulk.input.jdbc.AbstractJdbcInputPlugin;
+import org.embulk.input.postgresql.PostgreSQLInputConnection;
+
+public class PostgreSQLInputPlugin
+        extends AbstractJdbcInputPlugin
+{
+    private static final String DEFAULT_SCHEMA = "public";
+    private static final int DEFAULT_PORT = 5432;
+
+    private static final Driver driver = new org.postgresql.Driver();
+
+    @Override
+    protected PostgreSQLInputConnection newConnection(PluginTask task) throws SQLException
+    {
+        String url = String.format("jdbc:postgresql://%s:%d/%s",
+                task.getHost(), task.getPort().or(DEFAULT_PORT), task.getDatabase());
+
+        Properties props = new Properties();
+        props.setProperty("user", task.getUser());
+        props.setProperty("password", task.getPassword());
+        props.setProperty("loginTimeout",   "300"); // seconds
+        props.setProperty("socketTimeout", "1800"); // seconds
+
+        // Enable keepalive based on tcp_keepalive_time, tcp_keepalive_intvl and tcp_keepalive_probes kernel parameters.
+        // Socket options TCP_KEEPCNT, TCP_KEEPIDLE, and TCP_KEEPINTVL are not configurable.
+        props.setProperty("tcpKeepAlive", "true");
+
+        // TODO
+        //switch task.getSssl() {
+        //when "disable":
+        //    break;
+        //when "enable":
+        //    props.setProperty("sslfactory", "org.postgresql.ssl.NonValidatingFactory");  // disable server-side validation
+        //when "verify":
+        //    props.setProperty("ssl", "true");
+        //    break;
+        //}
+
+        props.putAll(task.getOptions());
+
+        Connection con = driver.connect(url, props);
+        try {
+            PostgreSQLInputConnection c = new PostgreSQLInputConnection(con, task.getSchema().or(DEFAULT_SCHEMA));
+            con = null;
+            return c;
+        } finally {
+            if (con != null) {
+                con.close();
+            }
+        }
+    }
+}

data/src/main/java/org/embulk/input/postgresql/PostgreSQLInputConnection.java

@@ -0,0 +1,60 @@
+package org.embulk.input.postgresql;
+
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import org.slf4j.Logger;
+import org.embulk.spi.Exec;
+import org.embulk.input.jdbc.JdbcInputConnection;
+
+public class PostgreSQLInputConnection
+        extends JdbcInputConnection
+{
+    private final Logger logger = Exec.getLogger(PostgreSQLInputConnection.class);
+
+    public PostgreSQLInputConnection(Connection connection, String schemaName)
+            throws SQLException
+    {
+        super(connection, schemaName);
+    }
+
+    @Override
+    protected CursorSelect newBatchSelect(String select, int fetchRows) throws SQLException
+    {
+        executeUpdate("DECLARE cur NO SCROLL CURSOR FOR "+select);
+
+        String fetchSql = "FETCH FORWARD "+fetchRows+" FROM cur";
+        return new CursorSelect(fetchSql, connection.prepareStatement(fetchSql));
+    }
+
+    public class CursorSelect
+            implements BatchSelect
+    {
+        private final String fetchSql;
+        private final PreparedStatement fetchStatement;
+
+        public CursorSelect(String fetchSql, PreparedStatement fetchStatement) throws SQLException
+        {
+            this.fetchSql = fetchSql;
+            this.fetchStatement = fetchStatement;
+        }
+
+        public ResultSet fetch() throws SQLException
+        {
+            logger.info("SQL: " + fetchSql);
+            long startTime = System.currentTimeMillis();
+
+            ResultSet rs = fetchStatement.executeQuery();
+
+            double seconds = (System.currentTimeMillis() - startTime) / 1000.0;
+            logger.info(String.format("> %.2f seconds", seconds));
+            return rs;
+        }
+
+        public void close() throws SQLException
+        {
+            // TODO close?
+        }
+    }
+}

metadata

@@ -0,0 +1,56 @@
+--- !ruby/object:Gem::Specification
+name: embulk-input-postgresql
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- FURUHASHI Sadayuki
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2015-02-16 00:00:00.000000000 Z
+dependencies: []
+description: JDBC input plugin is an Embulk plugin that loads records from JDBC so that any output plugins can receive the records. Search the output plugins by "embulk-output" keyword.
+email:
+- frsyuki@users.sourceforge.jp
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- README.md
+- build.gradle
+- lib/embulk/input/postgresql.rb
+- src/main/java/org/embulk/input/PostgreSQLInputPlugin.java
+- src/main/java/org/embulk/input/postgresql/PostgreSQLInputConnection.java
+- classpath/embulk-input-jdbc-0.1.0.jar
+- classpath/embulk-input-postgresql-0.1.0.jar
+- classpath/jna-4.1.0.jar
+- classpath/jna-platform-4.1.0.jar
+- classpath/postgresql-9.4-1200-jdbc41.jar
+- classpath/slf4j-simple-1.7.7.jar
+- classpath/waffle-jna-1.7.jar
+homepage: https://github.com/embulk/embulk-input-jdbc
+licenses:
+- Apache 2.0
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.1.9
+signing_key:
+specification_version: 4
+summary: JDBC input plugin for Embulk
+test_files: []