embulk-input-postgresql 0.4.0 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 36e483eaadb50c1495f7ca4af1e39b35b9fe53c3
-  data.tar.gz: 438714d06d86d73a47b107e1d9c6a013a68b56f5
+  metadata.gz: 6dcf39b8987aa913c05ae4784056021b1c18cb50
+  data.tar.gz: 345e58de19952c8d778f4ca4cc445207d6bd5e2c
 SHA512:
-  metadata.gz: 2d899f67bc54eac6cb39578f939d46f216bb09484d20be08979658c767a6a8419fc9a7f6f26da17fdb4a26a5ce311b5c8f7ab315d8da8e57755fd5b147c2cbf6
-  data.tar.gz: 07c6da64532ed9b5a9bdccfe15cd7de84351859839ae08fa6cccf858e085f9504fed8cb73ed603d7dfca78a976540bbb054e64c39d5a17d67bd616dfa05e2b20
+  metadata.gz: 868809537b9dcf5eb2449d10c12bc10354673da9f59f47b20106d76fab6d6bb466a64b2a7d9c44a4bf7af4e740e017a1bb3c3395010553eeabce5abd73651f95
+  data.tar.gz: 71079908e53545eea4e48725f7468b379f447ec49cb0ee2436d99e231ff9db625229290fd2c090a7d0a9c8aa0549a840cbe72d340225a8c7abc7318161049ea8

data/README.md

@@ -1,61 +1,89 @@
-# PostgreSQL input plugins for Embulk
-
-PostgreSQL input plugins for Embulk loads records from PostgreSQL.
-
-## Overview
-
-* **Plugin type**: input
-* **Resume supported**: yes
-
-## Configuration
-
-- **host**: database host name (string, required)
-- **port**: database port number (integer, 5432)
-- **user**: database login user name (string, required)
-- **password**: database login password (string, default: "")
-- **database**: destination database name (string, required)
-- **schema**: destination schema name (string, default: "public")
-- **fetch_rows**: number of rows to fetch one time (used for java.sql.Statement#setFetchSize) (integer, default: 10000)
-- **options**: extra JDBC properties (hash, default: {})
-- If you write SQL directly,
-  - **query**: SQL to run (string)
-- If **query** is not set,
-  - **table**: destination table name (string, required)
-  - **select**: comma-separated list of columns to select (string, default: "*")
-  - **where**: WHERE condition to filter the rows (string, default: no-condition)
-
-## Example
-
-```yaml
-in:
-  type: postgresql
-  host: localhost
-  user: myuser
-  password: ""
-  database: my_database
-  table: my_table
-  select: "col1, col2, col3"
-  where: "col4 != 'a'"
-```
-
-If you need a complex SQL,
-
-```yaml
-in:
-  type: postgresql
-  host: localhost
-  user: myuser
-  password: ""
-  database: my_database
-  query: |
-    SELECT t1.id, t1.name, t2.id AS t2_id, t2.name AS t2_name
-    FROM table1 AS t1
-    LEFT JOIN table2 AS t2
-      ON t1.id = t2.t1_id
-```
-
-## Build
-
-```
-$ ./gradlew gem
-```
+# PostgreSQL input plugins for Embulk
+
+PostgreSQL input plugins for Embulk loads records from PostgreSQL.
+
+## Overview
+
+* **Plugin type**: input
+* **Resume supported**: yes
+
+## Configuration
+
+- **host**: database host name (string, required)
+- **port**: database port number (integer, 5432)
+- **user**: database login user name (string, required)
+- **password**: database login password (string, default: "")
+- **database**: destination database name (string, required)
+- **schema**: destination schema name (string, default: "public")
+- **fetch_rows**: number of rows to fetch one time (used for java.sql.Statement#setFetchSize) (integer, default: 10000)
+- **options**: extra JDBC properties (hash, default: {})
+- If you write SQL directly,
+  - **query**: SQL to run (string)
+- If **query** is not set,
+  - **table**: destination table name (string, required)
+  - **select**: comma-separated list of columns to select (string, default: "*")
+  - **where**: WHERE condition to filter the rows (string, default: no-condition)
+- **default_timezone**: If the sql type of a column is `date`/`time`/`datetime` and the embulk type is `string`, column values are formatted int this default_timezone. You can overwrite timezone for each columns using column_options option. (string, default: `UTC`)
+- **column_options**: advanced: a key-value pairs where key is a column name and value is options for the column.
+  - **value_type**: embulk get values from database as this value_type. Typically, the value_type determines `getXXX` method of `java.sql.PreparedStatement`.
+  (string, default: depends on the sql type of the column. Available values options are: `long`, `double`, `float`, `decimal`, `boolean`, `string`, `date`, `time`, `timestamp`)
+  - **type**: Column values are converted to this embulk type.
+  Available values options are: `boolean`, `long`, `double`, `string`, `timestamp`).
+  By default, the embulk type is determined according to the sql type of the column (or value_type if specified).
+  - **timestamp_format**: If the sql type of the column is `date`/`time`/`datetime` and the embulk type is `string`, column values are formatted by this timestamp_format. And if the embulk type is `timestamp`, this timestamp_format will be used in the output plugin. (string, default : `%Y-%m-%d` for `date`, `%H:%M:%S` for `time`, `%Y-%m-%d %H:%M:%S` for `timestamp`)
+  - **timezone**: If the sql type of the column is `date`/`time`/`datetime` and the embulk type is `string`, column values are formatted in this timezone.
+(string, value of default_timezone option is used by default)
+
+## Example
+
+```yaml
+in:
+  type: postgresql
+  host: localhost
+  user: myuser
+  password: ""
+  database: my_database
+  table: my_table
+  select: "col1, col2, col3"
+  where: "col4 != 'a'"
+```
+
+If you need a complex SQL,
+
+```yaml
+in:
+  type: postgresql
+  host: localhost
+  user: myuser
+  password: ""
+  database: my_database
+  query: |
+    SELECT t1.id, t1.name, t2.id AS t2_id, t2.name AS t2_name
+    FROM table1 AS t1
+    LEFT JOIN table2 AS t2
+      ON t1.id = t2.t1_id
+```
+
+Advanced configuration:
+
+```yaml
+in:
+  type: postgresql
+  host: localhost
+  user: myuser
+  password: ""
+  database: my_database
+  table: "my_table"
+  select: "col1, col2, col3"
+  where: "col4 != 'a'"
+  column_options:
+    col1: {type: long}
+    col3: {type: string, timestamp_format: "%Y/%m/%d", timezone: "+0900"}
+
+```
+
+## Build
+
+```
+$ ./gradlew gem
+```

data/build.gradle

@@ -1,7 +1,7 @@
-dependencies {
-    compile project(':embulk-input-jdbc')
-
-    compile 'org.postgresql:postgresql:9.4-1200-jdbc41'
-
-    testCompile project(':embulk-input-jdbc').sourceSets.test.output
-}
+dependencies {
+    compile project(':embulk-input-jdbc')
+
+    compile 'org.postgresql:postgresql:9.4-1200-jdbc41'
+
+    testCompile project(':embulk-input-jdbc').sourceSets.test.output
+}

data/lib/embulk/input/postgresql.rb

@@ -1,3 +1,3 @@
-Embulk::JavaPlugin.register_input(
-  :postgresql, "org.embulk.input.PostgreSQLInputPlugin",
-  File.expand_path('../../../../classpath', __FILE__))
+Embulk::JavaPlugin.register_input(
+  :postgresql, "org.embulk.input.PostgreSQLInputPlugin",
+  File.expand_path('../../../../classpath', __FILE__))

data/src/main/java/org/embulk/input/PostgreSQLInputPlugin.java

@@ -1,91 +1,91 @@
-package org.embulk.input;
-
-import java.util.Properties;
-import java.sql.Connection;
-import java.sql.Driver;
-import java.sql.SQLException;
-import com.google.common.base.Throwables;
-import org.embulk.config.Config;
-import org.embulk.config.ConfigDefault;
-import org.embulk.input.jdbc.AbstractJdbcInputPlugin;
-import org.embulk.input.postgresql.PostgreSQLInputConnection;
-
-public class PostgreSQLInputPlugin
-        extends AbstractJdbcInputPlugin
-{
-    private static final Driver driver = new org.postgresql.Driver();
-
-    public interface PostgreSQLPluginTask
-            extends PluginTask
-    {
-        @Config("host")
-        public String getHost();
-
-        @Config("port")
-        @ConfigDefault("5432")
-        public int getPort();
-
-        @Config("user")
-        public String getUser();
-
-        @Config("password")
-        @ConfigDefault("\"\"")
-        public String getPassword();
-
-        @Config("database")
-        public String getDatabase();
-
-        @Config("schema")
-        @ConfigDefault("\"public\"")
-        public String getSchema();
-    }
-
-    @Override
-    protected Class<? extends PluginTask> getTaskClass()
-    {
-        return PostgreSQLPluginTask.class;
-    }
-
-    @Override
-    protected PostgreSQLInputConnection newConnection(PluginTask task) throws SQLException
-    {
-        PostgreSQLPluginTask t = (PostgreSQLPluginTask) task;
-
-        String url = String.format("jdbc:postgresql://%s:%d/%s",
-                t.getHost(), t.getPort(), t.getDatabase());
-
-        Properties props = new Properties();
-        props.setProperty("user", t.getUser());
-        props.setProperty("password", t.getPassword());
-        props.setProperty("loginTimeout",   "300"); // seconds
-        props.setProperty("socketTimeout", "1800"); // seconds
-
-        // Enable keepalive based on tcp_keepalive_time, tcp_keepalive_intvl and tcp_keepalive_probes kernel parameters.
-        // Socket options TCP_KEEPCNT, TCP_KEEPIDLE, and TCP_KEEPINTVL are not configurable.
-        props.setProperty("tcpKeepAlive", "true");
-
-        // TODO
-        //switch t.getSssl() {
-        //when "disable":
-        //    break;
-        //when "enable":
-        //    props.setProperty("sslfactory", "org.postgresql.ssl.NonValidatingFactory");  // disable server-side validation
-        //when "verify":
-        //    props.setProperty("ssl", "true");
-        //    break;
-        //}
-
-        props.putAll(t.getOptions());
-
-        Connection con = driver.connect(url, props);
-        try {
-            PostgreSQLInputConnection c = new PostgreSQLInputConnection(con, t.getSchema());
-            con = null;
-            return c;
-        } finally {
-            if (con != null) {
-                con.close();
-            }
-        }
-    }
-}
+package org.embulk.input;
+
+import java.util.Properties;
+import java.sql.Connection;
+import java.sql.Driver;
+import java.sql.SQLException;
+import com.google.common.base.Throwables;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.input.jdbc.AbstractJdbcInputPlugin;
+import org.embulk.input.postgresql.PostgreSQLInputConnection;
+
+public class PostgreSQLInputPlugin
+        extends AbstractJdbcInputPlugin
+{
+    private static final Driver driver = new org.postgresql.Driver();
+
+    public interface PostgreSQLPluginTask
+            extends PluginTask
+    {
+        @Config("host")
+        public String getHost();
+
+        @Config("port")
+        @ConfigDefault("5432")
+        public int getPort();
+
+        @Config("user")
+        public String getUser();
+
+        @Config("password")
+        @ConfigDefault("\"\"")
+        public String getPassword();
+
+        @Config("database")
+        public String getDatabase();
+
+        @Config("schema")
+        @ConfigDefault("\"public\"")
+        public String getSchema();
+    }
+
+    @Override
+    protected Class<? extends PluginTask> getTaskClass()
+    {
+        return PostgreSQLPluginTask.class;
+    }
+
+    @Override
+    protected PostgreSQLInputConnection newConnection(PluginTask task) throws SQLException
+    {
+        PostgreSQLPluginTask t = (PostgreSQLPluginTask) task;
+
+        String url = String.format("jdbc:postgresql://%s:%d/%s",
+                t.getHost(), t.getPort(), t.getDatabase());
+
+        Properties props = new Properties();
+        props.setProperty("user", t.getUser());
+        props.setProperty("password", t.getPassword());
+        props.setProperty("loginTimeout",   "300"); // seconds
+        props.setProperty("socketTimeout", "1800"); // seconds
+
+        // Enable keepalive based on tcp_keepalive_time, tcp_keepalive_intvl and tcp_keepalive_probes kernel parameters.
+        // Socket options TCP_KEEPCNT, TCP_KEEPIDLE, and TCP_KEEPINTVL are not configurable.
+        props.setProperty("tcpKeepAlive", "true");
+
+        // TODO
+        //switch t.getSssl() {
+        //when "disable":
+        //    break;
+        //when "enable":
+        //    props.setProperty("sslfactory", "org.postgresql.ssl.NonValidatingFactory");  // disable server-side validation
+        //when "verify":
+        //    props.setProperty("ssl", "true");
+        //    break;
+        //}
+
+        props.putAll(t.getOptions());
+
+        Connection con = driver.connect(url, props);
+        try {
+            PostgreSQLInputConnection c = new PostgreSQLInputConnection(con, t.getSchema());
+            con = null;
+            return c;
+        } finally {
+            if (con != null) {
+                con.close();
+            }
+        }
+    }
+}

data/src/main/java/org/embulk/input/postgresql/PostgreSQLInputConnection.java

@@ -1,60 +1,60 @@
-package org.embulk.input.postgresql;
-
-import java.sql.Connection;
-import java.sql.PreparedStatement;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import org.slf4j.Logger;
-import org.embulk.spi.Exec;
-import org.embulk.input.jdbc.JdbcInputConnection;
-
-public class PostgreSQLInputConnection
-        extends JdbcInputConnection
-{
-    private final Logger logger = Exec.getLogger(PostgreSQLInputConnection.class);
-
-    public PostgreSQLInputConnection(Connection connection, String schemaName)
-            throws SQLException
-    {
-        super(connection, schemaName);
-    }
-
-    @Override
-    protected CursorSelect newBatchSelect(String select, int fetchRows) throws SQLException
-    {
-        executeUpdate("DECLARE cur NO SCROLL CURSOR FOR "+select);
-
-        String fetchSql = "FETCH FORWARD "+fetchRows+" FROM cur";
-        return new CursorSelect(fetchSql, connection.prepareStatement(fetchSql));
-    }
-
-    public class CursorSelect
-            implements BatchSelect
-    {
-        private final String fetchSql;
-        private final PreparedStatement fetchStatement;
-
-        public CursorSelect(String fetchSql, PreparedStatement fetchStatement) throws SQLException
-        {
-            this.fetchSql = fetchSql;
-            this.fetchStatement = fetchStatement;
-        }
-
-        public ResultSet fetch() throws SQLException
-        {
-            logger.info("SQL: " + fetchSql);
-            long startTime = System.currentTimeMillis();
-
-            ResultSet rs = fetchStatement.executeQuery();
-
-            double seconds = (System.currentTimeMillis() - startTime) / 1000.0;
-            logger.info(String.format("> %.2f seconds", seconds));
-            return rs;
-        }
-
-        public void close() throws SQLException
-        {
-            // TODO close?
-        }
-    }
-}
+package org.embulk.input.postgresql;
+
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import org.slf4j.Logger;
+import org.embulk.spi.Exec;
+import org.embulk.input.jdbc.JdbcInputConnection;
+
+public class PostgreSQLInputConnection
+        extends JdbcInputConnection
+{
+    private final Logger logger = Exec.getLogger(PostgreSQLInputConnection.class);
+
+    public PostgreSQLInputConnection(Connection connection, String schemaName)
+            throws SQLException
+    {
+        super(connection, schemaName);
+    }
+
+    @Override
+    protected CursorSelect newBatchSelect(String select, int fetchRows) throws SQLException
+    {
+        executeUpdate("DECLARE cur NO SCROLL CURSOR FOR "+select);
+
+        String fetchSql = "FETCH FORWARD "+fetchRows+" FROM cur";
+        return new CursorSelect(fetchSql, connection.prepareStatement(fetchSql));
+    }
+
+    public class CursorSelect
+            implements BatchSelect
+    {
+        private final String fetchSql;
+        private final PreparedStatement fetchStatement;
+
+        public CursorSelect(String fetchSql, PreparedStatement fetchStatement) throws SQLException
+        {
+            this.fetchSql = fetchSql;
+            this.fetchStatement = fetchStatement;
+        }
+
+        public ResultSet fetch() throws SQLException
+        {
+            logger.info("SQL: " + fetchSql);
+            long startTime = System.currentTimeMillis();
+
+            ResultSet rs = fetchStatement.executeQuery();
+
+            double seconds = (System.currentTimeMillis() - startTime) / 1000.0;
+            logger.info(String.format("> %.2f seconds", seconds));
+            return rs;
+        }
+
+        public void close() throws SQLException
+        {
+            // TODO close?
+        }
+    }
+}

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-postgresql
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.5.0
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-03-06 00:00:00.000000000 Z
+date: 2015-06-17 00:00:00.000000000 Z
 dependencies: []
 description: Selects records from a table.
 email:
@@ -22,8 +22,8 @@ files:
 - lib/embulk/input/postgresql.rb
 - src/main/java/org/embulk/input/PostgreSQLInputPlugin.java
 - src/main/java/org/embulk/input/postgresql/PostgreSQLInputConnection.java
-- classpath/embulk-input-jdbc-0.4.0.jar
-- classpath/embulk-input-postgresql-0.4.0.jar
+- classpath/embulk-input-jdbc-0.5.0.jar
+- classpath/embulk-input-postgresql-0.5.0.jar
 - classpath/jna-4.1.0.jar
 - classpath/jna-platform-4.1.0.jar
 - classpath/postgresql-9.4-1200-jdbc41.jar