embulk-input-redshift 0.4.0 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3ad72eed1e485ab471dac2e9d44f01c1b4237bfb
-  data.tar.gz: c6b93b4021536b52e27a59d3af53cfcfa57fa8f5
+  metadata.gz: d420ce90518a621e71996d2ffc6d8b57246610ac
+  data.tar.gz: 5a3922ea637e1011cb04ea5387c15045d0c95e6d
 SHA512:
-  metadata.gz: 99dc1f9745c4afc6e43dd6539d22ef5366c8c19b07155402f910b8355d918ebe79c4bb529c307ae1191da9909c1ad7e48b92910b55470df9ed71c95de1e8fe69
-  data.tar.gz: 6f7a4ddc45b9dd93c8823a18be602b3e76cae9da96fc878a68a12379119d55b02cda628b6ae2c1caad67fefc564a03a43bdc8a3750ae24ff026fa5fb250abbb0
+  metadata.gz: 1fb1aeb87ce69650ecafd235856384f20aa3d3ea8eea090712178130d212f75c12bc0aff213d9c13334e871f4d7e2690570604b87d9148895fed9df75ada55a1
+  data.tar.gz: 8781224b4ee7d05b52d437b09022edee202f8b1e52543562c00daba7c8d10b98d38ec232534f634f45cdc52dcb5fd4d68846038cbdc50b4bef3c69081245bedc

data/README.md

@@ -1,62 +1,90 @@
-# Redshift input plugins for Embulk
-
-Redshift input plugins for Embulk loads records from Redshift.
-
-## Overview
-
-* **Plugin type**: input
-* **Resume supported**: yes
-
-## Configuration
-
-- **host**: database host name (string, required)
-- **port**: database port number (integer, 5439)
-- **user**: database login user name (string, required)
-- **password**: database login password (string, default: "")
-- **database**: destination database name (string, required)
-- **schema**: destination schema name (string, default: "public")
-- **where**: WHERE condition to filter the rows (string, default: no-condition)
-- **fetch_rows**: number of rows to fetch one time (used for java.sql.Statement#setFetchSize) (integer, default: 10000)
-- **options**: extra JDBC properties (hash, default: {})
-- If you write SQL directly,
-  - **query**: SQL to run (string)
-- If **query** is not set,
-  - **table**: destination table name (string, required)
-  - **select**: comma-separated list of columns to select (string, default: "*")
-  - **where**: WHERE condition to filter the rows (string, default: no-condition)
-
-## Example
-
-```yaml
-in:
-  type: redshift
-  host: localhost
-  user: myuser
-  password: ""
-  database: my_database
-  table: my_table
-  select: "col1, col2, col3"
-  where: "col4 != 'a'"
-```
-
-If you need a complex SQL,
-
-```yaml
-in:
-  type: redshift
-  host: localhost
-  user: myuser
-  password: ""
-  database: my_database
-  query: |
-    SELECT t1.id, t1.name, t2.id AS t2_id, t2.name AS t2_name
-    FROM table1 AS t1
-    LEFT JOIN table2 AS t2
-      ON t1.id = t2.t1_id
-```
-
-## Build
-
-```
-$ ./gradlew gem
-```
+# Redshift input plugins for Embulk
+
+Redshift input plugins for Embulk loads records from Redshift.
+
+## Overview
+
+* **Plugin type**: input
+* **Resume supported**: yes
+
+## Configuration
+
+- **host**: database host name (string, required)
+- **port**: database port number (integer, 5439)
+- **user**: database login user name (string, required)
+- **password**: database login password (string, default: "")
+- **database**: destination database name (string, required)
+- **schema**: destination schema name (string, default: "public")
+- **where**: WHERE condition to filter the rows (string, default: no-condition)
+- **fetch_rows**: number of rows to fetch one time (used for java.sql.Statement#setFetchSize) (integer, default: 10000)
+- **options**: extra JDBC properties (hash, default: {})
+- If you write SQL directly,
+  - **query**: SQL to run (string)
+- If **query** is not set,
+  - **table**: destination table name (string, required)
+  - **select**: comma-separated list of columns to select (string, default: "*")
+  - **where**: WHERE condition to filter the rows (string, default: no-condition)
+- **default_timezone**: If the sql type of a column is `date`/`time`/`datetime` and the embulk type is `string`, column values are formatted int this default_timezone. You can overwrite timezone for each columns using column_options option. (string, default: `UTC`)
+- **column_options**: advanced: a key-value pairs where key is a column name and value is options for the column.
+  - **value_type**: embulk get values from database as this value_type. Typically, the value_type determines `getXXX` method of `java.sql.PreparedStatement`.
+  (string, default: depends on the sql type of the column. Available values options are: `long`, `double`, `float`, `decimal`, `boolean`, `string`, `date`, `time`, `timestamp`)
+  - **type**: Column values are converted to this embulk type.
+  Available values options are: `boolean`, `long`, `double`, `string`, `timestamp`).
+  By default, the embulk type is determined according to the sql type of the column (or value_type if specified).
+  - **timestamp_format**: If the sql type of the column is `date`/`time`/`datetime` and the embulk type is `string`, column values are formatted by this timestamp_format. And if the embulk type is `timestamp`, this timestamp_format will be used in the output plugin. (string, default : `%Y-%m-%d` for `date`, `%H:%M:%S` for `time`, `%Y-%m-%d %H:%M:%S` for `timestamp`)
+  - **timezone**: If the sql type of the column is `date`/`time`/`datetime` and the embulk type is `string`, column values are formatted in this timezone.
+(string, value of default_timezone option is used by default)
+
+## Example
+
+```yaml
+in:
+  type: redshift
+  host: localhost
+  user: myuser
+  password: ""
+  database: my_database
+  table: my_table
+  select: "col1, col2, col3"
+  where: "col4 != 'a'"
+```
+
+If you need a complex SQL,
+
+```yaml
+in:
+  type: redshift
+  host: localhost
+  user: myuser
+  password: ""
+  database: my_database
+  query: |
+    SELECT t1.id, t1.name, t2.id AS t2_id, t2.name AS t2_name
+    FROM table1 AS t1
+    LEFT JOIN table2 AS t2
+      ON t1.id = t2.t1_id
+```
+
+Advanced configuration:
+
+```yaml
+in:
+  type: redshift
+  host: localhost
+  user: myuser
+  password: ""
+  database: my_database
+  table: "my_table"
+  select: "col1, col2, col3"
+  where: "col4 != 'a'"
+  column_options:
+    col1: {type: long}
+    col3: {type: string, timestamp_format: "%Y/%m/%d", timezone: "+0900"}
+
+```
+
+## Build
+
+```
+$ ./gradlew gem
+```

data/build.gradle

@@ -1,6 +1,6 @@
-dependencies {
-    compile project(':embulk-input-jdbc')
-    compile project(':embulk-input-postgresql')
-
-    testCompile project(':embulk-input-jdbc').sourceSets.test.output
-}
+dependencies {
+    compile project(':embulk-input-jdbc')
+    compile project(':embulk-input-postgresql')
+
+    testCompile project(':embulk-input-jdbc').sourceSets.test.output
+}

data/lib/embulk/input/redshift.rb

@@ -1,3 +1,3 @@
-Embulk::JavaPlugin.register_input(
-  :redshift, "org.embulk.input.RedshiftInputPlugin",
-  File.expand_path('../../../../classpath', __FILE__))
+Embulk::JavaPlugin.register_input(
+  :redshift, "org.embulk.input.RedshiftInputPlugin",
+  File.expand_path('../../../../classpath', __FILE__))

data/src/main/java/org/embulk/input/RedshiftInputPlugin.java

@@ -1,92 +1,92 @@
-package org.embulk.input;
-
-import java.util.Properties;
-import java.sql.Connection;
-import java.sql.Driver;
-import java.sql.SQLException;
-import com.google.common.base.Throwables;
-import org.embulk.config.Config;
-import org.embulk.config.ConfigDefault;
-import org.embulk.input.jdbc.AbstractJdbcInputPlugin;
-import org.embulk.input.jdbc.AbstractJdbcInputPlugin;
-import org.embulk.input.postgresql.PostgreSQLInputConnection;
-
-public class RedshiftInputPlugin
-        extends AbstractJdbcInputPlugin
-{
-    private static final Driver driver = new org.postgresql.Driver();
-
-    public interface RedshiftPluginTask
-            extends PluginTask
-    {
-        @Config("host")
-        public String getHost();
-
-        @Config("port")
-        @ConfigDefault("5439")
-        public int getPort();
-
-        @Config("user")
-        public String getUser();
-
-        @Config("password")
-        @ConfigDefault("\"\"")
-        public String getPassword();
-
-        @Config("database")
-        public String getDatabase();
-
-        @Config("schema")
-        @ConfigDefault("\"public\"")
-        public String getSchema();
-    }
-
-    @Override
-    protected Class<? extends PluginTask> getTaskClass()
-    {
-        return RedshiftPluginTask.class;
-    }
-
-    @Override
-    protected PostgreSQLInputConnection newConnection(PluginTask task) throws SQLException
-    {
-        RedshiftPluginTask t = (RedshiftPluginTask) task;
-
-        String url = String.format("jdbc:postgresql://%s:%d/%s",
-                t.getHost(), t.getPort(), t.getDatabase());
-
-        Properties props = new Properties();
-        props.setProperty("user", t.getUser());
-        props.setProperty("password", t.getPassword());
-        props.setProperty("loginTimeout",   "300"); // seconds
-        props.setProperty("socketTimeout", "1800"); // seconds
-
-        // Enable keepalive based on tcp_keepalive_time, tcp_keepalive_intvl and tcp_keepalive_probes kernel parameters.
-        // Socket options TCP_KEEPCNT, TCP_KEEPIDLE, and TCP_KEEPINTVL are not configurable.
-        props.setProperty("tcpKeepAlive", "true");
-
-        // TODO
-        //switch t.getSssl() {
-        //when "disable":
-        //    break;
-        //when "enable":
-        //    props.setProperty("sslfactory", "org.postgresql.ssl.NonValidatingFactory");  // disable server-side validation
-        //when "verify":
-        //    props.setProperty("ssl", "true");
-        //    break;
-        //}
-
-        props.putAll(t.getOptions());
-
-        Connection con = driver.connect(url, props);
-        try {
-            PostgreSQLInputConnection c = new PostgreSQLInputConnection(con, t.getSchema());
-            con = null;
-            return c;
-        } finally {
-            if (con != null) {
-                con.close();
-            }
-        }
-    }
-}
+package org.embulk.input;
+
+import java.util.Properties;
+import java.sql.Connection;
+import java.sql.Driver;
+import java.sql.SQLException;
+import com.google.common.base.Throwables;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.input.jdbc.AbstractJdbcInputPlugin;
+import org.embulk.input.jdbc.AbstractJdbcInputPlugin;
+import org.embulk.input.postgresql.PostgreSQLInputConnection;
+
+public class RedshiftInputPlugin
+        extends AbstractJdbcInputPlugin
+{
+    private static final Driver driver = new org.postgresql.Driver();
+
+    public interface RedshiftPluginTask
+            extends PluginTask
+    {
+        @Config("host")
+        public String getHost();
+
+        @Config("port")
+        @ConfigDefault("5439")
+        public int getPort();
+
+        @Config("user")
+        public String getUser();
+
+        @Config("password")
+        @ConfigDefault("\"\"")
+        public String getPassword();
+
+        @Config("database")
+        public String getDatabase();
+
+        @Config("schema")
+        @ConfigDefault("\"public\"")
+        public String getSchema();
+    }
+
+    @Override
+    protected Class<? extends PluginTask> getTaskClass()
+    {
+        return RedshiftPluginTask.class;
+    }
+
+    @Override
+    protected PostgreSQLInputConnection newConnection(PluginTask task) throws SQLException
+    {
+        RedshiftPluginTask t = (RedshiftPluginTask) task;
+
+        String url = String.format("jdbc:postgresql://%s:%d/%s",
+                t.getHost(), t.getPort(), t.getDatabase());
+
+        Properties props = new Properties();
+        props.setProperty("user", t.getUser());
+        props.setProperty("password", t.getPassword());
+        props.setProperty("loginTimeout",   "300"); // seconds
+        props.setProperty("socketTimeout", "1800"); // seconds
+
+        // Enable keepalive based on tcp_keepalive_time, tcp_keepalive_intvl and tcp_keepalive_probes kernel parameters.
+        // Socket options TCP_KEEPCNT, TCP_KEEPIDLE, and TCP_KEEPINTVL are not configurable.
+        props.setProperty("tcpKeepAlive", "true");
+
+        // TODO
+        //switch t.getSssl() {
+        //when "disable":
+        //    break;
+        //when "enable":
+        //    props.setProperty("sslfactory", "org.postgresql.ssl.NonValidatingFactory");  // disable server-side validation
+        //when "verify":
+        //    props.setProperty("ssl", "true");
+        //    break;
+        //}
+
+        props.putAll(t.getOptions());
+
+        Connection con = driver.connect(url, props);
+        try {
+            PostgreSQLInputConnection c = new PostgreSQLInputConnection(con, t.getSchema());
+            con = null;
+            return c;
+        } finally {
+            if (con != null) {
+                con.close();
+            }
+        }
+    }
+}

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-redshift
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.5.0
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-03-06 00:00:00.000000000 Z
+date: 2015-06-17 00:00:00.000000000 Z
 dependencies: []
 description: Selects records from a table.
 email:
@@ -21,9 +21,9 @@ files:
 - build.gradle
 - lib/embulk/input/redshift.rb
 - src/main/java/org/embulk/input/RedshiftInputPlugin.java
-- classpath/embulk-input-jdbc-0.4.0.jar
-- classpath/embulk-input-postgresql-0.4.0.jar
-- classpath/embulk-input-redshift-0.4.0.jar
+- classpath/embulk-input-jdbc-0.5.0.jar
+- classpath/embulk-input-postgresql-0.5.0.jar
+- classpath/embulk-input-redshift-0.5.0.jar
 - classpath/jna-4.1.0.jar
 - classpath/jna-platform-4.1.0.jar
 - classpath/postgresql-9.4-1200-jdbc41.jar