embulk-input-postgresql 0.7.2 → 0.7.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c855700b84c5cad076bb37c8ceb53f874be5d084
-  data.tar.gz: 0e9425e94c4bfad10f95e350f652097b92a03251
+  metadata.gz: 93171986c203df79e84845be61756890f6108e3c
+  data.tar.gz: f76cd155e82c075761ed4b5d3196de5d872af719
 SHA512:
-  metadata.gz: f829395a1ccd22d20e6a03d188a3297fb76dfdfa6e393eb333661d20021e6a7d0f1a6f56d9918a441b1dbda62884a10e3336b2c928215999071aaa0bfbca4b28
-  data.tar.gz: edda38237731d6e180935e8b3471c6d15e752d8bb74f68e723d12641e9049b643f36de201ccde504bddf7bf7ec676cc501c40efb7aed8ec8713d53400c69e425
+  metadata.gz: dc8b710ebcdbb08a5505a762095621c779f448b256398c1c40fa8c7a0739b587965ec64486f460c9377699649d699ee9ddec5cb5654cee90d597facefc364c80
+  data.tar.gz: fdc4e9af5ff63a5f93f17db6ee93792c897ee616a11db40e2f1a0d5fdff735ae51238950dfad3ac6c30e0d2557c504cbb41df79243939e3ae55805c27adb4f90

data/README.md

@@ -19,14 +19,18 @@ PostgreSQL input plugins for Embulk loads records from PostgreSQL.
 - **connect_timeout**: timeout for establishment of a database connection. (integer (seconds), default: 300)
 - **socket_timeout**: timeout for socket read operations. 0 means no timeout. (integer (seconds), default: 1800)
 - **ssl**: enables SSL. data will be encrypted but CA or certification will not be verified (boolean, default: false)
+- **application_name**: application name shown on pg_stat_activity. (string, default: "embulk-input-postgresql")
 - **options**: extra JDBC properties (hash, default: {})
 - If you write SQL directly,
   - **query**: SQL to run (string)
 - If **query** is not set,
   - **table**: destination table name (string, required)
-  - **select**: comma-separated list of columns to select (string, default: "*")
+  - **select**: expression of select (e.g. `id, created_at`) (string, default: "*")
   - **where**: WHERE condition to filter the rows (string, default: no-condition)
-  - **order_by**: name of the column that rows are sorted by (string, default: not sorted)
+  - **order_by**: expression of ORDER BY to sort rows (e.g. `created_at DESC, id ASC`) (string, default: not sorted)
+- **incremental**: if true, enables incremental loading. See next section for details (boolean, default: false)
+- **incremental_columns**: column names for incremental loading (array of strings, default: use primary keys)
+- **last_record**: values of the last record for incremental loading (array of objects, default: load all records)
 - **default_timezone**: If the sql type of a column is `date`/`time`/`datetime` and the embulk type is `string`, column values are formatted int this default_timezone. You can overwrite timezone for each columns using column_options option. (string, default: `UTC`)
 - **column_options**: advanced: a key-value pairs where key is a column name and value is options for the column.
   - **value_type**: embulk get values from database as this value_type. Typically, the value_type determines `getXXX` method of `java.sql.PreparedStatement`.
@@ -56,6 +60,42 @@ In addition, `json` type is supported for `hstore` column, and output will be as
 `value_type` is ignored.
 
 
+### Incremental loading
+
+Incremental loading uses monotonically increasing unique columns (such as auto-increment (serial / bigserial) column) to load records inserted (or updated) after last execution.
+
+First, if `incremental: true` is set, this plugin loads all records with additional ORDER BY. For example, if `incremental_columns: [updated_at, id]` option is set, query will be as following:
+
+```
+SELECT * FROM (
+  ...original query is here...
+)
+ORDER BY updated_at, id
+```
+
+When bulk data loading finishes successfully, it outputs `last_record: ` paramater as config-diff so that next execution uses it.
+
+At the next execution, when `last_record: ` is also set, this plugin generates additional WHERE conditions to load records larger than the last record. For example, if `last_record: ["2017-01-01 00:32:12", 5291]` is set,
+
+```
+SELECT * FROM (
+  ...original query is here...
+)
+WHERE created_at > '2017-01-01 00:32:12' OR (created_at = '2017-01-01 00:32:12' AND id > 5291)
+ORDER BY updated_at, id
+```
+
+Then, it updates `last_record: ` so that next execution uses the updated last_record.
+
+**IMPORTANT**: If you set `incremental_columns: ` option, make sure that there is an index on the columns to avoid full table scan. For this example, following index should be created:
+
+```
+CREATE INDEX embulk_incremental_loading_index ON table (updated_at, id);
+```
+
+Recommended usage is to leave `incremental_columns` unset and let this plugin automatically finds an auto-increment (serial / bigserial) primary key. Currently, only strings and integers are supported as incremental_columns.
+
+
 ## Example
 
 ```yaml
@@ -68,6 +108,16 @@ in:
   table: my_table
   select: "col1, col2, col3"
   where: "col4 != 'a'"
+  order_by: "col1 DESC"
+```
+
+This configuration will generate following SQL:
+
+```
+SELECT col1, col2, col3
+FROM "my_table"
+WHERE col4 != 'a'
+ORDER BY col1 DESC
 ```
 
 If you need a complex SQL,

data/src/main/java/org/embulk/input/PostgreSQLInputPlugin.java

@@ -48,6 +48,10 @@ public class PostgreSQLInputPlugin
         @Config("ssl")
         @ConfigDefault("false")
         public boolean getSsl();
+
+        @Config("application_name")
+        @ConfigDefault("\"embulk-input-postgresql\"")
+        public String getApplicationName();
     }
 
     @Override
@@ -82,6 +86,8 @@ public class PostgreSQLInputPlugin
         }
         // setting ssl=false enables SSL. See org.postgresql.core.v3.openConnectionImpl.
 
+        props.setProperty("ApplicationName", t.getApplicationName());
+
         props.putAll(t.getOptions());
 
         Connection con = driver.connect(url, props);

data/src/main/java/org/embulk/input/postgresql/PostgreSQLInputConnection.java

@@ -1,5 +1,6 @@
 package org.embulk.input.postgresql;
 
+import java.util.List;
 import java.sql.Connection;
 import java.sql.PreparedStatement;
 import java.sql.ResultSet;
@@ -7,6 +8,8 @@ import java.sql.SQLException;
 import org.slf4j.Logger;
 import org.embulk.spi.Exec;
 import org.embulk.input.jdbc.JdbcInputConnection;
+import org.embulk.input.jdbc.JdbcLiteral;
+import org.embulk.input.jdbc.getter.ColumnGetter;
 
 public class PostgreSQLInputConnection
         extends JdbcInputConnection
@@ -20,9 +23,24 @@ public class PostgreSQLInputConnection
     }
 
     @Override
-    protected CursorSelect newBatchSelect(String select, int fetchRows, int queryTimeout) throws SQLException
+    protected BatchSelect newBatchSelect(PreparedQuery preparedQuery,
+            List<ColumnGetter> getters,
+            int fetchRows, int queryTimeout) throws SQLException
     {
-        executeUpdate("DECLARE cur NO SCROLL CURSOR FOR "+select);
+        String query = "DECLARE cur NO SCROLL CURSOR FOR " + preparedQuery.getQuery();
+        List<JdbcLiteral> params = preparedQuery.getParameters();
+
+        logger.info("SQL: " + query);
+        PreparedStatement stmt = connection.prepareStatement(query);
+        try {
+            if (!params.isEmpty()) {
+                logger.info("Parameters: {}", params);
+                prepareParameters(stmt, getters, params);
+            }
+            stmt.executeUpdate();
+        } finally {
+            stmt.close();
+        }
 
         String fetchSql = "FETCH FORWARD "+fetchRows+" FROM cur";
         // Because socketTimeout is set in Connection, don't need to set quertyTimeout.

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-postgresql
 version: !ruby/object:Gem::Version
-  version: 0.7.2
+  version: 0.7.3
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-06-24 00:00:00.000000000 Z
+date: 2016-08-26 00:00:00.000000000 Z
 dependencies: []
 description: Selects records from a table.
 email:
@@ -19,6 +19,9 @@ extra_rdoc_files: []
 files:
 - README.md
 - build.gradle
+- classpath/embulk-input-jdbc-0.7.3.jar
+- classpath/embulk-input-postgresql-0.7.3.jar
+- classpath/postgresql-9.4-1205-jdbc41.jar
 - lib/embulk/input/postgresql.rb
 - src/main/java/org/embulk/input/PostgreSQLInputPlugin.java
 - src/main/java/org/embulk/input/postgresql/PostgreSQLInputConnection.java
@@ -27,9 +30,6 @@ files:
 - src/test/java/org/embulk/input/postgresql/PostgreSQLInputPluginTest.java
 - src/test/resources/yml/input_hstore.yml
 - src/test/resources/yml/input_hstore2.yml
-- classpath/embulk-input-jdbc-0.7.2.jar
-- classpath/embulk-input-postgresql-0.7.2.jar
-- classpath/postgresql-9.4-1205-jdbc41.jar
 homepage: https://github.com/embulk/embulk-input-jdbc
 licenses:
 - Apache 2.0
@@ -40,17 +40,17 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.1.9
+rubygems_version: 2.4.8
 signing_key:
 specification_version: 4
 summary: JDBC input plugin for Embulk