embulk-input-redshift 0.10.0 → 0.10.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9c54964bbf0155adb050eeb74010c7623121ed3c01e4cd61a7bb61924c9fdcd2
-  data.tar.gz: 6a8cda4203c69b5c5ff3045c75e833c06111cfacb419c257676c4fdc7f67fb4b
+  metadata.gz: d90c302f2b33a493831a26dcb2b4e0708a74d8678b592150575f242653404006
+  data.tar.gz: 5d5940e37cfced39c520bf65f0818dd83df69bccf18055359897672d2b84f312
 SHA512:
-  metadata.gz: 4882f793a635c81330746bab77fec9467cd3c250fd6c64107f2d0826cff2a7a37e31912dbeb9fcdac4d5dcf32cfcb64ad83b26b80deb72693361bdd5f8ed29fa
-  data.tar.gz: 518f1f94e44b5606251afe4616bf964b90b5e6be83760668c9e2237f344922de8f40e495dfffd663412e1232011d89f16323d079a5dfa36d37b3762786d88014
+  metadata.gz: 5e699117348eaffd2f0186af536fac8972c23c8866536b15ad683a196b1f2b5204b4c7e7f87e9601cce83bc3e2972452f6606d5cd4e5903681b5fd18153bae68
+  data.tar.gz: f4a057665b092d6fadbe533d92ea12b71ac654302b5be14050ecb0ec72c25da16d3c2d5e9e2c36255751fbc81c41fff2351f888e547e61bef6edf4d7b4febd78

data/README.md

@@ -30,7 +30,7 @@ Redshift input plugin for Embulk loads records from Redshift.
   - **where**: WHERE condition to filter the rows (string, default: no-condition)
   - **order_by**: expression of ORDER BY to sort rows (e.g. `created_at DESC, id ASC`) (string, default: not sorted)
 - **incremental**: if true, enables incremental loading. See next section for details (boolean, default: false)
-- **incremental_columns**: column names for incremental loading (array of strings, default: use primary keys)
+- **incremental_columns**: column names for incremental loading (array of strings, default: use primary keys). Columns of integer types, string types, `timestamp` and `timestamptz` are supported.
 - **last_record**: values of the last record for incremental loading (array of objects, default: load all records)
 - **default_timezone**: If the sql type of a column is `date`/`time`/`datetime` and the embulk type is `string`, column values are formatted int this default_timezone. You can overwrite timezone for each columns using column_options option. (string, default: `UTC`)
 - **default_column_options**: advanced: column_options for each JDBC type as default. key-value pairs where key is a JDBC type (e.g. 'DATE', 'BIGINT') and value is same as column_options's value.
@@ -63,13 +63,13 @@ ORDER BY updated_at, id
 
 When bulk data loading finishes successfully, it outputs `last_record: ` paramater as config-diff so that next execution uses it.
 
-At the next execution, when `last_record: ` is also set, this plugin generates additional WHERE conditions to load records larger than the last record. For example, if `last_record: ["2017-01-01 00:32:12", 5291]` is set,
+At the next execution, when `last_record: ` is also set, this plugin generates additional WHERE conditions to load records larger than the last record. For example, if `last_record: ["2017-01-01T00:32:12.487659", 5291]` is set,
 
 ```
 SELECT * FROM (
   ...original query is here...
 )
-WHERE updated_at > '2017-01-01 00:32:12' OR (updated_at = '2017-01-01 00:32:12' AND id > 5291)
+WHERE updated_at > '2017-01-01 00:32:12.487659' OR (updated_at = '2017-01-01 00:32:12.487659' AND id > 5291)
 ORDER BY updated_at, id
 ```
 

data/src/test/java/org/embulk/input/redshift/IncrementalTest.java

@@ -77,6 +77,38 @@ public class IncrementalTest
                 is((ConfigDiff) loadYamlResource(embulk, "int/expected_2.diff")));
     }
 
+    @Test
+    public void simpleChar() throws Exception
+    {
+        // setup first rows
+        execute(readResource("char/setup.sql"));
+
+        Path out1 = embulk.createTempFile("csv");
+        RunResult result1 = embulk.runInput(
+                baseConfig.merge(loadYamlResource(embulk, "char/config_1.yml")),
+                out1);
+        assertThat(
+                readSortedFile(out1),
+                is(readResource("char/expected_1.csv")));
+        assertThat(
+                result1.getConfigDiff(),
+                is((ConfigDiff) loadYamlResource(embulk, "char/expected_1.diff")));
+
+        // insert more rows
+        execute(readResource("char/insert_more.sql"));
+
+        Path out2 = embulk.createTempFile("csv");
+        RunResult result2 = embulk.runInput(
+                baseConfig.merge(loadYamlResource(embulk, "char/config_2.yml")),
+                out2);
+        assertThat(
+                readSortedFile(out2),
+                is(readResource("char/expected_2.csv")));
+        assertThat(
+                result2.getConfigDiff(),
+                is((ConfigDiff) loadYamlResource(embulk, "char/expected_2.diff")));
+    }
+
     @Test
     public void simpleTimestampWithoutTimeZone() throws Exception
     {

data/src/test/resources/org/embulk/input/redshift/test/expect/incremental/char/config_1.yml

@@ -0,0 +1,3 @@
+table: int_load
+incremental: true
+incremental_columns: [name]

data/src/test/resources/org/embulk/input/redshift/test/expect/incremental/char/config_2.yml

@@ -0,0 +1,4 @@
+table: int_load
+last_record: ['A4']
+incremental: true
+incremental_columns: [name]

data/src/test/resources/org/embulk/input/redshift/test/expect/incremental/char/expected_1.csv

@@ -0,0 +1,4 @@
+A1,first
+A2,first
+A3,first
+A4,first

data/src/test/resources/org/embulk/input/redshift/test/expect/incremental/char/expected_1.diff

@@ -0,0 +1,3 @@
+in:
+  last_record: ['A4']
+out: {}

data/src/test/resources/org/embulk/input/redshift/test/expect/incremental/char/expected_2.csv

@@ -0,0 +1,2 @@
+A5,more_load
+A9,more_load

data/src/test/resources/org/embulk/input/redshift/test/expect/incremental/char/expected_2.diff

@@ -0,0 +1,3 @@
+in:
+  last_record: ['A9']
+out: {}

data/src/test/resources/org/embulk/input/redshift/test/expect/incremental/char/insert_more.sql

@@ -0,0 +1,6 @@
+
+insert into int_load (name, note) values
+('A0', 'more_skip'),
+('A4', 'more_skip'),
+('A9', 'more_load'),
+('A5', 'more_load');

data/src/test/resources/org/embulk/input/redshift/test/expect/incremental/char/setup.sql

@@ -0,0 +1,12 @@
+drop table if exists int_load;
+
+create table int_load (
+    name char(2) not null,
+    note text
+);
+
+insert into int_load (name, note) values
+('A3', 'first'),
+('A4', 'first'),
+('A2', 'first'),
+('A1', 'first');

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-redshift
 version: !ruby/object:Gem::Version
-  version: 0.10.0
+  version: 0.10.1
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-05-23 00:00:00.000000000 Z
+date: 2019-09-06 00:00:00.000000000 Z
 dependencies: []
 description: Selects records from a table.
 email:
@@ -19,15 +19,23 @@ extra_rdoc_files: []
 files:
 - README.md
 - build.gradle
-- classpath/embulk-input-jdbc-0.10.0.jar
-- classpath/embulk-input-postgresql-0.10.0.jar
-- classpath/embulk-input-redshift-0.10.0.jar
+- classpath/embulk-input-jdbc-0.10.1.jar
+- classpath/embulk-input-postgresql-0.10.1.jar
+- classpath/embulk-input-redshift-0.10.1.jar
 - classpath/postgresql-9.4-1205-jdbc41.jar
 - lib/embulk/input/redshift.rb
 - src/main/java/org/embulk/input/RedshiftInputPlugin.java
 - src/main/java/org/embulk/input/redshift/getter/RedshiftColumnGetterFactory.java
 - src/test/java/org/embulk/input/redshift/IncrementalTest.java
 - src/test/java/org/embulk/input/redshift/RedshiftTests.java
+- src/test/resources/org/embulk/input/redshift/test/expect/incremental/char/config_1.yml
+- src/test/resources/org/embulk/input/redshift/test/expect/incremental/char/config_2.yml
+- src/test/resources/org/embulk/input/redshift/test/expect/incremental/char/expected_1.csv
+- src/test/resources/org/embulk/input/redshift/test/expect/incremental/char/expected_1.diff
+- src/test/resources/org/embulk/input/redshift/test/expect/incremental/char/expected_2.csv
+- src/test/resources/org/embulk/input/redshift/test/expect/incremental/char/expected_2.diff
+- src/test/resources/org/embulk/input/redshift/test/expect/incremental/char/insert_more.sql
+- src/test/resources/org/embulk/input/redshift/test/expect/incremental/char/setup.sql
 - src/test/resources/org/embulk/input/redshift/test/expect/incremental/int/config_1.yml
 - src/test/resources/org/embulk/input/redshift/test/expect/incremental/int/config_2.yml
 - src/test/resources/org/embulk/input/redshift/test/expect/incremental/int/expected_1.csv