embulk-input-redshift 0.8.2 → 0.8.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e0df12df1c6fb46147d37fffb8a7d85ffde42e96
-  data.tar.gz: b64af6c6f2f4fde4fd205fa33c2d20e82d123b4e
+  metadata.gz: 3c230871ead7faef54938ba29508c4806b4ce8af
+  data.tar.gz: 77d957464767609885fc1854c49b29cfdd382300
 SHA512:
-  metadata.gz: 457afd7d981acd354504b869d70798c7788bd779da60c03cedac3e0dd28afd73c5fabdf5ab51d171ba9e402501d323c7ef7cdb47bd88c2fd0f4e29138573dca3
-  data.tar.gz: 382eaf21e073ac9df4a103c9b29de1e434ecc52635ed18954d0ac2eb7eed16c65f40903a86cb409bf6f0620f91f548f1bb24d98c14980a32f544313cb2eec568
+  metadata.gz: b6fcc5c0786d941df213db00851f29f93d053eabf71d7d0c12dedf0e24c45ce3ad04d21bde674a582659a825a22d7acda93570007c85596ba26677e52e8bace6
+  data.tar.gz: 7ab2529173924e4b40be6c1b1a2adb454c4fb529910333696af580c14f949f03a44fc861673e3cd7702b4fa43ed1a25ed4d0bed6a8e178c2af27210897c52a12

data/README.md

@@ -144,3 +144,10 @@ in:
 ```
 $ ./gradlew gem
 ```
+
+Running tests:
+
+```
+$ cp ci/travis_redshift.yml ci/redshift.yml  # edit this file if necessary
+$ EMBULK_INPUT_REDSHIFT_TEST_CONFIG=`pwd`/ci/redshift.yml ./gradlew :embulk-input-redshift:check --info
+```

data/build.gradle

@@ -3,4 +3,6 @@ dependencies {
     compile project(':embulk-input-postgresql')
 
     testCompile project(':embulk-input-jdbc').sourceSets.test.output
+
+    testCompile 'org.embulk:embulk-standards:0.8.15'
 }

data/src/main/java/org/embulk/input/RedshiftInputPlugin.java

@@ -9,7 +9,11 @@ import java.sql.SQLException;
 import org.embulk.config.Config;
 import org.embulk.config.ConfigDefault;
 import org.embulk.input.jdbc.AbstractJdbcInputPlugin;
+import org.embulk.input.jdbc.getter.ColumnGetterFactory;
 import org.embulk.input.postgresql.PostgreSQLInputConnection;
+import org.embulk.input.redshift.getter.RedshiftColumnGetterFactory;
+import org.embulk.spi.PageBuilder;
+import org.joda.time.DateTimeZone;
 
 public class RedshiftInputPlugin
         extends AbstractJdbcInputPlugin
@@ -90,4 +94,10 @@ public class RedshiftInputPlugin
             }
         }
     }
+
+    @Override
+    protected ColumnGetterFactory newColumnGetterFactory(PageBuilder pageBuilder, DateTimeZone dateTimeZone)
+    {
+        return new RedshiftColumnGetterFactory(pageBuilder, dateTimeZone);
+    }
 }

data/src/main/java/org/embulk/input/redshift/getter/RedshiftColumnGetterFactory.java

@@ -0,0 +1,36 @@
+package org.embulk.input.redshift.getter;
+
+import org.embulk.input.jdbc.AbstractJdbcInputPlugin.PluginTask;
+import org.embulk.input.jdbc.JdbcColumn;
+import org.embulk.input.jdbc.JdbcColumnOption;
+import org.embulk.input.jdbc.JdbcInputConnection;
+import org.embulk.input.jdbc.getter.ColumnGetter;
+import org.embulk.input.jdbc.getter.ColumnGetterFactory;
+import org.embulk.input.jdbc.getter.TimestampWithTimeZoneIncrementalHandler;
+import org.embulk.input.jdbc.getter.TimestampWithoutTimeZoneIncrementalHandler;
+import org.embulk.spi.PageBuilder;
+import org.joda.time.DateTimeZone;
+
+public class RedshiftColumnGetterFactory extends ColumnGetterFactory
+{
+    public RedshiftColumnGetterFactory(PageBuilder to, DateTimeZone defaultTimeZone)
+    {
+        super(to, defaultTimeZone);
+    }
+
+    @Override
+    public ColumnGetter newColumnGetter(JdbcInputConnection con, PluginTask task, JdbcColumn column, JdbcColumnOption option)
+    {
+        ColumnGetter getter = super.newColumnGetter(con, task, column, option);
+
+        // incremental loading wrapper
+        switch (column.getTypeName()) {
+        case "timestamptz":
+            return new TimestampWithTimeZoneIncrementalHandler(getter);
+        case "timestamp":
+            return new TimestampWithoutTimeZoneIncrementalHandler(getter);
+        default:
+            return getter;
+        }
+    }
+}

data/src/test/java/org/embulk/input/redshift/IncrementalTest.java

@@ -0,0 +1,143 @@
+package org.embulk.input.redshift;
+
+import org.embulk.config.ConfigDiff;
+import org.embulk.config.ConfigSource;
+import org.embulk.input.RedshiftInputPlugin;
+import org.embulk.spi.InputPlugin;
+import org.embulk.test.EmbulkTests;
+import org.embulk.test.TestingEmbulk;
+import org.embulk.test.TestingEmbulk.RunResult;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+
+import java.nio.file.Path;
+
+import static org.embulk.input.redshift.RedshiftTests.execute;
+import static org.embulk.test.EmbulkTests.readSortedFile;
+import static org.hamcrest.Matchers.is;
+import static org.junit.Assert.assertThat;
+
+public class IncrementalTest
+{
+    private static final String BASIC_RESOURCE_PATH = "org/embulk/input/redshift/test/expect/incremental/";
+
+    private static ConfigSource loadYamlResource(TestingEmbulk embulk, String fileName)
+    {
+        return embulk.loadYamlResource(BASIC_RESOURCE_PATH + fileName);
+    }
+
+    private static String readResource(String fileName)
+    {
+        return EmbulkTests.readResource(BASIC_RESOURCE_PATH + fileName);
+    }
+
+    @Rule
+    public TestingEmbulk embulk = TestingEmbulk.builder()
+            .registerPlugin(InputPlugin.class, "redshift", RedshiftInputPlugin.class)
+            .build();
+
+    private ConfigSource baseConfig;
+
+    @Before
+    public void setup()
+    {
+        baseConfig = RedshiftTests.baseConfig();
+    }
+
+    @Test
+    public void simpleInt() throws Exception
+    {
+        // setup first rows
+        execute(readResource("int/setup.sql"));
+
+        Path out1 = embulk.createTempFile("csv");
+        RunResult result1 = embulk.runInput(
+                baseConfig.merge(loadYamlResource(embulk, "int/config_1.yml")),
+                out1);
+        assertThat(
+                readSortedFile(out1),
+                is(readResource("int/expected_1.csv")));
+        assertThat(
+                result1.getConfigDiff(),
+                is((ConfigDiff) loadYamlResource(embulk, "int/expected_1.diff")));
+
+        // insert more rows
+        execute(readResource("int/insert_more.sql"));
+
+        Path out2 = embulk.createTempFile("csv");
+        RunResult result2 = embulk.runInput(
+                baseConfig.merge(loadYamlResource(embulk, "int/config_2.yml")),
+                out2);
+        assertThat(
+                readSortedFile(out2),
+                is(readResource("int/expected_2.csv")));
+        assertThat(
+                result2.getConfigDiff(),
+                is((ConfigDiff) loadYamlResource(embulk, "int/expected_2.diff")));
+    }
+
+    @Test
+    public void simpleTimestampWithoutTimeZone() throws Exception
+    {
+        // setup first rows
+        execute(readResource("timestamp/setup.sql"));
+
+        Path out1 = embulk.createTempFile("csv");
+        RunResult result1 = embulk.runInput(
+                baseConfig.merge(loadYamlResource(embulk, "timestamp/config_1.yml")),
+                out1);
+        assertThat(
+                readSortedFile(out1),
+                is(readResource("timestamp/expected_1.csv")));
+        assertThat(
+                result1.getConfigDiff(),
+                is((ConfigDiff) loadYamlResource(embulk, "timestamp/expected_1.diff")));
+
+        // insert more rows
+        execute(readResource("timestamp/insert_more.sql"));
+
+        Path out2 = embulk.createTempFile("csv");
+        RunResult result2 = embulk.runInput(
+                baseConfig.merge(loadYamlResource(embulk, "timestamp/config_2.yml")),
+                out2);
+        assertThat(
+                readSortedFile(out2),
+                is(readResource("timestamp/expected_2.csv")));
+        assertThat(
+                result2.getConfigDiff(),
+                is((ConfigDiff) loadYamlResource(embulk, "timestamp/expected_2.diff")));
+    }
+
+    @Test
+    public void simpleTimestampWithTimeZone() throws Exception
+    {
+        // setup first rows
+        execute(readResource("timestamptz/setup.sql"));
+
+        Path out1 = embulk.createTempFile("csv");
+        RunResult result1 = embulk.runInput(
+                baseConfig.merge(loadYamlResource(embulk, "timestamptz/config_1.yml")),
+                out1);
+        assertThat(
+                readSortedFile(out1),
+                is(readResource("timestamptz/expected_1.csv")));
+        assertThat(
+                result1.getConfigDiff(),
+                is((ConfigDiff) loadYamlResource(embulk, "timestamptz/expected_1.diff")));
+
+        // insert more rows
+        execute(readResource("timestamptz/insert_more.sql"));
+
+        Path out2 = embulk.createTempFile("csv");
+        RunResult result2 = embulk.runInput(
+                baseConfig.merge(loadYamlResource(embulk, "timestamptz/config_2.yml")),
+                out2);
+        assertThat(
+                readSortedFile(out2),
+                is(readResource("timestamptz/expected_2.csv")));
+        assertThat(
+                result2.getConfigDiff(),
+                is((ConfigDiff) loadYamlResource(embulk, "timestamptz/expected_2.diff")));
+    }
+}

data/src/test/java/org/embulk/input/redshift/RedshiftTests.java

@@ -0,0 +1,43 @@
+package org.embulk.input.redshift;
+
+import com.google.common.base.Throwables;
+import com.google.common.io.ByteStreams;
+import org.embulk.config.ConfigSource;
+import org.embulk.test.EmbulkTests;
+
+import java.io.IOException;
+
+import static java.util.Locale.ENGLISH;
+
+public class RedshiftTests
+{
+    public static ConfigSource baseConfig()
+    {
+        return EmbulkTests.config("EMBULK_INPUT_REDSHIFT_TEST_CONFIG");
+    }
+
+    public static void execute(String sql)
+    {
+        ConfigSource config = baseConfig();
+        ProcessBuilder pb = new ProcessBuilder("psql", "-w", "--set", "ON_ERROR_STOP=1", "-c", sql);
+        pb.environment().put("PGUSER", config.get(String.class, "user"));
+        pb.environment().put("PGPASSWORD", config.get(String.class, "password"));
+        pb.environment().put("PGDATABASE", config.get(String.class, "database"));
+        pb.environment().put("PGHOST", config.get(String.class, "host", "localhost"));
+        pb.environment().put("PGPORT", config.get(String.class, "port", "5439"));
+        pb.redirectErrorStream(true);
+
+        int code;
+        try {
+            Process process = pb.start();
+            ByteStreams.copy(process.getInputStream(), System.out);
+            code = process.waitFor();
+        } catch (IOException | InterruptedException ex) {
+            throw Throwables.propagate(ex);
+        }
+        if (code != 0) {
+            throw new RuntimeException(String.format(ENGLISH,
+                        "Command finished with non-zero exit code. Exit code is %d.", code));
+        }
+    }
+}

data/src/test/resources/org/embulk/input/redshift/test/expect/incremental/int/config_1.yml

@@ -0,0 +1,3 @@
+table: int_load
+incremental: true
+incremental_columns: [num]

data/src/test/resources/org/embulk/input/redshift/test/expect/incremental/int/config_2.yml

@@ -0,0 +1,4 @@
+table: int_load
+last_record: [4]
+incremental: true
+incremental_columns: [num]

data/src/test/resources/org/embulk/input/redshift/test/expect/incremental/int/expected_1.csv

@@ -0,0 +1,4 @@
+1,first
+2,first
+3,first
+4,first

data/src/test/resources/org/embulk/input/redshift/test/expect/incremental/int/expected_1.diff

@@ -0,0 +1,3 @@
+in:
+  last_record: [4]
+out: {}

data/src/test/resources/org/embulk/input/redshift/test/expect/incremental/int/expected_2.csv

@@ -0,0 +1,2 @@
+5,more_load
+9,more_load

data/src/test/resources/org/embulk/input/redshift/test/expect/incremental/int/expected_2.diff

@@ -0,0 +1,3 @@
+in:
+  last_record: [9]
+out: {}

data/src/test/resources/org/embulk/input/redshift/test/expect/incremental/int/insert_more.sql

@@ -0,0 +1,6 @@
+
+insert into int_load (num, note) values
+(0, 'more_skip'),
+(4, 'more_skip'),
+(9, 'more_load'),
+(5, 'more_load');

data/src/test/resources/org/embulk/input/redshift/test/expect/incremental/int/setup.sql

@@ -0,0 +1,12 @@
+drop table if exists int_load;
+
+create table int_load (
+    num int not null,
+    note text
+);
+
+insert into int_load (num, note) values
+(3, 'first'),
+(4, 'first'),
+(2, 'first'),
+(1, 'first');

data/src/test/resources/org/embulk/input/redshift/test/expect/incremental/timestamp/config_1.yml

@@ -0,0 +1,4 @@
+table: load
+default_time_zone: +0300
+incremental: true
+incremental_columns: [time]

data/src/test/resources/org/embulk/input/redshift/test/expect/incremental/timestamp/config_2.yml

@@ -0,0 +1,4 @@
+table: load
+last_record: ['2016-11-02T04:00:05.333003']
+incremental: true
+incremental_columns: [time]

data/src/test/resources/org/embulk/input/redshift/test/expect/incremental/timestamp/expected_1.csv

@@ -0,0 +1,7 @@
+2016-11-01 23:00:01.000000 +0000,first
+2016-11-02 00:00:02.000000 +0000,first
+2016-11-02 01:00:03.000000 +0000,first
+2016-11-02 02:00:04.000000 +0000,first
+2016-11-02 02:00:05.111001 +0000,first
+2016-11-02 02:00:05.222002 +0000,first
+2016-11-02 02:00:05.333003 +0000,first

data/src/test/resources/org/embulk/input/redshift/test/expect/incremental/timestamp/expected_1.diff

@@ -0,0 +1,3 @@
+in:
+  last_record: ['2016-11-02T04:00:05.333003']
+out: {}

data/src/test/resources/org/embulk/input/redshift/test/expect/incremental/timestamp/expected_2.csv

@@ -0,0 +1,3 @@
+2016-11-02 02:00:05.333004 +0000,more_load
+2016-11-02 02:00:06.000000 +0000,more_load
+2016-11-02 02:00:06.000000 +0000,more_load

data/src/test/resources/org/embulk/input/redshift/test/expect/incremental/timestamp/expected_2.diff

@@ -0,0 +1,3 @@
+in:
+  last_record: ['2016-11-02T04:00:06.000000']
+out: {}

data/src/test/resources/org/embulk/input/redshift/test/expect/incremental/timestamp/insert_more.sql

@@ -0,0 +1,8 @@
+
+insert into load (time, note) values
+('2016-11-02 04:00:00', 'more_skip'),
+('2016-11-02 04:00:05.333000', 'more_skip'),
+('2016-11-02 04:00:05.333003', 'more_skip'),
+('2016-11-02 04:00:05.333004', 'more_load'),
+('2016-11-02 04:00:06', 'more_load'),
+('2016-11-02 04:00:06', 'more_load');

data/src/test/resources/org/embulk/input/redshift/test/expect/incremental/timestamp/setup.sql

@@ -0,0 +1,15 @@
+drop table if exists load;
+
+create table load (
+    time timestamp without time zone not null,
+    note text
+);
+
+insert into load (time, note) values
+('2016-11-02 01:00:01', 'first'),
+('2016-11-02 02:00:02', 'first'),
+('2016-11-02 03:00:03', 'first'),
+('2016-11-02 04:00:04', 'first'),
+('2016-11-02 04:00:05.111001', 'first'),
+('2016-11-02 04:00:05.222002', 'first'),
+('2016-11-02 04:00:05.333003', 'first');

data/src/test/resources/org/embulk/input/redshift/test/expect/incremental/timestamptz/config_1.yml

@@ -0,0 +1,4 @@
+table: load
+default_time_zone: +0300
+incremental: true
+incremental_columns: [time]

data/src/test/resources/org/embulk/input/redshift/test/expect/incremental/timestamptz/config_2.yml

@@ -0,0 +1,4 @@
+table: load
+last_record: ['2016-11-02T04:00:05.333003Z']
+incremental: true
+incremental_columns: [time]

data/src/test/resources/org/embulk/input/redshift/test/expect/incremental/timestamptz/expected_1.csv

@@ -0,0 +1,7 @@
+2016-11-02 01:00:01.000000 +0000,first
+2016-11-02 02:00:02.000000 +0000,first
+2016-11-02 03:00:03.000000 +0000,first
+2016-11-02 04:00:04.000000 +0000,first
+2016-11-02 04:00:05.111001 +0000,first
+2016-11-02 04:00:05.222002 +0000,first
+2016-11-02 04:00:05.333003 +0000,first

data/src/test/resources/org/embulk/input/redshift/test/expect/incremental/timestamptz/expected_1.diff

@@ -0,0 +1,3 @@
+in:
+  last_record: ['2016-11-02T04:00:05.333003Z']
+out: {}

data/src/test/resources/org/embulk/input/redshift/test/expect/incremental/timestamptz/expected_2.csv

@@ -0,0 +1,3 @@
+2016-11-02 04:00:05.333004 +0000,more_load
+2016-11-02 04:00:06.000000 +0000,more_load
+2016-11-02 04:00:06.000000 +0000,more_load

data/src/test/resources/org/embulk/input/redshift/test/expect/incremental/timestamptz/expected_2.diff

@@ -0,0 +1,3 @@
+in:
+  last_record: ['2016-11-02T04:00:06.000000Z']
+out: {}

data/src/test/resources/org/embulk/input/redshift/test/expect/incremental/timestamptz/insert_more.sql

@@ -0,0 +1,8 @@
+
+insert into load (time, note) values
+('2016-11-02 04:00:00+0000', 'more_skip'),
+('2016-11-02 04:00:05.333000+0000', 'more_skip'),
+('2016-11-02 04:00:05.333003+0000', 'more_skip'),
+('2016-11-02 04:00:05.333004+0000', 'more_load'),
+('2016-11-02 04:00:06+0000', 'more_load'),
+('2016-11-02 04:00:06+0000', 'more_load');

data/src/test/resources/org/embulk/input/redshift/test/expect/incremental/timestamptz/setup.sql

@@ -0,0 +1,15 @@
+drop table if exists load;
+
+create table load (
+    time timestamptz not null,
+    note text
+);
+
+insert into load (time, note) values
+('2016-11-02 01:00:01+0000', 'first'),
+('2016-11-02 02:00:02+0000', 'first'),
+('2016-11-02 03:00:03+0000', 'first'),
+('2016-11-02 04:00:04+0000', 'first'),
+('2016-11-02 04:00:05.111001+0000', 'first'),
+('2016-11-02 04:00:05.222002+0000', 'first'),
+('2016-11-02 04:00:05.333003+0000', 'first');

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-redshift
 version: !ruby/object:Gem::Version
-  version: 0.8.2
+  version: 0.8.3
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-02-10 00:00:00.000000000 Z
+date: 2017-05-26 00:00:00.000000000 Z
 dependencies: []
 description: Selects records from a table.
 email:
@@ -19,12 +19,39 @@ extra_rdoc_files: []
 files:
 - README.md
 - build.gradle
-- classpath/embulk-input-jdbc-0.8.2.jar
-- classpath/embulk-input-postgresql-0.8.2.jar
-- classpath/embulk-input-redshift-0.8.2.jar
+- classpath/embulk-input-jdbc-0.8.3.jar
+- classpath/embulk-input-postgresql-0.8.3.jar
+- classpath/embulk-input-redshift-0.8.3.jar
 - classpath/postgresql-9.4-1205-jdbc41.jar
 - lib/embulk/input/redshift.rb
 - src/main/java/org/embulk/input/RedshiftInputPlugin.java
+- src/main/java/org/embulk/input/redshift/getter/RedshiftColumnGetterFactory.java
+- src/test/java/org/embulk/input/redshift/IncrementalTest.java
+- src/test/java/org/embulk/input/redshift/RedshiftTests.java
+- src/test/resources/org/embulk/input/redshift/test/expect/incremental/int/config_1.yml
+- src/test/resources/org/embulk/input/redshift/test/expect/incremental/int/config_2.yml
+- src/test/resources/org/embulk/input/redshift/test/expect/incremental/int/expected_1.csv
+- src/test/resources/org/embulk/input/redshift/test/expect/incremental/int/expected_1.diff
+- src/test/resources/org/embulk/input/redshift/test/expect/incremental/int/expected_2.csv
+- src/test/resources/org/embulk/input/redshift/test/expect/incremental/int/expected_2.diff
+- src/test/resources/org/embulk/input/redshift/test/expect/incremental/int/insert_more.sql
+- src/test/resources/org/embulk/input/redshift/test/expect/incremental/int/setup.sql
+- src/test/resources/org/embulk/input/redshift/test/expect/incremental/timestamp/config_1.yml
+- src/test/resources/org/embulk/input/redshift/test/expect/incremental/timestamp/config_2.yml
+- src/test/resources/org/embulk/input/redshift/test/expect/incremental/timestamp/expected_1.csv
+- src/test/resources/org/embulk/input/redshift/test/expect/incremental/timestamp/expected_1.diff
+- src/test/resources/org/embulk/input/redshift/test/expect/incremental/timestamp/expected_2.csv
+- src/test/resources/org/embulk/input/redshift/test/expect/incremental/timestamp/expected_2.diff
+- src/test/resources/org/embulk/input/redshift/test/expect/incremental/timestamp/insert_more.sql
+- src/test/resources/org/embulk/input/redshift/test/expect/incremental/timestamp/setup.sql
+- src/test/resources/org/embulk/input/redshift/test/expect/incremental/timestamptz/config_1.yml
+- src/test/resources/org/embulk/input/redshift/test/expect/incremental/timestamptz/config_2.yml
+- src/test/resources/org/embulk/input/redshift/test/expect/incremental/timestamptz/expected_1.csv
+- src/test/resources/org/embulk/input/redshift/test/expect/incremental/timestamptz/expected_1.diff
+- src/test/resources/org/embulk/input/redshift/test/expect/incremental/timestamptz/expected_2.csv
+- src/test/resources/org/embulk/input/redshift/test/expect/incremental/timestamptz/expected_2.diff
+- src/test/resources/org/embulk/input/redshift/test/expect/incremental/timestamptz/insert_more.sql
+- src/test/resources/org/embulk/input/redshift/test/expect/incremental/timestamptz/setup.sql
 homepage: https://github.com/embulk/embulk-input-jdbc
 licenses:
 - Apache 2.0