embulk-input-postgresql 0.8.0 → 0.8.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2c06aab77369550b3a8fe22a06ed19f08813b552
-  data.tar.gz: fe540f426d5a51eab8768330faf2964fa0323410
+  metadata.gz: b02b6267f621dc419cbc4b72486ce4c69ec31dbf
+  data.tar.gz: 6257da06c3568e91fa9ffd6d7f4012300f5d4eed
 SHA512:
-  metadata.gz: 563d09b98cd013a35089223a1f88dbf413be9dd5d894c3cd567420e0f67bd5d390eab565cdc1ad2b4dcf7c70cde6f8c1461b19a35cfa6562ddadff88a1458fb4
-  data.tar.gz: 5166f3f915d6159c7a576fbf0db217ad5ee153b72eaa051feb6f9c2b2c7452f38ef0cce6c1cbb83ec2313716abe0e30eef66f38ac10c91937d9150f7896a0cd8
+  metadata.gz: 2d0a03f7ab55bd8a90de593d31525a210fa2fddd1271bc422eca6664234e353d9c376e6c8b0a6b3fda0b318a5ba3b5cf9b5f01e7deabbee85b11a87c14806dea
+  data.tar.gz: 3892fa790c926a25dbf4fd11c7e78e1e62113d3ce53e241ebc252385449b6bdb3bcc7f622f4371b8678c231ce4947ec0acecaf31d3646298a8827c31fcc83b65

data/README.md

@@ -160,3 +160,11 @@ in:
 ```
 $ ./gradlew gem
 ```
+
+Running tests:
+
+```
+$ cp ci/travis_postgresql.yml ci/postgresql.yml  # edit this file if necessary
+$ EMBULK_INPUT_POSTGRESQL_TEST_CONFIG=`pwd`/ci/postgresql.yml ./gradlew :embulk-input-postgresql:check --info
+```
+

data/build.gradle

@@ -3,6 +3,5 @@ dependencies {
 
     compile 'org.postgresql:postgresql:9.4-1205-jdbc41'
 
-    testCompile 'org.embulk:embulk-standards:0.8.8'
-    testCompile project(':embulk-input-jdbc').sourceSets.test.output
+    testCompile 'org.embulk:embulk-standards:0.8.15'
 }

data/src/main/java/org/embulk/input/postgresql/getter/{HstoreColumnGetter.java → HstoreToJsonColumnGetter.java}

@@ -2,62 +2,37 @@ package org.embulk.input.postgresql.getter;
 
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
-import org.embulk.input.jdbc.getter.AbstractColumnGetter;
+import org.embulk.input.jdbc.getter.JsonColumnGetter;
 import org.embulk.spi.Column;
 import org.embulk.spi.PageBuilder;
 import org.embulk.spi.json.JsonParseException;
-import org.embulk.spi.json.JsonParser;
 import org.embulk.spi.type.Type;
-import org.embulk.spi.type.Types;
 import org.msgpack.value.Value;
 import org.postgresql.util.HStoreConverter;
 
-import java.sql.ResultSet;
-import java.sql.SQLException;
 import java.util.Map;
 
-public class HstoreColumnGetter
-        extends AbstractColumnGetter
+public class HstoreToJsonColumnGetter
+        extends JsonColumnGetter
 {
-    private final JsonParser parser = new JsonParser();
     private final ObjectMapper mapper = new ObjectMapper();
 
-    private String value;
-
-    public HstoreColumnGetter(PageBuilder to, Type toType)
+    public HstoreToJsonColumnGetter(PageBuilder to, Type toType)
     {
         super(to, toType);
     }
 
-    @Override
-    protected void fetch(ResultSet from, int fromIndex) throws SQLException
-    {
-        value = from.getString(fromIndex);
-    }
-
-    @Override
-    protected Type getDefaultToType()
-    {
-        return Types.STRING;
-    }
-
     @Override
     public void jsonColumn(Column column)
     {
         Value v;
         try {
             Map map = HStoreConverter.fromString(value);
-            v = parser.parse(mapper.writeValueAsString(map));
+            v = jsonParser.parse(mapper.writeValueAsString(map));
         } catch (JsonProcessingException | JsonParseException e) {
             super.jsonColumn(column);
             return;
         }
         to.setJson(column, v);
     }
-
-    @Override
-    public void stringColumn(Column column)
-    {
-        to.setString(column, value);
-    }
 }

data/src/main/java/org/embulk/input/postgresql/getter/PostgreSQLColumnGetterFactory.java

@@ -1,10 +1,15 @@
 package org.embulk.input.postgresql.getter;
 
+import org.embulk.input.jdbc.AbstractJdbcInputPlugin.PluginTask;
 import org.embulk.input.jdbc.JdbcColumn;
 import org.embulk.input.jdbc.JdbcColumnOption;
+import org.embulk.input.jdbc.JdbcInputConnection;
 import org.embulk.input.jdbc.getter.ColumnGetter;
 import org.embulk.input.jdbc.getter.ColumnGetterFactory;
+import org.embulk.input.jdbc.getter.TimestampWithTimeZoneIncrementalHandler;
+import org.embulk.input.jdbc.getter.TimestampWithoutTimeZoneIncrementalHandler;
 import org.embulk.spi.PageBuilder;
+import org.embulk.spi.type.Types;
 import org.joda.time.DateTimeZone;
 
 public class PostgreSQLColumnGetterFactory extends ColumnGetterFactory
@@ -15,21 +20,37 @@ public class PostgreSQLColumnGetterFactory extends ColumnGetterFactory
     }
 
     @Override
-    public ColumnGetter newColumnGetter(JdbcColumn column, JdbcColumnOption option)
+    public ColumnGetter newColumnGetter(JdbcInputConnection con, PluginTask task, JdbcColumn column, JdbcColumnOption option)
     {
-        if (column.getTypeName().equals("hstore")) {
-            return new HstoreColumnGetter(to, getToType(option));
-        } else {
-            return super.newColumnGetter(column, option);
+        if (column.getTypeName().equals("hstore") && getToType(option) == Types.JSON) {
+            // converting hstore to json needs a special handling
+            return new HstoreToJsonColumnGetter(to, Types.JSON);
+        }
+
+        ColumnGetter getter = super.newColumnGetter(con, task, column, option);
+
+        // incremental loading wrapper
+        switch (column.getTypeName()) {
+        case "timestamptz":
+            return new TimestampWithTimeZoneIncrementalHandler(getter);
+        case "timestamp":
+            return new TimestampWithoutTimeZoneIncrementalHandler(getter);
+        default:
+            return getter;
         }
     }
 
     @Override
     protected String sqlTypeToValueType(JdbcColumn column, int sqlType)
     {
-        if (column.getTypeName().equals("json") || column.getTypeName().equals("jsonb")) {
+        switch(column.getTypeName()) {
+        case "json":
+        case "jsonb":
             return "json";
-        } else {
+        case "hstore":
+            // hstore is converted to string by default
+            return "string";
+        default:
             return super.sqlTypeToValueType(column, sqlType);
         }
     }

data/src/test/java/org/embulk/input/postgresql/HstoreTest.java

@@ -0,0 +1,75 @@
+package org.embulk.input.postgresql;
+
+import java.nio.file.Path;
+
+import org.embulk.config.ConfigDiff;
+import org.embulk.config.ConfigSource;
+import org.embulk.input.PostgreSQLInputPlugin;
+import org.embulk.spi.InputPlugin;
+import org.embulk.test.TestingEmbulk.RunResult;
+import org.embulk.test.EmbulkTests;
+import org.embulk.test.TestingEmbulk;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+
+import static org.embulk.input.postgresql.PostgreSQLTests.execute;
+import static org.embulk.test.EmbulkTests.readSortedFile;
+import static org.hamcrest.Matchers.is;
+import static org.junit.Assert.assertThat;
+
+public class HstoreTest
+{
+    private static final String BASIC_RESOURCE_PATH = "org/embulk/input/postgresql/test/expect/hstore/";
+
+    private static ConfigSource loadYamlResource(TestingEmbulk embulk, String fileName)
+    {
+        return embulk.loadYamlResource(BASIC_RESOURCE_PATH + fileName);
+    }
+
+    private static String readResource(String fileName)
+    {
+        return EmbulkTests.readResource(BASIC_RESOURCE_PATH + fileName);
+    }
+
+    @Rule
+    public TestingEmbulk embulk = TestingEmbulk.builder()
+        .registerPlugin(InputPlugin.class, "postgresql", PostgreSQLInputPlugin.class)
+        .build();
+
+    private ConfigSource baseConfig;
+
+    @Before
+    public void setup()
+    {
+        baseConfig = PostgreSQLTests.baseConfig();
+    }
+
+    @Test
+    public void loadAsStringByDefault() throws Exception
+    {
+        execute(readResource("setup.sql"));
+
+        Path out1 = embulk.createTempFile("csv");
+        RunResult result1 = embulk.runInput(
+                baseConfig.merge(loadYamlResource(embulk, "as_string.yml")),
+                out1);
+        assertThat(
+                readSortedFile(out1),
+                is(readResource("expected_string.csv")));
+    }
+
+    @Test
+    public void loadAsJson() throws Exception
+    {
+        execute(readResource("setup.sql"));
+
+        Path out1 = embulk.createTempFile("csv");
+        RunResult result1 = embulk.runInput(
+                baseConfig.merge(loadYamlResource(embulk, "as_json.yml")),
+                out1);
+        assertThat(
+                readSortedFile(out1),
+                is(readResource("expected_json.csv")));
+    }
+}

data/src/test/java/org/embulk/input/postgresql/IncrementalTest.java

@@ -0,0 +1,143 @@
+package org.embulk.input.postgresql;
+
+import java.nio.file.Path;
+
+import org.embulk.config.ConfigDiff;
+import org.embulk.config.ConfigSource;
+import org.embulk.input.PostgreSQLInputPlugin;
+import org.embulk.spi.InputPlugin;
+import org.embulk.test.TestingEmbulk.RunResult;
+import org.embulk.test.EmbulkTests;
+import org.embulk.test.TestingEmbulk;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+
+import static org.embulk.input.postgresql.PostgreSQLTests.execute;
+import static org.embulk.test.EmbulkTests.readSortedFile;
+import static org.hamcrest.Matchers.is;
+import static org.junit.Assert.assertThat;
+
+public class IncrementalTest
+{
+    private static final String BASIC_RESOURCE_PATH = "org/embulk/input/postgresql/test/expect/incremental/";
+
+    private static ConfigSource loadYamlResource(TestingEmbulk embulk, String fileName)
+    {
+        return embulk.loadYamlResource(BASIC_RESOURCE_PATH + fileName);
+    }
+
+    private static String readResource(String fileName)
+    {
+        return EmbulkTests.readResource(BASIC_RESOURCE_PATH + fileName);
+    }
+
+    @Rule
+    public TestingEmbulk embulk = TestingEmbulk.builder()
+        .registerPlugin(InputPlugin.class, "postgresql", PostgreSQLInputPlugin.class)
+        .build();
+
+    private ConfigSource baseConfig;
+
+    @Before
+    public void setup()
+    {
+        baseConfig = PostgreSQLTests.baseConfig();
+    }
+
+    @Test
+    public void simpleInt() throws Exception
+    {
+        // setup first rows
+        execute(readResource("int/setup.sql"));
+
+        Path out1 = embulk.createTempFile("csv");
+        RunResult result1 = embulk.runInput(
+                baseConfig.merge(loadYamlResource(embulk, "int/config_1.yml")),
+                out1);
+        assertThat(
+                readSortedFile(out1),
+                is(readResource("int/expected_1.csv")));
+        assertThat(
+                result1.getConfigDiff(),
+                is((ConfigDiff) loadYamlResource(embulk, "int/expected_1.diff")));
+
+        // insert more rows
+        execute(readResource("int/insert_more.sql"));
+
+        Path out2 = embulk.createTempFile("csv");
+        RunResult result2 = embulk.runInput(
+                baseConfig.merge(loadYamlResource(embulk, "int/config_2.yml")),
+                out2);
+        assertThat(
+                readSortedFile(out2),
+                is(readResource("int/expected_2.csv")));
+        assertThat(
+                result2.getConfigDiff(),
+                is((ConfigDiff) loadYamlResource(embulk, "int/expected_2.diff")));
+    }
+
+    @Test
+    public void simpleTimestampWithoutTimeZone() throws Exception
+    {
+        // setup first rows
+        execute(readResource("timestamp/setup.sql"));
+
+        Path out1 = embulk.createTempFile("csv");
+        RunResult result1 = embulk.runInput(
+                baseConfig.merge(loadYamlResource(embulk, "timestamp/config_1.yml")),
+                out1);
+        assertThat(
+                readSortedFile(out1),
+                is(readResource("timestamp/expected_1.csv")));
+        assertThat(
+                result1.getConfigDiff(),
+                is((ConfigDiff) loadYamlResource(embulk, "timestamp/expected_1.diff")));
+
+        // insert more rows
+        execute(readResource("timestamp/insert_more.sql"));
+
+        Path out2 = embulk.createTempFile("csv");
+        RunResult result2 = embulk.runInput(
+                baseConfig.merge(loadYamlResource(embulk, "timestamp/config_2.yml")),
+                out2);
+        assertThat(
+                readSortedFile(out2),
+                is(readResource("timestamp/expected_2.csv")));
+        assertThat(
+                result2.getConfigDiff(),
+                is((ConfigDiff) loadYamlResource(embulk, "timestamp/expected_2.diff")));
+    }
+
+    @Test
+    public void simpleTimestampWithTimeZone() throws Exception
+    {
+        // setup first rows
+        execute(readResource("timestamptz/setup.sql"));
+
+        Path out1 = embulk.createTempFile("csv");
+        RunResult result1 = embulk.runInput(
+                baseConfig.merge(loadYamlResource(embulk, "timestamptz/config_1.yml")),
+                out1);
+        assertThat(
+                readSortedFile(out1),
+                is(readResource("timestamptz/expected_1.csv")));
+        assertThat(
+                result1.getConfigDiff(),
+                is((ConfigDiff) loadYamlResource(embulk, "timestamptz/expected_1.diff")));
+
+        // insert more rows
+        execute(readResource("timestamptz/insert_more.sql"));
+
+        Path out2 = embulk.createTempFile("csv");
+        RunResult result2 = embulk.runInput(
+                baseConfig.merge(loadYamlResource(embulk, "timestamptz/config_2.yml")),
+                out2);
+        assertThat(
+                readSortedFile(out2),
+                is(readResource("timestamptz/expected_2.csv")));
+        assertThat(
+                result2.getConfigDiff(),
+                is((ConfigDiff) loadYamlResource(embulk, "timestamptz/expected_2.diff")));
+    }
+}

data/src/test/java/org/embulk/input/postgresql/PostgreSQLTests.java

@@ -0,0 +1,39 @@
+package org.embulk.input.postgresql;
+
+import org.embulk.test.EmbulkTests;
+import com.google.common.base.Throwables;
+import com.google.common.io.ByteStreams;
+import java.io.IOException;
+import org.embulk.config.ConfigSource;
+import static java.util.Locale.ENGLISH;
+
+public class PostgreSQLTests
+{
+    public static ConfigSource baseConfig()
+    {
+        return EmbulkTests.config("EMBULK_INPUT_POSTGRESQL_TEST_CONFIG");
+    }
+
+    public static void execute(String sql)
+    {
+        ConfigSource config = baseConfig();
+        ProcessBuilder pb = new ProcessBuilder("psql", "-w", "--set", "ON_ERROR_STOP=1", "-c", sql);
+        pb.environment().put("PGUSER", config.get(String.class, "user"));
+        pb.environment().put("PGPASSWORD", config.get(String.class, "password"));
+        pb.environment().put("PGDATABASE", config.get(String.class, "database"));
+        pb.environment().put("PGPORT", config.get(String.class, "port", "5432"));
+        pb.redirectErrorStream(true);
+        int code;
+        try {
+            Process process = pb.start();
+            ByteStreams.copy(process.getInputStream(), System.out);
+            code = process.waitFor();
+        } catch (IOException | InterruptedException ex) {
+            throw Throwables.propagate(ex);
+        }
+        if (code != 0) {
+            throw new RuntimeException(String.format(ENGLISH,
+                        "Command finished with non-zero exit code. Exit code is %d.", code));
+        }
+    }
+}

data/src/test/resources/org/embulk/input/postgresql/test/expect/hstore/as_json.yml

@@ -0,0 +1,3 @@
+table: input_hstore
+column_options:
+  c1: {type: json}

data/src/test/resources/org/embulk/input/postgresql/test/expect/hstore/as_string.yml

@@ -0,0 +1 @@
+table: input_hstore

data/src/test/resources/org/embulk/input/postgresql/test/expect/hstore/expected_json.csv

@@ -0,0 +1 @@
+"{""a"":""b""}"

data/src/test/resources/org/embulk/input/postgresql/test/expect/hstore/expected_string.csv

@@ -0,0 +1 @@
+"""a""=>""b"""

data/src/test/resources/org/embulk/input/postgresql/test/expect/hstore/setup.sql

@@ -0,0 +1,11 @@
+drop table if exists input_hstore;
+
+create extension if not exists hstore;
+
+create table input_hstore (
+    c1 hstore
+);
+
+insert into input_hstore (c1) values
+('"a" => "b"')
+;

data/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/int/config_1.yml

@@ -0,0 +1,3 @@
+table: int_load
+incremental: true
+incremental_columns: [num]

data/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/int/config_2.yml

@@ -0,0 +1,4 @@
+table: int_load
+last_record: [4]
+incremental: true
+incremental_columns: [num]

data/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/int/expected_1.csv

@@ -0,0 +1,4 @@
+1,first
+2,first
+3,first
+4,first

data/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/int/expected_1.diff

@@ -0,0 +1,3 @@
+in:
+  last_record: [4]
+out: {}

data/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/int/expected_2.csv

@@ -0,0 +1,2 @@
+5,more_load
+9,more_load

data/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/int/expected_2.diff

@@ -0,0 +1,3 @@
+in:
+  last_record: [9]
+out: {}

data/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/int/insert_more.sql

@@ -0,0 +1,7 @@
+
+insert into int_load (num, note) values
+(0, 'more_skip'),
+(4, 'more_skip'),
+(9, 'more_load'),
+(5, 'more_load');
+

data/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/int/setup.sql

@@ -0,0 +1,13 @@
+drop table if exists int_load;
+
+create table int_load (
+    num int not null,
+    note text
+);
+
+insert into int_load (num, note) values
+(3, 'first'),
+(4, 'first'),
+(2, 'first'),
+(1, 'first');
+

data/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/timestamp/config_1.yml

@@ -0,0 +1,4 @@
+table: load
+default_time_zone: +0300
+incremental: true
+incremental_columns: [time]

data/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/timestamp/config_2.yml

@@ -0,0 +1,4 @@
+table: load
+last_record: ['2016-11-02T04:00:05.333003']
+incremental: true
+incremental_columns: [time]

data/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/timestamp/expected_1.csv

@@ -0,0 +1,7 @@
+2016-11-01 23:00:01.000000 +0000,first
+2016-11-02 00:00:02.000000 +0000,first
+2016-11-02 01:00:03.000000 +0000,first
+2016-11-02 02:00:04.000000 +0000,first
+2016-11-02 02:00:05.111001 +0000,first
+2016-11-02 02:00:05.222002 +0000,first
+2016-11-02 02:00:05.333003 +0000,first

data/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/timestamp/expected_1.diff

@@ -0,0 +1,3 @@
+in:
+  last_record: ['2016-11-02T04:00:05.333003']
+out: {}

data/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/timestamp/expected_2.csv

@@ -0,0 +1,3 @@
+2016-11-02 02:00:05.333004 +0000,more_load
+2016-11-02 02:00:06.000000 +0000,more_load
+2016-11-02 02:00:06.000000 +0000,more_load

data/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/timestamp/expected_2.diff

@@ -0,0 +1,3 @@
+in:
+  last_record: ['2016-11-02T04:00:06.000000']
+out: {}

data/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/timestamp/insert_more.sql

@@ -0,0 +1,9 @@
+
+insert into load (time, note) values
+('2016-11-02 04:00:00', 'more_skip'),
+('2016-11-02 04:00:05.333000', 'more_skip'),
+('2016-11-02 04:00:05.333003', 'more_skip'),
+('2016-11-02 04:00:05.333004', 'more_load'),
+('2016-11-02 04:00:06', 'more_load'),
+('2016-11-02 04:00:06', 'more_load');
+

data/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/timestamp/setup.sql

@@ -0,0 +1,16 @@
+drop table if exists load;
+
+create table load (
+    time timestamp without time zone not null,
+    note text
+);
+
+insert into load (time, note) values
+('2016-11-02 01:00:01', 'first'),
+('2016-11-02 02:00:02', 'first'),
+('2016-11-02 03:00:03', 'first'),
+('2016-11-02 04:00:04', 'first'),
+('2016-11-02 04:00:05.111001', 'first'),
+('2016-11-02 04:00:05.222002', 'first'),
+('2016-11-02 04:00:05.333003', 'first');
+

data/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/timestamptz/config_1.yml

@@ -0,0 +1,4 @@
+table: load
+default_time_zone: +0300
+incremental: true
+incremental_columns: [time]

data/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/timestamptz/config_2.yml

@@ -0,0 +1,4 @@
+table: load
+last_record: ['2016-11-02T04:00:05.333003Z']
+incremental: true
+incremental_columns: [time]

data/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/timestamptz/expected_1.csv

@@ -0,0 +1,7 @@
+2016-11-02 01:00:01.000000 +0000,first
+2016-11-02 02:00:02.000000 +0000,first
+2016-11-02 03:00:03.000000 +0000,first
+2016-11-02 04:00:04.000000 +0000,first
+2016-11-02 04:00:05.111001 +0000,first
+2016-11-02 04:00:05.222002 +0000,first
+2016-11-02 04:00:05.333003 +0000,first

data/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/timestamptz/expected_1.diff

@@ -0,0 +1,3 @@
+in:
+  last_record: ['2016-11-02T04:00:05.333003Z']
+out: {}

data/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/timestamptz/expected_2.csv

@@ -0,0 +1,3 @@
+2016-11-02 04:00:05.333004 +0000,more_load
+2016-11-02 04:00:06.000000 +0000,more_load
+2016-11-02 04:00:06.000000 +0000,more_load

data/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/timestamptz/expected_2.diff

@@ -0,0 +1,3 @@
+in:
+  last_record: ['2016-11-02T04:00:06.000000Z']
+out: {}

data/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/timestamptz/insert_more.sql

@@ -0,0 +1,9 @@
+
+insert into load (time, note) values
+('2016-11-02 04:00:00+0000', 'more_skip'),
+('2016-11-02 04:00:05.333000+0000', 'more_skip'),
+('2016-11-02 04:00:05.333003+0000', 'more_skip'),
+('2016-11-02 04:00:05.333004+0000', 'more_load'),
+('2016-11-02 04:00:06+0000', 'more_load'),
+('2016-11-02 04:00:06+0000', 'more_load');
+

data/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/timestamptz/setup.sql

@@ -0,0 +1,16 @@
+drop table if exists load;
+
+create table load (
+    time timestamptz(6) not null,
+    note text
+);
+
+insert into load (time, note) values
+('2016-11-02 01:00:01+0000', 'first'),
+('2016-11-02 02:00:02+0000', 'first'),
+('2016-11-02 03:00:03+0000', 'first'),
+('2016-11-02 04:00:04+0000', 'first'),
+('2016-11-02 04:00:05.111001+0000', 'first'),
+('2016-11-02 04:00:05.222002+0000', 'first'),
+('2016-11-02 04:00:05.333003+0000', 'first');
+

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-postgresql
 version: !ruby/object:Gem::Version
-  version: 0.8.0
+  version: 0.8.1
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-10-13 00:00:00.000000000 Z
+date: 2017-02-10 00:00:00.000000000 Z
 dependencies: []
 description: Selects records from a table.
 email:
@@ -19,17 +19,46 @@ extra_rdoc_files: []
 files:
 - README.md
 - build.gradle
-- classpath/embulk-input-jdbc-0.8.0.jar
-- classpath/embulk-input-postgresql-0.8.0.jar
+- classpath/embulk-input-jdbc-0.8.1.jar
+- classpath/embulk-input-postgresql-0.8.1.jar
 - classpath/postgresql-9.4-1205-jdbc41.jar
 - lib/embulk/input/postgresql.rb
 - src/main/java/org/embulk/input/PostgreSQLInputPlugin.java
 - src/main/java/org/embulk/input/postgresql/PostgreSQLInputConnection.java
-- src/main/java/org/embulk/input/postgresql/getter/HstoreColumnGetter.java
+- src/main/java/org/embulk/input/postgresql/getter/HstoreToJsonColumnGetter.java
 - src/main/java/org/embulk/input/postgresql/getter/PostgreSQLColumnGetterFactory.java
-- src/test/java/org/embulk/input/postgresql/PostgreSQLInputPluginTest.java
-- src/test/resources/yml/input_hstore.yml
-- src/test/resources/yml/input_hstore2.yml
+- src/test/java/org/embulk/input/postgresql/HstoreTest.java
+- src/test/java/org/embulk/input/postgresql/IncrementalTest.java
+- src/test/java/org/embulk/input/postgresql/PostgreSQLTests.java
+- src/test/resources/org/embulk/input/postgresql/test/expect/hstore/as_json.yml
+- src/test/resources/org/embulk/input/postgresql/test/expect/hstore/as_string.yml
+- src/test/resources/org/embulk/input/postgresql/test/expect/hstore/expected_json.csv
+- src/test/resources/org/embulk/input/postgresql/test/expect/hstore/expected_string.csv
+- src/test/resources/org/embulk/input/postgresql/test/expect/hstore/setup.sql
+- src/test/resources/org/embulk/input/postgresql/test/expect/incremental/int/config_1.yml
+- src/test/resources/org/embulk/input/postgresql/test/expect/incremental/int/config_2.yml
+- src/test/resources/org/embulk/input/postgresql/test/expect/incremental/int/expected_1.csv
+- src/test/resources/org/embulk/input/postgresql/test/expect/incremental/int/expected_1.diff
+- src/test/resources/org/embulk/input/postgresql/test/expect/incremental/int/expected_2.csv
+- src/test/resources/org/embulk/input/postgresql/test/expect/incremental/int/expected_2.diff
+- src/test/resources/org/embulk/input/postgresql/test/expect/incremental/int/insert_more.sql
+- src/test/resources/org/embulk/input/postgresql/test/expect/incremental/int/setup.sql
+- src/test/resources/org/embulk/input/postgresql/test/expect/incremental/timestamp/config_1.yml
+- src/test/resources/org/embulk/input/postgresql/test/expect/incremental/timestamp/config_2.yml
+- src/test/resources/org/embulk/input/postgresql/test/expect/incremental/timestamp/expected_1.csv
+- src/test/resources/org/embulk/input/postgresql/test/expect/incremental/timestamp/expected_1.diff
+- src/test/resources/org/embulk/input/postgresql/test/expect/incremental/timestamp/expected_2.csv
+- src/test/resources/org/embulk/input/postgresql/test/expect/incremental/timestamp/expected_2.diff
+- src/test/resources/org/embulk/input/postgresql/test/expect/incremental/timestamp/insert_more.sql
+- src/test/resources/org/embulk/input/postgresql/test/expect/incremental/timestamp/setup.sql
+- src/test/resources/org/embulk/input/postgresql/test/expect/incremental/timestamptz/config_1.yml
+- src/test/resources/org/embulk/input/postgresql/test/expect/incremental/timestamptz/config_2.yml
+- src/test/resources/org/embulk/input/postgresql/test/expect/incremental/timestamptz/expected_1.csv
+- src/test/resources/org/embulk/input/postgresql/test/expect/incremental/timestamptz/expected_1.diff
+- src/test/resources/org/embulk/input/postgresql/test/expect/incremental/timestamptz/expected_2.csv
+- src/test/resources/org/embulk/input/postgresql/test/expect/incremental/timestamptz/expected_2.diff
+- src/test/resources/org/embulk/input/postgresql/test/expect/incremental/timestamptz/insert_more.sql
+- src/test/resources/org/embulk/input/postgresql/test/expect/incremental/timestamptz/setup.sql
 homepage: https://github.com/embulk/embulk-input-jdbc
 licenses:
 - Apache 2.0

data/src/test/java/org/embulk/input/postgresql/PostgreSQLInputPluginTest.java

@@ -1,102 +0,0 @@
-package org.embulk.input.postgresql;
-
-import static java.util.Locale.ENGLISH;
-import static org.junit.Assert.assertEquals;
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.sql.Connection;
-import java.sql.DriverManager;
-import java.sql.SQLException;
-import java.util.Arrays;
-
-import org.apache.commons.lang3.StringUtils;
-import org.embulk.input.AbstractJdbcInputPluginTest;
-import org.embulk.input.PostgreSQLInputPlugin;
-import org.embulk.spi.InputPlugin;
-import org.junit.Test;
-
-public class PostgreSQLInputPluginTest extends AbstractJdbcInputPluginTest
-{
-    @Override
-    protected void prepare() throws SQLException
-    {
-        tester.addPlugin(InputPlugin.class, "postgresql", PostgreSQLInputPlugin.class);
-
-        try {
-            // Create User and Database
-            psql(String.format(ENGLISH, "DROP DATABASE IF EXISTS %s;", getDatabase()));
-            psql(String.format(ENGLISH, "DROP USER IF EXISTS %s;", getUser()));
-            psql(String.format(ENGLISH, "CREATE USER %s WITH SUPERUSER PASSWORD '%s';", getUser(), getPassword()));
-            psql(String.format(ENGLISH, "CREATE DATABASE %s WITH OWNER %s;", getDatabase(), getUser()));
-        } catch (IOException e) {
-            System.err.println(e);
-            System.err.println("Warning: cannot prepare a database for testing embulk-input-postgresql.");
-            // 1. install postgresql.
-            // 2. add bin directory to path.
-            // 3. set environment variable PGPASSWORD or write pgpassword in tests.yml
-            return;
-        } catch (InterruptedException e) {
-            throw new RuntimeException(e);
-        }
-
-        enabled = true;
-
-        // Insert Data
-        String sql = "";
-        sql += "DROP TABLE IF EXISTS input_hstore;";
-        sql += "CREATE EXTENSION IF NOT EXISTS hstore WITH SCHEMA public;";
-        sql += "CREATE TABLE input_hstore (c1 hstore);";
-        sql += "INSERT INTO input_hstore (c1) VALUES('\"a\" => \"b\"');";
-        executeSQL(sql);
-    }
-
-    @Test
-    public void testHstoreAsString() throws Exception
-    {
-        if (enabled) {
-            test("/yml/input_hstore.yml");
-            assertEquals(Arrays.asList("c1", "\"\"\"a\"\"=>\"\"b\"\"\""),
-                    read("postgresql-input000.00.csv"));
-        }
-    }
-
-    @Test
-    public void testHstoreAsJson() throws Exception
-    {
-        if (enabled) {
-            test("/yml/input_hstore2.yml");
-            assertEquals(Arrays.asList("c1", "\"{\"\"a\"\":\"\"b\"\"}\""),
-                    read("postgresql-input000.00.csv"));
-        }
-    }
-
-    private void psql(String sql) throws IOException, InterruptedException {
-        ProcessBuilder pb = new ProcessBuilder("psql", "-w", "-c", sql);
-        String pgPassword = (String)getTestConfig("pgpassword", false);
-        if (!StringUtils.isEmpty(pgPassword)) {
-            pb.environment().put("PGPASSWORD", pgPassword);
-        }
-        System.out.println("PSQL: " + pb.command().toString());
-        final Process process = pb.start();
-        final int code = process.waitFor();
-        if (code != 0) {
-            try (BufferedReader reader = new BufferedReader(new InputStreamReader(process.getErrorStream()))) {
-                String line;
-                while ((line = reader.readLine()) != null) {
-                    System.err.println(line);
-                }
-            }
-            throw new IOException(String.format(
-                    "Command finished with non-zero exit code. Exit code is %d.", code));
-        }
-    }
-
-    @Override
-    protected Connection connect() throws SQLException
-    {
-        return DriverManager.getConnection(String.format(ENGLISH, "jdbc:postgresql://%s:%d/%s", getHost(), getPort(), getDatabase()),
-                getUser(), getPassword());
-    }
-}

data/src/test/resources/yml/input_hstore.yml

@@ -1,14 +0,0 @@
-in:
-  type: postgresql
-  host: #host#
-  database: #database#
-  user: #user#
-  password: #password#
-  table: input_hstore
-  select: "*"
-out:
-  type: file
-  path_prefix: postgresql-input
-  file_ext: csv
-  formatter:
-    type: csv

data/src/test/resources/yml/input_hstore2.yml

@@ -1,16 +0,0 @@
-in:
-  type: postgresql
-  host: #host#
-  database: #database#
-  user: #user#
-  password: #password#
-  table: input_hstore
-  select: "*"
-  column_options:
-    c1: {type: json}
-out:
-  type: file
-  path_prefix: postgresql-input
-  file_ext: csv
-  formatter:
-    type: csv