embulk-input-postgresql 0.8.5 → 0.8.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 06c0715712d158b155fdec049b1d902ad2c73286
-  data.tar.gz: 7d9c2e7114cc14883303756a2810111c151237c1
+  metadata.gz: f267963fed40c4b706f5c345b06fb29d2d4c04a7
+  data.tar.gz: 257c86658208b973aae246019f43a2b48ce4010a
 SHA512:
-  metadata.gz: 2d1012d143bf14b25ef74805e7784da11a6dd1b39e6da8fe4c17108aba4c204799d1326716fdfd3a161f6f022db073d71692c174a8f3fbcfa3bd3b8fedacdade
-  data.tar.gz: 02c1db91904b99b8305c9bd9e057cfd6b498daadc652fac00b2edfd343908e2fbc8684bf33b953627f7659c486b52343f8c0d883cb8cc0e721c74051ed299a61
+  metadata.gz: 15d1b0963f75aa994d747e28f2b33b1b3b6e1fdeb5060063237bd09b5506a22fd104e5472c7a8c07ed8118739e80b4d8d777ce15ed1688d628b03645733f5576
+  data.tar.gz: 7466b193d8105c41f5acc2d07b8f84bf10b956b64be7fafdabee1100a36b9b2a5932fde36f9a68edbcfc408a7d92a317a6050dcafc1dd25ac78d6ccce28cf9ac

data/README.md

@@ -35,7 +35,7 @@ PostgreSQL input plugin for Embulk loads records from PostgreSQL.
 - **default_timezone**: If the sql type of a column is `date`/`time`/`datetime` and the embulk type is `string`, column values are formatted int this default_timezone. You can overwrite timezone for each columns using column_options option. (string, default: `UTC`)
 - **column_options**: advanced: a key-value pairs where key is a column name and value is options for the column.
   - **value_type**: embulk get values from database as this value_type. Typically, the value_type determines `getXXX` method of `java.sql.PreparedStatement`.
-  (string, default: depends on the sql type of the column. Available values options are: `long`, `double`, `float`, `decimal`, `boolean`, `string`, `json`, `date`, `time`, `timestamp`)
+  (string, default: depends on the sql type of the column. Available values options are: `long`, `double`, `float`, `decimal`, `boolean`, `string`, `json`, `date`, `time`, `timestamp`, `array`)
   See below for `hstore` column.
   - **type**: Column values are converted to this embulk type.
   Available values options are: `boolean`, `long`, `double`, `string`, `json`, `timestamp`).
@@ -60,6 +60,24 @@ In addition, `json` type is supported for `hstore` column, and output will be as
 
 `value_type` is ignored.
 
+### Arrays column support
+
+PostgreSQL allows columns of a table to be defined as variable-length multidimensional arrays and this plugin supports converting its value into `string` or `json`.
+
+By default, `type` of `column_options` for `array` column is `string`, and output will be similar to what `psql` produces:
+```
+    {1000,2000,3000,4000}, {{red,green},{blue,cyan}}
+    {5000,6000,7000,8000}, {{yellow,magenta},{purple,"light,dark"}}
+```
+
+Output of `json` type will be as follow:
+```
+[1000,2000,3000,4000],[["red","green"],["blue","cyan"]]
+[5000,6000,7000,8000],[["yellow","magenta"],["purple"","light,dark"]]
+```
+However, the support for `json` type has the following limitations:
+- Postgres server version must be 8.3.0 and above
+- The value type of array element must be number, bool, or text, e.g. bool[], integer[], text[][], bigint[][][]...
 
 ### Incremental loading
 
@@ -82,7 +100,7 @@ At the next execution, when `last_record: ` is also set, this plugin generates a
 SELECT * FROM (
   ...original query is here...
 )
-WHERE created_at > '2017-01-01 00:32:12' OR (created_at = '2017-01-01 00:32:12' AND id > 5291)
+WHERE updated_at > '2017-01-01 00:32:12' OR (updated_at = '2017-01-01 00:32:12' AND id > 5291)
 ORDER BY updated_at, id
 ```
 

data/src/main/java/org/embulk/input/postgresql/getter/ArrayColumnGetter.java

@@ -0,0 +1,104 @@
+package org.embulk.input.postgresql.getter;
+
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.NullNode;
+import org.embulk.input.jdbc.getter.AbstractColumnGetter;
+import org.embulk.spi.Column;
+import org.embulk.spi.PageBuilder;
+import org.embulk.spi.json.JsonParseException;
+import org.embulk.spi.json.JsonParser;
+import org.embulk.spi.type.Type;
+import org.msgpack.value.Value;
+
+import java.sql.Array;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+
+import java.sql.Types;
+
+public class ArrayColumnGetter
+        extends AbstractColumnGetter
+{
+    protected Array value;
+
+    protected final JsonParser jsonParser = new JsonParser();
+
+    public ArrayColumnGetter(PageBuilder to, Type toType)
+    {
+        super(to, toType);
+    }
+
+    @Override
+    protected void fetch(ResultSet from, int fromIndex) throws SQLException
+    {
+        value = from.getArray(fromIndex);
+    }
+
+    private ArrayNode buildJsonArray(Object[] elements)
+            throws SQLException
+    {
+        ArrayNode arrayNode = jsonNodeFactory.arrayNode();
+        for (Object v : elements) {
+            if (v == null) {
+                arrayNode.add(NullNode.getInstance());
+                continue;
+            }
+            if (v.getClass().isArray()) {
+                arrayNode.add(buildJsonArray((Object[]) v));
+            }
+            else {
+                switch (value.getBaseType()) {
+                    case Types.TINYINT:
+                    case Types.SMALLINT:
+                    case Types.INTEGER:
+                    case Types.BIGINT:
+                        arrayNode.add(jsonNodeFactory.numberNode(((Number) v).longValue()));
+                        break;
+                    case Types.FLOAT:
+                    case Types.REAL:
+                    case Types.DOUBLE:
+                        arrayNode.add(jsonNodeFactory.numberNode(((Number) v).doubleValue()));
+                        break;
+                    case Types.BOOLEAN:
+                    case Types.BIT:  // JDBC BIT is boolean, unlike SQL-92
+                        arrayNode.add(jsonNodeFactory.booleanNode((Boolean) v));
+                        break;
+                    case Types.CHAR:
+                    case Types.VARCHAR:
+                    case Types.LONGVARCHAR:
+                    case Types.CLOB:
+                    case Types.NCHAR:
+                    case Types.NVARCHAR:
+                    case Types.LONGNVARCHAR:
+                        arrayNode.add(jsonNodeFactory.textNode((String) v));
+                        break;
+                }
+            }
+        }
+        return arrayNode;
+    }
+
+    @Override
+    protected Type getDefaultToType()
+    {
+        return org.embulk.spi.type.Types.STRING;
+    }
+
+    @Override
+    public void jsonColumn(Column column)
+    {
+        try {
+            Value v = jsonParser.parse(buildJsonArray((Object[]) value.getArray()).toString());
+            to.setJson(column, v);
+        }
+        catch (JsonParseException | SQLException | ClassCastException e) {
+            super.jsonColumn(column);
+        }
+    }
+
+    @Override
+    public void stringColumn(Column column)
+    {
+        to.setString(column, value.toString());
+    }
+}

data/src/main/java/org/embulk/input/postgresql/getter/PostgreSQLColumnGetterFactory.java

@@ -27,6 +27,10 @@ public class PostgreSQLColumnGetterFactory extends ColumnGetterFactory
             return new HstoreToJsonColumnGetter(to, Types.JSON);
         }
 
+        if (column.getSqlType() == java.sql.Types.ARRAY) {
+            return new ArrayColumnGetter(to, getToType(option));
+        }
+
         ColumnGetter getter = super.newColumnGetter(con, task, column, option);
 
         // incremental loading wrapper
@@ -48,7 +52,8 @@ public class PostgreSQLColumnGetterFactory extends ColumnGetterFactory
         case "jsonb":
             return "json";
         case "hstore":
-            // hstore is converted to string by default
+        case "array":
+            // array & hstore is converted to string by default
             return "string";
         default:
             return super.sqlTypeToValueType(column, sqlType);

data/src/test/java/org/embulk/input/postgresql/ArrayTest.java

@@ -0,0 +1,74 @@
+package org.embulk.input.postgresql;
+
+import org.embulk.config.ConfigSource;
+import org.embulk.input.PostgreSQLInputPlugin;
+import org.embulk.spi.InputPlugin;
+import org.embulk.test.EmbulkTests;
+import org.embulk.test.TestingEmbulk;
+import org.embulk.test.TestingEmbulk.RunResult;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+
+import java.nio.file.Path;
+
+import static org.embulk.input.postgresql.PostgreSQLTests.execute;
+import static org.embulk.test.EmbulkTests.readSortedFile;
+import static org.hamcrest.Matchers.is;
+import static org.junit.Assert.assertThat;
+
+public class ArrayTest
+{
+    private static final String BASIC_RESOURCE_PATH = "org/embulk/input/postgresql/test/expect/array/";
+
+    private static ConfigSource loadYamlResource(TestingEmbulk embulk, String fileName)
+    {
+        return embulk.loadYamlResource(BASIC_RESOURCE_PATH + fileName);
+    }
+
+    private static String readResource(String fileName)
+    {
+        return EmbulkTests.readResource(BASIC_RESOURCE_PATH + fileName);
+    }
+
+    @Rule
+    public TestingEmbulk embulk = TestingEmbulk.builder()
+        .registerPlugin(InputPlugin.class, "postgresql", PostgreSQLInputPlugin.class)
+        .build();
+
+    private ConfigSource baseConfig;
+
+    @Before
+    public void setup()
+    {
+        baseConfig = PostgreSQLTests.baseConfig();
+    }
+
+    @Test
+    public void loadAsStringByDefault() throws Exception
+    {
+        execute(readResource("setup.sql"));
+
+        Path out1 = embulk.createTempFile("csv");
+        embulk.runInput(
+                baseConfig.merge(loadYamlResource(embulk, "as_string.yml")),
+                out1);
+        assertThat(
+                readSortedFile(out1),
+                is(readResource("expected_string.csv")));
+    }
+
+    @Test
+    public void loadAsJson() throws Exception
+    {
+        execute(readResource("setup.sql"));
+
+        Path out1 = embulk.createTempFile("csv");
+        embulk.runInput(
+                baseConfig.merge(loadYamlResource(embulk, "as_json.yml")),
+                out1);
+        assertThat(
+                readSortedFile(out1),
+                is(readResource("expected_json.csv")));
+    }
+}

data/src/test/java/org/embulk/input/postgresql/PostgreSQLTests.java

@@ -4,6 +4,8 @@ import org.embulk.test.EmbulkTests;
 import com.google.common.base.Throwables;
 import com.google.common.io.ByteStreams;
 import java.io.IOException;
+import jnr.ffi.Platform;
+import jnr.ffi.Platform.OS;
 import org.embulk.config.ConfigSource;
 import static java.util.Locale.ENGLISH;
 
@@ -17,7 +19,7 @@ public class PostgreSQLTests
     public static void execute(String sql)
     {
         ConfigSource config = baseConfig();
-        ProcessBuilder pb = new ProcessBuilder("psql", "-w", "--set", "ON_ERROR_STOP=1", "-c", sql);
+        ProcessBuilder pb = new ProcessBuilder("psql", "-w", "--set", "ON_ERROR_STOP=1", "-c", convert(sql));
         pb.environment().put("PGUSER", config.get(String.class, "user"));
         pb.environment().put("PGPASSWORD", config.get(String.class, "password"));
         pb.environment().put("PGDATABASE", config.get(String.class, "database"));
@@ -36,4 +38,13 @@ public class PostgreSQLTests
                         "Command finished with non-zero exit code. Exit code is %d.", code));
         }
     }
+
+    private static String convert(String sql)
+    {
+        if (Platform.getNativePlatform().getOS().equals(OS.WINDOWS)) {
+            // '"' should be '\"' is Windows
+            return sql.replace("\"", "\\\"");
+        }
+        return sql;
+    }
 }

data/src/test/resources/org/embulk/input/postgresql/test/expect/array/as_json.yml

@@ -0,0 +1,5 @@
+table: input_array
+column_options:
+  c1: {type: json}
+  c2: {type: json}
+  c3: {type: json}

data/src/test/resources/org/embulk/input/postgresql/test/expect/array/as_string.yml

@@ -0,0 +1 @@
+table: input_array

data/src/test/resources/org/embulk/input/postgresql/test/expect/array/expected_json.csv

@@ -0,0 +1,2 @@
+"[1000,2000,3000,4000]","[[""red"",""green""],[""blue"",""cyan""]]",[[[true]]]
+"[5000,6000,7000,8000]","[[""yellow"",""magenta""],[""purple"",""light,dark""]]","[[[true,true],[false,false]],[[true,false],[false,true]]]"

data/src/test/resources/org/embulk/input/postgresql/test/expect/array/expected_string.csv

@@ -0,0 +1,2 @@
+"{1000,2000,3000,4000}","{{red,green},{blue,cyan}}",{{{t}}}
+"{5000,6000,7000,8000}","{{yellow,magenta},{purple,""light,dark""}}","{{{t,t},{f,f}},{{t,f},{f,t}}}"

data/src/test/resources/org/embulk/input/postgresql/test/expect/array/setup.sql

@@ -0,0 +1,11 @@
+drop table if exists input_array;
+
+create table input_array (
+    c1 integer[],
+    c2 text[][],
+    c3 bool[][][]
+);
+
+insert into input_array (c1, c2, c3) values ('{1000, 2000, 3000, 4000}', '{{"red", "green"}, {"blue", "cyan"}}', '{{{true}}}');
+
+insert into input_array (c1, c2, c3) values ('{5000, 6000, 7000, 8000}', '{{"yellow", "magenta"}, {"purple", "light,dark"}}', '{{{t,t},{f,f}},{{t,f},{f,t}}}');

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-postgresql
 version: !ruby/object:Gem::Version
-  version: 0.8.5
+  version: 0.8.6
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-07-31 00:00:00.000000000 Z
+date: 2017-11-24 00:00:00.000000000 Z
 dependencies: []
 description: Selects records from a table.
 email:
@@ -19,17 +19,24 @@ extra_rdoc_files: []
 files:
 - README.md
 - build.gradle
-- classpath/embulk-input-jdbc-0.8.5.jar
-- classpath/embulk-input-postgresql-0.8.5.jar
+- classpath/embulk-input-jdbc-0.8.6.jar
+- classpath/embulk-input-postgresql-0.8.6.jar
 - default_jdbc_driver/postgresql-9.4-1205-jdbc41.jar
 - lib/embulk/input/postgresql.rb
 - src/main/java/org/embulk/input/PostgreSQLInputPlugin.java
 - src/main/java/org/embulk/input/postgresql/PostgreSQLInputConnection.java
+- src/main/java/org/embulk/input/postgresql/getter/ArrayColumnGetter.java
 - src/main/java/org/embulk/input/postgresql/getter/HstoreToJsonColumnGetter.java
 - src/main/java/org/embulk/input/postgresql/getter/PostgreSQLColumnGetterFactory.java
+- src/test/java/org/embulk/input/postgresql/ArrayTest.java
 - src/test/java/org/embulk/input/postgresql/HstoreTest.java
 - src/test/java/org/embulk/input/postgresql/IncrementalTest.java
 - src/test/java/org/embulk/input/postgresql/PostgreSQLTests.java
+- src/test/resources/org/embulk/input/postgresql/test/expect/array/as_json.yml
+- src/test/resources/org/embulk/input/postgresql/test/expect/array/as_string.yml
+- src/test/resources/org/embulk/input/postgresql/test/expect/array/expected_json.csv
+- src/test/resources/org/embulk/input/postgresql/test/expect/array/expected_string.csv
+- src/test/resources/org/embulk/input/postgresql/test/expect/array/setup.sql
 - src/test/resources/org/embulk/input/postgresql/test/expect/hstore/as_json.yml
 - src/test/resources/org/embulk/input/postgresql/test/expect/hstore/as_string.yml
 - src/test/resources/org/embulk/input/postgresql/test/expect/hstore/expected_json.csv