embulk-input-postgresql 0.6.4 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b6b89fe67e85773011e8d1d3f4ae745ecbe13f36
-  data.tar.gz: a2ab692201c8e013f217fc32b5229bad05dc2475
+  metadata.gz: a8a52d399d27705849e5c548b2a4a2e26d3f8db4
+  data.tar.gz: ae9c01752906c2132c4dea6acdd47e49cedae5ae
 SHA512:
-  metadata.gz: c75a5dd2ad7b25a58d902b170e1a853be93e168b0699fa821a80e8068456a104d4aaf90348d7f15800e294e1e0bdcba22a8eda9d50aeb0a5b9a876cdee5a7d80
-  data.tar.gz: 0514090f496d88be229ed4a2299dedb88c745be2028b03a364d45b4332fe377364e71ca15245936b9b6f14fe87c00fbb61b89b7f2f5dd41fa05c85609b0ebe44
+  metadata.gz: 97da962ac8166bfb6929fdd4c375d1e8c24348a4c9ae0fd865435cc68974b7acff222fa5617bcc3ccacc71afe74cea3710da423903290011233dc178e9c6e173
+  data.tar.gz: cdf01ea2f741ceea6948202d56e241cb05430af539bba10574afb3918bcef3329e855c5ee7ca66cf90a56c7392526e69e478b08600c51b979c023edb7b978b22

data/README.md

@@ -29,13 +29,31 @@ PostgreSQL input plugins for Embulk loads records from PostgreSQL.
 - **default_timezone**: If the sql type of a column is `date`/`time`/`datetime` and the embulk type is `string`, column values are formatted int this default_timezone. You can overwrite timezone for each columns using column_options option. (string, default: `UTC`)
 - **column_options**: advanced: a key-value pairs where key is a column name and value is options for the column.
   - **value_type**: embulk get values from database as this value_type. Typically, the value_type determines `getXXX` method of `java.sql.PreparedStatement`.
-  (string, default: depends on the sql type of the column. Available values options are: `long`, `double`, `float`, `decimal`, `boolean`, `string`, `date`, `time`, `timestamp`)
+  (string, default: depends on the sql type of the column. Available values options are: `long`, `double`, `float`, `decimal`, `boolean`, `string`, `json`, `date`, `time`, `timestamp`)
+  See below for `hstore` column.
   - **type**: Column values are converted to this embulk type.
-  Available values options are: `boolean`, `long`, `double`, `string`, `timestamp`).
+  Available values options are: `boolean`, `long`, `double`, `string`, `json`, `timestamp`).
   By default, the embulk type is determined according to the sql type of the column (or value_type if specified).
+  See below for `hstore` column.
   - **timestamp_format**: If the sql type of the column is `date`/`time`/`datetime` and the embulk type is `string`, column values are formatted by this timestamp_format. And if the embulk type is `timestamp`, this timestamp_format may be used in the output plugin. For example, stdout plugin use the timestamp_format, but *csv formatter plugin doesn't use*. (string, default : `%Y-%m-%d` for `date`, `%H:%M:%S` for `time`, `%Y-%m-%d %H:%M:%S` for `timestamp`)
   - **timezone**: If the sql type of the column is `date`/`time`/`datetime` and the embulk type is `string`, column values are formatted in this timezone.
 (string, value of default_timezone option is used by default)
+- **after_select**: if set, this SQL will be executed after the SELECT query in the same transaction.
+
+### hstore column support
+
+By default, `type` of `column_options` for `hstore` column is `string`, and output will be as follows.
+```
+"key1"=>"value1", "key2"=>"value2"
+```
+
+In addition, `json` type is supported for `hstore` column, and output will be as follows.
+```
+{"key1": "value1", "key2": "value2"}
+```
+
+`value_type` is ignored.
+
 
 ## Example
 
@@ -82,6 +100,7 @@ in:
   column_options:
     col1: {type: long}
     col3: {type: string, timestamp_format: "%Y/%m/%d", timezone: "+0900"}
+  after_select: "update my_table set col5 = '1' where col4 != 'a'"
 
 ```
 

data/build.gradle

@@ -3,5 +3,6 @@ dependencies {
 
     compile 'org.postgresql:postgresql:9.4-1205-jdbc41'
 
+    testCompile 'org.embulk:embulk-standards:0.8.8'
     testCompile project(':embulk-input-jdbc').sourceSets.test.output
 }

data/src/main/java/org/embulk/input/PostgreSQLInputPlugin.java

@@ -1,13 +1,20 @@
 package org.embulk.input;
 
+import java.util.Map;
 import java.util.Properties;
 import java.sql.Connection;
 import java.sql.Driver;
 import java.sql.SQLException;
+
 import org.embulk.config.Config;
 import org.embulk.config.ConfigDefault;
 import org.embulk.input.jdbc.AbstractJdbcInputPlugin;
+import org.embulk.input.jdbc.JdbcColumnOption;
+import org.embulk.input.jdbc.getter.ColumnGetterFactory;
 import org.embulk.input.postgresql.PostgreSQLInputConnection;
+import org.embulk.input.postgresql.getter.PostgreSQLColumnGetterFactory;
+import org.embulk.spi.PageBuilder;
+import org.joda.time.DateTimeZone;
 
 public class PostgreSQLInputPlugin
         extends AbstractJdbcInputPlugin
@@ -88,4 +95,10 @@ public class PostgreSQLInputPlugin
             }
         }
     }
+
+    @Override
+    protected ColumnGetterFactory newColumnGetterFactory(PageBuilder pageBuilder, DateTimeZone dateTimeZone)
+    {
+        return new PostgreSQLColumnGetterFactory(pageBuilder, dateTimeZone);
+    }
 }

data/src/main/java/org/embulk/input/postgresql/getter/HstoreColumnGetter.java

@@ -0,0 +1,63 @@
+package org.embulk.input.postgresql.getter;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.embulk.input.jdbc.getter.AbstractColumnGetter;
+import org.embulk.spi.Column;
+import org.embulk.spi.PageBuilder;
+import org.embulk.spi.json.JsonParseException;
+import org.embulk.spi.json.JsonParser;
+import org.embulk.spi.type.Type;
+import org.embulk.spi.type.Types;
+import org.msgpack.value.Value;
+import org.postgresql.util.HStoreConverter;
+
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.Map;
+
+public class HstoreColumnGetter
+        extends AbstractColumnGetter
+{
+    private final JsonParser parser = new JsonParser();
+    private final ObjectMapper mapper = new ObjectMapper();
+
+    private String value;
+
+    public HstoreColumnGetter(PageBuilder to, Type toType)
+    {
+        super(to, toType);
+    }
+
+    @Override
+    protected void fetch(ResultSet from, int fromIndex) throws SQLException
+    {
+        value = from.getString(fromIndex);
+    }
+
+    @Override
+    protected Type getDefaultToType()
+    {
+        return Types.STRING;
+    }
+
+    @Override
+    public void jsonColumn(Column column)
+    {
+        Value v;
+        try {
+            Map map = HStoreConverter.fromString(value);
+            v = parser.parse(mapper.writeValueAsString(map));
+        } catch (JsonProcessingException | JsonParseException e) {
+            super.jsonColumn(column);
+            return;
+        }
+        to.setJson(column, v);
+    }
+
+    @Override
+    public void stringColumn(Column column)
+    {
+        to.setString(column, value);
+    }
+}

data/src/main/java/org/embulk/input/postgresql/getter/PostgreSQLColumnGetterFactory.java

@@ -0,0 +1,36 @@
+package org.embulk.input.postgresql.getter;
+
+import org.embulk.input.jdbc.JdbcColumn;
+import org.embulk.input.jdbc.JdbcColumnOption;
+import org.embulk.input.jdbc.getter.ColumnGetter;
+import org.embulk.input.jdbc.getter.ColumnGetterFactory;
+import org.embulk.spi.PageBuilder;
+import org.joda.time.DateTimeZone;
+
+public class PostgreSQLColumnGetterFactory extends ColumnGetterFactory
+{
+    public PostgreSQLColumnGetterFactory(PageBuilder to, DateTimeZone defaultTimeZone)
+    {
+        super(to, defaultTimeZone);
+    }
+
+    @Override
+    public ColumnGetter newColumnGetter(JdbcColumn column, JdbcColumnOption option)
+    {
+        if (column.getTypeName().equals("hstore")) {
+            return new HstoreColumnGetter(to, getToType(option));
+        } else {
+            return super.newColumnGetter(column, option);
+        }
+    }
+
+    @Override
+    protected String sqlTypeToValueType(JdbcColumn column, int sqlType)
+    {
+        if (column.getTypeName().equals("json") || column.getTypeName().equals("jsonb")) {
+            return "json";
+        } else {
+            return super.sqlTypeToValueType(column, sqlType);
+        }
+    }
+}

data/src/test/java/org/embulk/input/postgresql/PostgreSQLInputPluginTest.java

@@ -0,0 +1,126 @@
+package org.embulk.input.postgresql;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.net.URISyntaxException;
+import java.nio.charset.Charset;
+import java.nio.file.FileSystem;
+import java.nio.file.FileSystems;
+import java.nio.file.Files;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.Statement;
+import java.util.Arrays;
+import java.util.List;
+
+import org.embulk.input.EmbulkPluginTester;
+import org.embulk.input.PostgreSQLInputPlugin;
+import org.embulk.spi.InputPlugin;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class PostgreSQLInputPluginTest
+{
+    private static final String DATABASE = "test_db";
+    private static final String USER = "test_user";
+    private static final String PASSWORD = "test_pw";
+    private static final String URL = "jdbc:postgresql://localhost:5432/" + DATABASE;
+
+    private static boolean prepared = false;
+    private static EmbulkPluginTester tester = new EmbulkPluginTester(InputPlugin.class, "postgresql", PostgreSQLInputPlugin.class);
+
+    @BeforeClass
+    public static void prepare() throws Exception
+    {
+        try {
+            // Create User and Database
+            psql(String.format("DROP DATABASE IF EXISTS %s;", DATABASE));
+            psql(String.format("DROP USER IF EXISTS %s;", USER));
+            psql(String.format("CREATE USER %s WITH SUPERUSER PASSWORD '%s';", USER, PASSWORD));
+            psql(String.format("CREATE DATABASE %s WITH OWNER %s;", DATABASE, USER));
+        } catch (IOException e) {
+            System.err.println(e);
+            System.err.println("Warning: cannot prepare a database for testing embulk-input-postgresql.");
+            // 1. install postgresql.
+            // 2. add bin directory to path.
+            // 3. set environment variable PGPASSWORD
+            return;
+        }
+
+        prepared = true;
+
+        // Insert Data
+        try(Connection connection = DriverManager.getConnection(URL, USER, PASSWORD)) {
+            try (Statement statement = connection.createStatement()) {
+                String sql = "";
+                sql += "DROP TABLE IF EXISTS input_hstore;";
+                sql += "CREATE EXTENSION IF NOT EXISTS hstore WITH SCHEMA public;";
+                sql += "CREATE TABLE input_hstore (c1 hstore);";
+                sql += "INSERT INTO input_hstore (c1) VALUES('\"a\" => \"b\"');";
+                statement.execute(sql);
+            }
+        }
+    }
+
+    @AfterClass
+    public static void dispose()
+    {
+        tester.destroy();
+    }
+
+    @Test
+    public void testHstoreAsString() throws Exception
+    {
+        if (prepared) {
+            tester.run(convertPath("/yml/input_hstore.yml"));
+            assertEquals(Arrays.asList("c1", "\"\"\"a\"\"=>\"\"b\"\"\""),
+                    read("postgresql-input000.00.csv"));
+        }
+    }
+
+    @Test
+    public void testHstoreAsJson() throws Exception
+    {
+        if (prepared) {
+            tester.run(convertPath("/yml/input_hstore2.yml"));
+            assertEquals(Arrays.asList("c1", "\"{\"\"a\"\":\"\"b\"\"}\""),
+                    read("postgresql-input000.00.csv"));
+        }
+    }
+
+    private List<String> read(String path) throws IOException
+    {
+        FileSystem fs = FileSystems.getDefault();
+        return Files.readAllLines(fs.getPath(path), Charset.defaultCharset());
+    }
+
+    private String convertPath(String name) throws URISyntaxException
+    {
+        if (getClass().getResource(name) == null) {
+            return name;
+        }
+        return new File(getClass().getResource(name).toURI()).getAbsolutePath();
+    }
+
+    private static void psql(String sql) throws IOException, InterruptedException {
+        ProcessBuilder pb = new ProcessBuilder("psql", "-w", "-c", sql);
+        System.out.println("PSQL: " + pb.command().toString());
+        final Process process = pb.start();
+        final int code = process.waitFor();
+        if (code != 0) {
+            try (BufferedReader reader = new BufferedReader(new InputStreamReader(process.getErrorStream()))) {
+                String line;
+                while ((line = reader.readLine()) != null) {
+                    System.err.println(line);
+                }
+            }
+            throw new IOException(String.format(
+                    "Command finished with non-zero exit code. Exit code is %d.", code));
+        }
+    }
+}

data/src/test/resources/yml/input_hstore.yml

@@ -0,0 +1,14 @@
+in:
+  type: postgresql
+  host: localhost
+  database: test_db
+  user: test_user
+  password: test_pw
+  table: input_hstore
+  select: "*"
+out:
+  type: file
+  path_prefix: postgresql-input
+  file_ext: csv
+  formatter:
+    type: csv

data/src/test/resources/yml/input_hstore2.yml

@@ -0,0 +1,16 @@
+in:
+  type: postgresql
+  host: localhost
+  database: test_db
+  user: test_user
+  password: test_pw
+  table: input_hstore
+  select: "*"
+  column_options:
+    c1: {type: json}
+out:
+  type: file
+  path_prefix: postgresql-input
+  file_ext: csv
+  formatter:
+    type: csv

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-postgresql
 version: !ruby/object:Gem::Version
-  version: 0.6.4
+  version: 0.7.0
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-01-15 00:00:00.000000000 Z
+date: 2016-03-29 00:00:00.000000000 Z
 dependencies: []
 description: Selects records from a table.
 email:
@@ -22,8 +22,13 @@ files:
 - lib/embulk/input/postgresql.rb
 - src/main/java/org/embulk/input/PostgreSQLInputPlugin.java
 - src/main/java/org/embulk/input/postgresql/PostgreSQLInputConnection.java
-- classpath/embulk-input-jdbc-0.6.4.jar
-- classpath/embulk-input-postgresql-0.6.4.jar
+- src/main/java/org/embulk/input/postgresql/getter/HstoreColumnGetter.java
+- src/main/java/org/embulk/input/postgresql/getter/PostgreSQLColumnGetterFactory.java
+- src/test/java/org/embulk/input/postgresql/PostgreSQLInputPluginTest.java
+- src/test/resources/yml/input_hstore.yml
+- src/test/resources/yml/input_hstore2.yml
+- classpath/embulk-input-jdbc-0.7.0.jar
+- classpath/embulk-input-postgresql-0.7.0.jar
 - classpath/postgresql-9.4-1205-jdbc41.jar
 homepage: https://github.com/embulk/embulk-input-jdbc
 licenses: