embulk-output-postgresql 0.8.2 → 0.8.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c40968f0a5deb08818c904871fb92ed77a6be595
-  data.tar.gz: 583509046b2ba2c58b1a0f000c863e005fdf26d4
+  metadata.gz: 6905d4be0d280edee08d97a7a9a5fe4e93f6f93e
+  data.tar.gz: e25fcb5a9ba650b1ccbc636c8f7c432baa4b0c85
 SHA512:
-  metadata.gz: de8c4ea59948fe62389254cbad849802d6250080edd5024a51db8a53d4f4111c8ac7668997d9d7e9a1892f081fc73de38a5283590cccffcbb84cf805547c9e8b
-  data.tar.gz: 77879f4ab4996d2d2a15fde031f80d3ac6d6ca09cb4c743c2a92db4869d0fd94bbb9ee1a8692a4f042782a510abdeb3401ec40b4afa28a611a755c9f11734d2c
+  metadata.gz: a788977861a3fd4c194b8969795f4934f8c1b9bfa91d9ba9ca8860674aa147c9c8c358846da5d1a51ca9db877f2b84000161174df97027eec036ee3e46d3c8f8
+  data.tar.gz: 9f96b3b66f92bcdaa4202dca90ef537034527e2eaf7c95eed5703cadba09c0070fd4db3a36bd05c8f7c1454b4696f99ee844ae93e8322b577834ebfac781c597

data/README.md

@@ -21,8 +21,9 @@ PostgreSQL output plugin for Embulk loads records to PostgreSQL.
 - **table**: destination table name (string, required)
 - **create_table_constraint**: table constraint added to `CREATE TABLE` statement, like `CREATE TABLE <table_name> (<column1> <type1>, <column2> <type2>, ..., <create_table_constraint>) <create_table_option>`.
 - **create_table_option**: table option added to `CREATE TABLE` statement, like `CREATE TABLE <table_name> (<column1> <type1>, <column2> <type2>, ..., <create_table_constraint>) <create_table_option>`.
+- **transaction_isolation**: transaction isolation level for each connection ("read_uncommitted", "read_committed", "repeatable_read" or "serializable"). if not specified, database default value will be used.
 - **options**: extra connection properties (hash, default: {})
-- **retry_limit**: max retry count for database operations (integer, default: 12)
+- **retry_limit**: max retry count for database operations (integer, default: 12). When intermediate table to create already created by another process, this plugin will retry with another table name to avoid collision.
 - **retry_wait**: initial retry wait time in milliseconds (integer, default: 1000 (1 second))
 - **max_retry_wait**: upper limit of retry wait, which will be doubled at every retry (integer, default: 1800000 (30 minutes))
 - **mode**: "insert", "insert_direct", "truncate_insert", "replace", "merge" or "merge_direct". See below. (string, required)

data/build.gradle

@@ -4,5 +4,6 @@ dependencies {
     compile 'org.postgresql:postgresql:9.4-1205-jdbc41'
     defaultJdbcDriver 'org.postgresql:postgresql:9.4-1205-jdbc41'
 
+    testCompile 'org.embulk:embulk-standards:0.8.22'
     testCompile project(':embulk-output-jdbc').sourceSets.test.output
 }

data/src/main/java/org/embulk/output/PostgreSQLOutputPlugin.java

@@ -77,7 +77,7 @@ public class PostgreSQLOutputPlugin
     }
 
     @Override
-    protected PostgreSQLOutputConnector getConnector(PluginTask task, boolean retryableMetadataOperation)
+    protected JdbcOutputConnector getConnector(PluginTask task, boolean retryableMetadataOperation)
     {
         PostgreSQLPluginTask t = (PostgreSQLPluginTask) task;
 
@@ -114,7 +114,7 @@ public class PostgreSQLOutputPlugin
         props.setProperty("password", t.getPassword());
         logConnectionProperties(url, props);
 
-        return new PostgreSQLOutputConnector(url, props, t.getSchema());
+        return new PostgreSQLOutputConnector(url, props, t.getSchema(), t.getTransactionIsolation());
     }
 
     @Override

data/src/main/java/org/embulk/output/postgresql/AbstractPostgreSQLCopyBatchInsert.java

@@ -9,7 +9,6 @@ import java.io.OutputStreamWriter;
 import java.io.IOException;
 import java.nio.charset.Charset;
 import java.math.BigDecimal;
-import java.sql.SQLException;
 import org.embulk.spi.time.Timestamp;
 import org.embulk.output.jdbc.BatchInsert;
 
@@ -71,14 +70,6 @@ public abstract class AbstractPostgreSQLCopyBatchInsert
         }
     }
 
-    public void finish() throws IOException, SQLException
-    {
-        closeCurrentFile();  // this is necessary to make getBatchWeight() work
-        if (getBatchWeight() != 0) {
-            flush();
-        }
-    }
-
     public void add() throws IOException
     {
         writer.write(newLineString);

data/src/main/java/org/embulk/output/postgresql/PostgreSQLCopyBatchInsert.java

@@ -5,6 +5,7 @@ import java.io.FileInputStream;
 import java.io.IOException;
 import java.sql.SQLException;
 
+import org.embulk.output.jdbc.JdbcOutputConnector;
 import org.embulk.output.jdbc.JdbcSchema;
 import org.embulk.output.jdbc.TableIdentifier;
 import org.embulk.spi.Exec;
@@ -15,14 +16,14 @@ public class PostgreSQLCopyBatchInsert
         extends AbstractPostgreSQLCopyBatchInsert
 {
     private final Logger logger = Exec.getLogger(PostgreSQLCopyBatchInsert.class);
-    private final PostgreSQLOutputConnector connector;
+    private final JdbcOutputConnector connector;
 
     private PostgreSQLOutputConnection connection = null;
     private CopyManager copyManager = null;
     private String copySql = null;
     private long totalRows;
 
-    public PostgreSQLCopyBatchInsert(PostgreSQLOutputConnector connector) throws IOException, SQLException
+    public PostgreSQLCopyBatchInsert(JdbcOutputConnector connector) throws IOException, SQLException
     {
         super();
         this.connector = connector;
@@ -31,7 +32,7 @@ public class PostgreSQLCopyBatchInsert
     @Override
     public void prepare(TableIdentifier loadTable, JdbcSchema insertSchema) throws SQLException
     {
-        this.connection = connector.connect(true);
+        this.connection = (PostgreSQLOutputConnection)connector.connect(true);
         this.copySql = connection.buildCopySql(loadTable, insertSchema);
         this.copyManager = connection.newCopyManager();
         logger.info("Copy SQL: "+copySql);
@@ -41,6 +42,7 @@ public class PostgreSQLCopyBatchInsert
     public void flush() throws IOException, SQLException
     {
         File file = closeCurrentFile();  // flush buffered data in writer
+        if (getBatchWeight() == 0) return;
 
         logger.info(String.format("Loading %,d rows (%,d bytes)", batchRows, file.length()));
         long startTime = System.currentTimeMillis();
@@ -61,6 +63,11 @@ public class PostgreSQLCopyBatchInsert
         file.delete();
     }
 
+    @Override
+    public void finish() throws IOException ,SQLException
+    {
+    }
+
     @Override
     public void close() throws IOException, SQLException
     {

data/src/main/java/org/embulk/output/postgresql/PostgreSQLOutputConnection.java

@@ -18,11 +18,10 @@ public class PostgreSQLOutputConnection
 {
     private static final int MAX_NUMERIC_PRECISION = 1000;
 
-    public PostgreSQLOutputConnection(Connection connection, String schemaName, boolean autoCommit)
+    public PostgreSQLOutputConnection(Connection connection, String schemaName)
             throws SQLException
     {
         super(connection, schemaName);
-        connection.setAutoCommit(autoCommit);
     }
 
     public String buildCopySql(TableIdentifier toTable, JdbcSchema toTableSchema)

data/src/main/java/org/embulk/output/postgresql/PostgreSQLOutputConnector.java

@@ -5,28 +5,34 @@ import java.sql.Connection;
 import java.sql.DriverManager;
 import java.sql.SQLException;
 
-import org.embulk.output.jdbc.JdbcOutputConnector;
+import org.embulk.output.jdbc.JdbcOutputConnection;
+import org.embulk.output.jdbc.AbstractJdbcOutputConnector;
+import org.embulk.output.jdbc.TransactionIsolation;
+
+import com.google.common.base.Optional;
 
 public class PostgreSQLOutputConnector
-        implements JdbcOutputConnector
+        extends AbstractJdbcOutputConnector
 {
     private final String url;
     private final Properties properties;
     private final String schemaName;
 
-    public PostgreSQLOutputConnector(String url, Properties properties, String schemaName)
+    public PostgreSQLOutputConnector(String url, Properties properties, String schemaName,
+            Optional<TransactionIsolation> transactionIsolation)
     {
+        super(transactionIsolation);
         this.url = url;
         this.properties = properties;
         this.schemaName = schemaName;
     }
 
     @Override
-    public PostgreSQLOutputConnection connect(boolean autoCommit) throws SQLException
+    protected JdbcOutputConnection connect() throws SQLException
     {
         Connection c = DriverManager.getConnection(url, properties);
         try {
-            PostgreSQLOutputConnection con = new PostgreSQLOutputConnection(c, schemaName, autoCommit);
+            PostgreSQLOutputConnection con = new PostgreSQLOutputConnection(c, schemaName);
             c = null;
             return con;
         } finally {

data/src/test/java/org/embulk/output/postgresql/BasicTest.java

@@ -0,0 +1,103 @@
+package org.embulk.output.postgresql;
+
+import static org.embulk.output.postgresql.PostgreSQLTests.execute;
+import static org.embulk.output.postgresql.PostgreSQLTests.selectRecords;
+import static org.hamcrest.Matchers.is;
+import static org.junit.Assert.assertThat;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.nio.file.FileSystems;
+import java.nio.file.Path;
+
+import org.embulk.config.ConfigDiff;
+import org.embulk.config.ConfigSource;
+import org.embulk.output.PostgreSQLOutputPlugin;
+import org.embulk.spi.OutputPlugin;
+import org.embulk.test.EmbulkTests;
+import org.embulk.test.TestingEmbulk;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+
+import com.google.common.io.Resources;
+
+public class BasicTest
+{
+    private static final String BASIC_RESOURCE_PATH = "org/embulk/output/postgresql/test/expect/basic/";
+
+    private static ConfigSource loadYamlResource(TestingEmbulk embulk, String fileName)
+    {
+        return embulk.loadYamlResource(BASIC_RESOURCE_PATH + fileName);
+    }
+
+    private static String readResource(String fileName)
+    {
+        return EmbulkTests.readResource(BASIC_RESOURCE_PATH + fileName);
+    }
+
+    @Rule
+    public TestingEmbulk embulk = TestingEmbulk.builder()
+            .registerPlugin(OutputPlugin.class, "postgresql", PostgreSQLOutputPlugin.class)
+            .build();
+
+    private ConfigSource baseConfig;
+
+    @Before
+    public void setup() throws IOException
+    {
+        baseConfig = PostgreSQLTests.baseConfig();
+        execute(readResource("setup.sql")); // setup rows
+
+        if (System.getProperty("path.separator").equals(";")) {
+            // for Windows (because encoding will be set to UTF8 when executed by Eclipse)
+            System.setProperty("file.encoding", "MS932");
+        }
+    }
+
+    @Test
+    public void testNumber() throws Exception
+    {
+        Path in1 = toPath("test_number.csv");
+        TestingEmbulk.RunResult result1 = embulk.runOutput(baseConfig.merge(loadYamlResource(embulk, "test_number.yml")), in1);
+        assertThat(selectRecords(embulk, "test_number"), is(readResource("test_number_expected.csv")));
+        //assertThat(result1.getConfigDiff(), is((ConfigDiff) loadYamlResource(embulk, "test_expected.diff")));
+    }
+
+    @Test
+    public void testString() throws Exception
+    {
+        Path in1 = toPath("test_string.csv");
+        TestingEmbulk.RunResult result1 = embulk.runOutput(baseConfig.merge(loadYamlResource(embulk, "test_string.yml")), in1);
+        assertThat(selectRecords(embulk, "test_string"), is(readResource("test_string_expected.csv")));
+        //assertThat(result1.getConfigDiff(), is((ConfigDiff) loadYamlResource(embulk, "test_expected.diff")));
+    }
+
+    @Test
+    public void testTimestamp() throws Exception
+    {
+        Path in1 = toPath("test_timestamp.csv");
+        TestingEmbulk.RunResult result1 = embulk.runOutput(baseConfig.merge(loadYamlResource(embulk, "test_timestamp.yml")), in1);
+        assertThat(selectRecords(embulk, "test_timestamp"), is(readResource("test_timestamp_expected.csv")));
+        //assertThat(result1.getConfigDiff(), is((ConfigDiff) loadYamlResource(embulk, "test_expected.diff")));
+    }
+
+    @Test
+    public void testJson() throws Exception
+    {
+        Path in1 = toPath("test_json.csv");
+        TestingEmbulk.RunResult result1 = embulk.runOutput(baseConfig.merge(loadYamlResource(embulk, "test_json.yml")), in1);
+        assertThat(selectRecords(embulk, "test_json"), is(readResource("test_json_expected.csv")));
+        //assertThat(result1.getConfigDiff(), is((ConfigDiff) loadYamlResource(embulk, "test_expected.diff")));
+    }
+
+
+    private Path toPath(String fileName) throws URISyntaxException
+    {
+        URL url = Resources.getResource(BASIC_RESOURCE_PATH + fileName);
+        return FileSystems.getDefault().getPath(new File(url.toURI()).getAbsolutePath());
+    }
+
+}

data/src/test/java/org/embulk/output/postgresql/PostgreSQLTests.java

@@ -0,0 +1,68 @@
+package org.embulk.output.postgresql;
+
+import org.embulk.test.EmbulkTests;
+import org.embulk.test.TestingEmbulk;
+
+import com.google.common.base.Throwables;
+import com.google.common.io.ByteStreams;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+import jnr.ffi.Platform;
+import jnr.ffi.Platform.OS;
+
+import org.embulk.config.ConfigSource;
+
+import static java.util.Locale.ENGLISH;
+import static org.embulk.test.EmbulkTests.readSortedFile;
+
+public class PostgreSQLTests
+{
+    public static ConfigSource baseConfig()
+    {
+        return EmbulkTests.config("EMBULK_OUTPUT_POSTGRESQL_TEST_CONFIG");
+    }
+
+    public static void execute(String sql)
+    {
+        System.out.println(sql);
+        ConfigSource config = baseConfig();
+        ProcessBuilder pb = new ProcessBuilder("psql", "-w", "--set", "ON_ERROR_STOP=1", "-c", sql);
+        pb.environment().put("PGUSER", config.get(String.class, "user"));
+        pb.environment().put("PGPASSWORD", config.get(String.class, "password"));
+        pb.environment().put("PGDATABASE", config.get(String.class, "database"));
+        pb.environment().put("PGPORT", config.get(String.class, "port", "5432"));
+        pb.redirectErrorStream(true);
+        int code;
+        try {
+            Process process = pb.start();
+            ByteStreams.copy(process.getInputStream(), System.out);
+            code = process.waitFor();
+        } catch (IOException | InterruptedException ex) {
+            throw Throwables.propagate(ex);
+        }
+        if (code != 0) {
+            throw new RuntimeException(String.format(ENGLISH,
+                        "Command finished with non-zero exit code. Exit code is %d.", code));
+        }
+    }
+
+    private static String convert(String sql)
+    {
+        if (Platform.getNativePlatform().getOS().equals(OS.WINDOWS)) {
+            // '"' should be '\"' and '\' should be '\\' in Windows
+            return sql.replace("\\\\", "\\").replace("\\", "\\\\").replace("\"", "\\\"");
+        }
+        return sql;
+    }
+
+    public static String selectRecords(TestingEmbulk embulk, String tableName) throws IOException
+    {
+        Path temp = embulk.createTempFile("txt");
+        Files.delete(temp);
+        execute("\\copy " + tableName + " to '" + temp.toString().replace("\\", "\\\\") + "' delimiter ','");
+        return readSortedFile(temp);
+    }
+}

data/src/test/resources/org/embulk/output/postgresql/test/expect/basic/setup.sql

@@ -0,0 +1,44 @@
+drop table if exists test_number;
+
+create table test_number (
+    id            char(4),
+    bool_item     bool,
+    smallint_item smallint,
+    int_item      int,
+    bigint_item   bigint,
+    real_item     real,
+    double_item   double precision,
+    money_item    money,
+    numeric_item  numeric(8,3),
+    primary key (id)
+);
+
+drop table if exists test_string;
+
+create table test_string (
+    id            int,
+    char_item     char(4),
+    varchar_item  varchar(4),
+    text_item     text,
+    primary key (id)
+);
+
+drop table if exists test_timestamp;
+
+create table test_timestamp (
+    id             int,
+    date_item      date,
+    time_item      time,
+    timestamp_item timestamp,
+    primary key (id)
+);
+
+drop table if exists test_json;
+
+create table test_json (
+    id             int,
+    json_item      json,
+    jsonb_item     jsonb,
+    primary key (id)
+);
+

data/src/test/resources/org/embulk/output/postgresql/test/expect/basic/test_json.csv

@@ -0,0 +1,5 @@
+id:long,json_item:json,jsonb_item:json
+1,null,null
+2,88,99
+3,{"x":"@"},{"y":{"z":[5]}}
+4,,

data/src/test/resources/org/embulk/output/postgresql/test/expect/basic/test_json.yml

@@ -0,0 +1,2 @@
+table: test_json
+mode: insert

data/src/test/resources/org/embulk/output/postgresql/test/expect/basic/test_json_expected.csv

@@ -0,0 +1,4 @@
+1,null,null
+2,88,99
+3,{"x":"@"},{"y": {"z": [5]}}
+4,\N,\N

data/src/test/resources/org/embulk/output/postgresql/test/expect/basic/test_number.csv

@@ -0,0 +1,5 @@
+id:string,bool_item:long,smallint_item:long,int_item:long,bigint_item:long,real_item:double,double_item:double,money_item:string,numeric_item:string
+A001,0,0,0,0,0,0,0,0
+A002,1,12345,123456789,123456789012,123.45,123456.789,1234,1234.567
+A003,0,-1,-1,-1,-1,-1,-1,-1
+A004,,,,,,,,

data/src/test/resources/org/embulk/output/postgresql/test/expect/basic/test_number.yml

@@ -0,0 +1,2 @@
+table: test_number
+mode: insert

data/src/test/resources/org/embulk/output/postgresql/test/expect/basic/test_number_expected.csv

@@ -0,0 +1,4 @@
+A001,f,0,0,0,0,0,\\0,0.000
+A002,t,12345,123456789,123456789012,123.45,123456.789,\\1\,234,1234.567
+A003,f,-1,-1,-1,-1,-1,-\\1,-1.000
+A004,\N,\N,\N,\N,\N,\N,\N,\N

data/src/test/resources/org/embulk/output/postgresql/test/expect/basic/test_string.csv

@@ -0,0 +1,4 @@
+id:long,char_item:string,varchar_item:string,text_item:string
+1,a,b,c
+2,abc,def,ABCDEFGHIJ
+3,,,

data/src/test/resources/org/embulk/output/postgresql/test/expect/basic/test_string.yml

@@ -0,0 +1,2 @@
+table: test_string
+mode: insert

data/src/test/resources/org/embulk/output/postgresql/test/expect/basic/test_string_expected.csv

@@ -0,0 +1,3 @@
+1,a   ,b,c
+2,abc ,def,ABCDEFGHIJ
+3,\N,\N,\N

data/src/test/resources/org/embulk/output/postgresql/test/expect/basic/test_timestamp.csv

@@ -0,0 +1,3 @@
+id:long,date_item:timestamp,time_item:timestamp,timestamp_item:timestamp
+1,2019-1-2 00:00:00.000 +0900,2001-1-1 12:34:56.000 +0900,2019-12-31 23:59:59.000 +0900
+2,,,

data/src/test/resources/org/embulk/output/postgresql/test/expect/basic/test_timestamp.yml

@@ -0,0 +1,2 @@
+table: test_timestamp
+mode: insert

data/src/test/resources/org/embulk/output/postgresql/test/expect/basic/test_timestamp_expected.csv

@@ -0,0 +1,2 @@
+1,2019-01-01,03:34:56,2019-12-31 14:59:59
+2,\N,\N,\N

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-postgresql
 version: !ruby/object:Gem::Version
-  version: 0.8.2
+  version: 0.8.3
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-01-30 00:00:00.000000000 Z
+date: 2019-05-07 00:00:00.000000000 Z
 dependencies: []
 description: Inserts or updates records to a table.
 email:
@@ -19,8 +19,8 @@ extra_rdoc_files: []
 files:
 - README.md
 - build.gradle
-- classpath/embulk-output-jdbc-0.8.2.jar
-- classpath/embulk-output-postgresql-0.8.2.jar
+- classpath/embulk-output-jdbc-0.8.3.jar
+- classpath/embulk-output-postgresql-0.8.3.jar
 - default_jdbc_driver/postgresql-9.4-1205-jdbc41.jar
 - lib/embulk/output/postgresql.rb
 - src/main/java/org/embulk/output/PostgreSQLOutputPlugin.java
@@ -29,6 +29,21 @@ files:
 - src/main/java/org/embulk/output/postgresql/PostgreSQLOutputConnection.java
 - src/main/java/org/embulk/output/postgresql/PostgreSQLOutputConnector.java
 - src/main/java/org/embulk/output/postgresql/setter/PostgreSQLColumnSetterFactory.java
+- src/test/java/org/embulk/output/postgresql/BasicTest.java
+- src/test/java/org/embulk/output/postgresql/PostgreSQLTests.java
+- src/test/resources/org/embulk/output/postgresql/test/expect/basic/setup.sql
+- src/test/resources/org/embulk/output/postgresql/test/expect/basic/test_json.csv
+- src/test/resources/org/embulk/output/postgresql/test/expect/basic/test_json.yml
+- src/test/resources/org/embulk/output/postgresql/test/expect/basic/test_json_expected.csv
+- src/test/resources/org/embulk/output/postgresql/test/expect/basic/test_number.csv
+- src/test/resources/org/embulk/output/postgresql/test/expect/basic/test_number.yml
+- src/test/resources/org/embulk/output/postgresql/test/expect/basic/test_number_expected.csv
+- src/test/resources/org/embulk/output/postgresql/test/expect/basic/test_string.csv
+- src/test/resources/org/embulk/output/postgresql/test/expect/basic/test_string.yml
+- src/test/resources/org/embulk/output/postgresql/test/expect/basic/test_string_expected.csv
+- src/test/resources/org/embulk/output/postgresql/test/expect/basic/test_timestamp.csv
+- src/test/resources/org/embulk/output/postgresql/test/expect/basic/test_timestamp.yml
+- src/test/resources/org/embulk/output/postgresql/test/expect/basic/test_timestamp_expected.csv
 homepage: https://github.com/embulk/embulk-output-jdbc
 licenses:
 - Apache 2.0