embulk-output-postgresql 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 20e8259bf0d8db2e0a6486cf6d6e8122a35eac4e
-  data.tar.gz: c51d55c5847e97b55862f85bc7d7c36fbfd825be
+  metadata.gz: a92f3835ae4f58a46ab8e24b234cb58b89036498
+  data.tar.gz: fe2ff3d47c62be68df0ded07d8b6bbc4488cffa8
 SHA512:
-  metadata.gz: 77bddc402cbe94140137f501359842fd101206b26602908aba64f632c6b56a0e02c73e4bbc83c6a20b1a4047034fd95d5667a1ee0487c4792e1805724339ab1a
-  data.tar.gz: b8cc2770741d36bb12fb02868b164833f35d57f9d6015115af25f972b2a791e84cc86c4701521547d0004d377e39e4c171a89bcd5b30a6c1885231f85e7c2f77
+  metadata.gz: 5ddf59bd0f5d93c07dd5b64677e6d6ea924e9e6cf1f001101ff1a4d0f06c59835f6d669d7721ebef90b5f3ee1e8e56b35203def54c9e6e8792e1ffe36a93fd5b
+  data.tar.gz: b530d35eb951435f88913c0d051dbfca7d9eb86536aa604bfd3e8447e0918ea988b0e0a6c441f759f196f3d52b8b68c80517b1351956b908c9dedd7bbaed9167

data/README.md

@@ -20,12 +20,12 @@ PostgreSQL output plugins for Embulk loads records to PostgreSQL.
 - **options**: extra connection properties (hash, default: {})
 - **mode**: "replace", "merge" or "insert" (string, required)
 - **batch_size**: size of a single batch insert (integer, default: 16777216)
-- **default_timezone**: If input column type (embulk type) is timestamp and destination column type is `string` or `nstring`, this plugin needs to format the timestamp into a string. This default_timezone option is used to control the timezone. You can overwrite timezone for each columns using column_options option. (string, default: `UTC`)
+- **default_timezone**: If input column type (embulk type) is timestamp, this plugin needs to format the timestamp into a SQL string. This default_timezone option is used to control the timezone. You can overwrite timezone for each columns using column_options option. (string, default: `UTC`)
 - **column_options**: advanced: a key-value pairs where key is a column name and value is options for the column.
-  - **type**: type of a column when this plugin creates new tables (e.g. `VARCHAR(255)`, `INTEGER NOT NULL UNIQUE`). This used when this plugin creates intermediate tables (insert, truncate_insert and merge modes), when it creates the target table (insert_direct and replace modes), and when it creates nonexistent target table automatically. (string, default: depends on input column type. `BIGINT` if input column type is long, `BOOLEAN` if boolean, `DOUBLE PRECISION` if double, `CLOB` if string, `TIMESTAMP` if timestamp)
+  - **type**: type of a column when this plugin creates new tables (e.g. `VARCHAR(255)`, `INTEGER NOT NULL UNIQUE`). This used when this plugin creates intermediate tables (insert, truncate_insert and merge modes), when it creates the target table (insert_direct and replace modes), and when it creates nonexistent target table automatically. (string, default: depends on input column type. `BIGINT` if input column type is long, `BOOLEAN` if boolean, `DOUBLE PRECISION` if double, `CLOB` if string, `TIMESTAMP WITH TIME ZONE` if timestamp)
   - **value_type**: This plugin converts input column type (embulk type) into a database type to build a INSERT statement. This value_type option controls the type of the value in a INSERT statement. (string, default: depends on input column type. Available values options are: `byte`, `short`, `int`, `long`, `double`, `float`, `boolean`, `string`, `nstring`, `date`, `time`, `timestamp`, `decimal`, `null`, `pass`)
   - **timestamp_format**: If input column type (embulk type) is timestamp and value_type is `string` or `nstring`, this plugin needs to format the timestamp value into a string. This timestamp_format option is used to control the format of the timestamp. (string, default: `%Y-%m-%d %H:%M:%S.%6N`)
-  - **timezone**: If input column type (embulk type) is timestamp and value_type is `string` or `nstring`, this plugin needs to format the timestamp value into a string. And if the input column type is timestamp and value_type is `date`, this plugin needs to consider timezone. In those cases, this timezone option is used to control the timezone. (string, value of default_timezone option is used by default)
+  - **timezone**: If input column type (embulk type) is timestamp, this plugin needs to format the timestamp value into a SQL string. In this cases, this timezone option is used to control the timezone. (string, value of default_timezone option is used by default)
 
 ### Modes
 
@@ -42,7 +42,7 @@ PostgreSQL output plugins for Embulk loads records to PostgreSQL.
   * Transactional: Yes.
   * Resumable: Yes.
 * **merge**:
-  * Behavior: This mode writes rows to some intermediate tables first. If all those tasks run correctly, runs `INSERT INTO <target_table> SELECT * FROM <intermediate_table_1> UNION ALL SELECT * FROM <intermediate_table_2> UNION ALL ... ON DUPLICATE KEY UPDATE ...` query.
+  * Behavior: This mode writes rows to some intermediate tables first. If all those tasks run correctly, runs `with updated AS (UPDATE .... RETURNING ...) INSERT INTO ....` query.
   * Transactional: Yes.
   * Resumable: Yes.
 * **replace**:

data/src/main/java/org/embulk/output/PostgreSQLOutputPlugin.java

@@ -13,6 +13,14 @@ import org.embulk.output.jdbc.BatchInsert;
 import org.embulk.output.postgresql.PostgreSQLOutputConnector;
 import org.embulk.output.postgresql.PostgreSQLCopyBatchInsert;
 
+import com.google.common.collect.ImmutableList;
+import java.sql.Types;
+import org.embulk.spi.Schema;
+import org.embulk.spi.ColumnVisitor;
+import org.embulk.spi.Column;
+import org.embulk.output.jdbc.JdbcColumn;
+import org.embulk.output.jdbc.JdbcSchema;
+
 public class PostgreSQLOutputPlugin
         extends AbstractJdbcOutputPlugin
 {
@@ -106,4 +114,52 @@ public class PostgreSQLOutputPlugin
         }
         return new PostgreSQLCopyBatchInsert(getConnector(task, true));
     }
+
+    // TODO This is almost copy from AbstractJdbcOutputPlugin excepting type of TIMESTAMP -> TIMESTAMP WITH TIME ZONE.
+    //      AbstractJdbcOutputPlugin should have better extensibility.
+    @Override
+    protected JdbcSchema newJdbcSchemaForNewTable(Schema schema)
+    {
+        final ImmutableList.Builder<JdbcColumn> columns = ImmutableList.builder();
+        for (Column c : schema.getColumns()) {
+            final String columnName = c.getName();
+            c.visit(new ColumnVisitor() {
+                public void booleanColumn(Column column)
+                {
+                    columns.add(JdbcColumn.newGenericTypeColumn(
+                            columnName, Types.BOOLEAN, "BOOLEAN",
+                            1, 0, false, false));
+                }
+
+                public void longColumn(Column column)
+                {
+                    columns.add(JdbcColumn.newGenericTypeColumn(
+                            columnName, Types.BIGINT, "BIGINT",
+                            22, 0, false, false));
+                }
+
+                public void doubleColumn(Column column)
+                {
+                    columns.add(JdbcColumn.newGenericTypeColumn(
+                            columnName, Types.FLOAT, "DOUBLE PRECISION",
+                            24, 0, false, false));
+                }
+
+                public void stringColumn(Column column)
+                {
+                    columns.add(JdbcColumn.newGenericTypeColumn(
+                            columnName, Types.CLOB, "CLOB",
+                            4000, 0, false, false));  // TODO size type param
+                }
+
+                public void timestampColumn(Column column)
+                {
+                    columns.add(JdbcColumn.newGenericTypeColumn(
+                            columnName, Types.TIMESTAMP, "TIMESTAMP WITH TIME ZONE",
+                            26, 0, false, false));  // size type param is from postgresql
+                }
+            });
+        }
+        return new JdbcSchema(columns.build());
+    }
 }

data/src/main/java/org/embulk/output/postgresql/AbstractPostgreSQLCopyBatchInsert.java

@@ -1,5 +1,7 @@
 package org.embulk.output.postgresql;
 
+import java.util.Calendar;
+import java.util.Locale;
 import java.io.File;
 import java.io.FileOutputStream;
 import java.io.Writer;
@@ -10,8 +12,8 @@ import java.nio.charset.Charset;
 import java.math.BigDecimal;
 import java.sql.Date;
 import java.sql.Time;
-import java.sql.Timestamp;
 import java.sql.SQLException;
+import org.embulk.spi.time.Timestamp;
 import org.embulk.output.jdbc.BatchInsert;
 
 public abstract class AbstractPostgreSQLCopyBatchInsert
@@ -167,22 +169,50 @@ public abstract class AbstractPostgreSQLCopyBatchInsert
         setEscapedString(String.valueOf(v));
     }
 
-    public void setSqlDate(Date v, int sqlType) throws IOException
+    public void setSqlDate(Timestamp v, Calendar cal) throws IOException
     {
         appendDelimiter();
-        writer.write(v.toString());
+        cal.setTimeInMillis(v.getEpochSecond() * 1000);
+        String f = String.format(Locale.ENGLISH, "%02d-%02d-%02d",
+                cal.get(Calendar.YEAR),
+                cal.get(Calendar.MONTH) + 1,
+                cal.get(Calendar.DAY_OF_MONTH));
+        writer.write(f);
     }
 
-    public void setSqlTime(Time v, int sqlType) throws IOException
+    public void setSqlTime(Timestamp v, Calendar cal) throws IOException
     {
         appendDelimiter();
-        writer.write(v.toString());
+        cal.setTimeInMillis(v.getEpochSecond() * 1000);
+        String f = String.format(Locale.ENGLISH, "%02d:%02d:%02d.%06d",
+                cal.get(Calendar.HOUR_OF_DAY),
+                cal.get(Calendar.MINUTE),
+                cal.get(Calendar.SECOND),
+                v.getNano() / 1000);
+        writer.write(f);
     }
 
-    public void setSqlTimestamp(Timestamp v, int sqlType) throws IOException
+    public void setSqlTimestamp(Timestamp v, Calendar cal) throws IOException
     {
         appendDelimiter();
-        writer.write(v.toString());
+        cal.setTimeInMillis(v.getEpochSecond() * 1000);
+        int zoneOffset = cal.get(Calendar.ZONE_OFFSET) / 1000 / 60;  // zone offset considering DST in minute
+        String offset;
+        if (zoneOffset >= 0) {
+            offset = String.format(Locale.ENGLISH, "+%02d%02d", zoneOffset / 60, zoneOffset % 60);
+        } else {
+            offset = String.format(Locale.ENGLISH, "-%02d%02d", -zoneOffset / 60, -zoneOffset % 60);
+        }
+        String f = String.format(Locale.ENGLISH, "%d-%02d-%02d %02d:%02d:%02d.%06d%s",
+                cal.get(Calendar.YEAR),
+                cal.get(Calendar.MONTH) + 1,
+                cal.get(Calendar.DAY_OF_MONTH),
+                cal.get(Calendar.HOUR_OF_DAY),
+                cal.get(Calendar.MINUTE),
+                cal.get(Calendar.SECOND),
+                v.getNano() / 1000,
+                offset);
+        writer.write(f);
     }
 
     // Escape \, \n, \t, \r

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-postgresql
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-05-19 00:00:00.000000000 Z
+date: 2015-06-11 00:00:00.000000000 Z
 dependencies: []
 description: Inserts or updates records to a table.
 email:
@@ -25,8 +25,8 @@ files:
 - src/main/java/org/embulk/output/postgresql/PostgreSQLCopyBatchInsert.java
 - src/main/java/org/embulk/output/postgresql/PostgreSQLOutputConnection.java
 - src/main/java/org/embulk/output/postgresql/PostgreSQLOutputConnector.java
-- classpath/embulk-output-jdbc-0.3.0.jar
-- classpath/embulk-output-postgresql-0.3.0.jar
+- classpath/embulk-output-jdbc-0.4.0.jar
+- classpath/embulk-output-postgresql-0.4.0.jar
 - classpath/jna-4.1.0.jar
 - classpath/jna-platform-4.1.0.jar
 - classpath/postgresql-9.4-1200-jdbc41.jar