embulk-output-postgresql 0.3.0 → 0.4.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 20e8259bf0d8db2e0a6486cf6d6e8122a35eac4e
4
- data.tar.gz: c51d55c5847e97b55862f85bc7d7c36fbfd825be
3
+ metadata.gz: a92f3835ae4f58a46ab8e24b234cb58b89036498
4
+ data.tar.gz: fe2ff3d47c62be68df0ded07d8b6bbc4488cffa8
5
5
  SHA512:
6
- metadata.gz: 77bddc402cbe94140137f501359842fd101206b26602908aba64f632c6b56a0e02c73e4bbc83c6a20b1a4047034fd95d5667a1ee0487c4792e1805724339ab1a
7
- data.tar.gz: b8cc2770741d36bb12fb02868b164833f35d57f9d6015115af25f972b2a791e84cc86c4701521547d0004d377e39e4c171a89bcd5b30a6c1885231f85e7c2f77
6
+ metadata.gz: 5ddf59bd0f5d93c07dd5b64677e6d6ea924e9e6cf1f001101ff1a4d0f06c59835f6d669d7721ebef90b5f3ee1e8e56b35203def54c9e6e8792e1ffe36a93fd5b
7
+ data.tar.gz: b530d35eb951435f88913c0d051dbfca7d9eb86536aa604bfd3e8447e0918ea988b0e0a6c441f759f196f3d52b8b68c80517b1351956b908c9dedd7bbaed9167
data/README.md CHANGED
@@ -20,12 +20,12 @@ PostgreSQL output plugins for Embulk loads records to PostgreSQL.
20
20
  - **options**: extra connection properties (hash, default: {})
21
21
  - **mode**: "replace", "merge" or "insert" (string, required)
22
22
  - **batch_size**: size of a single batch insert (integer, default: 16777216)
23
- - **default_timezone**: If input column type (embulk type) is timestamp and destination column type is `string` or `nstring`, this plugin needs to format the timestamp into a string. This default_timezone option is used to control the timezone. You can overwrite timezone for each columns using column_options option. (string, default: `UTC`)
23
+ - **default_timezone**: If input column type (embulk type) is timestamp, this plugin needs to format the timestamp into a SQL string. This default_timezone option is used to control the timezone. You can overwrite timezone for each columns using column_options option. (string, default: `UTC`)
24
24
  - **column_options**: advanced: a key-value pairs where key is a column name and value is options for the column.
25
- - **type**: type of a column when this plugin creates new tables (e.g. `VARCHAR(255)`, `INTEGER NOT NULL UNIQUE`). This used when this plugin creates intermediate tables (insert, truncate_insert and merge modes), when it creates the target table (insert_direct and replace modes), and when it creates nonexistent target table automatically. (string, default: depends on input column type. `BIGINT` if input column type is long, `BOOLEAN` if boolean, `DOUBLE PRECISION` if double, `CLOB` if string, `TIMESTAMP` if timestamp)
25
+ - **type**: type of a column when this plugin creates new tables (e.g. `VARCHAR(255)`, `INTEGER NOT NULL UNIQUE`). This used when this plugin creates intermediate tables (insert, truncate_insert and merge modes), when it creates the target table (insert_direct and replace modes), and when it creates nonexistent target table automatically. (string, default: depends on input column type. `BIGINT` if input column type is long, `BOOLEAN` if boolean, `DOUBLE PRECISION` if double, `CLOB` if string, `TIMESTAMP WITH TIME ZONE` if timestamp)
26
26
  - **value_type**: This plugin converts input column type (embulk type) into a database type to build a INSERT statement. This value_type option controls the type of the value in a INSERT statement. (string, default: depends on input column type. Available values options are: `byte`, `short`, `int`, `long`, `double`, `float`, `boolean`, `string`, `nstring`, `date`, `time`, `timestamp`, `decimal`, `null`, `pass`)
27
27
  - **timestamp_format**: If input column type (embulk type) is timestamp and value_type is `string` or `nstring`, this plugin needs to format the timestamp value into a string. This timestamp_format option is used to control the format of the timestamp. (string, default: `%Y-%m-%d %H:%M:%S.%6N`)
28
- - **timezone**: If input column type (embulk type) is timestamp and value_type is `string` or `nstring`, this plugin needs to format the timestamp value into a string. And if the input column type is timestamp and value_type is `date`, this plugin needs to consider timezone. In those cases, this timezone option is used to control the timezone. (string, value of default_timezone option is used by default)
28
+ - **timezone**: If input column type (embulk type) is timestamp, this plugin needs to format the timestamp value into a SQL string. In this cases, this timezone option is used to control the timezone. (string, value of default_timezone option is used by default)
29
29
 
30
30
  ### Modes
31
31
 
@@ -42,7 +42,7 @@ PostgreSQL output plugins for Embulk loads records to PostgreSQL.
42
42
  * Transactional: Yes.
43
43
  * Resumable: Yes.
44
44
  * **merge**:
45
- * Behavior: This mode writes rows to some intermediate tables first. If all those tasks run correctly, runs `INSERT INTO <target_table> SELECT * FROM <intermediate_table_1> UNION ALL SELECT * FROM <intermediate_table_2> UNION ALL ... ON DUPLICATE KEY UPDATE ...` query.
45
+ * Behavior: This mode writes rows to some intermediate tables first. If all those tasks run correctly, runs `with updated AS (UPDATE .... RETURNING ...) INSERT INTO ....` query.
46
46
  * Transactional: Yes.
47
47
  * Resumable: Yes.
48
48
  * **replace**:
@@ -13,6 +13,14 @@ import org.embulk.output.jdbc.BatchInsert;
13
13
  import org.embulk.output.postgresql.PostgreSQLOutputConnector;
14
14
  import org.embulk.output.postgresql.PostgreSQLCopyBatchInsert;
15
15
 
16
+ import com.google.common.collect.ImmutableList;
17
+ import java.sql.Types;
18
+ import org.embulk.spi.Schema;
19
+ import org.embulk.spi.ColumnVisitor;
20
+ import org.embulk.spi.Column;
21
+ import org.embulk.output.jdbc.JdbcColumn;
22
+ import org.embulk.output.jdbc.JdbcSchema;
23
+
16
24
  public class PostgreSQLOutputPlugin
17
25
  extends AbstractJdbcOutputPlugin
18
26
  {
@@ -106,4 +114,52 @@ public class PostgreSQLOutputPlugin
106
114
  }
107
115
  return new PostgreSQLCopyBatchInsert(getConnector(task, true));
108
116
  }
117
+
118
+ // TODO This is almost copy from AbstractJdbcOutputPlugin excepting type of TIMESTAMP -> TIMESTAMP WITH TIME ZONE.
119
+ // AbstractJdbcOutputPlugin should have better extensibility.
120
+ @Override
121
+ protected JdbcSchema newJdbcSchemaForNewTable(Schema schema)
122
+ {
123
+ final ImmutableList.Builder<JdbcColumn> columns = ImmutableList.builder();
124
+ for (Column c : schema.getColumns()) {
125
+ final String columnName = c.getName();
126
+ c.visit(new ColumnVisitor() {
127
+ public void booleanColumn(Column column)
128
+ {
129
+ columns.add(JdbcColumn.newGenericTypeColumn(
130
+ columnName, Types.BOOLEAN, "BOOLEAN",
131
+ 1, 0, false, false));
132
+ }
133
+
134
+ public void longColumn(Column column)
135
+ {
136
+ columns.add(JdbcColumn.newGenericTypeColumn(
137
+ columnName, Types.BIGINT, "BIGINT",
138
+ 22, 0, false, false));
139
+ }
140
+
141
+ public void doubleColumn(Column column)
142
+ {
143
+ columns.add(JdbcColumn.newGenericTypeColumn(
144
+ columnName, Types.FLOAT, "DOUBLE PRECISION",
145
+ 24, 0, false, false));
146
+ }
147
+
148
+ public void stringColumn(Column column)
149
+ {
150
+ columns.add(JdbcColumn.newGenericTypeColumn(
151
+ columnName, Types.CLOB, "CLOB",
152
+ 4000, 0, false, false)); // TODO size type param
153
+ }
154
+
155
+ public void timestampColumn(Column column)
156
+ {
157
+ columns.add(JdbcColumn.newGenericTypeColumn(
158
+ columnName, Types.TIMESTAMP, "TIMESTAMP WITH TIME ZONE",
159
+ 26, 0, false, false)); // size type param is from postgresql
160
+ }
161
+ });
162
+ }
163
+ return new JdbcSchema(columns.build());
164
+ }
109
165
  }
@@ -1,5 +1,7 @@
1
1
  package org.embulk.output.postgresql;
2
2
 
3
+ import java.util.Calendar;
4
+ import java.util.Locale;
3
5
  import java.io.File;
4
6
  import java.io.FileOutputStream;
5
7
  import java.io.Writer;
@@ -10,8 +12,8 @@ import java.nio.charset.Charset;
10
12
  import java.math.BigDecimal;
11
13
  import java.sql.Date;
12
14
  import java.sql.Time;
13
- import java.sql.Timestamp;
14
15
  import java.sql.SQLException;
16
+ import org.embulk.spi.time.Timestamp;
15
17
  import org.embulk.output.jdbc.BatchInsert;
16
18
 
17
19
  public abstract class AbstractPostgreSQLCopyBatchInsert
@@ -167,22 +169,50 @@ public abstract class AbstractPostgreSQLCopyBatchInsert
167
169
  setEscapedString(String.valueOf(v));
168
170
  }
169
171
 
170
- public void setSqlDate(Date v, int sqlType) throws IOException
172
+ public void setSqlDate(Timestamp v, Calendar cal) throws IOException
171
173
  {
172
174
  appendDelimiter();
173
- writer.write(v.toString());
175
+ cal.setTimeInMillis(v.getEpochSecond() * 1000);
176
+ String f = String.format(Locale.ENGLISH, "%02d-%02d-%02d",
177
+ cal.get(Calendar.YEAR),
178
+ cal.get(Calendar.MONTH) + 1,
179
+ cal.get(Calendar.DAY_OF_MONTH));
180
+ writer.write(f);
174
181
  }
175
182
 
176
- public void setSqlTime(Time v, int sqlType) throws IOException
183
+ public void setSqlTime(Timestamp v, Calendar cal) throws IOException
177
184
  {
178
185
  appendDelimiter();
179
- writer.write(v.toString());
186
+ cal.setTimeInMillis(v.getEpochSecond() * 1000);
187
+ String f = String.format(Locale.ENGLISH, "%02d:%02d:%02d.%06d",
188
+ cal.get(Calendar.HOUR_OF_DAY),
189
+ cal.get(Calendar.MINUTE),
190
+ cal.get(Calendar.SECOND),
191
+ v.getNano() / 1000);
192
+ writer.write(f);
180
193
  }
181
194
 
182
- public void setSqlTimestamp(Timestamp v, int sqlType) throws IOException
195
+ public void setSqlTimestamp(Timestamp v, Calendar cal) throws IOException
183
196
  {
184
197
  appendDelimiter();
185
- writer.write(v.toString());
198
+ cal.setTimeInMillis(v.getEpochSecond() * 1000);
199
+ int zoneOffset = cal.get(Calendar.ZONE_OFFSET) / 1000 / 60; // zone offset considering DST in minute
200
+ String offset;
201
+ if (zoneOffset >= 0) {
202
+ offset = String.format(Locale.ENGLISH, "+%02d%02d", zoneOffset / 60, zoneOffset % 60);
203
+ } else {
204
+ offset = String.format(Locale.ENGLISH, "-%02d%02d", -zoneOffset / 60, -zoneOffset % 60);
205
+ }
206
+ String f = String.format(Locale.ENGLISH, "%d-%02d-%02d %02d:%02d:%02d.%06d%s",
207
+ cal.get(Calendar.YEAR),
208
+ cal.get(Calendar.MONTH) + 1,
209
+ cal.get(Calendar.DAY_OF_MONTH),
210
+ cal.get(Calendar.HOUR_OF_DAY),
211
+ cal.get(Calendar.MINUTE),
212
+ cal.get(Calendar.SECOND),
213
+ v.getNano() / 1000,
214
+ offset);
215
+ writer.write(f);
186
216
  }
187
217
 
188
218
  // Escape \, \n, \t, \r
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: embulk-output-postgresql
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.3.0
4
+ version: 0.4.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Sadayuki Furuhashi
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-05-19 00:00:00.000000000 Z
11
+ date: 2015-06-11 00:00:00.000000000 Z
12
12
  dependencies: []
13
13
  description: Inserts or updates records to a table.
14
14
  email:
@@ -25,8 +25,8 @@ files:
25
25
  - src/main/java/org/embulk/output/postgresql/PostgreSQLCopyBatchInsert.java
26
26
  - src/main/java/org/embulk/output/postgresql/PostgreSQLOutputConnection.java
27
27
  - src/main/java/org/embulk/output/postgresql/PostgreSQLOutputConnector.java
28
- - classpath/embulk-output-jdbc-0.3.0.jar
29
- - classpath/embulk-output-postgresql-0.3.0.jar
28
+ - classpath/embulk-output-jdbc-0.4.0.jar
29
+ - classpath/embulk-output-postgresql-0.4.0.jar
30
30
  - classpath/jna-4.1.0.jar
31
31
  - classpath/jna-platform-4.1.0.jar
32
32
  - classpath/postgresql-9.4-1200-jdbc41.jar
Binary file