embulk-output-postgresql 0.3.0 → 0.4.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +4 -4
- data/classpath/embulk-output-jdbc-0.4.0.jar +0 -0
- data/classpath/embulk-output-postgresql-0.4.0.jar +0 -0
- data/src/main/java/org/embulk/output/PostgreSQLOutputPlugin.java +56 -0
- data/src/main/java/org/embulk/output/postgresql/AbstractPostgreSQLCopyBatchInsert.java +37 -7
- metadata +4 -4
- data/classpath/embulk-output-jdbc-0.3.0.jar +0 -0
- data/classpath/embulk-output-postgresql-0.3.0.jar +0 -0
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: a92f3835ae4f58a46ab8e24b234cb58b89036498
|
4
|
+
data.tar.gz: fe2ff3d47c62be68df0ded07d8b6bbc4488cffa8
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 5ddf59bd0f5d93c07dd5b64677e6d6ea924e9e6cf1f001101ff1a4d0f06c59835f6d669d7721ebef90b5f3ee1e8e56b35203def54c9e6e8792e1ffe36a93fd5b
|
7
|
+
data.tar.gz: b530d35eb951435f88913c0d051dbfca7d9eb86536aa604bfd3e8447e0918ea988b0e0a6c441f759f196f3d52b8b68c80517b1351956b908c9dedd7bbaed9167
|
data/README.md
CHANGED
@@ -20,12 +20,12 @@ PostgreSQL output plugins for Embulk loads records to PostgreSQL.
|
|
20
20
|
- **options**: extra connection properties (hash, default: {})
|
21
21
|
- **mode**: "replace", "merge" or "insert" (string, required)
|
22
22
|
- **batch_size**: size of a single batch insert (integer, default: 16777216)
|
23
|
-
- **default_timezone**: If input column type (embulk type) is timestamp
|
23
|
+
- **default_timezone**: If input column type (embulk type) is timestamp, this plugin needs to format the timestamp into a SQL string. This default_timezone option is used to control the timezone. You can overwrite timezone for each columns using column_options option. (string, default: `UTC`)
|
24
24
|
- **column_options**: advanced: a key-value pairs where key is a column name and value is options for the column.
|
25
|
-
- **type**: type of a column when this plugin creates new tables (e.g. `VARCHAR(255)`, `INTEGER NOT NULL UNIQUE`). This used when this plugin creates intermediate tables (insert, truncate_insert and merge modes), when it creates the target table (insert_direct and replace modes), and when it creates nonexistent target table automatically. (string, default: depends on input column type. `BIGINT` if input column type is long, `BOOLEAN` if boolean, `DOUBLE PRECISION` if double, `CLOB` if string, `TIMESTAMP` if timestamp)
|
25
|
+
- **type**: type of a column when this plugin creates new tables (e.g. `VARCHAR(255)`, `INTEGER NOT NULL UNIQUE`). This used when this plugin creates intermediate tables (insert, truncate_insert and merge modes), when it creates the target table (insert_direct and replace modes), and when it creates nonexistent target table automatically. (string, default: depends on input column type. `BIGINT` if input column type is long, `BOOLEAN` if boolean, `DOUBLE PRECISION` if double, `CLOB` if string, `TIMESTAMP WITH TIME ZONE` if timestamp)
|
26
26
|
- **value_type**: This plugin converts input column type (embulk type) into a database type to build a INSERT statement. This value_type option controls the type of the value in a INSERT statement. (string, default: depends on input column type. Available values options are: `byte`, `short`, `int`, `long`, `double`, `float`, `boolean`, `string`, `nstring`, `date`, `time`, `timestamp`, `decimal`, `null`, `pass`)
|
27
27
|
- **timestamp_format**: If input column type (embulk type) is timestamp and value_type is `string` or `nstring`, this plugin needs to format the timestamp value into a string. This timestamp_format option is used to control the format of the timestamp. (string, default: `%Y-%m-%d %H:%M:%S.%6N`)
|
28
|
-
- **timezone**: If input column type (embulk type) is timestamp
|
28
|
+
- **timezone**: If input column type (embulk type) is timestamp, this plugin needs to format the timestamp value into a SQL string. In this cases, this timezone option is used to control the timezone. (string, value of default_timezone option is used by default)
|
29
29
|
|
30
30
|
### Modes
|
31
31
|
|
@@ -42,7 +42,7 @@ PostgreSQL output plugins for Embulk loads records to PostgreSQL.
|
|
42
42
|
* Transactional: Yes.
|
43
43
|
* Resumable: Yes.
|
44
44
|
* **merge**:
|
45
|
-
* Behavior: This mode writes rows to some intermediate tables first. If all those tasks run correctly, runs `
|
45
|
+
* Behavior: This mode writes rows to some intermediate tables first. If all those tasks run correctly, runs `with updated AS (UPDATE .... RETURNING ...) INSERT INTO ....` query.
|
46
46
|
* Transactional: Yes.
|
47
47
|
* Resumable: Yes.
|
48
48
|
* **replace**:
|
Binary file
|
Binary file
|
@@ -13,6 +13,14 @@ import org.embulk.output.jdbc.BatchInsert;
|
|
13
13
|
import org.embulk.output.postgresql.PostgreSQLOutputConnector;
|
14
14
|
import org.embulk.output.postgresql.PostgreSQLCopyBatchInsert;
|
15
15
|
|
16
|
+
import com.google.common.collect.ImmutableList;
|
17
|
+
import java.sql.Types;
|
18
|
+
import org.embulk.spi.Schema;
|
19
|
+
import org.embulk.spi.ColumnVisitor;
|
20
|
+
import org.embulk.spi.Column;
|
21
|
+
import org.embulk.output.jdbc.JdbcColumn;
|
22
|
+
import org.embulk.output.jdbc.JdbcSchema;
|
23
|
+
|
16
24
|
public class PostgreSQLOutputPlugin
|
17
25
|
extends AbstractJdbcOutputPlugin
|
18
26
|
{
|
@@ -106,4 +114,52 @@ public class PostgreSQLOutputPlugin
|
|
106
114
|
}
|
107
115
|
return new PostgreSQLCopyBatchInsert(getConnector(task, true));
|
108
116
|
}
|
117
|
+
|
118
|
+
// TODO This is almost copy from AbstractJdbcOutputPlugin excepting type of TIMESTAMP -> TIMESTAMP WITH TIME ZONE.
|
119
|
+
// AbstractJdbcOutputPlugin should have better extensibility.
|
120
|
+
@Override
|
121
|
+
protected JdbcSchema newJdbcSchemaForNewTable(Schema schema)
|
122
|
+
{
|
123
|
+
final ImmutableList.Builder<JdbcColumn> columns = ImmutableList.builder();
|
124
|
+
for (Column c : schema.getColumns()) {
|
125
|
+
final String columnName = c.getName();
|
126
|
+
c.visit(new ColumnVisitor() {
|
127
|
+
public void booleanColumn(Column column)
|
128
|
+
{
|
129
|
+
columns.add(JdbcColumn.newGenericTypeColumn(
|
130
|
+
columnName, Types.BOOLEAN, "BOOLEAN",
|
131
|
+
1, 0, false, false));
|
132
|
+
}
|
133
|
+
|
134
|
+
public void longColumn(Column column)
|
135
|
+
{
|
136
|
+
columns.add(JdbcColumn.newGenericTypeColumn(
|
137
|
+
columnName, Types.BIGINT, "BIGINT",
|
138
|
+
22, 0, false, false));
|
139
|
+
}
|
140
|
+
|
141
|
+
public void doubleColumn(Column column)
|
142
|
+
{
|
143
|
+
columns.add(JdbcColumn.newGenericTypeColumn(
|
144
|
+
columnName, Types.FLOAT, "DOUBLE PRECISION",
|
145
|
+
24, 0, false, false));
|
146
|
+
}
|
147
|
+
|
148
|
+
public void stringColumn(Column column)
|
149
|
+
{
|
150
|
+
columns.add(JdbcColumn.newGenericTypeColumn(
|
151
|
+
columnName, Types.CLOB, "CLOB",
|
152
|
+
4000, 0, false, false)); // TODO size type param
|
153
|
+
}
|
154
|
+
|
155
|
+
public void timestampColumn(Column column)
|
156
|
+
{
|
157
|
+
columns.add(JdbcColumn.newGenericTypeColumn(
|
158
|
+
columnName, Types.TIMESTAMP, "TIMESTAMP WITH TIME ZONE",
|
159
|
+
26, 0, false, false)); // size type param is from postgresql
|
160
|
+
}
|
161
|
+
});
|
162
|
+
}
|
163
|
+
return new JdbcSchema(columns.build());
|
164
|
+
}
|
109
165
|
}
|
@@ -1,5 +1,7 @@
|
|
1
1
|
package org.embulk.output.postgresql;
|
2
2
|
|
3
|
+
import java.util.Calendar;
|
4
|
+
import java.util.Locale;
|
3
5
|
import java.io.File;
|
4
6
|
import java.io.FileOutputStream;
|
5
7
|
import java.io.Writer;
|
@@ -10,8 +12,8 @@ import java.nio.charset.Charset;
|
|
10
12
|
import java.math.BigDecimal;
|
11
13
|
import java.sql.Date;
|
12
14
|
import java.sql.Time;
|
13
|
-
import java.sql.Timestamp;
|
14
15
|
import java.sql.SQLException;
|
16
|
+
import org.embulk.spi.time.Timestamp;
|
15
17
|
import org.embulk.output.jdbc.BatchInsert;
|
16
18
|
|
17
19
|
public abstract class AbstractPostgreSQLCopyBatchInsert
|
@@ -167,22 +169,50 @@ public abstract class AbstractPostgreSQLCopyBatchInsert
|
|
167
169
|
setEscapedString(String.valueOf(v));
|
168
170
|
}
|
169
171
|
|
170
|
-
public void setSqlDate(
|
172
|
+
public void setSqlDate(Timestamp v, Calendar cal) throws IOException
|
171
173
|
{
|
172
174
|
appendDelimiter();
|
173
|
-
|
175
|
+
cal.setTimeInMillis(v.getEpochSecond() * 1000);
|
176
|
+
String f = String.format(Locale.ENGLISH, "%02d-%02d-%02d",
|
177
|
+
cal.get(Calendar.YEAR),
|
178
|
+
cal.get(Calendar.MONTH) + 1,
|
179
|
+
cal.get(Calendar.DAY_OF_MONTH));
|
180
|
+
writer.write(f);
|
174
181
|
}
|
175
182
|
|
176
|
-
public void setSqlTime(
|
183
|
+
public void setSqlTime(Timestamp v, Calendar cal) throws IOException
|
177
184
|
{
|
178
185
|
appendDelimiter();
|
179
|
-
|
186
|
+
cal.setTimeInMillis(v.getEpochSecond() * 1000);
|
187
|
+
String f = String.format(Locale.ENGLISH, "%02d:%02d:%02d.%06d",
|
188
|
+
cal.get(Calendar.HOUR_OF_DAY),
|
189
|
+
cal.get(Calendar.MINUTE),
|
190
|
+
cal.get(Calendar.SECOND),
|
191
|
+
v.getNano() / 1000);
|
192
|
+
writer.write(f);
|
180
193
|
}
|
181
194
|
|
182
|
-
public void setSqlTimestamp(Timestamp v,
|
195
|
+
public void setSqlTimestamp(Timestamp v, Calendar cal) throws IOException
|
183
196
|
{
|
184
197
|
appendDelimiter();
|
185
|
-
|
198
|
+
cal.setTimeInMillis(v.getEpochSecond() * 1000);
|
199
|
+
int zoneOffset = cal.get(Calendar.ZONE_OFFSET) / 1000 / 60; // zone offset considering DST in minute
|
200
|
+
String offset;
|
201
|
+
if (zoneOffset >= 0) {
|
202
|
+
offset = String.format(Locale.ENGLISH, "+%02d%02d", zoneOffset / 60, zoneOffset % 60);
|
203
|
+
} else {
|
204
|
+
offset = String.format(Locale.ENGLISH, "-%02d%02d", -zoneOffset / 60, -zoneOffset % 60);
|
205
|
+
}
|
206
|
+
String f = String.format(Locale.ENGLISH, "%d-%02d-%02d %02d:%02d:%02d.%06d%s",
|
207
|
+
cal.get(Calendar.YEAR),
|
208
|
+
cal.get(Calendar.MONTH) + 1,
|
209
|
+
cal.get(Calendar.DAY_OF_MONTH),
|
210
|
+
cal.get(Calendar.HOUR_OF_DAY),
|
211
|
+
cal.get(Calendar.MINUTE),
|
212
|
+
cal.get(Calendar.SECOND),
|
213
|
+
v.getNano() / 1000,
|
214
|
+
offset);
|
215
|
+
writer.write(f);
|
186
216
|
}
|
187
217
|
|
188
218
|
// Escape \, \n, \t, \r
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-output-postgresql
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.4.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Sadayuki Furuhashi
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-
|
11
|
+
date: 2015-06-11 00:00:00.000000000 Z
|
12
12
|
dependencies: []
|
13
13
|
description: Inserts or updates records to a table.
|
14
14
|
email:
|
@@ -25,8 +25,8 @@ files:
|
|
25
25
|
- src/main/java/org/embulk/output/postgresql/PostgreSQLCopyBatchInsert.java
|
26
26
|
- src/main/java/org/embulk/output/postgresql/PostgreSQLOutputConnection.java
|
27
27
|
- src/main/java/org/embulk/output/postgresql/PostgreSQLOutputConnector.java
|
28
|
-
- classpath/embulk-output-jdbc-0.
|
29
|
-
- classpath/embulk-output-postgresql-0.
|
28
|
+
- classpath/embulk-output-jdbc-0.4.0.jar
|
29
|
+
- classpath/embulk-output-postgresql-0.4.0.jar
|
30
30
|
- classpath/jna-4.1.0.jar
|
31
31
|
- classpath/jna-platform-4.1.0.jar
|
32
32
|
- classpath/postgresql-9.4-1200-jdbc41.jar
|
Binary file
|
Binary file
|