embulk-output-postgresql 0.2.3 → 0.2.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +43 -43
- data/build.gradle +7 -7
- data/classpath/{embulk-output-jdbc-0.2.3.jar → embulk-output-jdbc-0.2.4.jar} +0 -0
- data/classpath/{embulk-output-postgresql-0.2.3.jar → embulk-output-postgresql-0.2.4.jar} +0 -0
- data/lib/embulk/output/postgresql.rb +3 -3
- data/src/main/java/org/embulk/output/PostgreSQLOutputPlugin.java +142 -138
- data/src/main/java/org/embulk/output/postgresql/AbstractPostgreSQLCopyBatchInsert.java +217 -217
- data/src/main/java/org/embulk/output/postgresql/PostgreSQLCopyBatchInsert.java +73 -73
- data/src/main/java/org/embulk/output/postgresql/PostgreSQLOutputConnector.java +40 -40
- metadata +6 -6
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 7401277e85bce3f1f2370d0aaa62d9811d8f4173
|
4
|
+
data.tar.gz: 2b308d4f179ad2191d1f2cd9ca4c0e506bdc7c1c
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 6fb3a523e6717894d079460c8a2c850ada1ed48f0fa484e38d7a7d6164a9a6600991a9b904273743ce084eee9fa219d8014954f81ed2bf9b0fd20df95cb3a06d
|
7
|
+
data.tar.gz: ad686f50fa358911d4a86fb0de4433cb75723e656ae71cbb598e43e4aefd074fa8e64319f24adc3bec17a2d90accf7d561099ae5309af6243803341492d8fa74
|
data/README.md
CHANGED
@@ -1,43 +1,43 @@
|
|
1
|
-
# PostgreSQL output plugins for Embulk
|
2
|
-
|
3
|
-
PostgreSQL output plugins for Embulk loads records to PostgreSQL.
|
4
|
-
|
5
|
-
## Overview
|
6
|
-
|
7
|
-
* **Plugin type**: output
|
8
|
-
* **Load all or nothing**: depnds on the mode:
|
9
|
-
* **insert**: no
|
10
|
-
* **replace**: yes
|
11
|
-
* **Resume supported**: no
|
12
|
-
|
13
|
-
## Configuration
|
14
|
-
|
15
|
-
- **host**: database host name (string, required)
|
16
|
-
- **port**: database port number (integer, default: 5432)
|
17
|
-
- **user**: database login user name (string, required)
|
18
|
-
- **password**: database login password (string, default: "")
|
19
|
-
- **database**: destination database name (string, required)
|
20
|
-
- **schema**: destination schema name (string, default: "public")
|
21
|
-
- **table**: destination table name (string, required)
|
22
|
-
- **mode**: "replace" or "insert" (string, required)
|
23
|
-
- **batch_size**: size of a single batch insert (integer, default: 16777216)
|
24
|
-
- **options**: extra connection properties (hash, default: {})
|
25
|
-
|
26
|
-
### Example
|
27
|
-
|
28
|
-
```yaml
|
29
|
-
out:
|
30
|
-
type: postgresql
|
31
|
-
host: localhost
|
32
|
-
user: pg
|
33
|
-
password: ""
|
34
|
-
database: my_database
|
35
|
-
table: my_table
|
36
|
-
mode: insert
|
37
|
-
```
|
38
|
-
|
39
|
-
### Build
|
40
|
-
|
41
|
-
```
|
42
|
-
$ ./gradlew gem
|
43
|
-
```
|
1
|
+
# PostgreSQL output plugins for Embulk
|
2
|
+
|
3
|
+
PostgreSQL output plugins for Embulk loads records to PostgreSQL.
|
4
|
+
|
5
|
+
## Overview
|
6
|
+
|
7
|
+
* **Plugin type**: output
|
8
|
+
* **Load all or nothing**: depnds on the mode:
|
9
|
+
* **insert**: no
|
10
|
+
* **replace**: yes
|
11
|
+
* **Resume supported**: no
|
12
|
+
|
13
|
+
## Configuration
|
14
|
+
|
15
|
+
- **host**: database host name (string, required)
|
16
|
+
- **port**: database port number (integer, default: 5432)
|
17
|
+
- **user**: database login user name (string, required)
|
18
|
+
- **password**: database login password (string, default: "")
|
19
|
+
- **database**: destination database name (string, required)
|
20
|
+
- **schema**: destination schema name (string, default: "public")
|
21
|
+
- **table**: destination table name (string, required)
|
22
|
+
- **mode**: "replace", "merge" or "insert" (string, required)
|
23
|
+
- **batch_size**: size of a single batch insert (integer, default: 16777216)
|
24
|
+
- **options**: extra connection properties (hash, default: {})
|
25
|
+
|
26
|
+
### Example
|
27
|
+
|
28
|
+
```yaml
|
29
|
+
out:
|
30
|
+
type: postgresql
|
31
|
+
host: localhost
|
32
|
+
user: pg
|
33
|
+
password: ""
|
34
|
+
database: my_database
|
35
|
+
table: my_table
|
36
|
+
mode: insert
|
37
|
+
```
|
38
|
+
|
39
|
+
### Build
|
40
|
+
|
41
|
+
```
|
42
|
+
$ ./gradlew gem
|
43
|
+
```
|
data/build.gradle
CHANGED
@@ -1,7 +1,7 @@
|
|
1
|
-
dependencies {
|
2
|
-
compile project(':embulk-output-jdbc')
|
3
|
-
|
4
|
-
compile 'org.postgresql:postgresql:9.4-1200-jdbc41'
|
5
|
-
|
6
|
-
testCompile project(':embulk-output-jdbc').sourceSets.test.output
|
7
|
-
}
|
1
|
+
dependencies {
|
2
|
+
compile project(':embulk-output-jdbc')
|
3
|
+
|
4
|
+
compile 'org.postgresql:postgresql:9.4-1200-jdbc41'
|
5
|
+
|
6
|
+
testCompile project(':embulk-output-jdbc').sourceSets.test.output
|
7
|
+
}
|
Binary file
|
Binary file
|
@@ -1,3 +1,3 @@
|
|
1
|
-
Embulk::JavaPlugin.register_output(
|
2
|
-
:postgresql, "org.embulk.output.PostgreSQLOutputPlugin",
|
3
|
-
File.expand_path('../../../../classpath', __FILE__))
|
1
|
+
Embulk::JavaPlugin.register_output(
|
2
|
+
:postgresql, "org.embulk.output.PostgreSQLOutputPlugin",
|
3
|
+
File.expand_path('../../../../classpath', __FILE__))
|
@@ -1,138 +1,142 @@
|
|
1
|
-
package org.embulk.output;
|
2
|
-
|
3
|
-
import java.util.List;
|
4
|
-
import java.util.Properties;
|
5
|
-
import java.io.IOException;
|
6
|
-
import java.sql.SQLException;
|
7
|
-
|
8
|
-
import org.embulk.output.jdbc.setter.ColumnSetter;
|
9
|
-
import org.embulk.output.postgresql.PostgresqlBatchUpsert;
|
10
|
-
import org.embulk.spi.Exec;
|
11
|
-
import org.embulk.config.Config;
|
12
|
-
import org.embulk.config.ConfigDefault;
|
13
|
-
import org.embulk.output.jdbc.AbstractJdbcOutputPlugin;
|
14
|
-
import org.embulk.output.jdbc.BatchInsert;
|
15
|
-
import org.embulk.output.postgresql.PostgreSQLOutputConnector;
|
16
|
-
import org.embulk.output.postgresql.PostgreSQLCopyBatchInsert;
|
17
|
-
import org.embulk.spi.PageReader;
|
18
|
-
|
19
|
-
public class PostgreSQLOutputPlugin
|
20
|
-
extends AbstractJdbcOutputPlugin
|
21
|
-
{
|
22
|
-
public interface PostgreSQLPluginTask
|
23
|
-
extends PluginTask
|
24
|
-
{
|
25
|
-
@Config("host")
|
26
|
-
public String getHost();
|
27
|
-
|
28
|
-
@Config("port")
|
29
|
-
@ConfigDefault("5432")
|
30
|
-
public int getPort();
|
31
|
-
|
32
|
-
@Config("user")
|
33
|
-
public String getUser();
|
34
|
-
|
35
|
-
@Config("password")
|
36
|
-
@ConfigDefault("\"\"")
|
37
|
-
public String getPassword();
|
38
|
-
|
39
|
-
@Config("database")
|
40
|
-
public String getDatabase();
|
41
|
-
|
42
|
-
@Config("schema")
|
43
|
-
@ConfigDefault("\"public\"")
|
44
|
-
public String getSchema();
|
45
|
-
}
|
46
|
-
|
47
|
-
@Override
|
48
|
-
protected Class<? extends PluginTask> getTaskClass()
|
49
|
-
{
|
50
|
-
return PostgreSQLPluginTask.class;
|
51
|
-
}
|
52
|
-
|
53
|
-
@Override
|
54
|
-
protected PostgreSQLOutputConnector getConnector(PluginTask task, boolean retryableMetadataOperation)
|
55
|
-
{
|
56
|
-
PostgreSQLPluginTask t = (PostgreSQLPluginTask) task;
|
57
|
-
|
58
|
-
String url = String.format("jdbc:postgresql://%s:%d/%s",
|
59
|
-
t.getHost(), t.getPort(), t.getDatabase());
|
60
|
-
|
61
|
-
Properties props = new Properties();
|
62
|
-
props.setProperty("user", t.getUser());
|
63
|
-
props.setProperty("password", t.getPassword());
|
64
|
-
props.setProperty("loginTimeout", "300"); // seconds
|
65
|
-
props.setProperty("socketTimeout", "1800"); // seconds
|
66
|
-
|
67
|
-
// Enable keepalive based on tcp_keepalive_time, tcp_keepalive_intvl and tcp_keepalive_probes kernel parameters.
|
68
|
-
// Socket options TCP_KEEPCNT, TCP_KEEPIDLE, and TCP_KEEPINTVL are not configurable.
|
69
|
-
props.setProperty("tcpKeepAlive", "true");
|
70
|
-
|
71
|
-
// TODO
|
72
|
-
//switch t.getSssl() {
|
73
|
-
//when "disable":
|
74
|
-
// break;
|
75
|
-
//when "enable":
|
76
|
-
// props.setProperty("sslfactory", "org.postgresql.ssl.NonValidatingFactory"); // disable server-side validation
|
77
|
-
//when "verify":
|
78
|
-
// props.setProperty("ssl", "true");
|
79
|
-
// break;
|
80
|
-
//}
|
81
|
-
|
82
|
-
if (!retryableMetadataOperation) {
|
83
|
-
// non-retryable batch operation uses longer timeout
|
84
|
-
props.setProperty("loginTimeout", "300"); // seconds
|
85
|
-
props.setProperty("socketTimeout", "28800"); // seconds
|
86
|
-
}
|
87
|
-
|
88
|
-
props.putAll(t.getOptions());
|
89
|
-
|
90
|
-
return new PostgreSQLOutputConnector(url, props, t.getSchema());
|
91
|
-
}
|
92
|
-
|
93
|
-
@Override
|
94
|
-
protected PluginPageOutput newPluginPageOutput(PageReader reader,
|
95
|
-
BatchInsert batch, List<ColumnSetter> columnSetters,
|
96
|
-
|
97
|
-
{
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
|
137
|
-
|
138
|
-
|
1
|
+
package org.embulk.output;
|
2
|
+
|
3
|
+
import java.util.List;
|
4
|
+
import java.util.Properties;
|
5
|
+
import java.io.IOException;
|
6
|
+
import java.sql.SQLException;
|
7
|
+
|
8
|
+
import org.embulk.output.jdbc.setter.ColumnSetter;
|
9
|
+
import org.embulk.output.postgresql.PostgresqlBatchUpsert;
|
10
|
+
import org.embulk.spi.Exec;
|
11
|
+
import org.embulk.config.Config;
|
12
|
+
import org.embulk.config.ConfigDefault;
|
13
|
+
import org.embulk.output.jdbc.AbstractJdbcOutputPlugin;
|
14
|
+
import org.embulk.output.jdbc.BatchInsert;
|
15
|
+
import org.embulk.output.postgresql.PostgreSQLOutputConnector;
|
16
|
+
import org.embulk.output.postgresql.PostgreSQLCopyBatchInsert;
|
17
|
+
import org.embulk.spi.PageReader;
|
18
|
+
|
19
|
+
public class PostgreSQLOutputPlugin
|
20
|
+
extends AbstractJdbcOutputPlugin
|
21
|
+
{
|
22
|
+
public interface PostgreSQLPluginTask
|
23
|
+
extends PluginTask
|
24
|
+
{
|
25
|
+
@Config("host")
|
26
|
+
public String getHost();
|
27
|
+
|
28
|
+
@Config("port")
|
29
|
+
@ConfigDefault("5432")
|
30
|
+
public int getPort();
|
31
|
+
|
32
|
+
@Config("user")
|
33
|
+
public String getUser();
|
34
|
+
|
35
|
+
@Config("password")
|
36
|
+
@ConfigDefault("\"\"")
|
37
|
+
public String getPassword();
|
38
|
+
|
39
|
+
@Config("database")
|
40
|
+
public String getDatabase();
|
41
|
+
|
42
|
+
@Config("schema")
|
43
|
+
@ConfigDefault("\"public\"")
|
44
|
+
public String getSchema();
|
45
|
+
}
|
46
|
+
|
47
|
+
@Override
|
48
|
+
protected Class<? extends PluginTask> getTaskClass()
|
49
|
+
{
|
50
|
+
return PostgreSQLPluginTask.class;
|
51
|
+
}
|
52
|
+
|
53
|
+
@Override
|
54
|
+
protected PostgreSQLOutputConnector getConnector(PluginTask task, boolean retryableMetadataOperation)
|
55
|
+
{
|
56
|
+
PostgreSQLPluginTask t = (PostgreSQLPluginTask) task;
|
57
|
+
|
58
|
+
String url = String.format("jdbc:postgresql://%s:%d/%s",
|
59
|
+
t.getHost(), t.getPort(), t.getDatabase());
|
60
|
+
|
61
|
+
Properties props = new Properties();
|
62
|
+
props.setProperty("user", t.getUser());
|
63
|
+
props.setProperty("password", t.getPassword());
|
64
|
+
props.setProperty("loginTimeout", "300"); // seconds
|
65
|
+
props.setProperty("socketTimeout", "1800"); // seconds
|
66
|
+
|
67
|
+
// Enable keepalive based on tcp_keepalive_time, tcp_keepalive_intvl and tcp_keepalive_probes kernel parameters.
|
68
|
+
// Socket options TCP_KEEPCNT, TCP_KEEPIDLE, and TCP_KEEPINTVL are not configurable.
|
69
|
+
props.setProperty("tcpKeepAlive", "true");
|
70
|
+
|
71
|
+
// TODO
|
72
|
+
//switch t.getSssl() {
|
73
|
+
//when "disable":
|
74
|
+
// break;
|
75
|
+
//when "enable":
|
76
|
+
// props.setProperty("sslfactory", "org.postgresql.ssl.NonValidatingFactory"); // disable server-side validation
|
77
|
+
//when "verify":
|
78
|
+
// props.setProperty("ssl", "true");
|
79
|
+
// break;
|
80
|
+
//}
|
81
|
+
|
82
|
+
if (!retryableMetadataOperation) {
|
83
|
+
// non-retryable batch operation uses longer timeout
|
84
|
+
props.setProperty("loginTimeout", "300"); // seconds
|
85
|
+
props.setProperty("socketTimeout", "28800"); // seconds
|
86
|
+
}
|
87
|
+
|
88
|
+
props.putAll(t.getOptions());
|
89
|
+
|
90
|
+
return new PostgreSQLOutputConnector(url, props, t.getSchema());
|
91
|
+
}
|
92
|
+
|
93
|
+
@Override
|
94
|
+
protected PluginPageOutput newPluginPageOutput(PageReader reader,
|
95
|
+
BatchInsert batch, List<ColumnSetter> columnSetters,
|
96
|
+
PluginTask task)
|
97
|
+
{
|
98
|
+
if (task.getMode().isMerge()) {
|
99
|
+
return new PostgresPluginPageOutput(reader, batch, columnSetters, task.getBatchSize());
|
100
|
+
}
|
101
|
+
return super.newPluginPageOutput(reader, batch, columnSetters, task);
|
102
|
+
}
|
103
|
+
|
104
|
+
public static class PostgresPluginPageOutput extends PluginPageOutput
|
105
|
+
{
|
106
|
+
|
107
|
+
public PostgresPluginPageOutput(PageReader pageReader, BatchInsert batch, List<ColumnSetter> columnSetters, int batchSize)
|
108
|
+
{
|
109
|
+
super(pageReader, batch, columnSetters, batchSize);
|
110
|
+
}
|
111
|
+
|
112
|
+
@Override
|
113
|
+
protected void handleColumnsSetters()
|
114
|
+
{
|
115
|
+
int size = columnSetters.size();
|
116
|
+
for (int i=0; i < size; i++) {
|
117
|
+
ColumnSetter columnSetter = columnSetters.get(i);
|
118
|
+
if (!columnSetter.getColumn().isPrimaryKey()) {
|
119
|
+
columns.get(i).visit(columnSetter);
|
120
|
+
}
|
121
|
+
}
|
122
|
+
for (int i=0; i < size; i++) {
|
123
|
+
ColumnSetter columnSetter = columnSetters.get(i);
|
124
|
+
if (columnSetter.getColumn().isPrimaryKey()) {
|
125
|
+
columns.get(i).visit(columnSetter);
|
126
|
+
}
|
127
|
+
}
|
128
|
+
for (int i=0; i < size; i++) {
|
129
|
+
columns.get(i).visit(columnSetters.get(i));
|
130
|
+
}
|
131
|
+
}
|
132
|
+
|
133
|
+
}
|
134
|
+
|
135
|
+
@Override
|
136
|
+
protected BatchInsert newBatchInsert(PluginTask task) throws IOException, SQLException
|
137
|
+
{
|
138
|
+
PostgreSQLOutputConnector connector = getConnector(task, true);
|
139
|
+
return task.getMode().isMerge() ? new PostgresqlBatchUpsert(connector) :
|
140
|
+
new PostgreSQLCopyBatchInsert(getConnector(task, true));
|
141
|
+
}
|
142
|
+
}
|
@@ -1,217 +1,217 @@
|
|
1
|
-
package org.embulk.output.postgresql;
|
2
|
-
|
3
|
-
import java.io.File;
|
4
|
-
import java.io.FileOutputStream;
|
5
|
-
import java.io.Writer;
|
6
|
-
import java.io.BufferedWriter;
|
7
|
-
import java.io.OutputStreamWriter;
|
8
|
-
import java.io.IOException;
|
9
|
-
import java.nio.charset.Charset;
|
10
|
-
import java.math.BigDecimal;
|
11
|
-
import java.sql.Date;
|
12
|
-
import java.sql.Time;
|
13
|
-
import java.sql.Timestamp;
|
14
|
-
import java.sql.SQLException;
|
15
|
-
import org.embulk.spi.Exec;
|
16
|
-
import org.embulk.output.jdbc.JdbcSchema;
|
17
|
-
import org.embulk.output.jdbc.BatchInsert;
|
18
|
-
|
19
|
-
public abstract class AbstractPostgreSQLCopyBatchInsert
|
20
|
-
implements BatchInsert
|
21
|
-
{
|
22
|
-
protected static final Charset FILE_CHARSET = Charset.forName("UTF-8");
|
23
|
-
|
24
|
-
protected static final String nullString = "\\N";
|
25
|
-
protected static final String newLineString = "\n";
|
26
|
-
protected static final String delimiterString = "\t";
|
27
|
-
|
28
|
-
protected File currentFile;
|
29
|
-
protected BufferedWriter writer;
|
30
|
-
protected int index;
|
31
|
-
protected int batchRows;
|
32
|
-
|
33
|
-
protected AbstractPostgreSQLCopyBatchInsert() throws IOException
|
34
|
-
{
|
35
|
-
this.index = 0;
|
36
|
-
openNewFile();
|
37
|
-
}
|
38
|
-
|
39
|
-
private File createTempFile() throws IOException
|
40
|
-
{
|
41
|
-
return File.createTempFile("embulk-output-postgres-copy-", ".tsv.tmp"); // TODO configurable temporary file path
|
42
|
-
}
|
43
|
-
|
44
|
-
protected File openNewFile() throws IOException
|
45
|
-
{
|
46
|
-
File newFile = createTempFile();
|
47
|
-
File oldFile = closeCurrentFile();
|
48
|
-
this.writer = openWriter(newFile);
|
49
|
-
currentFile = newFile;
|
50
|
-
return oldFile;
|
51
|
-
}
|
52
|
-
|
53
|
-
protected File closeCurrentFile() throws IOException
|
54
|
-
{
|
55
|
-
if(writer != null) {
|
56
|
-
writer.close();
|
57
|
-
writer = null;
|
58
|
-
}
|
59
|
-
return currentFile;
|
60
|
-
}
|
61
|
-
|
62
|
-
protected BufferedWriter openWriter(File newFile) throws IOException
|
63
|
-
{
|
64
|
-
return new BufferedWriter(new OutputStreamWriter(new FileOutputStream(newFile), FILE_CHARSET));
|
65
|
-
}
|
66
|
-
|
67
|
-
public int getBatchWeight()
|
68
|
-
{
|
69
|
-
long fsize = currentFile.length();
|
70
|
-
if (fsize > Integer.MAX_VALUE) {
|
71
|
-
return Integer.MAX_VALUE;
|
72
|
-
} else {
|
73
|
-
return (int) fsize;
|
74
|
-
}
|
75
|
-
}
|
76
|
-
|
77
|
-
public void finish() throws IOException, SQLException
|
78
|
-
{
|
79
|
-
closeCurrentFile(); // this is necessary to make getBatchWeight() work
|
80
|
-
if (getBatchWeight() != 0) {
|
81
|
-
flush();
|
82
|
-
}
|
83
|
-
}
|
84
|
-
|
85
|
-
public void add() throws IOException
|
86
|
-
{
|
87
|
-
writer.write(newLineString);
|
88
|
-
batchRows++;
|
89
|
-
index = 0;
|
90
|
-
}
|
91
|
-
|
92
|
-
private void appendDelimiter() throws IOException
|
93
|
-
{
|
94
|
-
if(index != 0) {
|
95
|
-
writer.write(delimiterString);
|
96
|
-
}
|
97
|
-
index++;
|
98
|
-
}
|
99
|
-
|
100
|
-
public void setNull(int sqlType) throws IOException
|
101
|
-
{
|
102
|
-
appendDelimiter();
|
103
|
-
writer.write(nullString);
|
104
|
-
}
|
105
|
-
|
106
|
-
public void setBoolean(boolean v) throws IOException
|
107
|
-
{
|
108
|
-
appendDelimiter();
|
109
|
-
writer.write(String.valueOf(v));
|
110
|
-
}
|
111
|
-
|
112
|
-
public void setByte(byte v) throws IOException
|
113
|
-
{
|
114
|
-
appendDelimiter();
|
115
|
-
setEscapedString(String.valueOf(v));
|
116
|
-
}
|
117
|
-
|
118
|
-
public void setShort(short v) throws IOException
|
119
|
-
{
|
120
|
-
appendDelimiter();
|
121
|
-
writer.write(String.valueOf(v));
|
122
|
-
}
|
123
|
-
|
124
|
-
public void setInt(int v) throws IOException
|
125
|
-
{
|
126
|
-
appendDelimiter();
|
127
|
-
writer.write(String.valueOf(v));
|
128
|
-
}
|
129
|
-
|
130
|
-
public void setLong(long v) throws IOException
|
131
|
-
{
|
132
|
-
appendDelimiter();
|
133
|
-
writer.write(String.valueOf(v));
|
134
|
-
}
|
135
|
-
|
136
|
-
public void setFloat(float v) throws IOException
|
137
|
-
{
|
138
|
-
appendDelimiter();
|
139
|
-
writer.write(String.valueOf(v));
|
140
|
-
}
|
141
|
-
|
142
|
-
public void setDouble(double v) throws IOException
|
143
|
-
{
|
144
|
-
appendDelimiter();
|
145
|
-
writer.write(String.valueOf(v));
|
146
|
-
}
|
147
|
-
|
148
|
-
public void setBigDecimal(BigDecimal v) throws IOException
|
149
|
-
{
|
150
|
-
appendDelimiter();
|
151
|
-
writer.write(String.valueOf(v));
|
152
|
-
}
|
153
|
-
|
154
|
-
public void setString(String v) throws IOException
|
155
|
-
{
|
156
|
-
appendDelimiter();
|
157
|
-
setEscapedString(v);
|
158
|
-
}
|
159
|
-
|
160
|
-
public void setNString(String v) throws IOException
|
161
|
-
{
|
162
|
-
appendDelimiter();
|
163
|
-
setEscapedString(v);
|
164
|
-
}
|
165
|
-
|
166
|
-
public void setBytes(byte[] v) throws IOException
|
167
|
-
{
|
168
|
-
appendDelimiter();
|
169
|
-
setEscapedString(String.valueOf(v));
|
170
|
-
}
|
171
|
-
|
172
|
-
public void setSqlDate(Date v, int sqlType) throws IOException
|
173
|
-
{
|
174
|
-
appendDelimiter();
|
175
|
-
writer.write(v.toString());
|
176
|
-
}
|
177
|
-
|
178
|
-
public void setSqlTime(Time v, int sqlType) throws IOException
|
179
|
-
{
|
180
|
-
appendDelimiter();
|
181
|
-
writer.write(v.toString());
|
182
|
-
}
|
183
|
-
|
184
|
-
public void setSqlTimestamp(Timestamp v, int sqlType) throws IOException
|
185
|
-
{
|
186
|
-
appendDelimiter();
|
187
|
-
writer.write(v.toString());
|
188
|
-
}
|
189
|
-
|
190
|
-
// Escape \, \n, \t, \r
|
191
|
-
// Remove \0
|
192
|
-
private void setEscapedString(String v) throws IOException{
|
193
|
-
for (char c : v.toCharArray()) {
|
194
|
-
String s;
|
195
|
-
switch (c) {
|
196
|
-
case '\\':
|
197
|
-
s = "\\\\";
|
198
|
-
break;
|
199
|
-
case '\n':
|
200
|
-
s = "\\n";
|
201
|
-
break;
|
202
|
-
case '\t':
|
203
|
-
s = "\\t";
|
204
|
-
break;
|
205
|
-
case '\r':
|
206
|
-
s = "\\r";
|
207
|
-
break;
|
208
|
-
case 0:
|
209
|
-
s = "";
|
210
|
-
break;
|
211
|
-
default:
|
212
|
-
s = String.valueOf(c);
|
213
|
-
}
|
214
|
-
writer.write(s);
|
215
|
-
}
|
216
|
-
}
|
217
|
-
}
|
1
|
+
package org.embulk.output.postgresql;
|
2
|
+
|
3
|
+
import java.io.File;
|
4
|
+
import java.io.FileOutputStream;
|
5
|
+
import java.io.Writer;
|
6
|
+
import java.io.BufferedWriter;
|
7
|
+
import java.io.OutputStreamWriter;
|
8
|
+
import java.io.IOException;
|
9
|
+
import java.nio.charset.Charset;
|
10
|
+
import java.math.BigDecimal;
|
11
|
+
import java.sql.Date;
|
12
|
+
import java.sql.Time;
|
13
|
+
import java.sql.Timestamp;
|
14
|
+
import java.sql.SQLException;
|
15
|
+
import org.embulk.spi.Exec;
|
16
|
+
import org.embulk.output.jdbc.JdbcSchema;
|
17
|
+
import org.embulk.output.jdbc.BatchInsert;
|
18
|
+
|
19
|
+
public abstract class AbstractPostgreSQLCopyBatchInsert
|
20
|
+
implements BatchInsert
|
21
|
+
{
|
22
|
+
protected static final Charset FILE_CHARSET = Charset.forName("UTF-8");
|
23
|
+
|
24
|
+
protected static final String nullString = "\\N";
|
25
|
+
protected static final String newLineString = "\n";
|
26
|
+
protected static final String delimiterString = "\t";
|
27
|
+
|
28
|
+
protected File currentFile;
|
29
|
+
protected BufferedWriter writer;
|
30
|
+
protected int index;
|
31
|
+
protected int batchRows;
|
32
|
+
|
33
|
+
protected AbstractPostgreSQLCopyBatchInsert() throws IOException
|
34
|
+
{
|
35
|
+
this.index = 0;
|
36
|
+
openNewFile();
|
37
|
+
}
|
38
|
+
|
39
|
+
private File createTempFile() throws IOException
|
40
|
+
{
|
41
|
+
return File.createTempFile("embulk-output-postgres-copy-", ".tsv.tmp"); // TODO configurable temporary file path
|
42
|
+
}
|
43
|
+
|
44
|
+
protected File openNewFile() throws IOException
|
45
|
+
{
|
46
|
+
File newFile = createTempFile();
|
47
|
+
File oldFile = closeCurrentFile();
|
48
|
+
this.writer = openWriter(newFile);
|
49
|
+
currentFile = newFile;
|
50
|
+
return oldFile;
|
51
|
+
}
|
52
|
+
|
53
|
+
protected File closeCurrentFile() throws IOException
|
54
|
+
{
|
55
|
+
if(writer != null) {
|
56
|
+
writer.close();
|
57
|
+
writer = null;
|
58
|
+
}
|
59
|
+
return currentFile;
|
60
|
+
}
|
61
|
+
|
62
|
+
protected BufferedWriter openWriter(File newFile) throws IOException
|
63
|
+
{
|
64
|
+
return new BufferedWriter(new OutputStreamWriter(new FileOutputStream(newFile), FILE_CHARSET));
|
65
|
+
}
|
66
|
+
|
67
|
+
public int getBatchWeight()
|
68
|
+
{
|
69
|
+
long fsize = currentFile.length();
|
70
|
+
if (fsize > Integer.MAX_VALUE) {
|
71
|
+
return Integer.MAX_VALUE;
|
72
|
+
} else {
|
73
|
+
return (int) fsize;
|
74
|
+
}
|
75
|
+
}
|
76
|
+
|
77
|
+
public void finish() throws IOException, SQLException
|
78
|
+
{
|
79
|
+
closeCurrentFile(); // this is necessary to make getBatchWeight() work
|
80
|
+
if (getBatchWeight() != 0) {
|
81
|
+
flush();
|
82
|
+
}
|
83
|
+
}
|
84
|
+
|
85
|
+
public void add() throws IOException
|
86
|
+
{
|
87
|
+
writer.write(newLineString);
|
88
|
+
batchRows++;
|
89
|
+
index = 0;
|
90
|
+
}
|
91
|
+
|
92
|
+
private void appendDelimiter() throws IOException
|
93
|
+
{
|
94
|
+
if(index != 0) {
|
95
|
+
writer.write(delimiterString);
|
96
|
+
}
|
97
|
+
index++;
|
98
|
+
}
|
99
|
+
|
100
|
+
public void setNull(int sqlType) throws IOException
|
101
|
+
{
|
102
|
+
appendDelimiter();
|
103
|
+
writer.write(nullString);
|
104
|
+
}
|
105
|
+
|
106
|
+
public void setBoolean(boolean v) throws IOException
|
107
|
+
{
|
108
|
+
appendDelimiter();
|
109
|
+
writer.write(String.valueOf(v));
|
110
|
+
}
|
111
|
+
|
112
|
+
public void setByte(byte v) throws IOException
|
113
|
+
{
|
114
|
+
appendDelimiter();
|
115
|
+
setEscapedString(String.valueOf(v));
|
116
|
+
}
|
117
|
+
|
118
|
+
public void setShort(short v) throws IOException
|
119
|
+
{
|
120
|
+
appendDelimiter();
|
121
|
+
writer.write(String.valueOf(v));
|
122
|
+
}
|
123
|
+
|
124
|
+
public void setInt(int v) throws IOException
|
125
|
+
{
|
126
|
+
appendDelimiter();
|
127
|
+
writer.write(String.valueOf(v));
|
128
|
+
}
|
129
|
+
|
130
|
+
public void setLong(long v) throws IOException
|
131
|
+
{
|
132
|
+
appendDelimiter();
|
133
|
+
writer.write(String.valueOf(v));
|
134
|
+
}
|
135
|
+
|
136
|
+
public void setFloat(float v) throws IOException
|
137
|
+
{
|
138
|
+
appendDelimiter();
|
139
|
+
writer.write(String.valueOf(v));
|
140
|
+
}
|
141
|
+
|
142
|
+
public void setDouble(double v) throws IOException
|
143
|
+
{
|
144
|
+
appendDelimiter();
|
145
|
+
writer.write(String.valueOf(v));
|
146
|
+
}
|
147
|
+
|
148
|
+
public void setBigDecimal(BigDecimal v) throws IOException
|
149
|
+
{
|
150
|
+
appendDelimiter();
|
151
|
+
writer.write(String.valueOf(v));
|
152
|
+
}
|
153
|
+
|
154
|
+
public void setString(String v) throws IOException
|
155
|
+
{
|
156
|
+
appendDelimiter();
|
157
|
+
setEscapedString(v);
|
158
|
+
}
|
159
|
+
|
160
|
+
public void setNString(String v) throws IOException
|
161
|
+
{
|
162
|
+
appendDelimiter();
|
163
|
+
setEscapedString(v);
|
164
|
+
}
|
165
|
+
|
166
|
+
public void setBytes(byte[] v) throws IOException
|
167
|
+
{
|
168
|
+
appendDelimiter();
|
169
|
+
setEscapedString(String.valueOf(v));
|
170
|
+
}
|
171
|
+
|
172
|
+
public void setSqlDate(Date v, int sqlType) throws IOException
|
173
|
+
{
|
174
|
+
appendDelimiter();
|
175
|
+
writer.write(v.toString());
|
176
|
+
}
|
177
|
+
|
178
|
+
public void setSqlTime(Time v, int sqlType) throws IOException
|
179
|
+
{
|
180
|
+
appendDelimiter();
|
181
|
+
writer.write(v.toString());
|
182
|
+
}
|
183
|
+
|
184
|
+
public void setSqlTimestamp(Timestamp v, int sqlType) throws IOException
|
185
|
+
{
|
186
|
+
appendDelimiter();
|
187
|
+
writer.write(v.toString());
|
188
|
+
}
|
189
|
+
|
190
|
+
// Escape \, \n, \t, \r
|
191
|
+
// Remove \0
|
192
|
+
private void setEscapedString(String v) throws IOException{
|
193
|
+
for (char c : v.toCharArray()) {
|
194
|
+
String s;
|
195
|
+
switch (c) {
|
196
|
+
case '\\':
|
197
|
+
s = "\\\\";
|
198
|
+
break;
|
199
|
+
case '\n':
|
200
|
+
s = "\\n";
|
201
|
+
break;
|
202
|
+
case '\t':
|
203
|
+
s = "\\t";
|
204
|
+
break;
|
205
|
+
case '\r':
|
206
|
+
s = "\\r";
|
207
|
+
break;
|
208
|
+
case 0:
|
209
|
+
s = "";
|
210
|
+
break;
|
211
|
+
default:
|
212
|
+
s = String.valueOf(c);
|
213
|
+
}
|
214
|
+
writer.write(s);
|
215
|
+
}
|
216
|
+
}
|
217
|
+
}
|
@@ -1,73 +1,73 @@
|
|
1
|
-
package org.embulk.output.postgresql;
|
2
|
-
|
3
|
-
import java.io.File;
|
4
|
-
import java.io.IOException;
|
5
|
-
import java.io.FileInputStream;
|
6
|
-
import java.sql.Connection;
|
7
|
-
import java.sql.SQLException;
|
8
|
-
import org.slf4j.Logger;
|
9
|
-
import org.postgresql.copy.CopyManager;
|
10
|
-
import org.postgresql.core.BaseConnection;
|
11
|
-
import org.embulk.spi.Exec;
|
12
|
-
import org.embulk.output.jdbc.JdbcSchema;
|
13
|
-
|
14
|
-
public class PostgreSQLCopyBatchInsert
|
15
|
-
extends AbstractPostgreSQLCopyBatchInsert
|
16
|
-
{
|
17
|
-
private final Logger logger = Exec.getLogger(PostgreSQLCopyBatchInsert.class);
|
18
|
-
private final PostgreSQLOutputConnector connector;
|
19
|
-
|
20
|
-
private PostgreSQLOutputConnection connection = null;
|
21
|
-
private CopyManager copyManager = null;
|
22
|
-
private String copySql = null;
|
23
|
-
private long totalRows;
|
24
|
-
|
25
|
-
public PostgreSQLCopyBatchInsert(PostgreSQLOutputConnector connector) throws IOException, SQLException
|
26
|
-
{
|
27
|
-
super();
|
28
|
-
this.connector = connector;
|
29
|
-
}
|
30
|
-
|
31
|
-
@Override
|
32
|
-
public void prepare(String loadTable, JdbcSchema insertSchema) throws SQLException
|
33
|
-
{
|
34
|
-
this.connection = connector.connect(true);
|
35
|
-
this.copySql = connection.buildCopySql(loadTable, insertSchema);
|
36
|
-
this.copyManager = connection.newCopyManager();
|
37
|
-
logger.info("Copy SQL: "+copySql);
|
38
|
-
}
|
39
|
-
|
40
|
-
@Override
|
41
|
-
public void flush() throws IOException, SQLException
|
42
|
-
{
|
43
|
-
File file = closeCurrentFile(); // flush buffered data in writer
|
44
|
-
|
45
|
-
logger.info(String.format("Loading %,d rows (%,d bytes)", batchRows, file.length()));
|
46
|
-
long startTime = System.currentTimeMillis();
|
47
|
-
FileInputStream in = new FileInputStream(file);
|
48
|
-
try {
|
49
|
-
// TODO check age of connection and call isValid if it's old and reconnect if it's invalid
|
50
|
-
copyManager.copyIn(copySql, in);
|
51
|
-
} finally {
|
52
|
-
in.close();
|
53
|
-
}
|
54
|
-
double seconds = (System.currentTimeMillis() - startTime) / 1000.0;
|
55
|
-
|
56
|
-
totalRows += batchRows;
|
57
|
-
batchRows = 0;
|
58
|
-
logger.info(String.format("> %.2f seconds (loaded %,d rows in total)", seconds, totalRows));
|
59
|
-
|
60
|
-
openNewFile();
|
61
|
-
file.delete();
|
62
|
-
}
|
63
|
-
|
64
|
-
@Override
|
65
|
-
public void close() throws IOException, SQLException
|
66
|
-
{
|
67
|
-
closeCurrentFile().delete();
|
68
|
-
if (connection != null) {
|
69
|
-
connection.close();
|
70
|
-
connection = null;
|
71
|
-
}
|
72
|
-
}
|
73
|
-
}
|
1
|
+
package org.embulk.output.postgresql;
|
2
|
+
|
3
|
+
import java.io.File;
|
4
|
+
import java.io.IOException;
|
5
|
+
import java.io.FileInputStream;
|
6
|
+
import java.sql.Connection;
|
7
|
+
import java.sql.SQLException;
|
8
|
+
import org.slf4j.Logger;
|
9
|
+
import org.postgresql.copy.CopyManager;
|
10
|
+
import org.postgresql.core.BaseConnection;
|
11
|
+
import org.embulk.spi.Exec;
|
12
|
+
import org.embulk.output.jdbc.JdbcSchema;
|
13
|
+
|
14
|
+
public class PostgreSQLCopyBatchInsert
|
15
|
+
extends AbstractPostgreSQLCopyBatchInsert
|
16
|
+
{
|
17
|
+
private final Logger logger = Exec.getLogger(PostgreSQLCopyBatchInsert.class);
|
18
|
+
private final PostgreSQLOutputConnector connector;
|
19
|
+
|
20
|
+
private PostgreSQLOutputConnection connection = null;
|
21
|
+
private CopyManager copyManager = null;
|
22
|
+
private String copySql = null;
|
23
|
+
private long totalRows;
|
24
|
+
|
25
|
+
public PostgreSQLCopyBatchInsert(PostgreSQLOutputConnector connector) throws IOException, SQLException
|
26
|
+
{
|
27
|
+
super();
|
28
|
+
this.connector = connector;
|
29
|
+
}
|
30
|
+
|
31
|
+
@Override
|
32
|
+
public void prepare(String loadTable, JdbcSchema insertSchema) throws SQLException
|
33
|
+
{
|
34
|
+
this.connection = connector.connect(true);
|
35
|
+
this.copySql = connection.buildCopySql(loadTable, insertSchema);
|
36
|
+
this.copyManager = connection.newCopyManager();
|
37
|
+
logger.info("Copy SQL: "+copySql);
|
38
|
+
}
|
39
|
+
|
40
|
+
@Override
|
41
|
+
public void flush() throws IOException, SQLException
|
42
|
+
{
|
43
|
+
File file = closeCurrentFile(); // flush buffered data in writer
|
44
|
+
|
45
|
+
logger.info(String.format("Loading %,d rows (%,d bytes)", batchRows, file.length()));
|
46
|
+
long startTime = System.currentTimeMillis();
|
47
|
+
FileInputStream in = new FileInputStream(file);
|
48
|
+
try {
|
49
|
+
// TODO check age of connection and call isValid if it's old and reconnect if it's invalid
|
50
|
+
copyManager.copyIn(copySql, in);
|
51
|
+
} finally {
|
52
|
+
in.close();
|
53
|
+
}
|
54
|
+
double seconds = (System.currentTimeMillis() - startTime) / 1000.0;
|
55
|
+
|
56
|
+
totalRows += batchRows;
|
57
|
+
batchRows = 0;
|
58
|
+
logger.info(String.format("> %.2f seconds (loaded %,d rows in total)", seconds, totalRows));
|
59
|
+
|
60
|
+
openNewFile();
|
61
|
+
file.delete();
|
62
|
+
}
|
63
|
+
|
64
|
+
@Override
|
65
|
+
public void close() throws IOException, SQLException
|
66
|
+
{
|
67
|
+
closeCurrentFile().delete();
|
68
|
+
if (connection != null) {
|
69
|
+
connection.close();
|
70
|
+
connection = null;
|
71
|
+
}
|
72
|
+
}
|
73
|
+
}
|
@@ -1,40 +1,40 @@
|
|
1
|
-
package org.embulk.output.postgresql;
|
2
|
-
|
3
|
-
import java.util.Properties;
|
4
|
-
import java.sql.Driver;
|
5
|
-
import java.sql.Connection;
|
6
|
-
import java.sql.SQLException;
|
7
|
-
import org.embulk.output.jdbc.JdbcOutputConnector;
|
8
|
-
import org.embulk.output.jdbc.JdbcOutputConnection;
|
9
|
-
|
10
|
-
public class PostgreSQLOutputConnector
|
11
|
-
implements JdbcOutputConnector
|
12
|
-
{
|
13
|
-
private static final Driver driver = new org.postgresql.Driver();
|
14
|
-
|
15
|
-
private final String url;
|
16
|
-
private final Properties properties;
|
17
|
-
private final String schemaName;
|
18
|
-
|
19
|
-
public PostgreSQLOutputConnector(String url, Properties properties, String schemaName)
|
20
|
-
{
|
21
|
-
this.url = url;
|
22
|
-
this.properties = properties;
|
23
|
-
this.schemaName = schemaName;
|
24
|
-
}
|
25
|
-
|
26
|
-
@Override
|
27
|
-
public PostgreSQLOutputConnection connect(boolean autoCommit) throws SQLException
|
28
|
-
{
|
29
|
-
Connection c = driver.connect(url, properties);
|
30
|
-
try {
|
31
|
-
PostgreSQLOutputConnection con = new PostgreSQLOutputConnection(c, schemaName, autoCommit);
|
32
|
-
c = null;
|
33
|
-
return con;
|
34
|
-
} finally {
|
35
|
-
if (c != null) {
|
36
|
-
c.close();
|
37
|
-
}
|
38
|
-
}
|
39
|
-
}
|
40
|
-
}
|
1
|
+
package org.embulk.output.postgresql;
|
2
|
+
|
3
|
+
import java.util.Properties;
|
4
|
+
import java.sql.Driver;
|
5
|
+
import java.sql.Connection;
|
6
|
+
import java.sql.SQLException;
|
7
|
+
import org.embulk.output.jdbc.JdbcOutputConnector;
|
8
|
+
import org.embulk.output.jdbc.JdbcOutputConnection;
|
9
|
+
|
10
|
+
public class PostgreSQLOutputConnector
|
11
|
+
implements JdbcOutputConnector
|
12
|
+
{
|
13
|
+
private static final Driver driver = new org.postgresql.Driver();
|
14
|
+
|
15
|
+
private final String url;
|
16
|
+
private final Properties properties;
|
17
|
+
private final String schemaName;
|
18
|
+
|
19
|
+
public PostgreSQLOutputConnector(String url, Properties properties, String schemaName)
|
20
|
+
{
|
21
|
+
this.url = url;
|
22
|
+
this.properties = properties;
|
23
|
+
this.schemaName = schemaName;
|
24
|
+
}
|
25
|
+
|
26
|
+
@Override
|
27
|
+
public PostgreSQLOutputConnection connect(boolean autoCommit) throws SQLException
|
28
|
+
{
|
29
|
+
Connection c = driver.connect(url, properties);
|
30
|
+
try {
|
31
|
+
PostgreSQLOutputConnection con = new PostgreSQLOutputConnection(c, schemaName, autoCommit);
|
32
|
+
c = null;
|
33
|
+
return con;
|
34
|
+
} finally {
|
35
|
+
if (c != null) {
|
36
|
+
c.close();
|
37
|
+
}
|
38
|
+
}
|
39
|
+
}
|
40
|
+
}
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-output-postgresql
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Sadayuki Furuhashi
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-
|
11
|
+
date: 2015-05-12 00:00:00.000000000 Z
|
12
12
|
dependencies: []
|
13
13
|
description: Inserts or updates records to a table.
|
14
14
|
email:
|
@@ -26,12 +26,12 @@ files:
|
|
26
26
|
- src/main/java/org/embulk/output/postgresql/PostgreSQLOutputConnection.java
|
27
27
|
- src/main/java/org/embulk/output/postgresql/PostgreSQLOutputConnector.java
|
28
28
|
- src/main/java/org/embulk/output/postgresql/PostgresqlBatchUpsert.java
|
29
|
-
- classpath/embulk-output-jdbc-0.2.
|
29
|
+
- classpath/embulk-output-jdbc-0.2.4.jar
|
30
|
+
- classpath/embulk-output-postgresql-0.2.4.jar
|
31
|
+
- classpath/jna-4.1.0.jar
|
32
|
+
- classpath/jna-platform-4.1.0.jar
|
30
33
|
- classpath/postgresql-9.4-1200-jdbc41.jar
|
31
|
-
- classpath/embulk-output-postgresql-0.2.3.jar
|
32
34
|
- classpath/slf4j-simple-1.7.7.jar
|
33
|
-
- classpath/jna-platform-4.1.0.jar
|
34
|
-
- classpath/jna-4.1.0.jar
|
35
35
|
- classpath/waffle-jna-1.7.jar
|
36
36
|
homepage: https://github.com/embulk/embulk-output-jdbc
|
37
37
|
licenses:
|