embulk-output-postgresql 0.2.3 → 0.2.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: a9e12ceb372cd836d2bda55df13f1bd78ca1400a
4
- data.tar.gz: 95cfefc30482d521bd3bcef520e494b7e5d82d39
3
+ metadata.gz: 7401277e85bce3f1f2370d0aaa62d9811d8f4173
4
+ data.tar.gz: 2b308d4f179ad2191d1f2cd9ca4c0e506bdc7c1c
5
5
  SHA512:
6
- metadata.gz: 29dde2105f5caf3e90cf6b423513403c82f87d031cf93df01320ed4ca0d4bec04edbcdbf6441bf9b95e9f5e274d1b8b3d32bd393f05aca02622d04a8b92222ef
7
- data.tar.gz: 0a6ef290f497fe919486dac90dfc87e0eac002914deca3dd02f0f58ec10a7d10ff7e998d81f2d09794c06363abd100137f28e2be2202b6c5fffb4408be049340
6
+ metadata.gz: 6fb3a523e6717894d079460c8a2c850ada1ed48f0fa484e38d7a7d6164a9a6600991a9b904273743ce084eee9fa219d8014954f81ed2bf9b0fd20df95cb3a06d
7
+ data.tar.gz: ad686f50fa358911d4a86fb0de4433cb75723e656ae71cbb598e43e4aefd074fa8e64319f24adc3bec17a2d90accf7d561099ae5309af6243803341492d8fa74
data/README.md CHANGED
@@ -1,43 +1,43 @@
1
- # PostgreSQL output plugins for Embulk
2
-
3
- PostgreSQL output plugins for Embulk loads records to PostgreSQL.
4
-
5
- ## Overview
6
-
7
- * **Plugin type**: output
8
- * **Load all or nothing**: depnds on the mode:
9
- * **insert**: no
10
- * **replace**: yes
11
- * **Resume supported**: no
12
-
13
- ## Configuration
14
-
15
- - **host**: database host name (string, required)
16
- - **port**: database port number (integer, default: 5432)
17
- - **user**: database login user name (string, required)
18
- - **password**: database login password (string, default: "")
19
- - **database**: destination database name (string, required)
20
- - **schema**: destination schema name (string, default: "public")
21
- - **table**: destination table name (string, required)
22
- - **mode**: "replace" or "insert" (string, required)
23
- - **batch_size**: size of a single batch insert (integer, default: 16777216)
24
- - **options**: extra connection properties (hash, default: {})
25
-
26
- ### Example
27
-
28
- ```yaml
29
- out:
30
- type: postgresql
31
- host: localhost
32
- user: pg
33
- password: ""
34
- database: my_database
35
- table: my_table
36
- mode: insert
37
- ```
38
-
39
- ### Build
40
-
41
- ```
42
- $ ./gradlew gem
43
- ```
1
+ # PostgreSQL output plugins for Embulk
2
+
3
+ PostgreSQL output plugins for Embulk loads records to PostgreSQL.
4
+
5
+ ## Overview
6
+
7
+ * **Plugin type**: output
8
+ * **Load all or nothing**: depnds on the mode:
9
+ * **insert**: no
10
+ * **replace**: yes
11
+ * **Resume supported**: no
12
+
13
+ ## Configuration
14
+
15
+ - **host**: database host name (string, required)
16
+ - **port**: database port number (integer, default: 5432)
17
+ - **user**: database login user name (string, required)
18
+ - **password**: database login password (string, default: "")
19
+ - **database**: destination database name (string, required)
20
+ - **schema**: destination schema name (string, default: "public")
21
+ - **table**: destination table name (string, required)
22
+ - **mode**: "replace", "merge" or "insert" (string, required)
23
+ - **batch_size**: size of a single batch insert (integer, default: 16777216)
24
+ - **options**: extra connection properties (hash, default: {})
25
+
26
+ ### Example
27
+
28
+ ```yaml
29
+ out:
30
+ type: postgresql
31
+ host: localhost
32
+ user: pg
33
+ password: ""
34
+ database: my_database
35
+ table: my_table
36
+ mode: insert
37
+ ```
38
+
39
+ ### Build
40
+
41
+ ```
42
+ $ ./gradlew gem
43
+ ```
data/build.gradle CHANGED
@@ -1,7 +1,7 @@
1
- dependencies {
2
- compile project(':embulk-output-jdbc')
3
-
4
- compile 'org.postgresql:postgresql:9.4-1200-jdbc41'
5
-
6
- testCompile project(':embulk-output-jdbc').sourceSets.test.output
7
- }
1
+ dependencies {
2
+ compile project(':embulk-output-jdbc')
3
+
4
+ compile 'org.postgresql:postgresql:9.4-1200-jdbc41'
5
+
6
+ testCompile project(':embulk-output-jdbc').sourceSets.test.output
7
+ }
@@ -1,3 +1,3 @@
1
- Embulk::JavaPlugin.register_output(
2
- :postgresql, "org.embulk.output.PostgreSQLOutputPlugin",
3
- File.expand_path('../../../../classpath', __FILE__))
1
+ Embulk::JavaPlugin.register_output(
2
+ :postgresql, "org.embulk.output.PostgreSQLOutputPlugin",
3
+ File.expand_path('../../../../classpath', __FILE__))
@@ -1,138 +1,142 @@
1
- package org.embulk.output;
2
-
3
- import java.util.List;
4
- import java.util.Properties;
5
- import java.io.IOException;
6
- import java.sql.SQLException;
7
-
8
- import org.embulk.output.jdbc.setter.ColumnSetter;
9
- import org.embulk.output.postgresql.PostgresqlBatchUpsert;
10
- import org.embulk.spi.Exec;
11
- import org.embulk.config.Config;
12
- import org.embulk.config.ConfigDefault;
13
- import org.embulk.output.jdbc.AbstractJdbcOutputPlugin;
14
- import org.embulk.output.jdbc.BatchInsert;
15
- import org.embulk.output.postgresql.PostgreSQLOutputConnector;
16
- import org.embulk.output.postgresql.PostgreSQLCopyBatchInsert;
17
- import org.embulk.spi.PageReader;
18
-
19
- public class PostgreSQLOutputPlugin
20
- extends AbstractJdbcOutputPlugin
21
- {
22
- public interface PostgreSQLPluginTask
23
- extends PluginTask
24
- {
25
- @Config("host")
26
- public String getHost();
27
-
28
- @Config("port")
29
- @ConfigDefault("5432")
30
- public int getPort();
31
-
32
- @Config("user")
33
- public String getUser();
34
-
35
- @Config("password")
36
- @ConfigDefault("\"\"")
37
- public String getPassword();
38
-
39
- @Config("database")
40
- public String getDatabase();
41
-
42
- @Config("schema")
43
- @ConfigDefault("\"public\"")
44
- public String getSchema();
45
- }
46
-
47
- @Override
48
- protected Class<? extends PluginTask> getTaskClass()
49
- {
50
- return PostgreSQLPluginTask.class;
51
- }
52
-
53
- @Override
54
- protected PostgreSQLOutputConnector getConnector(PluginTask task, boolean retryableMetadataOperation)
55
- {
56
- PostgreSQLPluginTask t = (PostgreSQLPluginTask) task;
57
-
58
- String url = String.format("jdbc:postgresql://%s:%d/%s",
59
- t.getHost(), t.getPort(), t.getDatabase());
60
-
61
- Properties props = new Properties();
62
- props.setProperty("user", t.getUser());
63
- props.setProperty("password", t.getPassword());
64
- props.setProperty("loginTimeout", "300"); // seconds
65
- props.setProperty("socketTimeout", "1800"); // seconds
66
-
67
- // Enable keepalive based on tcp_keepalive_time, tcp_keepalive_intvl and tcp_keepalive_probes kernel parameters.
68
- // Socket options TCP_KEEPCNT, TCP_KEEPIDLE, and TCP_KEEPINTVL are not configurable.
69
- props.setProperty("tcpKeepAlive", "true");
70
-
71
- // TODO
72
- //switch t.getSssl() {
73
- //when "disable":
74
- // break;
75
- //when "enable":
76
- // props.setProperty("sslfactory", "org.postgresql.ssl.NonValidatingFactory"); // disable server-side validation
77
- //when "verify":
78
- // props.setProperty("ssl", "true");
79
- // break;
80
- //}
81
-
82
- if (!retryableMetadataOperation) {
83
- // non-retryable batch operation uses longer timeout
84
- props.setProperty("loginTimeout", "300"); // seconds
85
- props.setProperty("socketTimeout", "28800"); // seconds
86
- }
87
-
88
- props.putAll(t.getOptions());
89
-
90
- return new PostgreSQLOutputConnector(url, props, t.getSchema());
91
- }
92
-
93
- @Override
94
- protected PluginPageOutput newPluginPageOutput(PageReader reader,
95
- BatchInsert batch, List<ColumnSetter> columnSetters,
96
- int batchSize)
97
- {
98
- return new PostgresPluginPageOutput(reader, batch, columnSetters, batchSize);
99
- }
100
-
101
- public static class PostgresPluginPageOutput extends PluginPageOutput
102
- {
103
-
104
- public PostgresPluginPageOutput(PageReader pageReader, BatchInsert batch, List<ColumnSetter> columnSetters, int batchSize) {
105
- super(pageReader, batch, columnSetters, batchSize);
106
- }
107
-
108
- @Override
109
- protected void handleColumnsSetters()
110
- {
111
- int size = columnSetters.size();
112
- for (int i=0; i < size; i++) {
113
- ColumnSetter columnSetter = columnSetters.get(i);
114
- if (!columnSetter.getColumn().isPrimaryKey()) {
115
- columns.get(i).visit(columnSetter);
116
- }
117
- }
118
- for (int i=0; i < size; i++) {
119
- ColumnSetter columnSetter = columnSetters.get(i);
120
- if (columnSetter.getColumn().isPrimaryKey()) {
121
- columns.get(i).visit(columnSetter);
122
- }
123
- }
124
- for (int i=0; i < size; i++) {
125
- columns.get(i).visit(columnSetters.get(i));
126
- }
127
- }
128
-
129
- }
130
-
131
- @Override
132
- protected BatchInsert newBatchInsert(PluginTask task) throws IOException, SQLException
133
- {
134
- PostgreSQLOutputConnector connector = getConnector(task, true);
135
- return task.getMode().isMerge() ? new PostgresqlBatchUpsert(connector) :
136
- new PostgreSQLCopyBatchInsert(getConnector(task, true));
137
- }
138
- }
1
+ package org.embulk.output;
2
+
3
+ import java.util.List;
4
+ import java.util.Properties;
5
+ import java.io.IOException;
6
+ import java.sql.SQLException;
7
+
8
+ import org.embulk.output.jdbc.setter.ColumnSetter;
9
+ import org.embulk.output.postgresql.PostgresqlBatchUpsert;
10
+ import org.embulk.spi.Exec;
11
+ import org.embulk.config.Config;
12
+ import org.embulk.config.ConfigDefault;
13
+ import org.embulk.output.jdbc.AbstractJdbcOutputPlugin;
14
+ import org.embulk.output.jdbc.BatchInsert;
15
+ import org.embulk.output.postgresql.PostgreSQLOutputConnector;
16
+ import org.embulk.output.postgresql.PostgreSQLCopyBatchInsert;
17
+ import org.embulk.spi.PageReader;
18
+
19
+ public class PostgreSQLOutputPlugin
20
+ extends AbstractJdbcOutputPlugin
21
+ {
22
+ public interface PostgreSQLPluginTask
23
+ extends PluginTask
24
+ {
25
+ @Config("host")
26
+ public String getHost();
27
+
28
+ @Config("port")
29
+ @ConfigDefault("5432")
30
+ public int getPort();
31
+
32
+ @Config("user")
33
+ public String getUser();
34
+
35
+ @Config("password")
36
+ @ConfigDefault("\"\"")
37
+ public String getPassword();
38
+
39
+ @Config("database")
40
+ public String getDatabase();
41
+
42
+ @Config("schema")
43
+ @ConfigDefault("\"public\"")
44
+ public String getSchema();
45
+ }
46
+
47
+ @Override
48
+ protected Class<? extends PluginTask> getTaskClass()
49
+ {
50
+ return PostgreSQLPluginTask.class;
51
+ }
52
+
53
+ @Override
54
+ protected PostgreSQLOutputConnector getConnector(PluginTask task, boolean retryableMetadataOperation)
55
+ {
56
+ PostgreSQLPluginTask t = (PostgreSQLPluginTask) task;
57
+
58
+ String url = String.format("jdbc:postgresql://%s:%d/%s",
59
+ t.getHost(), t.getPort(), t.getDatabase());
60
+
61
+ Properties props = new Properties();
62
+ props.setProperty("user", t.getUser());
63
+ props.setProperty("password", t.getPassword());
64
+ props.setProperty("loginTimeout", "300"); // seconds
65
+ props.setProperty("socketTimeout", "1800"); // seconds
66
+
67
+ // Enable keepalive based on tcp_keepalive_time, tcp_keepalive_intvl and tcp_keepalive_probes kernel parameters.
68
+ // Socket options TCP_KEEPCNT, TCP_KEEPIDLE, and TCP_KEEPINTVL are not configurable.
69
+ props.setProperty("tcpKeepAlive", "true");
70
+
71
+ // TODO
72
+ //switch t.getSssl() {
73
+ //when "disable":
74
+ // break;
75
+ //when "enable":
76
+ // props.setProperty("sslfactory", "org.postgresql.ssl.NonValidatingFactory"); // disable server-side validation
77
+ //when "verify":
78
+ // props.setProperty("ssl", "true");
79
+ // break;
80
+ //}
81
+
82
+ if (!retryableMetadataOperation) {
83
+ // non-retryable batch operation uses longer timeout
84
+ props.setProperty("loginTimeout", "300"); // seconds
85
+ props.setProperty("socketTimeout", "28800"); // seconds
86
+ }
87
+
88
+ props.putAll(t.getOptions());
89
+
90
+ return new PostgreSQLOutputConnector(url, props, t.getSchema());
91
+ }
92
+
93
+ @Override
94
+ protected PluginPageOutput newPluginPageOutput(PageReader reader,
95
+ BatchInsert batch, List<ColumnSetter> columnSetters,
96
+ PluginTask task)
97
+ {
98
+ if (task.getMode().isMerge()) {
99
+ return new PostgresPluginPageOutput(reader, batch, columnSetters, task.getBatchSize());
100
+ }
101
+ return super.newPluginPageOutput(reader, batch, columnSetters, task);
102
+ }
103
+
104
+ public static class PostgresPluginPageOutput extends PluginPageOutput
105
+ {
106
+
107
+ public PostgresPluginPageOutput(PageReader pageReader, BatchInsert batch, List<ColumnSetter> columnSetters, int batchSize)
108
+ {
109
+ super(pageReader, batch, columnSetters, batchSize);
110
+ }
111
+
112
+ @Override
113
+ protected void handleColumnsSetters()
114
+ {
115
+ int size = columnSetters.size();
116
+ for (int i=0; i < size; i++) {
117
+ ColumnSetter columnSetter = columnSetters.get(i);
118
+ if (!columnSetter.getColumn().isPrimaryKey()) {
119
+ columns.get(i).visit(columnSetter);
120
+ }
121
+ }
122
+ for (int i=0; i < size; i++) {
123
+ ColumnSetter columnSetter = columnSetters.get(i);
124
+ if (columnSetter.getColumn().isPrimaryKey()) {
125
+ columns.get(i).visit(columnSetter);
126
+ }
127
+ }
128
+ for (int i=0; i < size; i++) {
129
+ columns.get(i).visit(columnSetters.get(i));
130
+ }
131
+ }
132
+
133
+ }
134
+
135
+ @Override
136
+ protected BatchInsert newBatchInsert(PluginTask task) throws IOException, SQLException
137
+ {
138
+ PostgreSQLOutputConnector connector = getConnector(task, true);
139
+ return task.getMode().isMerge() ? new PostgresqlBatchUpsert(connector) :
140
+ new PostgreSQLCopyBatchInsert(getConnector(task, true));
141
+ }
142
+ }
@@ -1,217 +1,217 @@
1
- package org.embulk.output.postgresql;
2
-
3
- import java.io.File;
4
- import java.io.FileOutputStream;
5
- import java.io.Writer;
6
- import java.io.BufferedWriter;
7
- import java.io.OutputStreamWriter;
8
- import java.io.IOException;
9
- import java.nio.charset.Charset;
10
- import java.math.BigDecimal;
11
- import java.sql.Date;
12
- import java.sql.Time;
13
- import java.sql.Timestamp;
14
- import java.sql.SQLException;
15
- import org.embulk.spi.Exec;
16
- import org.embulk.output.jdbc.JdbcSchema;
17
- import org.embulk.output.jdbc.BatchInsert;
18
-
19
- public abstract class AbstractPostgreSQLCopyBatchInsert
20
- implements BatchInsert
21
- {
22
- protected static final Charset FILE_CHARSET = Charset.forName("UTF-8");
23
-
24
- protected static final String nullString = "\\N";
25
- protected static final String newLineString = "\n";
26
- protected static final String delimiterString = "\t";
27
-
28
- protected File currentFile;
29
- protected BufferedWriter writer;
30
- protected int index;
31
- protected int batchRows;
32
-
33
- protected AbstractPostgreSQLCopyBatchInsert() throws IOException
34
- {
35
- this.index = 0;
36
- openNewFile();
37
- }
38
-
39
- private File createTempFile() throws IOException
40
- {
41
- return File.createTempFile("embulk-output-postgres-copy-", ".tsv.tmp"); // TODO configurable temporary file path
42
- }
43
-
44
- protected File openNewFile() throws IOException
45
- {
46
- File newFile = createTempFile();
47
- File oldFile = closeCurrentFile();
48
- this.writer = openWriter(newFile);
49
- currentFile = newFile;
50
- return oldFile;
51
- }
52
-
53
- protected File closeCurrentFile() throws IOException
54
- {
55
- if(writer != null) {
56
- writer.close();
57
- writer = null;
58
- }
59
- return currentFile;
60
- }
61
-
62
- protected BufferedWriter openWriter(File newFile) throws IOException
63
- {
64
- return new BufferedWriter(new OutputStreamWriter(new FileOutputStream(newFile), FILE_CHARSET));
65
- }
66
-
67
- public int getBatchWeight()
68
- {
69
- long fsize = currentFile.length();
70
- if (fsize > Integer.MAX_VALUE) {
71
- return Integer.MAX_VALUE;
72
- } else {
73
- return (int) fsize;
74
- }
75
- }
76
-
77
- public void finish() throws IOException, SQLException
78
- {
79
- closeCurrentFile(); // this is necessary to make getBatchWeight() work
80
- if (getBatchWeight() != 0) {
81
- flush();
82
- }
83
- }
84
-
85
- public void add() throws IOException
86
- {
87
- writer.write(newLineString);
88
- batchRows++;
89
- index = 0;
90
- }
91
-
92
- private void appendDelimiter() throws IOException
93
- {
94
- if(index != 0) {
95
- writer.write(delimiterString);
96
- }
97
- index++;
98
- }
99
-
100
- public void setNull(int sqlType) throws IOException
101
- {
102
- appendDelimiter();
103
- writer.write(nullString);
104
- }
105
-
106
- public void setBoolean(boolean v) throws IOException
107
- {
108
- appendDelimiter();
109
- writer.write(String.valueOf(v));
110
- }
111
-
112
- public void setByte(byte v) throws IOException
113
- {
114
- appendDelimiter();
115
- setEscapedString(String.valueOf(v));
116
- }
117
-
118
- public void setShort(short v) throws IOException
119
- {
120
- appendDelimiter();
121
- writer.write(String.valueOf(v));
122
- }
123
-
124
- public void setInt(int v) throws IOException
125
- {
126
- appendDelimiter();
127
- writer.write(String.valueOf(v));
128
- }
129
-
130
- public void setLong(long v) throws IOException
131
- {
132
- appendDelimiter();
133
- writer.write(String.valueOf(v));
134
- }
135
-
136
- public void setFloat(float v) throws IOException
137
- {
138
- appendDelimiter();
139
- writer.write(String.valueOf(v));
140
- }
141
-
142
- public void setDouble(double v) throws IOException
143
- {
144
- appendDelimiter();
145
- writer.write(String.valueOf(v));
146
- }
147
-
148
- public void setBigDecimal(BigDecimal v) throws IOException
149
- {
150
- appendDelimiter();
151
- writer.write(String.valueOf(v));
152
- }
153
-
154
- public void setString(String v) throws IOException
155
- {
156
- appendDelimiter();
157
- setEscapedString(v);
158
- }
159
-
160
- public void setNString(String v) throws IOException
161
- {
162
- appendDelimiter();
163
- setEscapedString(v);
164
- }
165
-
166
- public void setBytes(byte[] v) throws IOException
167
- {
168
- appendDelimiter();
169
- setEscapedString(String.valueOf(v));
170
- }
171
-
172
- public void setSqlDate(Date v, int sqlType) throws IOException
173
- {
174
- appendDelimiter();
175
- writer.write(v.toString());
176
- }
177
-
178
- public void setSqlTime(Time v, int sqlType) throws IOException
179
- {
180
- appendDelimiter();
181
- writer.write(v.toString());
182
- }
183
-
184
- public void setSqlTimestamp(Timestamp v, int sqlType) throws IOException
185
- {
186
- appendDelimiter();
187
- writer.write(v.toString());
188
- }
189
-
190
- // Escape \, \n, \t, \r
191
- // Remove \0
192
- private void setEscapedString(String v) throws IOException{
193
- for (char c : v.toCharArray()) {
194
- String s;
195
- switch (c) {
196
- case '\\':
197
- s = "\\\\";
198
- break;
199
- case '\n':
200
- s = "\\n";
201
- break;
202
- case '\t':
203
- s = "\\t";
204
- break;
205
- case '\r':
206
- s = "\\r";
207
- break;
208
- case 0:
209
- s = "";
210
- break;
211
- default:
212
- s = String.valueOf(c);
213
- }
214
- writer.write(s);
215
- }
216
- }
217
- }
1
+ package org.embulk.output.postgresql;
2
+
3
+ import java.io.File;
4
+ import java.io.FileOutputStream;
5
+ import java.io.Writer;
6
+ import java.io.BufferedWriter;
7
+ import java.io.OutputStreamWriter;
8
+ import java.io.IOException;
9
+ import java.nio.charset.Charset;
10
+ import java.math.BigDecimal;
11
+ import java.sql.Date;
12
+ import java.sql.Time;
13
+ import java.sql.Timestamp;
14
+ import java.sql.SQLException;
15
+ import org.embulk.spi.Exec;
16
+ import org.embulk.output.jdbc.JdbcSchema;
17
+ import org.embulk.output.jdbc.BatchInsert;
18
+
19
+ public abstract class AbstractPostgreSQLCopyBatchInsert
20
+ implements BatchInsert
21
+ {
22
+ protected static final Charset FILE_CHARSET = Charset.forName("UTF-8");
23
+
24
+ protected static final String nullString = "\\N";
25
+ protected static final String newLineString = "\n";
26
+ protected static final String delimiterString = "\t";
27
+
28
+ protected File currentFile;
29
+ protected BufferedWriter writer;
30
+ protected int index;
31
+ protected int batchRows;
32
+
33
+ protected AbstractPostgreSQLCopyBatchInsert() throws IOException
34
+ {
35
+ this.index = 0;
36
+ openNewFile();
37
+ }
38
+
39
+ private File createTempFile() throws IOException
40
+ {
41
+ return File.createTempFile("embulk-output-postgres-copy-", ".tsv.tmp"); // TODO configurable temporary file path
42
+ }
43
+
44
+ protected File openNewFile() throws IOException
45
+ {
46
+ File newFile = createTempFile();
47
+ File oldFile = closeCurrentFile();
48
+ this.writer = openWriter(newFile);
49
+ currentFile = newFile;
50
+ return oldFile;
51
+ }
52
+
53
+ protected File closeCurrentFile() throws IOException
54
+ {
55
+ if(writer != null) {
56
+ writer.close();
57
+ writer = null;
58
+ }
59
+ return currentFile;
60
+ }
61
+
62
+ protected BufferedWriter openWriter(File newFile) throws IOException
63
+ {
64
+ return new BufferedWriter(new OutputStreamWriter(new FileOutputStream(newFile), FILE_CHARSET));
65
+ }
66
+
67
+ public int getBatchWeight()
68
+ {
69
+ long fsize = currentFile.length();
70
+ if (fsize > Integer.MAX_VALUE) {
71
+ return Integer.MAX_VALUE;
72
+ } else {
73
+ return (int) fsize;
74
+ }
75
+ }
76
+
77
+ public void finish() throws IOException, SQLException
78
+ {
79
+ closeCurrentFile(); // this is necessary to make getBatchWeight() work
80
+ if (getBatchWeight() != 0) {
81
+ flush();
82
+ }
83
+ }
84
+
85
+ public void add() throws IOException
86
+ {
87
+ writer.write(newLineString);
88
+ batchRows++;
89
+ index = 0;
90
+ }
91
+
92
+ private void appendDelimiter() throws IOException
93
+ {
94
+ if(index != 0) {
95
+ writer.write(delimiterString);
96
+ }
97
+ index++;
98
+ }
99
+
100
+ public void setNull(int sqlType) throws IOException
101
+ {
102
+ appendDelimiter();
103
+ writer.write(nullString);
104
+ }
105
+
106
+ public void setBoolean(boolean v) throws IOException
107
+ {
108
+ appendDelimiter();
109
+ writer.write(String.valueOf(v));
110
+ }
111
+
112
+ public void setByte(byte v) throws IOException
113
+ {
114
+ appendDelimiter();
115
+ setEscapedString(String.valueOf(v));
116
+ }
117
+
118
+ public void setShort(short v) throws IOException
119
+ {
120
+ appendDelimiter();
121
+ writer.write(String.valueOf(v));
122
+ }
123
+
124
+ public void setInt(int v) throws IOException
125
+ {
126
+ appendDelimiter();
127
+ writer.write(String.valueOf(v));
128
+ }
129
+
130
+ public void setLong(long v) throws IOException
131
+ {
132
+ appendDelimiter();
133
+ writer.write(String.valueOf(v));
134
+ }
135
+
136
+ public void setFloat(float v) throws IOException
137
+ {
138
+ appendDelimiter();
139
+ writer.write(String.valueOf(v));
140
+ }
141
+
142
+ public void setDouble(double v) throws IOException
143
+ {
144
+ appendDelimiter();
145
+ writer.write(String.valueOf(v));
146
+ }
147
+
148
+ public void setBigDecimal(BigDecimal v) throws IOException
149
+ {
150
+ appendDelimiter();
151
+ writer.write(String.valueOf(v));
152
+ }
153
+
154
+ public void setString(String v) throws IOException
155
+ {
156
+ appendDelimiter();
157
+ setEscapedString(v);
158
+ }
159
+
160
+ public void setNString(String v) throws IOException
161
+ {
162
+ appendDelimiter();
163
+ setEscapedString(v);
164
+ }
165
+
166
+ public void setBytes(byte[] v) throws IOException
167
+ {
168
+ appendDelimiter();
169
+ setEscapedString(String.valueOf(v));
170
+ }
171
+
172
+ public void setSqlDate(Date v, int sqlType) throws IOException
173
+ {
174
+ appendDelimiter();
175
+ writer.write(v.toString());
176
+ }
177
+
178
+ public void setSqlTime(Time v, int sqlType) throws IOException
179
+ {
180
+ appendDelimiter();
181
+ writer.write(v.toString());
182
+ }
183
+
184
+ public void setSqlTimestamp(Timestamp v, int sqlType) throws IOException
185
+ {
186
+ appendDelimiter();
187
+ writer.write(v.toString());
188
+ }
189
+
190
+ // Escape \, \n, \t, \r
191
+ // Remove \0
192
+ private void setEscapedString(String v) throws IOException{
193
+ for (char c : v.toCharArray()) {
194
+ String s;
195
+ switch (c) {
196
+ case '\\':
197
+ s = "\\\\";
198
+ break;
199
+ case '\n':
200
+ s = "\\n";
201
+ break;
202
+ case '\t':
203
+ s = "\\t";
204
+ break;
205
+ case '\r':
206
+ s = "\\r";
207
+ break;
208
+ case 0:
209
+ s = "";
210
+ break;
211
+ default:
212
+ s = String.valueOf(c);
213
+ }
214
+ writer.write(s);
215
+ }
216
+ }
217
+ }
@@ -1,73 +1,73 @@
1
- package org.embulk.output.postgresql;
2
-
3
- import java.io.File;
4
- import java.io.IOException;
5
- import java.io.FileInputStream;
6
- import java.sql.Connection;
7
- import java.sql.SQLException;
8
- import org.slf4j.Logger;
9
- import org.postgresql.copy.CopyManager;
10
- import org.postgresql.core.BaseConnection;
11
- import org.embulk.spi.Exec;
12
- import org.embulk.output.jdbc.JdbcSchema;
13
-
14
- public class PostgreSQLCopyBatchInsert
15
- extends AbstractPostgreSQLCopyBatchInsert
16
- {
17
- private final Logger logger = Exec.getLogger(PostgreSQLCopyBatchInsert.class);
18
- private final PostgreSQLOutputConnector connector;
19
-
20
- private PostgreSQLOutputConnection connection = null;
21
- private CopyManager copyManager = null;
22
- private String copySql = null;
23
- private long totalRows;
24
-
25
- public PostgreSQLCopyBatchInsert(PostgreSQLOutputConnector connector) throws IOException, SQLException
26
- {
27
- super();
28
- this.connector = connector;
29
- }
30
-
31
- @Override
32
- public void prepare(String loadTable, JdbcSchema insertSchema) throws SQLException
33
- {
34
- this.connection = connector.connect(true);
35
- this.copySql = connection.buildCopySql(loadTable, insertSchema);
36
- this.copyManager = connection.newCopyManager();
37
- logger.info("Copy SQL: "+copySql);
38
- }
39
-
40
- @Override
41
- public void flush() throws IOException, SQLException
42
- {
43
- File file = closeCurrentFile(); // flush buffered data in writer
44
-
45
- logger.info(String.format("Loading %,d rows (%,d bytes)", batchRows, file.length()));
46
- long startTime = System.currentTimeMillis();
47
- FileInputStream in = new FileInputStream(file);
48
- try {
49
- // TODO check age of connection and call isValid if it's old and reconnect if it's invalid
50
- copyManager.copyIn(copySql, in);
51
- } finally {
52
- in.close();
53
- }
54
- double seconds = (System.currentTimeMillis() - startTime) / 1000.0;
55
-
56
- totalRows += batchRows;
57
- batchRows = 0;
58
- logger.info(String.format("> %.2f seconds (loaded %,d rows in total)", seconds, totalRows));
59
-
60
- openNewFile();
61
- file.delete();
62
- }
63
-
64
- @Override
65
- public void close() throws IOException, SQLException
66
- {
67
- closeCurrentFile().delete();
68
- if (connection != null) {
69
- connection.close();
70
- connection = null;
71
- }
72
- }
73
- }
1
+ package org.embulk.output.postgresql;
2
+
3
+ import java.io.File;
4
+ import java.io.IOException;
5
+ import java.io.FileInputStream;
6
+ import java.sql.Connection;
7
+ import java.sql.SQLException;
8
+ import org.slf4j.Logger;
9
+ import org.postgresql.copy.CopyManager;
10
+ import org.postgresql.core.BaseConnection;
11
+ import org.embulk.spi.Exec;
12
+ import org.embulk.output.jdbc.JdbcSchema;
13
+
14
+ public class PostgreSQLCopyBatchInsert
15
+ extends AbstractPostgreSQLCopyBatchInsert
16
+ {
17
+ private final Logger logger = Exec.getLogger(PostgreSQLCopyBatchInsert.class);
18
+ private final PostgreSQLOutputConnector connector;
19
+
20
+ private PostgreSQLOutputConnection connection = null;
21
+ private CopyManager copyManager = null;
22
+ private String copySql = null;
23
+ private long totalRows;
24
+
25
+ public PostgreSQLCopyBatchInsert(PostgreSQLOutputConnector connector) throws IOException, SQLException
26
+ {
27
+ super();
28
+ this.connector = connector;
29
+ }
30
+
31
+ @Override
32
+ public void prepare(String loadTable, JdbcSchema insertSchema) throws SQLException
33
+ {
34
+ this.connection = connector.connect(true);
35
+ this.copySql = connection.buildCopySql(loadTable, insertSchema);
36
+ this.copyManager = connection.newCopyManager();
37
+ logger.info("Copy SQL: "+copySql);
38
+ }
39
+
40
+ @Override
41
+ public void flush() throws IOException, SQLException
42
+ {
43
+ File file = closeCurrentFile(); // flush buffered data in writer
44
+
45
+ logger.info(String.format("Loading %,d rows (%,d bytes)", batchRows, file.length()));
46
+ long startTime = System.currentTimeMillis();
47
+ FileInputStream in = new FileInputStream(file);
48
+ try {
49
+ // TODO check age of connection and call isValid if it's old and reconnect if it's invalid
50
+ copyManager.copyIn(copySql, in);
51
+ } finally {
52
+ in.close();
53
+ }
54
+ double seconds = (System.currentTimeMillis() - startTime) / 1000.0;
55
+
56
+ totalRows += batchRows;
57
+ batchRows = 0;
58
+ logger.info(String.format("> %.2f seconds (loaded %,d rows in total)", seconds, totalRows));
59
+
60
+ openNewFile();
61
+ file.delete();
62
+ }
63
+
64
+ @Override
65
+ public void close() throws IOException, SQLException
66
+ {
67
+ closeCurrentFile().delete();
68
+ if (connection != null) {
69
+ connection.close();
70
+ connection = null;
71
+ }
72
+ }
73
+ }
@@ -1,40 +1,40 @@
1
- package org.embulk.output.postgresql;
2
-
3
- import java.util.Properties;
4
- import java.sql.Driver;
5
- import java.sql.Connection;
6
- import java.sql.SQLException;
7
- import org.embulk.output.jdbc.JdbcOutputConnector;
8
- import org.embulk.output.jdbc.JdbcOutputConnection;
9
-
10
- public class PostgreSQLOutputConnector
11
- implements JdbcOutputConnector
12
- {
13
- private static final Driver driver = new org.postgresql.Driver();
14
-
15
- private final String url;
16
- private final Properties properties;
17
- private final String schemaName;
18
-
19
- public PostgreSQLOutputConnector(String url, Properties properties, String schemaName)
20
- {
21
- this.url = url;
22
- this.properties = properties;
23
- this.schemaName = schemaName;
24
- }
25
-
26
- @Override
27
- public PostgreSQLOutputConnection connect(boolean autoCommit) throws SQLException
28
- {
29
- Connection c = driver.connect(url, properties);
30
- try {
31
- PostgreSQLOutputConnection con = new PostgreSQLOutputConnection(c, schemaName, autoCommit);
32
- c = null;
33
- return con;
34
- } finally {
35
- if (c != null) {
36
- c.close();
37
- }
38
- }
39
- }
40
- }
1
+ package org.embulk.output.postgresql;
2
+
3
+ import java.util.Properties;
4
+ import java.sql.Driver;
5
+ import java.sql.Connection;
6
+ import java.sql.SQLException;
7
+ import org.embulk.output.jdbc.JdbcOutputConnector;
8
+ import org.embulk.output.jdbc.JdbcOutputConnection;
9
+
10
+ public class PostgreSQLOutputConnector
11
+ implements JdbcOutputConnector
12
+ {
13
+ private static final Driver driver = new org.postgresql.Driver();
14
+
15
+ private final String url;
16
+ private final Properties properties;
17
+ private final String schemaName;
18
+
19
+ public PostgreSQLOutputConnector(String url, Properties properties, String schemaName)
20
+ {
21
+ this.url = url;
22
+ this.properties = properties;
23
+ this.schemaName = schemaName;
24
+ }
25
+
26
+ @Override
27
+ public PostgreSQLOutputConnection connect(boolean autoCommit) throws SQLException
28
+ {
29
+ Connection c = driver.connect(url, properties);
30
+ try {
31
+ PostgreSQLOutputConnection con = new PostgreSQLOutputConnection(c, schemaName, autoCommit);
32
+ c = null;
33
+ return con;
34
+ } finally {
35
+ if (c != null) {
36
+ c.close();
37
+ }
38
+ }
39
+ }
40
+ }
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: embulk-output-postgresql
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.3
4
+ version: 0.2.4
5
5
  platform: ruby
6
6
  authors:
7
7
  - Sadayuki Furuhashi
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-04-23 00:00:00.000000000 Z
11
+ date: 2015-05-12 00:00:00.000000000 Z
12
12
  dependencies: []
13
13
  description: Inserts or updates records to a table.
14
14
  email:
@@ -26,12 +26,12 @@ files:
26
26
  - src/main/java/org/embulk/output/postgresql/PostgreSQLOutputConnection.java
27
27
  - src/main/java/org/embulk/output/postgresql/PostgreSQLOutputConnector.java
28
28
  - src/main/java/org/embulk/output/postgresql/PostgresqlBatchUpsert.java
29
- - classpath/embulk-output-jdbc-0.2.3.jar
29
+ - classpath/embulk-output-jdbc-0.2.4.jar
30
+ - classpath/embulk-output-postgresql-0.2.4.jar
31
+ - classpath/jna-4.1.0.jar
32
+ - classpath/jna-platform-4.1.0.jar
30
33
  - classpath/postgresql-9.4-1200-jdbc41.jar
31
- - classpath/embulk-output-postgresql-0.2.3.jar
32
34
  - classpath/slf4j-simple-1.7.7.jar
33
- - classpath/jna-platform-4.1.0.jar
34
- - classpath/jna-4.1.0.jar
35
35
  - classpath/waffle-jna-1.7.jar
36
36
  homepage: https://github.com/embulk/embulk-output-jdbc
37
37
  licenses: