embulk-output-postgresql 0.2.3 → 0.2.4

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: a9e12ceb372cd836d2bda55df13f1bd78ca1400a
4
- data.tar.gz: 95cfefc30482d521bd3bcef520e494b7e5d82d39
3
+ metadata.gz: 7401277e85bce3f1f2370d0aaa62d9811d8f4173
4
+ data.tar.gz: 2b308d4f179ad2191d1f2cd9ca4c0e506bdc7c1c
5
5
  SHA512:
6
- metadata.gz: 29dde2105f5caf3e90cf6b423513403c82f87d031cf93df01320ed4ca0d4bec04edbcdbf6441bf9b95e9f5e274d1b8b3d32bd393f05aca02622d04a8b92222ef
7
- data.tar.gz: 0a6ef290f497fe919486dac90dfc87e0eac002914deca3dd02f0f58ec10a7d10ff7e998d81f2d09794c06363abd100137f28e2be2202b6c5fffb4408be049340
6
+ metadata.gz: 6fb3a523e6717894d079460c8a2c850ada1ed48f0fa484e38d7a7d6164a9a6600991a9b904273743ce084eee9fa219d8014954f81ed2bf9b0fd20df95cb3a06d
7
+ data.tar.gz: ad686f50fa358911d4a86fb0de4433cb75723e656ae71cbb598e43e4aefd074fa8e64319f24adc3bec17a2d90accf7d561099ae5309af6243803341492d8fa74
data/README.md CHANGED
@@ -1,43 +1,43 @@
1
- # PostgreSQL output plugins for Embulk
2
-
3
- PostgreSQL output plugins for Embulk loads records to PostgreSQL.
4
-
5
- ## Overview
6
-
7
- * **Plugin type**: output
8
- * **Load all or nothing**: depnds on the mode:
9
- * **insert**: no
10
- * **replace**: yes
11
- * **Resume supported**: no
12
-
13
- ## Configuration
14
-
15
- - **host**: database host name (string, required)
16
- - **port**: database port number (integer, default: 5432)
17
- - **user**: database login user name (string, required)
18
- - **password**: database login password (string, default: "")
19
- - **database**: destination database name (string, required)
20
- - **schema**: destination schema name (string, default: "public")
21
- - **table**: destination table name (string, required)
22
- - **mode**: "replace" or "insert" (string, required)
23
- - **batch_size**: size of a single batch insert (integer, default: 16777216)
24
- - **options**: extra connection properties (hash, default: {})
25
-
26
- ### Example
27
-
28
- ```yaml
29
- out:
30
- type: postgresql
31
- host: localhost
32
- user: pg
33
- password: ""
34
- database: my_database
35
- table: my_table
36
- mode: insert
37
- ```
38
-
39
- ### Build
40
-
41
- ```
42
- $ ./gradlew gem
43
- ```
1
+ # PostgreSQL output plugins for Embulk
2
+
3
+ PostgreSQL output plugins for Embulk loads records to PostgreSQL.
4
+
5
+ ## Overview
6
+
7
+ * **Plugin type**: output
8
+ * **Load all or nothing**: depnds on the mode:
9
+ * **insert**: no
10
+ * **replace**: yes
11
+ * **Resume supported**: no
12
+
13
+ ## Configuration
14
+
15
+ - **host**: database host name (string, required)
16
+ - **port**: database port number (integer, default: 5432)
17
+ - **user**: database login user name (string, required)
18
+ - **password**: database login password (string, default: "")
19
+ - **database**: destination database name (string, required)
20
+ - **schema**: destination schema name (string, default: "public")
21
+ - **table**: destination table name (string, required)
22
+ - **mode**: "replace", "merge" or "insert" (string, required)
23
+ - **batch_size**: size of a single batch insert (integer, default: 16777216)
24
+ - **options**: extra connection properties (hash, default: {})
25
+
26
+ ### Example
27
+
28
+ ```yaml
29
+ out:
30
+ type: postgresql
31
+ host: localhost
32
+ user: pg
33
+ password: ""
34
+ database: my_database
35
+ table: my_table
36
+ mode: insert
37
+ ```
38
+
39
+ ### Build
40
+
41
+ ```
42
+ $ ./gradlew gem
43
+ ```
data/build.gradle CHANGED
@@ -1,7 +1,7 @@
1
- dependencies {
2
- compile project(':embulk-output-jdbc')
3
-
4
- compile 'org.postgresql:postgresql:9.4-1200-jdbc41'
5
-
6
- testCompile project(':embulk-output-jdbc').sourceSets.test.output
7
- }
1
+ dependencies {
2
+ compile project(':embulk-output-jdbc')
3
+
4
+ compile 'org.postgresql:postgresql:9.4-1200-jdbc41'
5
+
6
+ testCompile project(':embulk-output-jdbc').sourceSets.test.output
7
+ }
@@ -1,3 +1,3 @@
1
- Embulk::JavaPlugin.register_output(
2
- :postgresql, "org.embulk.output.PostgreSQLOutputPlugin",
3
- File.expand_path('../../../../classpath', __FILE__))
1
+ Embulk::JavaPlugin.register_output(
2
+ :postgresql, "org.embulk.output.PostgreSQLOutputPlugin",
3
+ File.expand_path('../../../../classpath', __FILE__))
@@ -1,138 +1,142 @@
1
- package org.embulk.output;
2
-
3
- import java.util.List;
4
- import java.util.Properties;
5
- import java.io.IOException;
6
- import java.sql.SQLException;
7
-
8
- import org.embulk.output.jdbc.setter.ColumnSetter;
9
- import org.embulk.output.postgresql.PostgresqlBatchUpsert;
10
- import org.embulk.spi.Exec;
11
- import org.embulk.config.Config;
12
- import org.embulk.config.ConfigDefault;
13
- import org.embulk.output.jdbc.AbstractJdbcOutputPlugin;
14
- import org.embulk.output.jdbc.BatchInsert;
15
- import org.embulk.output.postgresql.PostgreSQLOutputConnector;
16
- import org.embulk.output.postgresql.PostgreSQLCopyBatchInsert;
17
- import org.embulk.spi.PageReader;
18
-
19
- public class PostgreSQLOutputPlugin
20
- extends AbstractJdbcOutputPlugin
21
- {
22
- public interface PostgreSQLPluginTask
23
- extends PluginTask
24
- {
25
- @Config("host")
26
- public String getHost();
27
-
28
- @Config("port")
29
- @ConfigDefault("5432")
30
- public int getPort();
31
-
32
- @Config("user")
33
- public String getUser();
34
-
35
- @Config("password")
36
- @ConfigDefault("\"\"")
37
- public String getPassword();
38
-
39
- @Config("database")
40
- public String getDatabase();
41
-
42
- @Config("schema")
43
- @ConfigDefault("\"public\"")
44
- public String getSchema();
45
- }
46
-
47
- @Override
48
- protected Class<? extends PluginTask> getTaskClass()
49
- {
50
- return PostgreSQLPluginTask.class;
51
- }
52
-
53
- @Override
54
- protected PostgreSQLOutputConnector getConnector(PluginTask task, boolean retryableMetadataOperation)
55
- {
56
- PostgreSQLPluginTask t = (PostgreSQLPluginTask) task;
57
-
58
- String url = String.format("jdbc:postgresql://%s:%d/%s",
59
- t.getHost(), t.getPort(), t.getDatabase());
60
-
61
- Properties props = new Properties();
62
- props.setProperty("user", t.getUser());
63
- props.setProperty("password", t.getPassword());
64
- props.setProperty("loginTimeout", "300"); // seconds
65
- props.setProperty("socketTimeout", "1800"); // seconds
66
-
67
- // Enable keepalive based on tcp_keepalive_time, tcp_keepalive_intvl and tcp_keepalive_probes kernel parameters.
68
- // Socket options TCP_KEEPCNT, TCP_KEEPIDLE, and TCP_KEEPINTVL are not configurable.
69
- props.setProperty("tcpKeepAlive", "true");
70
-
71
- // TODO
72
- //switch t.getSssl() {
73
- //when "disable":
74
- // break;
75
- //when "enable":
76
- // props.setProperty("sslfactory", "org.postgresql.ssl.NonValidatingFactory"); // disable server-side validation
77
- //when "verify":
78
- // props.setProperty("ssl", "true");
79
- // break;
80
- //}
81
-
82
- if (!retryableMetadataOperation) {
83
- // non-retryable batch operation uses longer timeout
84
- props.setProperty("loginTimeout", "300"); // seconds
85
- props.setProperty("socketTimeout", "28800"); // seconds
86
- }
87
-
88
- props.putAll(t.getOptions());
89
-
90
- return new PostgreSQLOutputConnector(url, props, t.getSchema());
91
- }
92
-
93
- @Override
94
- protected PluginPageOutput newPluginPageOutput(PageReader reader,
95
- BatchInsert batch, List<ColumnSetter> columnSetters,
96
- int batchSize)
97
- {
98
- return new PostgresPluginPageOutput(reader, batch, columnSetters, batchSize);
99
- }
100
-
101
- public static class PostgresPluginPageOutput extends PluginPageOutput
102
- {
103
-
104
- public PostgresPluginPageOutput(PageReader pageReader, BatchInsert batch, List<ColumnSetter> columnSetters, int batchSize) {
105
- super(pageReader, batch, columnSetters, batchSize);
106
- }
107
-
108
- @Override
109
- protected void handleColumnsSetters()
110
- {
111
- int size = columnSetters.size();
112
- for (int i=0; i < size; i++) {
113
- ColumnSetter columnSetter = columnSetters.get(i);
114
- if (!columnSetter.getColumn().isPrimaryKey()) {
115
- columns.get(i).visit(columnSetter);
116
- }
117
- }
118
- for (int i=0; i < size; i++) {
119
- ColumnSetter columnSetter = columnSetters.get(i);
120
- if (columnSetter.getColumn().isPrimaryKey()) {
121
- columns.get(i).visit(columnSetter);
122
- }
123
- }
124
- for (int i=0; i < size; i++) {
125
- columns.get(i).visit(columnSetters.get(i));
126
- }
127
- }
128
-
129
- }
130
-
131
- @Override
132
- protected BatchInsert newBatchInsert(PluginTask task) throws IOException, SQLException
133
- {
134
- PostgreSQLOutputConnector connector = getConnector(task, true);
135
- return task.getMode().isMerge() ? new PostgresqlBatchUpsert(connector) :
136
- new PostgreSQLCopyBatchInsert(getConnector(task, true));
137
- }
138
- }
1
+ package org.embulk.output;
2
+
3
+ import java.util.List;
4
+ import java.util.Properties;
5
+ import java.io.IOException;
6
+ import java.sql.SQLException;
7
+
8
+ import org.embulk.output.jdbc.setter.ColumnSetter;
9
+ import org.embulk.output.postgresql.PostgresqlBatchUpsert;
10
+ import org.embulk.spi.Exec;
11
+ import org.embulk.config.Config;
12
+ import org.embulk.config.ConfigDefault;
13
+ import org.embulk.output.jdbc.AbstractJdbcOutputPlugin;
14
+ import org.embulk.output.jdbc.BatchInsert;
15
+ import org.embulk.output.postgresql.PostgreSQLOutputConnector;
16
+ import org.embulk.output.postgresql.PostgreSQLCopyBatchInsert;
17
+ import org.embulk.spi.PageReader;
18
+
19
+ public class PostgreSQLOutputPlugin
20
+ extends AbstractJdbcOutputPlugin
21
+ {
22
+ public interface PostgreSQLPluginTask
23
+ extends PluginTask
24
+ {
25
+ @Config("host")
26
+ public String getHost();
27
+
28
+ @Config("port")
29
+ @ConfigDefault("5432")
30
+ public int getPort();
31
+
32
+ @Config("user")
33
+ public String getUser();
34
+
35
+ @Config("password")
36
+ @ConfigDefault("\"\"")
37
+ public String getPassword();
38
+
39
+ @Config("database")
40
+ public String getDatabase();
41
+
42
+ @Config("schema")
43
+ @ConfigDefault("\"public\"")
44
+ public String getSchema();
45
+ }
46
+
47
+ @Override
48
+ protected Class<? extends PluginTask> getTaskClass()
49
+ {
50
+ return PostgreSQLPluginTask.class;
51
+ }
52
+
53
+ @Override
54
+ protected PostgreSQLOutputConnector getConnector(PluginTask task, boolean retryableMetadataOperation)
55
+ {
56
+ PostgreSQLPluginTask t = (PostgreSQLPluginTask) task;
57
+
58
+ String url = String.format("jdbc:postgresql://%s:%d/%s",
59
+ t.getHost(), t.getPort(), t.getDatabase());
60
+
61
+ Properties props = new Properties();
62
+ props.setProperty("user", t.getUser());
63
+ props.setProperty("password", t.getPassword());
64
+ props.setProperty("loginTimeout", "300"); // seconds
65
+ props.setProperty("socketTimeout", "1800"); // seconds
66
+
67
+ // Enable keepalive based on tcp_keepalive_time, tcp_keepalive_intvl and tcp_keepalive_probes kernel parameters.
68
+ // Socket options TCP_KEEPCNT, TCP_KEEPIDLE, and TCP_KEEPINTVL are not configurable.
69
+ props.setProperty("tcpKeepAlive", "true");
70
+
71
+ // TODO
72
+ //switch t.getSssl() {
73
+ //when "disable":
74
+ // break;
75
+ //when "enable":
76
+ // props.setProperty("sslfactory", "org.postgresql.ssl.NonValidatingFactory"); // disable server-side validation
77
+ //when "verify":
78
+ // props.setProperty("ssl", "true");
79
+ // break;
80
+ //}
81
+
82
+ if (!retryableMetadataOperation) {
83
+ // non-retryable batch operation uses longer timeout
84
+ props.setProperty("loginTimeout", "300"); // seconds
85
+ props.setProperty("socketTimeout", "28800"); // seconds
86
+ }
87
+
88
+ props.putAll(t.getOptions());
89
+
90
+ return new PostgreSQLOutputConnector(url, props, t.getSchema());
91
+ }
92
+
93
+ @Override
94
+ protected PluginPageOutput newPluginPageOutput(PageReader reader,
95
+ BatchInsert batch, List<ColumnSetter> columnSetters,
96
+ PluginTask task)
97
+ {
98
+ if (task.getMode().isMerge()) {
99
+ return new PostgresPluginPageOutput(reader, batch, columnSetters, task.getBatchSize());
100
+ }
101
+ return super.newPluginPageOutput(reader, batch, columnSetters, task);
102
+ }
103
+
104
+ public static class PostgresPluginPageOutput extends PluginPageOutput
105
+ {
106
+
107
+ public PostgresPluginPageOutput(PageReader pageReader, BatchInsert batch, List<ColumnSetter> columnSetters, int batchSize)
108
+ {
109
+ super(pageReader, batch, columnSetters, batchSize);
110
+ }
111
+
112
+ @Override
113
+ protected void handleColumnsSetters()
114
+ {
115
+ int size = columnSetters.size();
116
+ for (int i=0; i < size; i++) {
117
+ ColumnSetter columnSetter = columnSetters.get(i);
118
+ if (!columnSetter.getColumn().isPrimaryKey()) {
119
+ columns.get(i).visit(columnSetter);
120
+ }
121
+ }
122
+ for (int i=0; i < size; i++) {
123
+ ColumnSetter columnSetter = columnSetters.get(i);
124
+ if (columnSetter.getColumn().isPrimaryKey()) {
125
+ columns.get(i).visit(columnSetter);
126
+ }
127
+ }
128
+ for (int i=0; i < size; i++) {
129
+ columns.get(i).visit(columnSetters.get(i));
130
+ }
131
+ }
132
+
133
+ }
134
+
135
+ @Override
136
+ protected BatchInsert newBatchInsert(PluginTask task) throws IOException, SQLException
137
+ {
138
+ PostgreSQLOutputConnector connector = getConnector(task, true);
139
+ return task.getMode().isMerge() ? new PostgresqlBatchUpsert(connector) :
140
+ new PostgreSQLCopyBatchInsert(getConnector(task, true));
141
+ }
142
+ }
@@ -1,217 +1,217 @@
1
- package org.embulk.output.postgresql;
2
-
3
- import java.io.File;
4
- import java.io.FileOutputStream;
5
- import java.io.Writer;
6
- import java.io.BufferedWriter;
7
- import java.io.OutputStreamWriter;
8
- import java.io.IOException;
9
- import java.nio.charset.Charset;
10
- import java.math.BigDecimal;
11
- import java.sql.Date;
12
- import java.sql.Time;
13
- import java.sql.Timestamp;
14
- import java.sql.SQLException;
15
- import org.embulk.spi.Exec;
16
- import org.embulk.output.jdbc.JdbcSchema;
17
- import org.embulk.output.jdbc.BatchInsert;
18
-
19
- public abstract class AbstractPostgreSQLCopyBatchInsert
20
- implements BatchInsert
21
- {
22
- protected static final Charset FILE_CHARSET = Charset.forName("UTF-8");
23
-
24
- protected static final String nullString = "\\N";
25
- protected static final String newLineString = "\n";
26
- protected static final String delimiterString = "\t";
27
-
28
- protected File currentFile;
29
- protected BufferedWriter writer;
30
- protected int index;
31
- protected int batchRows;
32
-
33
- protected AbstractPostgreSQLCopyBatchInsert() throws IOException
34
- {
35
- this.index = 0;
36
- openNewFile();
37
- }
38
-
39
- private File createTempFile() throws IOException
40
- {
41
- return File.createTempFile("embulk-output-postgres-copy-", ".tsv.tmp"); // TODO configurable temporary file path
42
- }
43
-
44
- protected File openNewFile() throws IOException
45
- {
46
- File newFile = createTempFile();
47
- File oldFile = closeCurrentFile();
48
- this.writer = openWriter(newFile);
49
- currentFile = newFile;
50
- return oldFile;
51
- }
52
-
53
- protected File closeCurrentFile() throws IOException
54
- {
55
- if(writer != null) {
56
- writer.close();
57
- writer = null;
58
- }
59
- return currentFile;
60
- }
61
-
62
- protected BufferedWriter openWriter(File newFile) throws IOException
63
- {
64
- return new BufferedWriter(new OutputStreamWriter(new FileOutputStream(newFile), FILE_CHARSET));
65
- }
66
-
67
- public int getBatchWeight()
68
- {
69
- long fsize = currentFile.length();
70
- if (fsize > Integer.MAX_VALUE) {
71
- return Integer.MAX_VALUE;
72
- } else {
73
- return (int) fsize;
74
- }
75
- }
76
-
77
- public void finish() throws IOException, SQLException
78
- {
79
- closeCurrentFile(); // this is necessary to make getBatchWeight() work
80
- if (getBatchWeight() != 0) {
81
- flush();
82
- }
83
- }
84
-
85
- public void add() throws IOException
86
- {
87
- writer.write(newLineString);
88
- batchRows++;
89
- index = 0;
90
- }
91
-
92
- private void appendDelimiter() throws IOException
93
- {
94
- if(index != 0) {
95
- writer.write(delimiterString);
96
- }
97
- index++;
98
- }
99
-
100
- public void setNull(int sqlType) throws IOException
101
- {
102
- appendDelimiter();
103
- writer.write(nullString);
104
- }
105
-
106
- public void setBoolean(boolean v) throws IOException
107
- {
108
- appendDelimiter();
109
- writer.write(String.valueOf(v));
110
- }
111
-
112
- public void setByte(byte v) throws IOException
113
- {
114
- appendDelimiter();
115
- setEscapedString(String.valueOf(v));
116
- }
117
-
118
- public void setShort(short v) throws IOException
119
- {
120
- appendDelimiter();
121
- writer.write(String.valueOf(v));
122
- }
123
-
124
- public void setInt(int v) throws IOException
125
- {
126
- appendDelimiter();
127
- writer.write(String.valueOf(v));
128
- }
129
-
130
- public void setLong(long v) throws IOException
131
- {
132
- appendDelimiter();
133
- writer.write(String.valueOf(v));
134
- }
135
-
136
- public void setFloat(float v) throws IOException
137
- {
138
- appendDelimiter();
139
- writer.write(String.valueOf(v));
140
- }
141
-
142
- public void setDouble(double v) throws IOException
143
- {
144
- appendDelimiter();
145
- writer.write(String.valueOf(v));
146
- }
147
-
148
- public void setBigDecimal(BigDecimal v) throws IOException
149
- {
150
- appendDelimiter();
151
- writer.write(String.valueOf(v));
152
- }
153
-
154
- public void setString(String v) throws IOException
155
- {
156
- appendDelimiter();
157
- setEscapedString(v);
158
- }
159
-
160
- public void setNString(String v) throws IOException
161
- {
162
- appendDelimiter();
163
- setEscapedString(v);
164
- }
165
-
166
- public void setBytes(byte[] v) throws IOException
167
- {
168
- appendDelimiter();
169
- setEscapedString(String.valueOf(v));
170
- }
171
-
172
- public void setSqlDate(Date v, int sqlType) throws IOException
173
- {
174
- appendDelimiter();
175
- writer.write(v.toString());
176
- }
177
-
178
- public void setSqlTime(Time v, int sqlType) throws IOException
179
- {
180
- appendDelimiter();
181
- writer.write(v.toString());
182
- }
183
-
184
- public void setSqlTimestamp(Timestamp v, int sqlType) throws IOException
185
- {
186
- appendDelimiter();
187
- writer.write(v.toString());
188
- }
189
-
190
- // Escape \, \n, \t, \r
191
- // Remove \0
192
- private void setEscapedString(String v) throws IOException{
193
- for (char c : v.toCharArray()) {
194
- String s;
195
- switch (c) {
196
- case '\\':
197
- s = "\\\\";
198
- break;
199
- case '\n':
200
- s = "\\n";
201
- break;
202
- case '\t':
203
- s = "\\t";
204
- break;
205
- case '\r':
206
- s = "\\r";
207
- break;
208
- case 0:
209
- s = "";
210
- break;
211
- default:
212
- s = String.valueOf(c);
213
- }
214
- writer.write(s);
215
- }
216
- }
217
- }
1
+ package org.embulk.output.postgresql;
2
+
3
+ import java.io.File;
4
+ import java.io.FileOutputStream;
5
+ import java.io.Writer;
6
+ import java.io.BufferedWriter;
7
+ import java.io.OutputStreamWriter;
8
+ import java.io.IOException;
9
+ import java.nio.charset.Charset;
10
+ import java.math.BigDecimal;
11
+ import java.sql.Date;
12
+ import java.sql.Time;
13
+ import java.sql.Timestamp;
14
+ import java.sql.SQLException;
15
+ import org.embulk.spi.Exec;
16
+ import org.embulk.output.jdbc.JdbcSchema;
17
+ import org.embulk.output.jdbc.BatchInsert;
18
+
19
+ public abstract class AbstractPostgreSQLCopyBatchInsert
20
+ implements BatchInsert
21
+ {
22
+ protected static final Charset FILE_CHARSET = Charset.forName("UTF-8");
23
+
24
+ protected static final String nullString = "\\N";
25
+ protected static final String newLineString = "\n";
26
+ protected static final String delimiterString = "\t";
27
+
28
+ protected File currentFile;
29
+ protected BufferedWriter writer;
30
+ protected int index;
31
+ protected int batchRows;
32
+
33
+ protected AbstractPostgreSQLCopyBatchInsert() throws IOException
34
+ {
35
+ this.index = 0;
36
+ openNewFile();
37
+ }
38
+
39
+ private File createTempFile() throws IOException
40
+ {
41
+ return File.createTempFile("embulk-output-postgres-copy-", ".tsv.tmp"); // TODO configurable temporary file path
42
+ }
43
+
44
+ protected File openNewFile() throws IOException
45
+ {
46
+ File newFile = createTempFile();
47
+ File oldFile = closeCurrentFile();
48
+ this.writer = openWriter(newFile);
49
+ currentFile = newFile;
50
+ return oldFile;
51
+ }
52
+
53
+ protected File closeCurrentFile() throws IOException
54
+ {
55
+ if(writer != null) {
56
+ writer.close();
57
+ writer = null;
58
+ }
59
+ return currentFile;
60
+ }
61
+
62
+ protected BufferedWriter openWriter(File newFile) throws IOException
63
+ {
64
+ return new BufferedWriter(new OutputStreamWriter(new FileOutputStream(newFile), FILE_CHARSET));
65
+ }
66
+
67
+ public int getBatchWeight()
68
+ {
69
+ long fsize = currentFile.length();
70
+ if (fsize > Integer.MAX_VALUE) {
71
+ return Integer.MAX_VALUE;
72
+ } else {
73
+ return (int) fsize;
74
+ }
75
+ }
76
+
77
+ public void finish() throws IOException, SQLException
78
+ {
79
+ closeCurrentFile(); // this is necessary to make getBatchWeight() work
80
+ if (getBatchWeight() != 0) {
81
+ flush();
82
+ }
83
+ }
84
+
85
+ public void add() throws IOException
86
+ {
87
+ writer.write(newLineString);
88
+ batchRows++;
89
+ index = 0;
90
+ }
91
+
92
+ private void appendDelimiter() throws IOException
93
+ {
94
+ if(index != 0) {
95
+ writer.write(delimiterString);
96
+ }
97
+ index++;
98
+ }
99
+
100
+ public void setNull(int sqlType) throws IOException
101
+ {
102
+ appendDelimiter();
103
+ writer.write(nullString);
104
+ }
105
+
106
+ public void setBoolean(boolean v) throws IOException
107
+ {
108
+ appendDelimiter();
109
+ writer.write(String.valueOf(v));
110
+ }
111
+
112
+ public void setByte(byte v) throws IOException
113
+ {
114
+ appendDelimiter();
115
+ setEscapedString(String.valueOf(v));
116
+ }
117
+
118
+ public void setShort(short v) throws IOException
119
+ {
120
+ appendDelimiter();
121
+ writer.write(String.valueOf(v));
122
+ }
123
+
124
+ public void setInt(int v) throws IOException
125
+ {
126
+ appendDelimiter();
127
+ writer.write(String.valueOf(v));
128
+ }
129
+
130
+ public void setLong(long v) throws IOException
131
+ {
132
+ appendDelimiter();
133
+ writer.write(String.valueOf(v));
134
+ }
135
+
136
+ public void setFloat(float v) throws IOException
137
+ {
138
+ appendDelimiter();
139
+ writer.write(String.valueOf(v));
140
+ }
141
+
142
+ public void setDouble(double v) throws IOException
143
+ {
144
+ appendDelimiter();
145
+ writer.write(String.valueOf(v));
146
+ }
147
+
148
+ public void setBigDecimal(BigDecimal v) throws IOException
149
+ {
150
+ appendDelimiter();
151
+ writer.write(String.valueOf(v));
152
+ }
153
+
154
+ public void setString(String v) throws IOException
155
+ {
156
+ appendDelimiter();
157
+ setEscapedString(v);
158
+ }
159
+
160
+ public void setNString(String v) throws IOException
161
+ {
162
+ appendDelimiter();
163
+ setEscapedString(v);
164
+ }
165
+
166
+ public void setBytes(byte[] v) throws IOException
167
+ {
168
+ appendDelimiter();
169
+ setEscapedString(String.valueOf(v));
170
+ }
171
+
172
+ public void setSqlDate(Date v, int sqlType) throws IOException
173
+ {
174
+ appendDelimiter();
175
+ writer.write(v.toString());
176
+ }
177
+
178
+ public void setSqlTime(Time v, int sqlType) throws IOException
179
+ {
180
+ appendDelimiter();
181
+ writer.write(v.toString());
182
+ }
183
+
184
+ public void setSqlTimestamp(Timestamp v, int sqlType) throws IOException
185
+ {
186
+ appendDelimiter();
187
+ writer.write(v.toString());
188
+ }
189
+
190
+ // Escape \, \n, \t, \r
191
+ // Remove \0
192
+ private void setEscapedString(String v) throws IOException{
193
+ for (char c : v.toCharArray()) {
194
+ String s;
195
+ switch (c) {
196
+ case '\\':
197
+ s = "\\\\";
198
+ break;
199
+ case '\n':
200
+ s = "\\n";
201
+ break;
202
+ case '\t':
203
+ s = "\\t";
204
+ break;
205
+ case '\r':
206
+ s = "\\r";
207
+ break;
208
+ case 0:
209
+ s = "";
210
+ break;
211
+ default:
212
+ s = String.valueOf(c);
213
+ }
214
+ writer.write(s);
215
+ }
216
+ }
217
+ }
@@ -1,73 +1,73 @@
1
- package org.embulk.output.postgresql;
2
-
3
- import java.io.File;
4
- import java.io.IOException;
5
- import java.io.FileInputStream;
6
- import java.sql.Connection;
7
- import java.sql.SQLException;
8
- import org.slf4j.Logger;
9
- import org.postgresql.copy.CopyManager;
10
- import org.postgresql.core.BaseConnection;
11
- import org.embulk.spi.Exec;
12
- import org.embulk.output.jdbc.JdbcSchema;
13
-
14
- public class PostgreSQLCopyBatchInsert
15
- extends AbstractPostgreSQLCopyBatchInsert
16
- {
17
- private final Logger logger = Exec.getLogger(PostgreSQLCopyBatchInsert.class);
18
- private final PostgreSQLOutputConnector connector;
19
-
20
- private PostgreSQLOutputConnection connection = null;
21
- private CopyManager copyManager = null;
22
- private String copySql = null;
23
- private long totalRows;
24
-
25
- public PostgreSQLCopyBatchInsert(PostgreSQLOutputConnector connector) throws IOException, SQLException
26
- {
27
- super();
28
- this.connector = connector;
29
- }
30
-
31
- @Override
32
- public void prepare(String loadTable, JdbcSchema insertSchema) throws SQLException
33
- {
34
- this.connection = connector.connect(true);
35
- this.copySql = connection.buildCopySql(loadTable, insertSchema);
36
- this.copyManager = connection.newCopyManager();
37
- logger.info("Copy SQL: "+copySql);
38
- }
39
-
40
- @Override
41
- public void flush() throws IOException, SQLException
42
- {
43
- File file = closeCurrentFile(); // flush buffered data in writer
44
-
45
- logger.info(String.format("Loading %,d rows (%,d bytes)", batchRows, file.length()));
46
- long startTime = System.currentTimeMillis();
47
- FileInputStream in = new FileInputStream(file);
48
- try {
49
- // TODO check age of connection and call isValid if it's old and reconnect if it's invalid
50
- copyManager.copyIn(copySql, in);
51
- } finally {
52
- in.close();
53
- }
54
- double seconds = (System.currentTimeMillis() - startTime) / 1000.0;
55
-
56
- totalRows += batchRows;
57
- batchRows = 0;
58
- logger.info(String.format("> %.2f seconds (loaded %,d rows in total)", seconds, totalRows));
59
-
60
- openNewFile();
61
- file.delete();
62
- }
63
-
64
- @Override
65
- public void close() throws IOException, SQLException
66
- {
67
- closeCurrentFile().delete();
68
- if (connection != null) {
69
- connection.close();
70
- connection = null;
71
- }
72
- }
73
- }
1
+ package org.embulk.output.postgresql;
2
+
3
+ import java.io.File;
4
+ import java.io.IOException;
5
+ import java.io.FileInputStream;
6
+ import java.sql.Connection;
7
+ import java.sql.SQLException;
8
+ import org.slf4j.Logger;
9
+ import org.postgresql.copy.CopyManager;
10
+ import org.postgresql.core.BaseConnection;
11
+ import org.embulk.spi.Exec;
12
+ import org.embulk.output.jdbc.JdbcSchema;
13
+
14
+ public class PostgreSQLCopyBatchInsert
15
+ extends AbstractPostgreSQLCopyBatchInsert
16
+ {
17
+ private final Logger logger = Exec.getLogger(PostgreSQLCopyBatchInsert.class);
18
+ private final PostgreSQLOutputConnector connector;
19
+
20
+ private PostgreSQLOutputConnection connection = null;
21
+ private CopyManager copyManager = null;
22
+ private String copySql = null;
23
+ private long totalRows;
24
+
25
+ public PostgreSQLCopyBatchInsert(PostgreSQLOutputConnector connector) throws IOException, SQLException
26
+ {
27
+ super();
28
+ this.connector = connector;
29
+ }
30
+
31
+ @Override
32
+ public void prepare(String loadTable, JdbcSchema insertSchema) throws SQLException
33
+ {
34
+ this.connection = connector.connect(true);
35
+ this.copySql = connection.buildCopySql(loadTable, insertSchema);
36
+ this.copyManager = connection.newCopyManager();
37
+ logger.info("Copy SQL: "+copySql);
38
+ }
39
+
40
+ @Override
41
+ public void flush() throws IOException, SQLException
42
+ {
43
+ File file = closeCurrentFile(); // flush buffered data in writer
44
+
45
+ logger.info(String.format("Loading %,d rows (%,d bytes)", batchRows, file.length()));
46
+ long startTime = System.currentTimeMillis();
47
+ FileInputStream in = new FileInputStream(file);
48
+ try {
49
+ // TODO check age of connection and call isValid if it's old and reconnect if it's invalid
50
+ copyManager.copyIn(copySql, in);
51
+ } finally {
52
+ in.close();
53
+ }
54
+ double seconds = (System.currentTimeMillis() - startTime) / 1000.0;
55
+
56
+ totalRows += batchRows;
57
+ batchRows = 0;
58
+ logger.info(String.format("> %.2f seconds (loaded %,d rows in total)", seconds, totalRows));
59
+
60
+ openNewFile();
61
+ file.delete();
62
+ }
63
+
64
+ @Override
65
+ public void close() throws IOException, SQLException
66
+ {
67
+ closeCurrentFile().delete();
68
+ if (connection != null) {
69
+ connection.close();
70
+ connection = null;
71
+ }
72
+ }
73
+ }
@@ -1,40 +1,40 @@
1
- package org.embulk.output.postgresql;
2
-
3
- import java.util.Properties;
4
- import java.sql.Driver;
5
- import java.sql.Connection;
6
- import java.sql.SQLException;
7
- import org.embulk.output.jdbc.JdbcOutputConnector;
8
- import org.embulk.output.jdbc.JdbcOutputConnection;
9
-
10
- public class PostgreSQLOutputConnector
11
- implements JdbcOutputConnector
12
- {
13
- private static final Driver driver = new org.postgresql.Driver();
14
-
15
- private final String url;
16
- private final Properties properties;
17
- private final String schemaName;
18
-
19
- public PostgreSQLOutputConnector(String url, Properties properties, String schemaName)
20
- {
21
- this.url = url;
22
- this.properties = properties;
23
- this.schemaName = schemaName;
24
- }
25
-
26
- @Override
27
- public PostgreSQLOutputConnection connect(boolean autoCommit) throws SQLException
28
- {
29
- Connection c = driver.connect(url, properties);
30
- try {
31
- PostgreSQLOutputConnection con = new PostgreSQLOutputConnection(c, schemaName, autoCommit);
32
- c = null;
33
- return con;
34
- } finally {
35
- if (c != null) {
36
- c.close();
37
- }
38
- }
39
- }
40
- }
1
+ package org.embulk.output.postgresql;
2
+
3
+ import java.util.Properties;
4
+ import java.sql.Driver;
5
+ import java.sql.Connection;
6
+ import java.sql.SQLException;
7
+ import org.embulk.output.jdbc.JdbcOutputConnector;
8
+ import org.embulk.output.jdbc.JdbcOutputConnection;
9
+
10
+ public class PostgreSQLOutputConnector
11
+ implements JdbcOutputConnector
12
+ {
13
+ private static final Driver driver = new org.postgresql.Driver();
14
+
15
+ private final String url;
16
+ private final Properties properties;
17
+ private final String schemaName;
18
+
19
+ public PostgreSQLOutputConnector(String url, Properties properties, String schemaName)
20
+ {
21
+ this.url = url;
22
+ this.properties = properties;
23
+ this.schemaName = schemaName;
24
+ }
25
+
26
+ @Override
27
+ public PostgreSQLOutputConnection connect(boolean autoCommit) throws SQLException
28
+ {
29
+ Connection c = driver.connect(url, properties);
30
+ try {
31
+ PostgreSQLOutputConnection con = new PostgreSQLOutputConnection(c, schemaName, autoCommit);
32
+ c = null;
33
+ return con;
34
+ } finally {
35
+ if (c != null) {
36
+ c.close();
37
+ }
38
+ }
39
+ }
40
+ }
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: embulk-output-postgresql
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.3
4
+ version: 0.2.4
5
5
  platform: ruby
6
6
  authors:
7
7
  - Sadayuki Furuhashi
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-04-23 00:00:00.000000000 Z
11
+ date: 2015-05-12 00:00:00.000000000 Z
12
12
  dependencies: []
13
13
  description: Inserts or updates records to a table.
14
14
  email:
@@ -26,12 +26,12 @@ files:
26
26
  - src/main/java/org/embulk/output/postgresql/PostgreSQLOutputConnection.java
27
27
  - src/main/java/org/embulk/output/postgresql/PostgreSQLOutputConnector.java
28
28
  - src/main/java/org/embulk/output/postgresql/PostgresqlBatchUpsert.java
29
- - classpath/embulk-output-jdbc-0.2.3.jar
29
+ - classpath/embulk-output-jdbc-0.2.4.jar
30
+ - classpath/embulk-output-postgresql-0.2.4.jar
31
+ - classpath/jna-4.1.0.jar
32
+ - classpath/jna-platform-4.1.0.jar
30
33
  - classpath/postgresql-9.4-1200-jdbc41.jar
31
- - classpath/embulk-output-postgresql-0.2.3.jar
32
34
  - classpath/slf4j-simple-1.7.7.jar
33
- - classpath/jna-platform-4.1.0.jar
34
- - classpath/jna-4.1.0.jar
35
35
  - classpath/waffle-jna-1.7.jar
36
36
  homepage: https://github.com/embulk/embulk-output-jdbc
37
37
  licenses: