embulk-output-mysql 0.7.6 → 0.7.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +1 -0
- data/classpath/{embulk-output-jdbc-0.7.6.jar → embulk-output-jdbc-0.7.7.jar} +0 -0
- data/classpath/{embulk-output-mysql-0.7.6.jar → embulk-output-mysql-0.7.7.jar} +0 -0
- data/src/test/java/org/embulk/output/mysql/MySQLOutputPluginTest.java +229 -0
- data/src/test/resources/mysql/yml/test-insert-before-load.yml +22 -0
- data/src/test/resources/mysql/yml/test-insert-direct-before-load.yml +22 -0
- data/src/test/resources/mysql/yml/test-merge-before-load.yml +22 -0
- data/src/test/resources/mysql/yml/test-merge-direct-before-load.yml +22 -0
- data/src/test/resources/mysql/yml/test-truncate-insert-before-load.yml +22 -0
- metadata +9 -4
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: bf58d4882f474c11125f531945a5048f350c0f39
|
|
4
|
+
data.tar.gz: 2f28afb1f6bf1bcfdc700595f59eaaf0143984c7
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 227b8b0b5a95eb0c464a98a67a2824fb79d93c0491b97bcb729c0b15f624307d86095cdd5c1ed74afc2a424215ca87df070046fdcb2990f83b9398ac72e4e671
|
|
7
|
+
data.tar.gz: 9536f2f57ae9a11a48a0b594b4f67157a8ff35bb83ba3528db9cf05544fec85c60c4ae8a0fd17b979d92e35c5b413e058c41236e0751ad6786fda2e821519fcd
|
data/README.md
CHANGED
|
@@ -29,6 +29,7 @@ MySQL output plugin for Embulk loads records to MySQL.
|
|
|
29
29
|
- **value_type**: This plugin converts input column type (embulk type) into a database type to build a INSERT statement. This value_type option controls the type of the value in a INSERT statement. (string, default: depends on the sql type of the column. Available values options are: `byte`, `short`, `int`, `long`, `double`, `float`, `boolean`, `string`, `nstring`, `date`, `time`, `timestamp`, `decimal`, `json`, `null`, `pass`)
|
|
30
30
|
- **timestamp_format**: If input column type (embulk type) is timestamp and value_type is `string` or `nstring`, this plugin needs to format the timestamp value into a string. This timestamp_format option is used to control the format of the timestamp. (string, default: `%Y-%m-%d %H:%M:%S.%6N`)
|
|
31
31
|
- **timezone**: If input column type (embulk type) is timestamp, this plugin needs to format the timestamp value into a SQL string. In this cases, this timezone option is used to control the timezone. (string, value of default_timezone option is used by default)
|
|
32
|
+
- **before_load**: if set, this SQL will be executed before loading all records. In truncate_insert mode, the SQL will be executed after truncating. replace mode doesn't support this option.
|
|
32
33
|
- **after_load**: if set, this SQL will be executed after loading all records.
|
|
33
34
|
|
|
34
35
|
### Modes
|
|
Binary file
|
|
Binary file
|
|
@@ -34,6 +34,235 @@ public class MySQLOutputPluginTest extends AbstractJdbcOutputPluginTest
|
|
|
34
34
|
enabled = true;
|
|
35
35
|
}
|
|
36
36
|
|
|
37
|
+
@Test
|
|
38
|
+
public void testInsertDirectBeforeLoad() throws Exception
|
|
39
|
+
{
|
|
40
|
+
if (!enabled) {
|
|
41
|
+
return;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
String table = "test1";
|
|
45
|
+
|
|
46
|
+
dropTable(table);
|
|
47
|
+
createTable(table);
|
|
48
|
+
executeSQL(String.format("INSERT INTO %s VALUES('B001', 0, 'z')", table));
|
|
49
|
+
executeSQL(String.format("INSERT INTO %s VALUES('B002', 9, 'z')", table));
|
|
50
|
+
|
|
51
|
+
test("/mysql/yml/test-insert-direct-before-load.yml");
|
|
52
|
+
|
|
53
|
+
List<List<Object>> rows = select(table);
|
|
54
|
+
assertEquals(4, rows.size());
|
|
55
|
+
Iterator<List<Object>> i1 = rows.iterator();
|
|
56
|
+
{
|
|
57
|
+
Iterator<Object> i2 = i1.next().iterator();
|
|
58
|
+
assertEquals("A001", i2.next());
|
|
59
|
+
assertEquals(9, i2.next());
|
|
60
|
+
assertEquals("a", i2.next());
|
|
61
|
+
}
|
|
62
|
+
{
|
|
63
|
+
Iterator<Object> i2 = i1.next().iterator();
|
|
64
|
+
assertEquals("A002", i2.next());
|
|
65
|
+
assertEquals(0, i2.next());
|
|
66
|
+
assertEquals("b", i2.next());
|
|
67
|
+
}
|
|
68
|
+
{
|
|
69
|
+
Iterator<Object> i2 = i1.next().iterator();
|
|
70
|
+
assertEquals("A003", i2.next());
|
|
71
|
+
assertEquals(9, i2.next());
|
|
72
|
+
assertEquals("c", i2.next());
|
|
73
|
+
}
|
|
74
|
+
{
|
|
75
|
+
Iterator<Object> i2 = i1.next().iterator();
|
|
76
|
+
assertEquals("B001", i2.next());
|
|
77
|
+
assertEquals(0, i2.next());
|
|
78
|
+
assertEquals("z", i2.next());
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
@Test
|
|
83
|
+
public void testInsertBeforeLoad() throws Exception
|
|
84
|
+
{
|
|
85
|
+
if (!enabled) {
|
|
86
|
+
return;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
String table = "test1";
|
|
90
|
+
|
|
91
|
+
dropTable(table);
|
|
92
|
+
createTable(table);
|
|
93
|
+
executeSQL(String.format("INSERT INTO %s VALUES('B001', 0, 'z')", table));
|
|
94
|
+
executeSQL(String.format("INSERT INTO %s VALUES('B002', 9, 'z')", table));
|
|
95
|
+
|
|
96
|
+
test("/mysql/yml/test-insert-before-load.yml");
|
|
97
|
+
|
|
98
|
+
List<List<Object>> rows = select(table);
|
|
99
|
+
assertEquals(4, rows.size());
|
|
100
|
+
Iterator<List<Object>> i1 = rows.iterator();
|
|
101
|
+
{
|
|
102
|
+
Iterator<Object> i2 = i1.next().iterator();
|
|
103
|
+
assertEquals("A001", i2.next());
|
|
104
|
+
assertEquals(9, i2.next());
|
|
105
|
+
assertEquals("a", i2.next());
|
|
106
|
+
}
|
|
107
|
+
{
|
|
108
|
+
Iterator<Object> i2 = i1.next().iterator();
|
|
109
|
+
assertEquals("A002", i2.next());
|
|
110
|
+
assertEquals(0, i2.next());
|
|
111
|
+
assertEquals("b", i2.next());
|
|
112
|
+
}
|
|
113
|
+
{
|
|
114
|
+
Iterator<Object> i2 = i1.next().iterator();
|
|
115
|
+
assertEquals("A003", i2.next());
|
|
116
|
+
assertEquals(9, i2.next());
|
|
117
|
+
assertEquals("c", i2.next());
|
|
118
|
+
}
|
|
119
|
+
{
|
|
120
|
+
Iterator<Object> i2 = i1.next().iterator();
|
|
121
|
+
assertEquals("B001", i2.next());
|
|
122
|
+
assertEquals(0, i2.next());
|
|
123
|
+
assertEquals("z", i2.next());
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
@Test
|
|
128
|
+
public void testTruncateInsertBeforeLoad() throws Exception
|
|
129
|
+
{
|
|
130
|
+
if (!enabled) {
|
|
131
|
+
return;
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
String table = "test1";
|
|
135
|
+
|
|
136
|
+
dropTable(table);
|
|
137
|
+
createTable(table);
|
|
138
|
+
executeSQL(String.format("INSERT INTO %s VALUES('B001', 0, 'z')", table));
|
|
139
|
+
executeSQL(String.format("INSERT INTO %s VALUES('B002', 9, 'z')", table));
|
|
140
|
+
|
|
141
|
+
test("/mysql/yml/test-truncate-insert-before-load.yml");
|
|
142
|
+
|
|
143
|
+
List<List<Object>> rows = select(table);
|
|
144
|
+
assertEquals(4, rows.size());
|
|
145
|
+
Iterator<List<Object>> i1 = rows.iterator();
|
|
146
|
+
{
|
|
147
|
+
Iterator<Object> i2 = i1.next().iterator();
|
|
148
|
+
assertEquals("A001", i2.next());
|
|
149
|
+
assertEquals(9, i2.next());
|
|
150
|
+
assertEquals("a", i2.next());
|
|
151
|
+
}
|
|
152
|
+
{
|
|
153
|
+
Iterator<Object> i2 = i1.next().iterator();
|
|
154
|
+
assertEquals("A002", i2.next());
|
|
155
|
+
assertEquals(0, i2.next());
|
|
156
|
+
assertEquals("b", i2.next());
|
|
157
|
+
}
|
|
158
|
+
{
|
|
159
|
+
Iterator<Object> i2 = i1.next().iterator();
|
|
160
|
+
assertEquals("A003", i2.next());
|
|
161
|
+
assertEquals(9, i2.next());
|
|
162
|
+
assertEquals("c", i2.next());
|
|
163
|
+
}
|
|
164
|
+
{
|
|
165
|
+
Iterator<Object> i2 = i1.next().iterator();
|
|
166
|
+
assertEquals("C001", i2.next());
|
|
167
|
+
assertEquals(0, i2.next());
|
|
168
|
+
assertEquals("x", i2.next());
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
@Test
|
|
173
|
+
public void testMergeDirectBeforeLoad() throws Exception
|
|
174
|
+
{
|
|
175
|
+
if (!enabled) {
|
|
176
|
+
return;
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
String table = "test1";
|
|
180
|
+
|
|
181
|
+
dropTable(table);
|
|
182
|
+
createTable(table);
|
|
183
|
+
executeSQL(String.format("INSERT INTO %s VALUES('A002', 1, 'y')", table));
|
|
184
|
+
executeSQL(String.format("INSERT INTO %s VALUES('A003', 1, 'y')", table));
|
|
185
|
+
executeSQL(String.format("INSERT INTO %s VALUES('B001', 0, 'z')", table));
|
|
186
|
+
executeSQL(String.format("INSERT INTO %s VALUES('B002', 9, 'z')", table));
|
|
187
|
+
|
|
188
|
+
test("/mysql/yml/test-merge-direct-before-load.yml");
|
|
189
|
+
|
|
190
|
+
List<List<Object>> rows = select(table);
|
|
191
|
+
assertEquals(4, rows.size());
|
|
192
|
+
Iterator<List<Object>> i1 = rows.iterator();
|
|
193
|
+
{
|
|
194
|
+
Iterator<Object> i2 = i1.next().iterator();
|
|
195
|
+
assertEquals("A001", i2.next());
|
|
196
|
+
assertEquals(9, i2.next());
|
|
197
|
+
assertEquals("a", i2.next());
|
|
198
|
+
}
|
|
199
|
+
{
|
|
200
|
+
Iterator<Object> i2 = i1.next().iterator();
|
|
201
|
+
assertEquals("A002", i2.next());
|
|
202
|
+
assertEquals(0, i2.next());
|
|
203
|
+
assertEquals("b", i2.next());
|
|
204
|
+
}
|
|
205
|
+
{
|
|
206
|
+
Iterator<Object> i2 = i1.next().iterator();
|
|
207
|
+
assertEquals("A003", i2.next());
|
|
208
|
+
assertEquals(9, i2.next());
|
|
209
|
+
assertEquals("c", i2.next());
|
|
210
|
+
}
|
|
211
|
+
{
|
|
212
|
+
Iterator<Object> i2 = i1.next().iterator();
|
|
213
|
+
assertEquals("B001", i2.next());
|
|
214
|
+
assertEquals(0, i2.next());
|
|
215
|
+
assertEquals("z", i2.next());
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
@Test
|
|
220
|
+
public void testMergeBeforeLoad() throws Exception
|
|
221
|
+
{
|
|
222
|
+
if (!enabled) {
|
|
223
|
+
return;
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
String table = "test1";
|
|
227
|
+
|
|
228
|
+
dropTable(table);
|
|
229
|
+
createTable(table);
|
|
230
|
+
executeSQL(String.format("INSERT INTO %s VALUES('A002', 1, 'y')", table));
|
|
231
|
+
executeSQL(String.format("INSERT INTO %s VALUES('A003', 1, 'y')", table));
|
|
232
|
+
executeSQL(String.format("INSERT INTO %s VALUES('B001', 0, 'z')", table));
|
|
233
|
+
executeSQL(String.format("INSERT INTO %s VALUES('B002', 9, 'z')", table));
|
|
234
|
+
|
|
235
|
+
test("/mysql/yml/test-merge-before-load.yml");
|
|
236
|
+
|
|
237
|
+
List<List<Object>> rows = select(table);
|
|
238
|
+
assertEquals(4, rows.size());
|
|
239
|
+
Iterator<List<Object>> i1 = rows.iterator();
|
|
240
|
+
{
|
|
241
|
+
Iterator<Object> i2 = i1.next().iterator();
|
|
242
|
+
assertEquals("A001", i2.next());
|
|
243
|
+
assertEquals(9, i2.next());
|
|
244
|
+
assertEquals("a", i2.next());
|
|
245
|
+
}
|
|
246
|
+
{
|
|
247
|
+
Iterator<Object> i2 = i1.next().iterator();
|
|
248
|
+
assertEquals("A002", i2.next());
|
|
249
|
+
assertEquals(0, i2.next());
|
|
250
|
+
assertEquals("b", i2.next());
|
|
251
|
+
}
|
|
252
|
+
{
|
|
253
|
+
Iterator<Object> i2 = i1.next().iterator();
|
|
254
|
+
assertEquals("A003", i2.next());
|
|
255
|
+
assertEquals(9, i2.next());
|
|
256
|
+
assertEquals("c", i2.next());
|
|
257
|
+
}
|
|
258
|
+
{
|
|
259
|
+
Iterator<Object> i2 = i1.next().iterator();
|
|
260
|
+
assertEquals("B001", i2.next());
|
|
261
|
+
assertEquals(0, i2.next());
|
|
262
|
+
assertEquals("z", i2.next());
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
|
|
37
266
|
@Test
|
|
38
267
|
public void testInsertDirectAfterLoad() throws Exception
|
|
39
268
|
{
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
in:
|
|
2
|
+
type: file
|
|
3
|
+
path_prefix: '/mysql/data/test1.csv'
|
|
4
|
+
parser:
|
|
5
|
+
charset: UTF-8
|
|
6
|
+
newline: CRLF
|
|
7
|
+
type: csv
|
|
8
|
+
delimiter: ','
|
|
9
|
+
quote: ''
|
|
10
|
+
columns:
|
|
11
|
+
- {name: id, type: string}
|
|
12
|
+
- {name: int_item, type: long}
|
|
13
|
+
- {name: varchar_item, type: string}
|
|
14
|
+
out:
|
|
15
|
+
type: mysql
|
|
16
|
+
host: #host#
|
|
17
|
+
database: #database#
|
|
18
|
+
user: #user#
|
|
19
|
+
password: #password#
|
|
20
|
+
table: test1
|
|
21
|
+
mode: insert
|
|
22
|
+
before_load: "delete from test1 where int_item = 9"
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
in:
|
|
2
|
+
type: file
|
|
3
|
+
path_prefix: '/mysql/data/test1.csv'
|
|
4
|
+
parser:
|
|
5
|
+
charset: UTF-8
|
|
6
|
+
newline: CRLF
|
|
7
|
+
type: csv
|
|
8
|
+
delimiter: ','
|
|
9
|
+
quote: ''
|
|
10
|
+
columns:
|
|
11
|
+
- {name: id, type: string}
|
|
12
|
+
- {name: int_item, type: long}
|
|
13
|
+
- {name: varchar_item, type: string}
|
|
14
|
+
out:
|
|
15
|
+
type: mysql
|
|
16
|
+
host: #host#
|
|
17
|
+
database: #database#
|
|
18
|
+
user: #user#
|
|
19
|
+
password: #password#
|
|
20
|
+
table: test1
|
|
21
|
+
mode: insert_direct
|
|
22
|
+
before_load: "delete from test1 where int_item = 9"
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
in:
|
|
2
|
+
type: file
|
|
3
|
+
path_prefix: '/mysql/data/test1.csv'
|
|
4
|
+
parser:
|
|
5
|
+
charset: UTF-8
|
|
6
|
+
newline: CRLF
|
|
7
|
+
type: csv
|
|
8
|
+
delimiter: ','
|
|
9
|
+
quote: ''
|
|
10
|
+
columns:
|
|
11
|
+
- {name: id, type: string}
|
|
12
|
+
- {name: int_item, type: long}
|
|
13
|
+
- {name: varchar_item, type: string}
|
|
14
|
+
out:
|
|
15
|
+
type: mysql
|
|
16
|
+
host: #host#
|
|
17
|
+
database: #database#
|
|
18
|
+
user: #user#
|
|
19
|
+
password: #password#
|
|
20
|
+
table: test1
|
|
21
|
+
mode: merge
|
|
22
|
+
before_load: "delete from test1 where int_item = 9"
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
in:
|
|
2
|
+
type: file
|
|
3
|
+
path_prefix: '/mysql/data/test1.csv'
|
|
4
|
+
parser:
|
|
5
|
+
charset: UTF-8
|
|
6
|
+
newline: CRLF
|
|
7
|
+
type: csv
|
|
8
|
+
delimiter: ','
|
|
9
|
+
quote: ''
|
|
10
|
+
columns:
|
|
11
|
+
- {name: id, type: string}
|
|
12
|
+
- {name: int_item, type: long}
|
|
13
|
+
- {name: varchar_item, type: string}
|
|
14
|
+
out:
|
|
15
|
+
type: mysql
|
|
16
|
+
host: #host#
|
|
17
|
+
database: #database#
|
|
18
|
+
user: #user#
|
|
19
|
+
password: #password#
|
|
20
|
+
table: test1
|
|
21
|
+
mode: merge_direct
|
|
22
|
+
before_load: "delete from test1 where int_item = 9"
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
in:
|
|
2
|
+
type: file
|
|
3
|
+
path_prefix: '/mysql/data/test1.csv'
|
|
4
|
+
parser:
|
|
5
|
+
charset: UTF-8
|
|
6
|
+
newline: CRLF
|
|
7
|
+
type: csv
|
|
8
|
+
delimiter: ','
|
|
9
|
+
quote: ''
|
|
10
|
+
columns:
|
|
11
|
+
- {name: id, type: string}
|
|
12
|
+
- {name: int_item, type: long}
|
|
13
|
+
- {name: varchar_item, type: string}
|
|
14
|
+
out:
|
|
15
|
+
type: mysql
|
|
16
|
+
host: #host#
|
|
17
|
+
database: #database#
|
|
18
|
+
user: #user#
|
|
19
|
+
password: #password#
|
|
20
|
+
table: test1
|
|
21
|
+
mode: truncate_insert
|
|
22
|
+
before_load: "insert into test1 values('C001', 0, 'x')"
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: embulk-output-mysql
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.7.
|
|
4
|
+
version: 0.7.7
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Sadayuki Furuhashi
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2017-
|
|
11
|
+
date: 2017-04-13 00:00:00.000000000 Z
|
|
12
12
|
dependencies: []
|
|
13
13
|
description: Inserts or updates records to a table.
|
|
14
14
|
email:
|
|
@@ -19,8 +19,8 @@ extra_rdoc_files: []
|
|
|
19
19
|
files:
|
|
20
20
|
- README.md
|
|
21
21
|
- build.gradle
|
|
22
|
-
- classpath/embulk-output-jdbc-0.7.
|
|
23
|
-
- classpath/embulk-output-mysql-0.7.
|
|
22
|
+
- classpath/embulk-output-jdbc-0.7.7.jar
|
|
23
|
+
- classpath/embulk-output-mysql-0.7.7.jar
|
|
24
24
|
- classpath/mysql-connector-java-5.1.34.jar
|
|
25
25
|
- lib/embulk/output/mysql.rb
|
|
26
26
|
- src/main/java/org/embulk/output/MySQLOutputPlugin.java
|
|
@@ -30,11 +30,16 @@ files:
|
|
|
30
30
|
- src/test/java/org/embulk/output/mysql/MySQLOutputPluginTest.java
|
|
31
31
|
- src/test/resources/mysql/data/test1.csv
|
|
32
32
|
- src/test/resources/mysql/yml/test-insert-after-load.yml
|
|
33
|
+
- src/test/resources/mysql/yml/test-insert-before-load.yml
|
|
33
34
|
- src/test/resources/mysql/yml/test-insert-direct-after-load.yml
|
|
35
|
+
- src/test/resources/mysql/yml/test-insert-direct-before-load.yml
|
|
34
36
|
- src/test/resources/mysql/yml/test-merge-after-load.yml
|
|
37
|
+
- src/test/resources/mysql/yml/test-merge-before-load.yml
|
|
35
38
|
- src/test/resources/mysql/yml/test-merge-direct-after-load.yml
|
|
39
|
+
- src/test/resources/mysql/yml/test-merge-direct-before-load.yml
|
|
36
40
|
- src/test/resources/mysql/yml/test-replace-after-load.yml
|
|
37
41
|
- src/test/resources/mysql/yml/test-truncate-insert-after-load.yml
|
|
42
|
+
- src/test/resources/mysql/yml/test-truncate-insert-before-load.yml
|
|
38
43
|
homepage: https://github.com/embulk/embulk-output-jdbc
|
|
39
44
|
licenses:
|
|
40
45
|
- Apache 2.0
|