embulk-output-mysql 0.7.6 → 0.7.7
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +1 -0
- data/classpath/{embulk-output-jdbc-0.7.6.jar → embulk-output-jdbc-0.7.7.jar} +0 -0
- data/classpath/{embulk-output-mysql-0.7.6.jar → embulk-output-mysql-0.7.7.jar} +0 -0
- data/src/test/java/org/embulk/output/mysql/MySQLOutputPluginTest.java +229 -0
- data/src/test/resources/mysql/yml/test-insert-before-load.yml +22 -0
- data/src/test/resources/mysql/yml/test-insert-direct-before-load.yml +22 -0
- data/src/test/resources/mysql/yml/test-merge-before-load.yml +22 -0
- data/src/test/resources/mysql/yml/test-merge-direct-before-load.yml +22 -0
- data/src/test/resources/mysql/yml/test-truncate-insert-before-load.yml +22 -0
- metadata +9 -4
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: bf58d4882f474c11125f531945a5048f350c0f39
|
|
4
|
+
data.tar.gz: 2f28afb1f6bf1bcfdc700595f59eaaf0143984c7
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 227b8b0b5a95eb0c464a98a67a2824fb79d93c0491b97bcb729c0b15f624307d86095cdd5c1ed74afc2a424215ca87df070046fdcb2990f83b9398ac72e4e671
|
|
7
|
+
data.tar.gz: 9536f2f57ae9a11a48a0b594b4f67157a8ff35bb83ba3528db9cf05544fec85c60c4ae8a0fd17b979d92e35c5b413e058c41236e0751ad6786fda2e821519fcd
|
data/README.md
CHANGED
|
@@ -29,6 +29,7 @@ MySQL output plugin for Embulk loads records to MySQL.
|
|
|
29
29
|
- **value_type**: This plugin converts input column type (embulk type) into a database type to build a INSERT statement. This value_type option controls the type of the value in a INSERT statement. (string, default: depends on the sql type of the column. Available values options are: `byte`, `short`, `int`, `long`, `double`, `float`, `boolean`, `string`, `nstring`, `date`, `time`, `timestamp`, `decimal`, `json`, `null`, `pass`)
|
|
30
30
|
- **timestamp_format**: If input column type (embulk type) is timestamp and value_type is `string` or `nstring`, this plugin needs to format the timestamp value into a string. This timestamp_format option is used to control the format of the timestamp. (string, default: `%Y-%m-%d %H:%M:%S.%6N`)
|
|
31
31
|
- **timezone**: If input column type (embulk type) is timestamp, this plugin needs to format the timestamp value into a SQL string. In this cases, this timezone option is used to control the timezone. (string, value of default_timezone option is used by default)
|
|
32
|
+
- **before_load**: if set, this SQL will be executed before loading all records. In truncate_insert mode, the SQL will be executed after truncating. replace mode doesn't support this option.
|
|
32
33
|
- **after_load**: if set, this SQL will be executed after loading all records.
|
|
33
34
|
|
|
34
35
|
### Modes
|
|
Binary file
|
|
Binary file
|
|
@@ -34,6 +34,235 @@ public class MySQLOutputPluginTest extends AbstractJdbcOutputPluginTest
|
|
|
34
34
|
enabled = true;
|
|
35
35
|
}
|
|
36
36
|
|
|
37
|
+
@Test
|
|
38
|
+
public void testInsertDirectBeforeLoad() throws Exception
|
|
39
|
+
{
|
|
40
|
+
if (!enabled) {
|
|
41
|
+
return;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
String table = "test1";
|
|
45
|
+
|
|
46
|
+
dropTable(table);
|
|
47
|
+
createTable(table);
|
|
48
|
+
executeSQL(String.format("INSERT INTO %s VALUES('B001', 0, 'z')", table));
|
|
49
|
+
executeSQL(String.format("INSERT INTO %s VALUES('B002', 9, 'z')", table));
|
|
50
|
+
|
|
51
|
+
test("/mysql/yml/test-insert-direct-before-load.yml");
|
|
52
|
+
|
|
53
|
+
List<List<Object>> rows = select(table);
|
|
54
|
+
assertEquals(4, rows.size());
|
|
55
|
+
Iterator<List<Object>> i1 = rows.iterator();
|
|
56
|
+
{
|
|
57
|
+
Iterator<Object> i2 = i1.next().iterator();
|
|
58
|
+
assertEquals("A001", i2.next());
|
|
59
|
+
assertEquals(9, i2.next());
|
|
60
|
+
assertEquals("a", i2.next());
|
|
61
|
+
}
|
|
62
|
+
{
|
|
63
|
+
Iterator<Object> i2 = i1.next().iterator();
|
|
64
|
+
assertEquals("A002", i2.next());
|
|
65
|
+
assertEquals(0, i2.next());
|
|
66
|
+
assertEquals("b", i2.next());
|
|
67
|
+
}
|
|
68
|
+
{
|
|
69
|
+
Iterator<Object> i2 = i1.next().iterator();
|
|
70
|
+
assertEquals("A003", i2.next());
|
|
71
|
+
assertEquals(9, i2.next());
|
|
72
|
+
assertEquals("c", i2.next());
|
|
73
|
+
}
|
|
74
|
+
{
|
|
75
|
+
Iterator<Object> i2 = i1.next().iterator();
|
|
76
|
+
assertEquals("B001", i2.next());
|
|
77
|
+
assertEquals(0, i2.next());
|
|
78
|
+
assertEquals("z", i2.next());
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
@Test
|
|
83
|
+
public void testInsertBeforeLoad() throws Exception
|
|
84
|
+
{
|
|
85
|
+
if (!enabled) {
|
|
86
|
+
return;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
String table = "test1";
|
|
90
|
+
|
|
91
|
+
dropTable(table);
|
|
92
|
+
createTable(table);
|
|
93
|
+
executeSQL(String.format("INSERT INTO %s VALUES('B001', 0, 'z')", table));
|
|
94
|
+
executeSQL(String.format("INSERT INTO %s VALUES('B002', 9, 'z')", table));
|
|
95
|
+
|
|
96
|
+
test("/mysql/yml/test-insert-before-load.yml");
|
|
97
|
+
|
|
98
|
+
List<List<Object>> rows = select(table);
|
|
99
|
+
assertEquals(4, rows.size());
|
|
100
|
+
Iterator<List<Object>> i1 = rows.iterator();
|
|
101
|
+
{
|
|
102
|
+
Iterator<Object> i2 = i1.next().iterator();
|
|
103
|
+
assertEquals("A001", i2.next());
|
|
104
|
+
assertEquals(9, i2.next());
|
|
105
|
+
assertEquals("a", i2.next());
|
|
106
|
+
}
|
|
107
|
+
{
|
|
108
|
+
Iterator<Object> i2 = i1.next().iterator();
|
|
109
|
+
assertEquals("A002", i2.next());
|
|
110
|
+
assertEquals(0, i2.next());
|
|
111
|
+
assertEquals("b", i2.next());
|
|
112
|
+
}
|
|
113
|
+
{
|
|
114
|
+
Iterator<Object> i2 = i1.next().iterator();
|
|
115
|
+
assertEquals("A003", i2.next());
|
|
116
|
+
assertEquals(9, i2.next());
|
|
117
|
+
assertEquals("c", i2.next());
|
|
118
|
+
}
|
|
119
|
+
{
|
|
120
|
+
Iterator<Object> i2 = i1.next().iterator();
|
|
121
|
+
assertEquals("B001", i2.next());
|
|
122
|
+
assertEquals(0, i2.next());
|
|
123
|
+
assertEquals("z", i2.next());
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
@Test
|
|
128
|
+
public void testTruncateInsertBeforeLoad() throws Exception
|
|
129
|
+
{
|
|
130
|
+
if (!enabled) {
|
|
131
|
+
return;
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
String table = "test1";
|
|
135
|
+
|
|
136
|
+
dropTable(table);
|
|
137
|
+
createTable(table);
|
|
138
|
+
executeSQL(String.format("INSERT INTO %s VALUES('B001', 0, 'z')", table));
|
|
139
|
+
executeSQL(String.format("INSERT INTO %s VALUES('B002', 9, 'z')", table));
|
|
140
|
+
|
|
141
|
+
test("/mysql/yml/test-truncate-insert-before-load.yml");
|
|
142
|
+
|
|
143
|
+
List<List<Object>> rows = select(table);
|
|
144
|
+
assertEquals(4, rows.size());
|
|
145
|
+
Iterator<List<Object>> i1 = rows.iterator();
|
|
146
|
+
{
|
|
147
|
+
Iterator<Object> i2 = i1.next().iterator();
|
|
148
|
+
assertEquals("A001", i2.next());
|
|
149
|
+
assertEquals(9, i2.next());
|
|
150
|
+
assertEquals("a", i2.next());
|
|
151
|
+
}
|
|
152
|
+
{
|
|
153
|
+
Iterator<Object> i2 = i1.next().iterator();
|
|
154
|
+
assertEquals("A002", i2.next());
|
|
155
|
+
assertEquals(0, i2.next());
|
|
156
|
+
assertEquals("b", i2.next());
|
|
157
|
+
}
|
|
158
|
+
{
|
|
159
|
+
Iterator<Object> i2 = i1.next().iterator();
|
|
160
|
+
assertEquals("A003", i2.next());
|
|
161
|
+
assertEquals(9, i2.next());
|
|
162
|
+
assertEquals("c", i2.next());
|
|
163
|
+
}
|
|
164
|
+
{
|
|
165
|
+
Iterator<Object> i2 = i1.next().iterator();
|
|
166
|
+
assertEquals("C001", i2.next());
|
|
167
|
+
assertEquals(0, i2.next());
|
|
168
|
+
assertEquals("x", i2.next());
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
@Test
|
|
173
|
+
public void testMergeDirectBeforeLoad() throws Exception
|
|
174
|
+
{
|
|
175
|
+
if (!enabled) {
|
|
176
|
+
return;
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
String table = "test1";
|
|
180
|
+
|
|
181
|
+
dropTable(table);
|
|
182
|
+
createTable(table);
|
|
183
|
+
executeSQL(String.format("INSERT INTO %s VALUES('A002', 1, 'y')", table));
|
|
184
|
+
executeSQL(String.format("INSERT INTO %s VALUES('A003', 1, 'y')", table));
|
|
185
|
+
executeSQL(String.format("INSERT INTO %s VALUES('B001', 0, 'z')", table));
|
|
186
|
+
executeSQL(String.format("INSERT INTO %s VALUES('B002', 9, 'z')", table));
|
|
187
|
+
|
|
188
|
+
test("/mysql/yml/test-merge-direct-before-load.yml");
|
|
189
|
+
|
|
190
|
+
List<List<Object>> rows = select(table);
|
|
191
|
+
assertEquals(4, rows.size());
|
|
192
|
+
Iterator<List<Object>> i1 = rows.iterator();
|
|
193
|
+
{
|
|
194
|
+
Iterator<Object> i2 = i1.next().iterator();
|
|
195
|
+
assertEquals("A001", i2.next());
|
|
196
|
+
assertEquals(9, i2.next());
|
|
197
|
+
assertEquals("a", i2.next());
|
|
198
|
+
}
|
|
199
|
+
{
|
|
200
|
+
Iterator<Object> i2 = i1.next().iterator();
|
|
201
|
+
assertEquals("A002", i2.next());
|
|
202
|
+
assertEquals(0, i2.next());
|
|
203
|
+
assertEquals("b", i2.next());
|
|
204
|
+
}
|
|
205
|
+
{
|
|
206
|
+
Iterator<Object> i2 = i1.next().iterator();
|
|
207
|
+
assertEquals("A003", i2.next());
|
|
208
|
+
assertEquals(9, i2.next());
|
|
209
|
+
assertEquals("c", i2.next());
|
|
210
|
+
}
|
|
211
|
+
{
|
|
212
|
+
Iterator<Object> i2 = i1.next().iterator();
|
|
213
|
+
assertEquals("B001", i2.next());
|
|
214
|
+
assertEquals(0, i2.next());
|
|
215
|
+
assertEquals("z", i2.next());
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
@Test
|
|
220
|
+
public void testMergeBeforeLoad() throws Exception
|
|
221
|
+
{
|
|
222
|
+
if (!enabled) {
|
|
223
|
+
return;
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
String table = "test1";
|
|
227
|
+
|
|
228
|
+
dropTable(table);
|
|
229
|
+
createTable(table);
|
|
230
|
+
executeSQL(String.format("INSERT INTO %s VALUES('A002', 1, 'y')", table));
|
|
231
|
+
executeSQL(String.format("INSERT INTO %s VALUES('A003', 1, 'y')", table));
|
|
232
|
+
executeSQL(String.format("INSERT INTO %s VALUES('B001', 0, 'z')", table));
|
|
233
|
+
executeSQL(String.format("INSERT INTO %s VALUES('B002', 9, 'z')", table));
|
|
234
|
+
|
|
235
|
+
test("/mysql/yml/test-merge-before-load.yml");
|
|
236
|
+
|
|
237
|
+
List<List<Object>> rows = select(table);
|
|
238
|
+
assertEquals(4, rows.size());
|
|
239
|
+
Iterator<List<Object>> i1 = rows.iterator();
|
|
240
|
+
{
|
|
241
|
+
Iterator<Object> i2 = i1.next().iterator();
|
|
242
|
+
assertEquals("A001", i2.next());
|
|
243
|
+
assertEquals(9, i2.next());
|
|
244
|
+
assertEquals("a", i2.next());
|
|
245
|
+
}
|
|
246
|
+
{
|
|
247
|
+
Iterator<Object> i2 = i1.next().iterator();
|
|
248
|
+
assertEquals("A002", i2.next());
|
|
249
|
+
assertEquals(0, i2.next());
|
|
250
|
+
assertEquals("b", i2.next());
|
|
251
|
+
}
|
|
252
|
+
{
|
|
253
|
+
Iterator<Object> i2 = i1.next().iterator();
|
|
254
|
+
assertEquals("A003", i2.next());
|
|
255
|
+
assertEquals(9, i2.next());
|
|
256
|
+
assertEquals("c", i2.next());
|
|
257
|
+
}
|
|
258
|
+
{
|
|
259
|
+
Iterator<Object> i2 = i1.next().iterator();
|
|
260
|
+
assertEquals("B001", i2.next());
|
|
261
|
+
assertEquals(0, i2.next());
|
|
262
|
+
assertEquals("z", i2.next());
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
|
|
37
266
|
@Test
|
|
38
267
|
public void testInsertDirectAfterLoad() throws Exception
|
|
39
268
|
{
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
in:
|
|
2
|
+
type: file
|
|
3
|
+
path_prefix: '/mysql/data/test1.csv'
|
|
4
|
+
parser:
|
|
5
|
+
charset: UTF-8
|
|
6
|
+
newline: CRLF
|
|
7
|
+
type: csv
|
|
8
|
+
delimiter: ','
|
|
9
|
+
quote: ''
|
|
10
|
+
columns:
|
|
11
|
+
- {name: id, type: string}
|
|
12
|
+
- {name: int_item, type: long}
|
|
13
|
+
- {name: varchar_item, type: string}
|
|
14
|
+
out:
|
|
15
|
+
type: mysql
|
|
16
|
+
host: #host#
|
|
17
|
+
database: #database#
|
|
18
|
+
user: #user#
|
|
19
|
+
password: #password#
|
|
20
|
+
table: test1
|
|
21
|
+
mode: insert
|
|
22
|
+
before_load: "delete from test1 where int_item = 9"
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
in:
|
|
2
|
+
type: file
|
|
3
|
+
path_prefix: '/mysql/data/test1.csv'
|
|
4
|
+
parser:
|
|
5
|
+
charset: UTF-8
|
|
6
|
+
newline: CRLF
|
|
7
|
+
type: csv
|
|
8
|
+
delimiter: ','
|
|
9
|
+
quote: ''
|
|
10
|
+
columns:
|
|
11
|
+
- {name: id, type: string}
|
|
12
|
+
- {name: int_item, type: long}
|
|
13
|
+
- {name: varchar_item, type: string}
|
|
14
|
+
out:
|
|
15
|
+
type: mysql
|
|
16
|
+
host: #host#
|
|
17
|
+
database: #database#
|
|
18
|
+
user: #user#
|
|
19
|
+
password: #password#
|
|
20
|
+
table: test1
|
|
21
|
+
mode: insert_direct
|
|
22
|
+
before_load: "delete from test1 where int_item = 9"
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
in:
|
|
2
|
+
type: file
|
|
3
|
+
path_prefix: '/mysql/data/test1.csv'
|
|
4
|
+
parser:
|
|
5
|
+
charset: UTF-8
|
|
6
|
+
newline: CRLF
|
|
7
|
+
type: csv
|
|
8
|
+
delimiter: ','
|
|
9
|
+
quote: ''
|
|
10
|
+
columns:
|
|
11
|
+
- {name: id, type: string}
|
|
12
|
+
- {name: int_item, type: long}
|
|
13
|
+
- {name: varchar_item, type: string}
|
|
14
|
+
out:
|
|
15
|
+
type: mysql
|
|
16
|
+
host: #host#
|
|
17
|
+
database: #database#
|
|
18
|
+
user: #user#
|
|
19
|
+
password: #password#
|
|
20
|
+
table: test1
|
|
21
|
+
mode: merge
|
|
22
|
+
before_load: "delete from test1 where int_item = 9"
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
in:
|
|
2
|
+
type: file
|
|
3
|
+
path_prefix: '/mysql/data/test1.csv'
|
|
4
|
+
parser:
|
|
5
|
+
charset: UTF-8
|
|
6
|
+
newline: CRLF
|
|
7
|
+
type: csv
|
|
8
|
+
delimiter: ','
|
|
9
|
+
quote: ''
|
|
10
|
+
columns:
|
|
11
|
+
- {name: id, type: string}
|
|
12
|
+
- {name: int_item, type: long}
|
|
13
|
+
- {name: varchar_item, type: string}
|
|
14
|
+
out:
|
|
15
|
+
type: mysql
|
|
16
|
+
host: #host#
|
|
17
|
+
database: #database#
|
|
18
|
+
user: #user#
|
|
19
|
+
password: #password#
|
|
20
|
+
table: test1
|
|
21
|
+
mode: merge_direct
|
|
22
|
+
before_load: "delete from test1 where int_item = 9"
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
in:
|
|
2
|
+
type: file
|
|
3
|
+
path_prefix: '/mysql/data/test1.csv'
|
|
4
|
+
parser:
|
|
5
|
+
charset: UTF-8
|
|
6
|
+
newline: CRLF
|
|
7
|
+
type: csv
|
|
8
|
+
delimiter: ','
|
|
9
|
+
quote: ''
|
|
10
|
+
columns:
|
|
11
|
+
- {name: id, type: string}
|
|
12
|
+
- {name: int_item, type: long}
|
|
13
|
+
- {name: varchar_item, type: string}
|
|
14
|
+
out:
|
|
15
|
+
type: mysql
|
|
16
|
+
host: #host#
|
|
17
|
+
database: #database#
|
|
18
|
+
user: #user#
|
|
19
|
+
password: #password#
|
|
20
|
+
table: test1
|
|
21
|
+
mode: truncate_insert
|
|
22
|
+
before_load: "insert into test1 values('C001', 0, 'x')"
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: embulk-output-mysql
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.7.
|
|
4
|
+
version: 0.7.7
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Sadayuki Furuhashi
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2017-
|
|
11
|
+
date: 2017-04-13 00:00:00.000000000 Z
|
|
12
12
|
dependencies: []
|
|
13
13
|
description: Inserts or updates records to a table.
|
|
14
14
|
email:
|
|
@@ -19,8 +19,8 @@ extra_rdoc_files: []
|
|
|
19
19
|
files:
|
|
20
20
|
- README.md
|
|
21
21
|
- build.gradle
|
|
22
|
-
- classpath/embulk-output-jdbc-0.7.
|
|
23
|
-
- classpath/embulk-output-mysql-0.7.
|
|
22
|
+
- classpath/embulk-output-jdbc-0.7.7.jar
|
|
23
|
+
- classpath/embulk-output-mysql-0.7.7.jar
|
|
24
24
|
- classpath/mysql-connector-java-5.1.34.jar
|
|
25
25
|
- lib/embulk/output/mysql.rb
|
|
26
26
|
- src/main/java/org/embulk/output/MySQLOutputPlugin.java
|
|
@@ -30,11 +30,16 @@ files:
|
|
|
30
30
|
- src/test/java/org/embulk/output/mysql/MySQLOutputPluginTest.java
|
|
31
31
|
- src/test/resources/mysql/data/test1.csv
|
|
32
32
|
- src/test/resources/mysql/yml/test-insert-after-load.yml
|
|
33
|
+
- src/test/resources/mysql/yml/test-insert-before-load.yml
|
|
33
34
|
- src/test/resources/mysql/yml/test-insert-direct-after-load.yml
|
|
35
|
+
- src/test/resources/mysql/yml/test-insert-direct-before-load.yml
|
|
34
36
|
- src/test/resources/mysql/yml/test-merge-after-load.yml
|
|
37
|
+
- src/test/resources/mysql/yml/test-merge-before-load.yml
|
|
35
38
|
- src/test/resources/mysql/yml/test-merge-direct-after-load.yml
|
|
39
|
+
- src/test/resources/mysql/yml/test-merge-direct-before-load.yml
|
|
36
40
|
- src/test/resources/mysql/yml/test-replace-after-load.yml
|
|
37
41
|
- src/test/resources/mysql/yml/test-truncate-insert-after-load.yml
|
|
42
|
+
- src/test/resources/mysql/yml/test-truncate-insert-before-load.yml
|
|
38
43
|
homepage: https://github.com/embulk/embulk-output-jdbc
|
|
39
44
|
licenses:
|
|
40
45
|
- Apache 2.0
|