embulk-input-mysql 0.4.0 → 0.5.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +28 -0
- data/build.gradle +1 -0
- data/classpath/embulk-input-jdbc-0.5.0.jar +0 -0
- data/classpath/{embulk-input-mysql-0.4.0.jar → embulk-input-mysql-0.5.0.jar} +0 -0
- data/src/test/java/org/embulk/input/mysql/ChildFirstClassLoader.java +42 -0
- data/src/test/java/org/embulk/input/mysql/EmbulkPluginTester.java +81 -0
- data/src/test/java/org/embulk/input/mysql/EmptyConfigSource.java +100 -0
- data/src/test/java/org/embulk/input/mysql/MySQLInputPluginTest.java +269 -0
- data/src/test/resources/yml/input-boolean.yml +26 -0
- data/src/test/resources/yml/input-double.yml +26 -0
- data/src/test/resources/yml/input-long.yml +26 -0
- data/src/test/resources/yml/input-string.yml +26 -0
- data/src/test/resources/yml/input-timestamp1.yml +21 -0
- data/src/test/resources/yml/input-timestamp2.yml +21 -0
- data/src/test/resources/yml/input-timestamp3.yml +22 -0
- data/src/test/resources/yml/input-valuetype-decimal.yml +17 -0
- data/src/test/resources/yml/input-valuetype-string.yml +17 -0
- data/src/test/resources/yml/input.yml +15 -0
- metadata +18 -4
- data/classpath/embulk-input-jdbc-0.4.0.jar +0 -0
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 3f02975b24e6df7e4e3fe9eee4366448d02d17ef
|
4
|
+
data.tar.gz: a3134e8c89d0eb787d397ce2cb26ebd3a07e1b95
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: c4ee7883fa9a550063c961de66927cf4687169fc16a544167822e29bfc3f9a191aff0f071dd9a4229a55a99801cc5d5e98d2b1a06c6b8f3e1c404ac4bfcfc144
|
7
|
+
data.tar.gz: 3f15c1095f95c81882606f4315d1ae0d3ae0f8dad518745765fdfdd5856de175a013c0a0b666ef5b665bfa75210dd72b11d40d40366af9e6c2aed6d459ea5f97
|
data/README.md
CHANGED
@@ -31,6 +31,16 @@ MySQL input plugins for Embulk loads records from MySQL.
|
|
31
31
|
- It uses a client-side built statement and fetches all rows at once. This may cause OutOfMemoryError.
|
32
32
|
- Internally, `useCursorFetch=false` is used and `java.sql.Statement.setFetchSize` is not set.
|
33
33
|
- **options**: extra JDBC properties (hash, default: {})
|
34
|
+
- **default_timezone**: If the sql type of a column is `date`/`time`/`datetime` and the embulk type is `string`, column values are formatted int this default_timezone. You can overwrite timezone for each columns using column_options option. (string, default: `UTC`)
|
35
|
+
- **column_options**: advanced: a key-value pairs where key is a column name and value is options for the column.
|
36
|
+
- **value_type**: embulk get values from database as this value_type. Typically, the value_type determines `getXXX` method of `java.sql.PreparedStatement`.
|
37
|
+
(string, default: depends on the sql type of the column. Available values options are: `long`, `double`, `float`, `decimal`, `boolean`, `string`, `date`, `time`, `timestamp`)
|
38
|
+
- **type**: Column values are converted to this embulk type.
|
39
|
+
Available values options are: `boolean`, `long`, `double`, `string`, `timestamp`).
|
40
|
+
By default, the embulk type is determined according to the sql type of the column (or value_type if specified).
|
41
|
+
- **timestamp_format**: If the sql type of the column is `date`/`time`/`datetime` and the embulk type is `string`, column values are formatted by this timestamp_format. And if the embulk type is `timestamp`, this timestamp_format will be used in the output plugin. (string, default : `%Y-%m-%d` for `date`, `%H:%M:%S` for `time`, `%Y-%m-%d %H:%M:%S` for `timestamp`)
|
42
|
+
- **timezone**: If the sql type of the column is `date`/`time`/`datetime` and the embulk type is `string`, column values are formatted in this timezone.
|
43
|
+
(string, value of default_timezone option is used by default)
|
34
44
|
|
35
45
|
## Example
|
36
46
|
|
@@ -62,6 +72,24 @@ in:
|
|
62
72
|
ON t1.id = t2.t1_id
|
63
73
|
```
|
64
74
|
|
75
|
+
Advanced configuration:
|
76
|
+
|
77
|
+
```yaml
|
78
|
+
in:
|
79
|
+
type: mysql
|
80
|
+
host: localhost
|
81
|
+
user: myuser
|
82
|
+
password: ""
|
83
|
+
database: my_database
|
84
|
+
table: "my_table"
|
85
|
+
select: "col1, col2, col3"
|
86
|
+
where: "col4 != 'a'"
|
87
|
+
column_options:
|
88
|
+
col1: {type: long}
|
89
|
+
col3: {type: string, timestamp_format: "%Y/%m/%d", timezone: "+0900"}
|
90
|
+
|
91
|
+
```
|
92
|
+
|
65
93
|
## Build
|
66
94
|
|
67
95
|
```
|
data/build.gradle
CHANGED
Binary file
|
Binary file
|
@@ -0,0 +1,42 @@
|
|
1
|
+
package org.embulk.input.mysql;
|
2
|
+
|
3
|
+
import java.net.URL;
|
4
|
+
import java.net.URLClassLoader;
|
5
|
+
import java.util.List;
|
6
|
+
|
7
|
+
public class ChildFirstClassLoader
|
8
|
+
extends URLClassLoader
|
9
|
+
{
|
10
|
+
public ChildFirstClassLoader(List<URL> urls, ClassLoader parent)
|
11
|
+
{
|
12
|
+
super(urls.toArray(new URL[urls.size()]), parent);
|
13
|
+
}
|
14
|
+
|
15
|
+
@Override
|
16
|
+
protected Class<?> loadClass(String name, boolean resolve)
|
17
|
+
throws ClassNotFoundException
|
18
|
+
{
|
19
|
+
synchronized (getClassLoadingLock(name)) {
|
20
|
+
Class<?> loadedClass = findLoadedClass(name);
|
21
|
+
if (loadedClass != null) {
|
22
|
+
return resolveClass(loadedClass, resolve);
|
23
|
+
}
|
24
|
+
|
25
|
+
try {
|
26
|
+
return resolveClass(findClass(name), resolve);
|
27
|
+
} catch (ClassNotFoundException ignored) {
|
28
|
+
}
|
29
|
+
|
30
|
+
return super.loadClass(name, resolve);
|
31
|
+
}
|
32
|
+
}
|
33
|
+
|
34
|
+
private Class<?> resolveClass(Class<?> clazz, boolean resolve)
|
35
|
+
{
|
36
|
+
if (resolve) {
|
37
|
+
resolveClass(clazz);
|
38
|
+
}
|
39
|
+
return clazz;
|
40
|
+
}
|
41
|
+
|
42
|
+
}
|
@@ -0,0 +1,81 @@
|
|
1
|
+
package org.embulk.input.mysql;
|
2
|
+
|
3
|
+
import java.io.File;
|
4
|
+
import java.util.ArrayList;
|
5
|
+
import java.util.Arrays;
|
6
|
+
import java.util.List;
|
7
|
+
|
8
|
+
import org.embulk.EmbulkService;
|
9
|
+
import org.embulk.config.ConfigLoader;
|
10
|
+
import org.embulk.config.ConfigSource;
|
11
|
+
import org.embulk.exec.BulkLoader;
|
12
|
+
import org.embulk.exec.ExecutionResult;
|
13
|
+
import org.embulk.plugin.InjectedPluginSource;
|
14
|
+
import org.embulk.spi.ExecSession;
|
15
|
+
|
16
|
+
import com.google.inject.Binder;
|
17
|
+
import com.google.inject.Injector;
|
18
|
+
import com.google.inject.Module;
|
19
|
+
|
20
|
+
public class EmbulkPluginTester
|
21
|
+
{
|
22
|
+
private static class PluginDefinition
|
23
|
+
{
|
24
|
+
public final Class<?> iface;
|
25
|
+
public final String name;
|
26
|
+
public final Class<?> impl;
|
27
|
+
|
28
|
+
|
29
|
+
public PluginDefinition(Class<?> iface, String name, Class<?> impl)
|
30
|
+
{
|
31
|
+
this.iface = iface;
|
32
|
+
this.name = name;
|
33
|
+
this.impl = impl;
|
34
|
+
}
|
35
|
+
|
36
|
+
}
|
37
|
+
|
38
|
+
private final List<PluginDefinition> plugins = new ArrayList<PluginDefinition>();
|
39
|
+
|
40
|
+
|
41
|
+
|
42
|
+
public EmbulkPluginTester()
|
43
|
+
{
|
44
|
+
}
|
45
|
+
|
46
|
+
public EmbulkPluginTester(Class<?> iface, String name, Class<?> impl)
|
47
|
+
{
|
48
|
+
addPlugin(iface, name, impl);
|
49
|
+
}
|
50
|
+
|
51
|
+
public void addPlugin(Class<?> iface, String name, Class<?> impl)
|
52
|
+
{
|
53
|
+
plugins.add(new PluginDefinition(iface, name, impl));
|
54
|
+
}
|
55
|
+
|
56
|
+
public void run(String ymlPath) throws Exception
|
57
|
+
{
|
58
|
+
EmbulkService service = new EmbulkService(new EmptyConfigSource()) {
|
59
|
+
@Override
|
60
|
+
protected Iterable<? extends Module> getAdditionalModules(ConfigSource systemConfig)
|
61
|
+
{
|
62
|
+
return Arrays.asList(new Module()
|
63
|
+
{
|
64
|
+
@Override
|
65
|
+
public void configure(Binder binder)
|
66
|
+
{
|
67
|
+
for (PluginDefinition plugin : plugins) {
|
68
|
+
InjectedPluginSource.registerPluginTo(binder, plugin.iface, plugin.name, plugin.impl);
|
69
|
+
}
|
70
|
+
}
|
71
|
+
});
|
72
|
+
}
|
73
|
+
};
|
74
|
+
Injector injector = service.getInjector();
|
75
|
+
ConfigSource config = injector.getInstance(ConfigLoader.class).fromYamlFile(new File(ymlPath));
|
76
|
+
ExecSession session = new ExecSession(injector, config);
|
77
|
+
BulkLoader loader = injector.getInstance(BulkLoader.class);
|
78
|
+
ExecutionResult result = loader.run(session, config);
|
79
|
+
}
|
80
|
+
|
81
|
+
}
|
@@ -0,0 +1,100 @@
|
|
1
|
+
package org.embulk.input.mysql;
|
2
|
+
|
3
|
+
import java.util.Collections;
|
4
|
+
import java.util.List;
|
5
|
+
import java.util.Map.Entry;
|
6
|
+
|
7
|
+
import org.embulk.config.ConfigSource;
|
8
|
+
import org.embulk.config.DataSource;
|
9
|
+
|
10
|
+
import com.fasterxml.jackson.databind.JsonNode;
|
11
|
+
import com.fasterxml.jackson.databind.node.ObjectNode;
|
12
|
+
|
13
|
+
public class EmptyConfigSource implements ConfigSource
|
14
|
+
{
|
15
|
+
|
16
|
+
@Override
|
17
|
+
public <E> E get(Class<E> type, String attrName)
|
18
|
+
{
|
19
|
+
return null;
|
20
|
+
}
|
21
|
+
|
22
|
+
@Override
|
23
|
+
public <E> E get(Class<E> type, String attrName, E defaultValue)
|
24
|
+
{
|
25
|
+
return defaultValue;
|
26
|
+
}
|
27
|
+
|
28
|
+
@Override
|
29
|
+
public List<String> getAttributeNames()
|
30
|
+
{
|
31
|
+
return Collections.emptyList();
|
32
|
+
}
|
33
|
+
|
34
|
+
@Override
|
35
|
+
public Iterable<Entry<String, JsonNode>> getAttributes()
|
36
|
+
{
|
37
|
+
return Collections.emptyList();
|
38
|
+
}
|
39
|
+
|
40
|
+
@Override
|
41
|
+
public ObjectNode getObjectNode()
|
42
|
+
{
|
43
|
+
return null;
|
44
|
+
}
|
45
|
+
|
46
|
+
@Override
|
47
|
+
public boolean isEmpty()
|
48
|
+
{
|
49
|
+
return true;
|
50
|
+
}
|
51
|
+
|
52
|
+
@Override
|
53
|
+
public ConfigSource deepCopy()
|
54
|
+
{
|
55
|
+
return this;
|
56
|
+
}
|
57
|
+
|
58
|
+
@Override
|
59
|
+
public ConfigSource getNested(String s)
|
60
|
+
{
|
61
|
+
return null;
|
62
|
+
}
|
63
|
+
|
64
|
+
@Override
|
65
|
+
public ConfigSource getNestedOrSetEmpty(String s)
|
66
|
+
{
|
67
|
+
return null;
|
68
|
+
}
|
69
|
+
|
70
|
+
@Override
|
71
|
+
public <T> T loadConfig(Class<T> class1)
|
72
|
+
{
|
73
|
+
return null;
|
74
|
+
}
|
75
|
+
|
76
|
+
@Override
|
77
|
+
public ConfigSource merge(DataSource datasource)
|
78
|
+
{
|
79
|
+
return null;
|
80
|
+
}
|
81
|
+
|
82
|
+
@Override
|
83
|
+
public ConfigSource set(String s, Object obj)
|
84
|
+
{
|
85
|
+
return null;
|
86
|
+
}
|
87
|
+
|
88
|
+
@Override
|
89
|
+
public ConfigSource setAll(DataSource datasource)
|
90
|
+
{
|
91
|
+
return null;
|
92
|
+
}
|
93
|
+
|
94
|
+
@Override
|
95
|
+
public ConfigSource setNested(String s, DataSource datasource)
|
96
|
+
{
|
97
|
+
return null;
|
98
|
+
}
|
99
|
+
|
100
|
+
}
|
@@ -0,0 +1,269 @@
|
|
1
|
+
package org.embulk.input.mysql;
|
2
|
+
|
3
|
+
import java.io.File;
|
4
|
+
import java.io.IOException;
|
5
|
+
import java.net.URISyntaxException;
|
6
|
+
import java.nio.charset.Charset;
|
7
|
+
import java.nio.file.FileSystem;
|
8
|
+
import java.nio.file.FileSystems;
|
9
|
+
import java.nio.file.Files;
|
10
|
+
import java.sql.Connection;
|
11
|
+
import java.sql.DriverManager;
|
12
|
+
import java.sql.SQLException;
|
13
|
+
import java.sql.Statement;
|
14
|
+
import java.util.Arrays;
|
15
|
+
import java.util.List;
|
16
|
+
|
17
|
+
import org.embulk.input.MySQLInputPlugin;
|
18
|
+
import org.embulk.spi.InputPlugin;
|
19
|
+
import org.junit.BeforeClass;
|
20
|
+
import org.junit.Test;
|
21
|
+
|
22
|
+
import static org.junit.Assert.assertEquals;
|
23
|
+
|
24
|
+
public class MySQLInputPluginTest
|
25
|
+
{
|
26
|
+
private static boolean prepared = false;
|
27
|
+
|
28
|
+
@BeforeClass
|
29
|
+
public static void prepare() throws SQLException
|
30
|
+
{
|
31
|
+
Connection connection;
|
32
|
+
try {
|
33
|
+
connection = DriverManager.getConnection("jdbc:mysql://localhost/TESTDB", "TEST_USER", "test_pw");
|
34
|
+
} catch (SQLException e) {
|
35
|
+
System.err.println(e);
|
36
|
+
System.err.println("Warning: prepare a schema on MySQL (database = 'TESTDB', user = 'TEST_USER', password = 'test_pw').");
|
37
|
+
return;
|
38
|
+
}
|
39
|
+
|
40
|
+
try {
|
41
|
+
try (Statement statement = connection.createStatement()) {
|
42
|
+
String drop1 = "drop table if exists test1";
|
43
|
+
statement.execute(drop1);
|
44
|
+
|
45
|
+
String create1 =
|
46
|
+
"create table test1 ("
|
47
|
+
+ "c1 tinyint,"
|
48
|
+
+ "c2 smallint,"
|
49
|
+
+ "c3 int,"
|
50
|
+
+ "c4 bigint,"
|
51
|
+
+ "c5 float,"
|
52
|
+
+ "c6 double,"
|
53
|
+
+ "c7 decimal(4,0),"
|
54
|
+
+ "c8 decimal(20,2),"
|
55
|
+
+ "c9 char(4),"
|
56
|
+
+ "c10 varchar(4),"
|
57
|
+
+ "c11 date,"
|
58
|
+
+ "c12 datetime,"
|
59
|
+
+ "c13 timestamp,"
|
60
|
+
+ "c14 time,"
|
61
|
+
+ "c15 datetime(6));";
|
62
|
+
statement.execute(create1);
|
63
|
+
|
64
|
+
String insert1 =
|
65
|
+
"insert into test1 values("
|
66
|
+
+ "null,"
|
67
|
+
+ "null,"
|
68
|
+
+ "null,"
|
69
|
+
+ "null,"
|
70
|
+
+ "null,"
|
71
|
+
+ "null,"
|
72
|
+
+ "null,"
|
73
|
+
+ "null,"
|
74
|
+
+ "null,"
|
75
|
+
+ "null,"
|
76
|
+
+ "null,"
|
77
|
+
+ "null,"
|
78
|
+
+ "'2015-06-04 23:45:06',"
|
79
|
+
+ "null,"
|
80
|
+
+ "null);";
|
81
|
+
statement.executeUpdate(insert1);
|
82
|
+
|
83
|
+
String insert2 =
|
84
|
+
"insert into test1 values("
|
85
|
+
+ "99,"
|
86
|
+
+ "9999,"
|
87
|
+
+ "-99999999,"
|
88
|
+
+ "-9999999999999999,"
|
89
|
+
+ "1.2345,"
|
90
|
+
+ "1.234567890123,"
|
91
|
+
+ "-1234,"
|
92
|
+
+ "123456789012345678.12,"
|
93
|
+
+ "'5678',"
|
94
|
+
+ "'xy',"
|
95
|
+
+ "'2015-06-04',"
|
96
|
+
+ "'2015-06-04 12:34:56',"
|
97
|
+
+ "'2015-06-04 23:45:06',"
|
98
|
+
+ "'08:04:02',"
|
99
|
+
+ "'2015-06-04 01:02:03.123456');";
|
100
|
+
statement.executeUpdate(insert2);
|
101
|
+
|
102
|
+
String drop2 = "drop table if exists test2";
|
103
|
+
statement.execute(drop2);
|
104
|
+
|
105
|
+
String create2 = "create table test2 (c1 bigint unsigned);";
|
106
|
+
statement.execute(create2);
|
107
|
+
|
108
|
+
String insert3 = "insert into test2 values(18446744073709551615)";
|
109
|
+
statement.executeUpdate(insert3);
|
110
|
+
}
|
111
|
+
|
112
|
+
} finally {
|
113
|
+
connection.close();
|
114
|
+
prepared = true;
|
115
|
+
}
|
116
|
+
}
|
117
|
+
|
118
|
+
@Test
|
119
|
+
public void test() throws Exception
|
120
|
+
{
|
121
|
+
if (prepared) {
|
122
|
+
EmbulkPluginTester tester = new EmbulkPluginTester(InputPlugin.class, "mysql", MySQLInputPlugin.class);
|
123
|
+
tester.run(convertPath("/yml/input.yml"));
|
124
|
+
assertEquals(Arrays.asList(
|
125
|
+
"c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15",
|
126
|
+
",,,,,,,,,,,,2015-06-04 14:45:06,,",
|
127
|
+
"99,9999,-99999999,-9999999999999999,1.2345000505447388,1.234567890123,-1234.0,1.2345678901234568E17,5678,xy,2015-06-03,2015-06-04 03:34:56,2015-06-04 14:45:06,23:04:02,2015-06-03 16:02:03"),
|
128
|
+
read("mysql-input.000.00.csv"));
|
129
|
+
}
|
130
|
+
}
|
131
|
+
|
132
|
+
@Test
|
133
|
+
public void testString() throws Exception
|
134
|
+
{
|
135
|
+
if (prepared) {
|
136
|
+
EmbulkPluginTester tester = new EmbulkPluginTester(InputPlugin.class, "mysql", MySQLInputPlugin.class);
|
137
|
+
tester.run(convertPath("/yml/input-string.yml"));
|
138
|
+
assertEquals(Arrays.asList(
|
139
|
+
"c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15",
|
140
|
+
",,,,,,,,,,,,2015-06-04 14:45:06,,",
|
141
|
+
"99,9999,-99999999,-9999999999999999,1.2345,1.234567890123,-1234,123456789012345678.12,5678,xy,2015-06-03,2015-06-04 03:34:56,2015-06-04 14:45:06,23:04:02,2015-06-03 16:02:03"),
|
142
|
+
read("mysql-input.000.00.csv"));
|
143
|
+
}
|
144
|
+
}
|
145
|
+
|
146
|
+
@Test
|
147
|
+
public void testBoolean() throws Exception
|
148
|
+
{
|
149
|
+
if (prepared) {
|
150
|
+
EmbulkPluginTester tester = new EmbulkPluginTester(InputPlugin.class, "mysql", MySQLInputPlugin.class);
|
151
|
+
tester.run(convertPath("/yml/input-boolean.yml"));
|
152
|
+
assertEquals(Arrays.asList(
|
153
|
+
"c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15",
|
154
|
+
",,,,,,,,,,,,2015-06-04 14:45:06,,",
|
155
|
+
"true,true,false,false,true,true,false,true,,,2015-06-03,2015-06-04 03:34:56,2015-06-04 14:45:06,23:04:02,2015-06-03 16:02:03"),
|
156
|
+
read("mysql-input.000.00.csv"));
|
157
|
+
}
|
158
|
+
}
|
159
|
+
|
160
|
+
@Test
|
161
|
+
public void testLong() throws Exception
|
162
|
+
{
|
163
|
+
if (prepared) {
|
164
|
+
EmbulkPluginTester tester = new EmbulkPluginTester(InputPlugin.class, "mysql", MySQLInputPlugin.class);
|
165
|
+
tester.run(convertPath("/yml/input-long.yml"));
|
166
|
+
assertEquals(Arrays.asList(
|
167
|
+
"c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15",
|
168
|
+
",,,,,,,,,,,,2015-06-04 14:45:06,,",
|
169
|
+
"99,9999,-99999999,-9999999999999999,1,1,-1234,123456789012345678,5678,,2015-06-03,2015-06-04 03:34:56,2015-06-04 14:45:06,23:04:02,2015-06-03 16:02:03"),
|
170
|
+
read("mysql-input.000.00.csv"));
|
171
|
+
}
|
172
|
+
}
|
173
|
+
|
174
|
+
@Test
|
175
|
+
public void testDouble() throws Exception
|
176
|
+
{
|
177
|
+
if (prepared) {
|
178
|
+
EmbulkPluginTester tester = new EmbulkPluginTester(InputPlugin.class, "mysql", MySQLInputPlugin.class);
|
179
|
+
tester.run(convertPath("/yml/input-double.yml"));
|
180
|
+
assertEquals(Arrays.asList(
|
181
|
+
"c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15",
|
182
|
+
",,,,,,,,,,,,2015-06-04 14:45:06,,",
|
183
|
+
"99.0,9999.0,-9.9999999E7,-1.0E16,1.2345000505447388,1.234567890123,-1234.0,1.2345678901234568E17,5678.0,,2015-06-03,2015-06-04 03:34:56,2015-06-04 14:45:06,23:04:02,2015-06-03 16:02:03"),
|
184
|
+
read("mysql-input.000.00.csv"));
|
185
|
+
}
|
186
|
+
}
|
187
|
+
|
188
|
+
@Test
|
189
|
+
public void testTimestamp1() throws Exception
|
190
|
+
{
|
191
|
+
if (prepared) {
|
192
|
+
EmbulkPluginTester tester = new EmbulkPluginTester(InputPlugin.class, "mysql", MySQLInputPlugin.class);
|
193
|
+
tester.run(convertPath("/yml/input-timestamp1.yml"));
|
194
|
+
assertEquals(Arrays.asList(
|
195
|
+
"c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15",
|
196
|
+
",,,,,,,,,,,,2015/06/04 14:45:06,,",
|
197
|
+
"99,9999,-99999999,-9999999999999999,1.2345000505447388,1.234567890123,-1234.0,1.2345678901234568E17,5678,xy,2015/06/03,2015/06/04 03:34:56,2015/06/04 14:45:06,23-04-02,2015/06/03 16:02:03.123456"),
|
198
|
+
read("mysql-input.000.00.csv"));
|
199
|
+
}
|
200
|
+
}
|
201
|
+
|
202
|
+
@Test
|
203
|
+
public void testTimestamp2() throws Exception
|
204
|
+
{
|
205
|
+
if (prepared) {
|
206
|
+
EmbulkPluginTester tester = new EmbulkPluginTester(InputPlugin.class, "mysql", MySQLInputPlugin.class);
|
207
|
+
tester.run(convertPath("/yml/input-timestamp2.yml"));
|
208
|
+
assertEquals(Arrays.asList(
|
209
|
+
"c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15",
|
210
|
+
",,,,,,,,,,,,2015/06/04 23:45:06,,",
|
211
|
+
"99,9999,-99999999,-9999999999999999,1.2345000505447388,1.234567890123,-1234.0,1.2345678901234568E17,5678,xy,2015/06/03,2015/06/04 03:34:56,2015/06/04 23:45:06,08-04-02,2015/06/03 16:02:03.123456"),
|
212
|
+
read("mysql-input.000.00.csv"));
|
213
|
+
}
|
214
|
+
}
|
215
|
+
|
216
|
+
@Test
|
217
|
+
public void testTimestamp3() throws Exception
|
218
|
+
{
|
219
|
+
if (prepared) {
|
220
|
+
EmbulkPluginTester tester = new EmbulkPluginTester(InputPlugin.class, "mysql", MySQLInputPlugin.class);
|
221
|
+
tester.run(convertPath("/yml/input-timestamp3.yml"));
|
222
|
+
assertEquals(Arrays.asList(
|
223
|
+
"c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15",
|
224
|
+
",,,,,,,,,,,,2015/06/04 23:45:06,,",
|
225
|
+
"99,9999,-99999999,-9999999999999999,1.2345000505447388,1.234567890123,-1234.0,1.2345678901234568E17,5678,xy,2015/06/04,2015/06/04 12:34:56,2015/06/04 23:45:06,02-04-02,2015/06/04 01:02:03.123456"),
|
226
|
+
read("mysql-input.000.00.csv"));
|
227
|
+
}
|
228
|
+
}
|
229
|
+
|
230
|
+
@Test
|
231
|
+
public void testValueTypeString() throws Exception
|
232
|
+
{
|
233
|
+
if (prepared) {
|
234
|
+
EmbulkPluginTester tester = new EmbulkPluginTester(InputPlugin.class, "mysql", MySQLInputPlugin.class);
|
235
|
+
tester.run(convertPath("/yml/input-valuetype-string.yml"));
|
236
|
+
assertEquals(Arrays.asList(
|
237
|
+
"c1",
|
238
|
+
"18446744073709551615"),
|
239
|
+
read("mysql-input.000.00.csv"));
|
240
|
+
}
|
241
|
+
}
|
242
|
+
|
243
|
+
@Test
|
244
|
+
public void testValueTypeDecimal() throws Exception
|
245
|
+
{
|
246
|
+
if (prepared) {
|
247
|
+
EmbulkPluginTester tester = new EmbulkPluginTester(InputPlugin.class, "mysql", MySQLInputPlugin.class);
|
248
|
+
tester.run(convertPath("/yml/input-valuetype-decimal.yml"));
|
249
|
+
assertEquals(Arrays.asList(
|
250
|
+
"c1",
|
251
|
+
"1.8446744073709552E19"),
|
252
|
+
read("mysql-input.000.00.csv"));
|
253
|
+
}
|
254
|
+
}
|
255
|
+
|
256
|
+
private List<String> read(String path) throws IOException
|
257
|
+
{
|
258
|
+
FileSystem fs = FileSystems.getDefault();
|
259
|
+
return Files.readAllLines(fs.getPath(path), Charset.defaultCharset());
|
260
|
+
}
|
261
|
+
|
262
|
+
private String convertPath(String name) throws URISyntaxException
|
263
|
+
{
|
264
|
+
if (getClass().getResource(name) == null) {
|
265
|
+
return name;
|
266
|
+
}
|
267
|
+
return new File(getClass().getResource(name).toURI()).getAbsolutePath();
|
268
|
+
}
|
269
|
+
}
|
@@ -0,0 +1,26 @@
|
|
1
|
+
in:
|
2
|
+
type: mysql
|
3
|
+
host: localhost
|
4
|
+
database: TESTDB
|
5
|
+
user: TEST_USER
|
6
|
+
password: test_pw
|
7
|
+
table: test1
|
8
|
+
select: "*"
|
9
|
+
column_options:
|
10
|
+
c1: {type: boolean}
|
11
|
+
c2: {type: boolean}
|
12
|
+
c3: {type: boolean}
|
13
|
+
c4: {type: boolean}
|
14
|
+
c5: {type: boolean}
|
15
|
+
c6: {type: boolean}
|
16
|
+
c7: {type: boolean}
|
17
|
+
c8: {type: boolean}
|
18
|
+
c9: {type: boolean}
|
19
|
+
c10: {type: boolean}
|
20
|
+
|
21
|
+
out:
|
22
|
+
type: file
|
23
|
+
path_prefix: mysql-input
|
24
|
+
file_ext: .csv
|
25
|
+
formatter:
|
26
|
+
type: csv
|
@@ -0,0 +1,26 @@
|
|
1
|
+
in:
|
2
|
+
type: mysql
|
3
|
+
host: localhost
|
4
|
+
database: TESTDB
|
5
|
+
user: TEST_USER
|
6
|
+
password: test_pw
|
7
|
+
table: test1
|
8
|
+
select: "*"
|
9
|
+
column_options:
|
10
|
+
c1: {type: double}
|
11
|
+
c2: {type: double}
|
12
|
+
c3: {type: double}
|
13
|
+
c4: {type: double}
|
14
|
+
c5: {type: double}
|
15
|
+
c6: {type: double}
|
16
|
+
c7: {type: double}
|
17
|
+
c8: {type: double}
|
18
|
+
c9: {type: double}
|
19
|
+
c10: {type: double}
|
20
|
+
|
21
|
+
out:
|
22
|
+
type: file
|
23
|
+
path_prefix: mysql-input
|
24
|
+
file_ext: .csv
|
25
|
+
formatter:
|
26
|
+
type: csv
|
@@ -0,0 +1,26 @@
|
|
1
|
+
in:
|
2
|
+
type: mysql
|
3
|
+
host: localhost
|
4
|
+
database: TESTDB
|
5
|
+
user: TEST_USER
|
6
|
+
password: test_pw
|
7
|
+
table: test1
|
8
|
+
select: "*"
|
9
|
+
column_options:
|
10
|
+
c1: {type: long}
|
11
|
+
c2: {type: long}
|
12
|
+
c3: {type: long}
|
13
|
+
c4: {type: long}
|
14
|
+
c5: {type: long}
|
15
|
+
c6: {type: long}
|
16
|
+
c7: {type: long}
|
17
|
+
c8: {type: long}
|
18
|
+
c9: {type: long}
|
19
|
+
c10: {type: long}
|
20
|
+
|
21
|
+
out:
|
22
|
+
type: file
|
23
|
+
path_prefix: mysql-input
|
24
|
+
file_ext: .csv
|
25
|
+
formatter:
|
26
|
+
type: csv
|
@@ -0,0 +1,26 @@
|
|
1
|
+
in:
|
2
|
+
type: mysql
|
3
|
+
host: localhost
|
4
|
+
database: TESTDB
|
5
|
+
user: TEST_USER
|
6
|
+
password: test_pw
|
7
|
+
table: test1
|
8
|
+
select: "*"
|
9
|
+
column_options:
|
10
|
+
c1: {type: string}
|
11
|
+
c2: {type: string}
|
12
|
+
c3: {type: string}
|
13
|
+
c4: {type: string}
|
14
|
+
c5: {type: string}
|
15
|
+
c6: {type: string}
|
16
|
+
c7: {type: string}
|
17
|
+
c8: {type: string}
|
18
|
+
c9: {type: string}
|
19
|
+
c10: {type: string}
|
20
|
+
|
21
|
+
out:
|
22
|
+
type: file
|
23
|
+
path_prefix: mysql-input
|
24
|
+
file_ext: .csv
|
25
|
+
formatter:
|
26
|
+
type: csv
|
@@ -0,0 +1,21 @@
|
|
1
|
+
in:
|
2
|
+
type: mysql
|
3
|
+
host: localhost
|
4
|
+
database: TESTDB
|
5
|
+
user: TEST_USER
|
6
|
+
password: test_pw
|
7
|
+
table: test1
|
8
|
+
select: "*"
|
9
|
+
column_options:
|
10
|
+
c11: {type: timestamp, timestamp_format: '%Y/%m/%d'}
|
11
|
+
c12: {type: timestamp, timestamp_format: '%Y/%m/%d %H:%M:%S'}
|
12
|
+
c13: {type: timestamp, timestamp_format: '%Y/%m/%d %H:%M:%S', timezone: '+0900'}
|
13
|
+
c14: {type: timestamp, timestamp_format: '%H-%M-%S', timezone: '+0900'}
|
14
|
+
c15: {type: timestamp, timestamp_format: '%Y/%m/%d %H:%M:%S.%6N'}
|
15
|
+
|
16
|
+
out:
|
17
|
+
type: file
|
18
|
+
path_prefix: mysql-input
|
19
|
+
file_ext: .csv
|
20
|
+
formatter:
|
21
|
+
type: csv
|
@@ -0,0 +1,21 @@
|
|
1
|
+
in:
|
2
|
+
type: mysql
|
3
|
+
host: localhost
|
4
|
+
database: TESTDB
|
5
|
+
user: TEST_USER
|
6
|
+
password: test_pw
|
7
|
+
table: test1
|
8
|
+
select: "*"
|
9
|
+
column_options:
|
10
|
+
c11: {type: string, timestamp_format: '%Y/%m/%d'}
|
11
|
+
c12: {type: string, timestamp_format: '%Y/%m/%d %H:%M:%S'}
|
12
|
+
c13: {type: string, timestamp_format: '%Y/%m/%d %H:%M:%S', timezone: '+0900'}
|
13
|
+
c14: {type: string, timestamp_format: '%H-%M-%S', timezone: '+0900'}
|
14
|
+
c15: {type: string, timestamp_format: '%Y/%m/%d %H:%M:%S.%6N'}
|
15
|
+
|
16
|
+
out:
|
17
|
+
type: file
|
18
|
+
path_prefix: mysql-input
|
19
|
+
file_ext: .csv
|
20
|
+
formatter:
|
21
|
+
type: csv
|
@@ -0,0 +1,22 @@
|
|
1
|
+
in:
|
2
|
+
type: mysql
|
3
|
+
host: localhost
|
4
|
+
database: TESTDB
|
5
|
+
user: TEST_USER
|
6
|
+
password: test_pw
|
7
|
+
table: test1
|
8
|
+
select: "*"
|
9
|
+
default_timezone: '+0900'
|
10
|
+
column_options:
|
11
|
+
c11: {type: string, timestamp_format: '%Y/%m/%d'}
|
12
|
+
c12: {type: string, timestamp_format: '%Y/%m/%d %H:%M:%S'}
|
13
|
+
c13: {type: string, timestamp_format: '%Y/%m/%d %H:%M:%S'}
|
14
|
+
c14: {type: string, timestamp_format: '%H-%M-%S', timezone: '+0300'}
|
15
|
+
c15: {type: string, timestamp_format: '%Y/%m/%d %H:%M:%S.%6N'}
|
16
|
+
|
17
|
+
out:
|
18
|
+
type: file
|
19
|
+
path_prefix: mysql-input
|
20
|
+
file_ext: .csv
|
21
|
+
formatter:
|
22
|
+
type: csv
|
@@ -0,0 +1,17 @@
|
|
1
|
+
in:
|
2
|
+
type: mysql
|
3
|
+
host: localhost
|
4
|
+
database: TESTDB
|
5
|
+
user: TEST_USER
|
6
|
+
password: test_pw
|
7
|
+
table: test2
|
8
|
+
select: "*"
|
9
|
+
column_options:
|
10
|
+
c1: {value_type: decimal}
|
11
|
+
|
12
|
+
out:
|
13
|
+
type: file
|
14
|
+
path_prefix: mysql-input
|
15
|
+
file_ext: .csv
|
16
|
+
formatter:
|
17
|
+
type: csv
|
@@ -0,0 +1,17 @@
|
|
1
|
+
in:
|
2
|
+
type: mysql
|
3
|
+
host: localhost
|
4
|
+
database: TESTDB
|
5
|
+
user: TEST_USER
|
6
|
+
password: test_pw
|
7
|
+
table: test2
|
8
|
+
select: "*"
|
9
|
+
column_options:
|
10
|
+
c1: {value_type: string}
|
11
|
+
|
12
|
+
out:
|
13
|
+
type: file
|
14
|
+
path_prefix: mysql-input
|
15
|
+
file_ext: .csv
|
16
|
+
formatter:
|
17
|
+
type: csv
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-input-mysql
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.5.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Sadayuki Furuhashi
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-
|
11
|
+
date: 2015-06-17 00:00:00.000000000 Z
|
12
12
|
dependencies: []
|
13
13
|
description: Selects records from a table.
|
14
14
|
email:
|
@@ -22,8 +22,22 @@ files:
|
|
22
22
|
- lib/embulk/input/mysql.rb
|
23
23
|
- src/main/java/org/embulk/input/MySQLInputPlugin.java
|
24
24
|
- src/main/java/org/embulk/input/mysql/MySQLInputConnection.java
|
25
|
-
-
|
26
|
-
-
|
25
|
+
- src/test/java/org/embulk/input/mysql/ChildFirstClassLoader.java
|
26
|
+
- src/test/java/org/embulk/input/mysql/EmbulkPluginTester.java
|
27
|
+
- src/test/java/org/embulk/input/mysql/EmptyConfigSource.java
|
28
|
+
- src/test/java/org/embulk/input/mysql/MySQLInputPluginTest.java
|
29
|
+
- src/test/resources/yml/input-boolean.yml
|
30
|
+
- src/test/resources/yml/input-double.yml
|
31
|
+
- src/test/resources/yml/input-long.yml
|
32
|
+
- src/test/resources/yml/input-string.yml
|
33
|
+
- src/test/resources/yml/input-timestamp1.yml
|
34
|
+
- src/test/resources/yml/input-timestamp2.yml
|
35
|
+
- src/test/resources/yml/input-timestamp3.yml
|
36
|
+
- src/test/resources/yml/input-valuetype-decimal.yml
|
37
|
+
- src/test/resources/yml/input-valuetype-string.yml
|
38
|
+
- src/test/resources/yml/input.yml
|
39
|
+
- classpath/embulk-input-jdbc-0.5.0.jar
|
40
|
+
- classpath/embulk-input-mysql-0.5.0.jar
|
27
41
|
- classpath/mysql-connector-java-5.1.34.jar
|
28
42
|
homepage: https://github.com/embulk/embulk-input-jdbc
|
29
43
|
licenses:
|
Binary file
|