embulk-input-mysql 0.4.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +28 -0
- data/build.gradle +1 -0
- data/classpath/embulk-input-jdbc-0.5.0.jar +0 -0
- data/classpath/{embulk-input-mysql-0.4.0.jar → embulk-input-mysql-0.5.0.jar} +0 -0
- data/src/test/java/org/embulk/input/mysql/ChildFirstClassLoader.java +42 -0
- data/src/test/java/org/embulk/input/mysql/EmbulkPluginTester.java +81 -0
- data/src/test/java/org/embulk/input/mysql/EmptyConfigSource.java +100 -0
- data/src/test/java/org/embulk/input/mysql/MySQLInputPluginTest.java +269 -0
- data/src/test/resources/yml/input-boolean.yml +26 -0
- data/src/test/resources/yml/input-double.yml +26 -0
- data/src/test/resources/yml/input-long.yml +26 -0
- data/src/test/resources/yml/input-string.yml +26 -0
- data/src/test/resources/yml/input-timestamp1.yml +21 -0
- data/src/test/resources/yml/input-timestamp2.yml +21 -0
- data/src/test/resources/yml/input-timestamp3.yml +22 -0
- data/src/test/resources/yml/input-valuetype-decimal.yml +17 -0
- data/src/test/resources/yml/input-valuetype-string.yml +17 -0
- data/src/test/resources/yml/input.yml +15 -0
- metadata +18 -4
- data/classpath/embulk-input-jdbc-0.4.0.jar +0 -0
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 3f02975b24e6df7e4e3fe9eee4366448d02d17ef
|
4
|
+
data.tar.gz: a3134e8c89d0eb787d397ce2cb26ebd3a07e1b95
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: c4ee7883fa9a550063c961de66927cf4687169fc16a544167822e29bfc3f9a191aff0f071dd9a4229a55a99801cc5d5e98d2b1a06c6b8f3e1c404ac4bfcfc144
|
7
|
+
data.tar.gz: 3f15c1095f95c81882606f4315d1ae0d3ae0f8dad518745765fdfdd5856de175a013c0a0b666ef5b665bfa75210dd72b11d40d40366af9e6c2aed6d459ea5f97
|
data/README.md
CHANGED
@@ -31,6 +31,16 @@ MySQL input plugins for Embulk loads records from MySQL.
|
|
31
31
|
- It uses a client-side built statement and fetches all rows at once. This may cause OutOfMemoryError.
|
32
32
|
- Internally, `useCursorFetch=false` is used and `java.sql.Statement.setFetchSize` is not set.
|
33
33
|
- **options**: extra JDBC properties (hash, default: {})
|
34
|
+
- **default_timezone**: If the sql type of a column is `date`/`time`/`datetime` and the embulk type is `string`, column values are formatted int this default_timezone. You can overwrite timezone for each columns using column_options option. (string, default: `UTC`)
|
35
|
+
- **column_options**: advanced: a key-value pairs where key is a column name and value is options for the column.
|
36
|
+
- **value_type**: embulk get values from database as this value_type. Typically, the value_type determines `getXXX` method of `java.sql.PreparedStatement`.
|
37
|
+
(string, default: depends on the sql type of the column. Available values options are: `long`, `double`, `float`, `decimal`, `boolean`, `string`, `date`, `time`, `timestamp`)
|
38
|
+
- **type**: Column values are converted to this embulk type.
|
39
|
+
Available values options are: `boolean`, `long`, `double`, `string`, `timestamp`).
|
40
|
+
By default, the embulk type is determined according to the sql type of the column (or value_type if specified).
|
41
|
+
- **timestamp_format**: If the sql type of the column is `date`/`time`/`datetime` and the embulk type is `string`, column values are formatted by this timestamp_format. And if the embulk type is `timestamp`, this timestamp_format will be used in the output plugin. (string, default : `%Y-%m-%d` for `date`, `%H:%M:%S` for `time`, `%Y-%m-%d %H:%M:%S` for `timestamp`)
|
42
|
+
- **timezone**: If the sql type of the column is `date`/`time`/`datetime` and the embulk type is `string`, column values are formatted in this timezone.
|
43
|
+
(string, value of default_timezone option is used by default)
|
34
44
|
|
35
45
|
## Example
|
36
46
|
|
@@ -62,6 +72,24 @@ in:
|
|
62
72
|
ON t1.id = t2.t1_id
|
63
73
|
```
|
64
74
|
|
75
|
+
Advanced configuration:
|
76
|
+
|
77
|
+
```yaml
|
78
|
+
in:
|
79
|
+
type: mysql
|
80
|
+
host: localhost
|
81
|
+
user: myuser
|
82
|
+
password: ""
|
83
|
+
database: my_database
|
84
|
+
table: "my_table"
|
85
|
+
select: "col1, col2, col3"
|
86
|
+
where: "col4 != 'a'"
|
87
|
+
column_options:
|
88
|
+
col1: {type: long}
|
89
|
+
col3: {type: string, timestamp_format: "%Y/%m/%d", timezone: "+0900"}
|
90
|
+
|
91
|
+
```
|
92
|
+
|
65
93
|
## Build
|
66
94
|
|
67
95
|
```
|
data/build.gradle
CHANGED
Binary file
|
Binary file
|
@@ -0,0 +1,42 @@
|
|
1
|
+
package org.embulk.input.mysql;
|
2
|
+
|
3
|
+
import java.net.URL;
|
4
|
+
import java.net.URLClassLoader;
|
5
|
+
import java.util.List;
|
6
|
+
|
7
|
+
public class ChildFirstClassLoader
|
8
|
+
extends URLClassLoader
|
9
|
+
{
|
10
|
+
public ChildFirstClassLoader(List<URL> urls, ClassLoader parent)
|
11
|
+
{
|
12
|
+
super(urls.toArray(new URL[urls.size()]), parent);
|
13
|
+
}
|
14
|
+
|
15
|
+
@Override
|
16
|
+
protected Class<?> loadClass(String name, boolean resolve)
|
17
|
+
throws ClassNotFoundException
|
18
|
+
{
|
19
|
+
synchronized (getClassLoadingLock(name)) {
|
20
|
+
Class<?> loadedClass = findLoadedClass(name);
|
21
|
+
if (loadedClass != null) {
|
22
|
+
return resolveClass(loadedClass, resolve);
|
23
|
+
}
|
24
|
+
|
25
|
+
try {
|
26
|
+
return resolveClass(findClass(name), resolve);
|
27
|
+
} catch (ClassNotFoundException ignored) {
|
28
|
+
}
|
29
|
+
|
30
|
+
return super.loadClass(name, resolve);
|
31
|
+
}
|
32
|
+
}
|
33
|
+
|
34
|
+
private Class<?> resolveClass(Class<?> clazz, boolean resolve)
|
35
|
+
{
|
36
|
+
if (resolve) {
|
37
|
+
resolveClass(clazz);
|
38
|
+
}
|
39
|
+
return clazz;
|
40
|
+
}
|
41
|
+
|
42
|
+
}
|
@@ -0,0 +1,81 @@
|
|
1
|
+
package org.embulk.input.mysql;
|
2
|
+
|
3
|
+
import java.io.File;
|
4
|
+
import java.util.ArrayList;
|
5
|
+
import java.util.Arrays;
|
6
|
+
import java.util.List;
|
7
|
+
|
8
|
+
import org.embulk.EmbulkService;
|
9
|
+
import org.embulk.config.ConfigLoader;
|
10
|
+
import org.embulk.config.ConfigSource;
|
11
|
+
import org.embulk.exec.BulkLoader;
|
12
|
+
import org.embulk.exec.ExecutionResult;
|
13
|
+
import org.embulk.plugin.InjectedPluginSource;
|
14
|
+
import org.embulk.spi.ExecSession;
|
15
|
+
|
16
|
+
import com.google.inject.Binder;
|
17
|
+
import com.google.inject.Injector;
|
18
|
+
import com.google.inject.Module;
|
19
|
+
|
20
|
+
public class EmbulkPluginTester
|
21
|
+
{
|
22
|
+
private static class PluginDefinition
|
23
|
+
{
|
24
|
+
public final Class<?> iface;
|
25
|
+
public final String name;
|
26
|
+
public final Class<?> impl;
|
27
|
+
|
28
|
+
|
29
|
+
public PluginDefinition(Class<?> iface, String name, Class<?> impl)
|
30
|
+
{
|
31
|
+
this.iface = iface;
|
32
|
+
this.name = name;
|
33
|
+
this.impl = impl;
|
34
|
+
}
|
35
|
+
|
36
|
+
}
|
37
|
+
|
38
|
+
private final List<PluginDefinition> plugins = new ArrayList<PluginDefinition>();
|
39
|
+
|
40
|
+
|
41
|
+
|
42
|
+
public EmbulkPluginTester()
|
43
|
+
{
|
44
|
+
}
|
45
|
+
|
46
|
+
public EmbulkPluginTester(Class<?> iface, String name, Class<?> impl)
|
47
|
+
{
|
48
|
+
addPlugin(iface, name, impl);
|
49
|
+
}
|
50
|
+
|
51
|
+
public void addPlugin(Class<?> iface, String name, Class<?> impl)
|
52
|
+
{
|
53
|
+
plugins.add(new PluginDefinition(iface, name, impl));
|
54
|
+
}
|
55
|
+
|
56
|
+
public void run(String ymlPath) throws Exception
|
57
|
+
{
|
58
|
+
EmbulkService service = new EmbulkService(new EmptyConfigSource()) {
|
59
|
+
@Override
|
60
|
+
protected Iterable<? extends Module> getAdditionalModules(ConfigSource systemConfig)
|
61
|
+
{
|
62
|
+
return Arrays.asList(new Module()
|
63
|
+
{
|
64
|
+
@Override
|
65
|
+
public void configure(Binder binder)
|
66
|
+
{
|
67
|
+
for (PluginDefinition plugin : plugins) {
|
68
|
+
InjectedPluginSource.registerPluginTo(binder, plugin.iface, plugin.name, plugin.impl);
|
69
|
+
}
|
70
|
+
}
|
71
|
+
});
|
72
|
+
}
|
73
|
+
};
|
74
|
+
Injector injector = service.getInjector();
|
75
|
+
ConfigSource config = injector.getInstance(ConfigLoader.class).fromYamlFile(new File(ymlPath));
|
76
|
+
ExecSession session = new ExecSession(injector, config);
|
77
|
+
BulkLoader loader = injector.getInstance(BulkLoader.class);
|
78
|
+
ExecutionResult result = loader.run(session, config);
|
79
|
+
}
|
80
|
+
|
81
|
+
}
|
@@ -0,0 +1,100 @@
|
|
1
|
+
package org.embulk.input.mysql;
|
2
|
+
|
3
|
+
import java.util.Collections;
|
4
|
+
import java.util.List;
|
5
|
+
import java.util.Map.Entry;
|
6
|
+
|
7
|
+
import org.embulk.config.ConfigSource;
|
8
|
+
import org.embulk.config.DataSource;
|
9
|
+
|
10
|
+
import com.fasterxml.jackson.databind.JsonNode;
|
11
|
+
import com.fasterxml.jackson.databind.node.ObjectNode;
|
12
|
+
|
13
|
+
public class EmptyConfigSource implements ConfigSource
|
14
|
+
{
|
15
|
+
|
16
|
+
@Override
|
17
|
+
public <E> E get(Class<E> type, String attrName)
|
18
|
+
{
|
19
|
+
return null;
|
20
|
+
}
|
21
|
+
|
22
|
+
@Override
|
23
|
+
public <E> E get(Class<E> type, String attrName, E defaultValue)
|
24
|
+
{
|
25
|
+
return defaultValue;
|
26
|
+
}
|
27
|
+
|
28
|
+
@Override
|
29
|
+
public List<String> getAttributeNames()
|
30
|
+
{
|
31
|
+
return Collections.emptyList();
|
32
|
+
}
|
33
|
+
|
34
|
+
@Override
|
35
|
+
public Iterable<Entry<String, JsonNode>> getAttributes()
|
36
|
+
{
|
37
|
+
return Collections.emptyList();
|
38
|
+
}
|
39
|
+
|
40
|
+
@Override
|
41
|
+
public ObjectNode getObjectNode()
|
42
|
+
{
|
43
|
+
return null;
|
44
|
+
}
|
45
|
+
|
46
|
+
@Override
|
47
|
+
public boolean isEmpty()
|
48
|
+
{
|
49
|
+
return true;
|
50
|
+
}
|
51
|
+
|
52
|
+
@Override
|
53
|
+
public ConfigSource deepCopy()
|
54
|
+
{
|
55
|
+
return this;
|
56
|
+
}
|
57
|
+
|
58
|
+
@Override
|
59
|
+
public ConfigSource getNested(String s)
|
60
|
+
{
|
61
|
+
return null;
|
62
|
+
}
|
63
|
+
|
64
|
+
@Override
|
65
|
+
public ConfigSource getNestedOrSetEmpty(String s)
|
66
|
+
{
|
67
|
+
return null;
|
68
|
+
}
|
69
|
+
|
70
|
+
@Override
|
71
|
+
public <T> T loadConfig(Class<T> class1)
|
72
|
+
{
|
73
|
+
return null;
|
74
|
+
}
|
75
|
+
|
76
|
+
@Override
|
77
|
+
public ConfigSource merge(DataSource datasource)
|
78
|
+
{
|
79
|
+
return null;
|
80
|
+
}
|
81
|
+
|
82
|
+
@Override
|
83
|
+
public ConfigSource set(String s, Object obj)
|
84
|
+
{
|
85
|
+
return null;
|
86
|
+
}
|
87
|
+
|
88
|
+
@Override
|
89
|
+
public ConfigSource setAll(DataSource datasource)
|
90
|
+
{
|
91
|
+
return null;
|
92
|
+
}
|
93
|
+
|
94
|
+
@Override
|
95
|
+
public ConfigSource setNested(String s, DataSource datasource)
|
96
|
+
{
|
97
|
+
return null;
|
98
|
+
}
|
99
|
+
|
100
|
+
}
|
@@ -0,0 +1,269 @@
|
|
1
|
+
package org.embulk.input.mysql;
|
2
|
+
|
3
|
+
import java.io.File;
|
4
|
+
import java.io.IOException;
|
5
|
+
import java.net.URISyntaxException;
|
6
|
+
import java.nio.charset.Charset;
|
7
|
+
import java.nio.file.FileSystem;
|
8
|
+
import java.nio.file.FileSystems;
|
9
|
+
import java.nio.file.Files;
|
10
|
+
import java.sql.Connection;
|
11
|
+
import java.sql.DriverManager;
|
12
|
+
import java.sql.SQLException;
|
13
|
+
import java.sql.Statement;
|
14
|
+
import java.util.Arrays;
|
15
|
+
import java.util.List;
|
16
|
+
|
17
|
+
import org.embulk.input.MySQLInputPlugin;
|
18
|
+
import org.embulk.spi.InputPlugin;
|
19
|
+
import org.junit.BeforeClass;
|
20
|
+
import org.junit.Test;
|
21
|
+
|
22
|
+
import static org.junit.Assert.assertEquals;
|
23
|
+
|
24
|
+
public class MySQLInputPluginTest
|
25
|
+
{
|
26
|
+
private static boolean prepared = false;
|
27
|
+
|
28
|
+
@BeforeClass
|
29
|
+
public static void prepare() throws SQLException
|
30
|
+
{
|
31
|
+
Connection connection;
|
32
|
+
try {
|
33
|
+
connection = DriverManager.getConnection("jdbc:mysql://localhost/TESTDB", "TEST_USER", "test_pw");
|
34
|
+
} catch (SQLException e) {
|
35
|
+
System.err.println(e);
|
36
|
+
System.err.println("Warning: prepare a schema on MySQL (database = 'TESTDB', user = 'TEST_USER', password = 'test_pw').");
|
37
|
+
return;
|
38
|
+
}
|
39
|
+
|
40
|
+
try {
|
41
|
+
try (Statement statement = connection.createStatement()) {
|
42
|
+
String drop1 = "drop table if exists test1";
|
43
|
+
statement.execute(drop1);
|
44
|
+
|
45
|
+
String create1 =
|
46
|
+
"create table test1 ("
|
47
|
+
+ "c1 tinyint,"
|
48
|
+
+ "c2 smallint,"
|
49
|
+
+ "c3 int,"
|
50
|
+
+ "c4 bigint,"
|
51
|
+
+ "c5 float,"
|
52
|
+
+ "c6 double,"
|
53
|
+
+ "c7 decimal(4,0),"
|
54
|
+
+ "c8 decimal(20,2),"
|
55
|
+
+ "c9 char(4),"
|
56
|
+
+ "c10 varchar(4),"
|
57
|
+
+ "c11 date,"
|
58
|
+
+ "c12 datetime,"
|
59
|
+
+ "c13 timestamp,"
|
60
|
+
+ "c14 time,"
|
61
|
+
+ "c15 datetime(6));";
|
62
|
+
statement.execute(create1);
|
63
|
+
|
64
|
+
String insert1 =
|
65
|
+
"insert into test1 values("
|
66
|
+
+ "null,"
|
67
|
+
+ "null,"
|
68
|
+
+ "null,"
|
69
|
+
+ "null,"
|
70
|
+
+ "null,"
|
71
|
+
+ "null,"
|
72
|
+
+ "null,"
|
73
|
+
+ "null,"
|
74
|
+
+ "null,"
|
75
|
+
+ "null,"
|
76
|
+
+ "null,"
|
77
|
+
+ "null,"
|
78
|
+
+ "'2015-06-04 23:45:06',"
|
79
|
+
+ "null,"
|
80
|
+
+ "null);";
|
81
|
+
statement.executeUpdate(insert1);
|
82
|
+
|
83
|
+
String insert2 =
|
84
|
+
"insert into test1 values("
|
85
|
+
+ "99,"
|
86
|
+
+ "9999,"
|
87
|
+
+ "-99999999,"
|
88
|
+
+ "-9999999999999999,"
|
89
|
+
+ "1.2345,"
|
90
|
+
+ "1.234567890123,"
|
91
|
+
+ "-1234,"
|
92
|
+
+ "123456789012345678.12,"
|
93
|
+
+ "'5678',"
|
94
|
+
+ "'xy',"
|
95
|
+
+ "'2015-06-04',"
|
96
|
+
+ "'2015-06-04 12:34:56',"
|
97
|
+
+ "'2015-06-04 23:45:06',"
|
98
|
+
+ "'08:04:02',"
|
99
|
+
+ "'2015-06-04 01:02:03.123456');";
|
100
|
+
statement.executeUpdate(insert2);
|
101
|
+
|
102
|
+
String drop2 = "drop table if exists test2";
|
103
|
+
statement.execute(drop2);
|
104
|
+
|
105
|
+
String create2 = "create table test2 (c1 bigint unsigned);";
|
106
|
+
statement.execute(create2);
|
107
|
+
|
108
|
+
String insert3 = "insert into test2 values(18446744073709551615)";
|
109
|
+
statement.executeUpdate(insert3);
|
110
|
+
}
|
111
|
+
|
112
|
+
} finally {
|
113
|
+
connection.close();
|
114
|
+
prepared = true;
|
115
|
+
}
|
116
|
+
}
|
117
|
+
|
118
|
+
@Test
|
119
|
+
public void test() throws Exception
|
120
|
+
{
|
121
|
+
if (prepared) {
|
122
|
+
EmbulkPluginTester tester = new EmbulkPluginTester(InputPlugin.class, "mysql", MySQLInputPlugin.class);
|
123
|
+
tester.run(convertPath("/yml/input.yml"));
|
124
|
+
assertEquals(Arrays.asList(
|
125
|
+
"c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15",
|
126
|
+
",,,,,,,,,,,,2015-06-04 14:45:06,,",
|
127
|
+
"99,9999,-99999999,-9999999999999999,1.2345000505447388,1.234567890123,-1234.0,1.2345678901234568E17,5678,xy,2015-06-03,2015-06-04 03:34:56,2015-06-04 14:45:06,23:04:02,2015-06-03 16:02:03"),
|
128
|
+
read("mysql-input.000.00.csv"));
|
129
|
+
}
|
130
|
+
}
|
131
|
+
|
132
|
+
@Test
|
133
|
+
public void testString() throws Exception
|
134
|
+
{
|
135
|
+
if (prepared) {
|
136
|
+
EmbulkPluginTester tester = new EmbulkPluginTester(InputPlugin.class, "mysql", MySQLInputPlugin.class);
|
137
|
+
tester.run(convertPath("/yml/input-string.yml"));
|
138
|
+
assertEquals(Arrays.asList(
|
139
|
+
"c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15",
|
140
|
+
",,,,,,,,,,,,2015-06-04 14:45:06,,",
|
141
|
+
"99,9999,-99999999,-9999999999999999,1.2345,1.234567890123,-1234,123456789012345678.12,5678,xy,2015-06-03,2015-06-04 03:34:56,2015-06-04 14:45:06,23:04:02,2015-06-03 16:02:03"),
|
142
|
+
read("mysql-input.000.00.csv"));
|
143
|
+
}
|
144
|
+
}
|
145
|
+
|
146
|
+
@Test
|
147
|
+
public void testBoolean() throws Exception
|
148
|
+
{
|
149
|
+
if (prepared) {
|
150
|
+
EmbulkPluginTester tester = new EmbulkPluginTester(InputPlugin.class, "mysql", MySQLInputPlugin.class);
|
151
|
+
tester.run(convertPath("/yml/input-boolean.yml"));
|
152
|
+
assertEquals(Arrays.asList(
|
153
|
+
"c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15",
|
154
|
+
",,,,,,,,,,,,2015-06-04 14:45:06,,",
|
155
|
+
"true,true,false,false,true,true,false,true,,,2015-06-03,2015-06-04 03:34:56,2015-06-04 14:45:06,23:04:02,2015-06-03 16:02:03"),
|
156
|
+
read("mysql-input.000.00.csv"));
|
157
|
+
}
|
158
|
+
}
|
159
|
+
|
160
|
+
@Test
|
161
|
+
public void testLong() throws Exception
|
162
|
+
{
|
163
|
+
if (prepared) {
|
164
|
+
EmbulkPluginTester tester = new EmbulkPluginTester(InputPlugin.class, "mysql", MySQLInputPlugin.class);
|
165
|
+
tester.run(convertPath("/yml/input-long.yml"));
|
166
|
+
assertEquals(Arrays.asList(
|
167
|
+
"c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15",
|
168
|
+
",,,,,,,,,,,,2015-06-04 14:45:06,,",
|
169
|
+
"99,9999,-99999999,-9999999999999999,1,1,-1234,123456789012345678,5678,,2015-06-03,2015-06-04 03:34:56,2015-06-04 14:45:06,23:04:02,2015-06-03 16:02:03"),
|
170
|
+
read("mysql-input.000.00.csv"));
|
171
|
+
}
|
172
|
+
}
|
173
|
+
|
174
|
+
@Test
|
175
|
+
public void testDouble() throws Exception
|
176
|
+
{
|
177
|
+
if (prepared) {
|
178
|
+
EmbulkPluginTester tester = new EmbulkPluginTester(InputPlugin.class, "mysql", MySQLInputPlugin.class);
|
179
|
+
tester.run(convertPath("/yml/input-double.yml"));
|
180
|
+
assertEquals(Arrays.asList(
|
181
|
+
"c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15",
|
182
|
+
",,,,,,,,,,,,2015-06-04 14:45:06,,",
|
183
|
+
"99.0,9999.0,-9.9999999E7,-1.0E16,1.2345000505447388,1.234567890123,-1234.0,1.2345678901234568E17,5678.0,,2015-06-03,2015-06-04 03:34:56,2015-06-04 14:45:06,23:04:02,2015-06-03 16:02:03"),
|
184
|
+
read("mysql-input.000.00.csv"));
|
185
|
+
}
|
186
|
+
}
|
187
|
+
|
188
|
+
@Test
|
189
|
+
public void testTimestamp1() throws Exception
|
190
|
+
{
|
191
|
+
if (prepared) {
|
192
|
+
EmbulkPluginTester tester = new EmbulkPluginTester(InputPlugin.class, "mysql", MySQLInputPlugin.class);
|
193
|
+
tester.run(convertPath("/yml/input-timestamp1.yml"));
|
194
|
+
assertEquals(Arrays.asList(
|
195
|
+
"c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15",
|
196
|
+
",,,,,,,,,,,,2015/06/04 14:45:06,,",
|
197
|
+
"99,9999,-99999999,-9999999999999999,1.2345000505447388,1.234567890123,-1234.0,1.2345678901234568E17,5678,xy,2015/06/03,2015/06/04 03:34:56,2015/06/04 14:45:06,23-04-02,2015/06/03 16:02:03.123456"),
|
198
|
+
read("mysql-input.000.00.csv"));
|
199
|
+
}
|
200
|
+
}
|
201
|
+
|
202
|
+
@Test
|
203
|
+
public void testTimestamp2() throws Exception
|
204
|
+
{
|
205
|
+
if (prepared) {
|
206
|
+
EmbulkPluginTester tester = new EmbulkPluginTester(InputPlugin.class, "mysql", MySQLInputPlugin.class);
|
207
|
+
tester.run(convertPath("/yml/input-timestamp2.yml"));
|
208
|
+
assertEquals(Arrays.asList(
|
209
|
+
"c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15",
|
210
|
+
",,,,,,,,,,,,2015/06/04 23:45:06,,",
|
211
|
+
"99,9999,-99999999,-9999999999999999,1.2345000505447388,1.234567890123,-1234.0,1.2345678901234568E17,5678,xy,2015/06/03,2015/06/04 03:34:56,2015/06/04 23:45:06,08-04-02,2015/06/03 16:02:03.123456"),
|
212
|
+
read("mysql-input.000.00.csv"));
|
213
|
+
}
|
214
|
+
}
|
215
|
+
|
216
|
+
@Test
|
217
|
+
public void testTimestamp3() throws Exception
|
218
|
+
{
|
219
|
+
if (prepared) {
|
220
|
+
EmbulkPluginTester tester = new EmbulkPluginTester(InputPlugin.class, "mysql", MySQLInputPlugin.class);
|
221
|
+
tester.run(convertPath("/yml/input-timestamp3.yml"));
|
222
|
+
assertEquals(Arrays.asList(
|
223
|
+
"c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15",
|
224
|
+
",,,,,,,,,,,,2015/06/04 23:45:06,,",
|
225
|
+
"99,9999,-99999999,-9999999999999999,1.2345000505447388,1.234567890123,-1234.0,1.2345678901234568E17,5678,xy,2015/06/04,2015/06/04 12:34:56,2015/06/04 23:45:06,02-04-02,2015/06/04 01:02:03.123456"),
|
226
|
+
read("mysql-input.000.00.csv"));
|
227
|
+
}
|
228
|
+
}
|
229
|
+
|
230
|
+
@Test
|
231
|
+
public void testValueTypeString() throws Exception
|
232
|
+
{
|
233
|
+
if (prepared) {
|
234
|
+
EmbulkPluginTester tester = new EmbulkPluginTester(InputPlugin.class, "mysql", MySQLInputPlugin.class);
|
235
|
+
tester.run(convertPath("/yml/input-valuetype-string.yml"));
|
236
|
+
assertEquals(Arrays.asList(
|
237
|
+
"c1",
|
238
|
+
"18446744073709551615"),
|
239
|
+
read("mysql-input.000.00.csv"));
|
240
|
+
}
|
241
|
+
}
|
242
|
+
|
243
|
+
@Test
|
244
|
+
public void testValueTypeDecimal() throws Exception
|
245
|
+
{
|
246
|
+
if (prepared) {
|
247
|
+
EmbulkPluginTester tester = new EmbulkPluginTester(InputPlugin.class, "mysql", MySQLInputPlugin.class);
|
248
|
+
tester.run(convertPath("/yml/input-valuetype-decimal.yml"));
|
249
|
+
assertEquals(Arrays.asList(
|
250
|
+
"c1",
|
251
|
+
"1.8446744073709552E19"),
|
252
|
+
read("mysql-input.000.00.csv"));
|
253
|
+
}
|
254
|
+
}
|
255
|
+
|
256
|
+
private List<String> read(String path) throws IOException
|
257
|
+
{
|
258
|
+
FileSystem fs = FileSystems.getDefault();
|
259
|
+
return Files.readAllLines(fs.getPath(path), Charset.defaultCharset());
|
260
|
+
}
|
261
|
+
|
262
|
+
private String convertPath(String name) throws URISyntaxException
|
263
|
+
{
|
264
|
+
if (getClass().getResource(name) == null) {
|
265
|
+
return name;
|
266
|
+
}
|
267
|
+
return new File(getClass().getResource(name).toURI()).getAbsolutePath();
|
268
|
+
}
|
269
|
+
}
|
@@ -0,0 +1,26 @@
|
|
1
|
+
in:
|
2
|
+
type: mysql
|
3
|
+
host: localhost
|
4
|
+
database: TESTDB
|
5
|
+
user: TEST_USER
|
6
|
+
password: test_pw
|
7
|
+
table: test1
|
8
|
+
select: "*"
|
9
|
+
column_options:
|
10
|
+
c1: {type: boolean}
|
11
|
+
c2: {type: boolean}
|
12
|
+
c3: {type: boolean}
|
13
|
+
c4: {type: boolean}
|
14
|
+
c5: {type: boolean}
|
15
|
+
c6: {type: boolean}
|
16
|
+
c7: {type: boolean}
|
17
|
+
c8: {type: boolean}
|
18
|
+
c9: {type: boolean}
|
19
|
+
c10: {type: boolean}
|
20
|
+
|
21
|
+
out:
|
22
|
+
type: file
|
23
|
+
path_prefix: mysql-input
|
24
|
+
file_ext: .csv
|
25
|
+
formatter:
|
26
|
+
type: csv
|
@@ -0,0 +1,26 @@
|
|
1
|
+
in:
|
2
|
+
type: mysql
|
3
|
+
host: localhost
|
4
|
+
database: TESTDB
|
5
|
+
user: TEST_USER
|
6
|
+
password: test_pw
|
7
|
+
table: test1
|
8
|
+
select: "*"
|
9
|
+
column_options:
|
10
|
+
c1: {type: double}
|
11
|
+
c2: {type: double}
|
12
|
+
c3: {type: double}
|
13
|
+
c4: {type: double}
|
14
|
+
c5: {type: double}
|
15
|
+
c6: {type: double}
|
16
|
+
c7: {type: double}
|
17
|
+
c8: {type: double}
|
18
|
+
c9: {type: double}
|
19
|
+
c10: {type: double}
|
20
|
+
|
21
|
+
out:
|
22
|
+
type: file
|
23
|
+
path_prefix: mysql-input
|
24
|
+
file_ext: .csv
|
25
|
+
formatter:
|
26
|
+
type: csv
|
@@ -0,0 +1,26 @@
|
|
1
|
+
in:
|
2
|
+
type: mysql
|
3
|
+
host: localhost
|
4
|
+
database: TESTDB
|
5
|
+
user: TEST_USER
|
6
|
+
password: test_pw
|
7
|
+
table: test1
|
8
|
+
select: "*"
|
9
|
+
column_options:
|
10
|
+
c1: {type: long}
|
11
|
+
c2: {type: long}
|
12
|
+
c3: {type: long}
|
13
|
+
c4: {type: long}
|
14
|
+
c5: {type: long}
|
15
|
+
c6: {type: long}
|
16
|
+
c7: {type: long}
|
17
|
+
c8: {type: long}
|
18
|
+
c9: {type: long}
|
19
|
+
c10: {type: long}
|
20
|
+
|
21
|
+
out:
|
22
|
+
type: file
|
23
|
+
path_prefix: mysql-input
|
24
|
+
file_ext: .csv
|
25
|
+
formatter:
|
26
|
+
type: csv
|
@@ -0,0 +1,26 @@
|
|
1
|
+
in:
|
2
|
+
type: mysql
|
3
|
+
host: localhost
|
4
|
+
database: TESTDB
|
5
|
+
user: TEST_USER
|
6
|
+
password: test_pw
|
7
|
+
table: test1
|
8
|
+
select: "*"
|
9
|
+
column_options:
|
10
|
+
c1: {type: string}
|
11
|
+
c2: {type: string}
|
12
|
+
c3: {type: string}
|
13
|
+
c4: {type: string}
|
14
|
+
c5: {type: string}
|
15
|
+
c6: {type: string}
|
16
|
+
c7: {type: string}
|
17
|
+
c8: {type: string}
|
18
|
+
c9: {type: string}
|
19
|
+
c10: {type: string}
|
20
|
+
|
21
|
+
out:
|
22
|
+
type: file
|
23
|
+
path_prefix: mysql-input
|
24
|
+
file_ext: .csv
|
25
|
+
formatter:
|
26
|
+
type: csv
|
@@ -0,0 +1,21 @@
|
|
1
|
+
in:
|
2
|
+
type: mysql
|
3
|
+
host: localhost
|
4
|
+
database: TESTDB
|
5
|
+
user: TEST_USER
|
6
|
+
password: test_pw
|
7
|
+
table: test1
|
8
|
+
select: "*"
|
9
|
+
column_options:
|
10
|
+
c11: {type: timestamp, timestamp_format: '%Y/%m/%d'}
|
11
|
+
c12: {type: timestamp, timestamp_format: '%Y/%m/%d %H:%M:%S'}
|
12
|
+
c13: {type: timestamp, timestamp_format: '%Y/%m/%d %H:%M:%S', timezone: '+0900'}
|
13
|
+
c14: {type: timestamp, timestamp_format: '%H-%M-%S', timezone: '+0900'}
|
14
|
+
c15: {type: timestamp, timestamp_format: '%Y/%m/%d %H:%M:%S.%6N'}
|
15
|
+
|
16
|
+
out:
|
17
|
+
type: file
|
18
|
+
path_prefix: mysql-input
|
19
|
+
file_ext: .csv
|
20
|
+
formatter:
|
21
|
+
type: csv
|
@@ -0,0 +1,21 @@
|
|
1
|
+
in:
|
2
|
+
type: mysql
|
3
|
+
host: localhost
|
4
|
+
database: TESTDB
|
5
|
+
user: TEST_USER
|
6
|
+
password: test_pw
|
7
|
+
table: test1
|
8
|
+
select: "*"
|
9
|
+
column_options:
|
10
|
+
c11: {type: string, timestamp_format: '%Y/%m/%d'}
|
11
|
+
c12: {type: string, timestamp_format: '%Y/%m/%d %H:%M:%S'}
|
12
|
+
c13: {type: string, timestamp_format: '%Y/%m/%d %H:%M:%S', timezone: '+0900'}
|
13
|
+
c14: {type: string, timestamp_format: '%H-%M-%S', timezone: '+0900'}
|
14
|
+
c15: {type: string, timestamp_format: '%Y/%m/%d %H:%M:%S.%6N'}
|
15
|
+
|
16
|
+
out:
|
17
|
+
type: file
|
18
|
+
path_prefix: mysql-input
|
19
|
+
file_ext: .csv
|
20
|
+
formatter:
|
21
|
+
type: csv
|
@@ -0,0 +1,22 @@
|
|
1
|
+
in:
|
2
|
+
type: mysql
|
3
|
+
host: localhost
|
4
|
+
database: TESTDB
|
5
|
+
user: TEST_USER
|
6
|
+
password: test_pw
|
7
|
+
table: test1
|
8
|
+
select: "*"
|
9
|
+
default_timezone: '+0900'
|
10
|
+
column_options:
|
11
|
+
c11: {type: string, timestamp_format: '%Y/%m/%d'}
|
12
|
+
c12: {type: string, timestamp_format: '%Y/%m/%d %H:%M:%S'}
|
13
|
+
c13: {type: string, timestamp_format: '%Y/%m/%d %H:%M:%S'}
|
14
|
+
c14: {type: string, timestamp_format: '%H-%M-%S', timezone: '+0300'}
|
15
|
+
c15: {type: string, timestamp_format: '%Y/%m/%d %H:%M:%S.%6N'}
|
16
|
+
|
17
|
+
out:
|
18
|
+
type: file
|
19
|
+
path_prefix: mysql-input
|
20
|
+
file_ext: .csv
|
21
|
+
formatter:
|
22
|
+
type: csv
|
@@ -0,0 +1,17 @@
|
|
1
|
+
in:
|
2
|
+
type: mysql
|
3
|
+
host: localhost
|
4
|
+
database: TESTDB
|
5
|
+
user: TEST_USER
|
6
|
+
password: test_pw
|
7
|
+
table: test2
|
8
|
+
select: "*"
|
9
|
+
column_options:
|
10
|
+
c1: {value_type: decimal}
|
11
|
+
|
12
|
+
out:
|
13
|
+
type: file
|
14
|
+
path_prefix: mysql-input
|
15
|
+
file_ext: .csv
|
16
|
+
formatter:
|
17
|
+
type: csv
|
@@ -0,0 +1,17 @@
|
|
1
|
+
in:
|
2
|
+
type: mysql
|
3
|
+
host: localhost
|
4
|
+
database: TESTDB
|
5
|
+
user: TEST_USER
|
6
|
+
password: test_pw
|
7
|
+
table: test2
|
8
|
+
select: "*"
|
9
|
+
column_options:
|
10
|
+
c1: {value_type: string}
|
11
|
+
|
12
|
+
out:
|
13
|
+
type: file
|
14
|
+
path_prefix: mysql-input
|
15
|
+
file_ext: .csv
|
16
|
+
formatter:
|
17
|
+
type: csv
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-input-mysql
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.5.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Sadayuki Furuhashi
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-
|
11
|
+
date: 2015-06-17 00:00:00.000000000 Z
|
12
12
|
dependencies: []
|
13
13
|
description: Selects records from a table.
|
14
14
|
email:
|
@@ -22,8 +22,22 @@ files:
|
|
22
22
|
- lib/embulk/input/mysql.rb
|
23
23
|
- src/main/java/org/embulk/input/MySQLInputPlugin.java
|
24
24
|
- src/main/java/org/embulk/input/mysql/MySQLInputConnection.java
|
25
|
-
-
|
26
|
-
-
|
25
|
+
- src/test/java/org/embulk/input/mysql/ChildFirstClassLoader.java
|
26
|
+
- src/test/java/org/embulk/input/mysql/EmbulkPluginTester.java
|
27
|
+
- src/test/java/org/embulk/input/mysql/EmptyConfigSource.java
|
28
|
+
- src/test/java/org/embulk/input/mysql/MySQLInputPluginTest.java
|
29
|
+
- src/test/resources/yml/input-boolean.yml
|
30
|
+
- src/test/resources/yml/input-double.yml
|
31
|
+
- src/test/resources/yml/input-long.yml
|
32
|
+
- src/test/resources/yml/input-string.yml
|
33
|
+
- src/test/resources/yml/input-timestamp1.yml
|
34
|
+
- src/test/resources/yml/input-timestamp2.yml
|
35
|
+
- src/test/resources/yml/input-timestamp3.yml
|
36
|
+
- src/test/resources/yml/input-valuetype-decimal.yml
|
37
|
+
- src/test/resources/yml/input-valuetype-string.yml
|
38
|
+
- src/test/resources/yml/input.yml
|
39
|
+
- classpath/embulk-input-jdbc-0.5.0.jar
|
40
|
+
- classpath/embulk-input-mysql-0.5.0.jar
|
27
41
|
- classpath/mysql-connector-java-5.1.34.jar
|
28
42
|
homepage: https://github.com/embulk/embulk-input-jdbc
|
29
43
|
licenses:
|
Binary file
|