embulk-input-postgresql 0.8.0 → 0.8.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +8 -0
- data/build.gradle +1 -2
- data/classpath/embulk-input-jdbc-0.8.1.jar +0 -0
- data/classpath/{embulk-input-postgresql-0.8.0.jar → embulk-input-postgresql-0.8.1.jar} +0 -0
- data/src/main/java/org/embulk/input/postgresql/getter/{HstoreColumnGetter.java → HstoreToJsonColumnGetter.java} +5 -30
- data/src/main/java/org/embulk/input/postgresql/getter/PostgreSQLColumnGetterFactory.java +28 -7
- data/src/test/java/org/embulk/input/postgresql/HstoreTest.java +75 -0
- data/src/test/java/org/embulk/input/postgresql/IncrementalTest.java +143 -0
- data/src/test/java/org/embulk/input/postgresql/PostgreSQLTests.java +39 -0
- data/src/test/resources/org/embulk/input/postgresql/test/expect/hstore/as_json.yml +3 -0
- data/src/test/resources/org/embulk/input/postgresql/test/expect/hstore/as_string.yml +1 -0
- data/src/test/resources/org/embulk/input/postgresql/test/expect/hstore/expected_json.csv +1 -0
- data/src/test/resources/org/embulk/input/postgresql/test/expect/hstore/expected_string.csv +1 -0
- data/src/test/resources/org/embulk/input/postgresql/test/expect/hstore/setup.sql +11 -0
- data/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/int/config_1.yml +3 -0
- data/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/int/config_2.yml +4 -0
- data/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/int/expected_1.csv +4 -0
- data/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/int/expected_1.diff +3 -0
- data/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/int/expected_2.csv +2 -0
- data/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/int/expected_2.diff +3 -0
- data/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/int/insert_more.sql +7 -0
- data/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/int/setup.sql +13 -0
- data/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/timestamp/config_1.yml +4 -0
- data/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/timestamp/config_2.yml +4 -0
- data/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/timestamp/expected_1.csv +7 -0
- data/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/timestamp/expected_1.diff +3 -0
- data/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/timestamp/expected_2.csv +3 -0
- data/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/timestamp/expected_2.diff +3 -0
- data/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/timestamp/insert_more.sql +9 -0
- data/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/timestamp/setup.sql +16 -0
- data/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/timestamptz/config_1.yml +4 -0
- data/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/timestamptz/config_2.yml +4 -0
- data/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/timestamptz/expected_1.csv +7 -0
- data/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/timestamptz/expected_1.diff +3 -0
- data/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/timestamptz/expected_2.csv +3 -0
- data/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/timestamptz/expected_2.diff +3 -0
- data/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/timestamptz/insert_more.sql +9 -0
- data/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/timestamptz/setup.sql +16 -0
- metadata +37 -8
- data/classpath/embulk-input-jdbc-0.8.0.jar +0 -0
- data/src/test/java/org/embulk/input/postgresql/PostgreSQLInputPluginTest.java +0 -102
- data/src/test/resources/yml/input_hstore.yml +0 -14
- data/src/test/resources/yml/input_hstore2.yml +0 -16
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: b02b6267f621dc419cbc4b72486ce4c69ec31dbf
|
4
|
+
data.tar.gz: 6257da06c3568e91fa9ffd6d7f4012300f5d4eed
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 2d0a03f7ab55bd8a90de593d31525a210fa2fddd1271bc422eca6664234e353d9c376e6c8b0a6b3fda0b318a5ba3b5cf9b5f01e7deabbee85b11a87c14806dea
|
7
|
+
data.tar.gz: 3892fa790c926a25dbf4fd11c7e78e1e62113d3ce53e241ebc252385449b6bdb3bcc7f622f4371b8678c231ce4947ec0acecaf31d3646298a8827c31fcc83b65
|
data/README.md
CHANGED
@@ -160,3 +160,11 @@ in:
|
|
160
160
|
```
|
161
161
|
$ ./gradlew gem
|
162
162
|
```
|
163
|
+
|
164
|
+
Running tests:
|
165
|
+
|
166
|
+
```
|
167
|
+
$ cp ci/travis_postgresql.yml ci/postgresql.yml # edit this file if necessary
|
168
|
+
$ EMBULK_INPUT_POSTGRESQL_TEST_CONFIG=`pwd`/ci/postgresql.yml ./gradlew :embulk-input-postgresql:check --info
|
169
|
+
```
|
170
|
+
|
data/build.gradle
CHANGED
Binary file
|
Binary file
|
@@ -2,62 +2,37 @@ package org.embulk.input.postgresql.getter;
|
|
2
2
|
|
3
3
|
import com.fasterxml.jackson.core.JsonProcessingException;
|
4
4
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
5
|
-
import org.embulk.input.jdbc.getter.
|
5
|
+
import org.embulk.input.jdbc.getter.JsonColumnGetter;
|
6
6
|
import org.embulk.spi.Column;
|
7
7
|
import org.embulk.spi.PageBuilder;
|
8
8
|
import org.embulk.spi.json.JsonParseException;
|
9
|
-
import org.embulk.spi.json.JsonParser;
|
10
9
|
import org.embulk.spi.type.Type;
|
11
|
-
import org.embulk.spi.type.Types;
|
12
10
|
import org.msgpack.value.Value;
|
13
11
|
import org.postgresql.util.HStoreConverter;
|
14
12
|
|
15
|
-
import java.sql.ResultSet;
|
16
|
-
import java.sql.SQLException;
|
17
13
|
import java.util.Map;
|
18
14
|
|
19
|
-
public class
|
20
|
-
extends
|
15
|
+
public class HstoreToJsonColumnGetter
|
16
|
+
extends JsonColumnGetter
|
21
17
|
{
|
22
|
-
private final JsonParser parser = new JsonParser();
|
23
18
|
private final ObjectMapper mapper = new ObjectMapper();
|
24
19
|
|
25
|
-
|
26
|
-
|
27
|
-
public HstoreColumnGetter(PageBuilder to, Type toType)
|
20
|
+
public HstoreToJsonColumnGetter(PageBuilder to, Type toType)
|
28
21
|
{
|
29
22
|
super(to, toType);
|
30
23
|
}
|
31
24
|
|
32
|
-
@Override
|
33
|
-
protected void fetch(ResultSet from, int fromIndex) throws SQLException
|
34
|
-
{
|
35
|
-
value = from.getString(fromIndex);
|
36
|
-
}
|
37
|
-
|
38
|
-
@Override
|
39
|
-
protected Type getDefaultToType()
|
40
|
-
{
|
41
|
-
return Types.STRING;
|
42
|
-
}
|
43
|
-
|
44
25
|
@Override
|
45
26
|
public void jsonColumn(Column column)
|
46
27
|
{
|
47
28
|
Value v;
|
48
29
|
try {
|
49
30
|
Map map = HStoreConverter.fromString(value);
|
50
|
-
v =
|
31
|
+
v = jsonParser.parse(mapper.writeValueAsString(map));
|
51
32
|
} catch (JsonProcessingException | JsonParseException e) {
|
52
33
|
super.jsonColumn(column);
|
53
34
|
return;
|
54
35
|
}
|
55
36
|
to.setJson(column, v);
|
56
37
|
}
|
57
|
-
|
58
|
-
@Override
|
59
|
-
public void stringColumn(Column column)
|
60
|
-
{
|
61
|
-
to.setString(column, value);
|
62
|
-
}
|
63
38
|
}
|
@@ -1,10 +1,15 @@
|
|
1
1
|
package org.embulk.input.postgresql.getter;
|
2
2
|
|
3
|
+
import org.embulk.input.jdbc.AbstractJdbcInputPlugin.PluginTask;
|
3
4
|
import org.embulk.input.jdbc.JdbcColumn;
|
4
5
|
import org.embulk.input.jdbc.JdbcColumnOption;
|
6
|
+
import org.embulk.input.jdbc.JdbcInputConnection;
|
5
7
|
import org.embulk.input.jdbc.getter.ColumnGetter;
|
6
8
|
import org.embulk.input.jdbc.getter.ColumnGetterFactory;
|
9
|
+
import org.embulk.input.jdbc.getter.TimestampWithTimeZoneIncrementalHandler;
|
10
|
+
import org.embulk.input.jdbc.getter.TimestampWithoutTimeZoneIncrementalHandler;
|
7
11
|
import org.embulk.spi.PageBuilder;
|
12
|
+
import org.embulk.spi.type.Types;
|
8
13
|
import org.joda.time.DateTimeZone;
|
9
14
|
|
10
15
|
public class PostgreSQLColumnGetterFactory extends ColumnGetterFactory
|
@@ -15,21 +20,37 @@ public class PostgreSQLColumnGetterFactory extends ColumnGetterFactory
|
|
15
20
|
}
|
16
21
|
|
17
22
|
@Override
|
18
|
-
public ColumnGetter newColumnGetter(JdbcColumn column, JdbcColumnOption option)
|
23
|
+
public ColumnGetter newColumnGetter(JdbcInputConnection con, PluginTask task, JdbcColumn column, JdbcColumnOption option)
|
19
24
|
{
|
20
|
-
if (column.getTypeName().equals("hstore")) {
|
21
|
-
|
22
|
-
|
23
|
-
|
25
|
+
if (column.getTypeName().equals("hstore") && getToType(option) == Types.JSON) {
|
26
|
+
// converting hstore to json needs a special handling
|
27
|
+
return new HstoreToJsonColumnGetter(to, Types.JSON);
|
28
|
+
}
|
29
|
+
|
30
|
+
ColumnGetter getter = super.newColumnGetter(con, task, column, option);
|
31
|
+
|
32
|
+
// incremental loading wrapper
|
33
|
+
switch (column.getTypeName()) {
|
34
|
+
case "timestamptz":
|
35
|
+
return new TimestampWithTimeZoneIncrementalHandler(getter);
|
36
|
+
case "timestamp":
|
37
|
+
return new TimestampWithoutTimeZoneIncrementalHandler(getter);
|
38
|
+
default:
|
39
|
+
return getter;
|
24
40
|
}
|
25
41
|
}
|
26
42
|
|
27
43
|
@Override
|
28
44
|
protected String sqlTypeToValueType(JdbcColumn column, int sqlType)
|
29
45
|
{
|
30
|
-
|
46
|
+
switch(column.getTypeName()) {
|
47
|
+
case "json":
|
48
|
+
case "jsonb":
|
31
49
|
return "json";
|
32
|
-
|
50
|
+
case "hstore":
|
51
|
+
// hstore is converted to string by default
|
52
|
+
return "string";
|
53
|
+
default:
|
33
54
|
return super.sqlTypeToValueType(column, sqlType);
|
34
55
|
}
|
35
56
|
}
|
@@ -0,0 +1,75 @@
|
|
1
|
+
package org.embulk.input.postgresql;
|
2
|
+
|
3
|
+
import java.nio.file.Path;
|
4
|
+
|
5
|
+
import org.embulk.config.ConfigDiff;
|
6
|
+
import org.embulk.config.ConfigSource;
|
7
|
+
import org.embulk.input.PostgreSQLInputPlugin;
|
8
|
+
import org.embulk.spi.InputPlugin;
|
9
|
+
import org.embulk.test.TestingEmbulk.RunResult;
|
10
|
+
import org.embulk.test.EmbulkTests;
|
11
|
+
import org.embulk.test.TestingEmbulk;
|
12
|
+
import org.junit.Before;
|
13
|
+
import org.junit.Rule;
|
14
|
+
import org.junit.Test;
|
15
|
+
|
16
|
+
import static org.embulk.input.postgresql.PostgreSQLTests.execute;
|
17
|
+
import static org.embulk.test.EmbulkTests.readSortedFile;
|
18
|
+
import static org.hamcrest.Matchers.is;
|
19
|
+
import static org.junit.Assert.assertThat;
|
20
|
+
|
21
|
+
public class HstoreTest
|
22
|
+
{
|
23
|
+
private static final String BASIC_RESOURCE_PATH = "org/embulk/input/postgresql/test/expect/hstore/";
|
24
|
+
|
25
|
+
private static ConfigSource loadYamlResource(TestingEmbulk embulk, String fileName)
|
26
|
+
{
|
27
|
+
return embulk.loadYamlResource(BASIC_RESOURCE_PATH + fileName);
|
28
|
+
}
|
29
|
+
|
30
|
+
private static String readResource(String fileName)
|
31
|
+
{
|
32
|
+
return EmbulkTests.readResource(BASIC_RESOURCE_PATH + fileName);
|
33
|
+
}
|
34
|
+
|
35
|
+
@Rule
|
36
|
+
public TestingEmbulk embulk = TestingEmbulk.builder()
|
37
|
+
.registerPlugin(InputPlugin.class, "postgresql", PostgreSQLInputPlugin.class)
|
38
|
+
.build();
|
39
|
+
|
40
|
+
private ConfigSource baseConfig;
|
41
|
+
|
42
|
+
@Before
|
43
|
+
public void setup()
|
44
|
+
{
|
45
|
+
baseConfig = PostgreSQLTests.baseConfig();
|
46
|
+
}
|
47
|
+
|
48
|
+
@Test
|
49
|
+
public void loadAsStringByDefault() throws Exception
|
50
|
+
{
|
51
|
+
execute(readResource("setup.sql"));
|
52
|
+
|
53
|
+
Path out1 = embulk.createTempFile("csv");
|
54
|
+
RunResult result1 = embulk.runInput(
|
55
|
+
baseConfig.merge(loadYamlResource(embulk, "as_string.yml")),
|
56
|
+
out1);
|
57
|
+
assertThat(
|
58
|
+
readSortedFile(out1),
|
59
|
+
is(readResource("expected_string.csv")));
|
60
|
+
}
|
61
|
+
|
62
|
+
@Test
|
63
|
+
public void loadAsJson() throws Exception
|
64
|
+
{
|
65
|
+
execute(readResource("setup.sql"));
|
66
|
+
|
67
|
+
Path out1 = embulk.createTempFile("csv");
|
68
|
+
RunResult result1 = embulk.runInput(
|
69
|
+
baseConfig.merge(loadYamlResource(embulk, "as_json.yml")),
|
70
|
+
out1);
|
71
|
+
assertThat(
|
72
|
+
readSortedFile(out1),
|
73
|
+
is(readResource("expected_json.csv")));
|
74
|
+
}
|
75
|
+
}
|
@@ -0,0 +1,143 @@
|
|
1
|
+
package org.embulk.input.postgresql;
|
2
|
+
|
3
|
+
import java.nio.file.Path;
|
4
|
+
|
5
|
+
import org.embulk.config.ConfigDiff;
|
6
|
+
import org.embulk.config.ConfigSource;
|
7
|
+
import org.embulk.input.PostgreSQLInputPlugin;
|
8
|
+
import org.embulk.spi.InputPlugin;
|
9
|
+
import org.embulk.test.TestingEmbulk.RunResult;
|
10
|
+
import org.embulk.test.EmbulkTests;
|
11
|
+
import org.embulk.test.TestingEmbulk;
|
12
|
+
import org.junit.Before;
|
13
|
+
import org.junit.Rule;
|
14
|
+
import org.junit.Test;
|
15
|
+
|
16
|
+
import static org.embulk.input.postgresql.PostgreSQLTests.execute;
|
17
|
+
import static org.embulk.test.EmbulkTests.readSortedFile;
|
18
|
+
import static org.hamcrest.Matchers.is;
|
19
|
+
import static org.junit.Assert.assertThat;
|
20
|
+
|
21
|
+
public class IncrementalTest
|
22
|
+
{
|
23
|
+
private static final String BASIC_RESOURCE_PATH = "org/embulk/input/postgresql/test/expect/incremental/";
|
24
|
+
|
25
|
+
private static ConfigSource loadYamlResource(TestingEmbulk embulk, String fileName)
|
26
|
+
{
|
27
|
+
return embulk.loadYamlResource(BASIC_RESOURCE_PATH + fileName);
|
28
|
+
}
|
29
|
+
|
30
|
+
private static String readResource(String fileName)
|
31
|
+
{
|
32
|
+
return EmbulkTests.readResource(BASIC_RESOURCE_PATH + fileName);
|
33
|
+
}
|
34
|
+
|
35
|
+
@Rule
|
36
|
+
public TestingEmbulk embulk = TestingEmbulk.builder()
|
37
|
+
.registerPlugin(InputPlugin.class, "postgresql", PostgreSQLInputPlugin.class)
|
38
|
+
.build();
|
39
|
+
|
40
|
+
private ConfigSource baseConfig;
|
41
|
+
|
42
|
+
@Before
|
43
|
+
public void setup()
|
44
|
+
{
|
45
|
+
baseConfig = PostgreSQLTests.baseConfig();
|
46
|
+
}
|
47
|
+
|
48
|
+
@Test
|
49
|
+
public void simpleInt() throws Exception
|
50
|
+
{
|
51
|
+
// setup first rows
|
52
|
+
execute(readResource("int/setup.sql"));
|
53
|
+
|
54
|
+
Path out1 = embulk.createTempFile("csv");
|
55
|
+
RunResult result1 = embulk.runInput(
|
56
|
+
baseConfig.merge(loadYamlResource(embulk, "int/config_1.yml")),
|
57
|
+
out1);
|
58
|
+
assertThat(
|
59
|
+
readSortedFile(out1),
|
60
|
+
is(readResource("int/expected_1.csv")));
|
61
|
+
assertThat(
|
62
|
+
result1.getConfigDiff(),
|
63
|
+
is((ConfigDiff) loadYamlResource(embulk, "int/expected_1.diff")));
|
64
|
+
|
65
|
+
// insert more rows
|
66
|
+
execute(readResource("int/insert_more.sql"));
|
67
|
+
|
68
|
+
Path out2 = embulk.createTempFile("csv");
|
69
|
+
RunResult result2 = embulk.runInput(
|
70
|
+
baseConfig.merge(loadYamlResource(embulk, "int/config_2.yml")),
|
71
|
+
out2);
|
72
|
+
assertThat(
|
73
|
+
readSortedFile(out2),
|
74
|
+
is(readResource("int/expected_2.csv")));
|
75
|
+
assertThat(
|
76
|
+
result2.getConfigDiff(),
|
77
|
+
is((ConfigDiff) loadYamlResource(embulk, "int/expected_2.diff")));
|
78
|
+
}
|
79
|
+
|
80
|
+
@Test
|
81
|
+
public void simpleTimestampWithoutTimeZone() throws Exception
|
82
|
+
{
|
83
|
+
// setup first rows
|
84
|
+
execute(readResource("timestamp/setup.sql"));
|
85
|
+
|
86
|
+
Path out1 = embulk.createTempFile("csv");
|
87
|
+
RunResult result1 = embulk.runInput(
|
88
|
+
baseConfig.merge(loadYamlResource(embulk, "timestamp/config_1.yml")),
|
89
|
+
out1);
|
90
|
+
assertThat(
|
91
|
+
readSortedFile(out1),
|
92
|
+
is(readResource("timestamp/expected_1.csv")));
|
93
|
+
assertThat(
|
94
|
+
result1.getConfigDiff(),
|
95
|
+
is((ConfigDiff) loadYamlResource(embulk, "timestamp/expected_1.diff")));
|
96
|
+
|
97
|
+
// insert more rows
|
98
|
+
execute(readResource("timestamp/insert_more.sql"));
|
99
|
+
|
100
|
+
Path out2 = embulk.createTempFile("csv");
|
101
|
+
RunResult result2 = embulk.runInput(
|
102
|
+
baseConfig.merge(loadYamlResource(embulk, "timestamp/config_2.yml")),
|
103
|
+
out2);
|
104
|
+
assertThat(
|
105
|
+
readSortedFile(out2),
|
106
|
+
is(readResource("timestamp/expected_2.csv")));
|
107
|
+
assertThat(
|
108
|
+
result2.getConfigDiff(),
|
109
|
+
is((ConfigDiff) loadYamlResource(embulk, "timestamp/expected_2.diff")));
|
110
|
+
}
|
111
|
+
|
112
|
+
@Test
|
113
|
+
public void simpleTimestampWithTimeZone() throws Exception
|
114
|
+
{
|
115
|
+
// setup first rows
|
116
|
+
execute(readResource("timestamptz/setup.sql"));
|
117
|
+
|
118
|
+
Path out1 = embulk.createTempFile("csv");
|
119
|
+
RunResult result1 = embulk.runInput(
|
120
|
+
baseConfig.merge(loadYamlResource(embulk, "timestamptz/config_1.yml")),
|
121
|
+
out1);
|
122
|
+
assertThat(
|
123
|
+
readSortedFile(out1),
|
124
|
+
is(readResource("timestamptz/expected_1.csv")));
|
125
|
+
assertThat(
|
126
|
+
result1.getConfigDiff(),
|
127
|
+
is((ConfigDiff) loadYamlResource(embulk, "timestamptz/expected_1.diff")));
|
128
|
+
|
129
|
+
// insert more rows
|
130
|
+
execute(readResource("timestamptz/insert_more.sql"));
|
131
|
+
|
132
|
+
Path out2 = embulk.createTempFile("csv");
|
133
|
+
RunResult result2 = embulk.runInput(
|
134
|
+
baseConfig.merge(loadYamlResource(embulk, "timestamptz/config_2.yml")),
|
135
|
+
out2);
|
136
|
+
assertThat(
|
137
|
+
readSortedFile(out2),
|
138
|
+
is(readResource("timestamptz/expected_2.csv")));
|
139
|
+
assertThat(
|
140
|
+
result2.getConfigDiff(),
|
141
|
+
is((ConfigDiff) loadYamlResource(embulk, "timestamptz/expected_2.diff")));
|
142
|
+
}
|
143
|
+
}
|
@@ -0,0 +1,39 @@
|
|
1
|
+
package org.embulk.input.postgresql;
|
2
|
+
|
3
|
+
import org.embulk.test.EmbulkTests;
|
4
|
+
import com.google.common.base.Throwables;
|
5
|
+
import com.google.common.io.ByteStreams;
|
6
|
+
import java.io.IOException;
|
7
|
+
import org.embulk.config.ConfigSource;
|
8
|
+
import static java.util.Locale.ENGLISH;
|
9
|
+
|
10
|
+
public class PostgreSQLTests
|
11
|
+
{
|
12
|
+
public static ConfigSource baseConfig()
|
13
|
+
{
|
14
|
+
return EmbulkTests.config("EMBULK_INPUT_POSTGRESQL_TEST_CONFIG");
|
15
|
+
}
|
16
|
+
|
17
|
+
public static void execute(String sql)
|
18
|
+
{
|
19
|
+
ConfigSource config = baseConfig();
|
20
|
+
ProcessBuilder pb = new ProcessBuilder("psql", "-w", "--set", "ON_ERROR_STOP=1", "-c", sql);
|
21
|
+
pb.environment().put("PGUSER", config.get(String.class, "user"));
|
22
|
+
pb.environment().put("PGPASSWORD", config.get(String.class, "password"));
|
23
|
+
pb.environment().put("PGDATABASE", config.get(String.class, "database"));
|
24
|
+
pb.environment().put("PGPORT", config.get(String.class, "port", "5432"));
|
25
|
+
pb.redirectErrorStream(true);
|
26
|
+
int code;
|
27
|
+
try {
|
28
|
+
Process process = pb.start();
|
29
|
+
ByteStreams.copy(process.getInputStream(), System.out);
|
30
|
+
code = process.waitFor();
|
31
|
+
} catch (IOException | InterruptedException ex) {
|
32
|
+
throw Throwables.propagate(ex);
|
33
|
+
}
|
34
|
+
if (code != 0) {
|
35
|
+
throw new RuntimeException(String.format(ENGLISH,
|
36
|
+
"Command finished with non-zero exit code. Exit code is %d.", code));
|
37
|
+
}
|
38
|
+
}
|
39
|
+
}
|
@@ -0,0 +1 @@
|
|
1
|
+
table: input_hstore
|
@@ -0,0 +1 @@
|
|
1
|
+
"{""a"":""b""}"
|
@@ -0,0 +1 @@
|
|
1
|
+
"""a""=>""b"""
|
data/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/timestamp/expected_1.csv
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
2016-11-01 23:00:01.000000 +0000,first
|
2
|
+
2016-11-02 00:00:02.000000 +0000,first
|
3
|
+
2016-11-02 01:00:03.000000 +0000,first
|
4
|
+
2016-11-02 02:00:04.000000 +0000,first
|
5
|
+
2016-11-02 02:00:05.111001 +0000,first
|
6
|
+
2016-11-02 02:00:05.222002 +0000,first
|
7
|
+
2016-11-02 02:00:05.333003 +0000,first
|
@@ -0,0 +1,9 @@
|
|
1
|
+
|
2
|
+
insert into load (time, note) values
|
3
|
+
('2016-11-02 04:00:00', 'more_skip'),
|
4
|
+
('2016-11-02 04:00:05.333000', 'more_skip'),
|
5
|
+
('2016-11-02 04:00:05.333003', 'more_skip'),
|
6
|
+
('2016-11-02 04:00:05.333004', 'more_load'),
|
7
|
+
('2016-11-02 04:00:06', 'more_load'),
|
8
|
+
('2016-11-02 04:00:06', 'more_load');
|
9
|
+
|
data/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/timestamp/setup.sql
ADDED
@@ -0,0 +1,16 @@
|
|
1
|
+
drop table if exists load;
|
2
|
+
|
3
|
+
create table load (
|
4
|
+
time timestamp without time zone not null,
|
5
|
+
note text
|
6
|
+
);
|
7
|
+
|
8
|
+
insert into load (time, note) values
|
9
|
+
('2016-11-02 01:00:01', 'first'),
|
10
|
+
('2016-11-02 02:00:02', 'first'),
|
11
|
+
('2016-11-02 03:00:03', 'first'),
|
12
|
+
('2016-11-02 04:00:04', 'first'),
|
13
|
+
('2016-11-02 04:00:05.111001', 'first'),
|
14
|
+
('2016-11-02 04:00:05.222002', 'first'),
|
15
|
+
('2016-11-02 04:00:05.333003', 'first');
|
16
|
+
|
@@ -0,0 +1,7 @@
|
|
1
|
+
2016-11-02 01:00:01.000000 +0000,first
|
2
|
+
2016-11-02 02:00:02.000000 +0000,first
|
3
|
+
2016-11-02 03:00:03.000000 +0000,first
|
4
|
+
2016-11-02 04:00:04.000000 +0000,first
|
5
|
+
2016-11-02 04:00:05.111001 +0000,first
|
6
|
+
2016-11-02 04:00:05.222002 +0000,first
|
7
|
+
2016-11-02 04:00:05.333003 +0000,first
|
@@ -0,0 +1,9 @@
|
|
1
|
+
|
2
|
+
insert into load (time, note) values
|
3
|
+
('2016-11-02 04:00:00+0000', 'more_skip'),
|
4
|
+
('2016-11-02 04:00:05.333000+0000', 'more_skip'),
|
5
|
+
('2016-11-02 04:00:05.333003+0000', 'more_skip'),
|
6
|
+
('2016-11-02 04:00:05.333004+0000', 'more_load'),
|
7
|
+
('2016-11-02 04:00:06+0000', 'more_load'),
|
8
|
+
('2016-11-02 04:00:06+0000', 'more_load');
|
9
|
+
|
data/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/timestamptz/setup.sql
ADDED
@@ -0,0 +1,16 @@
|
|
1
|
+
drop table if exists load;
|
2
|
+
|
3
|
+
create table load (
|
4
|
+
time timestamptz(6) not null,
|
5
|
+
note text
|
6
|
+
);
|
7
|
+
|
8
|
+
insert into load (time, note) values
|
9
|
+
('2016-11-02 01:00:01+0000', 'first'),
|
10
|
+
('2016-11-02 02:00:02+0000', 'first'),
|
11
|
+
('2016-11-02 03:00:03+0000', 'first'),
|
12
|
+
('2016-11-02 04:00:04+0000', 'first'),
|
13
|
+
('2016-11-02 04:00:05.111001+0000', 'first'),
|
14
|
+
('2016-11-02 04:00:05.222002+0000', 'first'),
|
15
|
+
('2016-11-02 04:00:05.333003+0000', 'first');
|
16
|
+
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-input-postgresql
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.8.
|
4
|
+
version: 0.8.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Sadayuki Furuhashi
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2017-02-10 00:00:00.000000000 Z
|
12
12
|
dependencies: []
|
13
13
|
description: Selects records from a table.
|
14
14
|
email:
|
@@ -19,17 +19,46 @@ extra_rdoc_files: []
|
|
19
19
|
files:
|
20
20
|
- README.md
|
21
21
|
- build.gradle
|
22
|
-
- classpath/embulk-input-jdbc-0.8.
|
23
|
-
- classpath/embulk-input-postgresql-0.8.
|
22
|
+
- classpath/embulk-input-jdbc-0.8.1.jar
|
23
|
+
- classpath/embulk-input-postgresql-0.8.1.jar
|
24
24
|
- classpath/postgresql-9.4-1205-jdbc41.jar
|
25
25
|
- lib/embulk/input/postgresql.rb
|
26
26
|
- src/main/java/org/embulk/input/PostgreSQLInputPlugin.java
|
27
27
|
- src/main/java/org/embulk/input/postgresql/PostgreSQLInputConnection.java
|
28
|
-
- src/main/java/org/embulk/input/postgresql/getter/
|
28
|
+
- src/main/java/org/embulk/input/postgresql/getter/HstoreToJsonColumnGetter.java
|
29
29
|
- src/main/java/org/embulk/input/postgresql/getter/PostgreSQLColumnGetterFactory.java
|
30
|
-
- src/test/java/org/embulk/input/postgresql/
|
31
|
-
- src/test/
|
32
|
-
- src/test/
|
30
|
+
- src/test/java/org/embulk/input/postgresql/HstoreTest.java
|
31
|
+
- src/test/java/org/embulk/input/postgresql/IncrementalTest.java
|
32
|
+
- src/test/java/org/embulk/input/postgresql/PostgreSQLTests.java
|
33
|
+
- src/test/resources/org/embulk/input/postgresql/test/expect/hstore/as_json.yml
|
34
|
+
- src/test/resources/org/embulk/input/postgresql/test/expect/hstore/as_string.yml
|
35
|
+
- src/test/resources/org/embulk/input/postgresql/test/expect/hstore/expected_json.csv
|
36
|
+
- src/test/resources/org/embulk/input/postgresql/test/expect/hstore/expected_string.csv
|
37
|
+
- src/test/resources/org/embulk/input/postgresql/test/expect/hstore/setup.sql
|
38
|
+
- src/test/resources/org/embulk/input/postgresql/test/expect/incremental/int/config_1.yml
|
39
|
+
- src/test/resources/org/embulk/input/postgresql/test/expect/incremental/int/config_2.yml
|
40
|
+
- src/test/resources/org/embulk/input/postgresql/test/expect/incremental/int/expected_1.csv
|
41
|
+
- src/test/resources/org/embulk/input/postgresql/test/expect/incremental/int/expected_1.diff
|
42
|
+
- src/test/resources/org/embulk/input/postgresql/test/expect/incremental/int/expected_2.csv
|
43
|
+
- src/test/resources/org/embulk/input/postgresql/test/expect/incremental/int/expected_2.diff
|
44
|
+
- src/test/resources/org/embulk/input/postgresql/test/expect/incremental/int/insert_more.sql
|
45
|
+
- src/test/resources/org/embulk/input/postgresql/test/expect/incremental/int/setup.sql
|
46
|
+
- src/test/resources/org/embulk/input/postgresql/test/expect/incremental/timestamp/config_1.yml
|
47
|
+
- src/test/resources/org/embulk/input/postgresql/test/expect/incremental/timestamp/config_2.yml
|
48
|
+
- src/test/resources/org/embulk/input/postgresql/test/expect/incremental/timestamp/expected_1.csv
|
49
|
+
- src/test/resources/org/embulk/input/postgresql/test/expect/incremental/timestamp/expected_1.diff
|
50
|
+
- src/test/resources/org/embulk/input/postgresql/test/expect/incremental/timestamp/expected_2.csv
|
51
|
+
- src/test/resources/org/embulk/input/postgresql/test/expect/incremental/timestamp/expected_2.diff
|
52
|
+
- src/test/resources/org/embulk/input/postgresql/test/expect/incremental/timestamp/insert_more.sql
|
53
|
+
- src/test/resources/org/embulk/input/postgresql/test/expect/incremental/timestamp/setup.sql
|
54
|
+
- src/test/resources/org/embulk/input/postgresql/test/expect/incremental/timestamptz/config_1.yml
|
55
|
+
- src/test/resources/org/embulk/input/postgresql/test/expect/incremental/timestamptz/config_2.yml
|
56
|
+
- src/test/resources/org/embulk/input/postgresql/test/expect/incremental/timestamptz/expected_1.csv
|
57
|
+
- src/test/resources/org/embulk/input/postgresql/test/expect/incremental/timestamptz/expected_1.diff
|
58
|
+
- src/test/resources/org/embulk/input/postgresql/test/expect/incremental/timestamptz/expected_2.csv
|
59
|
+
- src/test/resources/org/embulk/input/postgresql/test/expect/incremental/timestamptz/expected_2.diff
|
60
|
+
- src/test/resources/org/embulk/input/postgresql/test/expect/incremental/timestamptz/insert_more.sql
|
61
|
+
- src/test/resources/org/embulk/input/postgresql/test/expect/incremental/timestamptz/setup.sql
|
33
62
|
homepage: https://github.com/embulk/embulk-input-jdbc
|
34
63
|
licenses:
|
35
64
|
- Apache 2.0
|
Binary file
|
@@ -1,102 +0,0 @@
|
|
1
|
-
package org.embulk.input.postgresql;
|
2
|
-
|
3
|
-
import static java.util.Locale.ENGLISH;
|
4
|
-
import static org.junit.Assert.assertEquals;
|
5
|
-
|
6
|
-
import java.io.BufferedReader;
|
7
|
-
import java.io.IOException;
|
8
|
-
import java.io.InputStreamReader;
|
9
|
-
import java.sql.Connection;
|
10
|
-
import java.sql.DriverManager;
|
11
|
-
import java.sql.SQLException;
|
12
|
-
import java.util.Arrays;
|
13
|
-
|
14
|
-
import org.apache.commons.lang3.StringUtils;
|
15
|
-
import org.embulk.input.AbstractJdbcInputPluginTest;
|
16
|
-
import org.embulk.input.PostgreSQLInputPlugin;
|
17
|
-
import org.embulk.spi.InputPlugin;
|
18
|
-
import org.junit.Test;
|
19
|
-
|
20
|
-
public class PostgreSQLInputPluginTest extends AbstractJdbcInputPluginTest
|
21
|
-
{
|
22
|
-
@Override
|
23
|
-
protected void prepare() throws SQLException
|
24
|
-
{
|
25
|
-
tester.addPlugin(InputPlugin.class, "postgresql", PostgreSQLInputPlugin.class);
|
26
|
-
|
27
|
-
try {
|
28
|
-
// Create User and Database
|
29
|
-
psql(String.format(ENGLISH, "DROP DATABASE IF EXISTS %s;", getDatabase()));
|
30
|
-
psql(String.format(ENGLISH, "DROP USER IF EXISTS %s;", getUser()));
|
31
|
-
psql(String.format(ENGLISH, "CREATE USER %s WITH SUPERUSER PASSWORD '%s';", getUser(), getPassword()));
|
32
|
-
psql(String.format(ENGLISH, "CREATE DATABASE %s WITH OWNER %s;", getDatabase(), getUser()));
|
33
|
-
} catch (IOException e) {
|
34
|
-
System.err.println(e);
|
35
|
-
System.err.println("Warning: cannot prepare a database for testing embulk-input-postgresql.");
|
36
|
-
// 1. install postgresql.
|
37
|
-
// 2. add bin directory to path.
|
38
|
-
// 3. set environment variable PGPASSWORD or write pgpassword in tests.yml
|
39
|
-
return;
|
40
|
-
} catch (InterruptedException e) {
|
41
|
-
throw new RuntimeException(e);
|
42
|
-
}
|
43
|
-
|
44
|
-
enabled = true;
|
45
|
-
|
46
|
-
// Insert Data
|
47
|
-
String sql = "";
|
48
|
-
sql += "DROP TABLE IF EXISTS input_hstore;";
|
49
|
-
sql += "CREATE EXTENSION IF NOT EXISTS hstore WITH SCHEMA public;";
|
50
|
-
sql += "CREATE TABLE input_hstore (c1 hstore);";
|
51
|
-
sql += "INSERT INTO input_hstore (c1) VALUES('\"a\" => \"b\"');";
|
52
|
-
executeSQL(sql);
|
53
|
-
}
|
54
|
-
|
55
|
-
@Test
|
56
|
-
public void testHstoreAsString() throws Exception
|
57
|
-
{
|
58
|
-
if (enabled) {
|
59
|
-
test("/yml/input_hstore.yml");
|
60
|
-
assertEquals(Arrays.asList("c1", "\"\"\"a\"\"=>\"\"b\"\"\""),
|
61
|
-
read("postgresql-input000.00.csv"));
|
62
|
-
}
|
63
|
-
}
|
64
|
-
|
65
|
-
@Test
|
66
|
-
public void testHstoreAsJson() throws Exception
|
67
|
-
{
|
68
|
-
if (enabled) {
|
69
|
-
test("/yml/input_hstore2.yml");
|
70
|
-
assertEquals(Arrays.asList("c1", "\"{\"\"a\"\":\"\"b\"\"}\""),
|
71
|
-
read("postgresql-input000.00.csv"));
|
72
|
-
}
|
73
|
-
}
|
74
|
-
|
75
|
-
private void psql(String sql) throws IOException, InterruptedException {
|
76
|
-
ProcessBuilder pb = new ProcessBuilder("psql", "-w", "-c", sql);
|
77
|
-
String pgPassword = (String)getTestConfig("pgpassword", false);
|
78
|
-
if (!StringUtils.isEmpty(pgPassword)) {
|
79
|
-
pb.environment().put("PGPASSWORD", pgPassword);
|
80
|
-
}
|
81
|
-
System.out.println("PSQL: " + pb.command().toString());
|
82
|
-
final Process process = pb.start();
|
83
|
-
final int code = process.waitFor();
|
84
|
-
if (code != 0) {
|
85
|
-
try (BufferedReader reader = new BufferedReader(new InputStreamReader(process.getErrorStream()))) {
|
86
|
-
String line;
|
87
|
-
while ((line = reader.readLine()) != null) {
|
88
|
-
System.err.println(line);
|
89
|
-
}
|
90
|
-
}
|
91
|
-
throw new IOException(String.format(
|
92
|
-
"Command finished with non-zero exit code. Exit code is %d.", code));
|
93
|
-
}
|
94
|
-
}
|
95
|
-
|
96
|
-
@Override
|
97
|
-
protected Connection connect() throws SQLException
|
98
|
-
{
|
99
|
-
return DriverManager.getConnection(String.format(ENGLISH, "jdbc:postgresql://%s:%d/%s", getHost(), getPort(), getDatabase()),
|
100
|
-
getUser(), getPassword());
|
101
|
-
}
|
102
|
-
}
|
@@ -1,16 +0,0 @@
|
|
1
|
-
in:
|
2
|
-
type: postgresql
|
3
|
-
host: #host#
|
4
|
-
database: #database#
|
5
|
-
user: #user#
|
6
|
-
password: #password#
|
7
|
-
table: input_hstore
|
8
|
-
select: "*"
|
9
|
-
column_options:
|
10
|
-
c1: {type: json}
|
11
|
-
out:
|
12
|
-
type: file
|
13
|
-
path_prefix: postgresql-input
|
14
|
-
file_ext: csv
|
15
|
-
formatter:
|
16
|
-
type: csv
|