embulk-input-postgresql 0.6.4 → 0.7.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +21 -2
- data/build.gradle +1 -0
- data/classpath/{embulk-input-jdbc-0.6.4.jar → embulk-input-jdbc-0.7.0.jar} +0 -0
- data/classpath/embulk-input-postgresql-0.7.0.jar +0 -0
- data/src/main/java/org/embulk/input/PostgreSQLInputPlugin.java +13 -0
- data/src/main/java/org/embulk/input/postgresql/getter/HstoreColumnGetter.java +63 -0
- data/src/main/java/org/embulk/input/postgresql/getter/PostgreSQLColumnGetterFactory.java +36 -0
- data/src/test/java/org/embulk/input/postgresql/PostgreSQLInputPluginTest.java +126 -0
- data/src/test/resources/yml/input_hstore.yml +14 -0
- data/src/test/resources/yml/input_hstore2.yml +16 -0
- metadata +9 -4
- data/classpath/embulk-input-postgresql-0.6.4.jar +0 -0
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: a8a52d399d27705849e5c548b2a4a2e26d3f8db4
|
4
|
+
data.tar.gz: ae9c01752906c2132c4dea6acdd47e49cedae5ae
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 97da962ac8166bfb6929fdd4c375d1e8c24348a4c9ae0fd865435cc68974b7acff222fa5617bcc3ccacc71afe74cea3710da423903290011233dc178e9c6e173
|
7
|
+
data.tar.gz: cdf01ea2f741ceea6948202d56e241cb05430af539bba10574afb3918bcef3329e855c5ee7ca66cf90a56c7392526e69e478b08600c51b979c023edb7b978b22
|
data/README.md
CHANGED
@@ -29,13 +29,31 @@ PostgreSQL input plugins for Embulk loads records from PostgreSQL.
|
|
29
29
|
- **default_timezone**: If the sql type of a column is `date`/`time`/`datetime` and the embulk type is `string`, column values are formatted int this default_timezone. You can overwrite timezone for each columns using column_options option. (string, default: `UTC`)
|
30
30
|
- **column_options**: advanced: a key-value pairs where key is a column name and value is options for the column.
|
31
31
|
- **value_type**: embulk get values from database as this value_type. Typically, the value_type determines `getXXX` method of `java.sql.PreparedStatement`.
|
32
|
-
(string, default: depends on the sql type of the column. Available values options are: `long`, `double`, `float`, `decimal`, `boolean`, `string`, `date`, `time`, `timestamp`)
|
32
|
+
(string, default: depends on the sql type of the column. Available values options are: `long`, `double`, `float`, `decimal`, `boolean`, `string`, `json`, `date`, `time`, `timestamp`)
|
33
|
+
See below for `hstore` column.
|
33
34
|
- **type**: Column values are converted to this embulk type.
|
34
|
-
Available values options are: `boolean`, `long`, `double`, `string`, `timestamp`).
|
35
|
+
Available values options are: `boolean`, `long`, `double`, `string`, `json`, `timestamp`).
|
35
36
|
By default, the embulk type is determined according to the sql type of the column (or value_type if specified).
|
37
|
+
See below for `hstore` column.
|
36
38
|
- **timestamp_format**: If the sql type of the column is `date`/`time`/`datetime` and the embulk type is `string`, column values are formatted by this timestamp_format. And if the embulk type is `timestamp`, this timestamp_format may be used in the output plugin. For example, stdout plugin use the timestamp_format, but *csv formatter plugin doesn't use*. (string, default : `%Y-%m-%d` for `date`, `%H:%M:%S` for `time`, `%Y-%m-%d %H:%M:%S` for `timestamp`)
|
37
39
|
- **timezone**: If the sql type of the column is `date`/`time`/`datetime` and the embulk type is `string`, column values are formatted in this timezone.
|
38
40
|
(string, value of default_timezone option is used by default)
|
41
|
+
- **after_select**: if set, this SQL will be executed after the SELECT query in the same transaction.
|
42
|
+
|
43
|
+
### hstore column support
|
44
|
+
|
45
|
+
By default, `type` of `column_options` for `hstore` column is `string`, and output will be as follows.
|
46
|
+
```
|
47
|
+
"key1"=>"value1", "key2"=>"value2"
|
48
|
+
```
|
49
|
+
|
50
|
+
In addition, `json` type is supported for `hstore` column, and output will be as follows.
|
51
|
+
```
|
52
|
+
{"key1": "value1", "key2": "value2"}
|
53
|
+
```
|
54
|
+
|
55
|
+
`value_type` is ignored.
|
56
|
+
|
39
57
|
|
40
58
|
## Example
|
41
59
|
|
@@ -82,6 +100,7 @@ in:
|
|
82
100
|
column_options:
|
83
101
|
col1: {type: long}
|
84
102
|
col3: {type: string, timestamp_format: "%Y/%m/%d", timezone: "+0900"}
|
103
|
+
after_select: "update my_table set col5 = '1' where col4 != 'a'"
|
85
104
|
|
86
105
|
```
|
87
106
|
|
data/build.gradle
CHANGED
Binary file
|
Binary file
|
@@ -1,13 +1,20 @@
|
|
1
1
|
package org.embulk.input;
|
2
2
|
|
3
|
+
import java.util.Map;
|
3
4
|
import java.util.Properties;
|
4
5
|
import java.sql.Connection;
|
5
6
|
import java.sql.Driver;
|
6
7
|
import java.sql.SQLException;
|
8
|
+
|
7
9
|
import org.embulk.config.Config;
|
8
10
|
import org.embulk.config.ConfigDefault;
|
9
11
|
import org.embulk.input.jdbc.AbstractJdbcInputPlugin;
|
12
|
+
import org.embulk.input.jdbc.JdbcColumnOption;
|
13
|
+
import org.embulk.input.jdbc.getter.ColumnGetterFactory;
|
10
14
|
import org.embulk.input.postgresql.PostgreSQLInputConnection;
|
15
|
+
import org.embulk.input.postgresql.getter.PostgreSQLColumnGetterFactory;
|
16
|
+
import org.embulk.spi.PageBuilder;
|
17
|
+
import org.joda.time.DateTimeZone;
|
11
18
|
|
12
19
|
public class PostgreSQLInputPlugin
|
13
20
|
extends AbstractJdbcInputPlugin
|
@@ -88,4 +95,10 @@ public class PostgreSQLInputPlugin
|
|
88
95
|
}
|
89
96
|
}
|
90
97
|
}
|
98
|
+
|
99
|
+
@Override
|
100
|
+
protected ColumnGetterFactory newColumnGetterFactory(PageBuilder pageBuilder, DateTimeZone dateTimeZone)
|
101
|
+
{
|
102
|
+
return new PostgreSQLColumnGetterFactory(pageBuilder, dateTimeZone);
|
103
|
+
}
|
91
104
|
}
|
@@ -0,0 +1,63 @@
|
|
1
|
+
package org.embulk.input.postgresql.getter;
|
2
|
+
|
3
|
+
import com.fasterxml.jackson.core.JsonProcessingException;
|
4
|
+
import com.fasterxml.jackson.databind.ObjectMapper;
|
5
|
+
import org.embulk.input.jdbc.getter.AbstractColumnGetter;
|
6
|
+
import org.embulk.spi.Column;
|
7
|
+
import org.embulk.spi.PageBuilder;
|
8
|
+
import org.embulk.spi.json.JsonParseException;
|
9
|
+
import org.embulk.spi.json.JsonParser;
|
10
|
+
import org.embulk.spi.type.Type;
|
11
|
+
import org.embulk.spi.type.Types;
|
12
|
+
import org.msgpack.value.Value;
|
13
|
+
import org.postgresql.util.HStoreConverter;
|
14
|
+
|
15
|
+
import java.sql.ResultSet;
|
16
|
+
import java.sql.SQLException;
|
17
|
+
import java.util.Map;
|
18
|
+
|
19
|
+
public class HstoreColumnGetter
|
20
|
+
extends AbstractColumnGetter
|
21
|
+
{
|
22
|
+
private final JsonParser parser = new JsonParser();
|
23
|
+
private final ObjectMapper mapper = new ObjectMapper();
|
24
|
+
|
25
|
+
private String value;
|
26
|
+
|
27
|
+
public HstoreColumnGetter(PageBuilder to, Type toType)
|
28
|
+
{
|
29
|
+
super(to, toType);
|
30
|
+
}
|
31
|
+
|
32
|
+
@Override
|
33
|
+
protected void fetch(ResultSet from, int fromIndex) throws SQLException
|
34
|
+
{
|
35
|
+
value = from.getString(fromIndex);
|
36
|
+
}
|
37
|
+
|
38
|
+
@Override
|
39
|
+
protected Type getDefaultToType()
|
40
|
+
{
|
41
|
+
return Types.STRING;
|
42
|
+
}
|
43
|
+
|
44
|
+
@Override
|
45
|
+
public void jsonColumn(Column column)
|
46
|
+
{
|
47
|
+
Value v;
|
48
|
+
try {
|
49
|
+
Map map = HStoreConverter.fromString(value);
|
50
|
+
v = parser.parse(mapper.writeValueAsString(map));
|
51
|
+
} catch (JsonProcessingException | JsonParseException e) {
|
52
|
+
super.jsonColumn(column);
|
53
|
+
return;
|
54
|
+
}
|
55
|
+
to.setJson(column, v);
|
56
|
+
}
|
57
|
+
|
58
|
+
@Override
|
59
|
+
public void stringColumn(Column column)
|
60
|
+
{
|
61
|
+
to.setString(column, value);
|
62
|
+
}
|
63
|
+
}
|
@@ -0,0 +1,36 @@
|
|
1
|
+
package org.embulk.input.postgresql.getter;
|
2
|
+
|
3
|
+
import org.embulk.input.jdbc.JdbcColumn;
|
4
|
+
import org.embulk.input.jdbc.JdbcColumnOption;
|
5
|
+
import org.embulk.input.jdbc.getter.ColumnGetter;
|
6
|
+
import org.embulk.input.jdbc.getter.ColumnGetterFactory;
|
7
|
+
import org.embulk.spi.PageBuilder;
|
8
|
+
import org.joda.time.DateTimeZone;
|
9
|
+
|
10
|
+
public class PostgreSQLColumnGetterFactory extends ColumnGetterFactory
|
11
|
+
{
|
12
|
+
public PostgreSQLColumnGetterFactory(PageBuilder to, DateTimeZone defaultTimeZone)
|
13
|
+
{
|
14
|
+
super(to, defaultTimeZone);
|
15
|
+
}
|
16
|
+
|
17
|
+
@Override
|
18
|
+
public ColumnGetter newColumnGetter(JdbcColumn column, JdbcColumnOption option)
|
19
|
+
{
|
20
|
+
if (column.getTypeName().equals("hstore")) {
|
21
|
+
return new HstoreColumnGetter(to, getToType(option));
|
22
|
+
} else {
|
23
|
+
return super.newColumnGetter(column, option);
|
24
|
+
}
|
25
|
+
}
|
26
|
+
|
27
|
+
@Override
|
28
|
+
protected String sqlTypeToValueType(JdbcColumn column, int sqlType)
|
29
|
+
{
|
30
|
+
if (column.getTypeName().equals("json") || column.getTypeName().equals("jsonb")) {
|
31
|
+
return "json";
|
32
|
+
} else {
|
33
|
+
return super.sqlTypeToValueType(column, sqlType);
|
34
|
+
}
|
35
|
+
}
|
36
|
+
}
|
@@ -0,0 +1,126 @@
|
|
1
|
+
package org.embulk.input.postgresql;
|
2
|
+
|
3
|
+
import static org.junit.Assert.assertEquals;
|
4
|
+
|
5
|
+
import java.io.BufferedReader;
|
6
|
+
import java.io.File;
|
7
|
+
import java.io.IOException;
|
8
|
+
import java.io.InputStreamReader;
|
9
|
+
import java.net.URISyntaxException;
|
10
|
+
import java.nio.charset.Charset;
|
11
|
+
import java.nio.file.FileSystem;
|
12
|
+
import java.nio.file.FileSystems;
|
13
|
+
import java.nio.file.Files;
|
14
|
+
import java.sql.Connection;
|
15
|
+
import java.sql.DriverManager;
|
16
|
+
import java.sql.Statement;
|
17
|
+
import java.util.Arrays;
|
18
|
+
import java.util.List;
|
19
|
+
|
20
|
+
import org.embulk.input.EmbulkPluginTester;
|
21
|
+
import org.embulk.input.PostgreSQLInputPlugin;
|
22
|
+
import org.embulk.spi.InputPlugin;
|
23
|
+
import org.junit.AfterClass;
|
24
|
+
import org.junit.BeforeClass;
|
25
|
+
import org.junit.Test;
|
26
|
+
|
27
|
+
public class PostgreSQLInputPluginTest
|
28
|
+
{
|
29
|
+
private static final String DATABASE = "test_db";
|
30
|
+
private static final String USER = "test_user";
|
31
|
+
private static final String PASSWORD = "test_pw";
|
32
|
+
private static final String URL = "jdbc:postgresql://localhost:5432/" + DATABASE;
|
33
|
+
|
34
|
+
private static boolean prepared = false;
|
35
|
+
private static EmbulkPluginTester tester = new EmbulkPluginTester(InputPlugin.class, "postgresql", PostgreSQLInputPlugin.class);
|
36
|
+
|
37
|
+
@BeforeClass
|
38
|
+
public static void prepare() throws Exception
|
39
|
+
{
|
40
|
+
try {
|
41
|
+
// Create User and Database
|
42
|
+
psql(String.format("DROP DATABASE IF EXISTS %s;", DATABASE));
|
43
|
+
psql(String.format("DROP USER IF EXISTS %s;", USER));
|
44
|
+
psql(String.format("CREATE USER %s WITH SUPERUSER PASSWORD '%s';", USER, PASSWORD));
|
45
|
+
psql(String.format("CREATE DATABASE %s WITH OWNER %s;", DATABASE, USER));
|
46
|
+
} catch (IOException e) {
|
47
|
+
System.err.println(e);
|
48
|
+
System.err.println("Warning: cannot prepare a database for testing embulk-input-postgresql.");
|
49
|
+
// 1. install postgresql.
|
50
|
+
// 2. add bin directory to path.
|
51
|
+
// 3. set environment variable PGPASSWORD
|
52
|
+
return;
|
53
|
+
}
|
54
|
+
|
55
|
+
prepared = true;
|
56
|
+
|
57
|
+
// Insert Data
|
58
|
+
try(Connection connection = DriverManager.getConnection(URL, USER, PASSWORD)) {
|
59
|
+
try (Statement statement = connection.createStatement()) {
|
60
|
+
String sql = "";
|
61
|
+
sql += "DROP TABLE IF EXISTS input_hstore;";
|
62
|
+
sql += "CREATE EXTENSION IF NOT EXISTS hstore WITH SCHEMA public;";
|
63
|
+
sql += "CREATE TABLE input_hstore (c1 hstore);";
|
64
|
+
sql += "INSERT INTO input_hstore (c1) VALUES('\"a\" => \"b\"');";
|
65
|
+
statement.execute(sql);
|
66
|
+
}
|
67
|
+
}
|
68
|
+
}
|
69
|
+
|
70
|
+
@AfterClass
|
71
|
+
public static void dispose()
|
72
|
+
{
|
73
|
+
tester.destroy();
|
74
|
+
}
|
75
|
+
|
76
|
+
@Test
|
77
|
+
public void testHstoreAsString() throws Exception
|
78
|
+
{
|
79
|
+
if (prepared) {
|
80
|
+
tester.run(convertPath("/yml/input_hstore.yml"));
|
81
|
+
assertEquals(Arrays.asList("c1", "\"\"\"a\"\"=>\"\"b\"\"\""),
|
82
|
+
read("postgresql-input000.00.csv"));
|
83
|
+
}
|
84
|
+
}
|
85
|
+
|
86
|
+
@Test
|
87
|
+
public void testHstoreAsJson() throws Exception
|
88
|
+
{
|
89
|
+
if (prepared) {
|
90
|
+
tester.run(convertPath("/yml/input_hstore2.yml"));
|
91
|
+
assertEquals(Arrays.asList("c1", "\"{\"\"a\"\":\"\"b\"\"}\""),
|
92
|
+
read("postgresql-input000.00.csv"));
|
93
|
+
}
|
94
|
+
}
|
95
|
+
|
96
|
+
private List<String> read(String path) throws IOException
|
97
|
+
{
|
98
|
+
FileSystem fs = FileSystems.getDefault();
|
99
|
+
return Files.readAllLines(fs.getPath(path), Charset.defaultCharset());
|
100
|
+
}
|
101
|
+
|
102
|
+
private String convertPath(String name) throws URISyntaxException
|
103
|
+
{
|
104
|
+
if (getClass().getResource(name) == null) {
|
105
|
+
return name;
|
106
|
+
}
|
107
|
+
return new File(getClass().getResource(name).toURI()).getAbsolutePath();
|
108
|
+
}
|
109
|
+
|
110
|
+
private static void psql(String sql) throws IOException, InterruptedException {
|
111
|
+
ProcessBuilder pb = new ProcessBuilder("psql", "-w", "-c", sql);
|
112
|
+
System.out.println("PSQL: " + pb.command().toString());
|
113
|
+
final Process process = pb.start();
|
114
|
+
final int code = process.waitFor();
|
115
|
+
if (code != 0) {
|
116
|
+
try (BufferedReader reader = new BufferedReader(new InputStreamReader(process.getErrorStream()))) {
|
117
|
+
String line;
|
118
|
+
while ((line = reader.readLine()) != null) {
|
119
|
+
System.err.println(line);
|
120
|
+
}
|
121
|
+
}
|
122
|
+
throw new IOException(String.format(
|
123
|
+
"Command finished with non-zero exit code. Exit code is %d.", code));
|
124
|
+
}
|
125
|
+
}
|
126
|
+
}
|
@@ -0,0 +1,16 @@
|
|
1
|
+
in:
|
2
|
+
type: postgresql
|
3
|
+
host: localhost
|
4
|
+
database: test_db
|
5
|
+
user: test_user
|
6
|
+
password: test_pw
|
7
|
+
table: input_hstore
|
8
|
+
select: "*"
|
9
|
+
column_options:
|
10
|
+
c1: {type: json}
|
11
|
+
out:
|
12
|
+
type: file
|
13
|
+
path_prefix: postgresql-input
|
14
|
+
file_ext: csv
|
15
|
+
formatter:
|
16
|
+
type: csv
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-input-postgresql
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.7.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Sadayuki Furuhashi
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-
|
11
|
+
date: 2016-03-29 00:00:00.000000000 Z
|
12
12
|
dependencies: []
|
13
13
|
description: Selects records from a table.
|
14
14
|
email:
|
@@ -22,8 +22,13 @@ files:
|
|
22
22
|
- lib/embulk/input/postgresql.rb
|
23
23
|
- src/main/java/org/embulk/input/PostgreSQLInputPlugin.java
|
24
24
|
- src/main/java/org/embulk/input/postgresql/PostgreSQLInputConnection.java
|
25
|
-
-
|
26
|
-
-
|
25
|
+
- src/main/java/org/embulk/input/postgresql/getter/HstoreColumnGetter.java
|
26
|
+
- src/main/java/org/embulk/input/postgresql/getter/PostgreSQLColumnGetterFactory.java
|
27
|
+
- src/test/java/org/embulk/input/postgresql/PostgreSQLInputPluginTest.java
|
28
|
+
- src/test/resources/yml/input_hstore.yml
|
29
|
+
- src/test/resources/yml/input_hstore2.yml
|
30
|
+
- classpath/embulk-input-jdbc-0.7.0.jar
|
31
|
+
- classpath/embulk-input-postgresql-0.7.0.jar
|
27
32
|
- classpath/postgresql-9.4-1205-jdbc41.jar
|
28
33
|
homepage: https://github.com/embulk/embulk-input-jdbc
|
29
34
|
licenses:
|
Binary file
|