embulk-input-postgresql 0.9.2 → 0.9.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +33 -0
- data/classpath/embulk-input-jdbc-0.9.3.jar +0 -0
- data/classpath/{embulk-input-postgresql-0.9.2.jar → embulk-input-postgresql-0.9.3.jar} +0 -0
- data/src/test/java/org/embulk/input/postgresql/IncrementalTest.java +64 -0
- data/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/query/config_1.yml +14 -0
- data/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/query/config_2.yml +14 -0
- data/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/query/expected_1.csv +4 -0
- data/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/query/expected_1.diff +3 -0
- data/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/query/expected_2.csv +2 -0
- data/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/query/expected_2.diff +3 -0
- data/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/query/insert_more.sql +6 -0
- data/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/query/multi_columns_config_1.yml +16 -0
- data/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/query/multi_columns_config_2.yml +16 -0
- data/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/query/multi_columns_expected_1.csv +4 -0
- data/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/query/multi_columns_expected_1.diff +3 -0
- data/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/query/multi_columns_expected_2.csv +2 -0
- data/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/query/multi_columns_expected_2.diff +3 -0
- data/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/query/setup.sql +13 -0
- metadata +18 -4
- data/classpath/embulk-input-jdbc-0.9.2.jar +0 -0
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: edb5cbc6680e6fdd62506d5274f43ae0fa95e89d
|
4
|
+
data.tar.gz: a961032ae6f7add861fcd88760fc8bbe9f09fe05
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: ece1c87b6a0b05919ba2054cdcdb8a10f59c3b4e4d9b6c6a1fb689f3964bfae1f013d5d0e17d8a453f1c49f7fe6a80218bfdb0bc2c19fd976a388cc517fb62cf
|
7
|
+
data.tar.gz: b0a055403e5a383316f4a39caee43fa24fd70207aa24a950779dfe3f08182e2d95e86a635057cc561b890626fd5d099603120fad0ae5bd308b80f3754c3c2e0b
|
data/README.md
CHANGED
@@ -24,6 +24,7 @@ PostgreSQL input plugin for Embulk loads records from PostgreSQL.
|
|
24
24
|
- **options**: extra JDBC properties (hash, default: {})
|
25
25
|
- If you write SQL directly,
|
26
26
|
- **query**: SQL to run (string)
|
27
|
+
- **use_raw_query_with_incremental**: If true, you can write optimized query using prepared statement by yourself. See [Use incremental loading with raw query](#use-incremental-loading-with-raw-query) for more detail (boolean, default: false)
|
27
28
|
- If **query** is not set,
|
28
29
|
- **table**: destination table name (string, required)
|
29
30
|
- **select**: expression of select (e.g. `id, created_at`) (string, default: "*")
|
@@ -114,6 +115,38 @@ CREATE INDEX embulk_incremental_loading_index ON table (updated_at, id);
|
|
114
115
|
|
115
116
|
Recommended usage is to leave `incremental_columns` unset and let this plugin automatically finds an auto-increment (serial / bigserial) primary key. Currently, only strings and integers are supported as incremental_columns.
|
116
117
|
|
118
|
+
### Use incremental loading with raw query
|
119
|
+
|
120
|
+
**IMPORTANT**: This is an advanced feature and assume you have an enough knowledge about incremental loading using Embulk and this plugin
|
121
|
+
|
122
|
+
Normally, you can't write your own query for incremental loading.
|
123
|
+
`use_raw_query_with_incremental` option allow you to write raw query for incremental loading. It might be well optimized and faster than SQL statement which is automatically generated by plugin.
|
124
|
+
|
125
|
+
Prepared statement starts with `:` is available instead of fixed value.
|
126
|
+
`last_record` value is necessary when you use this option.
|
127
|
+
Please use prepared statement that is well distinguishable in SQL statement. Using too simple prepared statement like `:a` might cause SQL parse failure.
|
128
|
+
|
129
|
+
In the following example, prepared statement `:foo_id` will be replaced with value "1" which is specified in `last_record`.
|
130
|
+
|
131
|
+
```yaml
|
132
|
+
in:
|
133
|
+
type: postgresql
|
134
|
+
query:
|
135
|
+
SELECT
|
136
|
+
foo.id as foo_id, bar.name
|
137
|
+
FROM
|
138
|
+
foo LEFT JOIN bar ON foo.id = bar.id
|
139
|
+
WHERE
|
140
|
+
foo.hoge IS NOT NULL
|
141
|
+
AND foo.id > :foo_id
|
142
|
+
ORDER BY
|
143
|
+
foo.id ASC
|
144
|
+
use_raw_query_with_incremental: true
|
145
|
+
incremental_columns:
|
146
|
+
- foo_id
|
147
|
+
incremental: true
|
148
|
+
last_record: [1]
|
149
|
+
```
|
117
150
|
|
118
151
|
## Example
|
119
152
|
|
Binary file
|
Binary file
|
@@ -140,4 +140,68 @@ public class IncrementalTest
|
|
140
140
|
result2.getConfigDiff(),
|
141
141
|
is((ConfigDiff) loadYamlResource(embulk, "timestamptz/expected_2.diff")));
|
142
142
|
}
|
143
|
+
|
144
|
+
@Test
|
145
|
+
public void simpleQueryWithPlaceholder() throws Exception
|
146
|
+
{
|
147
|
+
// setup first rows
|
148
|
+
execute(readResource("query/setup.sql"));
|
149
|
+
|
150
|
+
Path out1 = embulk.createTempFile("csv");
|
151
|
+
RunResult result1 = embulk.runInput(
|
152
|
+
baseConfig.merge(loadYamlResource(embulk, "query/config_1.yml")),
|
153
|
+
out1);
|
154
|
+
assertThat(
|
155
|
+
readSortedFile(out1),
|
156
|
+
is(readResource("query/expected_1.csv")));
|
157
|
+
assertThat(
|
158
|
+
result1.getConfigDiff(),
|
159
|
+
is((ConfigDiff) loadYamlResource(embulk, "query/expected_1.diff")));
|
160
|
+
|
161
|
+
// insert more rows
|
162
|
+
execute(readResource("query/insert_more.sql"));
|
163
|
+
|
164
|
+
Path out2 = embulk.createTempFile("csv");
|
165
|
+
RunResult result2 = embulk.runInput(
|
166
|
+
baseConfig.merge(loadYamlResource(embulk, "query/config_2.yml")),
|
167
|
+
out2);
|
168
|
+
assertThat(
|
169
|
+
readSortedFile(out2),
|
170
|
+
is(readResource("query/expected_2.csv")));
|
171
|
+
assertThat(
|
172
|
+
result2.getConfigDiff(),
|
173
|
+
is((ConfigDiff) loadYamlResource(embulk, "query/expected_2.diff")));
|
174
|
+
}
|
175
|
+
|
176
|
+
@Test
|
177
|
+
public void simpleQueryWithPlaceholderAndMultiColumns() throws Exception
|
178
|
+
{
|
179
|
+
// setup first rows
|
180
|
+
execute(readResource("query/setup.sql"));
|
181
|
+
|
182
|
+
Path out1 = embulk.createTempFile("csv");
|
183
|
+
RunResult result1 = embulk.runInput(
|
184
|
+
baseConfig.merge(loadYamlResource(embulk, "query/multi_columns_config_1.yml")),
|
185
|
+
out1);
|
186
|
+
assertThat(
|
187
|
+
readSortedFile(out1),
|
188
|
+
is(readResource("query/multi_columns_expected_1.csv")));
|
189
|
+
assertThat(
|
190
|
+
result1.getConfigDiff(),
|
191
|
+
is((ConfigDiff) loadYamlResource(embulk, "query/multi_columns_expected_1.diff")));
|
192
|
+
|
193
|
+
// insert more rows
|
194
|
+
execute(readResource("query/insert_more.sql"));
|
195
|
+
|
196
|
+
Path out2 = embulk.createTempFile("csv");
|
197
|
+
RunResult result2 = embulk.runInput(
|
198
|
+
baseConfig.merge(loadYamlResource(embulk, "query/multi_columns_config_2.yml")),
|
199
|
+
out2);
|
200
|
+
assertThat(
|
201
|
+
readSortedFile(out2),
|
202
|
+
is(readResource("query/multi_columns_expected_2.csv")));
|
203
|
+
assertThat(
|
204
|
+
result2.getConfigDiff(),
|
205
|
+
is((ConfigDiff) loadYamlResource(embulk, "query/multi_columns_expected_2.diff")));
|
206
|
+
}
|
143
207
|
}
|
@@ -0,0 +1,16 @@
|
|
1
|
+
last_record: [0,0]
|
2
|
+
incremental: true
|
3
|
+
incremental_columns: [num,num2]
|
4
|
+
use_raw_query_with_incremental: true
|
5
|
+
query: |
|
6
|
+
SELECT
|
7
|
+
*
|
8
|
+
FROM
|
9
|
+
query_load
|
10
|
+
WHERE
|
11
|
+
num IS NOT NULL
|
12
|
+
AND num > :num
|
13
|
+
OR (num = :num AND num2 > :num2)
|
14
|
+
ORDER BY
|
15
|
+
num ASC,
|
16
|
+
num2 ASC
|
@@ -0,0 +1,16 @@
|
|
1
|
+
last_record: [4,104]
|
2
|
+
incremental: true
|
3
|
+
incremental_columns: [num,num2]
|
4
|
+
use_raw_query_with_incremental: true
|
5
|
+
query: |
|
6
|
+
SELECT
|
7
|
+
*
|
8
|
+
FROM
|
9
|
+
query_load
|
10
|
+
WHERE
|
11
|
+
num IS NOT NULL
|
12
|
+
AND num > :num
|
13
|
+
OR (num = :num AND num2 > :num2)
|
14
|
+
ORDER BY
|
15
|
+
num ASC,
|
16
|
+
num2 ASC
|
@@ -0,0 +1,13 @@
|
|
1
|
+
drop table if exists query_load;
|
2
|
+
|
3
|
+
create table query_load (
|
4
|
+
num int not null,
|
5
|
+
num2 int not null,
|
6
|
+
note text
|
7
|
+
);
|
8
|
+
|
9
|
+
insert into query_load (num, num2, note) values
|
10
|
+
(3, 103, 'first'),
|
11
|
+
(4, 104, 'first'),
|
12
|
+
(2, 102, 'first'),
|
13
|
+
(1, 101, 'first');
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-input-postgresql
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.9.
|
4
|
+
version: 0.9.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Sadayuki Furuhashi
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2018-
|
11
|
+
date: 2018-08-10 00:00:00.000000000 Z
|
12
12
|
dependencies: []
|
13
13
|
description: Selects records from a table.
|
14
14
|
email:
|
@@ -19,8 +19,8 @@ extra_rdoc_files: []
|
|
19
19
|
files:
|
20
20
|
- README.md
|
21
21
|
- build.gradle
|
22
|
-
- classpath/embulk-input-jdbc-0.9.
|
23
|
-
- classpath/embulk-input-postgresql-0.9.
|
22
|
+
- classpath/embulk-input-jdbc-0.9.3.jar
|
23
|
+
- classpath/embulk-input-postgresql-0.9.3.jar
|
24
24
|
- default_jdbc_driver/postgresql-9.4-1205-jdbc41.jar
|
25
25
|
- lib/embulk/input/postgresql.rb
|
26
26
|
- src/main/java/org/embulk/input/PostgreSQLInputPlugin.java
|
@@ -51,6 +51,20 @@ files:
|
|
51
51
|
- src/test/resources/org/embulk/input/postgresql/test/expect/incremental/int/expected_2.diff
|
52
52
|
- src/test/resources/org/embulk/input/postgresql/test/expect/incremental/int/insert_more.sql
|
53
53
|
- src/test/resources/org/embulk/input/postgresql/test/expect/incremental/int/setup.sql
|
54
|
+
- src/test/resources/org/embulk/input/postgresql/test/expect/incremental/query/config_1.yml
|
55
|
+
- src/test/resources/org/embulk/input/postgresql/test/expect/incremental/query/config_2.yml
|
56
|
+
- src/test/resources/org/embulk/input/postgresql/test/expect/incremental/query/expected_1.csv
|
57
|
+
- src/test/resources/org/embulk/input/postgresql/test/expect/incremental/query/expected_1.diff
|
58
|
+
- src/test/resources/org/embulk/input/postgresql/test/expect/incremental/query/expected_2.csv
|
59
|
+
- src/test/resources/org/embulk/input/postgresql/test/expect/incremental/query/expected_2.diff
|
60
|
+
- src/test/resources/org/embulk/input/postgresql/test/expect/incremental/query/insert_more.sql
|
61
|
+
- src/test/resources/org/embulk/input/postgresql/test/expect/incremental/query/multi_columns_config_1.yml
|
62
|
+
- src/test/resources/org/embulk/input/postgresql/test/expect/incremental/query/multi_columns_config_2.yml
|
63
|
+
- src/test/resources/org/embulk/input/postgresql/test/expect/incremental/query/multi_columns_expected_1.csv
|
64
|
+
- src/test/resources/org/embulk/input/postgresql/test/expect/incremental/query/multi_columns_expected_1.diff
|
65
|
+
- src/test/resources/org/embulk/input/postgresql/test/expect/incremental/query/multi_columns_expected_2.csv
|
66
|
+
- src/test/resources/org/embulk/input/postgresql/test/expect/incremental/query/multi_columns_expected_2.diff
|
67
|
+
- src/test/resources/org/embulk/input/postgresql/test/expect/incremental/query/setup.sql
|
54
68
|
- src/test/resources/org/embulk/input/postgresql/test/expect/incremental/timestamp/config_1.yml
|
55
69
|
- src/test/resources/org/embulk/input/postgresql/test/expect/incremental/timestamp/config_2.yml
|
56
70
|
- src/test/resources/org/embulk/input/postgresql/test/expect/incremental/timestamp/expected_1.csv
|
Binary file
|