embulk-input-redshift 0.9.2 → 0.9.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +33 -0
- data/classpath/embulk-input-jdbc-0.9.3.jar +0 -0
- data/classpath/{embulk-input-postgresql-0.9.2.jar → embulk-input-postgresql-0.9.3.jar} +0 -0
- data/classpath/{embulk-input-redshift-0.9.2.jar → embulk-input-redshift-0.9.3.jar} +0 -0
- data/src/test/java/org/embulk/input/redshift/IncrementalTest.java +64 -0
- data/src/test/resources/org/embulk/input/redshift/test/expect/incremental/query/config_1.yml +14 -0
- data/src/test/resources/org/embulk/input/redshift/test/expect/incremental/query/config_2.yml +14 -0
- data/src/test/resources/org/embulk/input/redshift/test/expect/incremental/query/expected_1.csv +4 -0
- data/src/test/resources/org/embulk/input/redshift/test/expect/incremental/query/expected_1.diff +3 -0
- data/src/test/resources/org/embulk/input/redshift/test/expect/incremental/query/expected_2.csv +2 -0
- data/src/test/resources/org/embulk/input/redshift/test/expect/incremental/query/expected_2.diff +3 -0
- data/src/test/resources/org/embulk/input/redshift/test/expect/incremental/query/insert_more.sql +6 -0
- data/src/test/resources/org/embulk/input/redshift/test/expect/incremental/query/multi_columns_config_1.yml +16 -0
- data/src/test/resources/org/embulk/input/redshift/test/expect/incremental/query/multi_columns_config_2.yml +16 -0
- data/src/test/resources/org/embulk/input/redshift/test/expect/incremental/query/multi_columns_expected_1.csv +4 -0
- data/src/test/resources/org/embulk/input/redshift/test/expect/incremental/query/multi_columns_expected_1.diff +3 -0
- data/src/test/resources/org/embulk/input/redshift/test/expect/incremental/query/multi_columns_expected_2.csv +2 -0
- data/src/test/resources/org/embulk/input/redshift/test/expect/incremental/query/multi_columns_expected_2.diff +3 -0
- data/src/test/resources/org/embulk/input/redshift/test/expect/incremental/query/setup.sql +13 -0
- metadata +19 -5
- data/classpath/embulk-input-jdbc-0.9.2.jar +0 -0
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 6e8c344685265804a7323eab781eaf7bf87653bf
|
4
|
+
data.tar.gz: 95dd684c86cbc4fd44a10839a1949e7c44245053
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: c18d9405bbfb2609223c193679704bfcf6cfbc594d948603b9609b321a9b06e7710daf0e677287c9f972a1216790de77d6713b5d50158177531e558150762d4a
|
7
|
+
data.tar.gz: 71011ae9e8f7077807a73b3c3d72785c624a4336fae8a5e33e9a2695ea2ba87cd9f71f1ea4a74e549ec32444956ac6e6f49a99e079f9884044d1c2b1750d4397
|
data/README.md
CHANGED
@@ -23,6 +23,7 @@ Redshift input plugin for Embulk loads records from Redshift.
|
|
23
23
|
- **options**: extra JDBC properties (hash, default: {})
|
24
24
|
- If you write SQL directly,
|
25
25
|
- **query**: SQL to run (string)
|
26
|
+
- **use_raw_query_with_incremental**: If true, you can write optimized query using prepared statement by yourself. See [Use incremental loading with raw query](#use-incremental-loading-with-raw-query) for more detail (boolean, default: false)
|
26
27
|
- If **query** is not set,
|
27
28
|
- **table**: destination table name (string, required)
|
28
29
|
- **select**: expression of select (e.g. `id, created_at`) (string, default: "*")
|
@@ -79,6 +80,38 @@ CREATE INDEX embulk_incremental_loading_index ON table (updated_at, id);
|
|
79
80
|
|
80
81
|
Recommended usage is to leave `incremental_columns` unset and let this plugin automatically finds an auto-increment (IDENTITY) primary key. Currently, only strings and integers are supported as incremental_columns.
|
81
82
|
|
83
|
+
### Use incremental loading with raw query
|
84
|
+
|
85
|
+
**IMPORTANT**: This is an advanced feature and assume you have an enough knowledge about incremental loading using Embulk and this plugin
|
86
|
+
|
87
|
+
Normally, you can't write your own query for incremental loading.
|
88
|
+
`use_raw_query_with_incremental` option allow you to write raw query for incremental loading. It might be well optimized and faster than SQL statement which is automatically generated by plugin.
|
89
|
+
|
90
|
+
Prepared statement starts with `:` is available instead of fixed value.
|
91
|
+
`last_record` value is necessary when you use this option.
|
92
|
+
Please use prepared statement that is well distinguishable in SQL statement. Using too simple prepared statement like `:a` might cause SQL parse failure.
|
93
|
+
|
94
|
+
In the following example, prepared statement `:foo_id` will be replaced with value "1" which is specified in `last_record`.
|
95
|
+
|
96
|
+
```yaml
|
97
|
+
in:
|
98
|
+
type: redshift
|
99
|
+
query:
|
100
|
+
SELECT
|
101
|
+
foo.id as foo_id, bar.name
|
102
|
+
FROM
|
103
|
+
foo LEFT JOIN bar ON foo.id = bar.id
|
104
|
+
WHERE
|
105
|
+
foo.hoge IS NOT NULL
|
106
|
+
AND foo.id > :foo_id
|
107
|
+
ORDER BY
|
108
|
+
foo.id ASC
|
109
|
+
use_raw_query_with_incremental: true
|
110
|
+
incremental_columns:
|
111
|
+
- foo_id
|
112
|
+
incremental: true
|
113
|
+
last_record: [1]
|
114
|
+
```
|
82
115
|
|
83
116
|
## Example
|
84
117
|
|
Binary file
|
Binary file
|
Binary file
|
@@ -140,4 +140,68 @@ public class IncrementalTest
|
|
140
140
|
result2.getConfigDiff(),
|
141
141
|
is((ConfigDiff) loadYamlResource(embulk, "timestamptz/expected_2.diff")));
|
142
142
|
}
|
143
|
+
|
144
|
+
@Test
|
145
|
+
public void simpleQueryWithPlaceholder() throws Exception
|
146
|
+
{
|
147
|
+
// setup first rows
|
148
|
+
execute(readResource("query/setup.sql"));
|
149
|
+
|
150
|
+
Path out1 = embulk.createTempFile("csv");
|
151
|
+
RunResult result1 = embulk.runInput(
|
152
|
+
baseConfig.merge(loadYamlResource(embulk, "query/config_1.yml")),
|
153
|
+
out1);
|
154
|
+
assertThat(
|
155
|
+
readSortedFile(out1),
|
156
|
+
is(readResource("query/expected_1.csv")));
|
157
|
+
assertThat(
|
158
|
+
result1.getConfigDiff(),
|
159
|
+
is((ConfigDiff) loadYamlResource(embulk, "query/expected_1.diff")));
|
160
|
+
|
161
|
+
// insert more rows
|
162
|
+
execute(readResource("query/insert_more.sql"));
|
163
|
+
|
164
|
+
Path out2 = embulk.createTempFile("csv");
|
165
|
+
RunResult result2 = embulk.runInput(
|
166
|
+
baseConfig.merge(loadYamlResource(embulk, "query/config_2.yml")),
|
167
|
+
out2);
|
168
|
+
assertThat(
|
169
|
+
readSortedFile(out2),
|
170
|
+
is(readResource("query/expected_2.csv")));
|
171
|
+
assertThat(
|
172
|
+
result2.getConfigDiff(),
|
173
|
+
is((ConfigDiff) loadYamlResource(embulk, "query/expected_2.diff")));
|
174
|
+
}
|
175
|
+
|
176
|
+
@Test
|
177
|
+
public void simpleQueryWithPlaceholderAndMultiColumns() throws Exception
|
178
|
+
{
|
179
|
+
// setup first rows
|
180
|
+
execute(readResource("query/setup.sql"));
|
181
|
+
|
182
|
+
Path out1 = embulk.createTempFile("csv");
|
183
|
+
RunResult result1 = embulk.runInput(
|
184
|
+
baseConfig.merge(loadYamlResource(embulk, "query/multi_columns_config_1.yml")),
|
185
|
+
out1);
|
186
|
+
assertThat(
|
187
|
+
readSortedFile(out1),
|
188
|
+
is(readResource("query/multi_columns_expected_1.csv")));
|
189
|
+
assertThat(
|
190
|
+
result1.getConfigDiff(),
|
191
|
+
is((ConfigDiff) loadYamlResource(embulk, "query/multi_columns_expected_1.diff")));
|
192
|
+
|
193
|
+
// insert more rows
|
194
|
+
execute(readResource("query/insert_more.sql"));
|
195
|
+
|
196
|
+
Path out2 = embulk.createTempFile("csv");
|
197
|
+
RunResult result2 = embulk.runInput(
|
198
|
+
baseConfig.merge(loadYamlResource(embulk, "query/multi_columns_config_2.yml")),
|
199
|
+
out2);
|
200
|
+
assertThat(
|
201
|
+
readSortedFile(out2),
|
202
|
+
is(readResource("query/multi_columns_expected_2.csv")));
|
203
|
+
assertThat(
|
204
|
+
result2.getConfigDiff(),
|
205
|
+
is((ConfigDiff) loadYamlResource(embulk, "query/multi_columns_expected_2.diff")));
|
206
|
+
}
|
143
207
|
}
|
@@ -0,0 +1,16 @@
|
|
1
|
+
last_record: [0,0]
|
2
|
+
incremental: true
|
3
|
+
incremental_columns: [num,num2]
|
4
|
+
use_raw_query_with_incremental: true
|
5
|
+
query: |
|
6
|
+
SELECT
|
7
|
+
*
|
8
|
+
FROM
|
9
|
+
query_load
|
10
|
+
WHERE
|
11
|
+
num IS NOT NULL
|
12
|
+
AND num > :num
|
13
|
+
OR (num = :num AND num2 > :num2)
|
14
|
+
ORDER BY
|
15
|
+
num ASC,
|
16
|
+
num2 ASC
|
@@ -0,0 +1,16 @@
|
|
1
|
+
last_record: [4,104]
|
2
|
+
incremental: true
|
3
|
+
incremental_columns: [num,num2]
|
4
|
+
use_raw_query_with_incremental: true
|
5
|
+
query: |
|
6
|
+
SELECT
|
7
|
+
*
|
8
|
+
FROM
|
9
|
+
query_load
|
10
|
+
WHERE
|
11
|
+
num IS NOT NULL
|
12
|
+
AND num > :num
|
13
|
+
OR (num = :num AND num2 > :num2)
|
14
|
+
ORDER BY
|
15
|
+
num ASC,
|
16
|
+
num2 ASC
|
@@ -0,0 +1,13 @@
|
|
1
|
+
drop table if exists query_load;
|
2
|
+
|
3
|
+
create table query_load (
|
4
|
+
num int not null,
|
5
|
+
num2 int not null,
|
6
|
+
note text
|
7
|
+
);
|
8
|
+
|
9
|
+
insert into query_load (num, num2, note) values
|
10
|
+
(3, 103, 'first'),
|
11
|
+
(4, 104, 'first'),
|
12
|
+
(2, 102, 'first'),
|
13
|
+
(1, 101, 'first');
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-input-redshift
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.9.
|
4
|
+
version: 0.9.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Sadayuki Furuhashi
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2018-
|
11
|
+
date: 2018-08-10 00:00:00.000000000 Z
|
12
12
|
dependencies: []
|
13
13
|
description: Selects records from a table.
|
14
14
|
email:
|
@@ -19,9 +19,9 @@ extra_rdoc_files: []
|
|
19
19
|
files:
|
20
20
|
- README.md
|
21
21
|
- build.gradle
|
22
|
-
- classpath/embulk-input-jdbc-0.9.
|
23
|
-
- classpath/embulk-input-postgresql-0.9.
|
24
|
-
- classpath/embulk-input-redshift-0.9.
|
22
|
+
- classpath/embulk-input-jdbc-0.9.3.jar
|
23
|
+
- classpath/embulk-input-postgresql-0.9.3.jar
|
24
|
+
- classpath/embulk-input-redshift-0.9.3.jar
|
25
25
|
- classpath/postgresql-9.4-1205-jdbc41.jar
|
26
26
|
- lib/embulk/input/redshift.rb
|
27
27
|
- src/main/java/org/embulk/input/RedshiftInputPlugin.java
|
@@ -36,6 +36,20 @@ files:
|
|
36
36
|
- src/test/resources/org/embulk/input/redshift/test/expect/incremental/int/expected_2.diff
|
37
37
|
- src/test/resources/org/embulk/input/redshift/test/expect/incremental/int/insert_more.sql
|
38
38
|
- src/test/resources/org/embulk/input/redshift/test/expect/incremental/int/setup.sql
|
39
|
+
- src/test/resources/org/embulk/input/redshift/test/expect/incremental/query/config_1.yml
|
40
|
+
- src/test/resources/org/embulk/input/redshift/test/expect/incremental/query/config_2.yml
|
41
|
+
- src/test/resources/org/embulk/input/redshift/test/expect/incremental/query/expected_1.csv
|
42
|
+
- src/test/resources/org/embulk/input/redshift/test/expect/incremental/query/expected_1.diff
|
43
|
+
- src/test/resources/org/embulk/input/redshift/test/expect/incremental/query/expected_2.csv
|
44
|
+
- src/test/resources/org/embulk/input/redshift/test/expect/incremental/query/expected_2.diff
|
45
|
+
- src/test/resources/org/embulk/input/redshift/test/expect/incremental/query/insert_more.sql
|
46
|
+
- src/test/resources/org/embulk/input/redshift/test/expect/incremental/query/multi_columns_config_1.yml
|
47
|
+
- src/test/resources/org/embulk/input/redshift/test/expect/incremental/query/multi_columns_config_2.yml
|
48
|
+
- src/test/resources/org/embulk/input/redshift/test/expect/incremental/query/multi_columns_expected_1.csv
|
49
|
+
- src/test/resources/org/embulk/input/redshift/test/expect/incremental/query/multi_columns_expected_1.diff
|
50
|
+
- src/test/resources/org/embulk/input/redshift/test/expect/incremental/query/multi_columns_expected_2.csv
|
51
|
+
- src/test/resources/org/embulk/input/redshift/test/expect/incremental/query/multi_columns_expected_2.diff
|
52
|
+
- src/test/resources/org/embulk/input/redshift/test/expect/incremental/query/setup.sql
|
39
53
|
- src/test/resources/org/embulk/input/redshift/test/expect/incremental/timestamp/config_1.yml
|
40
54
|
- src/test/resources/org/embulk/input/redshift/test/expect/incremental/timestamp/config_2.yml
|
41
55
|
- src/test/resources/org/embulk/input/redshift/test/expect/incremental/timestamp/expected_1.csv
|
Binary file
|