embulk-input-redshift 0.9.2 → 0.9.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (22) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +33 -0
  3. data/classpath/embulk-input-jdbc-0.9.3.jar +0 -0
  4. data/classpath/{embulk-input-postgresql-0.9.2.jar → embulk-input-postgresql-0.9.3.jar} +0 -0
  5. data/classpath/{embulk-input-redshift-0.9.2.jar → embulk-input-redshift-0.9.3.jar} +0 -0
  6. data/src/test/java/org/embulk/input/redshift/IncrementalTest.java +64 -0
  7. data/src/test/resources/org/embulk/input/redshift/test/expect/incremental/query/config_1.yml +14 -0
  8. data/src/test/resources/org/embulk/input/redshift/test/expect/incremental/query/config_2.yml +14 -0
  9. data/src/test/resources/org/embulk/input/redshift/test/expect/incremental/query/expected_1.csv +4 -0
  10. data/src/test/resources/org/embulk/input/redshift/test/expect/incremental/query/expected_1.diff +3 -0
  11. data/src/test/resources/org/embulk/input/redshift/test/expect/incremental/query/expected_2.csv +2 -0
  12. data/src/test/resources/org/embulk/input/redshift/test/expect/incremental/query/expected_2.diff +3 -0
  13. data/src/test/resources/org/embulk/input/redshift/test/expect/incremental/query/insert_more.sql +6 -0
  14. data/src/test/resources/org/embulk/input/redshift/test/expect/incremental/query/multi_columns_config_1.yml +16 -0
  15. data/src/test/resources/org/embulk/input/redshift/test/expect/incremental/query/multi_columns_config_2.yml +16 -0
  16. data/src/test/resources/org/embulk/input/redshift/test/expect/incremental/query/multi_columns_expected_1.csv +4 -0
  17. data/src/test/resources/org/embulk/input/redshift/test/expect/incremental/query/multi_columns_expected_1.diff +3 -0
  18. data/src/test/resources/org/embulk/input/redshift/test/expect/incremental/query/multi_columns_expected_2.csv +2 -0
  19. data/src/test/resources/org/embulk/input/redshift/test/expect/incremental/query/multi_columns_expected_2.diff +3 -0
  20. data/src/test/resources/org/embulk/input/redshift/test/expect/incremental/query/setup.sql +13 -0
  21. metadata +19 -5
  22. data/classpath/embulk-input-jdbc-0.9.2.jar +0 -0
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: f1b87a75bee192f78b6a9c9486811c59ef5f58b3
4
- data.tar.gz: c1cf3328329376cc1b5aeb561fb47912f8956c09
3
+ metadata.gz: 6e8c344685265804a7323eab781eaf7bf87653bf
4
+ data.tar.gz: 95dd684c86cbc4fd44a10839a1949e7c44245053
5
5
  SHA512:
6
- metadata.gz: 176fa8df4324c4d527abc80a0c6045f1dd53111ae391015aeb741140e5a69002c10b932d3ae5e1f7a79aa9ae59db830142385371dfca13fb4511eb5b82c7518a
7
- data.tar.gz: 912cadb712b4c113e9d6e637098fb442bef885b917169aba0cacf269d342c76c91f6d0fd062e90f2ac7b7e94f70760069fc165fc26e4e39b14ab34a963fdfe89
6
+ metadata.gz: c18d9405bbfb2609223c193679704bfcf6cfbc594d948603b9609b321a9b06e7710daf0e677287c9f972a1216790de77d6713b5d50158177531e558150762d4a
7
+ data.tar.gz: 71011ae9e8f7077807a73b3c3d72785c624a4336fae8a5e33e9a2695ea2ba87cd9f71f1ea4a74e549ec32444956ac6e6f49a99e079f9884044d1c2b1750d4397
data/README.md CHANGED
@@ -23,6 +23,7 @@ Redshift input plugin for Embulk loads records from Redshift.
23
23
  - **options**: extra JDBC properties (hash, default: {})
24
24
  - If you write SQL directly,
25
25
  - **query**: SQL to run (string)
26
+ - **use_raw_query_with_incremental**: If true, you can write optimized query using prepared statement by yourself. See [Use incremental loading with raw query](#use-incremental-loading-with-raw-query) for more detail (boolean, default: false)
26
27
  - If **query** is not set,
27
28
  - **table**: destination table name (string, required)
28
29
  - **select**: expression of select (e.g. `id, created_at`) (string, default: "*")
@@ -79,6 +80,38 @@ CREATE INDEX embulk_incremental_loading_index ON table (updated_at, id);
79
80
 
80
81
  Recommended usage is to leave `incremental_columns` unset and let this plugin automatically finds an auto-increment (IDENTITY) primary key. Currently, only strings and integers are supported as incremental_columns.
81
82
 
83
+ ### Use incremental loading with raw query
84
+
85
+ **IMPORTANT**: This is an advanced feature and assume you have an enough knowledge about incremental loading using Embulk and this plugin
86
+
87
+ Normally, you can't write your own query for incremental loading.
88
+ `use_raw_query_with_incremental` option allow you to write raw query for incremental loading. It might be well optimized and faster than SQL statement which is automatically generated by plugin.
89
+
90
+ Prepared statement starts with `:` is available instead of fixed value.
91
+ `last_record` value is necessary when you use this option.
92
+ Please use prepared statement that is well distinguishable in SQL statement. Using too simple prepared statement like `:a` might cause SQL parse failure.
93
+
94
+ In the following example, prepared statement `:foo_id` will be replaced with value "1" which is specified in `last_record`.
95
+
96
+ ```yaml
97
+ in:
98
+ type: redshift
99
+ query:
100
+ SELECT
101
+ foo.id as foo_id, bar.name
102
+ FROM
103
+ foo LEFT JOIN bar ON foo.id = bar.id
104
+ WHERE
105
+ foo.hoge IS NOT NULL
106
+ AND foo.id > :foo_id
107
+ ORDER BY
108
+ foo.id ASC
109
+ use_raw_query_with_incremental: true
110
+ incremental_columns:
111
+ - foo_id
112
+ incremental: true
113
+ last_record: [1]
114
+ ```
82
115
 
83
116
  ## Example
84
117
 
@@ -140,4 +140,68 @@ public class IncrementalTest
140
140
  result2.getConfigDiff(),
141
141
  is((ConfigDiff) loadYamlResource(embulk, "timestamptz/expected_2.diff")));
142
142
  }
143
+
144
+ @Test
145
+ public void simpleQueryWithPlaceholder() throws Exception
146
+ {
147
+ // setup first rows
148
+ execute(readResource("query/setup.sql"));
149
+
150
+ Path out1 = embulk.createTempFile("csv");
151
+ RunResult result1 = embulk.runInput(
152
+ baseConfig.merge(loadYamlResource(embulk, "query/config_1.yml")),
153
+ out1);
154
+ assertThat(
155
+ readSortedFile(out1),
156
+ is(readResource("query/expected_1.csv")));
157
+ assertThat(
158
+ result1.getConfigDiff(),
159
+ is((ConfigDiff) loadYamlResource(embulk, "query/expected_1.diff")));
160
+
161
+ // insert more rows
162
+ execute(readResource("query/insert_more.sql"));
163
+
164
+ Path out2 = embulk.createTempFile("csv");
165
+ RunResult result2 = embulk.runInput(
166
+ baseConfig.merge(loadYamlResource(embulk, "query/config_2.yml")),
167
+ out2);
168
+ assertThat(
169
+ readSortedFile(out2),
170
+ is(readResource("query/expected_2.csv")));
171
+ assertThat(
172
+ result2.getConfigDiff(),
173
+ is((ConfigDiff) loadYamlResource(embulk, "query/expected_2.diff")));
174
+ }
175
+
176
+ @Test
177
+ public void simpleQueryWithPlaceholderAndMultiColumns() throws Exception
178
+ {
179
+ // setup first rows
180
+ execute(readResource("query/setup.sql"));
181
+
182
+ Path out1 = embulk.createTempFile("csv");
183
+ RunResult result1 = embulk.runInput(
184
+ baseConfig.merge(loadYamlResource(embulk, "query/multi_columns_config_1.yml")),
185
+ out1);
186
+ assertThat(
187
+ readSortedFile(out1),
188
+ is(readResource("query/multi_columns_expected_1.csv")));
189
+ assertThat(
190
+ result1.getConfigDiff(),
191
+ is((ConfigDiff) loadYamlResource(embulk, "query/multi_columns_expected_1.diff")));
192
+
193
+ // insert more rows
194
+ execute(readResource("query/insert_more.sql"));
195
+
196
+ Path out2 = embulk.createTempFile("csv");
197
+ RunResult result2 = embulk.runInput(
198
+ baseConfig.merge(loadYamlResource(embulk, "query/multi_columns_config_2.yml")),
199
+ out2);
200
+ assertThat(
201
+ readSortedFile(out2),
202
+ is(readResource("query/multi_columns_expected_2.csv")));
203
+ assertThat(
204
+ result2.getConfigDiff(),
205
+ is((ConfigDiff) loadYamlResource(embulk, "query/multi_columns_expected_2.diff")));
206
+ }
143
207
  }
@@ -0,0 +1,14 @@
1
+ last_record: [0]
2
+ incremental: true
3
+ incremental_columns: [num]
4
+ use_raw_query_with_incremental: true
5
+ query: |
6
+ SELECT
7
+ num,note
8
+ FROM
9
+ query_load
10
+ WHERE
11
+ num IS NOT NULL
12
+ AND num > :num
13
+ ORDER BY
14
+ num ASC
@@ -0,0 +1,14 @@
1
+ last_record: [4]
2
+ incremental: true
3
+ incremental_columns: [num]
4
+ use_raw_query_with_incremental: true
5
+ query: |
6
+ SELECT
7
+ num,note
8
+ FROM
9
+ query_load
10
+ WHERE
11
+ num IS NOT NULL
12
+ AND num > :num
13
+ ORDER BY
14
+ num ASC
@@ -0,0 +1,6 @@
1
+
2
+ insert into query_load (num, num2, note) values
3
+ (0, 100, 'more_skip'),
4
+ (4, 104, 'more_skip'),
5
+ (9, 109, 'more_load'),
6
+ (5, 105, 'more_load');
@@ -0,0 +1,16 @@
1
+ last_record: [0,0]
2
+ incremental: true
3
+ incremental_columns: [num,num2]
4
+ use_raw_query_with_incremental: true
5
+ query: |
6
+ SELECT
7
+ *
8
+ FROM
9
+ query_load
10
+ WHERE
11
+ num IS NOT NULL
12
+ AND num > :num
13
+ OR (num = :num AND num2 > :num2)
14
+ ORDER BY
15
+ num ASC,
16
+ num2 ASC
@@ -0,0 +1,16 @@
1
+ last_record: [4,104]
2
+ incremental: true
3
+ incremental_columns: [num,num2]
4
+ use_raw_query_with_incremental: true
5
+ query: |
6
+ SELECT
7
+ *
8
+ FROM
9
+ query_load
10
+ WHERE
11
+ num IS NOT NULL
12
+ AND num > :num
13
+ OR (num = :num AND num2 > :num2)
14
+ ORDER BY
15
+ num ASC,
16
+ num2 ASC
@@ -0,0 +1,4 @@
1
+ 1,101,first
2
+ 2,102,first
3
+ 3,103,first
4
+ 4,104,first
@@ -0,0 +1,13 @@
1
+ drop table if exists query_load;
2
+
3
+ create table query_load (
4
+ num int not null,
5
+ num2 int not null,
6
+ note text
7
+ );
8
+
9
+ insert into query_load (num, num2, note) values
10
+ (3, 103, 'first'),
11
+ (4, 104, 'first'),
12
+ (2, 102, 'first'),
13
+ (1, 101, 'first');
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: embulk-input-redshift
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.9.2
4
+ version: 0.9.3
5
5
  platform: ruby
6
6
  authors:
7
7
  - Sadayuki Furuhashi
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2018-07-03 00:00:00.000000000 Z
11
+ date: 2018-08-10 00:00:00.000000000 Z
12
12
  dependencies: []
13
13
  description: Selects records from a table.
14
14
  email:
@@ -19,9 +19,9 @@ extra_rdoc_files: []
19
19
  files:
20
20
  - README.md
21
21
  - build.gradle
22
- - classpath/embulk-input-jdbc-0.9.2.jar
23
- - classpath/embulk-input-postgresql-0.9.2.jar
24
- - classpath/embulk-input-redshift-0.9.2.jar
22
+ - classpath/embulk-input-jdbc-0.9.3.jar
23
+ - classpath/embulk-input-postgresql-0.9.3.jar
24
+ - classpath/embulk-input-redshift-0.9.3.jar
25
25
  - classpath/postgresql-9.4-1205-jdbc41.jar
26
26
  - lib/embulk/input/redshift.rb
27
27
  - src/main/java/org/embulk/input/RedshiftInputPlugin.java
@@ -36,6 +36,20 @@ files:
36
36
  - src/test/resources/org/embulk/input/redshift/test/expect/incremental/int/expected_2.diff
37
37
  - src/test/resources/org/embulk/input/redshift/test/expect/incremental/int/insert_more.sql
38
38
  - src/test/resources/org/embulk/input/redshift/test/expect/incremental/int/setup.sql
39
+ - src/test/resources/org/embulk/input/redshift/test/expect/incremental/query/config_1.yml
40
+ - src/test/resources/org/embulk/input/redshift/test/expect/incremental/query/config_2.yml
41
+ - src/test/resources/org/embulk/input/redshift/test/expect/incremental/query/expected_1.csv
42
+ - src/test/resources/org/embulk/input/redshift/test/expect/incremental/query/expected_1.diff
43
+ - src/test/resources/org/embulk/input/redshift/test/expect/incremental/query/expected_2.csv
44
+ - src/test/resources/org/embulk/input/redshift/test/expect/incremental/query/expected_2.diff
45
+ - src/test/resources/org/embulk/input/redshift/test/expect/incremental/query/insert_more.sql
46
+ - src/test/resources/org/embulk/input/redshift/test/expect/incremental/query/multi_columns_config_1.yml
47
+ - src/test/resources/org/embulk/input/redshift/test/expect/incremental/query/multi_columns_config_2.yml
48
+ - src/test/resources/org/embulk/input/redshift/test/expect/incremental/query/multi_columns_expected_1.csv
49
+ - src/test/resources/org/embulk/input/redshift/test/expect/incremental/query/multi_columns_expected_1.diff
50
+ - src/test/resources/org/embulk/input/redshift/test/expect/incremental/query/multi_columns_expected_2.csv
51
+ - src/test/resources/org/embulk/input/redshift/test/expect/incremental/query/multi_columns_expected_2.diff
52
+ - src/test/resources/org/embulk/input/redshift/test/expect/incremental/query/setup.sql
39
53
  - src/test/resources/org/embulk/input/redshift/test/expect/incremental/timestamp/config_1.yml
40
54
  - src/test/resources/org/embulk/input/redshift/test/expect/incremental/timestamp/config_2.yml
41
55
  - src/test/resources/org/embulk/input/redshift/test/expect/incremental/timestamp/expected_1.csv
Binary file