embulk-input-oracle 0.9.2 → 0.9.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (21) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +33 -0
  3. data/classpath/embulk-input-jdbc-0.9.3.jar +0 -0
  4. data/classpath/{embulk-input-oracle-0.9.2.jar → embulk-input-oracle-0.9.3.jar} +0 -0
  5. data/src/test/java/org/embulk/input/oracle/IncrementalTest.java +63 -0
  6. data/src/test/resources/org/embulk/input/oracle/test/expect/incremental/query/config_1.yml +16 -0
  7. data/src/test/resources/org/embulk/input/oracle/test/expect/incremental/query/config_2.yml +16 -0
  8. data/src/test/resources/org/embulk/input/oracle/test/expect/incremental/query/expected_1.csv +4 -0
  9. data/src/test/resources/org/embulk/input/oracle/test/expect/incremental/query/expected_1.diff +3 -0
  10. data/src/test/resources/org/embulk/input/oracle/test/expect/incremental/query/expected_2.csv +2 -0
  11. data/src/test/resources/org/embulk/input/oracle/test/expect/incremental/query/expected_2.diff +3 -0
  12. data/src/test/resources/org/embulk/input/oracle/test/expect/incremental/query/insert_more.sql +6 -0
  13. data/src/test/resources/org/embulk/input/oracle/test/expect/incremental/query/multi_columns_config_1.yml +19 -0
  14. data/src/test/resources/org/embulk/input/oracle/test/expect/incremental/query/multi_columns_config_2.yml +19 -0
  15. data/src/test/resources/org/embulk/input/oracle/test/expect/incremental/query/multi_columns_expected_1.csv +4 -0
  16. data/src/test/resources/org/embulk/input/oracle/test/expect/incremental/query/multi_columns_expected_1.diff +3 -0
  17. data/src/test/resources/org/embulk/input/oracle/test/expect/incremental/query/multi_columns_expected_2.csv +2 -0
  18. data/src/test/resources/org/embulk/input/oracle/test/expect/incremental/query/multi_columns_expected_2.diff +3 -0
  19. data/src/test/resources/org/embulk/input/oracle/test/expect/incremental/query/setup.sql +14 -0
  20. metadata +18 -4
  21. data/classpath/embulk-input-jdbc-0.9.2.jar +0 -0
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 793a1399c4c68e231e24b0ecb76c99f0a54a41b7
4
- data.tar.gz: 351b3f2920f4b5dfdb6ca04bbbd35d28f8545452
3
+ metadata.gz: 0ae776a1726302045e4d3beec5c476083fb68f38
4
+ data.tar.gz: 44df8eb7f1d3433a73cff5dab8dde1403c395596
5
5
  SHA512:
6
- metadata.gz: 0a433b3335d54052bd23adbdf76486658fd8ff1df2efa0845bfe281d372b7cfc8b43ea49509ea9e7b557add43997eac0a6734f9725a7a53414c94002d10e59c6
7
- data.tar.gz: 4b71b09c09068d97b95ab94a9a2a2d9746374781f123aea702bbbc1184aee033c47662eea6406888ff5c484bb6f62030b6e4841c2e723f545169e8d07fbd8f16
6
+ metadata.gz: 8da80d465996766d52b79abdb283e583dd244789511597e63e3bc969df671337483a0915db8a0b4ade5e65a205c13002d66c7c0ba07ea7a70ca5c75cf3bd73ad
7
+ data.tar.gz: ed2ee611f85075ce812621b25710a0f802b151906b0f7f140f2b6afd4f179d808a6847d2519d6edb992ad43f9262a54de1d310b183103143170c8dd46b6ef965
data/README.md CHANGED
@@ -21,6 +21,7 @@ Oracle input plugin for Embulk loads records from Oracle.
21
21
  - **net_service_name**: Oracle Net Service Name (string, optional)
22
22
  - If you write SQL directly,
23
23
  - **query**: SQL to run (string)
24
+ - **use_raw_query_with_incremental**: If true, you can write optimized query using prepared statement by yourself. See [Use incremental loading with raw query](#use-incremental-loading-with-raw-query) for more detail (boolean, default: false)
24
25
  - If **query** is not set,
25
26
  - **table**: destination table name (string, required)
26
27
  - **select**: expression of select (e.g. `id, created_at`) (string, default: "*")
@@ -81,6 +82,38 @@ CREATE INDEX embulk_incremental_loading_index ON table (updated_at, id);
81
82
 
82
83
  Recommended usage is to leave `incremental_columns` unset and let this plugin automatically finds a primary key. Currently, only strings and integers are supported as incremental_columns.
83
84
 
85
+ ### Use incremental loading with raw query
86
+
87
+ **IMPORTANT**: This is an advanced feature and assume you have an enough knowledge about incremental loading using Embulk and this plugin
88
+
89
+ Normally, you can't write your own query for incremental loading.
90
+ `use_raw_query_with_incremental` option allow you to write raw query for incremental loading. It might be well optimized and faster than SQL statement which is automatically generated by plugin.
91
+
92
+ Prepared statement starts with `:` is available instead of fixed value.
93
+ `last_record` value is necessary when you use this option.
94
+ Please use prepared statement that is well distinguishable in SQL statement. Using too simple prepared statement like `:a` might cause SQL parse failure.
95
+
96
+ In the following example, prepared statement `:foo_id` will be replaced with value "1" which is specified in `last_record`.
97
+
98
+ ```yaml
99
+ in:
100
+ type: oracle
101
+ query:
102
+ SELECT
103
+ foo.id as foo_id, bar.name
104
+ FROM
105
+ foo LEFT JOIN bar ON foo.id = bar.id
106
+ WHERE
107
+ foo.hoge IS NOT NULL
108
+ AND foo.id > :foo_id
109
+ ORDER BY
110
+ foo.id ASC
111
+ use_raw_query_with_incremental: true
112
+ incremental_columns:
113
+ - foo_id
114
+ incremental: true
115
+ last_record: [1]
116
+ ```
84
117
 
85
118
  ## Example
86
119
 
@@ -141,4 +141,67 @@ public class IncrementalTest
141
141
  is((ConfigDiff) loadYamlResource(embulk, "timestamp/expected_2.diff")));
142
142
  }
143
143
 
144
+ @Test
145
+ public void simpleQueryWithPlaceholder() throws Exception
146
+ {
147
+ // setup first rows
148
+ execute(embulk, readResource("query/setup.sql"));
149
+
150
+ Path out1 = embulk.createTempFile("csv");
151
+ RunResult result1 = embulk.runInput(
152
+ baseConfig.merge(loadYamlResource(embulk, "query/config_1.yml")),
153
+ out1);
154
+ assertThat(
155
+ readSortedFile(out1),
156
+ is(readResource("query/expected_1.csv")));
157
+ assertThat(
158
+ result1.getConfigDiff(),
159
+ is((ConfigDiff) loadYamlResource(embulk, "query/expected_1.diff")));
160
+
161
+ // insert more rows
162
+ execute(embulk, readResource("query/insert_more.sql"));
163
+
164
+ Path out2 = embulk.createTempFile("csv");
165
+ RunResult result2 = embulk.runInput(
166
+ baseConfig.merge(loadYamlResource(embulk, "query/config_2.yml")),
167
+ out2);
168
+ assertThat(
169
+ readSortedFile(out2),
170
+ is(readResource("query/expected_2.csv")));
171
+ assertThat(
172
+ result2.getConfigDiff(),
173
+ is((ConfigDiff) loadYamlResource(embulk, "query/expected_2.diff")));
174
+ }
175
+
176
+ @Test
177
+ public void simpleQueryWithPlaceholderAndMultiColumns() throws Exception
178
+ {
179
+ // setup first rows
180
+ execute(embulk, readResource("query/setup.sql"));
181
+
182
+ Path out1 = embulk.createTempFile("csv");
183
+ RunResult result1 = embulk.runInput(
184
+ baseConfig.merge(loadYamlResource(embulk, "query/multi_columns_config_1.yml")),
185
+ out1);
186
+ assertThat(
187
+ readSortedFile(out1),
188
+ is(readResource("query/multi_columns_expected_1.csv")));
189
+ assertThat(
190
+ result1.getConfigDiff(),
191
+ is((ConfigDiff) loadYamlResource(embulk, "query/multi_columns_expected_1.diff")));
192
+
193
+ // insert more rows
194
+ execute(embulk, readResource("query/insert_more.sql"));
195
+
196
+ Path out2 = embulk.createTempFile("csv");
197
+ RunResult result2 = embulk.runInput(
198
+ baseConfig.merge(loadYamlResource(embulk, "query/multi_columns_config_2.yml")),
199
+ out2);
200
+ assertThat(
201
+ readSortedFile(out2),
202
+ is(readResource("query/multi_columns_expected_2.csv")));
203
+ assertThat(
204
+ result2.getConfigDiff(),
205
+ is((ConfigDiff) loadYamlResource(embulk, "query/multi_columns_expected_2.diff")));
206
+ }
144
207
  }
@@ -0,0 +1,16 @@
1
+ column_options:
2
+ num: {value_type: long}
3
+ last_record: [0]
4
+ incremental: true
5
+ incremental_columns: [num]
6
+ use_raw_query_with_incremental: true
7
+ query: |
8
+ SELECT
9
+ num,note
10
+ FROM
11
+ query_load
12
+ WHERE
13
+ num IS NOT NULL
14
+ AND num > :num
15
+ ORDER BY
16
+ num ASC
@@ -0,0 +1,16 @@
1
+ column_options:
2
+ num: {type: long}
3
+ last_record: [4]
4
+ incremental: true
5
+ incremental_columns: [num]
6
+ use_raw_query_with_incremental: true
7
+ query: |
8
+ SELECT
9
+ num,note
10
+ FROM
11
+ query_load
12
+ WHERE
13
+ num IS NOT NULL
14
+ AND num > :num
15
+ ORDER BY
16
+ num ASC
@@ -0,0 +1,6 @@
1
+ insert into query_load (num, num2, note) values (0, 100, 'more_skip');
2
+ insert into query_load (num, num2, note) values (4, 104, 'more_skip');
3
+ insert into query_load (num, num2, note) values (9, 109, 'more_load');
4
+ insert into query_load (num, num2, note) values (5, 105, 'more_load');
5
+
6
+ EXIT;
@@ -0,0 +1,19 @@
1
+ column_options:
2
+ num: {value_type: long}
3
+ num2: {value_type: long}
4
+ last_record: [0,0]
5
+ incremental: true
6
+ incremental_columns: [num,num2]
7
+ use_raw_query_with_incremental: true
8
+ query: |
9
+ SELECT
10
+ *
11
+ FROM
12
+ query_load
13
+ WHERE
14
+ num IS NOT NULL
15
+ AND num > :num
16
+ OR (num = :num AND num2 > :num2)
17
+ ORDER BY
18
+ num ASC,
19
+ num2 ASC
@@ -0,0 +1,19 @@
1
+ column_options:
2
+ num: {type: long}
3
+ num2: {type: long}
4
+ last_record: [4,104]
5
+ incremental: true
6
+ incremental_columns: [num,num2]
7
+ use_raw_query_with_incremental: true
8
+ query: |
9
+ SELECT
10
+ *
11
+ FROM
12
+ query_load
13
+ WHERE
14
+ num IS NOT NULL
15
+ AND num > :num
16
+ OR (num = :num AND num2 > :num2)
17
+ ORDER BY
18
+ num ASC,
19
+ num2 ASC
@@ -0,0 +1,14 @@
1
+ drop table query_load;
2
+
3
+ create table query_load (
4
+ num NUMBER(10) not null,
5
+ num2 NUMBER(10) not null,
6
+ note VARCHAR2(10)
7
+ );
8
+
9
+ insert into query_load (num, num2, note) values (3, 103, 'first');
10
+ insert into query_load (num, num2, note) values (4, 104, 'first');
11
+ insert into query_load (num, num2, note) values (2, 102, 'first');
12
+ insert into query_load (num, num2, note) values (1, 101, 'first');
13
+
14
+ EXIT;
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: embulk-input-oracle
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.9.2
4
+ version: 0.9.3
5
5
  platform: ruby
6
6
  authors:
7
7
  - Sadayuki Furuhashi
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2018-07-03 00:00:00.000000000 Z
11
+ date: 2018-08-10 00:00:00.000000000 Z
12
12
  dependencies: []
13
13
  description: Selects records from a table.
14
14
  email:
@@ -19,8 +19,8 @@ extra_rdoc_files: []
19
19
  files:
20
20
  - README.md
21
21
  - build.gradle
22
- - classpath/embulk-input-jdbc-0.9.2.jar
23
- - classpath/embulk-input-oracle-0.9.2.jar
22
+ - classpath/embulk-input-jdbc-0.9.3.jar
23
+ - classpath/embulk-input-oracle-0.9.3.jar
24
24
  - lib/embulk/input/oracle.rb
25
25
  - src/main/java/org/embulk/input/OracleInputPlugin.java
26
26
  - src/main/java/org/embulk/input/oracle/OracleInputConnection.java
@@ -64,6 +64,20 @@ files:
64
64
  - src/test/resources/org/embulk/input/oracle/test/expect/incremental/int/expected_2.diff
65
65
  - src/test/resources/org/embulk/input/oracle/test/expect/incremental/int/insert_more.sql
66
66
  - src/test/resources/org/embulk/input/oracle/test/expect/incremental/int/setup.sql
67
+ - src/test/resources/org/embulk/input/oracle/test/expect/incremental/query/config_1.yml
68
+ - src/test/resources/org/embulk/input/oracle/test/expect/incremental/query/config_2.yml
69
+ - src/test/resources/org/embulk/input/oracle/test/expect/incremental/query/expected_1.csv
70
+ - src/test/resources/org/embulk/input/oracle/test/expect/incremental/query/expected_1.diff
71
+ - src/test/resources/org/embulk/input/oracle/test/expect/incremental/query/expected_2.csv
72
+ - src/test/resources/org/embulk/input/oracle/test/expect/incremental/query/expected_2.diff
73
+ - src/test/resources/org/embulk/input/oracle/test/expect/incremental/query/insert_more.sql
74
+ - src/test/resources/org/embulk/input/oracle/test/expect/incremental/query/multi_columns_config_1.yml
75
+ - src/test/resources/org/embulk/input/oracle/test/expect/incremental/query/multi_columns_config_2.yml
76
+ - src/test/resources/org/embulk/input/oracle/test/expect/incremental/query/multi_columns_expected_1.csv
77
+ - src/test/resources/org/embulk/input/oracle/test/expect/incremental/query/multi_columns_expected_1.diff
78
+ - src/test/resources/org/embulk/input/oracle/test/expect/incremental/query/multi_columns_expected_2.csv
79
+ - src/test/resources/org/embulk/input/oracle/test/expect/incremental/query/multi_columns_expected_2.diff
80
+ - src/test/resources/org/embulk/input/oracle/test/expect/incremental/query/setup.sql
67
81
  - src/test/resources/org/embulk/input/oracle/test/expect/incremental/timestamp/config_1.yml
68
82
  - src/test/resources/org/embulk/input/oracle/test/expect/incremental/timestamp/config_2.yml
69
83
  - src/test/resources/org/embulk/input/oracle/test/expect/incremental/timestamp/expected_1.csv