embulk-input-oracle 0.9.2 → 0.9.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +33 -0
- data/classpath/embulk-input-jdbc-0.9.3.jar +0 -0
- data/classpath/{embulk-input-oracle-0.9.2.jar → embulk-input-oracle-0.9.3.jar} +0 -0
- data/src/test/java/org/embulk/input/oracle/IncrementalTest.java +63 -0
- data/src/test/resources/org/embulk/input/oracle/test/expect/incremental/query/config_1.yml +16 -0
- data/src/test/resources/org/embulk/input/oracle/test/expect/incremental/query/config_2.yml +16 -0
- data/src/test/resources/org/embulk/input/oracle/test/expect/incremental/query/expected_1.csv +4 -0
- data/src/test/resources/org/embulk/input/oracle/test/expect/incremental/query/expected_1.diff +3 -0
- data/src/test/resources/org/embulk/input/oracle/test/expect/incremental/query/expected_2.csv +2 -0
- data/src/test/resources/org/embulk/input/oracle/test/expect/incremental/query/expected_2.diff +3 -0
- data/src/test/resources/org/embulk/input/oracle/test/expect/incremental/query/insert_more.sql +6 -0
- data/src/test/resources/org/embulk/input/oracle/test/expect/incremental/query/multi_columns_config_1.yml +19 -0
- data/src/test/resources/org/embulk/input/oracle/test/expect/incremental/query/multi_columns_config_2.yml +19 -0
- data/src/test/resources/org/embulk/input/oracle/test/expect/incremental/query/multi_columns_expected_1.csv +4 -0
- data/src/test/resources/org/embulk/input/oracle/test/expect/incremental/query/multi_columns_expected_1.diff +3 -0
- data/src/test/resources/org/embulk/input/oracle/test/expect/incremental/query/multi_columns_expected_2.csv +2 -0
- data/src/test/resources/org/embulk/input/oracle/test/expect/incremental/query/multi_columns_expected_2.diff +3 -0
- data/src/test/resources/org/embulk/input/oracle/test/expect/incremental/query/setup.sql +14 -0
- metadata +18 -4
- data/classpath/embulk-input-jdbc-0.9.2.jar +0 -0
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 0ae776a1726302045e4d3beec5c476083fb68f38
|
4
|
+
data.tar.gz: 44df8eb7f1d3433a73cff5dab8dde1403c395596
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 8da80d465996766d52b79abdb283e583dd244789511597e63e3bc969df671337483a0915db8a0b4ade5e65a205c13002d66c7c0ba07ea7a70ca5c75cf3bd73ad
|
7
|
+
data.tar.gz: ed2ee611f85075ce812621b25710a0f802b151906b0f7f140f2b6afd4f179d808a6847d2519d6edb992ad43f9262a54de1d310b183103143170c8dd46b6ef965
|
data/README.md
CHANGED
@@ -21,6 +21,7 @@ Oracle input plugin for Embulk loads records from Oracle.
|
|
21
21
|
- **net_service_name**: Oracle Net Service Name (string, optional)
|
22
22
|
- If you write SQL directly,
|
23
23
|
- **query**: SQL to run (string)
|
24
|
+
- **use_raw_query_with_incremental**: If true, you can write optimized query using prepared statement by yourself. See [Use incremental loading with raw query](#use-incremental-loading-with-raw-query) for more detail (boolean, default: false)
|
24
25
|
- If **query** is not set,
|
25
26
|
- **table**: destination table name (string, required)
|
26
27
|
- **select**: expression of select (e.g. `id, created_at`) (string, default: "*")
|
@@ -81,6 +82,38 @@ CREATE INDEX embulk_incremental_loading_index ON table (updated_at, id);
|
|
81
82
|
|
82
83
|
Recommended usage is to leave `incremental_columns` unset and let this plugin automatically finds a primary key. Currently, only strings and integers are supported as incremental_columns.
|
83
84
|
|
85
|
+
### Use incremental loading with raw query
|
86
|
+
|
87
|
+
**IMPORTANT**: This is an advanced feature and assume you have an enough knowledge about incremental loading using Embulk and this plugin
|
88
|
+
|
89
|
+
Normally, you can't write your own query for incremental loading.
|
90
|
+
`use_raw_query_with_incremental` option allow you to write raw query for incremental loading. It might be well optimized and faster than SQL statement which is automatically generated by plugin.
|
91
|
+
|
92
|
+
Prepared statement starts with `:` is available instead of fixed value.
|
93
|
+
`last_record` value is necessary when you use this option.
|
94
|
+
Please use prepared statement that is well distinguishable in SQL statement. Using too simple prepared statement like `:a` might cause SQL parse failure.
|
95
|
+
|
96
|
+
In the following example, prepared statement `:foo_id` will be replaced with value "1" which is specified in `last_record`.
|
97
|
+
|
98
|
+
```yaml
|
99
|
+
in:
|
100
|
+
type: oracle
|
101
|
+
query:
|
102
|
+
SELECT
|
103
|
+
foo.id as foo_id, bar.name
|
104
|
+
FROM
|
105
|
+
foo LEFT JOIN bar ON foo.id = bar.id
|
106
|
+
WHERE
|
107
|
+
foo.hoge IS NOT NULL
|
108
|
+
AND foo.id > :foo_id
|
109
|
+
ORDER BY
|
110
|
+
foo.id ASC
|
111
|
+
use_raw_query_with_incremental: true
|
112
|
+
incremental_columns:
|
113
|
+
- foo_id
|
114
|
+
incremental: true
|
115
|
+
last_record: [1]
|
116
|
+
```
|
84
117
|
|
85
118
|
## Example
|
86
119
|
|
Binary file
|
Binary file
|
@@ -141,4 +141,67 @@ public class IncrementalTest
|
|
141
141
|
is((ConfigDiff) loadYamlResource(embulk, "timestamp/expected_2.diff")));
|
142
142
|
}
|
143
143
|
|
144
|
+
@Test
|
145
|
+
public void simpleQueryWithPlaceholder() throws Exception
|
146
|
+
{
|
147
|
+
// setup first rows
|
148
|
+
execute(embulk, readResource("query/setup.sql"));
|
149
|
+
|
150
|
+
Path out1 = embulk.createTempFile("csv");
|
151
|
+
RunResult result1 = embulk.runInput(
|
152
|
+
baseConfig.merge(loadYamlResource(embulk, "query/config_1.yml")),
|
153
|
+
out1);
|
154
|
+
assertThat(
|
155
|
+
readSortedFile(out1),
|
156
|
+
is(readResource("query/expected_1.csv")));
|
157
|
+
assertThat(
|
158
|
+
result1.getConfigDiff(),
|
159
|
+
is((ConfigDiff) loadYamlResource(embulk, "query/expected_1.diff")));
|
160
|
+
|
161
|
+
// insert more rows
|
162
|
+
execute(embulk, readResource("query/insert_more.sql"));
|
163
|
+
|
164
|
+
Path out2 = embulk.createTempFile("csv");
|
165
|
+
RunResult result2 = embulk.runInput(
|
166
|
+
baseConfig.merge(loadYamlResource(embulk, "query/config_2.yml")),
|
167
|
+
out2);
|
168
|
+
assertThat(
|
169
|
+
readSortedFile(out2),
|
170
|
+
is(readResource("query/expected_2.csv")));
|
171
|
+
assertThat(
|
172
|
+
result2.getConfigDiff(),
|
173
|
+
is((ConfigDiff) loadYamlResource(embulk, "query/expected_2.diff")));
|
174
|
+
}
|
175
|
+
|
176
|
+
@Test
|
177
|
+
public void simpleQueryWithPlaceholderAndMultiColumns() throws Exception
|
178
|
+
{
|
179
|
+
// setup first rows
|
180
|
+
execute(embulk, readResource("query/setup.sql"));
|
181
|
+
|
182
|
+
Path out1 = embulk.createTempFile("csv");
|
183
|
+
RunResult result1 = embulk.runInput(
|
184
|
+
baseConfig.merge(loadYamlResource(embulk, "query/multi_columns_config_1.yml")),
|
185
|
+
out1);
|
186
|
+
assertThat(
|
187
|
+
readSortedFile(out1),
|
188
|
+
is(readResource("query/multi_columns_expected_1.csv")));
|
189
|
+
assertThat(
|
190
|
+
result1.getConfigDiff(),
|
191
|
+
is((ConfigDiff) loadYamlResource(embulk, "query/multi_columns_expected_1.diff")));
|
192
|
+
|
193
|
+
// insert more rows
|
194
|
+
execute(embulk, readResource("query/insert_more.sql"));
|
195
|
+
|
196
|
+
Path out2 = embulk.createTempFile("csv");
|
197
|
+
RunResult result2 = embulk.runInput(
|
198
|
+
baseConfig.merge(loadYamlResource(embulk, "query/multi_columns_config_2.yml")),
|
199
|
+
out2);
|
200
|
+
assertThat(
|
201
|
+
readSortedFile(out2),
|
202
|
+
is(readResource("query/multi_columns_expected_2.csv")));
|
203
|
+
assertThat(
|
204
|
+
result2.getConfigDiff(),
|
205
|
+
is((ConfigDiff) loadYamlResource(embulk, "query/multi_columns_expected_2.diff")));
|
206
|
+
}
|
144
207
|
}
|
@@ -0,0 +1,16 @@
|
|
1
|
+
column_options:
|
2
|
+
num: {value_type: long}
|
3
|
+
last_record: [0]
|
4
|
+
incremental: true
|
5
|
+
incremental_columns: [num]
|
6
|
+
use_raw_query_with_incremental: true
|
7
|
+
query: |
|
8
|
+
SELECT
|
9
|
+
num,note
|
10
|
+
FROM
|
11
|
+
query_load
|
12
|
+
WHERE
|
13
|
+
num IS NOT NULL
|
14
|
+
AND num > :num
|
15
|
+
ORDER BY
|
16
|
+
num ASC
|
@@ -0,0 +1,16 @@
|
|
1
|
+
column_options:
|
2
|
+
num: {type: long}
|
3
|
+
last_record: [4]
|
4
|
+
incremental: true
|
5
|
+
incremental_columns: [num]
|
6
|
+
use_raw_query_with_incremental: true
|
7
|
+
query: |
|
8
|
+
SELECT
|
9
|
+
num,note
|
10
|
+
FROM
|
11
|
+
query_load
|
12
|
+
WHERE
|
13
|
+
num IS NOT NULL
|
14
|
+
AND num > :num
|
15
|
+
ORDER BY
|
16
|
+
num ASC
|
@@ -0,0 +1,6 @@
|
|
1
|
+
insert into query_load (num, num2, note) values (0, 100, 'more_skip');
|
2
|
+
insert into query_load (num, num2, note) values (4, 104, 'more_skip');
|
3
|
+
insert into query_load (num, num2, note) values (9, 109, 'more_load');
|
4
|
+
insert into query_load (num, num2, note) values (5, 105, 'more_load');
|
5
|
+
|
6
|
+
EXIT;
|
@@ -0,0 +1,19 @@
|
|
1
|
+
column_options:
|
2
|
+
num: {value_type: long}
|
3
|
+
num2: {value_type: long}
|
4
|
+
last_record: [0,0]
|
5
|
+
incremental: true
|
6
|
+
incremental_columns: [num,num2]
|
7
|
+
use_raw_query_with_incremental: true
|
8
|
+
query: |
|
9
|
+
SELECT
|
10
|
+
*
|
11
|
+
FROM
|
12
|
+
query_load
|
13
|
+
WHERE
|
14
|
+
num IS NOT NULL
|
15
|
+
AND num > :num
|
16
|
+
OR (num = :num AND num2 > :num2)
|
17
|
+
ORDER BY
|
18
|
+
num ASC,
|
19
|
+
num2 ASC
|
@@ -0,0 +1,19 @@
|
|
1
|
+
column_options:
|
2
|
+
num: {type: long}
|
3
|
+
num2: {type: long}
|
4
|
+
last_record: [4,104]
|
5
|
+
incremental: true
|
6
|
+
incremental_columns: [num,num2]
|
7
|
+
use_raw_query_with_incremental: true
|
8
|
+
query: |
|
9
|
+
SELECT
|
10
|
+
*
|
11
|
+
FROM
|
12
|
+
query_load
|
13
|
+
WHERE
|
14
|
+
num IS NOT NULL
|
15
|
+
AND num > :num
|
16
|
+
OR (num = :num AND num2 > :num2)
|
17
|
+
ORDER BY
|
18
|
+
num ASC,
|
19
|
+
num2 ASC
|
@@ -0,0 +1,14 @@
|
|
1
|
+
drop table query_load;
|
2
|
+
|
3
|
+
create table query_load (
|
4
|
+
num NUMBER(10) not null,
|
5
|
+
num2 NUMBER(10) not null,
|
6
|
+
note VARCHAR2(10)
|
7
|
+
);
|
8
|
+
|
9
|
+
insert into query_load (num, num2, note) values (3, 103, 'first');
|
10
|
+
insert into query_load (num, num2, note) values (4, 104, 'first');
|
11
|
+
insert into query_load (num, num2, note) values (2, 102, 'first');
|
12
|
+
insert into query_load (num, num2, note) values (1, 101, 'first');
|
13
|
+
|
14
|
+
EXIT;
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-input-oracle
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.9.
|
4
|
+
version: 0.9.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Sadayuki Furuhashi
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2018-
|
11
|
+
date: 2018-08-10 00:00:00.000000000 Z
|
12
12
|
dependencies: []
|
13
13
|
description: Selects records from a table.
|
14
14
|
email:
|
@@ -19,8 +19,8 @@ extra_rdoc_files: []
|
|
19
19
|
files:
|
20
20
|
- README.md
|
21
21
|
- build.gradle
|
22
|
-
- classpath/embulk-input-jdbc-0.9.
|
23
|
-
- classpath/embulk-input-oracle-0.9.
|
22
|
+
- classpath/embulk-input-jdbc-0.9.3.jar
|
23
|
+
- classpath/embulk-input-oracle-0.9.3.jar
|
24
24
|
- lib/embulk/input/oracle.rb
|
25
25
|
- src/main/java/org/embulk/input/OracleInputPlugin.java
|
26
26
|
- src/main/java/org/embulk/input/oracle/OracleInputConnection.java
|
@@ -64,6 +64,20 @@ files:
|
|
64
64
|
- src/test/resources/org/embulk/input/oracle/test/expect/incremental/int/expected_2.diff
|
65
65
|
- src/test/resources/org/embulk/input/oracle/test/expect/incremental/int/insert_more.sql
|
66
66
|
- src/test/resources/org/embulk/input/oracle/test/expect/incremental/int/setup.sql
|
67
|
+
- src/test/resources/org/embulk/input/oracle/test/expect/incremental/query/config_1.yml
|
68
|
+
- src/test/resources/org/embulk/input/oracle/test/expect/incremental/query/config_2.yml
|
69
|
+
- src/test/resources/org/embulk/input/oracle/test/expect/incremental/query/expected_1.csv
|
70
|
+
- src/test/resources/org/embulk/input/oracle/test/expect/incremental/query/expected_1.diff
|
71
|
+
- src/test/resources/org/embulk/input/oracle/test/expect/incremental/query/expected_2.csv
|
72
|
+
- src/test/resources/org/embulk/input/oracle/test/expect/incremental/query/expected_2.diff
|
73
|
+
- src/test/resources/org/embulk/input/oracle/test/expect/incremental/query/insert_more.sql
|
74
|
+
- src/test/resources/org/embulk/input/oracle/test/expect/incremental/query/multi_columns_config_1.yml
|
75
|
+
- src/test/resources/org/embulk/input/oracle/test/expect/incremental/query/multi_columns_config_2.yml
|
76
|
+
- src/test/resources/org/embulk/input/oracle/test/expect/incremental/query/multi_columns_expected_1.csv
|
77
|
+
- src/test/resources/org/embulk/input/oracle/test/expect/incremental/query/multi_columns_expected_1.diff
|
78
|
+
- src/test/resources/org/embulk/input/oracle/test/expect/incremental/query/multi_columns_expected_2.csv
|
79
|
+
- src/test/resources/org/embulk/input/oracle/test/expect/incremental/query/multi_columns_expected_2.diff
|
80
|
+
- src/test/resources/org/embulk/input/oracle/test/expect/incremental/query/setup.sql
|
67
81
|
- src/test/resources/org/embulk/input/oracle/test/expect/incremental/timestamp/config_1.yml
|
68
82
|
- src/test/resources/org/embulk/input/oracle/test/expect/incremental/timestamp/config_2.yml
|
69
83
|
- src/test/resources/org/embulk/input/oracle/test/expect/incremental/timestamp/expected_1.csv
|
Binary file
|