embulk-input-oracle 0.9.2 → 0.9.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 793a1399c4c68e231e24b0ecb76c99f0a54a41b7
-  data.tar.gz: 351b3f2920f4b5dfdb6ca04bbbd35d28f8545452
+  metadata.gz: 0ae776a1726302045e4d3beec5c476083fb68f38
+  data.tar.gz: 44df8eb7f1d3433a73cff5dab8dde1403c395596
 SHA512:
-  metadata.gz: 0a433b3335d54052bd23adbdf76486658fd8ff1df2efa0845bfe281d372b7cfc8b43ea49509ea9e7b557add43997eac0a6734f9725a7a53414c94002d10e59c6
-  data.tar.gz: 4b71b09c09068d97b95ab94a9a2a2d9746374781f123aea702bbbc1184aee033c47662eea6406888ff5c484bb6f62030b6e4841c2e723f545169e8d07fbd8f16
+  metadata.gz: 8da80d465996766d52b79abdb283e583dd244789511597e63e3bc969df671337483a0915db8a0b4ade5e65a205c13002d66c7c0ba07ea7a70ca5c75cf3bd73ad
+  data.tar.gz: ed2ee611f85075ce812621b25710a0f802b151906b0f7f140f2b6afd4f179d808a6847d2519d6edb992ad43f9262a54de1d310b183103143170c8dd46b6ef965

data/README.md

@@ -21,6 +21,7 @@ Oracle input plugin for Embulk loads records from Oracle.
 - **net_service_name**: Oracle Net Service Name (string, optional)
 - If you write SQL directly,
   - **query**: SQL to run (string)
+  - **use_raw_query_with_incremental**: If true, you can write optimized query using prepared statement by yourself. See [Use incremental loading with raw query](#use-incremental-loading-with-raw-query) for more detail (boolean, default: false)
 - If **query** is not set,
   - **table**: destination table name (string, required)
   - **select**: expression of select (e.g. `id, created_at`) (string, default: "*")
@@ -81,6 +82,38 @@ CREATE INDEX embulk_incremental_loading_index ON table (updated_at, id);
 
 Recommended usage is to leave `incremental_columns` unset and let this plugin automatically finds a primary key. Currently, only strings and integers are supported as incremental_columns.
 
+### Use incremental loading with raw query
+
+**IMPORTANT**: This is an advanced feature and assume you have an enough knowledge about incremental loading using Embulk and this plugin
+
+Normally, you can't write your own query for incremental loading.
+`use_raw_query_with_incremental` option allow you to write raw query for incremental loading. It might be well optimized and faster than SQL statement which is automatically generated by plugin.
+
+Prepared statement starts with `:` is available instead of fixed value.
+`last_record` value is necessary when you use this option.
+Please use prepared statement that is well distinguishable in SQL statement. Using too simple prepared statement like `:a` might cause SQL parse failure.
+
+In the following example, prepared statement `:foo_id` will be replaced with value "1" which is specified in `last_record`.
+
+```yaml
+in:
+  type: oracle
+  query:
+    SELECT
+      foo.id as foo_id, bar.name
+    FROM
+      foo LEFT JOIN bar ON foo.id = bar.id
+    WHERE
+      foo.hoge IS NOT NULL
+      AND foo.id > :foo_id
+    ORDER BY
+      foo.id ASC
+  use_raw_query_with_incremental: true
+  incremental_columns:
+    - foo_id
+  incremental: true
+  last_record: [1]
+```
 
 ## Example
 

data/src/test/java/org/embulk/input/oracle/IncrementalTest.java

@@ -141,4 +141,67 @@ public class IncrementalTest
                 is((ConfigDiff) loadYamlResource(embulk, "timestamp/expected_2.diff")));
     }
 
+    @Test
+    public void simpleQueryWithPlaceholder() throws Exception
+    {
+        // setup first rows
+        execute(embulk, readResource("query/setup.sql"));
+
+        Path out1 = embulk.createTempFile("csv");
+        RunResult result1 = embulk.runInput(
+                baseConfig.merge(loadYamlResource(embulk, "query/config_1.yml")),
+                out1);
+        assertThat(
+                readSortedFile(out1),
+                is(readResource("query/expected_1.csv")));
+        assertThat(
+                result1.getConfigDiff(),
+                is((ConfigDiff) loadYamlResource(embulk, "query/expected_1.diff")));
+
+        // insert more rows
+        execute(embulk, readResource("query/insert_more.sql"));
+
+        Path out2 = embulk.createTempFile("csv");
+        RunResult result2 = embulk.runInput(
+                baseConfig.merge(loadYamlResource(embulk, "query/config_2.yml")),
+                out2);
+        assertThat(
+                readSortedFile(out2),
+                is(readResource("query/expected_2.csv")));
+        assertThat(
+                result2.getConfigDiff(),
+                is((ConfigDiff) loadYamlResource(embulk, "query/expected_2.diff")));
+    }
+
+    @Test
+    public void simpleQueryWithPlaceholderAndMultiColumns() throws Exception
+    {
+        // setup first rows
+        execute(embulk, readResource("query/setup.sql"));
+
+        Path out1 = embulk.createTempFile("csv");
+        RunResult result1 = embulk.runInput(
+                baseConfig.merge(loadYamlResource(embulk, "query/multi_columns_config_1.yml")),
+                out1);
+        assertThat(
+                readSortedFile(out1),
+                is(readResource("query/multi_columns_expected_1.csv")));
+        assertThat(
+                result1.getConfigDiff(),
+                is((ConfigDiff) loadYamlResource(embulk, "query/multi_columns_expected_1.diff")));
+
+        // insert more rows
+        execute(embulk, readResource("query/insert_more.sql"));
+
+        Path out2 = embulk.createTempFile("csv");
+        RunResult result2 = embulk.runInput(
+                baseConfig.merge(loadYamlResource(embulk, "query/multi_columns_config_2.yml")),
+                out2);
+        assertThat(
+                readSortedFile(out2),
+                is(readResource("query/multi_columns_expected_2.csv")));
+        assertThat(
+                result2.getConfigDiff(),
+                is((ConfigDiff) loadYamlResource(embulk, "query/multi_columns_expected_2.diff")));
+    }
 }

data/src/test/resources/org/embulk/input/oracle/test/expect/incremental/query/config_1.yml

@@ -0,0 +1,16 @@
+column_options:
+  num: {value_type: long}
+last_record: [0]
+incremental: true
+incremental_columns: [num]
+use_raw_query_with_incremental: true
+query: |
+  SELECT
+    num,note
+  FROM
+    query_load
+  WHERE
+    num IS NOT NULL
+    AND num > :num
+  ORDER BY
+    num ASC

data/src/test/resources/org/embulk/input/oracle/test/expect/incremental/query/config_2.yml

@@ -0,0 +1,16 @@
+column_options:
+  num: {type: long}
+last_record: [4]
+incremental: true
+incremental_columns: [num]
+use_raw_query_with_incremental: true
+query: |
+  SELECT
+    num,note
+  FROM
+    query_load
+  WHERE
+    num IS NOT NULL
+    AND num > :num
+  ORDER BY
+    num ASC

data/src/test/resources/org/embulk/input/oracle/test/expect/incremental/query/expected_1.csv

@@ -0,0 +1,4 @@
+1,first
+2,first
+3,first
+4,first

data/src/test/resources/org/embulk/input/oracle/test/expect/incremental/query/expected_1.diff

@@ -0,0 +1,3 @@
+in:
+  last_record: [4]
+out: {}

data/src/test/resources/org/embulk/input/oracle/test/expect/incremental/query/expected_2.csv

@@ -0,0 +1,2 @@
+5,more_load
+9,more_load

data/src/test/resources/org/embulk/input/oracle/test/expect/incremental/query/expected_2.diff

@@ -0,0 +1,3 @@
+in:
+  last_record: [9]
+out: {}

data/src/test/resources/org/embulk/input/oracle/test/expect/incremental/query/insert_more.sql

@@ -0,0 +1,6 @@
+insert into query_load (num, num2, note) values (0, 100, 'more_skip');
+insert into query_load (num, num2, note) values (4, 104, 'more_skip');
+insert into query_load (num, num2, note) values (9, 109, 'more_load');
+insert into query_load (num, num2, note) values (5, 105, 'more_load');
+
+EXIT;

data/src/test/resources/org/embulk/input/oracle/test/expect/incremental/query/multi_columns_config_1.yml

@@ -0,0 +1,19 @@
+column_options:
+  num: {value_type: long}
+  num2: {value_type: long}
+last_record: [0,0]
+incremental: true
+incremental_columns: [num,num2]
+use_raw_query_with_incremental: true
+query: |
+  SELECT
+    *
+  FROM
+    query_load
+  WHERE
+    num IS NOT NULL
+    AND num > :num
+    OR (num = :num AND num2 > :num2)
+  ORDER BY
+    num ASC,
+    num2 ASC

data/src/test/resources/org/embulk/input/oracle/test/expect/incremental/query/multi_columns_config_2.yml

@@ -0,0 +1,19 @@
+column_options:
+  num: {type: long}
+  num2: {type: long}
+last_record: [4,104]
+incremental: true
+incremental_columns: [num,num2]
+use_raw_query_with_incremental: true
+query: |
+  SELECT
+    *
+  FROM
+    query_load
+  WHERE
+    num IS NOT NULL
+    AND num > :num
+    OR (num = :num AND num2 > :num2)
+  ORDER BY
+    num ASC,
+    num2 ASC

data/src/test/resources/org/embulk/input/oracle/test/expect/incremental/query/multi_columns_expected_1.csv

@@ -0,0 +1,4 @@
+1,101,first
+2,102,first
+3,103,first
+4,104,first

data/src/test/resources/org/embulk/input/oracle/test/expect/incremental/query/multi_columns_expected_1.diff

@@ -0,0 +1,3 @@
+in:
+  last_record: [4,104]
+out: {}

data/src/test/resources/org/embulk/input/oracle/test/expect/incremental/query/multi_columns_expected_2.csv

@@ -0,0 +1,2 @@
+5,105,more_load
+9,109,more_load

data/src/test/resources/org/embulk/input/oracle/test/expect/incremental/query/multi_columns_expected_2.diff

@@ -0,0 +1,3 @@
+in:
+  last_record: [9,109]
+out: {}

data/src/test/resources/org/embulk/input/oracle/test/expect/incremental/query/setup.sql

@@ -0,0 +1,14 @@
+drop table query_load;
+
+create table query_load (
+    num NUMBER(10) not null,
+    num2 NUMBER(10) not null,
+    note VARCHAR2(10)
+);
+
+insert into query_load (num, num2, note) values (3, 103, 'first');
+insert into query_load (num, num2, note) values (4, 104, 'first');
+insert into query_load (num, num2, note) values (2, 102, 'first');
+insert into query_load (num, num2, note) values (1, 101, 'first');
+
+EXIT;

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-oracle
 version: !ruby/object:Gem::Version
-  version: 0.9.2
+  version: 0.9.3
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-07-03 00:00:00.000000000 Z
+date: 2018-08-10 00:00:00.000000000 Z
 dependencies: []
 description: Selects records from a table.
 email:
@@ -19,8 +19,8 @@ extra_rdoc_files: []
 files:
 - README.md
 - build.gradle
-- classpath/embulk-input-jdbc-0.9.2.jar
-- classpath/embulk-input-oracle-0.9.2.jar
+- classpath/embulk-input-jdbc-0.9.3.jar
+- classpath/embulk-input-oracle-0.9.3.jar
 - lib/embulk/input/oracle.rb
 - src/main/java/org/embulk/input/OracleInputPlugin.java
 - src/main/java/org/embulk/input/oracle/OracleInputConnection.java
@@ -64,6 +64,20 @@ files:
 - src/test/resources/org/embulk/input/oracle/test/expect/incremental/int/expected_2.diff
 - src/test/resources/org/embulk/input/oracle/test/expect/incremental/int/insert_more.sql
 - src/test/resources/org/embulk/input/oracle/test/expect/incremental/int/setup.sql
+- src/test/resources/org/embulk/input/oracle/test/expect/incremental/query/config_1.yml
+- src/test/resources/org/embulk/input/oracle/test/expect/incremental/query/config_2.yml
+- src/test/resources/org/embulk/input/oracle/test/expect/incremental/query/expected_1.csv
+- src/test/resources/org/embulk/input/oracle/test/expect/incremental/query/expected_1.diff
+- src/test/resources/org/embulk/input/oracle/test/expect/incremental/query/expected_2.csv
+- src/test/resources/org/embulk/input/oracle/test/expect/incremental/query/expected_2.diff
+- src/test/resources/org/embulk/input/oracle/test/expect/incremental/query/insert_more.sql
+- src/test/resources/org/embulk/input/oracle/test/expect/incremental/query/multi_columns_config_1.yml
+- src/test/resources/org/embulk/input/oracle/test/expect/incremental/query/multi_columns_config_2.yml
+- src/test/resources/org/embulk/input/oracle/test/expect/incremental/query/multi_columns_expected_1.csv
+- src/test/resources/org/embulk/input/oracle/test/expect/incremental/query/multi_columns_expected_1.diff
+- src/test/resources/org/embulk/input/oracle/test/expect/incremental/query/multi_columns_expected_2.csv
+- src/test/resources/org/embulk/input/oracle/test/expect/incremental/query/multi_columns_expected_2.diff
+- src/test/resources/org/embulk/input/oracle/test/expect/incremental/query/setup.sql
 - src/test/resources/org/embulk/input/oracle/test/expect/incremental/timestamp/config_1.yml
 - src/test/resources/org/embulk/input/oracle/test/expect/incremental/timestamp/config_2.yml
 - src/test/resources/org/embulk/input/oracle/test/expect/incremental/timestamp/expected_1.csv