embulk-input-postgresql 0.9.2 → 0.9.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (21) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +33 -0
  3. data/classpath/embulk-input-jdbc-0.9.3.jar +0 -0
  4. data/classpath/{embulk-input-postgresql-0.9.2.jar → embulk-input-postgresql-0.9.3.jar} +0 -0
  5. data/src/test/java/org/embulk/input/postgresql/IncrementalTest.java +64 -0
  6. data/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/query/config_1.yml +14 -0
  7. data/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/query/config_2.yml +14 -0
  8. data/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/query/expected_1.csv +4 -0
  9. data/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/query/expected_1.diff +3 -0
  10. data/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/query/expected_2.csv +2 -0
  11. data/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/query/expected_2.diff +3 -0
  12. data/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/query/insert_more.sql +6 -0
  13. data/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/query/multi_columns_config_1.yml +16 -0
  14. data/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/query/multi_columns_config_2.yml +16 -0
  15. data/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/query/multi_columns_expected_1.csv +4 -0
  16. data/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/query/multi_columns_expected_1.diff +3 -0
  17. data/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/query/multi_columns_expected_2.csv +2 -0
  18. data/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/query/multi_columns_expected_2.diff +3 -0
  19. data/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/query/setup.sql +13 -0
  20. metadata +18 -4
  21. data/classpath/embulk-input-jdbc-0.9.2.jar +0 -0
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 87724534f3076d4264cc4728fff2744c321c7982
4
- data.tar.gz: 9914d0dfe4ef6d469de2da431e26b61c03c566e6
3
+ metadata.gz: edb5cbc6680e6fdd62506d5274f43ae0fa95e89d
4
+ data.tar.gz: a961032ae6f7add861fcd88760fc8bbe9f09fe05
5
5
  SHA512:
6
- metadata.gz: 031259689b5a0f86e7a530f2ebdce24255b65d7da54b9e55cf56f2d9d4c65f62087bc6189cb887147db24f95c45d437e42e9923c5be607fd173b7af80c05ca5c
7
- data.tar.gz: 607b8e8e01d553badbeb7a4b881db6997ea721b2242bb373454785957fb188bbcbe2d416d5579e4f7dbe932de0bd5dcea6ad54e9c2c7f3f32a88c48608fbb088
6
+ metadata.gz: ece1c87b6a0b05919ba2054cdcdb8a10f59c3b4e4d9b6c6a1fb689f3964bfae1f013d5d0e17d8a453f1c49f7fe6a80218bfdb0bc2c19fd976a388cc517fb62cf
7
+ data.tar.gz: b0a055403e5a383316f4a39caee43fa24fd70207aa24a950779dfe3f08182e2d95e86a635057cc561b890626fd5d099603120fad0ae5bd308b80f3754c3c2e0b
data/README.md CHANGED
@@ -24,6 +24,7 @@ PostgreSQL input plugin for Embulk loads records from PostgreSQL.
24
24
  - **options**: extra JDBC properties (hash, default: {})
25
25
  - If you write SQL directly,
26
26
  - **query**: SQL to run (string)
27
+ - **use_raw_query_with_incremental**: If true, you can write optimized query using prepared statement by yourself. See [Use incremental loading with raw query](#use-incremental-loading-with-raw-query) for more detail (boolean, default: false)
27
28
  - If **query** is not set,
28
29
  - **table**: destination table name (string, required)
29
30
  - **select**: expression of select (e.g. `id, created_at`) (string, default: "*")
@@ -114,6 +115,38 @@ CREATE INDEX embulk_incremental_loading_index ON table (updated_at, id);
114
115
 
115
116
  Recommended usage is to leave `incremental_columns` unset and let this plugin automatically finds an auto-increment (serial / bigserial) primary key. Currently, only strings and integers are supported as incremental_columns.
116
117
 
118
+ ### Use incremental loading with raw query
119
+
120
+ **IMPORTANT**: This is an advanced feature and assume you have an enough knowledge about incremental loading using Embulk and this plugin
121
+
122
+ Normally, you can't write your own query for incremental loading.
123
+ `use_raw_query_with_incremental` option allow you to write raw query for incremental loading. It might be well optimized and faster than SQL statement which is automatically generated by plugin.
124
+
125
+ Prepared statement starts with `:` is available instead of fixed value.
126
+ `last_record` value is necessary when you use this option.
127
+ Please use prepared statement that is well distinguishable in SQL statement. Using too simple prepared statement like `:a` might cause SQL parse failure.
128
+
129
+ In the following example, prepared statement `:foo_id` will be replaced with value "1" which is specified in `last_record`.
130
+
131
+ ```yaml
132
+ in:
133
+ type: postgresql
134
+ query:
135
+ SELECT
136
+ foo.id as foo_id, bar.name
137
+ FROM
138
+ foo LEFT JOIN bar ON foo.id = bar.id
139
+ WHERE
140
+ foo.hoge IS NOT NULL
141
+ AND foo.id > :foo_id
142
+ ORDER BY
143
+ foo.id ASC
144
+ use_raw_query_with_incremental: true
145
+ incremental_columns:
146
+ - foo_id
147
+ incremental: true
148
+ last_record: [1]
149
+ ```
117
150
 
118
151
  ## Example
119
152
 
@@ -140,4 +140,68 @@ public class IncrementalTest
140
140
  result2.getConfigDiff(),
141
141
  is((ConfigDiff) loadYamlResource(embulk, "timestamptz/expected_2.diff")));
142
142
  }
143
+
144
+ @Test
145
+ public void simpleQueryWithPlaceholder() throws Exception
146
+ {
147
+ // setup first rows
148
+ execute(readResource("query/setup.sql"));
149
+
150
+ Path out1 = embulk.createTempFile("csv");
151
+ RunResult result1 = embulk.runInput(
152
+ baseConfig.merge(loadYamlResource(embulk, "query/config_1.yml")),
153
+ out1);
154
+ assertThat(
155
+ readSortedFile(out1),
156
+ is(readResource("query/expected_1.csv")));
157
+ assertThat(
158
+ result1.getConfigDiff(),
159
+ is((ConfigDiff) loadYamlResource(embulk, "query/expected_1.diff")));
160
+
161
+ // insert more rows
162
+ execute(readResource("query/insert_more.sql"));
163
+
164
+ Path out2 = embulk.createTempFile("csv");
165
+ RunResult result2 = embulk.runInput(
166
+ baseConfig.merge(loadYamlResource(embulk, "query/config_2.yml")),
167
+ out2);
168
+ assertThat(
169
+ readSortedFile(out2),
170
+ is(readResource("query/expected_2.csv")));
171
+ assertThat(
172
+ result2.getConfigDiff(),
173
+ is((ConfigDiff) loadYamlResource(embulk, "query/expected_2.diff")));
174
+ }
175
+
176
+ @Test
177
+ public void simpleQueryWithPlaceholderAndMultiColumns() throws Exception
178
+ {
179
+ // setup first rows
180
+ execute(readResource("query/setup.sql"));
181
+
182
+ Path out1 = embulk.createTempFile("csv");
183
+ RunResult result1 = embulk.runInput(
184
+ baseConfig.merge(loadYamlResource(embulk, "query/multi_columns_config_1.yml")),
185
+ out1);
186
+ assertThat(
187
+ readSortedFile(out1),
188
+ is(readResource("query/multi_columns_expected_1.csv")));
189
+ assertThat(
190
+ result1.getConfigDiff(),
191
+ is((ConfigDiff) loadYamlResource(embulk, "query/multi_columns_expected_1.diff")));
192
+
193
+ // insert more rows
194
+ execute(readResource("query/insert_more.sql"));
195
+
196
+ Path out2 = embulk.createTempFile("csv");
197
+ RunResult result2 = embulk.runInput(
198
+ baseConfig.merge(loadYamlResource(embulk, "query/multi_columns_config_2.yml")),
199
+ out2);
200
+ assertThat(
201
+ readSortedFile(out2),
202
+ is(readResource("query/multi_columns_expected_2.csv")));
203
+ assertThat(
204
+ result2.getConfigDiff(),
205
+ is((ConfigDiff) loadYamlResource(embulk, "query/multi_columns_expected_2.diff")));
206
+ }
143
207
  }
@@ -0,0 +1,14 @@
1
+ last_record: [0]
2
+ incremental: true
3
+ incremental_columns: [num]
4
+ use_raw_query_with_incremental: true
5
+ query: |
6
+ SELECT
7
+ num,note
8
+ FROM
9
+ query_load
10
+ WHERE
11
+ num IS NOT NULL
12
+ AND num > :num
13
+ ORDER BY
14
+ num ASC
@@ -0,0 +1,14 @@
1
+ last_record: [4]
2
+ incremental: true
3
+ incremental_columns: [num]
4
+ use_raw_query_with_incremental: true
5
+ query: |
6
+ SELECT
7
+ num,note
8
+ FROM
9
+ query_load
10
+ WHERE
11
+ num IS NOT NULL
12
+ AND num > :num
13
+ ORDER BY
14
+ num ASC
@@ -0,0 +1,6 @@
1
+
2
+ insert into query_load (num, num2, note) values
3
+ (0, 100, 'more_skip'),
4
+ (4, 104, 'more_skip'),
5
+ (9, 109, 'more_load'),
6
+ (5, 105, 'more_load');
@@ -0,0 +1,16 @@
1
+ last_record: [0,0]
2
+ incremental: true
3
+ incremental_columns: [num,num2]
4
+ use_raw_query_with_incremental: true
5
+ query: |
6
+ SELECT
7
+ *
8
+ FROM
9
+ query_load
10
+ WHERE
11
+ num IS NOT NULL
12
+ AND num > :num
13
+ OR (num = :num AND num2 > :num2)
14
+ ORDER BY
15
+ num ASC,
16
+ num2 ASC
@@ -0,0 +1,16 @@
1
+ last_record: [4,104]
2
+ incremental: true
3
+ incremental_columns: [num,num2]
4
+ use_raw_query_with_incremental: true
5
+ query: |
6
+ SELECT
7
+ *
8
+ FROM
9
+ query_load
10
+ WHERE
11
+ num IS NOT NULL
12
+ AND num > :num
13
+ OR (num = :num AND num2 > :num2)
14
+ ORDER BY
15
+ num ASC,
16
+ num2 ASC
@@ -0,0 +1,13 @@
1
+ drop table if exists query_load;
2
+
3
+ create table query_load (
4
+ num int not null,
5
+ num2 int not null,
6
+ note text
7
+ );
8
+
9
+ insert into query_load (num, num2, note) values
10
+ (3, 103, 'first'),
11
+ (4, 104, 'first'),
12
+ (2, 102, 'first'),
13
+ (1, 101, 'first');
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: embulk-input-postgresql
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.9.2
4
+ version: 0.9.3
5
5
  platform: ruby
6
6
  authors:
7
7
  - Sadayuki Furuhashi
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2018-07-03 00:00:00.000000000 Z
11
+ date: 2018-08-10 00:00:00.000000000 Z
12
12
  dependencies: []
13
13
  description: Selects records from a table.
14
14
  email:
@@ -19,8 +19,8 @@ extra_rdoc_files: []
19
19
  files:
20
20
  - README.md
21
21
  - build.gradle
22
- - classpath/embulk-input-jdbc-0.9.2.jar
23
- - classpath/embulk-input-postgresql-0.9.2.jar
22
+ - classpath/embulk-input-jdbc-0.9.3.jar
23
+ - classpath/embulk-input-postgresql-0.9.3.jar
24
24
  - default_jdbc_driver/postgresql-9.4-1205-jdbc41.jar
25
25
  - lib/embulk/input/postgresql.rb
26
26
  - src/main/java/org/embulk/input/PostgreSQLInputPlugin.java
@@ -51,6 +51,20 @@ files:
51
51
  - src/test/resources/org/embulk/input/postgresql/test/expect/incremental/int/expected_2.diff
52
52
  - src/test/resources/org/embulk/input/postgresql/test/expect/incremental/int/insert_more.sql
53
53
  - src/test/resources/org/embulk/input/postgresql/test/expect/incremental/int/setup.sql
54
+ - src/test/resources/org/embulk/input/postgresql/test/expect/incremental/query/config_1.yml
55
+ - src/test/resources/org/embulk/input/postgresql/test/expect/incremental/query/config_2.yml
56
+ - src/test/resources/org/embulk/input/postgresql/test/expect/incremental/query/expected_1.csv
57
+ - src/test/resources/org/embulk/input/postgresql/test/expect/incremental/query/expected_1.diff
58
+ - src/test/resources/org/embulk/input/postgresql/test/expect/incremental/query/expected_2.csv
59
+ - src/test/resources/org/embulk/input/postgresql/test/expect/incremental/query/expected_2.diff
60
+ - src/test/resources/org/embulk/input/postgresql/test/expect/incremental/query/insert_more.sql
61
+ - src/test/resources/org/embulk/input/postgresql/test/expect/incremental/query/multi_columns_config_1.yml
62
+ - src/test/resources/org/embulk/input/postgresql/test/expect/incremental/query/multi_columns_config_2.yml
63
+ - src/test/resources/org/embulk/input/postgresql/test/expect/incremental/query/multi_columns_expected_1.csv
64
+ - src/test/resources/org/embulk/input/postgresql/test/expect/incremental/query/multi_columns_expected_1.diff
65
+ - src/test/resources/org/embulk/input/postgresql/test/expect/incremental/query/multi_columns_expected_2.csv
66
+ - src/test/resources/org/embulk/input/postgresql/test/expect/incremental/query/multi_columns_expected_2.diff
67
+ - src/test/resources/org/embulk/input/postgresql/test/expect/incremental/query/setup.sql
54
68
  - src/test/resources/org/embulk/input/postgresql/test/expect/incremental/timestamp/config_1.yml
55
69
  - src/test/resources/org/embulk/input/postgresql/test/expect/incremental/timestamp/config_2.yml
56
70
  - src/test/resources/org/embulk/input/postgresql/test/expect/incremental/timestamp/expected_1.csv