embulk-output-kintone 0.2.2 → 0.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/ci.yml +52 -5
- data/README.md +3 -2
- data/build.gradle +1 -1
- data/classpath/{embulk-output-kintone-0.2.2.jar → embulk-output-kintone-0.3.2.jar} +0 -0
- data/src/main/java/org/embulk/output/kintone/KintoneColumnOption.java +4 -8
- data/src/main/java/org/embulk/output/kintone/KintoneColumnVisitor.java +30 -15
- data/src/main/java/org/embulk/output/kintone/KintoneOutputPlugin.java +6 -21
- data/src/main/java/org/embulk/output/kintone/KintonePageOutput.java +111 -156
- data/src/main/java/org/embulk/output/kintone/PluginTask.java +7 -3
- metadata +3 -3
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 12557e033d23e679cb3b6c54b74187083a36d9c4
|
|
4
|
+
data.tar.gz: 11bdecefdebb13d075410e56ed9ba3e42bdb51b7
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 9000840a27d24ad547435f6be018036c8958a522373d5bec83505ab14b05a26050c415e638b9c583fd19b400fb9b3a00599307805c4e7ad3a4134336ef57290a
|
|
7
|
+
data.tar.gz: 6e750bff291b90df9f7c90225b1f324d9c27289c3e487d856a607a6e5605dcb612c9fdecb33809e9c46970bb3686b2ac0c389b9a6c1bba8b68d5a6b9230507fd
|
data/.github/workflows/ci.yml
CHANGED
|
@@ -1,13 +1,60 @@
|
|
|
1
|
-
name:
|
|
1
|
+
name: main
|
|
2
2
|
|
|
3
|
-
on:
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches:
|
|
6
|
+
- 'master'
|
|
7
|
+
tags:
|
|
8
|
+
- '*'
|
|
9
|
+
pull_request:
|
|
10
|
+
branches:
|
|
11
|
+
- 'master'
|
|
12
|
+
types: [opened, synchronize]
|
|
4
13
|
|
|
5
14
|
jobs:
|
|
6
|
-
|
|
15
|
+
main:
|
|
7
16
|
runs-on: ubuntu-latest
|
|
8
17
|
steps:
|
|
9
18
|
- uses: actions/checkout@v2
|
|
10
|
-
-
|
|
19
|
+
- name: Set Up
|
|
20
|
+
uses: actions/setup-java@v1
|
|
11
21
|
with:
|
|
12
22
|
java-version: 1.8
|
|
13
|
-
|
|
23
|
+
|
|
24
|
+
- name: Test
|
|
25
|
+
run: ./gradlew test
|
|
26
|
+
|
|
27
|
+
- name: Build Gem
|
|
28
|
+
run: ./gradlew gem
|
|
29
|
+
|
|
30
|
+
- name: Set up JRuby
|
|
31
|
+
uses: ruby/setup-ruby@v1
|
|
32
|
+
with:
|
|
33
|
+
ruby-version: jruby
|
|
34
|
+
|
|
35
|
+
- name: Publish to GPR
|
|
36
|
+
if: startsWith( github.ref, 'refs/tags/' )
|
|
37
|
+
run: |
|
|
38
|
+
mkdir -p $HOME/.gem
|
|
39
|
+
touch $HOME/.gem/credentials
|
|
40
|
+
chmod 0600 $HOME/.gem/credentials
|
|
41
|
+
printf -- "---\n:github: ${GEM_HOST_API_KEY}\n" > $HOME/.gem/credentials
|
|
42
|
+
gem push --KEY github --host https://rubygems.pkg.github.com/${OWNER} ./pkg/*.gem
|
|
43
|
+
env:
|
|
44
|
+
GEM_HOST_API_KEY: "Bearer ${{secrets.GITHUB_TOKEN}}"
|
|
45
|
+
OWNER: ${{ github.repository_owner }}
|
|
46
|
+
|
|
47
|
+
- name: Publish to RubyGems
|
|
48
|
+
if: startsWith( github.ref, 'refs/tags/' )
|
|
49
|
+
run: |
|
|
50
|
+
mkdir -p $HOME/.gem
|
|
51
|
+
touch $HOME/.gem/credentials
|
|
52
|
+
chmod 0600 $HOME/.gem/credentials
|
|
53
|
+
printf -- "---\n:rubygems_api_key: ${GEM_HOST_API_KEY}\n" > $HOME/.gem/credentials
|
|
54
|
+
# TODO: If it is possible to accept input in the middle of a step, then the OTP Token should be inputted instead of generated.
|
|
55
|
+
gem install rotp -v 6.2.0
|
|
56
|
+
OTP_TOKEN=$(echo ${OTP_SECRET} | ruby -rtime -rrotp -e "puts ROTP::TOTP.new(STDIN.read.chomp, issuer: 'rubygems.org').at(Time.now)")
|
|
57
|
+
gem push --otp="${OTP_TOKEN}" ./pkg/*.gem
|
|
58
|
+
env:
|
|
59
|
+
GEM_HOST_API_KEY: "${{secrets.RUBYGEMS_API_KEY}}"
|
|
60
|
+
OTP_SECRET: "${{secrets.RUBYGEMS_OTP_SECRET}}"
|
data/README.md
CHANGED
|
@@ -17,11 +17,11 @@ kintone output plugin for Embulk stores app records from kintone.
|
|
|
17
17
|
- **basic_auth_password**: kintone basic auth password (string, optional)
|
|
18
18
|
- **guest_space_id**: kintone app belongs to guest space, guest space id is required. (integer, optional)
|
|
19
19
|
- **mode**: kintone mode (string, required)
|
|
20
|
+
- **update_key**: column name to set update key (string, required if mode is update or upsert)
|
|
20
21
|
- **column_options** advanced: a key-value pairs where key is a column name and value is options for the column.
|
|
21
22
|
- **field_code**: field code (string, required)
|
|
22
23
|
- **type**: field type (string, required)
|
|
23
24
|
- **timezone**: timezone to convert into `date` (string, default is `UTC`)
|
|
24
|
-
- **update_key**: update key (boolean, default is `false`)
|
|
25
25
|
- **val_sep**: Used to specify multiple checkbox values (string, default is `,`)
|
|
26
26
|
|
|
27
27
|
## Example
|
|
@@ -33,7 +33,8 @@ out:
|
|
|
33
33
|
username: username
|
|
34
34
|
password: password
|
|
35
35
|
app_id: 1
|
|
36
|
-
mode:
|
|
36
|
+
mode: upsert
|
|
37
|
+
update_key: id
|
|
37
38
|
column_options:
|
|
38
39
|
id: {field_code: "id", type: "NUMBER"}
|
|
39
40
|
name: {field_code: "name", type: "SINGLE_LINE_TEXT"}
|
data/build.gradle
CHANGED
|
Binary file
|
|
@@ -10,20 +10,16 @@ public interface KintoneColumnOption
|
|
|
10
10
|
extends Task
|
|
11
11
|
{
|
|
12
12
|
@Config("type")
|
|
13
|
-
|
|
13
|
+
String getType();
|
|
14
14
|
|
|
15
15
|
@Config("field_code")
|
|
16
|
-
|
|
16
|
+
String getFieldCode();
|
|
17
17
|
|
|
18
18
|
@Config("timezone")
|
|
19
19
|
@ConfigDefault("\"UTC\"")
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
@Config("update_key")
|
|
23
|
-
@ConfigDefault("false")
|
|
24
|
-
public boolean getUpdateKey();
|
|
20
|
+
Optional<String> getTimezone();
|
|
25
21
|
|
|
26
22
|
@Config("val_sep")
|
|
27
23
|
@ConfigDefault("\",\"")
|
|
28
|
-
|
|
24
|
+
String getValueSeparator();
|
|
29
25
|
}
|
|
@@ -22,14 +22,16 @@ import java.time.Instant;
|
|
|
22
22
|
import java.time.ZoneId;
|
|
23
23
|
import java.time.ZonedDateTime;
|
|
24
24
|
import java.util.Map;
|
|
25
|
+
import java.util.Objects;
|
|
25
26
|
|
|
26
27
|
public class KintoneColumnVisitor
|
|
27
28
|
implements ColumnVisitor
|
|
28
29
|
{
|
|
29
|
-
private PageReader pageReader;
|
|
30
|
+
private final PageReader pageReader;
|
|
30
31
|
private Record record;
|
|
31
32
|
private UpdateKey updateKey;
|
|
32
|
-
private Map<String, KintoneColumnOption> columnOptions;
|
|
33
|
+
private final Map<String, KintoneColumnOption> columnOptions;
|
|
34
|
+
private String updateKeyName;
|
|
33
35
|
|
|
34
36
|
public KintoneColumnVisitor(PageReader pageReader,
|
|
35
37
|
Map<String, KintoneColumnOption> columnOptions)
|
|
@@ -38,6 +40,15 @@ public class KintoneColumnVisitor
|
|
|
38
40
|
this.columnOptions = columnOptions;
|
|
39
41
|
}
|
|
40
42
|
|
|
43
|
+
public KintoneColumnVisitor(PageReader pageReader,
|
|
44
|
+
Map<String, KintoneColumnOption> columnOptions,
|
|
45
|
+
String updateKeyName)
|
|
46
|
+
{
|
|
47
|
+
this.pageReader = pageReader;
|
|
48
|
+
this.columnOptions = columnOptions;
|
|
49
|
+
this.updateKeyName = updateKeyName;
|
|
50
|
+
}
|
|
51
|
+
|
|
41
52
|
public void setRecord(Record record)
|
|
42
53
|
{
|
|
43
54
|
this.record = record;
|
|
@@ -50,20 +61,17 @@ public class KintoneColumnVisitor
|
|
|
50
61
|
|
|
51
62
|
private void setValue(String fieldCode, Object value, FieldType type, boolean isUpdateKey)
|
|
52
63
|
{
|
|
53
|
-
if (value == null) {
|
|
54
|
-
return;
|
|
55
|
-
}
|
|
56
|
-
|
|
57
64
|
if (isUpdateKey && updateKey != null) {
|
|
58
65
|
updateKey
|
|
59
66
|
.setField(fieldCode)
|
|
60
|
-
.setValue(
|
|
67
|
+
.setValue(Objects.toString(value, ""));
|
|
61
68
|
}
|
|
62
|
-
String stringValue =
|
|
63
|
-
FieldValue fieldValue
|
|
69
|
+
String stringValue = Objects.toString(value, "");
|
|
70
|
+
FieldValue fieldValue;
|
|
64
71
|
switch (type) {
|
|
65
72
|
case NUMBER:
|
|
66
|
-
|
|
73
|
+
BigDecimal setValue = stringValue.equals("") ? null : new BigDecimal(stringValue);
|
|
74
|
+
fieldValue = new NumberFieldValue(setValue);
|
|
67
75
|
break;
|
|
68
76
|
case MULTI_LINE_TEXT:
|
|
69
77
|
fieldValue = new MultiLineTextFieldValue(stringValue);
|
|
@@ -128,16 +136,19 @@ public class KintoneColumnVisitor
|
|
|
128
136
|
private ZoneId getZoneId(Column column)
|
|
129
137
|
{
|
|
130
138
|
KintoneColumnOption option = columnOptions.get(column.getName());
|
|
131
|
-
|
|
139
|
+
if (option == null) {
|
|
140
|
+
return ZoneId.of("UTC");
|
|
141
|
+
}
|
|
142
|
+
return ZoneId.of(option.getTimezone().orElse("UTC"));
|
|
132
143
|
}
|
|
133
144
|
|
|
134
145
|
private boolean isUpdateKey(Column column)
|
|
135
146
|
{
|
|
136
|
-
|
|
137
|
-
if (option == null) {
|
|
147
|
+
if (this.updateKeyName == null) {
|
|
138
148
|
return false;
|
|
139
149
|
}
|
|
140
|
-
|
|
150
|
+
|
|
151
|
+
return this.updateKeyName.equals(column.getName());
|
|
141
152
|
}
|
|
142
153
|
|
|
143
154
|
private String getValueSeparator(Column column)
|
|
@@ -162,7 +173,11 @@ public class KintoneColumnVisitor
|
|
|
162
173
|
{
|
|
163
174
|
String fieldCode = getFieldCode(column);
|
|
164
175
|
FieldType type = getType(column, FieldType.NUMBER);
|
|
165
|
-
|
|
176
|
+
if (pageReader.isNull(column)) {
|
|
177
|
+
setValue(fieldCode, null, type, isUpdateKey(column));
|
|
178
|
+
} else {
|
|
179
|
+
setValue(fieldCode, pageReader.getLong(column), type, isUpdateKey(column));
|
|
180
|
+
}
|
|
166
181
|
}
|
|
167
182
|
|
|
168
183
|
@Override
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
package org.embulk.output.kintone;
|
|
2
2
|
|
|
3
3
|
import org.embulk.config.ConfigDiff;
|
|
4
|
+
import org.embulk.config.ConfigException;
|
|
4
5
|
import org.embulk.config.ConfigSource;
|
|
5
6
|
import org.embulk.config.TaskReport;
|
|
6
7
|
import org.embulk.config.TaskSource;
|
|
@@ -9,7 +10,6 @@ import org.embulk.spi.OutputPlugin;
|
|
|
9
10
|
import org.embulk.spi.Schema;
|
|
10
11
|
import org.embulk.spi.TransactionalPageOutput;
|
|
11
12
|
|
|
12
|
-
import java.util.Collection;
|
|
13
13
|
import java.util.List;
|
|
14
14
|
|
|
15
15
|
public class KintoneOutputPlugin
|
|
@@ -45,37 +45,22 @@ public class KintoneOutputPlugin
|
|
|
45
45
|
public TransactionalPageOutput open(TaskSource taskSource, Schema schema, int taskIndex)
|
|
46
46
|
{
|
|
47
47
|
PluginTask task = taskSource.loadTask(PluginTask.class);
|
|
48
|
-
Collection<KintoneColumnOption> options = task.getColumnOptions().values();
|
|
49
48
|
|
|
50
49
|
KintoneMode mode = KintoneMode.getKintoneModeByValue(task.getMode());
|
|
51
50
|
switch (mode) {
|
|
52
51
|
case INSERT:
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
throw new IllegalArgumentException(
|
|
56
|
-
"when mode is insert, require no update_key.");
|
|
57
|
-
}
|
|
52
|
+
if (task.getUpdateKeyName().isPresent()) {
|
|
53
|
+
throw new ConfigException("when mode is insert, require no update_key.");
|
|
58
54
|
}
|
|
59
55
|
break;
|
|
60
56
|
case UPDATE:
|
|
61
57
|
case UPSERT:
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
if (option.getUpdateKey()) {
|
|
65
|
-
if (hasUpdateKey) {
|
|
66
|
-
throw new IllegalArgumentException(
|
|
67
|
-
"when mode is update and upsert, only one column can have an update_key.");
|
|
68
|
-
}
|
|
69
|
-
hasUpdateKey = true;
|
|
70
|
-
}
|
|
71
|
-
}
|
|
72
|
-
if (!hasUpdateKey) {
|
|
73
|
-
throw new IllegalArgumentException(
|
|
74
|
-
"when mode is update and upsert, require update_key.");
|
|
58
|
+
if (!task.getUpdateKeyName().isPresent()) {
|
|
59
|
+
throw new ConfigException("when mode is update or upsert, require update_key.");
|
|
75
60
|
}
|
|
76
61
|
break;
|
|
77
62
|
default:
|
|
78
|
-
throw new
|
|
63
|
+
throw new ConfigException(String.format(
|
|
79
64
|
"Unknown mode '%s'",
|
|
80
65
|
task.getMode()));
|
|
81
66
|
}
|
|
@@ -3,6 +3,7 @@ package org.embulk.output.kintone;
|
|
|
3
3
|
import com.kintone.client.KintoneClient;
|
|
4
4
|
import com.kintone.client.KintoneClientBuilder;
|
|
5
5
|
import com.kintone.client.api.record.GetRecordsByCursorResponseBody;
|
|
6
|
+
import com.kintone.client.model.record.FieldType;
|
|
6
7
|
import com.kintone.client.model.record.Record;
|
|
7
8
|
import com.kintone.client.model.record.RecordForUpdate;
|
|
8
9
|
import com.kintone.client.model.record.UpdateKey;
|
|
@@ -14,17 +15,18 @@ import org.embulk.spi.PageReader;
|
|
|
14
15
|
import org.embulk.spi.Schema;
|
|
15
16
|
import org.embulk.spi.TransactionalPageOutput;
|
|
16
17
|
|
|
17
|
-
import java.math.BigDecimal;
|
|
18
18
|
import java.util.ArrayList;
|
|
19
|
-
import java.util.
|
|
19
|
+
import java.util.Collections;
|
|
20
20
|
import java.util.List;
|
|
21
21
|
import java.util.stream.Collectors;
|
|
22
22
|
|
|
23
23
|
public class KintonePageOutput
|
|
24
24
|
implements TransactionalPageOutput
|
|
25
25
|
{
|
|
26
|
-
|
|
27
|
-
|
|
26
|
+
public static final int UPSERT_BATCH_SIZE = 10000;
|
|
27
|
+
public static final int CHUNK_SIZE = 100;
|
|
28
|
+
private final PageReader pageReader;
|
|
29
|
+
private final PluginTask task;
|
|
28
30
|
private KintoneClient client;
|
|
29
31
|
|
|
30
32
|
public KintonePageOutput(PluginTask task, Schema schema)
|
|
@@ -88,14 +90,14 @@ public class KintonePageOutput
|
|
|
88
90
|
|
|
89
91
|
public interface Consumer<T>
|
|
90
92
|
{
|
|
91
|
-
|
|
93
|
+
void accept(T t);
|
|
92
94
|
}
|
|
93
95
|
|
|
94
96
|
public void connect(final PluginTask task)
|
|
95
97
|
{
|
|
96
98
|
KintoneClientBuilder builder = KintoneClientBuilder.create("https://" + task.getDomain());
|
|
97
99
|
if (task.getGuestSpaceId().isPresent()) {
|
|
98
|
-
builder.setGuestSpaceId(task.getGuestSpaceId().
|
|
100
|
+
builder.setGuestSpaceId(task.getGuestSpaceId().orElse(-1));
|
|
99
101
|
}
|
|
100
102
|
if (task.getBasicAuthUsername().isPresent() && task.getBasicAuthPassword().isPresent()) {
|
|
101
103
|
builder.withBasicAuth(task.getBasicAuthUsername().get(),
|
|
@@ -136,7 +138,7 @@ public class KintonePageOutput
|
|
|
136
138
|
}
|
|
137
139
|
|
|
138
140
|
records.add(record);
|
|
139
|
-
if (records.size() ==
|
|
141
|
+
if (records.size() == CHUNK_SIZE) {
|
|
140
142
|
client.record().addRecords(task.getAppId(), records);
|
|
141
143
|
records.clear();
|
|
142
144
|
}
|
|
@@ -155,10 +157,12 @@ public class KintonePageOutput
|
|
|
155
157
|
{
|
|
156
158
|
execute(client -> {
|
|
157
159
|
try {
|
|
158
|
-
ArrayList<RecordForUpdate> updateRecords = new ArrayList
|
|
160
|
+
ArrayList<RecordForUpdate> updateRecords = new ArrayList<>();
|
|
159
161
|
pageReader.setPage(page);
|
|
162
|
+
|
|
160
163
|
KintoneColumnVisitor visitor = new KintoneColumnVisitor(pageReader,
|
|
161
|
-
task.getColumnOptions()
|
|
164
|
+
task.getColumnOptions(),
|
|
165
|
+
task.getUpdateKeyName().orElseThrow(() -> new RuntimeException("unreachable"))); // Already validated
|
|
162
166
|
while (pageReader.nextRecord()) {
|
|
163
167
|
Record record = new Record();
|
|
164
168
|
UpdateKey updateKey = new UpdateKey();
|
|
@@ -168,9 +172,13 @@ public class KintonePageOutput
|
|
|
168
172
|
column.visit(visitor);
|
|
169
173
|
}
|
|
170
174
|
|
|
175
|
+
if (updateKey.getValue() == "") {
|
|
176
|
+
continue;
|
|
177
|
+
}
|
|
178
|
+
|
|
171
179
|
record.removeField(updateKey.getField());
|
|
172
180
|
updateRecords.add(new RecordForUpdate(updateKey, record));
|
|
173
|
-
if (updateRecords.size() ==
|
|
181
|
+
if (updateRecords.size() == CHUNK_SIZE) {
|
|
174
182
|
client.record().updateRecords(task.getAppId(), updateRecords);
|
|
175
183
|
updateRecords.clear();
|
|
176
184
|
}
|
|
@@ -185,183 +193,130 @@ public class KintonePageOutput
|
|
|
185
193
|
});
|
|
186
194
|
}
|
|
187
195
|
|
|
188
|
-
private
|
|
196
|
+
private void upsertPage(final Page page)
|
|
189
197
|
{
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
fieldCode + " in (" + String.join(",", queryValues) + ")"
|
|
196
|
-
);
|
|
197
|
-
while (true) {
|
|
198
|
-
GetRecordsByCursorResponseBody resp = client.record().getRecordsByCursor(cursorId);
|
|
199
|
-
List<Record> records = resp.getRecords();
|
|
200
|
-
allRecords.addAll(records);
|
|
198
|
+
execute(client -> {
|
|
199
|
+
try {
|
|
200
|
+
ArrayList<Record> records = new ArrayList<>();
|
|
201
|
+
ArrayList<UpdateKey> updateKeys = new ArrayList<>();
|
|
202
|
+
pageReader.setPage(page);
|
|
201
203
|
|
|
202
|
-
|
|
203
|
-
|
|
204
|
+
KintoneColumnVisitor visitor = new KintoneColumnVisitor(pageReader,
|
|
205
|
+
task.getColumnOptions(),
|
|
206
|
+
task.getUpdateKeyName().orElseThrow(() -> new RuntimeException("unreachable"))); // Already validated
|
|
207
|
+
while (pageReader.nextRecord()) {
|
|
208
|
+
Record record = new Record();
|
|
209
|
+
UpdateKey updateKey = new UpdateKey();
|
|
210
|
+
visitor.setRecord(record);
|
|
211
|
+
visitor.setUpdateKey(updateKey);
|
|
212
|
+
for (Column column : pageReader.getSchema().getColumns()) {
|
|
213
|
+
column.visit(visitor);
|
|
214
|
+
}
|
|
215
|
+
records.add(record);
|
|
216
|
+
updateKeys.add(updateKey);
|
|
217
|
+
|
|
218
|
+
if (records.size() == UPSERT_BATCH_SIZE) {
|
|
219
|
+
upsert(records, updateKeys);
|
|
220
|
+
records.clear();
|
|
221
|
+
updateKeys.clear();
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
if (records.size() > 0) {
|
|
225
|
+
upsert(records, updateKeys);
|
|
226
|
+
}
|
|
204
227
|
}
|
|
205
|
-
|
|
206
|
-
|
|
228
|
+
catch (Exception e) {
|
|
229
|
+
throw new RuntimeException("kintone throw exception", e);
|
|
230
|
+
}
|
|
231
|
+
});
|
|
207
232
|
}
|
|
208
233
|
|
|
209
|
-
|
|
234
|
+
private void upsert(ArrayList<Record> records, ArrayList<UpdateKey> updateKeys)
|
|
210
235
|
{
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
public void upsert(ArrayList<Record> records, ArrayList<UpdateKey> updateKeys)
|
|
215
|
-
{
|
|
216
|
-
if (records.size() != updateKeys.size()) {
|
|
217
|
-
throw new RuntimeException("records.size() != updateKeys.size()");
|
|
218
|
-
}
|
|
219
|
-
|
|
220
|
-
List<String> queryValues = updateKeys
|
|
221
|
-
.stream()
|
|
222
|
-
.map(k -> "\"" + k.getValue().toString() + "\"")
|
|
223
|
-
.collect(Collectors.toList());
|
|
224
|
-
List<T> updateKeyValues = getUpdateKeyValues(queryValues);
|
|
236
|
+
if (records.size() != updateKeys.size()) {
|
|
237
|
+
throw new RuntimeException("records.size() != updateKeys.size()");
|
|
238
|
+
}
|
|
225
239
|
|
|
226
|
-
|
|
227
|
-
ArrayList<RecordForUpdate> updateRecords = new ArrayList<RecordForUpdate>();
|
|
228
|
-
for (int i = 0; i < records.size(); i++) {
|
|
229
|
-
Record record = records.get(i);
|
|
230
|
-
UpdateKey updateKey = updateKeys.get(i);
|
|
240
|
+
List<Record> existingRecords = getExistingRecordsByUpdateKey(updateKeys);
|
|
231
241
|
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
insertRecords.add(record);
|
|
238
|
-
}
|
|
242
|
+
ArrayList<Record> insertRecords = new ArrayList<>();
|
|
243
|
+
ArrayList<RecordForUpdate> updateRecords = new ArrayList<>();
|
|
244
|
+
for (int i = 0; i < records.size(); i++) {
|
|
245
|
+
Record record = records.get(i);
|
|
246
|
+
UpdateKey updateKey = updateKeys.get(i);
|
|
239
247
|
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
}
|
|
244
|
-
else if (updateRecords.size() == 100) {
|
|
245
|
-
client.record().updateRecords(task.getAppId(), updateRecords);
|
|
246
|
-
updateRecords.clear();
|
|
247
|
-
}
|
|
248
|
+
if (existsRecord(existingRecords, updateKey)) {
|
|
249
|
+
record.removeField(updateKey.getField());
|
|
250
|
+
updateRecords.add(new RecordForUpdate(updateKey, record));
|
|
248
251
|
}
|
|
249
|
-
|
|
252
|
+
else {
|
|
253
|
+
insertRecords.add(record);
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
if (insertRecords.size() == CHUNK_SIZE) {
|
|
250
257
|
client.record().addRecords(task.getAppId(), insertRecords);
|
|
258
|
+
insertRecords.clear();
|
|
251
259
|
}
|
|
252
|
-
if (updateRecords.size()
|
|
260
|
+
else if (updateRecords.size() == CHUNK_SIZE) {
|
|
253
261
|
client.record().updateRecords(task.getAppId(), updateRecords);
|
|
262
|
+
updateRecords.clear();
|
|
254
263
|
}
|
|
255
264
|
}
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
{
|
|
259
|
-
execute(client -> {
|
|
260
|
-
try {
|
|
261
|
-
ArrayList<Record> records = new ArrayList<>();
|
|
262
|
-
ArrayList<UpdateKey> updateKeys = new ArrayList<>();
|
|
263
|
-
|
|
264
|
-
pageReader.setPage(page);
|
|
265
|
-
KintoneColumnVisitor visitor = new KintoneColumnVisitor(pageReader,
|
|
266
|
-
task.getColumnOptions());
|
|
267
|
-
while (pageReader.nextRecord()) {
|
|
268
|
-
Record record = new Record();
|
|
269
|
-
UpdateKey updateKey = new UpdateKey();
|
|
270
|
-
visitor.setRecord(record);
|
|
271
|
-
visitor.setUpdateKey(updateKey);
|
|
272
|
-
for (Column column : pageReader.getSchema().getColumns()) {
|
|
273
|
-
column.visit(visitor);
|
|
274
|
-
}
|
|
275
|
-
records.add(record);
|
|
276
|
-
updateKeys.add(updateKey);
|
|
277
|
-
|
|
278
|
-
if (records.size() == 10000) {
|
|
279
|
-
upsert(records, updateKeys);
|
|
280
|
-
records.clear();
|
|
281
|
-
updateKeys.clear();
|
|
282
|
-
}
|
|
283
|
-
}
|
|
284
|
-
if (records.size() > 0) {
|
|
285
|
-
upsert(records, updateKeys);
|
|
286
|
-
}
|
|
287
|
-
}
|
|
288
|
-
catch (Exception e) {
|
|
289
|
-
throw new RuntimeException("kintone throw exception", e);
|
|
290
|
-
}
|
|
291
|
-
});
|
|
292
|
-
}
|
|
293
|
-
}
|
|
294
|
-
|
|
295
|
-
class UpsertPageByStringKey extends UpsertPage<String>
|
|
296
|
-
{
|
|
297
|
-
private String fieldCode;
|
|
298
|
-
|
|
299
|
-
public UpsertPageByStringKey(String fieldCode)
|
|
300
|
-
{
|
|
301
|
-
this.fieldCode = fieldCode;
|
|
302
|
-
}
|
|
303
|
-
|
|
304
|
-
public List<String> getUpdateKeyValues(List<String> queryValues)
|
|
305
|
-
{
|
|
306
|
-
return getRecordsByUpdateKey(fieldCode, queryValues)
|
|
307
|
-
.stream()
|
|
308
|
-
.map(r -> r.getSingleLineTextFieldValue(fieldCode))
|
|
309
|
-
.collect(Collectors.toList());
|
|
265
|
+
if (insertRecords.size() > 0) {
|
|
266
|
+
client.record().addRecords(task.getAppId(), insertRecords);
|
|
310
267
|
}
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
{
|
|
314
|
-
return updateKeyValues.contains(record.getSingleLineTextFieldValue(fieldCode));
|
|
268
|
+
if (updateRecords.size() > 0) {
|
|
269
|
+
client.record().updateRecords(task.getAppId(), updateRecords);
|
|
315
270
|
}
|
|
316
271
|
}
|
|
317
272
|
|
|
318
|
-
class UpsertPageByNumberKey extends UpsertPage<BigDecimal>
|
|
319
|
-
{
|
|
320
|
-
private String fieldCode;
|
|
321
|
-
|
|
322
|
-
public UpsertPageByNumberKey(String fieldCode)
|
|
323
|
-
{
|
|
324
|
-
this.fieldCode = fieldCode;
|
|
325
|
-
}
|
|
326
273
|
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
274
|
+
private List<Record> getExistingRecordsByUpdateKey(ArrayList<UpdateKey> updateKeys)
|
|
275
|
+
{
|
|
276
|
+
String fieldCode = updateKeys.get(0).getField();
|
|
277
|
+
List<String> queryValues = updateKeys
|
|
330
278
|
.stream()
|
|
331
|
-
.
|
|
279
|
+
.filter(k -> k.getValue() != "")
|
|
280
|
+
.map(k -> "\"" + k.getValue().toString() + "\"")
|
|
332
281
|
.collect(Collectors.toList());
|
|
333
|
-
}
|
|
334
282
|
|
|
335
|
-
|
|
336
|
-
{
|
|
337
|
-
return
|
|
283
|
+
List<Record> allRecords = new ArrayList<>();
|
|
284
|
+
if (queryValues.isEmpty()) {
|
|
285
|
+
return allRecords;
|
|
338
286
|
}
|
|
339
|
-
|
|
287
|
+
String cursorId = client.record().createCursor(
|
|
288
|
+
task.getAppId(),
|
|
289
|
+
Collections.singletonList(fieldCode),
|
|
290
|
+
fieldCode + " in (" + String.join(",", queryValues) + ")"
|
|
291
|
+
);
|
|
292
|
+
while (true) {
|
|
293
|
+
GetRecordsByCursorResponseBody resp = client.record().getRecordsByCursor(cursorId);
|
|
294
|
+
List<Record> records = resp.getRecords();
|
|
295
|
+
allRecords.addAll(records);
|
|
340
296
|
|
|
341
|
-
|
|
342
|
-
{
|
|
343
|
-
KintoneColumnOption updateKeyColumn = null;
|
|
344
|
-
for (KintoneColumnOption v : task.getColumnOptions().values()) {
|
|
345
|
-
if (v.getUpdateKey()) {
|
|
346
|
-
updateKeyColumn = v;
|
|
297
|
+
if (!resp.hasNext()) {
|
|
347
298
|
break;
|
|
348
299
|
}
|
|
349
300
|
}
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
}
|
|
301
|
+
return allRecords;
|
|
302
|
+
}
|
|
353
303
|
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
case
|
|
360
|
-
|
|
361
|
-
|
|
304
|
+
private boolean existsRecord(List<Record> distRecords, UpdateKey updateKey)
|
|
305
|
+
{
|
|
306
|
+
String fieldCode = updateKey.getField();
|
|
307
|
+
FieldType type = client.app().getFormFields(task.getAppId()).get(fieldCode).getType();
|
|
308
|
+
switch (type) {
|
|
309
|
+
case SINGLE_LINE_TEXT:
|
|
310
|
+
return distRecords
|
|
311
|
+
.stream()
|
|
312
|
+
.anyMatch(d -> d.getSingleLineTextFieldValue(fieldCode).equals(updateKey.getValue().toString()));
|
|
313
|
+
case NUMBER:
|
|
314
|
+
return distRecords
|
|
315
|
+
.stream()
|
|
316
|
+
.anyMatch(d -> d.getNumberFieldValue(fieldCode).toPlainString()
|
|
317
|
+
.equals(updateKey.getValue().toString()));
|
|
362
318
|
default:
|
|
363
319
|
throw new RuntimeException("The update_key must be 'SINGLE_LINE_TEXT' or 'NUMBER'.");
|
|
364
320
|
}
|
|
365
|
-
runner.run(page);
|
|
366
321
|
}
|
|
367
322
|
}
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
package org.embulk.output.kintone;
|
|
2
2
|
|
|
3
|
-
import com.google.common.base.Optional;
|
|
4
3
|
import org.embulk.config.Config;
|
|
5
4
|
import org.embulk.config.ConfigDefault;
|
|
6
5
|
import org.embulk.config.Task;
|
|
7
6
|
|
|
8
7
|
import java.util.Map;
|
|
8
|
+
import java.util.Optional;
|
|
9
9
|
|
|
10
10
|
public interface PluginTask
|
|
11
11
|
extends Task
|
|
@@ -42,9 +42,13 @@ public interface PluginTask
|
|
|
42
42
|
|
|
43
43
|
@Config("column_options")
|
|
44
44
|
@ConfigDefault("{}")
|
|
45
|
-
|
|
45
|
+
Map<String, KintoneColumnOption> getColumnOptions();
|
|
46
46
|
|
|
47
47
|
@Config("mode")
|
|
48
48
|
@ConfigDefault("insert")
|
|
49
|
-
|
|
49
|
+
String getMode();
|
|
50
|
+
|
|
51
|
+
@Config("update_key")
|
|
52
|
+
@ConfigDefault("null")
|
|
53
|
+
Optional<String> getUpdateKeyName();
|
|
50
54
|
}
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: embulk-output-kintone
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.3.2
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- takeshi fujita
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date:
|
|
11
|
+
date: 2022-03-23 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -52,7 +52,7 @@ files:
|
|
|
52
52
|
- README.md
|
|
53
53
|
- Rakefile
|
|
54
54
|
- build.gradle
|
|
55
|
-
- classpath/embulk-output-kintone-0.
|
|
55
|
+
- classpath/embulk-output-kintone-0.3.2.jar
|
|
56
56
|
- config/checkstyle/checkstyle.xml
|
|
57
57
|
- config/checkstyle/default.xml
|
|
58
58
|
- gradle/wrapper/gradle-wrapper.jar
|