embulk-output-kintone 0.2.2 → 0.3.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/ci.yml +52 -5
- data/README.md +3 -2
- data/build.gradle +1 -1
- data/classpath/{embulk-output-kintone-0.2.2.jar → embulk-output-kintone-0.3.2.jar} +0 -0
- data/src/main/java/org/embulk/output/kintone/KintoneColumnOption.java +4 -8
- data/src/main/java/org/embulk/output/kintone/KintoneColumnVisitor.java +30 -15
- data/src/main/java/org/embulk/output/kintone/KintoneOutputPlugin.java +6 -21
- data/src/main/java/org/embulk/output/kintone/KintonePageOutput.java +111 -156
- data/src/main/java/org/embulk/output/kintone/PluginTask.java +7 -3
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 12557e033d23e679cb3b6c54b74187083a36d9c4
|
4
|
+
data.tar.gz: 11bdecefdebb13d075410e56ed9ba3e42bdb51b7
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 9000840a27d24ad547435f6be018036c8958a522373d5bec83505ab14b05a26050c415e638b9c583fd19b400fb9b3a00599307805c4e7ad3a4134336ef57290a
|
7
|
+
data.tar.gz: 6e750bff291b90df9f7c90225b1f324d9c27289c3e487d856a607a6e5605dcb612c9fdecb33809e9c46970bb3686b2ac0c389b9a6c1bba8b68d5a6b9230507fd
|
data/.github/workflows/ci.yml
CHANGED
@@ -1,13 +1,60 @@
|
|
1
|
-
name:
|
1
|
+
name: main
|
2
2
|
|
3
|
-
on:
|
3
|
+
on:
|
4
|
+
push:
|
5
|
+
branches:
|
6
|
+
- 'master'
|
7
|
+
tags:
|
8
|
+
- '*'
|
9
|
+
pull_request:
|
10
|
+
branches:
|
11
|
+
- 'master'
|
12
|
+
types: [opened, synchronize]
|
4
13
|
|
5
14
|
jobs:
|
6
|
-
|
15
|
+
main:
|
7
16
|
runs-on: ubuntu-latest
|
8
17
|
steps:
|
9
18
|
- uses: actions/checkout@v2
|
10
|
-
-
|
19
|
+
- name: Set Up
|
20
|
+
uses: actions/setup-java@v1
|
11
21
|
with:
|
12
22
|
java-version: 1.8
|
13
|
-
|
23
|
+
|
24
|
+
- name: Test
|
25
|
+
run: ./gradlew test
|
26
|
+
|
27
|
+
- name: Build Gem
|
28
|
+
run: ./gradlew gem
|
29
|
+
|
30
|
+
- name: Set up JRuby
|
31
|
+
uses: ruby/setup-ruby@v1
|
32
|
+
with:
|
33
|
+
ruby-version: jruby
|
34
|
+
|
35
|
+
- name: Publish to GPR
|
36
|
+
if: startsWith( github.ref, 'refs/tags/' )
|
37
|
+
run: |
|
38
|
+
mkdir -p $HOME/.gem
|
39
|
+
touch $HOME/.gem/credentials
|
40
|
+
chmod 0600 $HOME/.gem/credentials
|
41
|
+
printf -- "---\n:github: ${GEM_HOST_API_KEY}\n" > $HOME/.gem/credentials
|
42
|
+
gem push --KEY github --host https://rubygems.pkg.github.com/${OWNER} ./pkg/*.gem
|
43
|
+
env:
|
44
|
+
GEM_HOST_API_KEY: "Bearer ${{secrets.GITHUB_TOKEN}}"
|
45
|
+
OWNER: ${{ github.repository_owner }}
|
46
|
+
|
47
|
+
- name: Publish to RubyGems
|
48
|
+
if: startsWith( github.ref, 'refs/tags/' )
|
49
|
+
run: |
|
50
|
+
mkdir -p $HOME/.gem
|
51
|
+
touch $HOME/.gem/credentials
|
52
|
+
chmod 0600 $HOME/.gem/credentials
|
53
|
+
printf -- "---\n:rubygems_api_key: ${GEM_HOST_API_KEY}\n" > $HOME/.gem/credentials
|
54
|
+
# TODO: If it is possible to accept input in the middle of a step, then the OTP Token should be inputted instead of generated.
|
55
|
+
gem install rotp -v 6.2.0
|
56
|
+
OTP_TOKEN=$(echo ${OTP_SECRET} | ruby -rtime -rrotp -e "puts ROTP::TOTP.new(STDIN.read.chomp, issuer: 'rubygems.org').at(Time.now)")
|
57
|
+
gem push --otp="${OTP_TOKEN}" ./pkg/*.gem
|
58
|
+
env:
|
59
|
+
GEM_HOST_API_KEY: "${{secrets.RUBYGEMS_API_KEY}}"
|
60
|
+
OTP_SECRET: "${{secrets.RUBYGEMS_OTP_SECRET}}"
|
data/README.md
CHANGED
@@ -17,11 +17,11 @@ kintone output plugin for Embulk stores app records from kintone.
|
|
17
17
|
- **basic_auth_password**: kintone basic auth password (string, optional)
|
18
18
|
- **guest_space_id**: kintone app belongs to guest space, guest space id is required. (integer, optional)
|
19
19
|
- **mode**: kintone mode (string, required)
|
20
|
+
- **update_key**: column name to set update key (string, required if mode is update or upsert)
|
20
21
|
- **column_options** advanced: a key-value pairs where key is a column name and value is options for the column.
|
21
22
|
- **field_code**: field code (string, required)
|
22
23
|
- **type**: field type (string, required)
|
23
24
|
- **timezone**: timezone to convert into `date` (string, default is `UTC`)
|
24
|
-
- **update_key**: update key (boolean, default is `false`)
|
25
25
|
- **val_sep**: Used to specify multiple checkbox values (string, default is `,`)
|
26
26
|
|
27
27
|
## Example
|
@@ -33,7 +33,8 @@ out:
|
|
33
33
|
username: username
|
34
34
|
password: password
|
35
35
|
app_id: 1
|
36
|
-
mode:
|
36
|
+
mode: upsert
|
37
|
+
update_key: id
|
37
38
|
column_options:
|
38
39
|
id: {field_code: "id", type: "NUMBER"}
|
39
40
|
name: {field_code: "name", type: "SINGLE_LINE_TEXT"}
|
data/build.gradle
CHANGED
Binary file
|
@@ -10,20 +10,16 @@ public interface KintoneColumnOption
|
|
10
10
|
extends Task
|
11
11
|
{
|
12
12
|
@Config("type")
|
13
|
-
|
13
|
+
String getType();
|
14
14
|
|
15
15
|
@Config("field_code")
|
16
|
-
|
16
|
+
String getFieldCode();
|
17
17
|
|
18
18
|
@Config("timezone")
|
19
19
|
@ConfigDefault("\"UTC\"")
|
20
|
-
|
21
|
-
|
22
|
-
@Config("update_key")
|
23
|
-
@ConfigDefault("false")
|
24
|
-
public boolean getUpdateKey();
|
20
|
+
Optional<String> getTimezone();
|
25
21
|
|
26
22
|
@Config("val_sep")
|
27
23
|
@ConfigDefault("\",\"")
|
28
|
-
|
24
|
+
String getValueSeparator();
|
29
25
|
}
|
@@ -22,14 +22,16 @@ import java.time.Instant;
|
|
22
22
|
import java.time.ZoneId;
|
23
23
|
import java.time.ZonedDateTime;
|
24
24
|
import java.util.Map;
|
25
|
+
import java.util.Objects;
|
25
26
|
|
26
27
|
public class KintoneColumnVisitor
|
27
28
|
implements ColumnVisitor
|
28
29
|
{
|
29
|
-
private PageReader pageReader;
|
30
|
+
private final PageReader pageReader;
|
30
31
|
private Record record;
|
31
32
|
private UpdateKey updateKey;
|
32
|
-
private Map<String, KintoneColumnOption> columnOptions;
|
33
|
+
private final Map<String, KintoneColumnOption> columnOptions;
|
34
|
+
private String updateKeyName;
|
33
35
|
|
34
36
|
public KintoneColumnVisitor(PageReader pageReader,
|
35
37
|
Map<String, KintoneColumnOption> columnOptions)
|
@@ -38,6 +40,15 @@ public class KintoneColumnVisitor
|
|
38
40
|
this.columnOptions = columnOptions;
|
39
41
|
}
|
40
42
|
|
43
|
+
public KintoneColumnVisitor(PageReader pageReader,
|
44
|
+
Map<String, KintoneColumnOption> columnOptions,
|
45
|
+
String updateKeyName)
|
46
|
+
{
|
47
|
+
this.pageReader = pageReader;
|
48
|
+
this.columnOptions = columnOptions;
|
49
|
+
this.updateKeyName = updateKeyName;
|
50
|
+
}
|
51
|
+
|
41
52
|
public void setRecord(Record record)
|
42
53
|
{
|
43
54
|
this.record = record;
|
@@ -50,20 +61,17 @@ public class KintoneColumnVisitor
|
|
50
61
|
|
51
62
|
private void setValue(String fieldCode, Object value, FieldType type, boolean isUpdateKey)
|
52
63
|
{
|
53
|
-
if (value == null) {
|
54
|
-
return;
|
55
|
-
}
|
56
|
-
|
57
64
|
if (isUpdateKey && updateKey != null) {
|
58
65
|
updateKey
|
59
66
|
.setField(fieldCode)
|
60
|
-
.setValue(
|
67
|
+
.setValue(Objects.toString(value, ""));
|
61
68
|
}
|
62
|
-
String stringValue =
|
63
|
-
FieldValue fieldValue
|
69
|
+
String stringValue = Objects.toString(value, "");
|
70
|
+
FieldValue fieldValue;
|
64
71
|
switch (type) {
|
65
72
|
case NUMBER:
|
66
|
-
|
73
|
+
BigDecimal setValue = stringValue.equals("") ? null : new BigDecimal(stringValue);
|
74
|
+
fieldValue = new NumberFieldValue(setValue);
|
67
75
|
break;
|
68
76
|
case MULTI_LINE_TEXT:
|
69
77
|
fieldValue = new MultiLineTextFieldValue(stringValue);
|
@@ -128,16 +136,19 @@ public class KintoneColumnVisitor
|
|
128
136
|
private ZoneId getZoneId(Column column)
|
129
137
|
{
|
130
138
|
KintoneColumnOption option = columnOptions.get(column.getName());
|
131
|
-
|
139
|
+
if (option == null) {
|
140
|
+
return ZoneId.of("UTC");
|
141
|
+
}
|
142
|
+
return ZoneId.of(option.getTimezone().orElse("UTC"));
|
132
143
|
}
|
133
144
|
|
134
145
|
private boolean isUpdateKey(Column column)
|
135
146
|
{
|
136
|
-
|
137
|
-
if (option == null) {
|
147
|
+
if (this.updateKeyName == null) {
|
138
148
|
return false;
|
139
149
|
}
|
140
|
-
|
150
|
+
|
151
|
+
return this.updateKeyName.equals(column.getName());
|
141
152
|
}
|
142
153
|
|
143
154
|
private String getValueSeparator(Column column)
|
@@ -162,7 +173,11 @@ public class KintoneColumnVisitor
|
|
162
173
|
{
|
163
174
|
String fieldCode = getFieldCode(column);
|
164
175
|
FieldType type = getType(column, FieldType.NUMBER);
|
165
|
-
|
176
|
+
if (pageReader.isNull(column)) {
|
177
|
+
setValue(fieldCode, null, type, isUpdateKey(column));
|
178
|
+
} else {
|
179
|
+
setValue(fieldCode, pageReader.getLong(column), type, isUpdateKey(column));
|
180
|
+
}
|
166
181
|
}
|
167
182
|
|
168
183
|
@Override
|
@@ -1,6 +1,7 @@
|
|
1
1
|
package org.embulk.output.kintone;
|
2
2
|
|
3
3
|
import org.embulk.config.ConfigDiff;
|
4
|
+
import org.embulk.config.ConfigException;
|
4
5
|
import org.embulk.config.ConfigSource;
|
5
6
|
import org.embulk.config.TaskReport;
|
6
7
|
import org.embulk.config.TaskSource;
|
@@ -9,7 +10,6 @@ import org.embulk.spi.OutputPlugin;
|
|
9
10
|
import org.embulk.spi.Schema;
|
10
11
|
import org.embulk.spi.TransactionalPageOutput;
|
11
12
|
|
12
|
-
import java.util.Collection;
|
13
13
|
import java.util.List;
|
14
14
|
|
15
15
|
public class KintoneOutputPlugin
|
@@ -45,37 +45,22 @@ public class KintoneOutputPlugin
|
|
45
45
|
public TransactionalPageOutput open(TaskSource taskSource, Schema schema, int taskIndex)
|
46
46
|
{
|
47
47
|
PluginTask task = taskSource.loadTask(PluginTask.class);
|
48
|
-
Collection<KintoneColumnOption> options = task.getColumnOptions().values();
|
49
48
|
|
50
49
|
KintoneMode mode = KintoneMode.getKintoneModeByValue(task.getMode());
|
51
50
|
switch (mode) {
|
52
51
|
case INSERT:
|
53
|
-
|
54
|
-
|
55
|
-
throw new IllegalArgumentException(
|
56
|
-
"when mode is insert, require no update_key.");
|
57
|
-
}
|
52
|
+
if (task.getUpdateKeyName().isPresent()) {
|
53
|
+
throw new ConfigException("when mode is insert, require no update_key.");
|
58
54
|
}
|
59
55
|
break;
|
60
56
|
case UPDATE:
|
61
57
|
case UPSERT:
|
62
|
-
|
63
|
-
|
64
|
-
if (option.getUpdateKey()) {
|
65
|
-
if (hasUpdateKey) {
|
66
|
-
throw new IllegalArgumentException(
|
67
|
-
"when mode is update and upsert, only one column can have an update_key.");
|
68
|
-
}
|
69
|
-
hasUpdateKey = true;
|
70
|
-
}
|
71
|
-
}
|
72
|
-
if (!hasUpdateKey) {
|
73
|
-
throw new IllegalArgumentException(
|
74
|
-
"when mode is update and upsert, require update_key.");
|
58
|
+
if (!task.getUpdateKeyName().isPresent()) {
|
59
|
+
throw new ConfigException("when mode is update or upsert, require update_key.");
|
75
60
|
}
|
76
61
|
break;
|
77
62
|
default:
|
78
|
-
throw new
|
63
|
+
throw new ConfigException(String.format(
|
79
64
|
"Unknown mode '%s'",
|
80
65
|
task.getMode()));
|
81
66
|
}
|
@@ -3,6 +3,7 @@ package org.embulk.output.kintone;
|
|
3
3
|
import com.kintone.client.KintoneClient;
|
4
4
|
import com.kintone.client.KintoneClientBuilder;
|
5
5
|
import com.kintone.client.api.record.GetRecordsByCursorResponseBody;
|
6
|
+
import com.kintone.client.model.record.FieldType;
|
6
7
|
import com.kintone.client.model.record.Record;
|
7
8
|
import com.kintone.client.model.record.RecordForUpdate;
|
8
9
|
import com.kintone.client.model.record.UpdateKey;
|
@@ -14,17 +15,18 @@ import org.embulk.spi.PageReader;
|
|
14
15
|
import org.embulk.spi.Schema;
|
15
16
|
import org.embulk.spi.TransactionalPageOutput;
|
16
17
|
|
17
|
-
import java.math.BigDecimal;
|
18
18
|
import java.util.ArrayList;
|
19
|
-
import java.util.
|
19
|
+
import java.util.Collections;
|
20
20
|
import java.util.List;
|
21
21
|
import java.util.stream.Collectors;
|
22
22
|
|
23
23
|
public class KintonePageOutput
|
24
24
|
implements TransactionalPageOutput
|
25
25
|
{
|
26
|
-
|
27
|
-
|
26
|
+
public static final int UPSERT_BATCH_SIZE = 10000;
|
27
|
+
public static final int CHUNK_SIZE = 100;
|
28
|
+
private final PageReader pageReader;
|
29
|
+
private final PluginTask task;
|
28
30
|
private KintoneClient client;
|
29
31
|
|
30
32
|
public KintonePageOutput(PluginTask task, Schema schema)
|
@@ -88,14 +90,14 @@ public class KintonePageOutput
|
|
88
90
|
|
89
91
|
public interface Consumer<T>
|
90
92
|
{
|
91
|
-
|
93
|
+
void accept(T t);
|
92
94
|
}
|
93
95
|
|
94
96
|
public void connect(final PluginTask task)
|
95
97
|
{
|
96
98
|
KintoneClientBuilder builder = KintoneClientBuilder.create("https://" + task.getDomain());
|
97
99
|
if (task.getGuestSpaceId().isPresent()) {
|
98
|
-
builder.setGuestSpaceId(task.getGuestSpaceId().
|
100
|
+
builder.setGuestSpaceId(task.getGuestSpaceId().orElse(-1));
|
99
101
|
}
|
100
102
|
if (task.getBasicAuthUsername().isPresent() && task.getBasicAuthPassword().isPresent()) {
|
101
103
|
builder.withBasicAuth(task.getBasicAuthUsername().get(),
|
@@ -136,7 +138,7 @@ public class KintonePageOutput
|
|
136
138
|
}
|
137
139
|
|
138
140
|
records.add(record);
|
139
|
-
if (records.size() ==
|
141
|
+
if (records.size() == CHUNK_SIZE) {
|
140
142
|
client.record().addRecords(task.getAppId(), records);
|
141
143
|
records.clear();
|
142
144
|
}
|
@@ -155,10 +157,12 @@ public class KintonePageOutput
|
|
155
157
|
{
|
156
158
|
execute(client -> {
|
157
159
|
try {
|
158
|
-
ArrayList<RecordForUpdate> updateRecords = new ArrayList
|
160
|
+
ArrayList<RecordForUpdate> updateRecords = new ArrayList<>();
|
159
161
|
pageReader.setPage(page);
|
162
|
+
|
160
163
|
KintoneColumnVisitor visitor = new KintoneColumnVisitor(pageReader,
|
161
|
-
task.getColumnOptions()
|
164
|
+
task.getColumnOptions(),
|
165
|
+
task.getUpdateKeyName().orElseThrow(() -> new RuntimeException("unreachable"))); // Already validated
|
162
166
|
while (pageReader.nextRecord()) {
|
163
167
|
Record record = new Record();
|
164
168
|
UpdateKey updateKey = new UpdateKey();
|
@@ -168,9 +172,13 @@ public class KintonePageOutput
|
|
168
172
|
column.visit(visitor);
|
169
173
|
}
|
170
174
|
|
175
|
+
if (updateKey.getValue() == "") {
|
176
|
+
continue;
|
177
|
+
}
|
178
|
+
|
171
179
|
record.removeField(updateKey.getField());
|
172
180
|
updateRecords.add(new RecordForUpdate(updateKey, record));
|
173
|
-
if (updateRecords.size() ==
|
181
|
+
if (updateRecords.size() == CHUNK_SIZE) {
|
174
182
|
client.record().updateRecords(task.getAppId(), updateRecords);
|
175
183
|
updateRecords.clear();
|
176
184
|
}
|
@@ -185,183 +193,130 @@ public class KintonePageOutput
|
|
185
193
|
});
|
186
194
|
}
|
187
195
|
|
188
|
-
private
|
196
|
+
private void upsertPage(final Page page)
|
189
197
|
{
|
190
|
-
|
191
|
-
|
192
|
-
|
193
|
-
|
194
|
-
|
195
|
-
fieldCode + " in (" + String.join(",", queryValues) + ")"
|
196
|
-
);
|
197
|
-
while (true) {
|
198
|
-
GetRecordsByCursorResponseBody resp = client.record().getRecordsByCursor(cursorId);
|
199
|
-
List<Record> records = resp.getRecords();
|
200
|
-
allRecords.addAll(records);
|
198
|
+
execute(client -> {
|
199
|
+
try {
|
200
|
+
ArrayList<Record> records = new ArrayList<>();
|
201
|
+
ArrayList<UpdateKey> updateKeys = new ArrayList<>();
|
202
|
+
pageReader.setPage(page);
|
201
203
|
|
202
|
-
|
203
|
-
|
204
|
+
KintoneColumnVisitor visitor = new KintoneColumnVisitor(pageReader,
|
205
|
+
task.getColumnOptions(),
|
206
|
+
task.getUpdateKeyName().orElseThrow(() -> new RuntimeException("unreachable"))); // Already validated
|
207
|
+
while (pageReader.nextRecord()) {
|
208
|
+
Record record = new Record();
|
209
|
+
UpdateKey updateKey = new UpdateKey();
|
210
|
+
visitor.setRecord(record);
|
211
|
+
visitor.setUpdateKey(updateKey);
|
212
|
+
for (Column column : pageReader.getSchema().getColumns()) {
|
213
|
+
column.visit(visitor);
|
214
|
+
}
|
215
|
+
records.add(record);
|
216
|
+
updateKeys.add(updateKey);
|
217
|
+
|
218
|
+
if (records.size() == UPSERT_BATCH_SIZE) {
|
219
|
+
upsert(records, updateKeys);
|
220
|
+
records.clear();
|
221
|
+
updateKeys.clear();
|
222
|
+
}
|
223
|
+
}
|
224
|
+
if (records.size() > 0) {
|
225
|
+
upsert(records, updateKeys);
|
226
|
+
}
|
204
227
|
}
|
205
|
-
|
206
|
-
|
228
|
+
catch (Exception e) {
|
229
|
+
throw new RuntimeException("kintone throw exception", e);
|
230
|
+
}
|
231
|
+
});
|
207
232
|
}
|
208
233
|
|
209
|
-
|
234
|
+
private void upsert(ArrayList<Record> records, ArrayList<UpdateKey> updateKeys)
|
210
235
|
{
|
211
|
-
|
212
|
-
|
213
|
-
|
214
|
-
public void upsert(ArrayList<Record> records, ArrayList<UpdateKey> updateKeys)
|
215
|
-
{
|
216
|
-
if (records.size() != updateKeys.size()) {
|
217
|
-
throw new RuntimeException("records.size() != updateKeys.size()");
|
218
|
-
}
|
219
|
-
|
220
|
-
List<String> queryValues = updateKeys
|
221
|
-
.stream()
|
222
|
-
.map(k -> "\"" + k.getValue().toString() + "\"")
|
223
|
-
.collect(Collectors.toList());
|
224
|
-
List<T> updateKeyValues = getUpdateKeyValues(queryValues);
|
236
|
+
if (records.size() != updateKeys.size()) {
|
237
|
+
throw new RuntimeException("records.size() != updateKeys.size()");
|
238
|
+
}
|
225
239
|
|
226
|
-
|
227
|
-
ArrayList<RecordForUpdate> updateRecords = new ArrayList<RecordForUpdate>();
|
228
|
-
for (int i = 0; i < records.size(); i++) {
|
229
|
-
Record record = records.get(i);
|
230
|
-
UpdateKey updateKey = updateKeys.get(i);
|
240
|
+
List<Record> existingRecords = getExistingRecordsByUpdateKey(updateKeys);
|
231
241
|
|
232
|
-
|
233
|
-
|
234
|
-
|
235
|
-
|
236
|
-
|
237
|
-
insertRecords.add(record);
|
238
|
-
}
|
242
|
+
ArrayList<Record> insertRecords = new ArrayList<>();
|
243
|
+
ArrayList<RecordForUpdate> updateRecords = new ArrayList<>();
|
244
|
+
for (int i = 0; i < records.size(); i++) {
|
245
|
+
Record record = records.get(i);
|
246
|
+
UpdateKey updateKey = updateKeys.get(i);
|
239
247
|
|
240
|
-
|
241
|
-
|
242
|
-
|
243
|
-
}
|
244
|
-
else if (updateRecords.size() == 100) {
|
245
|
-
client.record().updateRecords(task.getAppId(), updateRecords);
|
246
|
-
updateRecords.clear();
|
247
|
-
}
|
248
|
+
if (existsRecord(existingRecords, updateKey)) {
|
249
|
+
record.removeField(updateKey.getField());
|
250
|
+
updateRecords.add(new RecordForUpdate(updateKey, record));
|
248
251
|
}
|
249
|
-
|
252
|
+
else {
|
253
|
+
insertRecords.add(record);
|
254
|
+
}
|
255
|
+
|
256
|
+
if (insertRecords.size() == CHUNK_SIZE) {
|
250
257
|
client.record().addRecords(task.getAppId(), insertRecords);
|
258
|
+
insertRecords.clear();
|
251
259
|
}
|
252
|
-
if (updateRecords.size()
|
260
|
+
else if (updateRecords.size() == CHUNK_SIZE) {
|
253
261
|
client.record().updateRecords(task.getAppId(), updateRecords);
|
262
|
+
updateRecords.clear();
|
254
263
|
}
|
255
264
|
}
|
256
|
-
|
257
|
-
|
258
|
-
{
|
259
|
-
execute(client -> {
|
260
|
-
try {
|
261
|
-
ArrayList<Record> records = new ArrayList<>();
|
262
|
-
ArrayList<UpdateKey> updateKeys = new ArrayList<>();
|
263
|
-
|
264
|
-
pageReader.setPage(page);
|
265
|
-
KintoneColumnVisitor visitor = new KintoneColumnVisitor(pageReader,
|
266
|
-
task.getColumnOptions());
|
267
|
-
while (pageReader.nextRecord()) {
|
268
|
-
Record record = new Record();
|
269
|
-
UpdateKey updateKey = new UpdateKey();
|
270
|
-
visitor.setRecord(record);
|
271
|
-
visitor.setUpdateKey(updateKey);
|
272
|
-
for (Column column : pageReader.getSchema().getColumns()) {
|
273
|
-
column.visit(visitor);
|
274
|
-
}
|
275
|
-
records.add(record);
|
276
|
-
updateKeys.add(updateKey);
|
277
|
-
|
278
|
-
if (records.size() == 10000) {
|
279
|
-
upsert(records, updateKeys);
|
280
|
-
records.clear();
|
281
|
-
updateKeys.clear();
|
282
|
-
}
|
283
|
-
}
|
284
|
-
if (records.size() > 0) {
|
285
|
-
upsert(records, updateKeys);
|
286
|
-
}
|
287
|
-
}
|
288
|
-
catch (Exception e) {
|
289
|
-
throw new RuntimeException("kintone throw exception", e);
|
290
|
-
}
|
291
|
-
});
|
292
|
-
}
|
293
|
-
}
|
294
|
-
|
295
|
-
class UpsertPageByStringKey extends UpsertPage<String>
|
296
|
-
{
|
297
|
-
private String fieldCode;
|
298
|
-
|
299
|
-
public UpsertPageByStringKey(String fieldCode)
|
300
|
-
{
|
301
|
-
this.fieldCode = fieldCode;
|
302
|
-
}
|
303
|
-
|
304
|
-
public List<String> getUpdateKeyValues(List<String> queryValues)
|
305
|
-
{
|
306
|
-
return getRecordsByUpdateKey(fieldCode, queryValues)
|
307
|
-
.stream()
|
308
|
-
.map(r -> r.getSingleLineTextFieldValue(fieldCode))
|
309
|
-
.collect(Collectors.toList());
|
265
|
+
if (insertRecords.size() > 0) {
|
266
|
+
client.record().addRecords(task.getAppId(), insertRecords);
|
310
267
|
}
|
311
|
-
|
312
|
-
|
313
|
-
{
|
314
|
-
return updateKeyValues.contains(record.getSingleLineTextFieldValue(fieldCode));
|
268
|
+
if (updateRecords.size() > 0) {
|
269
|
+
client.record().updateRecords(task.getAppId(), updateRecords);
|
315
270
|
}
|
316
271
|
}
|
317
272
|
|
318
|
-
class UpsertPageByNumberKey extends UpsertPage<BigDecimal>
|
319
|
-
{
|
320
|
-
private String fieldCode;
|
321
|
-
|
322
|
-
public UpsertPageByNumberKey(String fieldCode)
|
323
|
-
{
|
324
|
-
this.fieldCode = fieldCode;
|
325
|
-
}
|
326
273
|
|
327
|
-
|
328
|
-
|
329
|
-
|
274
|
+
private List<Record> getExistingRecordsByUpdateKey(ArrayList<UpdateKey> updateKeys)
|
275
|
+
{
|
276
|
+
String fieldCode = updateKeys.get(0).getField();
|
277
|
+
List<String> queryValues = updateKeys
|
330
278
|
.stream()
|
331
|
-
.
|
279
|
+
.filter(k -> k.getValue() != "")
|
280
|
+
.map(k -> "\"" + k.getValue().toString() + "\"")
|
332
281
|
.collect(Collectors.toList());
|
333
|
-
}
|
334
282
|
|
335
|
-
|
336
|
-
{
|
337
|
-
return
|
283
|
+
List<Record> allRecords = new ArrayList<>();
|
284
|
+
if (queryValues.isEmpty()) {
|
285
|
+
return allRecords;
|
338
286
|
}
|
339
|
-
|
287
|
+
String cursorId = client.record().createCursor(
|
288
|
+
task.getAppId(),
|
289
|
+
Collections.singletonList(fieldCode),
|
290
|
+
fieldCode + " in (" + String.join(",", queryValues) + ")"
|
291
|
+
);
|
292
|
+
while (true) {
|
293
|
+
GetRecordsByCursorResponseBody resp = client.record().getRecordsByCursor(cursorId);
|
294
|
+
List<Record> records = resp.getRecords();
|
295
|
+
allRecords.addAll(records);
|
340
296
|
|
341
|
-
|
342
|
-
{
|
343
|
-
KintoneColumnOption updateKeyColumn = null;
|
344
|
-
for (KintoneColumnOption v : task.getColumnOptions().values()) {
|
345
|
-
if (v.getUpdateKey()) {
|
346
|
-
updateKeyColumn = v;
|
297
|
+
if (!resp.hasNext()) {
|
347
298
|
break;
|
348
299
|
}
|
349
300
|
}
|
350
|
-
|
351
|
-
|
352
|
-
}
|
301
|
+
return allRecords;
|
302
|
+
}
|
353
303
|
|
354
|
-
|
355
|
-
|
356
|
-
|
357
|
-
|
358
|
-
|
359
|
-
case
|
360
|
-
|
361
|
-
|
304
|
+
private boolean existsRecord(List<Record> distRecords, UpdateKey updateKey)
|
305
|
+
{
|
306
|
+
String fieldCode = updateKey.getField();
|
307
|
+
FieldType type = client.app().getFormFields(task.getAppId()).get(fieldCode).getType();
|
308
|
+
switch (type) {
|
309
|
+
case SINGLE_LINE_TEXT:
|
310
|
+
return distRecords
|
311
|
+
.stream()
|
312
|
+
.anyMatch(d -> d.getSingleLineTextFieldValue(fieldCode).equals(updateKey.getValue().toString()));
|
313
|
+
case NUMBER:
|
314
|
+
return distRecords
|
315
|
+
.stream()
|
316
|
+
.anyMatch(d -> d.getNumberFieldValue(fieldCode).toPlainString()
|
317
|
+
.equals(updateKey.getValue().toString()));
|
362
318
|
default:
|
363
319
|
throw new RuntimeException("The update_key must be 'SINGLE_LINE_TEXT' or 'NUMBER'.");
|
364
320
|
}
|
365
|
-
runner.run(page);
|
366
321
|
}
|
367
322
|
}
|
@@ -1,11 +1,11 @@
|
|
1
1
|
package org.embulk.output.kintone;
|
2
2
|
|
3
|
-
import com.google.common.base.Optional;
|
4
3
|
import org.embulk.config.Config;
|
5
4
|
import org.embulk.config.ConfigDefault;
|
6
5
|
import org.embulk.config.Task;
|
7
6
|
|
8
7
|
import java.util.Map;
|
8
|
+
import java.util.Optional;
|
9
9
|
|
10
10
|
public interface PluginTask
|
11
11
|
extends Task
|
@@ -42,9 +42,13 @@ public interface PluginTask
|
|
42
42
|
|
43
43
|
@Config("column_options")
|
44
44
|
@ConfigDefault("{}")
|
45
|
-
|
45
|
+
Map<String, KintoneColumnOption> getColumnOptions();
|
46
46
|
|
47
47
|
@Config("mode")
|
48
48
|
@ConfigDefault("insert")
|
49
|
-
|
49
|
+
String getMode();
|
50
|
+
|
51
|
+
@Config("update_key")
|
52
|
+
@ConfigDefault("null")
|
53
|
+
Optional<String> getUpdateKeyName();
|
50
54
|
}
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-output-kintone
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.3.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- takeshi fujita
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2022-03-23 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -52,7 +52,7 @@ files:
|
|
52
52
|
- README.md
|
53
53
|
- Rakefile
|
54
54
|
- build.gradle
|
55
|
-
- classpath/embulk-output-kintone-0.
|
55
|
+
- classpath/embulk-output-kintone-0.3.2.jar
|
56
56
|
- config/checkstyle/checkstyle.xml
|
57
57
|
- config/checkstyle/default.xml
|
58
58
|
- gradle/wrapper/gradle-wrapper.jar
|