embulk-output-kintone 0.4.1 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +8 -6
- data/classpath/embulk-output-kintone-1.0.0.jar +0 -0
- data/classpath/{shadow-kintone-java-client-0.4.1-all.jar → shadow-kintone-java-client-1.0.0-all.jar} +0 -0
- data/src/main/java/org/embulk/output/kintone/KintoneColumnOption.java +1 -2
- data/src/main/java/org/embulk/output/kintone/KintoneColumnType.java +368 -0
- data/src/main/java/org/embulk/output/kintone/KintoneColumnVisitor.java +195 -135
- data/src/main/java/org/embulk/output/kintone/KintoneMode.java +0 -1
- data/src/main/java/org/embulk/output/kintone/KintoneOutputPlugin.java +0 -2
- data/src/main/java/org/embulk/output/kintone/KintonePageOutput.java +169 -157
- data/src/main/java/org/embulk/output/kintone/PluginTask.java +8 -0
- data/src/test/java/org/embulk/output/kintone/KintoneColumnOptionBuilder.java +2 -3
- data/src/test/java/org/embulk/output/kintone/KintoneColumnVisitorTest.java +563 -40
- data/src/test/java/org/embulk/output/kintone/KintoneColumnVisitorVerifier.java +35 -14
- data/src/test/java/org/embulk/output/kintone/KintonePageOutputVerifier.java +2 -7
- data/src/test/java/org/embulk/output/kintone/TestKintoneOutputPlugin.java +22 -9
- data/src/test/java/org/embulk/output/kintone/TestTaskMode.java +12 -0
- data/src/test/resources/org/embulk/output/kintone/task/mode/config.yml +104 -0
- data/src/test/resources/org/embulk/output/kintone/task/mode/input.csv +7 -7
- data/src/test/resources/org/embulk/output/kintone/task/mode/insert_add_ignore_nulls_records.jsonl +6 -0
- data/src/test/resources/org/embulk/output/kintone/task/mode/insert_add_prefer_nulls_records.jsonl +6 -0
- data/src/test/resources/org/embulk/output/kintone/task/mode/insert_add_records.jsonl +6 -6
- data/src/test/resources/org/embulk/output/kintone/task/mode/update_update_ignore_nulls_records.jsonl +3 -0
- data/src/test/resources/org/embulk/output/kintone/task/mode/update_update_prefer_nulls_records.jsonl +3 -0
- data/src/test/resources/org/embulk/output/kintone/task/mode/update_update_records.jsonl +6 -3
- data/src/test/resources/org/embulk/output/kintone/task/mode/upsert_add_ignore_nulls_records.jsonl +3 -0
- data/src/test/resources/org/embulk/output/kintone/task/mode/upsert_add_prefer_nulls_records.jsonl +3 -0
- data/src/test/resources/org/embulk/output/kintone/task/mode/upsert_add_records.jsonl +2 -2
- data/src/test/resources/org/embulk/output/kintone/task/mode/upsert_update_ignore_nulls_records.jsonl +3 -0
- data/src/test/resources/org/embulk/output/kintone/task/mode/upsert_update_prefer_nulls_records.jsonl +3 -0
- data/src/test/resources/org/embulk/output/kintone/task/mode/upsert_update_records.jsonl +4 -4
- data/src/test/resources/org/embulk/output/kintone/task/mode/values_ignore_nulls.json +1 -0
- data/src/test/resources/org/embulk/output/kintone/task/mode/values_prefer_nulls.json +1 -0
- metadata +15 -4
- data/classpath/embulk-output-kintone-0.4.1.jar +0 -0
@@ -4,24 +4,32 @@ import static org.embulk.spi.util.RetryExecutor.retryExecutor;
|
|
4
4
|
|
5
5
|
import com.fasterxml.jackson.databind.JsonNode;
|
6
6
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
7
|
+
import com.google.common.collect.Maps;
|
7
8
|
import com.kintone.client.KintoneClient;
|
8
9
|
import com.kintone.client.KintoneClientBuilder;
|
9
10
|
import com.kintone.client.api.record.GetRecordsByCursorResponseBody;
|
10
11
|
import com.kintone.client.exception.KintoneApiRuntimeException;
|
12
|
+
import com.kintone.client.model.app.field.FieldProperty;
|
11
13
|
import com.kintone.client.model.record.FieldType;
|
12
14
|
import com.kintone.client.model.record.Record;
|
13
15
|
import com.kintone.client.model.record.RecordForUpdate;
|
14
16
|
import com.kintone.client.model.record.UpdateKey;
|
15
17
|
import java.io.IOException;
|
18
|
+
import java.lang.invoke.MethodHandles;
|
16
19
|
import java.math.BigDecimal;
|
17
20
|
import java.util.ArrayList;
|
18
21
|
import java.util.Arrays;
|
19
22
|
import java.util.Collections;
|
20
23
|
import java.util.List;
|
24
|
+
import java.util.Map;
|
25
|
+
import java.util.Objects;
|
26
|
+
import java.util.TreeMap;
|
27
|
+
import java.util.function.Function;
|
28
|
+
import java.util.function.Supplier;
|
21
29
|
import java.util.stream.Collectors;
|
30
|
+
import org.apache.commons.lang3.tuple.Pair;
|
22
31
|
import org.embulk.config.ConfigException;
|
23
32
|
import org.embulk.config.TaskReport;
|
24
|
-
import org.embulk.spi.Column;
|
25
33
|
import org.embulk.spi.Exec;
|
26
34
|
import org.embulk.spi.Page;
|
27
35
|
import org.embulk.spi.PageReader;
|
@@ -33,22 +41,25 @@ import org.slf4j.Logger;
|
|
33
41
|
import org.slf4j.LoggerFactory;
|
34
42
|
|
35
43
|
public class KintonePageOutput implements TransactionalPageOutput {
|
36
|
-
|
37
|
-
|
38
|
-
private static final Logger LOGGER = LoggerFactory.getLogger(KintonePageOutput.class);
|
44
|
+
private static final Logger LOGGER =
|
45
|
+
LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
|
39
46
|
private static final List<String> RETRYABLE_ERROR_CODES =
|
40
47
|
Arrays.asList(
|
41
|
-
"GAIA_TM12", //
|
48
|
+
"GAIA_TM12", // 作成できるカーソルの上限に達しているため、カーソルを作成できません。不要なカーソルを削除するか、しばらく経ってから再実行してください。
|
42
49
|
"GAIA_RE18", // データベースのロックに失敗したため、変更を保存できませんでした。時間をおいて再度お試しください。
|
43
50
|
"GAIA_DA02" // データベースのロックに失敗したため、変更を保存できませんでした。時間をおいて再度お試しください。
|
44
51
|
);
|
45
|
-
private final
|
52
|
+
private static final int UPSERT_BATCH_SIZE = 10000;
|
53
|
+
private static final int CHUNK_SIZE = 100;
|
54
|
+
private final Map<String, Pair<FieldType, FieldType>> wrongTypeFields = new TreeMap<>();
|
46
55
|
private final PluginTask task;
|
56
|
+
private final PageReader reader;
|
47
57
|
private KintoneClient client;
|
58
|
+
private Map<String, FieldProperty> formFields;
|
48
59
|
|
49
60
|
public KintonePageOutput(PluginTask task, Schema schema) {
|
50
|
-
this.pageReader = new PageReader(schema);
|
51
61
|
this.task = task;
|
62
|
+
reader = new PageReader(schema);
|
52
63
|
}
|
53
64
|
|
54
65
|
@Override
|
@@ -76,11 +87,11 @@ public class KintonePageOutput implements TransactionalPageOutput {
|
|
76
87
|
|
77
88
|
@Override
|
78
89
|
public void close() {
|
79
|
-
if (
|
80
|
-
return;
|
90
|
+
if (client == null) {
|
91
|
+
return; // Not connected
|
81
92
|
}
|
82
93
|
try {
|
83
|
-
|
94
|
+
client.close();
|
84
95
|
} catch (Exception e) {
|
85
96
|
throw new RuntimeException("kintone throw exception", e);
|
86
97
|
}
|
@@ -93,78 +104,73 @@ public class KintonePageOutput implements TransactionalPageOutput {
|
|
93
104
|
|
94
105
|
@Override
|
95
106
|
public TaskReport commit() {
|
107
|
+
wrongTypeFields.forEach(
|
108
|
+
(key, value) ->
|
109
|
+
LOGGER.warn(
|
110
|
+
String.format(
|
111
|
+
"Field type of %s is expected %s but actual %s",
|
112
|
+
key, value.getLeft(), value.getRight())));
|
96
113
|
return Exec.newTaskReport();
|
97
114
|
}
|
98
115
|
|
99
|
-
public
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
public void connect(final PluginTask task) {
|
116
|
+
public void connectIfNeeded() {
|
117
|
+
if (client != null) {
|
118
|
+
return; // Already connected
|
119
|
+
}
|
104
120
|
KintoneClientBuilder builder = KintoneClientBuilder.create("https://" + task.getDomain());
|
105
121
|
if (task.getGuestSpaceId().isPresent()) {
|
106
|
-
builder.setGuestSpaceId(task.getGuestSpaceId().
|
122
|
+
builder.setGuestSpaceId(task.getGuestSpaceId().get());
|
107
123
|
}
|
108
124
|
if (task.getBasicAuthUsername().isPresent() && task.getBasicAuthPassword().isPresent()) {
|
109
125
|
builder.withBasicAuth(task.getBasicAuthUsername().get(), task.getBasicAuthPassword().get());
|
110
126
|
}
|
111
|
-
|
112
127
|
if (task.getUsername().isPresent() && task.getPassword().isPresent()) {
|
113
|
-
|
114
|
-
builder.authByPassword(task.getUsername().get(), task.getPassword().get()).build();
|
128
|
+
builder.authByPassword(task.getUsername().get(), task.getPassword().get());
|
115
129
|
} else if (task.getToken().isPresent()) {
|
116
|
-
|
130
|
+
builder.authByApiToken(task.getToken().get());
|
131
|
+
} else {
|
132
|
+
throw new ConfigException("Username and password or token must be configured.");
|
117
133
|
}
|
134
|
+
client = builder.build();
|
135
|
+
formFields = client.app().getFormFields(task.getAppId());
|
118
136
|
}
|
119
137
|
|
120
|
-
private void
|
121
|
-
|
122
|
-
client -> {
|
123
|
-
client.record().updateRecords(task.getAppId(), records);
|
124
|
-
});
|
138
|
+
private void insert(List<Record> records) {
|
139
|
+
executeWithRetry(() -> client.record().addRecords(task.getAppId(), records));
|
125
140
|
}
|
126
141
|
|
127
|
-
private void
|
128
|
-
|
129
|
-
client -> {
|
130
|
-
client.record().addRecords(task.getAppId(), records);
|
131
|
-
});
|
142
|
+
private void update(List<RecordForUpdate> records) {
|
143
|
+
executeWithRetry(() -> client.record().updateRecords(task.getAppId(), records));
|
132
144
|
}
|
133
145
|
|
134
|
-
private
|
135
|
-
|
136
|
-
if (this.client == null) {
|
137
|
-
throw new RuntimeException("Failed to connect to kintone.");
|
138
|
-
}
|
146
|
+
private <T> T executeWithRetry(Supplier<T> operation) {
|
147
|
+
connectIfNeeded();
|
139
148
|
KintoneRetryOption retryOption = task.getRetryOptions();
|
140
149
|
try {
|
141
|
-
retryExecutor()
|
150
|
+
return retryExecutor()
|
142
151
|
.withRetryLimit(retryOption.getLimit())
|
143
152
|
.withInitialRetryWait(retryOption.getInitialWaitMillis())
|
144
153
|
.withMaxRetryWait(retryOption.getMaxWaitMillis())
|
145
154
|
.runInterruptible(
|
146
|
-
new Retryable<
|
147
|
-
|
155
|
+
new Retryable<T>() {
|
148
156
|
@Override
|
149
|
-
public
|
150
|
-
operation.
|
151
|
-
return null;
|
157
|
+
public T call() throws Exception {
|
158
|
+
return operation.get();
|
152
159
|
}
|
153
160
|
|
154
161
|
@Override
|
155
|
-
public boolean isRetryableException(Exception
|
156
|
-
if (!(
|
162
|
+
public boolean isRetryableException(Exception exception) {
|
163
|
+
if (!(exception instanceof KintoneApiRuntimeException)) {
|
157
164
|
return false;
|
158
165
|
}
|
159
|
-
|
160
166
|
try {
|
161
167
|
ObjectMapper mapper = new ObjectMapper();
|
162
168
|
JsonNode content =
|
163
|
-
mapper.readTree(((KintoneApiRuntimeException)
|
169
|
+
mapper.readTree(((KintoneApiRuntimeException) exception).getContent());
|
164
170
|
String code = content.get("code").textValue();
|
165
171
|
return RETRYABLE_ERROR_CODES.contains(code);
|
166
|
-
} catch (IOException
|
167
|
-
throw new RuntimeException(
|
172
|
+
} catch (IOException e) {
|
173
|
+
throw new RuntimeException(e);
|
168
174
|
}
|
169
175
|
}
|
170
176
|
|
@@ -192,143 +198,108 @@ public class KintonePageOutput implements TransactionalPageOutput {
|
|
192
198
|
}
|
193
199
|
}
|
194
200
|
|
195
|
-
private void insertPage(
|
196
|
-
|
197
|
-
|
198
|
-
|
199
|
-
|
200
|
-
|
201
|
+
private void insertPage(Page page) {
|
202
|
+
List<Record> records = new ArrayList<>();
|
203
|
+
reader.setPage(page);
|
204
|
+
KintoneColumnVisitor visitor =
|
205
|
+
new KintoneColumnVisitor(
|
206
|
+
reader, task.getColumnOptions(), task.getPreferNulls(), task.getIgnoreNulls());
|
207
|
+
while (reader.nextRecord()) {
|
201
208
|
Record record = new Record();
|
202
209
|
visitor.setRecord(record);
|
203
|
-
|
204
|
-
|
205
|
-
}
|
206
|
-
|
210
|
+
reader.getSchema().visitColumns(visitor);
|
211
|
+
putWrongTypeFields(record);
|
207
212
|
records.add(record);
|
208
213
|
if (records.size() == CHUNK_SIZE) {
|
209
214
|
insert(records);
|
210
215
|
records.clear();
|
211
216
|
}
|
212
217
|
}
|
213
|
-
if (records.
|
218
|
+
if (!records.isEmpty()) {
|
214
219
|
insert(records);
|
215
220
|
}
|
216
221
|
}
|
217
222
|
|
218
|
-
private void updatePage(
|
219
|
-
|
220
|
-
|
221
|
-
|
223
|
+
private void updatePage(Page page) {
|
224
|
+
List<RecordForUpdate> records = new ArrayList<>();
|
225
|
+
reader.setPage(page);
|
222
226
|
KintoneColumnVisitor visitor =
|
223
227
|
new KintoneColumnVisitor(
|
224
|
-
|
228
|
+
reader,
|
225
229
|
task.getColumnOptions(),
|
230
|
+
task.getPreferNulls(),
|
231
|
+
task.getIgnoreNulls(),
|
226
232
|
task.getUpdateKeyName()
|
227
233
|
.orElseThrow(() -> new RuntimeException("unreachable"))); // Already validated
|
228
|
-
while (
|
234
|
+
while (reader.nextRecord()) {
|
229
235
|
Record record = new Record();
|
230
236
|
UpdateKey updateKey = new UpdateKey();
|
231
237
|
visitor.setRecord(record);
|
232
238
|
visitor.setUpdateKey(updateKey);
|
233
|
-
|
234
|
-
|
235
|
-
|
236
|
-
|
237
|
-
if (updateKey.getValue() == "") {
|
239
|
+
reader.getSchema().visitColumns(visitor);
|
240
|
+
putWrongTypeFields(record);
|
241
|
+
if (updateKey.getValue() == null || updateKey.getValue().toString().isEmpty()) {
|
242
|
+
LOGGER.warn("Record skipped because no update key value was specified");
|
238
243
|
continue;
|
239
244
|
}
|
240
|
-
|
241
|
-
|
242
|
-
|
243
|
-
|
244
|
-
update(updateRecords);
|
245
|
-
updateRecords.clear();
|
245
|
+
records.add(new RecordForUpdate(updateKey, record.removeField(updateKey.getField())));
|
246
|
+
if (records.size() == CHUNK_SIZE) {
|
247
|
+
update(records);
|
248
|
+
records.clear();
|
246
249
|
}
|
247
250
|
}
|
248
|
-
if (
|
249
|
-
update(
|
251
|
+
if (!records.isEmpty()) {
|
252
|
+
update(records);
|
250
253
|
}
|
251
254
|
}
|
252
255
|
|
253
|
-
private void upsertPage(
|
254
|
-
|
255
|
-
|
256
|
-
|
257
|
-
|
258
|
-
|
259
|
-
|
260
|
-
|
261
|
-
|
262
|
-
|
263
|
-
|
264
|
-
|
265
|
-
|
266
|
-
|
267
|
-
|
268
|
-
|
269
|
-
|
270
|
-
|
271
|
-
|
272
|
-
|
273
|
-
|
274
|
-
|
275
|
-
|
276
|
-
|
277
|
-
|
278
|
-
|
279
|
-
|
280
|
-
|
281
|
-
|
282
|
-
|
283
|
-
if (records.size() > 0) {
|
284
|
-
upsert(records, updateKeys);
|
285
|
-
}
|
286
|
-
});
|
256
|
+
private void upsertPage(Page page) {
|
257
|
+
List<Record> records = new ArrayList<>();
|
258
|
+
List<UpdateKey> updateKeys = new ArrayList<>();
|
259
|
+
reader.setPage(page);
|
260
|
+
KintoneColumnVisitor visitor =
|
261
|
+
new KintoneColumnVisitor(
|
262
|
+
reader,
|
263
|
+
task.getColumnOptions(),
|
264
|
+
task.getPreferNulls(),
|
265
|
+
task.getIgnoreNulls(),
|
266
|
+
task.getUpdateKeyName()
|
267
|
+
.orElseThrow(() -> new RuntimeException("unreachable"))); // Already validated
|
268
|
+
while (reader.nextRecord()) {
|
269
|
+
Record record = new Record();
|
270
|
+
UpdateKey updateKey = new UpdateKey();
|
271
|
+
visitor.setRecord(record);
|
272
|
+
visitor.setUpdateKey(updateKey);
|
273
|
+
reader.getSchema().visitColumns(visitor);
|
274
|
+
putWrongTypeFields(record);
|
275
|
+
records.add(record);
|
276
|
+
updateKeys.add(updateKey);
|
277
|
+
if (records.size() == UPSERT_BATCH_SIZE) {
|
278
|
+
upsert(records, updateKeys);
|
279
|
+
records.clear();
|
280
|
+
updateKeys.clear();
|
281
|
+
}
|
282
|
+
}
|
283
|
+
if (!records.isEmpty()) {
|
284
|
+
upsert(records, updateKeys);
|
285
|
+
}
|
287
286
|
}
|
288
287
|
|
289
|
-
private void upsert(
|
288
|
+
private void upsert(List<Record> records, List<UpdateKey> updateKeys) {
|
290
289
|
if (records.size() != updateKeys.size()) {
|
291
290
|
throw new RuntimeException("records.size() != updateKeys.size()");
|
292
291
|
}
|
293
|
-
|
294
|
-
|
295
|
-
|
296
|
-
throw new ConfigException("The update_key must be 'SINGLE_LINE_TEXT' or 'NUMBER'.");
|
297
|
-
}
|
298
|
-
|
299
|
-
List<Record> existingRecords = getExistingRecordsByUpdateKey(updateKeys);
|
300
|
-
String updateField = updateKeys.get(0).getField();
|
301
|
-
List<String> existingValues =
|
302
|
-
existingRecords.stream()
|
303
|
-
.map(
|
304
|
-
(r) -> {
|
305
|
-
switch (updateKeyFieldType) {
|
306
|
-
case SINGLE_LINE_TEXT:
|
307
|
-
String s = r.getSingleLineTextFieldValue(updateField);
|
308
|
-
return s == null ? null : s.toString();
|
309
|
-
case NUMBER:
|
310
|
-
BigDecimal bd = r.getNumberFieldValue(updateField);
|
311
|
-
return bd == null ? null : bd.toPlainString();
|
312
|
-
default:
|
313
|
-
return null;
|
314
|
-
}
|
315
|
-
})
|
316
|
-
.filter(v -> v != null)
|
317
|
-
.collect(Collectors.toList());
|
318
|
-
|
319
|
-
ArrayList<Record> insertRecords = new ArrayList<>();
|
320
|
-
ArrayList<RecordForUpdate> updateRecords = new ArrayList<>();
|
292
|
+
List<String> existingValues = executeWithRetry(() -> getExistingValuesByUpdateKey(updateKeys));
|
293
|
+
List<Record> insertRecords = new ArrayList<>();
|
294
|
+
List<RecordForUpdate> updateRecords = new ArrayList<>();
|
321
295
|
for (int i = 0; i < records.size(); i++) {
|
322
296
|
Record record = records.get(i);
|
323
297
|
UpdateKey updateKey = updateKeys.get(i);
|
324
|
-
|
325
298
|
if (existsRecord(existingValues, updateKey)) {
|
326
|
-
record.removeField(updateKey.getField());
|
327
|
-
updateRecords.add(new RecordForUpdate(updateKey, record));
|
299
|
+
updateRecords.add(new RecordForUpdate(updateKey, record.removeField(updateKey.getField())));
|
328
300
|
} else {
|
329
301
|
insertRecords.add(record);
|
330
302
|
}
|
331
|
-
|
332
303
|
if (insertRecords.size() == CHUNK_SIZE) {
|
333
304
|
insert(insertRecords);
|
334
305
|
insertRecords.clear();
|
@@ -337,25 +308,40 @@ public class KintonePageOutput implements TransactionalPageOutput {
|
|
337
308
|
updateRecords.clear();
|
338
309
|
}
|
339
310
|
}
|
340
|
-
if (insertRecords.
|
311
|
+
if (!insertRecords.isEmpty()) {
|
341
312
|
insert(insertRecords);
|
342
313
|
}
|
343
|
-
if (updateRecords.
|
314
|
+
if (!updateRecords.isEmpty()) {
|
344
315
|
update(updateRecords);
|
345
316
|
}
|
346
317
|
}
|
347
318
|
|
348
|
-
private List<
|
349
|
-
String fieldCode =
|
319
|
+
private List<String> getExistingValuesByUpdateKey(List<UpdateKey> updateKeys) {
|
320
|
+
String fieldCode =
|
321
|
+
updateKeys.stream()
|
322
|
+
.map(UpdateKey::getField)
|
323
|
+
.filter(Objects::nonNull)
|
324
|
+
.findFirst()
|
325
|
+
.orElse(null);
|
326
|
+
if (fieldCode == null) {
|
327
|
+
return Collections.emptyList();
|
328
|
+
}
|
329
|
+
Function<Record, String> fieldValueAsString;
|
330
|
+
FieldType fieldType = getFieldType(fieldCode);
|
331
|
+
if (fieldType == FieldType.SINGLE_LINE_TEXT) {
|
332
|
+
fieldValueAsString = record -> record.getSingleLineTextFieldValue(fieldCode);
|
333
|
+
} else if (fieldType == FieldType.NUMBER) {
|
334
|
+
fieldValueAsString = record -> toString(record.getNumberFieldValue(fieldCode));
|
335
|
+
} else {
|
336
|
+
throw new ConfigException("The update_key must be 'SINGLE_LINE_TEXT' or 'NUMBER'.");
|
337
|
+
}
|
350
338
|
List<String> queryValues =
|
351
339
|
updateKeys.stream()
|
352
|
-
.filter(k -> k.getValue() !=
|
353
|
-
.map(k -> "\"" + k.getValue()
|
340
|
+
.filter(k -> k.getValue() != null && !k.getValue().toString().isEmpty())
|
341
|
+
.map(k -> "\"" + k.getValue() + "\"")
|
354
342
|
.collect(Collectors.toList());
|
355
|
-
|
356
|
-
List<Record> allRecords = new ArrayList<>();
|
357
343
|
if (queryValues.isEmpty()) {
|
358
|
-
return
|
344
|
+
return Collections.emptyList();
|
359
345
|
}
|
360
346
|
String cursorId =
|
361
347
|
client
|
@@ -364,19 +350,45 @@ public class KintonePageOutput implements TransactionalPageOutput {
|
|
364
350
|
task.getAppId(),
|
365
351
|
Collections.singletonList(fieldCode),
|
366
352
|
fieldCode + " in (" + String.join(",", queryValues) + ")");
|
353
|
+
List<Record> allRecords = new ArrayList<>();
|
367
354
|
while (true) {
|
368
355
|
GetRecordsByCursorResponseBody resp = client.record().getRecordsByCursor(cursorId);
|
369
356
|
List<Record> records = resp.getRecords();
|
370
357
|
allRecords.addAll(records);
|
371
|
-
|
372
358
|
if (!resp.hasNext()) {
|
373
359
|
break;
|
374
360
|
}
|
375
361
|
}
|
376
|
-
return allRecords
|
362
|
+
return allRecords.stream()
|
363
|
+
.map(fieldValueAsString)
|
364
|
+
.filter(Objects::nonNull)
|
365
|
+
.collect(Collectors.toList());
|
366
|
+
}
|
367
|
+
|
368
|
+
private boolean existsRecord(List<String> existingValues, UpdateKey updateKey) {
|
369
|
+
String value = toString(updateKey.getValue());
|
370
|
+
return value != null && existingValues.stream().anyMatch(v -> v.equals(value));
|
371
|
+
}
|
372
|
+
|
373
|
+
private void putWrongTypeFields(Record record) {
|
374
|
+
record.getFieldCodes(true).stream()
|
375
|
+
.map(
|
376
|
+
fieldCode ->
|
377
|
+
Maps.immutableEntry(
|
378
|
+
fieldCode, Pair.of(record.getFieldType(fieldCode), getFieldType(fieldCode))))
|
379
|
+
.filter(entry -> entry.getValue().getLeft() != entry.getValue().getRight())
|
380
|
+
.forEach(entry -> wrongTypeFields.put(entry.getKey(), entry.getValue()));
|
381
|
+
}
|
382
|
+
|
383
|
+
private FieldType getFieldType(String fieldCode) {
|
384
|
+
connectIfNeeded();
|
385
|
+
FieldProperty field = formFields.get(fieldCode);
|
386
|
+
return field == null ? null : field.getType();
|
377
387
|
}
|
378
388
|
|
379
|
-
private
|
380
|
-
return
|
389
|
+
private static String toString(Object value) {
|
390
|
+
return value == null
|
391
|
+
? null
|
392
|
+
: value instanceof BigDecimal ? ((BigDecimal) value).toPlainString() : value.toString();
|
381
393
|
}
|
382
394
|
}
|
@@ -41,6 +41,14 @@ public interface PluginTask extends Task {
|
|
41
41
|
@ConfigDefault("{}")
|
42
42
|
Map<String, KintoneColumnOption> getColumnOptions();
|
43
43
|
|
44
|
+
@Config("prefer_nulls")
|
45
|
+
@ConfigDefault("\"false\"")
|
46
|
+
boolean getPreferNulls();
|
47
|
+
|
48
|
+
@Config("ignore_nulls")
|
49
|
+
@ConfigDefault("\"false\"")
|
50
|
+
boolean getIgnoreNulls();
|
51
|
+
|
44
52
|
@Config("mode")
|
45
53
|
@ConfigDefault("\"insert\"")
|
46
54
|
String getMode();
|
@@ -1,6 +1,5 @@
|
|
1
1
|
package org.embulk.output.kintone;
|
2
2
|
|
3
|
-
import java.util.Optional;
|
4
3
|
import org.embulk.config.TaskSource;
|
5
4
|
|
6
5
|
public class KintoneColumnOptionBuilder {
|
@@ -42,8 +41,8 @@ public class KintoneColumnOptionBuilder {
|
|
42
41
|
}
|
43
42
|
|
44
43
|
@Override
|
45
|
-
public
|
46
|
-
return
|
44
|
+
public String getTimezone() {
|
45
|
+
return timezone;
|
47
46
|
}
|
48
47
|
|
49
48
|
@Override
|