embulk-output-kintone 0.3.3 → 0.3.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/ci.yml +11 -4
- data/build.gradle +11 -0
- data/classpath/{embulk-output-kintone-0.3.3.jar → embulk-output-kintone-0.3.6.jar} +0 -0
- data/src/main/java/org/embulk/output/kintone/KintoneColumnOption.java +12 -15
- data/src/main/java/org/embulk/output/kintone/KintoneColumnVisitor.java +186 -211
- data/src/main/java/org/embulk/output/kintone/KintoneMode.java +19 -21
- data/src/main/java/org/embulk/output/kintone/KintoneOutputPlugin.java +41 -54
- data/src/main/java/org/embulk/output/kintone/KintonePageOutput.java +277 -284
- data/src/main/java/org/embulk/output/kintone/PluginTask.java +34 -37
- data/src/test/java/org/embulk/output/kintone/TestKintoneOutputPlugin.java +1 -3
- metadata +3 -3
@@ -7,6 +7,13 @@ import com.kintone.client.model.record.FieldType;
|
|
7
7
|
import com.kintone.client.model.record.Record;
|
8
8
|
import com.kintone.client.model.record.RecordForUpdate;
|
9
9
|
import com.kintone.client.model.record.UpdateKey;
|
10
|
+
import java.math.BigDecimal;
|
11
|
+
import java.util.ArrayList;
|
12
|
+
import java.util.Arrays;
|
13
|
+
import java.util.Collections;
|
14
|
+
import java.util.List;
|
15
|
+
import java.util.stream.Collectors;
|
16
|
+
import org.embulk.config.ConfigException;
|
10
17
|
import org.embulk.config.TaskReport;
|
11
18
|
import org.embulk.spi.Column;
|
12
19
|
import org.embulk.spi.Exec;
|
@@ -15,312 +22,298 @@ import org.embulk.spi.PageReader;
|
|
15
22
|
import org.embulk.spi.Schema;
|
16
23
|
import org.embulk.spi.TransactionalPageOutput;
|
17
24
|
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
{
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
{
|
34
|
-
|
35
|
-
|
25
|
+
public class KintonePageOutput implements TransactionalPageOutput {
|
26
|
+
public static final int UPSERT_BATCH_SIZE = 10000;
|
27
|
+
public static final int CHUNK_SIZE = 100;
|
28
|
+
private final PageReader pageReader;
|
29
|
+
private final PluginTask task;
|
30
|
+
private KintoneClient client;
|
31
|
+
|
32
|
+
public KintonePageOutput(PluginTask task, Schema schema) {
|
33
|
+
this.pageReader = new PageReader(schema);
|
34
|
+
this.task = task;
|
35
|
+
}
|
36
|
+
|
37
|
+
@Override
|
38
|
+
public void add(Page page) {
|
39
|
+
KintoneMode mode = KintoneMode.getKintoneModeByValue(task.getMode());
|
40
|
+
switch (mode) {
|
41
|
+
case INSERT:
|
42
|
+
insertPage(page);
|
43
|
+
break;
|
44
|
+
case UPDATE:
|
45
|
+
updatePage(page);
|
46
|
+
break;
|
47
|
+
case UPSERT:
|
48
|
+
upsertPage(page);
|
49
|
+
break;
|
50
|
+
default:
|
51
|
+
throw new UnsupportedOperationException(String.format("Unknown mode '%s'", task.getMode()));
|
36
52
|
}
|
53
|
+
}
|
37
54
|
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
switch (mode) {
|
43
|
-
case INSERT:
|
44
|
-
insertPage(page);
|
45
|
-
break;
|
46
|
-
case UPDATE:
|
47
|
-
updatePage(page);
|
48
|
-
break;
|
49
|
-
case UPSERT:
|
50
|
-
upsertPage(page);
|
51
|
-
break;
|
52
|
-
default:
|
53
|
-
throw new UnsupportedOperationException(String.format(
|
54
|
-
"Unknown mode '%s'",
|
55
|
-
task.getMode()));
|
56
|
-
}
|
57
|
-
}
|
55
|
+
@Override
|
56
|
+
public void finish() {
|
57
|
+
// noop
|
58
|
+
}
|
58
59
|
|
59
|
-
|
60
|
-
|
61
|
-
{
|
62
|
-
|
63
|
-
}
|
64
|
-
|
65
|
-
@Override
|
66
|
-
public void close()
|
67
|
-
{
|
68
|
-
if (this.client == null) {
|
69
|
-
return;
|
70
|
-
}
|
71
|
-
try {
|
72
|
-
this.client.close();
|
73
|
-
}
|
74
|
-
catch (Exception e) {
|
75
|
-
throw new RuntimeException("kintone throw exception", e);
|
76
|
-
}
|
60
|
+
@Override
|
61
|
+
public void close() {
|
62
|
+
if (this.client == null) {
|
63
|
+
return;
|
77
64
|
}
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
// noop
|
65
|
+
try {
|
66
|
+
this.client.close();
|
67
|
+
} catch (Exception e) {
|
68
|
+
throw new RuntimeException("kintone throw exception", e);
|
83
69
|
}
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
70
|
+
}
|
71
|
+
|
72
|
+
@Override
|
73
|
+
public void abort() {
|
74
|
+
// noop
|
75
|
+
}
|
76
|
+
|
77
|
+
@Override
|
78
|
+
public TaskReport commit() {
|
79
|
+
return Exec.newTaskReport();
|
80
|
+
}
|
81
|
+
|
82
|
+
public interface Consumer<T> {
|
83
|
+
void accept(T t);
|
84
|
+
}
|
85
|
+
|
86
|
+
public void connect(final PluginTask task) {
|
87
|
+
KintoneClientBuilder builder = KintoneClientBuilder.create("https://" + task.getDomain());
|
88
|
+
if (task.getGuestSpaceId().isPresent()) {
|
89
|
+
builder.setGuestSpaceId(task.getGuestSpaceId().orElse(-1));
|
89
90
|
}
|
90
|
-
|
91
|
-
|
92
|
-
{
|
93
|
-
void accept(T t);
|
91
|
+
if (task.getBasicAuthUsername().isPresent() && task.getBasicAuthPassword().isPresent()) {
|
92
|
+
builder.withBasicAuth(task.getBasicAuthUsername().get(), task.getBasicAuthPassword().get());
|
94
93
|
}
|
95
94
|
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
}
|
102
|
-
if (task.getBasicAuthUsername().isPresent() && task.getBasicAuthPassword().isPresent()) {
|
103
|
-
builder.withBasicAuth(task.getBasicAuthUsername().get(),
|
104
|
-
task.getBasicAuthPassword().get());
|
105
|
-
}
|
106
|
-
|
107
|
-
if (task.getUsername().isPresent() && task.getPassword().isPresent()) {
|
108
|
-
this.client = builder
|
109
|
-
.authByPassword(task.getUsername().get(), task.getPassword().get())
|
110
|
-
.build();
|
111
|
-
}
|
112
|
-
else if (task.getToken().isPresent()) {
|
113
|
-
this.client = builder
|
114
|
-
.authByApiToken(task.getToken().get())
|
115
|
-
.build();
|
116
|
-
}
|
95
|
+
if (task.getUsername().isPresent() && task.getPassword().isPresent()) {
|
96
|
+
this.client =
|
97
|
+
builder.authByPassword(task.getUsername().get(), task.getPassword().get()).build();
|
98
|
+
} else if (task.getToken().isPresent()) {
|
99
|
+
this.client = builder.authByApiToken(task.getToken().get()).build();
|
117
100
|
}
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
}
|
101
|
+
}
|
102
|
+
|
103
|
+
private void execute(Consumer<KintoneClient> operation) {
|
104
|
+
connect(task);
|
105
|
+
if (this.client != null) {
|
106
|
+
operation.accept(this.client);
|
107
|
+
} else {
|
108
|
+
throw new RuntimeException("Failed to connect to kintone.");
|
127
109
|
}
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
|
148
|
-
|
149
|
-
|
150
|
-
if (records.size() > 0) {
|
151
|
-
client.record().addRecords(task.getAppId(), records);
|
152
|
-
}
|
110
|
+
}
|
111
|
+
|
112
|
+
private void insertPage(final Page page) {
|
113
|
+
execute(
|
114
|
+
client -> {
|
115
|
+
try {
|
116
|
+
ArrayList<Record> records = new ArrayList<>();
|
117
|
+
pageReader.setPage(page);
|
118
|
+
KintoneColumnVisitor visitor =
|
119
|
+
new KintoneColumnVisitor(pageReader, task.getColumnOptions());
|
120
|
+
while (pageReader.nextRecord()) {
|
121
|
+
Record record = new Record();
|
122
|
+
visitor.setRecord(record);
|
123
|
+
for (Column column : pageReader.getSchema().getColumns()) {
|
124
|
+
column.visit(visitor);
|
125
|
+
}
|
126
|
+
|
127
|
+
records.add(record);
|
128
|
+
if (records.size() == CHUNK_SIZE) {
|
129
|
+
client.record().addRecords(task.getAppId(), records);
|
130
|
+
records.clear();
|
131
|
+
}
|
153
132
|
}
|
154
|
-
|
155
|
-
|
133
|
+
if (records.size() > 0) {
|
134
|
+
client.record().addRecords(task.getAppId(), records);
|
156
135
|
}
|
136
|
+
} catch (Exception e) {
|
137
|
+
throw new RuntimeException("kintone throw exception", e);
|
138
|
+
}
|
157
139
|
});
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
168
|
-
|
169
|
-
|
170
|
-
|
171
|
-
|
172
|
-
|
173
|
-
|
174
|
-
|
175
|
-
|
176
|
-
|
177
|
-
|
178
|
-
|
179
|
-
|
180
|
-
|
181
|
-
|
182
|
-
|
183
|
-
|
184
|
-
|
185
|
-
|
186
|
-
|
187
|
-
|
188
|
-
|
189
|
-
|
190
|
-
|
191
|
-
|
192
|
-
|
140
|
+
}
|
141
|
+
|
142
|
+
private void updatePage(final Page page) {
|
143
|
+
execute(
|
144
|
+
client -> {
|
145
|
+
try {
|
146
|
+
ArrayList<RecordForUpdate> updateRecords = new ArrayList<>();
|
147
|
+
pageReader.setPage(page);
|
148
|
+
|
149
|
+
KintoneColumnVisitor visitor =
|
150
|
+
new KintoneColumnVisitor(
|
151
|
+
pageReader,
|
152
|
+
task.getColumnOptions(),
|
153
|
+
task.getUpdateKeyName()
|
154
|
+
.orElseThrow(
|
155
|
+
() -> new RuntimeException("unreachable"))); // Already validated
|
156
|
+
while (pageReader.nextRecord()) {
|
157
|
+
Record record = new Record();
|
158
|
+
UpdateKey updateKey = new UpdateKey();
|
159
|
+
visitor.setRecord(record);
|
160
|
+
visitor.setUpdateKey(updateKey);
|
161
|
+
for (Column column : pageReader.getSchema().getColumns()) {
|
162
|
+
column.visit(visitor);
|
163
|
+
}
|
164
|
+
|
165
|
+
if (updateKey.getValue() == "") {
|
166
|
+
continue;
|
167
|
+
}
|
168
|
+
|
169
|
+
record.removeField(updateKey.getField());
|
170
|
+
updateRecords.add(new RecordForUpdate(updateKey, record));
|
171
|
+
if (updateRecords.size() == CHUNK_SIZE) {
|
172
|
+
client.record().updateRecords(task.getAppId(), updateRecords);
|
173
|
+
updateRecords.clear();
|
174
|
+
}
|
193
175
|
}
|
194
|
-
|
195
|
-
|
176
|
+
if (updateRecords.size() > 0) {
|
177
|
+
client.record().updateRecords(task.getAppId(), updateRecords);
|
196
178
|
}
|
179
|
+
} catch (Exception e) {
|
180
|
+
throw new RuntimeException("kintone throw exception", e);
|
181
|
+
}
|
197
182
|
});
|
198
|
-
|
199
|
-
|
200
|
-
|
201
|
-
|
202
|
-
|
203
|
-
|
204
|
-
|
205
|
-
|
206
|
-
|
207
|
-
|
208
|
-
|
209
|
-
|
210
|
-
|
211
|
-
|
212
|
-
|
213
|
-
|
214
|
-
|
215
|
-
|
216
|
-
|
217
|
-
|
218
|
-
|
219
|
-
|
220
|
-
|
221
|
-
|
222
|
-
|
223
|
-
|
224
|
-
|
225
|
-
|
226
|
-
|
227
|
-
|
228
|
-
|
229
|
-
|
230
|
-
|
183
|
+
}
|
184
|
+
|
185
|
+
private void upsertPage(final Page page) {
|
186
|
+
execute(
|
187
|
+
client -> {
|
188
|
+
try {
|
189
|
+
ArrayList<Record> records = new ArrayList<>();
|
190
|
+
ArrayList<UpdateKey> updateKeys = new ArrayList<>();
|
191
|
+
pageReader.setPage(page);
|
192
|
+
|
193
|
+
KintoneColumnVisitor visitor =
|
194
|
+
new KintoneColumnVisitor(
|
195
|
+
pageReader,
|
196
|
+
task.getColumnOptions(),
|
197
|
+
task.getUpdateKeyName()
|
198
|
+
.orElseThrow(
|
199
|
+
() -> new RuntimeException("unreachable"))); // Already validated
|
200
|
+
while (pageReader.nextRecord()) {
|
201
|
+
Record record = new Record();
|
202
|
+
UpdateKey updateKey = new UpdateKey();
|
203
|
+
visitor.setRecord(record);
|
204
|
+
visitor.setUpdateKey(updateKey);
|
205
|
+
for (Column column : pageReader.getSchema().getColumns()) {
|
206
|
+
column.visit(visitor);
|
207
|
+
}
|
208
|
+
records.add(record);
|
209
|
+
updateKeys.add(updateKey);
|
210
|
+
|
211
|
+
if (records.size() == UPSERT_BATCH_SIZE) {
|
212
|
+
upsert(records, updateKeys);
|
213
|
+
records.clear();
|
214
|
+
updateKeys.clear();
|
215
|
+
}
|
231
216
|
}
|
232
|
-
|
233
|
-
|
217
|
+
if (records.size() > 0) {
|
218
|
+
upsert(records, updateKeys);
|
234
219
|
}
|
220
|
+
} catch (Exception e) {
|
221
|
+
throw new RuntimeException("kintone throw exception", e);
|
222
|
+
}
|
235
223
|
});
|
236
|
-
|
237
|
-
|
238
|
-
private void upsert(ArrayList<Record> records, ArrayList<UpdateKey> updateKeys)
|
239
|
-
{
|
240
|
-
if (records.size() != updateKeys.size()) {
|
241
|
-
throw new RuntimeException("records.size() != updateKeys.size()");
|
242
|
-
}
|
243
|
-
|
244
|
-
List<Record> existingRecords = getExistingRecordsByUpdateKey(updateKeys);
|
245
|
-
|
246
|
-
ArrayList<Record> insertRecords = new ArrayList<>();
|
247
|
-
ArrayList<RecordForUpdate> updateRecords = new ArrayList<>();
|
248
|
-
for (int i = 0; i < records.size(); i++) {
|
249
|
-
Record record = records.get(i);
|
250
|
-
UpdateKey updateKey = updateKeys.get(i);
|
251
|
-
|
252
|
-
if (existsRecord(existingRecords, updateKey)) {
|
253
|
-
record.removeField(updateKey.getField());
|
254
|
-
updateRecords.add(new RecordForUpdate(updateKey, record));
|
255
|
-
}
|
256
|
-
else {
|
257
|
-
insertRecords.add(record);
|
258
|
-
}
|
224
|
+
}
|
259
225
|
|
260
|
-
|
261
|
-
|
262
|
-
|
263
|
-
}
|
264
|
-
else if (updateRecords.size() == CHUNK_SIZE) {
|
265
|
-
client.record().updateRecords(task.getAppId(), updateRecords);
|
266
|
-
updateRecords.clear();
|
267
|
-
}
|
268
|
-
}
|
269
|
-
if (insertRecords.size() > 0) {
|
270
|
-
client.record().addRecords(task.getAppId(), insertRecords);
|
271
|
-
}
|
272
|
-
if (updateRecords.size() > 0) {
|
273
|
-
client.record().updateRecords(task.getAppId(), updateRecords);
|
274
|
-
}
|
226
|
+
private void upsert(ArrayList<Record> records, ArrayList<UpdateKey> updateKeys) {
|
227
|
+
if (records.size() != updateKeys.size()) {
|
228
|
+
throw new RuntimeException("records.size() != updateKeys.size()");
|
275
229
|
}
|
276
|
-
|
277
|
-
|
278
|
-
|
279
|
-
|
280
|
-
String fieldCode = updateKeys.get(0).getField();
|
281
|
-
List<String> queryValues = updateKeys
|
282
|
-
.stream()
|
283
|
-
.filter(k -> k.getValue() != "")
|
284
|
-
.map(k -> "\"" + k.getValue().toString() + "\"")
|
285
|
-
.collect(Collectors.toList());
|
286
|
-
|
287
|
-
List<Record> allRecords = new ArrayList<>();
|
288
|
-
if (queryValues.isEmpty()) {
|
289
|
-
return allRecords;
|
290
|
-
}
|
291
|
-
String cursorId = client.record().createCursor(
|
292
|
-
task.getAppId(),
|
293
|
-
Collections.singletonList(fieldCode),
|
294
|
-
fieldCode + " in (" + String.join(",", queryValues) + ")"
|
295
|
-
);
|
296
|
-
while (true) {
|
297
|
-
GetRecordsByCursorResponseBody resp = client.record().getRecordsByCursor(cursorId);
|
298
|
-
List<Record> records = resp.getRecords();
|
299
|
-
allRecords.addAll(records);
|
300
|
-
|
301
|
-
if (!resp.hasNext()) {
|
302
|
-
break;
|
303
|
-
}
|
304
|
-
}
|
305
|
-
return allRecords;
|
230
|
+
FieldType updateKeyFieldType =
|
231
|
+
client.app().getFormFields(task.getAppId()).get(updateKeys.get(0).getField()).getType();
|
232
|
+
if (!Arrays.asList(FieldType.SINGLE_LINE_TEXT, FieldType.NUMBER).contains(updateKeyFieldType)) {
|
233
|
+
throw new ConfigException("The update_key must be 'SINGLE_LINE_TEXT' or 'NUMBER'.");
|
306
234
|
}
|
307
235
|
|
308
|
-
|
309
|
-
|
310
|
-
|
311
|
-
|
312
|
-
|
313
|
-
|
314
|
-
|
315
|
-
|
316
|
-
|
317
|
-
|
318
|
-
|
319
|
-
|
320
|
-
|
321
|
-
|
322
|
-
|
323
|
-
|
324
|
-
|
236
|
+
List<Record> existingRecords = getExistingRecordsByUpdateKey(updateKeys);
|
237
|
+
String updateField = updateKeys.get(0).getField();
|
238
|
+
List<String> existingValues =
|
239
|
+
existingRecords.stream()
|
240
|
+
.map(
|
241
|
+
(r) -> {
|
242
|
+
switch (updateKeyFieldType) {
|
243
|
+
case SINGLE_LINE_TEXT:
|
244
|
+
String s = r.getSingleLineTextFieldValue(updateField);
|
245
|
+
return s == null ? null : s.toString();
|
246
|
+
case NUMBER:
|
247
|
+
BigDecimal bd = r.getNumberFieldValue(updateField);
|
248
|
+
return bd == null ? null : bd.toPlainString();
|
249
|
+
default:
|
250
|
+
return null;
|
251
|
+
}
|
252
|
+
})
|
253
|
+
.filter(v -> v != null)
|
254
|
+
.collect(Collectors.toList());
|
255
|
+
|
256
|
+
ArrayList<Record> insertRecords = new ArrayList<>();
|
257
|
+
ArrayList<RecordForUpdate> updateRecords = new ArrayList<>();
|
258
|
+
for (int i = 0; i < records.size(); i++) {
|
259
|
+
Record record = records.get(i);
|
260
|
+
UpdateKey updateKey = updateKeys.get(i);
|
261
|
+
|
262
|
+
if (existsRecord(existingValues, updateKey)) {
|
263
|
+
record.removeField(updateKey.getField());
|
264
|
+
updateRecords.add(new RecordForUpdate(updateKey, record));
|
265
|
+
} else {
|
266
|
+
insertRecords.add(record);
|
267
|
+
}
|
268
|
+
|
269
|
+
if (insertRecords.size() == CHUNK_SIZE) {
|
270
|
+
client.record().addRecords(task.getAppId(), insertRecords);
|
271
|
+
insertRecords.clear();
|
272
|
+
} else if (updateRecords.size() == CHUNK_SIZE) {
|
273
|
+
client.record().updateRecords(task.getAppId(), updateRecords);
|
274
|
+
updateRecords.clear();
|
275
|
+
}
|
276
|
+
}
|
277
|
+
if (insertRecords.size() > 0) {
|
278
|
+
client.record().addRecords(task.getAppId(), insertRecords);
|
279
|
+
}
|
280
|
+
if (updateRecords.size() > 0) {
|
281
|
+
client.record().updateRecords(task.getAppId(), updateRecords);
|
325
282
|
}
|
283
|
+
}
|
284
|
+
|
285
|
+
private List<Record> getExistingRecordsByUpdateKey(ArrayList<UpdateKey> updateKeys) {
|
286
|
+
String fieldCode = updateKeys.get(0).getField();
|
287
|
+
List<String> queryValues =
|
288
|
+
updateKeys.stream()
|
289
|
+
.filter(k -> k.getValue() != "")
|
290
|
+
.map(k -> "\"" + k.getValue().toString() + "\"")
|
291
|
+
.collect(Collectors.toList());
|
292
|
+
|
293
|
+
List<Record> allRecords = new ArrayList<>();
|
294
|
+
if (queryValues.isEmpty()) {
|
295
|
+
return allRecords;
|
296
|
+
}
|
297
|
+
String cursorId =
|
298
|
+
client
|
299
|
+
.record()
|
300
|
+
.createCursor(
|
301
|
+
task.getAppId(),
|
302
|
+
Collections.singletonList(fieldCode),
|
303
|
+
fieldCode + " in (" + String.join(",", queryValues) + ")");
|
304
|
+
while (true) {
|
305
|
+
GetRecordsByCursorResponseBody resp = client.record().getRecordsByCursor(cursorId);
|
306
|
+
List<Record> records = resp.getRecords();
|
307
|
+
allRecords.addAll(records);
|
308
|
+
|
309
|
+
if (!resp.hasNext()) {
|
310
|
+
break;
|
311
|
+
}
|
312
|
+
}
|
313
|
+
return allRecords;
|
314
|
+
}
|
315
|
+
|
316
|
+
private boolean existsRecord(List<String> distValues, UpdateKey updateKey) {
|
317
|
+
return distValues.stream().anyMatch(v -> v.equals(updateKey.getValue().toString()));
|
318
|
+
}
|
326
319
|
}
|