embulk-output-kintone 0.3.2 → 0.3.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (12) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/ci.yml +11 -4
  3. data/build.gradle +21 -1
  4. data/classpath/{embulk-output-kintone-0.3.2.jar → embulk-output-kintone-0.3.6.jar} +0 -0
  5. data/src/main/java/org/embulk/output/kintone/KintoneColumnOption.java +12 -15
  6. data/src/main/java/org/embulk/output/kintone/KintoneColumnVisitor.java +186 -205
  7. data/src/main/java/org/embulk/output/kintone/KintoneMode.java +19 -21
  8. data/src/main/java/org/embulk/output/kintone/KintoneOutputPlugin.java +41 -54
  9. data/src/main/java/org/embulk/output/kintone/KintonePageOutput.java +277 -280
  10. data/src/main/java/org/embulk/output/kintone/PluginTask.java +34 -37
  11. data/src/test/java/org/embulk/output/kintone/TestKintoneOutputPlugin.java +1 -3
  12. metadata +3 -3
data/src/main/java/org/embulk/output/kintone/KintonePageOutput.java CHANGED
@@ -7,6 +7,13 @@ import com.kintone.client.model.record.FieldType;
7
7
  import com.kintone.client.model.record.Record;
8
8
  import com.kintone.client.model.record.RecordForUpdate;
9
9
  import com.kintone.client.model.record.UpdateKey;
10
+ import java.math.BigDecimal;
11
+ import java.util.ArrayList;
12
+ import java.util.Arrays;
13
+ import java.util.Collections;
14
+ import java.util.List;
15
+ import java.util.stream.Collectors;
16
+ import org.embulk.config.ConfigException;
10
17
  import org.embulk.config.TaskReport;
11
18
  import org.embulk.spi.Column;
12
19
  import org.embulk.spi.Exec;
@@ -15,308 +22,298 @@ import org.embulk.spi.PageReader;
15
22
  import org.embulk.spi.Schema;
16
23
  import org.embulk.spi.TransactionalPageOutput;
17
24
 
18
- import java.util.ArrayList;
19
- import java.util.Collections;
20
- import java.util.List;
21
- import java.util.stream.Collectors;
22
-
23
- public class KintonePageOutput
24
- implements TransactionalPageOutput
25
- {
26
- public static final int UPSERT_BATCH_SIZE = 10000;
27
- public static final int CHUNK_SIZE = 100;
28
- private final PageReader pageReader;
29
- private final PluginTask task;
30
- private KintoneClient client;
31
-
32
- public KintonePageOutput(PluginTask task, Schema schema)
33
- {
34
- this.pageReader = new PageReader(schema);
35
- this.task = task;
25
+ public class KintonePageOutput implements TransactionalPageOutput {
26
+ public static final int UPSERT_BATCH_SIZE = 10000;
27
+ public static final int CHUNK_SIZE = 100;
28
+ private final PageReader pageReader;
29
+ private final PluginTask task;
30
+ private KintoneClient client;
31
+
32
+ public KintonePageOutput(PluginTask task, Schema schema) {
33
+ this.pageReader = new PageReader(schema);
34
+ this.task = task;
35
+ }
36
+
37
+ @Override
38
+ public void add(Page page) {
39
+ KintoneMode mode = KintoneMode.getKintoneModeByValue(task.getMode());
40
+ switch (mode) {
41
+ case INSERT:
42
+ insertPage(page);
43
+ break;
44
+ case UPDATE:
45
+ updatePage(page);
46
+ break;
47
+ case UPSERT:
48
+ upsertPage(page);
49
+ break;
50
+ default:
51
+ throw new UnsupportedOperationException(String.format("Unknown mode '%s'", task.getMode()));
36
52
  }
53
+ }
37
54
 
38
- @Override
39
- public void add(Page page)
40
- {
41
- KintoneMode mode = KintoneMode.getKintoneModeByValue(task.getMode());
42
- switch (mode) {
43
- case INSERT:
44
- insertPage(page);
45
- break;
46
- case UPDATE:
47
- updatePage(page);
48
- break;
49
- case UPSERT:
50
- upsertPage(page);
51
- break;
52
- default:
53
- throw new UnsupportedOperationException(String.format(
54
- "Unknown mode '%s'",
55
- task.getMode()));
56
- }
57
- }
55
+ @Override
56
+ public void finish() {
57
+ // noop
58
+ }
58
59
 
59
- @Override
60
- public void finish()
61
- {
62
- // noop
63
- }
64
-
65
- @Override
66
- public void close()
67
- {
68
- if (this.client == null) {
69
- return;
70
- }
71
- try {
72
- this.client.close();
73
- }
74
- catch (Exception e) {
75
- throw new RuntimeException("kintone throw exception", e);
76
- }
60
+ @Override
61
+ public void close() {
62
+ if (this.client == null) {
63
+ return;
77
64
  }
78
-
79
- @Override
80
- public void abort()
81
- {
82
- // noop
65
+ try {
66
+ this.client.close();
67
+ } catch (Exception e) {
68
+ throw new RuntimeException("kintone throw exception", e);
83
69
  }
84
-
85
- @Override
86
- public TaskReport commit()
87
- {
88
- return Exec.newTaskReport();
70
+ }
71
+
72
+ @Override
73
+ public void abort() {
74
+ // noop
75
+ }
76
+
77
+ @Override
78
+ public TaskReport commit() {
79
+ return Exec.newTaskReport();
80
+ }
81
+
82
+ public interface Consumer<T> {
83
+ void accept(T t);
84
+ }
85
+
86
+ public void connect(final PluginTask task) {
87
+ KintoneClientBuilder builder = KintoneClientBuilder.create("https://" + task.getDomain());
88
+ if (task.getGuestSpaceId().isPresent()) {
89
+ builder.setGuestSpaceId(task.getGuestSpaceId().orElse(-1));
89
90
  }
90
-
91
- public interface Consumer<T>
92
- {
93
- void accept(T t);
91
+ if (task.getBasicAuthUsername().isPresent() && task.getBasicAuthPassword().isPresent()) {
92
+ builder.withBasicAuth(task.getBasicAuthUsername().get(), task.getBasicAuthPassword().get());
94
93
  }
95
94
 
96
- public void connect(final PluginTask task)
97
- {
98
- KintoneClientBuilder builder = KintoneClientBuilder.create("https://" + task.getDomain());
99
- if (task.getGuestSpaceId().isPresent()) {
100
- builder.setGuestSpaceId(task.getGuestSpaceId().orElse(-1));
101
- }
102
- if (task.getBasicAuthUsername().isPresent() && task.getBasicAuthPassword().isPresent()) {
103
- builder.withBasicAuth(task.getBasicAuthUsername().get(),
104
- task.getBasicAuthPassword().get());
105
- }
106
-
107
- if (task.getUsername().isPresent() && task.getPassword().isPresent()) {
108
- this.client = builder
109
- .authByPassword(task.getUsername().get(), task.getPassword().get())
110
- .build();
111
- }
112
- else if (task.getToken().isPresent()) {
113
- this.client = builder
114
- .authByApiToken(task.getToken().get())
115
- .build();
116
- }
95
+ if (task.getUsername().isPresent() && task.getPassword().isPresent()) {
96
+ this.client =
97
+ builder.authByPassword(task.getUsername().get(), task.getPassword().get()).build();
98
+ } else if (task.getToken().isPresent()) {
99
+ this.client = builder.authByApiToken(task.getToken().get()).build();
117
100
  }
118
-
119
- private void execute(Consumer<KintoneClient> operation)
120
- {
121
- connect(task);
122
- operation.accept(this.client);
101
+ }
102
+
103
+ private void execute(Consumer<KintoneClient> operation) {
104
+ connect(task);
105
+ if (this.client != null) {
106
+ operation.accept(this.client);
107
+ } else {
108
+ throw new RuntimeException("Failed to connect to kintone.");
123
109
  }
124
-
125
- private void insertPage(final Page page)
126
- {
127
- execute(client -> {
128
- try {
129
- ArrayList<Record> records = new ArrayList<>();
130
- pageReader.setPage(page);
131
- KintoneColumnVisitor visitor = new KintoneColumnVisitor(pageReader,
132
- task.getColumnOptions());
133
- while (pageReader.nextRecord()) {
134
- Record record = new Record();
135
- visitor.setRecord(record);
136
- for (Column column : pageReader.getSchema().getColumns()) {
137
- column.visit(visitor);
138
- }
139
-
140
- records.add(record);
141
- if (records.size() == CHUNK_SIZE) {
142
- client.record().addRecords(task.getAppId(), records);
143
- records.clear();
144
- }
145
- }
146
- if (records.size() > 0) {
147
- client.record().addRecords(task.getAppId(), records);
148
- }
110
+ }
111
+
112
+ private void insertPage(final Page page) {
113
+ execute(
114
+ client -> {
115
+ try {
116
+ ArrayList<Record> records = new ArrayList<>();
117
+ pageReader.setPage(page);
118
+ KintoneColumnVisitor visitor =
119
+ new KintoneColumnVisitor(pageReader, task.getColumnOptions());
120
+ while (pageReader.nextRecord()) {
121
+ Record record = new Record();
122
+ visitor.setRecord(record);
123
+ for (Column column : pageReader.getSchema().getColumns()) {
124
+ column.visit(visitor);
125
+ }
126
+
127
+ records.add(record);
128
+ if (records.size() == CHUNK_SIZE) {
129
+ client.record().addRecords(task.getAppId(), records);
130
+ records.clear();
131
+ }
149
132
  }
150
- catch (Exception e) {
151
- throw new RuntimeException("kintone throw exception", e);
133
+ if (records.size() > 0) {
134
+ client.record().addRecords(task.getAppId(), records);
152
135
  }
136
+ } catch (Exception e) {
137
+ throw new RuntimeException("kintone throw exception", e);
138
+ }
153
139
  });
154
- }
155
-
156
- private void updatePage(final Page page)
157
- {
158
- execute(client -> {
159
- try {
160
- ArrayList<RecordForUpdate> updateRecords = new ArrayList<>();
161
- pageReader.setPage(page);
162
-
163
- KintoneColumnVisitor visitor = new KintoneColumnVisitor(pageReader,
164
- task.getColumnOptions(),
165
- task.getUpdateKeyName().orElseThrow(() -> new RuntimeException("unreachable"))); // Already validated
166
- while (pageReader.nextRecord()) {
167
- Record record = new Record();
168
- UpdateKey updateKey = new UpdateKey();
169
- visitor.setRecord(record);
170
- visitor.setUpdateKey(updateKey);
171
- for (Column column : pageReader.getSchema().getColumns()) {
172
- column.visit(visitor);
173
- }
174
-
175
- if (updateKey.getValue() == "") {
176
- continue;
177
- }
178
-
179
- record.removeField(updateKey.getField());
180
- updateRecords.add(new RecordForUpdate(updateKey, record));
181
- if (updateRecords.size() == CHUNK_SIZE) {
182
- client.record().updateRecords(task.getAppId(), updateRecords);
183
- updateRecords.clear();
184
- }
185
- }
186
- if (updateRecords.size() > 0) {
187
- client.record().updateRecords(task.getAppId(), updateRecords);
188
- }
140
+ }
141
+
142
+ private void updatePage(final Page page) {
143
+ execute(
144
+ client -> {
145
+ try {
146
+ ArrayList<RecordForUpdate> updateRecords = new ArrayList<>();
147
+ pageReader.setPage(page);
148
+
149
+ KintoneColumnVisitor visitor =
150
+ new KintoneColumnVisitor(
151
+ pageReader,
152
+ task.getColumnOptions(),
153
+ task.getUpdateKeyName()
154
+ .orElseThrow(
155
+ () -> new RuntimeException("unreachable"))); // Already validated
156
+ while (pageReader.nextRecord()) {
157
+ Record record = new Record();
158
+ UpdateKey updateKey = new UpdateKey();
159
+ visitor.setRecord(record);
160
+ visitor.setUpdateKey(updateKey);
161
+ for (Column column : pageReader.getSchema().getColumns()) {
162
+ column.visit(visitor);
163
+ }
164
+
165
+ if (updateKey.getValue() == "") {
166
+ continue;
167
+ }
168
+
169
+ record.removeField(updateKey.getField());
170
+ updateRecords.add(new RecordForUpdate(updateKey, record));
171
+ if (updateRecords.size() == CHUNK_SIZE) {
172
+ client.record().updateRecords(task.getAppId(), updateRecords);
173
+ updateRecords.clear();
174
+ }
189
175
  }
190
- catch (Exception e) {
191
- throw new RuntimeException("kintone throw exception", e);
176
+ if (updateRecords.size() > 0) {
177
+ client.record().updateRecords(task.getAppId(), updateRecords);
192
178
  }
179
+ } catch (Exception e) {
180
+ throw new RuntimeException("kintone throw exception", e);
181
+ }
193
182
  });
194
- }
195
-
196
- private void upsertPage(final Page page)
197
- {
198
- execute(client -> {
199
- try {
200
- ArrayList<Record> records = new ArrayList<>();
201
- ArrayList<UpdateKey> updateKeys = new ArrayList<>();
202
- pageReader.setPage(page);
203
-
204
- KintoneColumnVisitor visitor = new KintoneColumnVisitor(pageReader,
205
- task.getColumnOptions(),
206
- task.getUpdateKeyName().orElseThrow(() -> new RuntimeException("unreachable"))); // Already validated
207
- while (pageReader.nextRecord()) {
208
- Record record = new Record();
209
- UpdateKey updateKey = new UpdateKey();
210
- visitor.setRecord(record);
211
- visitor.setUpdateKey(updateKey);
212
- for (Column column : pageReader.getSchema().getColumns()) {
213
- column.visit(visitor);
214
- }
215
- records.add(record);
216
- updateKeys.add(updateKey);
217
-
218
- if (records.size() == UPSERT_BATCH_SIZE) {
219
- upsert(records, updateKeys);
220
- records.clear();
221
- updateKeys.clear();
222
- }
223
- }
224
- if (records.size() > 0) {
225
- upsert(records, updateKeys);
226
- }
183
+ }
184
+
185
+ private void upsertPage(final Page page) {
186
+ execute(
187
+ client -> {
188
+ try {
189
+ ArrayList<Record> records = new ArrayList<>();
190
+ ArrayList<UpdateKey> updateKeys = new ArrayList<>();
191
+ pageReader.setPage(page);
192
+
193
+ KintoneColumnVisitor visitor =
194
+ new KintoneColumnVisitor(
195
+ pageReader,
196
+ task.getColumnOptions(),
197
+ task.getUpdateKeyName()
198
+ .orElseThrow(
199
+ () -> new RuntimeException("unreachable"))); // Already validated
200
+ while (pageReader.nextRecord()) {
201
+ Record record = new Record();
202
+ UpdateKey updateKey = new UpdateKey();
203
+ visitor.setRecord(record);
204
+ visitor.setUpdateKey(updateKey);
205
+ for (Column column : pageReader.getSchema().getColumns()) {
206
+ column.visit(visitor);
207
+ }
208
+ records.add(record);
209
+ updateKeys.add(updateKey);
210
+
211
+ if (records.size() == UPSERT_BATCH_SIZE) {
212
+ upsert(records, updateKeys);
213
+ records.clear();
214
+ updateKeys.clear();
215
+ }
227
216
  }
228
- catch (Exception e) {
229
- throw new RuntimeException("kintone throw exception", e);
217
+ if (records.size() > 0) {
218
+ upsert(records, updateKeys);
230
219
  }
220
+ } catch (Exception e) {
221
+ throw new RuntimeException("kintone throw exception", e);
222
+ }
231
223
  });
232
- }
233
-
234
- private void upsert(ArrayList<Record> records, ArrayList<UpdateKey> updateKeys)
235
- {
236
- if (records.size() != updateKeys.size()) {
237
- throw new RuntimeException("records.size() != updateKeys.size()");
238
- }
239
-
240
- List<Record> existingRecords = getExistingRecordsByUpdateKey(updateKeys);
241
-
242
- ArrayList<Record> insertRecords = new ArrayList<>();
243
- ArrayList<RecordForUpdate> updateRecords = new ArrayList<>();
244
- for (int i = 0; i < records.size(); i++) {
245
- Record record = records.get(i);
246
- UpdateKey updateKey = updateKeys.get(i);
247
-
248
- if (existsRecord(existingRecords, updateKey)) {
249
- record.removeField(updateKey.getField());
250
- updateRecords.add(new RecordForUpdate(updateKey, record));
251
- }
252
- else {
253
- insertRecords.add(record);
254
- }
224
+ }
255
225
 
256
- if (insertRecords.size() == CHUNK_SIZE) {
257
- client.record().addRecords(task.getAppId(), insertRecords);
258
- insertRecords.clear();
259
- }
260
- else if (updateRecords.size() == CHUNK_SIZE) {
261
- client.record().updateRecords(task.getAppId(), updateRecords);
262
- updateRecords.clear();
263
- }
264
- }
265
- if (insertRecords.size() > 0) {
266
- client.record().addRecords(task.getAppId(), insertRecords);
267
- }
268
- if (updateRecords.size() > 0) {
269
- client.record().updateRecords(task.getAppId(), updateRecords);
270
- }
226
+ private void upsert(ArrayList<Record> records, ArrayList<UpdateKey> updateKeys) {
227
+ if (records.size() != updateKeys.size()) {
228
+ throw new RuntimeException("records.size() != updateKeys.size()");
271
229
  }
272
-
273
-
274
- private List<Record> getExistingRecordsByUpdateKey(ArrayList<UpdateKey> updateKeys)
275
- {
276
- String fieldCode = updateKeys.get(0).getField();
277
- List<String> queryValues = updateKeys
278
- .stream()
279
- .filter(k -> k.getValue() != "")
280
- .map(k -> "\"" + k.getValue().toString() + "\"")
281
- .collect(Collectors.toList());
282
-
283
- List<Record> allRecords = new ArrayList<>();
284
- if (queryValues.isEmpty()) {
285
- return allRecords;
286
- }
287
- String cursorId = client.record().createCursor(
288
- task.getAppId(),
289
- Collections.singletonList(fieldCode),
290
- fieldCode + " in (" + String.join(",", queryValues) + ")"
291
- );
292
- while (true) {
293
- GetRecordsByCursorResponseBody resp = client.record().getRecordsByCursor(cursorId);
294
- List<Record> records = resp.getRecords();
295
- allRecords.addAll(records);
296
-
297
- if (!resp.hasNext()) {
298
- break;
299
- }
300
- }
301
- return allRecords;
230
+ FieldType updateKeyFieldType =
231
+ client.app().getFormFields(task.getAppId()).get(updateKeys.get(0).getField()).getType();
232
+ if (!Arrays.asList(FieldType.SINGLE_LINE_TEXT, FieldType.NUMBER).contains(updateKeyFieldType)) {
233
+ throw new ConfigException("The update_key must be 'SINGLE_LINE_TEXT' or 'NUMBER'.");
302
234
  }
303
235
 
304
- private boolean existsRecord(List<Record> distRecords, UpdateKey updateKey)
305
- {
306
- String fieldCode = updateKey.getField();
307
- FieldType type = client.app().getFormFields(task.getAppId()).get(fieldCode).getType();
308
- switch (type) {
309
- case SINGLE_LINE_TEXT:
310
- return distRecords
311
- .stream()
312
- .anyMatch(d -> d.getSingleLineTextFieldValue(fieldCode).equals(updateKey.getValue().toString()));
313
- case NUMBER:
314
- return distRecords
315
- .stream()
316
- .anyMatch(d -> d.getNumberFieldValue(fieldCode).toPlainString()
317
- .equals(updateKey.getValue().toString()));
318
- default:
319
- throw new RuntimeException("The update_key must be 'SINGLE_LINE_TEXT' or 'NUMBER'.");
320
- }
236
+ List<Record> existingRecords = getExistingRecordsByUpdateKey(updateKeys);
237
+ String updateField = updateKeys.get(0).getField();
238
+ List<String> existingValues =
239
+ existingRecords.stream()
240
+ .map(
241
+ (r) -> {
242
+ switch (updateKeyFieldType) {
243
+ case SINGLE_LINE_TEXT:
244
+ String s = r.getSingleLineTextFieldValue(updateField);
245
+ return s == null ? null : s.toString();
246
+ case NUMBER:
247
+ BigDecimal bd = r.getNumberFieldValue(updateField);
248
+ return bd == null ? null : bd.toPlainString();
249
+ default:
250
+ return null;
251
+ }
252
+ })
253
+ .filter(v -> v != null)
254
+ .collect(Collectors.toList());
255
+
256
+ ArrayList<Record> insertRecords = new ArrayList<>();
257
+ ArrayList<RecordForUpdate> updateRecords = new ArrayList<>();
258
+ for (int i = 0; i < records.size(); i++) {
259
+ Record record = records.get(i);
260
+ UpdateKey updateKey = updateKeys.get(i);
261
+
262
+ if (existsRecord(existingValues, updateKey)) {
263
+ record.removeField(updateKey.getField());
264
+ updateRecords.add(new RecordForUpdate(updateKey, record));
265
+ } else {
266
+ insertRecords.add(record);
267
+ }
268
+
269
+ if (insertRecords.size() == CHUNK_SIZE) {
270
+ client.record().addRecords(task.getAppId(), insertRecords);
271
+ insertRecords.clear();
272
+ } else if (updateRecords.size() == CHUNK_SIZE) {
273
+ client.record().updateRecords(task.getAppId(), updateRecords);
274
+ updateRecords.clear();
275
+ }
276
+ }
277
+ if (insertRecords.size() > 0) {
278
+ client.record().addRecords(task.getAppId(), insertRecords);
279
+ }
280
+ if (updateRecords.size() > 0) {
281
+ client.record().updateRecords(task.getAppId(), updateRecords);
321
282
  }
283
+ }
284
+
285
+ private List<Record> getExistingRecordsByUpdateKey(ArrayList<UpdateKey> updateKeys) {
286
+ String fieldCode = updateKeys.get(0).getField();
287
+ List<String> queryValues =
288
+ updateKeys.stream()
289
+ .filter(k -> k.getValue() != "")
290
+ .map(k -> "\"" + k.getValue().toString() + "\"")
291
+ .collect(Collectors.toList());
292
+
293
+ List<Record> allRecords = new ArrayList<>();
294
+ if (queryValues.isEmpty()) {
295
+ return allRecords;
296
+ }
297
+ String cursorId =
298
+ client
299
+ .record()
300
+ .createCursor(
301
+ task.getAppId(),
302
+ Collections.singletonList(fieldCode),
303
+ fieldCode + " in (" + String.join(",", queryValues) + ")");
304
+ while (true) {
305
+ GetRecordsByCursorResponseBody resp = client.record().getRecordsByCursor(cursorId);
306
+ List<Record> records = resp.getRecords();
307
+ allRecords.addAll(records);
308
+
309
+ if (!resp.hasNext()) {
310
+ break;
311
+ }
312
+ }
313
+ return allRecords;
314
+ }
315
+
316
+ private boolean existsRecord(List<String> distValues, UpdateKey updateKey) {
317
+ return distValues.stream().anyMatch(v -> v.equals(updateKey.getValue().toString()));
318
+ }
322
319
  }