embulk-output-kintone 0.3.3 → 0.3.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -7,6 +7,13 @@ import com.kintone.client.model.record.FieldType;
7
7
  import com.kintone.client.model.record.Record;
8
8
  import com.kintone.client.model.record.RecordForUpdate;
9
9
  import com.kintone.client.model.record.UpdateKey;
10
+ import java.math.BigDecimal;
11
+ import java.util.ArrayList;
12
+ import java.util.Arrays;
13
+ import java.util.Collections;
14
+ import java.util.List;
15
+ import java.util.stream.Collectors;
16
+ import org.embulk.config.ConfigException;
10
17
  import org.embulk.config.TaskReport;
11
18
  import org.embulk.spi.Column;
12
19
  import org.embulk.spi.Exec;
@@ -15,312 +22,298 @@ import org.embulk.spi.PageReader;
15
22
  import org.embulk.spi.Schema;
16
23
  import org.embulk.spi.TransactionalPageOutput;
17
24
 
18
- import java.util.ArrayList;
19
- import java.util.Collections;
20
- import java.util.List;
21
- import java.util.stream.Collectors;
22
-
23
- public class KintonePageOutput
24
- implements TransactionalPageOutput
25
- {
26
- public static final int UPSERT_BATCH_SIZE = 10000;
27
- public static final int CHUNK_SIZE = 100;
28
- private final PageReader pageReader;
29
- private final PluginTask task;
30
- private KintoneClient client;
31
-
32
- public KintonePageOutput(PluginTask task, Schema schema)
33
- {
34
- this.pageReader = new PageReader(schema);
35
- this.task = task;
25
+ public class KintonePageOutput implements TransactionalPageOutput {
26
+ public static final int UPSERT_BATCH_SIZE = 10000;
27
+ public static final int CHUNK_SIZE = 100;
28
+ private final PageReader pageReader;
29
+ private final PluginTask task;
30
+ private KintoneClient client;
31
+
32
+ public KintonePageOutput(PluginTask task, Schema schema) {
33
+ this.pageReader = new PageReader(schema);
34
+ this.task = task;
35
+ }
36
+
37
+ @Override
38
+ public void add(Page page) {
39
+ KintoneMode mode = KintoneMode.getKintoneModeByValue(task.getMode());
40
+ switch (mode) {
41
+ case INSERT:
42
+ insertPage(page);
43
+ break;
44
+ case UPDATE:
45
+ updatePage(page);
46
+ break;
47
+ case UPSERT:
48
+ upsertPage(page);
49
+ break;
50
+ default:
51
+ throw new UnsupportedOperationException(String.format("Unknown mode '%s'", task.getMode()));
36
52
  }
53
+ }
37
54
 
38
- @Override
39
- public void add(Page page)
40
- {
41
- KintoneMode mode = KintoneMode.getKintoneModeByValue(task.getMode());
42
- switch (mode) {
43
- case INSERT:
44
- insertPage(page);
45
- break;
46
- case UPDATE:
47
- updatePage(page);
48
- break;
49
- case UPSERT:
50
- upsertPage(page);
51
- break;
52
- default:
53
- throw new UnsupportedOperationException(String.format(
54
- "Unknown mode '%s'",
55
- task.getMode()));
56
- }
57
- }
55
+ @Override
56
+ public void finish() {
57
+ // noop
58
+ }
58
59
 
59
- @Override
60
- public void finish()
61
- {
62
- // noop
63
- }
64
-
65
- @Override
66
- public void close()
67
- {
68
- if (this.client == null) {
69
- return;
70
- }
71
- try {
72
- this.client.close();
73
- }
74
- catch (Exception e) {
75
- throw new RuntimeException("kintone throw exception", e);
76
- }
60
+ @Override
61
+ public void close() {
62
+ if (this.client == null) {
63
+ return;
77
64
  }
78
-
79
- @Override
80
- public void abort()
81
- {
82
- // noop
65
+ try {
66
+ this.client.close();
67
+ } catch (Exception e) {
68
+ throw new RuntimeException("kintone throw exception", e);
83
69
  }
84
-
85
- @Override
86
- public TaskReport commit()
87
- {
88
- return Exec.newTaskReport();
70
+ }
71
+
72
+ @Override
73
+ public void abort() {
74
+ // noop
75
+ }
76
+
77
+ @Override
78
+ public TaskReport commit() {
79
+ return Exec.newTaskReport();
80
+ }
81
+
82
+ public interface Consumer<T> {
83
+ void accept(T t);
84
+ }
85
+
86
+ public void connect(final PluginTask task) {
87
+ KintoneClientBuilder builder = KintoneClientBuilder.create("https://" + task.getDomain());
88
+ if (task.getGuestSpaceId().isPresent()) {
89
+ builder.setGuestSpaceId(task.getGuestSpaceId().orElse(-1));
89
90
  }
90
-
91
- public interface Consumer<T>
92
- {
93
- void accept(T t);
91
+ if (task.getBasicAuthUsername().isPresent() && task.getBasicAuthPassword().isPresent()) {
92
+ builder.withBasicAuth(task.getBasicAuthUsername().get(), task.getBasicAuthPassword().get());
94
93
  }
95
94
 
96
- public void connect(final PluginTask task)
97
- {
98
- KintoneClientBuilder builder = KintoneClientBuilder.create("https://" + task.getDomain());
99
- if (task.getGuestSpaceId().isPresent()) {
100
- builder.setGuestSpaceId(task.getGuestSpaceId().orElse(-1));
101
- }
102
- if (task.getBasicAuthUsername().isPresent() && task.getBasicAuthPassword().isPresent()) {
103
- builder.withBasicAuth(task.getBasicAuthUsername().get(),
104
- task.getBasicAuthPassword().get());
105
- }
106
-
107
- if (task.getUsername().isPresent() && task.getPassword().isPresent()) {
108
- this.client = builder
109
- .authByPassword(task.getUsername().get(), task.getPassword().get())
110
- .build();
111
- }
112
- else if (task.getToken().isPresent()) {
113
- this.client = builder
114
- .authByApiToken(task.getToken().get())
115
- .build();
116
- }
95
+ if (task.getUsername().isPresent() && task.getPassword().isPresent()) {
96
+ this.client =
97
+ builder.authByPassword(task.getUsername().get(), task.getPassword().get()).build();
98
+ } else if (task.getToken().isPresent()) {
99
+ this.client = builder.authByApiToken(task.getToken().get()).build();
117
100
  }
118
-
119
- private void execute(Consumer<KintoneClient> operation)
120
- {
121
- connect(task);
122
- if (this.client != null) {
123
- operation.accept(this.client);
124
- } else {
125
- throw new RuntimeException("Failed to connect to kintone.");
126
- }
101
+ }
102
+
103
+ private void execute(Consumer<KintoneClient> operation) {
104
+ connect(task);
105
+ if (this.client != null) {
106
+ operation.accept(this.client);
107
+ } else {
108
+ throw new RuntimeException("Failed to connect to kintone.");
127
109
  }
128
-
129
- private void insertPage(final Page page)
130
- {
131
- execute(client -> {
132
- try {
133
- ArrayList<Record> records = new ArrayList<>();
134
- pageReader.setPage(page);
135
- KintoneColumnVisitor visitor = new KintoneColumnVisitor(pageReader,
136
- task.getColumnOptions());
137
- while (pageReader.nextRecord()) {
138
- Record record = new Record();
139
- visitor.setRecord(record);
140
- for (Column column : pageReader.getSchema().getColumns()) {
141
- column.visit(visitor);
142
- }
143
-
144
- records.add(record);
145
- if (records.size() == CHUNK_SIZE) {
146
- client.record().addRecords(task.getAppId(), records);
147
- records.clear();
148
- }
149
- }
150
- if (records.size() > 0) {
151
- client.record().addRecords(task.getAppId(), records);
152
- }
110
+ }
111
+
112
+ private void insertPage(final Page page) {
113
+ execute(
114
+ client -> {
115
+ try {
116
+ ArrayList<Record> records = new ArrayList<>();
117
+ pageReader.setPage(page);
118
+ KintoneColumnVisitor visitor =
119
+ new KintoneColumnVisitor(pageReader, task.getColumnOptions());
120
+ while (pageReader.nextRecord()) {
121
+ Record record = new Record();
122
+ visitor.setRecord(record);
123
+ for (Column column : pageReader.getSchema().getColumns()) {
124
+ column.visit(visitor);
125
+ }
126
+
127
+ records.add(record);
128
+ if (records.size() == CHUNK_SIZE) {
129
+ client.record().addRecords(task.getAppId(), records);
130
+ records.clear();
131
+ }
153
132
  }
154
- catch (Exception e) {
155
- throw new RuntimeException("kintone throw exception", e);
133
+ if (records.size() > 0) {
134
+ client.record().addRecords(task.getAppId(), records);
156
135
  }
136
+ } catch (Exception e) {
137
+ throw new RuntimeException("kintone throw exception", e);
138
+ }
157
139
  });
158
- }
159
-
160
- private void updatePage(final Page page)
161
- {
162
- execute(client -> {
163
- try {
164
- ArrayList<RecordForUpdate> updateRecords = new ArrayList<>();
165
- pageReader.setPage(page);
166
-
167
- KintoneColumnVisitor visitor = new KintoneColumnVisitor(pageReader,
168
- task.getColumnOptions(),
169
- task.getUpdateKeyName().orElseThrow(() -> new RuntimeException("unreachable"))); // Already validated
170
- while (pageReader.nextRecord()) {
171
- Record record = new Record();
172
- UpdateKey updateKey = new UpdateKey();
173
- visitor.setRecord(record);
174
- visitor.setUpdateKey(updateKey);
175
- for (Column column : pageReader.getSchema().getColumns()) {
176
- column.visit(visitor);
177
- }
178
-
179
- if (updateKey.getValue() == "") {
180
- continue;
181
- }
182
-
183
- record.removeField(updateKey.getField());
184
- updateRecords.add(new RecordForUpdate(updateKey, record));
185
- if (updateRecords.size() == CHUNK_SIZE) {
186
- client.record().updateRecords(task.getAppId(), updateRecords);
187
- updateRecords.clear();
188
- }
189
- }
190
- if (updateRecords.size() > 0) {
191
- client.record().updateRecords(task.getAppId(), updateRecords);
192
- }
140
+ }
141
+
142
+ private void updatePage(final Page page) {
143
+ execute(
144
+ client -> {
145
+ try {
146
+ ArrayList<RecordForUpdate> updateRecords = new ArrayList<>();
147
+ pageReader.setPage(page);
148
+
149
+ KintoneColumnVisitor visitor =
150
+ new KintoneColumnVisitor(
151
+ pageReader,
152
+ task.getColumnOptions(),
153
+ task.getUpdateKeyName()
154
+ .orElseThrow(
155
+ () -> new RuntimeException("unreachable"))); // Already validated
156
+ while (pageReader.nextRecord()) {
157
+ Record record = new Record();
158
+ UpdateKey updateKey = new UpdateKey();
159
+ visitor.setRecord(record);
160
+ visitor.setUpdateKey(updateKey);
161
+ for (Column column : pageReader.getSchema().getColumns()) {
162
+ column.visit(visitor);
163
+ }
164
+
165
+ if (updateKey.getValue() == "") {
166
+ continue;
167
+ }
168
+
169
+ record.removeField(updateKey.getField());
170
+ updateRecords.add(new RecordForUpdate(updateKey, record));
171
+ if (updateRecords.size() == CHUNK_SIZE) {
172
+ client.record().updateRecords(task.getAppId(), updateRecords);
173
+ updateRecords.clear();
174
+ }
193
175
  }
194
- catch (Exception e) {
195
- throw new RuntimeException("kintone throw exception", e);
176
+ if (updateRecords.size() > 0) {
177
+ client.record().updateRecords(task.getAppId(), updateRecords);
196
178
  }
179
+ } catch (Exception e) {
180
+ throw new RuntimeException("kintone throw exception", e);
181
+ }
197
182
  });
198
- }
199
-
200
- private void upsertPage(final Page page)
201
- {
202
- execute(client -> {
203
- try {
204
- ArrayList<Record> records = new ArrayList<>();
205
- ArrayList<UpdateKey> updateKeys = new ArrayList<>();
206
- pageReader.setPage(page);
207
-
208
- KintoneColumnVisitor visitor = new KintoneColumnVisitor(pageReader,
209
- task.getColumnOptions(),
210
- task.getUpdateKeyName().orElseThrow(() -> new RuntimeException("unreachable"))); // Already validated
211
- while (pageReader.nextRecord()) {
212
- Record record = new Record();
213
- UpdateKey updateKey = new UpdateKey();
214
- visitor.setRecord(record);
215
- visitor.setUpdateKey(updateKey);
216
- for (Column column : pageReader.getSchema().getColumns()) {
217
- column.visit(visitor);
218
- }
219
- records.add(record);
220
- updateKeys.add(updateKey);
221
-
222
- if (records.size() == UPSERT_BATCH_SIZE) {
223
- upsert(records, updateKeys);
224
- records.clear();
225
- updateKeys.clear();
226
- }
227
- }
228
- if (records.size() > 0) {
229
- upsert(records, updateKeys);
230
- }
183
+ }
184
+
185
+ private void upsertPage(final Page page) {
186
+ execute(
187
+ client -> {
188
+ try {
189
+ ArrayList<Record> records = new ArrayList<>();
190
+ ArrayList<UpdateKey> updateKeys = new ArrayList<>();
191
+ pageReader.setPage(page);
192
+
193
+ KintoneColumnVisitor visitor =
194
+ new KintoneColumnVisitor(
195
+ pageReader,
196
+ task.getColumnOptions(),
197
+ task.getUpdateKeyName()
198
+ .orElseThrow(
199
+ () -> new RuntimeException("unreachable"))); // Already validated
200
+ while (pageReader.nextRecord()) {
201
+ Record record = new Record();
202
+ UpdateKey updateKey = new UpdateKey();
203
+ visitor.setRecord(record);
204
+ visitor.setUpdateKey(updateKey);
205
+ for (Column column : pageReader.getSchema().getColumns()) {
206
+ column.visit(visitor);
207
+ }
208
+ records.add(record);
209
+ updateKeys.add(updateKey);
210
+
211
+ if (records.size() == UPSERT_BATCH_SIZE) {
212
+ upsert(records, updateKeys);
213
+ records.clear();
214
+ updateKeys.clear();
215
+ }
231
216
  }
232
- catch (Exception e) {
233
- throw new RuntimeException("kintone throw exception", e);
217
+ if (records.size() > 0) {
218
+ upsert(records, updateKeys);
234
219
  }
220
+ } catch (Exception e) {
221
+ throw new RuntimeException("kintone throw exception", e);
222
+ }
235
223
  });
236
- }
237
-
238
- private void upsert(ArrayList<Record> records, ArrayList<UpdateKey> updateKeys)
239
- {
240
- if (records.size() != updateKeys.size()) {
241
- throw new RuntimeException("records.size() != updateKeys.size()");
242
- }
243
-
244
- List<Record> existingRecords = getExistingRecordsByUpdateKey(updateKeys);
245
-
246
- ArrayList<Record> insertRecords = new ArrayList<>();
247
- ArrayList<RecordForUpdate> updateRecords = new ArrayList<>();
248
- for (int i = 0; i < records.size(); i++) {
249
- Record record = records.get(i);
250
- UpdateKey updateKey = updateKeys.get(i);
251
-
252
- if (existsRecord(existingRecords, updateKey)) {
253
- record.removeField(updateKey.getField());
254
- updateRecords.add(new RecordForUpdate(updateKey, record));
255
- }
256
- else {
257
- insertRecords.add(record);
258
- }
224
+ }
259
225
 
260
- if (insertRecords.size() == CHUNK_SIZE) {
261
- client.record().addRecords(task.getAppId(), insertRecords);
262
- insertRecords.clear();
263
- }
264
- else if (updateRecords.size() == CHUNK_SIZE) {
265
- client.record().updateRecords(task.getAppId(), updateRecords);
266
- updateRecords.clear();
267
- }
268
- }
269
- if (insertRecords.size() > 0) {
270
- client.record().addRecords(task.getAppId(), insertRecords);
271
- }
272
- if (updateRecords.size() > 0) {
273
- client.record().updateRecords(task.getAppId(), updateRecords);
274
- }
226
+ private void upsert(ArrayList<Record> records, ArrayList<UpdateKey> updateKeys) {
227
+ if (records.size() != updateKeys.size()) {
228
+ throw new RuntimeException("records.size() != updateKeys.size()");
275
229
  }
276
-
277
-
278
- private List<Record> getExistingRecordsByUpdateKey(ArrayList<UpdateKey> updateKeys)
279
- {
280
- String fieldCode = updateKeys.get(0).getField();
281
- List<String> queryValues = updateKeys
282
- .stream()
283
- .filter(k -> k.getValue() != "")
284
- .map(k -> "\"" + k.getValue().toString() + "\"")
285
- .collect(Collectors.toList());
286
-
287
- List<Record> allRecords = new ArrayList<>();
288
- if (queryValues.isEmpty()) {
289
- return allRecords;
290
- }
291
- String cursorId = client.record().createCursor(
292
- task.getAppId(),
293
- Collections.singletonList(fieldCode),
294
- fieldCode + " in (" + String.join(",", queryValues) + ")"
295
- );
296
- while (true) {
297
- GetRecordsByCursorResponseBody resp = client.record().getRecordsByCursor(cursorId);
298
- List<Record> records = resp.getRecords();
299
- allRecords.addAll(records);
300
-
301
- if (!resp.hasNext()) {
302
- break;
303
- }
304
- }
305
- return allRecords;
230
+ FieldType updateKeyFieldType =
231
+ client.app().getFormFields(task.getAppId()).get(updateKeys.get(0).getField()).getType();
232
+ if (!Arrays.asList(FieldType.SINGLE_LINE_TEXT, FieldType.NUMBER).contains(updateKeyFieldType)) {
233
+ throw new ConfigException("The update_key must be 'SINGLE_LINE_TEXT' or 'NUMBER'.");
306
234
  }
307
235
 
308
- private boolean existsRecord(List<Record> distRecords, UpdateKey updateKey)
309
- {
310
- String fieldCode = updateKey.getField();
311
- FieldType type = client.app().getFormFields(task.getAppId()).get(fieldCode).getType();
312
- switch (type) {
313
- case SINGLE_LINE_TEXT:
314
- return distRecords
315
- .stream()
316
- .anyMatch(d -> d.getSingleLineTextFieldValue(fieldCode).equals(updateKey.getValue().toString()));
317
- case NUMBER:
318
- return distRecords
319
- .stream()
320
- .anyMatch(d -> d.getNumberFieldValue(fieldCode).toPlainString()
321
- .equals(updateKey.getValue().toString()));
322
- default:
323
- throw new RuntimeException("The update_key must be 'SINGLE_LINE_TEXT' or 'NUMBER'.");
324
- }
236
+ List<Record> existingRecords = getExistingRecordsByUpdateKey(updateKeys);
237
+ String updateField = updateKeys.get(0).getField();
238
+ List<String> existingValues =
239
+ existingRecords.stream()
240
+ .map(
241
+ (r) -> {
242
+ switch (updateKeyFieldType) {
243
+ case SINGLE_LINE_TEXT:
244
+ String s = r.getSingleLineTextFieldValue(updateField);
245
+ return s == null ? null : s.toString();
246
+ case NUMBER:
247
+ BigDecimal bd = r.getNumberFieldValue(updateField);
248
+ return bd == null ? null : bd.toPlainString();
249
+ default:
250
+ return null;
251
+ }
252
+ })
253
+ .filter(v -> v != null)
254
+ .collect(Collectors.toList());
255
+
256
+ ArrayList<Record> insertRecords = new ArrayList<>();
257
+ ArrayList<RecordForUpdate> updateRecords = new ArrayList<>();
258
+ for (int i = 0; i < records.size(); i++) {
259
+ Record record = records.get(i);
260
+ UpdateKey updateKey = updateKeys.get(i);
261
+
262
+ if (existsRecord(existingValues, updateKey)) {
263
+ record.removeField(updateKey.getField());
264
+ updateRecords.add(new RecordForUpdate(updateKey, record));
265
+ } else {
266
+ insertRecords.add(record);
267
+ }
268
+
269
+ if (insertRecords.size() == CHUNK_SIZE) {
270
+ client.record().addRecords(task.getAppId(), insertRecords);
271
+ insertRecords.clear();
272
+ } else if (updateRecords.size() == CHUNK_SIZE) {
273
+ client.record().updateRecords(task.getAppId(), updateRecords);
274
+ updateRecords.clear();
275
+ }
276
+ }
277
+ if (insertRecords.size() > 0) {
278
+ client.record().addRecords(task.getAppId(), insertRecords);
279
+ }
280
+ if (updateRecords.size() > 0) {
281
+ client.record().updateRecords(task.getAppId(), updateRecords);
325
282
  }
283
+ }
284
+
285
+ private List<Record> getExistingRecordsByUpdateKey(ArrayList<UpdateKey> updateKeys) {
286
+ String fieldCode = updateKeys.get(0).getField();
287
+ List<String> queryValues =
288
+ updateKeys.stream()
289
+ .filter(k -> k.getValue() != "")
290
+ .map(k -> "\"" + k.getValue().toString() + "\"")
291
+ .collect(Collectors.toList());
292
+
293
+ List<Record> allRecords = new ArrayList<>();
294
+ if (queryValues.isEmpty()) {
295
+ return allRecords;
296
+ }
297
+ String cursorId =
298
+ client
299
+ .record()
300
+ .createCursor(
301
+ task.getAppId(),
302
+ Collections.singletonList(fieldCode),
303
+ fieldCode + " in (" + String.join(",", queryValues) + ")");
304
+ while (true) {
305
+ GetRecordsByCursorResponseBody resp = client.record().getRecordsByCursor(cursorId);
306
+ List<Record> records = resp.getRecords();
307
+ allRecords.addAll(records);
308
+
309
+ if (!resp.hasNext()) {
310
+ break;
311
+ }
312
+ }
313
+ return allRecords;
314
+ }
315
+
316
+ private boolean existsRecord(List<String> distValues, UpdateKey updateKey) {
317
+ return distValues.stream().anyMatch(v -> v.equals(updateKey.getValue().toString()));
318
+ }
326
319
  }