embulk-output-kintone 0.2.1 → 0.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: e8c2d570242a7103d8d86149c3e31e8daa4bd593
4
- data.tar.gz: bf24b0324e3d4a65dc6f6fd2bc000314317a1036
3
+ metadata.gz: 59e8a8850c43f8ff293ca6b63d908f6e7e04b1ac
4
+ data.tar.gz: 42f8ae842db0e3177b1123dc45dd7919f2cb903d
5
5
  SHA512:
6
- metadata.gz: c38ba0d3dcfe715347f2a1efbf78ec4015eaff6c6c9bb80c29cdc3cbe3bfe21e0ebe8bbde199386cce9fb15b50f3a35e0a5b705a224a2a55f27eb02703c380dc
7
- data.tar.gz: de192f098f8b9f91c153d14e19b1e3f3c78cff543d305bf749e3cd6ed35cd498bf26e54009cd6f0a6fa82966850695675d0b4d7b9266c182ea963e822cb09d61
6
+ metadata.gz: 447d0a6901e2bf7d4993fb9f20291f236199bd53e3ae4924b85444c64e8b4fa78a2bccf3c42083edde6c6d863953a6d6f1e70be4d629dd7f35ddc63ab81276de
7
+ data.tar.gz: 8a6fb5c6cde0a75377b95641cadacc243ca7ab0ddd03d641df37f055c574a33552bd045b13cfa6236f49d570188c490d83dedb5715bd2908a53746af721c060c
@@ -14,7 +14,7 @@ configurations {
14
14
  provided
15
15
  }
16
16
 
17
- version = "0.2.1"
17
+ version = "0.2.2"
18
18
 
19
19
  sourceCompatibility = 1.8
20
20
  targetCompatibility = 1.8
@@ -17,9 +17,7 @@ import org.embulk.spi.TransactionalPageOutput;
17
17
  import java.math.BigDecimal;
18
18
  import java.util.ArrayList;
19
19
  import java.util.Arrays;
20
- import java.util.HashMap;
21
20
  import java.util.List;
22
- import java.util.Map;
23
21
  import java.util.stream.Collectors;
24
22
 
25
23
  public class KintonePageOutput
@@ -187,11 +185,15 @@ public class KintonePageOutput
187
185
  });
188
186
  }
189
187
 
190
- private List<Record> getAllRecords(String fieldCode)
188
+ private List<Record> getRecordsByUpdateKey(String fieldCode, List<String> queryValues)
191
189
  {
192
190
  List<Record> allRecords = new ArrayList<Record>();
193
191
  List<String> fields = Arrays.asList(fieldCode);
194
- String cursorId = client.record().createCursor(task.getAppId(), fields, null);
192
+ String cursorId = client.record().createCursor(
193
+ task.getAppId(),
194
+ fields,
195
+ fieldCode + " in (" + String.join(",", queryValues) + ")"
196
+ );
195
197
  while (true) {
196
198
  GetRecordsByCursorResponseBody resp = client.record().getRecordsByCursor(cursorId);
197
199
  List<Record> records = resp.getRecords();
@@ -206,17 +208,59 @@ public class KintonePageOutput
206
208
 
207
209
  abstract class UpsertPage<T>
208
210
  {
209
- public abstract List<T> getUpdateKeyValues();
211
+ public abstract List<T> getUpdateKeyValues(List<String> queryValues);
210
212
  public abstract boolean existsRecord(List<T> updateKeyValues, Record record);
211
213
 
214
+ public void upsert(ArrayList<Record> records, ArrayList<UpdateKey> updateKeys)
215
+ {
216
+ if (records.size() != updateKeys.size()) {
217
+ throw new RuntimeException("records.size() != updateKeys.size()");
218
+ }
219
+
220
+ List<String> queryValues = updateKeys
221
+ .stream()
222
+ .map(k -> "\"" + k.getValue().toString() + "\"")
223
+ .collect(Collectors.toList());
224
+ List<T> updateKeyValues = getUpdateKeyValues(queryValues);
225
+
226
+ ArrayList<Record> insertRecords = new ArrayList<>();
227
+ ArrayList<RecordForUpdate> updateRecords = new ArrayList<RecordForUpdate>();
228
+ for (int i = 0; i < records.size(); i++) {
229
+ Record record = records.get(i);
230
+ UpdateKey updateKey = updateKeys.get(i);
231
+
232
+ if (existsRecord(updateKeyValues, record)) {
233
+ record.removeField(updateKey.getField());
234
+ updateRecords.add(new RecordForUpdate(updateKey, record));
235
+ }
236
+ else {
237
+ insertRecords.add(record);
238
+ }
239
+
240
+ if (insertRecords.size() == 100) {
241
+ client.record().addRecords(task.getAppId(), insertRecords);
242
+ insertRecords.clear();
243
+ }
244
+ else if (updateRecords.size() == 100) {
245
+ client.record().updateRecords(task.getAppId(), updateRecords);
246
+ updateRecords.clear();
247
+ }
248
+ }
249
+ if (insertRecords.size() > 0) {
250
+ client.record().addRecords(task.getAppId(), insertRecords);
251
+ }
252
+ if (updateRecords.size() > 0) {
253
+ client.record().updateRecords(task.getAppId(), updateRecords);
254
+ }
255
+ }
256
+
212
257
  public void run(final Page page)
213
258
  {
214
259
  execute(client -> {
215
260
  try {
216
- List<T> updateKeyValues = getUpdateKeyValues();
261
+ ArrayList<Record> records = new ArrayList<>();
262
+ ArrayList<UpdateKey> updateKeys = new ArrayList<>();
217
263
 
218
- ArrayList<Record> insertRecords = new ArrayList<>();
219
- ArrayList<RecordForUpdate> updateRecords = new ArrayList<RecordForUpdate>();
220
264
  pageReader.setPage(page);
221
265
  KintoneColumnVisitor visitor = new KintoneColumnVisitor(pageReader,
222
266
  task.getColumnOptions());
@@ -228,30 +272,17 @@ public class KintonePageOutput
228
272
  for (Column column : pageReader.getSchema().getColumns()) {
229
273
  column.visit(visitor);
230
274
  }
275
+ records.add(record);
276
+ updateKeys.add(updateKey);
231
277
 
232
- if (existsRecord(updateKeyValues, record)) {
233
- record.removeField(updateKey.getField());
234
- updateRecords.add(new RecordForUpdate(updateKey, record));
235
- }
236
- else {
237
- insertRecords.add(record);
238
- }
239
-
240
- if (insertRecords.size() == 100) {
241
- client.record().addRecords(task.getAppId(), insertRecords);
242
- insertRecords.clear();
243
- }
244
- else if (updateRecords.size() == 100) {
245
- client.record().updateRecords(task.getAppId(), updateRecords);
246
- updateRecords.clear();
278
+ if (records.size() == 10000) {
279
+ upsert(records, updateKeys);
280
+ records.clear();
281
+ updateKeys.clear();
247
282
  }
248
283
  }
249
-
250
- if (insertRecords.size() > 0) {
251
- client.record().addRecords(task.getAppId(), insertRecords);
252
- }
253
- if (updateRecords.size() > 0) {
254
- client.record().updateRecords(task.getAppId(), updateRecords);
284
+ if (records.size() > 0) {
285
+ upsert(records, updateKeys);
255
286
  }
256
287
  }
257
288
  catch (Exception e) {
@@ -270,9 +301,9 @@ public class KintonePageOutput
270
301
  this.fieldCode = fieldCode;
271
302
  }
272
303
 
273
- public List<String> getUpdateKeyValues()
304
+ public List<String> getUpdateKeyValues(List<String> queryValues)
274
305
  {
275
- return getAllRecords(fieldCode)
306
+ return getRecordsByUpdateKey(fieldCode, queryValues)
276
307
  .stream()
277
308
  .map(r -> r.getSingleLineTextFieldValue(fieldCode))
278
309
  .collect(Collectors.toList());
@@ -293,9 +324,9 @@ public class KintonePageOutput
293
324
  this.fieldCode = fieldCode;
294
325
  }
295
326
 
296
- public List<BigDecimal> getUpdateKeyValues()
327
+ public List<BigDecimal> getUpdateKeyValues(List<String> queryValues)
297
328
  {
298
- return getAllRecords(fieldCode)
329
+ return getRecordsByUpdateKey(fieldCode, queryValues)
299
330
  .stream()
300
331
  .map(r -> r.getNumberFieldValue(fieldCode))
301
332
  .collect(Collectors.toList());
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: embulk-output-kintone
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.1
4
+ version: 0.2.2
5
5
  platform: ruby
6
6
  authors:
7
7
  - takeshi fujita
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2020-09-26 00:00:00.000000000 Z
11
+ date: 2020-11-18 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
@@ -52,7 +52,7 @@ files:
52
52
  - README.md
53
53
  - Rakefile
54
54
  - build.gradle
55
- - classpath/embulk-output-kintone-0.2.1.jar
55
+ - classpath/embulk-output-kintone-0.2.2.jar
56
56
  - config/checkstyle/checkstyle.xml
57
57
  - config/checkstyle/default.xml
58
58
  - gradle/wrapper/gradle-wrapper.jar