embulk-output-kintone 0.2.1 → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/build.gradle +1 -1
- data/src/main/java/org/embulk/output/kintone/KintonePageOutput.java +64 -33
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 59e8a8850c43f8ff293ca6b63d908f6e7e04b1ac
|
4
|
+
data.tar.gz: 42f8ae842db0e3177b1123dc45dd7919f2cb903d
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 447d0a6901e2bf7d4993fb9f20291f236199bd53e3ae4924b85444c64e8b4fa78a2bccf3c42083edde6c6d863953a6d6f1e70be4d629dd7f35ddc63ab81276de
|
7
|
+
data.tar.gz: 8a6fb5c6cde0a75377b95641cadacc243ca7ab0ddd03d641df37f055c574a33552bd045b13cfa6236f49d570188c490d83dedb5715bd2908a53746af721c060c
|
data/build.gradle
CHANGED
@@ -17,9 +17,7 @@ import org.embulk.spi.TransactionalPageOutput;
|
|
17
17
|
import java.math.BigDecimal;
|
18
18
|
import java.util.ArrayList;
|
19
19
|
import java.util.Arrays;
|
20
|
-
import java.util.HashMap;
|
21
20
|
import java.util.List;
|
22
|
-
import java.util.Map;
|
23
21
|
import java.util.stream.Collectors;
|
24
22
|
|
25
23
|
public class KintonePageOutput
|
@@ -187,11 +185,15 @@ public class KintonePageOutput
|
|
187
185
|
});
|
188
186
|
}
|
189
187
|
|
190
|
-
private List<Record>
|
188
|
+
private List<Record> getRecordsByUpdateKey(String fieldCode, List<String> queryValues)
|
191
189
|
{
|
192
190
|
List<Record> allRecords = new ArrayList<Record>();
|
193
191
|
List<String> fields = Arrays.asList(fieldCode);
|
194
|
-
String cursorId = client.record().createCursor(
|
192
|
+
String cursorId = client.record().createCursor(
|
193
|
+
task.getAppId(),
|
194
|
+
fields,
|
195
|
+
fieldCode + " in (" + String.join(",", queryValues) + ")"
|
196
|
+
);
|
195
197
|
while (true) {
|
196
198
|
GetRecordsByCursorResponseBody resp = client.record().getRecordsByCursor(cursorId);
|
197
199
|
List<Record> records = resp.getRecords();
|
@@ -206,17 +208,59 @@ public class KintonePageOutput
|
|
206
208
|
|
207
209
|
abstract class UpsertPage<T>
|
208
210
|
{
|
209
|
-
public abstract List<T> getUpdateKeyValues();
|
211
|
+
public abstract List<T> getUpdateKeyValues(List<String> queryValues);
|
210
212
|
public abstract boolean existsRecord(List<T> updateKeyValues, Record record);
|
211
213
|
|
214
|
+
public void upsert(ArrayList<Record> records, ArrayList<UpdateKey> updateKeys)
|
215
|
+
{
|
216
|
+
if (records.size() != updateKeys.size()) {
|
217
|
+
throw new RuntimeException("records.size() != updateKeys.size()");
|
218
|
+
}
|
219
|
+
|
220
|
+
List<String> queryValues = updateKeys
|
221
|
+
.stream()
|
222
|
+
.map(k -> "\"" + k.getValue().toString() + "\"")
|
223
|
+
.collect(Collectors.toList());
|
224
|
+
List<T> updateKeyValues = getUpdateKeyValues(queryValues);
|
225
|
+
|
226
|
+
ArrayList<Record> insertRecords = new ArrayList<>();
|
227
|
+
ArrayList<RecordForUpdate> updateRecords = new ArrayList<RecordForUpdate>();
|
228
|
+
for (int i = 0; i < records.size(); i++) {
|
229
|
+
Record record = records.get(i);
|
230
|
+
UpdateKey updateKey = updateKeys.get(i);
|
231
|
+
|
232
|
+
if (existsRecord(updateKeyValues, record)) {
|
233
|
+
record.removeField(updateKey.getField());
|
234
|
+
updateRecords.add(new RecordForUpdate(updateKey, record));
|
235
|
+
}
|
236
|
+
else {
|
237
|
+
insertRecords.add(record);
|
238
|
+
}
|
239
|
+
|
240
|
+
if (insertRecords.size() == 100) {
|
241
|
+
client.record().addRecords(task.getAppId(), insertRecords);
|
242
|
+
insertRecords.clear();
|
243
|
+
}
|
244
|
+
else if (updateRecords.size() == 100) {
|
245
|
+
client.record().updateRecords(task.getAppId(), updateRecords);
|
246
|
+
updateRecords.clear();
|
247
|
+
}
|
248
|
+
}
|
249
|
+
if (insertRecords.size() > 0) {
|
250
|
+
client.record().addRecords(task.getAppId(), insertRecords);
|
251
|
+
}
|
252
|
+
if (updateRecords.size() > 0) {
|
253
|
+
client.record().updateRecords(task.getAppId(), updateRecords);
|
254
|
+
}
|
255
|
+
}
|
256
|
+
|
212
257
|
public void run(final Page page)
|
213
258
|
{
|
214
259
|
execute(client -> {
|
215
260
|
try {
|
216
|
-
|
261
|
+
ArrayList<Record> records = new ArrayList<>();
|
262
|
+
ArrayList<UpdateKey> updateKeys = new ArrayList<>();
|
217
263
|
|
218
|
-
ArrayList<Record> insertRecords = new ArrayList<>();
|
219
|
-
ArrayList<RecordForUpdate> updateRecords = new ArrayList<RecordForUpdate>();
|
220
264
|
pageReader.setPage(page);
|
221
265
|
KintoneColumnVisitor visitor = new KintoneColumnVisitor(pageReader,
|
222
266
|
task.getColumnOptions());
|
@@ -228,30 +272,17 @@ public class KintonePageOutput
|
|
228
272
|
for (Column column : pageReader.getSchema().getColumns()) {
|
229
273
|
column.visit(visitor);
|
230
274
|
}
|
275
|
+
records.add(record);
|
276
|
+
updateKeys.add(updateKey);
|
231
277
|
|
232
|
-
if (
|
233
|
-
|
234
|
-
|
235
|
-
|
236
|
-
else {
|
237
|
-
insertRecords.add(record);
|
238
|
-
}
|
239
|
-
|
240
|
-
if (insertRecords.size() == 100) {
|
241
|
-
client.record().addRecords(task.getAppId(), insertRecords);
|
242
|
-
insertRecords.clear();
|
243
|
-
}
|
244
|
-
else if (updateRecords.size() == 100) {
|
245
|
-
client.record().updateRecords(task.getAppId(), updateRecords);
|
246
|
-
updateRecords.clear();
|
278
|
+
if (records.size() == 10000) {
|
279
|
+
upsert(records, updateKeys);
|
280
|
+
records.clear();
|
281
|
+
updateKeys.clear();
|
247
282
|
}
|
248
283
|
}
|
249
|
-
|
250
|
-
|
251
|
-
client.record().addRecords(task.getAppId(), insertRecords);
|
252
|
-
}
|
253
|
-
if (updateRecords.size() > 0) {
|
254
|
-
client.record().updateRecords(task.getAppId(), updateRecords);
|
284
|
+
if (records.size() > 0) {
|
285
|
+
upsert(records, updateKeys);
|
255
286
|
}
|
256
287
|
}
|
257
288
|
catch (Exception e) {
|
@@ -270,9 +301,9 @@ public class KintonePageOutput
|
|
270
301
|
this.fieldCode = fieldCode;
|
271
302
|
}
|
272
303
|
|
273
|
-
public List<String> getUpdateKeyValues()
|
304
|
+
public List<String> getUpdateKeyValues(List<String> queryValues)
|
274
305
|
{
|
275
|
-
return
|
306
|
+
return getRecordsByUpdateKey(fieldCode, queryValues)
|
276
307
|
.stream()
|
277
308
|
.map(r -> r.getSingleLineTextFieldValue(fieldCode))
|
278
309
|
.collect(Collectors.toList());
|
@@ -293,9 +324,9 @@ public class KintonePageOutput
|
|
293
324
|
this.fieldCode = fieldCode;
|
294
325
|
}
|
295
326
|
|
296
|
-
public List<BigDecimal> getUpdateKeyValues()
|
327
|
+
public List<BigDecimal> getUpdateKeyValues(List<String> queryValues)
|
297
328
|
{
|
298
|
-
return
|
329
|
+
return getRecordsByUpdateKey(fieldCode, queryValues)
|
299
330
|
.stream()
|
300
331
|
.map(r -> r.getNumberFieldValue(fieldCode))
|
301
332
|
.collect(Collectors.toList());
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-output-kintone
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- takeshi fujita
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-
|
11
|
+
date: 2020-11-18 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -52,7 +52,7 @@ files:
|
|
52
52
|
- README.md
|
53
53
|
- Rakefile
|
54
54
|
- build.gradle
|
55
|
-
- classpath/embulk-output-kintone-0.2.
|
55
|
+
- classpath/embulk-output-kintone-0.2.2.jar
|
56
56
|
- config/checkstyle/checkstyle.xml
|
57
57
|
- config/checkstyle/default.xml
|
58
58
|
- gradle/wrapper/gradle-wrapper.jar
|