embulk-output-kintone 0.2.1 → 0.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e8c2d570242a7103d8d86149c3e31e8daa4bd593
-  data.tar.gz: bf24b0324e3d4a65dc6f6fd2bc000314317a1036
+  metadata.gz: 59e8a8850c43f8ff293ca6b63d908f6e7e04b1ac
+  data.tar.gz: 42f8ae842db0e3177b1123dc45dd7919f2cb903d
 SHA512:
-  metadata.gz: c38ba0d3dcfe715347f2a1efbf78ec4015eaff6c6c9bb80c29cdc3cbe3bfe21e0ebe8bbde199386cce9fb15b50f3a35e0a5b705a224a2a55f27eb02703c380dc
-  data.tar.gz: de192f098f8b9f91c153d14e19b1e3f3c78cff543d305bf749e3cd6ed35cd498bf26e54009cd6f0a6fa82966850695675d0b4d7b9266c182ea963e822cb09d61
+  metadata.gz: 447d0a6901e2bf7d4993fb9f20291f236199bd53e3ae4924b85444c64e8b4fa78a2bccf3c42083edde6c6d863953a6d6f1e70be4d629dd7f35ddc63ab81276de
+  data.tar.gz: 8a6fb5c6cde0a75377b95641cadacc243ca7ab0ddd03d641df37f055c574a33552bd045b13cfa6236f49d570188c490d83dedb5715bd2908a53746af721c060c

data/build.gradle

@@ -14,7 +14,7 @@ configurations {
     provided
 }
 
-version = "0.2.1"
+version = "0.2.2"
 
 sourceCompatibility = 1.8
 targetCompatibility = 1.8

data/src/main/java/org/embulk/output/kintone/KintonePageOutput.java

@@ -17,9 +17,7 @@ import org.embulk.spi.TransactionalPageOutput;
 import java.math.BigDecimal;
 import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.HashMap;
 import java.util.List;
-import java.util.Map;
 import java.util.stream.Collectors;
 
 public class KintonePageOutput
@@ -187,11 +185,15 @@ public class KintonePageOutput
         });
     }
 
-    private List<Record> getAllRecords(String fieldCode)
+    private List<Record> getRecordsByUpdateKey(String fieldCode, List<String> queryValues)
     {
         List<Record> allRecords = new ArrayList<Record>();
         List<String> fields = Arrays.asList(fieldCode);
-        String cursorId = client.record().createCursor(task.getAppId(), fields, null);
+        String cursorId = client.record().createCursor(
+            task.getAppId(),
+            fields,
+            fieldCode + " in (" + String.join(",", queryValues) + ")"
+        );
         while (true) {
             GetRecordsByCursorResponseBody resp = client.record().getRecordsByCursor(cursorId);
             List<Record> records = resp.getRecords();
@@ -206,17 +208,59 @@ public class KintonePageOutput
 
     abstract class UpsertPage<T>
     {
-        public abstract List<T> getUpdateKeyValues();
+        public abstract List<T> getUpdateKeyValues(List<String> queryValues);
         public abstract boolean existsRecord(List<T> updateKeyValues, Record record);
 
+        public void upsert(ArrayList<Record> records, ArrayList<UpdateKey> updateKeys)
+        {
+            if (records.size() != updateKeys.size()) {
+                throw new RuntimeException("records.size() != updateKeys.size()");
+            }
+
+            List<String> queryValues = updateKeys
+                .stream()
+                .map(k -> "\"" + k.getValue().toString() + "\"")
+                .collect(Collectors.toList());
+            List<T> updateKeyValues = getUpdateKeyValues(queryValues);
+
+            ArrayList<Record> insertRecords = new ArrayList<>();
+            ArrayList<RecordForUpdate> updateRecords = new ArrayList<RecordForUpdate>();
+            for (int i = 0; i < records.size(); i++) {
+                Record record = records.get(i);
+                UpdateKey updateKey = updateKeys.get(i);
+
+                if (existsRecord(updateKeyValues, record)) {
+                    record.removeField(updateKey.getField());
+                    updateRecords.add(new RecordForUpdate(updateKey, record));
+                }
+                else {
+                    insertRecords.add(record);
+                }
+
+                if (insertRecords.size() == 100) {
+                    client.record().addRecords(task.getAppId(), insertRecords);
+                    insertRecords.clear();
+                }
+                else if (updateRecords.size() == 100) {
+                    client.record().updateRecords(task.getAppId(), updateRecords);
+                    updateRecords.clear();
+                }
+            }
+            if (insertRecords.size() > 0) {
+                client.record().addRecords(task.getAppId(), insertRecords);
+            }
+            if (updateRecords.size() > 0) {
+                client.record().updateRecords(task.getAppId(), updateRecords);
+            }
+        }
+
         public void run(final Page page)
         {
             execute(client -> {
                 try {
-                    List<T> updateKeyValues = getUpdateKeyValues();
+                    ArrayList<Record> records = new ArrayList<>();
+                    ArrayList<UpdateKey> updateKeys = new ArrayList<>();
 
-                    ArrayList<Record> insertRecords = new ArrayList<>();
-                    ArrayList<RecordForUpdate> updateRecords = new ArrayList<RecordForUpdate>();
                     pageReader.setPage(page);
                     KintoneColumnVisitor visitor = new KintoneColumnVisitor(pageReader,
                             task.getColumnOptions());
@@ -228,30 +272,17 @@ public class KintonePageOutput
                         for (Column column : pageReader.getSchema().getColumns()) {
                             column.visit(visitor);
                         }
+                        records.add(record);
+                        updateKeys.add(updateKey);
 
-                        if (existsRecord(updateKeyValues, record)) {
-                            record.removeField(updateKey.getField());
-                            updateRecords.add(new RecordForUpdate(updateKey, record));
-                        }
-                        else {
-                            insertRecords.add(record);
-                        }
-
-                        if (insertRecords.size() == 100) {
-                            client.record().addRecords(task.getAppId(), insertRecords);
-                            insertRecords.clear();
-                        }
-                        else if (updateRecords.size() == 100) {
-                            client.record().updateRecords(task.getAppId(), updateRecords);
-                            updateRecords.clear();
+                        if (records.size() == 10000) {
+                            upsert(records, updateKeys);
+                            records.clear();
+                            updateKeys.clear();
                         }
                     }
-
-                    if (insertRecords.size() > 0) {
-                        client.record().addRecords(task.getAppId(), insertRecords);
-                    }
-                    if (updateRecords.size() > 0) {
-                        client.record().updateRecords(task.getAppId(), updateRecords);
+                    if (records.size() > 0) {
+                        upsert(records, updateKeys);
                     }
                 }
                 catch (Exception e) {
@@ -270,9 +301,9 @@ public class KintonePageOutput
             this.fieldCode = fieldCode;
         }
 
-        public List<String> getUpdateKeyValues()
+        public List<String> getUpdateKeyValues(List<String> queryValues)
         {
-            return getAllRecords(fieldCode)
+            return getRecordsByUpdateKey(fieldCode, queryValues)
                 .stream()
                 .map(r -> r.getSingleLineTextFieldValue(fieldCode))
                 .collect(Collectors.toList());
@@ -293,9 +324,9 @@ public class KintonePageOutput
             this.fieldCode = fieldCode;
         }
 
-        public List<BigDecimal> getUpdateKeyValues()
+        public List<BigDecimal> getUpdateKeyValues(List<String> queryValues)
         {
-            return getAllRecords(fieldCode)
+            return getRecordsByUpdateKey(fieldCode, queryValues)
                 .stream()
                 .map(r -> r.getNumberFieldValue(fieldCode))
                 .collect(Collectors.toList());

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-kintone
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.2.2
 platform: ruby
 authors:
 - takeshi fujita
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-09-26 00:00:00.000000000 Z
+date: 2020-11-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -52,7 +52,7 @@ files:
 - README.md
 - Rakefile
 - build.gradle
-- classpath/embulk-output-kintone-0.2.1.jar
+- classpath/embulk-output-kintone-0.2.2.jar
 - config/checkstyle/checkstyle.xml
 - config/checkstyle/default.xml
 - gradle/wrapper/gradle-wrapper.jar