embulk-output-kintone 0.2.2 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 59e8a8850c43f8ff293ca6b63d908f6e7e04b1ac
-  data.tar.gz: 42f8ae842db0e3177b1123dc45dd7919f2cb903d
+  metadata.gz: 2004bfc2e202eeb6c57f3149bc7311f90343a67e
+  data.tar.gz: 86e036bc863ac7d8ca632488d0eb84c645658c43
 SHA512:
-  metadata.gz: 447d0a6901e2bf7d4993fb9f20291f236199bd53e3ae4924b85444c64e8b4fa78a2bccf3c42083edde6c6d863953a6d6f1e70be4d629dd7f35ddc63ab81276de
-  data.tar.gz: 8a6fb5c6cde0a75377b95641cadacc243ca7ab0ddd03d641df37f055c574a33552bd045b13cfa6236f49d570188c490d83dedb5715bd2908a53746af721c060c
+  metadata.gz: 723f4b6b9e5341a7e0bef48c6de38528860985d46d9a139c2d3e133c59ab867c9a4507b0c98a431b5b654f332178f90038168bc0143d9d9dcb2916dfe1956e44
+  data.tar.gz: 84441877b034060e5876ae67d8afd1d01f15a91d1aab8a76065bd4549a85c7de1639b0557a814482edc8312bbc11cbcbdc0565fa87aedfb794493966064201b7

data/.github/workflows/ci.yml

@@ -1,13 +1,60 @@
-name: Java CI
+name: main
 
-on: [push, pull_request]
+on:
+  push:
+    branches:
+      - 'master'
+    tags:
+      - '*'
+  pull_request:
+    branches:
+      - 'master'
+    types: [opened, synchronize]
 
 jobs:
-  build:
+  main:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - uses: actions/setup-java@v1
+      - name: Set Up
+        uses: actions/setup-java@v1
         with:
           java-version: 1.8
-      - run: ./gradlew build
+
+      - name: Test
+        run: ./gradlew test
+
+      - name: Build Gem
+        run: ./gradlew gem
+
+      - name: Set up JRuby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: jruby
+
+      - name: Publish to GPR
+        if: startsWith( github.ref, 'refs/tags/' )
+        run: |
+          mkdir -p $HOME/.gem
+          touch $HOME/.gem/credentials
+          chmod 0600 $HOME/.gem/credentials
+          printf -- "---\n:github: ${GEM_HOST_API_KEY}\n" > $HOME/.gem/credentials
+          gem push --KEY github --host https://rubygems.pkg.github.com/${OWNER} ./pkg/*.gem
+        env:
+          GEM_HOST_API_KEY: "Bearer ${{secrets.GITHUB_TOKEN}}"
+          OWNER: ${{ github.repository_owner }}
+
+      - name: Publish to RubyGems
+        if: startsWith( github.ref, 'refs/tags/' )
+        run: |
+          mkdir -p $HOME/.gem
+          touch $HOME/.gem/credentials
+          chmod 0600 $HOME/.gem/credentials
+          printf -- "---\n:rubygems_api_key: ${GEM_HOST_API_KEY}\n" > $HOME/.gem/credentials
+          # TODO: If it is possible to accept input in the middle of a step, then the OTP Token should be inputted instead of generated.
+          gem install rotp -v 6.2.0
+          OTP_TOKEN=$(echo ${OTP_SECRET} | ruby -rtime -rrotp -e "puts ROTP::TOTP.new(STDIN.read.chomp, issuer: 'rubygems.org').at(Time.now)")
+          gem push --otp="${OTP_TOKEN}" ./pkg/*.gem
+        env:
+          GEM_HOST_API_KEY: "${{secrets.RUBYGEMS_API_KEY}}"
+          OTP_SECRET: "${{secrets.RUBYGEMS_OTP_SECRET}}"

data/README.md

@@ -17,11 +17,11 @@ kintone output plugin for Embulk stores app records from kintone.
 - **basic_auth_password**:  kintone basic auth password (string, optional)
 - **guest_space_id**: kintone app belongs to guest space, guest space id is required. (integer, optional)
 - **mode**: kintone mode (string, required)
+- **update_key**: column name to set update key (string, required if mode is update or upsert)
 - **column_options** advanced: a key-value pairs where key is a column name and value is options for the column.
     - **field_code**: field code (string, required)
     - **type**: field type (string, required)
     - **timezone**: timezone to convert into `date` (string, default is `UTC`)
-    - **update_key**: update key (boolean, default is `false`)
     - **val_sep**: Used to specify multiple checkbox values (string, default is `,`)
 
 ## Example
@@ -33,7 +33,8 @@ out:
   username: username
   password: password
   app_id: 1
-  mode: insert
+  mode: upsert
+  update_key: id
   column_options:
     id: {field_code: "id", type: "NUMBER"}
     name: {field_code: "name", type: "SINGLE_LINE_TEXT"}

data/build.gradle

@@ -14,7 +14,7 @@ configurations {
     provided
 }
 
-version = "0.2.2"
+version = "0.3.0"
 
 sourceCompatibility = 1.8
 targetCompatibility = 1.8

data/src/main/java/org/embulk/output/kintone/KintoneColumnOption.java

@@ -19,10 +19,6 @@ public interface KintoneColumnOption
     @ConfigDefault("\"UTC\"")
     public Optional<String> getTimezone();
 
-    @Config("update_key")
-    @ConfigDefault("false")
-    public boolean getUpdateKey();
-
     @Config("val_sep")
     @ConfigDefault("\",\"")
     public String getValueSeparator();

data/src/main/java/org/embulk/output/kintone/KintoneColumnVisitor.java

@@ -22,6 +22,7 @@ import java.time.Instant;
 import java.time.ZoneId;
 import java.time.ZonedDateTime;
 import java.util.Map;
+import java.util.Objects;
 
 public class KintoneColumnVisitor
         implements ColumnVisitor
@@ -30,6 +31,7 @@ public class KintoneColumnVisitor
     private Record record;
     private UpdateKey updateKey;
     private Map<String, KintoneColumnOption> columnOptions;
+    private String updateKeyName;
 
     public KintoneColumnVisitor(PageReader pageReader,
                                 Map<String, KintoneColumnOption> columnOptions)
@@ -38,6 +40,15 @@ public class KintoneColumnVisitor
         this.columnOptions = columnOptions;
     }
 
+    public KintoneColumnVisitor(PageReader pageReader,
+                                Map<String, KintoneColumnOption> columnOptions,
+                                String updateKeyName)
+    {
+        this.pageReader = pageReader;
+        this.columnOptions = columnOptions;
+        this.updateKeyName = updateKeyName;
+    }
+
     public void setRecord(Record record)
     {
         this.record = record;
@@ -50,20 +61,17 @@ public class KintoneColumnVisitor
 
     private void setValue(String fieldCode, Object value, FieldType type, boolean isUpdateKey)
     {
-        if (value == null) {
-            return;
-        }
-
         if (isUpdateKey && updateKey != null) {
             updateKey
                 .setField(fieldCode)
-                .setValue(String.valueOf(value));
+                .setValue(Objects.toString(value, ""));
         }
-        String stringValue = String.valueOf(value);
-        FieldValue fieldValue = null;
+        String stringValue = Objects.toString(value, "");
+        FieldValue fieldValue;
         switch (type) {
             case NUMBER:
-                fieldValue = new NumberFieldValue(new BigDecimal(stringValue));
+                BigDecimal setValue = stringValue.equals("") ? null : new BigDecimal(stringValue);
+                fieldValue = new NumberFieldValue(setValue);
                 break;
             case MULTI_LINE_TEXT:
                 fieldValue = new MultiLineTextFieldValue(stringValue);
@@ -133,11 +141,11 @@ public class KintoneColumnVisitor
 
     private boolean isUpdateKey(Column column)
     {
-        KintoneColumnOption option = columnOptions.get(column.getName());
-        if (option == null) {
+        if (this.updateKeyName == null) {
             return false;
         }
-        return option.getUpdateKey();
+
+        return this.updateKeyName.equals(column.getName());
     }
 
     private String getValueSeparator(Column column)
@@ -162,7 +170,11 @@ public class KintoneColumnVisitor
     {
         String fieldCode = getFieldCode(column);
         FieldType type = getType(column, FieldType.NUMBER);
-        setValue(fieldCode, pageReader.getLong(column), type, isUpdateKey(column));
+        if (pageReader.isNull(column)) {
+            setValue(fieldCode, null, type, isUpdateKey(column));
+        } else {
+            setValue(fieldCode, pageReader.getLong(column), type, isUpdateKey(column));
+        }
     }
 
     @Override

data/src/main/java/org/embulk/output/kintone/KintoneOutputPlugin.java

@@ -1,6 +1,7 @@
 package org.embulk.output.kintone;
 
 import org.embulk.config.ConfigDiff;
+import org.embulk.config.ConfigException;
 import org.embulk.config.ConfigSource;
 import org.embulk.config.TaskReport;
 import org.embulk.config.TaskSource;
@@ -50,32 +51,18 @@ public class KintoneOutputPlugin
         KintoneMode mode = KintoneMode.getKintoneModeByValue(task.getMode());
         switch (mode) {
             case INSERT:
-                for (KintoneColumnOption option : options) {
-                    if (option.getUpdateKey()) {
-                        throw new IllegalArgumentException(
-                                "when mode is insert, require no update_key.");
-                    }
+                if (task.getUpdateKeyName().isPresent()) {
+                    throw new ConfigException("when mode is insert, require no update_key.");
                 }
                 break;
             case UPDATE:
             case UPSERT:
-                boolean hasUpdateKey = false;
-                for (KintoneColumnOption option : options) {
-                    if (option.getUpdateKey()) {
-                        if (hasUpdateKey) {
-                            throw new IllegalArgumentException(
-                                    "when mode is update and upsert, only one column can have an update_key.");
-                        }
-                        hasUpdateKey = true;
-                    }
-                }
-                if (!hasUpdateKey) {
-                    throw new IllegalArgumentException(
-                            "when mode is update and upsert, require update_key.");
+                if (!task.getUpdateKeyName().isPresent()) {
+                    throw new ConfigException("when mode is update or upsert, require update_key.");
                 }
                 break;
             default:
-                throw new IllegalArgumentException(String.format(
+                throw new ConfigException(String.format(
                         "Unknown mode '%s'",
                         task.getMode()));
         }

data/src/main/java/org/embulk/output/kintone/KintonePageOutput.java

@@ -3,6 +3,7 @@ package org.embulk.output.kintone;
 import com.kintone.client.KintoneClient;
 import com.kintone.client.KintoneClientBuilder;
 import com.kintone.client.api.record.GetRecordsByCursorResponseBody;
+import com.kintone.client.model.record.FieldType;
 import com.kintone.client.model.record.Record;
 import com.kintone.client.model.record.RecordForUpdate;
 import com.kintone.client.model.record.UpdateKey;
@@ -14,7 +15,6 @@ import org.embulk.spi.PageReader;
 import org.embulk.spi.Schema;
 import org.embulk.spi.TransactionalPageOutput;
 
-import java.math.BigDecimal;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
@@ -23,6 +23,8 @@ import java.util.stream.Collectors;
 public class KintonePageOutput
         implements TransactionalPageOutput
 {
+    public static final int UPSERT_BATCH_SIZE = 10000;
+    public static final int CHUNK_SIZE = 100;
     private PageReader pageReader;
     private PluginTask task;
     private KintoneClient client;
@@ -136,7 +138,7 @@ public class KintonePageOutput
                     }
 
                     records.add(record);
-                    if (records.size() == 100) {
+                    if (records.size() == CHUNK_SIZE) {
                         client.record().addRecords(task.getAppId(), records);
                         records.clear();
                     }
@@ -155,10 +157,16 @@ public class KintonePageOutput
     {
         execute(client -> {
             try {
-                ArrayList<RecordForUpdate> updateRecords = new ArrayList<RecordForUpdate>();
+                if (!task.getUpdateKeyName().isPresent()) {
+                    // already validated in KintoneOutputPlugin.open()
+                    throw new RuntimeException("unreachable error");
+                }
+                ArrayList<RecordForUpdate> updateRecords = new ArrayList<>();
                 pageReader.setPage(page);
+
                 KintoneColumnVisitor visitor = new KintoneColumnVisitor(pageReader,
-                        task.getColumnOptions());
+                        task.getColumnOptions(),
+                        task.getUpdateKeyName().get());
                 while (pageReader.nextRecord()) {
                     Record record = new Record();
                     UpdateKey updateKey = new UpdateKey();
@@ -168,9 +176,13 @@ public class KintonePageOutput
                         column.visit(visitor);
                     }
 
+                    if (updateKey.getValue() == "") {
+                        continue;
+                    }
+
                     record.removeField(updateKey.getField());
                     updateRecords.add(new RecordForUpdate(updateKey, record));
-                    if (updateRecords.size() == 100) {
+                    if (updateRecords.size() == CHUNK_SIZE) {
                         client.record().updateRecords(task.getAppId(), updateRecords);
                         updateRecords.clear();
                     }
@@ -185,183 +197,133 @@ public class KintonePageOutput
         });
     }
 
-    private List<Record> getRecordsByUpdateKey(String fieldCode, List<String> queryValues)
+    private void upsertPage(final Page page)
     {
-        List<Record> allRecords = new ArrayList<Record>();
-        List<String> fields = Arrays.asList(fieldCode);
-        String cursorId = client.record().createCursor(
-            task.getAppId(),
-            fields,
-            fieldCode + " in (" + String.join(",", queryValues) + ")"
-        );
-        while (true) {
-            GetRecordsByCursorResponseBody resp = client.record().getRecordsByCursor(cursorId);
-            List<Record> records = resp.getRecords();
-            allRecords.addAll(records);
+        execute(client -> {
+            try {
+                if (!task.getUpdateKeyName().isPresent()) {
+                    // already validated in KintoneOutputPlugin.open()
+                    throw new RuntimeException("unreachable error");
+                }
+                ArrayList<Record> records = new ArrayList<>();
+                ArrayList<UpdateKey> updateKeys = new ArrayList<>();
+                pageReader.setPage(page);
 
-            if (!resp.hasNext()) {
-                break;
+                KintoneColumnVisitor visitor = new KintoneColumnVisitor(pageReader,
+                        task.getColumnOptions(),
+                        task.getUpdateKeyName().get());
+                while (pageReader.nextRecord()) {
+                    Record record = new Record();
+                    UpdateKey updateKey = new UpdateKey();
+                    visitor.setRecord(record);
+                    visitor.setUpdateKey(updateKey);
+                    for (Column column : pageReader.getSchema().getColumns()) {
+                        column.visit(visitor);
+                    }
+                    records.add(record);
+                    updateKeys.add(updateKey);
+
+                    if (records.size() == UPSERT_BATCH_SIZE) {
+                        upsert(records, updateKeys);
+                        records.clear();
+                        updateKeys.clear();
+                    }
+                }
+                if (records.size() > 0) {
+                    upsert(records, updateKeys);
+                }
             }
-        }
-        return allRecords;
+            catch (Exception e) {
+                throw new RuntimeException("kintone throw exception", e);
+            }
+        });
     }
 
-    abstract class UpsertPage<T>
+    private void upsert(ArrayList<Record> records, ArrayList<UpdateKey> updateKeys)
     {
-        public abstract List<T> getUpdateKeyValues(List<String> queryValues);
-        public abstract boolean existsRecord(List<T> updateKeyValues, Record record);
-
-        public void upsert(ArrayList<Record> records, ArrayList<UpdateKey> updateKeys)
-        {
-            if (records.size() != updateKeys.size()) {
-                throw new RuntimeException("records.size() != updateKeys.size()");
-            }
-
-            List<String> queryValues = updateKeys
-                .stream()
-                .map(k -> "\"" + k.getValue().toString() + "\"")
-                .collect(Collectors.toList());
-            List<T> updateKeyValues = getUpdateKeyValues(queryValues);
+        if (records.size() != updateKeys.size()) {
+            throw new RuntimeException("records.size() != updateKeys.size()");
+        }
 
-            ArrayList<Record> insertRecords = new ArrayList<>();
-            ArrayList<RecordForUpdate> updateRecords = new ArrayList<RecordForUpdate>();
-            for (int i = 0; i < records.size(); i++) {
-                Record record = records.get(i);
-                UpdateKey updateKey = updateKeys.get(i);
+        List<Record> existingRecords = getExistingRecordsByUpdateKey(updateKeys);
 
-                if (existsRecord(updateKeyValues, record)) {
-                    record.removeField(updateKey.getField());
-                    updateRecords.add(new RecordForUpdate(updateKey, record));
-                }
-                else {
-                    insertRecords.add(record);
-                }
+        ArrayList<Record> insertRecords = new ArrayList<>();
+        ArrayList<RecordForUpdate> updateRecords = new ArrayList<>();
+        for (int i = 0; i < records.size(); i++) {
+            Record record = records.get(i);
+            UpdateKey updateKey = updateKeys.get(i);
 
-                if (insertRecords.size() == 100) {
-                    client.record().addRecords(task.getAppId(), insertRecords);
-                    insertRecords.clear();
-                }
-                else if (updateRecords.size() == 100) {
-                    client.record().updateRecords(task.getAppId(), updateRecords);
-                    updateRecords.clear();
-                }
+            if (existsRecord(existingRecords, updateKey)) {
+                record.removeField(updateKey.getField());
+                updateRecords.add(new RecordForUpdate(updateKey, record));
+            }
+            else {
+                insertRecords.add(record);
             }
-            if (insertRecords.size() > 0) {
+
+            if (insertRecords.size() == CHUNK_SIZE) {
                 client.record().addRecords(task.getAppId(), insertRecords);
+                insertRecords.clear();
             }
-            if (updateRecords.size() > 0) {
+            else if (updateRecords.size() == CHUNK_SIZE) {
                 client.record().updateRecords(task.getAppId(), updateRecords);
+                updateRecords.clear();
             }
         }
-
-        public void run(final Page page)
-        {
-            execute(client -> {
-                try {
-                    ArrayList<Record> records = new ArrayList<>();
-                    ArrayList<UpdateKey> updateKeys = new ArrayList<>();
-
-                    pageReader.setPage(page);
-                    KintoneColumnVisitor visitor = new KintoneColumnVisitor(pageReader,
-                            task.getColumnOptions());
-                    while (pageReader.nextRecord()) {
-                        Record record = new Record();
-                        UpdateKey updateKey = new UpdateKey();
-                        visitor.setRecord(record);
-                        visitor.setUpdateKey(updateKey);
-                        for (Column column : pageReader.getSchema().getColumns()) {
-                            column.visit(visitor);
-                        }
-                        records.add(record);
-                        updateKeys.add(updateKey);
-
-                        if (records.size() == 10000) {
-                            upsert(records, updateKeys);
-                            records.clear();
-                            updateKeys.clear();
-                        }
-                    }
-                    if (records.size() > 0) {
-                        upsert(records, updateKeys);
-                    }
-                }
-                catch (Exception e) {
-                    throw new RuntimeException("kintone throw exception", e);
-                }
-            });
-        }
-    }
-
-    class UpsertPageByStringKey extends UpsertPage<String>
-    {
-        private String fieldCode;
-
-        public UpsertPageByStringKey(String fieldCode)
-        {
-            this.fieldCode = fieldCode;
-        }
-
-        public List<String> getUpdateKeyValues(List<String> queryValues)
-        {
-            return getRecordsByUpdateKey(fieldCode, queryValues)
-                .stream()
-                .map(r -> r.getSingleLineTextFieldValue(fieldCode))
-                .collect(Collectors.toList());
+        if (insertRecords.size() > 0) {
+            client.record().addRecords(task.getAppId(), insertRecords);
         }
-
-        public boolean existsRecord(List<String> updateKeyValues, Record record)
-        {
-            return updateKeyValues.contains(record.getSingleLineTextFieldValue(fieldCode));
+        if (updateRecords.size() > 0) {
+            client.record().updateRecords(task.getAppId(), updateRecords);
         }
     }
 
-    class UpsertPageByNumberKey extends UpsertPage<BigDecimal>
-    {
-        private String fieldCode;
-
-        public UpsertPageByNumberKey(String fieldCode)
-        {
-            this.fieldCode = fieldCode;
-        }
 
-        public List<BigDecimal> getUpdateKeyValues(List<String> queryValues)
-        {
-            return getRecordsByUpdateKey(fieldCode, queryValues)
+    private List<Record> getExistingRecordsByUpdateKey(ArrayList<UpdateKey> updateKeys)
+    {
+        String fieldCode = updateKeys.get(0).getField();
+        List<String> queryValues = updateKeys
                 .stream()
-                .map(r -> r.getNumberFieldValue(fieldCode))
+                .filter(k -> k.getValue() != "")
+                .map(k -> "\"" + k.getValue().toString() + "\"")
                 .collect(Collectors.toList());
-        }
 
-        public boolean existsRecord(List<BigDecimal> updateKeyValues, Record record)
-        {
-            return updateKeyValues.contains(record.getNumberFieldValue(fieldCode));
+        List<Record> allRecords = new ArrayList<>();
+        if (queryValues.isEmpty()) {
+            return allRecords;
         }
-    }
+        String cursorId = client.record().createCursor(
+            task.getAppId(),
+            Arrays.asList(fieldCode),
+            fieldCode + " in (" + String.join(",", queryValues) + ")"
+        );
+        while (true) {
+            GetRecordsByCursorResponseBody resp = client.record().getRecordsByCursor(cursorId);
+            List<Record> records = resp.getRecords();
+            allRecords.addAll(records);
 
-    private void upsertPage(final Page page)
-    {
-        KintoneColumnOption updateKeyColumn = null;
-        for (KintoneColumnOption v : task.getColumnOptions().values()) {
-            if (v.getUpdateKey()) {
-                updateKeyColumn = v;
+            if (!resp.hasNext()) {
                 break;
             }
         }
-        if (updateKeyColumn == null) {
-            throw new RuntimeException("when mode is upsert, require update_key");
-        }
+        return allRecords;
+    }
 
-        UpsertPage runner;
-        switch(updateKeyColumn.getType()) {
-            case "SINGLE_LINE_TEXT":
-                runner = new UpsertPageByStringKey(updateKeyColumn.getFieldCode());
-                break;
-            case "NUMBER":
-                runner = new UpsertPageByNumberKey(updateKeyColumn.getFieldCode());
-                break;
+    private boolean existsRecord(List<Record> distRecords, UpdateKey updateKey)
+    {
+        String fieldCode = updateKey.getField();
+        FieldType type = client.app().getFormFields(task.getAppId()).get(fieldCode).getType();
+        switch (type) {
+            case SINGLE_LINE_TEXT:
+                return distRecords
+                        .stream()
+                        .anyMatch(d -> d.getSingleLineTextFieldValue(fieldCode).equals(updateKey.getValue()));
+            case NUMBER:
+                return distRecords
+                        .stream()
+                        .anyMatch(d -> d.getNumberFieldValue(fieldCode).equals(updateKey.getValue()));
             default:
                 throw new RuntimeException("The update_key must be 'SINGLE_LINE_TEXT' or 'NUMBER'.");
         }
-        runner.run(page);
     }
 }

data/src/main/java/org/embulk/output/kintone/PluginTask.java

@@ -47,4 +47,8 @@ public interface PluginTask
     @Config("mode")
     @ConfigDefault("insert")
     public String getMode();
+
+    @Config("update_key")
+    @ConfigDefault("null")
+    Optional<String> getUpdateKeyName();
 }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-kintone
 version: !ruby/object:Gem::Version
-  version: 0.2.2
+  version: 0.3.0
 platform: ruby
 authors:
 - takeshi fujita
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-11-18 00:00:00.000000000 Z
+date: 2022-03-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -52,7 +52,7 @@ files:
 - README.md
 - Rakefile
 - build.gradle
-- classpath/embulk-output-kintone-0.2.2.jar
+- classpath/embulk-output-kintone-0.3.0.jar
 - config/checkstyle/checkstyle.xml
 - config/checkstyle/default.xml
 - gradle/wrapper/gradle-wrapper.jar