embulk-output-kintone 0.1.1 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA256:
-  metadata.gz: 256bcb9a1f785c4811ab072062b8bca8cf7d55cfc6dd01a883313ac341abf971
-  data.tar.gz: bec48d54a62afbee336ef60c271e5795d6567d19b93a3029857403572a8418dd
+SHA1:
+  metadata.gz: 12c249d873d4feb22cad46c5d7bd8858f21c6138
+  data.tar.gz: fe0ff0d7a8433caa4afe30a90c2acd7bdffb73ac
 SHA512:
-  metadata.gz: 4fc949cbe0b4f631005b7de223902cee34c23099484d412e89bd5fad58ec099ca4e8ed536121348043cfa2b813630efc34f6274858d93ddb066b85c7b53b980a
-  data.tar.gz: 2216fee31203a3f34dfa422b488d87d966848500c43db5045fb056780f082375d434d4b3ce82645ae6ee1b39973112714608f8dfa9eaf6bc2f6edc325fb7994e
+  metadata.gz: 6f6a9c530c8c9ea1b3324aace5dbac3a9cbb6eaf9a800a064942f1229565a86cfb0ad51bd95ce53cadfc014320981c4e126addf699a32e16b90e7667a2745004
+  data.tar.gz: 96f86db9027769370cf6af39f2ff157d7ca6b691d1712fa2ec112e04a464dc87e6db5b3d664ef45d69db73f213ce5e7007f0321c86809a5d5d17c11b1bb0479f

data/.github/workflows/ci.yml

@@ -0,0 +1,13 @@
+name: Java CI
+
+on: [push, pull_request]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-java@v1
+        with:
+          java-version: 1.8
+      - run: ./gradlew build

data/.gitignore

@@ -10,3 +10,4 @@ build/
 /.metadata/
 .classpath
 .project
+config.yml

data/README.md

@@ -5,7 +5,6 @@
 ## Overview
 
 kintone output plugin for Embulk stores app records from kintone.
-embulk 0.9 is only supported due to the dependency of kintone-java-sdk 0.4.0, which requires java 8
 
 ## Configuration
 
@@ -23,23 +22,24 @@ embulk 0.9 is only supported due to the dependency of kintone-java-sdk 0.4.0, wh
     - **type**: field type (string, required)
     - **timezone**: timezone to convert into `date` (string, default is `UTC`)
     - **update_key**: update key (boolean, default is `false`)
+    - **val_sep**: Used to specify multiple checkbox values (string, default is `,`)
 
 ## Example
 
 ```yaml
 out:
   type: kintone
-    domain: example.cybozu.com
-    username: username
-    password: password
-    app_id: 1
-    mode: insert
-    column_options:
-      id: {field_code: "id", type: "NUMBER"}
-      name: {field_code: "name", type: "SINGLE_LINE_TEXT"}
-      number: {field_code: "num", type: "NUMBER"}
-      date: {field_code: "date", type: "DATE"}
-      date_time: {field_code: "datetime", type: "DATETIME"}
+  domain: example.cybozu.com
+  username: username
+  password: password
+  app_id: 1
+  mode: insert
+  column_options:
+    id: {field_code: "id", type: "NUMBER"}
+    name: {field_code: "name", type: "SINGLE_LINE_TEXT"}
+    number: {field_code: "num", type: "NUMBER"}
+    date: {field_code: "date", type: "DATE"}
+    date_time: {field_code: "datetime", type: "DATETIME"}
 ```
 
 

data/build.gradle

@@ -1,6 +1,7 @@
 plugins {
     id "com.jfrog.bintray" version "1.1"
     id "com.github.jruby-gradle.base" version "1.5.0"
+    id "com.github.johnrengelman.shadow" version "4.0.3"
     id "java"
     id "checkstyle"
 }
@@ -13,23 +14,31 @@ configurations {
     provided
 }
 
-version = "0.1.1"
+version = "0.2.0"
 
 sourceCompatibility = 1.8
 targetCompatibility = 1.8
 
+shadowJar {
+    exclude "org/embulk/plugin/**"
+    relocate "com.fasterxml.jackson", "embulk.kintone.com.fasterxml.jackson"
+}
+
 dependencies {
     compile  "org.embulk:embulk-core:0.9.20"
     provided "org.embulk:embulk-core:0.9.20"
-    compile group: 'com.cybozu.kintone', name: 'kintone-sdk', version: '0.4.0'
+    compile "com.kintone:kintone-java-client:1.0.2"
 
     // compile "YOUR_JAR_DEPENDENCY_GROUP:YOUR_JAR_DEPENDENCY_MODULE:YOUR_JAR_DEPENDENCY_VERSION"
     testCompile "junit:junit:4.+"
 }
 
-task classpath(type: Copy, dependsOn: ["jar"]) {
+task classpath(type: Copy, dependsOn: ["jar", "shadowJar"]) {
     doFirst { file("classpath").deleteDir() }
-    from (configurations.runtime - configurations.provided + files(jar.archivePath))
+    from (configurations.runtime - configurations.provided
+            + configurations.shadow
+            - files(shadowJar.getIncludedDependencies())
+            + files(shadowJar.archivePath))
     into "classpath"
 }
 clean { delete "classpath" }

data/gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +1,5 @@
-#Wed Dec 11 14:01:39 JST 2019
-distributionUrl=https\://services.gradle.org/distributions/gradle-4.1-all.zip
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-zipStorePath=wrapper/dists
+distributionUrl=https\://services.gradle.org/distributions/gradle-6.6-bin.zip
 zipStoreBase=GRADLE_USER_HOME
+zipStorePath=wrapper/dists

data/gradlew

@@ -1,5 +1,21 @@
 #!/usr/bin/env sh
 
+#
+# Copyright 2015 the original author or authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
 ##############################################################################
 ##
 ##  Gradle start up script for UN*X
@@ -28,7 +44,7 @@ APP_NAME="Gradle"
 APP_BASE_NAME=`basename "$0"`
 
 # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
-DEFAULT_JVM_OPTS=""
+DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
 
 # Use the maximum available, or set MAX_FD != -1 to use that value.
 MAX_FD="maximum"
@@ -66,6 +82,7 @@ esac
 
 CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
 
+
 # Determine the Java command to use to start the JVM.
 if [ -n "$JAVA_HOME" ] ; then
     if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
@@ -109,10 +126,11 @@ if $darwin; then
     GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\""
 fi
 
-# For Cygwin, switch paths to Windows format before running java
-if $cygwin ; then
+# For Cygwin or MSYS, switch paths to Windows format before running java
+if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then
     APP_HOME=`cygpath --path --mixed "$APP_HOME"`
     CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`
+
     JAVACMD=`cygpath --unix "$JAVACMD"`
 
     # We build the pattern for arguments to be converted via cygpath
@@ -138,19 +156,19 @@ if $cygwin ; then
         else
             eval `echo args$i`="\"$arg\""
         fi
-        i=$((i+1))
+        i=`expr $i + 1`
     done
     case $i in
-        (0) set -- ;;
-        (1) set -- "$args0" ;;
-        (2) set -- "$args0" "$args1" ;;
-        (3) set -- "$args0" "$args1" "$args2" ;;
-        (4) set -- "$args0" "$args1" "$args2" "$args3" ;;
-        (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;;
-        (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;;
-        (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;;
-        (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;;
-        (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;;
+        0) set -- ;;
+        1) set -- "$args0" ;;
+        2) set -- "$args0" "$args1" ;;
+        3) set -- "$args0" "$args1" "$args2" ;;
+        4) set -- "$args0" "$args1" "$args2" "$args3" ;;
+        5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;;
+        6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;;
+        7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;;
+        8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;;
+        9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;;
     esac
 fi
 
@@ -159,14 +177,9 @@ save () {
     for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done
     echo " "
 }
-APP_ARGS=$(save "$@")
+APP_ARGS=`save "$@"`
 
 # Collect all arguments for the java command, following the shell quoting and substitution rules
 eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS"
 
-# by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong
-if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then
-  cd "$(dirname "$0")"
-fi
-
 exec "$JAVACMD" "$@"

data/gradlew.bat

@@ -1,3 +1,19 @@
+@rem
+@rem Copyright 2015 the original author or authors.
+@rem
+@rem Licensed under the Apache License, Version 2.0 (the "License");
+@rem you may not use this file except in compliance with the License.
+@rem You may obtain a copy of the License at
+@rem
+@rem      https://www.apache.org/licenses/LICENSE-2.0
+@rem
+@rem Unless required by applicable law or agreed to in writing, software
+@rem distributed under the License is distributed on an "AS IS" BASIS,
+@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+@rem See the License for the specific language governing permissions and
+@rem limitations under the License.
+@rem
+
 @if "%DEBUG%" == "" @echo off
 @rem ##########################################################################
 @rem
@@ -13,15 +29,18 @@ if "%DIRNAME%" == "" set DIRNAME=.
 set APP_BASE_NAME=%~n0
 set APP_HOME=%DIRNAME%
 
+@rem Resolve any "." and ".." in APP_HOME to make it shorter.
+for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi
+
 @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
-set DEFAULT_JVM_OPTS=
+set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m"
 
 @rem Find java.exe
 if defined JAVA_HOME goto findJavaFromJavaHome
 
 set JAVA_EXE=java.exe
 %JAVA_EXE% -version >NUL 2>&1
-if "%ERRORLEVEL%" == "0" goto init
+if "%ERRORLEVEL%" == "0" goto execute
 
 echo.
 echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
@@ -35,7 +54,7 @@ goto fail
 set JAVA_HOME=%JAVA_HOME:"=%
 set JAVA_EXE=%JAVA_HOME%/bin/java.exe
 
-if exist "%JAVA_EXE%" goto init
+if exist "%JAVA_EXE%" goto execute
 
 echo.
 echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
@@ -45,28 +64,14 @@ echo location of your Java installation.
 
 goto fail
 
-:init
-@rem Get command-line arguments, handling Windows variants
-
-if not "%OS%" == "Windows_NT" goto win9xME_args
-
-:win9xME_args
-@rem Slurp the command line arguments.
-set CMD_LINE_ARGS=
-set _SKIP=2
-
-:win9xME_args_slurp
-if "x%~1" == "x" goto execute
-
-set CMD_LINE_ARGS=%*
-
 :execute
 @rem Setup the command line
 
 set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
 
+
 @rem Execute Gradle
-"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
+"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %*
 
 :end
 @rem End local scope for the variables with windows NT shell

data/src/main/java/org/embulk/output/kintone/KintoneColumnOption.java

@@ -22,4 +22,8 @@ public interface KintoneColumnOption
     @Config("update_key")
     @ConfigDefault("false")
     public boolean getUpdateKey();
+
+    @Config("val_sep")
+    @ConfigDefault("\",\"")
+    public String getValueSeparator();
 }

data/src/main/java/org/embulk/output/kintone/KintoneColumnVisitor.java

@@ -1,22 +1,34 @@
 package org.embulk.output.kintone;
 
-import com.cybozu.kintone.client.model.app.form.FieldType;
-import com.cybozu.kintone.client.model.record.field.FieldValue;
+import com.kintone.client.model.record.CheckBoxFieldValue;
+import com.kintone.client.model.record.DateFieldValue;
+import com.kintone.client.model.record.DateTimeFieldValue;
+import com.kintone.client.model.record.DropDownFieldValue;
+import com.kintone.client.model.record.FieldType;
+import com.kintone.client.model.record.FieldValue;
+import com.kintone.client.model.record.LinkFieldValue;
+import com.kintone.client.model.record.MultiLineTextFieldValue;
+import com.kintone.client.model.record.NumberFieldValue;
+import com.kintone.client.model.record.Record;
+import com.kintone.client.model.record.SingleLineTextFieldValue;
+import com.kintone.client.model.record.UpdateKey;
 import org.embulk.spi.Column;
 import org.embulk.spi.ColumnVisitor;
 import org.embulk.spi.PageReader;
 import org.embulk.spi.time.Timestamp;
-import org.embulk.spi.time.TimestampFormatter;
-import org.joda.time.DateTimeZone;
 
-import java.util.HashMap;
+import java.math.BigDecimal;
+import java.time.Instant;
+import java.time.ZoneId;
+import java.time.ZonedDateTime;
 import java.util.Map;
 
 public class KintoneColumnVisitor
         implements ColumnVisitor
 {
     private PageReader pageReader;
-    private HashMap record;
+    private Record record;
+    private UpdateKey updateKey;
     private Map<String, KintoneColumnOption> columnOptions;
 
     public KintoneColumnVisitor(PageReader pageReader,
@@ -26,20 +38,71 @@ public class KintoneColumnVisitor
         this.columnOptions = columnOptions;
     }
 
-    public void setRecord(HashMap record)
+    public void setRecord(Record record)
     {
         this.record = record;
     }
 
-    private void setValue(String fieldCode, Object value, FieldType type)
+    public void setUpdateKey(UpdateKey updateKey)
+    {
+        this.updateKey = updateKey;
+    }
+
+    private void setValue(String fieldCode, Object value, FieldType type, boolean isUpdateKey)
     {
         if (value == null) {
             return;
         }
-        FieldValue fieldValue = new FieldValue();
-        fieldValue.setType(type);
-        record.put(fieldCode, fieldValue);
-        fieldValue.setValue(String.valueOf(value));
+
+        if (isUpdateKey && updateKey != null) {
+            updateKey
+                .setField(fieldCode)
+                .setValue(String.valueOf(value));
+        }
+        else {
+            String stringValue = String.valueOf(value);
+            FieldValue fieldValue = null;
+            switch (type) {
+                case NUMBER:
+                    fieldValue = new NumberFieldValue(new BigDecimal(stringValue));
+                    break;
+                case MULTI_LINE_TEXT:
+                    fieldValue = new MultiLineTextFieldValue(stringValue);
+                    break;
+                case DROP_DOWN:
+                    fieldValue = new DropDownFieldValue(stringValue);
+                    break;
+                case LINK:
+                    fieldValue = new LinkFieldValue(stringValue);
+                    break;
+                default:
+                    fieldValue = new SingleLineTextFieldValue(stringValue);
+            }
+            record.putField(fieldCode, fieldValue);
+        }
+    }
+
+    private void setTimestampValue(String fieldCode, Instant instant, ZoneId zoneId, FieldType type)
+    {
+        FieldValue fieldValue = null;
+        ZonedDateTime datetime = instant.atZone(zoneId);
+        switch (type) {
+            case DATE:
+                fieldValue = new DateFieldValue(datetime.toLocalDate());
+                break;
+            case DATETIME:
+                fieldValue = new DateTimeFieldValue(datetime);
+        }
+        record.putField(fieldCode, fieldValue);
+    }
+
+    private void setCheckBoxValue(String fieldCode, Object value, String valueSeparator)
+    {
+        String str = String.valueOf(value);
+        record.putField(
+            fieldCode,
+            new CheckBoxFieldValue(str.split(valueSeparator, 0))
+        );
     }
 
     private FieldType getType(Column column, FieldType defaultType)
@@ -64,10 +127,28 @@ public class KintoneColumnVisitor
         }
     }
 
-    private DateTimeZone getTimezone(Column column)
+    private ZoneId getZoneId(Column column)
+    {
+        KintoneColumnOption option = columnOptions.get(column.getName());
+        return ZoneId.of(option.getTimezone().get());
+    }
+
+    private boolean isUpdateKey(Column column)
+    {
+        KintoneColumnOption option = columnOptions.get(column.getName());
+        if (option == null) {
+            return false;
+        }
+        return option.getUpdateKey();
+    }
+
+    private String getValueSeparator(Column column)
     {
         KintoneColumnOption option = columnOptions.get(column.getName());
-        return DateTimeZone.forID(option.getTimezone().get());
+        if (option == null) {
+            return ",";
+        }
+        return option.getValueSeparator();
     }
 
     @Override
@@ -75,7 +156,7 @@ public class KintoneColumnVisitor
     {
         String fieldCode = getFieldCode(column);
         FieldType type = getType(column, FieldType.NUMBER);
-        setValue(fieldCode, pageReader.getBoolean(column), type);
+        setValue(fieldCode, pageReader.getBoolean(column), type, isUpdateKey(column));
     }
 
     @Override
@@ -83,7 +164,7 @@ public class KintoneColumnVisitor
     {
         String fieldCode = getFieldCode(column);
         FieldType type = getType(column, FieldType.NUMBER);
-        setValue(fieldCode, pageReader.getLong(column), type);
+        setValue(fieldCode, pageReader.getLong(column), type, isUpdateKey(column));
     }
 
     @Override
@@ -91,7 +172,7 @@ public class KintoneColumnVisitor
     {
         String fieldCode = getFieldCode(column);
         FieldType type = getType(column, FieldType.NUMBER);
-        setValue(fieldCode, pageReader.getDouble(column), type);
+        setValue(fieldCode, pageReader.getDouble(column), type, isUpdateKey(column));
     }
 
     @Override
@@ -99,39 +180,28 @@ public class KintoneColumnVisitor
     {
         String fieldCode = getFieldCode(column);
         FieldType type = getType(column, FieldType.MULTI_LINE_TEXT);
-        setValue(fieldCode, pageReader.getString(column), type);
+        if (type == FieldType.CHECK_BOX) {
+            setCheckBoxValue(fieldCode, pageReader.getString(column), getValueSeparator(column));
+            return;
+        }
+        setValue(fieldCode, pageReader.getString(column), type, isUpdateKey(column));
     }
 
     @Override
     public void timestampColumn(Column column)
     {
-        String fieldCode = getFieldCode(column);
-        FieldType type = getType(column, FieldType.DATETIME);
         Timestamp value = pageReader.getTimestamp(column);
         if (value == null) {
             return;
         }
-        switch (type) {
-            case DATE: {
-                String format = "%Y-%m-%d";
-                DateTimeZone timezone = getTimezone(column);
-                TimestampFormatter formatter = new TimestampFormatter(format, timezone);
-                String date = formatter.format(value);
-                setValue(fieldCode, date, type);
-                break;
-            }
-            case DATETIME: {
-                String format = "%Y-%m-%dT%H:%M:%S%z";
-                DateTimeZone timezone = DateTimeZone.forID("UTC");
-                TimestampFormatter formatter = new TimestampFormatter(format, timezone);
-                String dateTime = formatter.format(value);
-                setValue(fieldCode, dateTime, type);
-                break;
-            }
-            default: {
-                setValue(fieldCode, value, type);
-            }
+
+        String fieldCode = getFieldCode(column);
+        FieldType type = getType(column, FieldType.DATETIME);
+        ZoneId zoneId = getZoneId(column);
+        if (type == FieldType.DATETIME) {
+            zoneId = ZoneId.of("UTC");
         }
+        setTimestampValue(fieldCode, value.getInstant(), zoneId, type);
     }
 
     @Override
@@ -139,6 +209,6 @@ public class KintoneColumnVisitor
     {
         String fieldCode = getFieldCode(column);
         FieldType type = getType(column, FieldType.MULTI_LINE_TEXT);
-        setValue(fieldCode, pageReader.getJson(column), type);
+        setValue(fieldCode, pageReader.getJson(column), type, isUpdateKey(column));
     }
 }

data/src/main/java/org/embulk/output/kintone/KintoneMode.java

@@ -1,36 +1,31 @@
 package org.embulk.output.kintone;
 
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonValue;
 import org.embulk.config.ConfigException;
 
-import java.util.Locale;
-
 public enum KintoneMode
 {
-    INSERT, UPDATE, UPSERT;
+    INSERT("insert"), UPDATE("update"), UPSERT("upsert");
+
+    private final String value;
 
-    @JsonCreator
-    public static KintoneMode fromString(String value)
+    KintoneMode(String value)
     {
-        switch (value) {
-            case "insert":
-                return INSERT;
-            case "update":
-                return UPDATE;
-            case "upsert":
-                return UPSERT;
-            default:
-                throw new ConfigException(String.format(
-                        "Unknown mode '%s'. Supported modes are insert",
-                        value));
-        }
+        this.value = value;
     }
 
-    @JsonValue
     @Override
     public String toString()
     {
-        return name().toLowerCase(Locale.ENGLISH);
+        return value;
+    }
+
+    public static KintoneMode getKintoneModeByValue(String value)
+    {
+        for (KintoneMode mode : values()) {
+            if (mode.toString().equals(value)) {
+                return mode;
+            }
+        }
+        throw new ConfigException(String.format("Unknown mode '%s'", value));
     }
 }

data/src/main/java/org/embulk/output/kintone/KintoneOutputPlugin.java

@@ -45,10 +45,11 @@ public class KintoneOutputPlugin
     public TransactionalPageOutput open(TaskSource taskSource, Schema schema, int taskIndex)
     {
         PluginTask task = taskSource.loadTask(PluginTask.class);
+        Collection<KintoneColumnOption> options = task.getColumnOptions().values();
 
-        switch (task.getMode()) {
+        KintoneMode mode = KintoneMode.getKintoneModeByValue(task.getMode());
+        switch (mode) {
             case INSERT:
-                Collection<KintoneColumnOption> options = task.getColumnOptions().values();
                 for (KintoneColumnOption option : options) {
                     if (option.getUpdateKey()) {
                         throw new IllegalArgumentException(
@@ -57,12 +58,26 @@ public class KintoneOutputPlugin
                 }
                 break;
             case UPDATE:
-                // TODO updatePage
             case UPSERT:
-                // TODO upsertPage
+                boolean hasUpdateKey = false;
+                for (KintoneColumnOption option : options) {
+                    if (option.getUpdateKey()) {
+                        if (hasUpdateKey) {
+                            throw new IllegalArgumentException(
+                                    "when mode is update and upsert, only one column can have an update_key.");
+                        }
+                        hasUpdateKey = true;
+                    }
+                }
+                if (!hasUpdateKey) {
+                    throw new IllegalArgumentException(
+                            "when mode is update and upsert, require update_key.");
+                }
+                break;
             default:
-                throw new IllegalArgumentException(
-                        "mode is insert only.");
+                throw new IllegalArgumentException(String.format(
+                        "Unknown mode '%s'",
+                        task.getMode()));
         }
         return new KintonePageOutput(task, schema);
     }

data/src/main/java/org/embulk/output/kintone/KintonePageOutput.java

@@ -1,9 +1,11 @@
 package org.embulk.output.kintone;
 
-import com.cybozu.kintone.client.authentication.Auth;
-import com.cybozu.kintone.client.connection.Connection;
-import com.cybozu.kintone.client.model.record.field.FieldValue;
-import com.cybozu.kintone.client.module.record.Record;
+import com.kintone.client.KintoneClient;
+import com.kintone.client.KintoneClientBuilder;
+import com.kintone.client.api.record.GetRecordsByCursorResponseBody;
+import com.kintone.client.model.record.Record;
+import com.kintone.client.model.record.RecordForUpdate;
+import com.kintone.client.model.record.UpdateKey;
 import org.embulk.config.TaskReport;
 import org.embulk.spi.Column;
 import org.embulk.spi.Exec;
@@ -12,15 +14,20 @@ import org.embulk.spi.PageReader;
 import org.embulk.spi.Schema;
 import org.embulk.spi.TransactionalPageOutput;
 
+import java.math.BigDecimal;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
 
 public class KintonePageOutput
         implements TransactionalPageOutput
 {
     private PageReader pageReader;
     private PluginTask task;
-    private Connection conn;
+    private KintoneClient client;
 
     public KintonePageOutput(PluginTask task, Schema schema)
     {
@@ -31,17 +38,21 @@ public class KintonePageOutput
     @Override
     public void add(Page page)
     {
-        switch (task.getMode()) {
+        KintoneMode mode = KintoneMode.getKintoneModeByValue(task.getMode());
+        switch (mode) {
             case INSERT:
                 insertPage(page);
                 break;
             case UPDATE:
-                // TODO updatePage
+                updatePage(page);
+                break;
             case UPSERT:
-                // TODO upsertPage
+                upsertPage(page);
+                break;
             default:
-                throw new UnsupportedOperationException(
-                        "kintone output plugin does not support update, upsert");
+                throw new UnsupportedOperationException(String.format(
+                        "Unknown mode '%s'",
+                        task.getMode()));
         }
     }
 
@@ -54,7 +65,15 @@ public class KintonePageOutput
     @Override
     public void close()
     {
-        // noop
+        if (this.client == null) {
+            return;
+        }
+        try {
+            this.client.close();
+        }
+        catch (Exception e) {
+            throw new RuntimeException("kintone throw exception", e);
+        }
     }
 
     @Override
@@ -76,56 +95,90 @@ public class KintonePageOutput
 
     public void connect(final PluginTask task)
     {
-        Auth kintoneAuth = new Auth();
-        if (task.getUsername().isPresent() && task.getPassword().isPresent()) {
-            kintoneAuth.setPasswordAuth(task.getUsername().get(), task.getPassword().get());
-        }
-        else if (task.getToken().isPresent()) {
-            kintoneAuth.setApiToken(task.getToken().get());
+        KintoneClientBuilder builder = KintoneClientBuilder.create("https://" + task.getDomain());
+        if (task.getGuestSpaceId().isPresent()) {
+            builder.setGuestSpaceId(task.getGuestSpaceId().or(-1));
         }
-
         if (task.getBasicAuthUsername().isPresent() && task.getBasicAuthPassword().isPresent()) {
-            kintoneAuth.setBasicAuth(task.getBasicAuthUsername().get(),
+            builder.withBasicAuth(task.getBasicAuthUsername().get(),
                     task.getBasicAuthPassword().get());
         }
 
-        if (task.getGuestSpaceId().isPresent()) {
-            this.conn = new Connection(task.getDomain(), kintoneAuth, task.getGuestSpaceId().or(-1));
+        if (task.getUsername().isPresent() && task.getPassword().isPresent()) {
+            this.client = builder
+                .authByPassword(task.getUsername().get(), task.getPassword().get())
+                .build();
         }
-        else {
-            this.conn = new Connection(task.getDomain(), kintoneAuth);
+        else if (task.getToken().isPresent()) {
+            this.client = builder
+                .authByApiToken(task.getToken().get())
+                .build();
         }
     }
 
-    private void execute(Consumer<Connection> operation)
+    private void execute(Consumer<KintoneClient> operation)
     {
         connect(task);
-        operation.accept(this.conn);
+        operation.accept(this.client);
     }
 
     private void insertPage(final Page page)
     {
-        execute(conn -> {
+        execute(client -> {
             try {
-                ArrayList<HashMap<String, FieldValue>> records = new ArrayList<>();
+                ArrayList<Record> records = new ArrayList<>();
                 pageReader.setPage(page);
                 KintoneColumnVisitor visitor = new KintoneColumnVisitor(pageReader,
                         task.getColumnOptions());
-                Record kintoneRecordManager = new Record(conn);
                 while (pageReader.nextRecord()) {
-                    HashMap record = new HashMap();
+                    Record record = new Record();
                     visitor.setRecord(record);
                     for (Column column : pageReader.getSchema().getColumns()) {
                         column.visit(visitor);
                     }
+
                     records.add(record);
                     if (records.size() == 100) {
-                        kintoneRecordManager.addRecords(task.getAppId(), records);
+                        client.record().addRecords(task.getAppId(), records);
                         records.clear();
                     }
                 }
                 if (records.size() > 0) {
-                    kintoneRecordManager.addRecords(task.getAppId(), records);
+                    client.record().addRecords(task.getAppId(), records);
+                }
+            }
+            catch (Exception e) {
+                throw new RuntimeException("kintone throw exception", e);
+            }
+        });
+    }
+
+    private void updatePage(final Page page)
+    {
+        execute(client -> {
+            try {
+                ArrayList<RecordForUpdate> updateRecords = new ArrayList<RecordForUpdate>();
+                pageReader.setPage(page);
+                KintoneColumnVisitor visitor = new KintoneColumnVisitor(pageReader,
+                        task.getColumnOptions());
+                while (pageReader.nextRecord()) {
+                    Record record = new Record();
+                    UpdateKey updateKey = new UpdateKey();
+                    visitor.setRecord(record);
+                    visitor.setUpdateKey(updateKey);
+                    for (Column column : pageReader.getSchema().getColumns()) {
+                        column.visit(visitor);
+                    }
+
+                    record.removeField(updateKey.getField());
+                    updateRecords.add(new RecordForUpdate(updateKey, record));
+                    if (updateRecords.size() == 100) {
+                        client.record().updateRecords(task.getAppId(), updateRecords);
+                        updateRecords.clear();
+                    }
+                }
+                if (updateRecords.size() > 0) {
+                    client.record().updateRecords(task.getAppId(), updateRecords);
                 }
             }
             catch (Exception e) {
@@ -133,4 +186,151 @@ public class KintonePageOutput
             }
         });
     }
+
+    private List<Record> getAllRecords(String fieldCode)
+    {
+        List<Record> allRecords = new ArrayList<Record>();
+        List<String> fields = Arrays.asList(fieldCode);
+        String cursorId = client.record().createCursor(task.getAppId(), fields, null);
+        while (true) {
+            GetRecordsByCursorResponseBody resp = client.record().getRecordsByCursor(cursorId);
+            List<Record> records = resp.getRecords();
+            allRecords.addAll(records);
+
+            if (!resp.hasNext()) {
+                break;
+            }
+        }
+        return allRecords;
+    }
+
+    abstract class UpsertPage<T>
+    {
+        public abstract List<T> getUpdateKeyValues();
+        public abstract boolean existsRecord(List<T> updateKeyValues, Record record);
+
+        public void run(final Page page)
+        {
+            execute(client -> {
+                try {
+                    List<T> updateKeyValues = getUpdateKeyValues();
+
+                    ArrayList<Record> insertRecords = new ArrayList<>();
+                    ArrayList<RecordForUpdate> updateRecords = new ArrayList<RecordForUpdate>();
+                    pageReader.setPage(page);
+                    KintoneColumnVisitor visitor = new KintoneColumnVisitor(pageReader,
+                            task.getColumnOptions());
+                    while (pageReader.nextRecord()) {
+                        Record record = new Record();
+                        UpdateKey updateKey = new UpdateKey();
+                        visitor.setRecord(record);
+                        visitor.setUpdateKey(updateKey);
+                        for (Column column : pageReader.getSchema().getColumns()) {
+                            column.visit(visitor);
+                        }
+
+                        if (existsRecord(updateKeyValues, record)) {
+                            record.removeField(updateKey.getField());
+                            updateRecords.add(new RecordForUpdate(updateKey, record));
+                        }
+                        else {
+                            insertRecords.add(record);
+                        }
+
+                        if (insertRecords.size() == 100) {
+                            client.record().addRecords(task.getAppId(), insertRecords);
+                            insertRecords.clear();
+                        }
+                        else if (updateRecords.size() == 100) {
+                            client.record().updateRecords(task.getAppId(), updateRecords);
+                            updateRecords.clear();
+                        }
+                    }
+
+                    if (insertRecords.size() > 0) {
+                        client.record().addRecords(task.getAppId(), insertRecords);
+                    }
+                    if (updateRecords.size() > 0) {
+                        client.record().updateRecords(task.getAppId(), updateRecords);
+                    }
+                }
+                catch (Exception e) {
+                    throw new RuntimeException("kintone throw exception", e);
+                }
+            });
+        }
+    }
+
+    class UpsertPageByStringKey extends UpsertPage<String>
+    {
+        private String fieldCode;
+
+        public UpsertPageByStringKey(String fieldCode)
+        {
+            this.fieldCode = fieldCode;
+        }
+
+        public List<String> getUpdateKeyValues()
+        {
+            return getAllRecords(fieldCode)
+                .stream()
+                .map(r -> r.getSingleLineTextFieldValue(fieldCode))
+                .collect(Collectors.toList());
+        }
+
+        public boolean existsRecord(List<String> updateKeyValues, Record record)
+        {
+            return updateKeyValues.contains(record.getSingleLineTextFieldValue(fieldCode));
+        }
+    }
+
+    class UpsertPageByNumberKey extends UpsertPage<BigDecimal>
+    {
+        private String fieldCode;
+
+        public UpsertPageByNumberKey(String fieldCode)
+        {
+            this.fieldCode = fieldCode;
+        }
+
+        public List<BigDecimal> getUpdateKeyValues()
+        {
+            return getAllRecords(fieldCode)
+                .stream()
+                .map(r -> r.getNumberFieldValue(fieldCode))
+                .collect(Collectors.toList());
+        }
+
+        public boolean existsRecord(List<BigDecimal> updateKeyValues, Record record)
+        {
+            return updateKeyValues.contains(record.getNumberFieldValue(fieldCode));
+        }
+    }
+
+    private void upsertPage(final Page page)
+    {
+        KintoneColumnOption updateKeyColumn = null;
+        for (KintoneColumnOption v : task.getColumnOptions().values()) {
+            if (v.getUpdateKey()) {
+                updateKeyColumn = v;
+                break;
+            }
+        }
+        if (updateKeyColumn == null) {
+            throw new RuntimeException("when mode is upsert, require update_key");
+        }
+
+        UpsertPage runner;
+        switch(updateKeyColumn.getType()) {
+            case "SINGLE_LINE_TEXT":
+                runner = new UpsertPageByStringKey(updateKeyColumn.getFieldCode());
+                break;
+            case "NUMBER":
+                runner = new UpsertPageByNumberKey(updateKeyColumn.getFieldCode());
+                break;
+            default:
+                throw new RuntimeException("The update_key must be 'SINGLE_LINE_TEXT' or 'NUMBER'.");
+        }
+        runner.run(page);
+    }
 }

data/src/main/java/org/embulk/output/kintone/PluginTask.java

@@ -46,5 +46,5 @@ public interface PluginTask
 
     @Config("mode")
     @ConfigDefault("insert")
-    public KintoneMode getMode();
+    public String getMode();
 }

metadata

@@ -1,38 +1,38 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-kintone
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.2.0
 platform: ruby
 authors:
 - takeshi fujita
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-12-17 00:00:00.000000000 Z
+date: 2020-09-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
-  type: :development
+  name: bundler
   prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
 - !ruby/object:Gem::Dependency
-  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '12.0'
-  type: :development
+  name: rake
   prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -45,15 +45,14 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/workflows/ci.yml"
 - ".gitignore"
 - Gemfile
 - LICENSE.txt
 - README.md
 - Rakefile
 - build.gradle
-- classpath/embulk-output-kintone-0.1.0.jar
-- classpath/gson-2.8.2.jar
-- classpath/kintone-sdk-0.4.0.jar
+- classpath/embulk-output-kintone-0.2.0.jar
 - config/checkstyle/checkstyle.xml
 - config/checkstyle/default.xml
 - gradle/wrapper/gradle-wrapper.jar
@@ -72,7 +71,7 @@ homepage: https://github.com/trocco-io/embulk-output-kintone
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -87,8 +86,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
-signing_key: 
+rubyforge_project:
+rubygems_version: 2.6.8
+signing_key:
 specification_version: 4
 summary: kintone output plugin for Embulk
 test_files: []