embulk-output-kintone 0.4.0 → 0.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d405f264473164aaf9581956d1a5343fe993a4a1
-  data.tar.gz: 85988a65971c21fbc058401a9af1e600b8029f1d
+  metadata.gz: c677006a4f3dfdbec29c97e039045874614b3e99
+  data.tar.gz: 9c1fff33d958a4ee8be940da64cc7e10af04eb92
 SHA512:
-  metadata.gz: bc2eb92ad6fb1a200f4567aeb4e57baebbdf636a94ab40834c4864d48dc83891fa2fc23152e86e283fe28d7fa9248c1bd08497fd7f0ab4887def5ca3c6f5dafb
-  data.tar.gz: 9a8295b11401e64357a67ffb733e51f45e8321b68377e21e3d66cd50ea3c20f62bbf7c9d28e58d44b9fb805eb483b35af96e72868ecae383fb81d53935c0c6e5
+  metadata.gz: d6e53840dad2b56c75e7422a769f0ae2eb8b24ce6034cc269255952e22442c77164c3bdac78a08f28086e5318df2318f99ad924d1bd85af8966ae83e5d57d627
+  data.tar.gz: 641e40a2986787204ef8e0feec1b916a2d4d0b5c0cc1d5e9d0137f38eaf8ed43b3b5b0b2f19f3304ee8736668b049d0a115836a62d8d3cd6efa20fe61ecc3be5

data/.github/workflows/ci.yml

@@ -1,6 +1,7 @@
 name: main
 
 on:
+  workflow_dispatch:
   push:
     branches:
       - "master"

data/.gitignore

@@ -10,4 +10,3 @@ build/
 /.metadata/
 .classpath
 .project
-config.yml

data/build.gradle

@@ -1,7 +1,7 @@
 plugins {
     id "com.jfrog.bintray" version "1.1"
     id "com.github.jruby-gradle.base" version "1.5.0"
-    id "com.github.johnrengelman.shadow" version "4.0.3"
+    id "com.github.johnrengelman.shadow" version "6.1.0" apply false
     id "java"
     id "checkstyle"
     id "com.palantir.git-version" version "0.12.3"
@@ -12,9 +12,6 @@ repositories {
     mavenCentral()
     jcenter()
 }
-configurations {
-    provided
-}
 
 version = {
     def vd = versionDetails()
@@ -29,26 +26,23 @@ version = {
 sourceCompatibility = 1.8
 targetCompatibility = 1.8
 
-shadowJar {
-    exclude "org/embulk/plugin/**"
-    relocate "com.fasterxml.jackson", "embulk.kintone.com.fasterxml.jackson"
-}
-
 dependencies {
-    compile  "org.embulk:embulk-core:0.9.20"
-    provided "org.embulk:embulk-core:0.9.20"
-    compile "com.kintone:kintone-java-client:1.0.2"
+    compileOnly "org.embulk:embulk-core:0.9.23"
+    implementation project(path: ":shadow-kintone-java-client", configuration: "shadow")
 
-    // compile "YOUR_JAR_DEPENDENCY_GROUP:YOUR_JAR_DEPENDENCY_MODULE:YOUR_JAR_DEPENDENCY_VERSION"
-    testCompile "junit:junit:4.+"
+    testImplementation "junit:junit:4.+"
+    testImplementation "org.embulk:embulk-test:0.9.23"
+    testImplementation "org.embulk:embulk-standards:0.9.23"
+    testImplementation "org.embulk:embulk-deps-buffer:0.9.23"
+    testImplementation "org.embulk:embulk-deps-config:0.9.23"
+    testImplementation "org.mockito:mockito-inline:4.11.0"
+    testImplementation "ch.qos.logback:logback-classic:1.3.8"
+    testImplementation "net.jcip:jcip-annotations:1.0"
 }
 
-task classpath(type: Copy, dependsOn: ["jar", "shadowJar"]) {
+task classpath(type: Copy, dependsOn: ["jar"]) {
     doFirst { file("classpath").deleteDir() }
-    from (configurations.runtime - configurations.provided
-            + configurations.shadow
-            - files(shadowJar.getIncludedDependencies())
-            + files(shadowJar.archivePath))
+    from (configurations.runtimeClasspath + files(jar.archivePath))
     into "classpath"
 }
 clean { delete "classpath" }
@@ -127,3 +121,8 @@ spotless {
         toggleOffOn()
     }
 }
+
+test {
+    forkEvery = 1
+    maxHeapSize = "1g"
+}

data/gradle/wrapper/gradle-wrapper.properties

@@ -1,5 +1,5 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-6.6-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-6.9.2-bin.zip
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists

data/settings.gradle

@@ -0,0 +1,2 @@
+rootProject.name = "embulk-output-kintone"
+include "shadow-kintone-java-client"

data/shadow-kintone-java-client/build.gradle

@@ -0,0 +1,35 @@
+plugins {
+    id "java"
+    id "maven-publish"
+    id "com.github.johnrengelman.shadow"
+}
+
+repositories {
+    mavenCentral()
+}
+
+group = "${rootProject.group}"
+version = "${rootProject.version}"
+
+sourceCompatibility = 1.8
+targetCompatibility = 1.8
+
+configurations {
+    runtimeClasspath {
+        resolutionStrategy.activateDependencyLocking()
+    }
+    shadow {
+        resolutionStrategy.activateDependencyLocking()
+        transitive = false
+    }
+}
+
+dependencies {
+    implementation("com.kintone:kintone-java-client:1.4.0") {
+        exclude group: "org.slf4j", module: "slf4j-api"
+    }
+}
+
+shadowJar {
+    relocate "com.fasterxml.jackson", "embulk.kintone.com.fasterxml.jackson"
+}

data/src/main/java/org/embulk/output/kintone/KintonePageOutput.java

@@ -1,18 +1,23 @@
 package org.embulk.output.kintone;
 
+import static org.embulk.spi.util.RetryExecutor.retryExecutor;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
 import com.kintone.client.KintoneClient;
 import com.kintone.client.KintoneClientBuilder;
 import com.kintone.client.api.record.GetRecordsByCursorResponseBody;
+import com.kintone.client.exception.KintoneApiRuntimeException;
 import com.kintone.client.model.record.FieldType;
 import com.kintone.client.model.record.Record;
 import com.kintone.client.model.record.RecordForUpdate;
 import com.kintone.client.model.record.UpdateKey;
+import java.io.IOException;
 import java.math.BigDecimal;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
-import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 import org.embulk.config.ConfigException;
 import org.embulk.config.TaskReport;
@@ -22,6 +27,8 @@ import org.embulk.spi.Page;
 import org.embulk.spi.PageReader;
 import org.embulk.spi.Schema;
 import org.embulk.spi.TransactionalPageOutput;
+import org.embulk.spi.util.RetryExecutor.RetryGiveupException;
+import org.embulk.spi.util.RetryExecutor.Retryable;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -29,6 +36,12 @@ public class KintonePageOutput implements TransactionalPageOutput {
   public static final int UPSERT_BATCH_SIZE = 10000;
   public static final int CHUNK_SIZE = 100;
   private static final Logger LOGGER = LoggerFactory.getLogger(KintonePageOutput.class);
+  private static final List<String> RETRYABLE_ERROR_CODES =
+      Arrays.asList(
+          "GAIA_TM12", // 作成できるカーソルの上限に達しているため、カーソ ルを作成できません。不要なカーソルを削除するか、しばらく経ってから再実行してください。
+          "GAIA_RE18", // データベースのロックに失敗したため、変更を保存できませんでした。時間をおいて再度お試しください。
+          "GAIA_DA02" // データベースのロックに失敗したため、変更を保存できませんでした。時間をおいて再度お試しください。
+          );
   private final PageReader pageReader;
   private final PluginTask task;
   private KintoneClient client;
@@ -104,133 +117,176 @@ public class KintonePageOutput implements TransactionalPageOutput {
     }
   }
 
+  private void update(ArrayList<RecordForUpdate> records) {
+    execute(
+        client -> {
+          client.record().updateRecords(task.getAppId(), records);
+        });
+  }
+
+  private void insert(ArrayList<Record> records) {
+    execute(
+        client -> {
+          client.record().addRecords(task.getAppId(), records);
+        });
+  }
+
   private void execute(Consumer<KintoneClient> operation) {
     connect(task);
-    if (this.client != null) {
-      operation.accept(this.client);
-    } else {
+    if (this.client == null) {
       throw new RuntimeException("Failed to connect to kintone.");
     }
+    KintoneRetryOption retryOption = task.getRetryOptions();
+    try {
+      retryExecutor()
+          .withRetryLimit(retryOption.getLimit())
+          .withInitialRetryWait(retryOption.getInitialWaitMillis())
+          .withMaxRetryWait(retryOption.getMaxWaitMillis())
+          .runInterruptible(
+              new Retryable<Void>() {
+
+                @Override
+                public Void call() throws Exception {
+                  operation.accept(client);
+                  return null;
+                }
+
+                @Override
+                public boolean isRetryableException(Exception e) {
+                  if (!(e instanceof KintoneApiRuntimeException)) {
+                    return false;
+                  }
+
+                  try {
+                    ObjectMapper mapper = new ObjectMapper();
+                    JsonNode content =
+                        mapper.readTree(((KintoneApiRuntimeException) e).getContent());
+                    String code = content.get("code").textValue();
+                    return RETRYABLE_ERROR_CODES.contains(code);
+                  } catch (IOException ex) {
+                    throw new RuntimeException(ex);
+                  }
+                }
+
+                @Override
+                public void onRetry(
+                    Exception exception, int retryCount, int retryLimit, int retryWait)
+                    throws RetryGiveupException {
+                  String message =
+                      String.format(
+                          "Retrying %d/%d after %d seconds. Message: %s",
+                          retryCount, retryLimit, retryWait / 1000, exception.getMessage());
+                  if (retryCount % 3 == 0) {
+                    LOGGER.warn(message, exception);
+                  } else {
+                    LOGGER.warn(message);
+                  }
+                }
+
+                @Override
+                public void onGiveup(Exception firstException, Exception lastException)
+                    throws RetryGiveupException {}
+              });
+    } catch (RetryGiveupException | InterruptedException e) {
+      throw new RuntimeException("kintone throw exception", e);
+    }
   }
 
   private void insertPage(final Page page) {
-    execute(
-        client -> {
-          try {
-            ArrayList<Record> records = new ArrayList<>();
-            pageReader.setPage(page);
-            KintoneColumnVisitor visitor =
-                new KintoneColumnVisitor(pageReader, task.getColumnOptions());
-            while (pageReader.nextRecord()) {
-              Record record = new Record();
-              visitor.setRecord(record);
-              for (Column column : pageReader.getSchema().getColumns()) {
-                column.visit(visitor);
-              }
-
-              records.add(record);
-              if (records.size() == CHUNK_SIZE) {
-                client.record().addRecords(task.getAppId(), records);
-                records.clear();
-                sleep();
-              }
-            }
-            if (records.size() > 0) {
-              client.record().addRecords(task.getAppId(), records);
-            }
-          } catch (Exception e) {
-            throw new RuntimeException("kintone throw exception", e);
-          }
-        });
+
+    ArrayList<Record> records = new ArrayList<>();
+    pageReader.setPage(page);
+    KintoneColumnVisitor visitor = new KintoneColumnVisitor(pageReader, task.getColumnOptions());
+    while (pageReader.nextRecord()) {
+      Record record = new Record();
+      visitor.setRecord(record);
+      for (Column column : pageReader.getSchema().getColumns()) {
+        column.visit(visitor);
+      }
+
+      records.add(record);
+      if (records.size() == CHUNK_SIZE) {
+        insert(records);
+        records.clear();
+      }
+    }
+    if (records.size() > 0) {
+      insert(records);
+    }
   }
 
   private void updatePage(final Page page) {
-    execute(
-        client -> {
-          try {
-            ArrayList<RecordForUpdate> updateRecords = new ArrayList<>();
-            pageReader.setPage(page);
-
-            KintoneColumnVisitor visitor =
-                new KintoneColumnVisitor(
-                    pageReader,
-                    task.getColumnOptions(),
-                    task.getUpdateKeyName()
-                        .orElseThrow(
-                            () -> new RuntimeException("unreachable"))); // Already validated
-            while (pageReader.nextRecord()) {
-              Record record = new Record();
-              UpdateKey updateKey = new UpdateKey();
-              visitor.setRecord(record);
-              visitor.setUpdateKey(updateKey);
-              for (Column column : pageReader.getSchema().getColumns()) {
-                column.visit(visitor);
-              }
-
-              if (updateKey.getValue() == "") {
-                continue;
-              }
-
-              record.removeField(updateKey.getField());
-              updateRecords.add(new RecordForUpdate(updateKey, record));
-              if (updateRecords.size() == CHUNK_SIZE) {
-                client.record().updateRecords(task.getAppId(), updateRecords);
-                updateRecords.clear();
-                sleep();
-              }
-            }
-            if (updateRecords.size() > 0) {
-              client.record().updateRecords(task.getAppId(), updateRecords);
-            }
-          } catch (Exception e) {
-            throw new RuntimeException("kintone throw exception", e);
-          }
-        });
+    ArrayList<RecordForUpdate> updateRecords = new ArrayList<>();
+    pageReader.setPage(page);
+
+    KintoneColumnVisitor visitor =
+        new KintoneColumnVisitor(
+            pageReader,
+            task.getColumnOptions(),
+            task.getUpdateKeyName()
+                .orElseThrow(() -> new RuntimeException("unreachable"))); // Already validated
+    while (pageReader.nextRecord()) {
+      Record record = new Record();
+      UpdateKey updateKey = new UpdateKey();
+      visitor.setRecord(record);
+      visitor.setUpdateKey(updateKey);
+      for (Column column : pageReader.getSchema().getColumns()) {
+        column.visit(visitor);
+      }
+
+      if (updateKey.getValue() == "") {
+        continue;
+      }
+
+      record.removeField(updateKey.getField());
+      updateRecords.add(new RecordForUpdate(updateKey, record));
+      if (updateRecords.size() == CHUNK_SIZE) {
+        update(updateRecords);
+        updateRecords.clear();
+      }
+    }
+    if (updateRecords.size() > 0) {
+      update(updateRecords);
+    }
   }
 
   private void upsertPage(final Page page) {
     execute(
         client -> {
-          try {
-            ArrayList<Record> records = new ArrayList<>();
-            ArrayList<UpdateKey> updateKeys = new ArrayList<>();
-            pageReader.setPage(page);
-
-            KintoneColumnVisitor visitor =
-                new KintoneColumnVisitor(
-                    pageReader,
-                    task.getColumnOptions(),
-                    task.getUpdateKeyName()
-                        .orElseThrow(
-                            () -> new RuntimeException("unreachable"))); // Already validated
-            while (pageReader.nextRecord()) {
-              Record record = new Record();
-              UpdateKey updateKey = new UpdateKey();
-              visitor.setRecord(record);
-              visitor.setUpdateKey(updateKey);
-              for (Column column : pageReader.getSchema().getColumns()) {
-                column.visit(visitor);
-              }
-              records.add(record);
-              updateKeys.add(updateKey);
-
-              if (records.size() == UPSERT_BATCH_SIZE) {
-                upsert(records, updateKeys);
-                records.clear();
-                updateKeys.clear();
-              }
+          ArrayList<Record> records = new ArrayList<>();
+          ArrayList<UpdateKey> updateKeys = new ArrayList<>();
+          pageReader.setPage(page);
+
+          KintoneColumnVisitor visitor =
+              new KintoneColumnVisitor(
+                  pageReader,
+                  task.getColumnOptions(),
+                  task.getUpdateKeyName()
+                      .orElseThrow(() -> new RuntimeException("unreachable"))); // Already validated
+          while (pageReader.nextRecord()) {
+            Record record = new Record();
+            UpdateKey updateKey = new UpdateKey();
+            visitor.setRecord(record);
+            visitor.setUpdateKey(updateKey);
+            for (Column column : pageReader.getSchema().getColumns()) {
+              column.visit(visitor);
             }
-            if (records.size() > 0) {
+            records.add(record);
+            updateKeys.add(updateKey);
+
+            if (records.size() == UPSERT_BATCH_SIZE) {
               upsert(records, updateKeys);
+              records.clear();
+              updateKeys.clear();
             }
-          } catch (Exception e) {
-            throw new RuntimeException("kintone throw exception", e);
+          }
+          if (records.size() > 0) {
+            upsert(records, updateKeys);
           }
         });
   }
 
-  private void upsert(ArrayList<Record> records, ArrayList<UpdateKey> updateKeys)
-      throws InterruptedException {
+  private void upsert(ArrayList<Record> records, ArrayList<UpdateKey> updateKeys) {
     if (records.size() != updateKeys.size()) {
       throw new RuntimeException("records.size() != updateKeys.size()");
     }
@@ -274,20 +330,18 @@ public class KintonePageOutput implements TransactionalPageOutput {
       }
 
       if (insertRecords.size() == CHUNK_SIZE) {
-        client.record().addRecords(task.getAppId(), insertRecords);
+        insert(insertRecords);
         insertRecords.clear();
-        sleep();
       } else if (updateRecords.size() == CHUNK_SIZE) {
-        client.record().updateRecords(task.getAppId(), updateRecords);
+        update(updateRecords);
         updateRecords.clear();
-        sleep();
       }
     }
     if (insertRecords.size() > 0) {
-      client.record().addRecords(task.getAppId(), insertRecords);
+      insert(insertRecords);
     }
     if (updateRecords.size() > 0) {
-      client.record().updateRecords(task.getAppId(), updateRecords);
+      update(updateRecords);
     }
   }
 
@@ -325,13 +379,4 @@ public class KintonePageOutput implements TransactionalPageOutput {
   private boolean existsRecord(List<String> distValues, UpdateKey updateKey) {
     return distValues.stream().anyMatch(v -> v.equals(updateKey.getValue().toString()));
   }
-
-  private void sleep() throws InterruptedException {
-    if (!task.getIntervalSeconds().isPresent()) {
-      return;
-    }
-    Integer interval = task.getIntervalSeconds().get();
-    LOGGER.info(String.format("sleep %d seconds.", interval));
-    TimeUnit.SECONDS.sleep(interval);
-  }
 }

data/src/main/java/org/embulk/output/kintone/KintoneRetryOption.java

@@ -0,0 +1,19 @@
+package org.embulk.output.kintone;
+
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.config.Task;
+
+public interface KintoneRetryOption extends Task {
+  @Config("limit")
+  @ConfigDefault("10")
+  Integer getLimit();
+
+  @Config("initial_wait_millis")
+  @ConfigDefault("1000")
+  Integer getInitialWaitMillis();
+
+  @Config("max_wait_millis")
+  @ConfigDefault("60000")
+  Integer getMaxWaitMillis();
+}

data/src/main/java/org/embulk/output/kintone/PluginTask.java

@@ -49,7 +49,7 @@ public interface PluginTask extends Task {
   @ConfigDefault("null")
   Optional<String> getUpdateKeyName();
 
-  @Config("interval_seconds")
-  @ConfigDefault("null")
-  Optional<Integer> getIntervalSeconds();
+  @Config("retry_options")
+  @ConfigDefault("{}")
+  KintoneRetryOption getRetryOptions();
 }

data/src/test/java/com/kintone/client/Json.java

@@ -0,0 +1,16 @@
+package com.kintone.client;
+
+import java.io.ByteArrayInputStream;
+import java.nio.charset.StandardCharsets;
+
+public class Json {
+  private static final JsonMapper MAPPER = new JsonMapper();
+
+  public static String format(Object o) {
+    return new String(MAPPER.format(o), StandardCharsets.UTF_8);
+  }
+
+  public static <T> T parse(String s, Class<T> clazz) {
+    return MAPPER.parse(new ByteArrayInputStream(s.getBytes(StandardCharsets.UTF_8)), clazz);
+  }
+}

data/src/test/java/org/embulk/output/kintone/KintoneColumnOptionBuilder.java

@@ -0,0 +1,63 @@
+package org.embulk.output.kintone;
+
+import java.util.Optional;
+import org.embulk.config.TaskSource;
+
+public class KintoneColumnOptionBuilder {
+  private String type;
+  private String fieldCode;
+  private String timezone;
+  private String valueSeparator;
+
+  public KintoneColumnOptionBuilder setType(String type) {
+    this.type = type;
+    return this;
+  }
+
+  public KintoneColumnOptionBuilder setFieldCode(String fieldCode) {
+    this.fieldCode = fieldCode;
+    return this;
+  }
+
+  public KintoneColumnOptionBuilder setTimezone(String timezone) {
+    this.timezone = timezone;
+    return this;
+  }
+
+  public KintoneColumnOptionBuilder setValueSeparator(String valueSeparator) {
+    this.valueSeparator = valueSeparator;
+    return this;
+  }
+
+  public KintoneColumnOption build() {
+    return new KintoneColumnOption() {
+      @Override
+      public String getType() {
+        return type;
+      }
+
+      @Override
+      public String getFieldCode() {
+        return fieldCode;
+      }
+
+      @Override
+      public Optional<String> getTimezone() {
+        return Optional.ofNullable(timezone);
+      }
+
+      @Override
+      public String getValueSeparator() {
+        return valueSeparator;
+      }
+
+      @Override
+      public void validate() {}
+
+      @Override
+      public TaskSource dump() {
+        return null;
+      }
+    };
+  }
+}