embulk-output-kintone 0.4.0 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d405f264473164aaf9581956d1a5343fe993a4a1
-  data.tar.gz: 85988a65971c21fbc058401a9af1e600b8029f1d
+  metadata.gz: 11c7f1cbe588863b206b1581bf2ec89adeafeea4
+  data.tar.gz: c17ae7651f3409a6e51999e65233a96da2d87bf4
 SHA512:
-  metadata.gz: bc2eb92ad6fb1a200f4567aeb4e57baebbdf636a94ab40834c4864d48dc83891fa2fc23152e86e283fe28d7fa9248c1bd08497fd7f0ab4887def5ca3c6f5dafb
-  data.tar.gz: 9a8295b11401e64357a67ffb733e51f45e8321b68377e21e3d66cd50ea3c20f62bbf7c9d28e58d44b9fb805eb483b35af96e72868ecae383fb81d53935c0c6e5
+  metadata.gz: 4e1bc2fe10bc224c643d88c06a37a19bfe4849fb46318fede913ef3a3923e1315ce48b78aea8bbdcc5ea9a5b4760058776c569e8acedf8f02691b80b330c5657
+  data.tar.gz: 9261603e49b45d2c8b72bf013d2995057293dd7f98ae0588f4dfa2f7e962b0ba2eac70990cdd7658eedf72a5caabd9cbb8f3135bc80cb91a7d250706c06081b4

data/.github/workflows/ci.yml

@@ -1,6 +1,7 @@
 name: main
 
 on:
+  workflow_dispatch:
   push:
     branches:
       - "master"

data/.gitignore

@@ -10,4 +10,3 @@ build/
 /.metadata/
 .classpath
 .project
-config.yml

data/README.md

@@ -8,19 +8,22 @@ kintone output plugin for Embulk stores app records from kintone.
 
 ## Configuration
 
-- **domain**: kintone domain(FQDN) e.g. devfoo.cybozu.com (string, required)
+- **domain**: kintone domain (FQDN) e.g. devfoo.cybozu.com (string, required)
 - **username**: kintone username (string, optional)
 - **password**: kintone password (string, optional)
 - **token**: kintone app token. Username and password or token must be configured. If all of them are provided, this plugin uses username and password (string, optional)
 - **app_id**: kintone app id (integer, required)
-- **basic_auth_username**:  kintone basic auth username Please see kintone basic auth [here](https://jp.cybozu.help/general/en/admin/list_security/list_ip_basic/basic_auth.html) (string, optional)
-- **basic_auth_password**:  kintone basic auth password (string, optional)
+- **basic_auth_username**: kintone basic auth username Please see kintone basic auth [here](https://jp.cybozu.help/general/en/admin/list_security/list_ip_basic/basic_auth.html) (string, optional)
+- **basic_auth_password**: kintone basic auth password (string, optional)
 - **guest_space_id**: kintone app belongs to guest space, guest space id is required. (integer, optional)
 - **mode**: kintone mode (string, required)
-- **update_key**: column name to set update key (string, required if mode is update or upsert)
+- **update_key**: Column name to set update key (string, required if mode is update or upsert)
+- **prefer_nulls**: Whether to set fields to null instead of default value of type when column is null (boolean, default is `false`)
+- **ignore_nulls**: Whether to completely ignore fields when column is null (boolean, default is `false`)
 - **column_options** advanced: a key-value pairs where key is a column name and value is options for the column.
     - **field_code**: field code (string, required)
-    - **type**: field type (string, required)
+    - **type**: field type (string, required). See [this page](https://cybozu.dev/ja/kintone/docs/overview/field-types/#field-type-update) for list of available types. However, following types are not yet supported
+        - `USER_SELECT`, `ORGANIZATION_SELECT`, `GROUP_SELECT`, `FILE`, `SUBTABLE`
     - **timezone**: timezone to convert into `date` (string, default is `UTC`)
     - **val_sep**: Used to specify multiple checkbox values (string, default is `,`)
 
@@ -43,7 +46,6 @@ out:
     date_time: {field_code: "datetime", type: "DATETIME"}
 ```
 
-
 ## Build
 
 ```

data/build.gradle

@@ -1,7 +1,7 @@
 plugins {
     id "com.jfrog.bintray" version "1.1"
     id "com.github.jruby-gradle.base" version "1.5.0"
-    id "com.github.johnrengelman.shadow" version "4.0.3"
+    id "com.github.johnrengelman.shadow" version "6.1.0" apply false
     id "java"
     id "checkstyle"
     id "com.palantir.git-version" version "0.12.3"
@@ -12,9 +12,6 @@ repositories {
     mavenCentral()
     jcenter()
 }
-configurations {
-    provided
-}
 
 version = {
     def vd = versionDetails()
@@ -29,26 +26,23 @@ version = {
 sourceCompatibility = 1.8
 targetCompatibility = 1.8
 
-shadowJar {
-    exclude "org/embulk/plugin/**"
-    relocate "com.fasterxml.jackson", "embulk.kintone.com.fasterxml.jackson"
-}
-
 dependencies {
-    compile  "org.embulk:embulk-core:0.9.20"
-    provided "org.embulk:embulk-core:0.9.20"
-    compile "com.kintone:kintone-java-client:1.0.2"
+    compileOnly "org.embulk:embulk-core:0.9.23"
+    implementation project(path: ":shadow-kintone-java-client", configuration: "shadow")
 
-    // compile "YOUR_JAR_DEPENDENCY_GROUP:YOUR_JAR_DEPENDENCY_MODULE:YOUR_JAR_DEPENDENCY_VERSION"
-    testCompile "junit:junit:4.+"
+    testImplementation "junit:junit:4.+"
+    testImplementation "org.embulk:embulk-test:0.9.23"
+    testImplementation "org.embulk:embulk-standards:0.9.23"
+    testImplementation "org.embulk:embulk-deps-buffer:0.9.23"
+    testImplementation "org.embulk:embulk-deps-config:0.9.23"
+    testImplementation "org.mockito:mockito-inline:4.11.0"
+    testImplementation "ch.qos.logback:logback-classic:1.3.8"
+    testImplementation "net.jcip:jcip-annotations:1.0"
 }
 
-task classpath(type: Copy, dependsOn: ["jar", "shadowJar"]) {
+task classpath(type: Copy, dependsOn: ["jar"]) {
     doFirst { file("classpath").deleteDir() }
-    from (configurations.runtime - configurations.provided
-            + configurations.shadow
-            - files(shadowJar.getIncludedDependencies())
-            + files(shadowJar.archivePath))
+    from (configurations.runtimeClasspath + files(jar.archivePath))
     into "classpath"
 }
 clean { delete "classpath" }
@@ -127,3 +121,8 @@ spotless {
         toggleOffOn()
     }
 }
+
+test {
+    forkEvery = 1
+    maxHeapSize = "1g"
+}

data/gradle/wrapper/gradle-wrapper.properties

@@ -1,5 +1,5 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-6.6-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-6.9.2-bin.zip
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists

data/settings.gradle

@@ -0,0 +1,2 @@
+rootProject.name = "embulk-output-kintone"
+include "shadow-kintone-java-client"

data/shadow-kintone-java-client/build.gradle

@@ -0,0 +1,35 @@
+plugins {
+    id "java"
+    id "maven-publish"
+    id "com.github.johnrengelman.shadow"
+}
+
+repositories {
+    mavenCentral()
+}
+
+group = "${rootProject.group}"
+version = "${rootProject.version}"
+
+sourceCompatibility = 1.8
+targetCompatibility = 1.8
+
+configurations {
+    runtimeClasspath {
+        resolutionStrategy.activateDependencyLocking()
+    }
+    shadow {
+        resolutionStrategy.activateDependencyLocking()
+        transitive = false
+    }
+}
+
+dependencies {
+    implementation("com.kintone:kintone-java-client:1.4.0") {
+        exclude group: "org.slf4j", module: "slf4j-api"
+    }
+}
+
+shadowJar {
+    relocate "com.fasterxml.jackson", "embulk.kintone.com.fasterxml.jackson"
+}

data/src/main/java/org/embulk/output/kintone/KintoneColumnOption.java

@@ -1,6 +1,5 @@
 package org.embulk.output.kintone;
 
-import java.util.Optional;
 import org.embulk.config.Config;
 import org.embulk.config.ConfigDefault;
 import org.embulk.config.Task;
@@ -14,7 +13,7 @@ public interface KintoneColumnOption extends Task {
 
   @Config("timezone")
   @ConfigDefault("\"UTC\"")
-  Optional<String> getTimezone();
+  String getTimezone();
 
   @Config("val_sep")
   @ConfigDefault("\",\"")

data/src/main/java/org/embulk/output/kintone/KintoneColumnType.java

@@ -0,0 +1,368 @@
+package org.embulk.output.kintone;
+
+import com.kintone.client.model.record.CheckBoxFieldValue;
+import com.kintone.client.model.record.DateFieldValue;
+import com.kintone.client.model.record.DateTimeFieldValue;
+import com.kintone.client.model.record.DropDownFieldValue;
+import com.kintone.client.model.record.FieldValue;
+import com.kintone.client.model.record.FileFieldValue;
+import com.kintone.client.model.record.GroupSelectFieldValue;
+import com.kintone.client.model.record.LinkFieldValue;
+import com.kintone.client.model.record.MultiLineTextFieldValue;
+import com.kintone.client.model.record.MultiSelectFieldValue;
+import com.kintone.client.model.record.NumberFieldValue;
+import com.kintone.client.model.record.OrganizationSelectFieldValue;
+import com.kintone.client.model.record.RadioButtonFieldValue;
+import com.kintone.client.model.record.RichTextFieldValue;
+import com.kintone.client.model.record.SingleLineTextFieldValue;
+import com.kintone.client.model.record.SubtableFieldValue;
+import com.kintone.client.model.record.TimeFieldValue;
+import com.kintone.client.model.record.UpdateKey;
+import com.kintone.client.model.record.UserSelectFieldValue;
+import java.math.BigDecimal;
+import java.time.Instant;
+import java.time.LocalDate;
+import java.time.LocalTime;
+import java.time.ZoneId;
+import java.time.ZoneOffset;
+import java.time.ZonedDateTime;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import org.embulk.spi.time.Timestamp;
+import org.msgpack.value.Value;
+
+public enum KintoneColumnType {
+  SINGLE_LINE_TEXT {
+    @Override
+    public SingleLineTextFieldValue getFieldValue() {
+      return new SingleLineTextFieldValue(null);
+    }
+
+    @Override
+    public SingleLineTextFieldValue getFieldValue(String value, KintoneColumnOption option) {
+      return new SingleLineTextFieldValue(value);
+    }
+
+    @Override
+    public void setUpdateKey(UpdateKey updateKey, String field) {
+      updateKey.setField(field);
+    }
+
+    @Override
+    public void setUpdateKey(UpdateKey updateKey, String field, FieldValue value) {
+      updateKey.setField(field).setValue(((SingleLineTextFieldValue) value).getValue());
+    }
+  },
+  MULTI_LINE_TEXT {
+    @Override
+    public MultiLineTextFieldValue getFieldValue() {
+      return new MultiLineTextFieldValue(null);
+    }
+
+    @Override
+    public MultiLineTextFieldValue getFieldValue(String value, KintoneColumnOption option) {
+      return new MultiLineTextFieldValue(value);
+    }
+  },
+  RICH_TEXT {
+    @Override
+    public RichTextFieldValue getFieldValue() {
+      return new RichTextFieldValue(null);
+    }
+
+    @Override
+    public RichTextFieldValue getFieldValue(String value, KintoneColumnOption option) {
+      return new RichTextFieldValue(value);
+    }
+  },
+  NUMBER {
+    @Override
+    public NumberFieldValue getFieldValue() {
+      return new NumberFieldValue(null);
+    }
+
+    @Override
+    public NumberFieldValue getFieldValue(boolean value, KintoneColumnOption option) {
+      return getFieldValue(value ? "1" : "0", option);
+    }
+
+    @Override
+    public NumberFieldValue getFieldValue(String value, KintoneColumnOption option) {
+      return new NumberFieldValue(new BigDecimal(value.isEmpty() ? "0" : value));
+    }
+
+    @Override
+    public NumberFieldValue getFieldValue(Timestamp value, KintoneColumnOption option) {
+      return (NumberFieldValue) getFieldValue(value.getEpochSecond(), option);
+    }
+
+    @Override
+    public void setUpdateKey(UpdateKey updateKey, String field) {
+      updateKey.setField(field);
+    }
+
+    @Override
+    public void setUpdateKey(UpdateKey updateKey, String field, FieldValue value) {
+      updateKey.setField(field).setValue(((NumberFieldValue) value).getValue());
+    }
+  },
+  CHECK_BOX {
+    @Override
+    public CheckBoxFieldValue getFieldValue() {
+      return new CheckBoxFieldValue();
+    }
+
+    @Override
+    public CheckBoxFieldValue getFieldValue(String value, KintoneColumnOption option) {
+      return new CheckBoxFieldValue(asList(value, option));
+    }
+  },
+  RADIO_BUTTON {
+    @Override
+    public RadioButtonFieldValue getFieldValue() {
+      return new RadioButtonFieldValue(null);
+    }
+
+    @Override
+    public RadioButtonFieldValue getFieldValue(String value, KintoneColumnOption option) {
+      return new RadioButtonFieldValue(value);
+    }
+  },
+  MULTI_SELECT {
+    @Override
+    public MultiSelectFieldValue getFieldValue() {
+      return new MultiSelectFieldValue();
+    }
+
+    @Override
+    public MultiSelectFieldValue getFieldValue(String value, KintoneColumnOption option) {
+      return new MultiSelectFieldValue(asList(value, option));
+    }
+  },
+  DROP_DOWN {
+    @Override
+    public DropDownFieldValue getFieldValue() {
+      return new DropDownFieldValue(null);
+    }
+
+    @Override
+    public DropDownFieldValue getFieldValue(String value, KintoneColumnOption option) {
+      return new DropDownFieldValue(value);
+    }
+  },
+  USER_SELECT {
+    @Override
+    public UserSelectFieldValue getFieldValue() {
+      return new UserSelectFieldValue();
+    }
+
+    @Override
+    public UserSelectFieldValue getFieldValue(String value, KintoneColumnOption option) {
+      throw new UnsupportedOperationException();
+    }
+  },
+  ORGANIZATION_SELECT {
+    @Override
+    public OrganizationSelectFieldValue getFieldValue() {
+      return new OrganizationSelectFieldValue();
+    }
+
+    @Override
+    public OrganizationSelectFieldValue getFieldValue(String value, KintoneColumnOption option) {
+      throw new UnsupportedOperationException();
+    }
+  },
+  GROUP_SELECT {
+    @Override
+    public GroupSelectFieldValue getFieldValue() {
+      return new GroupSelectFieldValue();
+    }
+
+    @Override
+    public GroupSelectFieldValue getFieldValue(String value, KintoneColumnOption option) {
+      throw new UnsupportedOperationException();
+    }
+  },
+  DATE {
+    @Override
+    public DateFieldValue getFieldValue() {
+      return new DateFieldValue(null);
+    }
+
+    @Override
+    public DateFieldValue getFieldValue(long value, KintoneColumnOption option) {
+      return getFieldValue(Timestamp.ofEpochSecond(value), option);
+    }
+
+    @Override
+    public DateFieldValue getFieldValue(double value, KintoneColumnOption option) {
+      return getFieldValue(Double.valueOf(value).longValue(), option);
+    }
+
+    @Override
+    public DateFieldValue getFieldValue(String value, KintoneColumnOption option) {
+      return value.isEmpty()
+          ? getFieldValue(EPOCH, option)
+          : new DateFieldValue(
+              LocalDate.parse(value)
+                  .atStartOfDay(ZoneOffset.UTC)
+                  .withZoneSameInstant(getZoneId(option))
+                  .toLocalDate());
+    }
+
+    @Override
+    public DateFieldValue getFieldValue(Timestamp value, KintoneColumnOption option) {
+      return new DateFieldValue(value.getInstant().atZone(getZoneId(option)).toLocalDate());
+    }
+  },
+  TIME {
+    @Override
+    public TimeFieldValue getFieldValue() {
+      return new TimeFieldValue(null);
+    }
+
+    @Override
+    public TimeFieldValue getFieldValue(long value, KintoneColumnOption option) {
+      return getFieldValue(Timestamp.ofEpochSecond(value), option);
+    }
+
+    @Override
+    public TimeFieldValue getFieldValue(double value, KintoneColumnOption option) {
+      return getFieldValue(Double.valueOf(value).longValue(), option);
+    }
+
+    @Override
+    public TimeFieldValue getFieldValue(String value, KintoneColumnOption option) {
+      return value.isEmpty()
+          ? getFieldValue(EPOCH, option)
+          : new TimeFieldValue(
+              LocalTime.parse(value)
+                  .atOffset(ZoneOffset.UTC)
+                  .withOffsetSameInstant(getZoneOffset(option))
+                  .toLocalTime());
+    }
+
+    @Override
+    public TimeFieldValue getFieldValue(Timestamp value, KintoneColumnOption option) {
+      return new TimeFieldValue(value.getInstant().atZone(getZoneId(option)).toLocalTime());
+    }
+  },
+  DATETIME {
+    @Override
+    public DateTimeFieldValue getFieldValue() {
+      return new DateTimeFieldValue(null);
+    }
+
+    @Override
+    public DateTimeFieldValue getFieldValue(long value, KintoneColumnOption option) {
+      return getFieldValue(Timestamp.ofEpochSecond(value), option);
+    }
+
+    @Override
+    public DateTimeFieldValue getFieldValue(double value, KintoneColumnOption option) {
+      return getFieldValue(Double.valueOf(value).longValue(), option);
+    }
+
+    @Override
+    public DateTimeFieldValue getFieldValue(String value, KintoneColumnOption option) {
+      return value.isEmpty()
+          ? getFieldValue(EPOCH, option)
+          : new DateTimeFieldValue(ZonedDateTime.parse(value));
+    }
+
+    @Override
+    public DateTimeFieldValue getFieldValue(Timestamp value, KintoneColumnOption option) {
+      return new DateTimeFieldValue(value.getInstant().atZone(ZoneOffset.UTC));
+    }
+  },
+  LINK {
+    @Override
+    public LinkFieldValue getFieldValue() {
+      return new LinkFieldValue(null);
+    }
+
+    @Override
+    public LinkFieldValue getFieldValue(String value, KintoneColumnOption option) {
+      return new LinkFieldValue(value);
+    }
+  },
+  FILE {
+    @Override
+    public FileFieldValue getFieldValue() {
+      return new FileFieldValue();
+    }
+
+    @Override
+    public FileFieldValue getFieldValue(String value, KintoneColumnOption option) {
+      throw new UnsupportedOperationException();
+    }
+  },
+  SUBTABLE {
+    @Override
+    public SubtableFieldValue getFieldValue() {
+      return new SubtableFieldValue();
+    }
+
+    @Override
+    public SubtableFieldValue getFieldValue(String value, KintoneColumnOption option) {
+      throw new UnsupportedOperationException();
+    }
+  };
+  private static final Timestamp EPOCH = Timestamp.ofInstant(Instant.EPOCH);
+
+  public static KintoneColumnType getType(
+      KintoneColumnOption option, KintoneColumnType defaultType) {
+    return option != null ? valueOf(option.getType()) : defaultType;
+  }
+
+  public abstract FieldValue getFieldValue();
+
+  public FieldValue getFieldValue(boolean value, KintoneColumnOption option) {
+    return getFieldValue(String.valueOf(value), option);
+  }
+
+  public FieldValue getFieldValue(long value, KintoneColumnOption option) {
+    return getFieldValue(String.valueOf(value), option);
+  }
+
+  public FieldValue getFieldValue(double value, KintoneColumnOption option) {
+    return getFieldValue(String.valueOf(value), option);
+  }
+
+  public abstract FieldValue getFieldValue(String value, KintoneColumnOption option);
+
+  public FieldValue getFieldValue(Timestamp value, KintoneColumnOption option) {
+    return getFieldValue(value.getInstant().toString(), option);
+  }
+
+  public FieldValue getFieldValue(Value value, KintoneColumnOption option) {
+    return getFieldValue(value.toJson(), option);
+  }
+
+  public void setUpdateKey(UpdateKey updateKey, String field) {
+    throw new UnsupportedOperationException();
+  }
+
+  public void setUpdateKey(UpdateKey updateKey, String field, FieldValue value) {
+    throw new UnsupportedOperationException();
+  }
+
+  private static List<String> asList(String value, KintoneColumnOption option) {
+    return value.isEmpty()
+        ? Collections.emptyList()
+        : Arrays.asList(value.split(getValueSeparator(option), 0));
+  }
+
+  private static ZoneOffset getZoneOffset(KintoneColumnOption option) {
+    return getZoneId(option).getRules().getOffset(Instant.EPOCH);
+  }
+
+  private static ZoneId getZoneId(KintoneColumnOption option) {
+    return ZoneId.of(option != null ? option.getTimezone() : "UTC");
+  }
+
+  private static String getValueSeparator(KintoneColumnOption option) {
+    return option != null ? option.getValueSeparator() : ",";
+  }
+}