embulk-output-kintone 1.1.0 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 38d9d369925e0cab9fa558716b9a939be4b1387e
-  data.tar.gz: f4ae4a741548a48b042e1c4b79bf7e533ded0d31
+  metadata.gz: abd297c9f29bedc191c3abc3837ff3955fdde7df
+  data.tar.gz: 5be528c62b5f1985b756f311699f74370b05a52b
 SHA512:
-  metadata.gz: 06d103f7c4036dd458f62955bee9b2b82aa2bb2719cb59f18a2a762bab5c892025b4dd7fb0b41647e0e9cf66e012916f60de1a48c1a6d41414af0447d8affb06
-  data.tar.gz: ee848b7cfb53cc27c731724d32b6d5ffa9fe3336742fc1bbf24bd2b49d07f6f99439b478a0342051810ef94597471caded0f81caaf3aaff78203ce0ff20485d6
+  metadata.gz: 040f72edfe4cfe79b4bc0c303653efc6d1fb93b84db8d009df9bf845e6bd1e712d4c222412db5c22bf21a9b1315cf6029b30c359294c664972c0fe369ed50345
+  data.tar.gz: 808579e76d0a6d854583d0d834f487f9a102b7cb827030d5857f4f0268f310f5f854135790904b9551cca9b489c58311434b4117a53426581642c79acced3ff0

data/README.md

@@ -26,6 +26,14 @@ kintone output plugin for Embulk stores app records from kintone.
 - **max_sort_memory**: Maximum memory usage for sorting input records (bytes in long, default is the estimated available memory, which is the approximate value of the JVM's current free memory)
 - **prefer_nulls**: Whether to set fields to null instead of default value of type when column is null (boolean, default is `false`)
 - **ignore_nulls**: Whether to completely ignore fields when column is null (boolean, default is `false`)
+- **skip_if_non_existing_id_or_update_key**: The skip policy if the record corresponding to the id or update key does not exist (string `auto`, `never` or `always`, default is `auto`). No effect for insert mode.
+    - **auto**:
+        - **update mode**: Skip the record if no id or update key value is specified.
+        - **upsert mode**: Skip the record if corresponds to the id does not exist or no update key value is specified.
+    - **never**: Never skip the record even if corresponds to the id or update key does not exist.
+        - **update mode**: Throw exception if no id or update key value is specified.
+        - **upsert mode**: Insert the record if corresponds to the id or update key does not exist (also, if no id or update key value is specified).
+    - **always**: Always skip the record if corresponds to the id or update key does not exist (also, if no id or update key value is specified). update mode and upsert mode will the same behavior (only updated, never inserted).
 - **column_options** advanced: a key-value pairs where key is a column name and value is options for the column.
     - **field_code**: field code (string, required)
     - **type**: field type (string, required). See [this page](https://cybozu.dev/ja/kintone/docs/overview/field-types/#field-type-update) for list of available types. However, following types are not yet supported

data/src/main/java/org/embulk/output/kintone/KintoneClient.java

@@ -0,0 +1,117 @@
+package org.embulk.output.kintone;
+
+import com.kintone.client.KintoneClientBuilder;
+import com.kintone.client.RecordClient;
+import com.kintone.client.model.app.field.FieldProperty;
+import com.kintone.client.model.record.FieldType;
+import java.io.IOException;
+import java.util.Map;
+import java.util.function.Supplier;
+import org.embulk.config.ConfigException;
+import org.embulk.output.kintone.record.Id;
+import org.embulk.output.kintone.util.Lazy;
+import org.embulk.spi.Column;
+import org.embulk.spi.Schema;
+import org.embulk.spi.type.Type;
+import org.embulk.spi.type.Types;
+
+public class KintoneClient implements AutoCloseable {
+  private final PluginTask task;
+  private final Schema schema;
+  private final com.kintone.client.KintoneClient client;
+  private final Map<String, FieldProperty> fields;
+
+  public static Lazy<KintoneClient> lazy(Supplier<PluginTask> task, Schema schema) {
+    return new Lazy<KintoneClient>() {
+      @Override
+      protected KintoneClient initialValue() {
+        return new KintoneClient(task.get(), schema);
+      }
+    };
+  }
+
+  private KintoneClient(PluginTask task, Schema schema) {
+    this.task = task;
+    this.schema = schema;
+    KintoneClientBuilder builder = KintoneClientBuilder.create("https://" + task.getDomain());
+    if (task.getGuestSpaceId().isPresent()) {
+      builder.setGuestSpaceId(task.getGuestSpaceId().get());
+    }
+    if (task.getBasicAuthUsername().isPresent() && task.getBasicAuthPassword().isPresent()) {
+      builder.withBasicAuth(task.getBasicAuthUsername().get(), task.getBasicAuthPassword().get());
+    }
+    if (task.getUsername().isPresent() && task.getPassword().isPresent()) {
+      builder.authByPassword(task.getUsername().get(), task.getPassword().get());
+    } else if (task.getToken().isPresent()) {
+      builder.authByApiToken(task.getToken().get());
+    } else {
+      throw new ConfigException("Username and password or token must be configured.");
+    }
+    client = builder.build();
+    fields = client.app().getFormFields(task.getAppId());
+    KintoneMode.of(task).validate(task, this);
+  }
+
+  public void validateIdOrUpdateKey(String columnName) {
+    Column column = getColumn(columnName);
+    if (column == null) {
+      throw new ConfigException("The column '" + columnName + "' for update does not exist.");
+    }
+    validateId(column);
+    validateUpdateKey(column);
+  }
+
+  public Column getColumn(String columnName) {
+    return schema.getColumns().stream()
+        .filter(column -> column.getName().equals(columnName))
+        .findFirst()
+        .orElse(null);
+  }
+
+  public FieldType getFieldType(String fieldCode) {
+    FieldProperty field = fields.get(fieldCode);
+    return field == null ? null : field.getType();
+  }
+
+  public RecordClient record() {
+    return client.record();
+  }
+
+  @Override
+  public void close() {
+    try {
+      client.close();
+    } catch (IOException e) {
+      throw new RuntimeException("kintone throw exception", e);
+    }
+  }
+
+  private void validateId(Column column) {
+    if (!column.getName().equals(Id.FIELD)) {
+      return;
+    }
+    Type type = column.getType();
+    if (!type.equals(Types.LONG)) {
+      throw new ConfigException("The id column must be 'long'.");
+    }
+  }
+
+  private void validateUpdateKey(Column column) {
+    if (column.getName().equals(Id.FIELD)) {
+      return;
+    }
+    String fieldCode = getFieldCode(column);
+    FieldType fieldType = getFieldType(fieldCode);
+    if (fieldType == null) {
+      throw new ConfigException("The field '" + fieldCode + "' for update does not exist.");
+    }
+    if (fieldType != FieldType.SINGLE_LINE_TEXT && fieldType != FieldType.NUMBER) {
+      throw new ConfigException("The update_key must be 'SINGLE_LINE_TEXT' or 'NUMBER'.");
+    }
+  }
+
+  private String getFieldCode(Column column) {
+    KintoneColumnOption option = task.getColumnOptions().get(column.getName());
+    return option != null ? option.getFieldCode() : column.getName();
+  }
+}

data/src/main/java/org/embulk/output/kintone/KintoneColumnType.java

@@ -13,6 +13,7 @@ import com.kintone.client.model.record.MultiSelectFieldValue;
 import com.kintone.client.model.record.NumberFieldValue;
 import com.kintone.client.model.record.OrganizationSelectFieldValue;
 import com.kintone.client.model.record.RadioButtonFieldValue;
+import com.kintone.client.model.record.Record;
 import com.kintone.client.model.record.RichTextFieldValue;
 import com.kintone.client.model.record.SingleLineTextFieldValue;
 import com.kintone.client.model.record.SubtableFieldValue;
@@ -34,6 +35,8 @@ import org.embulk.output.kintone.deserializer.Deserializer;
 import org.embulk.spi.time.Timestamp;
 import org.embulk.spi.type.Type;
 import org.embulk.spi.type.Types;
+import org.msgpack.value.ArrayValue;
+import org.msgpack.value.StringValue;
 import org.msgpack.value.Value;
 import org.msgpack.value.ValueFactory;
 
@@ -49,6 +52,11 @@ public enum KintoneColumnType {
       return new SingleLineTextFieldValue(value);
     }
 
+    @Override
+    public String getValue(Record record, String fieldCode) {
+      return record.getSingleLineTextFieldValue(fieldCode);
+    }
+
     @Override
     public void setUpdateKey(UpdateKey updateKey, String field) {
       updateKey.setField(field);
@@ -60,7 +68,7 @@ public enum KintoneColumnType {
     }
 
     @Override
-    public Value asValue(FieldValue value) {
+    public StringValue asValue(FieldValue value) {
       return ValueFactory.newString(((SingleLineTextFieldValue) value).getValue());
     }
 
@@ -81,7 +89,7 @@ public enum KintoneColumnType {
     }
 
     @Override
-    public Value asValue(FieldValue value) {
+    public StringValue asValue(FieldValue value) {
       return ValueFactory.newString(((MultiLineTextFieldValue) value).getValue());
     }
 
@@ -102,7 +110,7 @@ public enum KintoneColumnType {
     }
 
     @Override
-    public Value asValue(FieldValue value) {
+    public StringValue asValue(FieldValue value) {
       return ValueFactory.newString(((RichTextFieldValue) value).getValue());
     }
 
@@ -132,6 +140,11 @@ public enum KintoneColumnType {
       return (NumberFieldValue) getFieldValue(value.getEpochSecond(), option);
     }
 
+    @Override
+    public BigDecimal getValue(Record record, String fieldCode) {
+      return record.getNumberFieldValue(fieldCode);
+    }
+
     @Override
     public void setUpdateKey(UpdateKey updateKey, String field) {
       updateKey.setField(field);
@@ -143,8 +156,8 @@ public enum KintoneColumnType {
     }
 
     @Override
-    public Value asValue(FieldValue value) {
-      return ValueFactory.newString(((NumberFieldValue) value).getValue().toString());
+    public StringValue asValue(FieldValue value) {
+      return ValueFactory.newString(((NumberFieldValue) value).getValue().toPlainString());
     }
 
     @Override
@@ -164,7 +177,7 @@ public enum KintoneColumnType {
     }
 
     @Override
-    public Value asValue(FieldValue value) {
+    public ArrayValue asValue(FieldValue value) {
       return ValueFactory.newArray(
           ((CheckBoxFieldValue) value)
               .getValues().stream().map(ValueFactory::newString).collect(Collectors.toList()));
@@ -187,7 +200,7 @@ public enum KintoneColumnType {
     }
 
     @Override
-    public Value asValue(FieldValue value) {
+    public StringValue asValue(FieldValue value) {
       return ValueFactory.newString(((RadioButtonFieldValue) value).getValue());
     }
 
@@ -208,7 +221,7 @@ public enum KintoneColumnType {
     }
 
     @Override
-    public Value asValue(FieldValue value) {
+    public ArrayValue asValue(FieldValue value) {
       return ValueFactory.newArray(
           ((MultiSelectFieldValue) value)
               .getValues().stream().map(ValueFactory::newString).collect(Collectors.toList()));
@@ -231,7 +244,7 @@ public enum KintoneColumnType {
     }
 
     @Override
-    public Value asValue(FieldValue value) {
+    public StringValue asValue(FieldValue value) {
       return ValueFactory.newString(((DropDownFieldValue) value).getValue());
     }
 
@@ -336,7 +349,7 @@ public enum KintoneColumnType {
     }
 
     @Override
-    public Value asValue(FieldValue value) {
+    public StringValue asValue(FieldValue value) {
       return ValueFactory.newString(((DateFieldValue) value).getValue().toString());
     }
 
@@ -378,7 +391,7 @@ public enum KintoneColumnType {
     }
 
     @Override
-    public Value asValue(FieldValue value) {
+    public StringValue asValue(FieldValue value) {
       return ValueFactory.newString(((TimeFieldValue) value).getValue().toString());
     }
 
@@ -416,7 +429,7 @@ public enum KintoneColumnType {
     }
 
     @Override
-    public Value asValue(FieldValue value) {
+    public StringValue asValue(FieldValue value) {
       return ValueFactory.newString(((DateTimeFieldValue) value).getValue().toString());
     }
 
@@ -437,7 +450,7 @@ public enum KintoneColumnType {
     }
 
     @Override
-    public Value asValue(FieldValue value) {
+    public StringValue asValue(FieldValue value) {
       return ValueFactory.newString(((LinkFieldValue) value).getValue());
     }
 
@@ -540,6 +553,10 @@ public enum KintoneColumnType {
     }
   }
 
+  public Object getValue(Record Record, String fieldCode) {
+    throw new UnsupportedOperationException();
+  }
+
   public void setUpdateKey(UpdateKey updateKey, String field) {
     throw new UnsupportedOperationException();
   }

data/src/main/java/org/embulk/output/kintone/KintoneColumnVisitor.java

@@ -4,8 +4,12 @@ import com.kintone.client.model.record.FieldValue;
 import com.kintone.client.model.record.Record;
 import com.kintone.client.model.record.UpdateKey;
 import java.time.Instant;
+import java.util.Arrays;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import org.embulk.output.kintone.record.Id;
+import org.embulk.output.kintone.record.IdOrUpdateKey;
 import org.embulk.spi.Column;
 import org.embulk.spi.ColumnVisitor;
 import org.embulk.spi.PageReader;
@@ -14,6 +18,7 @@ import org.msgpack.value.Value;
 import org.msgpack.value.ValueFactory;
 
 public class KintoneColumnVisitor implements ColumnVisitor {
+  private static final List<String> BUILTIN_FIELD_CODES = Arrays.asList(Id.FIELD, "$revision");
   private final PageReader reader;
   private final Set<Column> derived;
   private final Map<String, KintoneColumnOption> options;
@@ -22,7 +27,7 @@ public class KintoneColumnVisitor implements ColumnVisitor {
   private final String reduceKeyName;
   private final String updateKeyName;
   private Record record;
-  private UpdateKey updateKey;
+  private IdOrUpdateKey idOrUpdateKey;
 
   public KintoneColumnVisitor(
       PageReader reader,
@@ -55,8 +60,8 @@ public class KintoneColumnVisitor implements ColumnVisitor {
     this.record = record;
   }
 
-  public void setUpdateKey(UpdateKey updateKey) {
-    this.updateKey = updateKey;
+  public void setIdOrUpdateKey(IdOrUpdateKey idOrUpdateKey) {
+    this.idOrUpdateKey = idOrUpdateKey;
   }
 
   @Override
@@ -68,6 +73,9 @@ public class KintoneColumnVisitor implements ColumnVisitor {
     UpdateKey updateKey = getUpdateKey(column);
     KintoneColumnType type = KintoneColumnType.getType(option, KintoneColumnType.NUMBER);
     String fieldCode = getFieldCode(column);
+    if (isBuiltin(fieldCode)) {
+      return;
+    }
     if (isPreferNull(column)) {
       setNull(type, fieldCode, updateKey);
     } else if (reader.isNull(column)) {
@@ -82,10 +90,18 @@ public class KintoneColumnVisitor implements ColumnVisitor {
     if (isReduced(column) || isIgnoreNull(column)) {
       return;
     }
+    Id id = getId(column);
+    if (id != null) {
+      setIdValue(id, column);
+      return;
+    }
     KintoneColumnOption option = getOption(column);
     UpdateKey updateKey = getUpdateKey(column);
     KintoneColumnType type = KintoneColumnType.getType(option, KintoneColumnType.NUMBER);
     String fieldCode = getFieldCode(column);
+    if (isBuiltin(fieldCode)) {
+      return;
+    }
     if (isPreferNull(column)) {
       setNull(type, fieldCode, updateKey);
     } else if (reader.isNull(column)) {
@@ -104,6 +120,9 @@ public class KintoneColumnVisitor implements ColumnVisitor {
     UpdateKey updateKey = getUpdateKey(column);
     KintoneColumnType type = KintoneColumnType.getType(option, KintoneColumnType.NUMBER);
     String fieldCode = getFieldCode(column);
+    if (isBuiltin(fieldCode)) {
+      return;
+    }
     if (isPreferNull(column)) {
       setNull(type, fieldCode, updateKey);
     } else if (reader.isNull(column)) {
@@ -124,6 +143,9 @@ public class KintoneColumnVisitor implements ColumnVisitor {
         updateKey != null ? KintoneColumnType.SINGLE_LINE_TEXT : KintoneColumnType.MULTI_LINE_TEXT;
     KintoneColumnType type = KintoneColumnType.getType(option, defaultType);
     String fieldCode = getFieldCode(column);
+    if (isBuiltin(fieldCode)) {
+      return;
+    }
     if (isPreferNull(column)) {
       setNull(type, fieldCode, updateKey);
     } else if (reader.isNull(column)) {
@@ -142,6 +164,9 @@ public class KintoneColumnVisitor implements ColumnVisitor {
     UpdateKey updateKey = getUpdateKey(column);
     KintoneColumnType type = KintoneColumnType.getType(option, KintoneColumnType.DATETIME);
     String fieldCode = getFieldCode(column);
+    if (isBuiltin(fieldCode)) {
+      return;
+    }
     if (isPreferNull(column)) {
       setNull(type, fieldCode, updateKey);
     } else if (reader.isNull(column)) {
@@ -162,6 +187,9 @@ public class KintoneColumnVisitor implements ColumnVisitor {
         isDerived(column) ? KintoneColumnType.SUBTABLE : KintoneColumnType.MULTI_LINE_TEXT;
     KintoneColumnType type = KintoneColumnType.getType(option, defaultType);
     String fieldCode = getFieldCode(column);
+    if (isBuiltin(fieldCode)) {
+      return;
+    }
     if (isPreferNull(column)) {
       setNull(type, fieldCode, updateKey);
     } else if (reader.isNull(column)) {
@@ -171,6 +199,16 @@ public class KintoneColumnVisitor implements ColumnVisitor {
     }
   }
 
+  private void setIdValue(Id id, Column column) {
+    if (isPreferNull(column)) {
+      id.setValue(null);
+    } else if (reader.isNull(column)) {
+      id.setValue(0L);
+    } else {
+      id.setValue(reader.getLong(column));
+    }
+  }
+
   private void setNull(KintoneColumnType type, String fieldCode, UpdateKey updateKey) {
     if (updateKey != null) {
       type.setUpdateKey(updateKey, fieldCode);
@@ -265,8 +303,16 @@ public class KintoneColumnVisitor implements ColumnVisitor {
     return options.get(column.getName());
   }
 
+  private Id getId(Column column) {
+    return column.getName().equals(updateKeyName) && updateKeyName.equals(Id.FIELD)
+        ? idOrUpdateKey.getId()
+        : null;
+  }
+
   private UpdateKey getUpdateKey(Column column) {
-    return updateKeyName != null && updateKeyName.equals(column.getName()) ? updateKey : null;
+    return column.getName().equals(updateKeyName) && !updateKeyName.equals(Id.FIELD)
+        ? idOrUpdateKey.getUpdateKey()
+        : null;
   }
 
   private boolean isReduced(Column column) {
@@ -284,4 +330,8 @@ public class KintoneColumnVisitor implements ColumnVisitor {
   private boolean isPreferNull(Column column) {
     return preferNulls && reader.isNull(column);
   }
+
+  private static boolean isBuiltin(String fieldCode) {
+    return BUILTIN_FIELD_CODES.contains(fieldCode);
+  }
 }

data/src/main/java/org/embulk/output/kintone/KintoneMode.java

@@ -1,23 +1,71 @@
 package org.embulk.output.kintone;
 
+import java.util.function.Consumer;
 import org.embulk.config.ConfigException;
+import org.embulk.output.kintone.record.Id;
+import org.embulk.output.kintone.record.Skip;
+import org.embulk.spi.Page;
 
 public enum KintoneMode {
-  INSERT("insert"),
-  UPDATE("update"),
-  UPSERT("upsert");
+  INSERT("insert") {
+    @Override
+    public void validate(PluginTask task, KintoneClient client) {
+      if (task.getUpdateKeyName().isPresent()) {
+        throw new ConfigException("When mode is insert, require no update_key.");
+      }
+    }
+
+    @Override
+    public void add(Page page, Skip skip, KintonePageOutput output) {
+      output.insertPage(page);
+    }
+  },
+  UPDATE("update") {
+    @Override
+    public void validate(PluginTask task, KintoneClient client) {
+      if (!task.getUpdateKeyName().isPresent() && client.getColumn(Id.FIELD) == null) {
+        throw new ConfigException("When mode is update, require update_key or id column.");
+      }
+      client.validateIdOrUpdateKey(task.getUpdateKeyName().orElse(Id.FIELD));
+    }
+
+    @Override
+    public void add(Page page, Skip skip, KintonePageOutput output) {
+      Consumer<Page> consumer = skip == Skip.ALWAYS ? output::upsertPage : output::updatePage;
+      consumer.accept(page);
+    }
+  },
+  UPSERT("upsert") {
+    @Override
+    public void validate(PluginTask task, KintoneClient client) {
+      if (!task.getUpdateKeyName().isPresent() && client.getColumn(Id.FIELD) == null) {
+        throw new ConfigException("When mode is upsert, require update_key or id column.");
+      }
+      client.validateIdOrUpdateKey(task.getUpdateKeyName().orElse(Id.FIELD));
+    }
+
+    @Override
+    public void add(Page page, Skip skip, KintonePageOutput output) {
+      output.upsertPage(page);
+    }
+  };
   private final String value;
 
   KintoneMode(String value) {
     this.value = value;
   }
 
+  public abstract void validate(PluginTask task, KintoneClient client);
+
+  public abstract void add(Page page, Skip skip, KintonePageOutput output);
+
   @Override
   public String toString() {
     return value;
   }
 
-  public static KintoneMode getKintoneModeByValue(String value) {
+  public static KintoneMode of(PluginTask task) {
+    String value = task.getMode();
     for (KintoneMode mode : values()) {
       if (mode.toString().equals(value)) {
         return mode;

data/src/main/java/org/embulk/output/kintone/KintoneOutputPlugin.java

@@ -3,7 +3,6 @@ package org.embulk.output.kintone;
 import java.util.Collections;
 import java.util.List;
 import org.embulk.config.ConfigDiff;
-import org.embulk.config.ConfigException;
 import org.embulk.config.ConfigSource;
 import org.embulk.config.TaskReport;
 import org.embulk.config.TaskSource;
@@ -40,22 +39,6 @@ public class KintoneOutputPlugin implements OutputPlugin {
   @Override
   public TransactionalPageOutput open(TaskSource taskSource, Schema schema, int taskIndex) {
     PluginTask task = taskSource.loadTask(PluginTask.class);
-    KintoneMode mode = KintoneMode.getKintoneModeByValue(task.getMode());
-    switch (mode) {
-      case INSERT:
-        if (task.getUpdateKeyName().isPresent()) {
-          throw new ConfigException("when mode is insert, require no update_key.");
-        }
-        break;
-      case UPDATE:
-      case UPSERT:
-        if (!task.getUpdateKeyName().isPresent()) {
-          throw new ConfigException("when mode is update or upsert, require update_key.");
-        }
-        break;
-      default:
-        throw new ConfigException(String.format("Unknown mode '%s'", task.getMode()));
-    }
     return task.getReduceKeyName().isPresent()
         ? new ReducedPageOutput(schema, taskIndex)
         : new KintonePageOutput(task, schema);