embulk-output-kintone 0.4.1 → 1.1.0

data/src/main/java/org/embulk/output/kintone/KintoneColumnVisitor.java

@@ -1,47 +1,53 @@
 package org.embulk.output.kintone;
 
-import com.kintone.client.model.record.CheckBoxFieldValue;
-import com.kintone.client.model.record.DateFieldValue;
-import com.kintone.client.model.record.DateTimeFieldValue;
-import com.kintone.client.model.record.DropDownFieldValue;
-import com.kintone.client.model.record.FieldType;
 import com.kintone.client.model.record.FieldValue;
-import com.kintone.client.model.record.LinkFieldValue;
-import com.kintone.client.model.record.MultiLineTextFieldValue;
-import com.kintone.client.model.record.NumberFieldValue;
 import com.kintone.client.model.record.Record;
-import com.kintone.client.model.record.SingleLineTextFieldValue;
 import com.kintone.client.model.record.UpdateKey;
-import java.math.BigDecimal;
 import java.time.Instant;
-import java.time.ZoneId;
-import java.time.ZonedDateTime;
-import java.util.Arrays;
-import java.util.List;
 import java.util.Map;
-import java.util.Objects;
+import java.util.Set;
 import org.embulk.spi.Column;
 import org.embulk.spi.ColumnVisitor;
 import org.embulk.spi.PageReader;
 import org.embulk.spi.time.Timestamp;
+import org.msgpack.value.Value;
+import org.msgpack.value.ValueFactory;
 
 public class KintoneColumnVisitor implements ColumnVisitor {
-  private final PageReader pageReader;
+  private final PageReader reader;
+  private final Set<Column> derived;
+  private final Map<String, KintoneColumnOption> options;
+  private final boolean preferNulls;
+  private final boolean ignoreNulls;
+  private final String reduceKeyName;
+  private final String updateKeyName;
   private Record record;
   private UpdateKey updateKey;
-  private final Map<String, KintoneColumnOption> columnOptions;
-  private String updateKeyName;
 
   public KintoneColumnVisitor(
-      PageReader pageReader, Map<String, KintoneColumnOption> columnOptions) {
-    this.pageReader = pageReader;
-    this.columnOptions = columnOptions;
+      PageReader reader,
+      Set<Column> derived,
+      Map<String, KintoneColumnOption> options,
+      boolean preferNulls,
+      boolean ignoreNulls,
+      String reduceKeyName) {
+    this(reader, derived, options, preferNulls, ignoreNulls, reduceKeyName, null);
   }
 
   public KintoneColumnVisitor(
-      PageReader pageReader, Map<String, KintoneColumnOption> columnOptions, String updateKeyName) {
-    this.pageReader = pageReader;
-    this.columnOptions = columnOptions;
+      PageReader reader,
+      Set<Column> derived,
+      Map<String, KintoneColumnOption> options,
+      boolean preferNulls,
+      boolean ignoreNulls,
+      String reduceKeyName,
+      String updateKeyName) {
+    this.reader = reader;
+    this.derived = derived;
+    this.options = options;
+    this.preferNulls = preferNulls;
+    this.ignoreNulls = ignoreNulls;
+    this.reduceKeyName = reduceKeyName;
     this.updateKeyName = updateKeyName;
   }
 
@@ -53,156 +59,229 @@ public class KintoneColumnVisitor implements ColumnVisitor {
     this.updateKey = updateKey;
   }
 
-  private void setValue(String fieldCode, Object value, FieldType type, boolean isUpdateKey) {
-    if (isUpdateKey && updateKey != null) {
-      updateKey.setField(fieldCode).setValue(Objects.toString(value, ""));
-    }
-    String stringValue = Objects.toString(value, "");
-    FieldValue fieldValue;
-    switch (type) {
-      case NUMBER:
-        BigDecimal setValue = stringValue.equals("") ? null : new BigDecimal(stringValue);
-        fieldValue = new NumberFieldValue(setValue);
-        break;
-      case MULTI_LINE_TEXT:
-        fieldValue = new MultiLineTextFieldValue(stringValue);
-        break;
-      case DROP_DOWN:
-        fieldValue = new DropDownFieldValue(stringValue);
-        break;
-      case LINK:
-        fieldValue = new LinkFieldValue(stringValue);
-        break;
-      default:
-        fieldValue = new SingleLineTextFieldValue(stringValue);
+  @Override
+  public void booleanColumn(Column column) {
+    if (isReduced(column) || isIgnoreNull(column)) {
+      return;
+    }
+    KintoneColumnOption option = getOption(column);
+    UpdateKey updateKey = getUpdateKey(column);
+    KintoneColumnType type = KintoneColumnType.getType(option, KintoneColumnType.NUMBER);
+    String fieldCode = getFieldCode(column);
+    if (isPreferNull(column)) {
+      setNull(type, fieldCode, updateKey);
+    } else if (reader.isNull(column)) {
+      setBoolean(type, fieldCode, updateKey, false, option);
+    } else {
+      setBoolean(type, fieldCode, updateKey, reader.getBoolean(column), option);
     }
-    record.putField(fieldCode, fieldValue);
   }
 
-  private void setTimestampValue(String fieldCode, Instant instant, ZoneId zoneId, FieldType type) {
-    FieldValue fieldValue = null;
-    ZonedDateTime datetime = instant.atZone(zoneId);
-    switch (type) {
-      case DATE:
-        fieldValue = new DateFieldValue(datetime.toLocalDate());
-        break;
-      case DATETIME:
-        fieldValue = new DateTimeFieldValue(datetime);
+  @Override
+  public void longColumn(Column column) {
+    if (isReduced(column) || isIgnoreNull(column)) {
+      return;
+    }
+    KintoneColumnOption option = getOption(column);
+    UpdateKey updateKey = getUpdateKey(column);
+    KintoneColumnType type = KintoneColumnType.getType(option, KintoneColumnType.NUMBER);
+    String fieldCode = getFieldCode(column);
+    if (isPreferNull(column)) {
+      setNull(type, fieldCode, updateKey);
+    } else if (reader.isNull(column)) {
+      setLong(type, fieldCode, updateKey, 0, option);
+    } else {
+      setLong(type, fieldCode, updateKey, reader.getLong(column), option);
     }
-    record.putField(fieldCode, fieldValue);
   }
 
-  private void setCheckBoxValue(String fieldCode, Object value, String valueSeparator) {
-    String str = String.valueOf(value);
-    CheckBoxFieldValue checkBoxFieldValue = new CheckBoxFieldValue();
-
-    if (str != null && !str.equals("")) {
-      List<String> values = Arrays.asList(str.split(valueSeparator, 0));
-      checkBoxFieldValue = new CheckBoxFieldValue(values);
+  @Override
+  public void doubleColumn(Column column) {
+    if (isReduced(column) || isIgnoreNull(column)) {
+      return;
+    }
+    KintoneColumnOption option = getOption(column);
+    UpdateKey updateKey = getUpdateKey(column);
+    KintoneColumnType type = KintoneColumnType.getType(option, KintoneColumnType.NUMBER);
+    String fieldCode = getFieldCode(column);
+    if (isPreferNull(column)) {
+      setNull(type, fieldCode, updateKey);
+    } else if (reader.isNull(column)) {
+      setDouble(type, fieldCode, updateKey, 0, option);
+    } else {
+      setDouble(type, fieldCode, updateKey, reader.getDouble(column), option);
     }
-    record.putField(fieldCode, checkBoxFieldValue);
   }
 
-  private FieldType getType(Column column, FieldType defaultType) {
-    KintoneColumnOption option = columnOptions.get(column.getName());
-    if (option == null) {
-      return defaultType;
+  @Override
+  public void stringColumn(Column column) {
+    if (isReduced(column) || isIgnoreNull(column)) {
+      return;
+    }
+    KintoneColumnOption option = getOption(column);
+    UpdateKey updateKey = getUpdateKey(column);
+    KintoneColumnType defaultType =
+        updateKey != null ? KintoneColumnType.SINGLE_LINE_TEXT : KintoneColumnType.MULTI_LINE_TEXT;
+    KintoneColumnType type = KintoneColumnType.getType(option, defaultType);
+    String fieldCode = getFieldCode(column);
+    if (isPreferNull(column)) {
+      setNull(type, fieldCode, updateKey);
+    } else if (reader.isNull(column)) {
+      setString(type, fieldCode, updateKey, "", option);
     } else {
-      return FieldType.valueOf(option.getType());
+      setString(type, fieldCode, updateKey, reader.getString(column), option);
     }
   }
 
-  private String getFieldCode(Column column) {
-    KintoneColumnOption option = columnOptions.get(column.getName());
-    if (option == null) {
-      return column.getName();
+  @Override
+  public void timestampColumn(Column column) {
+    if (isReduced(column) || isIgnoreNull(column)) {
+      return;
+    }
+    KintoneColumnOption option = getOption(column);
+    UpdateKey updateKey = getUpdateKey(column);
+    KintoneColumnType type = KintoneColumnType.getType(option, KintoneColumnType.DATETIME);
+    String fieldCode = getFieldCode(column);
+    if (isPreferNull(column)) {
+      setNull(type, fieldCode, updateKey);
+    } else if (reader.isNull(column)) {
+      setTimestamp(type, fieldCode, updateKey, Timestamp.ofInstant(Instant.EPOCH), option);
     } else {
-      return option.getFieldCode();
+      setTimestamp(type, fieldCode, updateKey, reader.getTimestamp(column), option);
     }
   }
 
-  private ZoneId getZoneId(Column column) {
-    KintoneColumnOption option = columnOptions.get(column.getName());
-    if (option == null) {
-      return ZoneId.of("UTC");
+  @Override
+  public void jsonColumn(Column column) {
+    if (isReduced(column) || isIgnoreNull(column)) {
+      return;
+    }
+    KintoneColumnOption option = getOption(column);
+    UpdateKey updateKey = getUpdateKey(column);
+    KintoneColumnType defaultType =
+        isDerived(column) ? KintoneColumnType.SUBTABLE : KintoneColumnType.MULTI_LINE_TEXT;
+    KintoneColumnType type = KintoneColumnType.getType(option, defaultType);
+    String fieldCode = getFieldCode(column);
+    if (isPreferNull(column)) {
+      setNull(type, fieldCode, updateKey);
+    } else if (reader.isNull(column)) {
+      setJson(type, fieldCode, updateKey, ValueFactory.newString(""), option);
+    } else {
+      setJson(type, fieldCode, updateKey, reader.getJson(column), option);
     }
-    return ZoneId.of(option.getTimezone().orElse("UTC"));
   }
 
-  private boolean isUpdateKey(Column column) {
-    if (this.updateKeyName == null) {
-      return false;
+  private void setNull(KintoneColumnType type, String fieldCode, UpdateKey updateKey) {
+    if (updateKey != null) {
+      type.setUpdateKey(updateKey, fieldCode);
     }
-
-    return this.updateKeyName.equals(column.getName());
+    record.putField(fieldCode, type.getFieldValue());
   }
 
-  private String getValueSeparator(Column column) {
-    KintoneColumnOption option = columnOptions.get(column.getName());
-    if (option == null) {
-      return ",";
+  private void setBoolean(
+      KintoneColumnType type,
+      String fieldCode,
+      UpdateKey updateKey,
+      boolean value,
+      KintoneColumnOption option) {
+    FieldValue fieldValue = type.getFieldValue(value, option);
+    if (updateKey != null) {
+      type.setUpdateKey(updateKey, fieldCode, fieldValue);
     }
-    return option.getValueSeparator();
+    record.putField(fieldCode, fieldValue);
   }
 
-  @Override
-  public void booleanColumn(Column column) {
-    String fieldCode = getFieldCode(column);
-    FieldType type = getType(column, FieldType.NUMBER);
-    setValue(fieldCode, pageReader.getBoolean(column), type, isUpdateKey(column));
+  private void setLong(
+      KintoneColumnType type,
+      String fieldCode,
+      UpdateKey updateKey,
+      long value,
+      KintoneColumnOption option) {
+    FieldValue fieldValue = type.getFieldValue(value, option);
+    if (updateKey != null) {
+      type.setUpdateKey(updateKey, fieldCode, fieldValue);
+    }
+    record.putField(fieldCode, fieldValue);
   }
 
-  @Override
-  public void longColumn(Column column) {
-    String fieldCode = getFieldCode(column);
-    FieldType type = getType(column, FieldType.NUMBER);
-    if (pageReader.isNull(column)) {
-      setValue(fieldCode, null, type, isUpdateKey(column));
-    } else {
-      setValue(fieldCode, pageReader.getLong(column), type, isUpdateKey(column));
+  private void setDouble(
+      KintoneColumnType type,
+      String fieldCode,
+      UpdateKey updateKey,
+      double value,
+      KintoneColumnOption option) {
+    FieldValue fieldValue = type.getFieldValue(value, option);
+    if (updateKey != null) {
+      type.setUpdateKey(updateKey, fieldCode, fieldValue);
     }
+    record.putField(fieldCode, fieldValue);
   }
 
-  @Override
-  public void doubleColumn(Column column) {
-    String fieldCode = getFieldCode(column);
-    FieldType type = getType(column, FieldType.NUMBER);
-    setValue(fieldCode, pageReader.getDouble(column), type, isUpdateKey(column));
+  private void setString(
+      KintoneColumnType type,
+      String fieldCode,
+      UpdateKey updateKey,
+      String value,
+      KintoneColumnOption option) {
+    FieldValue fieldValue = type.getFieldValue(value, option);
+    if (updateKey != null) {
+      type.setUpdateKey(updateKey, fieldCode, fieldValue);
+    }
+    record.putField(fieldCode, fieldValue);
   }
 
-  @Override
-  public void stringColumn(Column column) {
-    String fieldCode = getFieldCode(column);
-    FieldType type = getType(column, FieldType.MULTI_LINE_TEXT);
-    Object value = pageReader.getString(column);
-    if (type == FieldType.CHECK_BOX) {
-      String stringValue = Objects.toString(value, "");
-      setCheckBoxValue(fieldCode, value, getValueSeparator(column));
-      return;
+  private void setTimestamp(
+      KintoneColumnType type,
+      String fieldCode,
+      UpdateKey updateKey,
+      Timestamp value,
+      KintoneColumnOption option) {
+    FieldValue fieldValue = type.getFieldValue(value, option);
+    if (updateKey != null) {
+      type.setUpdateKey(updateKey, fieldCode, fieldValue);
     }
-    setValue(fieldCode, value, type, isUpdateKey(column));
+    record.putField(fieldCode, fieldValue);
   }
 
-  @Override
-  public void timestampColumn(Column column) {
-    Timestamp value = pageReader.getTimestamp(column);
-    if (value == null) {
-      return;
+  private void setJson(
+      KintoneColumnType type,
+      String fieldCode,
+      UpdateKey updateKey,
+      Value value,
+      KintoneColumnOption option) {
+    FieldValue fieldValue = type.getFieldValue(value, option);
+    if (updateKey != null) {
+      type.setUpdateKey(updateKey, fieldCode, fieldValue);
     }
+    record.putField(fieldCode, fieldValue);
+  }
 
-    String fieldCode = getFieldCode(column);
-    FieldType type = getType(column, FieldType.DATETIME);
-    ZoneId zoneId = getZoneId(column);
-    if (type == FieldType.DATETIME) {
-      zoneId = ZoneId.of("UTC");
-    }
-    setTimestampValue(fieldCode, value.getInstant(), zoneId, type);
+  private String getFieldCode(Column column) {
+    KintoneColumnOption option = getOption(column);
+    return option != null ? option.getFieldCode() : column.getName();
   }
 
-  @Override
-  public void jsonColumn(Column column) {
-    String fieldCode = getFieldCode(column);
-    FieldType type = getType(column, FieldType.MULTI_LINE_TEXT);
-    setValue(fieldCode, pageReader.getJson(column), type, isUpdateKey(column));
+  private KintoneColumnOption getOption(Column column) {
+    return options.get(column.getName());
+  }
+
+  private UpdateKey getUpdateKey(Column column) {
+    return updateKeyName != null && updateKeyName.equals(column.getName()) ? updateKey : null;
+  }
+
+  private boolean isReduced(Column column) {
+    return reduceKeyName != null && column.getName().matches("^.*\\..*$");
+  }
+
+  private boolean isDerived(Column column) {
+    return reduceKeyName != null && derived.contains(column);
+  }
+
+  private boolean isIgnoreNull(Column column) {
+    return ignoreNulls && reader.isNull(column);
+  }
+
+  private boolean isPreferNull(Column column) {
+    return preferNulls && reader.isNull(column);
   }
 }

data/src/main/java/org/embulk/output/kintone/KintoneMode.java

@@ -6,7 +6,6 @@ public enum KintoneMode {
   INSERT("insert"),
   UPDATE("update"),
   UPSERT("upsert");
-
   private final String value;
 
   KintoneMode(String value) {

data/src/main/java/org/embulk/output/kintone/KintoneOutputPlugin.java

@@ -1,11 +1,14 @@
 package org.embulk.output.kintone;
 
+import java.util.Collections;
 import java.util.List;
 import org.embulk.config.ConfigDiff;
 import org.embulk.config.ConfigException;
 import org.embulk.config.ConfigSource;
 import org.embulk.config.TaskReport;
 import org.embulk.config.TaskSource;
+import org.embulk.output.kintone.reducer.ReducedPageOutput;
+import org.embulk.output.kintone.reducer.Reducer;
 import org.embulk.spi.Exec;
 import org.embulk.spi.OutputPlugin;
 import org.embulk.spi.Schema;
@@ -16,9 +19,12 @@ public class KintoneOutputPlugin implements OutputPlugin {
   public ConfigDiff transaction(
       ConfigSource config, Schema schema, int taskCount, OutputPlugin.Control control) {
     PluginTask task = config.loadConfig(PluginTask.class);
-
-    control.run(task.dump());
-    return Exec.newConfigDiff();
+    task.setDerivedColumns(Collections.emptySet());
+    List<TaskReport> taskReports = control.run(task.dump());
+    return task.getReduceKeyName().isPresent()
+        ? new Reducer(task, schema)
+            .reduce(taskReports, schema.lookupColumn(task.getReduceKeyName().get()))
+        : Exec.newConfigDiff();
   }
 
   @Override
@@ -34,7 +40,6 @@ public class KintoneOutputPlugin implements OutputPlugin {
   @Override
   public TransactionalPageOutput open(TaskSource taskSource, Schema schema, int taskIndex) {
     PluginTask task = taskSource.loadTask(PluginTask.class);
-
     KintoneMode mode = KintoneMode.getKintoneModeByValue(task.getMode());
     switch (mode) {
       case INSERT:
@@ -51,6 +56,8 @@ public class KintoneOutputPlugin implements OutputPlugin {
       default:
         throw new ConfigException(String.format("Unknown mode '%s'", task.getMode()));
     }
-    return new KintonePageOutput(task, schema);
+    return task.getReduceKeyName().isPresent()
+        ? new ReducedPageOutput(schema, taskIndex)
+        : new KintonePageOutput(task, schema);
   }
 }