embulk-output-kintone 1.0.0 → 1.2.0

data/src/main/java/org/embulk/output/kintone/KintoneColumnVisitor.java

@@ -4,7 +4,12 @@ import com.kintone.client.model.record.FieldValue;
 import com.kintone.client.model.record.Record;
 import com.kintone.client.model.record.UpdateKey;
 import java.time.Instant;
+import java.util.Arrays;
+import java.util.List;
 import java.util.Map;
+import java.util.Set;
+import org.embulk.output.kintone.record.Id;
+import org.embulk.output.kintone.record.IdOrUpdateKey;
 import org.embulk.spi.Column;
 import org.embulk.spi.ColumnVisitor;
 import org.embulk.spi.PageReader;
@@ -13,32 +18,41 @@ import org.msgpack.value.Value;
 import org.msgpack.value.ValueFactory;
 
 public class KintoneColumnVisitor implements ColumnVisitor {
+  private static final List<String> BUILTIN_FIELD_CODES = Arrays.asList(Id.FIELD, "$revision");
   private final PageReader reader;
+  private final Set<Column> derived;
   private final Map<String, KintoneColumnOption> options;
   private final boolean preferNulls;
   private final boolean ignoreNulls;
+  private final String reduceKeyName;
   private final String updateKeyName;
   private Record record;
-  private UpdateKey updateKey;
+  private IdOrUpdateKey idOrUpdateKey;
 
   public KintoneColumnVisitor(
       PageReader reader,
+      Set<Column> derived,
       Map<String, KintoneColumnOption> options,
       boolean preferNulls,
-      boolean ignoreNulls) {
-    this(reader, options, preferNulls, ignoreNulls, null);
+      boolean ignoreNulls,
+      String reduceKeyName) {
+    this(reader, derived, options, preferNulls, ignoreNulls, reduceKeyName, null);
   }
 
   public KintoneColumnVisitor(
       PageReader reader,
+      Set<Column> derived,
       Map<String, KintoneColumnOption> options,
       boolean preferNulls,
       boolean ignoreNulls,
+      String reduceKeyName,
       String updateKeyName) {
     this.reader = reader;
+    this.derived = derived;
     this.options = options;
     this.preferNulls = preferNulls;
     this.ignoreNulls = ignoreNulls;
+    this.reduceKeyName = reduceKeyName;
     this.updateKeyName = updateKeyName;
   }
 
@@ -46,19 +60,22 @@ public class KintoneColumnVisitor implements ColumnVisitor {
     this.record = record;
   }
 
-  public void setUpdateKey(UpdateKey updateKey) {
-    this.updateKey = updateKey;
+  public void setIdOrUpdateKey(IdOrUpdateKey idOrUpdateKey) {
+    this.idOrUpdateKey = idOrUpdateKey;
   }
 
   @Override
   public void booleanColumn(Column column) {
-    if (isIgnoreNull(column)) {
+    if (isReduced(column) || isIgnoreNull(column)) {
       return;
     }
     KintoneColumnOption option = getOption(column);
     UpdateKey updateKey = getUpdateKey(column);
     KintoneColumnType type = KintoneColumnType.getType(option, KintoneColumnType.NUMBER);
     String fieldCode = getFieldCode(column);
+    if (isBuiltin(fieldCode)) {
+      return;
+    }
     if (isPreferNull(column)) {
       setNull(type, fieldCode, updateKey);
     } else if (reader.isNull(column)) {
@@ -70,13 +87,21 @@ public class KintoneColumnVisitor implements ColumnVisitor {
 
   @Override
   public void longColumn(Column column) {
-    if (isIgnoreNull(column)) {
+    if (isReduced(column) || isIgnoreNull(column)) {
+      return;
+    }
+    Id id = getId(column);
+    if (id != null) {
+      setIdValue(id, column);
       return;
     }
     KintoneColumnOption option = getOption(column);
     UpdateKey updateKey = getUpdateKey(column);
     KintoneColumnType type = KintoneColumnType.getType(option, KintoneColumnType.NUMBER);
     String fieldCode = getFieldCode(column);
+    if (isBuiltin(fieldCode)) {
+      return;
+    }
     if (isPreferNull(column)) {
       setNull(type, fieldCode, updateKey);
     } else if (reader.isNull(column)) {
@@ -88,13 +113,16 @@ public class KintoneColumnVisitor implements ColumnVisitor {
 
   @Override
   public void doubleColumn(Column column) {
-    if (isIgnoreNull(column)) {
+    if (isReduced(column) || isIgnoreNull(column)) {
       return;
     }
     KintoneColumnOption option = getOption(column);
     UpdateKey updateKey = getUpdateKey(column);
     KintoneColumnType type = KintoneColumnType.getType(option, KintoneColumnType.NUMBER);
     String fieldCode = getFieldCode(column);
+    if (isBuiltin(fieldCode)) {
+      return;
+    }
     if (isPreferNull(column)) {
       setNull(type, fieldCode, updateKey);
     } else if (reader.isNull(column)) {
@@ -106,7 +134,7 @@ public class KintoneColumnVisitor implements ColumnVisitor {
 
   @Override
   public void stringColumn(Column column) {
-    if (isIgnoreNull(column)) {
+    if (isReduced(column) || isIgnoreNull(column)) {
       return;
     }
     KintoneColumnOption option = getOption(column);
@@ -115,6 +143,9 @@ public class KintoneColumnVisitor implements ColumnVisitor {
         updateKey != null ? KintoneColumnType.SINGLE_LINE_TEXT : KintoneColumnType.MULTI_LINE_TEXT;
     KintoneColumnType type = KintoneColumnType.getType(option, defaultType);
     String fieldCode = getFieldCode(column);
+    if (isBuiltin(fieldCode)) {
+      return;
+    }
     if (isPreferNull(column)) {
       setNull(type, fieldCode, updateKey);
     } else if (reader.isNull(column)) {
@@ -126,13 +157,16 @@ public class KintoneColumnVisitor implements ColumnVisitor {
 
   @Override
   public void timestampColumn(Column column) {
-    if (isIgnoreNull(column)) {
+    if (isReduced(column) || isIgnoreNull(column)) {
       return;
     }
     KintoneColumnOption option = getOption(column);
     UpdateKey updateKey = getUpdateKey(column);
     KintoneColumnType type = KintoneColumnType.getType(option, KintoneColumnType.DATETIME);
     String fieldCode = getFieldCode(column);
+    if (isBuiltin(fieldCode)) {
+      return;
+    }
     if (isPreferNull(column)) {
       setNull(type, fieldCode, updateKey);
     } else if (reader.isNull(column)) {
@@ -144,13 +178,18 @@ public class KintoneColumnVisitor implements ColumnVisitor {
 
   @Override
   public void jsonColumn(Column column) {
-    if (isIgnoreNull(column)) {
+    if (isReduced(column) || isIgnoreNull(column)) {
       return;
     }
     KintoneColumnOption option = getOption(column);
     UpdateKey updateKey = getUpdateKey(column);
-    KintoneColumnType type = KintoneColumnType.getType(option, KintoneColumnType.MULTI_LINE_TEXT);
+    KintoneColumnType defaultType =
+        isDerived(column) ? KintoneColumnType.SUBTABLE : KintoneColumnType.MULTI_LINE_TEXT;
+    KintoneColumnType type = KintoneColumnType.getType(option, defaultType);
     String fieldCode = getFieldCode(column);
+    if (isBuiltin(fieldCode)) {
+      return;
+    }
     if (isPreferNull(column)) {
       setNull(type, fieldCode, updateKey);
     } else if (reader.isNull(column)) {
@@ -160,6 +199,16 @@ public class KintoneColumnVisitor implements ColumnVisitor {
     }
   }
 
+  private void setIdValue(Id id, Column column) {
+    if (isPreferNull(column)) {
+      id.setValue(null);
+    } else if (reader.isNull(column)) {
+      id.setValue(0L);
+    } else {
+      id.setValue(reader.getLong(column));
+    }
+  }
+
   private void setNull(KintoneColumnType type, String fieldCode, UpdateKey updateKey) {
     if (updateKey != null) {
       type.setUpdateKey(updateKey, fieldCode);
@@ -254,8 +303,24 @@ public class KintoneColumnVisitor implements ColumnVisitor {
     return options.get(column.getName());
   }
 
+  private Id getId(Column column) {
+    return column.getName().equals(updateKeyName) && updateKeyName.equals(Id.FIELD)
+        ? idOrUpdateKey.getId()
+        : null;
+  }
+
   private UpdateKey getUpdateKey(Column column) {
-    return updateKeyName != null && updateKeyName.equals(column.getName()) ? updateKey : null;
+    return column.getName().equals(updateKeyName) && !updateKeyName.equals(Id.FIELD)
+        ? idOrUpdateKey.getUpdateKey()
+        : null;
+  }
+
+  private boolean isReduced(Column column) {
+    return reduceKeyName != null && column.getName().matches("^.*\\..*$");
+  }
+
+  private boolean isDerived(Column column) {
+    return reduceKeyName != null && derived.contains(column);
   }
 
   private boolean isIgnoreNull(Column column) {
@@ -265,4 +330,8 @@ public class KintoneColumnVisitor implements ColumnVisitor {
   private boolean isPreferNull(Column column) {
     return preferNulls && reader.isNull(column);
   }
+
+  private static boolean isBuiltin(String fieldCode) {
+    return BUILTIN_FIELD_CODES.contains(fieldCode);
+  }
 }

data/src/main/java/org/embulk/output/kintone/KintoneMode.java

@@ -1,23 +1,71 @@
 package org.embulk.output.kintone;
 
+import java.util.function.Consumer;
 import org.embulk.config.ConfigException;
+import org.embulk.output.kintone.record.Id;
+import org.embulk.output.kintone.record.Skip;
+import org.embulk.spi.Page;
 
 public enum KintoneMode {
-  INSERT("insert"),
-  UPDATE("update"),
-  UPSERT("upsert");
+  INSERT("insert") {
+    @Override
+    public void validate(PluginTask task, KintoneClient client) {
+      if (task.getUpdateKeyName().isPresent()) {
+        throw new ConfigException("When mode is insert, require no update_key.");
+      }
+    }
+
+    @Override
+    public void add(Page page, Skip skip, KintonePageOutput output) {
+      output.insertPage(page);
+    }
+  },
+  UPDATE("update") {
+    @Override
+    public void validate(PluginTask task, KintoneClient client) {
+      if (!task.getUpdateKeyName().isPresent() && client.getColumn(Id.FIELD) == null) {
+        throw new ConfigException("When mode is update, require update_key or id column.");
+      }
+      client.validateIdOrUpdateKey(task.getUpdateKeyName().orElse(Id.FIELD));
+    }
+
+    @Override
+    public void add(Page page, Skip skip, KintonePageOutput output) {
+      Consumer<Page> consumer = skip == Skip.ALWAYS ? output::upsertPage : output::updatePage;
+      consumer.accept(page);
+    }
+  },
+  UPSERT("upsert") {
+    @Override
+    public void validate(PluginTask task, KintoneClient client) {
+      if (!task.getUpdateKeyName().isPresent() && client.getColumn(Id.FIELD) == null) {
+        throw new ConfigException("When mode is upsert, require update_key or id column.");
+      }
+      client.validateIdOrUpdateKey(task.getUpdateKeyName().orElse(Id.FIELD));
+    }
+
+    @Override
+    public void add(Page page, Skip skip, KintonePageOutput output) {
+      output.upsertPage(page);
+    }
+  };
   private final String value;
 
   KintoneMode(String value) {
     this.value = value;
   }
 
+  public abstract void validate(PluginTask task, KintoneClient client);
+
+  public abstract void add(Page page, Skip skip, KintonePageOutput output);
+
   @Override
   public String toString() {
     return value;
   }
 
-  public static KintoneMode getKintoneModeByValue(String value) {
+  public static KintoneMode of(PluginTask task) {
+    String value = task.getMode();
     for (KintoneMode mode : values()) {
       if (mode.toString().equals(value)) {
         return mode;

data/src/main/java/org/embulk/output/kintone/KintoneOutputPlugin.java

@@ -1,11 +1,13 @@
 package org.embulk.output.kintone;
 
+import java.util.Collections;
 import java.util.List;
 import org.embulk.config.ConfigDiff;
-import org.embulk.config.ConfigException;
 import org.embulk.config.ConfigSource;
 import org.embulk.config.TaskReport;
 import org.embulk.config.TaskSource;
+import org.embulk.output.kintone.reducer.ReducedPageOutput;
+import org.embulk.output.kintone.reducer.Reducer;
 import org.embulk.spi.Exec;
 import org.embulk.spi.OutputPlugin;
 import org.embulk.spi.Schema;
@@ -16,8 +18,12 @@ public class KintoneOutputPlugin implements OutputPlugin {
   public ConfigDiff transaction(
       ConfigSource config, Schema schema, int taskCount, OutputPlugin.Control control) {
     PluginTask task = config.loadConfig(PluginTask.class);
-    control.run(task.dump());
-    return Exec.newConfigDiff();
+    task.setDerivedColumns(Collections.emptySet());
+    List<TaskReport> taskReports = control.run(task.dump());
+    return task.getReduceKeyName().isPresent()
+        ? new Reducer(task, schema)
+            .reduce(taskReports, schema.lookupColumn(task.getReduceKeyName().get()))
+        : Exec.newConfigDiff();
   }
 
   @Override
@@ -33,22 +39,8 @@ public class KintoneOutputPlugin implements OutputPlugin {
   @Override
   public TransactionalPageOutput open(TaskSource taskSource, Schema schema, int taskIndex) {
     PluginTask task = taskSource.loadTask(PluginTask.class);
-    KintoneMode mode = KintoneMode.getKintoneModeByValue(task.getMode());
-    switch (mode) {
-      case INSERT:
-        if (task.getUpdateKeyName().isPresent()) {
-          throw new ConfigException("when mode is insert, require no update_key.");
-        }
-        break;
-      case UPDATE:
-      case UPSERT:
-        if (!task.getUpdateKeyName().isPresent()) {
-          throw new ConfigException("when mode is update or upsert, require update_key.");
-        }
-        break;
-      default:
-        throw new ConfigException(String.format("Unknown mode '%s'", task.getMode()));
-    }
-    return new KintonePageOutput(task, schema);
+    return task.getReduceKeyName().isPresent()
+        ? new ReducedPageOutput(schema, taskIndex)
+        : new KintonePageOutput(task, schema);
   }
 }