embulk-output-kintone 1.0.0 → 1.1.0

data/src/main/java/org/embulk/output/kintone/KintonePageOutput.java

@@ -50,7 +50,6 @@ public class KintonePageOutput implements TransactionalPageOutput {
           "GAIA_DA02" // データベースのロックに失敗したため、変更を保存できませんでした。時間をおいて再度お試しください。
           );
   private static final int UPSERT_BATCH_SIZE = 10000;
-  private static final int CHUNK_SIZE = 100;
   private final Map<String, Pair<FieldType, FieldType>> wrongTypeFields = new TreeMap<>();
   private final PluginTask task;
   private final PageReader reader;
@@ -113,7 +112,7 @@ public class KintonePageOutput implements TransactionalPageOutput {
     return Exec.newTaskReport();
   }
 
-  public void connectIfNeeded() {
+  private void connectIfNeeded() {
     if (client != null) {
       return; // Already connected
     }
@@ -203,14 +202,19 @@ public class KintonePageOutput implements TransactionalPageOutput {
     reader.setPage(page);
     KintoneColumnVisitor visitor =
         new KintoneColumnVisitor(
-            reader, task.getColumnOptions(), task.getPreferNulls(), task.getIgnoreNulls());
+            reader,
+            task.getDerivedColumns(),
+            task.getColumnOptions(),
+            task.getPreferNulls(),
+            task.getIgnoreNulls(),
+            task.getReduceKeyName().orElse(null));
     while (reader.nextRecord()) {
       Record record = new Record();
       visitor.setRecord(record);
       reader.getSchema().visitColumns(visitor);
       putWrongTypeFields(record);
       records.add(record);
-      if (records.size() == CHUNK_SIZE) {
+      if (records.size() == task.getChunkSize()) {
         insert(records);
         records.clear();
       }
@@ -226,9 +230,11 @@ public class KintonePageOutput implements TransactionalPageOutput {
     KintoneColumnVisitor visitor =
         new KintoneColumnVisitor(
             reader,
+            task.getDerivedColumns(),
             task.getColumnOptions(),
             task.getPreferNulls(),
             task.getIgnoreNulls(),
+            task.getReduceKeyName().orElse(null),
             task.getUpdateKeyName()
                 .orElseThrow(() -> new RuntimeException("unreachable"))); // Already validated
     while (reader.nextRecord()) {
@@ -243,7 +249,7 @@ public class KintonePageOutput implements TransactionalPageOutput {
         continue;
       }
       records.add(new RecordForUpdate(updateKey, record.removeField(updateKey.getField())));
-      if (records.size() == CHUNK_SIZE) {
+      if (records.size() == task.getChunkSize()) {
         update(records);
         records.clear();
       }
@@ -260,9 +266,11 @@ public class KintonePageOutput implements TransactionalPageOutput {
     KintoneColumnVisitor visitor =
         new KintoneColumnVisitor(
             reader,
+            task.getDerivedColumns(),
             task.getColumnOptions(),
             task.getPreferNulls(),
             task.getIgnoreNulls(),
+            task.getReduceKeyName().orElse(null),
             task.getUpdateKeyName()
                 .orElseThrow(() -> new RuntimeException("unreachable"))); // Already validated
     while (reader.nextRecord()) {
@@ -300,10 +308,10 @@ public class KintonePageOutput implements TransactionalPageOutput {
       } else {
         insertRecords.add(record);
       }
-      if (insertRecords.size() == CHUNK_SIZE) {
+      if (insertRecords.size() == task.getChunkSize()) {
         insert(insertRecords);
         insertRecords.clear();
-      } else if (updateRecords.size() == CHUNK_SIZE) {
+      } else if (updateRecords.size() == task.getChunkSize()) {
         update(updateRecords);
         updateRecords.clear();
       }
@@ -365,11 +373,6 @@ public class KintonePageOutput implements TransactionalPageOutput {
         .collect(Collectors.toList());
   }
 
-  private boolean existsRecord(List<String> existingValues, UpdateKey updateKey) {
-    String value = toString(updateKey.getValue());
-    return value != null && existingValues.stream().anyMatch(v -> v.equals(value));
-  }
-
   private void putWrongTypeFields(Record record) {
     record.getFieldCodes(true).stream()
         .map(
@@ -386,6 +389,11 @@ public class KintonePageOutput implements TransactionalPageOutput {
     return field == null ? null : field.getType();
   }
 
+  private static boolean existsRecord(List<String> existingValues, UpdateKey updateKey) {
+    String value = toString(updateKey.getValue());
+    return value != null && existingValues.stream().anyMatch(v -> v.equals(value));
+  }
+
   private static String toString(Object value) {
     return value == null
         ? null

data/src/main/java/org/embulk/output/kintone/KintoneSortColumn.java

@@ -0,0 +1,33 @@
+package org.embulk.output.kintone;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+public class KintoneSortColumn {
+  private final String name;
+  private final Order order;
+
+  @JsonCreator
+  public KintoneSortColumn(@JsonProperty("name") String name, @JsonProperty("order") Order order) {
+    this.name = name;
+    this.order = order;
+  }
+
+  public String getName() {
+    return name;
+  }
+
+  public Order getOrder() {
+    return order;
+  }
+
+  public enum Order {
+    ASC,
+    DESC;
+
+    @JsonCreator
+    public static Order of(String name) {
+      return valueOf(name.toUpperCase());
+    }
+  }
+}

data/src/main/java/org/embulk/output/kintone/PluginTask.java

@@ -1,10 +1,13 @@
 package org.embulk.output.kintone;
 
+import java.util.List;
 import java.util.Map;
 import java.util.Optional;
+import java.util.Set;
 import org.embulk.config.Config;
 import org.embulk.config.ConfigDefault;
 import org.embulk.config.Task;
+import org.embulk.spi.Column;
 
 public interface PluginTask extends Task {
   @Config("domain")
@@ -57,7 +60,31 @@ public interface PluginTask extends Task {
   @ConfigDefault("null")
   Optional<String> getUpdateKeyName();
 
+  @Config("reduce_key")
+  @ConfigDefault("null")
+  Optional<String> getReduceKeyName();
+
+  @Config("sort_columns")
+  @ConfigDefault("[]")
+  List<KintoneSortColumn> getSortColumns();
+
+  @Config("max_sort_tmp_files")
+  @ConfigDefault("null")
+  Optional<Integer> getMaxSortTmpFiles();
+
+  @Config("max_sort_memory")
+  @ConfigDefault("null")
+  Optional<Long> getMaxSortMemory();
+
+  @Config("chunk_size")
+  @ConfigDefault("100")
+  Integer getChunkSize();
+
   @Config("retry_options")
   @ConfigDefault("{}")
   KintoneRetryOption getRetryOptions();
+
+  Set<Column> getDerivedColumns();
+
+  void setDerivedColumns(Set<Column> columns);
 }

data/src/main/java/org/embulk/output/kintone/deserializer/DeserializeApplier.java

@@ -0,0 +1,19 @@
+package org.embulk.output.kintone.deserializer;
+
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.databind.DeserializationContext;
+import com.fasterxml.jackson.databind.JsonDeserializer;
+import java.util.function.BiFunction;
+
+public class DeserializeApplier<T> extends JsonDeserializer<T> {
+  private final BiFunction<JsonParser, DeserializationContext, T> deserializer;
+
+  public DeserializeApplier(BiFunction<JsonParser, DeserializationContext, T> deserializer) {
+    this.deserializer = deserializer;
+  }
+
+  @Override
+  public T deserialize(JsonParser parser, DeserializationContext context) {
+    return deserializer.apply(parser, context);
+  }
+}

data/src/main/java/org/embulk/output/kintone/deserializer/DeserializeException.java

@@ -0,0 +1,7 @@
+package org.embulk.output.kintone.deserializer;
+
+public class DeserializeException extends RuntimeException {
+  public DeserializeException(Throwable cause) {
+    super(cause);
+  }
+}

data/src/main/java/org/embulk/output/kintone/deserializer/Deserializer.java

@@ -0,0 +1,279 @@
+package org.embulk.output.kintone.deserializer;
+
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.ObjectCodec;
+import com.fasterxml.jackson.databind.DeserializationContext;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.module.SimpleModule;
+import com.fasterxml.jackson.databind.node.TextNode;
+import com.kintone.client.model.FileBody;
+import com.kintone.client.model.Group;
+import com.kintone.client.model.Organization;
+import com.kintone.client.model.User;
+import com.kintone.client.model.record.CalcFieldValue;
+import com.kintone.client.model.record.CheckBoxFieldValue;
+import com.kintone.client.model.record.DateFieldValue;
+import com.kintone.client.model.record.DateTimeFieldValue;
+import com.kintone.client.model.record.DropDownFieldValue;
+import com.kintone.client.model.record.FieldType;
+import com.kintone.client.model.record.FieldValue;
+import com.kintone.client.model.record.FileFieldValue;
+import com.kintone.client.model.record.GroupSelectFieldValue;
+import com.kintone.client.model.record.LinkFieldValue;
+import com.kintone.client.model.record.MultiLineTextFieldValue;
+import com.kintone.client.model.record.MultiSelectFieldValue;
+import com.kintone.client.model.record.NumberFieldValue;
+import com.kintone.client.model.record.OrganizationSelectFieldValue;
+import com.kintone.client.model.record.RadioButtonFieldValue;
+import com.kintone.client.model.record.RichTextFieldValue;
+import com.kintone.client.model.record.SingleLineTextFieldValue;
+import com.kintone.client.model.record.SubtableFieldValue;
+import com.kintone.client.model.record.TableRow;
+import com.kintone.client.model.record.TimeFieldValue;
+import com.kintone.client.model.record.UserSelectFieldValue;
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.math.BigDecimal;
+import java.time.LocalDate;
+import java.time.LocalTime;
+import java.time.ZonedDateTime;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Spliterator;
+import java.util.Spliterators;
+import java.util.function.BiFunction;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import java.util.stream.StreamSupport;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class Deserializer {
+  private static final Logger LOGGER =
+      LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+  private static final JsonNode NULL = new TextNode(null);
+  private final ObjectMapper mapper = new ObjectMapper();
+
+  public Deserializer() {
+    SimpleModule module = new SimpleModule();
+    // spotless:off
+    addDeserializer(module, SubtableFieldValue.class, this::deserializeSubtable);
+    addDeserializer(module, TableRow.class, this::deserializeTableRow);
+    addDeserializer(module, SingleLineTextFieldValue.class, this::deserializeSingleLineText);
+    addDeserializer(module, MultiLineTextFieldValue.class, this::deserializeMultiLineText);
+    addDeserializer(module, RichTextFieldValue.class, this::deserializeRichText);
+    addDeserializer(module, NumberFieldValue.class, this::deserializeNumber);
+    addDeserializer(module, CalcFieldValue.class, this::deserializeCalc);
+    addDeserializer(module, CheckBoxFieldValue.class, this::deserializeCheckBox);
+    addDeserializer(module, RadioButtonFieldValue.class, this::deserializeRadioButton);
+    addDeserializer(module, MultiSelectFieldValue.class, this::deserializeMultiSelect);
+    addDeserializer(module, DropDownFieldValue.class, this::deserializeDropDown);
+    addDeserializer(module, UserSelectFieldValue.class, this::deserializeUserSelect);
+    addDeserializer(module, User.class, this::deserializeUser);
+    addDeserializer(module, OrganizationSelectFieldValue.class, this::deserializeOrganizationSelect);
+    addDeserializer(module, Organization.class, this::deserializeOrganization);
+    addDeserializer(module, GroupSelectFieldValue.class, this::deserializeGroupSelect);
+    addDeserializer(module, Group.class, this::deserializeGroup);
+    addDeserializer(module, DateFieldValue.class, this::deserializeDate);
+    addDeserializer(module, TimeFieldValue.class, this::deserializeTime);
+    addDeserializer(module, DateTimeFieldValue.class, this::deserializeDateTime);
+    addDeserializer(module, LinkFieldValue.class, this::deserializeLink);
+    addDeserializer(module, FileFieldValue.class, this::deserializeFile);
+    addDeserializer(module, FileBody.class, this::deserializeFileBody);
+    // spotless:on
+    mapper.registerModule(module);
+  }
+
+  public <T> T deserialize(String content, Class<T> type) {
+    try {
+      return mapper.readValue(content, type);
+    } catch (IOException e) {
+      throw new DeserializeException(e);
+    }
+  }
+
+  private <T> void addDeserializer(
+      SimpleModule module,
+      Class<T> type,
+      BiFunction<JsonParser, DeserializationContext, T> deserializer) {
+    module.addDeserializer(type, new DeserializeApplier<>(deserializer));
+  }
+
+  private SubtableFieldValue deserializeSubtable(
+      JsonParser parser, DeserializationContext context) {
+    return new SubtableFieldValue(readList(parser, TableRow.class));
+  }
+
+  private TableRow deserializeTableRow(JsonParser parser, DeserializationContext context) {
+    ObjectCodec codec = parser.getCodec();
+    JsonNode node = readTree(codec, parser);
+    TableRow row = new TableRow(get(node, "id", JsonNode::asLong));
+    stream(node.get("value").fields())
+        .forEach(entry -> row.putField(entry.getKey(), readValue(entry.getValue(), codec)));
+    return row;
+  }
+
+  private SingleLineTextFieldValue deserializeSingleLineText(
+      JsonParser parser, DeserializationContext context) {
+    return new SingleLineTextFieldValue(readText(parser));
+  }
+
+  private MultiLineTextFieldValue deserializeMultiLineText(
+      JsonParser parser, DeserializationContext context) {
+    return new MultiLineTextFieldValue(readText(parser));
+  }
+
+  private RichTextFieldValue deserializeRichText(
+      JsonParser parser, DeserializationContext context) {
+    return new RichTextFieldValue(readText(parser));
+  }
+
+  private NumberFieldValue deserializeNumber(JsonParser parser, DeserializationContext context) {
+    return new NumberFieldValue(read(parser, BigDecimal::new));
+  }
+
+  private CalcFieldValue deserializeCalc(JsonParser parser, DeserializationContext context) {
+    LOGGER.warn(
+        String.format(
+            "Ignore field: %s", new CalcFieldValue((BigDecimal) read(parser, BigDecimal::new))));
+    return null;
+  }
+
+  private CheckBoxFieldValue deserializeCheckBox(
+      JsonParser parser, DeserializationContext context) {
+    return new CheckBoxFieldValue(readList(parser, String.class));
+  }
+
+  private RadioButtonFieldValue deserializeRadioButton(
+      JsonParser parser, DeserializationContext context) {
+    return new RadioButtonFieldValue(readText(parser));
+  }
+
+  private MultiSelectFieldValue deserializeMultiSelect(
+      JsonParser parser, DeserializationContext context) {
+    return new MultiSelectFieldValue(readList(parser, String.class));
+  }
+
+  private DropDownFieldValue deserializeDropDown(
+      JsonParser parser, DeserializationContext context) {
+    return new DropDownFieldValue(readText(parser));
+  }
+
+  private UserSelectFieldValue deserializeUserSelect(
+      JsonParser parser, DeserializationContext context) {
+    return new UserSelectFieldValue(readList(parser, User.class));
+  }
+
+  private User deserializeUser(JsonParser parser, DeserializationContext context) {
+    JsonNode node = readTree(parser);
+    return new User(get(node, "name", JsonNode::asText), get(node, "code", JsonNode::asText));
+  }
+
+  private OrganizationSelectFieldValue deserializeOrganizationSelect(
+      JsonParser parser, DeserializationContext context) {
+    return new OrganizationSelectFieldValue(readList(parser, Organization.class));
+  }
+
+  private Organization deserializeOrganization(JsonParser parser, DeserializationContext context) {
+    JsonNode node = readTree(parser);
+    return new Organization(
+        get(node, "name", JsonNode::asText), get(node, "code", JsonNode::asText));
+  }
+
+  private GroupSelectFieldValue deserializeGroupSelect(
+      JsonParser parser, DeserializationContext context) {
+    return new GroupSelectFieldValue(readList(parser, Group.class));
+  }
+
+  private Group deserializeGroup(JsonParser parser, DeserializationContext context) {
+    JsonNode node = readTree(parser);
+    return new Group(get(node, "name", JsonNode::asText), get(node, "code", JsonNode::asText));
+  }
+
+  private DateFieldValue deserializeDate(JsonParser parser, DeserializationContext context) {
+    return new DateFieldValue(read(parser, LocalDate::parse));
+  }
+
+  private TimeFieldValue deserializeTime(JsonParser parser, DeserializationContext context) {
+    return new TimeFieldValue(read(parser, LocalTime::parse));
+  }
+
+  private DateTimeFieldValue deserializeDateTime(
+      JsonParser parser, DeserializationContext context) {
+    return new DateTimeFieldValue(read(parser, ZonedDateTime::parse));
+  }
+
+  private LinkFieldValue deserializeLink(JsonParser parser, DeserializationContext context) {
+    return new LinkFieldValue(readText(parser));
+  }
+
+  private FileFieldValue deserializeFile(JsonParser parser, DeserializationContext context) {
+    return new FileFieldValue(readList(parser, FileBody.class));
+  }
+
+  private FileBody deserializeFileBody(JsonParser parser, DeserializationContext context) {
+    JsonNode node = readTree(parser);
+    return new FileBody()
+        .setContentType(get(node, "contentType", JsonNode::asText))
+        .setFileKey(get(node, "fileKey", JsonNode::asText))
+        .setName(get(node, "name", JsonNode::asText))
+        .setSize(get(node, "size", JsonNode::asInt));
+  }
+
+  private static <T> T get(JsonNode node, String name, Function<JsonNode, T> as) {
+    return node.has(name) ? as.apply(node.get(name)) : null;
+  }
+
+  private static FieldValue readValue(JsonNode node, ObjectCodec codec) {
+    return node == null
+        ? null
+        : readValueAs(
+            node.has("value") ? node.get("value") : NULL,
+            codec,
+            FieldType.valueOf(node.get("type").asText()).getFieldValueClass());
+  }
+
+  private static <T> List<T> readList(JsonParser parser, Class<T> type) {
+    ObjectCodec codec = parser.getCodec();
+    return stream(readTree(codec, parser).elements())
+        .map(node -> readValueAs(node, codec, type))
+        .collect(Collectors.toList());
+  }
+
+  private static <T> Stream<T> stream(Iterator<T> iterator) {
+    return StreamSupport.stream(
+        Spliterators.spliteratorUnknownSize(iterator, Spliterator.ORDERED), false);
+  }
+
+  private static <T> T read(JsonParser parser, Function<String, T> as) {
+    String text = readText(parser);
+    return text == null || text.isEmpty() ? null : as.apply(text);
+  }
+
+  private static String readText(JsonParser parser) {
+    JsonNode node = readTree(parser);
+    return node == null || node.isNull() ? null : node.asText();
+  }
+
+  private static JsonNode readTree(JsonParser parser) {
+    return readTree(parser.getCodec(), parser);
+  }
+
+  private static JsonNode readTree(ObjectCodec codec, JsonParser parser) {
+    try {
+      return codec.readTree(parser);
+    } catch (IOException e) {
+      throw new DeserializeException(e);
+    }
+  }
+
+  private static <T> T readValueAs(JsonNode node, ObjectCodec codec, Class<T> type) {
+    try (JsonParser parser = node.traverse(codec)) {
+      return parser.readValueAs(type);
+    } catch (IOException e) {
+      throw new DeserializeException(e);
+    }
+  }
+}

data/src/main/java/org/embulk/output/kintone/reducer/CSVInputColumnVisitor.java

@@ -0,0 +1,78 @@
+package org.embulk.output.kintone.reducer;
+
+import java.time.Instant;
+import java.util.List;
+import org.embulk.spi.Column;
+import org.embulk.spi.ColumnVisitor;
+import org.embulk.spi.PageBuilder;
+import org.embulk.spi.time.Timestamp;
+
+public class CSVInputColumnVisitor implements ColumnVisitor {
+  private final PageBuilder builder;
+  private final List<String> values;
+
+  public CSVInputColumnVisitor(PageBuilder builder, List<String> values) {
+    this.builder = builder;
+    this.values = values;
+  }
+
+  @Override
+  public void booleanColumn(Column column) {
+    String value = values.get(column.getIndex());
+    if (value == null) {
+      builder.setNull(column);
+    } else {
+      builder.setBoolean(column, Boolean.parseBoolean(value));
+    }
+  }
+
+  @Override
+  public void longColumn(Column column) {
+    String value = values.get(column.getIndex());
+    if (value == null) {
+      builder.setNull(column);
+    } else {
+      builder.setLong(column, Long.parseLong(value));
+    }
+  }
+
+  @Override
+  public void doubleColumn(Column column) {
+    String value = values.get(column.getIndex());
+    if (value == null) {
+      builder.setNull(column);
+    } else {
+      builder.setDouble(column, Double.parseDouble(value));
+    }
+  }
+
+  @Override
+  public void stringColumn(Column column) {
+    String value = values.get(column.getIndex());
+    if (value == null) {
+      builder.setNull(column);
+    } else {
+      builder.setString(column, value);
+    }
+  }
+
+  @Override
+  public void timestampColumn(Column column) {
+    String value = values.get(column.getIndex());
+    if (value == null) {
+      builder.setNull(column);
+    } else {
+      builder.setTimestamp(column, Timestamp.ofInstant(Instant.parse(value)));
+    }
+  }
+
+  @Override
+  public void jsonColumn(Column column) {
+    String value = values.get(column.getIndex());
+    if (value == null) {
+      builder.setNull(column);
+    } else {
+      builder.setJson(column, Reducer.PARSER.parse(value));
+    }
+  }
+}

data/src/main/java/org/embulk/output/kintone/reducer/CSVOutputColumnVisitor.java

@@ -0,0 +1,79 @@
+package org.embulk.output.kintone.reducer;
+
+import java.io.IOException;
+import org.apache.commons.csv.CSVPrinter;
+import org.embulk.spi.Column;
+import org.embulk.spi.ColumnVisitor;
+import org.embulk.spi.PageReader;
+
+public class CSVOutputColumnVisitor implements ColumnVisitor {
+  private final PageReader reader;
+  private final CSVPrinter printer;
+
+  public CSVOutputColumnVisitor(PageReader reader, CSVPrinter printer) {
+    this.reader = reader;
+    this.printer = printer;
+  }
+
+  @Override
+  public void booleanColumn(Column column) {
+    if (reader.isNull(column)) {
+      print(null);
+    } else {
+      print(reader.getBoolean(column));
+    }
+  }
+
+  @Override
+  public void longColumn(Column column) {
+    if (reader.isNull(column)) {
+      print(null);
+    } else {
+      print(reader.getLong(column));
+    }
+  }
+
+  @Override
+  public void doubleColumn(Column column) {
+    if (reader.isNull(column)) {
+      print(null);
+    } else {
+      print(reader.getDouble(column));
+    }
+  }
+
+  @Override
+  public void stringColumn(Column column) {
+    if (reader.isNull(column)) {
+      print(null);
+    } else {
+      print(reader.getString(column));
+    }
+  }
+
+  @Override
+  public void timestampColumn(Column column) {
+    if (reader.isNull(column)) {
+      print(null);
+    } else {
+      print(reader.getTimestamp(column).getInstant());
+    }
+  }
+
+  @Override
+  public void jsonColumn(Column column) {
+    if (reader.isNull(column)) {
+      print(null);
+    } else {
+      print(reader.getJson(column).toJson());
+    }
+  }
+
+  private void print(Object value) {
+    try {
+      printer.print(value);
+    } catch (IOException e) {
+      throw new ReduceException(e);
+    }
+  }
+}

data/src/main/java/org/embulk/output/kintone/reducer/ReduceException.java

@@ -0,0 +1,11 @@
+package org.embulk.output.kintone.reducer;
+
+public class ReduceException extends RuntimeException {
+  public ReduceException(String message) {
+    super(message);
+  }
+
+  public ReduceException(Throwable cause) {
+    super(cause);
+  }
+}