embulk-output-kintone 1.0.0 → 1.2.0

data/src/main/java/org/embulk/output/kintone/deserializer/Deserializer.java

@@ -0,0 +1,279 @@
+package org.embulk.output.kintone.deserializer;
+
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.ObjectCodec;
+import com.fasterxml.jackson.databind.DeserializationContext;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.module.SimpleModule;
+import com.fasterxml.jackson.databind.node.TextNode;
+import com.kintone.client.model.FileBody;
+import com.kintone.client.model.Group;
+import com.kintone.client.model.Organization;
+import com.kintone.client.model.User;
+import com.kintone.client.model.record.CalcFieldValue;
+import com.kintone.client.model.record.CheckBoxFieldValue;
+import com.kintone.client.model.record.DateFieldValue;
+import com.kintone.client.model.record.DateTimeFieldValue;
+import com.kintone.client.model.record.DropDownFieldValue;
+import com.kintone.client.model.record.FieldType;
+import com.kintone.client.model.record.FieldValue;
+import com.kintone.client.model.record.FileFieldValue;
+import com.kintone.client.model.record.GroupSelectFieldValue;
+import com.kintone.client.model.record.LinkFieldValue;
+import com.kintone.client.model.record.MultiLineTextFieldValue;
+import com.kintone.client.model.record.MultiSelectFieldValue;
+import com.kintone.client.model.record.NumberFieldValue;
+import com.kintone.client.model.record.OrganizationSelectFieldValue;
+import com.kintone.client.model.record.RadioButtonFieldValue;
+import com.kintone.client.model.record.RichTextFieldValue;
+import com.kintone.client.model.record.SingleLineTextFieldValue;
+import com.kintone.client.model.record.SubtableFieldValue;
+import com.kintone.client.model.record.TableRow;
+import com.kintone.client.model.record.TimeFieldValue;
+import com.kintone.client.model.record.UserSelectFieldValue;
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.math.BigDecimal;
+import java.time.LocalDate;
+import java.time.LocalTime;
+import java.time.ZonedDateTime;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Spliterator;
+import java.util.Spliterators;
+import java.util.function.BiFunction;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import java.util.stream.StreamSupport;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class Deserializer {
+  private static final Logger LOGGER =
+      LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+  private static final JsonNode NULL = new TextNode(null);
+  private final ObjectMapper mapper = new ObjectMapper();
+
+  public Deserializer() {
+    SimpleModule module = new SimpleModule();
+    // spotless:off
+    addDeserializer(module, SubtableFieldValue.class, this::deserializeSubtable);
+    addDeserializer(module, TableRow.class, this::deserializeTableRow);
+    addDeserializer(module, SingleLineTextFieldValue.class, this::deserializeSingleLineText);
+    addDeserializer(module, MultiLineTextFieldValue.class, this::deserializeMultiLineText);
+    addDeserializer(module, RichTextFieldValue.class, this::deserializeRichText);
+    addDeserializer(module, NumberFieldValue.class, this::deserializeNumber);
+    addDeserializer(module, CalcFieldValue.class, this::deserializeCalc);
+    addDeserializer(module, CheckBoxFieldValue.class, this::deserializeCheckBox);
+    addDeserializer(module, RadioButtonFieldValue.class, this::deserializeRadioButton);
+    addDeserializer(module, MultiSelectFieldValue.class, this::deserializeMultiSelect);
+    addDeserializer(module, DropDownFieldValue.class, this::deserializeDropDown);
+    addDeserializer(module, UserSelectFieldValue.class, this::deserializeUserSelect);
+    addDeserializer(module, User.class, this::deserializeUser);
+    addDeserializer(module, OrganizationSelectFieldValue.class, this::deserializeOrganizationSelect);
+    addDeserializer(module, Organization.class, this::deserializeOrganization);
+    addDeserializer(module, GroupSelectFieldValue.class, this::deserializeGroupSelect);
+    addDeserializer(module, Group.class, this::deserializeGroup);
+    addDeserializer(module, DateFieldValue.class, this::deserializeDate);
+    addDeserializer(module, TimeFieldValue.class, this::deserializeTime);
+    addDeserializer(module, DateTimeFieldValue.class, this::deserializeDateTime);
+    addDeserializer(module, LinkFieldValue.class, this::deserializeLink);
+    addDeserializer(module, FileFieldValue.class, this::deserializeFile);
+    addDeserializer(module, FileBody.class, this::deserializeFileBody);
+    // spotless:on
+    mapper.registerModule(module);
+  }
+
+  public <T> T deserialize(String content, Class<T> type) {
+    try {
+      return mapper.readValue(content, type);
+    } catch (IOException e) {
+      throw new DeserializeException(e);
+    }
+  }
+
+  private <T> void addDeserializer(
+      SimpleModule module,
+      Class<T> type,
+      BiFunction<JsonParser, DeserializationContext, T> deserializer) {
+    module.addDeserializer(type, new DeserializeApplier<>(deserializer));
+  }
+
+  private SubtableFieldValue deserializeSubtable(
+      JsonParser parser, DeserializationContext context) {
+    return new SubtableFieldValue(readList(parser, TableRow.class));
+  }
+
+  private TableRow deserializeTableRow(JsonParser parser, DeserializationContext context) {
+    ObjectCodec codec = parser.getCodec();
+    JsonNode node = readTree(codec, parser);
+    TableRow row = new TableRow(get(node, "id", JsonNode::asLong));
+    stream(node.get("value").fields())
+        .forEach(entry -> row.putField(entry.getKey(), readValue(entry.getValue(), codec)));
+    return row;
+  }
+
+  private SingleLineTextFieldValue deserializeSingleLineText(
+      JsonParser parser, DeserializationContext context) {
+    return new SingleLineTextFieldValue(readText(parser));
+  }
+
+  private MultiLineTextFieldValue deserializeMultiLineText(
+      JsonParser parser, DeserializationContext context) {
+    return new MultiLineTextFieldValue(readText(parser));
+  }
+
+  private RichTextFieldValue deserializeRichText(
+      JsonParser parser, DeserializationContext context) {
+    return new RichTextFieldValue(readText(parser));
+  }
+
+  private NumberFieldValue deserializeNumber(JsonParser parser, DeserializationContext context) {
+    return new NumberFieldValue(read(parser, BigDecimal::new));
+  }
+
+  private CalcFieldValue deserializeCalc(JsonParser parser, DeserializationContext context) {
+    LOGGER.warn(
+        String.format(
+            "Ignore field: %s", new CalcFieldValue((BigDecimal) read(parser, BigDecimal::new))));
+    return null;
+  }
+
+  private CheckBoxFieldValue deserializeCheckBox(
+      JsonParser parser, DeserializationContext context) {
+    return new CheckBoxFieldValue(readList(parser, String.class));
+  }
+
+  private RadioButtonFieldValue deserializeRadioButton(
+      JsonParser parser, DeserializationContext context) {
+    return new RadioButtonFieldValue(readText(parser));
+  }
+
+  private MultiSelectFieldValue deserializeMultiSelect(
+      JsonParser parser, DeserializationContext context) {
+    return new MultiSelectFieldValue(readList(parser, String.class));
+  }
+
+  private DropDownFieldValue deserializeDropDown(
+      JsonParser parser, DeserializationContext context) {
+    return new DropDownFieldValue(readText(parser));
+  }
+
+  private UserSelectFieldValue deserializeUserSelect(
+      JsonParser parser, DeserializationContext context) {
+    return new UserSelectFieldValue(readList(parser, User.class));
+  }
+
+  private User deserializeUser(JsonParser parser, DeserializationContext context) {
+    JsonNode node = readTree(parser);
+    return new User(get(node, "name", JsonNode::asText), get(node, "code", JsonNode::asText));
+  }
+
+  private OrganizationSelectFieldValue deserializeOrganizationSelect(
+      JsonParser parser, DeserializationContext context) {
+    return new OrganizationSelectFieldValue(readList(parser, Organization.class));
+  }
+
+  private Organization deserializeOrganization(JsonParser parser, DeserializationContext context) {
+    JsonNode node = readTree(parser);
+    return new Organization(
+        get(node, "name", JsonNode::asText), get(node, "code", JsonNode::asText));
+  }
+
+  private GroupSelectFieldValue deserializeGroupSelect(
+      JsonParser parser, DeserializationContext context) {
+    return new GroupSelectFieldValue(readList(parser, Group.class));
+  }
+
+  private Group deserializeGroup(JsonParser parser, DeserializationContext context) {
+    JsonNode node = readTree(parser);
+    return new Group(get(node, "name", JsonNode::asText), get(node, "code", JsonNode::asText));
+  }
+
+  private DateFieldValue deserializeDate(JsonParser parser, DeserializationContext context) {
+    return new DateFieldValue(read(parser, LocalDate::parse));
+  }
+
+  private TimeFieldValue deserializeTime(JsonParser parser, DeserializationContext context) {
+    return new TimeFieldValue(read(parser, LocalTime::parse));
+  }
+
+  private DateTimeFieldValue deserializeDateTime(
+      JsonParser parser, DeserializationContext context) {
+    return new DateTimeFieldValue(read(parser, ZonedDateTime::parse));
+  }
+
+  private LinkFieldValue deserializeLink(JsonParser parser, DeserializationContext context) {
+    return new LinkFieldValue(readText(parser));
+  }
+
+  private FileFieldValue deserializeFile(JsonParser parser, DeserializationContext context) {
+    return new FileFieldValue(readList(parser, FileBody.class));
+  }
+
+  private FileBody deserializeFileBody(JsonParser parser, DeserializationContext context) {
+    JsonNode node = readTree(parser);
+    return new FileBody()
+        .setContentType(get(node, "contentType", JsonNode::asText))
+        .setFileKey(get(node, "fileKey", JsonNode::asText))
+        .setName(get(node, "name", JsonNode::asText))
+        .setSize(get(node, "size", JsonNode::asInt));
+  }
+
+  private static <T> T get(JsonNode node, String name, Function<JsonNode, T> as) {
+    return node.has(name) ? as.apply(node.get(name)) : null;
+  }
+
+  private static FieldValue readValue(JsonNode node, ObjectCodec codec) {
+    return node == null
+        ? null
+        : readValueAs(
+            node.has("value") ? node.get("value") : NULL,
+            codec,
+            FieldType.valueOf(node.get("type").asText()).getFieldValueClass());
+  }
+
+  private static <T> List<T> readList(JsonParser parser, Class<T> type) {
+    ObjectCodec codec = parser.getCodec();
+    return stream(readTree(codec, parser).elements())
+        .map(node -> readValueAs(node, codec, type))
+        .collect(Collectors.toList());
+  }
+
+  private static <T> Stream<T> stream(Iterator<T> iterator) {
+    return StreamSupport.stream(
+        Spliterators.spliteratorUnknownSize(iterator, Spliterator.ORDERED), false);
+  }
+
+  private static <T> T read(JsonParser parser, Function<String, T> as) {
+    String text = readText(parser);
+    return text == null || text.isEmpty() ? null : as.apply(text);
+  }
+
+  private static String readText(JsonParser parser) {
+    JsonNode node = readTree(parser);
+    return node == null || node.isNull() ? null : node.asText();
+  }
+
+  private static JsonNode readTree(JsonParser parser) {
+    return readTree(parser.getCodec(), parser);
+  }
+
+  private static JsonNode readTree(ObjectCodec codec, JsonParser parser) {
+    try {
+      return codec.readTree(parser);
+    } catch (IOException e) {
+      throw new DeserializeException(e);
+    }
+  }
+
+  private static <T> T readValueAs(JsonNode node, ObjectCodec codec, Class<T> type) {
+    try (JsonParser parser = node.traverse(codec)) {
+      return parser.readValueAs(type);
+    } catch (IOException e) {
+      throw new DeserializeException(e);
+    }
+  }
+}

data/src/main/java/org/embulk/output/kintone/record/Id.java

@@ -0,0 +1,18 @@
+package org.embulk.output.kintone.record;
+
+public class Id {
+  public static final String FIELD = "$id";
+  private Long value;
+
+  public void setValue(Long value) {
+    this.value = value;
+  }
+
+  public Long getValue() {
+    return value;
+  }
+
+  public boolean isPresent() {
+    return value != null;
+  }
+}

data/src/main/java/org/embulk/output/kintone/record/IdOrUpdateKey.java

@@ -0,0 +1,45 @@
+package org.embulk.output.kintone.record;
+
+import com.kintone.client.model.record.Record;
+import com.kintone.client.model.record.RecordForUpdate;
+import com.kintone.client.model.record.UpdateKey;
+
+public class IdOrUpdateKey {
+  private final Id id = new Id();
+  private final UpdateKey updateKey = new UpdateKey();
+
+  public Id getId() {
+    return id;
+  }
+
+  public UpdateKey getUpdateKey() {
+    return updateKey;
+  }
+
+  public String getField() {
+    return isIdPresent() ? Id.FIELD : updateKey.getField();
+  }
+
+  public Object getValue() {
+    return isIdPresent() ? id.getValue() : updateKey.getValue();
+  }
+
+  public RecordForUpdate forUpdate(Record record) {
+    return isIdPresent()
+        ? new RecordForUpdate(id.getValue(), record)
+        : new RecordForUpdate(updateKey, record.removeField(updateKey.getField()));
+  }
+
+  public boolean isPresent() {
+    return isIdPresent() || isUpdateKeyPresent();
+  }
+
+  public boolean isIdPresent() {
+    return id.isPresent();
+  }
+
+  public boolean isUpdateKeyPresent() {
+    Object value = updateKey.getValue();
+    return value != null && !value.toString().isEmpty();
+  }
+}

data/src/main/java/org/embulk/output/kintone/record/Skip.java

@@ -0,0 +1,14 @@
+package org.embulk.output.kintone.record;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+
+public enum Skip {
+  AUTO,
+  NEVER,
+  ALWAYS;
+
+  @JsonCreator
+  public static Skip of(String name) {
+    return valueOf(name.toUpperCase());
+  }
+}

data/src/main/java/org/embulk/output/kintone/reducer/CSVInputColumnVisitor.java

@@ -0,0 +1,78 @@
+package org.embulk.output.kintone.reducer;
+
+import java.time.Instant;
+import java.util.List;
+import org.embulk.spi.Column;
+import org.embulk.spi.ColumnVisitor;
+import org.embulk.spi.PageBuilder;
+import org.embulk.spi.time.Timestamp;
+
+public class CSVInputColumnVisitor implements ColumnVisitor {
+  private final PageBuilder builder;
+  private final List<String> values;
+
+  public CSVInputColumnVisitor(PageBuilder builder, List<String> values) {
+    this.builder = builder;
+    this.values = values;
+  }
+
+  @Override
+  public void booleanColumn(Column column) {
+    String value = values.get(column.getIndex());
+    if (value == null) {
+      builder.setNull(column);
+    } else {
+      builder.setBoolean(column, Boolean.parseBoolean(value));
+    }
+  }
+
+  @Override
+  public void longColumn(Column column) {
+    String value = values.get(column.getIndex());
+    if (value == null) {
+      builder.setNull(column);
+    } else {
+      builder.setLong(column, Long.parseLong(value));
+    }
+  }
+
+  @Override
+  public void doubleColumn(Column column) {
+    String value = values.get(column.getIndex());
+    if (value == null) {
+      builder.setNull(column);
+    } else {
+      builder.setDouble(column, Double.parseDouble(value));
+    }
+  }
+
+  @Override
+  public void stringColumn(Column column) {
+    String value = values.get(column.getIndex());
+    if (value == null) {
+      builder.setNull(column);
+    } else {
+      builder.setString(column, value);
+    }
+  }
+
+  @Override
+  public void timestampColumn(Column column) {
+    String value = values.get(column.getIndex());
+    if (value == null) {
+      builder.setNull(column);
+    } else {
+      builder.setTimestamp(column, Timestamp.ofInstant(Instant.parse(value)));
+    }
+  }
+
+  @Override
+  public void jsonColumn(Column column) {
+    String value = values.get(column.getIndex());
+    if (value == null) {
+      builder.setNull(column);
+    } else {
+      builder.setJson(column, Reducer.PARSER.parse(value));
+    }
+  }
+}

data/src/main/java/org/embulk/output/kintone/reducer/CSVOutputColumnVisitor.java

@@ -0,0 +1,79 @@
+package org.embulk.output.kintone.reducer;
+
+import java.io.IOException;
+import org.apache.commons.csv.CSVPrinter;
+import org.embulk.spi.Column;
+import org.embulk.spi.ColumnVisitor;
+import org.embulk.spi.PageReader;
+
+public class CSVOutputColumnVisitor implements ColumnVisitor {
+  private final PageReader reader;
+  private final CSVPrinter printer;
+
+  public CSVOutputColumnVisitor(PageReader reader, CSVPrinter printer) {
+    this.reader = reader;
+    this.printer = printer;
+  }
+
+  @Override
+  public void booleanColumn(Column column) {
+    if (reader.isNull(column)) {
+      print(null);
+    } else {
+      print(reader.getBoolean(column));
+    }
+  }
+
+  @Override
+  public void longColumn(Column column) {
+    if (reader.isNull(column)) {
+      print(null);
+    } else {
+      print(reader.getLong(column));
+    }
+  }
+
+  @Override
+  public void doubleColumn(Column column) {
+    if (reader.isNull(column)) {
+      print(null);
+    } else {
+      print(reader.getDouble(column));
+    }
+  }
+
+  @Override
+  public void stringColumn(Column column) {
+    if (reader.isNull(column)) {
+      print(null);
+    } else {
+      print(reader.getString(column));
+    }
+  }
+
+  @Override
+  public void timestampColumn(Column column) {
+    if (reader.isNull(column)) {
+      print(null);
+    } else {
+      print(reader.getTimestamp(column).getInstant());
+    }
+  }
+
+  @Override
+  public void jsonColumn(Column column) {
+    if (reader.isNull(column)) {
+      print(null);
+    } else {
+      print(reader.getJson(column).toJson());
+    }
+  }
+
+  private void print(Object value) {
+    try {
+      printer.print(value);
+    } catch (IOException e) {
+      throw new ReduceException(e);
+    }
+  }
+}

data/src/main/java/org/embulk/output/kintone/reducer/ReduceException.java

@@ -0,0 +1,11 @@
+package org.embulk.output.kintone.reducer;
+
+public class ReduceException extends RuntimeException {
+  public ReduceException(String message) {
+    super(message);
+  }
+
+  public ReduceException(Throwable cause) {
+    super(cause);
+  }
+}