embulk-output-kintone 0.4.1 → 1.1.0

data/src/main/java/org/embulk/output/kintone/deserializer/Deserializer.java

@@ -0,0 +1,279 @@
+package org.embulk.output.kintone.deserializer;
+
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.ObjectCodec;
+import com.fasterxml.jackson.databind.DeserializationContext;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.module.SimpleModule;
+import com.fasterxml.jackson.databind.node.TextNode;
+import com.kintone.client.model.FileBody;
+import com.kintone.client.model.Group;
+import com.kintone.client.model.Organization;
+import com.kintone.client.model.User;
+import com.kintone.client.model.record.CalcFieldValue;
+import com.kintone.client.model.record.CheckBoxFieldValue;
+import com.kintone.client.model.record.DateFieldValue;
+import com.kintone.client.model.record.DateTimeFieldValue;
+import com.kintone.client.model.record.DropDownFieldValue;
+import com.kintone.client.model.record.FieldType;
+import com.kintone.client.model.record.FieldValue;
+import com.kintone.client.model.record.FileFieldValue;
+import com.kintone.client.model.record.GroupSelectFieldValue;
+import com.kintone.client.model.record.LinkFieldValue;
+import com.kintone.client.model.record.MultiLineTextFieldValue;
+import com.kintone.client.model.record.MultiSelectFieldValue;
+import com.kintone.client.model.record.NumberFieldValue;
+import com.kintone.client.model.record.OrganizationSelectFieldValue;
+import com.kintone.client.model.record.RadioButtonFieldValue;
+import com.kintone.client.model.record.RichTextFieldValue;
+import com.kintone.client.model.record.SingleLineTextFieldValue;
+import com.kintone.client.model.record.SubtableFieldValue;
+import com.kintone.client.model.record.TableRow;
+import com.kintone.client.model.record.TimeFieldValue;
+import com.kintone.client.model.record.UserSelectFieldValue;
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.math.BigDecimal;
+import java.time.LocalDate;
+import java.time.LocalTime;
+import java.time.ZonedDateTime;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Spliterator;
+import java.util.Spliterators;
+import java.util.function.BiFunction;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import java.util.stream.StreamSupport;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class Deserializer {
+  private static final Logger LOGGER =
+      LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+  private static final JsonNode NULL = new TextNode(null);
+  private final ObjectMapper mapper = new ObjectMapper();
+
+  public Deserializer() {
+    SimpleModule module = new SimpleModule();
+    // spotless:off
+    addDeserializer(module, SubtableFieldValue.class, this::deserializeSubtable);
+    addDeserializer(module, TableRow.class, this::deserializeTableRow);
+    addDeserializer(module, SingleLineTextFieldValue.class, this::deserializeSingleLineText);
+    addDeserializer(module, MultiLineTextFieldValue.class, this::deserializeMultiLineText);
+    addDeserializer(module, RichTextFieldValue.class, this::deserializeRichText);
+    addDeserializer(module, NumberFieldValue.class, this::deserializeNumber);
+    addDeserializer(module, CalcFieldValue.class, this::deserializeCalc);
+    addDeserializer(module, CheckBoxFieldValue.class, this::deserializeCheckBox);
+    addDeserializer(module, RadioButtonFieldValue.class, this::deserializeRadioButton);
+    addDeserializer(module, MultiSelectFieldValue.class, this::deserializeMultiSelect);
+    addDeserializer(module, DropDownFieldValue.class, this::deserializeDropDown);
+    addDeserializer(module, UserSelectFieldValue.class, this::deserializeUserSelect);
+    addDeserializer(module, User.class, this::deserializeUser);
+    addDeserializer(module, OrganizationSelectFieldValue.class, this::deserializeOrganizationSelect);
+    addDeserializer(module, Organization.class, this::deserializeOrganization);
+    addDeserializer(module, GroupSelectFieldValue.class, this::deserializeGroupSelect);
+    addDeserializer(module, Group.class, this::deserializeGroup);
+    addDeserializer(module, DateFieldValue.class, this::deserializeDate);
+    addDeserializer(module, TimeFieldValue.class, this::deserializeTime);
+    addDeserializer(module, DateTimeFieldValue.class, this::deserializeDateTime);
+    addDeserializer(module, LinkFieldValue.class, this::deserializeLink);
+    addDeserializer(module, FileFieldValue.class, this::deserializeFile);
+    addDeserializer(module, FileBody.class, this::deserializeFileBody);
+    // spotless:on
+    mapper.registerModule(module);
+  }
+
+  public <T> T deserialize(String content, Class<T> type) {
+    try {
+      return mapper.readValue(content, type);
+    } catch (IOException e) {
+      throw new DeserializeException(e);
+    }
+  }
+
+  private <T> void addDeserializer(
+      SimpleModule module,
+      Class<T> type,
+      BiFunction<JsonParser, DeserializationContext, T> deserializer) {
+    module.addDeserializer(type, new DeserializeApplier<>(deserializer));
+  }
+
+  private SubtableFieldValue deserializeSubtable(
+      JsonParser parser, DeserializationContext context) {
+    return new SubtableFieldValue(readList(parser, TableRow.class));
+  }
+
+  private TableRow deserializeTableRow(JsonParser parser, DeserializationContext context) {
+    ObjectCodec codec = parser.getCodec();
+    JsonNode node = readTree(codec, parser);
+    TableRow row = new TableRow(get(node, "id", JsonNode::asLong));
+    stream(node.get("value").fields())
+        .forEach(entry -> row.putField(entry.getKey(), readValue(entry.getValue(), codec)));
+    return row;
+  }
+
+  private SingleLineTextFieldValue deserializeSingleLineText(
+      JsonParser parser, DeserializationContext context) {
+    return new SingleLineTextFieldValue(readText(parser));
+  }
+
+  private MultiLineTextFieldValue deserializeMultiLineText(
+      JsonParser parser, DeserializationContext context) {
+    return new MultiLineTextFieldValue(readText(parser));
+  }
+
+  private RichTextFieldValue deserializeRichText(
+      JsonParser parser, DeserializationContext context) {
+    return new RichTextFieldValue(readText(parser));
+  }
+
+  private NumberFieldValue deserializeNumber(JsonParser parser, DeserializationContext context) {
+    return new NumberFieldValue(read(parser, BigDecimal::new));
+  }
+
+  private CalcFieldValue deserializeCalc(JsonParser parser, DeserializationContext context) {
+    LOGGER.warn(
+        String.format(
+            "Ignore field: %s", new CalcFieldValue((BigDecimal) read(parser, BigDecimal::new))));
+    return null;
+  }
+
+  private CheckBoxFieldValue deserializeCheckBox(
+      JsonParser parser, DeserializationContext context) {
+    return new CheckBoxFieldValue(readList(parser, String.class));
+  }
+
+  private RadioButtonFieldValue deserializeRadioButton(
+      JsonParser parser, DeserializationContext context) {
+    return new RadioButtonFieldValue(readText(parser));
+  }
+
+  private MultiSelectFieldValue deserializeMultiSelect(
+      JsonParser parser, DeserializationContext context) {
+    return new MultiSelectFieldValue(readList(parser, String.class));
+  }
+
+  private DropDownFieldValue deserializeDropDown(
+      JsonParser parser, DeserializationContext context) {
+    return new DropDownFieldValue(readText(parser));
+  }
+
+  private UserSelectFieldValue deserializeUserSelect(
+      JsonParser parser, DeserializationContext context) {
+    return new UserSelectFieldValue(readList(parser, User.class));
+  }
+
+  private User deserializeUser(JsonParser parser, DeserializationContext context) {
+    JsonNode node = readTree(parser);
+    return new User(get(node, "name", JsonNode::asText), get(node, "code", JsonNode::asText));
+  }
+
+  private OrganizationSelectFieldValue deserializeOrganizationSelect(
+      JsonParser parser, DeserializationContext context) {
+    return new OrganizationSelectFieldValue(readList(parser, Organization.class));
+  }
+
+  private Organization deserializeOrganization(JsonParser parser, DeserializationContext context) {
+    JsonNode node = readTree(parser);
+    return new Organization(
+        get(node, "name", JsonNode::asText), get(node, "code", JsonNode::asText));
+  }
+
+  private GroupSelectFieldValue deserializeGroupSelect(
+      JsonParser parser, DeserializationContext context) {
+    return new GroupSelectFieldValue(readList(parser, Group.class));
+  }
+
+  private Group deserializeGroup(JsonParser parser, DeserializationContext context) {
+    JsonNode node = readTree(parser);
+    return new Group(get(node, "name", JsonNode::asText), get(node, "code", JsonNode::asText));
+  }
+
+  private DateFieldValue deserializeDate(JsonParser parser, DeserializationContext context) {
+    return new DateFieldValue(read(parser, LocalDate::parse));
+  }
+
+  private TimeFieldValue deserializeTime(JsonParser parser, DeserializationContext context) {
+    return new TimeFieldValue(read(parser, LocalTime::parse));
+  }
+
+  private DateTimeFieldValue deserializeDateTime(
+      JsonParser parser, DeserializationContext context) {
+    return new DateTimeFieldValue(read(parser, ZonedDateTime::parse));
+  }
+
+  private LinkFieldValue deserializeLink(JsonParser parser, DeserializationContext context) {
+    return new LinkFieldValue(readText(parser));
+  }
+
+  private FileFieldValue deserializeFile(JsonParser parser, DeserializationContext context) {
+    return new FileFieldValue(readList(parser, FileBody.class));
+  }
+
+  private FileBody deserializeFileBody(JsonParser parser, DeserializationContext context) {
+    JsonNode node = readTree(parser);
+    return new FileBody()
+        .setContentType(get(node, "contentType", JsonNode::asText))
+        .setFileKey(get(node, "fileKey", JsonNode::asText))
+        .setName(get(node, "name", JsonNode::asText))
+        .setSize(get(node, "size", JsonNode::asInt));
+  }
+
+  private static <T> T get(JsonNode node, String name, Function<JsonNode, T> as) {
+    return node.has(name) ? as.apply(node.get(name)) : null;
+  }
+
+  private static FieldValue readValue(JsonNode node, ObjectCodec codec) {
+    return node == null
+        ? null
+        : readValueAs(
+            node.has("value") ? node.get("value") : NULL,
+            codec,
+            FieldType.valueOf(node.get("type").asText()).getFieldValueClass());
+  }
+
+  private static <T> List<T> readList(JsonParser parser, Class<T> type) {
+    ObjectCodec codec = parser.getCodec();
+    return stream(readTree(codec, parser).elements())
+        .map(node -> readValueAs(node, codec, type))
+        .collect(Collectors.toList());
+  }
+
+  private static <T> Stream<T> stream(Iterator<T> iterator) {
+    return StreamSupport.stream(
+        Spliterators.spliteratorUnknownSize(iterator, Spliterator.ORDERED), false);
+  }
+
+  private static <T> T read(JsonParser parser, Function<String, T> as) {
+    String text = readText(parser);
+    return text == null || text.isEmpty() ? null : as.apply(text);
+  }
+
+  private static String readText(JsonParser parser) {
+    JsonNode node = readTree(parser);
+    return node == null || node.isNull() ? null : node.asText();
+  }
+
+  private static JsonNode readTree(JsonParser parser) {
+    return readTree(parser.getCodec(), parser);
+  }
+
+  private static JsonNode readTree(ObjectCodec codec, JsonParser parser) {
+    try {
+      return codec.readTree(parser);
+    } catch (IOException e) {
+      throw new DeserializeException(e);
+    }
+  }
+
+  private static <T> T readValueAs(JsonNode node, ObjectCodec codec, Class<T> type) {
+    try (JsonParser parser = node.traverse(codec)) {
+      return parser.readValueAs(type);
+    } catch (IOException e) {
+      throw new DeserializeException(e);
+    }
+  }
+}

data/src/main/java/org/embulk/output/kintone/reducer/CSVInputColumnVisitor.java

@@ -0,0 +1,78 @@
+package org.embulk.output.kintone.reducer;
+
+import java.time.Instant;
+import java.util.List;
+import org.embulk.spi.Column;
+import org.embulk.spi.ColumnVisitor;
+import org.embulk.spi.PageBuilder;
+import org.embulk.spi.time.Timestamp;
+
+public class CSVInputColumnVisitor implements ColumnVisitor {
+  private final PageBuilder builder;
+  private final List<String> values;
+
+  public CSVInputColumnVisitor(PageBuilder builder, List<String> values) {
+    this.builder = builder;
+    this.values = values;
+  }
+
+  @Override
+  public void booleanColumn(Column column) {
+    String value = values.get(column.getIndex());
+    if (value == null) {
+      builder.setNull(column);
+    } else {
+      builder.setBoolean(column, Boolean.parseBoolean(value));
+    }
+  }
+
+  @Override
+  public void longColumn(Column column) {
+    String value = values.get(column.getIndex());
+    if (value == null) {
+      builder.setNull(column);
+    } else {
+      builder.setLong(column, Long.parseLong(value));
+    }
+  }
+
+  @Override
+  public void doubleColumn(Column column) {
+    String value = values.get(column.getIndex());
+    if (value == null) {
+      builder.setNull(column);
+    } else {
+      builder.setDouble(column, Double.parseDouble(value));
+    }
+  }
+
+  @Override
+  public void stringColumn(Column column) {
+    String value = values.get(column.getIndex());
+    if (value == null) {
+      builder.setNull(column);
+    } else {
+      builder.setString(column, value);
+    }
+  }
+
+  @Override
+  public void timestampColumn(Column column) {
+    String value = values.get(column.getIndex());
+    if (value == null) {
+      builder.setNull(column);
+    } else {
+      builder.setTimestamp(column, Timestamp.ofInstant(Instant.parse(value)));
+    }
+  }
+
+  @Override
+  public void jsonColumn(Column column) {
+    String value = values.get(column.getIndex());
+    if (value == null) {
+      builder.setNull(column);
+    } else {
+      builder.setJson(column, Reducer.PARSER.parse(value));
+    }
+  }
+}

data/src/main/java/org/embulk/output/kintone/reducer/CSVOutputColumnVisitor.java

@@ -0,0 +1,79 @@
+package org.embulk.output.kintone.reducer;
+
+import java.io.IOException;
+import org.apache.commons.csv.CSVPrinter;
+import org.embulk.spi.Column;
+import org.embulk.spi.ColumnVisitor;
+import org.embulk.spi.PageReader;
+
+public class CSVOutputColumnVisitor implements ColumnVisitor {
+  private final PageReader reader;
+  private final CSVPrinter printer;
+
+  public CSVOutputColumnVisitor(PageReader reader, CSVPrinter printer) {
+    this.reader = reader;
+    this.printer = printer;
+  }
+
+  @Override
+  public void booleanColumn(Column column) {
+    if (reader.isNull(column)) {
+      print(null);
+    } else {
+      print(reader.getBoolean(column));
+    }
+  }
+
+  @Override
+  public void longColumn(Column column) {
+    if (reader.isNull(column)) {
+      print(null);
+    } else {
+      print(reader.getLong(column));
+    }
+  }
+
+  @Override
+  public void doubleColumn(Column column) {
+    if (reader.isNull(column)) {
+      print(null);
+    } else {
+      print(reader.getDouble(column));
+    }
+  }
+
+  @Override
+  public void stringColumn(Column column) {
+    if (reader.isNull(column)) {
+      print(null);
+    } else {
+      print(reader.getString(column));
+    }
+  }
+
+  @Override
+  public void timestampColumn(Column column) {
+    if (reader.isNull(column)) {
+      print(null);
+    } else {
+      print(reader.getTimestamp(column).getInstant());
+    }
+  }
+
+  @Override
+  public void jsonColumn(Column column) {
+    if (reader.isNull(column)) {
+      print(null);
+    } else {
+      print(reader.getJson(column).toJson());
+    }
+  }
+
+  private void print(Object value) {
+    try {
+      printer.print(value);
+    } catch (IOException e) {
+      throw new ReduceException(e);
+    }
+  }
+}

data/src/main/java/org/embulk/output/kintone/reducer/ReduceException.java

@@ -0,0 +1,11 @@
+package org.embulk.output.kintone.reducer;
+
+public class ReduceException extends RuntimeException {
+  public ReduceException(String message) {
+    super(message);
+  }
+
+  public ReduceException(Throwable cause) {
+    super(cause);
+  }
+}

data/src/main/java/org/embulk/output/kintone/reducer/ReduceType.java

@@ -0,0 +1,190 @@
+package org.embulk.output.kintone.reducer;
+
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Supplier;
+import org.embulk.output.kintone.KintoneColumnOption;
+import org.embulk.output.kintone.KintoneColumnType;
+import org.embulk.output.kintone.KintoneSortColumn;
+import org.embulk.spi.Column;
+import org.embulk.spi.time.Timestamp;
+import org.msgpack.value.ArrayValue;
+import org.msgpack.value.MapValue;
+import org.msgpack.value.Value;
+import org.msgpack.value.ValueFactory;
+
+public enum ReduceType {
+  BOOLEAN {
+    @Override
+    public MapValue value(String value, KintoneColumnOption option) {
+      KintoneColumnType type = KintoneColumnType.getType(option, KintoneColumnType.NUMBER);
+      Supplier<Value> supplier =
+          () -> type.asValue(type.getFieldValue(Boolean.parseBoolean(value), option));
+      return value(type, value, supplier);
+    }
+
+    @Override
+    public Comparator<String> comparator(KintoneSortColumn.Order order) {
+      return Comparator.comparing(Boolean::parseBoolean, order(order));
+    }
+  },
+  LONG {
+    @Override
+    public MapValue value(String value, KintoneColumnOption option) {
+      KintoneColumnType type = KintoneColumnType.getType(option, KintoneColumnType.NUMBER);
+      Supplier<Value> supplier =
+          () -> type.asValue(type.getFieldValue(Long.parseLong(value), option));
+      return value(type, value, supplier);
+    }
+
+    @Override
+    public Comparator<String> comparator(KintoneSortColumn.Order order) {
+      return Comparator.comparing(Long::parseLong, order(order));
+    }
+  },
+  DOUBLE {
+    @Override
+    public MapValue value(String value, KintoneColumnOption option) {
+      KintoneColumnType type = KintoneColumnType.getType(option, KintoneColumnType.NUMBER);
+      Supplier<Value> supplier =
+          () -> type.asValue(type.getFieldValue(Double.parseDouble(value), option));
+      return value(type, value, supplier);
+    }
+
+    @Override
+    public Comparator<String> comparator(KintoneSortColumn.Order order) {
+      return Comparator.comparing(Double::parseDouble, order(order));
+    }
+  },
+  STRING {
+    @Override
+    public MapValue value(String value, KintoneColumnOption option) {
+      KintoneColumnType type = KintoneColumnType.getType(option, KintoneColumnType.MULTI_LINE_TEXT);
+      Supplier<Value> supplier = () -> type.asValue(type.getFieldValue(value, option));
+      return value(type, value, supplier);
+    }
+
+    @Override
+    public Comparator<String> comparator(KintoneSortColumn.Order order) {
+      return order(order);
+    }
+  },
+  TIMESTAMP {
+    @Override
+    public MapValue value(String value, KintoneColumnOption option) {
+      KintoneColumnType type = KintoneColumnType.getType(option, KintoneColumnType.DATETIME);
+      Supplier<Value> supplier =
+          () -> type.asValue(type.getFieldValue(Timestamp.ofInstant(Instant.parse(value)), option));
+      return value(type, value, supplier);
+    }
+
+    @Override
+    public Comparator<String> comparator(KintoneSortColumn.Order order) {
+      return Comparator.comparing(Instant::parse, order(order));
+    }
+  },
+  JSON {
+    @Override
+    public MapValue value(String value, KintoneColumnOption option) {
+      KintoneColumnType type = KintoneColumnType.getType(option, KintoneColumnType.MULTI_LINE_TEXT);
+      Supplier<Value> supplier =
+          () -> type.asValue(type.getFieldValue(Reducer.PARSER.parse(value), option));
+      return value(type, value, supplier);
+    }
+
+    @Override
+    public Comparator<String> comparator(KintoneSortColumn.Order order) {
+      return order(order);
+    }
+  };
+  private static final Value NIL = ValueFactory.newNil();
+  private static final Value ID = ValueFactory.newString("id");
+  private static final Value TYPE = ValueFactory.newString("type");
+  private static final Value VALUE = ValueFactory.newString("value");
+  private static final Value KEY_SET = ValueFactory.newString("$$key_set");
+  private static final Value SORT_VALUE = ValueFactory.newString("$$sort_value");
+
+  public abstract MapValue value(String value, KintoneColumnOption option);
+
+  public abstract Comparator<String> comparator(KintoneSortColumn.Order order);
+
+  public static Comparator<String> comparator(Column column, KintoneSortColumn.Order order) {
+    return valueOf(column).comparator(order);
+  }
+
+  public static String asString(Value value, KintoneSortColumn sortColumn) {
+    return asString(sortValue(value).map().get(value(sortColumn.getName())));
+  }
+
+  public static boolean isEmpty(MapValue value) {
+    return value.values().stream()
+        .map(Value::asMapValue)
+        .map(MapValue::map)
+        .map(map -> map.get(VALUE))
+        .allMatch(Value::isNilValue);
+  }
+
+  public static Value value(String value) {
+    return value == null ? NIL : ValueFactory.newString(value);
+  }
+
+  public static MapValue value(Value value) {
+    ValueFactory.MapBuilder builder = ValueFactory.newMapBuilder();
+    Map<Value, Value> map = value.asMapValue().map();
+    builder.put(ID, map.get(ID));
+    builder.put(VALUE, value(map.get(VALUE).asMapValue().map(), map.get(KEY_SET).asArrayValue()));
+    return builder.build();
+  }
+
+  public static MapValue value(Long id, MapValue value, MapValue sortValue) {
+    ValueFactory.MapBuilder builder = ValueFactory.newMapBuilder();
+    builder.put(ID, id == null ? NIL : ValueFactory.newString(id.toString()));
+    builder.put(VALUE, value == null ? ValueFactory.emptyMap() : value);
+    builder.put(KEY_SET, value == null ? ValueFactory.emptyArray() : keySet(value));
+    builder.put(SORT_VALUE, sortValue == null ? ValueFactory.emptyMap() : sortValue);
+    return builder.build();
+  }
+
+  public static MapValue value(Column column, List<String> values, KintoneColumnOption option) {
+    return valueOf(column).value(values.get(column.getIndex()), option);
+  }
+
+  protected static MapValue value(KintoneColumnType type, String value, Supplier<Value> supplier) {
+    ValueFactory.MapBuilder builder = ValueFactory.newMapBuilder();
+    builder.put(TYPE, value(type.name()));
+    builder.put(VALUE, value == null ? NIL : supplier.get());
+    return builder.build();
+  }
+
+  private static <T extends Comparable<? super T>> Comparator<T> order(
+      KintoneSortColumn.Order order) {
+    return order == KintoneSortColumn.Order.DESC
+        ? Comparator.reverseOrder()
+        : Comparator.naturalOrder();
+  }
+
+  private static String asString(Value value) {
+    return value.isNilValue() ? null : value.asStringValue().asString();
+  }
+
+  private static MapValue sortValue(Value value) {
+    return value.asMapValue().map().get(SORT_VALUE).asMapValue();
+  }
+
+  private static MapValue value(Map<Value, Value> map, ArrayValue keySet) {
+    ValueFactory.MapBuilder builder = ValueFactory.newMapBuilder();
+    keySet.forEach(key -> builder.put(key, map.get(key)));
+    return builder.build();
+  }
+
+  private static ArrayValue keySet(MapValue value) {
+    return ValueFactory.newArray(new ArrayList<>(value.asMapValue().keySet()));
+  }
+
+  private static ReduceType valueOf(Column column) {
+    return valueOf(column.getType().getName().toUpperCase());
+  }
+}