embulk-output-kintone 0.4.1 → 1.1.0

data/src/test/java/org/embulk/output/kintone/KintoneColumnVisitorVerifier.java

@@ -4,7 +4,9 @@ import com.kintone.client.model.record.FieldType;
 import com.kintone.client.model.record.Record;
 import com.kintone.client.model.record.UpdateKey;
 import java.util.Map;
+import java.util.Set;
 import java.util.function.BiConsumer;
+import javax.validation.UnexpectedTypeException;
 import org.embulk.spi.Column;
 import org.embulk.spi.Page;
 import org.embulk.spi.PageReader;
@@ -12,24 +14,48 @@ import org.embulk.spi.Schema;
 
 public class KintoneColumnVisitorVerifier {
   private final Schema schema;
-  private final Map<String, KintoneColumnOption> columnOptions;
-  private final PageReader pageReader;
+  private final Map<String, KintoneColumnOption> options;
+  private final PageReader reader;
   private final KintoneColumnVisitor visitor;
 
   public KintoneColumnVisitorVerifier(
       Schema schema,
-      Map<String, KintoneColumnOption> columnOptions,
+      Set<Column> derived,
+      Map<String, KintoneColumnOption> options,
+      String reduceKeyName,
+      String updateKeyName,
+      Page page) {
+    this(schema, derived, options, false, false, reduceKeyName, updateKeyName, page);
+  }
+
+  public KintoneColumnVisitorVerifier(
+      Schema schema,
+      Set<Column> derived,
+      Map<String, KintoneColumnOption> options,
+      boolean preferNulls,
+      boolean ignoreNulls,
+      String reduceKeyName,
       String updateKeyName,
       Page page) {
     this.schema = schema;
-    this.columnOptions = columnOptions;
-    pageReader = new PageReader(schema);
-    pageReader.setPage(page);
-    visitor = new KintoneColumnVisitor(pageReader, columnOptions, updateKeyName);
+    this.options = options;
+    reader = new PageReader(schema);
+    reader.setPage(page);
+    visitor =
+        new KintoneColumnVisitor(
+            reader, derived, options, preferNulls, ignoreNulls, reduceKeyName, updateKeyName);
+  }
+
+  public void verify() {
+    verify((record, updateKey) -> {});
   }
 
   public void verify(BiConsumer<Record, UpdateKey> consumer) {
-    if (!pageReader.nextRecord()) {
+    verify(consumer, false);
+  }
+
+  public void verify(BiConsumer<Record, UpdateKey> consumer, boolean nullable) {
+    if (!reader.nextRecord()) {
       throw new IllegalStateException();
     }
     Record record = new Record();
@@ -37,16 +63,21 @@ public class KintoneColumnVisitorVerifier {
     UpdateKey updateKey = new UpdateKey();
     visitor.setUpdateKey(updateKey);
     schema.visitColumns(visitor);
-    schema.getColumns().forEach(column -> verify(record, column));
+    schema.getColumns().forEach(column -> verify(record, column, nullable));
     consumer.accept(record, updateKey);
   }
 
-  private void verify(Record record, Column column) {
-    FieldType expected = FieldType.valueOf(columnOptions.get(column.getName()).getType());
+  private void verify(Record record, Column column, boolean nullable) {
+    FieldType expected = FieldType.valueOf(options.get(column.getName()).getType());
     FieldType actual = record.getFieldType(column.getName());
-    if (!expected.equals(actual)) {
-      System.out.printf(
-          "%s: Expected type is %s, but actual type is %s%n", column.getName(), expected, actual);
+    if (actual == null && nullable) {
+      return;
+    }
+    if (expected != actual) {
+      throw new UnexpectedTypeException(
+          String.format(
+              "%s: Expected type is %s, but actual type is %s%n",
+              column.getName(), expected, actual));
     }
   }
 }

data/src/test/java/org/embulk/output/kintone/KintonePageOutputVerifier.java

@@ -1,10 +1,12 @@
 package org.embulk.output.kintone;
 
+import static org.embulk.output.kintone.deserializer.DeserializerTest.assertTableRows;
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.Matchers.is;
 import static org.mockito.ArgumentMatchers.anyList;
 import static org.mockito.ArgumentMatchers.eq;
 import static org.mockito.ArgumentMatchers.matches;
+import static org.mockito.Mockito.CALLS_REAL_METHODS;
 import static org.mockito.Mockito.atLeast;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.mockStatic;
@@ -19,16 +21,15 @@ import com.kintone.client.api.record.GetRecordsByCursorResponseBody;
 import com.kintone.client.model.app.field.FieldProperty;
 import com.kintone.client.model.app.field.NumberFieldProperty;
 import com.kintone.client.model.app.field.SingleLineTextFieldProperty;
-import com.kintone.client.model.record.DateTimeFieldValue;
 import com.kintone.client.model.record.FieldType;
 import com.kintone.client.model.record.FieldValue;
 import com.kintone.client.model.record.NumberFieldValue;
 import com.kintone.client.model.record.Record;
 import com.kintone.client.model.record.RecordForUpdate;
 import com.kintone.client.model.record.SingleLineTextFieldValue;
+import com.kintone.client.model.record.SubtableFieldValue;
 import com.kintone.client.model.record.UpdateKey;
 import java.math.BigDecimal;
-import java.time.ZoneId;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
@@ -36,6 +37,9 @@ import java.util.Map;
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 import org.embulk.config.TaskReport;
+import org.embulk.deps.buffer.PooledBufferAllocator;
+import org.embulk.spi.BufferAllocator;
+import org.embulk.spi.Exec;
 import org.embulk.spi.Page;
 import org.embulk.spi.TransactionalPageOutput;
 import org.mockito.ArgumentCaptor;
@@ -49,6 +53,15 @@ public class KintonePageOutputVerifier implements TransactionalPageOutput {
   private final List<Record> addRecords;
   private final List<RecordForUpdate> updateRecords;
 
+  public KintonePageOutputVerifier(
+      String domain,
+      String field,
+      List<String> values,
+      List<Record> addRecords,
+      List<RecordForUpdate> updateRecords) {
+    this(null, domain, field, values, addRecords, updateRecords);
+  }
+
   public KintonePageOutputVerifier(
       TransactionalPageOutput transactionalPageOutput,
       String domain,
@@ -76,6 +89,9 @@ public class KintonePageOutputVerifier implements TransactionalPageOutput {
 
   @Override
   public void close() {
+    if (transactionalPageOutput == null) {
+      return;
+    }
     transactionalPageOutput.close();
   }
 
@@ -91,12 +107,20 @@ public class KintonePageOutputVerifier implements TransactionalPageOutput {
 
   public void runWithMock(Runnable runnable) {
     try {
-      runWithMockClient(runnable);
+      runWithMockExec(runnable);
     } catch (Exception e) {
       throw new RuntimeException(e);
     }
   }
 
+  private void runWithMockExec(Runnable runnable) throws Exception {
+    BufferAllocator bufferAllocator = PooledBufferAllocator.create(1024 * 1024);
+    try (MockedStatic<Exec> mocked = mockStatic(Exec.class, CALLS_REAL_METHODS)) {
+      mocked.when(Exec::getBufferAllocator).thenReturn(bufferAllocator);
+      runWithMockClient(runnable);
+    }
+  }
+
   private void runWithMockClient(Runnable runnable) throws Exception {
     @SuppressWarnings("unchecked")
     Map<String, FieldProperty> mockFormFields = mock(Map.class);
@@ -184,15 +208,29 @@ public class KintonePageOutputVerifier implements TransactionalPageOutput {
     assertThat(reason, actual.getFieldCodes(true), is(expected.getFieldCodes(true)));
     // spotless:off
     actual.getFieldCodes(true).forEach(fieldCode -> assertFieldValue(domain, index, fieldCode, actual.getFieldValue(fieldCode), expected.getFieldValue(fieldCode)));
+    actual.getFieldCodes(true).forEach(fieldCode -> assertSubtableFieldValue(domain, index, fieldCode, actual.getFieldValue(fieldCode), expected.getFieldValue(fieldCode)));
     // spotless:on
   }
 
   private static void assertFieldValue(
       String domain, int index, String fieldCode, FieldValue actual, FieldValue expected) {
+    if (actual.getType() == FieldType.SUBTABLE) {
+      return;
+    }
+    String reason = String.format("%s:%d:%s", domain, index, fieldCode);
+    assertThat(reason, actual.getType(), is(expected.getType()));
+    assertThat(reason, actual, is(expected));
+  }
+
+  private static void assertSubtableFieldValue(
+      String domain, int index, String fieldCode, FieldValue actual, FieldValue expected) {
+    if (actual.getType() != FieldType.SUBTABLE) {
+      return;
+    }
     String reason = String.format("%s:%d:%s", domain, index, fieldCode);
     assertThat(reason, actual.getType(), is(expected.getType()));
     // spotless:off
-    assertThat(reason, actual, is(expected.getType().equals(FieldType.DATETIME) ? new DateTimeFieldValue(((DateTimeFieldValue) expected).getValue().withZoneSameInstant(ZoneId.of("UTC"))) : expected));
+    assertTableRows(reason, ((SubtableFieldValue) actual).getRows(), ((SubtableFieldValue) expected).getRows());
     // spotless:on
   }
 
@@ -217,7 +255,7 @@ public class KintonePageOutputVerifier implements TransactionalPageOutput {
       String domain, int index, UpdateKey actual, UpdateKey expected) {
     String reason = String.format("%s:%d", domain, index);
     assertThat(reason, actual.getField(), is(expected.getField()));
-    assertThat(reason, actual.getValue(), is(expected.getValue().toString()));
+    assertThat(reason, actual.getValue(), is(expected.getValue()));
   }
 
   public interface Runnable {

data/src/test/java/org/embulk/output/kintone/TestKintoneOutputPlugin.java

@@ -1,5 +1,10 @@
 package org.embulk.output.kintone;
 
+import static org.mockito.Mockito.inOrder;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
 import com.google.common.io.Resources;
 import com.kintone.client.Json;
 import com.kintone.client.model.record.Record;
@@ -14,10 +19,15 @@ import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Function;
 import java.util.stream.Collectors;
+import org.embulk.config.ConfigDiff;
 import org.embulk.config.ConfigSource;
 import org.embulk.config.TaskSource;
+import org.embulk.spi.Column;
+import org.embulk.spi.Exec;
 import org.embulk.spi.OutputPlugin;
 import org.embulk.spi.Schema;
 import org.embulk.spi.TransactionalPageOutput;
@@ -25,6 +35,7 @@ import org.embulk.spi.json.JsonParser;
 import org.embulk.test.EmbulkTests;
 import org.embulk.test.TestingEmbulk;
 import org.junit.Rule;
+import org.mockito.InOrder;
 import org.msgpack.value.Value;
 
 public class TestKintoneOutputPlugin extends KintoneOutputPlugin {
@@ -36,18 +47,19 @@ public class TestKintoneOutputPlugin extends KintoneOutputPlugin {
           .registerPlugin(OutputPlugin.class, "kintone", TestKintoneOutputPlugin.class)
           .build();
 
+  @Override
+  public ConfigDiff transaction(
+      ConfigSource config, Schema schema, int taskCount, Control control) {
+    return config.get(String.class, "reduce_key", null) == null
+        ? super.transaction(config, schema, taskCount, control)
+        : transactionWithVerifier(config, schema, taskCount, control);
+  }
+
   @Override
   public TransactionalPageOutput open(TaskSource taskSource, Schema schema, int taskIndex) {
-    String test = taskSource.get(String.class, "Domain");
-    String mode = taskSource.get(String.class, "Mode");
-    String field = taskSource.get(String.class, "UpdateKeyName");
-    return new KintonePageOutputVerifier(
-        super.open(taskSource, schema, taskIndex),
-        test,
-        field,
-        getValues(test),
-        getAddRecords(test, mode),
-        getUpdateRecords(test, mode, field));
+    return taskSource.get(String.class, "ReduceKeyName") == null
+        ? openWithVerifier(taskSource, schema, taskIndex)
+        : super.open(taskSource, schema, taskIndex);
   }
 
   protected void runOutput(String configName, String inputName) throws Exception {
@@ -89,8 +101,78 @@ public class TestKintoneOutputPlugin extends KintoneOutputPlugin {
     return embulk.configLoader().fromYamlString(string);
   }
 
-  private static List<String> getValues(String test) {
-    String name = String.format("%s/values.json", test);
+  private ConfigDiff transactionWithVerifier(
+      ConfigSource config, Schema schema, int taskCount, Control control) {
+    try (KintonePageOutputVerifier verifier = verifier(config)) {
+      return runWithMock(verifier, config, schema, taskCount, control);
+    }
+  }
+
+  private ConfigDiff runWithMock(
+      KintonePageOutputVerifier verifier,
+      ConfigSource config,
+      Schema schema,
+      int taskCount,
+      OutputPlugin.Control control) {
+    String test = config.get(String.class, "domain");
+    PluginTask spyTask = spy(config.loadConfig(PluginTask.class));
+    ConfigSource spyConfig = spy(config);
+    when(spyConfig.loadConfig(PluginTask.class)).thenReturn(spyTask);
+    AtomicReference<ConfigDiff> configDiff = new AtomicReference<>();
+    verifier.runWithMock(
+        () -> configDiff.set(super.transaction(spyConfig, schema, taskCount, control)));
+    verify(spyConfig).loadConfig(PluginTask.class);
+    InOrder inOrderTask = inOrder(spyTask);
+    inOrderTask.verify(spyTask).setDerivedColumns(Collections.emptySet());
+    inOrderTask.verify(spyTask).setDerivedColumns(getDerivedColumns(test));
+    return configDiff.get();
+  }
+
+  private KintonePageOutputVerifier verifier(ConfigSource config) {
+    String test = config.get(String.class, "domain");
+    String mode = config.get(String.class, "mode");
+    String field = config.get(String.class, "update_key", null);
+    boolean preferNulls = config.get(boolean.class, "prefer_nulls", false);
+    boolean ignoreNulls = config.get(boolean.class, "ignore_nulls", false);
+    return new KintonePageOutputVerifier(
+        test,
+        field,
+        getValues(test, preferNulls, ignoreNulls),
+        getAddRecords(test, mode, preferNulls, ignoreNulls),
+        getUpdateRecords(test, mode, preferNulls, ignoreNulls, field));
+  }
+
+  private TransactionalPageOutput openWithVerifier(
+      TaskSource taskSource, Schema schema, int taskIndex) {
+    String test = taskSource.get(String.class, "Domain");
+    String mode = taskSource.get(String.class, "Mode");
+    String field = taskSource.get(String.class, "UpdateKeyName");
+    boolean preferNulls = taskSource.get(boolean.class, "PreferNulls");
+    boolean ignoreNulls = taskSource.get(boolean.class, "IgnoreNulls");
+    return new KintonePageOutputVerifier(
+        super.open(taskSource, schema, taskIndex),
+        test,
+        field,
+        getValues(test, preferNulls, ignoreNulls),
+        getAddRecords(test, mode, preferNulls, ignoreNulls),
+        getUpdateRecords(test, mode, preferNulls, ignoreNulls, field));
+  }
+
+  private static Set<Column> getDerivedColumns(String test) {
+    String name = String.format("%s/derived_columns.json", test);
+    String json = existsResource(name) ? readResource(name) : null;
+    return json == null || json.isEmpty()
+        ? Collections.emptySet()
+        : PARSER.parse(json).asArrayValue().list().stream()
+            .map(value -> Exec.getModelManager().readObject(Column.class, value.toJson()))
+            .collect(Collectors.toSet());
+  }
+
+  private static List<String> getValues(String test, boolean preferNulls, boolean ignoreNulls) {
+    String name =
+        String.format(
+            "%s/values%s.json",
+            test, ignoreNulls ? "_ignore_nulls" : preferNulls ? "_prefer_nulls" : "");
     String json = existsResource(name) ? readResource(name) : null;
     return json == null || json.isEmpty()
         ? Collections.emptyList()
@@ -99,8 +181,12 @@ public class TestKintoneOutputPlugin extends KintoneOutputPlugin {
             .collect(Collectors.toList());
   }
 
-  private static List<Record> getAddRecords(String test, String mode) {
-    String name = String.format("%s/%s_add_records.jsonl", test, mode);
+  private static List<Record> getAddRecords(
+      String test, String mode, boolean preferNulls, boolean ignoreNulls) {
+    String name =
+        String.format(
+            "%s/%s_add%s_records.jsonl",
+            test, mode, ignoreNulls ? "_ignore_nulls" : preferNulls ? "_prefer_nulls" : "");
     String jsonl = existsResource(name) ? readResource(name) : null;
     return jsonl == null || jsonl.isEmpty()
         ? Collections.emptyList()
@@ -109,9 +195,13 @@ public class TestKintoneOutputPlugin extends KintoneOutputPlugin {
             .collect(Collectors.toList());
   }
 
-  private static List<RecordForUpdate> getUpdateRecords(String test, String mode, String field) {
+  private static List<RecordForUpdate> getUpdateRecords(
+      String test, String mode, boolean preferNulls, boolean ignoreNulls, String field) {
     Function<Record, UpdateKey> key = getKey(field);
-    String name = String.format("%s/%s_update_records.jsonl", test, mode);
+    String name =
+        String.format(
+            "%s/%s_update%s_records.jsonl",
+            test, mode, ignoreNulls ? "_ignore_nulls" : preferNulls ? "_prefer_nulls" : "");
     String jsonl = existsResource(name) ? readResource(name) : null;
     return jsonl == null || jsonl.isEmpty()
         ? Collections.emptyList()

data/src/test/java/org/embulk/output/kintone/TestTaskMode.java

@@ -15,17 +15,29 @@ public class TestTaskMode extends TestTask {
   public void testInsert() throws Exception {
     merge(config("mode: insert"));
     runOutput();
+    merge(config("prefer_nulls: true"));
+    runOutput();
+    merge(config("ignore_nulls: true"));
+    runOutput();
   }
 
   @Test
   public void testUpdate() throws Exception {
     merge(config("mode: update", "update_key: string_number"));
     runOutput();
+    merge(config("prefer_nulls: true"));
+    runOutput();
+    merge(config("ignore_nulls: true"));
+    runOutput();
   }
 
   @Test
   public void testUpsert() throws Exception {
     merge(config("mode: upsert", "update_key: double_single_line_text"));
     runOutput();
+    merge(config("prefer_nulls: true"));
+    runOutput();
+    merge(config("ignore_nulls: true"));
+    runOutput();
   }
 }

data/src/test/java/org/embulk/output/kintone/TestTaskReduce.java

@@ -0,0 +1,46 @@
+package org.embulk.output.kintone;
+
+import net.jcip.annotations.NotThreadSafe;
+import org.junit.Test;
+
+@NotThreadSafe
+public class TestTaskReduce extends TestTask {
+  @Override
+  public void before() {
+    super.before();
+    merge(config("domain: task/reduce"));
+  }
+
+  @Test
+  public void testInsert() throws Exception {
+    merge(config("mode: insert"));
+    merge(config("reduce_key: timestamp"));
+    runOutput();
+    merge(config("prefer_nulls: true"));
+    runOutput();
+    merge(config("ignore_nulls: true"));
+    runOutput();
+  }
+
+  @Test
+  public void testUpdate() throws Exception {
+    merge(config("mode: update", "update_key: string_number"));
+    merge(config("reduce_key: double_single_line_text"));
+    runOutput();
+    merge(config("prefer_nulls: true"));
+    runOutput();
+    merge(config("ignore_nulls: true"));
+    runOutput();
+  }
+
+  @Test
+  public void testUpsert() throws Exception {
+    merge(config("mode: upsert", "update_key: double_single_line_text"));
+    merge(config("reduce_key: string_number"));
+    runOutput();
+    merge(config("prefer_nulls: true"));
+    runOutput();
+    merge(config("ignore_nulls: true"));
+    runOutput();
+  }
+}

data/src/test/java/org/embulk/output/kintone/TestTaskReduceException.java

@@ -0,0 +1,50 @@
+package org.embulk.output.kintone;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.is;
+import static org.junit.Assert.assertThrows;
+
+import net.jcip.annotations.NotThreadSafe;
+import org.embulk.exec.PartialExecutionException;
+import org.embulk.output.kintone.reducer.ReduceException;
+import org.junit.Test;
+
+@NotThreadSafe
+public class TestTaskReduceException extends TestTask {
+  @Override
+  public void before() {
+    super.before();
+    merge(config("domain: task/reduce_exception"));
+  }
+
+  @Test
+  public void test() throws Exception {
+    merge(config("mode: insert"));
+    merge(config("reduce_key: double_single_line_text"));
+    runOutput();
+    merge(config("mode: update", "update_key: double_single_line_text"));
+    merge(config("reduce_key: string_number"));
+    runOutput();
+    merge(config("mode: upsert", "update_key: string_number"));
+    merge(config("reduce_key: json.double_single_line_text"));
+    assertReduceException(
+        "Couldn't reduce because column json.double_single_line_text is not unique to [json, json_subtable]\n[double_single_line_text, string_number] expected [123.0, 456] but actual [456.0, 123]");
+    merge(config("reduce_key: json.string_number"));
+    assertReduceException(
+        "Couldn't reduce because column json.string_number is not unique to [json, json_subtable]\n[double_single_line_text, string_number] expected [123.0, 456] but actual [456.0, 123]");
+    merge(config("reduce_key: json_subtable.double_single_line_text"));
+    assertReduceException(
+        "Couldn't reduce because column json_subtable.double_single_line_text is not unique to [json, json_subtable]\n[double_single_line_text, string_number] expected [123.0, 456] but actual [456.0, 123]");
+    merge(config("reduce_key: json_subtable.string_number"));
+    assertReduceException(
+        "Couldn't reduce because column json_subtable.string_number is not unique to [json, json_subtable]\n[double_single_line_text, string_number] expected [123.0, 456] but actual [456.0, 123]");
+  }
+
+  private void assertReduceException(String message) {
+    Exception e = assertThrows(PartialExecutionException.class, this::runOutput);
+    assertThat(e.getCause(), is(instanceOf(RuntimeException.class)));
+    assertThat(e.getCause().getCause(), is(instanceOf(ReduceException.class)));
+    assertThat(e.getCause().getCause().getMessage(), is(message));
+  }
+}

data/src/test/java/org/embulk/output/kintone/TestTaskReduceSubtable.java

@@ -0,0 +1,46 @@
+package org.embulk.output.kintone;
+
+import net.jcip.annotations.NotThreadSafe;
+import org.junit.Test;
+
+@NotThreadSafe
+public class TestTaskReduceSubtable extends TestTask {
+  @Override
+  public void before() {
+    super.before();
+    merge(config("domain: task/reduce_subtable"));
+  }
+
+  @Test
+  public void testInsert() throws Exception {
+    merge(config("mode: insert"));
+    merge(config("reduce_key: timestamp"));
+    runOutput();
+    merge(config("prefer_nulls: true"));
+    runOutput();
+    merge(config("ignore_nulls: true"));
+    runOutput();
+  }
+
+  @Test
+  public void testUpdate() throws Exception {
+    merge(config("mode: update", "update_key: string_number"));
+    merge(config("reduce_key: string_number"));
+    runOutput();
+    merge(config("prefer_nulls: true"));
+    runOutput();
+    merge(config("ignore_nulls: true"));
+    runOutput();
+  }
+
+  @Test
+  public void testUpsert() throws Exception {
+    merge(config("mode: upsert", "update_key: double_single_line_text"));
+    merge(config("reduce_key: double_single_line_text"));
+    runOutput();
+    merge(config("prefer_nulls: true"));
+    runOutput();
+    merge(config("ignore_nulls: true"));
+    runOutput();
+  }
+}