embulk-output-kintone 1.0.0 → 1.1.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (90) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +57 -1
  3. data/build.gradle +1 -0
  4. data/classpath/commons-csv-1.9.0.jar +0 -0
  5. data/classpath/embulk-output-kintone-1.1.0.jar +0 -0
  6. data/classpath/externalsortinginjava-0.6.2.jar +0 -0
  7. data/classpath/{shadow-kintone-java-client-1.0.0-all.jar → shadow-kintone-java-client-1.1.0-all.jar} +0 -0
  8. data/src/main/java/org/embulk/output/kintone/KintoneColumnOption.java +5 -0
  9. data/src/main/java/org/embulk/output/kintone/KintoneColumnType.java +209 -5
  10. data/src/main/java/org/embulk/output/kintone/KintoneColumnVisitor.java +28 -9
  11. data/src/main/java/org/embulk/output/kintone/KintoneOutputPlugin.java +12 -3
  12. data/src/main/java/org/embulk/output/kintone/KintonePageOutput.java +20 -12
  13. data/src/main/java/org/embulk/output/kintone/KintoneSortColumn.java +33 -0
  14. data/src/main/java/org/embulk/output/kintone/PluginTask.java +27 -0
  15. data/src/main/java/org/embulk/output/kintone/deserializer/DeserializeApplier.java +19 -0
  16. data/src/main/java/org/embulk/output/kintone/deserializer/DeserializeException.java +7 -0
  17. data/src/main/java/org/embulk/output/kintone/deserializer/Deserializer.java +279 -0
  18. data/src/main/java/org/embulk/output/kintone/reducer/CSVInputColumnVisitor.java +78 -0
  19. data/src/main/java/org/embulk/output/kintone/reducer/CSVOutputColumnVisitor.java +79 -0
  20. data/src/main/java/org/embulk/output/kintone/reducer/ReduceException.java +11 -0
  21. data/src/main/java/org/embulk/output/kintone/reducer/ReduceType.java +190 -0
  22. data/src/main/java/org/embulk/output/kintone/reducer/ReducedPageOutput.java +100 -0
  23. data/src/main/java/org/embulk/output/kintone/reducer/Reducer.java +355 -0
  24. data/src/test/java/org/embulk/output/kintone/KintoneColumnOptionBuilder.java +7 -0
  25. data/src/test/java/org/embulk/output/kintone/KintoneColumnTypeTest.java +194 -0
  26. data/src/test/java/org/embulk/output/kintone/KintoneColumnVisitorTest.java +153 -34
  27. data/src/test/java/org/embulk/output/kintone/KintoneColumnVisitorVerifier.java +13 -3
  28. data/src/test/java/org/embulk/output/kintone/KintonePageOutputVerifier.java +44 -1
  29. data/src/test/java/org/embulk/output/kintone/TestKintoneOutputPlugin.java +89 -12
  30. data/src/test/java/org/embulk/output/kintone/TestTaskReduce.java +46 -0
  31. data/src/test/java/org/embulk/output/kintone/TestTaskReduceException.java +50 -0
  32. data/src/test/java/org/embulk/output/kintone/TestTaskReduceSubtable.java +46 -0
  33. data/src/test/java/org/embulk/output/kintone/deserializer/DeserializerTest.java +165 -0
  34. data/src/test/java/org/embulk/output/kintone/reducer/ReduceTypeTest.java +154 -0
  35. data/src/test/resources/org/embulk/output/kintone/task/config.yml +1 -1
  36. data/src/test/resources/org/embulk/output/kintone/task/mode/config.yml +6 -0
  37. data/src/test/resources/org/embulk/output/kintone/task/mode/input.csv +7 -7
  38. data/src/test/resources/org/embulk/output/kintone/task/mode/insert_add_ignore_nulls_records.jsonl +2 -2
  39. data/src/test/resources/org/embulk/output/kintone/task/mode/insert_add_prefer_nulls_records.jsonl +6 -6
  40. data/src/test/resources/org/embulk/output/kintone/task/mode/insert_add_records.jsonl +6 -6
  41. data/src/test/resources/org/embulk/output/kintone/task/mode/update_update_ignore_nulls_records.jsonl +2 -2
  42. data/src/test/resources/org/embulk/output/kintone/task/mode/update_update_prefer_nulls_records.jsonl +3 -3
  43. data/src/test/resources/org/embulk/output/kintone/task/mode/update_update_records.jsonl +6 -6
  44. data/src/test/resources/org/embulk/output/kintone/task/mode/upsert_add_prefer_nulls_records.jsonl +3 -3
  45. data/src/test/resources/org/embulk/output/kintone/task/mode/upsert_add_records.jsonl +2 -2
  46. data/src/test/resources/org/embulk/output/kintone/task/mode/upsert_update_ignore_nulls_records.jsonl +2 -2
  47. data/src/test/resources/org/embulk/output/kintone/task/mode/upsert_update_prefer_nulls_records.jsonl +3 -3
  48. data/src/test/resources/org/embulk/output/kintone/task/mode/upsert_update_records.jsonl +4 -4
  49. data/src/test/resources/org/embulk/output/kintone/task/reduce/config.yml +171 -0
  50. data/src/test/resources/org/embulk/output/kintone/task/reduce/input.csv +7 -0
  51. data/src/test/resources/org/embulk/output/kintone/task/reduce/insert_add_ignore_nulls_records.jsonl +6 -0
  52. data/src/test/resources/org/embulk/output/kintone/task/reduce/insert_add_prefer_nulls_records.jsonl +6 -0
  53. data/src/test/resources/org/embulk/output/kintone/task/reduce/insert_add_records.jsonl +6 -0
  54. data/src/test/resources/org/embulk/output/kintone/task/reduce/update_update_ignore_nulls_records.jsonl +3 -0
  55. data/src/test/resources/org/embulk/output/kintone/task/reduce/update_update_prefer_nulls_records.jsonl +3 -0
  56. data/src/test/resources/org/embulk/output/kintone/task/reduce/update_update_records.jsonl +6 -0
  57. data/src/test/resources/org/embulk/output/kintone/task/reduce/upsert_add_ignore_nulls_records.jsonl +3 -0
  58. data/src/test/resources/org/embulk/output/kintone/task/reduce/upsert_add_prefer_nulls_records.jsonl +3 -0
  59. data/src/test/resources/org/embulk/output/kintone/task/reduce/upsert_add_records.jsonl +2 -0
  60. data/src/test/resources/org/embulk/output/kintone/task/reduce/upsert_update_ignore_nulls_records.jsonl +3 -0
  61. data/src/test/resources/org/embulk/output/kintone/task/reduce/upsert_update_prefer_nulls_records.jsonl +3 -0
  62. data/src/test/resources/org/embulk/output/kintone/task/reduce/upsert_update_records.jsonl +4 -0
  63. data/src/test/resources/org/embulk/output/kintone/task/reduce/values.json +1 -0
  64. data/src/test/resources/org/embulk/output/kintone/task/reduce/values_ignore_nulls.json +1 -0
  65. data/src/test/resources/org/embulk/output/kintone/task/reduce/values_prefer_nulls.json +1 -0
  66. data/src/test/resources/org/embulk/output/kintone/task/reduce_exception/config.yml +36 -0
  67. data/src/test/resources/org/embulk/output/kintone/task/reduce_exception/derived_columns.json +1 -0
  68. data/src/test/resources/org/embulk/output/kintone/task/reduce_exception/input.csv +13 -0
  69. data/src/test/resources/org/embulk/output/kintone/task/reduce_exception/insert_add_records.jsonl +2 -0
  70. data/src/test/resources/org/embulk/output/kintone/task/reduce_exception/update_update_records.jsonl +2 -0
  71. data/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/config.yml +343 -0
  72. data/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/derived_columns.json +1 -0
  73. data/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/input.csv +13 -0
  74. data/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/insert_add_ignore_nulls_records.jsonl +6 -0
  75. data/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/insert_add_prefer_nulls_records.jsonl +6 -0
  76. data/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/insert_add_records.jsonl +6 -0
  77. data/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/update_update_ignore_nulls_records.jsonl +3 -0
  78. data/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/update_update_prefer_nulls_records.jsonl +3 -0
  79. data/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/update_update_records.jsonl +6 -0
  80. data/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/upsert_add_ignore_nulls_records.jsonl +3 -0
  81. data/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/upsert_add_prefer_nulls_records.jsonl +3 -0
  82. data/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/upsert_add_records.jsonl +0 -0
  83. data/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/upsert_update_ignore_nulls_records.jsonl +3 -0
  84. data/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/upsert_update_prefer_nulls_records.jsonl +3 -0
  85. data/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/upsert_update_records.jsonl +6 -0
  86. data/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/values.json +1 -0
  87. data/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/values_ignore_nulls.json +1 -0
  88. data/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/values_prefer_nulls.json +1 -0
  89. metadata +62 -4
  90. data/classpath/embulk-output-kintone-1.0.0.jar +0 -0
@@ -4,6 +4,7 @@ import com.kintone.client.model.record.FieldType;
4
4
  import com.kintone.client.model.record.Record;
5
5
  import com.kintone.client.model.record.UpdateKey;
6
6
  import java.util.Map;
7
+ import java.util.Set;
7
8
  import java.util.function.BiConsumer;
8
9
  import javax.validation.UnexpectedTypeException;
9
10
  import org.embulk.spi.Column;
@@ -18,22 +19,31 @@ public class KintoneColumnVisitorVerifier {
18
19
  private final KintoneColumnVisitor visitor;
19
20
 
20
21
  public KintoneColumnVisitorVerifier(
21
- Schema schema, Map<String, KintoneColumnOption> options, String updateKeyName, Page page) {
22
- this(schema, options, false, false, updateKeyName, page);
22
+ Schema schema,
23
+ Set<Column> derived,
24
+ Map<String, KintoneColumnOption> options,
25
+ String reduceKeyName,
26
+ String updateKeyName,
27
+ Page page) {
28
+ this(schema, derived, options, false, false, reduceKeyName, updateKeyName, page);
23
29
  }
24
30
 
25
31
  public KintoneColumnVisitorVerifier(
26
32
  Schema schema,
33
+ Set<Column> derived,
27
34
  Map<String, KintoneColumnOption> options,
28
35
  boolean preferNulls,
29
36
  boolean ignoreNulls,
37
+ String reduceKeyName,
30
38
  String updateKeyName,
31
39
  Page page) {
32
40
  this.schema = schema;
33
41
  this.options = options;
34
42
  reader = new PageReader(schema);
35
43
  reader.setPage(page);
36
- visitor = new KintoneColumnVisitor(reader, options, preferNulls, ignoreNulls, updateKeyName);
44
+ visitor =
45
+ new KintoneColumnVisitor(
46
+ reader, derived, options, preferNulls, ignoreNulls, reduceKeyName, updateKeyName);
37
47
  }
38
48
 
39
49
  public void verify() {
@@ -1,10 +1,12 @@
1
1
  package org.embulk.output.kintone;
2
2
 
3
+ import static org.embulk.output.kintone.deserializer.DeserializerTest.assertTableRows;
3
4
  import static org.hamcrest.MatcherAssert.assertThat;
4
5
  import static org.hamcrest.Matchers.is;
5
6
  import static org.mockito.ArgumentMatchers.anyList;
6
7
  import static org.mockito.ArgumentMatchers.eq;
7
8
  import static org.mockito.ArgumentMatchers.matches;
9
+ import static org.mockito.Mockito.CALLS_REAL_METHODS;
8
10
  import static org.mockito.Mockito.atLeast;
9
11
  import static org.mockito.Mockito.mock;
10
12
  import static org.mockito.Mockito.mockStatic;
@@ -19,11 +21,13 @@ import com.kintone.client.api.record.GetRecordsByCursorResponseBody;
19
21
  import com.kintone.client.model.app.field.FieldProperty;
20
22
  import com.kintone.client.model.app.field.NumberFieldProperty;
21
23
  import com.kintone.client.model.app.field.SingleLineTextFieldProperty;
24
+ import com.kintone.client.model.record.FieldType;
22
25
  import com.kintone.client.model.record.FieldValue;
23
26
  import com.kintone.client.model.record.NumberFieldValue;
24
27
  import com.kintone.client.model.record.Record;
25
28
  import com.kintone.client.model.record.RecordForUpdate;
26
29
  import com.kintone.client.model.record.SingleLineTextFieldValue;
30
+ import com.kintone.client.model.record.SubtableFieldValue;
27
31
  import com.kintone.client.model.record.UpdateKey;
28
32
  import java.math.BigDecimal;
29
33
  import java.util.Collection;
@@ -33,6 +37,9 @@ import java.util.Map;
33
37
  import java.util.stream.Collectors;
34
38
  import java.util.stream.IntStream;
35
39
  import org.embulk.config.TaskReport;
40
+ import org.embulk.deps.buffer.PooledBufferAllocator;
41
+ import org.embulk.spi.BufferAllocator;
42
+ import org.embulk.spi.Exec;
36
43
  import org.embulk.spi.Page;
37
44
  import org.embulk.spi.TransactionalPageOutput;
38
45
  import org.mockito.ArgumentCaptor;
@@ -46,6 +53,15 @@ public class KintonePageOutputVerifier implements TransactionalPageOutput {
46
53
  private final List<Record> addRecords;
47
54
  private final List<RecordForUpdate> updateRecords;
48
55
 
56
+ public KintonePageOutputVerifier(
57
+ String domain,
58
+ String field,
59
+ List<String> values,
60
+ List<Record> addRecords,
61
+ List<RecordForUpdate> updateRecords) {
62
+ this(null, domain, field, values, addRecords, updateRecords);
63
+ }
64
+
49
65
  public KintonePageOutputVerifier(
50
66
  TransactionalPageOutput transactionalPageOutput,
51
67
  String domain,
@@ -73,6 +89,9 @@ public class KintonePageOutputVerifier implements TransactionalPageOutput {
73
89
 
74
90
  @Override
75
91
  public void close() {
92
+ if (transactionalPageOutput == null) {
93
+ return;
94
+ }
76
95
  transactionalPageOutput.close();
77
96
  }
78
97
 
@@ -88,12 +107,20 @@ public class KintonePageOutputVerifier implements TransactionalPageOutput {
88
107
 
89
108
  public void runWithMock(Runnable runnable) {
90
109
  try {
91
- runWithMockClient(runnable);
110
+ runWithMockExec(runnable);
92
111
  } catch (Exception e) {
93
112
  throw new RuntimeException(e);
94
113
  }
95
114
  }
96
115
 
116
+ private void runWithMockExec(Runnable runnable) throws Exception {
117
+ BufferAllocator bufferAllocator = PooledBufferAllocator.create(1024 * 1024);
118
+ try (MockedStatic<Exec> mocked = mockStatic(Exec.class, CALLS_REAL_METHODS)) {
119
+ mocked.when(Exec::getBufferAllocator).thenReturn(bufferAllocator);
120
+ runWithMockClient(runnable);
121
+ }
122
+ }
123
+
97
124
  private void runWithMockClient(Runnable runnable) throws Exception {
98
125
  @SuppressWarnings("unchecked")
99
126
  Map<String, FieldProperty> mockFormFields = mock(Map.class);
@@ -181,16 +208,32 @@ public class KintonePageOutputVerifier implements TransactionalPageOutput {
181
208
  assertThat(reason, actual.getFieldCodes(true), is(expected.getFieldCodes(true)));
182
209
  // spotless:off
183
210
  actual.getFieldCodes(true).forEach(fieldCode -> assertFieldValue(domain, index, fieldCode, actual.getFieldValue(fieldCode), expected.getFieldValue(fieldCode)));
211
+ actual.getFieldCodes(true).forEach(fieldCode -> assertSubtableFieldValue(domain, index, fieldCode, actual.getFieldValue(fieldCode), expected.getFieldValue(fieldCode)));
184
212
  // spotless:on
185
213
  }
186
214
 
187
215
  private static void assertFieldValue(
188
216
  String domain, int index, String fieldCode, FieldValue actual, FieldValue expected) {
217
+ if (actual.getType() == FieldType.SUBTABLE) {
218
+ return;
219
+ }
189
220
  String reason = String.format("%s:%d:%s", domain, index, fieldCode);
190
221
  assertThat(reason, actual.getType(), is(expected.getType()));
191
222
  assertThat(reason, actual, is(expected));
192
223
  }
193
224
 
225
+ private static void assertSubtableFieldValue(
226
+ String domain, int index, String fieldCode, FieldValue actual, FieldValue expected) {
227
+ if (actual.getType() != FieldType.SUBTABLE) {
228
+ return;
229
+ }
230
+ String reason = String.format("%s:%d:%s", domain, index, fieldCode);
231
+ assertThat(reason, actual.getType(), is(expected.getType()));
232
+ // spotless:off
233
+ assertTableRows(reason, ((SubtableFieldValue) actual).getRows(), ((SubtableFieldValue) expected).getRows());
234
+ // spotless:on
235
+ }
236
+
194
237
  private static void assertRecordForUpdates(
195
238
  String domain, List<RecordForUpdate> actual, List<RecordForUpdate> expected) {
196
239
  assertThat(domain, actual.size(), is(expected.size()));
@@ -1,5 +1,10 @@
1
1
  package org.embulk.output.kintone;
2
2
 
3
+ import static org.mockito.Mockito.inOrder;
4
+ import static org.mockito.Mockito.spy;
5
+ import static org.mockito.Mockito.verify;
6
+ import static org.mockito.Mockito.when;
7
+
3
8
  import com.google.common.io.Resources;
4
9
  import com.kintone.client.Json;
5
10
  import com.kintone.client.model.record.Record;
@@ -14,10 +19,15 @@ import java.util.Arrays;
14
19
  import java.util.Collections;
15
20
  import java.util.List;
16
21
  import java.util.Objects;
22
+ import java.util.Set;
23
+ import java.util.concurrent.atomic.AtomicReference;
17
24
  import java.util.function.Function;
18
25
  import java.util.stream.Collectors;
26
+ import org.embulk.config.ConfigDiff;
19
27
  import org.embulk.config.ConfigSource;
20
28
  import org.embulk.config.TaskSource;
29
+ import org.embulk.spi.Column;
30
+ import org.embulk.spi.Exec;
21
31
  import org.embulk.spi.OutputPlugin;
22
32
  import org.embulk.spi.Schema;
23
33
  import org.embulk.spi.TransactionalPageOutput;
@@ -25,6 +35,7 @@ import org.embulk.spi.json.JsonParser;
25
35
  import org.embulk.test.EmbulkTests;
26
36
  import org.embulk.test.TestingEmbulk;
27
37
  import org.junit.Rule;
38
+ import org.mockito.InOrder;
28
39
  import org.msgpack.value.Value;
29
40
 
30
41
  public class TestKintoneOutputPlugin extends KintoneOutputPlugin {
@@ -36,20 +47,19 @@ public class TestKintoneOutputPlugin extends KintoneOutputPlugin {
36
47
  .registerPlugin(OutputPlugin.class, "kintone", TestKintoneOutputPlugin.class)
37
48
  .build();
38
49
 
50
+ @Override
51
+ public ConfigDiff transaction(
52
+ ConfigSource config, Schema schema, int taskCount, Control control) {
53
+ return config.get(String.class, "reduce_key", null) == null
54
+ ? super.transaction(config, schema, taskCount, control)
55
+ : transactionWithVerifier(config, schema, taskCount, control);
56
+ }
57
+
39
58
  @Override
40
59
  public TransactionalPageOutput open(TaskSource taskSource, Schema schema, int taskIndex) {
41
- String test = taskSource.get(String.class, "Domain");
42
- String mode = taskSource.get(String.class, "Mode");
43
- String field = taskSource.get(String.class, "UpdateKeyName");
44
- boolean preferNulls = taskSource.get(boolean.class, "PreferNulls");
45
- boolean ignoreNulls = taskSource.get(boolean.class, "IgnoreNulls");
46
- return new KintonePageOutputVerifier(
47
- super.open(taskSource, schema, taskIndex),
48
- test,
49
- field,
50
- getValues(test, preferNulls, ignoreNulls),
51
- getAddRecords(test, mode, preferNulls, ignoreNulls),
52
- getUpdateRecords(test, mode, preferNulls, ignoreNulls, field));
60
+ return taskSource.get(String.class, "ReduceKeyName") == null
61
+ ? openWithVerifier(taskSource, schema, taskIndex)
62
+ : super.open(taskSource, schema, taskIndex);
53
63
  }
54
64
 
55
65
  protected void runOutput(String configName, String inputName) throws Exception {
@@ -91,6 +101,73 @@ public class TestKintoneOutputPlugin extends KintoneOutputPlugin {
91
101
  return embulk.configLoader().fromYamlString(string);
92
102
  }
93
103
 
104
+ private ConfigDiff transactionWithVerifier(
105
+ ConfigSource config, Schema schema, int taskCount, Control control) {
106
+ try (KintonePageOutputVerifier verifier = verifier(config)) {
107
+ return runWithMock(verifier, config, schema, taskCount, control);
108
+ }
109
+ }
110
+
111
+ private ConfigDiff runWithMock(
112
+ KintonePageOutputVerifier verifier,
113
+ ConfigSource config,
114
+ Schema schema,
115
+ int taskCount,
116
+ OutputPlugin.Control control) {
117
+ String test = config.get(String.class, "domain");
118
+ PluginTask spyTask = spy(config.loadConfig(PluginTask.class));
119
+ ConfigSource spyConfig = spy(config);
120
+ when(spyConfig.loadConfig(PluginTask.class)).thenReturn(spyTask);
121
+ AtomicReference<ConfigDiff> configDiff = new AtomicReference<>();
122
+ verifier.runWithMock(
123
+ () -> configDiff.set(super.transaction(spyConfig, schema, taskCount, control)));
124
+ verify(spyConfig).loadConfig(PluginTask.class);
125
+ InOrder inOrderTask = inOrder(spyTask);
126
+ inOrderTask.verify(spyTask).setDerivedColumns(Collections.emptySet());
127
+ inOrderTask.verify(spyTask).setDerivedColumns(getDerivedColumns(test));
128
+ return configDiff.get();
129
+ }
130
+
131
+ private KintonePageOutputVerifier verifier(ConfigSource config) {
132
+ String test = config.get(String.class, "domain");
133
+ String mode = config.get(String.class, "mode");
134
+ String field = config.get(String.class, "update_key", null);
135
+ boolean preferNulls = config.get(boolean.class, "prefer_nulls", false);
136
+ boolean ignoreNulls = config.get(boolean.class, "ignore_nulls", false);
137
+ return new KintonePageOutputVerifier(
138
+ test,
139
+ field,
140
+ getValues(test, preferNulls, ignoreNulls),
141
+ getAddRecords(test, mode, preferNulls, ignoreNulls),
142
+ getUpdateRecords(test, mode, preferNulls, ignoreNulls, field));
143
+ }
144
+
145
+ private TransactionalPageOutput openWithVerifier(
146
+ TaskSource taskSource, Schema schema, int taskIndex) {
147
+ String test = taskSource.get(String.class, "Domain");
148
+ String mode = taskSource.get(String.class, "Mode");
149
+ String field = taskSource.get(String.class, "UpdateKeyName");
150
+ boolean preferNulls = taskSource.get(boolean.class, "PreferNulls");
151
+ boolean ignoreNulls = taskSource.get(boolean.class, "IgnoreNulls");
152
+ return new KintonePageOutputVerifier(
153
+ super.open(taskSource, schema, taskIndex),
154
+ test,
155
+ field,
156
+ getValues(test, preferNulls, ignoreNulls),
157
+ getAddRecords(test, mode, preferNulls, ignoreNulls),
158
+ getUpdateRecords(test, mode, preferNulls, ignoreNulls, field));
159
+ }
160
+
161
+ private static Set<Column> getDerivedColumns(String test) {
162
+ String name = String.format("%s/derived_columns.json", test);
163
+ String json = existsResource(name) ? readResource(name) : null;
164
+ return json == null || json.isEmpty()
165
+ ? Collections.emptySet()
166
+ : PARSER.parse(json).asArrayValue().list().stream()
167
+ .map(value -> Exec.getModelManager().readObject(Column.class, value.toJson()))
168
+ .collect(Collectors.toSet());
169
+ }
170
+
94
171
  private static List<String> getValues(String test, boolean preferNulls, boolean ignoreNulls) {
95
172
  String name =
96
173
  String.format(
@@ -0,0 +1,46 @@
1
+ package org.embulk.output.kintone;
2
+
3
+ import net.jcip.annotations.NotThreadSafe;
4
+ import org.junit.Test;
5
+
6
+ @NotThreadSafe
7
+ public class TestTaskReduce extends TestTask {
8
+ @Override
9
+ public void before() {
10
+ super.before();
11
+ merge(config("domain: task/reduce"));
12
+ }
13
+
14
+ @Test
15
+ public void testInsert() throws Exception {
16
+ merge(config("mode: insert"));
17
+ merge(config("reduce_key: timestamp"));
18
+ runOutput();
19
+ merge(config("prefer_nulls: true"));
20
+ runOutput();
21
+ merge(config("ignore_nulls: true"));
22
+ runOutput();
23
+ }
24
+
25
+ @Test
26
+ public void testUpdate() throws Exception {
27
+ merge(config("mode: update", "update_key: string_number"));
28
+ merge(config("reduce_key: double_single_line_text"));
29
+ runOutput();
30
+ merge(config("prefer_nulls: true"));
31
+ runOutput();
32
+ merge(config("ignore_nulls: true"));
33
+ runOutput();
34
+ }
35
+
36
+ @Test
37
+ public void testUpsert() throws Exception {
38
+ merge(config("mode: upsert", "update_key: double_single_line_text"));
39
+ merge(config("reduce_key: string_number"));
40
+ runOutput();
41
+ merge(config("prefer_nulls: true"));
42
+ runOutput();
43
+ merge(config("ignore_nulls: true"));
44
+ runOutput();
45
+ }
46
+ }
@@ -0,0 +1,50 @@
1
+ package org.embulk.output.kintone;
2
+
3
+ import static org.hamcrest.MatcherAssert.assertThat;
4
+ import static org.hamcrest.Matchers.instanceOf;
5
+ import static org.hamcrest.Matchers.is;
6
+ import static org.junit.Assert.assertThrows;
7
+
8
+ import net.jcip.annotations.NotThreadSafe;
9
+ import org.embulk.exec.PartialExecutionException;
10
+ import org.embulk.output.kintone.reducer.ReduceException;
11
+ import org.junit.Test;
12
+
13
+ @NotThreadSafe
14
+ public class TestTaskReduceException extends TestTask {
15
+ @Override
16
+ public void before() {
17
+ super.before();
18
+ merge(config("domain: task/reduce_exception"));
19
+ }
20
+
21
+ @Test
22
+ public void test() throws Exception {
23
+ merge(config("mode: insert"));
24
+ merge(config("reduce_key: double_single_line_text"));
25
+ runOutput();
26
+ merge(config("mode: update", "update_key: double_single_line_text"));
27
+ merge(config("reduce_key: string_number"));
28
+ runOutput();
29
+ merge(config("mode: upsert", "update_key: string_number"));
30
+ merge(config("reduce_key: json.double_single_line_text"));
31
+ assertReduceException(
32
+ "Couldn't reduce because column json.double_single_line_text is not unique to [json, json_subtable]\n[double_single_line_text, string_number] expected [123.0, 456] but actual [456.0, 123]");
33
+ merge(config("reduce_key: json.string_number"));
34
+ assertReduceException(
35
+ "Couldn't reduce because column json.string_number is not unique to [json, json_subtable]\n[double_single_line_text, string_number] expected [123.0, 456] but actual [456.0, 123]");
36
+ merge(config("reduce_key: json_subtable.double_single_line_text"));
37
+ assertReduceException(
38
+ "Couldn't reduce because column json_subtable.double_single_line_text is not unique to [json, json_subtable]\n[double_single_line_text, string_number] expected [123.0, 456] but actual [456.0, 123]");
39
+ merge(config("reduce_key: json_subtable.string_number"));
40
+ assertReduceException(
41
+ "Couldn't reduce because column json_subtable.string_number is not unique to [json, json_subtable]\n[double_single_line_text, string_number] expected [123.0, 456] but actual [456.0, 123]");
42
+ }
43
+
44
+ private void assertReduceException(String message) {
45
+ Exception e = assertThrows(PartialExecutionException.class, this::runOutput);
46
+ assertThat(e.getCause(), is(instanceOf(RuntimeException.class)));
47
+ assertThat(e.getCause().getCause(), is(instanceOf(ReduceException.class)));
48
+ assertThat(e.getCause().getCause().getMessage(), is(message));
49
+ }
50
+ }
@@ -0,0 +1,46 @@
1
+ package org.embulk.output.kintone;
2
+
3
+ import net.jcip.annotations.NotThreadSafe;
4
+ import org.junit.Test;
5
+
6
+ @NotThreadSafe
7
+ public class TestTaskReduceSubtable extends TestTask {
8
+ @Override
9
+ public void before() {
10
+ super.before();
11
+ merge(config("domain: task/reduce_subtable"));
12
+ }
13
+
14
+ @Test
15
+ public void testInsert() throws Exception {
16
+ merge(config("mode: insert"));
17
+ merge(config("reduce_key: timestamp"));
18
+ runOutput();
19
+ merge(config("prefer_nulls: true"));
20
+ runOutput();
21
+ merge(config("ignore_nulls: true"));
22
+ runOutput();
23
+ }
24
+
25
+ @Test
26
+ public void testUpdate() throws Exception {
27
+ merge(config("mode: update", "update_key: string_number"));
28
+ merge(config("reduce_key: string_number"));
29
+ runOutput();
30
+ merge(config("prefer_nulls: true"));
31
+ runOutput();
32
+ merge(config("ignore_nulls: true"));
33
+ runOutput();
34
+ }
35
+
36
+ @Test
37
+ public void testUpsert() throws Exception {
38
+ merge(config("mode: upsert", "update_key: double_single_line_text"));
39
+ merge(config("reduce_key: double_single_line_text"));
40
+ runOutput();
41
+ merge(config("prefer_nulls: true"));
42
+ runOutput();
43
+ merge(config("ignore_nulls: true"));
44
+ runOutput();
45
+ }
46
+ }
@@ -0,0 +1,165 @@
1
+ package org.embulk.output.kintone.deserializer;
2
+
3
+ import static org.hamcrest.MatcherAssert.assertThat;
4
+ import static org.hamcrest.Matchers.is;
5
+
6
+ import com.kintone.client.model.FileBody;
7
+ import com.kintone.client.model.Group;
8
+ import com.kintone.client.model.Organization;
9
+ import com.kintone.client.model.User;
10
+ import com.kintone.client.model.record.CheckBoxFieldValue;
11
+ import com.kintone.client.model.record.DateFieldValue;
12
+ import com.kintone.client.model.record.DateTimeFieldValue;
13
+ import com.kintone.client.model.record.DropDownFieldValue;
14
+ import com.kintone.client.model.record.FieldValue;
15
+ import com.kintone.client.model.record.FileFieldValue;
16
+ import com.kintone.client.model.record.GroupSelectFieldValue;
17
+ import com.kintone.client.model.record.LinkFieldValue;
18
+ import com.kintone.client.model.record.MultiLineTextFieldValue;
19
+ import com.kintone.client.model.record.MultiSelectFieldValue;
20
+ import com.kintone.client.model.record.NumberFieldValue;
21
+ import com.kintone.client.model.record.OrganizationSelectFieldValue;
22
+ import com.kintone.client.model.record.RadioButtonFieldValue;
23
+ import com.kintone.client.model.record.RichTextFieldValue;
24
+ import com.kintone.client.model.record.SingleLineTextFieldValue;
25
+ import com.kintone.client.model.record.SubtableFieldValue;
26
+ import com.kintone.client.model.record.TableRow;
27
+ import com.kintone.client.model.record.TimeFieldValue;
28
+ import com.kintone.client.model.record.UserSelectFieldValue;
29
+ import java.math.BigDecimal;
30
+ import java.time.LocalDate;
31
+ import java.time.LocalTime;
32
+ import java.time.ZonedDateTime;
33
+ import java.util.List;
34
+ import java.util.function.Function;
35
+ import java.util.stream.IntStream;
36
+ import org.junit.Test;
37
+
38
+ public class DeserializerTest {
39
+ // spotless:off
40
+ public static final Function<Long, String> TABLE_ROW = (id) -> String.format("{\"id\":%d,\"value\":{\"リッチエディター\":{\"type\":\"RICH_TEXT\",\"value\":\"\\u003ca href\\u003d\\\"https://www.cybozu.com\\\"\\u003eサイボウズ\\u003c/a\\u003e\"},\"グループ選択\":{\"type\":\"GROUP_SELECT\",\"value\":[{\"name\":\"プロジェクトマネージャー\",\"code\":\"project_manager\"},{\"name\":\"チームリーダー\",\"code\":\"team_leader\"}]},\"文字列(1行)\":{\"type\":\"SINGLE_LINE_TEXT\",\"value\":\"テストです。\"},\"ラジオボタン\":{\"type\":\"RADIO_BUTTON\",\"value\":\"選択肢3\"},\"ドロップダウン\":{\"type\":\"DROP_DOWN\",\"value\":\"選択肢6\"},\"組織選択\":{\"type\":\"ORGANIZATION_SELECT\",\"value\":[{\"name\":\"開発部\",\"code\":\"kaihatsu\"},{\"name\":\"人事部\",\"code\":\"jinji\"}]},\"ユーザー選択\":{\"type\":\"USER_SELECT\",\"value\":[{\"name\":\"Noboru Sato\",\"code\":\"guest/sato@cybozu.com\"},{\"name\":\"Misaki Kato\",\"code\":\"kato\"}]},\"日時\":{\"type\":\"DATETIME\",\"value\":\"2012-01-11T11:30:00Z\"},\"文字列(複数行)\":{\"type\":\"MULTI_LINE_TEXT\",\"value\":\"テスト\\nです。\"},\"時刻\":{\"type\":\"TIME\",\"value\":\"11:30\"},\"チェックボックス\":{\"type\":\"CHECK_BOX\",\"value\":[\"選択肢1\",\"選択肢2\"]},\"複数選択\":{\"type\":\"MULTI_SELECT\",\"value\":[\"選択肢4\",\"選択肢5\"]},\"数値\":{\"type\":\"NUMBER\",\"value\":\"123\"},\"添付ファイル\":{\"type\":\"FILE\",\"value\":[{\"contentType\":\"text/plain\",\"fileKey\":\"201202061155587E339F9067544F1A92C743460E3D12B3297\",\"name\":\"17to20_VerupLog (1).txt\",\"size\":\"23175\"},{\"contentType\":\"application/json\",\"fileKey\":\"201202061155583C763E30196F419E83E91D2E4A03746C273\",\"name\":\"17to20_VerupLog.txt\",\"size\":\"23176\"}]},\"リンク\":{\"type\":\"LINK\",\"value\":\"https://cybozu.co.jp/\"},\"計算\":{\"type\":\"CALC\",\"value\":\"456\"},\"日付\":{\"type\":\"DATE\",\"value\":\"2012-01-11\"}}}", id);
41
+ private static final String NULL_TABLE_ROW = "{\"value\":{\"添付ファイル(null要素)\":{\"type\":\"FILE\",\"value\":[null,null]},\"添付ファイル(null項目)\":{\"type\":\"FILE\",\"value\":[{},{}]},\"複数選択(空)\":{\"type\":\"MULTI_SELECT\",\"value\":[]},\"リッチエディター\":{\"type\":\"RICH_TEXT\"},\"文字列(1行)\":{\"type\":\"SINGLE_LINE_TEXT\"},\"ユーザー選択(null項目)\":{\"type\":\"USER_SELECT\",\"value\":[{},{}]},\"文字列(複数行)\":{\"type\":\"MULTI_LINE_TEXT\"},\"ユーザー選択(空)\":{\"type\":\"USER_SELECT\",\"value\":[]},\"チェックボックス(null要素)\":{\"type\":\"CHECK_BOX\",\"value\":[null,null]},\"組織選択(null項目)\":{\"type\":\"ORGANIZATION_SELECT\",\"value\":[{},{}]},\"計算\":{\"type\":\"CALC\"},\"日付\":{\"type\":\"DATE\"},\"組織選択(空)\":{\"type\":\"ORGANIZATION_SELECT\",\"value\":[]},\"添付ファイル(空)\":{\"type\":\"FILE\",\"value\":[]},\"ラジオボタン\":{\"type\":\"RADIO_BUTTON\"},\"グループ選択(null項目)\":{\"type\":\"GROUP_SELECT\",\"value\":[{},{}]},\"複数選択(null要素)\":{\"type\":\"MULTI_SELECT\",\"value\":[null,null]},\"ドロップダウン\":{\"type\":\"DROP_DOWN\"},\"日時\":{\"type\":\"DATETIME\"},\"組織選択(null要素)\":{\"type\":\"ORGANIZATION_SELECT\",\"value\":[null,null]},\"時刻\":{\"type\":\"TIME\"},\"グループ選択(空)\":{\"type\":\"GROUP_SELECT\",\"value\":[]},\"数値\":{\"type\":\"NUMBER\"},\"ユーザー選択(null要素)\":{\"type\":\"USER_SELECT\",\"value\":[null,null]},\"グループ選択(null要素)\":{\"type\":\"GROUP_SELECT\",\"value\":[null,null]},\"リンク\":{\"type\":\"LINK\"},\"チェックボックス(空)\":{\"type\":\"CHECK_BOX\",\"value\":[]}}}";
42
+ // spotless:on
43
+ @Test
44
+ public void deserialize() {
45
+ // spotless:off
46
+ SubtableFieldValue actual = new Deserializer().deserialize(String.format("[%s,%s]", TABLE_ROW.apply(48290L), TABLE_ROW.apply(48291L)), SubtableFieldValue.class);
47
+ // spotless:on
48
+ SubtableFieldValue expected = new SubtableFieldValue(tableRow(48290L), tableRow(48291L));
49
+ assertTableRows(actual.getRows(), expected.getRows());
50
+ }
51
+
52
+ @Test
53
+ public void deserializeNull() {
54
+ // spotless:off
55
+ SubtableFieldValue actual = new Deserializer().deserialize(String.format("[%s,%s]", NULL_TABLE_ROW, NULL_TABLE_ROW), SubtableFieldValue.class);
56
+ // spotless:on
57
+ SubtableFieldValue expected = new SubtableFieldValue(nullTableRow(), nullTableRow());
58
+ assertTableRows(actual.getRows(), expected.getRows());
59
+ }
60
+
61
+ public static void assertTableRows(List<TableRow> actual, List<TableRow> expected) {
62
+ assertTableRows("", actual, expected);
63
+ }
64
+
65
+ public static void assertTableRows(
66
+ String domain, List<TableRow> actual, List<TableRow> expected) {
67
+ assertThat(domain, actual.size(), is(expected.size()));
68
+ // spotless:off
69
+ IntStream.range(0, actual.size()).forEach(index -> assertTableRow(domain, index, actual.get(index), expected.get(index)));
70
+ // spotless:on
71
+ }
72
+
73
+ private static void assertTableRow(String domain, int index, TableRow actual, TableRow expected) {
74
+ String reason = String.format("%s:%d", domain, index);
75
+ assertThat(reason, actual.getId(), is(expected.getId()));
76
+ assertThat(reason, actual.getFieldCodes(), is(expected.getFieldCodes()));
77
+ // spotless:off
78
+ actual.getFieldCodes().forEach(fieldCode -> assertFieldValue(domain, index, fieldCode, actual.getFieldValue(fieldCode), expected.getFieldValue(fieldCode)));
79
+ // spotless:on
80
+ }
81
+
82
+ private static void assertFieldValue(
83
+ String domain, int index, String fieldCode, FieldValue actual, FieldValue expected) {
84
+ String reason = String.format("%s:%d:%s", domain, index, fieldCode);
85
+ assertThat(reason, actual.getType(), is(expected.getType()));
86
+ assertThat(reason, actual, is(expected));
87
+ }
88
+
89
+ public static TableRow tableRow(Long id) {
90
+ TableRow tableRow = new TableRow(id);
91
+ // spotless:off
92
+ tableRow.putField("文字列(1行)", new SingleLineTextFieldValue("テストです。"));
93
+ tableRow.putField("文字列(複数行)", new MultiLineTextFieldValue("テスト\nです。"));
94
+ tableRow.putField("リッチエディター", new RichTextFieldValue("<a href=\"https://www.cybozu.com\">サイボウズ</a>"));
95
+ tableRow.putField("数値", new NumberFieldValue(new BigDecimal("123")));
96
+ tableRow.putField("チェックボックス", new CheckBoxFieldValue("選択肢1", "選択肢2"));
97
+ tableRow.putField("ラジオボタン", new RadioButtonFieldValue("選択肢3"));
98
+ tableRow.putField("複数選択", new MultiSelectFieldValue("選択肢4", "選択肢5"));
99
+ tableRow.putField("ドロップダウン", new DropDownFieldValue("選択肢6"));
100
+ tableRow.putField("ユーザー選択", new UserSelectFieldValue(user("guest/sato@cybozu.com", "Noboru Sato"), user("kato", "Misaki Kato")));
101
+ tableRow.putField("組織選択", new OrganizationSelectFieldValue(organization("kaihatsu", "開発部"), organization("jinji", "人事部")));
102
+ tableRow.putField("グループ選択", new GroupSelectFieldValue(group("project_manager", "プロジェクトマネージャー"), group("team_leader", "チームリーダー")));
103
+ tableRow.putField("日付", new DateFieldValue(LocalDate.parse("2012-01-11")));
104
+ tableRow.putField("時刻", new TimeFieldValue(LocalTime.parse("11:30")));
105
+ tableRow.putField("日時", new DateTimeFieldValue(ZonedDateTime.parse("2012-01-11T11:30:00Z")));
106
+ tableRow.putField("リンク", new LinkFieldValue("https://cybozu.co.jp/"));
107
+ tableRow.putField("添付ファイル", new FileFieldValue(fileBody("text/plain", "201202061155587E339F9067544F1A92C743460E3D12B3297", "17to20_VerupLog (1).txt", "23175"), fileBody("application/json", "201202061155583C763E30196F419E83E91D2E4A03746C273", "17to20_VerupLog.txt", "23176")));
108
+ // spotless:on
109
+ return tableRow;
110
+ }
111
+
112
+ private static TableRow nullTableRow() {
113
+ TableRow tableRow = new TableRow();
114
+ // spotless:off
115
+ tableRow.putField("文字列(1行)", new SingleLineTextFieldValue(null));
116
+ tableRow.putField("文字列(複数行)", new MultiLineTextFieldValue(null));
117
+ tableRow.putField("リッチエディター", new RichTextFieldValue(null));
118
+ tableRow.putField("数値", new NumberFieldValue(null));
119
+ tableRow.putField("チェックボックス(空)", new CheckBoxFieldValue());
120
+ tableRow.putField("チェックボックス(null要素)", new CheckBoxFieldValue(null, null));
121
+ tableRow.putField("ラジオボタン", new RadioButtonFieldValue(null));
122
+ tableRow.putField("複数選択(空)", new MultiSelectFieldValue());
123
+ tableRow.putField("複数選択(null要素)", new MultiSelectFieldValue(null, null));
124
+ tableRow.putField("ドロップダウン", new DropDownFieldValue(null));
125
+ tableRow.putField("ユーザー選択(空)", new UserSelectFieldValue());
126
+ tableRow.putField("ユーザー選択(null要素)", new UserSelectFieldValue(null, null));
127
+ tableRow.putField("ユーザー選択(null項目)", new UserSelectFieldValue(user(null, null), user(null, null)));
128
+ tableRow.putField("組織選択(空)", new OrganizationSelectFieldValue());
129
+ tableRow.putField("組織選択(null要素)", new OrganizationSelectFieldValue(null, null));
130
+ tableRow.putField("組織選択(null項目)", new OrganizationSelectFieldValue(organization(null, null), organization(null, null)));
131
+ tableRow.putField("グループ選択(空)", new GroupSelectFieldValue());
132
+ tableRow.putField("グループ選択(null要素)", new GroupSelectFieldValue(null, null));
133
+ tableRow.putField("グループ選択(null項目)", new GroupSelectFieldValue(group(null, null), group(null, null)));
134
+ tableRow.putField("日付", new DateFieldValue(null));
135
+ tableRow.putField("時刻", new TimeFieldValue(null));
136
+ tableRow.putField("日時", new DateTimeFieldValue(null));
137
+ tableRow.putField("リンク", new LinkFieldValue(null));
138
+ tableRow.putField("添付ファイル(空)", new FileFieldValue());
139
+ tableRow.putField("添付ファイル(null要素)", new FileFieldValue(null, null));
140
+ tableRow.putField("添付ファイル(null項目)", new FileFieldValue(fileBody(null, null, null, null), fileBody(null, null, null, null)));
141
+ // spotless:on
142
+ return tableRow;
143
+ }
144
+
145
+ private static User user(String code, String name) {
146
+ return new User(name, code);
147
+ }
148
+
149
+ private static Organization organization(String code, String name) {
150
+ return new Organization(name, code);
151
+ }
152
+
153
+ private static Group group(String code, String name) {
154
+ return new Group(name, code);
155
+ }
156
+
157
+ private static FileBody fileBody(String contentType, String fileKey, String name, String size) {
158
+ FileBody fileBody = new FileBody();
159
+ fileBody.setContentType(contentType);
160
+ fileBody.setFileKey(fileKey);
161
+ fileBody.setName(name);
162
+ fileBody.setSize(size == null ? null : Integer.valueOf(size));
163
+ return fileBody;
164
+ }
165
+ }