embulk-output-kintone 0.4.0 → 0.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (33) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/ci.yml +1 -0
  3. data/.gitignore +0 -1
  4. data/build.gradle +18 -19
  5. data/classpath/embulk-output-kintone-0.4.1.jar +0 -0
  6. data/classpath/shadow-kintone-java-client-0.4.1-all.jar +0 -0
  7. data/gradle/wrapper/gradle-wrapper.properties +1 -1
  8. data/settings.gradle +2 -0
  9. data/shadow-kintone-java-client/build.gradle +35 -0
  10. data/src/main/java/org/embulk/output/kintone/KintonePageOutput.java +166 -121
  11. data/src/main/java/org/embulk/output/kintone/KintoneRetryOption.java +19 -0
  12. data/src/main/java/org/embulk/output/kintone/PluginTask.java +3 -3
  13. data/src/test/java/com/kintone/client/Json.java +16 -0
  14. data/src/test/java/org/embulk/output/kintone/KintoneColumnOptionBuilder.java +63 -0
  15. data/src/test/java/org/embulk/output/kintone/KintoneColumnVisitorTest.java +297 -0
  16. data/src/test/java/org/embulk/output/kintone/KintoneColumnVisitorVerifier.java +52 -0
  17. data/src/test/java/org/embulk/output/kintone/KintonePageOutputVerifier.java +226 -0
  18. data/src/test/java/org/embulk/output/kintone/OutputPageBuilder.java +92 -0
  19. data/src/test/java/org/embulk/output/kintone/TestKintoneOutputPlugin.java +163 -1
  20. data/src/test/java/org/embulk/output/kintone/TestTask.java +40 -0
  21. data/src/test/java/org/embulk/output/kintone/TestTaskMode.java +31 -0
  22. data/src/test/resources/logback-test.xml +14 -0
  23. data/src/test/resources/org/embulk/output/kintone/config.yml +4 -0
  24. data/src/test/resources/org/embulk/output/kintone/task/config.yml +1 -0
  25. data/src/test/resources/org/embulk/output/kintone/task/mode/config.yml +54 -0
  26. data/src/test/resources/org/embulk/output/kintone/task/mode/input.csv +7 -0
  27. data/src/test/resources/org/embulk/output/kintone/task/mode/insert_add_records.jsonl +6 -0
  28. data/src/test/resources/org/embulk/output/kintone/task/mode/update_update_records.jsonl +3 -0
  29. data/src/test/resources/org/embulk/output/kintone/task/mode/upsert_add_records.jsonl +2 -0
  30. data/src/test/resources/org/embulk/output/kintone/task/mode/upsert_update_records.jsonl +4 -0
  31. data/src/test/resources/org/embulk/output/kintone/task/mode/values.json +1 -0
  32. metadata +25 -3
  33. data/classpath/embulk-output-kintone-0.4.0.jar +0 -0
data/src/test/java/org/embulk/output/kintone/KintoneColumnVisitorTest.java ADDED
@@ -0,0 +1,297 @@
1
+ package org.embulk.output.kintone;
2
+
3
+ import static org.hamcrest.MatcherAssert.assertThat;
4
+ import static org.hamcrest.Matchers.is;
5
+ import static org.hamcrest.Matchers.nullValue;
6
+
7
+ import com.google.common.collect.ImmutableMap;
8
+ import java.math.BigDecimal;
9
+ import java.time.Instant;
10
+ import java.time.LocalDate;
11
+ import java.time.ZoneId;
12
+ import java.time.ZonedDateTime;
13
+ import java.util.AbstractMap;
14
+ import java.util.Arrays;
15
+ import java.util.List;
16
+ import java.util.Map;
17
+ import java.util.function.Function;
18
+ import org.embulk.spi.Page;
19
+ import org.embulk.spi.Schema;
20
+ import org.embulk.spi.time.Timestamp;
21
+ import org.embulk.spi.type.Types;
22
+ import org.junit.Test;
23
+ import org.msgpack.value.ValueFactory;
24
+
25
+ public class KintoneColumnVisitorTest {
26
+ @Test
27
+ public void test() {
28
+ Schema schema = build(Schema.builder());
29
+ KintoneColumnVisitorVerifier verifier =
30
+ new KintoneColumnVisitorVerifier(
31
+ schema,
32
+ build(ImmutableMap.builder()),
33
+ "STRING|SINGLE_LINE_TEXT",
34
+ OutputPageBuilder.build(schema, KintoneColumnVisitorTest::build));
35
+ verifier.verify(
36
+ (record, updateKey) -> {
37
+ assertThat(record.getSingleLineTextFieldValue("BOOLEAN|SINGLE_LINE_TEXT"), is("false"));
38
+ assertThat(record.getNumberFieldValue("BOOLEAN"), nullValue());
39
+ assertThat(record.getSingleLineTextFieldValue("LONG|SINGLE_LINE_TEXT"), is(""));
40
+ assertThat(record.getNumberFieldValue("LONG"), nullValue());
41
+ assertThat(record.getSingleLineTextFieldValue("DOUBLE|SINGLE_LINE_TEXT"), is("0.0"));
42
+ assertThat(record.getNumberFieldValue("DOUBLE"), is(number("0.0")));
43
+ assertThat(record.getSingleLineTextFieldValue("STRING|SINGLE_LINE_TEXT"), is(""));
44
+ assertThat(record.getMultiLineTextFieldValue("STRING"), is(""));
45
+ assertThat(record.getNumberFieldValue("STRING|NUMBER"), nullValue());
46
+ assertThat(record.getCheckBoxFieldValue("STRING|CHECK_BOX"), is(list("null")));
47
+ assertThat(record.getDropDownFieldValue("STRING|DROP_DOWN"), is(""));
48
+ assertThat(record.getLinkFieldValue("STRING|LINK"), is(""));
49
+ assertThat(record.getSingleLineTextFieldValue("TIMESTAMP|SINGLE_LINE_TEXT"), nullValue());
50
+ assertThat(record.getDateFieldValue("TIMESTAMP|DATE"), nullValue());
51
+ assertThat(record.getDateFieldValue("TIMESTAMP|DATE|JST"), nullValue());
52
+ assertThat(record.getDateFieldValue("TIMESTAMP|DATE|PST"), nullValue());
53
+ assertThat(record.getDateTimeFieldValue("TIMESTAMP"), nullValue());
54
+ assertThat(record.getSingleLineTextFieldValue("JSON|SINGLE_LINE_TEXT"), is(""));
55
+ assertThat(record.getMultiLineTextFieldValue("JSON"), is(""));
56
+ assertThat(updateKey.getField(), is("STRING|SINGLE_LINE_TEXT"));
57
+ assertThat(updateKey.getValue(), is(""));
58
+ });
59
+ verifier.verify(
60
+ (record, updateKey) -> {
61
+ assertThat(record.getSingleLineTextFieldValue("BOOLEAN|SINGLE_LINE_TEXT"), is("false"));
62
+ assertThat(record.getNumberFieldValue("BOOLEAN"), nullValue());
63
+ assertThat(record.getSingleLineTextFieldValue("LONG|SINGLE_LINE_TEXT"), is("0"));
64
+ assertThat(record.getNumberFieldValue("LONG"), is(number("0")));
65
+ assertThat(record.getSingleLineTextFieldValue("DOUBLE|SINGLE_LINE_TEXT"), is("0.0"));
66
+ assertThat(record.getNumberFieldValue("DOUBLE"), is(number("0.0")));
67
+ assertThat(record.getSingleLineTextFieldValue("STRING|SINGLE_LINE_TEXT"), is(""));
68
+ assertThat(record.getMultiLineTextFieldValue("STRING"), is(""));
69
+ assertThat(record.getNumberFieldValue("STRING|NUMBER"), nullValue());
70
+ assertThat(record.getCheckBoxFieldValue("STRING|CHECK_BOX"), is(list()));
71
+ assertThat(record.getDropDownFieldValue("STRING|DROP_DOWN"), is(""));
72
+ assertThat(record.getLinkFieldValue("STRING|LINK"), is(""));
73
+ assertThat(record.getSingleLineTextFieldValue("TIMESTAMP|SINGLE_LINE_TEXT"), nullValue());
74
+ assertThat(record.getDateFieldValue("TIMESTAMP|DATE"), is(date("1970-01-01")));
75
+ assertThat(record.getDateFieldValue("TIMESTAMP|DATE|JST"), is(date("1970-01-01")));
76
+ assertThat(record.getDateFieldValue("TIMESTAMP|DATE|PST"), is(date("1969-12-31")));
77
+ assertThat(
78
+ record.getDateTimeFieldValue("TIMESTAMP"), is(dateTime("1970-01-01T00:00:00Z")));
79
+ assertThat(record.getSingleLineTextFieldValue("JSON|SINGLE_LINE_TEXT"), is(""));
80
+ assertThat(record.getMultiLineTextFieldValue("JSON"), is(""));
81
+ assertThat(updateKey.getField(), is("STRING|SINGLE_LINE_TEXT"));
82
+ assertThat(updateKey.getValue(), is(""));
83
+ });
84
+ verifier.verify(
85
+ (record, updateKey) -> {
86
+ assertThat(record.getSingleLineTextFieldValue("BOOLEAN|SINGLE_LINE_TEXT"), is("true"));
87
+ assertThat(record.getNumberFieldValue("BOOLEAN"), nullValue());
88
+ assertThat(record.getSingleLineTextFieldValue("LONG|SINGLE_LINE_TEXT"), is("123"));
89
+ assertThat(record.getNumberFieldValue("LONG"), is(number("123")));
90
+ assertThat(record.getSingleLineTextFieldValue("DOUBLE|SINGLE_LINE_TEXT"), is("123.0"));
91
+ assertThat(record.getNumberFieldValue("DOUBLE"), is(number("123.0")));
92
+ assertThat(record.getSingleLineTextFieldValue("STRING|SINGLE_LINE_TEXT"), is("abc"));
93
+ assertThat(record.getMultiLineTextFieldValue("STRING"), is("abc"));
94
+ assertThat(record.getNumberFieldValue("STRING|NUMBER"), is(number("123")));
95
+ assertThat(record.getCheckBoxFieldValue("STRING|CHECK_BOX"), is(list("123", "abc")));
96
+ assertThat(record.getDropDownFieldValue("STRING|DROP_DOWN"), is("abc"));
97
+ assertThat(record.getLinkFieldValue("STRING|LINK"), is("abc"));
98
+ assertThat(record.getSingleLineTextFieldValue("TIMESTAMP|SINGLE_LINE_TEXT"), nullValue());
99
+ assertThat(record.getDateFieldValue("TIMESTAMP|DATE"), is(date("1999-12-31")));
100
+ assertThat(record.getDateFieldValue("TIMESTAMP|DATE|JST"), is(date("2000-01-01")));
101
+ assertThat(record.getDateFieldValue("TIMESTAMP|DATE|PST"), is(date("1999-12-31")));
102
+ assertThat(
103
+ record.getDateTimeFieldValue("TIMESTAMP"), is(dateTime("1999-12-31T23:59:59Z")));
104
+ assertThat(record.getSingleLineTextFieldValue("JSON|SINGLE_LINE_TEXT"), is("abc"));
105
+ assertThat(record.getMultiLineTextFieldValue("JSON"), is("abc"));
106
+ assertThat(updateKey.getField(), is("STRING|SINGLE_LINE_TEXT"));
107
+ assertThat(updateKey.getValue(), is("abc"));
108
+ });
109
+ verifier.verify(
110
+ (record, updateKey) -> {
111
+ assertThat(record.getSingleLineTextFieldValue("BOOLEAN|SINGLE_LINE_TEXT"), is("false"));
112
+ assertThat(record.getNumberFieldValue("BOOLEAN"), nullValue());
113
+ assertThat(record.getSingleLineTextFieldValue("LONG|SINGLE_LINE_TEXT"), is("456"));
114
+ assertThat(record.getNumberFieldValue("LONG"), is(number("456")));
115
+ assertThat(record.getSingleLineTextFieldValue("DOUBLE|SINGLE_LINE_TEXT"), is("456.0"));
116
+ assertThat(record.getNumberFieldValue("DOUBLE"), is(number("456.0")));
117
+ assertThat(record.getSingleLineTextFieldValue("STRING|SINGLE_LINE_TEXT"), is("def"));
118
+ assertThat(record.getMultiLineTextFieldValue("STRING"), is("def"));
119
+ assertThat(record.getNumberFieldValue("STRING|NUMBER"), is(number("456")));
120
+ assertThat(record.getCheckBoxFieldValue("STRING|CHECK_BOX"), is(list("456", "def")));
121
+ assertThat(record.getDropDownFieldValue("STRING|DROP_DOWN"), is("def"));
122
+ assertThat(record.getLinkFieldValue("STRING|LINK"), is("def"));
123
+ assertThat(record.getSingleLineTextFieldValue("TIMESTAMP|SINGLE_LINE_TEXT"), nullValue());
124
+ assertThat(record.getDateFieldValue("TIMESTAMP|DATE"), is(date("2000-01-01")));
125
+ assertThat(record.getDateFieldValue("TIMESTAMP|DATE|JST"), is(date("2000-01-01")));
126
+ assertThat(record.getDateFieldValue("TIMESTAMP|DATE|PST"), is(date("1999-12-31")));
127
+ assertThat(
128
+ record.getDateTimeFieldValue("TIMESTAMP"), is(dateTime("2000-01-01T00:00:00Z")));
129
+ assertThat(record.getSingleLineTextFieldValue("JSON|SINGLE_LINE_TEXT"), is("def"));
130
+ assertThat(record.getMultiLineTextFieldValue("JSON"), is("def"));
131
+ assertThat(updateKey.getField(), is("STRING|SINGLE_LINE_TEXT"));
132
+ assertThat(updateKey.getValue(), is("def"));
133
+ });
134
+ }
135
+
136
+ private static Schema build(Schema.Builder builder) {
137
+ return builder
138
+ .add("BOOLEAN|SINGLE_LINE_TEXT", Types.BOOLEAN)
139
+ // java.lang.NumberFormatException at java.math.BigDecimal.<init>
140
+ // .add("BOOLEAN", Types.BOOLEAN)
141
+ .add("LONG|SINGLE_LINE_TEXT", Types.LONG)
142
+ .add("LONG", Types.LONG)
143
+ .add("DOUBLE|SINGLE_LINE_TEXT", Types.DOUBLE)
144
+ .add("DOUBLE", Types.DOUBLE)
145
+ .add("STRING|SINGLE_LINE_TEXT", Types.STRING)
146
+ .add("STRING", Types.STRING)
147
+ .add("STRING|NUMBER", Types.STRING)
148
+ .add("STRING|CHECK_BOX", Types.STRING)
149
+ .add("STRING|DROP_DOWN", Types.STRING)
150
+ .add("STRING|LINK", Types.STRING)
151
+ .add("TIMESTAMP|SINGLE_LINE_TEXT", Types.TIMESTAMP)
152
+ .add("TIMESTAMP|DATE", Types.TIMESTAMP)
153
+ .add("TIMESTAMP|DATE|JST", Types.TIMESTAMP)
154
+ .add("TIMESTAMP|DATE|PST", Types.TIMESTAMP)
155
+ .add("TIMESTAMP", Types.TIMESTAMP)
156
+ .add("JSON|SINGLE_LINE_TEXT", Types.JSON)
157
+ .add("JSON", Types.JSON)
158
+ .build();
159
+ }
160
+
161
+ private static Map<String, KintoneColumnOption> build(
162
+ ImmutableMap.Builder<String, KintoneColumnOption> builder) {
163
+ return builder
164
+ .put(build("BOOLEAN|SINGLE_LINE_TEXT", it -> it.setType("SINGLE_LINE_TEXT")))
165
+ .put(build("BOOLEAN", it -> it.setType("NUMBER")))
166
+ .put(build("LONG|SINGLE_LINE_TEXT", it -> it.setType("SINGLE_LINE_TEXT")))
167
+ .put(build("LONG", it -> it.setType("NUMBER")))
168
+ .put(build("DOUBLE|SINGLE_LINE_TEXT", it -> it.setType("SINGLE_LINE_TEXT")))
169
+ .put(build("DOUBLE", it -> it.setType("NUMBER")))
170
+ .put(build("STRING|SINGLE_LINE_TEXT", it -> it.setType("SINGLE_LINE_TEXT")))
171
+ .put(build("STRING", it -> it.setType("MULTI_LINE_TEXT")))
172
+ .put(build("STRING|NUMBER", it -> it.setType("NUMBER")))
173
+ .put(build("STRING|CHECK_BOX", it -> it.setType("CHECK_BOX").setValueSeparator(",")))
174
+ .put(build("STRING|DROP_DOWN", it -> it.setType("DROP_DOWN")))
175
+ .put(build("STRING|LINK", it -> it.setType("LINK")))
176
+ .put(build("TIMESTAMP|SINGLE_LINE_TEXT", it -> it.setType("SINGLE_LINE_TEXT")))
177
+ .put(build("TIMESTAMP|DATE", it -> it.setType("DATE").setTimezone("UTC")))
178
+ .put(build("TIMESTAMP|DATE|JST", it -> it.setType("DATE").setTimezone("Asia/Tokyo")))
179
+ .put(build("TIMESTAMP|DATE|PST", it -> it.setType("DATE").setTimezone("US/Pacific")))
180
+ .put(build("TIMESTAMP", it -> it.setType("DATETIME")))
181
+ .put(build("JSON|SINGLE_LINE_TEXT", it -> it.setType("SINGLE_LINE_TEXT")))
182
+ .put(build("JSON", it -> it.setType("MULTI_LINE_TEXT")))
183
+ .build();
184
+ }
185
+
186
+ private static Map.Entry<String, KintoneColumnOption> build(
187
+ String name, Function<KintoneColumnOptionBuilder, KintoneColumnOptionBuilder> function) {
188
+ return new AbstractMap.SimpleImmutableEntry<>(
189
+ name, function.apply(new KintoneColumnOptionBuilder().setFieldCode(name)).build());
190
+ }
191
+
192
+ private static Page build(OutputPageBuilder builder) {
193
+ return builder
194
+ .setNull("BOOLEAN|SINGLE_LINE_TEXT")
195
+ // .setNull("BOOLEAN")
196
+ .setNull("LONG|SINGLE_LINE_TEXT")
197
+ .setNull("LONG")
198
+ .setNull("DOUBLE|SINGLE_LINE_TEXT")
199
+ .setNull("DOUBLE")
200
+ .setNull("STRING|SINGLE_LINE_TEXT")
201
+ .setNull("STRING")
202
+ .setNull("STRING|NUMBER")
203
+ .setNull("STRING|CHECK_BOX")
204
+ .setNull("STRING|DROP_DOWN")
205
+ .setNull("STRING|LINK")
206
+ .setNull("TIMESTAMP|SINGLE_LINE_TEXT")
207
+ .setNull("TIMESTAMP|DATE")
208
+ .setNull("TIMESTAMP|DATE|JST")
209
+ .setNull("TIMESTAMP|DATE|PST")
210
+ .setNull("TIMESTAMP")
211
+ .setNull("JSON|SINGLE_LINE_TEXT")
212
+ .setNull("JSON")
213
+ .addRecord()
214
+ .setBoolean("BOOLEAN|SINGLE_LINE_TEXT", false)
215
+ // .setBoolean("BOOLEAN", false)
216
+ .setLong("LONG|SINGLE_LINE_TEXT", 0)
217
+ .setLong("LONG", 0)
218
+ .setDouble("DOUBLE|SINGLE_LINE_TEXT", 0)
219
+ .setDouble("DOUBLE", 0)
220
+ .setString("STRING|SINGLE_LINE_TEXT", "")
221
+ .setString("STRING", "")
222
+ .setString("STRING|NUMBER", "")
223
+ .setString("STRING|CHECK_BOX", "")
224
+ .setString("STRING|DROP_DOWN", "")
225
+ .setString("STRING|LINK", "")
226
+ .setTimestamp("TIMESTAMP|SINGLE_LINE_TEXT", Timestamp.ofInstant(Instant.EPOCH))
227
+ .setTimestamp("TIMESTAMP|DATE", Timestamp.ofInstant(Instant.EPOCH))
228
+ .setTimestamp("TIMESTAMP|DATE|JST", Timestamp.ofInstant(Instant.EPOCH))
229
+ .setTimestamp("TIMESTAMP|DATE|PST", Timestamp.ofInstant(Instant.EPOCH))
230
+ .setTimestamp("TIMESTAMP", Timestamp.ofInstant(Instant.EPOCH))
231
+ .setJson("JSON|SINGLE_LINE_TEXT", ValueFactory.newString(""))
232
+ .setJson("JSON", ValueFactory.newString(""))
233
+ .addRecord()
234
+ .setBoolean("BOOLEAN|SINGLE_LINE_TEXT", true)
235
+ // .setBoolean("BOOLEAN", true)
236
+ .setLong("LONG|SINGLE_LINE_TEXT", 123)
237
+ .setLong("LONG", 123)
238
+ .setDouble("DOUBLE|SINGLE_LINE_TEXT", 123)
239
+ .setDouble("DOUBLE", 123)
240
+ .setString("STRING|SINGLE_LINE_TEXT", "abc")
241
+ .setString("STRING", "abc")
242
+ .setString("STRING|NUMBER", "123")
243
+ .setString("STRING|CHECK_BOX", "123,abc")
244
+ .setString("STRING|DROP_DOWN", "abc")
245
+ .setString("STRING|LINK", "abc")
246
+ .setTimestamp("TIMESTAMP|SINGLE_LINE_TEXT", timestamp("1999-12-31T23:59:59Z"))
247
+ .setTimestamp("TIMESTAMP|DATE", timestamp("1999-12-31T23:59:59Z"))
248
+ .setTimestamp("TIMESTAMP|DATE|JST", timestamp("1999-12-31T23:59:59Z"))
249
+ .setTimestamp("TIMESTAMP|DATE|PST", timestamp("1999-12-31T23:59:59Z"))
250
+ .setTimestamp("TIMESTAMP", timestamp("1999-12-31T23:59:59Z"))
251
+ .setJson("JSON|SINGLE_LINE_TEXT", ValueFactory.newString("abc"))
252
+ .setJson("JSON", ValueFactory.newString("abc"))
253
+ .addRecord()
254
+ .setBoolean("BOOLEAN|SINGLE_LINE_TEXT", false)
255
+ // .setBoolean("BOOLEAN", false)
256
+ .setLong("LONG|SINGLE_LINE_TEXT", 456)
257
+ .setLong("LONG", 456)
258
+ .setDouble("DOUBLE|SINGLE_LINE_TEXT", 456)
259
+ .setDouble("DOUBLE", 456)
260
+ .setString("STRING|SINGLE_LINE_TEXT", "def")
261
+ .setString("STRING", "def")
262
+ .setString("STRING|NUMBER", "456")
263
+ .setString("STRING|CHECK_BOX", "456,def")
264
+ .setString("STRING|DROP_DOWN", "def")
265
+ .setString("STRING|LINK", "def")
266
+ .setTimestamp("TIMESTAMP|SINGLE_LINE_TEXT", timestamp("2000-01-01T00:00:00Z"))
267
+ .setTimestamp("TIMESTAMP|DATE", timestamp("2000-01-01T00:00:00Z"))
268
+ .setTimestamp("TIMESTAMP|DATE|JST", timestamp("2000-01-01T00:00:00Z"))
269
+ .setTimestamp("TIMESTAMP|DATE|PST", timestamp("2000-01-01T00:00:00Z"))
270
+ .setTimestamp("TIMESTAMP", timestamp("2000-01-01T00:00:00Z"))
271
+ .setJson("JSON|SINGLE_LINE_TEXT", ValueFactory.newString("def"))
272
+ .setJson("JSON", ValueFactory.newString("def"))
273
+ .addRecord()
274
+ .build();
275
+ }
276
+
277
+ private static BigDecimal number(String value) {
278
+ return new BigDecimal(value);
279
+ }
280
+
281
+ @SafeVarargs
282
+ private static <T> List<T> list(T... a) {
283
+ return Arrays.asList(a);
284
+ }
285
+
286
+ private static LocalDate date(CharSequence text) {
287
+ return LocalDate.parse(text);
288
+ }
289
+
290
+ private static ZonedDateTime dateTime(CharSequence text) {
291
+ return ZonedDateTime.parse(text).withZoneSameInstant(ZoneId.of("UTC"));
292
+ }
293
+
294
+ private static Timestamp timestamp(CharSequence text) {
295
+ return Timestamp.ofInstant(Instant.parse(text));
296
+ }
297
+ }
data/src/test/java/org/embulk/output/kintone/KintoneColumnVisitorVerifier.java ADDED
@@ -0,0 +1,52 @@
1
+ package org.embulk.output.kintone;
2
+
3
+ import com.kintone.client.model.record.FieldType;
4
+ import com.kintone.client.model.record.Record;
5
+ import com.kintone.client.model.record.UpdateKey;
6
+ import java.util.Map;
7
+ import java.util.function.BiConsumer;
8
+ import org.embulk.spi.Column;
9
+ import org.embulk.spi.Page;
10
+ import org.embulk.spi.PageReader;
11
+ import org.embulk.spi.Schema;
12
+
13
+ public class KintoneColumnVisitorVerifier {
14
+ private final Schema schema;
15
+ private final Map<String, KintoneColumnOption> columnOptions;
16
+ private final PageReader pageReader;
17
+ private final KintoneColumnVisitor visitor;
18
+
19
+ public KintoneColumnVisitorVerifier(
20
+ Schema schema,
21
+ Map<String, KintoneColumnOption> columnOptions,
22
+ String updateKeyName,
23
+ Page page) {
24
+ this.schema = schema;
25
+ this.columnOptions = columnOptions;
26
+ pageReader = new PageReader(schema);
27
+ pageReader.setPage(page);
28
+ visitor = new KintoneColumnVisitor(pageReader, columnOptions, updateKeyName);
29
+ }
30
+
31
+ public void verify(BiConsumer<Record, UpdateKey> consumer) {
32
+ if (!pageReader.nextRecord()) {
33
+ throw new IllegalStateException();
34
+ }
35
+ Record record = new Record();
36
+ visitor.setRecord(record);
37
+ UpdateKey updateKey = new UpdateKey();
38
+ visitor.setUpdateKey(updateKey);
39
+ schema.visitColumns(visitor);
40
+ schema.getColumns().forEach(column -> verify(record, column));
41
+ consumer.accept(record, updateKey);
42
+ }
43
+
44
+ private void verify(Record record, Column column) {
45
+ FieldType expected = FieldType.valueOf(columnOptions.get(column.getName()).getType());
46
+ FieldType actual = record.getFieldType(column.getName());
47
+ if (!expected.equals(actual)) {
48
+ System.out.printf(
49
+ "%s: Expected type is %s, but actual type is %s%n", column.getName(), expected, actual);
50
+ }
51
+ }
52
+ }
data/src/test/java/org/embulk/output/kintone/KintonePageOutputVerifier.java ADDED
@@ -0,0 +1,226 @@
1
+ package org.embulk.output.kintone;
2
+
3
+ import static org.hamcrest.MatcherAssert.assertThat;
4
+ import static org.hamcrest.Matchers.is;
5
+ import static org.mockito.ArgumentMatchers.anyList;
6
+ import static org.mockito.ArgumentMatchers.eq;
7
+ import static org.mockito.ArgumentMatchers.matches;
8
+ import static org.mockito.Mockito.atLeast;
9
+ import static org.mockito.Mockito.mock;
10
+ import static org.mockito.Mockito.mockStatic;
11
+ import static org.mockito.Mockito.verify;
12
+ import static org.mockito.Mockito.when;
13
+
14
+ import com.kintone.client.AppClient;
15
+ import com.kintone.client.KintoneClient;
16
+ import com.kintone.client.KintoneClientBuilder;
17
+ import com.kintone.client.RecordClient;
18
+ import com.kintone.client.api.record.GetRecordsByCursorResponseBody;
19
+ import com.kintone.client.model.app.field.FieldProperty;
20
+ import com.kintone.client.model.app.field.NumberFieldProperty;
21
+ import com.kintone.client.model.app.field.SingleLineTextFieldProperty;
22
+ import com.kintone.client.model.record.DateTimeFieldValue;
23
+ import com.kintone.client.model.record.FieldType;
24
+ import com.kintone.client.model.record.FieldValue;
25
+ import com.kintone.client.model.record.NumberFieldValue;
26
+ import com.kintone.client.model.record.Record;
27
+ import com.kintone.client.model.record.RecordForUpdate;
28
+ import com.kintone.client.model.record.SingleLineTextFieldValue;
29
+ import com.kintone.client.model.record.UpdateKey;
30
+ import java.math.BigDecimal;
31
+ import java.time.ZoneId;
32
+ import java.util.Collection;
33
+ import java.util.Collections;
34
+ import java.util.List;
35
+ import java.util.Map;
36
+ import java.util.stream.Collectors;
37
+ import java.util.stream.IntStream;
38
+ import org.embulk.config.TaskReport;
39
+ import org.embulk.spi.Page;
40
+ import org.embulk.spi.TransactionalPageOutput;
41
+ import org.mockito.ArgumentCaptor;
42
+ import org.mockito.MockedStatic;
43
+
44
+ public class KintonePageOutputVerifier implements TransactionalPageOutput {
45
+ private final TransactionalPageOutput transactionalPageOutput;
46
+ private final String domain;
47
+ private final String field;
48
+ private final List<String> values;
49
+ private final List<Record> addRecords;
50
+ private final List<RecordForUpdate> updateRecords;
51
+
52
+ public KintonePageOutputVerifier(
53
+ TransactionalPageOutput transactionalPageOutput,
54
+ String domain,
55
+ String field,
56
+ List<String> values,
57
+ List<Record> addRecords,
58
+ List<RecordForUpdate> updateRecords) {
59
+ this.transactionalPageOutput = transactionalPageOutput;
60
+ this.domain = domain;
61
+ this.field = field;
62
+ this.values = values;
63
+ this.addRecords = addRecords;
64
+ this.updateRecords = updateRecords;
65
+ }
66
+
67
+ @Override
68
+ public void add(Page page) {
69
+ runWithMock(() -> transactionalPageOutput.add(page));
70
+ }
71
+
72
+ @Override
73
+ public void finish() {
74
+ transactionalPageOutput.finish();
75
+ }
76
+
77
+ @Override
78
+ public void close() {
79
+ transactionalPageOutput.close();
80
+ }
81
+
82
+ @Override
83
+ public void abort() {
84
+ transactionalPageOutput.abort();
85
+ }
86
+
87
+ @Override
88
+ public TaskReport commit() {
89
+ return transactionalPageOutput.commit();
90
+ }
91
+
92
+ public void runWithMock(Runnable runnable) {
93
+ try {
94
+ runWithMockClient(runnable);
95
+ } catch (Exception e) {
96
+ throw new RuntimeException(e);
97
+ }
98
+ }
99
+
100
+ private void runWithMockClient(Runnable runnable) throws Exception {
101
+ @SuppressWarnings("unchecked")
102
+ Map<String, FieldProperty> mockFormFields = mock(Map.class);
103
+ when(mockFormFields.get(matches("^.*_single_line_text$")))
104
+ .thenReturn(new SingleLineTextFieldProperty());
105
+ when(mockFormFields.get(matches("^.*_number$"))).thenReturn(new NumberFieldProperty());
106
+ AppClient mockAppClient = mock(AppClient.class);
107
+ when(mockAppClient.getFormFields(eq(0L))).thenReturn(mockFormFields);
108
+ GetRecordsByCursorResponseBody mockGetRecordsByCursorResponseBody =
109
+ mock(GetRecordsByCursorResponseBody.class);
110
+ when(mockGetRecordsByCursorResponseBody.getRecords())
111
+ .thenReturn(updateRecords.stream().map(this::getRecord).collect(Collectors.toList()));
112
+ when(mockGetRecordsByCursorResponseBody.hasNext()).thenReturn(false);
113
+ RecordClient mockRecordClient = mock(RecordClient.class);
114
+ when(mockRecordClient.createCursor(eq(0L), eq(getFields()), eq(getQuery()))).thenReturn("id");
115
+ when(mockRecordClient.getRecordsByCursor(eq("id")))
116
+ .thenReturn(mockGetRecordsByCursorResponseBody);
117
+ when(mockRecordClient.addRecords(eq(0L), anyList())).thenReturn(Collections.emptyList());
118
+ when(mockRecordClient.updateRecords(eq(0L), anyList())).thenReturn(Collections.emptyList());
119
+ KintoneClient mockKintoneClient = mock(KintoneClient.class);
120
+ when(mockKintoneClient.app()).thenReturn(mockAppClient);
121
+ when(mockKintoneClient.record()).thenReturn(mockRecordClient);
122
+ KintoneClientBuilder mockKintoneClientBuilder = mock(KintoneClientBuilder.class);
123
+ when(mockKintoneClientBuilder.authByApiToken(eq("token"))).thenReturn(mockKintoneClientBuilder);
124
+ when(mockKintoneClientBuilder.build()).thenReturn(mockKintoneClient);
125
+ try (MockedStatic<KintoneClientBuilder> mocked = mockStatic(KintoneClientBuilder.class)) {
126
+ mocked
127
+ .when(() -> KintoneClientBuilder.create(String.format("https://%s", domain)))
128
+ .thenReturn(mockKintoneClientBuilder);
129
+ runnable.run();
130
+ }
131
+ @SuppressWarnings("unchecked")
132
+ ArgumentCaptor<List<Record>> addRecordsArgumentCaptor = ArgumentCaptor.forClass(List.class);
133
+ verify(mockRecordClient, atLeast(0)).addRecords(eq(0L), addRecordsArgumentCaptor.capture());
134
+ assertRecords(
135
+ domain,
136
+ addRecordsArgumentCaptor.getAllValues().stream()
137
+ .flatMap(Collection::stream)
138
+ .collect(Collectors.toList()),
139
+ addRecords);
140
+ @SuppressWarnings("unchecked")
141
+ ArgumentCaptor<List<RecordForUpdate>> updateRecordsArgumentCaptor =
142
+ ArgumentCaptor.forClass(List.class);
143
+ verify(mockRecordClient, atLeast(0))
144
+ .updateRecords(eq(0L), updateRecordsArgumentCaptor.capture());
145
+ assertRecordForUpdates(
146
+ domain,
147
+ updateRecordsArgumentCaptor.getAllValues().stream()
148
+ .flatMap(Collection::stream)
149
+ .collect(Collectors.toList()),
150
+ updateRecords);
151
+ }
152
+
153
+ private Record getRecord(RecordForUpdate updateRecord) {
154
+ UpdateKey key = updateRecord.getUpdateKey();
155
+ String field = key.getField();
156
+ Object value = key.getValue();
157
+ return new Record()
158
+ .putField(
159
+ field,
160
+ field.matches("^.*_number$")
161
+ ? new NumberFieldValue((BigDecimal) value)
162
+ : new SingleLineTextFieldValue((String) value));
163
+ }
164
+
165
+ private List<String> getFields() {
166
+ return Collections.singletonList(field);
167
+ }
168
+
169
+ private String getQuery() {
170
+ return String.format("%s in (%s)", field, String.join(",", values));
171
+ }
172
+
173
+ private static void assertRecords(String domain, List<Record> actual, List<Record> expected) {
174
+ assertThat(domain, actual.size(), is(expected.size()));
175
+ // spotless:off
176
+ IntStream.range(0, actual.size()).forEach(index -> assertRecord(domain, index, actual.get(index), expected.get(index)));
177
+ // spotless:on
178
+ }
179
+
180
+ private static void assertRecord(String domain, int index, Record actual, Record expected) {
181
+ String reason = String.format("%s:%d", domain, index);
182
+ assertThat(reason, actual.getId(), is(expected.getId()));
183
+ assertThat(reason, actual.getRevision(), is(expected.getRevision()));
184
+ assertThat(reason, actual.getFieldCodes(true), is(expected.getFieldCodes(true)));
185
+ // spotless:off
186
+ actual.getFieldCodes(true).forEach(fieldCode -> assertFieldValue(domain, index, fieldCode, actual.getFieldValue(fieldCode), expected.getFieldValue(fieldCode)));
187
+ // spotless:on
188
+ }
189
+
190
+ private static void assertFieldValue(
191
+ String domain, int index, String fieldCode, FieldValue actual, FieldValue expected) {
192
+ String reason = String.format("%s:%d:%s", domain, index, fieldCode);
193
+ assertThat(reason, actual.getType(), is(expected.getType()));
194
+ // spotless:off
195
+ assertThat(reason, actual, is(expected.getType().equals(FieldType.DATETIME) ? new DateTimeFieldValue(((DateTimeFieldValue) expected).getValue().withZoneSameInstant(ZoneId.of("UTC"))) : expected));
196
+ // spotless:on
197
+ }
198
+
199
+ private static void assertRecordForUpdates(
200
+ String domain, List<RecordForUpdate> actual, List<RecordForUpdate> expected) {
201
+ assertThat(domain, actual.size(), is(expected.size()));
202
+ // spotless:off
203
+ IntStream.range(0, actual.size()).forEach(index -> assertRecordForUpdate(domain, index, actual.get(index), expected.get(index)));
204
+ // spotless:on
205
+ }
206
+
207
+ private static void assertRecordForUpdate(
208
+ String domain, int index, RecordForUpdate actual, RecordForUpdate expected) {
209
+ String reason = String.format("%s:%d", domain, index);
210
+ assertThat(reason, actual.getId(), is(expected.getId()));
211
+ assertUpdateKey(domain, index, actual.getUpdateKey(), expected.getUpdateKey());
212
+ assertRecord(domain, index, actual.getRecord(), expected.getRecord());
213
+ assertThat(reason, actual.getRevision(), is(expected.getRevision()));
214
+ }
215
+
216
+ private static void assertUpdateKey(
217
+ String domain, int index, UpdateKey actual, UpdateKey expected) {
218
+ String reason = String.format("%s:%d", domain, index);
219
+ assertThat(reason, actual.getField(), is(expected.getField()));
220
+ assertThat(reason, actual.getValue(), is(expected.getValue().toString()));
221
+ }
222
+
223
+ public interface Runnable {
224
+ void run() throws Exception;
225
+ }
226
+ }
data/src/test/java/org/embulk/output/kintone/OutputPageBuilder.java ADDED
@@ -0,0 +1,92 @@
1
+ package org.embulk.output.kintone;
2
+
3
+ import java.util.List;
4
+ import java.util.function.Function;
5
+ import java.util.stream.Collectors;
6
+ import org.embulk.deps.buffer.PooledBufferAllocator;
7
+ import org.embulk.spi.Column;
8
+ import org.embulk.spi.Page;
9
+ import org.embulk.spi.PageBuilder;
10
+ import org.embulk.spi.PageOutput;
11
+ import org.embulk.spi.Schema;
12
+ import org.embulk.spi.time.Timestamp;
13
+ import org.msgpack.value.Value;
14
+
15
+ public class OutputPageBuilder implements PageOutput {
16
+ private final List<String> names;
17
+ private final PageBuilder builder;
18
+ private Page page;
19
+
20
+ public static Page build(Schema schema, Function<OutputPageBuilder, Page> function) {
21
+ Page page;
22
+ try (OutputPageBuilder builder = new OutputPageBuilder(schema)) {
23
+ page = function.apply(builder);
24
+ }
25
+ return page;
26
+ }
27
+
28
+ public OutputPageBuilder(Schema schema) {
29
+ names = schema.getColumns().stream().map(Column::getName).collect(Collectors.toList());
30
+ builder = new PageBuilder(PooledBufferAllocator.create(), schema, this);
31
+ }
32
+
33
+ public OutputPageBuilder setNull(String name) {
34
+ builder.setNull(names.indexOf(name));
35
+ return this;
36
+ }
37
+
38
+ public OutputPageBuilder setBoolean(String name, boolean value) {
39
+ builder.setBoolean(names.indexOf(name), value);
40
+ return this;
41
+ }
42
+
43
+ public OutputPageBuilder setLong(String name, long value) {
44
+ builder.setLong(names.indexOf(name), value);
45
+ return this;
46
+ }
47
+
48
+ public OutputPageBuilder setDouble(String name, double value) {
49
+ builder.setDouble(names.indexOf(name), value);
50
+ return this;
51
+ }
52
+
53
+ public OutputPageBuilder setString(String name, String value) {
54
+ builder.setString(names.indexOf(name), value);
55
+ return this;
56
+ }
57
+
58
+ public OutputPageBuilder setJson(String name, Value value) {
59
+ builder.setJson(names.indexOf(name), value);
60
+ return this;
61
+ }
62
+
63
+ public OutputPageBuilder setTimestamp(String name, Timestamp value) {
64
+ builder.setTimestamp(names.indexOf(name), value);
65
+ return this;
66
+ }
67
+
68
+ public OutputPageBuilder addRecord() {
69
+ builder.addRecord();
70
+ return this;
71
+ }
72
+
73
+ public Page build() {
74
+ builder.flush();
75
+ builder.close();
76
+ return page;
77
+ }
78
+
79
+ @Override
80
+ public void add(Page page) {
81
+ if (this.page != null) {
82
+ throw new IllegalStateException();
83
+ }
84
+ this.page = page;
85
+ }
86
+
87
+ @Override
88
+ public void finish() {}
89
+
90
+ @Override
91
+ public void close() {}
92
+ }