embulk-output-kintone 0.3.6 → 0.4.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/ci.yml +1 -0
- data/.gitignore +0 -1
- data/build.gradle +18 -19
- data/classpath/embulk-output-kintone-0.4.1.jar +0 -0
- data/classpath/shadow-kintone-java-client-0.4.1-all.jar +0 -0
- data/gradle/wrapper/gradle-wrapper.properties +1 -1
- data/settings.gradle +2 -0
- data/shadow-kintone-java-client/build.gradle +35 -0
- data/src/main/java/org/embulk/output/kintone/KintonePageOutput.java +168 -105
- data/src/main/java/org/embulk/output/kintone/KintoneRetryOption.java +19 -0
- data/src/main/java/org/embulk/output/kintone/PluginTask.java +4 -0
- data/src/test/java/com/kintone/client/Json.java +16 -0
- data/src/test/java/org/embulk/output/kintone/KintoneColumnOptionBuilder.java +63 -0
- data/src/test/java/org/embulk/output/kintone/KintoneColumnVisitorTest.java +297 -0
- data/src/test/java/org/embulk/output/kintone/KintoneColumnVisitorVerifier.java +52 -0
- data/src/test/java/org/embulk/output/kintone/KintonePageOutputVerifier.java +226 -0
- data/src/test/java/org/embulk/output/kintone/OutputPageBuilder.java +92 -0
- data/src/test/java/org/embulk/output/kintone/TestKintoneOutputPlugin.java +163 -1
- data/src/test/java/org/embulk/output/kintone/TestTask.java +40 -0
- data/src/test/java/org/embulk/output/kintone/TestTaskMode.java +31 -0
- data/src/test/resources/logback-test.xml +14 -0
- data/src/test/resources/org/embulk/output/kintone/config.yml +4 -0
- data/src/test/resources/org/embulk/output/kintone/task/config.yml +1 -0
- data/src/test/resources/org/embulk/output/kintone/task/mode/config.yml +54 -0
- data/src/test/resources/org/embulk/output/kintone/task/mode/input.csv +7 -0
- data/src/test/resources/org/embulk/output/kintone/task/mode/insert_add_records.jsonl +6 -0
- data/src/test/resources/org/embulk/output/kintone/task/mode/update_update_records.jsonl +3 -0
- data/src/test/resources/org/embulk/output/kintone/task/mode/upsert_add_records.jsonl +2 -0
- data/src/test/resources/org/embulk/output/kintone/task/mode/upsert_update_records.jsonl +4 -0
- data/src/test/resources/org/embulk/output/kintone/task/mode/values.json +1 -0
- metadata +25 -3
- data/classpath/embulk-output-kintone-0.3.6.jar +0 -0
@@ -0,0 +1,297 @@
|
|
1
|
+
package org.embulk.output.kintone;
|
2
|
+
|
3
|
+
import static org.hamcrest.MatcherAssert.assertThat;
|
4
|
+
import static org.hamcrest.Matchers.is;
|
5
|
+
import static org.hamcrest.Matchers.nullValue;
|
6
|
+
|
7
|
+
import com.google.common.collect.ImmutableMap;
|
8
|
+
import java.math.BigDecimal;
|
9
|
+
import java.time.Instant;
|
10
|
+
import java.time.LocalDate;
|
11
|
+
import java.time.ZoneId;
|
12
|
+
import java.time.ZonedDateTime;
|
13
|
+
import java.util.AbstractMap;
|
14
|
+
import java.util.Arrays;
|
15
|
+
import java.util.List;
|
16
|
+
import java.util.Map;
|
17
|
+
import java.util.function.Function;
|
18
|
+
import org.embulk.spi.Page;
|
19
|
+
import org.embulk.spi.Schema;
|
20
|
+
import org.embulk.spi.time.Timestamp;
|
21
|
+
import org.embulk.spi.type.Types;
|
22
|
+
import org.junit.Test;
|
23
|
+
import org.msgpack.value.ValueFactory;
|
24
|
+
|
25
|
+
public class KintoneColumnVisitorTest {
|
26
|
+
@Test
|
27
|
+
public void test() {
|
28
|
+
Schema schema = build(Schema.builder());
|
29
|
+
KintoneColumnVisitorVerifier verifier =
|
30
|
+
new KintoneColumnVisitorVerifier(
|
31
|
+
schema,
|
32
|
+
build(ImmutableMap.builder()),
|
33
|
+
"STRING|SINGLE_LINE_TEXT",
|
34
|
+
OutputPageBuilder.build(schema, KintoneColumnVisitorTest::build));
|
35
|
+
verifier.verify(
|
36
|
+
(record, updateKey) -> {
|
37
|
+
assertThat(record.getSingleLineTextFieldValue("BOOLEAN|SINGLE_LINE_TEXT"), is("false"));
|
38
|
+
assertThat(record.getNumberFieldValue("BOOLEAN"), nullValue());
|
39
|
+
assertThat(record.getSingleLineTextFieldValue("LONG|SINGLE_LINE_TEXT"), is(""));
|
40
|
+
assertThat(record.getNumberFieldValue("LONG"), nullValue());
|
41
|
+
assertThat(record.getSingleLineTextFieldValue("DOUBLE|SINGLE_LINE_TEXT"), is("0.0"));
|
42
|
+
assertThat(record.getNumberFieldValue("DOUBLE"), is(number("0.0")));
|
43
|
+
assertThat(record.getSingleLineTextFieldValue("STRING|SINGLE_LINE_TEXT"), is(""));
|
44
|
+
assertThat(record.getMultiLineTextFieldValue("STRING"), is(""));
|
45
|
+
assertThat(record.getNumberFieldValue("STRING|NUMBER"), nullValue());
|
46
|
+
assertThat(record.getCheckBoxFieldValue("STRING|CHECK_BOX"), is(list("null")));
|
47
|
+
assertThat(record.getDropDownFieldValue("STRING|DROP_DOWN"), is(""));
|
48
|
+
assertThat(record.getLinkFieldValue("STRING|LINK"), is(""));
|
49
|
+
assertThat(record.getSingleLineTextFieldValue("TIMESTAMP|SINGLE_LINE_TEXT"), nullValue());
|
50
|
+
assertThat(record.getDateFieldValue("TIMESTAMP|DATE"), nullValue());
|
51
|
+
assertThat(record.getDateFieldValue("TIMESTAMP|DATE|JST"), nullValue());
|
52
|
+
assertThat(record.getDateFieldValue("TIMESTAMP|DATE|PST"), nullValue());
|
53
|
+
assertThat(record.getDateTimeFieldValue("TIMESTAMP"), nullValue());
|
54
|
+
assertThat(record.getSingleLineTextFieldValue("JSON|SINGLE_LINE_TEXT"), is(""));
|
55
|
+
assertThat(record.getMultiLineTextFieldValue("JSON"), is(""));
|
56
|
+
assertThat(updateKey.getField(), is("STRING|SINGLE_LINE_TEXT"));
|
57
|
+
assertThat(updateKey.getValue(), is(""));
|
58
|
+
});
|
59
|
+
verifier.verify(
|
60
|
+
(record, updateKey) -> {
|
61
|
+
assertThat(record.getSingleLineTextFieldValue("BOOLEAN|SINGLE_LINE_TEXT"), is("false"));
|
62
|
+
assertThat(record.getNumberFieldValue("BOOLEAN"), nullValue());
|
63
|
+
assertThat(record.getSingleLineTextFieldValue("LONG|SINGLE_LINE_TEXT"), is("0"));
|
64
|
+
assertThat(record.getNumberFieldValue("LONG"), is(number("0")));
|
65
|
+
assertThat(record.getSingleLineTextFieldValue("DOUBLE|SINGLE_LINE_TEXT"), is("0.0"));
|
66
|
+
assertThat(record.getNumberFieldValue("DOUBLE"), is(number("0.0")));
|
67
|
+
assertThat(record.getSingleLineTextFieldValue("STRING|SINGLE_LINE_TEXT"), is(""));
|
68
|
+
assertThat(record.getMultiLineTextFieldValue("STRING"), is(""));
|
69
|
+
assertThat(record.getNumberFieldValue("STRING|NUMBER"), nullValue());
|
70
|
+
assertThat(record.getCheckBoxFieldValue("STRING|CHECK_BOX"), is(list()));
|
71
|
+
assertThat(record.getDropDownFieldValue("STRING|DROP_DOWN"), is(""));
|
72
|
+
assertThat(record.getLinkFieldValue("STRING|LINK"), is(""));
|
73
|
+
assertThat(record.getSingleLineTextFieldValue("TIMESTAMP|SINGLE_LINE_TEXT"), nullValue());
|
74
|
+
assertThat(record.getDateFieldValue("TIMESTAMP|DATE"), is(date("1970-01-01")));
|
75
|
+
assertThat(record.getDateFieldValue("TIMESTAMP|DATE|JST"), is(date("1970-01-01")));
|
76
|
+
assertThat(record.getDateFieldValue("TIMESTAMP|DATE|PST"), is(date("1969-12-31")));
|
77
|
+
assertThat(
|
78
|
+
record.getDateTimeFieldValue("TIMESTAMP"), is(dateTime("1970-01-01T00:00:00Z")));
|
79
|
+
assertThat(record.getSingleLineTextFieldValue("JSON|SINGLE_LINE_TEXT"), is(""));
|
80
|
+
assertThat(record.getMultiLineTextFieldValue("JSON"), is(""));
|
81
|
+
assertThat(updateKey.getField(), is("STRING|SINGLE_LINE_TEXT"));
|
82
|
+
assertThat(updateKey.getValue(), is(""));
|
83
|
+
});
|
84
|
+
verifier.verify(
|
85
|
+
(record, updateKey) -> {
|
86
|
+
assertThat(record.getSingleLineTextFieldValue("BOOLEAN|SINGLE_LINE_TEXT"), is("true"));
|
87
|
+
assertThat(record.getNumberFieldValue("BOOLEAN"), nullValue());
|
88
|
+
assertThat(record.getSingleLineTextFieldValue("LONG|SINGLE_LINE_TEXT"), is("123"));
|
89
|
+
assertThat(record.getNumberFieldValue("LONG"), is(number("123")));
|
90
|
+
assertThat(record.getSingleLineTextFieldValue("DOUBLE|SINGLE_LINE_TEXT"), is("123.0"));
|
91
|
+
assertThat(record.getNumberFieldValue("DOUBLE"), is(number("123.0")));
|
92
|
+
assertThat(record.getSingleLineTextFieldValue("STRING|SINGLE_LINE_TEXT"), is("abc"));
|
93
|
+
assertThat(record.getMultiLineTextFieldValue("STRING"), is("abc"));
|
94
|
+
assertThat(record.getNumberFieldValue("STRING|NUMBER"), is(number("123")));
|
95
|
+
assertThat(record.getCheckBoxFieldValue("STRING|CHECK_BOX"), is(list("123", "abc")));
|
96
|
+
assertThat(record.getDropDownFieldValue("STRING|DROP_DOWN"), is("abc"));
|
97
|
+
assertThat(record.getLinkFieldValue("STRING|LINK"), is("abc"));
|
98
|
+
assertThat(record.getSingleLineTextFieldValue("TIMESTAMP|SINGLE_LINE_TEXT"), nullValue());
|
99
|
+
assertThat(record.getDateFieldValue("TIMESTAMP|DATE"), is(date("1999-12-31")));
|
100
|
+
assertThat(record.getDateFieldValue("TIMESTAMP|DATE|JST"), is(date("2000-01-01")));
|
101
|
+
assertThat(record.getDateFieldValue("TIMESTAMP|DATE|PST"), is(date("1999-12-31")));
|
102
|
+
assertThat(
|
103
|
+
record.getDateTimeFieldValue("TIMESTAMP"), is(dateTime("1999-12-31T23:59:59Z")));
|
104
|
+
assertThat(record.getSingleLineTextFieldValue("JSON|SINGLE_LINE_TEXT"), is("abc"));
|
105
|
+
assertThat(record.getMultiLineTextFieldValue("JSON"), is("abc"));
|
106
|
+
assertThat(updateKey.getField(), is("STRING|SINGLE_LINE_TEXT"));
|
107
|
+
assertThat(updateKey.getValue(), is("abc"));
|
108
|
+
});
|
109
|
+
verifier.verify(
|
110
|
+
(record, updateKey) -> {
|
111
|
+
assertThat(record.getSingleLineTextFieldValue("BOOLEAN|SINGLE_LINE_TEXT"), is("false"));
|
112
|
+
assertThat(record.getNumberFieldValue("BOOLEAN"), nullValue());
|
113
|
+
assertThat(record.getSingleLineTextFieldValue("LONG|SINGLE_LINE_TEXT"), is("456"));
|
114
|
+
assertThat(record.getNumberFieldValue("LONG"), is(number("456")));
|
115
|
+
assertThat(record.getSingleLineTextFieldValue("DOUBLE|SINGLE_LINE_TEXT"), is("456.0"));
|
116
|
+
assertThat(record.getNumberFieldValue("DOUBLE"), is(number("456.0")));
|
117
|
+
assertThat(record.getSingleLineTextFieldValue("STRING|SINGLE_LINE_TEXT"), is("def"));
|
118
|
+
assertThat(record.getMultiLineTextFieldValue("STRING"), is("def"));
|
119
|
+
assertThat(record.getNumberFieldValue("STRING|NUMBER"), is(number("456")));
|
120
|
+
assertThat(record.getCheckBoxFieldValue("STRING|CHECK_BOX"), is(list("456", "def")));
|
121
|
+
assertThat(record.getDropDownFieldValue("STRING|DROP_DOWN"), is("def"));
|
122
|
+
assertThat(record.getLinkFieldValue("STRING|LINK"), is("def"));
|
123
|
+
assertThat(record.getSingleLineTextFieldValue("TIMESTAMP|SINGLE_LINE_TEXT"), nullValue());
|
124
|
+
assertThat(record.getDateFieldValue("TIMESTAMP|DATE"), is(date("2000-01-01")));
|
125
|
+
assertThat(record.getDateFieldValue("TIMESTAMP|DATE|JST"), is(date("2000-01-01")));
|
126
|
+
assertThat(record.getDateFieldValue("TIMESTAMP|DATE|PST"), is(date("1999-12-31")));
|
127
|
+
assertThat(
|
128
|
+
record.getDateTimeFieldValue("TIMESTAMP"), is(dateTime("2000-01-01T00:00:00Z")));
|
129
|
+
assertThat(record.getSingleLineTextFieldValue("JSON|SINGLE_LINE_TEXT"), is("def"));
|
130
|
+
assertThat(record.getMultiLineTextFieldValue("JSON"), is("def"));
|
131
|
+
assertThat(updateKey.getField(), is("STRING|SINGLE_LINE_TEXT"));
|
132
|
+
assertThat(updateKey.getValue(), is("def"));
|
133
|
+
});
|
134
|
+
}
|
135
|
+
|
136
|
+
private static Schema build(Schema.Builder builder) {
|
137
|
+
return builder
|
138
|
+
.add("BOOLEAN|SINGLE_LINE_TEXT", Types.BOOLEAN)
|
139
|
+
// java.lang.NumberFormatException at java.math.BigDecimal.<init>
|
140
|
+
// .add("BOOLEAN", Types.BOOLEAN)
|
141
|
+
.add("LONG|SINGLE_LINE_TEXT", Types.LONG)
|
142
|
+
.add("LONG", Types.LONG)
|
143
|
+
.add("DOUBLE|SINGLE_LINE_TEXT", Types.DOUBLE)
|
144
|
+
.add("DOUBLE", Types.DOUBLE)
|
145
|
+
.add("STRING|SINGLE_LINE_TEXT", Types.STRING)
|
146
|
+
.add("STRING", Types.STRING)
|
147
|
+
.add("STRING|NUMBER", Types.STRING)
|
148
|
+
.add("STRING|CHECK_BOX", Types.STRING)
|
149
|
+
.add("STRING|DROP_DOWN", Types.STRING)
|
150
|
+
.add("STRING|LINK", Types.STRING)
|
151
|
+
.add("TIMESTAMP|SINGLE_LINE_TEXT", Types.TIMESTAMP)
|
152
|
+
.add("TIMESTAMP|DATE", Types.TIMESTAMP)
|
153
|
+
.add("TIMESTAMP|DATE|JST", Types.TIMESTAMP)
|
154
|
+
.add("TIMESTAMP|DATE|PST", Types.TIMESTAMP)
|
155
|
+
.add("TIMESTAMP", Types.TIMESTAMP)
|
156
|
+
.add("JSON|SINGLE_LINE_TEXT", Types.JSON)
|
157
|
+
.add("JSON", Types.JSON)
|
158
|
+
.build();
|
159
|
+
}
|
160
|
+
|
161
|
+
private static Map<String, KintoneColumnOption> build(
|
162
|
+
ImmutableMap.Builder<String, KintoneColumnOption> builder) {
|
163
|
+
return builder
|
164
|
+
.put(build("BOOLEAN|SINGLE_LINE_TEXT", it -> it.setType("SINGLE_LINE_TEXT")))
|
165
|
+
.put(build("BOOLEAN", it -> it.setType("NUMBER")))
|
166
|
+
.put(build("LONG|SINGLE_LINE_TEXT", it -> it.setType("SINGLE_LINE_TEXT")))
|
167
|
+
.put(build("LONG", it -> it.setType("NUMBER")))
|
168
|
+
.put(build("DOUBLE|SINGLE_LINE_TEXT", it -> it.setType("SINGLE_LINE_TEXT")))
|
169
|
+
.put(build("DOUBLE", it -> it.setType("NUMBER")))
|
170
|
+
.put(build("STRING|SINGLE_LINE_TEXT", it -> it.setType("SINGLE_LINE_TEXT")))
|
171
|
+
.put(build("STRING", it -> it.setType("MULTI_LINE_TEXT")))
|
172
|
+
.put(build("STRING|NUMBER", it -> it.setType("NUMBER")))
|
173
|
+
.put(build("STRING|CHECK_BOX", it -> it.setType("CHECK_BOX").setValueSeparator(",")))
|
174
|
+
.put(build("STRING|DROP_DOWN", it -> it.setType("DROP_DOWN")))
|
175
|
+
.put(build("STRING|LINK", it -> it.setType("LINK")))
|
176
|
+
.put(build("TIMESTAMP|SINGLE_LINE_TEXT", it -> it.setType("SINGLE_LINE_TEXT")))
|
177
|
+
.put(build("TIMESTAMP|DATE", it -> it.setType("DATE").setTimezone("UTC")))
|
178
|
+
.put(build("TIMESTAMP|DATE|JST", it -> it.setType("DATE").setTimezone("Asia/Tokyo")))
|
179
|
+
.put(build("TIMESTAMP|DATE|PST", it -> it.setType("DATE").setTimezone("US/Pacific")))
|
180
|
+
.put(build("TIMESTAMP", it -> it.setType("DATETIME")))
|
181
|
+
.put(build("JSON|SINGLE_LINE_TEXT", it -> it.setType("SINGLE_LINE_TEXT")))
|
182
|
+
.put(build("JSON", it -> it.setType("MULTI_LINE_TEXT")))
|
183
|
+
.build();
|
184
|
+
}
|
185
|
+
|
186
|
+
private static Map.Entry<String, KintoneColumnOption> build(
|
187
|
+
String name, Function<KintoneColumnOptionBuilder, KintoneColumnOptionBuilder> function) {
|
188
|
+
return new AbstractMap.SimpleImmutableEntry<>(
|
189
|
+
name, function.apply(new KintoneColumnOptionBuilder().setFieldCode(name)).build());
|
190
|
+
}
|
191
|
+
|
192
|
+
private static Page build(OutputPageBuilder builder) {
|
193
|
+
return builder
|
194
|
+
.setNull("BOOLEAN|SINGLE_LINE_TEXT")
|
195
|
+
// .setNull("BOOLEAN")
|
196
|
+
.setNull("LONG|SINGLE_LINE_TEXT")
|
197
|
+
.setNull("LONG")
|
198
|
+
.setNull("DOUBLE|SINGLE_LINE_TEXT")
|
199
|
+
.setNull("DOUBLE")
|
200
|
+
.setNull("STRING|SINGLE_LINE_TEXT")
|
201
|
+
.setNull("STRING")
|
202
|
+
.setNull("STRING|NUMBER")
|
203
|
+
.setNull("STRING|CHECK_BOX")
|
204
|
+
.setNull("STRING|DROP_DOWN")
|
205
|
+
.setNull("STRING|LINK")
|
206
|
+
.setNull("TIMESTAMP|SINGLE_LINE_TEXT")
|
207
|
+
.setNull("TIMESTAMP|DATE")
|
208
|
+
.setNull("TIMESTAMP|DATE|JST")
|
209
|
+
.setNull("TIMESTAMP|DATE|PST")
|
210
|
+
.setNull("TIMESTAMP")
|
211
|
+
.setNull("JSON|SINGLE_LINE_TEXT")
|
212
|
+
.setNull("JSON")
|
213
|
+
.addRecord()
|
214
|
+
.setBoolean("BOOLEAN|SINGLE_LINE_TEXT", false)
|
215
|
+
// .setBoolean("BOOLEAN", false)
|
216
|
+
.setLong("LONG|SINGLE_LINE_TEXT", 0)
|
217
|
+
.setLong("LONG", 0)
|
218
|
+
.setDouble("DOUBLE|SINGLE_LINE_TEXT", 0)
|
219
|
+
.setDouble("DOUBLE", 0)
|
220
|
+
.setString("STRING|SINGLE_LINE_TEXT", "")
|
221
|
+
.setString("STRING", "")
|
222
|
+
.setString("STRING|NUMBER", "")
|
223
|
+
.setString("STRING|CHECK_BOX", "")
|
224
|
+
.setString("STRING|DROP_DOWN", "")
|
225
|
+
.setString("STRING|LINK", "")
|
226
|
+
.setTimestamp("TIMESTAMP|SINGLE_LINE_TEXT", Timestamp.ofInstant(Instant.EPOCH))
|
227
|
+
.setTimestamp("TIMESTAMP|DATE", Timestamp.ofInstant(Instant.EPOCH))
|
228
|
+
.setTimestamp("TIMESTAMP|DATE|JST", Timestamp.ofInstant(Instant.EPOCH))
|
229
|
+
.setTimestamp("TIMESTAMP|DATE|PST", Timestamp.ofInstant(Instant.EPOCH))
|
230
|
+
.setTimestamp("TIMESTAMP", Timestamp.ofInstant(Instant.EPOCH))
|
231
|
+
.setJson("JSON|SINGLE_LINE_TEXT", ValueFactory.newString(""))
|
232
|
+
.setJson("JSON", ValueFactory.newString(""))
|
233
|
+
.addRecord()
|
234
|
+
.setBoolean("BOOLEAN|SINGLE_LINE_TEXT", true)
|
235
|
+
// .setBoolean("BOOLEAN", true)
|
236
|
+
.setLong("LONG|SINGLE_LINE_TEXT", 123)
|
237
|
+
.setLong("LONG", 123)
|
238
|
+
.setDouble("DOUBLE|SINGLE_LINE_TEXT", 123)
|
239
|
+
.setDouble("DOUBLE", 123)
|
240
|
+
.setString("STRING|SINGLE_LINE_TEXT", "abc")
|
241
|
+
.setString("STRING", "abc")
|
242
|
+
.setString("STRING|NUMBER", "123")
|
243
|
+
.setString("STRING|CHECK_BOX", "123,abc")
|
244
|
+
.setString("STRING|DROP_DOWN", "abc")
|
245
|
+
.setString("STRING|LINK", "abc")
|
246
|
+
.setTimestamp("TIMESTAMP|SINGLE_LINE_TEXT", timestamp("1999-12-31T23:59:59Z"))
|
247
|
+
.setTimestamp("TIMESTAMP|DATE", timestamp("1999-12-31T23:59:59Z"))
|
248
|
+
.setTimestamp("TIMESTAMP|DATE|JST", timestamp("1999-12-31T23:59:59Z"))
|
249
|
+
.setTimestamp("TIMESTAMP|DATE|PST", timestamp("1999-12-31T23:59:59Z"))
|
250
|
+
.setTimestamp("TIMESTAMP", timestamp("1999-12-31T23:59:59Z"))
|
251
|
+
.setJson("JSON|SINGLE_LINE_TEXT", ValueFactory.newString("abc"))
|
252
|
+
.setJson("JSON", ValueFactory.newString("abc"))
|
253
|
+
.addRecord()
|
254
|
+
.setBoolean("BOOLEAN|SINGLE_LINE_TEXT", false)
|
255
|
+
// .setBoolean("BOOLEAN", false)
|
256
|
+
.setLong("LONG|SINGLE_LINE_TEXT", 456)
|
257
|
+
.setLong("LONG", 456)
|
258
|
+
.setDouble("DOUBLE|SINGLE_LINE_TEXT", 456)
|
259
|
+
.setDouble("DOUBLE", 456)
|
260
|
+
.setString("STRING|SINGLE_LINE_TEXT", "def")
|
261
|
+
.setString("STRING", "def")
|
262
|
+
.setString("STRING|NUMBER", "456")
|
263
|
+
.setString("STRING|CHECK_BOX", "456,def")
|
264
|
+
.setString("STRING|DROP_DOWN", "def")
|
265
|
+
.setString("STRING|LINK", "def")
|
266
|
+
.setTimestamp("TIMESTAMP|SINGLE_LINE_TEXT", timestamp("2000-01-01T00:00:00Z"))
|
267
|
+
.setTimestamp("TIMESTAMP|DATE", timestamp("2000-01-01T00:00:00Z"))
|
268
|
+
.setTimestamp("TIMESTAMP|DATE|JST", timestamp("2000-01-01T00:00:00Z"))
|
269
|
+
.setTimestamp("TIMESTAMP|DATE|PST", timestamp("2000-01-01T00:00:00Z"))
|
270
|
+
.setTimestamp("TIMESTAMP", timestamp("2000-01-01T00:00:00Z"))
|
271
|
+
.setJson("JSON|SINGLE_LINE_TEXT", ValueFactory.newString("def"))
|
272
|
+
.setJson("JSON", ValueFactory.newString("def"))
|
273
|
+
.addRecord()
|
274
|
+
.build();
|
275
|
+
}
|
276
|
+
|
277
|
+
private static BigDecimal number(String value) {
|
278
|
+
return new BigDecimal(value);
|
279
|
+
}
|
280
|
+
|
281
|
+
@SafeVarargs
|
282
|
+
private static <T> List<T> list(T... a) {
|
283
|
+
return Arrays.asList(a);
|
284
|
+
}
|
285
|
+
|
286
|
+
private static LocalDate date(CharSequence text) {
|
287
|
+
return LocalDate.parse(text);
|
288
|
+
}
|
289
|
+
|
290
|
+
private static ZonedDateTime dateTime(CharSequence text) {
|
291
|
+
return ZonedDateTime.parse(text).withZoneSameInstant(ZoneId.of("UTC"));
|
292
|
+
}
|
293
|
+
|
294
|
+
private static Timestamp timestamp(CharSequence text) {
|
295
|
+
return Timestamp.ofInstant(Instant.parse(text));
|
296
|
+
}
|
297
|
+
}
|
@@ -0,0 +1,52 @@
|
|
1
|
+
package org.embulk.output.kintone;
|
2
|
+
|
3
|
+
import com.kintone.client.model.record.FieldType;
|
4
|
+
import com.kintone.client.model.record.Record;
|
5
|
+
import com.kintone.client.model.record.UpdateKey;
|
6
|
+
import java.util.Map;
|
7
|
+
import java.util.function.BiConsumer;
|
8
|
+
import org.embulk.spi.Column;
|
9
|
+
import org.embulk.spi.Page;
|
10
|
+
import org.embulk.spi.PageReader;
|
11
|
+
import org.embulk.spi.Schema;
|
12
|
+
|
13
|
+
public class KintoneColumnVisitorVerifier {
|
14
|
+
private final Schema schema;
|
15
|
+
private final Map<String, KintoneColumnOption> columnOptions;
|
16
|
+
private final PageReader pageReader;
|
17
|
+
private final KintoneColumnVisitor visitor;
|
18
|
+
|
19
|
+
public KintoneColumnVisitorVerifier(
|
20
|
+
Schema schema,
|
21
|
+
Map<String, KintoneColumnOption> columnOptions,
|
22
|
+
String updateKeyName,
|
23
|
+
Page page) {
|
24
|
+
this.schema = schema;
|
25
|
+
this.columnOptions = columnOptions;
|
26
|
+
pageReader = new PageReader(schema);
|
27
|
+
pageReader.setPage(page);
|
28
|
+
visitor = new KintoneColumnVisitor(pageReader, columnOptions, updateKeyName);
|
29
|
+
}
|
30
|
+
|
31
|
+
public void verify(BiConsumer<Record, UpdateKey> consumer) {
|
32
|
+
if (!pageReader.nextRecord()) {
|
33
|
+
throw new IllegalStateException();
|
34
|
+
}
|
35
|
+
Record record = new Record();
|
36
|
+
visitor.setRecord(record);
|
37
|
+
UpdateKey updateKey = new UpdateKey();
|
38
|
+
visitor.setUpdateKey(updateKey);
|
39
|
+
schema.visitColumns(visitor);
|
40
|
+
schema.getColumns().forEach(column -> verify(record, column));
|
41
|
+
consumer.accept(record, updateKey);
|
42
|
+
}
|
43
|
+
|
44
|
+
private void verify(Record record, Column column) {
|
45
|
+
FieldType expected = FieldType.valueOf(columnOptions.get(column.getName()).getType());
|
46
|
+
FieldType actual = record.getFieldType(column.getName());
|
47
|
+
if (!expected.equals(actual)) {
|
48
|
+
System.out.printf(
|
49
|
+
"%s: Expected type is %s, but actual type is %s%n", column.getName(), expected, actual);
|
50
|
+
}
|
51
|
+
}
|
52
|
+
}
|
@@ -0,0 +1,226 @@
|
|
1
|
+
package org.embulk.output.kintone;
|
2
|
+
|
3
|
+
import static org.hamcrest.MatcherAssert.assertThat;
|
4
|
+
import static org.hamcrest.Matchers.is;
|
5
|
+
import static org.mockito.ArgumentMatchers.anyList;
|
6
|
+
import static org.mockito.ArgumentMatchers.eq;
|
7
|
+
import static org.mockito.ArgumentMatchers.matches;
|
8
|
+
import static org.mockito.Mockito.atLeast;
|
9
|
+
import static org.mockito.Mockito.mock;
|
10
|
+
import static org.mockito.Mockito.mockStatic;
|
11
|
+
import static org.mockito.Mockito.verify;
|
12
|
+
import static org.mockito.Mockito.when;
|
13
|
+
|
14
|
+
import com.kintone.client.AppClient;
|
15
|
+
import com.kintone.client.KintoneClient;
|
16
|
+
import com.kintone.client.KintoneClientBuilder;
|
17
|
+
import com.kintone.client.RecordClient;
|
18
|
+
import com.kintone.client.api.record.GetRecordsByCursorResponseBody;
|
19
|
+
import com.kintone.client.model.app.field.FieldProperty;
|
20
|
+
import com.kintone.client.model.app.field.NumberFieldProperty;
|
21
|
+
import com.kintone.client.model.app.field.SingleLineTextFieldProperty;
|
22
|
+
import com.kintone.client.model.record.DateTimeFieldValue;
|
23
|
+
import com.kintone.client.model.record.FieldType;
|
24
|
+
import com.kintone.client.model.record.FieldValue;
|
25
|
+
import com.kintone.client.model.record.NumberFieldValue;
|
26
|
+
import com.kintone.client.model.record.Record;
|
27
|
+
import com.kintone.client.model.record.RecordForUpdate;
|
28
|
+
import com.kintone.client.model.record.SingleLineTextFieldValue;
|
29
|
+
import com.kintone.client.model.record.UpdateKey;
|
30
|
+
import java.math.BigDecimal;
|
31
|
+
import java.time.ZoneId;
|
32
|
+
import java.util.Collection;
|
33
|
+
import java.util.Collections;
|
34
|
+
import java.util.List;
|
35
|
+
import java.util.Map;
|
36
|
+
import java.util.stream.Collectors;
|
37
|
+
import java.util.stream.IntStream;
|
38
|
+
import org.embulk.config.TaskReport;
|
39
|
+
import org.embulk.spi.Page;
|
40
|
+
import org.embulk.spi.TransactionalPageOutput;
|
41
|
+
import org.mockito.ArgumentCaptor;
|
42
|
+
import org.mockito.MockedStatic;
|
43
|
+
|
44
|
+
public class KintonePageOutputVerifier implements TransactionalPageOutput {
|
45
|
+
private final TransactionalPageOutput transactionalPageOutput;
|
46
|
+
private final String domain;
|
47
|
+
private final String field;
|
48
|
+
private final List<String> values;
|
49
|
+
private final List<Record> addRecords;
|
50
|
+
private final List<RecordForUpdate> updateRecords;
|
51
|
+
|
52
|
+
public KintonePageOutputVerifier(
|
53
|
+
TransactionalPageOutput transactionalPageOutput,
|
54
|
+
String domain,
|
55
|
+
String field,
|
56
|
+
List<String> values,
|
57
|
+
List<Record> addRecords,
|
58
|
+
List<RecordForUpdate> updateRecords) {
|
59
|
+
this.transactionalPageOutput = transactionalPageOutput;
|
60
|
+
this.domain = domain;
|
61
|
+
this.field = field;
|
62
|
+
this.values = values;
|
63
|
+
this.addRecords = addRecords;
|
64
|
+
this.updateRecords = updateRecords;
|
65
|
+
}
|
66
|
+
|
67
|
+
@Override
|
68
|
+
public void add(Page page) {
|
69
|
+
runWithMock(() -> transactionalPageOutput.add(page));
|
70
|
+
}
|
71
|
+
|
72
|
+
@Override
|
73
|
+
public void finish() {
|
74
|
+
transactionalPageOutput.finish();
|
75
|
+
}
|
76
|
+
|
77
|
+
@Override
|
78
|
+
public void close() {
|
79
|
+
transactionalPageOutput.close();
|
80
|
+
}
|
81
|
+
|
82
|
+
@Override
|
83
|
+
public void abort() {
|
84
|
+
transactionalPageOutput.abort();
|
85
|
+
}
|
86
|
+
|
87
|
+
@Override
|
88
|
+
public TaskReport commit() {
|
89
|
+
return transactionalPageOutput.commit();
|
90
|
+
}
|
91
|
+
|
92
|
+
public void runWithMock(Runnable runnable) {
|
93
|
+
try {
|
94
|
+
runWithMockClient(runnable);
|
95
|
+
} catch (Exception e) {
|
96
|
+
throw new RuntimeException(e);
|
97
|
+
}
|
98
|
+
}
|
99
|
+
|
100
|
+
private void runWithMockClient(Runnable runnable) throws Exception {
|
101
|
+
@SuppressWarnings("unchecked")
|
102
|
+
Map<String, FieldProperty> mockFormFields = mock(Map.class);
|
103
|
+
when(mockFormFields.get(matches("^.*_single_line_text$")))
|
104
|
+
.thenReturn(new SingleLineTextFieldProperty());
|
105
|
+
when(mockFormFields.get(matches("^.*_number$"))).thenReturn(new NumberFieldProperty());
|
106
|
+
AppClient mockAppClient = mock(AppClient.class);
|
107
|
+
when(mockAppClient.getFormFields(eq(0L))).thenReturn(mockFormFields);
|
108
|
+
GetRecordsByCursorResponseBody mockGetRecordsByCursorResponseBody =
|
109
|
+
mock(GetRecordsByCursorResponseBody.class);
|
110
|
+
when(mockGetRecordsByCursorResponseBody.getRecords())
|
111
|
+
.thenReturn(updateRecords.stream().map(this::getRecord).collect(Collectors.toList()));
|
112
|
+
when(mockGetRecordsByCursorResponseBody.hasNext()).thenReturn(false);
|
113
|
+
RecordClient mockRecordClient = mock(RecordClient.class);
|
114
|
+
when(mockRecordClient.createCursor(eq(0L), eq(getFields()), eq(getQuery()))).thenReturn("id");
|
115
|
+
when(mockRecordClient.getRecordsByCursor(eq("id")))
|
116
|
+
.thenReturn(mockGetRecordsByCursorResponseBody);
|
117
|
+
when(mockRecordClient.addRecords(eq(0L), anyList())).thenReturn(Collections.emptyList());
|
118
|
+
when(mockRecordClient.updateRecords(eq(0L), anyList())).thenReturn(Collections.emptyList());
|
119
|
+
KintoneClient mockKintoneClient = mock(KintoneClient.class);
|
120
|
+
when(mockKintoneClient.app()).thenReturn(mockAppClient);
|
121
|
+
when(mockKintoneClient.record()).thenReturn(mockRecordClient);
|
122
|
+
KintoneClientBuilder mockKintoneClientBuilder = mock(KintoneClientBuilder.class);
|
123
|
+
when(mockKintoneClientBuilder.authByApiToken(eq("token"))).thenReturn(mockKintoneClientBuilder);
|
124
|
+
when(mockKintoneClientBuilder.build()).thenReturn(mockKintoneClient);
|
125
|
+
try (MockedStatic<KintoneClientBuilder> mocked = mockStatic(KintoneClientBuilder.class)) {
|
126
|
+
mocked
|
127
|
+
.when(() -> KintoneClientBuilder.create(String.format("https://%s", domain)))
|
128
|
+
.thenReturn(mockKintoneClientBuilder);
|
129
|
+
runnable.run();
|
130
|
+
}
|
131
|
+
@SuppressWarnings("unchecked")
|
132
|
+
ArgumentCaptor<List<Record>> addRecordsArgumentCaptor = ArgumentCaptor.forClass(List.class);
|
133
|
+
verify(mockRecordClient, atLeast(0)).addRecords(eq(0L), addRecordsArgumentCaptor.capture());
|
134
|
+
assertRecords(
|
135
|
+
domain,
|
136
|
+
addRecordsArgumentCaptor.getAllValues().stream()
|
137
|
+
.flatMap(Collection::stream)
|
138
|
+
.collect(Collectors.toList()),
|
139
|
+
addRecords);
|
140
|
+
@SuppressWarnings("unchecked")
|
141
|
+
ArgumentCaptor<List<RecordForUpdate>> updateRecordsArgumentCaptor =
|
142
|
+
ArgumentCaptor.forClass(List.class);
|
143
|
+
verify(mockRecordClient, atLeast(0))
|
144
|
+
.updateRecords(eq(0L), updateRecordsArgumentCaptor.capture());
|
145
|
+
assertRecordForUpdates(
|
146
|
+
domain,
|
147
|
+
updateRecordsArgumentCaptor.getAllValues().stream()
|
148
|
+
.flatMap(Collection::stream)
|
149
|
+
.collect(Collectors.toList()),
|
150
|
+
updateRecords);
|
151
|
+
}
|
152
|
+
|
153
|
+
private Record getRecord(RecordForUpdate updateRecord) {
|
154
|
+
UpdateKey key = updateRecord.getUpdateKey();
|
155
|
+
String field = key.getField();
|
156
|
+
Object value = key.getValue();
|
157
|
+
return new Record()
|
158
|
+
.putField(
|
159
|
+
field,
|
160
|
+
field.matches("^.*_number$")
|
161
|
+
? new NumberFieldValue((BigDecimal) value)
|
162
|
+
: new SingleLineTextFieldValue((String) value));
|
163
|
+
}
|
164
|
+
|
165
|
+
private List<String> getFields() {
|
166
|
+
return Collections.singletonList(field);
|
167
|
+
}
|
168
|
+
|
169
|
+
private String getQuery() {
|
170
|
+
return String.format("%s in (%s)", field, String.join(",", values));
|
171
|
+
}
|
172
|
+
|
173
|
+
private static void assertRecords(String domain, List<Record> actual, List<Record> expected) {
|
174
|
+
assertThat(domain, actual.size(), is(expected.size()));
|
175
|
+
// spotless:off
|
176
|
+
IntStream.range(0, actual.size()).forEach(index -> assertRecord(domain, index, actual.get(index), expected.get(index)));
|
177
|
+
// spotless:on
|
178
|
+
}
|
179
|
+
|
180
|
+
private static void assertRecord(String domain, int index, Record actual, Record expected) {
|
181
|
+
String reason = String.format("%s:%d", domain, index);
|
182
|
+
assertThat(reason, actual.getId(), is(expected.getId()));
|
183
|
+
assertThat(reason, actual.getRevision(), is(expected.getRevision()));
|
184
|
+
assertThat(reason, actual.getFieldCodes(true), is(expected.getFieldCodes(true)));
|
185
|
+
// spotless:off
|
186
|
+
actual.getFieldCodes(true).forEach(fieldCode -> assertFieldValue(domain, index, fieldCode, actual.getFieldValue(fieldCode), expected.getFieldValue(fieldCode)));
|
187
|
+
// spotless:on
|
188
|
+
}
|
189
|
+
|
190
|
+
private static void assertFieldValue(
|
191
|
+
String domain, int index, String fieldCode, FieldValue actual, FieldValue expected) {
|
192
|
+
String reason = String.format("%s:%d:%s", domain, index, fieldCode);
|
193
|
+
assertThat(reason, actual.getType(), is(expected.getType()));
|
194
|
+
// spotless:off
|
195
|
+
assertThat(reason, actual, is(expected.getType().equals(FieldType.DATETIME) ? new DateTimeFieldValue(((DateTimeFieldValue) expected).getValue().withZoneSameInstant(ZoneId.of("UTC"))) : expected));
|
196
|
+
// spotless:on
|
197
|
+
}
|
198
|
+
|
199
|
+
private static void assertRecordForUpdates(
|
200
|
+
String domain, List<RecordForUpdate> actual, List<RecordForUpdate> expected) {
|
201
|
+
assertThat(domain, actual.size(), is(expected.size()));
|
202
|
+
// spotless:off
|
203
|
+
IntStream.range(0, actual.size()).forEach(index -> assertRecordForUpdate(domain, index, actual.get(index), expected.get(index)));
|
204
|
+
// spotless:on
|
205
|
+
}
|
206
|
+
|
207
|
+
private static void assertRecordForUpdate(
|
208
|
+
String domain, int index, RecordForUpdate actual, RecordForUpdate expected) {
|
209
|
+
String reason = String.format("%s:%d", domain, index);
|
210
|
+
assertThat(reason, actual.getId(), is(expected.getId()));
|
211
|
+
assertUpdateKey(domain, index, actual.getUpdateKey(), expected.getUpdateKey());
|
212
|
+
assertRecord(domain, index, actual.getRecord(), expected.getRecord());
|
213
|
+
assertThat(reason, actual.getRevision(), is(expected.getRevision()));
|
214
|
+
}
|
215
|
+
|
216
|
+
private static void assertUpdateKey(
|
217
|
+
String domain, int index, UpdateKey actual, UpdateKey expected) {
|
218
|
+
String reason = String.format("%s:%d", domain, index);
|
219
|
+
assertThat(reason, actual.getField(), is(expected.getField()));
|
220
|
+
assertThat(reason, actual.getValue(), is(expected.getValue().toString()));
|
221
|
+
}
|
222
|
+
|
223
|
+
public interface Runnable {
|
224
|
+
void run() throws Exception;
|
225
|
+
}
|
226
|
+
}
|
@@ -0,0 +1,92 @@
|
|
1
|
+
package org.embulk.output.kintone;
|
2
|
+
|
3
|
+
import java.util.List;
|
4
|
+
import java.util.function.Function;
|
5
|
+
import java.util.stream.Collectors;
|
6
|
+
import org.embulk.deps.buffer.PooledBufferAllocator;
|
7
|
+
import org.embulk.spi.Column;
|
8
|
+
import org.embulk.spi.Page;
|
9
|
+
import org.embulk.spi.PageBuilder;
|
10
|
+
import org.embulk.spi.PageOutput;
|
11
|
+
import org.embulk.spi.Schema;
|
12
|
+
import org.embulk.spi.time.Timestamp;
|
13
|
+
import org.msgpack.value.Value;
|
14
|
+
|
15
|
+
public class OutputPageBuilder implements PageOutput {
|
16
|
+
private final List<String> names;
|
17
|
+
private final PageBuilder builder;
|
18
|
+
private Page page;
|
19
|
+
|
20
|
+
public static Page build(Schema schema, Function<OutputPageBuilder, Page> function) {
|
21
|
+
Page page;
|
22
|
+
try (OutputPageBuilder builder = new OutputPageBuilder(schema)) {
|
23
|
+
page = function.apply(builder);
|
24
|
+
}
|
25
|
+
return page;
|
26
|
+
}
|
27
|
+
|
28
|
+
public OutputPageBuilder(Schema schema) {
|
29
|
+
names = schema.getColumns().stream().map(Column::getName).collect(Collectors.toList());
|
30
|
+
builder = new PageBuilder(PooledBufferAllocator.create(), schema, this);
|
31
|
+
}
|
32
|
+
|
33
|
+
public OutputPageBuilder setNull(String name) {
|
34
|
+
builder.setNull(names.indexOf(name));
|
35
|
+
return this;
|
36
|
+
}
|
37
|
+
|
38
|
+
public OutputPageBuilder setBoolean(String name, boolean value) {
|
39
|
+
builder.setBoolean(names.indexOf(name), value);
|
40
|
+
return this;
|
41
|
+
}
|
42
|
+
|
43
|
+
public OutputPageBuilder setLong(String name, long value) {
|
44
|
+
builder.setLong(names.indexOf(name), value);
|
45
|
+
return this;
|
46
|
+
}
|
47
|
+
|
48
|
+
public OutputPageBuilder setDouble(String name, double value) {
|
49
|
+
builder.setDouble(names.indexOf(name), value);
|
50
|
+
return this;
|
51
|
+
}
|
52
|
+
|
53
|
+
public OutputPageBuilder setString(String name, String value) {
|
54
|
+
builder.setString(names.indexOf(name), value);
|
55
|
+
return this;
|
56
|
+
}
|
57
|
+
|
58
|
+
public OutputPageBuilder setJson(String name, Value value) {
|
59
|
+
builder.setJson(names.indexOf(name), value);
|
60
|
+
return this;
|
61
|
+
}
|
62
|
+
|
63
|
+
public OutputPageBuilder setTimestamp(String name, Timestamp value) {
|
64
|
+
builder.setTimestamp(names.indexOf(name), value);
|
65
|
+
return this;
|
66
|
+
}
|
67
|
+
|
68
|
+
public OutputPageBuilder addRecord() {
|
69
|
+
builder.addRecord();
|
70
|
+
return this;
|
71
|
+
}
|
72
|
+
|
73
|
+
public Page build() {
|
74
|
+
builder.flush();
|
75
|
+
builder.close();
|
76
|
+
return page;
|
77
|
+
}
|
78
|
+
|
79
|
+
@Override
|
80
|
+
public void add(Page page) {
|
81
|
+
if (this.page != null) {
|
82
|
+
throw new IllegalStateException();
|
83
|
+
}
|
84
|
+
this.page = page;
|
85
|
+
}
|
86
|
+
|
87
|
+
@Override
|
88
|
+
public void finish() {}
|
89
|
+
|
90
|
+
@Override
|
91
|
+
public void close() {}
|
92
|
+
}
|