embulk-output-kintone 0.4.0 → 1.0.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (49) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/ci.yml +1 -0
  3. data/.gitignore +0 -1
  4. data/README.md +8 -6
  5. data/build.gradle +18 -19
  6. data/classpath/embulk-output-kintone-1.0.0.jar +0 -0
  7. data/classpath/shadow-kintone-java-client-1.0.0-all.jar +0 -0
  8. data/gradle/wrapper/gradle-wrapper.properties +1 -1
  9. data/settings.gradle +2 -0
  10. data/shadow-kintone-java-client/build.gradle +35 -0
  11. data/src/main/java/org/embulk/output/kintone/KintoneColumnOption.java +1 -2
  12. data/src/main/java/org/embulk/output/kintone/KintoneColumnType.java +368 -0
  13. data/src/main/java/org/embulk/output/kintone/KintoneColumnVisitor.java +195 -135
  14. data/src/main/java/org/embulk/output/kintone/KintoneMode.java +0 -1
  15. data/src/main/java/org/embulk/output/kintone/KintoneOutputPlugin.java +0 -2
  16. data/src/main/java/org/embulk/output/kintone/KintonePageOutput.java +249 -192
  17. data/src/main/java/org/embulk/output/kintone/KintoneRetryOption.java +19 -0
  18. data/src/main/java/org/embulk/output/kintone/PluginTask.java +11 -3
  19. data/src/test/java/com/kintone/client/Json.java +16 -0
  20. data/src/test/java/org/embulk/output/kintone/KintoneColumnOptionBuilder.java +62 -0
  21. data/src/test/java/org/embulk/output/kintone/KintoneColumnVisitorTest.java +820 -0
  22. data/src/test/java/org/embulk/output/kintone/KintoneColumnVisitorVerifier.java +73 -0
  23. data/src/test/java/org/embulk/output/kintone/KintonePageOutputVerifier.java +221 -0
  24. data/src/test/java/org/embulk/output/kintone/OutputPageBuilder.java +92 -0
  25. data/src/test/java/org/embulk/output/kintone/TestKintoneOutputPlugin.java +176 -1
  26. data/src/test/java/org/embulk/output/kintone/TestTask.java +40 -0
  27. data/src/test/java/org/embulk/output/kintone/TestTaskMode.java +43 -0
  28. data/src/test/resources/logback-test.xml +14 -0
  29. data/src/test/resources/org/embulk/output/kintone/config.yml +4 -0
  30. data/src/test/resources/org/embulk/output/kintone/task/config.yml +1 -0
  31. data/src/test/resources/org/embulk/output/kintone/task/mode/config.yml +158 -0
  32. data/src/test/resources/org/embulk/output/kintone/task/mode/input.csv +7 -0
  33. data/src/test/resources/org/embulk/output/kintone/task/mode/insert_add_ignore_nulls_records.jsonl +6 -0
  34. data/src/test/resources/org/embulk/output/kintone/task/mode/insert_add_prefer_nulls_records.jsonl +6 -0
  35. data/src/test/resources/org/embulk/output/kintone/task/mode/insert_add_records.jsonl +6 -0
  36. data/src/test/resources/org/embulk/output/kintone/task/mode/update_update_ignore_nulls_records.jsonl +3 -0
  37. data/src/test/resources/org/embulk/output/kintone/task/mode/update_update_prefer_nulls_records.jsonl +3 -0
  38. data/src/test/resources/org/embulk/output/kintone/task/mode/update_update_records.jsonl +6 -0
  39. data/src/test/resources/org/embulk/output/kintone/task/mode/upsert_add_ignore_nulls_records.jsonl +3 -0
  40. data/src/test/resources/org/embulk/output/kintone/task/mode/upsert_add_prefer_nulls_records.jsonl +3 -0
  41. data/src/test/resources/org/embulk/output/kintone/task/mode/upsert_add_records.jsonl +2 -0
  42. data/src/test/resources/org/embulk/output/kintone/task/mode/upsert_update_ignore_nulls_records.jsonl +3 -0
  43. data/src/test/resources/org/embulk/output/kintone/task/mode/upsert_update_prefer_nulls_records.jsonl +3 -0
  44. data/src/test/resources/org/embulk/output/kintone/task/mode/upsert_update_records.jsonl +4 -0
  45. data/src/test/resources/org/embulk/output/kintone/task/mode/values.json +1 -0
  46. data/src/test/resources/org/embulk/output/kintone/task/mode/values_ignore_nulls.json +1 -0
  47. data/src/test/resources/org/embulk/output/kintone/task/mode/values_prefer_nulls.json +1 -0
  48. metadata +36 -3
  49. data/classpath/embulk-output-kintone-0.4.0.jar +0 -0
@@ -0,0 +1,73 @@
1
+ package org.embulk.output.kintone;
2
+
3
+ import com.kintone.client.model.record.FieldType;
4
+ import com.kintone.client.model.record.Record;
5
+ import com.kintone.client.model.record.UpdateKey;
6
+ import java.util.Map;
7
+ import java.util.function.BiConsumer;
8
+ import javax.validation.UnexpectedTypeException;
9
+ import org.embulk.spi.Column;
10
+ import org.embulk.spi.Page;
11
+ import org.embulk.spi.PageReader;
12
+ import org.embulk.spi.Schema;
13
+
14
+ public class KintoneColumnVisitorVerifier {
15
+ private final Schema schema;
16
+ private final Map<String, KintoneColumnOption> options;
17
+ private final PageReader reader;
18
+ private final KintoneColumnVisitor visitor;
19
+
20
+ public KintoneColumnVisitorVerifier(
21
+ Schema schema, Map<String, KintoneColumnOption> options, String updateKeyName, Page page) {
22
+ this(schema, options, false, false, updateKeyName, page);
23
+ }
24
+
25
+ public KintoneColumnVisitorVerifier(
26
+ Schema schema,
27
+ Map<String, KintoneColumnOption> options,
28
+ boolean preferNulls,
29
+ boolean ignoreNulls,
30
+ String updateKeyName,
31
+ Page page) {
32
+ this.schema = schema;
33
+ this.options = options;
34
+ reader = new PageReader(schema);
35
+ reader.setPage(page);
36
+ visitor = new KintoneColumnVisitor(reader, options, preferNulls, ignoreNulls, updateKeyName);
37
+ }
38
+
39
+ public void verify() {
40
+ verify((record, updateKey) -> {});
41
+ }
42
+
43
+ public void verify(BiConsumer<Record, UpdateKey> consumer) {
44
+ verify(consumer, false);
45
+ }
46
+
47
+ public void verify(BiConsumer<Record, UpdateKey> consumer, boolean nullable) {
48
+ if (!reader.nextRecord()) {
49
+ throw new IllegalStateException();
50
+ }
51
+ Record record = new Record();
52
+ visitor.setRecord(record);
53
+ UpdateKey updateKey = new UpdateKey();
54
+ visitor.setUpdateKey(updateKey);
55
+ schema.visitColumns(visitor);
56
+ schema.getColumns().forEach(column -> verify(record, column, nullable));
57
+ consumer.accept(record, updateKey);
58
+ }
59
+
60
+ private void verify(Record record, Column column, boolean nullable) {
61
+ FieldType expected = FieldType.valueOf(options.get(column.getName()).getType());
62
+ FieldType actual = record.getFieldType(column.getName());
63
+ if (actual == null && nullable) {
64
+ return;
65
+ }
66
+ if (expected != actual) {
67
+ throw new UnexpectedTypeException(
68
+ String.format(
69
+ "%s: Expected type is %s, but actual type is %s%n",
70
+ column.getName(), expected, actual));
71
+ }
72
+ }
73
+ }
@@ -0,0 +1,221 @@
1
+ package org.embulk.output.kintone;
2
+
3
+ import static org.hamcrest.MatcherAssert.assertThat;
4
+ import static org.hamcrest.Matchers.is;
5
+ import static org.mockito.ArgumentMatchers.anyList;
6
+ import static org.mockito.ArgumentMatchers.eq;
7
+ import static org.mockito.ArgumentMatchers.matches;
8
+ import static org.mockito.Mockito.atLeast;
9
+ import static org.mockito.Mockito.mock;
10
+ import static org.mockito.Mockito.mockStatic;
11
+ import static org.mockito.Mockito.verify;
12
+ import static org.mockito.Mockito.when;
13
+
14
+ import com.kintone.client.AppClient;
15
+ import com.kintone.client.KintoneClient;
16
+ import com.kintone.client.KintoneClientBuilder;
17
+ import com.kintone.client.RecordClient;
18
+ import com.kintone.client.api.record.GetRecordsByCursorResponseBody;
19
+ import com.kintone.client.model.app.field.FieldProperty;
20
+ import com.kintone.client.model.app.field.NumberFieldProperty;
21
+ import com.kintone.client.model.app.field.SingleLineTextFieldProperty;
22
+ import com.kintone.client.model.record.FieldValue;
23
+ import com.kintone.client.model.record.NumberFieldValue;
24
+ import com.kintone.client.model.record.Record;
25
+ import com.kintone.client.model.record.RecordForUpdate;
26
+ import com.kintone.client.model.record.SingleLineTextFieldValue;
27
+ import com.kintone.client.model.record.UpdateKey;
28
+ import java.math.BigDecimal;
29
+ import java.util.Collection;
30
+ import java.util.Collections;
31
+ import java.util.List;
32
+ import java.util.Map;
33
+ import java.util.stream.Collectors;
34
+ import java.util.stream.IntStream;
35
+ import org.embulk.config.TaskReport;
36
+ import org.embulk.spi.Page;
37
+ import org.embulk.spi.TransactionalPageOutput;
38
+ import org.mockito.ArgumentCaptor;
39
+ import org.mockito.MockedStatic;
40
+
41
+ public class KintonePageOutputVerifier implements TransactionalPageOutput {
42
+ private final TransactionalPageOutput transactionalPageOutput;
43
+ private final String domain;
44
+ private final String field;
45
+ private final List<String> values;
46
+ private final List<Record> addRecords;
47
+ private final List<RecordForUpdate> updateRecords;
48
+
49
+ public KintonePageOutputVerifier(
50
+ TransactionalPageOutput transactionalPageOutput,
51
+ String domain,
52
+ String field,
53
+ List<String> values,
54
+ List<Record> addRecords,
55
+ List<RecordForUpdate> updateRecords) {
56
+ this.transactionalPageOutput = transactionalPageOutput;
57
+ this.domain = domain;
58
+ this.field = field;
59
+ this.values = values;
60
+ this.addRecords = addRecords;
61
+ this.updateRecords = updateRecords;
62
+ }
63
+
64
+ @Override
65
+ public void add(Page page) {
66
+ runWithMock(() -> transactionalPageOutput.add(page));
67
+ }
68
+
69
+ @Override
70
+ public void finish() {
71
+ transactionalPageOutput.finish();
72
+ }
73
+
74
+ @Override
75
+ public void close() {
76
+ transactionalPageOutput.close();
77
+ }
78
+
79
+ @Override
80
+ public void abort() {
81
+ transactionalPageOutput.abort();
82
+ }
83
+
84
+ @Override
85
+ public TaskReport commit() {
86
+ return transactionalPageOutput.commit();
87
+ }
88
+
89
+ public void runWithMock(Runnable runnable) {
90
+ try {
91
+ runWithMockClient(runnable);
92
+ } catch (Exception e) {
93
+ throw new RuntimeException(e);
94
+ }
95
+ }
96
+
97
+ private void runWithMockClient(Runnable runnable) throws Exception {
98
+ @SuppressWarnings("unchecked")
99
+ Map<String, FieldProperty> mockFormFields = mock(Map.class);
100
+ when(mockFormFields.get(matches("^.*_single_line_text$")))
101
+ .thenReturn(new SingleLineTextFieldProperty());
102
+ when(mockFormFields.get(matches("^.*_number$"))).thenReturn(new NumberFieldProperty());
103
+ AppClient mockAppClient = mock(AppClient.class);
104
+ when(mockAppClient.getFormFields(eq(0L))).thenReturn(mockFormFields);
105
+ GetRecordsByCursorResponseBody mockGetRecordsByCursorResponseBody =
106
+ mock(GetRecordsByCursorResponseBody.class);
107
+ when(mockGetRecordsByCursorResponseBody.getRecords())
108
+ .thenReturn(updateRecords.stream().map(this::getRecord).collect(Collectors.toList()));
109
+ when(mockGetRecordsByCursorResponseBody.hasNext()).thenReturn(false);
110
+ RecordClient mockRecordClient = mock(RecordClient.class);
111
+ when(mockRecordClient.createCursor(eq(0L), eq(getFields()), eq(getQuery()))).thenReturn("id");
112
+ when(mockRecordClient.getRecordsByCursor(eq("id")))
113
+ .thenReturn(mockGetRecordsByCursorResponseBody);
114
+ when(mockRecordClient.addRecords(eq(0L), anyList())).thenReturn(Collections.emptyList());
115
+ when(mockRecordClient.updateRecords(eq(0L), anyList())).thenReturn(Collections.emptyList());
116
+ KintoneClient mockKintoneClient = mock(KintoneClient.class);
117
+ when(mockKintoneClient.app()).thenReturn(mockAppClient);
118
+ when(mockKintoneClient.record()).thenReturn(mockRecordClient);
119
+ KintoneClientBuilder mockKintoneClientBuilder = mock(KintoneClientBuilder.class);
120
+ when(mockKintoneClientBuilder.authByApiToken(eq("token"))).thenReturn(mockKintoneClientBuilder);
121
+ when(mockKintoneClientBuilder.build()).thenReturn(mockKintoneClient);
122
+ try (MockedStatic<KintoneClientBuilder> mocked = mockStatic(KintoneClientBuilder.class)) {
123
+ mocked
124
+ .when(() -> KintoneClientBuilder.create(String.format("https://%s", domain)))
125
+ .thenReturn(mockKintoneClientBuilder);
126
+ runnable.run();
127
+ }
128
+ @SuppressWarnings("unchecked")
129
+ ArgumentCaptor<List<Record>> addRecordsArgumentCaptor = ArgumentCaptor.forClass(List.class);
130
+ verify(mockRecordClient, atLeast(0)).addRecords(eq(0L), addRecordsArgumentCaptor.capture());
131
+ assertRecords(
132
+ domain,
133
+ addRecordsArgumentCaptor.getAllValues().stream()
134
+ .flatMap(Collection::stream)
135
+ .collect(Collectors.toList()),
136
+ addRecords);
137
+ @SuppressWarnings("unchecked")
138
+ ArgumentCaptor<List<RecordForUpdate>> updateRecordsArgumentCaptor =
139
+ ArgumentCaptor.forClass(List.class);
140
+ verify(mockRecordClient, atLeast(0))
141
+ .updateRecords(eq(0L), updateRecordsArgumentCaptor.capture());
142
+ assertRecordForUpdates(
143
+ domain,
144
+ updateRecordsArgumentCaptor.getAllValues().stream()
145
+ .flatMap(Collection::stream)
146
+ .collect(Collectors.toList()),
147
+ updateRecords);
148
+ }
149
+
150
+ private Record getRecord(RecordForUpdate updateRecord) {
151
+ UpdateKey key = updateRecord.getUpdateKey();
152
+ String field = key.getField();
153
+ Object value = key.getValue();
154
+ return new Record()
155
+ .putField(
156
+ field,
157
+ field.matches("^.*_number$")
158
+ ? new NumberFieldValue((BigDecimal) value)
159
+ : new SingleLineTextFieldValue((String) value));
160
+ }
161
+
162
+ private List<String> getFields() {
163
+ return Collections.singletonList(field);
164
+ }
165
+
166
+ private String getQuery() {
167
+ return String.format("%s in (%s)", field, String.join(",", values));
168
+ }
169
+
170
+ private static void assertRecords(String domain, List<Record> actual, List<Record> expected) {
171
+ assertThat(domain, actual.size(), is(expected.size()));
172
+ // spotless:off
173
+ IntStream.range(0, actual.size()).forEach(index -> assertRecord(domain, index, actual.get(index), expected.get(index)));
174
+ // spotless:on
175
+ }
176
+
177
+ private static void assertRecord(String domain, int index, Record actual, Record expected) {
178
+ String reason = String.format("%s:%d", domain, index);
179
+ assertThat(reason, actual.getId(), is(expected.getId()));
180
+ assertThat(reason, actual.getRevision(), is(expected.getRevision()));
181
+ assertThat(reason, actual.getFieldCodes(true), is(expected.getFieldCodes(true)));
182
+ // spotless:off
183
+ actual.getFieldCodes(true).forEach(fieldCode -> assertFieldValue(domain, index, fieldCode, actual.getFieldValue(fieldCode), expected.getFieldValue(fieldCode)));
184
+ // spotless:on
185
+ }
186
+
187
+ private static void assertFieldValue(
188
+ String domain, int index, String fieldCode, FieldValue actual, FieldValue expected) {
189
+ String reason = String.format("%s:%d:%s", domain, index, fieldCode);
190
+ assertThat(reason, actual.getType(), is(expected.getType()));
191
+ assertThat(reason, actual, is(expected));
192
+ }
193
+
194
+ private static void assertRecordForUpdates(
195
+ String domain, List<RecordForUpdate> actual, List<RecordForUpdate> expected) {
196
+ assertThat(domain, actual.size(), is(expected.size()));
197
+ // spotless:off
198
+ IntStream.range(0, actual.size()).forEach(index -> assertRecordForUpdate(domain, index, actual.get(index), expected.get(index)));
199
+ // spotless:on
200
+ }
201
+
202
+ private static void assertRecordForUpdate(
203
+ String domain, int index, RecordForUpdate actual, RecordForUpdate expected) {
204
+ String reason = String.format("%s:%d", domain, index);
205
+ assertThat(reason, actual.getId(), is(expected.getId()));
206
+ assertUpdateKey(domain, index, actual.getUpdateKey(), expected.getUpdateKey());
207
+ assertRecord(domain, index, actual.getRecord(), expected.getRecord());
208
+ assertThat(reason, actual.getRevision(), is(expected.getRevision()));
209
+ }
210
+
211
+ private static void assertUpdateKey(
212
+ String domain, int index, UpdateKey actual, UpdateKey expected) {
213
+ String reason = String.format("%s:%d", domain, index);
214
+ assertThat(reason, actual.getField(), is(expected.getField()));
215
+ assertThat(reason, actual.getValue(), is(expected.getValue()));
216
+ }
217
+
218
+ public interface Runnable {
219
+ void run() throws Exception;
220
+ }
221
+ }
@@ -0,0 +1,92 @@
1
+ package org.embulk.output.kintone;
2
+
3
+ import java.util.List;
4
+ import java.util.function.Function;
5
+ import java.util.stream.Collectors;
6
+ import org.embulk.deps.buffer.PooledBufferAllocator;
7
+ import org.embulk.spi.Column;
8
+ import org.embulk.spi.Page;
9
+ import org.embulk.spi.PageBuilder;
10
+ import org.embulk.spi.PageOutput;
11
+ import org.embulk.spi.Schema;
12
+ import org.embulk.spi.time.Timestamp;
13
+ import org.msgpack.value.Value;
14
+
15
+ public class OutputPageBuilder implements PageOutput {
16
+ private final List<String> names;
17
+ private final PageBuilder builder;
18
+ private Page page;
19
+
20
+ public static Page build(Schema schema, Function<OutputPageBuilder, Page> function) {
21
+ Page page;
22
+ try (OutputPageBuilder builder = new OutputPageBuilder(schema)) {
23
+ page = function.apply(builder);
24
+ }
25
+ return page;
26
+ }
27
+
28
+ public OutputPageBuilder(Schema schema) {
29
+ names = schema.getColumns().stream().map(Column::getName).collect(Collectors.toList());
30
+ builder = new PageBuilder(PooledBufferAllocator.create(), schema, this);
31
+ }
32
+
33
+ public OutputPageBuilder setNull(String name) {
34
+ builder.setNull(names.indexOf(name));
35
+ return this;
36
+ }
37
+
38
+ public OutputPageBuilder setBoolean(String name, boolean value) {
39
+ builder.setBoolean(names.indexOf(name), value);
40
+ return this;
41
+ }
42
+
43
+ public OutputPageBuilder setLong(String name, long value) {
44
+ builder.setLong(names.indexOf(name), value);
45
+ return this;
46
+ }
47
+
48
+ public OutputPageBuilder setDouble(String name, double value) {
49
+ builder.setDouble(names.indexOf(name), value);
50
+ return this;
51
+ }
52
+
53
+ public OutputPageBuilder setString(String name, String value) {
54
+ builder.setString(names.indexOf(name), value);
55
+ return this;
56
+ }
57
+
58
+ public OutputPageBuilder setJson(String name, Value value) {
59
+ builder.setJson(names.indexOf(name), value);
60
+ return this;
61
+ }
62
+
63
+ public OutputPageBuilder setTimestamp(String name, Timestamp value) {
64
+ builder.setTimestamp(names.indexOf(name), value);
65
+ return this;
66
+ }
67
+
68
+ public OutputPageBuilder addRecord() {
69
+ builder.addRecord();
70
+ return this;
71
+ }
72
+
73
+ public Page build() {
74
+ builder.flush();
75
+ builder.close();
76
+ return page;
77
+ }
78
+
79
+ @Override
80
+ public void add(Page page) {
81
+ if (this.page != null) {
82
+ throw new IllegalStateException();
83
+ }
84
+ this.page = page;
85
+ }
86
+
87
+ @Override
88
+ public void finish() {}
89
+
90
+ @Override
91
+ public void close() {}
92
+ }
@@ -1,3 +1,178 @@
1
1
  package org.embulk.output.kintone;
2
2
 
3
- public class TestKintoneOutputPlugin {}
3
+ import com.google.common.io.Resources;
4
+ import com.kintone.client.Json;
5
+ import com.kintone.client.model.record.Record;
6
+ import com.kintone.client.model.record.RecordForUpdate;
7
+ import com.kintone.client.model.record.UpdateKey;
8
+ import java.io.File;
9
+ import java.net.URISyntaxException;
10
+ import java.net.URL;
11
+ import java.nio.file.Path;
12
+ import java.nio.file.Paths;
13
+ import java.util.Arrays;
14
+ import java.util.Collections;
15
+ import java.util.List;
16
+ import java.util.Objects;
17
+ import java.util.function.Function;
18
+ import java.util.stream.Collectors;
19
+ import org.embulk.config.ConfigSource;
20
+ import org.embulk.config.TaskSource;
21
+ import org.embulk.spi.OutputPlugin;
22
+ import org.embulk.spi.Schema;
23
+ import org.embulk.spi.TransactionalPageOutput;
24
+ import org.embulk.spi.json.JsonParser;
25
+ import org.embulk.test.EmbulkTests;
26
+ import org.embulk.test.TestingEmbulk;
27
+ import org.junit.Rule;
28
+ import org.msgpack.value.Value;
29
+
30
+ public class TestKintoneOutputPlugin extends KintoneOutputPlugin {
31
+ private static final JsonParser PARSER = new JsonParser();
32
+
33
+ @Rule
34
+ public final TestingEmbulk embulk =
35
+ TestingEmbulk.builder()
36
+ .registerPlugin(OutputPlugin.class, "kintone", TestKintoneOutputPlugin.class)
37
+ .build();
38
+
39
+ @Override
40
+ public TransactionalPageOutput open(TaskSource taskSource, Schema schema, int taskIndex) {
41
+ String test = taskSource.get(String.class, "Domain");
42
+ String mode = taskSource.get(String.class, "Mode");
43
+ String field = taskSource.get(String.class, "UpdateKeyName");
44
+ boolean preferNulls = taskSource.get(boolean.class, "PreferNulls");
45
+ boolean ignoreNulls = taskSource.get(boolean.class, "IgnoreNulls");
46
+ return new KintonePageOutputVerifier(
47
+ super.open(taskSource, schema, taskIndex),
48
+ test,
49
+ field,
50
+ getValues(test, preferNulls, ignoreNulls),
51
+ getAddRecords(test, mode, preferNulls, ignoreNulls),
52
+ getUpdateRecords(test, mode, preferNulls, ignoreNulls, field));
53
+ }
54
+
55
+ protected void runOutput(String configName, String inputName) throws Exception {
56
+ System.gc();
57
+ ConfigSource outConfig = loadConfigYaml(configName);
58
+ Path inputPath = getResourceFile(inputName).toPath();
59
+ embulk.runOutput(outConfig, inputPath);
60
+ }
61
+
62
+ protected String getConfigName() {
63
+ return getName("config.yml");
64
+ }
65
+
66
+ protected String getInputName() {
67
+ return getName("input.csv");
68
+ }
69
+
70
+ protected String getName(String name) {
71
+ return name;
72
+ }
73
+
74
+ protected ConfigSource config(String... strings) {
75
+ return Arrays.stream(strings)
76
+ .map(this::fromYamlString)
77
+ .reduce(ConfigSource::merge)
78
+ .orElseGet(() -> fromYamlString("{}"));
79
+ }
80
+
81
+ protected ConfigSource loadConfigYaml(String name) {
82
+ ConfigSource config = loadYamlResource("config.yml");
83
+ return config.merge(loadYamlResource(name));
84
+ }
85
+
86
+ protected ConfigSource loadYamlResource(String name) {
87
+ return embulk.loadYamlResource(getResourceName(name));
88
+ }
89
+
90
+ protected ConfigSource fromYamlString(String string) {
91
+ return embulk.configLoader().fromYamlString(string);
92
+ }
93
+
94
+ private static List<String> getValues(String test, boolean preferNulls, boolean ignoreNulls) {
95
+ String name =
96
+ String.format(
97
+ "%s/values%s.json",
98
+ test, ignoreNulls ? "_ignore_nulls" : preferNulls ? "_prefer_nulls" : "");
99
+ String json = existsResource(name) ? readResource(name) : null;
100
+ return json == null || json.isEmpty()
101
+ ? Collections.emptyList()
102
+ : PARSER.parse(json).asArrayValue().list().stream()
103
+ .map(Value::toJson)
104
+ .collect(Collectors.toList());
105
+ }
106
+
107
+ private static List<Record> getAddRecords(
108
+ String test, String mode, boolean preferNulls, boolean ignoreNulls) {
109
+ String name =
110
+ String.format(
111
+ "%s/%s_add%s_records.jsonl",
112
+ test, mode, ignoreNulls ? "_ignore_nulls" : preferNulls ? "_prefer_nulls" : "");
113
+ String jsonl = existsResource(name) ? readResource(name) : null;
114
+ return jsonl == null || jsonl.isEmpty()
115
+ ? Collections.emptyList()
116
+ : Arrays.stream(jsonl.split("\\r?\\n|\\r"))
117
+ .map(s -> Json.parse(s, Record.class))
118
+ .collect(Collectors.toList());
119
+ }
120
+
121
+ private static List<RecordForUpdate> getUpdateRecords(
122
+ String test, String mode, boolean preferNulls, boolean ignoreNulls, String field) {
123
+ Function<Record, UpdateKey> key = getKey(field);
124
+ String name =
125
+ String.format(
126
+ "%s/%s_update%s_records.jsonl",
127
+ test, mode, ignoreNulls ? "_ignore_nulls" : preferNulls ? "_prefer_nulls" : "");
128
+ String jsonl = existsResource(name) ? readResource(name) : null;
129
+ return jsonl == null || jsonl.isEmpty()
130
+ ? Collections.emptyList()
131
+ : Arrays.stream(jsonl.split("\\r?\\n|\\r"))
132
+ .map(s -> Json.parse(s, Record.class))
133
+ .map(record -> new RecordForUpdate(key.apply(record), record.removeField(field)))
134
+ .collect(Collectors.toList());
135
+ }
136
+
137
+ private static Function<Record, UpdateKey> getKey(String field) {
138
+ return field == null
139
+ ? record -> null
140
+ : record ->
141
+ field.matches("^.*_number$")
142
+ ? new UpdateKey(field, record.getNumberFieldValue(field))
143
+ : new UpdateKey(field, record.getSingleLineTextFieldValue(field));
144
+ }
145
+
146
+ private static File getResourceFile(String name) {
147
+ return toPath(Objects.requireNonNull(getResource(getResourceName(name)))).toFile();
148
+ }
149
+
150
+ private static String readResource(String name) {
151
+ return EmbulkTests.readResource(getResourceName(name));
152
+ }
153
+
154
+ private static boolean existsResource(String name) {
155
+ return getResource(getResourceName(name)) != null;
156
+ }
157
+
158
+ private static String getResourceName(String name) {
159
+ return String.format("org/embulk/output/kintone/%s", name);
160
+ }
161
+
162
+ private static Path toPath(URL url) {
163
+ try {
164
+ return Paths.get(url.toURI());
165
+ } catch (URISyntaxException e) {
166
+ throw new RuntimeException(e);
167
+ }
168
+ }
169
+
170
+ @SuppressWarnings("UnstableApiUsage")
171
+ private static URL getResource(String resourceName) {
172
+ try {
173
+ return Resources.getResource(resourceName);
174
+ } catch (IllegalArgumentException e) {
175
+ return null;
176
+ }
177
+ }
178
+ }
@@ -0,0 +1,40 @@
1
+ package org.embulk.output.kintone;
2
+
3
+ import org.embulk.config.ConfigSource;
4
+ import org.junit.Before;
5
+
6
+ public class TestTask extends TestKintoneOutputPlugin {
7
+ private ConfigSource config;
8
+
9
+ @Before
10
+ public void before() {
11
+ config = fromYamlString("{}");
12
+ }
13
+
14
+ @Override
15
+ protected ConfigSource loadConfigYaml(String name) {
16
+ ConfigSource config = super.loadConfigYaml("task/config.yml");
17
+ return config.merge(loadYamlResource(name)).merge(this.config);
18
+ }
19
+
20
+ protected void merge(ConfigSource config) {
21
+ this.config.merge(config);
22
+ }
23
+
24
+ protected void runOutput() throws Exception {
25
+ String test = config.get(String.class, "domain");
26
+ runOutput(getConfigName(test), getInputName(test));
27
+ }
28
+
29
+ private String getConfigName(String test) {
30
+ return getName(test, getConfigName());
31
+ }
32
+
33
+ private String getInputName(String test) {
34
+ return getName(test, getInputName());
35
+ }
36
+
37
+ private static String getName(String test, String name) {
38
+ return String.format("%s/%s", test, name);
39
+ }
40
+ }
@@ -0,0 +1,43 @@
1
+ package org.embulk.output.kintone;
2
+
3
+ import net.jcip.annotations.NotThreadSafe;
4
+ import org.junit.Test;
5
+
6
+ @NotThreadSafe
7
+ public class TestTaskMode extends TestTask {
8
+ @Override
9
+ public void before() {
10
+ super.before();
11
+ merge(config("domain: task/mode"));
12
+ }
13
+
14
+ @Test
15
+ public void testInsert() throws Exception {
16
+ merge(config("mode: insert"));
17
+ runOutput();
18
+ merge(config("prefer_nulls: true"));
19
+ runOutput();
20
+ merge(config("ignore_nulls: true"));
21
+ runOutput();
22
+ }
23
+
24
+ @Test
25
+ public void testUpdate() throws Exception {
26
+ merge(config("mode: update", "update_key: string_number"));
27
+ runOutput();
28
+ merge(config("prefer_nulls: true"));
29
+ runOutput();
30
+ merge(config("ignore_nulls: true"));
31
+ runOutput();
32
+ }
33
+
34
+ @Test
35
+ public void testUpsert() throws Exception {
36
+ merge(config("mode: upsert", "update_key: double_single_line_text"));
37
+ runOutput();
38
+ merge(config("prefer_nulls: true"));
39
+ runOutput();
40
+ merge(config("ignore_nulls: true"));
41
+ runOutput();
42
+ }
43
+ }
@@ -0,0 +1,14 @@
1
+ <?xml version="1.0" encoding="UTF-8" ?>
2
+ <!DOCTYPE configuration>
3
+ <configuration>
4
+ <import class="ch.qos.logback.classic.encoder.PatternLayoutEncoder"/>
5
+ <import class="ch.qos.logback.core.ConsoleAppender"/>
6
+ <appender name="STDOUT" class="ConsoleAppender">
7
+ <encoder class="PatternLayoutEncoder">
8
+ <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} -%kvp- %msg%n</pattern>
9
+ </encoder>
10
+ </appender>
11
+ <root level="debug">
12
+ <appender-ref ref="STDOUT"/>
13
+ </root>
14
+ </configuration>
@@ -0,0 +1,4 @@
1
+ type: kintone
2
+ domain: domain
3
+ app_id: 0
4
+ token: token