embulk-output-kintone 0.4.0 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/ci.yml +1 -0
- data/.gitignore +0 -1
- data/README.md +8 -6
- data/build.gradle +18 -19
- data/classpath/embulk-output-kintone-1.0.0.jar +0 -0
- data/classpath/shadow-kintone-java-client-1.0.0-all.jar +0 -0
- data/gradle/wrapper/gradle-wrapper.properties +1 -1
- data/settings.gradle +2 -0
- data/shadow-kintone-java-client/build.gradle +35 -0
- data/src/main/java/org/embulk/output/kintone/KintoneColumnOption.java +1 -2
- data/src/main/java/org/embulk/output/kintone/KintoneColumnType.java +368 -0
- data/src/main/java/org/embulk/output/kintone/KintoneColumnVisitor.java +195 -135
- data/src/main/java/org/embulk/output/kintone/KintoneMode.java +0 -1
- data/src/main/java/org/embulk/output/kintone/KintoneOutputPlugin.java +0 -2
- data/src/main/java/org/embulk/output/kintone/KintonePageOutput.java +249 -192
- data/src/main/java/org/embulk/output/kintone/KintoneRetryOption.java +19 -0
- data/src/main/java/org/embulk/output/kintone/PluginTask.java +11 -3
- data/src/test/java/com/kintone/client/Json.java +16 -0
- data/src/test/java/org/embulk/output/kintone/KintoneColumnOptionBuilder.java +62 -0
- data/src/test/java/org/embulk/output/kintone/KintoneColumnVisitorTest.java +820 -0
- data/src/test/java/org/embulk/output/kintone/KintoneColumnVisitorVerifier.java +73 -0
- data/src/test/java/org/embulk/output/kintone/KintonePageOutputVerifier.java +221 -0
- data/src/test/java/org/embulk/output/kintone/OutputPageBuilder.java +92 -0
- data/src/test/java/org/embulk/output/kintone/TestKintoneOutputPlugin.java +176 -1
- data/src/test/java/org/embulk/output/kintone/TestTask.java +40 -0
- data/src/test/java/org/embulk/output/kintone/TestTaskMode.java +43 -0
- data/src/test/resources/logback-test.xml +14 -0
- data/src/test/resources/org/embulk/output/kintone/config.yml +4 -0
- data/src/test/resources/org/embulk/output/kintone/task/config.yml +1 -0
- data/src/test/resources/org/embulk/output/kintone/task/mode/config.yml +158 -0
- data/src/test/resources/org/embulk/output/kintone/task/mode/input.csv +7 -0
- data/src/test/resources/org/embulk/output/kintone/task/mode/insert_add_ignore_nulls_records.jsonl +6 -0
- data/src/test/resources/org/embulk/output/kintone/task/mode/insert_add_prefer_nulls_records.jsonl +6 -0
- data/src/test/resources/org/embulk/output/kintone/task/mode/insert_add_records.jsonl +6 -0
- data/src/test/resources/org/embulk/output/kintone/task/mode/update_update_ignore_nulls_records.jsonl +3 -0
- data/src/test/resources/org/embulk/output/kintone/task/mode/update_update_prefer_nulls_records.jsonl +3 -0
- data/src/test/resources/org/embulk/output/kintone/task/mode/update_update_records.jsonl +6 -0
- data/src/test/resources/org/embulk/output/kintone/task/mode/upsert_add_ignore_nulls_records.jsonl +3 -0
- data/src/test/resources/org/embulk/output/kintone/task/mode/upsert_add_prefer_nulls_records.jsonl +3 -0
- data/src/test/resources/org/embulk/output/kintone/task/mode/upsert_add_records.jsonl +2 -0
- data/src/test/resources/org/embulk/output/kintone/task/mode/upsert_update_ignore_nulls_records.jsonl +3 -0
- data/src/test/resources/org/embulk/output/kintone/task/mode/upsert_update_prefer_nulls_records.jsonl +3 -0
- data/src/test/resources/org/embulk/output/kintone/task/mode/upsert_update_records.jsonl +4 -0
- data/src/test/resources/org/embulk/output/kintone/task/mode/values.json +1 -0
- data/src/test/resources/org/embulk/output/kintone/task/mode/values_ignore_nulls.json +1 -0
- data/src/test/resources/org/embulk/output/kintone/task/mode/values_prefer_nulls.json +1 -0
- metadata +36 -3
- data/classpath/embulk-output-kintone-0.4.0.jar +0 -0
@@ -0,0 +1,73 @@
|
|
1
|
+
package org.embulk.output.kintone;
|
2
|
+
|
3
|
+
import com.kintone.client.model.record.FieldType;
|
4
|
+
import com.kintone.client.model.record.Record;
|
5
|
+
import com.kintone.client.model.record.UpdateKey;
|
6
|
+
import java.util.Map;
|
7
|
+
import java.util.function.BiConsumer;
|
8
|
+
import javax.validation.UnexpectedTypeException;
|
9
|
+
import org.embulk.spi.Column;
|
10
|
+
import org.embulk.spi.Page;
|
11
|
+
import org.embulk.spi.PageReader;
|
12
|
+
import org.embulk.spi.Schema;
|
13
|
+
|
14
|
+
public class KintoneColumnVisitorVerifier {
|
15
|
+
private final Schema schema;
|
16
|
+
private final Map<String, KintoneColumnOption> options;
|
17
|
+
private final PageReader reader;
|
18
|
+
private final KintoneColumnVisitor visitor;
|
19
|
+
|
20
|
+
public KintoneColumnVisitorVerifier(
|
21
|
+
Schema schema, Map<String, KintoneColumnOption> options, String updateKeyName, Page page) {
|
22
|
+
this(schema, options, false, false, updateKeyName, page);
|
23
|
+
}
|
24
|
+
|
25
|
+
public KintoneColumnVisitorVerifier(
|
26
|
+
Schema schema,
|
27
|
+
Map<String, KintoneColumnOption> options,
|
28
|
+
boolean preferNulls,
|
29
|
+
boolean ignoreNulls,
|
30
|
+
String updateKeyName,
|
31
|
+
Page page) {
|
32
|
+
this.schema = schema;
|
33
|
+
this.options = options;
|
34
|
+
reader = new PageReader(schema);
|
35
|
+
reader.setPage(page);
|
36
|
+
visitor = new KintoneColumnVisitor(reader, options, preferNulls, ignoreNulls, updateKeyName);
|
37
|
+
}
|
38
|
+
|
39
|
+
public void verify() {
|
40
|
+
verify((record, updateKey) -> {});
|
41
|
+
}
|
42
|
+
|
43
|
+
public void verify(BiConsumer<Record, UpdateKey> consumer) {
|
44
|
+
verify(consumer, false);
|
45
|
+
}
|
46
|
+
|
47
|
+
public void verify(BiConsumer<Record, UpdateKey> consumer, boolean nullable) {
|
48
|
+
if (!reader.nextRecord()) {
|
49
|
+
throw new IllegalStateException();
|
50
|
+
}
|
51
|
+
Record record = new Record();
|
52
|
+
visitor.setRecord(record);
|
53
|
+
UpdateKey updateKey = new UpdateKey();
|
54
|
+
visitor.setUpdateKey(updateKey);
|
55
|
+
schema.visitColumns(visitor);
|
56
|
+
schema.getColumns().forEach(column -> verify(record, column, nullable));
|
57
|
+
consumer.accept(record, updateKey);
|
58
|
+
}
|
59
|
+
|
60
|
+
private void verify(Record record, Column column, boolean nullable) {
|
61
|
+
FieldType expected = FieldType.valueOf(options.get(column.getName()).getType());
|
62
|
+
FieldType actual = record.getFieldType(column.getName());
|
63
|
+
if (actual == null && nullable) {
|
64
|
+
return;
|
65
|
+
}
|
66
|
+
if (expected != actual) {
|
67
|
+
throw new UnexpectedTypeException(
|
68
|
+
String.format(
|
69
|
+
"%s: Expected type is %s, but actual type is %s%n",
|
70
|
+
column.getName(), expected, actual));
|
71
|
+
}
|
72
|
+
}
|
73
|
+
}
|
@@ -0,0 +1,221 @@
|
|
1
|
+
package org.embulk.output.kintone;
|
2
|
+
|
3
|
+
import static org.hamcrest.MatcherAssert.assertThat;
|
4
|
+
import static org.hamcrest.Matchers.is;
|
5
|
+
import static org.mockito.ArgumentMatchers.anyList;
|
6
|
+
import static org.mockito.ArgumentMatchers.eq;
|
7
|
+
import static org.mockito.ArgumentMatchers.matches;
|
8
|
+
import static org.mockito.Mockito.atLeast;
|
9
|
+
import static org.mockito.Mockito.mock;
|
10
|
+
import static org.mockito.Mockito.mockStatic;
|
11
|
+
import static org.mockito.Mockito.verify;
|
12
|
+
import static org.mockito.Mockito.when;
|
13
|
+
|
14
|
+
import com.kintone.client.AppClient;
|
15
|
+
import com.kintone.client.KintoneClient;
|
16
|
+
import com.kintone.client.KintoneClientBuilder;
|
17
|
+
import com.kintone.client.RecordClient;
|
18
|
+
import com.kintone.client.api.record.GetRecordsByCursorResponseBody;
|
19
|
+
import com.kintone.client.model.app.field.FieldProperty;
|
20
|
+
import com.kintone.client.model.app.field.NumberFieldProperty;
|
21
|
+
import com.kintone.client.model.app.field.SingleLineTextFieldProperty;
|
22
|
+
import com.kintone.client.model.record.FieldValue;
|
23
|
+
import com.kintone.client.model.record.NumberFieldValue;
|
24
|
+
import com.kintone.client.model.record.Record;
|
25
|
+
import com.kintone.client.model.record.RecordForUpdate;
|
26
|
+
import com.kintone.client.model.record.SingleLineTextFieldValue;
|
27
|
+
import com.kintone.client.model.record.UpdateKey;
|
28
|
+
import java.math.BigDecimal;
|
29
|
+
import java.util.Collection;
|
30
|
+
import java.util.Collections;
|
31
|
+
import java.util.List;
|
32
|
+
import java.util.Map;
|
33
|
+
import java.util.stream.Collectors;
|
34
|
+
import java.util.stream.IntStream;
|
35
|
+
import org.embulk.config.TaskReport;
|
36
|
+
import org.embulk.spi.Page;
|
37
|
+
import org.embulk.spi.TransactionalPageOutput;
|
38
|
+
import org.mockito.ArgumentCaptor;
|
39
|
+
import org.mockito.MockedStatic;
|
40
|
+
|
41
|
+
public class KintonePageOutputVerifier implements TransactionalPageOutput {
|
42
|
+
private final TransactionalPageOutput transactionalPageOutput;
|
43
|
+
private final String domain;
|
44
|
+
private final String field;
|
45
|
+
private final List<String> values;
|
46
|
+
private final List<Record> addRecords;
|
47
|
+
private final List<RecordForUpdate> updateRecords;
|
48
|
+
|
49
|
+
public KintonePageOutputVerifier(
|
50
|
+
TransactionalPageOutput transactionalPageOutput,
|
51
|
+
String domain,
|
52
|
+
String field,
|
53
|
+
List<String> values,
|
54
|
+
List<Record> addRecords,
|
55
|
+
List<RecordForUpdate> updateRecords) {
|
56
|
+
this.transactionalPageOutput = transactionalPageOutput;
|
57
|
+
this.domain = domain;
|
58
|
+
this.field = field;
|
59
|
+
this.values = values;
|
60
|
+
this.addRecords = addRecords;
|
61
|
+
this.updateRecords = updateRecords;
|
62
|
+
}
|
63
|
+
|
64
|
+
@Override
|
65
|
+
public void add(Page page) {
|
66
|
+
runWithMock(() -> transactionalPageOutput.add(page));
|
67
|
+
}
|
68
|
+
|
69
|
+
@Override
|
70
|
+
public void finish() {
|
71
|
+
transactionalPageOutput.finish();
|
72
|
+
}
|
73
|
+
|
74
|
+
@Override
|
75
|
+
public void close() {
|
76
|
+
transactionalPageOutput.close();
|
77
|
+
}
|
78
|
+
|
79
|
+
@Override
|
80
|
+
public void abort() {
|
81
|
+
transactionalPageOutput.abort();
|
82
|
+
}
|
83
|
+
|
84
|
+
@Override
|
85
|
+
public TaskReport commit() {
|
86
|
+
return transactionalPageOutput.commit();
|
87
|
+
}
|
88
|
+
|
89
|
+
public void runWithMock(Runnable runnable) {
|
90
|
+
try {
|
91
|
+
runWithMockClient(runnable);
|
92
|
+
} catch (Exception e) {
|
93
|
+
throw new RuntimeException(e);
|
94
|
+
}
|
95
|
+
}
|
96
|
+
|
97
|
+
private void runWithMockClient(Runnable runnable) throws Exception {
|
98
|
+
@SuppressWarnings("unchecked")
|
99
|
+
Map<String, FieldProperty> mockFormFields = mock(Map.class);
|
100
|
+
when(mockFormFields.get(matches("^.*_single_line_text$")))
|
101
|
+
.thenReturn(new SingleLineTextFieldProperty());
|
102
|
+
when(mockFormFields.get(matches("^.*_number$"))).thenReturn(new NumberFieldProperty());
|
103
|
+
AppClient mockAppClient = mock(AppClient.class);
|
104
|
+
when(mockAppClient.getFormFields(eq(0L))).thenReturn(mockFormFields);
|
105
|
+
GetRecordsByCursorResponseBody mockGetRecordsByCursorResponseBody =
|
106
|
+
mock(GetRecordsByCursorResponseBody.class);
|
107
|
+
when(mockGetRecordsByCursorResponseBody.getRecords())
|
108
|
+
.thenReturn(updateRecords.stream().map(this::getRecord).collect(Collectors.toList()));
|
109
|
+
when(mockGetRecordsByCursorResponseBody.hasNext()).thenReturn(false);
|
110
|
+
RecordClient mockRecordClient = mock(RecordClient.class);
|
111
|
+
when(mockRecordClient.createCursor(eq(0L), eq(getFields()), eq(getQuery()))).thenReturn("id");
|
112
|
+
when(mockRecordClient.getRecordsByCursor(eq("id")))
|
113
|
+
.thenReturn(mockGetRecordsByCursorResponseBody);
|
114
|
+
when(mockRecordClient.addRecords(eq(0L), anyList())).thenReturn(Collections.emptyList());
|
115
|
+
when(mockRecordClient.updateRecords(eq(0L), anyList())).thenReturn(Collections.emptyList());
|
116
|
+
KintoneClient mockKintoneClient = mock(KintoneClient.class);
|
117
|
+
when(mockKintoneClient.app()).thenReturn(mockAppClient);
|
118
|
+
when(mockKintoneClient.record()).thenReturn(mockRecordClient);
|
119
|
+
KintoneClientBuilder mockKintoneClientBuilder = mock(KintoneClientBuilder.class);
|
120
|
+
when(mockKintoneClientBuilder.authByApiToken(eq("token"))).thenReturn(mockKintoneClientBuilder);
|
121
|
+
when(mockKintoneClientBuilder.build()).thenReturn(mockKintoneClient);
|
122
|
+
try (MockedStatic<KintoneClientBuilder> mocked = mockStatic(KintoneClientBuilder.class)) {
|
123
|
+
mocked
|
124
|
+
.when(() -> KintoneClientBuilder.create(String.format("https://%s", domain)))
|
125
|
+
.thenReturn(mockKintoneClientBuilder);
|
126
|
+
runnable.run();
|
127
|
+
}
|
128
|
+
@SuppressWarnings("unchecked")
|
129
|
+
ArgumentCaptor<List<Record>> addRecordsArgumentCaptor = ArgumentCaptor.forClass(List.class);
|
130
|
+
verify(mockRecordClient, atLeast(0)).addRecords(eq(0L), addRecordsArgumentCaptor.capture());
|
131
|
+
assertRecords(
|
132
|
+
domain,
|
133
|
+
addRecordsArgumentCaptor.getAllValues().stream()
|
134
|
+
.flatMap(Collection::stream)
|
135
|
+
.collect(Collectors.toList()),
|
136
|
+
addRecords);
|
137
|
+
@SuppressWarnings("unchecked")
|
138
|
+
ArgumentCaptor<List<RecordForUpdate>> updateRecordsArgumentCaptor =
|
139
|
+
ArgumentCaptor.forClass(List.class);
|
140
|
+
verify(mockRecordClient, atLeast(0))
|
141
|
+
.updateRecords(eq(0L), updateRecordsArgumentCaptor.capture());
|
142
|
+
assertRecordForUpdates(
|
143
|
+
domain,
|
144
|
+
updateRecordsArgumentCaptor.getAllValues().stream()
|
145
|
+
.flatMap(Collection::stream)
|
146
|
+
.collect(Collectors.toList()),
|
147
|
+
updateRecords);
|
148
|
+
}
|
149
|
+
|
150
|
+
private Record getRecord(RecordForUpdate updateRecord) {
|
151
|
+
UpdateKey key = updateRecord.getUpdateKey();
|
152
|
+
String field = key.getField();
|
153
|
+
Object value = key.getValue();
|
154
|
+
return new Record()
|
155
|
+
.putField(
|
156
|
+
field,
|
157
|
+
field.matches("^.*_number$")
|
158
|
+
? new NumberFieldValue((BigDecimal) value)
|
159
|
+
: new SingleLineTextFieldValue((String) value));
|
160
|
+
}
|
161
|
+
|
162
|
+
private List<String> getFields() {
|
163
|
+
return Collections.singletonList(field);
|
164
|
+
}
|
165
|
+
|
166
|
+
private String getQuery() {
|
167
|
+
return String.format("%s in (%s)", field, String.join(",", values));
|
168
|
+
}
|
169
|
+
|
170
|
+
private static void assertRecords(String domain, List<Record> actual, List<Record> expected) {
|
171
|
+
assertThat(domain, actual.size(), is(expected.size()));
|
172
|
+
// spotless:off
|
173
|
+
IntStream.range(0, actual.size()).forEach(index -> assertRecord(domain, index, actual.get(index), expected.get(index)));
|
174
|
+
// spotless:on
|
175
|
+
}
|
176
|
+
|
177
|
+
private static void assertRecord(String domain, int index, Record actual, Record expected) {
|
178
|
+
String reason = String.format("%s:%d", domain, index);
|
179
|
+
assertThat(reason, actual.getId(), is(expected.getId()));
|
180
|
+
assertThat(reason, actual.getRevision(), is(expected.getRevision()));
|
181
|
+
assertThat(reason, actual.getFieldCodes(true), is(expected.getFieldCodes(true)));
|
182
|
+
// spotless:off
|
183
|
+
actual.getFieldCodes(true).forEach(fieldCode -> assertFieldValue(domain, index, fieldCode, actual.getFieldValue(fieldCode), expected.getFieldValue(fieldCode)));
|
184
|
+
// spotless:on
|
185
|
+
}
|
186
|
+
|
187
|
+
private static void assertFieldValue(
|
188
|
+
String domain, int index, String fieldCode, FieldValue actual, FieldValue expected) {
|
189
|
+
String reason = String.format("%s:%d:%s", domain, index, fieldCode);
|
190
|
+
assertThat(reason, actual.getType(), is(expected.getType()));
|
191
|
+
assertThat(reason, actual, is(expected));
|
192
|
+
}
|
193
|
+
|
194
|
+
private static void assertRecordForUpdates(
|
195
|
+
String domain, List<RecordForUpdate> actual, List<RecordForUpdate> expected) {
|
196
|
+
assertThat(domain, actual.size(), is(expected.size()));
|
197
|
+
// spotless:off
|
198
|
+
IntStream.range(0, actual.size()).forEach(index -> assertRecordForUpdate(domain, index, actual.get(index), expected.get(index)));
|
199
|
+
// spotless:on
|
200
|
+
}
|
201
|
+
|
202
|
+
private static void assertRecordForUpdate(
|
203
|
+
String domain, int index, RecordForUpdate actual, RecordForUpdate expected) {
|
204
|
+
String reason = String.format("%s:%d", domain, index);
|
205
|
+
assertThat(reason, actual.getId(), is(expected.getId()));
|
206
|
+
assertUpdateKey(domain, index, actual.getUpdateKey(), expected.getUpdateKey());
|
207
|
+
assertRecord(domain, index, actual.getRecord(), expected.getRecord());
|
208
|
+
assertThat(reason, actual.getRevision(), is(expected.getRevision()));
|
209
|
+
}
|
210
|
+
|
211
|
+
private static void assertUpdateKey(
|
212
|
+
String domain, int index, UpdateKey actual, UpdateKey expected) {
|
213
|
+
String reason = String.format("%s:%d", domain, index);
|
214
|
+
assertThat(reason, actual.getField(), is(expected.getField()));
|
215
|
+
assertThat(reason, actual.getValue(), is(expected.getValue()));
|
216
|
+
}
|
217
|
+
|
218
|
+
public interface Runnable {
|
219
|
+
void run() throws Exception;
|
220
|
+
}
|
221
|
+
}
|
@@ -0,0 +1,92 @@
|
|
1
|
+
package org.embulk.output.kintone;
|
2
|
+
|
3
|
+
import java.util.List;
|
4
|
+
import java.util.function.Function;
|
5
|
+
import java.util.stream.Collectors;
|
6
|
+
import org.embulk.deps.buffer.PooledBufferAllocator;
|
7
|
+
import org.embulk.spi.Column;
|
8
|
+
import org.embulk.spi.Page;
|
9
|
+
import org.embulk.spi.PageBuilder;
|
10
|
+
import org.embulk.spi.PageOutput;
|
11
|
+
import org.embulk.spi.Schema;
|
12
|
+
import org.embulk.spi.time.Timestamp;
|
13
|
+
import org.msgpack.value.Value;
|
14
|
+
|
15
|
+
public class OutputPageBuilder implements PageOutput {
|
16
|
+
private final List<String> names;
|
17
|
+
private final PageBuilder builder;
|
18
|
+
private Page page;
|
19
|
+
|
20
|
+
public static Page build(Schema schema, Function<OutputPageBuilder, Page> function) {
|
21
|
+
Page page;
|
22
|
+
try (OutputPageBuilder builder = new OutputPageBuilder(schema)) {
|
23
|
+
page = function.apply(builder);
|
24
|
+
}
|
25
|
+
return page;
|
26
|
+
}
|
27
|
+
|
28
|
+
public OutputPageBuilder(Schema schema) {
|
29
|
+
names = schema.getColumns().stream().map(Column::getName).collect(Collectors.toList());
|
30
|
+
builder = new PageBuilder(PooledBufferAllocator.create(), schema, this);
|
31
|
+
}
|
32
|
+
|
33
|
+
public OutputPageBuilder setNull(String name) {
|
34
|
+
builder.setNull(names.indexOf(name));
|
35
|
+
return this;
|
36
|
+
}
|
37
|
+
|
38
|
+
public OutputPageBuilder setBoolean(String name, boolean value) {
|
39
|
+
builder.setBoolean(names.indexOf(name), value);
|
40
|
+
return this;
|
41
|
+
}
|
42
|
+
|
43
|
+
public OutputPageBuilder setLong(String name, long value) {
|
44
|
+
builder.setLong(names.indexOf(name), value);
|
45
|
+
return this;
|
46
|
+
}
|
47
|
+
|
48
|
+
public OutputPageBuilder setDouble(String name, double value) {
|
49
|
+
builder.setDouble(names.indexOf(name), value);
|
50
|
+
return this;
|
51
|
+
}
|
52
|
+
|
53
|
+
public OutputPageBuilder setString(String name, String value) {
|
54
|
+
builder.setString(names.indexOf(name), value);
|
55
|
+
return this;
|
56
|
+
}
|
57
|
+
|
58
|
+
public OutputPageBuilder setJson(String name, Value value) {
|
59
|
+
builder.setJson(names.indexOf(name), value);
|
60
|
+
return this;
|
61
|
+
}
|
62
|
+
|
63
|
+
public OutputPageBuilder setTimestamp(String name, Timestamp value) {
|
64
|
+
builder.setTimestamp(names.indexOf(name), value);
|
65
|
+
return this;
|
66
|
+
}
|
67
|
+
|
68
|
+
public OutputPageBuilder addRecord() {
|
69
|
+
builder.addRecord();
|
70
|
+
return this;
|
71
|
+
}
|
72
|
+
|
73
|
+
public Page build() {
|
74
|
+
builder.flush();
|
75
|
+
builder.close();
|
76
|
+
return page;
|
77
|
+
}
|
78
|
+
|
79
|
+
@Override
|
80
|
+
public void add(Page page) {
|
81
|
+
if (this.page != null) {
|
82
|
+
throw new IllegalStateException();
|
83
|
+
}
|
84
|
+
this.page = page;
|
85
|
+
}
|
86
|
+
|
87
|
+
@Override
|
88
|
+
public void finish() {}
|
89
|
+
|
90
|
+
@Override
|
91
|
+
public void close() {}
|
92
|
+
}
|
@@ -1,3 +1,178 @@
|
|
1
1
|
package org.embulk.output.kintone;
|
2
2
|
|
3
|
-
|
3
|
+
import com.google.common.io.Resources;
|
4
|
+
import com.kintone.client.Json;
|
5
|
+
import com.kintone.client.model.record.Record;
|
6
|
+
import com.kintone.client.model.record.RecordForUpdate;
|
7
|
+
import com.kintone.client.model.record.UpdateKey;
|
8
|
+
import java.io.File;
|
9
|
+
import java.net.URISyntaxException;
|
10
|
+
import java.net.URL;
|
11
|
+
import java.nio.file.Path;
|
12
|
+
import java.nio.file.Paths;
|
13
|
+
import java.util.Arrays;
|
14
|
+
import java.util.Collections;
|
15
|
+
import java.util.List;
|
16
|
+
import java.util.Objects;
|
17
|
+
import java.util.function.Function;
|
18
|
+
import java.util.stream.Collectors;
|
19
|
+
import org.embulk.config.ConfigSource;
|
20
|
+
import org.embulk.config.TaskSource;
|
21
|
+
import org.embulk.spi.OutputPlugin;
|
22
|
+
import org.embulk.spi.Schema;
|
23
|
+
import org.embulk.spi.TransactionalPageOutput;
|
24
|
+
import org.embulk.spi.json.JsonParser;
|
25
|
+
import org.embulk.test.EmbulkTests;
|
26
|
+
import org.embulk.test.TestingEmbulk;
|
27
|
+
import org.junit.Rule;
|
28
|
+
import org.msgpack.value.Value;
|
29
|
+
|
30
|
+
public class TestKintoneOutputPlugin extends KintoneOutputPlugin {
|
31
|
+
private static final JsonParser PARSER = new JsonParser();
|
32
|
+
|
33
|
+
@Rule
|
34
|
+
public final TestingEmbulk embulk =
|
35
|
+
TestingEmbulk.builder()
|
36
|
+
.registerPlugin(OutputPlugin.class, "kintone", TestKintoneOutputPlugin.class)
|
37
|
+
.build();
|
38
|
+
|
39
|
+
@Override
|
40
|
+
public TransactionalPageOutput open(TaskSource taskSource, Schema schema, int taskIndex) {
|
41
|
+
String test = taskSource.get(String.class, "Domain");
|
42
|
+
String mode = taskSource.get(String.class, "Mode");
|
43
|
+
String field = taskSource.get(String.class, "UpdateKeyName");
|
44
|
+
boolean preferNulls = taskSource.get(boolean.class, "PreferNulls");
|
45
|
+
boolean ignoreNulls = taskSource.get(boolean.class, "IgnoreNulls");
|
46
|
+
return new KintonePageOutputVerifier(
|
47
|
+
super.open(taskSource, schema, taskIndex),
|
48
|
+
test,
|
49
|
+
field,
|
50
|
+
getValues(test, preferNulls, ignoreNulls),
|
51
|
+
getAddRecords(test, mode, preferNulls, ignoreNulls),
|
52
|
+
getUpdateRecords(test, mode, preferNulls, ignoreNulls, field));
|
53
|
+
}
|
54
|
+
|
55
|
+
protected void runOutput(String configName, String inputName) throws Exception {
|
56
|
+
System.gc();
|
57
|
+
ConfigSource outConfig = loadConfigYaml(configName);
|
58
|
+
Path inputPath = getResourceFile(inputName).toPath();
|
59
|
+
embulk.runOutput(outConfig, inputPath);
|
60
|
+
}
|
61
|
+
|
62
|
+
protected String getConfigName() {
|
63
|
+
return getName("config.yml");
|
64
|
+
}
|
65
|
+
|
66
|
+
protected String getInputName() {
|
67
|
+
return getName("input.csv");
|
68
|
+
}
|
69
|
+
|
70
|
+
protected String getName(String name) {
|
71
|
+
return name;
|
72
|
+
}
|
73
|
+
|
74
|
+
protected ConfigSource config(String... strings) {
|
75
|
+
return Arrays.stream(strings)
|
76
|
+
.map(this::fromYamlString)
|
77
|
+
.reduce(ConfigSource::merge)
|
78
|
+
.orElseGet(() -> fromYamlString("{}"));
|
79
|
+
}
|
80
|
+
|
81
|
+
protected ConfigSource loadConfigYaml(String name) {
|
82
|
+
ConfigSource config = loadYamlResource("config.yml");
|
83
|
+
return config.merge(loadYamlResource(name));
|
84
|
+
}
|
85
|
+
|
86
|
+
protected ConfigSource loadYamlResource(String name) {
|
87
|
+
return embulk.loadYamlResource(getResourceName(name));
|
88
|
+
}
|
89
|
+
|
90
|
+
protected ConfigSource fromYamlString(String string) {
|
91
|
+
return embulk.configLoader().fromYamlString(string);
|
92
|
+
}
|
93
|
+
|
94
|
+
private static List<String> getValues(String test, boolean preferNulls, boolean ignoreNulls) {
|
95
|
+
String name =
|
96
|
+
String.format(
|
97
|
+
"%s/values%s.json",
|
98
|
+
test, ignoreNulls ? "_ignore_nulls" : preferNulls ? "_prefer_nulls" : "");
|
99
|
+
String json = existsResource(name) ? readResource(name) : null;
|
100
|
+
return json == null || json.isEmpty()
|
101
|
+
? Collections.emptyList()
|
102
|
+
: PARSER.parse(json).asArrayValue().list().stream()
|
103
|
+
.map(Value::toJson)
|
104
|
+
.collect(Collectors.toList());
|
105
|
+
}
|
106
|
+
|
107
|
+
private static List<Record> getAddRecords(
|
108
|
+
String test, String mode, boolean preferNulls, boolean ignoreNulls) {
|
109
|
+
String name =
|
110
|
+
String.format(
|
111
|
+
"%s/%s_add%s_records.jsonl",
|
112
|
+
test, mode, ignoreNulls ? "_ignore_nulls" : preferNulls ? "_prefer_nulls" : "");
|
113
|
+
String jsonl = existsResource(name) ? readResource(name) : null;
|
114
|
+
return jsonl == null || jsonl.isEmpty()
|
115
|
+
? Collections.emptyList()
|
116
|
+
: Arrays.stream(jsonl.split("\\r?\\n|\\r"))
|
117
|
+
.map(s -> Json.parse(s, Record.class))
|
118
|
+
.collect(Collectors.toList());
|
119
|
+
}
|
120
|
+
|
121
|
+
private static List<RecordForUpdate> getUpdateRecords(
|
122
|
+
String test, String mode, boolean preferNulls, boolean ignoreNulls, String field) {
|
123
|
+
Function<Record, UpdateKey> key = getKey(field);
|
124
|
+
String name =
|
125
|
+
String.format(
|
126
|
+
"%s/%s_update%s_records.jsonl",
|
127
|
+
test, mode, ignoreNulls ? "_ignore_nulls" : preferNulls ? "_prefer_nulls" : "");
|
128
|
+
String jsonl = existsResource(name) ? readResource(name) : null;
|
129
|
+
return jsonl == null || jsonl.isEmpty()
|
130
|
+
? Collections.emptyList()
|
131
|
+
: Arrays.stream(jsonl.split("\\r?\\n|\\r"))
|
132
|
+
.map(s -> Json.parse(s, Record.class))
|
133
|
+
.map(record -> new RecordForUpdate(key.apply(record), record.removeField(field)))
|
134
|
+
.collect(Collectors.toList());
|
135
|
+
}
|
136
|
+
|
137
|
+
private static Function<Record, UpdateKey> getKey(String field) {
|
138
|
+
return field == null
|
139
|
+
? record -> null
|
140
|
+
: record ->
|
141
|
+
field.matches("^.*_number$")
|
142
|
+
? new UpdateKey(field, record.getNumberFieldValue(field))
|
143
|
+
: new UpdateKey(field, record.getSingleLineTextFieldValue(field));
|
144
|
+
}
|
145
|
+
|
146
|
+
private static File getResourceFile(String name) {
|
147
|
+
return toPath(Objects.requireNonNull(getResource(getResourceName(name)))).toFile();
|
148
|
+
}
|
149
|
+
|
150
|
+
private static String readResource(String name) {
|
151
|
+
return EmbulkTests.readResource(getResourceName(name));
|
152
|
+
}
|
153
|
+
|
154
|
+
private static boolean existsResource(String name) {
|
155
|
+
return getResource(getResourceName(name)) != null;
|
156
|
+
}
|
157
|
+
|
158
|
+
private static String getResourceName(String name) {
|
159
|
+
return String.format("org/embulk/output/kintone/%s", name);
|
160
|
+
}
|
161
|
+
|
162
|
+
private static Path toPath(URL url) {
|
163
|
+
try {
|
164
|
+
return Paths.get(url.toURI());
|
165
|
+
} catch (URISyntaxException e) {
|
166
|
+
throw new RuntimeException(e);
|
167
|
+
}
|
168
|
+
}
|
169
|
+
|
170
|
+
@SuppressWarnings("UnstableApiUsage")
|
171
|
+
private static URL getResource(String resourceName) {
|
172
|
+
try {
|
173
|
+
return Resources.getResource(resourceName);
|
174
|
+
} catch (IllegalArgumentException e) {
|
175
|
+
return null;
|
176
|
+
}
|
177
|
+
}
|
178
|
+
}
|
@@ -0,0 +1,40 @@
|
|
1
|
+
package org.embulk.output.kintone;
|
2
|
+
|
3
|
+
import org.embulk.config.ConfigSource;
|
4
|
+
import org.junit.Before;
|
5
|
+
|
6
|
+
public class TestTask extends TestKintoneOutputPlugin {
|
7
|
+
private ConfigSource config;
|
8
|
+
|
9
|
+
@Before
|
10
|
+
public void before() {
|
11
|
+
config = fromYamlString("{}");
|
12
|
+
}
|
13
|
+
|
14
|
+
@Override
|
15
|
+
protected ConfigSource loadConfigYaml(String name) {
|
16
|
+
ConfigSource config = super.loadConfigYaml("task/config.yml");
|
17
|
+
return config.merge(loadYamlResource(name)).merge(this.config);
|
18
|
+
}
|
19
|
+
|
20
|
+
protected void merge(ConfigSource config) {
|
21
|
+
this.config.merge(config);
|
22
|
+
}
|
23
|
+
|
24
|
+
protected void runOutput() throws Exception {
|
25
|
+
String test = config.get(String.class, "domain");
|
26
|
+
runOutput(getConfigName(test), getInputName(test));
|
27
|
+
}
|
28
|
+
|
29
|
+
private String getConfigName(String test) {
|
30
|
+
return getName(test, getConfigName());
|
31
|
+
}
|
32
|
+
|
33
|
+
private String getInputName(String test) {
|
34
|
+
return getName(test, getInputName());
|
35
|
+
}
|
36
|
+
|
37
|
+
private static String getName(String test, String name) {
|
38
|
+
return String.format("%s/%s", test, name);
|
39
|
+
}
|
40
|
+
}
|
@@ -0,0 +1,43 @@
|
|
1
|
+
package org.embulk.output.kintone;
|
2
|
+
|
3
|
+
import net.jcip.annotations.NotThreadSafe;
|
4
|
+
import org.junit.Test;
|
5
|
+
|
6
|
+
@NotThreadSafe
|
7
|
+
public class TestTaskMode extends TestTask {
|
8
|
+
@Override
|
9
|
+
public void before() {
|
10
|
+
super.before();
|
11
|
+
merge(config("domain: task/mode"));
|
12
|
+
}
|
13
|
+
|
14
|
+
@Test
|
15
|
+
public void testInsert() throws Exception {
|
16
|
+
merge(config("mode: insert"));
|
17
|
+
runOutput();
|
18
|
+
merge(config("prefer_nulls: true"));
|
19
|
+
runOutput();
|
20
|
+
merge(config("ignore_nulls: true"));
|
21
|
+
runOutput();
|
22
|
+
}
|
23
|
+
|
24
|
+
@Test
|
25
|
+
public void testUpdate() throws Exception {
|
26
|
+
merge(config("mode: update", "update_key: string_number"));
|
27
|
+
runOutput();
|
28
|
+
merge(config("prefer_nulls: true"));
|
29
|
+
runOutput();
|
30
|
+
merge(config("ignore_nulls: true"));
|
31
|
+
runOutput();
|
32
|
+
}
|
33
|
+
|
34
|
+
@Test
|
35
|
+
public void testUpsert() throws Exception {
|
36
|
+
merge(config("mode: upsert", "update_key: double_single_line_text"));
|
37
|
+
runOutput();
|
38
|
+
merge(config("prefer_nulls: true"));
|
39
|
+
runOutput();
|
40
|
+
merge(config("ignore_nulls: true"));
|
41
|
+
runOutput();
|
42
|
+
}
|
43
|
+
}
|
@@ -0,0 +1,14 @@
|
|
1
|
+
<?xml version="1.0" encoding="UTF-8" ?>
|
2
|
+
<!DOCTYPE configuration>
|
3
|
+
<configuration>
|
4
|
+
<import class="ch.qos.logback.classic.encoder.PatternLayoutEncoder"/>
|
5
|
+
<import class="ch.qos.logback.core.ConsoleAppender"/>
|
6
|
+
<appender name="STDOUT" class="ConsoleAppender">
|
7
|
+
<encoder class="PatternLayoutEncoder">
|
8
|
+
<pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} -%kvp- %msg%n</pattern>
|
9
|
+
</encoder>
|
10
|
+
</appender>
|
11
|
+
<root level="debug">
|
12
|
+
<appender-ref ref="STDOUT"/>
|
13
|
+
</root>
|
14
|
+
</configuration>
|
@@ -0,0 +1 @@
|
|
1
|
+
{}
|