embulk-output-kintone 0.4.0 → 1.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/ci.yml +1 -0
- data/.gitignore +0 -1
- data/README.md +8 -6
- data/build.gradle +18 -19
- data/classpath/embulk-output-kintone-1.0.0.jar +0 -0
- data/classpath/shadow-kintone-java-client-1.0.0-all.jar +0 -0
- data/gradle/wrapper/gradle-wrapper.properties +1 -1
- data/settings.gradle +2 -0
- data/shadow-kintone-java-client/build.gradle +35 -0
- data/src/main/java/org/embulk/output/kintone/KintoneColumnOption.java +1 -2
- data/src/main/java/org/embulk/output/kintone/KintoneColumnType.java +368 -0
- data/src/main/java/org/embulk/output/kintone/KintoneColumnVisitor.java +195 -135
- data/src/main/java/org/embulk/output/kintone/KintoneMode.java +0 -1
- data/src/main/java/org/embulk/output/kintone/KintoneOutputPlugin.java +0 -2
- data/src/main/java/org/embulk/output/kintone/KintonePageOutput.java +249 -192
- data/src/main/java/org/embulk/output/kintone/KintoneRetryOption.java +19 -0
- data/src/main/java/org/embulk/output/kintone/PluginTask.java +11 -3
- data/src/test/java/com/kintone/client/Json.java +16 -0
- data/src/test/java/org/embulk/output/kintone/KintoneColumnOptionBuilder.java +62 -0
- data/src/test/java/org/embulk/output/kintone/KintoneColumnVisitorTest.java +820 -0
- data/src/test/java/org/embulk/output/kintone/KintoneColumnVisitorVerifier.java +73 -0
- data/src/test/java/org/embulk/output/kintone/KintonePageOutputVerifier.java +221 -0
- data/src/test/java/org/embulk/output/kintone/OutputPageBuilder.java +92 -0
- data/src/test/java/org/embulk/output/kintone/TestKintoneOutputPlugin.java +176 -1
- data/src/test/java/org/embulk/output/kintone/TestTask.java +40 -0
- data/src/test/java/org/embulk/output/kintone/TestTaskMode.java +43 -0
- data/src/test/resources/logback-test.xml +14 -0
- data/src/test/resources/org/embulk/output/kintone/config.yml +4 -0
- data/src/test/resources/org/embulk/output/kintone/task/config.yml +1 -0
- data/src/test/resources/org/embulk/output/kintone/task/mode/config.yml +158 -0
- data/src/test/resources/org/embulk/output/kintone/task/mode/input.csv +7 -0
- data/src/test/resources/org/embulk/output/kintone/task/mode/insert_add_ignore_nulls_records.jsonl +6 -0
- data/src/test/resources/org/embulk/output/kintone/task/mode/insert_add_prefer_nulls_records.jsonl +6 -0
- data/src/test/resources/org/embulk/output/kintone/task/mode/insert_add_records.jsonl +6 -0
- data/src/test/resources/org/embulk/output/kintone/task/mode/update_update_ignore_nulls_records.jsonl +3 -0
- data/src/test/resources/org/embulk/output/kintone/task/mode/update_update_prefer_nulls_records.jsonl +3 -0
- data/src/test/resources/org/embulk/output/kintone/task/mode/update_update_records.jsonl +6 -0
- data/src/test/resources/org/embulk/output/kintone/task/mode/upsert_add_ignore_nulls_records.jsonl +3 -0
- data/src/test/resources/org/embulk/output/kintone/task/mode/upsert_add_prefer_nulls_records.jsonl +3 -0
- data/src/test/resources/org/embulk/output/kintone/task/mode/upsert_add_records.jsonl +2 -0
- data/src/test/resources/org/embulk/output/kintone/task/mode/upsert_update_ignore_nulls_records.jsonl +3 -0
- data/src/test/resources/org/embulk/output/kintone/task/mode/upsert_update_prefer_nulls_records.jsonl +3 -0
- data/src/test/resources/org/embulk/output/kintone/task/mode/upsert_update_records.jsonl +4 -0
- data/src/test/resources/org/embulk/output/kintone/task/mode/values.json +1 -0
- data/src/test/resources/org/embulk/output/kintone/task/mode/values_ignore_nulls.json +1 -0
- data/src/test/resources/org/embulk/output/kintone/task/mode/values_prefer_nulls.json +1 -0
- metadata +36 -3
- data/classpath/embulk-output-kintone-0.4.0.jar +0 -0
@@ -0,0 +1,73 @@
|
|
1
|
+
package org.embulk.output.kintone;
|
2
|
+
|
3
|
+
import com.kintone.client.model.record.FieldType;
|
4
|
+
import com.kintone.client.model.record.Record;
|
5
|
+
import com.kintone.client.model.record.UpdateKey;
|
6
|
+
import java.util.Map;
|
7
|
+
import java.util.function.BiConsumer;
|
8
|
+
import javax.validation.UnexpectedTypeException;
|
9
|
+
import org.embulk.spi.Column;
|
10
|
+
import org.embulk.spi.Page;
|
11
|
+
import org.embulk.spi.PageReader;
|
12
|
+
import org.embulk.spi.Schema;
|
13
|
+
|
14
|
+
public class KintoneColumnVisitorVerifier {
|
15
|
+
private final Schema schema;
|
16
|
+
private final Map<String, KintoneColumnOption> options;
|
17
|
+
private final PageReader reader;
|
18
|
+
private final KintoneColumnVisitor visitor;
|
19
|
+
|
20
|
+
public KintoneColumnVisitorVerifier(
|
21
|
+
Schema schema, Map<String, KintoneColumnOption> options, String updateKeyName, Page page) {
|
22
|
+
this(schema, options, false, false, updateKeyName, page);
|
23
|
+
}
|
24
|
+
|
25
|
+
public KintoneColumnVisitorVerifier(
|
26
|
+
Schema schema,
|
27
|
+
Map<String, KintoneColumnOption> options,
|
28
|
+
boolean preferNulls,
|
29
|
+
boolean ignoreNulls,
|
30
|
+
String updateKeyName,
|
31
|
+
Page page) {
|
32
|
+
this.schema = schema;
|
33
|
+
this.options = options;
|
34
|
+
reader = new PageReader(schema);
|
35
|
+
reader.setPage(page);
|
36
|
+
visitor = new KintoneColumnVisitor(reader, options, preferNulls, ignoreNulls, updateKeyName);
|
37
|
+
}
|
38
|
+
|
39
|
+
public void verify() {
|
40
|
+
verify((record, updateKey) -> {});
|
41
|
+
}
|
42
|
+
|
43
|
+
public void verify(BiConsumer<Record, UpdateKey> consumer) {
|
44
|
+
verify(consumer, false);
|
45
|
+
}
|
46
|
+
|
47
|
+
public void verify(BiConsumer<Record, UpdateKey> consumer, boolean nullable) {
|
48
|
+
if (!reader.nextRecord()) {
|
49
|
+
throw new IllegalStateException();
|
50
|
+
}
|
51
|
+
Record record = new Record();
|
52
|
+
visitor.setRecord(record);
|
53
|
+
UpdateKey updateKey = new UpdateKey();
|
54
|
+
visitor.setUpdateKey(updateKey);
|
55
|
+
schema.visitColumns(visitor);
|
56
|
+
schema.getColumns().forEach(column -> verify(record, column, nullable));
|
57
|
+
consumer.accept(record, updateKey);
|
58
|
+
}
|
59
|
+
|
60
|
+
private void verify(Record record, Column column, boolean nullable) {
|
61
|
+
FieldType expected = FieldType.valueOf(options.get(column.getName()).getType());
|
62
|
+
FieldType actual = record.getFieldType(column.getName());
|
63
|
+
if (actual == null && nullable) {
|
64
|
+
return;
|
65
|
+
}
|
66
|
+
if (expected != actual) {
|
67
|
+
throw new UnexpectedTypeException(
|
68
|
+
String.format(
|
69
|
+
"%s: Expected type is %s, but actual type is %s%n",
|
70
|
+
column.getName(), expected, actual));
|
71
|
+
}
|
72
|
+
}
|
73
|
+
}
|
@@ -0,0 +1,221 @@
|
|
1
|
+
package org.embulk.output.kintone;
|
2
|
+
|
3
|
+
import static org.hamcrest.MatcherAssert.assertThat;
|
4
|
+
import static org.hamcrest.Matchers.is;
|
5
|
+
import static org.mockito.ArgumentMatchers.anyList;
|
6
|
+
import static org.mockito.ArgumentMatchers.eq;
|
7
|
+
import static org.mockito.ArgumentMatchers.matches;
|
8
|
+
import static org.mockito.Mockito.atLeast;
|
9
|
+
import static org.mockito.Mockito.mock;
|
10
|
+
import static org.mockito.Mockito.mockStatic;
|
11
|
+
import static org.mockito.Mockito.verify;
|
12
|
+
import static org.mockito.Mockito.when;
|
13
|
+
|
14
|
+
import com.kintone.client.AppClient;
|
15
|
+
import com.kintone.client.KintoneClient;
|
16
|
+
import com.kintone.client.KintoneClientBuilder;
|
17
|
+
import com.kintone.client.RecordClient;
|
18
|
+
import com.kintone.client.api.record.GetRecordsByCursorResponseBody;
|
19
|
+
import com.kintone.client.model.app.field.FieldProperty;
|
20
|
+
import com.kintone.client.model.app.field.NumberFieldProperty;
|
21
|
+
import com.kintone.client.model.app.field.SingleLineTextFieldProperty;
|
22
|
+
import com.kintone.client.model.record.FieldValue;
|
23
|
+
import com.kintone.client.model.record.NumberFieldValue;
|
24
|
+
import com.kintone.client.model.record.Record;
|
25
|
+
import com.kintone.client.model.record.RecordForUpdate;
|
26
|
+
import com.kintone.client.model.record.SingleLineTextFieldValue;
|
27
|
+
import com.kintone.client.model.record.UpdateKey;
|
28
|
+
import java.math.BigDecimal;
|
29
|
+
import java.util.Collection;
|
30
|
+
import java.util.Collections;
|
31
|
+
import java.util.List;
|
32
|
+
import java.util.Map;
|
33
|
+
import java.util.stream.Collectors;
|
34
|
+
import java.util.stream.IntStream;
|
35
|
+
import org.embulk.config.TaskReport;
|
36
|
+
import org.embulk.spi.Page;
|
37
|
+
import org.embulk.spi.TransactionalPageOutput;
|
38
|
+
import org.mockito.ArgumentCaptor;
|
39
|
+
import org.mockito.MockedStatic;
|
40
|
+
|
41
|
+
public class KintonePageOutputVerifier implements TransactionalPageOutput {
|
42
|
+
private final TransactionalPageOutput transactionalPageOutput;
|
43
|
+
private final String domain;
|
44
|
+
private final String field;
|
45
|
+
private final List<String> values;
|
46
|
+
private final List<Record> addRecords;
|
47
|
+
private final List<RecordForUpdate> updateRecords;
|
48
|
+
|
49
|
+
public KintonePageOutputVerifier(
|
50
|
+
TransactionalPageOutput transactionalPageOutput,
|
51
|
+
String domain,
|
52
|
+
String field,
|
53
|
+
List<String> values,
|
54
|
+
List<Record> addRecords,
|
55
|
+
List<RecordForUpdate> updateRecords) {
|
56
|
+
this.transactionalPageOutput = transactionalPageOutput;
|
57
|
+
this.domain = domain;
|
58
|
+
this.field = field;
|
59
|
+
this.values = values;
|
60
|
+
this.addRecords = addRecords;
|
61
|
+
this.updateRecords = updateRecords;
|
62
|
+
}
|
63
|
+
|
64
|
+
@Override
|
65
|
+
public void add(Page page) {
|
66
|
+
runWithMock(() -> transactionalPageOutput.add(page));
|
67
|
+
}
|
68
|
+
|
69
|
+
@Override
|
70
|
+
public void finish() {
|
71
|
+
transactionalPageOutput.finish();
|
72
|
+
}
|
73
|
+
|
74
|
+
@Override
|
75
|
+
public void close() {
|
76
|
+
transactionalPageOutput.close();
|
77
|
+
}
|
78
|
+
|
79
|
+
@Override
|
80
|
+
public void abort() {
|
81
|
+
transactionalPageOutput.abort();
|
82
|
+
}
|
83
|
+
|
84
|
+
@Override
|
85
|
+
public TaskReport commit() {
|
86
|
+
return transactionalPageOutput.commit();
|
87
|
+
}
|
88
|
+
|
89
|
+
public void runWithMock(Runnable runnable) {
|
90
|
+
try {
|
91
|
+
runWithMockClient(runnable);
|
92
|
+
} catch (Exception e) {
|
93
|
+
throw new RuntimeException(e);
|
94
|
+
}
|
95
|
+
}
|
96
|
+
|
97
|
+
private void runWithMockClient(Runnable runnable) throws Exception {
|
98
|
+
@SuppressWarnings("unchecked")
|
99
|
+
Map<String, FieldProperty> mockFormFields = mock(Map.class);
|
100
|
+
when(mockFormFields.get(matches("^.*_single_line_text$")))
|
101
|
+
.thenReturn(new SingleLineTextFieldProperty());
|
102
|
+
when(mockFormFields.get(matches("^.*_number$"))).thenReturn(new NumberFieldProperty());
|
103
|
+
AppClient mockAppClient = mock(AppClient.class);
|
104
|
+
when(mockAppClient.getFormFields(eq(0L))).thenReturn(mockFormFields);
|
105
|
+
GetRecordsByCursorResponseBody mockGetRecordsByCursorResponseBody =
|
106
|
+
mock(GetRecordsByCursorResponseBody.class);
|
107
|
+
when(mockGetRecordsByCursorResponseBody.getRecords())
|
108
|
+
.thenReturn(updateRecords.stream().map(this::getRecord).collect(Collectors.toList()));
|
109
|
+
when(mockGetRecordsByCursorResponseBody.hasNext()).thenReturn(false);
|
110
|
+
RecordClient mockRecordClient = mock(RecordClient.class);
|
111
|
+
when(mockRecordClient.createCursor(eq(0L), eq(getFields()), eq(getQuery()))).thenReturn("id");
|
112
|
+
when(mockRecordClient.getRecordsByCursor(eq("id")))
|
113
|
+
.thenReturn(mockGetRecordsByCursorResponseBody);
|
114
|
+
when(mockRecordClient.addRecords(eq(0L), anyList())).thenReturn(Collections.emptyList());
|
115
|
+
when(mockRecordClient.updateRecords(eq(0L), anyList())).thenReturn(Collections.emptyList());
|
116
|
+
KintoneClient mockKintoneClient = mock(KintoneClient.class);
|
117
|
+
when(mockKintoneClient.app()).thenReturn(mockAppClient);
|
118
|
+
when(mockKintoneClient.record()).thenReturn(mockRecordClient);
|
119
|
+
KintoneClientBuilder mockKintoneClientBuilder = mock(KintoneClientBuilder.class);
|
120
|
+
when(mockKintoneClientBuilder.authByApiToken(eq("token"))).thenReturn(mockKintoneClientBuilder);
|
121
|
+
when(mockKintoneClientBuilder.build()).thenReturn(mockKintoneClient);
|
122
|
+
try (MockedStatic<KintoneClientBuilder> mocked = mockStatic(KintoneClientBuilder.class)) {
|
123
|
+
mocked
|
124
|
+
.when(() -> KintoneClientBuilder.create(String.format("https://%s", domain)))
|
125
|
+
.thenReturn(mockKintoneClientBuilder);
|
126
|
+
runnable.run();
|
127
|
+
}
|
128
|
+
@SuppressWarnings("unchecked")
|
129
|
+
ArgumentCaptor<List<Record>> addRecordsArgumentCaptor = ArgumentCaptor.forClass(List.class);
|
130
|
+
verify(mockRecordClient, atLeast(0)).addRecords(eq(0L), addRecordsArgumentCaptor.capture());
|
131
|
+
assertRecords(
|
132
|
+
domain,
|
133
|
+
addRecordsArgumentCaptor.getAllValues().stream()
|
134
|
+
.flatMap(Collection::stream)
|
135
|
+
.collect(Collectors.toList()),
|
136
|
+
addRecords);
|
137
|
+
@SuppressWarnings("unchecked")
|
138
|
+
ArgumentCaptor<List<RecordForUpdate>> updateRecordsArgumentCaptor =
|
139
|
+
ArgumentCaptor.forClass(List.class);
|
140
|
+
verify(mockRecordClient, atLeast(0))
|
141
|
+
.updateRecords(eq(0L), updateRecordsArgumentCaptor.capture());
|
142
|
+
assertRecordForUpdates(
|
143
|
+
domain,
|
144
|
+
updateRecordsArgumentCaptor.getAllValues().stream()
|
145
|
+
.flatMap(Collection::stream)
|
146
|
+
.collect(Collectors.toList()),
|
147
|
+
updateRecords);
|
148
|
+
}
|
149
|
+
|
150
|
+
private Record getRecord(RecordForUpdate updateRecord) {
|
151
|
+
UpdateKey key = updateRecord.getUpdateKey();
|
152
|
+
String field = key.getField();
|
153
|
+
Object value = key.getValue();
|
154
|
+
return new Record()
|
155
|
+
.putField(
|
156
|
+
field,
|
157
|
+
field.matches("^.*_number$")
|
158
|
+
? new NumberFieldValue((BigDecimal) value)
|
159
|
+
: new SingleLineTextFieldValue((String) value));
|
160
|
+
}
|
161
|
+
|
162
|
+
private List<String> getFields() {
|
163
|
+
return Collections.singletonList(field);
|
164
|
+
}
|
165
|
+
|
166
|
+
private String getQuery() {
|
167
|
+
return String.format("%s in (%s)", field, String.join(",", values));
|
168
|
+
}
|
169
|
+
|
170
|
+
private static void assertRecords(String domain, List<Record> actual, List<Record> expected) {
|
171
|
+
assertThat(domain, actual.size(), is(expected.size()));
|
172
|
+
// spotless:off
|
173
|
+
IntStream.range(0, actual.size()).forEach(index -> assertRecord(domain, index, actual.get(index), expected.get(index)));
|
174
|
+
// spotless:on
|
175
|
+
}
|
176
|
+
|
177
|
+
private static void assertRecord(String domain, int index, Record actual, Record expected) {
|
178
|
+
String reason = String.format("%s:%d", domain, index);
|
179
|
+
assertThat(reason, actual.getId(), is(expected.getId()));
|
180
|
+
assertThat(reason, actual.getRevision(), is(expected.getRevision()));
|
181
|
+
assertThat(reason, actual.getFieldCodes(true), is(expected.getFieldCodes(true)));
|
182
|
+
// spotless:off
|
183
|
+
actual.getFieldCodes(true).forEach(fieldCode -> assertFieldValue(domain, index, fieldCode, actual.getFieldValue(fieldCode), expected.getFieldValue(fieldCode)));
|
184
|
+
// spotless:on
|
185
|
+
}
|
186
|
+
|
187
|
+
private static void assertFieldValue(
|
188
|
+
String domain, int index, String fieldCode, FieldValue actual, FieldValue expected) {
|
189
|
+
String reason = String.format("%s:%d:%s", domain, index, fieldCode);
|
190
|
+
assertThat(reason, actual.getType(), is(expected.getType()));
|
191
|
+
assertThat(reason, actual, is(expected));
|
192
|
+
}
|
193
|
+
|
194
|
+
private static void assertRecordForUpdates(
|
195
|
+
String domain, List<RecordForUpdate> actual, List<RecordForUpdate> expected) {
|
196
|
+
assertThat(domain, actual.size(), is(expected.size()));
|
197
|
+
// spotless:off
|
198
|
+
IntStream.range(0, actual.size()).forEach(index -> assertRecordForUpdate(domain, index, actual.get(index), expected.get(index)));
|
199
|
+
// spotless:on
|
200
|
+
}
|
201
|
+
|
202
|
+
private static void assertRecordForUpdate(
|
203
|
+
String domain, int index, RecordForUpdate actual, RecordForUpdate expected) {
|
204
|
+
String reason = String.format("%s:%d", domain, index);
|
205
|
+
assertThat(reason, actual.getId(), is(expected.getId()));
|
206
|
+
assertUpdateKey(domain, index, actual.getUpdateKey(), expected.getUpdateKey());
|
207
|
+
assertRecord(domain, index, actual.getRecord(), expected.getRecord());
|
208
|
+
assertThat(reason, actual.getRevision(), is(expected.getRevision()));
|
209
|
+
}
|
210
|
+
|
211
|
+
private static void assertUpdateKey(
|
212
|
+
String domain, int index, UpdateKey actual, UpdateKey expected) {
|
213
|
+
String reason = String.format("%s:%d", domain, index);
|
214
|
+
assertThat(reason, actual.getField(), is(expected.getField()));
|
215
|
+
assertThat(reason, actual.getValue(), is(expected.getValue()));
|
216
|
+
}
|
217
|
+
|
218
|
+
public interface Runnable {
|
219
|
+
void run() throws Exception;
|
220
|
+
}
|
221
|
+
}
|
@@ -0,0 +1,92 @@
|
|
1
|
+
package org.embulk.output.kintone;
|
2
|
+
|
3
|
+
import java.util.List;
|
4
|
+
import java.util.function.Function;
|
5
|
+
import java.util.stream.Collectors;
|
6
|
+
import org.embulk.deps.buffer.PooledBufferAllocator;
|
7
|
+
import org.embulk.spi.Column;
|
8
|
+
import org.embulk.spi.Page;
|
9
|
+
import org.embulk.spi.PageBuilder;
|
10
|
+
import org.embulk.spi.PageOutput;
|
11
|
+
import org.embulk.spi.Schema;
|
12
|
+
import org.embulk.spi.time.Timestamp;
|
13
|
+
import org.msgpack.value.Value;
|
14
|
+
|
15
|
+
public class OutputPageBuilder implements PageOutput {
|
16
|
+
private final List<String> names;
|
17
|
+
private final PageBuilder builder;
|
18
|
+
private Page page;
|
19
|
+
|
20
|
+
public static Page build(Schema schema, Function<OutputPageBuilder, Page> function) {
|
21
|
+
Page page;
|
22
|
+
try (OutputPageBuilder builder = new OutputPageBuilder(schema)) {
|
23
|
+
page = function.apply(builder);
|
24
|
+
}
|
25
|
+
return page;
|
26
|
+
}
|
27
|
+
|
28
|
+
public OutputPageBuilder(Schema schema) {
|
29
|
+
names = schema.getColumns().stream().map(Column::getName).collect(Collectors.toList());
|
30
|
+
builder = new PageBuilder(PooledBufferAllocator.create(), schema, this);
|
31
|
+
}
|
32
|
+
|
33
|
+
public OutputPageBuilder setNull(String name) {
|
34
|
+
builder.setNull(names.indexOf(name));
|
35
|
+
return this;
|
36
|
+
}
|
37
|
+
|
38
|
+
public OutputPageBuilder setBoolean(String name, boolean value) {
|
39
|
+
builder.setBoolean(names.indexOf(name), value);
|
40
|
+
return this;
|
41
|
+
}
|
42
|
+
|
43
|
+
public OutputPageBuilder setLong(String name, long value) {
|
44
|
+
builder.setLong(names.indexOf(name), value);
|
45
|
+
return this;
|
46
|
+
}
|
47
|
+
|
48
|
+
public OutputPageBuilder setDouble(String name, double value) {
|
49
|
+
builder.setDouble(names.indexOf(name), value);
|
50
|
+
return this;
|
51
|
+
}
|
52
|
+
|
53
|
+
public OutputPageBuilder setString(String name, String value) {
|
54
|
+
builder.setString(names.indexOf(name), value);
|
55
|
+
return this;
|
56
|
+
}
|
57
|
+
|
58
|
+
public OutputPageBuilder setJson(String name, Value value) {
|
59
|
+
builder.setJson(names.indexOf(name), value);
|
60
|
+
return this;
|
61
|
+
}
|
62
|
+
|
63
|
+
public OutputPageBuilder setTimestamp(String name, Timestamp value) {
|
64
|
+
builder.setTimestamp(names.indexOf(name), value);
|
65
|
+
return this;
|
66
|
+
}
|
67
|
+
|
68
|
+
public OutputPageBuilder addRecord() {
|
69
|
+
builder.addRecord();
|
70
|
+
return this;
|
71
|
+
}
|
72
|
+
|
73
|
+
public Page build() {
|
74
|
+
builder.flush();
|
75
|
+
builder.close();
|
76
|
+
return page;
|
77
|
+
}
|
78
|
+
|
79
|
+
@Override
|
80
|
+
public void add(Page page) {
|
81
|
+
if (this.page != null) {
|
82
|
+
throw new IllegalStateException();
|
83
|
+
}
|
84
|
+
this.page = page;
|
85
|
+
}
|
86
|
+
|
87
|
+
@Override
|
88
|
+
public void finish() {}
|
89
|
+
|
90
|
+
@Override
|
91
|
+
public void close() {}
|
92
|
+
}
|
@@ -1,3 +1,178 @@
|
|
1
1
|
package org.embulk.output.kintone;
|
2
2
|
|
3
|
-
|
3
|
+
import com.google.common.io.Resources;
|
4
|
+
import com.kintone.client.Json;
|
5
|
+
import com.kintone.client.model.record.Record;
|
6
|
+
import com.kintone.client.model.record.RecordForUpdate;
|
7
|
+
import com.kintone.client.model.record.UpdateKey;
|
8
|
+
import java.io.File;
|
9
|
+
import java.net.URISyntaxException;
|
10
|
+
import java.net.URL;
|
11
|
+
import java.nio.file.Path;
|
12
|
+
import java.nio.file.Paths;
|
13
|
+
import java.util.Arrays;
|
14
|
+
import java.util.Collections;
|
15
|
+
import java.util.List;
|
16
|
+
import java.util.Objects;
|
17
|
+
import java.util.function.Function;
|
18
|
+
import java.util.stream.Collectors;
|
19
|
+
import org.embulk.config.ConfigSource;
|
20
|
+
import org.embulk.config.TaskSource;
|
21
|
+
import org.embulk.spi.OutputPlugin;
|
22
|
+
import org.embulk.spi.Schema;
|
23
|
+
import org.embulk.spi.TransactionalPageOutput;
|
24
|
+
import org.embulk.spi.json.JsonParser;
|
25
|
+
import org.embulk.test.EmbulkTests;
|
26
|
+
import org.embulk.test.TestingEmbulk;
|
27
|
+
import org.junit.Rule;
|
28
|
+
import org.msgpack.value.Value;
|
29
|
+
|
30
|
+
public class TestKintoneOutputPlugin extends KintoneOutputPlugin {
|
31
|
+
private static final JsonParser PARSER = new JsonParser();
|
32
|
+
|
33
|
+
@Rule
|
34
|
+
public final TestingEmbulk embulk =
|
35
|
+
TestingEmbulk.builder()
|
36
|
+
.registerPlugin(OutputPlugin.class, "kintone", TestKintoneOutputPlugin.class)
|
37
|
+
.build();
|
38
|
+
|
39
|
+
@Override
|
40
|
+
public TransactionalPageOutput open(TaskSource taskSource, Schema schema, int taskIndex) {
|
41
|
+
String test = taskSource.get(String.class, "Domain");
|
42
|
+
String mode = taskSource.get(String.class, "Mode");
|
43
|
+
String field = taskSource.get(String.class, "UpdateKeyName");
|
44
|
+
boolean preferNulls = taskSource.get(boolean.class, "PreferNulls");
|
45
|
+
boolean ignoreNulls = taskSource.get(boolean.class, "IgnoreNulls");
|
46
|
+
return new KintonePageOutputVerifier(
|
47
|
+
super.open(taskSource, schema, taskIndex),
|
48
|
+
test,
|
49
|
+
field,
|
50
|
+
getValues(test, preferNulls, ignoreNulls),
|
51
|
+
getAddRecords(test, mode, preferNulls, ignoreNulls),
|
52
|
+
getUpdateRecords(test, mode, preferNulls, ignoreNulls, field));
|
53
|
+
}
|
54
|
+
|
55
|
+
protected void runOutput(String configName, String inputName) throws Exception {
|
56
|
+
System.gc();
|
57
|
+
ConfigSource outConfig = loadConfigYaml(configName);
|
58
|
+
Path inputPath = getResourceFile(inputName).toPath();
|
59
|
+
embulk.runOutput(outConfig, inputPath);
|
60
|
+
}
|
61
|
+
|
62
|
+
protected String getConfigName() {
|
63
|
+
return getName("config.yml");
|
64
|
+
}
|
65
|
+
|
66
|
+
protected String getInputName() {
|
67
|
+
return getName("input.csv");
|
68
|
+
}
|
69
|
+
|
70
|
+
protected String getName(String name) {
|
71
|
+
return name;
|
72
|
+
}
|
73
|
+
|
74
|
+
protected ConfigSource config(String... strings) {
|
75
|
+
return Arrays.stream(strings)
|
76
|
+
.map(this::fromYamlString)
|
77
|
+
.reduce(ConfigSource::merge)
|
78
|
+
.orElseGet(() -> fromYamlString("{}"));
|
79
|
+
}
|
80
|
+
|
81
|
+
protected ConfigSource loadConfigYaml(String name) {
|
82
|
+
ConfigSource config = loadYamlResource("config.yml");
|
83
|
+
return config.merge(loadYamlResource(name));
|
84
|
+
}
|
85
|
+
|
86
|
+
protected ConfigSource loadYamlResource(String name) {
|
87
|
+
return embulk.loadYamlResource(getResourceName(name));
|
88
|
+
}
|
89
|
+
|
90
|
+
protected ConfigSource fromYamlString(String string) {
|
91
|
+
return embulk.configLoader().fromYamlString(string);
|
92
|
+
}
|
93
|
+
|
94
|
+
private static List<String> getValues(String test, boolean preferNulls, boolean ignoreNulls) {
|
95
|
+
String name =
|
96
|
+
String.format(
|
97
|
+
"%s/values%s.json",
|
98
|
+
test, ignoreNulls ? "_ignore_nulls" : preferNulls ? "_prefer_nulls" : "");
|
99
|
+
String json = existsResource(name) ? readResource(name) : null;
|
100
|
+
return json == null || json.isEmpty()
|
101
|
+
? Collections.emptyList()
|
102
|
+
: PARSER.parse(json).asArrayValue().list().stream()
|
103
|
+
.map(Value::toJson)
|
104
|
+
.collect(Collectors.toList());
|
105
|
+
}
|
106
|
+
|
107
|
+
private static List<Record> getAddRecords(
|
108
|
+
String test, String mode, boolean preferNulls, boolean ignoreNulls) {
|
109
|
+
String name =
|
110
|
+
String.format(
|
111
|
+
"%s/%s_add%s_records.jsonl",
|
112
|
+
test, mode, ignoreNulls ? "_ignore_nulls" : preferNulls ? "_prefer_nulls" : "");
|
113
|
+
String jsonl = existsResource(name) ? readResource(name) : null;
|
114
|
+
return jsonl == null || jsonl.isEmpty()
|
115
|
+
? Collections.emptyList()
|
116
|
+
: Arrays.stream(jsonl.split("\\r?\\n|\\r"))
|
117
|
+
.map(s -> Json.parse(s, Record.class))
|
118
|
+
.collect(Collectors.toList());
|
119
|
+
}
|
120
|
+
|
121
|
+
private static List<RecordForUpdate> getUpdateRecords(
|
122
|
+
String test, String mode, boolean preferNulls, boolean ignoreNulls, String field) {
|
123
|
+
Function<Record, UpdateKey> key = getKey(field);
|
124
|
+
String name =
|
125
|
+
String.format(
|
126
|
+
"%s/%s_update%s_records.jsonl",
|
127
|
+
test, mode, ignoreNulls ? "_ignore_nulls" : preferNulls ? "_prefer_nulls" : "");
|
128
|
+
String jsonl = existsResource(name) ? readResource(name) : null;
|
129
|
+
return jsonl == null || jsonl.isEmpty()
|
130
|
+
? Collections.emptyList()
|
131
|
+
: Arrays.stream(jsonl.split("\\r?\\n|\\r"))
|
132
|
+
.map(s -> Json.parse(s, Record.class))
|
133
|
+
.map(record -> new RecordForUpdate(key.apply(record), record.removeField(field)))
|
134
|
+
.collect(Collectors.toList());
|
135
|
+
}
|
136
|
+
|
137
|
+
private static Function<Record, UpdateKey> getKey(String field) {
|
138
|
+
return field == null
|
139
|
+
? record -> null
|
140
|
+
: record ->
|
141
|
+
field.matches("^.*_number$")
|
142
|
+
? new UpdateKey(field, record.getNumberFieldValue(field))
|
143
|
+
: new UpdateKey(field, record.getSingleLineTextFieldValue(field));
|
144
|
+
}
|
145
|
+
|
146
|
+
private static File getResourceFile(String name) {
|
147
|
+
return toPath(Objects.requireNonNull(getResource(getResourceName(name)))).toFile();
|
148
|
+
}
|
149
|
+
|
150
|
+
private static String readResource(String name) {
|
151
|
+
return EmbulkTests.readResource(getResourceName(name));
|
152
|
+
}
|
153
|
+
|
154
|
+
private static boolean existsResource(String name) {
|
155
|
+
return getResource(getResourceName(name)) != null;
|
156
|
+
}
|
157
|
+
|
158
|
+
private static String getResourceName(String name) {
|
159
|
+
return String.format("org/embulk/output/kintone/%s", name);
|
160
|
+
}
|
161
|
+
|
162
|
+
private static Path toPath(URL url) {
|
163
|
+
try {
|
164
|
+
return Paths.get(url.toURI());
|
165
|
+
} catch (URISyntaxException e) {
|
166
|
+
throw new RuntimeException(e);
|
167
|
+
}
|
168
|
+
}
|
169
|
+
|
170
|
+
@SuppressWarnings("UnstableApiUsage")
|
171
|
+
private static URL getResource(String resourceName) {
|
172
|
+
try {
|
173
|
+
return Resources.getResource(resourceName);
|
174
|
+
} catch (IllegalArgumentException e) {
|
175
|
+
return null;
|
176
|
+
}
|
177
|
+
}
|
178
|
+
}
|
@@ -0,0 +1,40 @@
|
|
1
|
+
package org.embulk.output.kintone;
|
2
|
+
|
3
|
+
import org.embulk.config.ConfigSource;
|
4
|
+
import org.junit.Before;
|
5
|
+
|
6
|
+
public class TestTask extends TestKintoneOutputPlugin {
|
7
|
+
private ConfigSource config;
|
8
|
+
|
9
|
+
@Before
|
10
|
+
public void before() {
|
11
|
+
config = fromYamlString("{}");
|
12
|
+
}
|
13
|
+
|
14
|
+
@Override
|
15
|
+
protected ConfigSource loadConfigYaml(String name) {
|
16
|
+
ConfigSource config = super.loadConfigYaml("task/config.yml");
|
17
|
+
return config.merge(loadYamlResource(name)).merge(this.config);
|
18
|
+
}
|
19
|
+
|
20
|
+
protected void merge(ConfigSource config) {
|
21
|
+
this.config.merge(config);
|
22
|
+
}
|
23
|
+
|
24
|
+
protected void runOutput() throws Exception {
|
25
|
+
String test = config.get(String.class, "domain");
|
26
|
+
runOutput(getConfigName(test), getInputName(test));
|
27
|
+
}
|
28
|
+
|
29
|
+
private String getConfigName(String test) {
|
30
|
+
return getName(test, getConfigName());
|
31
|
+
}
|
32
|
+
|
33
|
+
private String getInputName(String test) {
|
34
|
+
return getName(test, getInputName());
|
35
|
+
}
|
36
|
+
|
37
|
+
private static String getName(String test, String name) {
|
38
|
+
return String.format("%s/%s", test, name);
|
39
|
+
}
|
40
|
+
}
|
@@ -0,0 +1,43 @@
|
|
1
|
+
package org.embulk.output.kintone;
|
2
|
+
|
3
|
+
import net.jcip.annotations.NotThreadSafe;
|
4
|
+
import org.junit.Test;
|
5
|
+
|
6
|
+
@NotThreadSafe
|
7
|
+
public class TestTaskMode extends TestTask {
|
8
|
+
@Override
|
9
|
+
public void before() {
|
10
|
+
super.before();
|
11
|
+
merge(config("domain: task/mode"));
|
12
|
+
}
|
13
|
+
|
14
|
+
@Test
|
15
|
+
public void testInsert() throws Exception {
|
16
|
+
merge(config("mode: insert"));
|
17
|
+
runOutput();
|
18
|
+
merge(config("prefer_nulls: true"));
|
19
|
+
runOutput();
|
20
|
+
merge(config("ignore_nulls: true"));
|
21
|
+
runOutput();
|
22
|
+
}
|
23
|
+
|
24
|
+
@Test
|
25
|
+
public void testUpdate() throws Exception {
|
26
|
+
merge(config("mode: update", "update_key: string_number"));
|
27
|
+
runOutput();
|
28
|
+
merge(config("prefer_nulls: true"));
|
29
|
+
runOutput();
|
30
|
+
merge(config("ignore_nulls: true"));
|
31
|
+
runOutput();
|
32
|
+
}
|
33
|
+
|
34
|
+
@Test
|
35
|
+
public void testUpsert() throws Exception {
|
36
|
+
merge(config("mode: upsert", "update_key: double_single_line_text"));
|
37
|
+
runOutput();
|
38
|
+
merge(config("prefer_nulls: true"));
|
39
|
+
runOutput();
|
40
|
+
merge(config("ignore_nulls: true"));
|
41
|
+
runOutput();
|
42
|
+
}
|
43
|
+
}
|
@@ -0,0 +1,14 @@
|
|
1
|
+
<?xml version="1.0" encoding="UTF-8" ?>
|
2
|
+
<!DOCTYPE configuration>
|
3
|
+
<configuration>
|
4
|
+
<import class="ch.qos.logback.classic.encoder.PatternLayoutEncoder"/>
|
5
|
+
<import class="ch.qos.logback.core.ConsoleAppender"/>
|
6
|
+
<appender name="STDOUT" class="ConsoleAppender">
|
7
|
+
<encoder class="PatternLayoutEncoder">
|
8
|
+
<pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} -%kvp- %msg%n</pattern>
|
9
|
+
</encoder>
|
10
|
+
</appender>
|
11
|
+
<root level="debug">
|
12
|
+
<appender-ref ref="STDOUT"/>
|
13
|
+
</root>
|
14
|
+
</configuration>
|
@@ -0,0 +1 @@
|
|
1
|
+
{}
|