embulk-output-kintone 0.4.1 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +8 -6
- data/classpath/embulk-output-kintone-1.0.0.jar +0 -0
- data/classpath/{shadow-kintone-java-client-0.4.1-all.jar → shadow-kintone-java-client-1.0.0-all.jar} +0 -0
- data/src/main/java/org/embulk/output/kintone/KintoneColumnOption.java +1 -2
- data/src/main/java/org/embulk/output/kintone/KintoneColumnType.java +368 -0
- data/src/main/java/org/embulk/output/kintone/KintoneColumnVisitor.java +195 -135
- data/src/main/java/org/embulk/output/kintone/KintoneMode.java +0 -1
- data/src/main/java/org/embulk/output/kintone/KintoneOutputPlugin.java +0 -2
- data/src/main/java/org/embulk/output/kintone/KintonePageOutput.java +169 -157
- data/src/main/java/org/embulk/output/kintone/PluginTask.java +8 -0
- data/src/test/java/org/embulk/output/kintone/KintoneColumnOptionBuilder.java +2 -3
- data/src/test/java/org/embulk/output/kintone/KintoneColumnVisitorTest.java +563 -40
- data/src/test/java/org/embulk/output/kintone/KintoneColumnVisitorVerifier.java +35 -14
- data/src/test/java/org/embulk/output/kintone/KintonePageOutputVerifier.java +2 -7
- data/src/test/java/org/embulk/output/kintone/TestKintoneOutputPlugin.java +22 -9
- data/src/test/java/org/embulk/output/kintone/TestTaskMode.java +12 -0
- data/src/test/resources/org/embulk/output/kintone/task/mode/config.yml +104 -0
- data/src/test/resources/org/embulk/output/kintone/task/mode/input.csv +7 -7
- data/src/test/resources/org/embulk/output/kintone/task/mode/insert_add_ignore_nulls_records.jsonl +6 -0
- data/src/test/resources/org/embulk/output/kintone/task/mode/insert_add_prefer_nulls_records.jsonl +6 -0
- data/src/test/resources/org/embulk/output/kintone/task/mode/insert_add_records.jsonl +6 -6
- data/src/test/resources/org/embulk/output/kintone/task/mode/update_update_ignore_nulls_records.jsonl +3 -0
- data/src/test/resources/org/embulk/output/kintone/task/mode/update_update_prefer_nulls_records.jsonl +3 -0
- data/src/test/resources/org/embulk/output/kintone/task/mode/update_update_records.jsonl +6 -3
- data/src/test/resources/org/embulk/output/kintone/task/mode/upsert_add_ignore_nulls_records.jsonl +3 -0
- data/src/test/resources/org/embulk/output/kintone/task/mode/upsert_add_prefer_nulls_records.jsonl +3 -0
- data/src/test/resources/org/embulk/output/kintone/task/mode/upsert_add_records.jsonl +2 -2
- data/src/test/resources/org/embulk/output/kintone/task/mode/upsert_update_ignore_nulls_records.jsonl +3 -0
- data/src/test/resources/org/embulk/output/kintone/task/mode/upsert_update_prefer_nulls_records.jsonl +3 -0
- data/src/test/resources/org/embulk/output/kintone/task/mode/upsert_update_records.jsonl +4 -4
- data/src/test/resources/org/embulk/output/kintone/task/mode/values_ignore_nulls.json +1 -0
- data/src/test/resources/org/embulk/output/kintone/task/mode/values_prefer_nulls.json +1 -0
- metadata +15 -4
- data/classpath/embulk-output-kintone-0.4.1.jar +0 -0
@@ -1,47 +1,44 @@
|
|
1
1
|
package org.embulk.output.kintone;
|
2
2
|
|
3
|
-
import com.kintone.client.model.record.CheckBoxFieldValue;
|
4
|
-
import com.kintone.client.model.record.DateFieldValue;
|
5
|
-
import com.kintone.client.model.record.DateTimeFieldValue;
|
6
|
-
import com.kintone.client.model.record.DropDownFieldValue;
|
7
|
-
import com.kintone.client.model.record.FieldType;
|
8
3
|
import com.kintone.client.model.record.FieldValue;
|
9
|
-
import com.kintone.client.model.record.LinkFieldValue;
|
10
|
-
import com.kintone.client.model.record.MultiLineTextFieldValue;
|
11
|
-
import com.kintone.client.model.record.NumberFieldValue;
|
12
4
|
import com.kintone.client.model.record.Record;
|
13
|
-
import com.kintone.client.model.record.SingleLineTextFieldValue;
|
14
5
|
import com.kintone.client.model.record.UpdateKey;
|
15
|
-
import java.math.BigDecimal;
|
16
6
|
import java.time.Instant;
|
17
|
-
import java.time.ZoneId;
|
18
|
-
import java.time.ZonedDateTime;
|
19
|
-
import java.util.Arrays;
|
20
|
-
import java.util.List;
|
21
7
|
import java.util.Map;
|
22
|
-
import java.util.Objects;
|
23
8
|
import org.embulk.spi.Column;
|
24
9
|
import org.embulk.spi.ColumnVisitor;
|
25
10
|
import org.embulk.spi.PageReader;
|
26
11
|
import org.embulk.spi.time.Timestamp;
|
12
|
+
import org.msgpack.value.Value;
|
13
|
+
import org.msgpack.value.ValueFactory;
|
27
14
|
|
28
15
|
public class KintoneColumnVisitor implements ColumnVisitor {
|
29
|
-
private final PageReader
|
16
|
+
private final PageReader reader;
|
17
|
+
private final Map<String, KintoneColumnOption> options;
|
18
|
+
private final boolean preferNulls;
|
19
|
+
private final boolean ignoreNulls;
|
20
|
+
private final String updateKeyName;
|
30
21
|
private Record record;
|
31
22
|
private UpdateKey updateKey;
|
32
|
-
private final Map<String, KintoneColumnOption> columnOptions;
|
33
|
-
private String updateKeyName;
|
34
23
|
|
35
24
|
public KintoneColumnVisitor(
|
36
|
-
PageReader
|
37
|
-
|
38
|
-
|
25
|
+
PageReader reader,
|
26
|
+
Map<String, KintoneColumnOption> options,
|
27
|
+
boolean preferNulls,
|
28
|
+
boolean ignoreNulls) {
|
29
|
+
this(reader, options, preferNulls, ignoreNulls, null);
|
39
30
|
}
|
40
31
|
|
41
32
|
public KintoneColumnVisitor(
|
42
|
-
PageReader
|
43
|
-
|
44
|
-
|
33
|
+
PageReader reader,
|
34
|
+
Map<String, KintoneColumnOption> options,
|
35
|
+
boolean preferNulls,
|
36
|
+
boolean ignoreNulls,
|
37
|
+
String updateKeyName) {
|
38
|
+
this.reader = reader;
|
39
|
+
this.options = options;
|
40
|
+
this.preferNulls = preferNulls;
|
41
|
+
this.ignoreNulls = ignoreNulls;
|
45
42
|
this.updateKeyName = updateKeyName;
|
46
43
|
}
|
47
44
|
|
@@ -53,156 +50,219 @@ public class KintoneColumnVisitor implements ColumnVisitor {
|
|
53
50
|
this.updateKey = updateKey;
|
54
51
|
}
|
55
52
|
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
fieldValue = new DropDownFieldValue(stringValue);
|
72
|
-
break;
|
73
|
-
case LINK:
|
74
|
-
fieldValue = new LinkFieldValue(stringValue);
|
75
|
-
break;
|
76
|
-
default:
|
77
|
-
fieldValue = new SingleLineTextFieldValue(stringValue);
|
53
|
+
@Override
|
54
|
+
public void booleanColumn(Column column) {
|
55
|
+
if (isIgnoreNull(column)) {
|
56
|
+
return;
|
57
|
+
}
|
58
|
+
KintoneColumnOption option = getOption(column);
|
59
|
+
UpdateKey updateKey = getUpdateKey(column);
|
60
|
+
KintoneColumnType type = KintoneColumnType.getType(option, KintoneColumnType.NUMBER);
|
61
|
+
String fieldCode = getFieldCode(column);
|
62
|
+
if (isPreferNull(column)) {
|
63
|
+
setNull(type, fieldCode, updateKey);
|
64
|
+
} else if (reader.isNull(column)) {
|
65
|
+
setBoolean(type, fieldCode, updateKey, false, option);
|
66
|
+
} else {
|
67
|
+
setBoolean(type, fieldCode, updateKey, reader.getBoolean(column), option);
|
78
68
|
}
|
79
|
-
record.putField(fieldCode, fieldValue);
|
80
69
|
}
|
81
70
|
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
71
|
+
@Override
|
72
|
+
public void longColumn(Column column) {
|
73
|
+
if (isIgnoreNull(column)) {
|
74
|
+
return;
|
75
|
+
}
|
76
|
+
KintoneColumnOption option = getOption(column);
|
77
|
+
UpdateKey updateKey = getUpdateKey(column);
|
78
|
+
KintoneColumnType type = KintoneColumnType.getType(option, KintoneColumnType.NUMBER);
|
79
|
+
String fieldCode = getFieldCode(column);
|
80
|
+
if (isPreferNull(column)) {
|
81
|
+
setNull(type, fieldCode, updateKey);
|
82
|
+
} else if (reader.isNull(column)) {
|
83
|
+
setLong(type, fieldCode, updateKey, 0, option);
|
84
|
+
} else {
|
85
|
+
setLong(type, fieldCode, updateKey, reader.getLong(column), option);
|
91
86
|
}
|
92
|
-
record.putField(fieldCode, fieldValue);
|
93
87
|
}
|
94
88
|
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
89
|
+
@Override
|
90
|
+
public void doubleColumn(Column column) {
|
91
|
+
if (isIgnoreNull(column)) {
|
92
|
+
return;
|
93
|
+
}
|
94
|
+
KintoneColumnOption option = getOption(column);
|
95
|
+
UpdateKey updateKey = getUpdateKey(column);
|
96
|
+
KintoneColumnType type = KintoneColumnType.getType(option, KintoneColumnType.NUMBER);
|
97
|
+
String fieldCode = getFieldCode(column);
|
98
|
+
if (isPreferNull(column)) {
|
99
|
+
setNull(type, fieldCode, updateKey);
|
100
|
+
} else if (reader.isNull(column)) {
|
101
|
+
setDouble(type, fieldCode, updateKey, 0, option);
|
102
|
+
} else {
|
103
|
+
setDouble(type, fieldCode, updateKey, reader.getDouble(column), option);
|
102
104
|
}
|
103
|
-
record.putField(fieldCode, checkBoxFieldValue);
|
104
105
|
}
|
105
106
|
|
106
|
-
|
107
|
-
|
108
|
-
if (
|
109
|
-
return
|
107
|
+
@Override
|
108
|
+
public void stringColumn(Column column) {
|
109
|
+
if (isIgnoreNull(column)) {
|
110
|
+
return;
|
111
|
+
}
|
112
|
+
KintoneColumnOption option = getOption(column);
|
113
|
+
UpdateKey updateKey = getUpdateKey(column);
|
114
|
+
KintoneColumnType defaultType =
|
115
|
+
updateKey != null ? KintoneColumnType.SINGLE_LINE_TEXT : KintoneColumnType.MULTI_LINE_TEXT;
|
116
|
+
KintoneColumnType type = KintoneColumnType.getType(option, defaultType);
|
117
|
+
String fieldCode = getFieldCode(column);
|
118
|
+
if (isPreferNull(column)) {
|
119
|
+
setNull(type, fieldCode, updateKey);
|
120
|
+
} else if (reader.isNull(column)) {
|
121
|
+
setString(type, fieldCode, updateKey, "", option);
|
110
122
|
} else {
|
111
|
-
|
123
|
+
setString(type, fieldCode, updateKey, reader.getString(column), option);
|
112
124
|
}
|
113
125
|
}
|
114
126
|
|
115
|
-
|
116
|
-
|
117
|
-
if (
|
118
|
-
return
|
127
|
+
@Override
|
128
|
+
public void timestampColumn(Column column) {
|
129
|
+
if (isIgnoreNull(column)) {
|
130
|
+
return;
|
131
|
+
}
|
132
|
+
KintoneColumnOption option = getOption(column);
|
133
|
+
UpdateKey updateKey = getUpdateKey(column);
|
134
|
+
KintoneColumnType type = KintoneColumnType.getType(option, KintoneColumnType.DATETIME);
|
135
|
+
String fieldCode = getFieldCode(column);
|
136
|
+
if (isPreferNull(column)) {
|
137
|
+
setNull(type, fieldCode, updateKey);
|
138
|
+
} else if (reader.isNull(column)) {
|
139
|
+
setTimestamp(type, fieldCode, updateKey, Timestamp.ofInstant(Instant.EPOCH), option);
|
119
140
|
} else {
|
120
|
-
|
141
|
+
setTimestamp(type, fieldCode, updateKey, reader.getTimestamp(column), option);
|
121
142
|
}
|
122
143
|
}
|
123
144
|
|
124
|
-
|
125
|
-
|
126
|
-
if (
|
127
|
-
return
|
145
|
+
@Override
|
146
|
+
public void jsonColumn(Column column) {
|
147
|
+
if (isIgnoreNull(column)) {
|
148
|
+
return;
|
149
|
+
}
|
150
|
+
KintoneColumnOption option = getOption(column);
|
151
|
+
UpdateKey updateKey = getUpdateKey(column);
|
152
|
+
KintoneColumnType type = KintoneColumnType.getType(option, KintoneColumnType.MULTI_LINE_TEXT);
|
153
|
+
String fieldCode = getFieldCode(column);
|
154
|
+
if (isPreferNull(column)) {
|
155
|
+
setNull(type, fieldCode, updateKey);
|
156
|
+
} else if (reader.isNull(column)) {
|
157
|
+
setJson(type, fieldCode, updateKey, ValueFactory.newString(""), option);
|
158
|
+
} else {
|
159
|
+
setJson(type, fieldCode, updateKey, reader.getJson(column), option);
|
128
160
|
}
|
129
|
-
return ZoneId.of(option.getTimezone().orElse("UTC"));
|
130
161
|
}
|
131
162
|
|
132
|
-
private
|
133
|
-
if (
|
134
|
-
|
163
|
+
private void setNull(KintoneColumnType type, String fieldCode, UpdateKey updateKey) {
|
164
|
+
if (updateKey != null) {
|
165
|
+
type.setUpdateKey(updateKey, fieldCode);
|
135
166
|
}
|
136
|
-
|
137
|
-
return this.updateKeyName.equals(column.getName());
|
167
|
+
record.putField(fieldCode, type.getFieldValue());
|
138
168
|
}
|
139
169
|
|
140
|
-
private
|
141
|
-
|
142
|
-
|
143
|
-
|
170
|
+
private void setBoolean(
|
171
|
+
KintoneColumnType type,
|
172
|
+
String fieldCode,
|
173
|
+
UpdateKey updateKey,
|
174
|
+
boolean value,
|
175
|
+
KintoneColumnOption option) {
|
176
|
+
FieldValue fieldValue = type.getFieldValue(value, option);
|
177
|
+
if (updateKey != null) {
|
178
|
+
type.setUpdateKey(updateKey, fieldCode, fieldValue);
|
144
179
|
}
|
145
|
-
|
180
|
+
record.putField(fieldCode, fieldValue);
|
146
181
|
}
|
147
182
|
|
148
|
-
|
149
|
-
|
150
|
-
|
151
|
-
|
152
|
-
|
183
|
+
private void setLong(
|
184
|
+
KintoneColumnType type,
|
185
|
+
String fieldCode,
|
186
|
+
UpdateKey updateKey,
|
187
|
+
long value,
|
188
|
+
KintoneColumnOption option) {
|
189
|
+
FieldValue fieldValue = type.getFieldValue(value, option);
|
190
|
+
if (updateKey != null) {
|
191
|
+
type.setUpdateKey(updateKey, fieldCode, fieldValue);
|
192
|
+
}
|
193
|
+
record.putField(fieldCode, fieldValue);
|
153
194
|
}
|
154
195
|
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
|
196
|
+
private void setDouble(
|
197
|
+
KintoneColumnType type,
|
198
|
+
String fieldCode,
|
199
|
+
UpdateKey updateKey,
|
200
|
+
double value,
|
201
|
+
KintoneColumnOption option) {
|
202
|
+
FieldValue fieldValue = type.getFieldValue(value, option);
|
203
|
+
if (updateKey != null) {
|
204
|
+
type.setUpdateKey(updateKey, fieldCode, fieldValue);
|
163
205
|
}
|
206
|
+
record.putField(fieldCode, fieldValue);
|
164
207
|
}
|
165
208
|
|
166
|
-
|
167
|
-
|
168
|
-
|
169
|
-
|
170
|
-
|
209
|
+
private void setString(
|
210
|
+
KintoneColumnType type,
|
211
|
+
String fieldCode,
|
212
|
+
UpdateKey updateKey,
|
213
|
+
String value,
|
214
|
+
KintoneColumnOption option) {
|
215
|
+
FieldValue fieldValue = type.getFieldValue(value, option);
|
216
|
+
if (updateKey != null) {
|
217
|
+
type.setUpdateKey(updateKey, fieldCode, fieldValue);
|
218
|
+
}
|
219
|
+
record.putField(fieldCode, fieldValue);
|
171
220
|
}
|
172
221
|
|
173
|
-
|
174
|
-
|
175
|
-
|
176
|
-
|
177
|
-
|
178
|
-
|
179
|
-
|
180
|
-
|
181
|
-
|
222
|
+
private void setTimestamp(
|
223
|
+
KintoneColumnType type,
|
224
|
+
String fieldCode,
|
225
|
+
UpdateKey updateKey,
|
226
|
+
Timestamp value,
|
227
|
+
KintoneColumnOption option) {
|
228
|
+
FieldValue fieldValue = type.getFieldValue(value, option);
|
229
|
+
if (updateKey != null) {
|
230
|
+
type.setUpdateKey(updateKey, fieldCode, fieldValue);
|
182
231
|
}
|
183
|
-
|
232
|
+
record.putField(fieldCode, fieldValue);
|
184
233
|
}
|
185
234
|
|
186
|
-
|
187
|
-
|
188
|
-
|
189
|
-
|
190
|
-
|
235
|
+
private void setJson(
|
236
|
+
KintoneColumnType type,
|
237
|
+
String fieldCode,
|
238
|
+
UpdateKey updateKey,
|
239
|
+
Value value,
|
240
|
+
KintoneColumnOption option) {
|
241
|
+
FieldValue fieldValue = type.getFieldValue(value, option);
|
242
|
+
if (updateKey != null) {
|
243
|
+
type.setUpdateKey(updateKey, fieldCode, fieldValue);
|
191
244
|
}
|
245
|
+
record.putField(fieldCode, fieldValue);
|
246
|
+
}
|
192
247
|
|
193
|
-
|
194
|
-
|
195
|
-
|
196
|
-
if (type == FieldType.DATETIME) {
|
197
|
-
zoneId = ZoneId.of("UTC");
|
198
|
-
}
|
199
|
-
setTimestampValue(fieldCode, value.getInstant(), zoneId, type);
|
248
|
+
private String getFieldCode(Column column) {
|
249
|
+
KintoneColumnOption option = getOption(column);
|
250
|
+
return option != null ? option.getFieldCode() : column.getName();
|
200
251
|
}
|
201
252
|
|
202
|
-
|
203
|
-
|
204
|
-
|
205
|
-
|
206
|
-
|
253
|
+
private KintoneColumnOption getOption(Column column) {
|
254
|
+
return options.get(column.getName());
|
255
|
+
}
|
256
|
+
|
257
|
+
private UpdateKey getUpdateKey(Column column) {
|
258
|
+
return updateKeyName != null && updateKeyName.equals(column.getName()) ? updateKey : null;
|
259
|
+
}
|
260
|
+
|
261
|
+
private boolean isIgnoreNull(Column column) {
|
262
|
+
return ignoreNulls && reader.isNull(column);
|
263
|
+
}
|
264
|
+
|
265
|
+
private boolean isPreferNull(Column column) {
|
266
|
+
return preferNulls && reader.isNull(column);
|
207
267
|
}
|
208
268
|
}
|
@@ -16,7 +16,6 @@ public class KintoneOutputPlugin implements OutputPlugin {
|
|
16
16
|
public ConfigDiff transaction(
|
17
17
|
ConfigSource config, Schema schema, int taskCount, OutputPlugin.Control control) {
|
18
18
|
PluginTask task = config.loadConfig(PluginTask.class);
|
19
|
-
|
20
19
|
control.run(task.dump());
|
21
20
|
return Exec.newConfigDiff();
|
22
21
|
}
|
@@ -34,7 +33,6 @@ public class KintoneOutputPlugin implements OutputPlugin {
|
|
34
33
|
@Override
|
35
34
|
public TransactionalPageOutput open(TaskSource taskSource, Schema schema, int taskIndex) {
|
36
35
|
PluginTask task = taskSource.loadTask(PluginTask.class);
|
37
|
-
|
38
36
|
KintoneMode mode = KintoneMode.getKintoneModeByValue(task.getMode());
|
39
37
|
switch (mode) {
|
40
38
|
case INSERT:
|