embulk-output-kintone 0.4.1 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +8 -6
  3. data/classpath/embulk-output-kintone-1.0.0.jar +0 -0
  4. data/classpath/{shadow-kintone-java-client-0.4.1-all.jar → shadow-kintone-java-client-1.0.0-all.jar} +0 -0
  5. data/src/main/java/org/embulk/output/kintone/KintoneColumnOption.java +1 -2
  6. data/src/main/java/org/embulk/output/kintone/KintoneColumnType.java +368 -0
  7. data/src/main/java/org/embulk/output/kintone/KintoneColumnVisitor.java +195 -135
  8. data/src/main/java/org/embulk/output/kintone/KintoneMode.java +0 -1
  9. data/src/main/java/org/embulk/output/kintone/KintoneOutputPlugin.java +0 -2
  10. data/src/main/java/org/embulk/output/kintone/KintonePageOutput.java +169 -157
  11. data/src/main/java/org/embulk/output/kintone/PluginTask.java +8 -0
  12. data/src/test/java/org/embulk/output/kintone/KintoneColumnOptionBuilder.java +2 -3
  13. data/src/test/java/org/embulk/output/kintone/KintoneColumnVisitorTest.java +563 -40
  14. data/src/test/java/org/embulk/output/kintone/KintoneColumnVisitorVerifier.java +35 -14
  15. data/src/test/java/org/embulk/output/kintone/KintonePageOutputVerifier.java +2 -7
  16. data/src/test/java/org/embulk/output/kintone/TestKintoneOutputPlugin.java +22 -9
  17. data/src/test/java/org/embulk/output/kintone/TestTaskMode.java +12 -0
  18. data/src/test/resources/org/embulk/output/kintone/task/mode/config.yml +104 -0
  19. data/src/test/resources/org/embulk/output/kintone/task/mode/input.csv +7 -7
  20. data/src/test/resources/org/embulk/output/kintone/task/mode/insert_add_ignore_nulls_records.jsonl +6 -0
  21. data/src/test/resources/org/embulk/output/kintone/task/mode/insert_add_prefer_nulls_records.jsonl +6 -0
  22. data/src/test/resources/org/embulk/output/kintone/task/mode/insert_add_records.jsonl +6 -6
  23. data/src/test/resources/org/embulk/output/kintone/task/mode/update_update_ignore_nulls_records.jsonl +3 -0
  24. data/src/test/resources/org/embulk/output/kintone/task/mode/update_update_prefer_nulls_records.jsonl +3 -0
  25. data/src/test/resources/org/embulk/output/kintone/task/mode/update_update_records.jsonl +6 -3
  26. data/src/test/resources/org/embulk/output/kintone/task/mode/upsert_add_ignore_nulls_records.jsonl +3 -0
  27. data/src/test/resources/org/embulk/output/kintone/task/mode/upsert_add_prefer_nulls_records.jsonl +3 -0
  28. data/src/test/resources/org/embulk/output/kintone/task/mode/upsert_add_records.jsonl +2 -2
  29. data/src/test/resources/org/embulk/output/kintone/task/mode/upsert_update_ignore_nulls_records.jsonl +3 -0
  30. data/src/test/resources/org/embulk/output/kintone/task/mode/upsert_update_prefer_nulls_records.jsonl +3 -0
  31. data/src/test/resources/org/embulk/output/kintone/task/mode/upsert_update_records.jsonl +4 -4
  32. data/src/test/resources/org/embulk/output/kintone/task/mode/values_ignore_nulls.json +1 -0
  33. data/src/test/resources/org/embulk/output/kintone/task/mode/values_prefer_nulls.json +1 -0
  34. metadata +15 -4
  35. data/classpath/embulk-output-kintone-0.4.1.jar +0 -0
@@ -1,47 +1,44 @@
1
1
  package org.embulk.output.kintone;
2
2
 
3
- import com.kintone.client.model.record.CheckBoxFieldValue;
4
- import com.kintone.client.model.record.DateFieldValue;
5
- import com.kintone.client.model.record.DateTimeFieldValue;
6
- import com.kintone.client.model.record.DropDownFieldValue;
7
- import com.kintone.client.model.record.FieldType;
8
3
  import com.kintone.client.model.record.FieldValue;
9
- import com.kintone.client.model.record.LinkFieldValue;
10
- import com.kintone.client.model.record.MultiLineTextFieldValue;
11
- import com.kintone.client.model.record.NumberFieldValue;
12
4
  import com.kintone.client.model.record.Record;
13
- import com.kintone.client.model.record.SingleLineTextFieldValue;
14
5
  import com.kintone.client.model.record.UpdateKey;
15
- import java.math.BigDecimal;
16
6
  import java.time.Instant;
17
- import java.time.ZoneId;
18
- import java.time.ZonedDateTime;
19
- import java.util.Arrays;
20
- import java.util.List;
21
7
  import java.util.Map;
22
- import java.util.Objects;
23
8
  import org.embulk.spi.Column;
24
9
  import org.embulk.spi.ColumnVisitor;
25
10
  import org.embulk.spi.PageReader;
26
11
  import org.embulk.spi.time.Timestamp;
12
+ import org.msgpack.value.Value;
13
+ import org.msgpack.value.ValueFactory;
27
14
 
28
15
  public class KintoneColumnVisitor implements ColumnVisitor {
29
- private final PageReader pageReader;
16
+ private final PageReader reader;
17
+ private final Map<String, KintoneColumnOption> options;
18
+ private final boolean preferNulls;
19
+ private final boolean ignoreNulls;
20
+ private final String updateKeyName;
30
21
  private Record record;
31
22
  private UpdateKey updateKey;
32
- private final Map<String, KintoneColumnOption> columnOptions;
33
- private String updateKeyName;
34
23
 
35
24
  public KintoneColumnVisitor(
36
- PageReader pageReader, Map<String, KintoneColumnOption> columnOptions) {
37
- this.pageReader = pageReader;
38
- this.columnOptions = columnOptions;
25
+ PageReader reader,
26
+ Map<String, KintoneColumnOption> options,
27
+ boolean preferNulls,
28
+ boolean ignoreNulls) {
29
+ this(reader, options, preferNulls, ignoreNulls, null);
39
30
  }
40
31
 
41
32
  public KintoneColumnVisitor(
42
- PageReader pageReader, Map<String, KintoneColumnOption> columnOptions, String updateKeyName) {
43
- this.pageReader = pageReader;
44
- this.columnOptions = columnOptions;
33
+ PageReader reader,
34
+ Map<String, KintoneColumnOption> options,
35
+ boolean preferNulls,
36
+ boolean ignoreNulls,
37
+ String updateKeyName) {
38
+ this.reader = reader;
39
+ this.options = options;
40
+ this.preferNulls = preferNulls;
41
+ this.ignoreNulls = ignoreNulls;
45
42
  this.updateKeyName = updateKeyName;
46
43
  }
47
44
 
@@ -53,156 +50,219 @@ public class KintoneColumnVisitor implements ColumnVisitor {
53
50
  this.updateKey = updateKey;
54
51
  }
55
52
 
56
- private void setValue(String fieldCode, Object value, FieldType type, boolean isUpdateKey) {
57
- if (isUpdateKey && updateKey != null) {
58
- updateKey.setField(fieldCode).setValue(Objects.toString(value, ""));
59
- }
60
- String stringValue = Objects.toString(value, "");
61
- FieldValue fieldValue;
62
- switch (type) {
63
- case NUMBER:
64
- BigDecimal setValue = stringValue.equals("") ? null : new BigDecimal(stringValue);
65
- fieldValue = new NumberFieldValue(setValue);
66
- break;
67
- case MULTI_LINE_TEXT:
68
- fieldValue = new MultiLineTextFieldValue(stringValue);
69
- break;
70
- case DROP_DOWN:
71
- fieldValue = new DropDownFieldValue(stringValue);
72
- break;
73
- case LINK:
74
- fieldValue = new LinkFieldValue(stringValue);
75
- break;
76
- default:
77
- fieldValue = new SingleLineTextFieldValue(stringValue);
53
+ @Override
54
+ public void booleanColumn(Column column) {
55
+ if (isIgnoreNull(column)) {
56
+ return;
57
+ }
58
+ KintoneColumnOption option = getOption(column);
59
+ UpdateKey updateKey = getUpdateKey(column);
60
+ KintoneColumnType type = KintoneColumnType.getType(option, KintoneColumnType.NUMBER);
61
+ String fieldCode = getFieldCode(column);
62
+ if (isPreferNull(column)) {
63
+ setNull(type, fieldCode, updateKey);
64
+ } else if (reader.isNull(column)) {
65
+ setBoolean(type, fieldCode, updateKey, false, option);
66
+ } else {
67
+ setBoolean(type, fieldCode, updateKey, reader.getBoolean(column), option);
78
68
  }
79
- record.putField(fieldCode, fieldValue);
80
69
  }
81
70
 
82
- private void setTimestampValue(String fieldCode, Instant instant, ZoneId zoneId, FieldType type) {
83
- FieldValue fieldValue = null;
84
- ZonedDateTime datetime = instant.atZone(zoneId);
85
- switch (type) {
86
- case DATE:
87
- fieldValue = new DateFieldValue(datetime.toLocalDate());
88
- break;
89
- case DATETIME:
90
- fieldValue = new DateTimeFieldValue(datetime);
71
+ @Override
72
+ public void longColumn(Column column) {
73
+ if (isIgnoreNull(column)) {
74
+ return;
75
+ }
76
+ KintoneColumnOption option = getOption(column);
77
+ UpdateKey updateKey = getUpdateKey(column);
78
+ KintoneColumnType type = KintoneColumnType.getType(option, KintoneColumnType.NUMBER);
79
+ String fieldCode = getFieldCode(column);
80
+ if (isPreferNull(column)) {
81
+ setNull(type, fieldCode, updateKey);
82
+ } else if (reader.isNull(column)) {
83
+ setLong(type, fieldCode, updateKey, 0, option);
84
+ } else {
85
+ setLong(type, fieldCode, updateKey, reader.getLong(column), option);
91
86
  }
92
- record.putField(fieldCode, fieldValue);
93
87
  }
94
88
 
95
- private void setCheckBoxValue(String fieldCode, Object value, String valueSeparator) {
96
- String str = String.valueOf(value);
97
- CheckBoxFieldValue checkBoxFieldValue = new CheckBoxFieldValue();
98
-
99
- if (str != null && !str.equals("")) {
100
- List<String> values = Arrays.asList(str.split(valueSeparator, 0));
101
- checkBoxFieldValue = new CheckBoxFieldValue(values);
89
+ @Override
90
+ public void doubleColumn(Column column) {
91
+ if (isIgnoreNull(column)) {
92
+ return;
93
+ }
94
+ KintoneColumnOption option = getOption(column);
95
+ UpdateKey updateKey = getUpdateKey(column);
96
+ KintoneColumnType type = KintoneColumnType.getType(option, KintoneColumnType.NUMBER);
97
+ String fieldCode = getFieldCode(column);
98
+ if (isPreferNull(column)) {
99
+ setNull(type, fieldCode, updateKey);
100
+ } else if (reader.isNull(column)) {
101
+ setDouble(type, fieldCode, updateKey, 0, option);
102
+ } else {
103
+ setDouble(type, fieldCode, updateKey, reader.getDouble(column), option);
102
104
  }
103
- record.putField(fieldCode, checkBoxFieldValue);
104
105
  }
105
106
 
106
- private FieldType getType(Column column, FieldType defaultType) {
107
- KintoneColumnOption option = columnOptions.get(column.getName());
108
- if (option == null) {
109
- return defaultType;
107
+ @Override
108
+ public void stringColumn(Column column) {
109
+ if (isIgnoreNull(column)) {
110
+ return;
111
+ }
112
+ KintoneColumnOption option = getOption(column);
113
+ UpdateKey updateKey = getUpdateKey(column);
114
+ KintoneColumnType defaultType =
115
+ updateKey != null ? KintoneColumnType.SINGLE_LINE_TEXT : KintoneColumnType.MULTI_LINE_TEXT;
116
+ KintoneColumnType type = KintoneColumnType.getType(option, defaultType);
117
+ String fieldCode = getFieldCode(column);
118
+ if (isPreferNull(column)) {
119
+ setNull(type, fieldCode, updateKey);
120
+ } else if (reader.isNull(column)) {
121
+ setString(type, fieldCode, updateKey, "", option);
110
122
  } else {
111
- return FieldType.valueOf(option.getType());
123
+ setString(type, fieldCode, updateKey, reader.getString(column), option);
112
124
  }
113
125
  }
114
126
 
115
- private String getFieldCode(Column column) {
116
- KintoneColumnOption option = columnOptions.get(column.getName());
117
- if (option == null) {
118
- return column.getName();
127
+ @Override
128
+ public void timestampColumn(Column column) {
129
+ if (isIgnoreNull(column)) {
130
+ return;
131
+ }
132
+ KintoneColumnOption option = getOption(column);
133
+ UpdateKey updateKey = getUpdateKey(column);
134
+ KintoneColumnType type = KintoneColumnType.getType(option, KintoneColumnType.DATETIME);
135
+ String fieldCode = getFieldCode(column);
136
+ if (isPreferNull(column)) {
137
+ setNull(type, fieldCode, updateKey);
138
+ } else if (reader.isNull(column)) {
139
+ setTimestamp(type, fieldCode, updateKey, Timestamp.ofInstant(Instant.EPOCH), option);
119
140
  } else {
120
- return option.getFieldCode();
141
+ setTimestamp(type, fieldCode, updateKey, reader.getTimestamp(column), option);
121
142
  }
122
143
  }
123
144
 
124
- private ZoneId getZoneId(Column column) {
125
- KintoneColumnOption option = columnOptions.get(column.getName());
126
- if (option == null) {
127
- return ZoneId.of("UTC");
145
+ @Override
146
+ public void jsonColumn(Column column) {
147
+ if (isIgnoreNull(column)) {
148
+ return;
149
+ }
150
+ KintoneColumnOption option = getOption(column);
151
+ UpdateKey updateKey = getUpdateKey(column);
152
+ KintoneColumnType type = KintoneColumnType.getType(option, KintoneColumnType.MULTI_LINE_TEXT);
153
+ String fieldCode = getFieldCode(column);
154
+ if (isPreferNull(column)) {
155
+ setNull(type, fieldCode, updateKey);
156
+ } else if (reader.isNull(column)) {
157
+ setJson(type, fieldCode, updateKey, ValueFactory.newString(""), option);
158
+ } else {
159
+ setJson(type, fieldCode, updateKey, reader.getJson(column), option);
128
160
  }
129
- return ZoneId.of(option.getTimezone().orElse("UTC"));
130
161
  }
131
162
 
132
- private boolean isUpdateKey(Column column) {
133
- if (this.updateKeyName == null) {
134
- return false;
163
+ private void setNull(KintoneColumnType type, String fieldCode, UpdateKey updateKey) {
164
+ if (updateKey != null) {
165
+ type.setUpdateKey(updateKey, fieldCode);
135
166
  }
136
-
137
- return this.updateKeyName.equals(column.getName());
167
+ record.putField(fieldCode, type.getFieldValue());
138
168
  }
139
169
 
140
- private String getValueSeparator(Column column) {
141
- KintoneColumnOption option = columnOptions.get(column.getName());
142
- if (option == null) {
143
- return ",";
170
+ private void setBoolean(
171
+ KintoneColumnType type,
172
+ String fieldCode,
173
+ UpdateKey updateKey,
174
+ boolean value,
175
+ KintoneColumnOption option) {
176
+ FieldValue fieldValue = type.getFieldValue(value, option);
177
+ if (updateKey != null) {
178
+ type.setUpdateKey(updateKey, fieldCode, fieldValue);
144
179
  }
145
- return option.getValueSeparator();
180
+ record.putField(fieldCode, fieldValue);
146
181
  }
147
182
 
148
- @Override
149
- public void booleanColumn(Column column) {
150
- String fieldCode = getFieldCode(column);
151
- FieldType type = getType(column, FieldType.NUMBER);
152
- setValue(fieldCode, pageReader.getBoolean(column), type, isUpdateKey(column));
183
+ private void setLong(
184
+ KintoneColumnType type,
185
+ String fieldCode,
186
+ UpdateKey updateKey,
187
+ long value,
188
+ KintoneColumnOption option) {
189
+ FieldValue fieldValue = type.getFieldValue(value, option);
190
+ if (updateKey != null) {
191
+ type.setUpdateKey(updateKey, fieldCode, fieldValue);
192
+ }
193
+ record.putField(fieldCode, fieldValue);
153
194
  }
154
195
 
155
- @Override
156
- public void longColumn(Column column) {
157
- String fieldCode = getFieldCode(column);
158
- FieldType type = getType(column, FieldType.NUMBER);
159
- if (pageReader.isNull(column)) {
160
- setValue(fieldCode, null, type, isUpdateKey(column));
161
- } else {
162
- setValue(fieldCode, pageReader.getLong(column), type, isUpdateKey(column));
196
+ private void setDouble(
197
+ KintoneColumnType type,
198
+ String fieldCode,
199
+ UpdateKey updateKey,
200
+ double value,
201
+ KintoneColumnOption option) {
202
+ FieldValue fieldValue = type.getFieldValue(value, option);
203
+ if (updateKey != null) {
204
+ type.setUpdateKey(updateKey, fieldCode, fieldValue);
163
205
  }
206
+ record.putField(fieldCode, fieldValue);
164
207
  }
165
208
 
166
- @Override
167
- public void doubleColumn(Column column) {
168
- String fieldCode = getFieldCode(column);
169
- FieldType type = getType(column, FieldType.NUMBER);
170
- setValue(fieldCode, pageReader.getDouble(column), type, isUpdateKey(column));
209
+ private void setString(
210
+ KintoneColumnType type,
211
+ String fieldCode,
212
+ UpdateKey updateKey,
213
+ String value,
214
+ KintoneColumnOption option) {
215
+ FieldValue fieldValue = type.getFieldValue(value, option);
216
+ if (updateKey != null) {
217
+ type.setUpdateKey(updateKey, fieldCode, fieldValue);
218
+ }
219
+ record.putField(fieldCode, fieldValue);
171
220
  }
172
221
 
173
- @Override
174
- public void stringColumn(Column column) {
175
- String fieldCode = getFieldCode(column);
176
- FieldType type = getType(column, FieldType.MULTI_LINE_TEXT);
177
- Object value = pageReader.getString(column);
178
- if (type == FieldType.CHECK_BOX) {
179
- String stringValue = Objects.toString(value, "");
180
- setCheckBoxValue(fieldCode, value, getValueSeparator(column));
181
- return;
222
+ private void setTimestamp(
223
+ KintoneColumnType type,
224
+ String fieldCode,
225
+ UpdateKey updateKey,
226
+ Timestamp value,
227
+ KintoneColumnOption option) {
228
+ FieldValue fieldValue = type.getFieldValue(value, option);
229
+ if (updateKey != null) {
230
+ type.setUpdateKey(updateKey, fieldCode, fieldValue);
182
231
  }
183
- setValue(fieldCode, value, type, isUpdateKey(column));
232
+ record.putField(fieldCode, fieldValue);
184
233
  }
185
234
 
186
- @Override
187
- public void timestampColumn(Column column) {
188
- Timestamp value = pageReader.getTimestamp(column);
189
- if (value == null) {
190
- return;
235
+ private void setJson(
236
+ KintoneColumnType type,
237
+ String fieldCode,
238
+ UpdateKey updateKey,
239
+ Value value,
240
+ KintoneColumnOption option) {
241
+ FieldValue fieldValue = type.getFieldValue(value, option);
242
+ if (updateKey != null) {
243
+ type.setUpdateKey(updateKey, fieldCode, fieldValue);
191
244
  }
245
+ record.putField(fieldCode, fieldValue);
246
+ }
192
247
 
193
- String fieldCode = getFieldCode(column);
194
- FieldType type = getType(column, FieldType.DATETIME);
195
- ZoneId zoneId = getZoneId(column);
196
- if (type == FieldType.DATETIME) {
197
- zoneId = ZoneId.of("UTC");
198
- }
199
- setTimestampValue(fieldCode, value.getInstant(), zoneId, type);
248
+ private String getFieldCode(Column column) {
249
+ KintoneColumnOption option = getOption(column);
250
+ return option != null ? option.getFieldCode() : column.getName();
200
251
  }
201
252
 
202
- @Override
203
- public void jsonColumn(Column column) {
204
- String fieldCode = getFieldCode(column);
205
- FieldType type = getType(column, FieldType.MULTI_LINE_TEXT);
206
- setValue(fieldCode, pageReader.getJson(column), type, isUpdateKey(column));
253
+ private KintoneColumnOption getOption(Column column) {
254
+ return options.get(column.getName());
255
+ }
256
+
257
+ private UpdateKey getUpdateKey(Column column) {
258
+ return updateKeyName != null && updateKeyName.equals(column.getName()) ? updateKey : null;
259
+ }
260
+
261
+ private boolean isIgnoreNull(Column column) {
262
+ return ignoreNulls && reader.isNull(column);
263
+ }
264
+
265
+ private boolean isPreferNull(Column column) {
266
+ return preferNulls && reader.isNull(column);
207
267
  }
208
268
  }
@@ -6,7 +6,6 @@ public enum KintoneMode {
6
6
  INSERT("insert"),
7
7
  UPDATE("update"),
8
8
  UPSERT("upsert");
9
-
10
9
  private final String value;
11
10
 
12
11
  KintoneMode(String value) {
@@ -16,7 +16,6 @@ public class KintoneOutputPlugin implements OutputPlugin {
16
16
  public ConfigDiff transaction(
17
17
  ConfigSource config, Schema schema, int taskCount, OutputPlugin.Control control) {
18
18
  PluginTask task = config.loadConfig(PluginTask.class);
19
-
20
19
  control.run(task.dump());
21
20
  return Exec.newConfigDiff();
22
21
  }
@@ -34,7 +33,6 @@ public class KintoneOutputPlugin implements OutputPlugin {
34
33
  @Override
35
34
  public TransactionalPageOutput open(TaskSource taskSource, Schema schema, int taskIndex) {
36
35
  PluginTask task = taskSource.loadTask(PluginTask.class);
37
-
38
36
  KintoneMode mode = KintoneMode.getKintoneModeByValue(task.getMode());
39
37
  switch (mode) {
40
38
  case INSERT: