embulk-output-kintone 1.0.0 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (90) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +57 -1
  3. data/build.gradle +1 -0
  4. data/classpath/commons-csv-1.9.0.jar +0 -0
  5. data/classpath/embulk-output-kintone-1.1.0.jar +0 -0
  6. data/classpath/externalsortinginjava-0.6.2.jar +0 -0
  7. data/classpath/{shadow-kintone-java-client-1.0.0-all.jar → shadow-kintone-java-client-1.1.0-all.jar} +0 -0
  8. data/src/main/java/org/embulk/output/kintone/KintoneColumnOption.java +5 -0
  9. data/src/main/java/org/embulk/output/kintone/KintoneColumnType.java +209 -5
  10. data/src/main/java/org/embulk/output/kintone/KintoneColumnVisitor.java +28 -9
  11. data/src/main/java/org/embulk/output/kintone/KintoneOutputPlugin.java +12 -3
  12. data/src/main/java/org/embulk/output/kintone/KintonePageOutput.java +20 -12
  13. data/src/main/java/org/embulk/output/kintone/KintoneSortColumn.java +33 -0
  14. data/src/main/java/org/embulk/output/kintone/PluginTask.java +27 -0
  15. data/src/main/java/org/embulk/output/kintone/deserializer/DeserializeApplier.java +19 -0
  16. data/src/main/java/org/embulk/output/kintone/deserializer/DeserializeException.java +7 -0
  17. data/src/main/java/org/embulk/output/kintone/deserializer/Deserializer.java +279 -0
  18. data/src/main/java/org/embulk/output/kintone/reducer/CSVInputColumnVisitor.java +78 -0
  19. data/src/main/java/org/embulk/output/kintone/reducer/CSVOutputColumnVisitor.java +79 -0
  20. data/src/main/java/org/embulk/output/kintone/reducer/ReduceException.java +11 -0
  21. data/src/main/java/org/embulk/output/kintone/reducer/ReduceType.java +190 -0
  22. data/src/main/java/org/embulk/output/kintone/reducer/ReducedPageOutput.java +100 -0
  23. data/src/main/java/org/embulk/output/kintone/reducer/Reducer.java +355 -0
  24. data/src/test/java/org/embulk/output/kintone/KintoneColumnOptionBuilder.java +7 -0
  25. data/src/test/java/org/embulk/output/kintone/KintoneColumnTypeTest.java +194 -0
  26. data/src/test/java/org/embulk/output/kintone/KintoneColumnVisitorTest.java +153 -34
  27. data/src/test/java/org/embulk/output/kintone/KintoneColumnVisitorVerifier.java +13 -3
  28. data/src/test/java/org/embulk/output/kintone/KintonePageOutputVerifier.java +44 -1
  29. data/src/test/java/org/embulk/output/kintone/TestKintoneOutputPlugin.java +89 -12
  30. data/src/test/java/org/embulk/output/kintone/TestTaskReduce.java +46 -0
  31. data/src/test/java/org/embulk/output/kintone/TestTaskReduceException.java +50 -0
  32. data/src/test/java/org/embulk/output/kintone/TestTaskReduceSubtable.java +46 -0
  33. data/src/test/java/org/embulk/output/kintone/deserializer/DeserializerTest.java +165 -0
  34. data/src/test/java/org/embulk/output/kintone/reducer/ReduceTypeTest.java +154 -0
  35. data/src/test/resources/org/embulk/output/kintone/task/config.yml +1 -1
  36. data/src/test/resources/org/embulk/output/kintone/task/mode/config.yml +6 -0
  37. data/src/test/resources/org/embulk/output/kintone/task/mode/input.csv +7 -7
  38. data/src/test/resources/org/embulk/output/kintone/task/mode/insert_add_ignore_nulls_records.jsonl +2 -2
  39. data/src/test/resources/org/embulk/output/kintone/task/mode/insert_add_prefer_nulls_records.jsonl +6 -6
  40. data/src/test/resources/org/embulk/output/kintone/task/mode/insert_add_records.jsonl +6 -6
  41. data/src/test/resources/org/embulk/output/kintone/task/mode/update_update_ignore_nulls_records.jsonl +2 -2
  42. data/src/test/resources/org/embulk/output/kintone/task/mode/update_update_prefer_nulls_records.jsonl +3 -3
  43. data/src/test/resources/org/embulk/output/kintone/task/mode/update_update_records.jsonl +6 -6
  44. data/src/test/resources/org/embulk/output/kintone/task/mode/upsert_add_prefer_nulls_records.jsonl +3 -3
  45. data/src/test/resources/org/embulk/output/kintone/task/mode/upsert_add_records.jsonl +2 -2
  46. data/src/test/resources/org/embulk/output/kintone/task/mode/upsert_update_ignore_nulls_records.jsonl +2 -2
  47. data/src/test/resources/org/embulk/output/kintone/task/mode/upsert_update_prefer_nulls_records.jsonl +3 -3
  48. data/src/test/resources/org/embulk/output/kintone/task/mode/upsert_update_records.jsonl +4 -4
  49. data/src/test/resources/org/embulk/output/kintone/task/reduce/config.yml +171 -0
  50. data/src/test/resources/org/embulk/output/kintone/task/reduce/input.csv +7 -0
  51. data/src/test/resources/org/embulk/output/kintone/task/reduce/insert_add_ignore_nulls_records.jsonl +6 -0
  52. data/src/test/resources/org/embulk/output/kintone/task/reduce/insert_add_prefer_nulls_records.jsonl +6 -0
  53. data/src/test/resources/org/embulk/output/kintone/task/reduce/insert_add_records.jsonl +6 -0
  54. data/src/test/resources/org/embulk/output/kintone/task/reduce/update_update_ignore_nulls_records.jsonl +3 -0
  55. data/src/test/resources/org/embulk/output/kintone/task/reduce/update_update_prefer_nulls_records.jsonl +3 -0
  56. data/src/test/resources/org/embulk/output/kintone/task/reduce/update_update_records.jsonl +6 -0
  57. data/src/test/resources/org/embulk/output/kintone/task/reduce/upsert_add_ignore_nulls_records.jsonl +3 -0
  58. data/src/test/resources/org/embulk/output/kintone/task/reduce/upsert_add_prefer_nulls_records.jsonl +3 -0
  59. data/src/test/resources/org/embulk/output/kintone/task/reduce/upsert_add_records.jsonl +2 -0
  60. data/src/test/resources/org/embulk/output/kintone/task/reduce/upsert_update_ignore_nulls_records.jsonl +3 -0
  61. data/src/test/resources/org/embulk/output/kintone/task/reduce/upsert_update_prefer_nulls_records.jsonl +3 -0
  62. data/src/test/resources/org/embulk/output/kintone/task/reduce/upsert_update_records.jsonl +4 -0
  63. data/src/test/resources/org/embulk/output/kintone/task/reduce/values.json +1 -0
  64. data/src/test/resources/org/embulk/output/kintone/task/reduce/values_ignore_nulls.json +1 -0
  65. data/src/test/resources/org/embulk/output/kintone/task/reduce/values_prefer_nulls.json +1 -0
  66. data/src/test/resources/org/embulk/output/kintone/task/reduce_exception/config.yml +36 -0
  67. data/src/test/resources/org/embulk/output/kintone/task/reduce_exception/derived_columns.json +1 -0
  68. data/src/test/resources/org/embulk/output/kintone/task/reduce_exception/input.csv +13 -0
  69. data/src/test/resources/org/embulk/output/kintone/task/reduce_exception/insert_add_records.jsonl +2 -0
  70. data/src/test/resources/org/embulk/output/kintone/task/reduce_exception/update_update_records.jsonl +2 -0
  71. data/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/config.yml +343 -0
  72. data/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/derived_columns.json +1 -0
  73. data/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/input.csv +13 -0
  74. data/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/insert_add_ignore_nulls_records.jsonl +6 -0
  75. data/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/insert_add_prefer_nulls_records.jsonl +6 -0
  76. data/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/insert_add_records.jsonl +6 -0
  77. data/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/update_update_ignore_nulls_records.jsonl +3 -0
  78. data/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/update_update_prefer_nulls_records.jsonl +3 -0
  79. data/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/update_update_records.jsonl +6 -0
  80. data/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/upsert_add_ignore_nulls_records.jsonl +3 -0
  81. data/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/upsert_add_prefer_nulls_records.jsonl +3 -0
  82. data/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/upsert_add_records.jsonl +0 -0
  83. data/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/upsert_update_ignore_nulls_records.jsonl +3 -0
  84. data/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/upsert_update_prefer_nulls_records.jsonl +3 -0
  85. data/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/upsert_update_records.jsonl +6 -0
  86. data/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/values.json +1 -0
  87. data/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/values_ignore_nulls.json +1 -0
  88. data/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/values_prefer_nulls.json +1 -0
  89. metadata +62 -4
  90. data/classpath/embulk-output-kintone-1.0.0.jar +0 -0
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 11c7f1cbe588863b206b1581bf2ec89adeafeea4
4
- data.tar.gz: c17ae7651f3409a6e51999e65233a96da2d87bf4
3
+ metadata.gz: 38d9d369925e0cab9fa558716b9a939be4b1387e
4
+ data.tar.gz: f4ae4a741548a48b042e1c4b79bf7e533ded0d31
5
5
  SHA512:
6
- metadata.gz: 4e1bc2fe10bc224c643d88c06a37a19bfe4849fb46318fede913ef3a3923e1315ce48b78aea8bbdcc5ea9a5b4760058776c569e8acedf8f02691b80b330c5657
7
- data.tar.gz: 9261603e49b45d2c8b72bf013d2995057293dd7f98ae0588f4dfa2f7e962b0ba2eac70990cdd7658eedf72a5caabd9cbb8f3135bc80cb91a7d250706c06081b4
6
+ metadata.gz: 06d103f7c4036dd458f62955bee9b2b82aa2bb2719cb59f18a2a762bab5c892025b4dd7fb0b41647e0e9cf66e012916f60de1a48c1a6d41414af0447d8affb06
7
+ data.tar.gz: ee848b7cfb53cc27c731724d32b6d5ffa9fe3336742fc1bbf24bd2b49d07f6f99439b478a0342051810ef94597471caded0f81caaf3aaff78203ce0ff20485d6
data/README.md CHANGED
@@ -18,14 +18,24 @@ kintone output plugin for Embulk stores app records from kintone.
18
18
  - **guest_space_id**: kintone app belongs to guest space, guest space id is required. (integer, optional)
19
19
  - **mode**: kintone mode (string, required)
20
20
  - **update_key**: Column name to set update key (string, required if mode is update or upsert)
21
+ - **reduce_key**: Key column name to reduce expanded SUBTABLE (string, optional)
22
+ - **sort_columns**: List of columns for sorting input records (array of objects, optional)
23
+ - **name**: Column name (string, required)
24
+ - **order**: Sort order (string `asc` or `desc`, required)
25
+ - **max_sort_tmp_files**: Maximum number of temporary files for sorting input records (integer, default is `1024`)
26
+ - **max_sort_memory**: Maximum memory usage for sorting input records (bytes in long, default is the estimated available memory, which is the approximate value of the JVM's current free memory)
21
27
  - **prefer_nulls**: Whether to set fields to null instead of default value of type when column is null (boolean, default is `false`)
22
28
  - **ignore_nulls**: Whether to completely ignore fields when column is null (boolean, default is `false`)
23
29
  - **column_options** advanced: a key-value pairs where key is a column name and value is options for the column.
24
30
  - **field_code**: field code (string, required)
25
31
  - **type**: field type (string, required). See [this page](https://cybozu.dev/ja/kintone/docs/overview/field-types/#field-type-update) for list of available types. However, following types are not yet supported
26
- - `USER_SELECT`, `ORGANIZATION_SELECT`, `GROUP_SELECT`, `FILE`, `SUBTABLE`
32
+ - `USER_SELECT`, `ORGANIZATION_SELECT`, `GROUP_SELECT`, `FILE`
27
33
  - **timezone**: timezone to convert into `date` (string, default is `UTC`)
28
34
  - **val_sep**: Used to specify multiple checkbox values (string, default is `,`)
35
+ - **sort_columns**: List of columns for sorting rows in SUBTABLE. Available only if type is `SUBTABLE` (array of objects, optional)
36
+ - **name**: Column name (string, required)
37
+ - **order**: Sort order (string `asc` or `desc`, required)
38
+ - **chunk_size**: Maximum number of records to request at once (integer, default is `100`)
29
39
 
30
40
  ## Example
31
41
 
@@ -46,6 +56,52 @@ out:
46
56
  date_time: {field_code: "datetime", type: "DATETIME"}
47
57
  ```
48
58
 
59
+ ### For reduce expanded SUBTABLE
60
+
61
+ ```yaml
62
+ out:
63
+ ...
64
+ reduce_key: id
65
+ column_options:
66
+ id: {field_code: "id", type: "NUMBER"}
67
+ ...
68
+ table: {field_code: "table", type: "SUBTABLE", sort_columns: [{name: number, order: asc}, {name: text, order: desc}]}
69
+ table.number: {field_code: "number_in_table", type: "NUMBER"}
70
+ table.text: {field_code: "text_in_table", type: "SINGLE_LINE_TEXT"}
71
+ ```
72
+
73
+ #### KINTONE
74
+
75
+ | Form | Field Code |
76
+ | ------------------------- | --------------- |
77
+ | Table's own | table |
78
+ | NUMBER In Table | number_in_table |
79
+ | SINGLE_LINE_TEXT In Table | text_in_table |
80
+
81
+ #### INPUT CSV
82
+
83
+ ```csv
84
+ id,table.number,table.text
85
+ 1,0,test0
86
+ 1,1,test1
87
+ 2,0,test0
88
+ ```
89
+
90
+ #### RESULT
91
+
92
+ ID:1
93
+
94
+ | NUMBER | SINGLE_LINE_TEXT |
95
+ | ------------- | ---------------- |
96
+ | 0 | test0 |
97
+ | 1 | test1 |
98
+
99
+ ID:2
100
+
101
+ | NUMBER | SINGLE_LINE_TEXT |
102
+ | ------------- | ---------------- |
103
+ | 0 | test0 |
104
+
49
105
  ## Build
50
106
 
51
107
  ```
data/build.gradle CHANGED
@@ -28,6 +28,7 @@ targetCompatibility = 1.8
28
28
 
29
29
  dependencies {
30
30
  compileOnly "org.embulk:embulk-core:0.9.23"
31
+ implementation "com.google.code.externalsortinginjava:externalsortinginjava:0.6.2"
31
32
  implementation project(path: ":shadow-kintone-java-client", configuration: "shadow")
32
33
 
33
34
  testImplementation "junit:junit:4.+"
Binary file
@@ -1,5 +1,6 @@
1
1
  package org.embulk.output.kintone;
2
2
 
3
+ import java.util.List;
3
4
  import org.embulk.config.Config;
4
5
  import org.embulk.config.ConfigDefault;
5
6
  import org.embulk.config.Task;
@@ -18,4 +19,8 @@ public interface KintoneColumnOption extends Task {
18
19
  @Config("val_sep")
19
20
  @ConfigDefault("\",\"")
20
21
  String getValueSeparator();
22
+
23
+ @Config("sort_columns")
24
+ @ConfigDefault("[]")
25
+ List<KintoneSortColumn> getSortColumns();
21
26
  }
@@ -29,8 +29,13 @@ import java.time.ZonedDateTime;
29
29
  import java.util.Arrays;
30
30
  import java.util.Collections;
31
31
  import java.util.List;
32
+ import java.util.stream.Collectors;
33
+ import org.embulk.output.kintone.deserializer.Deserializer;
32
34
  import org.embulk.spi.time.Timestamp;
35
+ import org.embulk.spi.type.Type;
36
+ import org.embulk.spi.type.Types;
33
37
  import org.msgpack.value.Value;
38
+ import org.msgpack.value.ValueFactory;
34
39
 
35
40
  public enum KintoneColumnType {
36
41
  SINGLE_LINE_TEXT {
@@ -53,6 +58,16 @@ public enum KintoneColumnType {
53
58
  public void setUpdateKey(UpdateKey updateKey, String field, FieldValue value) {
54
59
  updateKey.setField(field).setValue(((SingleLineTextFieldValue) value).getValue());
55
60
  }
61
+
62
+ @Override
63
+ public Value asValue(FieldValue value) {
64
+ return ValueFactory.newString(((SingleLineTextFieldValue) value).getValue());
65
+ }
66
+
67
+ @Override
68
+ protected List<Type> getSupportedTypes() {
69
+ return Arrays.asList(Types.BOOLEAN, Types.LONG, Types.DOUBLE, Types.TIMESTAMP, Types.JSON);
70
+ }
56
71
  },
57
72
  MULTI_LINE_TEXT {
58
73
  @Override
@@ -64,6 +79,16 @@ public enum KintoneColumnType {
64
79
  public MultiLineTextFieldValue getFieldValue(String value, KintoneColumnOption option) {
65
80
  return new MultiLineTextFieldValue(value);
66
81
  }
82
+
83
+ @Override
84
+ public Value asValue(FieldValue value) {
85
+ return ValueFactory.newString(((MultiLineTextFieldValue) value).getValue());
86
+ }
87
+
88
+ @Override
89
+ protected List<Type> getSupportedTypes() {
90
+ return Arrays.asList(Types.BOOLEAN, Types.LONG, Types.DOUBLE, Types.TIMESTAMP, Types.JSON);
91
+ }
67
92
  },
68
93
  RICH_TEXT {
69
94
  @Override
@@ -75,6 +100,16 @@ public enum KintoneColumnType {
75
100
  public RichTextFieldValue getFieldValue(String value, KintoneColumnOption option) {
76
101
  return new RichTextFieldValue(value);
77
102
  }
103
+
104
+ @Override
105
+ public Value asValue(FieldValue value) {
106
+ return ValueFactory.newString(((RichTextFieldValue) value).getValue());
107
+ }
108
+
109
+ @Override
110
+ protected List<Type> getSupportedTypes() {
111
+ return Arrays.asList(Types.BOOLEAN, Types.LONG, Types.DOUBLE, Types.TIMESTAMP, Types.JSON);
112
+ }
78
113
  },
79
114
  NUMBER {
80
115
  @Override
@@ -106,6 +141,16 @@ public enum KintoneColumnType {
106
141
  public void setUpdateKey(UpdateKey updateKey, String field, FieldValue value) {
107
142
  updateKey.setField(field).setValue(((NumberFieldValue) value).getValue());
108
143
  }
144
+
145
+ @Override
146
+ public Value asValue(FieldValue value) {
147
+ return ValueFactory.newString(((NumberFieldValue) value).getValue().toString());
148
+ }
149
+
150
+ @Override
151
+ protected List<Type> getSupportedTypes() {
152
+ return Arrays.asList(Types.BOOLEAN, Types.LONG, Types.DOUBLE, Types.TIMESTAMP);
153
+ }
109
154
  },
110
155
  CHECK_BOX {
111
156
  @Override
@@ -117,6 +162,18 @@ public enum KintoneColumnType {
117
162
  public CheckBoxFieldValue getFieldValue(String value, KintoneColumnOption option) {
118
163
  return new CheckBoxFieldValue(asList(value, option));
119
164
  }
165
+
166
+ @Override
167
+ public Value asValue(FieldValue value) {
168
+ return ValueFactory.newArray(
169
+ ((CheckBoxFieldValue) value)
170
+ .getValues().stream().map(ValueFactory::newString).collect(Collectors.toList()));
171
+ }
172
+
173
+ @Override
174
+ protected List<Type> getSupportedTypes() {
175
+ return Collections.emptyList();
176
+ }
120
177
  },
121
178
  RADIO_BUTTON {
122
179
  @Override
@@ -128,6 +185,16 @@ public enum KintoneColumnType {
128
185
  public RadioButtonFieldValue getFieldValue(String value, KintoneColumnOption option) {
129
186
  return new RadioButtonFieldValue(value);
130
187
  }
188
+
189
+ @Override
190
+ public Value asValue(FieldValue value) {
191
+ return ValueFactory.newString(((RadioButtonFieldValue) value).getValue());
192
+ }
193
+
194
+ @Override
195
+ protected List<Type> getSupportedTypes() {
196
+ return Collections.emptyList();
197
+ }
131
198
  },
132
199
  MULTI_SELECT {
133
200
  @Override
@@ -139,6 +206,18 @@ public enum KintoneColumnType {
139
206
  public MultiSelectFieldValue getFieldValue(String value, KintoneColumnOption option) {
140
207
  return new MultiSelectFieldValue(asList(value, option));
141
208
  }
209
+
210
+ @Override
211
+ public Value asValue(FieldValue value) {
212
+ return ValueFactory.newArray(
213
+ ((MultiSelectFieldValue) value)
214
+ .getValues().stream().map(ValueFactory::newString).collect(Collectors.toList()));
215
+ }
216
+
217
+ @Override
218
+ protected List<Type> getSupportedTypes() {
219
+ return Collections.emptyList();
220
+ }
142
221
  },
143
222
  DROP_DOWN {
144
223
  @Override
@@ -150,6 +229,16 @@ public enum KintoneColumnType {
150
229
  public DropDownFieldValue getFieldValue(String value, KintoneColumnOption option) {
151
230
  return new DropDownFieldValue(value);
152
231
  }
232
+
233
+ @Override
234
+ public Value asValue(FieldValue value) {
235
+ return ValueFactory.newString(((DropDownFieldValue) value).getValue());
236
+ }
237
+
238
+ @Override
239
+ protected List<Type> getSupportedTypes() {
240
+ return Collections.emptyList();
241
+ }
153
242
  },
154
243
  USER_SELECT {
155
244
  @Override
@@ -161,6 +250,16 @@ public enum KintoneColumnType {
161
250
  public UserSelectFieldValue getFieldValue(String value, KintoneColumnOption option) {
162
251
  throw new UnsupportedOperationException();
163
252
  }
253
+
254
+ @Override
255
+ public Value asValue(FieldValue value) {
256
+ throw new UnsupportedOperationException();
257
+ }
258
+
259
+ @Override
260
+ protected List<Type> getSupportedTypes() {
261
+ return Collections.emptyList();
262
+ }
164
263
  },
165
264
  ORGANIZATION_SELECT {
166
265
  @Override
@@ -172,6 +271,16 @@ public enum KintoneColumnType {
172
271
  public OrganizationSelectFieldValue getFieldValue(String value, KintoneColumnOption option) {
173
272
  throw new UnsupportedOperationException();
174
273
  }
274
+
275
+ @Override
276
+ public Value asValue(FieldValue value) {
277
+ throw new UnsupportedOperationException();
278
+ }
279
+
280
+ @Override
281
+ protected List<Type> getSupportedTypes() {
282
+ return Collections.emptyList();
283
+ }
175
284
  },
176
285
  GROUP_SELECT {
177
286
  @Override
@@ -183,6 +292,16 @@ public enum KintoneColumnType {
183
292
  public GroupSelectFieldValue getFieldValue(String value, KintoneColumnOption option) {
184
293
  throw new UnsupportedOperationException();
185
294
  }
295
+
296
+ @Override
297
+ public Value asValue(FieldValue value) {
298
+ throw new UnsupportedOperationException();
299
+ }
300
+
301
+ @Override
302
+ protected List<Type> getSupportedTypes() {
303
+ return Collections.emptyList();
304
+ }
186
305
  },
187
306
  DATE {
188
307
  @Override
@@ -215,6 +334,16 @@ public enum KintoneColumnType {
215
334
  public DateFieldValue getFieldValue(Timestamp value, KintoneColumnOption option) {
216
335
  return new DateFieldValue(value.getInstant().atZone(getZoneId(option)).toLocalDate());
217
336
  }
337
+
338
+ @Override
339
+ public Value asValue(FieldValue value) {
340
+ return ValueFactory.newString(((DateFieldValue) value).getValue().toString());
341
+ }
342
+
343
+ @Override
344
+ protected List<Type> getSupportedTypes() {
345
+ return Arrays.asList(Types.LONG, Types.DOUBLE, Types.TIMESTAMP);
346
+ }
218
347
  },
219
348
  TIME {
220
349
  @Override
@@ -247,6 +376,16 @@ public enum KintoneColumnType {
247
376
  public TimeFieldValue getFieldValue(Timestamp value, KintoneColumnOption option) {
248
377
  return new TimeFieldValue(value.getInstant().atZone(getZoneId(option)).toLocalTime());
249
378
  }
379
+
380
+ @Override
381
+ public Value asValue(FieldValue value) {
382
+ return ValueFactory.newString(((TimeFieldValue) value).getValue().toString());
383
+ }
384
+
385
+ @Override
386
+ protected List<Type> getSupportedTypes() {
387
+ return Arrays.asList(Types.LONG, Types.DOUBLE, Types.TIMESTAMP);
388
+ }
250
389
  },
251
390
  DATETIME {
252
391
  @Override
@@ -275,6 +414,16 @@ public enum KintoneColumnType {
275
414
  public DateTimeFieldValue getFieldValue(Timestamp value, KintoneColumnOption option) {
276
415
  return new DateTimeFieldValue(value.getInstant().atZone(ZoneOffset.UTC));
277
416
  }
417
+
418
+ @Override
419
+ public Value asValue(FieldValue value) {
420
+ return ValueFactory.newString(((DateTimeFieldValue) value).getValue().toString());
421
+ }
422
+
423
+ @Override
424
+ protected List<Type> getSupportedTypes() {
425
+ return Arrays.asList(Types.LONG, Types.DOUBLE, Types.TIMESTAMP);
426
+ }
278
427
  },
279
428
  LINK {
280
429
  @Override
@@ -286,6 +435,16 @@ public enum KintoneColumnType {
286
435
  public LinkFieldValue getFieldValue(String value, KintoneColumnOption option) {
287
436
  return new LinkFieldValue(value);
288
437
  }
438
+
439
+ @Override
440
+ public Value asValue(FieldValue value) {
441
+ return ValueFactory.newString(((LinkFieldValue) value).getValue());
442
+ }
443
+
444
+ @Override
445
+ protected List<Type> getSupportedTypes() {
446
+ return Collections.emptyList();
447
+ }
289
448
  },
290
449
  FILE {
291
450
  @Override
@@ -297,6 +456,16 @@ public enum KintoneColumnType {
297
456
  public FileFieldValue getFieldValue(String value, KintoneColumnOption option) {
298
457
  throw new UnsupportedOperationException();
299
458
  }
459
+
460
+ @Override
461
+ public Value asValue(FieldValue value) {
462
+ throw new UnsupportedOperationException();
463
+ }
464
+
465
+ @Override
466
+ protected List<Type> getSupportedTypes() {
467
+ return Collections.emptyList();
468
+ }
300
469
  },
301
470
  SUBTABLE {
302
471
  @Override
@@ -306,9 +475,20 @@ public enum KintoneColumnType {
306
475
 
307
476
  @Override
308
477
  public SubtableFieldValue getFieldValue(String value, KintoneColumnOption option) {
478
+ return DESERIALIZER.deserialize(value.isEmpty() ? "[]" : value, SubtableFieldValue.class);
479
+ }
480
+
481
+ @Override
482
+ public Value asValue(FieldValue value) {
309
483
  throw new UnsupportedOperationException();
310
484
  }
485
+
486
+ @Override
487
+ protected List<Type> getSupportedTypes() {
488
+ return Collections.singletonList(Types.JSON);
489
+ }
311
490
  };
491
+ private static final Deserializer DESERIALIZER = new Deserializer();
312
492
  private static final Timestamp EPOCH = Timestamp.ofInstant(Instant.EPOCH);
313
493
 
314
494
  public static KintoneColumnType getType(
@@ -319,25 +499,45 @@ public enum KintoneColumnType {
319
499
  public abstract FieldValue getFieldValue();
320
500
 
321
501
  public FieldValue getFieldValue(boolean value, KintoneColumnOption option) {
322
- return getFieldValue(String.valueOf(value), option);
502
+ if (getSupportedTypes().contains(Types.BOOLEAN)) {
503
+ return getFieldValue(String.valueOf(value), option);
504
+ } else {
505
+ throw new UnsupportedOperationException();
506
+ }
323
507
  }
324
508
 
325
509
  public FieldValue getFieldValue(long value, KintoneColumnOption option) {
326
- return getFieldValue(String.valueOf(value), option);
510
+ if (getSupportedTypes().contains(Types.LONG)) {
511
+ return getFieldValue(String.valueOf(value), option);
512
+ } else {
513
+ throw new UnsupportedOperationException();
514
+ }
327
515
  }
328
516
 
329
517
  public FieldValue getFieldValue(double value, KintoneColumnOption option) {
330
- return getFieldValue(String.valueOf(value), option);
518
+ if (getSupportedTypes().contains(Types.DOUBLE)) {
519
+ return getFieldValue(String.valueOf(value), option);
520
+ } else {
521
+ throw new UnsupportedOperationException();
522
+ }
331
523
  }
332
524
 
333
525
  public abstract FieldValue getFieldValue(String value, KintoneColumnOption option);
334
526
 
335
527
  public FieldValue getFieldValue(Timestamp value, KintoneColumnOption option) {
336
- return getFieldValue(value.getInstant().toString(), option);
528
+ if (getSupportedTypes().contains(Types.TIMESTAMP)) {
529
+ return getFieldValue(value.getInstant().toString(), option);
530
+ } else {
531
+ throw new UnsupportedOperationException();
532
+ }
337
533
  }
338
534
 
339
535
  public FieldValue getFieldValue(Value value, KintoneColumnOption option) {
340
- return getFieldValue(value.toJson(), option);
536
+ if (getSupportedTypes().contains(Types.JSON)) {
537
+ return getFieldValue(value.toJson(), option);
538
+ } else {
539
+ throw new UnsupportedOperationException();
540
+ }
341
541
  }
342
542
 
343
543
  public void setUpdateKey(UpdateKey updateKey, String field) {
@@ -348,6 +548,10 @@ public enum KintoneColumnType {
348
548
  throw new UnsupportedOperationException();
349
549
  }
350
550
 
551
+ public abstract Value asValue(FieldValue value);
552
+
553
+ protected abstract List<Type> getSupportedTypes();
554
+
351
555
  private static List<String> asList(String value, KintoneColumnOption option) {
352
556
  return value.isEmpty()
353
557
  ? Collections.emptyList()
@@ -5,6 +5,7 @@ import com.kintone.client.model.record.Record;
5
5
  import com.kintone.client.model.record.UpdateKey;
6
6
  import java.time.Instant;
7
7
  import java.util.Map;
8
+ import java.util.Set;
8
9
  import org.embulk.spi.Column;
9
10
  import org.embulk.spi.ColumnVisitor;
10
11
  import org.embulk.spi.PageReader;
@@ -14,31 +15,39 @@ import org.msgpack.value.ValueFactory;
14
15
 
15
16
  public class KintoneColumnVisitor implements ColumnVisitor {
16
17
  private final PageReader reader;
18
+ private final Set<Column> derived;
17
19
  private final Map<String, KintoneColumnOption> options;
18
20
  private final boolean preferNulls;
19
21
  private final boolean ignoreNulls;
22
+ private final String reduceKeyName;
20
23
  private final String updateKeyName;
21
24
  private Record record;
22
25
  private UpdateKey updateKey;
23
26
 
24
27
  public KintoneColumnVisitor(
25
28
  PageReader reader,
29
+ Set<Column> derived,
26
30
  Map<String, KintoneColumnOption> options,
27
31
  boolean preferNulls,
28
- boolean ignoreNulls) {
29
- this(reader, options, preferNulls, ignoreNulls, null);
32
+ boolean ignoreNulls,
33
+ String reduceKeyName) {
34
+ this(reader, derived, options, preferNulls, ignoreNulls, reduceKeyName, null);
30
35
  }
31
36
 
32
37
  public KintoneColumnVisitor(
33
38
  PageReader reader,
39
+ Set<Column> derived,
34
40
  Map<String, KintoneColumnOption> options,
35
41
  boolean preferNulls,
36
42
  boolean ignoreNulls,
43
+ String reduceKeyName,
37
44
  String updateKeyName) {
38
45
  this.reader = reader;
46
+ this.derived = derived;
39
47
  this.options = options;
40
48
  this.preferNulls = preferNulls;
41
49
  this.ignoreNulls = ignoreNulls;
50
+ this.reduceKeyName = reduceKeyName;
42
51
  this.updateKeyName = updateKeyName;
43
52
  }
44
53
 
@@ -52,7 +61,7 @@ public class KintoneColumnVisitor implements ColumnVisitor {
52
61
 
53
62
  @Override
54
63
  public void booleanColumn(Column column) {
55
- if (isIgnoreNull(column)) {
64
+ if (isReduced(column) || isIgnoreNull(column)) {
56
65
  return;
57
66
  }
58
67
  KintoneColumnOption option = getOption(column);
@@ -70,7 +79,7 @@ public class KintoneColumnVisitor implements ColumnVisitor {
70
79
 
71
80
  @Override
72
81
  public void longColumn(Column column) {
73
- if (isIgnoreNull(column)) {
82
+ if (isReduced(column) || isIgnoreNull(column)) {
74
83
  return;
75
84
  }
76
85
  KintoneColumnOption option = getOption(column);
@@ -88,7 +97,7 @@ public class KintoneColumnVisitor implements ColumnVisitor {
88
97
 
89
98
  @Override
90
99
  public void doubleColumn(Column column) {
91
- if (isIgnoreNull(column)) {
100
+ if (isReduced(column) || isIgnoreNull(column)) {
92
101
  return;
93
102
  }
94
103
  KintoneColumnOption option = getOption(column);
@@ -106,7 +115,7 @@ public class KintoneColumnVisitor implements ColumnVisitor {
106
115
 
107
116
  @Override
108
117
  public void stringColumn(Column column) {
109
- if (isIgnoreNull(column)) {
118
+ if (isReduced(column) || isIgnoreNull(column)) {
110
119
  return;
111
120
  }
112
121
  KintoneColumnOption option = getOption(column);
@@ -126,7 +135,7 @@ public class KintoneColumnVisitor implements ColumnVisitor {
126
135
 
127
136
  @Override
128
137
  public void timestampColumn(Column column) {
129
- if (isIgnoreNull(column)) {
138
+ if (isReduced(column) || isIgnoreNull(column)) {
130
139
  return;
131
140
  }
132
141
  KintoneColumnOption option = getOption(column);
@@ -144,12 +153,14 @@ public class KintoneColumnVisitor implements ColumnVisitor {
144
153
 
145
154
  @Override
146
155
  public void jsonColumn(Column column) {
147
- if (isIgnoreNull(column)) {
156
+ if (isReduced(column) || isIgnoreNull(column)) {
148
157
  return;
149
158
  }
150
159
  KintoneColumnOption option = getOption(column);
151
160
  UpdateKey updateKey = getUpdateKey(column);
152
- KintoneColumnType type = KintoneColumnType.getType(option, KintoneColumnType.MULTI_LINE_TEXT);
161
+ KintoneColumnType defaultType =
162
+ isDerived(column) ? KintoneColumnType.SUBTABLE : KintoneColumnType.MULTI_LINE_TEXT;
163
+ KintoneColumnType type = KintoneColumnType.getType(option, defaultType);
153
164
  String fieldCode = getFieldCode(column);
154
165
  if (isPreferNull(column)) {
155
166
  setNull(type, fieldCode, updateKey);
@@ -258,6 +269,14 @@ public class KintoneColumnVisitor implements ColumnVisitor {
258
269
  return updateKeyName != null && updateKeyName.equals(column.getName()) ? updateKey : null;
259
270
  }
260
271
 
272
+ private boolean isReduced(Column column) {
273
+ return reduceKeyName != null && column.getName().matches("^.*\\..*$");
274
+ }
275
+
276
+ private boolean isDerived(Column column) {
277
+ return reduceKeyName != null && derived.contains(column);
278
+ }
279
+
261
280
  private boolean isIgnoreNull(Column column) {
262
281
  return ignoreNulls && reader.isNull(column);
263
282
  }
@@ -1,11 +1,14 @@
1
1
  package org.embulk.output.kintone;
2
2
 
3
+ import java.util.Collections;
3
4
  import java.util.List;
4
5
  import org.embulk.config.ConfigDiff;
5
6
  import org.embulk.config.ConfigException;
6
7
  import org.embulk.config.ConfigSource;
7
8
  import org.embulk.config.TaskReport;
8
9
  import org.embulk.config.TaskSource;
10
+ import org.embulk.output.kintone.reducer.ReducedPageOutput;
11
+ import org.embulk.output.kintone.reducer.Reducer;
9
12
  import org.embulk.spi.Exec;
10
13
  import org.embulk.spi.OutputPlugin;
11
14
  import org.embulk.spi.Schema;
@@ -16,8 +19,12 @@ public class KintoneOutputPlugin implements OutputPlugin {
16
19
  public ConfigDiff transaction(
17
20
  ConfigSource config, Schema schema, int taskCount, OutputPlugin.Control control) {
18
21
  PluginTask task = config.loadConfig(PluginTask.class);
19
- control.run(task.dump());
20
- return Exec.newConfigDiff();
22
+ task.setDerivedColumns(Collections.emptySet());
23
+ List<TaskReport> taskReports = control.run(task.dump());
24
+ return task.getReduceKeyName().isPresent()
25
+ ? new Reducer(task, schema)
26
+ .reduce(taskReports, schema.lookupColumn(task.getReduceKeyName().get()))
27
+ : Exec.newConfigDiff();
21
28
  }
22
29
 
23
30
  @Override
@@ -49,6 +56,8 @@ public class KintoneOutputPlugin implements OutputPlugin {
49
56
  default:
50
57
  throw new ConfigException(String.format("Unknown mode '%s'", task.getMode()));
51
58
  }
52
- return new KintonePageOutput(task, schema);
59
+ return task.getReduceKeyName().isPresent()
60
+ ? new ReducedPageOutput(schema, taskIndex)
61
+ : new KintonePageOutput(task, schema);
53
62
  }
54
63
  }