embulk-output-kintone 1.0.0 → 1.1.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (90) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +57 -1
  3. data/build.gradle +1 -0
  4. data/classpath/commons-csv-1.9.0.jar +0 -0
  5. data/classpath/embulk-output-kintone-1.1.0.jar +0 -0
  6. data/classpath/externalsortinginjava-0.6.2.jar +0 -0
  7. data/classpath/{shadow-kintone-java-client-1.0.0-all.jar → shadow-kintone-java-client-1.1.0-all.jar} +0 -0
  8. data/src/main/java/org/embulk/output/kintone/KintoneColumnOption.java +5 -0
  9. data/src/main/java/org/embulk/output/kintone/KintoneColumnType.java +209 -5
  10. data/src/main/java/org/embulk/output/kintone/KintoneColumnVisitor.java +28 -9
  11. data/src/main/java/org/embulk/output/kintone/KintoneOutputPlugin.java +12 -3
  12. data/src/main/java/org/embulk/output/kintone/KintonePageOutput.java +20 -12
  13. data/src/main/java/org/embulk/output/kintone/KintoneSortColumn.java +33 -0
  14. data/src/main/java/org/embulk/output/kintone/PluginTask.java +27 -0
  15. data/src/main/java/org/embulk/output/kintone/deserializer/DeserializeApplier.java +19 -0
  16. data/src/main/java/org/embulk/output/kintone/deserializer/DeserializeException.java +7 -0
  17. data/src/main/java/org/embulk/output/kintone/deserializer/Deserializer.java +279 -0
  18. data/src/main/java/org/embulk/output/kintone/reducer/CSVInputColumnVisitor.java +78 -0
  19. data/src/main/java/org/embulk/output/kintone/reducer/CSVOutputColumnVisitor.java +79 -0
  20. data/src/main/java/org/embulk/output/kintone/reducer/ReduceException.java +11 -0
  21. data/src/main/java/org/embulk/output/kintone/reducer/ReduceType.java +190 -0
  22. data/src/main/java/org/embulk/output/kintone/reducer/ReducedPageOutput.java +100 -0
  23. data/src/main/java/org/embulk/output/kintone/reducer/Reducer.java +355 -0
  24. data/src/test/java/org/embulk/output/kintone/KintoneColumnOptionBuilder.java +7 -0
  25. data/src/test/java/org/embulk/output/kintone/KintoneColumnTypeTest.java +194 -0
  26. data/src/test/java/org/embulk/output/kintone/KintoneColumnVisitorTest.java +153 -34
  27. data/src/test/java/org/embulk/output/kintone/KintoneColumnVisitorVerifier.java +13 -3
  28. data/src/test/java/org/embulk/output/kintone/KintonePageOutputVerifier.java +44 -1
  29. data/src/test/java/org/embulk/output/kintone/TestKintoneOutputPlugin.java +89 -12
  30. data/src/test/java/org/embulk/output/kintone/TestTaskReduce.java +46 -0
  31. data/src/test/java/org/embulk/output/kintone/TestTaskReduceException.java +50 -0
  32. data/src/test/java/org/embulk/output/kintone/TestTaskReduceSubtable.java +46 -0
  33. data/src/test/java/org/embulk/output/kintone/deserializer/DeserializerTest.java +165 -0
  34. data/src/test/java/org/embulk/output/kintone/reducer/ReduceTypeTest.java +154 -0
  35. data/src/test/resources/org/embulk/output/kintone/task/config.yml +1 -1
  36. data/src/test/resources/org/embulk/output/kintone/task/mode/config.yml +6 -0
  37. data/src/test/resources/org/embulk/output/kintone/task/mode/input.csv +7 -7
  38. data/src/test/resources/org/embulk/output/kintone/task/mode/insert_add_ignore_nulls_records.jsonl +2 -2
  39. data/src/test/resources/org/embulk/output/kintone/task/mode/insert_add_prefer_nulls_records.jsonl +6 -6
  40. data/src/test/resources/org/embulk/output/kintone/task/mode/insert_add_records.jsonl +6 -6
  41. data/src/test/resources/org/embulk/output/kintone/task/mode/update_update_ignore_nulls_records.jsonl +2 -2
  42. data/src/test/resources/org/embulk/output/kintone/task/mode/update_update_prefer_nulls_records.jsonl +3 -3
  43. data/src/test/resources/org/embulk/output/kintone/task/mode/update_update_records.jsonl +6 -6
  44. data/src/test/resources/org/embulk/output/kintone/task/mode/upsert_add_prefer_nulls_records.jsonl +3 -3
  45. data/src/test/resources/org/embulk/output/kintone/task/mode/upsert_add_records.jsonl +2 -2
  46. data/src/test/resources/org/embulk/output/kintone/task/mode/upsert_update_ignore_nulls_records.jsonl +2 -2
  47. data/src/test/resources/org/embulk/output/kintone/task/mode/upsert_update_prefer_nulls_records.jsonl +3 -3
  48. data/src/test/resources/org/embulk/output/kintone/task/mode/upsert_update_records.jsonl +4 -4
  49. data/src/test/resources/org/embulk/output/kintone/task/reduce/config.yml +171 -0
  50. data/src/test/resources/org/embulk/output/kintone/task/reduce/input.csv +7 -0
  51. data/src/test/resources/org/embulk/output/kintone/task/reduce/insert_add_ignore_nulls_records.jsonl +6 -0
  52. data/src/test/resources/org/embulk/output/kintone/task/reduce/insert_add_prefer_nulls_records.jsonl +6 -0
  53. data/src/test/resources/org/embulk/output/kintone/task/reduce/insert_add_records.jsonl +6 -0
  54. data/src/test/resources/org/embulk/output/kintone/task/reduce/update_update_ignore_nulls_records.jsonl +3 -0
  55. data/src/test/resources/org/embulk/output/kintone/task/reduce/update_update_prefer_nulls_records.jsonl +3 -0
  56. data/src/test/resources/org/embulk/output/kintone/task/reduce/update_update_records.jsonl +6 -0
  57. data/src/test/resources/org/embulk/output/kintone/task/reduce/upsert_add_ignore_nulls_records.jsonl +3 -0
  58. data/src/test/resources/org/embulk/output/kintone/task/reduce/upsert_add_prefer_nulls_records.jsonl +3 -0
  59. data/src/test/resources/org/embulk/output/kintone/task/reduce/upsert_add_records.jsonl +2 -0
  60. data/src/test/resources/org/embulk/output/kintone/task/reduce/upsert_update_ignore_nulls_records.jsonl +3 -0
  61. data/src/test/resources/org/embulk/output/kintone/task/reduce/upsert_update_prefer_nulls_records.jsonl +3 -0
  62. data/src/test/resources/org/embulk/output/kintone/task/reduce/upsert_update_records.jsonl +4 -0
  63. data/src/test/resources/org/embulk/output/kintone/task/reduce/values.json +1 -0
  64. data/src/test/resources/org/embulk/output/kintone/task/reduce/values_ignore_nulls.json +1 -0
  65. data/src/test/resources/org/embulk/output/kintone/task/reduce/values_prefer_nulls.json +1 -0
  66. data/src/test/resources/org/embulk/output/kintone/task/reduce_exception/config.yml +36 -0
  67. data/src/test/resources/org/embulk/output/kintone/task/reduce_exception/derived_columns.json +1 -0
  68. data/src/test/resources/org/embulk/output/kintone/task/reduce_exception/input.csv +13 -0
  69. data/src/test/resources/org/embulk/output/kintone/task/reduce_exception/insert_add_records.jsonl +2 -0
  70. data/src/test/resources/org/embulk/output/kintone/task/reduce_exception/update_update_records.jsonl +2 -0
  71. data/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/config.yml +343 -0
  72. data/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/derived_columns.json +1 -0
  73. data/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/input.csv +13 -0
  74. data/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/insert_add_ignore_nulls_records.jsonl +6 -0
  75. data/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/insert_add_prefer_nulls_records.jsonl +6 -0
  76. data/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/insert_add_records.jsonl +6 -0
  77. data/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/update_update_ignore_nulls_records.jsonl +3 -0
  78. data/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/update_update_prefer_nulls_records.jsonl +3 -0
  79. data/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/update_update_records.jsonl +6 -0
  80. data/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/upsert_add_ignore_nulls_records.jsonl +3 -0
  81. data/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/upsert_add_prefer_nulls_records.jsonl +3 -0
  82. data/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/upsert_add_records.jsonl +0 -0
  83. data/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/upsert_update_ignore_nulls_records.jsonl +3 -0
  84. data/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/upsert_update_prefer_nulls_records.jsonl +3 -0
  85. data/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/upsert_update_records.jsonl +6 -0
  86. data/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/values.json +1 -0
  87. data/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/values_ignore_nulls.json +1 -0
  88. data/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/values_prefer_nulls.json +1 -0
  89. metadata +62 -4
  90. data/classpath/embulk-output-kintone-1.0.0.jar +0 -0
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 11c7f1cbe588863b206b1581bf2ec89adeafeea4
4
- data.tar.gz: c17ae7651f3409a6e51999e65233a96da2d87bf4
3
+ metadata.gz: 38d9d369925e0cab9fa558716b9a939be4b1387e
4
+ data.tar.gz: f4ae4a741548a48b042e1c4b79bf7e533ded0d31
5
5
  SHA512:
6
- metadata.gz: 4e1bc2fe10bc224c643d88c06a37a19bfe4849fb46318fede913ef3a3923e1315ce48b78aea8bbdcc5ea9a5b4760058776c569e8acedf8f02691b80b330c5657
7
- data.tar.gz: 9261603e49b45d2c8b72bf013d2995057293dd7f98ae0588f4dfa2f7e962b0ba2eac70990cdd7658eedf72a5caabd9cbb8f3135bc80cb91a7d250706c06081b4
6
+ metadata.gz: 06d103f7c4036dd458f62955bee9b2b82aa2bb2719cb59f18a2a762bab5c892025b4dd7fb0b41647e0e9cf66e012916f60de1a48c1a6d41414af0447d8affb06
7
+ data.tar.gz: ee848b7cfb53cc27c731724d32b6d5ffa9fe3336742fc1bbf24bd2b49d07f6f99439b478a0342051810ef94597471caded0f81caaf3aaff78203ce0ff20485d6
data/README.md CHANGED
@@ -18,14 +18,24 @@ kintone output plugin for Embulk stores app records from kintone.
18
18
  - **guest_space_id**: kintone app belongs to guest space, guest space id is required. (integer, optional)
19
19
  - **mode**: kintone mode (string, required)
20
20
  - **update_key**: Column name to set update key (string, required if mode is update or upsert)
21
+ - **reduce_key**: Key column name to reduce expanded SUBTABLE (string, optional)
22
+ - **sort_columns**: List of columns for sorting input records (array of objects, optional)
23
+ - **name**: Column name (string, required)
24
+ - **order**: Sort order (string `asc` or `desc`, required)
25
+ - **max_sort_tmp_files**: Maximum number of temporary files for sorting input records (integer, default is `1024`)
26
+ - **max_sort_memory**: Maximum memory usage for sorting input records (bytes in long, default is the estimated available memory, which is the approximate value of the JVM's current free memory)
21
27
  - **prefer_nulls**: Whether to set fields to null instead of default value of type when column is null (boolean, default is `false`)
22
28
  - **ignore_nulls**: Whether to completely ignore fields when column is null (boolean, default is `false`)
23
29
  - **column_options** advanced: a key-value pairs where key is a column name and value is options for the column.
24
30
  - **field_code**: field code (string, required)
25
31
  - **type**: field type (string, required). See [this page](https://cybozu.dev/ja/kintone/docs/overview/field-types/#field-type-update) for list of available types. However, following types are not yet supported
26
- - `USER_SELECT`, `ORGANIZATION_SELECT`, `GROUP_SELECT`, `FILE`, `SUBTABLE`
32
+ - `USER_SELECT`, `ORGANIZATION_SELECT`, `GROUP_SELECT`, `FILE`
27
33
  - **timezone**: timezone to convert into `date` (string, default is `UTC`)
28
34
  - **val_sep**: Used to specify multiple checkbox values (string, default is `,`)
35
+ - **sort_columns**: List of columns for sorting rows in SUBTABLE. Available only if type is `SUBTABLE` (array of objects, optional)
36
+ - **name**: Column name (string, required)
37
+ - **order**: Sort order (string `asc` or `desc`, required)
38
+ - **chunk_size**: Maximum number of records to request at once (integer, default is `100`)
29
39
 
30
40
  ## Example
31
41
 
@@ -46,6 +56,52 @@ out:
46
56
  date_time: {field_code: "datetime", type: "DATETIME"}
47
57
  ```
48
58
 
59
+ ### For reduce expanded SUBTABLE
60
+
61
+ ```yaml
62
+ out:
63
+ ...
64
+ reduce_key: id
65
+ column_options:
66
+ id: {field_code: "id", type: "NUMBER"}
67
+ ...
68
+ table: {field_code: "table", type: "SUBTABLE", sort_columns: [{name: number, order: asc}, {name: text, order: desc}]}
69
+ table.number: {field_code: "number_in_table", type: "NUMBER"}
70
+ table.text: {field_code: "text_in_table", type: "SINGLE_LINE_TEXT"}
71
+ ```
72
+
73
+ #### KINTONE
74
+
75
+ | Form | Field Code |
76
+ | ------------------------- | --------------- |
77
+ | Table's own | table |
78
+ | NUMBER In Table | number_in_table |
79
+ | SINGLE_LINE_TEXT In Table | text_in_table |
80
+
81
+ #### INPUT CSV
82
+
83
+ ```csv
84
+ id,table.number,table.text
85
+ 1,0,test0
86
+ 1,1,test1
87
+ 2,0,test0
88
+ ```
89
+
90
+ #### RESULT
91
+
92
+ ID:1
93
+
94
+ | NUMBER | SINGLE_LINE_TEXT |
95
+ | ------------- | ---------------- |
96
+ | 0 | test0 |
97
+ | 1 | test1 |
98
+
99
+ ID:2
100
+
101
+ | NUMBER | SINGLE_LINE_TEXT |
102
+ | ------------- | ---------------- |
103
+ | 0 | test0 |
104
+
49
105
  ## Build
50
106
 
51
107
  ```
data/build.gradle CHANGED
@@ -28,6 +28,7 @@ targetCompatibility = 1.8
28
28
 
29
29
  dependencies {
30
30
  compileOnly "org.embulk:embulk-core:0.9.23"
31
+ implementation "com.google.code.externalsortinginjava:externalsortinginjava:0.6.2"
31
32
  implementation project(path: ":shadow-kintone-java-client", configuration: "shadow")
32
33
 
33
34
  testImplementation "junit:junit:4.+"
Binary file
@@ -1,5 +1,6 @@
1
1
  package org.embulk.output.kintone;
2
2
 
3
+ import java.util.List;
3
4
  import org.embulk.config.Config;
4
5
  import org.embulk.config.ConfigDefault;
5
6
  import org.embulk.config.Task;
@@ -18,4 +19,8 @@ public interface KintoneColumnOption extends Task {
18
19
  @Config("val_sep")
19
20
  @ConfigDefault("\",\"")
20
21
  String getValueSeparator();
22
+
23
+ @Config("sort_columns")
24
+ @ConfigDefault("[]")
25
+ List<KintoneSortColumn> getSortColumns();
21
26
  }
@@ -29,8 +29,13 @@ import java.time.ZonedDateTime;
29
29
  import java.util.Arrays;
30
30
  import java.util.Collections;
31
31
  import java.util.List;
32
+ import java.util.stream.Collectors;
33
+ import org.embulk.output.kintone.deserializer.Deserializer;
32
34
  import org.embulk.spi.time.Timestamp;
35
+ import org.embulk.spi.type.Type;
36
+ import org.embulk.spi.type.Types;
33
37
  import org.msgpack.value.Value;
38
+ import org.msgpack.value.ValueFactory;
34
39
 
35
40
  public enum KintoneColumnType {
36
41
  SINGLE_LINE_TEXT {
@@ -53,6 +58,16 @@ public enum KintoneColumnType {
53
58
  public void setUpdateKey(UpdateKey updateKey, String field, FieldValue value) {
54
59
  updateKey.setField(field).setValue(((SingleLineTextFieldValue) value).getValue());
55
60
  }
61
+
62
+ @Override
63
+ public Value asValue(FieldValue value) {
64
+ return ValueFactory.newString(((SingleLineTextFieldValue) value).getValue());
65
+ }
66
+
67
+ @Override
68
+ protected List<Type> getSupportedTypes() {
69
+ return Arrays.asList(Types.BOOLEAN, Types.LONG, Types.DOUBLE, Types.TIMESTAMP, Types.JSON);
70
+ }
56
71
  },
57
72
  MULTI_LINE_TEXT {
58
73
  @Override
@@ -64,6 +79,16 @@ public enum KintoneColumnType {
64
79
  public MultiLineTextFieldValue getFieldValue(String value, KintoneColumnOption option) {
65
80
  return new MultiLineTextFieldValue(value);
66
81
  }
82
+
83
+ @Override
84
+ public Value asValue(FieldValue value) {
85
+ return ValueFactory.newString(((MultiLineTextFieldValue) value).getValue());
86
+ }
87
+
88
+ @Override
89
+ protected List<Type> getSupportedTypes() {
90
+ return Arrays.asList(Types.BOOLEAN, Types.LONG, Types.DOUBLE, Types.TIMESTAMP, Types.JSON);
91
+ }
67
92
  },
68
93
  RICH_TEXT {
69
94
  @Override
@@ -75,6 +100,16 @@ public enum KintoneColumnType {
75
100
  public RichTextFieldValue getFieldValue(String value, KintoneColumnOption option) {
76
101
  return new RichTextFieldValue(value);
77
102
  }
103
+
104
+ @Override
105
+ public Value asValue(FieldValue value) {
106
+ return ValueFactory.newString(((RichTextFieldValue) value).getValue());
107
+ }
108
+
109
+ @Override
110
+ protected List<Type> getSupportedTypes() {
111
+ return Arrays.asList(Types.BOOLEAN, Types.LONG, Types.DOUBLE, Types.TIMESTAMP, Types.JSON);
112
+ }
78
113
  },
79
114
  NUMBER {
80
115
  @Override
@@ -106,6 +141,16 @@ public enum KintoneColumnType {
106
141
  public void setUpdateKey(UpdateKey updateKey, String field, FieldValue value) {
107
142
  updateKey.setField(field).setValue(((NumberFieldValue) value).getValue());
108
143
  }
144
+
145
+ @Override
146
+ public Value asValue(FieldValue value) {
147
+ return ValueFactory.newString(((NumberFieldValue) value).getValue().toString());
148
+ }
149
+
150
+ @Override
151
+ protected List<Type> getSupportedTypes() {
152
+ return Arrays.asList(Types.BOOLEAN, Types.LONG, Types.DOUBLE, Types.TIMESTAMP);
153
+ }
109
154
  },
110
155
  CHECK_BOX {
111
156
  @Override
@@ -117,6 +162,18 @@ public enum KintoneColumnType {
117
162
  public CheckBoxFieldValue getFieldValue(String value, KintoneColumnOption option) {
118
163
  return new CheckBoxFieldValue(asList(value, option));
119
164
  }
165
+
166
+ @Override
167
+ public Value asValue(FieldValue value) {
168
+ return ValueFactory.newArray(
169
+ ((CheckBoxFieldValue) value)
170
+ .getValues().stream().map(ValueFactory::newString).collect(Collectors.toList()));
171
+ }
172
+
173
+ @Override
174
+ protected List<Type> getSupportedTypes() {
175
+ return Collections.emptyList();
176
+ }
120
177
  },
121
178
  RADIO_BUTTON {
122
179
  @Override
@@ -128,6 +185,16 @@ public enum KintoneColumnType {
128
185
  public RadioButtonFieldValue getFieldValue(String value, KintoneColumnOption option) {
129
186
  return new RadioButtonFieldValue(value);
130
187
  }
188
+
189
+ @Override
190
+ public Value asValue(FieldValue value) {
191
+ return ValueFactory.newString(((RadioButtonFieldValue) value).getValue());
192
+ }
193
+
194
+ @Override
195
+ protected List<Type> getSupportedTypes() {
196
+ return Collections.emptyList();
197
+ }
131
198
  },
132
199
  MULTI_SELECT {
133
200
  @Override
@@ -139,6 +206,18 @@ public enum KintoneColumnType {
139
206
  public MultiSelectFieldValue getFieldValue(String value, KintoneColumnOption option) {
140
207
  return new MultiSelectFieldValue(asList(value, option));
141
208
  }
209
+
210
+ @Override
211
+ public Value asValue(FieldValue value) {
212
+ return ValueFactory.newArray(
213
+ ((MultiSelectFieldValue) value)
214
+ .getValues().stream().map(ValueFactory::newString).collect(Collectors.toList()));
215
+ }
216
+
217
+ @Override
218
+ protected List<Type> getSupportedTypes() {
219
+ return Collections.emptyList();
220
+ }
142
221
  },
143
222
  DROP_DOWN {
144
223
  @Override
@@ -150,6 +229,16 @@ public enum KintoneColumnType {
150
229
  public DropDownFieldValue getFieldValue(String value, KintoneColumnOption option) {
151
230
  return new DropDownFieldValue(value);
152
231
  }
232
+
233
+ @Override
234
+ public Value asValue(FieldValue value) {
235
+ return ValueFactory.newString(((DropDownFieldValue) value).getValue());
236
+ }
237
+
238
+ @Override
239
+ protected List<Type> getSupportedTypes() {
240
+ return Collections.emptyList();
241
+ }
153
242
  },
154
243
  USER_SELECT {
155
244
  @Override
@@ -161,6 +250,16 @@ public enum KintoneColumnType {
161
250
  public UserSelectFieldValue getFieldValue(String value, KintoneColumnOption option) {
162
251
  throw new UnsupportedOperationException();
163
252
  }
253
+
254
+ @Override
255
+ public Value asValue(FieldValue value) {
256
+ throw new UnsupportedOperationException();
257
+ }
258
+
259
+ @Override
260
+ protected List<Type> getSupportedTypes() {
261
+ return Collections.emptyList();
262
+ }
164
263
  },
165
264
  ORGANIZATION_SELECT {
166
265
  @Override
@@ -172,6 +271,16 @@ public enum KintoneColumnType {
172
271
  public OrganizationSelectFieldValue getFieldValue(String value, KintoneColumnOption option) {
173
272
  throw new UnsupportedOperationException();
174
273
  }
274
+
275
+ @Override
276
+ public Value asValue(FieldValue value) {
277
+ throw new UnsupportedOperationException();
278
+ }
279
+
280
+ @Override
281
+ protected List<Type> getSupportedTypes() {
282
+ return Collections.emptyList();
283
+ }
175
284
  },
176
285
  GROUP_SELECT {
177
286
  @Override
@@ -183,6 +292,16 @@ public enum KintoneColumnType {
183
292
  public GroupSelectFieldValue getFieldValue(String value, KintoneColumnOption option) {
184
293
  throw new UnsupportedOperationException();
185
294
  }
295
+
296
+ @Override
297
+ public Value asValue(FieldValue value) {
298
+ throw new UnsupportedOperationException();
299
+ }
300
+
301
+ @Override
302
+ protected List<Type> getSupportedTypes() {
303
+ return Collections.emptyList();
304
+ }
186
305
  },
187
306
  DATE {
188
307
  @Override
@@ -215,6 +334,16 @@ public enum KintoneColumnType {
215
334
  public DateFieldValue getFieldValue(Timestamp value, KintoneColumnOption option) {
216
335
  return new DateFieldValue(value.getInstant().atZone(getZoneId(option)).toLocalDate());
217
336
  }
337
+
338
+ @Override
339
+ public Value asValue(FieldValue value) {
340
+ return ValueFactory.newString(((DateFieldValue) value).getValue().toString());
341
+ }
342
+
343
+ @Override
344
+ protected List<Type> getSupportedTypes() {
345
+ return Arrays.asList(Types.LONG, Types.DOUBLE, Types.TIMESTAMP);
346
+ }
218
347
  },
219
348
  TIME {
220
349
  @Override
@@ -247,6 +376,16 @@ public enum KintoneColumnType {
247
376
  public TimeFieldValue getFieldValue(Timestamp value, KintoneColumnOption option) {
248
377
  return new TimeFieldValue(value.getInstant().atZone(getZoneId(option)).toLocalTime());
249
378
  }
379
+
380
+ @Override
381
+ public Value asValue(FieldValue value) {
382
+ return ValueFactory.newString(((TimeFieldValue) value).getValue().toString());
383
+ }
384
+
385
+ @Override
386
+ protected List<Type> getSupportedTypes() {
387
+ return Arrays.asList(Types.LONG, Types.DOUBLE, Types.TIMESTAMP);
388
+ }
250
389
  },
251
390
  DATETIME {
252
391
  @Override
@@ -275,6 +414,16 @@ public enum KintoneColumnType {
275
414
  public DateTimeFieldValue getFieldValue(Timestamp value, KintoneColumnOption option) {
276
415
  return new DateTimeFieldValue(value.getInstant().atZone(ZoneOffset.UTC));
277
416
  }
417
+
418
+ @Override
419
+ public Value asValue(FieldValue value) {
420
+ return ValueFactory.newString(((DateTimeFieldValue) value).getValue().toString());
421
+ }
422
+
423
+ @Override
424
+ protected List<Type> getSupportedTypes() {
425
+ return Arrays.asList(Types.LONG, Types.DOUBLE, Types.TIMESTAMP);
426
+ }
278
427
  },
279
428
  LINK {
280
429
  @Override
@@ -286,6 +435,16 @@ public enum KintoneColumnType {
286
435
  public LinkFieldValue getFieldValue(String value, KintoneColumnOption option) {
287
436
  return new LinkFieldValue(value);
288
437
  }
438
+
439
+ @Override
440
+ public Value asValue(FieldValue value) {
441
+ return ValueFactory.newString(((LinkFieldValue) value).getValue());
442
+ }
443
+
444
+ @Override
445
+ protected List<Type> getSupportedTypes() {
446
+ return Collections.emptyList();
447
+ }
289
448
  },
290
449
  FILE {
291
450
  @Override
@@ -297,6 +456,16 @@ public enum KintoneColumnType {
297
456
  public FileFieldValue getFieldValue(String value, KintoneColumnOption option) {
298
457
  throw new UnsupportedOperationException();
299
458
  }
459
+
460
+ @Override
461
+ public Value asValue(FieldValue value) {
462
+ throw new UnsupportedOperationException();
463
+ }
464
+
465
+ @Override
466
+ protected List<Type> getSupportedTypes() {
467
+ return Collections.emptyList();
468
+ }
300
469
  },
301
470
  SUBTABLE {
302
471
  @Override
@@ -306,9 +475,20 @@ public enum KintoneColumnType {
306
475
 
307
476
  @Override
308
477
  public SubtableFieldValue getFieldValue(String value, KintoneColumnOption option) {
478
+ return DESERIALIZER.deserialize(value.isEmpty() ? "[]" : value, SubtableFieldValue.class);
479
+ }
480
+
481
+ @Override
482
+ public Value asValue(FieldValue value) {
309
483
  throw new UnsupportedOperationException();
310
484
  }
485
+
486
+ @Override
487
+ protected List<Type> getSupportedTypes() {
488
+ return Collections.singletonList(Types.JSON);
489
+ }
311
490
  };
491
+ private static final Deserializer DESERIALIZER = new Deserializer();
312
492
  private static final Timestamp EPOCH = Timestamp.ofInstant(Instant.EPOCH);
313
493
 
314
494
  public static KintoneColumnType getType(
@@ -319,25 +499,45 @@ public enum KintoneColumnType {
319
499
  public abstract FieldValue getFieldValue();
320
500
 
321
501
  public FieldValue getFieldValue(boolean value, KintoneColumnOption option) {
322
- return getFieldValue(String.valueOf(value), option);
502
+ if (getSupportedTypes().contains(Types.BOOLEAN)) {
503
+ return getFieldValue(String.valueOf(value), option);
504
+ } else {
505
+ throw new UnsupportedOperationException();
506
+ }
323
507
  }
324
508
 
325
509
  public FieldValue getFieldValue(long value, KintoneColumnOption option) {
326
- return getFieldValue(String.valueOf(value), option);
510
+ if (getSupportedTypes().contains(Types.LONG)) {
511
+ return getFieldValue(String.valueOf(value), option);
512
+ } else {
513
+ throw new UnsupportedOperationException();
514
+ }
327
515
  }
328
516
 
329
517
  public FieldValue getFieldValue(double value, KintoneColumnOption option) {
330
- return getFieldValue(String.valueOf(value), option);
518
+ if (getSupportedTypes().contains(Types.DOUBLE)) {
519
+ return getFieldValue(String.valueOf(value), option);
520
+ } else {
521
+ throw new UnsupportedOperationException();
522
+ }
331
523
  }
332
524
 
333
525
  public abstract FieldValue getFieldValue(String value, KintoneColumnOption option);
334
526
 
335
527
  public FieldValue getFieldValue(Timestamp value, KintoneColumnOption option) {
336
- return getFieldValue(value.getInstant().toString(), option);
528
+ if (getSupportedTypes().contains(Types.TIMESTAMP)) {
529
+ return getFieldValue(value.getInstant().toString(), option);
530
+ } else {
531
+ throw new UnsupportedOperationException();
532
+ }
337
533
  }
338
534
 
339
535
  public FieldValue getFieldValue(Value value, KintoneColumnOption option) {
340
- return getFieldValue(value.toJson(), option);
536
+ if (getSupportedTypes().contains(Types.JSON)) {
537
+ return getFieldValue(value.toJson(), option);
538
+ } else {
539
+ throw new UnsupportedOperationException();
540
+ }
341
541
  }
342
542
 
343
543
  public void setUpdateKey(UpdateKey updateKey, String field) {
@@ -348,6 +548,10 @@ public enum KintoneColumnType {
348
548
  throw new UnsupportedOperationException();
349
549
  }
350
550
 
551
+ public abstract Value asValue(FieldValue value);
552
+
553
+ protected abstract List<Type> getSupportedTypes();
554
+
351
555
  private static List<String> asList(String value, KintoneColumnOption option) {
352
556
  return value.isEmpty()
353
557
  ? Collections.emptyList()
@@ -5,6 +5,7 @@ import com.kintone.client.model.record.Record;
5
5
  import com.kintone.client.model.record.UpdateKey;
6
6
  import java.time.Instant;
7
7
  import java.util.Map;
8
+ import java.util.Set;
8
9
  import org.embulk.spi.Column;
9
10
  import org.embulk.spi.ColumnVisitor;
10
11
  import org.embulk.spi.PageReader;
@@ -14,31 +15,39 @@ import org.msgpack.value.ValueFactory;
14
15
 
15
16
  public class KintoneColumnVisitor implements ColumnVisitor {
16
17
  private final PageReader reader;
18
+ private final Set<Column> derived;
17
19
  private final Map<String, KintoneColumnOption> options;
18
20
  private final boolean preferNulls;
19
21
  private final boolean ignoreNulls;
22
+ private final String reduceKeyName;
20
23
  private final String updateKeyName;
21
24
  private Record record;
22
25
  private UpdateKey updateKey;
23
26
 
24
27
  public KintoneColumnVisitor(
25
28
  PageReader reader,
29
+ Set<Column> derived,
26
30
  Map<String, KintoneColumnOption> options,
27
31
  boolean preferNulls,
28
- boolean ignoreNulls) {
29
- this(reader, options, preferNulls, ignoreNulls, null);
32
+ boolean ignoreNulls,
33
+ String reduceKeyName) {
34
+ this(reader, derived, options, preferNulls, ignoreNulls, reduceKeyName, null);
30
35
  }
31
36
 
32
37
  public KintoneColumnVisitor(
33
38
  PageReader reader,
39
+ Set<Column> derived,
34
40
  Map<String, KintoneColumnOption> options,
35
41
  boolean preferNulls,
36
42
  boolean ignoreNulls,
43
+ String reduceKeyName,
37
44
  String updateKeyName) {
38
45
  this.reader = reader;
46
+ this.derived = derived;
39
47
  this.options = options;
40
48
  this.preferNulls = preferNulls;
41
49
  this.ignoreNulls = ignoreNulls;
50
+ this.reduceKeyName = reduceKeyName;
42
51
  this.updateKeyName = updateKeyName;
43
52
  }
44
53
 
@@ -52,7 +61,7 @@ public class KintoneColumnVisitor implements ColumnVisitor {
52
61
 
53
62
  @Override
54
63
  public void booleanColumn(Column column) {
55
- if (isIgnoreNull(column)) {
64
+ if (isReduced(column) || isIgnoreNull(column)) {
56
65
  return;
57
66
  }
58
67
  KintoneColumnOption option = getOption(column);
@@ -70,7 +79,7 @@ public class KintoneColumnVisitor implements ColumnVisitor {
70
79
 
71
80
  @Override
72
81
  public void longColumn(Column column) {
73
- if (isIgnoreNull(column)) {
82
+ if (isReduced(column) || isIgnoreNull(column)) {
74
83
  return;
75
84
  }
76
85
  KintoneColumnOption option = getOption(column);
@@ -88,7 +97,7 @@ public class KintoneColumnVisitor implements ColumnVisitor {
88
97
 
89
98
  @Override
90
99
  public void doubleColumn(Column column) {
91
- if (isIgnoreNull(column)) {
100
+ if (isReduced(column) || isIgnoreNull(column)) {
92
101
  return;
93
102
  }
94
103
  KintoneColumnOption option = getOption(column);
@@ -106,7 +115,7 @@ public class KintoneColumnVisitor implements ColumnVisitor {
106
115
 
107
116
  @Override
108
117
  public void stringColumn(Column column) {
109
- if (isIgnoreNull(column)) {
118
+ if (isReduced(column) || isIgnoreNull(column)) {
110
119
  return;
111
120
  }
112
121
  KintoneColumnOption option = getOption(column);
@@ -126,7 +135,7 @@ public class KintoneColumnVisitor implements ColumnVisitor {
126
135
 
127
136
  @Override
128
137
  public void timestampColumn(Column column) {
129
- if (isIgnoreNull(column)) {
138
+ if (isReduced(column) || isIgnoreNull(column)) {
130
139
  return;
131
140
  }
132
141
  KintoneColumnOption option = getOption(column);
@@ -144,12 +153,14 @@ public class KintoneColumnVisitor implements ColumnVisitor {
144
153
 
145
154
  @Override
146
155
  public void jsonColumn(Column column) {
147
- if (isIgnoreNull(column)) {
156
+ if (isReduced(column) || isIgnoreNull(column)) {
148
157
  return;
149
158
  }
150
159
  KintoneColumnOption option = getOption(column);
151
160
  UpdateKey updateKey = getUpdateKey(column);
152
- KintoneColumnType type = KintoneColumnType.getType(option, KintoneColumnType.MULTI_LINE_TEXT);
161
+ KintoneColumnType defaultType =
162
+ isDerived(column) ? KintoneColumnType.SUBTABLE : KintoneColumnType.MULTI_LINE_TEXT;
163
+ KintoneColumnType type = KintoneColumnType.getType(option, defaultType);
153
164
  String fieldCode = getFieldCode(column);
154
165
  if (isPreferNull(column)) {
155
166
  setNull(type, fieldCode, updateKey);
@@ -258,6 +269,14 @@ public class KintoneColumnVisitor implements ColumnVisitor {
258
269
  return updateKeyName != null && updateKeyName.equals(column.getName()) ? updateKey : null;
259
270
  }
260
271
 
272
+ private boolean isReduced(Column column) {
273
+ return reduceKeyName != null && column.getName().matches("^.*\\..*$");
274
+ }
275
+
276
+ private boolean isDerived(Column column) {
277
+ return reduceKeyName != null && derived.contains(column);
278
+ }
279
+
261
280
  private boolean isIgnoreNull(Column column) {
262
281
  return ignoreNulls && reader.isNull(column);
263
282
  }
@@ -1,11 +1,14 @@
1
1
  package org.embulk.output.kintone;
2
2
 
3
+ import java.util.Collections;
3
4
  import java.util.List;
4
5
  import org.embulk.config.ConfigDiff;
5
6
  import org.embulk.config.ConfigException;
6
7
  import org.embulk.config.ConfigSource;
7
8
  import org.embulk.config.TaskReport;
8
9
  import org.embulk.config.TaskSource;
10
+ import org.embulk.output.kintone.reducer.ReducedPageOutput;
11
+ import org.embulk.output.kintone.reducer.Reducer;
9
12
  import org.embulk.spi.Exec;
10
13
  import org.embulk.spi.OutputPlugin;
11
14
  import org.embulk.spi.Schema;
@@ -16,8 +19,12 @@ public class KintoneOutputPlugin implements OutputPlugin {
16
19
  public ConfigDiff transaction(
17
20
  ConfigSource config, Schema schema, int taskCount, OutputPlugin.Control control) {
18
21
  PluginTask task = config.loadConfig(PluginTask.class);
19
- control.run(task.dump());
20
- return Exec.newConfigDiff();
22
+ task.setDerivedColumns(Collections.emptySet());
23
+ List<TaskReport> taskReports = control.run(task.dump());
24
+ return task.getReduceKeyName().isPresent()
25
+ ? new Reducer(task, schema)
26
+ .reduce(taskReports, schema.lookupColumn(task.getReduceKeyName().get()))
27
+ : Exec.newConfigDiff();
21
28
  }
22
29
 
23
30
  @Override
@@ -49,6 +56,8 @@ public class KintoneOutputPlugin implements OutputPlugin {
49
56
  default:
50
57
  throw new ConfigException(String.format("Unknown mode '%s'", task.getMode()));
51
58
  }
52
- return new KintonePageOutput(task, schema);
59
+ return task.getReduceKeyName().isPresent()
60
+ ? new ReducedPageOutput(schema, taskIndex)
61
+ : new KintonePageOutput(task, schema);
53
62
  }
54
63
  }