embulk-output-kintone 1.0.0 → 1.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +57 -1
- data/build.gradle +1 -0
- data/classpath/commons-csv-1.9.0.jar +0 -0
- data/classpath/embulk-output-kintone-1.1.0.jar +0 -0
- data/classpath/externalsortinginjava-0.6.2.jar +0 -0
- data/classpath/{shadow-kintone-java-client-1.0.0-all.jar → shadow-kintone-java-client-1.1.0-all.jar} +0 -0
- data/src/main/java/org/embulk/output/kintone/KintoneColumnOption.java +5 -0
- data/src/main/java/org/embulk/output/kintone/KintoneColumnType.java +209 -5
- data/src/main/java/org/embulk/output/kintone/KintoneColumnVisitor.java +28 -9
- data/src/main/java/org/embulk/output/kintone/KintoneOutputPlugin.java +12 -3
- data/src/main/java/org/embulk/output/kintone/KintonePageOutput.java +20 -12
- data/src/main/java/org/embulk/output/kintone/KintoneSortColumn.java +33 -0
- data/src/main/java/org/embulk/output/kintone/PluginTask.java +27 -0
- data/src/main/java/org/embulk/output/kintone/deserializer/DeserializeApplier.java +19 -0
- data/src/main/java/org/embulk/output/kintone/deserializer/DeserializeException.java +7 -0
- data/src/main/java/org/embulk/output/kintone/deserializer/Deserializer.java +279 -0
- data/src/main/java/org/embulk/output/kintone/reducer/CSVInputColumnVisitor.java +78 -0
- data/src/main/java/org/embulk/output/kintone/reducer/CSVOutputColumnVisitor.java +79 -0
- data/src/main/java/org/embulk/output/kintone/reducer/ReduceException.java +11 -0
- data/src/main/java/org/embulk/output/kintone/reducer/ReduceType.java +190 -0
- data/src/main/java/org/embulk/output/kintone/reducer/ReducedPageOutput.java +100 -0
- data/src/main/java/org/embulk/output/kintone/reducer/Reducer.java +355 -0
- data/src/test/java/org/embulk/output/kintone/KintoneColumnOptionBuilder.java +7 -0
- data/src/test/java/org/embulk/output/kintone/KintoneColumnTypeTest.java +194 -0
- data/src/test/java/org/embulk/output/kintone/KintoneColumnVisitorTest.java +153 -34
- data/src/test/java/org/embulk/output/kintone/KintoneColumnVisitorVerifier.java +13 -3
- data/src/test/java/org/embulk/output/kintone/KintonePageOutputVerifier.java +44 -1
- data/src/test/java/org/embulk/output/kintone/TestKintoneOutputPlugin.java +89 -12
- data/src/test/java/org/embulk/output/kintone/TestTaskReduce.java +46 -0
- data/src/test/java/org/embulk/output/kintone/TestTaskReduceException.java +50 -0
- data/src/test/java/org/embulk/output/kintone/TestTaskReduceSubtable.java +46 -0
- data/src/test/java/org/embulk/output/kintone/deserializer/DeserializerTest.java +165 -0
- data/src/test/java/org/embulk/output/kintone/reducer/ReduceTypeTest.java +154 -0
- data/src/test/resources/org/embulk/output/kintone/task/config.yml +1 -1
- data/src/test/resources/org/embulk/output/kintone/task/mode/config.yml +6 -0
- data/src/test/resources/org/embulk/output/kintone/task/mode/input.csv +7 -7
- data/src/test/resources/org/embulk/output/kintone/task/mode/insert_add_ignore_nulls_records.jsonl +2 -2
- data/src/test/resources/org/embulk/output/kintone/task/mode/insert_add_prefer_nulls_records.jsonl +6 -6
- data/src/test/resources/org/embulk/output/kintone/task/mode/insert_add_records.jsonl +6 -6
- data/src/test/resources/org/embulk/output/kintone/task/mode/update_update_ignore_nulls_records.jsonl +2 -2
- data/src/test/resources/org/embulk/output/kintone/task/mode/update_update_prefer_nulls_records.jsonl +3 -3
- data/src/test/resources/org/embulk/output/kintone/task/mode/update_update_records.jsonl +6 -6
- data/src/test/resources/org/embulk/output/kintone/task/mode/upsert_add_prefer_nulls_records.jsonl +3 -3
- data/src/test/resources/org/embulk/output/kintone/task/mode/upsert_add_records.jsonl +2 -2
- data/src/test/resources/org/embulk/output/kintone/task/mode/upsert_update_ignore_nulls_records.jsonl +2 -2
- data/src/test/resources/org/embulk/output/kintone/task/mode/upsert_update_prefer_nulls_records.jsonl +3 -3
- data/src/test/resources/org/embulk/output/kintone/task/mode/upsert_update_records.jsonl +4 -4
- data/src/test/resources/org/embulk/output/kintone/task/reduce/config.yml +171 -0
- data/src/test/resources/org/embulk/output/kintone/task/reduce/input.csv +7 -0
- data/src/test/resources/org/embulk/output/kintone/task/reduce/insert_add_ignore_nulls_records.jsonl +6 -0
- data/src/test/resources/org/embulk/output/kintone/task/reduce/insert_add_prefer_nulls_records.jsonl +6 -0
- data/src/test/resources/org/embulk/output/kintone/task/reduce/insert_add_records.jsonl +6 -0
- data/src/test/resources/org/embulk/output/kintone/task/reduce/update_update_ignore_nulls_records.jsonl +3 -0
- data/src/test/resources/org/embulk/output/kintone/task/reduce/update_update_prefer_nulls_records.jsonl +3 -0
- data/src/test/resources/org/embulk/output/kintone/task/reduce/update_update_records.jsonl +6 -0
- data/src/test/resources/org/embulk/output/kintone/task/reduce/upsert_add_ignore_nulls_records.jsonl +3 -0
- data/src/test/resources/org/embulk/output/kintone/task/reduce/upsert_add_prefer_nulls_records.jsonl +3 -0
- data/src/test/resources/org/embulk/output/kintone/task/reduce/upsert_add_records.jsonl +2 -0
- data/src/test/resources/org/embulk/output/kintone/task/reduce/upsert_update_ignore_nulls_records.jsonl +3 -0
- data/src/test/resources/org/embulk/output/kintone/task/reduce/upsert_update_prefer_nulls_records.jsonl +3 -0
- data/src/test/resources/org/embulk/output/kintone/task/reduce/upsert_update_records.jsonl +4 -0
- data/src/test/resources/org/embulk/output/kintone/task/reduce/values.json +1 -0
- data/src/test/resources/org/embulk/output/kintone/task/reduce/values_ignore_nulls.json +1 -0
- data/src/test/resources/org/embulk/output/kintone/task/reduce/values_prefer_nulls.json +1 -0
- data/src/test/resources/org/embulk/output/kintone/task/reduce_exception/config.yml +36 -0
- data/src/test/resources/org/embulk/output/kintone/task/reduce_exception/derived_columns.json +1 -0
- data/src/test/resources/org/embulk/output/kintone/task/reduce_exception/input.csv +13 -0
- data/src/test/resources/org/embulk/output/kintone/task/reduce_exception/insert_add_records.jsonl +2 -0
- data/src/test/resources/org/embulk/output/kintone/task/reduce_exception/update_update_records.jsonl +2 -0
- data/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/config.yml +343 -0
- data/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/derived_columns.json +1 -0
- data/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/input.csv +13 -0
- data/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/insert_add_ignore_nulls_records.jsonl +6 -0
- data/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/insert_add_prefer_nulls_records.jsonl +6 -0
- data/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/insert_add_records.jsonl +6 -0
- data/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/update_update_ignore_nulls_records.jsonl +3 -0
- data/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/update_update_prefer_nulls_records.jsonl +3 -0
- data/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/update_update_records.jsonl +6 -0
- data/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/upsert_add_ignore_nulls_records.jsonl +3 -0
- data/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/upsert_add_prefer_nulls_records.jsonl +3 -0
- data/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/upsert_add_records.jsonl +0 -0
- data/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/upsert_update_ignore_nulls_records.jsonl +3 -0
- data/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/upsert_update_prefer_nulls_records.jsonl +3 -0
- data/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/upsert_update_records.jsonl +6 -0
- data/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/values.json +1 -0
- data/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/values_ignore_nulls.json +1 -0
- data/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/values_prefer_nulls.json +1 -0
- metadata +62 -4
- data/classpath/embulk-output-kintone-1.0.0.jar +0 -0
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 38d9d369925e0cab9fa558716b9a939be4b1387e
|
4
|
+
data.tar.gz: f4ae4a741548a48b042e1c4b79bf7e533ded0d31
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 06d103f7c4036dd458f62955bee9b2b82aa2bb2719cb59f18a2a762bab5c892025b4dd7fb0b41647e0e9cf66e012916f60de1a48c1a6d41414af0447d8affb06
|
7
|
+
data.tar.gz: ee848b7cfb53cc27c731724d32b6d5ffa9fe3336742fc1bbf24bd2b49d07f6f99439b478a0342051810ef94597471caded0f81caaf3aaff78203ce0ff20485d6
|
data/README.md
CHANGED
@@ -18,14 +18,24 @@ kintone output plugin for Embulk stores app records from kintone.
|
|
18
18
|
- **guest_space_id**: kintone app belongs to guest space, guest space id is required. (integer, optional)
|
19
19
|
- **mode**: kintone mode (string, required)
|
20
20
|
- **update_key**: Column name to set update key (string, required if mode is update or upsert)
|
21
|
+
- **reduce_key**: Key column name to reduce expanded SUBTABLE (string, optional)
|
22
|
+
- **sort_columns**: List of columns for sorting input records (array of objects, optional)
|
23
|
+
- **name**: Column name (string, required)
|
24
|
+
- **order**: Sort order (string `asc` or `desc`, required)
|
25
|
+
- **max_sort_tmp_files**: Maximum number of temporary files for sorting input records (integer, default is `1024`)
|
26
|
+
- **max_sort_memory**: Maximum memory usage for sorting input records (bytes in long, default is the estimated available memory, which is the approximate value of the JVM's current free memory)
|
21
27
|
- **prefer_nulls**: Whether to set fields to null instead of default value of type when column is null (boolean, default is `false`)
|
22
28
|
- **ignore_nulls**: Whether to completely ignore fields when column is null (boolean, default is `false`)
|
23
29
|
- **column_options** advanced: a key-value pairs where key is a column name and value is options for the column.
|
24
30
|
- **field_code**: field code (string, required)
|
25
31
|
- **type**: field type (string, required). See [this page](https://cybozu.dev/ja/kintone/docs/overview/field-types/#field-type-update) for list of available types. However, following types are not yet supported
|
26
|
-
- `USER_SELECT`, `ORGANIZATION_SELECT`, `GROUP_SELECT`, `FILE
|
32
|
+
- `USER_SELECT`, `ORGANIZATION_SELECT`, `GROUP_SELECT`, `FILE`
|
27
33
|
- **timezone**: timezone to convert into `date` (string, default is `UTC`)
|
28
34
|
- **val_sep**: Used to specify multiple checkbox values (string, default is `,`)
|
35
|
+
- **sort_columns**: List of columns for sorting rows in SUBTABLE. Available only if type is `SUBTABLE` (array of objects, optional)
|
36
|
+
- **name**: Column name (string, required)
|
37
|
+
- **order**: Sort order (string `asc` or `desc`, required)
|
38
|
+
- **chunk_size**: Maximum number of records to request at once (integer, default is `100`)
|
29
39
|
|
30
40
|
## Example
|
31
41
|
|
@@ -46,6 +56,52 @@ out:
|
|
46
56
|
date_time: {field_code: "datetime", type: "DATETIME"}
|
47
57
|
```
|
48
58
|
|
59
|
+
### For reduce expanded SUBTABLE
|
60
|
+
|
61
|
+
```yaml
|
62
|
+
out:
|
63
|
+
...
|
64
|
+
reduce_key: id
|
65
|
+
column_options:
|
66
|
+
id: {field_code: "id", type: "NUMBER"}
|
67
|
+
...
|
68
|
+
table: {field_code: "table", type: "SUBTABLE", sort_columns: [{name: number, order: asc}, {name: text, order: desc}]}
|
69
|
+
table.number: {field_code: "number_in_table", type: "NUMBER"}
|
70
|
+
table.text: {field_code: "text_in_table", type: "SINGLE_LINE_TEXT"}
|
71
|
+
```
|
72
|
+
|
73
|
+
#### KINTONE
|
74
|
+
|
75
|
+
| Form | Field Code |
|
76
|
+
| ------------------------- | --------------- |
|
77
|
+
| Table's own | table |
|
78
|
+
| NUMBER In Table | number_in_table |
|
79
|
+
| SINGLE_LINE_TEXT In Table | text_in_table |
|
80
|
+
|
81
|
+
#### INPUT CSV
|
82
|
+
|
83
|
+
```csv
|
84
|
+
id,table.number,table.text
|
85
|
+
1,0,test0
|
86
|
+
1,1,test1
|
87
|
+
2,0,test0
|
88
|
+
```
|
89
|
+
|
90
|
+
#### RESULT
|
91
|
+
|
92
|
+
ID:1
|
93
|
+
|
94
|
+
| NUMBER | SINGLE_LINE_TEXT |
|
95
|
+
| ------------- | ---------------- |
|
96
|
+
| 0 | test0 |
|
97
|
+
| 1 | test1 |
|
98
|
+
|
99
|
+
ID:2
|
100
|
+
|
101
|
+
| NUMBER | SINGLE_LINE_TEXT |
|
102
|
+
| ------------- | ---------------- |
|
103
|
+
| 0 | test0 |
|
104
|
+
|
49
105
|
## Build
|
50
106
|
|
51
107
|
```
|
data/build.gradle
CHANGED
@@ -28,6 +28,7 @@ targetCompatibility = 1.8
|
|
28
28
|
|
29
29
|
dependencies {
|
30
30
|
compileOnly "org.embulk:embulk-core:0.9.23"
|
31
|
+
implementation "com.google.code.externalsortinginjava:externalsortinginjava:0.6.2"
|
31
32
|
implementation project(path: ":shadow-kintone-java-client", configuration: "shadow")
|
32
33
|
|
33
34
|
testImplementation "junit:junit:4.+"
|
Binary file
|
Binary file
|
Binary file
|
data/classpath/{shadow-kintone-java-client-1.0.0-all.jar → shadow-kintone-java-client-1.1.0-all.jar}
RENAMED
Binary file
|
@@ -1,5 +1,6 @@
|
|
1
1
|
package org.embulk.output.kintone;
|
2
2
|
|
3
|
+
import java.util.List;
|
3
4
|
import org.embulk.config.Config;
|
4
5
|
import org.embulk.config.ConfigDefault;
|
5
6
|
import org.embulk.config.Task;
|
@@ -18,4 +19,8 @@ public interface KintoneColumnOption extends Task {
|
|
18
19
|
@Config("val_sep")
|
19
20
|
@ConfigDefault("\",\"")
|
20
21
|
String getValueSeparator();
|
22
|
+
|
23
|
+
@Config("sort_columns")
|
24
|
+
@ConfigDefault("[]")
|
25
|
+
List<KintoneSortColumn> getSortColumns();
|
21
26
|
}
|
@@ -29,8 +29,13 @@ import java.time.ZonedDateTime;
|
|
29
29
|
import java.util.Arrays;
|
30
30
|
import java.util.Collections;
|
31
31
|
import java.util.List;
|
32
|
+
import java.util.stream.Collectors;
|
33
|
+
import org.embulk.output.kintone.deserializer.Deserializer;
|
32
34
|
import org.embulk.spi.time.Timestamp;
|
35
|
+
import org.embulk.spi.type.Type;
|
36
|
+
import org.embulk.spi.type.Types;
|
33
37
|
import org.msgpack.value.Value;
|
38
|
+
import org.msgpack.value.ValueFactory;
|
34
39
|
|
35
40
|
public enum KintoneColumnType {
|
36
41
|
SINGLE_LINE_TEXT {
|
@@ -53,6 +58,16 @@ public enum KintoneColumnType {
|
|
53
58
|
public void setUpdateKey(UpdateKey updateKey, String field, FieldValue value) {
|
54
59
|
updateKey.setField(field).setValue(((SingleLineTextFieldValue) value).getValue());
|
55
60
|
}
|
61
|
+
|
62
|
+
@Override
|
63
|
+
public Value asValue(FieldValue value) {
|
64
|
+
return ValueFactory.newString(((SingleLineTextFieldValue) value).getValue());
|
65
|
+
}
|
66
|
+
|
67
|
+
@Override
|
68
|
+
protected List<Type> getSupportedTypes() {
|
69
|
+
return Arrays.asList(Types.BOOLEAN, Types.LONG, Types.DOUBLE, Types.TIMESTAMP, Types.JSON);
|
70
|
+
}
|
56
71
|
},
|
57
72
|
MULTI_LINE_TEXT {
|
58
73
|
@Override
|
@@ -64,6 +79,16 @@ public enum KintoneColumnType {
|
|
64
79
|
public MultiLineTextFieldValue getFieldValue(String value, KintoneColumnOption option) {
|
65
80
|
return new MultiLineTextFieldValue(value);
|
66
81
|
}
|
82
|
+
|
83
|
+
@Override
|
84
|
+
public Value asValue(FieldValue value) {
|
85
|
+
return ValueFactory.newString(((MultiLineTextFieldValue) value).getValue());
|
86
|
+
}
|
87
|
+
|
88
|
+
@Override
|
89
|
+
protected List<Type> getSupportedTypes() {
|
90
|
+
return Arrays.asList(Types.BOOLEAN, Types.LONG, Types.DOUBLE, Types.TIMESTAMP, Types.JSON);
|
91
|
+
}
|
67
92
|
},
|
68
93
|
RICH_TEXT {
|
69
94
|
@Override
|
@@ -75,6 +100,16 @@ public enum KintoneColumnType {
|
|
75
100
|
public RichTextFieldValue getFieldValue(String value, KintoneColumnOption option) {
|
76
101
|
return new RichTextFieldValue(value);
|
77
102
|
}
|
103
|
+
|
104
|
+
@Override
|
105
|
+
public Value asValue(FieldValue value) {
|
106
|
+
return ValueFactory.newString(((RichTextFieldValue) value).getValue());
|
107
|
+
}
|
108
|
+
|
109
|
+
@Override
|
110
|
+
protected List<Type> getSupportedTypes() {
|
111
|
+
return Arrays.asList(Types.BOOLEAN, Types.LONG, Types.DOUBLE, Types.TIMESTAMP, Types.JSON);
|
112
|
+
}
|
78
113
|
},
|
79
114
|
NUMBER {
|
80
115
|
@Override
|
@@ -106,6 +141,16 @@ public enum KintoneColumnType {
|
|
106
141
|
public void setUpdateKey(UpdateKey updateKey, String field, FieldValue value) {
|
107
142
|
updateKey.setField(field).setValue(((NumberFieldValue) value).getValue());
|
108
143
|
}
|
144
|
+
|
145
|
+
@Override
|
146
|
+
public Value asValue(FieldValue value) {
|
147
|
+
return ValueFactory.newString(((NumberFieldValue) value).getValue().toString());
|
148
|
+
}
|
149
|
+
|
150
|
+
@Override
|
151
|
+
protected List<Type> getSupportedTypes() {
|
152
|
+
return Arrays.asList(Types.BOOLEAN, Types.LONG, Types.DOUBLE, Types.TIMESTAMP);
|
153
|
+
}
|
109
154
|
},
|
110
155
|
CHECK_BOX {
|
111
156
|
@Override
|
@@ -117,6 +162,18 @@ public enum KintoneColumnType {
|
|
117
162
|
public CheckBoxFieldValue getFieldValue(String value, KintoneColumnOption option) {
|
118
163
|
return new CheckBoxFieldValue(asList(value, option));
|
119
164
|
}
|
165
|
+
|
166
|
+
@Override
|
167
|
+
public Value asValue(FieldValue value) {
|
168
|
+
return ValueFactory.newArray(
|
169
|
+
((CheckBoxFieldValue) value)
|
170
|
+
.getValues().stream().map(ValueFactory::newString).collect(Collectors.toList()));
|
171
|
+
}
|
172
|
+
|
173
|
+
@Override
|
174
|
+
protected List<Type> getSupportedTypes() {
|
175
|
+
return Collections.emptyList();
|
176
|
+
}
|
120
177
|
},
|
121
178
|
RADIO_BUTTON {
|
122
179
|
@Override
|
@@ -128,6 +185,16 @@ public enum KintoneColumnType {
|
|
128
185
|
public RadioButtonFieldValue getFieldValue(String value, KintoneColumnOption option) {
|
129
186
|
return new RadioButtonFieldValue(value);
|
130
187
|
}
|
188
|
+
|
189
|
+
@Override
|
190
|
+
public Value asValue(FieldValue value) {
|
191
|
+
return ValueFactory.newString(((RadioButtonFieldValue) value).getValue());
|
192
|
+
}
|
193
|
+
|
194
|
+
@Override
|
195
|
+
protected List<Type> getSupportedTypes() {
|
196
|
+
return Collections.emptyList();
|
197
|
+
}
|
131
198
|
},
|
132
199
|
MULTI_SELECT {
|
133
200
|
@Override
|
@@ -139,6 +206,18 @@ public enum KintoneColumnType {
|
|
139
206
|
public MultiSelectFieldValue getFieldValue(String value, KintoneColumnOption option) {
|
140
207
|
return new MultiSelectFieldValue(asList(value, option));
|
141
208
|
}
|
209
|
+
|
210
|
+
@Override
|
211
|
+
public Value asValue(FieldValue value) {
|
212
|
+
return ValueFactory.newArray(
|
213
|
+
((MultiSelectFieldValue) value)
|
214
|
+
.getValues().stream().map(ValueFactory::newString).collect(Collectors.toList()));
|
215
|
+
}
|
216
|
+
|
217
|
+
@Override
|
218
|
+
protected List<Type> getSupportedTypes() {
|
219
|
+
return Collections.emptyList();
|
220
|
+
}
|
142
221
|
},
|
143
222
|
DROP_DOWN {
|
144
223
|
@Override
|
@@ -150,6 +229,16 @@ public enum KintoneColumnType {
|
|
150
229
|
public DropDownFieldValue getFieldValue(String value, KintoneColumnOption option) {
|
151
230
|
return new DropDownFieldValue(value);
|
152
231
|
}
|
232
|
+
|
233
|
+
@Override
|
234
|
+
public Value asValue(FieldValue value) {
|
235
|
+
return ValueFactory.newString(((DropDownFieldValue) value).getValue());
|
236
|
+
}
|
237
|
+
|
238
|
+
@Override
|
239
|
+
protected List<Type> getSupportedTypes() {
|
240
|
+
return Collections.emptyList();
|
241
|
+
}
|
153
242
|
},
|
154
243
|
USER_SELECT {
|
155
244
|
@Override
|
@@ -161,6 +250,16 @@ public enum KintoneColumnType {
|
|
161
250
|
public UserSelectFieldValue getFieldValue(String value, KintoneColumnOption option) {
|
162
251
|
throw new UnsupportedOperationException();
|
163
252
|
}
|
253
|
+
|
254
|
+
@Override
|
255
|
+
public Value asValue(FieldValue value) {
|
256
|
+
throw new UnsupportedOperationException();
|
257
|
+
}
|
258
|
+
|
259
|
+
@Override
|
260
|
+
protected List<Type> getSupportedTypes() {
|
261
|
+
return Collections.emptyList();
|
262
|
+
}
|
164
263
|
},
|
165
264
|
ORGANIZATION_SELECT {
|
166
265
|
@Override
|
@@ -172,6 +271,16 @@ public enum KintoneColumnType {
|
|
172
271
|
public OrganizationSelectFieldValue getFieldValue(String value, KintoneColumnOption option) {
|
173
272
|
throw new UnsupportedOperationException();
|
174
273
|
}
|
274
|
+
|
275
|
+
@Override
|
276
|
+
public Value asValue(FieldValue value) {
|
277
|
+
throw new UnsupportedOperationException();
|
278
|
+
}
|
279
|
+
|
280
|
+
@Override
|
281
|
+
protected List<Type> getSupportedTypes() {
|
282
|
+
return Collections.emptyList();
|
283
|
+
}
|
175
284
|
},
|
176
285
|
GROUP_SELECT {
|
177
286
|
@Override
|
@@ -183,6 +292,16 @@ public enum KintoneColumnType {
|
|
183
292
|
public GroupSelectFieldValue getFieldValue(String value, KintoneColumnOption option) {
|
184
293
|
throw new UnsupportedOperationException();
|
185
294
|
}
|
295
|
+
|
296
|
+
@Override
|
297
|
+
public Value asValue(FieldValue value) {
|
298
|
+
throw new UnsupportedOperationException();
|
299
|
+
}
|
300
|
+
|
301
|
+
@Override
|
302
|
+
protected List<Type> getSupportedTypes() {
|
303
|
+
return Collections.emptyList();
|
304
|
+
}
|
186
305
|
},
|
187
306
|
DATE {
|
188
307
|
@Override
|
@@ -215,6 +334,16 @@ public enum KintoneColumnType {
|
|
215
334
|
public DateFieldValue getFieldValue(Timestamp value, KintoneColumnOption option) {
|
216
335
|
return new DateFieldValue(value.getInstant().atZone(getZoneId(option)).toLocalDate());
|
217
336
|
}
|
337
|
+
|
338
|
+
@Override
|
339
|
+
public Value asValue(FieldValue value) {
|
340
|
+
return ValueFactory.newString(((DateFieldValue) value).getValue().toString());
|
341
|
+
}
|
342
|
+
|
343
|
+
@Override
|
344
|
+
protected List<Type> getSupportedTypes() {
|
345
|
+
return Arrays.asList(Types.LONG, Types.DOUBLE, Types.TIMESTAMP);
|
346
|
+
}
|
218
347
|
},
|
219
348
|
TIME {
|
220
349
|
@Override
|
@@ -247,6 +376,16 @@ public enum KintoneColumnType {
|
|
247
376
|
public TimeFieldValue getFieldValue(Timestamp value, KintoneColumnOption option) {
|
248
377
|
return new TimeFieldValue(value.getInstant().atZone(getZoneId(option)).toLocalTime());
|
249
378
|
}
|
379
|
+
|
380
|
+
@Override
|
381
|
+
public Value asValue(FieldValue value) {
|
382
|
+
return ValueFactory.newString(((TimeFieldValue) value).getValue().toString());
|
383
|
+
}
|
384
|
+
|
385
|
+
@Override
|
386
|
+
protected List<Type> getSupportedTypes() {
|
387
|
+
return Arrays.asList(Types.LONG, Types.DOUBLE, Types.TIMESTAMP);
|
388
|
+
}
|
250
389
|
},
|
251
390
|
DATETIME {
|
252
391
|
@Override
|
@@ -275,6 +414,16 @@ public enum KintoneColumnType {
|
|
275
414
|
public DateTimeFieldValue getFieldValue(Timestamp value, KintoneColumnOption option) {
|
276
415
|
return new DateTimeFieldValue(value.getInstant().atZone(ZoneOffset.UTC));
|
277
416
|
}
|
417
|
+
|
418
|
+
@Override
|
419
|
+
public Value asValue(FieldValue value) {
|
420
|
+
return ValueFactory.newString(((DateTimeFieldValue) value).getValue().toString());
|
421
|
+
}
|
422
|
+
|
423
|
+
@Override
|
424
|
+
protected List<Type> getSupportedTypes() {
|
425
|
+
return Arrays.asList(Types.LONG, Types.DOUBLE, Types.TIMESTAMP);
|
426
|
+
}
|
278
427
|
},
|
279
428
|
LINK {
|
280
429
|
@Override
|
@@ -286,6 +435,16 @@ public enum KintoneColumnType {
|
|
286
435
|
public LinkFieldValue getFieldValue(String value, KintoneColumnOption option) {
|
287
436
|
return new LinkFieldValue(value);
|
288
437
|
}
|
438
|
+
|
439
|
+
@Override
|
440
|
+
public Value asValue(FieldValue value) {
|
441
|
+
return ValueFactory.newString(((LinkFieldValue) value).getValue());
|
442
|
+
}
|
443
|
+
|
444
|
+
@Override
|
445
|
+
protected List<Type> getSupportedTypes() {
|
446
|
+
return Collections.emptyList();
|
447
|
+
}
|
289
448
|
},
|
290
449
|
FILE {
|
291
450
|
@Override
|
@@ -297,6 +456,16 @@ public enum KintoneColumnType {
|
|
297
456
|
public FileFieldValue getFieldValue(String value, KintoneColumnOption option) {
|
298
457
|
throw new UnsupportedOperationException();
|
299
458
|
}
|
459
|
+
|
460
|
+
@Override
|
461
|
+
public Value asValue(FieldValue value) {
|
462
|
+
throw new UnsupportedOperationException();
|
463
|
+
}
|
464
|
+
|
465
|
+
@Override
|
466
|
+
protected List<Type> getSupportedTypes() {
|
467
|
+
return Collections.emptyList();
|
468
|
+
}
|
300
469
|
},
|
301
470
|
SUBTABLE {
|
302
471
|
@Override
|
@@ -306,9 +475,20 @@ public enum KintoneColumnType {
|
|
306
475
|
|
307
476
|
@Override
|
308
477
|
public SubtableFieldValue getFieldValue(String value, KintoneColumnOption option) {
|
478
|
+
return DESERIALIZER.deserialize(value.isEmpty() ? "[]" : value, SubtableFieldValue.class);
|
479
|
+
}
|
480
|
+
|
481
|
+
@Override
|
482
|
+
public Value asValue(FieldValue value) {
|
309
483
|
throw new UnsupportedOperationException();
|
310
484
|
}
|
485
|
+
|
486
|
+
@Override
|
487
|
+
protected List<Type> getSupportedTypes() {
|
488
|
+
return Collections.singletonList(Types.JSON);
|
489
|
+
}
|
311
490
|
};
|
491
|
+
private static final Deserializer DESERIALIZER = new Deserializer();
|
312
492
|
private static final Timestamp EPOCH = Timestamp.ofInstant(Instant.EPOCH);
|
313
493
|
|
314
494
|
public static KintoneColumnType getType(
|
@@ -319,25 +499,45 @@ public enum KintoneColumnType {
|
|
319
499
|
public abstract FieldValue getFieldValue();
|
320
500
|
|
321
501
|
public FieldValue getFieldValue(boolean value, KintoneColumnOption option) {
|
322
|
-
|
502
|
+
if (getSupportedTypes().contains(Types.BOOLEAN)) {
|
503
|
+
return getFieldValue(String.valueOf(value), option);
|
504
|
+
} else {
|
505
|
+
throw new UnsupportedOperationException();
|
506
|
+
}
|
323
507
|
}
|
324
508
|
|
325
509
|
public FieldValue getFieldValue(long value, KintoneColumnOption option) {
|
326
|
-
|
510
|
+
if (getSupportedTypes().contains(Types.LONG)) {
|
511
|
+
return getFieldValue(String.valueOf(value), option);
|
512
|
+
} else {
|
513
|
+
throw new UnsupportedOperationException();
|
514
|
+
}
|
327
515
|
}
|
328
516
|
|
329
517
|
public FieldValue getFieldValue(double value, KintoneColumnOption option) {
|
330
|
-
|
518
|
+
if (getSupportedTypes().contains(Types.DOUBLE)) {
|
519
|
+
return getFieldValue(String.valueOf(value), option);
|
520
|
+
} else {
|
521
|
+
throw new UnsupportedOperationException();
|
522
|
+
}
|
331
523
|
}
|
332
524
|
|
333
525
|
public abstract FieldValue getFieldValue(String value, KintoneColumnOption option);
|
334
526
|
|
335
527
|
public FieldValue getFieldValue(Timestamp value, KintoneColumnOption option) {
|
336
|
-
|
528
|
+
if (getSupportedTypes().contains(Types.TIMESTAMP)) {
|
529
|
+
return getFieldValue(value.getInstant().toString(), option);
|
530
|
+
} else {
|
531
|
+
throw new UnsupportedOperationException();
|
532
|
+
}
|
337
533
|
}
|
338
534
|
|
339
535
|
public FieldValue getFieldValue(Value value, KintoneColumnOption option) {
|
340
|
-
|
536
|
+
if (getSupportedTypes().contains(Types.JSON)) {
|
537
|
+
return getFieldValue(value.toJson(), option);
|
538
|
+
} else {
|
539
|
+
throw new UnsupportedOperationException();
|
540
|
+
}
|
341
541
|
}
|
342
542
|
|
343
543
|
public void setUpdateKey(UpdateKey updateKey, String field) {
|
@@ -348,6 +548,10 @@ public enum KintoneColumnType {
|
|
348
548
|
throw new UnsupportedOperationException();
|
349
549
|
}
|
350
550
|
|
551
|
+
public abstract Value asValue(FieldValue value);
|
552
|
+
|
553
|
+
protected abstract List<Type> getSupportedTypes();
|
554
|
+
|
351
555
|
private static List<String> asList(String value, KintoneColumnOption option) {
|
352
556
|
return value.isEmpty()
|
353
557
|
? Collections.emptyList()
|
@@ -5,6 +5,7 @@ import com.kintone.client.model.record.Record;
|
|
5
5
|
import com.kintone.client.model.record.UpdateKey;
|
6
6
|
import java.time.Instant;
|
7
7
|
import java.util.Map;
|
8
|
+
import java.util.Set;
|
8
9
|
import org.embulk.spi.Column;
|
9
10
|
import org.embulk.spi.ColumnVisitor;
|
10
11
|
import org.embulk.spi.PageReader;
|
@@ -14,31 +15,39 @@ import org.msgpack.value.ValueFactory;
|
|
14
15
|
|
15
16
|
public class KintoneColumnVisitor implements ColumnVisitor {
|
16
17
|
private final PageReader reader;
|
18
|
+
private final Set<Column> derived;
|
17
19
|
private final Map<String, KintoneColumnOption> options;
|
18
20
|
private final boolean preferNulls;
|
19
21
|
private final boolean ignoreNulls;
|
22
|
+
private final String reduceKeyName;
|
20
23
|
private final String updateKeyName;
|
21
24
|
private Record record;
|
22
25
|
private UpdateKey updateKey;
|
23
26
|
|
24
27
|
public KintoneColumnVisitor(
|
25
28
|
PageReader reader,
|
29
|
+
Set<Column> derived,
|
26
30
|
Map<String, KintoneColumnOption> options,
|
27
31
|
boolean preferNulls,
|
28
|
-
boolean ignoreNulls
|
29
|
-
|
32
|
+
boolean ignoreNulls,
|
33
|
+
String reduceKeyName) {
|
34
|
+
this(reader, derived, options, preferNulls, ignoreNulls, reduceKeyName, null);
|
30
35
|
}
|
31
36
|
|
32
37
|
public KintoneColumnVisitor(
|
33
38
|
PageReader reader,
|
39
|
+
Set<Column> derived,
|
34
40
|
Map<String, KintoneColumnOption> options,
|
35
41
|
boolean preferNulls,
|
36
42
|
boolean ignoreNulls,
|
43
|
+
String reduceKeyName,
|
37
44
|
String updateKeyName) {
|
38
45
|
this.reader = reader;
|
46
|
+
this.derived = derived;
|
39
47
|
this.options = options;
|
40
48
|
this.preferNulls = preferNulls;
|
41
49
|
this.ignoreNulls = ignoreNulls;
|
50
|
+
this.reduceKeyName = reduceKeyName;
|
42
51
|
this.updateKeyName = updateKeyName;
|
43
52
|
}
|
44
53
|
|
@@ -52,7 +61,7 @@ public class KintoneColumnVisitor implements ColumnVisitor {
|
|
52
61
|
|
53
62
|
@Override
|
54
63
|
public void booleanColumn(Column column) {
|
55
|
-
if (isIgnoreNull(column)) {
|
64
|
+
if (isReduced(column) || isIgnoreNull(column)) {
|
56
65
|
return;
|
57
66
|
}
|
58
67
|
KintoneColumnOption option = getOption(column);
|
@@ -70,7 +79,7 @@ public class KintoneColumnVisitor implements ColumnVisitor {
|
|
70
79
|
|
71
80
|
@Override
|
72
81
|
public void longColumn(Column column) {
|
73
|
-
if (isIgnoreNull(column)) {
|
82
|
+
if (isReduced(column) || isIgnoreNull(column)) {
|
74
83
|
return;
|
75
84
|
}
|
76
85
|
KintoneColumnOption option = getOption(column);
|
@@ -88,7 +97,7 @@ public class KintoneColumnVisitor implements ColumnVisitor {
|
|
88
97
|
|
89
98
|
@Override
|
90
99
|
public void doubleColumn(Column column) {
|
91
|
-
if (isIgnoreNull(column)) {
|
100
|
+
if (isReduced(column) || isIgnoreNull(column)) {
|
92
101
|
return;
|
93
102
|
}
|
94
103
|
KintoneColumnOption option = getOption(column);
|
@@ -106,7 +115,7 @@ public class KintoneColumnVisitor implements ColumnVisitor {
|
|
106
115
|
|
107
116
|
@Override
|
108
117
|
public void stringColumn(Column column) {
|
109
|
-
if (isIgnoreNull(column)) {
|
118
|
+
if (isReduced(column) || isIgnoreNull(column)) {
|
110
119
|
return;
|
111
120
|
}
|
112
121
|
KintoneColumnOption option = getOption(column);
|
@@ -126,7 +135,7 @@ public class KintoneColumnVisitor implements ColumnVisitor {
|
|
126
135
|
|
127
136
|
@Override
|
128
137
|
public void timestampColumn(Column column) {
|
129
|
-
if (isIgnoreNull(column)) {
|
138
|
+
if (isReduced(column) || isIgnoreNull(column)) {
|
130
139
|
return;
|
131
140
|
}
|
132
141
|
KintoneColumnOption option = getOption(column);
|
@@ -144,12 +153,14 @@ public class KintoneColumnVisitor implements ColumnVisitor {
|
|
144
153
|
|
145
154
|
@Override
|
146
155
|
public void jsonColumn(Column column) {
|
147
|
-
if (isIgnoreNull(column)) {
|
156
|
+
if (isReduced(column) || isIgnoreNull(column)) {
|
148
157
|
return;
|
149
158
|
}
|
150
159
|
KintoneColumnOption option = getOption(column);
|
151
160
|
UpdateKey updateKey = getUpdateKey(column);
|
152
|
-
KintoneColumnType
|
161
|
+
KintoneColumnType defaultType =
|
162
|
+
isDerived(column) ? KintoneColumnType.SUBTABLE : KintoneColumnType.MULTI_LINE_TEXT;
|
163
|
+
KintoneColumnType type = KintoneColumnType.getType(option, defaultType);
|
153
164
|
String fieldCode = getFieldCode(column);
|
154
165
|
if (isPreferNull(column)) {
|
155
166
|
setNull(type, fieldCode, updateKey);
|
@@ -258,6 +269,14 @@ public class KintoneColumnVisitor implements ColumnVisitor {
|
|
258
269
|
return updateKeyName != null && updateKeyName.equals(column.getName()) ? updateKey : null;
|
259
270
|
}
|
260
271
|
|
272
|
+
private boolean isReduced(Column column) {
|
273
|
+
return reduceKeyName != null && column.getName().matches("^.*\\..*$");
|
274
|
+
}
|
275
|
+
|
276
|
+
private boolean isDerived(Column column) {
|
277
|
+
return reduceKeyName != null && derived.contains(column);
|
278
|
+
}
|
279
|
+
|
261
280
|
private boolean isIgnoreNull(Column column) {
|
262
281
|
return ignoreNulls && reader.isNull(column);
|
263
282
|
}
|
@@ -1,11 +1,14 @@
|
|
1
1
|
package org.embulk.output.kintone;
|
2
2
|
|
3
|
+
import java.util.Collections;
|
3
4
|
import java.util.List;
|
4
5
|
import org.embulk.config.ConfigDiff;
|
5
6
|
import org.embulk.config.ConfigException;
|
6
7
|
import org.embulk.config.ConfigSource;
|
7
8
|
import org.embulk.config.TaskReport;
|
8
9
|
import org.embulk.config.TaskSource;
|
10
|
+
import org.embulk.output.kintone.reducer.ReducedPageOutput;
|
11
|
+
import org.embulk.output.kintone.reducer.Reducer;
|
9
12
|
import org.embulk.spi.Exec;
|
10
13
|
import org.embulk.spi.OutputPlugin;
|
11
14
|
import org.embulk.spi.Schema;
|
@@ -16,8 +19,12 @@ public class KintoneOutputPlugin implements OutputPlugin {
|
|
16
19
|
public ConfigDiff transaction(
|
17
20
|
ConfigSource config, Schema schema, int taskCount, OutputPlugin.Control control) {
|
18
21
|
PluginTask task = config.loadConfig(PluginTask.class);
|
19
|
-
|
20
|
-
|
22
|
+
task.setDerivedColumns(Collections.emptySet());
|
23
|
+
List<TaskReport> taskReports = control.run(task.dump());
|
24
|
+
return task.getReduceKeyName().isPresent()
|
25
|
+
? new Reducer(task, schema)
|
26
|
+
.reduce(taskReports, schema.lookupColumn(task.getReduceKeyName().get()))
|
27
|
+
: Exec.newConfigDiff();
|
21
28
|
}
|
22
29
|
|
23
30
|
@Override
|
@@ -49,6 +56,8 @@ public class KintoneOutputPlugin implements OutputPlugin {
|
|
49
56
|
default:
|
50
57
|
throw new ConfigException(String.format("Unknown mode '%s'", task.getMode()));
|
51
58
|
}
|
52
|
-
return
|
59
|
+
return task.getReduceKeyName().isPresent()
|
60
|
+
? new ReducedPageOutput(schema, taskIndex)
|
61
|
+
: new KintonePageOutput(task, schema);
|
53
62
|
}
|
54
63
|
}
|