embulk-output-kintone 1.0.0 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +57 -1
- data/build.gradle +1 -0
- data/classpath/commons-csv-1.9.0.jar +0 -0
- data/classpath/embulk-output-kintone-1.1.0.jar +0 -0
- data/classpath/externalsortinginjava-0.6.2.jar +0 -0
- data/classpath/{shadow-kintone-java-client-1.0.0-all.jar → shadow-kintone-java-client-1.1.0-all.jar} +0 -0
- data/src/main/java/org/embulk/output/kintone/KintoneColumnOption.java +5 -0
- data/src/main/java/org/embulk/output/kintone/KintoneColumnType.java +209 -5
- data/src/main/java/org/embulk/output/kintone/KintoneColumnVisitor.java +28 -9
- data/src/main/java/org/embulk/output/kintone/KintoneOutputPlugin.java +12 -3
- data/src/main/java/org/embulk/output/kintone/KintonePageOutput.java +20 -12
- data/src/main/java/org/embulk/output/kintone/KintoneSortColumn.java +33 -0
- data/src/main/java/org/embulk/output/kintone/PluginTask.java +27 -0
- data/src/main/java/org/embulk/output/kintone/deserializer/DeserializeApplier.java +19 -0
- data/src/main/java/org/embulk/output/kintone/deserializer/DeserializeException.java +7 -0
- data/src/main/java/org/embulk/output/kintone/deserializer/Deserializer.java +279 -0
- data/src/main/java/org/embulk/output/kintone/reducer/CSVInputColumnVisitor.java +78 -0
- data/src/main/java/org/embulk/output/kintone/reducer/CSVOutputColumnVisitor.java +79 -0
- data/src/main/java/org/embulk/output/kintone/reducer/ReduceException.java +11 -0
- data/src/main/java/org/embulk/output/kintone/reducer/ReduceType.java +190 -0
- data/src/main/java/org/embulk/output/kintone/reducer/ReducedPageOutput.java +100 -0
- data/src/main/java/org/embulk/output/kintone/reducer/Reducer.java +355 -0
- data/src/test/java/org/embulk/output/kintone/KintoneColumnOptionBuilder.java +7 -0
- data/src/test/java/org/embulk/output/kintone/KintoneColumnTypeTest.java +194 -0
- data/src/test/java/org/embulk/output/kintone/KintoneColumnVisitorTest.java +153 -34
- data/src/test/java/org/embulk/output/kintone/KintoneColumnVisitorVerifier.java +13 -3
- data/src/test/java/org/embulk/output/kintone/KintonePageOutputVerifier.java +44 -1
- data/src/test/java/org/embulk/output/kintone/TestKintoneOutputPlugin.java +89 -12
- data/src/test/java/org/embulk/output/kintone/TestTaskReduce.java +46 -0
- data/src/test/java/org/embulk/output/kintone/TestTaskReduceException.java +50 -0
- data/src/test/java/org/embulk/output/kintone/TestTaskReduceSubtable.java +46 -0
- data/src/test/java/org/embulk/output/kintone/deserializer/DeserializerTest.java +165 -0
- data/src/test/java/org/embulk/output/kintone/reducer/ReduceTypeTest.java +154 -0
- data/src/test/resources/org/embulk/output/kintone/task/config.yml +1 -1
- data/src/test/resources/org/embulk/output/kintone/task/mode/config.yml +6 -0
- data/src/test/resources/org/embulk/output/kintone/task/mode/input.csv +7 -7
- data/src/test/resources/org/embulk/output/kintone/task/mode/insert_add_ignore_nulls_records.jsonl +2 -2
- data/src/test/resources/org/embulk/output/kintone/task/mode/insert_add_prefer_nulls_records.jsonl +6 -6
- data/src/test/resources/org/embulk/output/kintone/task/mode/insert_add_records.jsonl +6 -6
- data/src/test/resources/org/embulk/output/kintone/task/mode/update_update_ignore_nulls_records.jsonl +2 -2
- data/src/test/resources/org/embulk/output/kintone/task/mode/update_update_prefer_nulls_records.jsonl +3 -3
- data/src/test/resources/org/embulk/output/kintone/task/mode/update_update_records.jsonl +6 -6
- data/src/test/resources/org/embulk/output/kintone/task/mode/upsert_add_prefer_nulls_records.jsonl +3 -3
- data/src/test/resources/org/embulk/output/kintone/task/mode/upsert_add_records.jsonl +2 -2
- data/src/test/resources/org/embulk/output/kintone/task/mode/upsert_update_ignore_nulls_records.jsonl +2 -2
- data/src/test/resources/org/embulk/output/kintone/task/mode/upsert_update_prefer_nulls_records.jsonl +3 -3
- data/src/test/resources/org/embulk/output/kintone/task/mode/upsert_update_records.jsonl +4 -4
- data/src/test/resources/org/embulk/output/kintone/task/reduce/config.yml +171 -0
- data/src/test/resources/org/embulk/output/kintone/task/reduce/input.csv +7 -0
- data/src/test/resources/org/embulk/output/kintone/task/reduce/insert_add_ignore_nulls_records.jsonl +6 -0
- data/src/test/resources/org/embulk/output/kintone/task/reduce/insert_add_prefer_nulls_records.jsonl +6 -0
- data/src/test/resources/org/embulk/output/kintone/task/reduce/insert_add_records.jsonl +6 -0
- data/src/test/resources/org/embulk/output/kintone/task/reduce/update_update_ignore_nulls_records.jsonl +3 -0
- data/src/test/resources/org/embulk/output/kintone/task/reduce/update_update_prefer_nulls_records.jsonl +3 -0
- data/src/test/resources/org/embulk/output/kintone/task/reduce/update_update_records.jsonl +6 -0
- data/src/test/resources/org/embulk/output/kintone/task/reduce/upsert_add_ignore_nulls_records.jsonl +3 -0
- data/src/test/resources/org/embulk/output/kintone/task/reduce/upsert_add_prefer_nulls_records.jsonl +3 -0
- data/src/test/resources/org/embulk/output/kintone/task/reduce/upsert_add_records.jsonl +2 -0
- data/src/test/resources/org/embulk/output/kintone/task/reduce/upsert_update_ignore_nulls_records.jsonl +3 -0
- data/src/test/resources/org/embulk/output/kintone/task/reduce/upsert_update_prefer_nulls_records.jsonl +3 -0
- data/src/test/resources/org/embulk/output/kintone/task/reduce/upsert_update_records.jsonl +4 -0
- data/src/test/resources/org/embulk/output/kintone/task/reduce/values.json +1 -0
- data/src/test/resources/org/embulk/output/kintone/task/reduce/values_ignore_nulls.json +1 -0
- data/src/test/resources/org/embulk/output/kintone/task/reduce/values_prefer_nulls.json +1 -0
- data/src/test/resources/org/embulk/output/kintone/task/reduce_exception/config.yml +36 -0
- data/src/test/resources/org/embulk/output/kintone/task/reduce_exception/derived_columns.json +1 -0
- data/src/test/resources/org/embulk/output/kintone/task/reduce_exception/input.csv +13 -0
- data/src/test/resources/org/embulk/output/kintone/task/reduce_exception/insert_add_records.jsonl +2 -0
- data/src/test/resources/org/embulk/output/kintone/task/reduce_exception/update_update_records.jsonl +2 -0
- data/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/config.yml +343 -0
- data/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/derived_columns.json +1 -0
- data/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/input.csv +13 -0
- data/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/insert_add_ignore_nulls_records.jsonl +6 -0
- data/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/insert_add_prefer_nulls_records.jsonl +6 -0
- data/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/insert_add_records.jsonl +6 -0
- data/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/update_update_ignore_nulls_records.jsonl +3 -0
- data/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/update_update_prefer_nulls_records.jsonl +3 -0
- data/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/update_update_records.jsonl +6 -0
- data/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/upsert_add_ignore_nulls_records.jsonl +3 -0
- data/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/upsert_add_prefer_nulls_records.jsonl +3 -0
- data/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/upsert_add_records.jsonl +0 -0
- data/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/upsert_update_ignore_nulls_records.jsonl +3 -0
- data/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/upsert_update_prefer_nulls_records.jsonl +3 -0
- data/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/upsert_update_records.jsonl +6 -0
- data/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/values.json +1 -0
- data/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/values_ignore_nulls.json +1 -0
- data/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/values_prefer_nulls.json +1 -0
- metadata +62 -4
- data/classpath/embulk-output-kintone-1.0.0.jar +0 -0
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 38d9d369925e0cab9fa558716b9a939be4b1387e
|
|
4
|
+
data.tar.gz: f4ae4a741548a48b042e1c4b79bf7e533ded0d31
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 06d103f7c4036dd458f62955bee9b2b82aa2bb2719cb59f18a2a762bab5c892025b4dd7fb0b41647e0e9cf66e012916f60de1a48c1a6d41414af0447d8affb06
|
|
7
|
+
data.tar.gz: ee848b7cfb53cc27c731724d32b6d5ffa9fe3336742fc1bbf24bd2b49d07f6f99439b478a0342051810ef94597471caded0f81caaf3aaff78203ce0ff20485d6
|
data/README.md
CHANGED
|
@@ -18,14 +18,24 @@ kintone output plugin for Embulk stores app records from kintone.
|
|
|
18
18
|
- **guest_space_id**: kintone app belongs to guest space, guest space id is required. (integer, optional)
|
|
19
19
|
- **mode**: kintone mode (string, required)
|
|
20
20
|
- **update_key**: Column name to set update key (string, required if mode is update or upsert)
|
|
21
|
+
- **reduce_key**: Key column name to reduce expanded SUBTABLE (string, optional)
|
|
22
|
+
- **sort_columns**: List of columns for sorting input records (array of objects, optional)
|
|
23
|
+
- **name**: Column name (string, required)
|
|
24
|
+
- **order**: Sort order (string `asc` or `desc`, required)
|
|
25
|
+
- **max_sort_tmp_files**: Maximum number of temporary files for sorting input records (integer, default is `1024`)
|
|
26
|
+
- **max_sort_memory**: Maximum memory usage for sorting input records (bytes in long, default is the estimated available memory, which is the approximate value of the JVM's current free memory)
|
|
21
27
|
- **prefer_nulls**: Whether to set fields to null instead of default value of type when column is null (boolean, default is `false`)
|
|
22
28
|
- **ignore_nulls**: Whether to completely ignore fields when column is null (boolean, default is `false`)
|
|
23
29
|
- **column_options** advanced: a key-value pairs where key is a column name and value is options for the column.
|
|
24
30
|
- **field_code**: field code (string, required)
|
|
25
31
|
- **type**: field type (string, required). See [this page](https://cybozu.dev/ja/kintone/docs/overview/field-types/#field-type-update) for list of available types. However, following types are not yet supported
|
|
26
|
-
- `USER_SELECT`, `ORGANIZATION_SELECT`, `GROUP_SELECT`, `FILE
|
|
32
|
+
- `USER_SELECT`, `ORGANIZATION_SELECT`, `GROUP_SELECT`, `FILE`
|
|
27
33
|
- **timezone**: timezone to convert into `date` (string, default is `UTC`)
|
|
28
34
|
- **val_sep**: Used to specify multiple checkbox values (string, default is `,`)
|
|
35
|
+
- **sort_columns**: List of columns for sorting rows in SUBTABLE. Available only if type is `SUBTABLE` (array of objects, optional)
|
|
36
|
+
- **name**: Column name (string, required)
|
|
37
|
+
- **order**: Sort order (string `asc` or `desc`, required)
|
|
38
|
+
- **chunk_size**: Maximum number of records to request at once (integer, default is `100`)
|
|
29
39
|
|
|
30
40
|
## Example
|
|
31
41
|
|
|
@@ -46,6 +56,52 @@ out:
|
|
|
46
56
|
date_time: {field_code: "datetime", type: "DATETIME"}
|
|
47
57
|
```
|
|
48
58
|
|
|
59
|
+
### For reduce expanded SUBTABLE
|
|
60
|
+
|
|
61
|
+
```yaml
|
|
62
|
+
out:
|
|
63
|
+
...
|
|
64
|
+
reduce_key: id
|
|
65
|
+
column_options:
|
|
66
|
+
id: {field_code: "id", type: "NUMBER"}
|
|
67
|
+
...
|
|
68
|
+
table: {field_code: "table", type: "SUBTABLE", sort_columns: [{name: number, order: asc}, {name: text, order: desc}]}
|
|
69
|
+
table.number: {field_code: "number_in_table", type: "NUMBER"}
|
|
70
|
+
table.text: {field_code: "text_in_table", type: "SINGLE_LINE_TEXT"}
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
#### KINTONE
|
|
74
|
+
|
|
75
|
+
| Form | Field Code |
|
|
76
|
+
| ------------------------- | --------------- |
|
|
77
|
+
| Table's own | table |
|
|
78
|
+
| NUMBER In Table | number_in_table |
|
|
79
|
+
| SINGLE_LINE_TEXT In Table | text_in_table |
|
|
80
|
+
|
|
81
|
+
#### INPUT CSV
|
|
82
|
+
|
|
83
|
+
```csv
|
|
84
|
+
id,table.number,table.text
|
|
85
|
+
1,0,test0
|
|
86
|
+
1,1,test1
|
|
87
|
+
2,0,test0
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
#### RESULT
|
|
91
|
+
|
|
92
|
+
ID:1
|
|
93
|
+
|
|
94
|
+
| NUMBER | SINGLE_LINE_TEXT |
|
|
95
|
+
| ------------- | ---------------- |
|
|
96
|
+
| 0 | test0 |
|
|
97
|
+
| 1 | test1 |
|
|
98
|
+
|
|
99
|
+
ID:2
|
|
100
|
+
|
|
101
|
+
| NUMBER | SINGLE_LINE_TEXT |
|
|
102
|
+
| ------------- | ---------------- |
|
|
103
|
+
| 0 | test0 |
|
|
104
|
+
|
|
49
105
|
## Build
|
|
50
106
|
|
|
51
107
|
```
|
data/build.gradle
CHANGED
|
@@ -28,6 +28,7 @@ targetCompatibility = 1.8
|
|
|
28
28
|
|
|
29
29
|
dependencies {
|
|
30
30
|
compileOnly "org.embulk:embulk-core:0.9.23"
|
|
31
|
+
implementation "com.google.code.externalsortinginjava:externalsortinginjava:0.6.2"
|
|
31
32
|
implementation project(path: ":shadow-kintone-java-client", configuration: "shadow")
|
|
32
33
|
|
|
33
34
|
testImplementation "junit:junit:4.+"
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
data/classpath/{shadow-kintone-java-client-1.0.0-all.jar → shadow-kintone-java-client-1.1.0-all.jar}
RENAMED
|
Binary file
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
package org.embulk.output.kintone;
|
|
2
2
|
|
|
3
|
+
import java.util.List;
|
|
3
4
|
import org.embulk.config.Config;
|
|
4
5
|
import org.embulk.config.ConfigDefault;
|
|
5
6
|
import org.embulk.config.Task;
|
|
@@ -18,4 +19,8 @@ public interface KintoneColumnOption extends Task {
|
|
|
18
19
|
@Config("val_sep")
|
|
19
20
|
@ConfigDefault("\",\"")
|
|
20
21
|
String getValueSeparator();
|
|
22
|
+
|
|
23
|
+
@Config("sort_columns")
|
|
24
|
+
@ConfigDefault("[]")
|
|
25
|
+
List<KintoneSortColumn> getSortColumns();
|
|
21
26
|
}
|
|
@@ -29,8 +29,13 @@ import java.time.ZonedDateTime;
|
|
|
29
29
|
import java.util.Arrays;
|
|
30
30
|
import java.util.Collections;
|
|
31
31
|
import java.util.List;
|
|
32
|
+
import java.util.stream.Collectors;
|
|
33
|
+
import org.embulk.output.kintone.deserializer.Deserializer;
|
|
32
34
|
import org.embulk.spi.time.Timestamp;
|
|
35
|
+
import org.embulk.spi.type.Type;
|
|
36
|
+
import org.embulk.spi.type.Types;
|
|
33
37
|
import org.msgpack.value.Value;
|
|
38
|
+
import org.msgpack.value.ValueFactory;
|
|
34
39
|
|
|
35
40
|
public enum KintoneColumnType {
|
|
36
41
|
SINGLE_LINE_TEXT {
|
|
@@ -53,6 +58,16 @@ public enum KintoneColumnType {
|
|
|
53
58
|
public void setUpdateKey(UpdateKey updateKey, String field, FieldValue value) {
|
|
54
59
|
updateKey.setField(field).setValue(((SingleLineTextFieldValue) value).getValue());
|
|
55
60
|
}
|
|
61
|
+
|
|
62
|
+
@Override
|
|
63
|
+
public Value asValue(FieldValue value) {
|
|
64
|
+
return ValueFactory.newString(((SingleLineTextFieldValue) value).getValue());
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
@Override
|
|
68
|
+
protected List<Type> getSupportedTypes() {
|
|
69
|
+
return Arrays.asList(Types.BOOLEAN, Types.LONG, Types.DOUBLE, Types.TIMESTAMP, Types.JSON);
|
|
70
|
+
}
|
|
56
71
|
},
|
|
57
72
|
MULTI_LINE_TEXT {
|
|
58
73
|
@Override
|
|
@@ -64,6 +79,16 @@ public enum KintoneColumnType {
|
|
|
64
79
|
public MultiLineTextFieldValue getFieldValue(String value, KintoneColumnOption option) {
|
|
65
80
|
return new MultiLineTextFieldValue(value);
|
|
66
81
|
}
|
|
82
|
+
|
|
83
|
+
@Override
|
|
84
|
+
public Value asValue(FieldValue value) {
|
|
85
|
+
return ValueFactory.newString(((MultiLineTextFieldValue) value).getValue());
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
@Override
|
|
89
|
+
protected List<Type> getSupportedTypes() {
|
|
90
|
+
return Arrays.asList(Types.BOOLEAN, Types.LONG, Types.DOUBLE, Types.TIMESTAMP, Types.JSON);
|
|
91
|
+
}
|
|
67
92
|
},
|
|
68
93
|
RICH_TEXT {
|
|
69
94
|
@Override
|
|
@@ -75,6 +100,16 @@ public enum KintoneColumnType {
|
|
|
75
100
|
public RichTextFieldValue getFieldValue(String value, KintoneColumnOption option) {
|
|
76
101
|
return new RichTextFieldValue(value);
|
|
77
102
|
}
|
|
103
|
+
|
|
104
|
+
@Override
|
|
105
|
+
public Value asValue(FieldValue value) {
|
|
106
|
+
return ValueFactory.newString(((RichTextFieldValue) value).getValue());
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
@Override
|
|
110
|
+
protected List<Type> getSupportedTypes() {
|
|
111
|
+
return Arrays.asList(Types.BOOLEAN, Types.LONG, Types.DOUBLE, Types.TIMESTAMP, Types.JSON);
|
|
112
|
+
}
|
|
78
113
|
},
|
|
79
114
|
NUMBER {
|
|
80
115
|
@Override
|
|
@@ -106,6 +141,16 @@ public enum KintoneColumnType {
|
|
|
106
141
|
public void setUpdateKey(UpdateKey updateKey, String field, FieldValue value) {
|
|
107
142
|
updateKey.setField(field).setValue(((NumberFieldValue) value).getValue());
|
|
108
143
|
}
|
|
144
|
+
|
|
145
|
+
@Override
|
|
146
|
+
public Value asValue(FieldValue value) {
|
|
147
|
+
return ValueFactory.newString(((NumberFieldValue) value).getValue().toString());
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
@Override
|
|
151
|
+
protected List<Type> getSupportedTypes() {
|
|
152
|
+
return Arrays.asList(Types.BOOLEAN, Types.LONG, Types.DOUBLE, Types.TIMESTAMP);
|
|
153
|
+
}
|
|
109
154
|
},
|
|
110
155
|
CHECK_BOX {
|
|
111
156
|
@Override
|
|
@@ -117,6 +162,18 @@ public enum KintoneColumnType {
|
|
|
117
162
|
public CheckBoxFieldValue getFieldValue(String value, KintoneColumnOption option) {
|
|
118
163
|
return new CheckBoxFieldValue(asList(value, option));
|
|
119
164
|
}
|
|
165
|
+
|
|
166
|
+
@Override
|
|
167
|
+
public Value asValue(FieldValue value) {
|
|
168
|
+
return ValueFactory.newArray(
|
|
169
|
+
((CheckBoxFieldValue) value)
|
|
170
|
+
.getValues().stream().map(ValueFactory::newString).collect(Collectors.toList()));
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
@Override
|
|
174
|
+
protected List<Type> getSupportedTypes() {
|
|
175
|
+
return Collections.emptyList();
|
|
176
|
+
}
|
|
120
177
|
},
|
|
121
178
|
RADIO_BUTTON {
|
|
122
179
|
@Override
|
|
@@ -128,6 +185,16 @@ public enum KintoneColumnType {
|
|
|
128
185
|
public RadioButtonFieldValue getFieldValue(String value, KintoneColumnOption option) {
|
|
129
186
|
return new RadioButtonFieldValue(value);
|
|
130
187
|
}
|
|
188
|
+
|
|
189
|
+
@Override
|
|
190
|
+
public Value asValue(FieldValue value) {
|
|
191
|
+
return ValueFactory.newString(((RadioButtonFieldValue) value).getValue());
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
@Override
|
|
195
|
+
protected List<Type> getSupportedTypes() {
|
|
196
|
+
return Collections.emptyList();
|
|
197
|
+
}
|
|
131
198
|
},
|
|
132
199
|
MULTI_SELECT {
|
|
133
200
|
@Override
|
|
@@ -139,6 +206,18 @@ public enum KintoneColumnType {
|
|
|
139
206
|
public MultiSelectFieldValue getFieldValue(String value, KintoneColumnOption option) {
|
|
140
207
|
return new MultiSelectFieldValue(asList(value, option));
|
|
141
208
|
}
|
|
209
|
+
|
|
210
|
+
@Override
|
|
211
|
+
public Value asValue(FieldValue value) {
|
|
212
|
+
return ValueFactory.newArray(
|
|
213
|
+
((MultiSelectFieldValue) value)
|
|
214
|
+
.getValues().stream().map(ValueFactory::newString).collect(Collectors.toList()));
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
@Override
|
|
218
|
+
protected List<Type> getSupportedTypes() {
|
|
219
|
+
return Collections.emptyList();
|
|
220
|
+
}
|
|
142
221
|
},
|
|
143
222
|
DROP_DOWN {
|
|
144
223
|
@Override
|
|
@@ -150,6 +229,16 @@ public enum KintoneColumnType {
|
|
|
150
229
|
public DropDownFieldValue getFieldValue(String value, KintoneColumnOption option) {
|
|
151
230
|
return new DropDownFieldValue(value);
|
|
152
231
|
}
|
|
232
|
+
|
|
233
|
+
@Override
|
|
234
|
+
public Value asValue(FieldValue value) {
|
|
235
|
+
return ValueFactory.newString(((DropDownFieldValue) value).getValue());
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
@Override
|
|
239
|
+
protected List<Type> getSupportedTypes() {
|
|
240
|
+
return Collections.emptyList();
|
|
241
|
+
}
|
|
153
242
|
},
|
|
154
243
|
USER_SELECT {
|
|
155
244
|
@Override
|
|
@@ -161,6 +250,16 @@ public enum KintoneColumnType {
|
|
|
161
250
|
public UserSelectFieldValue getFieldValue(String value, KintoneColumnOption option) {
|
|
162
251
|
throw new UnsupportedOperationException();
|
|
163
252
|
}
|
|
253
|
+
|
|
254
|
+
@Override
|
|
255
|
+
public Value asValue(FieldValue value) {
|
|
256
|
+
throw new UnsupportedOperationException();
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
@Override
|
|
260
|
+
protected List<Type> getSupportedTypes() {
|
|
261
|
+
return Collections.emptyList();
|
|
262
|
+
}
|
|
164
263
|
},
|
|
165
264
|
ORGANIZATION_SELECT {
|
|
166
265
|
@Override
|
|
@@ -172,6 +271,16 @@ public enum KintoneColumnType {
|
|
|
172
271
|
public OrganizationSelectFieldValue getFieldValue(String value, KintoneColumnOption option) {
|
|
173
272
|
throw new UnsupportedOperationException();
|
|
174
273
|
}
|
|
274
|
+
|
|
275
|
+
@Override
|
|
276
|
+
public Value asValue(FieldValue value) {
|
|
277
|
+
throw new UnsupportedOperationException();
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
@Override
|
|
281
|
+
protected List<Type> getSupportedTypes() {
|
|
282
|
+
return Collections.emptyList();
|
|
283
|
+
}
|
|
175
284
|
},
|
|
176
285
|
GROUP_SELECT {
|
|
177
286
|
@Override
|
|
@@ -183,6 +292,16 @@ public enum KintoneColumnType {
|
|
|
183
292
|
public GroupSelectFieldValue getFieldValue(String value, KintoneColumnOption option) {
|
|
184
293
|
throw new UnsupportedOperationException();
|
|
185
294
|
}
|
|
295
|
+
|
|
296
|
+
@Override
|
|
297
|
+
public Value asValue(FieldValue value) {
|
|
298
|
+
throw new UnsupportedOperationException();
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
@Override
|
|
302
|
+
protected List<Type> getSupportedTypes() {
|
|
303
|
+
return Collections.emptyList();
|
|
304
|
+
}
|
|
186
305
|
},
|
|
187
306
|
DATE {
|
|
188
307
|
@Override
|
|
@@ -215,6 +334,16 @@ public enum KintoneColumnType {
|
|
|
215
334
|
public DateFieldValue getFieldValue(Timestamp value, KintoneColumnOption option) {
|
|
216
335
|
return new DateFieldValue(value.getInstant().atZone(getZoneId(option)).toLocalDate());
|
|
217
336
|
}
|
|
337
|
+
|
|
338
|
+
@Override
|
|
339
|
+
public Value asValue(FieldValue value) {
|
|
340
|
+
return ValueFactory.newString(((DateFieldValue) value).getValue().toString());
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
@Override
|
|
344
|
+
protected List<Type> getSupportedTypes() {
|
|
345
|
+
return Arrays.asList(Types.LONG, Types.DOUBLE, Types.TIMESTAMP);
|
|
346
|
+
}
|
|
218
347
|
},
|
|
219
348
|
TIME {
|
|
220
349
|
@Override
|
|
@@ -247,6 +376,16 @@ public enum KintoneColumnType {
|
|
|
247
376
|
public TimeFieldValue getFieldValue(Timestamp value, KintoneColumnOption option) {
|
|
248
377
|
return new TimeFieldValue(value.getInstant().atZone(getZoneId(option)).toLocalTime());
|
|
249
378
|
}
|
|
379
|
+
|
|
380
|
+
@Override
|
|
381
|
+
public Value asValue(FieldValue value) {
|
|
382
|
+
return ValueFactory.newString(((TimeFieldValue) value).getValue().toString());
|
|
383
|
+
}
|
|
384
|
+
|
|
385
|
+
@Override
|
|
386
|
+
protected List<Type> getSupportedTypes() {
|
|
387
|
+
return Arrays.asList(Types.LONG, Types.DOUBLE, Types.TIMESTAMP);
|
|
388
|
+
}
|
|
250
389
|
},
|
|
251
390
|
DATETIME {
|
|
252
391
|
@Override
|
|
@@ -275,6 +414,16 @@ public enum KintoneColumnType {
|
|
|
275
414
|
public DateTimeFieldValue getFieldValue(Timestamp value, KintoneColumnOption option) {
|
|
276
415
|
return new DateTimeFieldValue(value.getInstant().atZone(ZoneOffset.UTC));
|
|
277
416
|
}
|
|
417
|
+
|
|
418
|
+
@Override
|
|
419
|
+
public Value asValue(FieldValue value) {
|
|
420
|
+
return ValueFactory.newString(((DateTimeFieldValue) value).getValue().toString());
|
|
421
|
+
}
|
|
422
|
+
|
|
423
|
+
@Override
|
|
424
|
+
protected List<Type> getSupportedTypes() {
|
|
425
|
+
return Arrays.asList(Types.LONG, Types.DOUBLE, Types.TIMESTAMP);
|
|
426
|
+
}
|
|
278
427
|
},
|
|
279
428
|
LINK {
|
|
280
429
|
@Override
|
|
@@ -286,6 +435,16 @@ public enum KintoneColumnType {
|
|
|
286
435
|
public LinkFieldValue getFieldValue(String value, KintoneColumnOption option) {
|
|
287
436
|
return new LinkFieldValue(value);
|
|
288
437
|
}
|
|
438
|
+
|
|
439
|
+
@Override
|
|
440
|
+
public Value asValue(FieldValue value) {
|
|
441
|
+
return ValueFactory.newString(((LinkFieldValue) value).getValue());
|
|
442
|
+
}
|
|
443
|
+
|
|
444
|
+
@Override
|
|
445
|
+
protected List<Type> getSupportedTypes() {
|
|
446
|
+
return Collections.emptyList();
|
|
447
|
+
}
|
|
289
448
|
},
|
|
290
449
|
FILE {
|
|
291
450
|
@Override
|
|
@@ -297,6 +456,16 @@ public enum KintoneColumnType {
|
|
|
297
456
|
public FileFieldValue getFieldValue(String value, KintoneColumnOption option) {
|
|
298
457
|
throw new UnsupportedOperationException();
|
|
299
458
|
}
|
|
459
|
+
|
|
460
|
+
@Override
|
|
461
|
+
public Value asValue(FieldValue value) {
|
|
462
|
+
throw new UnsupportedOperationException();
|
|
463
|
+
}
|
|
464
|
+
|
|
465
|
+
@Override
|
|
466
|
+
protected List<Type> getSupportedTypes() {
|
|
467
|
+
return Collections.emptyList();
|
|
468
|
+
}
|
|
300
469
|
},
|
|
301
470
|
SUBTABLE {
|
|
302
471
|
@Override
|
|
@@ -306,9 +475,20 @@ public enum KintoneColumnType {
|
|
|
306
475
|
|
|
307
476
|
@Override
|
|
308
477
|
public SubtableFieldValue getFieldValue(String value, KintoneColumnOption option) {
|
|
478
|
+
return DESERIALIZER.deserialize(value.isEmpty() ? "[]" : value, SubtableFieldValue.class);
|
|
479
|
+
}
|
|
480
|
+
|
|
481
|
+
@Override
|
|
482
|
+
public Value asValue(FieldValue value) {
|
|
309
483
|
throw new UnsupportedOperationException();
|
|
310
484
|
}
|
|
485
|
+
|
|
486
|
+
@Override
|
|
487
|
+
protected List<Type> getSupportedTypes() {
|
|
488
|
+
return Collections.singletonList(Types.JSON);
|
|
489
|
+
}
|
|
311
490
|
};
|
|
491
|
+
private static final Deserializer DESERIALIZER = new Deserializer();
|
|
312
492
|
private static final Timestamp EPOCH = Timestamp.ofInstant(Instant.EPOCH);
|
|
313
493
|
|
|
314
494
|
public static KintoneColumnType getType(
|
|
@@ -319,25 +499,45 @@ public enum KintoneColumnType {
|
|
|
319
499
|
public abstract FieldValue getFieldValue();
|
|
320
500
|
|
|
321
501
|
public FieldValue getFieldValue(boolean value, KintoneColumnOption option) {
|
|
322
|
-
|
|
502
|
+
if (getSupportedTypes().contains(Types.BOOLEAN)) {
|
|
503
|
+
return getFieldValue(String.valueOf(value), option);
|
|
504
|
+
} else {
|
|
505
|
+
throw new UnsupportedOperationException();
|
|
506
|
+
}
|
|
323
507
|
}
|
|
324
508
|
|
|
325
509
|
public FieldValue getFieldValue(long value, KintoneColumnOption option) {
|
|
326
|
-
|
|
510
|
+
if (getSupportedTypes().contains(Types.LONG)) {
|
|
511
|
+
return getFieldValue(String.valueOf(value), option);
|
|
512
|
+
} else {
|
|
513
|
+
throw new UnsupportedOperationException();
|
|
514
|
+
}
|
|
327
515
|
}
|
|
328
516
|
|
|
329
517
|
public FieldValue getFieldValue(double value, KintoneColumnOption option) {
|
|
330
|
-
|
|
518
|
+
if (getSupportedTypes().contains(Types.DOUBLE)) {
|
|
519
|
+
return getFieldValue(String.valueOf(value), option);
|
|
520
|
+
} else {
|
|
521
|
+
throw new UnsupportedOperationException();
|
|
522
|
+
}
|
|
331
523
|
}
|
|
332
524
|
|
|
333
525
|
public abstract FieldValue getFieldValue(String value, KintoneColumnOption option);
|
|
334
526
|
|
|
335
527
|
public FieldValue getFieldValue(Timestamp value, KintoneColumnOption option) {
|
|
336
|
-
|
|
528
|
+
if (getSupportedTypes().contains(Types.TIMESTAMP)) {
|
|
529
|
+
return getFieldValue(value.getInstant().toString(), option);
|
|
530
|
+
} else {
|
|
531
|
+
throw new UnsupportedOperationException();
|
|
532
|
+
}
|
|
337
533
|
}
|
|
338
534
|
|
|
339
535
|
public FieldValue getFieldValue(Value value, KintoneColumnOption option) {
|
|
340
|
-
|
|
536
|
+
if (getSupportedTypes().contains(Types.JSON)) {
|
|
537
|
+
return getFieldValue(value.toJson(), option);
|
|
538
|
+
} else {
|
|
539
|
+
throw new UnsupportedOperationException();
|
|
540
|
+
}
|
|
341
541
|
}
|
|
342
542
|
|
|
343
543
|
public void setUpdateKey(UpdateKey updateKey, String field) {
|
|
@@ -348,6 +548,10 @@ public enum KintoneColumnType {
|
|
|
348
548
|
throw new UnsupportedOperationException();
|
|
349
549
|
}
|
|
350
550
|
|
|
551
|
+
public abstract Value asValue(FieldValue value);
|
|
552
|
+
|
|
553
|
+
protected abstract List<Type> getSupportedTypes();
|
|
554
|
+
|
|
351
555
|
private static List<String> asList(String value, KintoneColumnOption option) {
|
|
352
556
|
return value.isEmpty()
|
|
353
557
|
? Collections.emptyList()
|
|
@@ -5,6 +5,7 @@ import com.kintone.client.model.record.Record;
|
|
|
5
5
|
import com.kintone.client.model.record.UpdateKey;
|
|
6
6
|
import java.time.Instant;
|
|
7
7
|
import java.util.Map;
|
|
8
|
+
import java.util.Set;
|
|
8
9
|
import org.embulk.spi.Column;
|
|
9
10
|
import org.embulk.spi.ColumnVisitor;
|
|
10
11
|
import org.embulk.spi.PageReader;
|
|
@@ -14,31 +15,39 @@ import org.msgpack.value.ValueFactory;
|
|
|
14
15
|
|
|
15
16
|
public class KintoneColumnVisitor implements ColumnVisitor {
|
|
16
17
|
private final PageReader reader;
|
|
18
|
+
private final Set<Column> derived;
|
|
17
19
|
private final Map<String, KintoneColumnOption> options;
|
|
18
20
|
private final boolean preferNulls;
|
|
19
21
|
private final boolean ignoreNulls;
|
|
22
|
+
private final String reduceKeyName;
|
|
20
23
|
private final String updateKeyName;
|
|
21
24
|
private Record record;
|
|
22
25
|
private UpdateKey updateKey;
|
|
23
26
|
|
|
24
27
|
public KintoneColumnVisitor(
|
|
25
28
|
PageReader reader,
|
|
29
|
+
Set<Column> derived,
|
|
26
30
|
Map<String, KintoneColumnOption> options,
|
|
27
31
|
boolean preferNulls,
|
|
28
|
-
boolean ignoreNulls
|
|
29
|
-
|
|
32
|
+
boolean ignoreNulls,
|
|
33
|
+
String reduceKeyName) {
|
|
34
|
+
this(reader, derived, options, preferNulls, ignoreNulls, reduceKeyName, null);
|
|
30
35
|
}
|
|
31
36
|
|
|
32
37
|
public KintoneColumnVisitor(
|
|
33
38
|
PageReader reader,
|
|
39
|
+
Set<Column> derived,
|
|
34
40
|
Map<String, KintoneColumnOption> options,
|
|
35
41
|
boolean preferNulls,
|
|
36
42
|
boolean ignoreNulls,
|
|
43
|
+
String reduceKeyName,
|
|
37
44
|
String updateKeyName) {
|
|
38
45
|
this.reader = reader;
|
|
46
|
+
this.derived = derived;
|
|
39
47
|
this.options = options;
|
|
40
48
|
this.preferNulls = preferNulls;
|
|
41
49
|
this.ignoreNulls = ignoreNulls;
|
|
50
|
+
this.reduceKeyName = reduceKeyName;
|
|
42
51
|
this.updateKeyName = updateKeyName;
|
|
43
52
|
}
|
|
44
53
|
|
|
@@ -52,7 +61,7 @@ public class KintoneColumnVisitor implements ColumnVisitor {
|
|
|
52
61
|
|
|
53
62
|
@Override
|
|
54
63
|
public void booleanColumn(Column column) {
|
|
55
|
-
if (isIgnoreNull(column)) {
|
|
64
|
+
if (isReduced(column) || isIgnoreNull(column)) {
|
|
56
65
|
return;
|
|
57
66
|
}
|
|
58
67
|
KintoneColumnOption option = getOption(column);
|
|
@@ -70,7 +79,7 @@ public class KintoneColumnVisitor implements ColumnVisitor {
|
|
|
70
79
|
|
|
71
80
|
@Override
|
|
72
81
|
public void longColumn(Column column) {
|
|
73
|
-
if (isIgnoreNull(column)) {
|
|
82
|
+
if (isReduced(column) || isIgnoreNull(column)) {
|
|
74
83
|
return;
|
|
75
84
|
}
|
|
76
85
|
KintoneColumnOption option = getOption(column);
|
|
@@ -88,7 +97,7 @@ public class KintoneColumnVisitor implements ColumnVisitor {
|
|
|
88
97
|
|
|
89
98
|
@Override
|
|
90
99
|
public void doubleColumn(Column column) {
|
|
91
|
-
if (isIgnoreNull(column)) {
|
|
100
|
+
if (isReduced(column) || isIgnoreNull(column)) {
|
|
92
101
|
return;
|
|
93
102
|
}
|
|
94
103
|
KintoneColumnOption option = getOption(column);
|
|
@@ -106,7 +115,7 @@ public class KintoneColumnVisitor implements ColumnVisitor {
|
|
|
106
115
|
|
|
107
116
|
@Override
|
|
108
117
|
public void stringColumn(Column column) {
|
|
109
|
-
if (isIgnoreNull(column)) {
|
|
118
|
+
if (isReduced(column) || isIgnoreNull(column)) {
|
|
110
119
|
return;
|
|
111
120
|
}
|
|
112
121
|
KintoneColumnOption option = getOption(column);
|
|
@@ -126,7 +135,7 @@ public class KintoneColumnVisitor implements ColumnVisitor {
|
|
|
126
135
|
|
|
127
136
|
@Override
|
|
128
137
|
public void timestampColumn(Column column) {
|
|
129
|
-
if (isIgnoreNull(column)) {
|
|
138
|
+
if (isReduced(column) || isIgnoreNull(column)) {
|
|
130
139
|
return;
|
|
131
140
|
}
|
|
132
141
|
KintoneColumnOption option = getOption(column);
|
|
@@ -144,12 +153,14 @@ public class KintoneColumnVisitor implements ColumnVisitor {
|
|
|
144
153
|
|
|
145
154
|
@Override
|
|
146
155
|
public void jsonColumn(Column column) {
|
|
147
|
-
if (isIgnoreNull(column)) {
|
|
156
|
+
if (isReduced(column) || isIgnoreNull(column)) {
|
|
148
157
|
return;
|
|
149
158
|
}
|
|
150
159
|
KintoneColumnOption option = getOption(column);
|
|
151
160
|
UpdateKey updateKey = getUpdateKey(column);
|
|
152
|
-
KintoneColumnType
|
|
161
|
+
KintoneColumnType defaultType =
|
|
162
|
+
isDerived(column) ? KintoneColumnType.SUBTABLE : KintoneColumnType.MULTI_LINE_TEXT;
|
|
163
|
+
KintoneColumnType type = KintoneColumnType.getType(option, defaultType);
|
|
153
164
|
String fieldCode = getFieldCode(column);
|
|
154
165
|
if (isPreferNull(column)) {
|
|
155
166
|
setNull(type, fieldCode, updateKey);
|
|
@@ -258,6 +269,14 @@ public class KintoneColumnVisitor implements ColumnVisitor {
|
|
|
258
269
|
return updateKeyName != null && updateKeyName.equals(column.getName()) ? updateKey : null;
|
|
259
270
|
}
|
|
260
271
|
|
|
272
|
+
private boolean isReduced(Column column) {
|
|
273
|
+
return reduceKeyName != null && column.getName().matches("^.*\\..*$");
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
private boolean isDerived(Column column) {
|
|
277
|
+
return reduceKeyName != null && derived.contains(column);
|
|
278
|
+
}
|
|
279
|
+
|
|
261
280
|
private boolean isIgnoreNull(Column column) {
|
|
262
281
|
return ignoreNulls && reader.isNull(column);
|
|
263
282
|
}
|
|
@@ -1,11 +1,14 @@
|
|
|
1
1
|
package org.embulk.output.kintone;
|
|
2
2
|
|
|
3
|
+
import java.util.Collections;
|
|
3
4
|
import java.util.List;
|
|
4
5
|
import org.embulk.config.ConfigDiff;
|
|
5
6
|
import org.embulk.config.ConfigException;
|
|
6
7
|
import org.embulk.config.ConfigSource;
|
|
7
8
|
import org.embulk.config.TaskReport;
|
|
8
9
|
import org.embulk.config.TaskSource;
|
|
10
|
+
import org.embulk.output.kintone.reducer.ReducedPageOutput;
|
|
11
|
+
import org.embulk.output.kintone.reducer.Reducer;
|
|
9
12
|
import org.embulk.spi.Exec;
|
|
10
13
|
import org.embulk.spi.OutputPlugin;
|
|
11
14
|
import org.embulk.spi.Schema;
|
|
@@ -16,8 +19,12 @@ public class KintoneOutputPlugin implements OutputPlugin {
|
|
|
16
19
|
public ConfigDiff transaction(
|
|
17
20
|
ConfigSource config, Schema schema, int taskCount, OutputPlugin.Control control) {
|
|
18
21
|
PluginTask task = config.loadConfig(PluginTask.class);
|
|
19
|
-
|
|
20
|
-
|
|
22
|
+
task.setDerivedColumns(Collections.emptySet());
|
|
23
|
+
List<TaskReport> taskReports = control.run(task.dump());
|
|
24
|
+
return task.getReduceKeyName().isPresent()
|
|
25
|
+
? new Reducer(task, schema)
|
|
26
|
+
.reduce(taskReports, schema.lookupColumn(task.getReduceKeyName().get()))
|
|
27
|
+
: Exec.newConfigDiff();
|
|
21
28
|
}
|
|
22
29
|
|
|
23
30
|
@Override
|
|
@@ -49,6 +56,8 @@ public class KintoneOutputPlugin implements OutputPlugin {
|
|
|
49
56
|
default:
|
|
50
57
|
throw new ConfigException(String.format("Unknown mode '%s'", task.getMode()));
|
|
51
58
|
}
|
|
52
|
-
return
|
|
59
|
+
return task.getReduceKeyName().isPresent()
|
|
60
|
+
? new ReducedPageOutput(schema, taskIndex)
|
|
61
|
+
: new KintonePageOutput(task, schema);
|
|
53
62
|
}
|
|
54
63
|
}
|