embulk-output-kintone 0.2.2 → 0.3.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 59e8a8850c43f8ff293ca6b63d908f6e7e04b1ac
4
- data.tar.gz: 42f8ae842db0e3177b1123dc45dd7919f2cb903d
3
+ metadata.gz: 12557e033d23e679cb3b6c54b74187083a36d9c4
4
+ data.tar.gz: 11bdecefdebb13d075410e56ed9ba3e42bdb51b7
5
5
  SHA512:
6
- metadata.gz: 447d0a6901e2bf7d4993fb9f20291f236199bd53e3ae4924b85444c64e8b4fa78a2bccf3c42083edde6c6d863953a6d6f1e70be4d629dd7f35ddc63ab81276de
7
- data.tar.gz: 8a6fb5c6cde0a75377b95641cadacc243ca7ab0ddd03d641df37f055c574a33552bd045b13cfa6236f49d570188c490d83dedb5715bd2908a53746af721c060c
6
+ metadata.gz: 9000840a27d24ad547435f6be018036c8958a522373d5bec83505ab14b05a26050c415e638b9c583fd19b400fb9b3a00599307805c4e7ad3a4134336ef57290a
7
+ data.tar.gz: 6e750bff291b90df9f7c90225b1f324d9c27289c3e487d856a607a6e5605dcb612c9fdecb33809e9c46970bb3686b2ac0c389b9a6c1bba8b68d5a6b9230507fd
@@ -1,13 +1,60 @@
1
- name: Java CI
1
+ name: main
2
2
 
3
- on: [push, pull_request]
3
+ on:
4
+ push:
5
+ branches:
6
+ - 'master'
7
+ tags:
8
+ - '*'
9
+ pull_request:
10
+ branches:
11
+ - 'master'
12
+ types: [opened, synchronize]
4
13
 
5
14
  jobs:
6
- build:
15
+ main:
7
16
  runs-on: ubuntu-latest
8
17
  steps:
9
18
  - uses: actions/checkout@v2
10
- - uses: actions/setup-java@v1
19
+ - name: Set Up
20
+ uses: actions/setup-java@v1
11
21
  with:
12
22
  java-version: 1.8
13
- - run: ./gradlew build
23
+
24
+ - name: Test
25
+ run: ./gradlew test
26
+
27
+ - name: Build Gem
28
+ run: ./gradlew gem
29
+
30
+ - name: Set up JRuby
31
+ uses: ruby/setup-ruby@v1
32
+ with:
33
+ ruby-version: jruby
34
+
35
+ - name: Publish to GPR
36
+ if: startsWith( github.ref, 'refs/tags/' )
37
+ run: |
38
+ mkdir -p $HOME/.gem
39
+ touch $HOME/.gem/credentials
40
+ chmod 0600 $HOME/.gem/credentials
41
+ printf -- "---\n:github: ${GEM_HOST_API_KEY}\n" > $HOME/.gem/credentials
42
+ gem push --KEY github --host https://rubygems.pkg.github.com/${OWNER} ./pkg/*.gem
43
+ env:
44
+ GEM_HOST_API_KEY: "Bearer ${{secrets.GITHUB_TOKEN}}"
45
+ OWNER: ${{ github.repository_owner }}
46
+
47
+ - name: Publish to RubyGems
48
+ if: startsWith( github.ref, 'refs/tags/' )
49
+ run: |
50
+ mkdir -p $HOME/.gem
51
+ touch $HOME/.gem/credentials
52
+ chmod 0600 $HOME/.gem/credentials
53
+ printf -- "---\n:rubygems_api_key: ${GEM_HOST_API_KEY}\n" > $HOME/.gem/credentials
54
+ # TODO: If it is possible to accept input in the middle of a step, then the OTP Token should be inputted instead of generated.
55
+ gem install rotp -v 6.2.0
56
+ OTP_TOKEN=$(echo ${OTP_SECRET} | ruby -rtime -rrotp -e "puts ROTP::TOTP.new(STDIN.read.chomp, issuer: 'rubygems.org').at(Time.now)")
57
+ gem push --otp="${OTP_TOKEN}" ./pkg/*.gem
58
+ env:
59
+ GEM_HOST_API_KEY: "${{secrets.RUBYGEMS_API_KEY}}"
60
+ OTP_SECRET: "${{secrets.RUBYGEMS_OTP_SECRET}}"
data/README.md CHANGED
@@ -17,11 +17,11 @@ kintone output plugin for Embulk stores app records from kintone.
17
17
  - **basic_auth_password**: kintone basic auth password (string, optional)
18
18
  - **guest_space_id**: kintone app belongs to guest space, guest space id is required. (integer, optional)
19
19
  - **mode**: kintone mode (string, required)
20
+ - **update_key**: column name to set update key (string, required if mode is update or upsert)
20
21
  - **column_options** advanced: a key-value pairs where key is a column name and value is options for the column.
21
22
  - **field_code**: field code (string, required)
22
23
  - **type**: field type (string, required)
23
24
  - **timezone**: timezone to convert into `date` (string, default is `UTC`)
24
- - **update_key**: update key (boolean, default is `false`)
25
25
  - **val_sep**: Used to specify multiple checkbox values (string, default is `,`)
26
26
 
27
27
  ## Example
@@ -33,7 +33,8 @@ out:
33
33
  username: username
34
34
  password: password
35
35
  app_id: 1
36
- mode: insert
36
+ mode: upsert
37
+ update_key: id
37
38
  column_options:
38
39
  id: {field_code: "id", type: "NUMBER"}
39
40
  name: {field_code: "name", type: "SINGLE_LINE_TEXT"}
data/build.gradle CHANGED
@@ -14,7 +14,7 @@ configurations {
14
14
  provided
15
15
  }
16
16
 
17
- version = "0.2.2"
17
+ version = "0.3.2"
18
18
 
19
19
  sourceCompatibility = 1.8
20
20
  targetCompatibility = 1.8
@@ -10,20 +10,16 @@ public interface KintoneColumnOption
10
10
  extends Task
11
11
  {
12
12
  @Config("type")
13
- public String getType();
13
+ String getType();
14
14
 
15
15
  @Config("field_code")
16
- public String getFieldCode();
16
+ String getFieldCode();
17
17
 
18
18
  @Config("timezone")
19
19
  @ConfigDefault("\"UTC\"")
20
- public Optional<String> getTimezone();
21
-
22
- @Config("update_key")
23
- @ConfigDefault("false")
24
- public boolean getUpdateKey();
20
+ Optional<String> getTimezone();
25
21
 
26
22
  @Config("val_sep")
27
23
  @ConfigDefault("\",\"")
28
- public String getValueSeparator();
24
+ String getValueSeparator();
29
25
  }
@@ -22,14 +22,16 @@ import java.time.Instant;
22
22
  import java.time.ZoneId;
23
23
  import java.time.ZonedDateTime;
24
24
  import java.util.Map;
25
+ import java.util.Objects;
25
26
 
26
27
  public class KintoneColumnVisitor
27
28
  implements ColumnVisitor
28
29
  {
29
- private PageReader pageReader;
30
+ private final PageReader pageReader;
30
31
  private Record record;
31
32
  private UpdateKey updateKey;
32
- private Map<String, KintoneColumnOption> columnOptions;
33
+ private final Map<String, KintoneColumnOption> columnOptions;
34
+ private String updateKeyName;
33
35
 
34
36
  public KintoneColumnVisitor(PageReader pageReader,
35
37
  Map<String, KintoneColumnOption> columnOptions)
@@ -38,6 +40,15 @@ public class KintoneColumnVisitor
38
40
  this.columnOptions = columnOptions;
39
41
  }
40
42
 
43
+ public KintoneColumnVisitor(PageReader pageReader,
44
+ Map<String, KintoneColumnOption> columnOptions,
45
+ String updateKeyName)
46
+ {
47
+ this.pageReader = pageReader;
48
+ this.columnOptions = columnOptions;
49
+ this.updateKeyName = updateKeyName;
50
+ }
51
+
41
52
  public void setRecord(Record record)
42
53
  {
43
54
  this.record = record;
@@ -50,20 +61,17 @@ public class KintoneColumnVisitor
50
61
 
51
62
  private void setValue(String fieldCode, Object value, FieldType type, boolean isUpdateKey)
52
63
  {
53
- if (value == null) {
54
- return;
55
- }
56
-
57
64
  if (isUpdateKey && updateKey != null) {
58
65
  updateKey
59
66
  .setField(fieldCode)
60
- .setValue(String.valueOf(value));
67
+ .setValue(Objects.toString(value, ""));
61
68
  }
62
- String stringValue = String.valueOf(value);
63
- FieldValue fieldValue = null;
69
+ String stringValue = Objects.toString(value, "");
70
+ FieldValue fieldValue;
64
71
  switch (type) {
65
72
  case NUMBER:
66
- fieldValue = new NumberFieldValue(new BigDecimal(stringValue));
73
+ BigDecimal setValue = stringValue.equals("") ? null : new BigDecimal(stringValue);
74
+ fieldValue = new NumberFieldValue(setValue);
67
75
  break;
68
76
  case MULTI_LINE_TEXT:
69
77
  fieldValue = new MultiLineTextFieldValue(stringValue);
@@ -128,16 +136,19 @@ public class KintoneColumnVisitor
128
136
  private ZoneId getZoneId(Column column)
129
137
  {
130
138
  KintoneColumnOption option = columnOptions.get(column.getName());
131
- return ZoneId.of(option.getTimezone().get());
139
+ if (option == null) {
140
+ return ZoneId.of("UTC");
141
+ }
142
+ return ZoneId.of(option.getTimezone().orElse("UTC"));
132
143
  }
133
144
 
134
145
  private boolean isUpdateKey(Column column)
135
146
  {
136
- KintoneColumnOption option = columnOptions.get(column.getName());
137
- if (option == null) {
147
+ if (this.updateKeyName == null) {
138
148
  return false;
139
149
  }
140
- return option.getUpdateKey();
150
+
151
+ return this.updateKeyName.equals(column.getName());
141
152
  }
142
153
 
143
154
  private String getValueSeparator(Column column)
@@ -162,7 +173,11 @@ public class KintoneColumnVisitor
162
173
  {
163
174
  String fieldCode = getFieldCode(column);
164
175
  FieldType type = getType(column, FieldType.NUMBER);
165
- setValue(fieldCode, pageReader.getLong(column), type, isUpdateKey(column));
176
+ if (pageReader.isNull(column)) {
177
+ setValue(fieldCode, null, type, isUpdateKey(column));
178
+ } else {
179
+ setValue(fieldCode, pageReader.getLong(column), type, isUpdateKey(column));
180
+ }
166
181
  }
167
182
 
168
183
  @Override
@@ -1,6 +1,7 @@
1
1
  package org.embulk.output.kintone;
2
2
 
3
3
  import org.embulk.config.ConfigDiff;
4
+ import org.embulk.config.ConfigException;
4
5
  import org.embulk.config.ConfigSource;
5
6
  import org.embulk.config.TaskReport;
6
7
  import org.embulk.config.TaskSource;
@@ -9,7 +10,6 @@ import org.embulk.spi.OutputPlugin;
9
10
  import org.embulk.spi.Schema;
10
11
  import org.embulk.spi.TransactionalPageOutput;
11
12
 
12
- import java.util.Collection;
13
13
  import java.util.List;
14
14
 
15
15
  public class KintoneOutputPlugin
@@ -45,37 +45,22 @@ public class KintoneOutputPlugin
45
45
  public TransactionalPageOutput open(TaskSource taskSource, Schema schema, int taskIndex)
46
46
  {
47
47
  PluginTask task = taskSource.loadTask(PluginTask.class);
48
- Collection<KintoneColumnOption> options = task.getColumnOptions().values();
49
48
 
50
49
  KintoneMode mode = KintoneMode.getKintoneModeByValue(task.getMode());
51
50
  switch (mode) {
52
51
  case INSERT:
53
- for (KintoneColumnOption option : options) {
54
- if (option.getUpdateKey()) {
55
- throw new IllegalArgumentException(
56
- "when mode is insert, require no update_key.");
57
- }
52
+ if (task.getUpdateKeyName().isPresent()) {
53
+ throw new ConfigException("when mode is insert, require no update_key.");
58
54
  }
59
55
  break;
60
56
  case UPDATE:
61
57
  case UPSERT:
62
- boolean hasUpdateKey = false;
63
- for (KintoneColumnOption option : options) {
64
- if (option.getUpdateKey()) {
65
- if (hasUpdateKey) {
66
- throw new IllegalArgumentException(
67
- "when mode is update and upsert, only one column can have an update_key.");
68
- }
69
- hasUpdateKey = true;
70
- }
71
- }
72
- if (!hasUpdateKey) {
73
- throw new IllegalArgumentException(
74
- "when mode is update and upsert, require update_key.");
58
+ if (!task.getUpdateKeyName().isPresent()) {
59
+ throw new ConfigException("when mode is update or upsert, require update_key.");
75
60
  }
76
61
  break;
77
62
  default:
78
- throw new IllegalArgumentException(String.format(
63
+ throw new ConfigException(String.format(
79
64
  "Unknown mode '%s'",
80
65
  task.getMode()));
81
66
  }
@@ -3,6 +3,7 @@ package org.embulk.output.kintone;
3
3
  import com.kintone.client.KintoneClient;
4
4
  import com.kintone.client.KintoneClientBuilder;
5
5
  import com.kintone.client.api.record.GetRecordsByCursorResponseBody;
6
+ import com.kintone.client.model.record.FieldType;
6
7
  import com.kintone.client.model.record.Record;
7
8
  import com.kintone.client.model.record.RecordForUpdate;
8
9
  import com.kintone.client.model.record.UpdateKey;
@@ -14,17 +15,18 @@ import org.embulk.spi.PageReader;
14
15
  import org.embulk.spi.Schema;
15
16
  import org.embulk.spi.TransactionalPageOutput;
16
17
 
17
- import java.math.BigDecimal;
18
18
  import java.util.ArrayList;
19
- import java.util.Arrays;
19
+ import java.util.Collections;
20
20
  import java.util.List;
21
21
  import java.util.stream.Collectors;
22
22
 
23
23
  public class KintonePageOutput
24
24
  implements TransactionalPageOutput
25
25
  {
26
- private PageReader pageReader;
27
- private PluginTask task;
26
+ public static final int UPSERT_BATCH_SIZE = 10000;
27
+ public static final int CHUNK_SIZE = 100;
28
+ private final PageReader pageReader;
29
+ private final PluginTask task;
28
30
  private KintoneClient client;
29
31
 
30
32
  public KintonePageOutput(PluginTask task, Schema schema)
@@ -88,14 +90,14 @@ public class KintonePageOutput
88
90
 
89
91
  public interface Consumer<T>
90
92
  {
91
- public void accept(T t);
93
+ void accept(T t);
92
94
  }
93
95
 
94
96
  public void connect(final PluginTask task)
95
97
  {
96
98
  KintoneClientBuilder builder = KintoneClientBuilder.create("https://" + task.getDomain());
97
99
  if (task.getGuestSpaceId().isPresent()) {
98
- builder.setGuestSpaceId(task.getGuestSpaceId().or(-1));
100
+ builder.setGuestSpaceId(task.getGuestSpaceId().orElse(-1));
99
101
  }
100
102
  if (task.getBasicAuthUsername().isPresent() && task.getBasicAuthPassword().isPresent()) {
101
103
  builder.withBasicAuth(task.getBasicAuthUsername().get(),
@@ -136,7 +138,7 @@ public class KintonePageOutput
136
138
  }
137
139
 
138
140
  records.add(record);
139
- if (records.size() == 100) {
141
+ if (records.size() == CHUNK_SIZE) {
140
142
  client.record().addRecords(task.getAppId(), records);
141
143
  records.clear();
142
144
  }
@@ -155,10 +157,12 @@ public class KintonePageOutput
155
157
  {
156
158
  execute(client -> {
157
159
  try {
158
- ArrayList<RecordForUpdate> updateRecords = new ArrayList<RecordForUpdate>();
160
+ ArrayList<RecordForUpdate> updateRecords = new ArrayList<>();
159
161
  pageReader.setPage(page);
162
+
160
163
  KintoneColumnVisitor visitor = new KintoneColumnVisitor(pageReader,
161
- task.getColumnOptions());
164
+ task.getColumnOptions(),
165
+ task.getUpdateKeyName().orElseThrow(() -> new RuntimeException("unreachable"))); // Already validated
162
166
  while (pageReader.nextRecord()) {
163
167
  Record record = new Record();
164
168
  UpdateKey updateKey = new UpdateKey();
@@ -168,9 +172,13 @@ public class KintonePageOutput
168
172
  column.visit(visitor);
169
173
  }
170
174
 
175
+ if (updateKey.getValue() == "") {
176
+ continue;
177
+ }
178
+
171
179
  record.removeField(updateKey.getField());
172
180
  updateRecords.add(new RecordForUpdate(updateKey, record));
173
- if (updateRecords.size() == 100) {
181
+ if (updateRecords.size() == CHUNK_SIZE) {
174
182
  client.record().updateRecords(task.getAppId(), updateRecords);
175
183
  updateRecords.clear();
176
184
  }
@@ -185,183 +193,130 @@ public class KintonePageOutput
185
193
  });
186
194
  }
187
195
 
188
- private List<Record> getRecordsByUpdateKey(String fieldCode, List<String> queryValues)
196
+ private void upsertPage(final Page page)
189
197
  {
190
- List<Record> allRecords = new ArrayList<Record>();
191
- List<String> fields = Arrays.asList(fieldCode);
192
- String cursorId = client.record().createCursor(
193
- task.getAppId(),
194
- fields,
195
- fieldCode + " in (" + String.join(",", queryValues) + ")"
196
- );
197
- while (true) {
198
- GetRecordsByCursorResponseBody resp = client.record().getRecordsByCursor(cursorId);
199
- List<Record> records = resp.getRecords();
200
- allRecords.addAll(records);
198
+ execute(client -> {
199
+ try {
200
+ ArrayList<Record> records = new ArrayList<>();
201
+ ArrayList<UpdateKey> updateKeys = new ArrayList<>();
202
+ pageReader.setPage(page);
201
203
 
202
- if (!resp.hasNext()) {
203
- break;
204
+ KintoneColumnVisitor visitor = new KintoneColumnVisitor(pageReader,
205
+ task.getColumnOptions(),
206
+ task.getUpdateKeyName().orElseThrow(() -> new RuntimeException("unreachable"))); // Already validated
207
+ while (pageReader.nextRecord()) {
208
+ Record record = new Record();
209
+ UpdateKey updateKey = new UpdateKey();
210
+ visitor.setRecord(record);
211
+ visitor.setUpdateKey(updateKey);
212
+ for (Column column : pageReader.getSchema().getColumns()) {
213
+ column.visit(visitor);
214
+ }
215
+ records.add(record);
216
+ updateKeys.add(updateKey);
217
+
218
+ if (records.size() == UPSERT_BATCH_SIZE) {
219
+ upsert(records, updateKeys);
220
+ records.clear();
221
+ updateKeys.clear();
222
+ }
223
+ }
224
+ if (records.size() > 0) {
225
+ upsert(records, updateKeys);
226
+ }
204
227
  }
205
- }
206
- return allRecords;
228
+ catch (Exception e) {
229
+ throw new RuntimeException("kintone throw exception", e);
230
+ }
231
+ });
207
232
  }
208
233
 
209
- abstract class UpsertPage<T>
234
+ private void upsert(ArrayList<Record> records, ArrayList<UpdateKey> updateKeys)
210
235
  {
211
- public abstract List<T> getUpdateKeyValues(List<String> queryValues);
212
- public abstract boolean existsRecord(List<T> updateKeyValues, Record record);
213
-
214
- public void upsert(ArrayList<Record> records, ArrayList<UpdateKey> updateKeys)
215
- {
216
- if (records.size() != updateKeys.size()) {
217
- throw new RuntimeException("records.size() != updateKeys.size()");
218
- }
219
-
220
- List<String> queryValues = updateKeys
221
- .stream()
222
- .map(k -> "\"" + k.getValue().toString() + "\"")
223
- .collect(Collectors.toList());
224
- List<T> updateKeyValues = getUpdateKeyValues(queryValues);
236
+ if (records.size() != updateKeys.size()) {
237
+ throw new RuntimeException("records.size() != updateKeys.size()");
238
+ }
225
239
 
226
- ArrayList<Record> insertRecords = new ArrayList<>();
227
- ArrayList<RecordForUpdate> updateRecords = new ArrayList<RecordForUpdate>();
228
- for (int i = 0; i < records.size(); i++) {
229
- Record record = records.get(i);
230
- UpdateKey updateKey = updateKeys.get(i);
240
+ List<Record> existingRecords = getExistingRecordsByUpdateKey(updateKeys);
231
241
 
232
- if (existsRecord(updateKeyValues, record)) {
233
- record.removeField(updateKey.getField());
234
- updateRecords.add(new RecordForUpdate(updateKey, record));
235
- }
236
- else {
237
- insertRecords.add(record);
238
- }
242
+ ArrayList<Record> insertRecords = new ArrayList<>();
243
+ ArrayList<RecordForUpdate> updateRecords = new ArrayList<>();
244
+ for (int i = 0; i < records.size(); i++) {
245
+ Record record = records.get(i);
246
+ UpdateKey updateKey = updateKeys.get(i);
239
247
 
240
- if (insertRecords.size() == 100) {
241
- client.record().addRecords(task.getAppId(), insertRecords);
242
- insertRecords.clear();
243
- }
244
- else if (updateRecords.size() == 100) {
245
- client.record().updateRecords(task.getAppId(), updateRecords);
246
- updateRecords.clear();
247
- }
248
+ if (existsRecord(existingRecords, updateKey)) {
249
+ record.removeField(updateKey.getField());
250
+ updateRecords.add(new RecordForUpdate(updateKey, record));
248
251
  }
249
- if (insertRecords.size() > 0) {
252
+ else {
253
+ insertRecords.add(record);
254
+ }
255
+
256
+ if (insertRecords.size() == CHUNK_SIZE) {
250
257
  client.record().addRecords(task.getAppId(), insertRecords);
258
+ insertRecords.clear();
251
259
  }
252
- if (updateRecords.size() > 0) {
260
+ else if (updateRecords.size() == CHUNK_SIZE) {
253
261
  client.record().updateRecords(task.getAppId(), updateRecords);
262
+ updateRecords.clear();
254
263
  }
255
264
  }
256
-
257
- public void run(final Page page)
258
- {
259
- execute(client -> {
260
- try {
261
- ArrayList<Record> records = new ArrayList<>();
262
- ArrayList<UpdateKey> updateKeys = new ArrayList<>();
263
-
264
- pageReader.setPage(page);
265
- KintoneColumnVisitor visitor = new KintoneColumnVisitor(pageReader,
266
- task.getColumnOptions());
267
- while (pageReader.nextRecord()) {
268
- Record record = new Record();
269
- UpdateKey updateKey = new UpdateKey();
270
- visitor.setRecord(record);
271
- visitor.setUpdateKey(updateKey);
272
- for (Column column : pageReader.getSchema().getColumns()) {
273
- column.visit(visitor);
274
- }
275
- records.add(record);
276
- updateKeys.add(updateKey);
277
-
278
- if (records.size() == 10000) {
279
- upsert(records, updateKeys);
280
- records.clear();
281
- updateKeys.clear();
282
- }
283
- }
284
- if (records.size() > 0) {
285
- upsert(records, updateKeys);
286
- }
287
- }
288
- catch (Exception e) {
289
- throw new RuntimeException("kintone throw exception", e);
290
- }
291
- });
292
- }
293
- }
294
-
295
- class UpsertPageByStringKey extends UpsertPage<String>
296
- {
297
- private String fieldCode;
298
-
299
- public UpsertPageByStringKey(String fieldCode)
300
- {
301
- this.fieldCode = fieldCode;
302
- }
303
-
304
- public List<String> getUpdateKeyValues(List<String> queryValues)
305
- {
306
- return getRecordsByUpdateKey(fieldCode, queryValues)
307
- .stream()
308
- .map(r -> r.getSingleLineTextFieldValue(fieldCode))
309
- .collect(Collectors.toList());
265
+ if (insertRecords.size() > 0) {
266
+ client.record().addRecords(task.getAppId(), insertRecords);
310
267
  }
311
-
312
- public boolean existsRecord(List<String> updateKeyValues, Record record)
313
- {
314
- return updateKeyValues.contains(record.getSingleLineTextFieldValue(fieldCode));
268
+ if (updateRecords.size() > 0) {
269
+ client.record().updateRecords(task.getAppId(), updateRecords);
315
270
  }
316
271
  }
317
272
 
318
- class UpsertPageByNumberKey extends UpsertPage<BigDecimal>
319
- {
320
- private String fieldCode;
321
-
322
- public UpsertPageByNumberKey(String fieldCode)
323
- {
324
- this.fieldCode = fieldCode;
325
- }
326
273
 
327
- public List<BigDecimal> getUpdateKeyValues(List<String> queryValues)
328
- {
329
- return getRecordsByUpdateKey(fieldCode, queryValues)
274
+ private List<Record> getExistingRecordsByUpdateKey(ArrayList<UpdateKey> updateKeys)
275
+ {
276
+ String fieldCode = updateKeys.get(0).getField();
277
+ List<String> queryValues = updateKeys
330
278
  .stream()
331
- .map(r -> r.getNumberFieldValue(fieldCode))
279
+ .filter(k -> k.getValue() != "")
280
+ .map(k -> "\"" + k.getValue().toString() + "\"")
332
281
  .collect(Collectors.toList());
333
- }
334
282
 
335
- public boolean existsRecord(List<BigDecimal> updateKeyValues, Record record)
336
- {
337
- return updateKeyValues.contains(record.getNumberFieldValue(fieldCode));
283
+ List<Record> allRecords = new ArrayList<>();
284
+ if (queryValues.isEmpty()) {
285
+ return allRecords;
338
286
  }
339
- }
287
+ String cursorId = client.record().createCursor(
288
+ task.getAppId(),
289
+ Collections.singletonList(fieldCode),
290
+ fieldCode + " in (" + String.join(",", queryValues) + ")"
291
+ );
292
+ while (true) {
293
+ GetRecordsByCursorResponseBody resp = client.record().getRecordsByCursor(cursorId);
294
+ List<Record> records = resp.getRecords();
295
+ allRecords.addAll(records);
340
296
 
341
- private void upsertPage(final Page page)
342
- {
343
- KintoneColumnOption updateKeyColumn = null;
344
- for (KintoneColumnOption v : task.getColumnOptions().values()) {
345
- if (v.getUpdateKey()) {
346
- updateKeyColumn = v;
297
+ if (!resp.hasNext()) {
347
298
  break;
348
299
  }
349
300
  }
350
- if (updateKeyColumn == null) {
351
- throw new RuntimeException("when mode is upsert, require update_key");
352
- }
301
+ return allRecords;
302
+ }
353
303
 
354
- UpsertPage runner;
355
- switch(updateKeyColumn.getType()) {
356
- case "SINGLE_LINE_TEXT":
357
- runner = new UpsertPageByStringKey(updateKeyColumn.getFieldCode());
358
- break;
359
- case "NUMBER":
360
- runner = new UpsertPageByNumberKey(updateKeyColumn.getFieldCode());
361
- break;
304
+ private boolean existsRecord(List<Record> distRecords, UpdateKey updateKey)
305
+ {
306
+ String fieldCode = updateKey.getField();
307
+ FieldType type = client.app().getFormFields(task.getAppId()).get(fieldCode).getType();
308
+ switch (type) {
309
+ case SINGLE_LINE_TEXT:
310
+ return distRecords
311
+ .stream()
312
+ .anyMatch(d -> d.getSingleLineTextFieldValue(fieldCode).equals(updateKey.getValue().toString()));
313
+ case NUMBER:
314
+ return distRecords
315
+ .stream()
316
+ .anyMatch(d -> d.getNumberFieldValue(fieldCode).toPlainString()
317
+ .equals(updateKey.getValue().toString()));
362
318
  default:
363
319
  throw new RuntimeException("The update_key must be 'SINGLE_LINE_TEXT' or 'NUMBER'.");
364
320
  }
365
- runner.run(page);
366
321
  }
367
322
  }
@@ -1,11 +1,11 @@
1
1
  package org.embulk.output.kintone;
2
2
 
3
- import com.google.common.base.Optional;
4
3
  import org.embulk.config.Config;
5
4
  import org.embulk.config.ConfigDefault;
6
5
  import org.embulk.config.Task;
7
6
 
8
7
  import java.util.Map;
8
+ import java.util.Optional;
9
9
 
10
10
  public interface PluginTask
11
11
  extends Task
@@ -42,9 +42,13 @@ public interface PluginTask
42
42
 
43
43
  @Config("column_options")
44
44
  @ConfigDefault("{}")
45
- public Map<String, KintoneColumnOption> getColumnOptions();
45
+ Map<String, KintoneColumnOption> getColumnOptions();
46
46
 
47
47
  @Config("mode")
48
48
  @ConfigDefault("insert")
49
- public String getMode();
49
+ String getMode();
50
+
51
+ @Config("update_key")
52
+ @ConfigDefault("null")
53
+ Optional<String> getUpdateKeyName();
50
54
  }
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: embulk-output-kintone
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.2
4
+ version: 0.3.2
5
5
  platform: ruby
6
6
  authors:
7
7
  - takeshi fujita
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2020-11-18 00:00:00.000000000 Z
11
+ date: 2022-03-23 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
@@ -52,7 +52,7 @@ files:
52
52
  - README.md
53
53
  - Rakefile
54
54
  - build.gradle
55
- - classpath/embulk-output-kintone-0.2.2.jar
55
+ - classpath/embulk-output-kintone-0.3.2.jar
56
56
  - config/checkstyle/checkstyle.xml
57
57
  - config/checkstyle/default.xml
58
58
  - gradle/wrapper/gradle-wrapper.jar