embulk-output-kintone 0.2.2 → 0.3.2

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 59e8a8850c43f8ff293ca6b63d908f6e7e04b1ac
4
- data.tar.gz: 42f8ae842db0e3177b1123dc45dd7919f2cb903d
3
+ metadata.gz: 12557e033d23e679cb3b6c54b74187083a36d9c4
4
+ data.tar.gz: 11bdecefdebb13d075410e56ed9ba3e42bdb51b7
5
5
  SHA512:
6
- metadata.gz: 447d0a6901e2bf7d4993fb9f20291f236199bd53e3ae4924b85444c64e8b4fa78a2bccf3c42083edde6c6d863953a6d6f1e70be4d629dd7f35ddc63ab81276de
7
- data.tar.gz: 8a6fb5c6cde0a75377b95641cadacc243ca7ab0ddd03d641df37f055c574a33552bd045b13cfa6236f49d570188c490d83dedb5715bd2908a53746af721c060c
6
+ metadata.gz: 9000840a27d24ad547435f6be018036c8958a522373d5bec83505ab14b05a26050c415e638b9c583fd19b400fb9b3a00599307805c4e7ad3a4134336ef57290a
7
+ data.tar.gz: 6e750bff291b90df9f7c90225b1f324d9c27289c3e487d856a607a6e5605dcb612c9fdecb33809e9c46970bb3686b2ac0c389b9a6c1bba8b68d5a6b9230507fd
@@ -1,13 +1,60 @@
1
- name: Java CI
1
+ name: main
2
2
 
3
- on: [push, pull_request]
3
+ on:
4
+ push:
5
+ branches:
6
+ - 'master'
7
+ tags:
8
+ - '*'
9
+ pull_request:
10
+ branches:
11
+ - 'master'
12
+ types: [opened, synchronize]
4
13
 
5
14
  jobs:
6
- build:
15
+ main:
7
16
  runs-on: ubuntu-latest
8
17
  steps:
9
18
  - uses: actions/checkout@v2
10
- - uses: actions/setup-java@v1
19
+ - name: Set Up
20
+ uses: actions/setup-java@v1
11
21
  with:
12
22
  java-version: 1.8
13
- - run: ./gradlew build
23
+
24
+ - name: Test
25
+ run: ./gradlew test
26
+
27
+ - name: Build Gem
28
+ run: ./gradlew gem
29
+
30
+ - name: Set up JRuby
31
+ uses: ruby/setup-ruby@v1
32
+ with:
33
+ ruby-version: jruby
34
+
35
+ - name: Publish to GPR
36
+ if: startsWith( github.ref, 'refs/tags/' )
37
+ run: |
38
+ mkdir -p $HOME/.gem
39
+ touch $HOME/.gem/credentials
40
+ chmod 0600 $HOME/.gem/credentials
41
+ printf -- "---\n:github: ${GEM_HOST_API_KEY}\n" > $HOME/.gem/credentials
42
+ gem push --KEY github --host https://rubygems.pkg.github.com/${OWNER} ./pkg/*.gem
43
+ env:
44
+ GEM_HOST_API_KEY: "Bearer ${{secrets.GITHUB_TOKEN}}"
45
+ OWNER: ${{ github.repository_owner }}
46
+
47
+ - name: Publish to RubyGems
48
+ if: startsWith( github.ref, 'refs/tags/' )
49
+ run: |
50
+ mkdir -p $HOME/.gem
51
+ touch $HOME/.gem/credentials
52
+ chmod 0600 $HOME/.gem/credentials
53
+ printf -- "---\n:rubygems_api_key: ${GEM_HOST_API_KEY}\n" > $HOME/.gem/credentials
54
+ # TODO: If it is possible to accept input in the middle of a step, then the OTP Token should be inputted instead of generated.
55
+ gem install rotp -v 6.2.0
56
+ OTP_TOKEN=$(echo ${OTP_SECRET} | ruby -rtime -rrotp -e "puts ROTP::TOTP.new(STDIN.read.chomp, issuer: 'rubygems.org').at(Time.now)")
57
+ gem push --otp="${OTP_TOKEN}" ./pkg/*.gem
58
+ env:
59
+ GEM_HOST_API_KEY: "${{secrets.RUBYGEMS_API_KEY}}"
60
+ OTP_SECRET: "${{secrets.RUBYGEMS_OTP_SECRET}}"
data/README.md CHANGED
@@ -17,11 +17,11 @@ kintone output plugin for Embulk stores app records from kintone.
17
17
  - **basic_auth_password**: kintone basic auth password (string, optional)
18
18
  - **guest_space_id**: kintone app belongs to guest space, guest space id is required. (integer, optional)
19
19
  - **mode**: kintone mode (string, required)
20
+ - **update_key**: column name to set update key (string, required if mode is update or upsert)
20
21
  - **column_options** advanced: a key-value pairs where key is a column name and value is options for the column.
21
22
  - **field_code**: field code (string, required)
22
23
  - **type**: field type (string, required)
23
24
  - **timezone**: timezone to convert into `date` (string, default is `UTC`)
24
- - **update_key**: update key (boolean, default is `false`)
25
25
  - **val_sep**: Used to specify multiple checkbox values (string, default is `,`)
26
26
 
27
27
  ## Example
@@ -33,7 +33,8 @@ out:
33
33
  username: username
34
34
  password: password
35
35
  app_id: 1
36
- mode: insert
36
+ mode: upsert
37
+ update_key: id
37
38
  column_options:
38
39
  id: {field_code: "id", type: "NUMBER"}
39
40
  name: {field_code: "name", type: "SINGLE_LINE_TEXT"}
data/build.gradle CHANGED
@@ -14,7 +14,7 @@ configurations {
14
14
  provided
15
15
  }
16
16
 
17
- version = "0.2.2"
17
+ version = "0.3.2"
18
18
 
19
19
  sourceCompatibility = 1.8
20
20
  targetCompatibility = 1.8
@@ -10,20 +10,16 @@ public interface KintoneColumnOption
10
10
  extends Task
11
11
  {
12
12
  @Config("type")
13
- public String getType();
13
+ String getType();
14
14
 
15
15
  @Config("field_code")
16
- public String getFieldCode();
16
+ String getFieldCode();
17
17
 
18
18
  @Config("timezone")
19
19
  @ConfigDefault("\"UTC\"")
20
- public Optional<String> getTimezone();
21
-
22
- @Config("update_key")
23
- @ConfigDefault("false")
24
- public boolean getUpdateKey();
20
+ Optional<String> getTimezone();
25
21
 
26
22
  @Config("val_sep")
27
23
  @ConfigDefault("\",\"")
28
- public String getValueSeparator();
24
+ String getValueSeparator();
29
25
  }
@@ -22,14 +22,16 @@ import java.time.Instant;
22
22
  import java.time.ZoneId;
23
23
  import java.time.ZonedDateTime;
24
24
  import java.util.Map;
25
+ import java.util.Objects;
25
26
 
26
27
  public class KintoneColumnVisitor
27
28
  implements ColumnVisitor
28
29
  {
29
- private PageReader pageReader;
30
+ private final PageReader pageReader;
30
31
  private Record record;
31
32
  private UpdateKey updateKey;
32
- private Map<String, KintoneColumnOption> columnOptions;
33
+ private final Map<String, KintoneColumnOption> columnOptions;
34
+ private String updateKeyName;
33
35
 
34
36
  public KintoneColumnVisitor(PageReader pageReader,
35
37
  Map<String, KintoneColumnOption> columnOptions)
@@ -38,6 +40,15 @@ public class KintoneColumnVisitor
38
40
  this.columnOptions = columnOptions;
39
41
  }
40
42
 
43
+ public KintoneColumnVisitor(PageReader pageReader,
44
+ Map<String, KintoneColumnOption> columnOptions,
45
+ String updateKeyName)
46
+ {
47
+ this.pageReader = pageReader;
48
+ this.columnOptions = columnOptions;
49
+ this.updateKeyName = updateKeyName;
50
+ }
51
+
41
52
  public void setRecord(Record record)
42
53
  {
43
54
  this.record = record;
@@ -50,20 +61,17 @@ public class KintoneColumnVisitor
50
61
 
51
62
  private void setValue(String fieldCode, Object value, FieldType type, boolean isUpdateKey)
52
63
  {
53
- if (value == null) {
54
- return;
55
- }
56
-
57
64
  if (isUpdateKey && updateKey != null) {
58
65
  updateKey
59
66
  .setField(fieldCode)
60
- .setValue(String.valueOf(value));
67
+ .setValue(Objects.toString(value, ""));
61
68
  }
62
- String stringValue = String.valueOf(value);
63
- FieldValue fieldValue = null;
69
+ String stringValue = Objects.toString(value, "");
70
+ FieldValue fieldValue;
64
71
  switch (type) {
65
72
  case NUMBER:
66
- fieldValue = new NumberFieldValue(new BigDecimal(stringValue));
73
+ BigDecimal setValue = stringValue.equals("") ? null : new BigDecimal(stringValue);
74
+ fieldValue = new NumberFieldValue(setValue);
67
75
  break;
68
76
  case MULTI_LINE_TEXT:
69
77
  fieldValue = new MultiLineTextFieldValue(stringValue);
@@ -128,16 +136,19 @@ public class KintoneColumnVisitor
128
136
  private ZoneId getZoneId(Column column)
129
137
  {
130
138
  KintoneColumnOption option = columnOptions.get(column.getName());
131
- return ZoneId.of(option.getTimezone().get());
139
+ if (option == null) {
140
+ return ZoneId.of("UTC");
141
+ }
142
+ return ZoneId.of(option.getTimezone().orElse("UTC"));
132
143
  }
133
144
 
134
145
  private boolean isUpdateKey(Column column)
135
146
  {
136
- KintoneColumnOption option = columnOptions.get(column.getName());
137
- if (option == null) {
147
+ if (this.updateKeyName == null) {
138
148
  return false;
139
149
  }
140
- return option.getUpdateKey();
150
+
151
+ return this.updateKeyName.equals(column.getName());
141
152
  }
142
153
 
143
154
  private String getValueSeparator(Column column)
@@ -162,7 +173,11 @@ public class KintoneColumnVisitor
162
173
  {
163
174
  String fieldCode = getFieldCode(column);
164
175
  FieldType type = getType(column, FieldType.NUMBER);
165
- setValue(fieldCode, pageReader.getLong(column), type, isUpdateKey(column));
176
+ if (pageReader.isNull(column)) {
177
+ setValue(fieldCode, null, type, isUpdateKey(column));
178
+ } else {
179
+ setValue(fieldCode, pageReader.getLong(column), type, isUpdateKey(column));
180
+ }
166
181
  }
167
182
 
168
183
  @Override
@@ -1,6 +1,7 @@
1
1
  package org.embulk.output.kintone;
2
2
 
3
3
  import org.embulk.config.ConfigDiff;
4
+ import org.embulk.config.ConfigException;
4
5
  import org.embulk.config.ConfigSource;
5
6
  import org.embulk.config.TaskReport;
6
7
  import org.embulk.config.TaskSource;
@@ -9,7 +10,6 @@ import org.embulk.spi.OutputPlugin;
9
10
  import org.embulk.spi.Schema;
10
11
  import org.embulk.spi.TransactionalPageOutput;
11
12
 
12
- import java.util.Collection;
13
13
  import java.util.List;
14
14
 
15
15
  public class KintoneOutputPlugin
@@ -45,37 +45,22 @@ public class KintoneOutputPlugin
45
45
  public TransactionalPageOutput open(TaskSource taskSource, Schema schema, int taskIndex)
46
46
  {
47
47
  PluginTask task = taskSource.loadTask(PluginTask.class);
48
- Collection<KintoneColumnOption> options = task.getColumnOptions().values();
49
48
 
50
49
  KintoneMode mode = KintoneMode.getKintoneModeByValue(task.getMode());
51
50
  switch (mode) {
52
51
  case INSERT:
53
- for (KintoneColumnOption option : options) {
54
- if (option.getUpdateKey()) {
55
- throw new IllegalArgumentException(
56
- "when mode is insert, require no update_key.");
57
- }
52
+ if (task.getUpdateKeyName().isPresent()) {
53
+ throw new ConfigException("when mode is insert, require no update_key.");
58
54
  }
59
55
  break;
60
56
  case UPDATE:
61
57
  case UPSERT:
62
- boolean hasUpdateKey = false;
63
- for (KintoneColumnOption option : options) {
64
- if (option.getUpdateKey()) {
65
- if (hasUpdateKey) {
66
- throw new IllegalArgumentException(
67
- "when mode is update and upsert, only one column can have an update_key.");
68
- }
69
- hasUpdateKey = true;
70
- }
71
- }
72
- if (!hasUpdateKey) {
73
- throw new IllegalArgumentException(
74
- "when mode is update and upsert, require update_key.");
58
+ if (!task.getUpdateKeyName().isPresent()) {
59
+ throw new ConfigException("when mode is update or upsert, require update_key.");
75
60
  }
76
61
  break;
77
62
  default:
78
- throw new IllegalArgumentException(String.format(
63
+ throw new ConfigException(String.format(
79
64
  "Unknown mode '%s'",
80
65
  task.getMode()));
81
66
  }
@@ -3,6 +3,7 @@ package org.embulk.output.kintone;
3
3
  import com.kintone.client.KintoneClient;
4
4
  import com.kintone.client.KintoneClientBuilder;
5
5
  import com.kintone.client.api.record.GetRecordsByCursorResponseBody;
6
+ import com.kintone.client.model.record.FieldType;
6
7
  import com.kintone.client.model.record.Record;
7
8
  import com.kintone.client.model.record.RecordForUpdate;
8
9
  import com.kintone.client.model.record.UpdateKey;
@@ -14,17 +15,18 @@ import org.embulk.spi.PageReader;
14
15
  import org.embulk.spi.Schema;
15
16
  import org.embulk.spi.TransactionalPageOutput;
16
17
 
17
- import java.math.BigDecimal;
18
18
  import java.util.ArrayList;
19
- import java.util.Arrays;
19
+ import java.util.Collections;
20
20
  import java.util.List;
21
21
  import java.util.stream.Collectors;
22
22
 
23
23
  public class KintonePageOutput
24
24
  implements TransactionalPageOutput
25
25
  {
26
- private PageReader pageReader;
27
- private PluginTask task;
26
+ public static final int UPSERT_BATCH_SIZE = 10000;
27
+ public static final int CHUNK_SIZE = 100;
28
+ private final PageReader pageReader;
29
+ private final PluginTask task;
28
30
  private KintoneClient client;
29
31
 
30
32
  public KintonePageOutput(PluginTask task, Schema schema)
@@ -88,14 +90,14 @@ public class KintonePageOutput
88
90
 
89
91
  public interface Consumer<T>
90
92
  {
91
- public void accept(T t);
93
+ void accept(T t);
92
94
  }
93
95
 
94
96
  public void connect(final PluginTask task)
95
97
  {
96
98
  KintoneClientBuilder builder = KintoneClientBuilder.create("https://" + task.getDomain());
97
99
  if (task.getGuestSpaceId().isPresent()) {
98
- builder.setGuestSpaceId(task.getGuestSpaceId().or(-1));
100
+ builder.setGuestSpaceId(task.getGuestSpaceId().orElse(-1));
99
101
  }
100
102
  if (task.getBasicAuthUsername().isPresent() && task.getBasicAuthPassword().isPresent()) {
101
103
  builder.withBasicAuth(task.getBasicAuthUsername().get(),
@@ -136,7 +138,7 @@ public class KintonePageOutput
136
138
  }
137
139
 
138
140
  records.add(record);
139
- if (records.size() == 100) {
141
+ if (records.size() == CHUNK_SIZE) {
140
142
  client.record().addRecords(task.getAppId(), records);
141
143
  records.clear();
142
144
  }
@@ -155,10 +157,12 @@ public class KintonePageOutput
155
157
  {
156
158
  execute(client -> {
157
159
  try {
158
- ArrayList<RecordForUpdate> updateRecords = new ArrayList<RecordForUpdate>();
160
+ ArrayList<RecordForUpdate> updateRecords = new ArrayList<>();
159
161
  pageReader.setPage(page);
162
+
160
163
  KintoneColumnVisitor visitor = new KintoneColumnVisitor(pageReader,
161
- task.getColumnOptions());
164
+ task.getColumnOptions(),
165
+ task.getUpdateKeyName().orElseThrow(() -> new RuntimeException("unreachable"))); // Already validated
162
166
  while (pageReader.nextRecord()) {
163
167
  Record record = new Record();
164
168
  UpdateKey updateKey = new UpdateKey();
@@ -168,9 +172,13 @@ public class KintonePageOutput
168
172
  column.visit(visitor);
169
173
  }
170
174
 
175
+ if (updateKey.getValue() == "") {
176
+ continue;
177
+ }
178
+
171
179
  record.removeField(updateKey.getField());
172
180
  updateRecords.add(new RecordForUpdate(updateKey, record));
173
- if (updateRecords.size() == 100) {
181
+ if (updateRecords.size() == CHUNK_SIZE) {
174
182
  client.record().updateRecords(task.getAppId(), updateRecords);
175
183
  updateRecords.clear();
176
184
  }
@@ -185,183 +193,130 @@ public class KintonePageOutput
185
193
  });
186
194
  }
187
195
 
188
- private List<Record> getRecordsByUpdateKey(String fieldCode, List<String> queryValues)
196
+ private void upsertPage(final Page page)
189
197
  {
190
- List<Record> allRecords = new ArrayList<Record>();
191
- List<String> fields = Arrays.asList(fieldCode);
192
- String cursorId = client.record().createCursor(
193
- task.getAppId(),
194
- fields,
195
- fieldCode + " in (" + String.join(",", queryValues) + ")"
196
- );
197
- while (true) {
198
- GetRecordsByCursorResponseBody resp = client.record().getRecordsByCursor(cursorId);
199
- List<Record> records = resp.getRecords();
200
- allRecords.addAll(records);
198
+ execute(client -> {
199
+ try {
200
+ ArrayList<Record> records = new ArrayList<>();
201
+ ArrayList<UpdateKey> updateKeys = new ArrayList<>();
202
+ pageReader.setPage(page);
201
203
 
202
- if (!resp.hasNext()) {
203
- break;
204
+ KintoneColumnVisitor visitor = new KintoneColumnVisitor(pageReader,
205
+ task.getColumnOptions(),
206
+ task.getUpdateKeyName().orElseThrow(() -> new RuntimeException("unreachable"))); // Already validated
207
+ while (pageReader.nextRecord()) {
208
+ Record record = new Record();
209
+ UpdateKey updateKey = new UpdateKey();
210
+ visitor.setRecord(record);
211
+ visitor.setUpdateKey(updateKey);
212
+ for (Column column : pageReader.getSchema().getColumns()) {
213
+ column.visit(visitor);
214
+ }
215
+ records.add(record);
216
+ updateKeys.add(updateKey);
217
+
218
+ if (records.size() == UPSERT_BATCH_SIZE) {
219
+ upsert(records, updateKeys);
220
+ records.clear();
221
+ updateKeys.clear();
222
+ }
223
+ }
224
+ if (records.size() > 0) {
225
+ upsert(records, updateKeys);
226
+ }
204
227
  }
205
- }
206
- return allRecords;
228
+ catch (Exception e) {
229
+ throw new RuntimeException("kintone throw exception", e);
230
+ }
231
+ });
207
232
  }
208
233
 
209
- abstract class UpsertPage<T>
234
+ private void upsert(ArrayList<Record> records, ArrayList<UpdateKey> updateKeys)
210
235
  {
211
- public abstract List<T> getUpdateKeyValues(List<String> queryValues);
212
- public abstract boolean existsRecord(List<T> updateKeyValues, Record record);
213
-
214
- public void upsert(ArrayList<Record> records, ArrayList<UpdateKey> updateKeys)
215
- {
216
- if (records.size() != updateKeys.size()) {
217
- throw new RuntimeException("records.size() != updateKeys.size()");
218
- }
219
-
220
- List<String> queryValues = updateKeys
221
- .stream()
222
- .map(k -> "\"" + k.getValue().toString() + "\"")
223
- .collect(Collectors.toList());
224
- List<T> updateKeyValues = getUpdateKeyValues(queryValues);
236
+ if (records.size() != updateKeys.size()) {
237
+ throw new RuntimeException("records.size() != updateKeys.size()");
238
+ }
225
239
 
226
- ArrayList<Record> insertRecords = new ArrayList<>();
227
- ArrayList<RecordForUpdate> updateRecords = new ArrayList<RecordForUpdate>();
228
- for (int i = 0; i < records.size(); i++) {
229
- Record record = records.get(i);
230
- UpdateKey updateKey = updateKeys.get(i);
240
+ List<Record> existingRecords = getExistingRecordsByUpdateKey(updateKeys);
231
241
 
232
- if (existsRecord(updateKeyValues, record)) {
233
- record.removeField(updateKey.getField());
234
- updateRecords.add(new RecordForUpdate(updateKey, record));
235
- }
236
- else {
237
- insertRecords.add(record);
238
- }
242
+ ArrayList<Record> insertRecords = new ArrayList<>();
243
+ ArrayList<RecordForUpdate> updateRecords = new ArrayList<>();
244
+ for (int i = 0; i < records.size(); i++) {
245
+ Record record = records.get(i);
246
+ UpdateKey updateKey = updateKeys.get(i);
239
247
 
240
- if (insertRecords.size() == 100) {
241
- client.record().addRecords(task.getAppId(), insertRecords);
242
- insertRecords.clear();
243
- }
244
- else if (updateRecords.size() == 100) {
245
- client.record().updateRecords(task.getAppId(), updateRecords);
246
- updateRecords.clear();
247
- }
248
+ if (existsRecord(existingRecords, updateKey)) {
249
+ record.removeField(updateKey.getField());
250
+ updateRecords.add(new RecordForUpdate(updateKey, record));
248
251
  }
249
- if (insertRecords.size() > 0) {
252
+ else {
253
+ insertRecords.add(record);
254
+ }
255
+
256
+ if (insertRecords.size() == CHUNK_SIZE) {
250
257
  client.record().addRecords(task.getAppId(), insertRecords);
258
+ insertRecords.clear();
251
259
  }
252
- if (updateRecords.size() > 0) {
260
+ else if (updateRecords.size() == CHUNK_SIZE) {
253
261
  client.record().updateRecords(task.getAppId(), updateRecords);
262
+ updateRecords.clear();
254
263
  }
255
264
  }
256
-
257
- public void run(final Page page)
258
- {
259
- execute(client -> {
260
- try {
261
- ArrayList<Record> records = new ArrayList<>();
262
- ArrayList<UpdateKey> updateKeys = new ArrayList<>();
263
-
264
- pageReader.setPage(page);
265
- KintoneColumnVisitor visitor = new KintoneColumnVisitor(pageReader,
266
- task.getColumnOptions());
267
- while (pageReader.nextRecord()) {
268
- Record record = new Record();
269
- UpdateKey updateKey = new UpdateKey();
270
- visitor.setRecord(record);
271
- visitor.setUpdateKey(updateKey);
272
- for (Column column : pageReader.getSchema().getColumns()) {
273
- column.visit(visitor);
274
- }
275
- records.add(record);
276
- updateKeys.add(updateKey);
277
-
278
- if (records.size() == 10000) {
279
- upsert(records, updateKeys);
280
- records.clear();
281
- updateKeys.clear();
282
- }
283
- }
284
- if (records.size() > 0) {
285
- upsert(records, updateKeys);
286
- }
287
- }
288
- catch (Exception e) {
289
- throw new RuntimeException("kintone throw exception", e);
290
- }
291
- });
292
- }
293
- }
294
-
295
- class UpsertPageByStringKey extends UpsertPage<String>
296
- {
297
- private String fieldCode;
298
-
299
- public UpsertPageByStringKey(String fieldCode)
300
- {
301
- this.fieldCode = fieldCode;
302
- }
303
-
304
- public List<String> getUpdateKeyValues(List<String> queryValues)
305
- {
306
- return getRecordsByUpdateKey(fieldCode, queryValues)
307
- .stream()
308
- .map(r -> r.getSingleLineTextFieldValue(fieldCode))
309
- .collect(Collectors.toList());
265
+ if (insertRecords.size() > 0) {
266
+ client.record().addRecords(task.getAppId(), insertRecords);
310
267
  }
311
-
312
- public boolean existsRecord(List<String> updateKeyValues, Record record)
313
- {
314
- return updateKeyValues.contains(record.getSingleLineTextFieldValue(fieldCode));
268
+ if (updateRecords.size() > 0) {
269
+ client.record().updateRecords(task.getAppId(), updateRecords);
315
270
  }
316
271
  }
317
272
 
318
- class UpsertPageByNumberKey extends UpsertPage<BigDecimal>
319
- {
320
- private String fieldCode;
321
-
322
- public UpsertPageByNumberKey(String fieldCode)
323
- {
324
- this.fieldCode = fieldCode;
325
- }
326
273
 
327
- public List<BigDecimal> getUpdateKeyValues(List<String> queryValues)
328
- {
329
- return getRecordsByUpdateKey(fieldCode, queryValues)
274
+ private List<Record> getExistingRecordsByUpdateKey(ArrayList<UpdateKey> updateKeys)
275
+ {
276
+ String fieldCode = updateKeys.get(0).getField();
277
+ List<String> queryValues = updateKeys
330
278
  .stream()
331
- .map(r -> r.getNumberFieldValue(fieldCode))
279
+ .filter(k -> k.getValue() != "")
280
+ .map(k -> "\"" + k.getValue().toString() + "\"")
332
281
  .collect(Collectors.toList());
333
- }
334
282
 
335
- public boolean existsRecord(List<BigDecimal> updateKeyValues, Record record)
336
- {
337
- return updateKeyValues.contains(record.getNumberFieldValue(fieldCode));
283
+ List<Record> allRecords = new ArrayList<>();
284
+ if (queryValues.isEmpty()) {
285
+ return allRecords;
338
286
  }
339
- }
287
+ String cursorId = client.record().createCursor(
288
+ task.getAppId(),
289
+ Collections.singletonList(fieldCode),
290
+ fieldCode + " in (" + String.join(",", queryValues) + ")"
291
+ );
292
+ while (true) {
293
+ GetRecordsByCursorResponseBody resp = client.record().getRecordsByCursor(cursorId);
294
+ List<Record> records = resp.getRecords();
295
+ allRecords.addAll(records);
340
296
 
341
- private void upsertPage(final Page page)
342
- {
343
- KintoneColumnOption updateKeyColumn = null;
344
- for (KintoneColumnOption v : task.getColumnOptions().values()) {
345
- if (v.getUpdateKey()) {
346
- updateKeyColumn = v;
297
+ if (!resp.hasNext()) {
347
298
  break;
348
299
  }
349
300
  }
350
- if (updateKeyColumn == null) {
351
- throw new RuntimeException("when mode is upsert, require update_key");
352
- }
301
+ return allRecords;
302
+ }
353
303
 
354
- UpsertPage runner;
355
- switch(updateKeyColumn.getType()) {
356
- case "SINGLE_LINE_TEXT":
357
- runner = new UpsertPageByStringKey(updateKeyColumn.getFieldCode());
358
- break;
359
- case "NUMBER":
360
- runner = new UpsertPageByNumberKey(updateKeyColumn.getFieldCode());
361
- break;
304
+ private boolean existsRecord(List<Record> distRecords, UpdateKey updateKey)
305
+ {
306
+ String fieldCode = updateKey.getField();
307
+ FieldType type = client.app().getFormFields(task.getAppId()).get(fieldCode).getType();
308
+ switch (type) {
309
+ case SINGLE_LINE_TEXT:
310
+ return distRecords
311
+ .stream()
312
+ .anyMatch(d -> d.getSingleLineTextFieldValue(fieldCode).equals(updateKey.getValue().toString()));
313
+ case NUMBER:
314
+ return distRecords
315
+ .stream()
316
+ .anyMatch(d -> d.getNumberFieldValue(fieldCode).toPlainString()
317
+ .equals(updateKey.getValue().toString()));
362
318
  default:
363
319
  throw new RuntimeException("The update_key must be 'SINGLE_LINE_TEXT' or 'NUMBER'.");
364
320
  }
365
- runner.run(page);
366
321
  }
367
322
  }
@@ -1,11 +1,11 @@
1
1
  package org.embulk.output.kintone;
2
2
 
3
- import com.google.common.base.Optional;
4
3
  import org.embulk.config.Config;
5
4
  import org.embulk.config.ConfigDefault;
6
5
  import org.embulk.config.Task;
7
6
 
8
7
  import java.util.Map;
8
+ import java.util.Optional;
9
9
 
10
10
  public interface PluginTask
11
11
  extends Task
@@ -42,9 +42,13 @@ public interface PluginTask
42
42
 
43
43
  @Config("column_options")
44
44
  @ConfigDefault("{}")
45
- public Map<String, KintoneColumnOption> getColumnOptions();
45
+ Map<String, KintoneColumnOption> getColumnOptions();
46
46
 
47
47
  @Config("mode")
48
48
  @ConfigDefault("insert")
49
- public String getMode();
49
+ String getMode();
50
+
51
+ @Config("update_key")
52
+ @ConfigDefault("null")
53
+ Optional<String> getUpdateKeyName();
50
54
  }
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: embulk-output-kintone
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.2
4
+ version: 0.3.2
5
5
  platform: ruby
6
6
  authors:
7
7
  - takeshi fujita
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2020-11-18 00:00:00.000000000 Z
11
+ date: 2022-03-23 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
@@ -52,7 +52,7 @@ files:
52
52
  - README.md
53
53
  - Rakefile
54
54
  - build.gradle
55
- - classpath/embulk-output-kintone-0.2.2.jar
55
+ - classpath/embulk-output-kintone-0.3.2.jar
56
56
  - config/checkstyle/checkstyle.xml
57
57
  - config/checkstyle/default.xml
58
58
  - gradle/wrapper/gradle-wrapper.jar