embulk-input-kintone 0.1.2 → 0.1.3

Sign up to get free protection for your applications and to get access to all the features.
Files changed (6) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +7 -4
  3. data/build.gradle +1 -1
  4. data/src/main/java/org/embulk/input/kintone/KintoneClient.java +22 -8
  5. data/src/main/java/org/embulk/input/kintone/KintoneInputPlugin.java +10 -12
  6. metadata +3 -3
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 7228024b39d9a99ed12bb46ed0533fbf0e8bc203
4
- data.tar.gz: 79da56d30429a42c0f286c6e898f9f67b8ee17c1
3
+ metadata.gz: 69c23aa1d03cb2ccfd39918eda208847ed5dcf11
4
+ data.tar.gz: 1c24d9b3210d4e30d2c23b48ddb08e2b8f515e2d
5
5
  SHA512:
6
- metadata.gz: 8dd98237c2e4d4141466ff8f1405eb3c36e4b6f531f0fa3571abc17336a72c8f04cbcc8eecfa305d0c25470c4ae15cdb142d259d74c6c6867c82b3b81f69050d
7
- data.tar.gz: ba3f6ca0f421e40119a68f4b076085d2b405e4663a2a733ad49bc7147fac4be05e835ad48bcc853088767f565576f712b6092a800b68f4f64963b9e43e036367
6
+ metadata.gz: 632505db9038c63988f56c8ec4bc961cc11ba28f8e2320a1f22228add31be29ca0df913337ccc786fdfa86cb93e4f4a386cd85aeee53e7352ba2abc86de3b468
7
+ data.tar.gz: c858872b535a14af20adb30023388c7d3acb03159b3c4d7d4db547d45e15f17eca76eb68bb853b0742e13de0b63eb93e2595fca279610d46aa91be37b2949766
data/README.md CHANGED
@@ -1,10 +1,13 @@
1
- # Kintone input plugin for Embulk
1
+ # kintone input plugin for Embulk
2
2
  [![Build Status](https://travis-ci.org/trocco-io/embulk-input-kintone.svg?branch=master)](https://travis-ci.org/trocco-io/embulk-input-kintone)
3
3
 
4
4
  ## Overview
5
- Kintone input plugin for Embulk loads app records from Kintone.
5
+ kintone input plugin for Embulk loads app records from kintone.
6
6
  embulk 0.9 is only supported due to the dependency of kintone-java-sdk 0.4.0, which requires java 8
7
7
 
8
+ This plugin uses [cursor API](https://developer.kintone.io/hc/en-us/articles/360000280322). See the limitation on this page.
9
+ e.g. limit, offset are not supported.
10
+
8
11
  * **Plugin type**: input
9
12
  * **Resume supported**: no
10
13
  * **Cleanup supported**: no
@@ -32,7 +35,7 @@ embulk 0.9 is only supported due to the dependency of kintone-java-sdk 0.4.0, wh
32
35
  - **type** Column values are converted to this embulk type. Available values options are: boolean, long, double, string, json, timestamp)
33
36
  - **format** Format of the timestamp if type is timestamp. The format for kintone DATETIME is `%Y-%m-%dT%H:%M:%S%z`.
34
37
 
35
- Kintone API has the limitation, therefore this plugin also faces it. See [official documentation](https://developer.kintone.io/hc/en-us/articles/212495188/)
38
+ kintone API has the limitation, therefore this plugin also faces it. See [official documentation](https://developer.kintone.io/hc/en-us/articles/212495188/)
36
39
 
37
40
  ## Example
38
41
 
@@ -60,7 +63,7 @@ in:
60
63
  username: user
61
64
  password: password
62
65
  app_id: 1
63
- query: Time > 10:00 and Time < 19:00 and Created_datatime = TODAY() order by $id asc limit 10
66
+ query: Time > 10:00 and Time < 19:00 and Created_datatime = TODAY() order by $id asc
64
67
  fields:
65
68
  - {name: $id, type: long}
66
69
  - {name: $revision, type: long}
data/build.gradle CHANGED
@@ -13,7 +13,7 @@ configurations {
13
13
  provided
14
14
  }
15
15
 
16
- version = "0.1.2"
16
+ version = "0.1.3"
17
17
 
18
18
  sourceCompatibility = 1.8
19
19
  targetCompatibility = 1.8
data/src/main/java/org/embulk/input/kintone/KintoneClient.java CHANGED
@@ -2,11 +2,12 @@ package org.embulk.input.kintone;
2
2
 
3
3
  import com.cybozu.kintone.client.authentication.Auth;
4
4
  import com.cybozu.kintone.client.connection.Connection;
5
+ import com.cybozu.kintone.client.exception.KintoneAPIException;
6
+ import com.cybozu.kintone.client.model.cursor.CreateRecordCursorResponse;
5
7
  import com.cybozu.kintone.client.model.record.GetRecordsResponse;
6
- import com.cybozu.kintone.client.module.record.Record;
8
+ import com.cybozu.kintone.client.module.recordCursor.RecordCursor;
7
9
  import org.embulk.config.ConfigException;
8
- import org.embulk.spi.*;
9
-
10
+ import org.embulk.spi.ColumnConfig;
10
11
  import org.slf4j.Logger;
11
12
  import org.slf4j.LoggerFactory;
12
13
 
@@ -14,9 +15,11 @@ import java.util.ArrayList;
14
15
 
15
16
  public class KintoneClient {
16
17
  private final Logger logger = LoggerFactory.getLogger(KintoneClient.class);
18
+ private static final int FETCH_SIZE = 500;
17
19
  private Auth kintoneAuth;
18
- private Record kintoneRecordManager;
20
+ private RecordCursor kintoneRecordManager;
19
21
  private Connection con;
22
+ private CreateRecordCursorResponse cursor;
20
23
 
21
24
  public KintoneClient(){
22
25
  this.kintoneAuth = new Auth();
@@ -49,7 +52,7 @@ public class KintoneClient {
49
52
  } else {
50
53
  this.con = new Connection(task.getDomain(), this.kintoneAuth);
51
54
  }
52
- this.kintoneRecordManager = new Record(con);
55
+ this.kintoneRecordManager = new RecordCursor(con);
53
56
  }
54
57
 
55
58
 
@@ -60,11 +63,22 @@ public class KintoneClient {
60
63
  fields.add(c.getName());
61
64
  }
62
65
  try {
63
- return kintoneRecordManager.getAllRecordsByQuery(
64
- task.getAppId(), task.getQuery().or(""), fields);
65
- } catch (Exception e) {
66
+ this.cursor = this.kintoneRecordManager.createCursor(task.getAppId(),
67
+ fields, task.getQuery().or(""), FETCH_SIZE);
68
+ return this.kintoneRecordManager.getAllRecords(cursor.getId());
69
+ }catch (KintoneAPIException e){
70
+ if (this.cursor != null) {
71
+ this.deleteCursor();
72
+ }
66
73
  throw new RuntimeException(e);
67
74
  }
68
75
  }
69
76
 
77
+ public void deleteCursor() {
78
+ try {
79
+ this.kintoneRecordManager.deleteCursor(this.cursor.getId());
80
+ }catch (KintoneAPIException e){
81
+ this.logger.error(e.toString());
82
+ }
83
+ }
70
84
  }
data/src/main/java/org/embulk/input/kintone/KintoneInputPlugin.java CHANGED
@@ -1,21 +1,19 @@
1
1
  package org.embulk.input.kintone;
2
2
 
3
- import java.util.HashMap;
4
- import java.util.List;
5
-
6
- import com.google.common.annotations.VisibleForTesting;
7
- import org.embulk.config.*;
8
- import org.embulk.spi.Exec;
9
- import org.embulk.spi.PageBuilder;
10
- import org.embulk.spi.InputPlugin;
11
- import org.embulk.spi.Schema;
12
- import org.embulk.spi.PageOutput;
13
-
14
- import com.cybozu.kintone.client.model.record.field.FieldValue;
15
3
  import com.cybozu.kintone.client.model.record.GetRecordsResponse;
4
+ import com.cybozu.kintone.client.model.record.field.FieldValue;
5
+ import com.google.common.annotations.VisibleForTesting;
6
+ import org.embulk.config.ConfigDiff;
7
+ import org.embulk.config.ConfigSource;
8
+ import org.embulk.config.TaskReport;
9
+ import org.embulk.config.TaskSource;
10
+ import org.embulk.spi.*;
16
11
  import org.slf4j.Logger;
17
12
  import org.slf4j.LoggerFactory;
18
13
 
14
+ import java.util.HashMap;
15
+ import java.util.List;
16
+
19
17
  public class KintoneInputPlugin
20
18
  implements InputPlugin {
21
19
  private final Logger logger = LoggerFactory.getLogger(KintoneInputPlugin.class);
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: embulk-input-kintone
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.2
4
+ version: 0.1.3
5
5
  platform: ruby
6
6
  authors:
7
7
  - giwa
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2019-10-23 00:00:00.000000000 Z
11
+ date: 2019-11-12 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
@@ -50,7 +50,7 @@ files:
50
50
  - LICENSE.txt
51
51
  - README.md
52
52
  - build.gradle
53
- - classpath/embulk-input-kintone-0.1.2.jar
53
+ - classpath/embulk-input-kintone-0.1.3.jar
54
54
  - classpath/gson-2.8.2.jar
55
55
  - classpath/kintone-sdk-0.4.0.jar
56
56
  - config/checkstyle/checkstyle.xml