embulk-input-kintone 0.1.0 → 0.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.travis.yml +7 -0
- data/README.md +19 -9
- data/build.gradle +7 -3
- data/gradle/wrapper/gradle-wrapper.properties +2 -1
- data/gradlew.bat +84 -84
- data/src/main/java/org/embulk/input/kintone/KintoneAccessor.java +8 -4
- data/src/main/java/org/embulk/input/kintone/KintoneClient.java +65 -31
- data/src/main/java/org/embulk/input/kintone/KintoneInputColumnVisitor.java +1 -1
- data/src/main/java/org/embulk/input/kintone/KintoneInputPlugin.java +39 -20
- data/src/test/java/org/embulk/input/kintone/TestHelper.java +17 -0
- data/src/test/java/org/embulk/input/kintone/TestKintoneAccessor.java +100 -0
- data/src/test/java/org/embulk/input/kintone/TestKintoneClient.java +80 -0
- data/src/test/java/org/embulk/input/kintone/TestKintoneInputPlugin.java +147 -2
- data/src/test/resources/org/embulk/input/kintone/base.yml +11 -0
- metadata +9 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 646c14f2c41e56b23174e35fcbc2147072d49ba5
|
4
|
+
data.tar.gz: 90477839161c398082fb0cceae2c2cb14e1a2bc3
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: f0abfe7044ab966a6ad27158bdb56941b6857be9c1bd97714a5c9be87faff45e5c0c88e0f21746965cf7fcfeaad1e63542729ecb5b1ab1655c28c8d7c6946ca4
|
7
|
+
data.tar.gz: da7f3422eccc3a7cd49a3ded84c37fabb02c90489623ec553443ac6c36b9891be2317456b1e1b1a8bf5724efea75d4e08ba755e68fa2e159fd3076b6194befb9
|
data/.travis.yml
ADDED
data/README.md
CHANGED
@@ -1,9 +1,13 @@
|
|
1
|
-
#
|
1
|
+
# kintone input plugin for Embulk
|
2
|
+
[](https://travis-ci.org/trocco-io/embulk-input-kintone)
|
2
3
|
|
3
4
|
## Overview
|
4
|
-
|
5
|
+
kintone input plugin for Embulk loads app records from kintone.
|
5
6
|
embulk 0.9 is only supported due to the dependency of kintone-java-sdk 0.4.0, which requires java 8
|
6
7
|
|
8
|
+
This plugin uses [cursor API](https://developer.kintone.io/hc/en-us/articles/360000280322). See the limitation on this page.
|
9
|
+
e.g. limit, offset are not supported.
|
10
|
+
|
7
11
|
* **Plugin type**: input
|
8
12
|
* **Resume supported**: no
|
9
13
|
* **Cleanup supported**: no
|
@@ -11,9 +15,7 @@ embulk 0.9 is only supported due to the dependency of kintone-java-sdk 0.4.0, wh
|
|
11
15
|
|
12
16
|
## Road Map
|
13
17
|
- [ ] Guess
|
14
|
-
- [ ] Subtable data type support
|
15
18
|
- [ ] field name mapping
|
16
|
-
- [ ] timestamp_format in fields
|
17
19
|
- [ ] handle certification fot authentication
|
18
20
|
|
19
21
|
## Configuration
|
@@ -27,11 +29,12 @@ embulk 0.9 is only supported due to the dependency of kintone-java-sdk 0.4.0, wh
|
|
27
29
|
- **basic_auth_username**: Kintone basic auth username Please see Kintone basic auth [here](https://jp.cybozu.help/general/en/admin/list_security/list_ip_basic/basic_auth.html) (string, optional)
|
28
30
|
- **basic_auth_password**: Kintone basic auth password (string, optional)
|
29
31
|
- **guest_space_id**: Kintone app belongs to guest space, guest space id is required. (integer, optional)
|
30
|
-
- **fields**
|
32
|
+
- **fields** (required)
|
31
33
|
- **name** the field code of Kintone app record will be retrieved.
|
32
|
-
- **type** Column values are converted to this embulk type. Available values options are: boolean, long, double, string, json, timestamp)
|
34
|
+
- **type** Column values are converted to this embulk type. Available values options are: boolean, long, double, string, json, timestamp) Kintone `SUBTABLE` type is loaded as json text.
|
35
|
+
- **format** Format of the timestamp if type is timestamp. The format for kintone DATETIME is `%Y-%m-%dT%H:%M:%S%z`.
|
33
36
|
|
34
|
-
|
37
|
+
kintone API has the limitation, therefore this plugin also faces it. See [official documentation](https://developer.kintone.io/hc/en-us/articles/212495188/)
|
35
38
|
|
36
39
|
## Example
|
37
40
|
|
@@ -59,13 +62,14 @@ in:
|
|
59
62
|
username: user
|
60
63
|
password: password
|
61
64
|
app_id: 1
|
62
|
-
query: Time > 10:00 and Time < 19:00 and Created_datatime = TODAY() order by $id asc
|
65
|
+
query: Time > 10:00 and Time < 19:00 and Created_datatime = TODAY() order by $id asc
|
63
66
|
fields:
|
64
67
|
- {name: $id, type: long}
|
65
68
|
- {name: $revision, type: long}
|
66
69
|
- {name: Time, type: string}
|
67
|
-
- {name: Created_datatime, type:
|
70
|
+
- {name: Created_datatime, type: string}
|
68
71
|
- {name: foo, type: string}
|
72
|
+
- {name: datetime, type: timestamp, format: '%Y-%m-%dT%H:%M:%S%z'}
|
69
73
|
```
|
70
74
|
|
71
75
|
## Build
|
@@ -73,3 +77,9 @@ in:
|
|
73
77
|
```
|
74
78
|
$ ./gradlew gem # -t to watch change of files and rebuild continuously
|
75
79
|
```
|
80
|
+
|
81
|
+
## Development
|
82
|
+
```
|
83
|
+
$ ./gradew build
|
84
|
+
$ ./gradew test
|
85
|
+
```
|
data/build.gradle
CHANGED
@@ -13,7 +13,7 @@ configurations {
|
|
13
13
|
provided
|
14
14
|
}
|
15
15
|
|
16
|
-
version = "0.1.
|
16
|
+
version = "0.1.5"
|
17
17
|
|
18
18
|
sourceCompatibility = 1.8
|
19
19
|
targetCompatibility = 1.8
|
@@ -22,8 +22,12 @@ dependencies {
|
|
22
22
|
compile "org.embulk:embulk-core:0.9.12"
|
23
23
|
provided "org.embulk:embulk-core:0.9.12"
|
24
24
|
compile group: 'com.cybozu.kintone', name: 'kintone-sdk', version: '0.4.0'
|
25
|
-
|
25
|
+
|
26
26
|
testCompile "junit:junit:4.+"
|
27
|
+
testCompile 'org.embulk:embulk-standards:0.9.12'
|
28
|
+
testCompile 'org.embulk:embulk-test:0.9.12'
|
29
|
+
testCompile "org.mockito:mockito-core:1.+"
|
30
|
+
testCompile "org.embulk:embulk-core:0.9.12:tests"
|
27
31
|
}
|
28
32
|
|
29
33
|
task classpath(type: Copy, dependsOn: ["jar"]) {
|
@@ -83,7 +87,7 @@ Gem::Specification.new do |spec|
|
|
83
87
|
spec.description = %[Loads records from Kintone.]
|
84
88
|
spec.email = ["ugw.gi.world@gmail.com"]
|
85
89
|
spec.licenses = ["MIT"]
|
86
|
-
|
90
|
+
spec.homepage = "https://github.com/trocco-io/embulk-input-kintone"
|
87
91
|
|
88
92
|
spec.files = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
|
89
93
|
spec.test_files = spec.files.grep(%r"^(test|spec)/")
|
@@ -1,5 +1,6 @@
|
|
1
|
+
#Fri Oct 11 16:04:33 JST 2019
|
1
2
|
distributionBase=GRADLE_USER_HOME
|
2
3
|
distributionPath=wrapper/dists
|
3
4
|
zipStoreBase=GRADLE_USER_HOME
|
4
5
|
zipStorePath=wrapper/dists
|
5
|
-
distributionUrl=https\://services.gradle.org/distributions/gradle-4.1-
|
6
|
+
distributionUrl=https\://services.gradle.org/distributions/gradle-4.1-all.zip
|
data/gradlew.bat
CHANGED
@@ -1,84 +1,84 @@
|
|
1
|
-
@if "%DEBUG%" == "" @echo off
|
2
|
-
@rem ##########################################################################
|
3
|
-
@rem
|
4
|
-
@rem Gradle startup script for Windows
|
5
|
-
@rem
|
6
|
-
@rem ##########################################################################
|
7
|
-
|
8
|
-
@rem Set local scope for the variables with windows NT shell
|
9
|
-
if "%OS%"=="Windows_NT" setlocal
|
10
|
-
|
11
|
-
set DIRNAME=%~dp0
|
12
|
-
if "%DIRNAME%" == "" set DIRNAME=.
|
13
|
-
set APP_BASE_NAME=%~n0
|
14
|
-
set APP_HOME=%DIRNAME%
|
15
|
-
|
16
|
-
@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
|
17
|
-
set DEFAULT_JVM_OPTS=
|
18
|
-
|
19
|
-
@rem Find java.exe
|
20
|
-
if defined JAVA_HOME goto findJavaFromJavaHome
|
21
|
-
|
22
|
-
set JAVA_EXE=java.exe
|
23
|
-
%JAVA_EXE% -version >NUL 2>&1
|
24
|
-
if "%ERRORLEVEL%" == "0" goto init
|
25
|
-
|
26
|
-
echo.
|
27
|
-
echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
|
28
|
-
echo.
|
29
|
-
echo Please set the JAVA_HOME variable in your environment to match the
|
30
|
-
echo location of your Java installation.
|
31
|
-
|
32
|
-
goto fail
|
33
|
-
|
34
|
-
:findJavaFromJavaHome
|
35
|
-
set JAVA_HOME=%JAVA_HOME:"=%
|
36
|
-
set JAVA_EXE=%JAVA_HOME%/bin/java.exe
|
37
|
-
|
38
|
-
if exist "%JAVA_EXE%" goto init
|
39
|
-
|
40
|
-
echo.
|
41
|
-
echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
|
42
|
-
echo.
|
43
|
-
echo Please set the JAVA_HOME variable in your environment to match the
|
44
|
-
echo location of your Java installation.
|
45
|
-
|
46
|
-
goto fail
|
47
|
-
|
48
|
-
:init
|
49
|
-
@rem Get command-line arguments, handling Windows variants
|
50
|
-
|
51
|
-
if not "%OS%" == "Windows_NT" goto win9xME_args
|
52
|
-
|
53
|
-
:win9xME_args
|
54
|
-
@rem Slurp the command line arguments.
|
55
|
-
set CMD_LINE_ARGS=
|
56
|
-
set _SKIP=2
|
57
|
-
|
58
|
-
:win9xME_args_slurp
|
59
|
-
if "x%~1" == "x" goto execute
|
60
|
-
|
61
|
-
set CMD_LINE_ARGS=%*
|
62
|
-
|
63
|
-
:execute
|
64
|
-
@rem Setup the command line
|
65
|
-
|
66
|
-
set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
|
67
|
-
|
68
|
-
@rem Execute Gradle
|
69
|
-
"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
|
70
|
-
|
71
|
-
:end
|
72
|
-
@rem End local scope for the variables with windows NT shell
|
73
|
-
if "%ERRORLEVEL%"=="0" goto mainEnd
|
74
|
-
|
75
|
-
:fail
|
76
|
-
rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
|
77
|
-
rem the _cmd.exe /c_ return code!
|
78
|
-
if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
|
79
|
-
exit /b 1
|
80
|
-
|
81
|
-
:mainEnd
|
82
|
-
if "%OS%"=="Windows_NT" endlocal
|
83
|
-
|
84
|
-
:omega
|
1
|
+
@if "%DEBUG%" == "" @echo off
|
2
|
+
@rem ##########################################################################
|
3
|
+
@rem
|
4
|
+
@rem Gradle startup script for Windows
|
5
|
+
@rem
|
6
|
+
@rem ##########################################################################
|
7
|
+
|
8
|
+
@rem Set local scope for the variables with windows NT shell
|
9
|
+
if "%OS%"=="Windows_NT" setlocal
|
10
|
+
|
11
|
+
set DIRNAME=%~dp0
|
12
|
+
if "%DIRNAME%" == "" set DIRNAME=.
|
13
|
+
set APP_BASE_NAME=%~n0
|
14
|
+
set APP_HOME=%DIRNAME%
|
15
|
+
|
16
|
+
@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
|
17
|
+
set DEFAULT_JVM_OPTS=
|
18
|
+
|
19
|
+
@rem Find java.exe
|
20
|
+
if defined JAVA_HOME goto findJavaFromJavaHome
|
21
|
+
|
22
|
+
set JAVA_EXE=java.exe
|
23
|
+
%JAVA_EXE% -version >NUL 2>&1
|
24
|
+
if "%ERRORLEVEL%" == "0" goto init
|
25
|
+
|
26
|
+
echo.
|
27
|
+
echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
|
28
|
+
echo.
|
29
|
+
echo Please set the JAVA_HOME variable in your environment to match the
|
30
|
+
echo location of your Java installation.
|
31
|
+
|
32
|
+
goto fail
|
33
|
+
|
34
|
+
:findJavaFromJavaHome
|
35
|
+
set JAVA_HOME=%JAVA_HOME:"=%
|
36
|
+
set JAVA_EXE=%JAVA_HOME%/bin/java.exe
|
37
|
+
|
38
|
+
if exist "%JAVA_EXE%" goto init
|
39
|
+
|
40
|
+
echo.
|
41
|
+
echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
|
42
|
+
echo.
|
43
|
+
echo Please set the JAVA_HOME variable in your environment to match the
|
44
|
+
echo location of your Java installation.
|
45
|
+
|
46
|
+
goto fail
|
47
|
+
|
48
|
+
:init
|
49
|
+
@rem Get command-line arguments, handling Windows variants
|
50
|
+
|
51
|
+
if not "%OS%" == "Windows_NT" goto win9xME_args
|
52
|
+
|
53
|
+
:win9xME_args
|
54
|
+
@rem Slurp the command line arguments.
|
55
|
+
set CMD_LINE_ARGS=
|
56
|
+
set _SKIP=2
|
57
|
+
|
58
|
+
:win9xME_args_slurp
|
59
|
+
if "x%~1" == "x" goto execute
|
60
|
+
|
61
|
+
set CMD_LINE_ARGS=%*
|
62
|
+
|
63
|
+
:execute
|
64
|
+
@rem Setup the command line
|
65
|
+
|
66
|
+
set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
|
67
|
+
|
68
|
+
@rem Execute Gradle
|
69
|
+
"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
|
70
|
+
|
71
|
+
:end
|
72
|
+
@rem End local scope for the variables with windows NT shell
|
73
|
+
if "%ERRORLEVEL%"=="0" goto mainEnd
|
74
|
+
|
75
|
+
:fail
|
76
|
+
rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
|
77
|
+
rem the _cmd.exe /c_ return code!
|
78
|
+
if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
|
79
|
+
exit /b 1
|
80
|
+
|
81
|
+
:mainEnd
|
82
|
+
if "%OS%"=="Windows_NT" endlocal
|
83
|
+
|
84
|
+
:omega
|
@@ -5,12 +5,14 @@ import org.slf4j.Logger;
|
|
5
5
|
import org.slf4j.LoggerFactory;
|
6
6
|
import com.cybozu.kintone.client.model.record.field.FieldValue;
|
7
7
|
import com.cybozu.kintone.client.model.member.Member;
|
8
|
+
import com.google.gson.Gson;
|
8
9
|
|
9
10
|
import java.util.ArrayList;
|
10
11
|
import java.util.HashMap;
|
11
12
|
|
12
13
|
public class KintoneAccessor {
|
13
14
|
private final Logger logger = LoggerFactory.getLogger(KintoneAccessor.class);
|
15
|
+
private final Gson gson = new Gson();
|
14
16
|
|
15
17
|
private final HashMap<String, FieldValue> record;
|
16
18
|
private final String delimiter = "\n";
|
@@ -26,12 +28,12 @@ public class KintoneAccessor {
|
|
26
28
|
case GROUP_SELECT:
|
27
29
|
case STATUS_ASSIGNEE:
|
28
30
|
ArrayList<Member> members = (ArrayList<Member>) this.record.get(name).getValue();
|
29
|
-
return members.stream().map(
|
31
|
+
return members.stream().map(Member::getCode)
|
30
32
|
.reduce((accum, value) -> accum + this.delimiter + value)
|
31
33
|
.orElse("");
|
32
34
|
case SUBTABLE:
|
33
|
-
|
34
|
-
return
|
35
|
+
Object subTableValueItem = this.record.get(name).getValue();
|
36
|
+
return gson.toJson(subTableValueItem);
|
35
37
|
case CREATOR:
|
36
38
|
case MODIFIER:
|
37
39
|
Member m = (Member) this.record.get(name).getValue();
|
@@ -45,9 +47,11 @@ public class KintoneAccessor {
|
|
45
47
|
.orElse("");
|
46
48
|
case FILE:
|
47
49
|
ArrayList<FileModel> cbFileList = (ArrayList<FileModel>) this.record.get(name).getValue();
|
48
|
-
return cbFileList.stream().map(
|
50
|
+
return cbFileList.stream().map(FileModel::getFileKey)
|
49
51
|
.reduce((accum, value) -> accum + this.delimiter + value)
|
50
52
|
.orElse("");
|
53
|
+
case NUMBER:
|
54
|
+
return String.valueOf(this.record.get(name).getValue());
|
51
55
|
default:
|
52
56
|
return (String) this.record.get(name).getValue();
|
53
57
|
}
|
@@ -2,11 +2,13 @@ package org.embulk.input.kintone;
|
|
2
2
|
|
3
3
|
import com.cybozu.kintone.client.authentication.Auth;
|
4
4
|
import com.cybozu.kintone.client.connection.Connection;
|
5
|
+
import com.cybozu.kintone.client.exception.KintoneAPIException;
|
6
|
+
import com.cybozu.kintone.client.model.cursor.CreateRecordCursorResponse;
|
7
|
+
import com.cybozu.kintone.client.model.cursor.GetRecordCursorResponse;
|
5
8
|
import com.cybozu.kintone.client.model.record.GetRecordsResponse;
|
6
|
-
import com.cybozu.kintone.client.module.
|
9
|
+
import com.cybozu.kintone.client.module.recordCursor.RecordCursor;
|
7
10
|
import org.embulk.config.ConfigException;
|
8
|
-
import org.embulk.spi
|
9
|
-
|
11
|
+
import org.embulk.spi.ColumnConfig;
|
10
12
|
import org.slf4j.Logger;
|
11
13
|
import org.slf4j.LoggerFactory;
|
12
14
|
|
@@ -14,52 +16,84 @@ import java.util.ArrayList;
|
|
14
16
|
|
15
17
|
public class KintoneClient {
|
16
18
|
private final Logger logger = LoggerFactory.getLogger(KintoneClient.class);
|
17
|
-
private final
|
18
|
-
private ArrayList<String> fields;
|
19
|
+
private static final int FETCH_SIZE = 500;
|
19
20
|
private Auth kintoneAuth;
|
20
|
-
private
|
21
|
+
private RecordCursor kintoneRecordManager;
|
21
22
|
private Connection con;
|
22
23
|
|
23
|
-
public KintoneClient(
|
24
|
-
this.pluginTask = pluginTask;
|
24
|
+
public KintoneClient(){
|
25
25
|
this.kintoneAuth = new Auth();
|
26
|
+
}
|
26
27
|
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
this.setAuth();
|
33
|
-
if (pluginTask.getGuestSpaceId().isPresent()) {
|
34
|
-
this.con = new Connection(pluginTask.getDomain(), this.kintoneAuth, pluginTask.getGuestSpaceId().or(-1));
|
28
|
+
public void validateAuth(final PluginTask task) throws ConfigException{
|
29
|
+
if (task.getUsername().isPresent() && task.getPassword().isPresent()) {
|
30
|
+
return;
|
31
|
+
} else if (task.getToken().isPresent()) {
|
32
|
+
return;
|
35
33
|
} else {
|
36
|
-
|
34
|
+
throw new ConfigException("Username and password or token must be provided");
|
37
35
|
}
|
38
|
-
this.kintoneRecordManager = new Record(con);
|
39
36
|
}
|
40
37
|
|
41
|
-
|
42
|
-
if (
|
43
|
-
this.kintoneAuth.setPasswordAuth(
|
44
|
-
} else if (
|
45
|
-
this.kintoneAuth.setApiToken(
|
38
|
+
public void connect(final PluginTask task) {
|
39
|
+
if (task.getUsername().isPresent() && task.getPassword().isPresent()) {
|
40
|
+
this.kintoneAuth.setPasswordAuth(task.getUsername().get(), task.getPassword().get());
|
41
|
+
} else if (task.getToken().isPresent()) {
|
42
|
+
this.kintoneAuth.setApiToken(task.getToken().get());
|
43
|
+
}
|
44
|
+
|
45
|
+
if (task.getBasicAuthUsername().isPresent() && task.getBasicAuthPassword().isPresent()) {
|
46
|
+
this.kintoneAuth.setBasicAuth(task.getBasicAuthUsername().get(),
|
47
|
+
task.getBasicAuthPassword().get());
|
48
|
+
}
|
49
|
+
|
50
|
+
if (task.getGuestSpaceId().isPresent()) {
|
51
|
+
this.con = new Connection(task.getDomain(), this.kintoneAuth, task.getGuestSpaceId().or(-1));
|
46
52
|
} else {
|
47
|
-
|
53
|
+
this.con = new Connection(task.getDomain(), this.kintoneAuth);
|
48
54
|
}
|
55
|
+
this.kintoneRecordManager = new RecordCursor(con);
|
56
|
+
}
|
57
|
+
|
49
58
|
|
50
|
-
|
51
|
-
|
52
|
-
|
59
|
+
public GetRecordsResponse getResponse(final PluginTask task) {
|
60
|
+
CreateRecordCursorResponse cursor = this.createCursor(task);
|
61
|
+
try {
|
62
|
+
return this.kintoneRecordManager.getAllRecords(cursor.getId());
|
63
|
+
}catch (KintoneAPIException e){
|
64
|
+
this.deleteCursor(cursor);
|
65
|
+
throw new RuntimeException(e);
|
53
66
|
}
|
54
67
|
}
|
55
68
|
|
56
|
-
public
|
69
|
+
public GetRecordCursorResponse getRecordsByCursor(CreateRecordCursorResponse cursor){
|
57
70
|
try {
|
58
|
-
return kintoneRecordManager.
|
59
|
-
|
60
|
-
|
71
|
+
return this.kintoneRecordManager.getRecords(cursor.getId());
|
72
|
+
}catch (KintoneAPIException e){
|
73
|
+
this.deleteCursor(cursor);
|
61
74
|
throw new RuntimeException(e);
|
62
75
|
}
|
63
76
|
}
|
64
77
|
|
78
|
+
public CreateRecordCursorResponse createCursor(final PluginTask task){
|
79
|
+
ArrayList<String> fields = new ArrayList<>();
|
80
|
+
for (ColumnConfig c : task.getFields().getColumns()
|
81
|
+
) {
|
82
|
+
fields.add(c.getName());
|
83
|
+
}
|
84
|
+
|
85
|
+
try{
|
86
|
+
return this.kintoneRecordManager.createCursor(task.getAppId(), fields, task.getQuery().or(""), FETCH_SIZE);
|
87
|
+
}catch (KintoneAPIException e) {
|
88
|
+
throw new RuntimeException(e);
|
89
|
+
}
|
90
|
+
}
|
91
|
+
|
92
|
+
public void deleteCursor(CreateRecordCursorResponse cursor) {
|
93
|
+
try {
|
94
|
+
this.kintoneRecordManager.deleteCursor(cursor.getId());
|
95
|
+
}catch (KintoneAPIException e){
|
96
|
+
this.logger.error(e.toString());
|
97
|
+
}
|
98
|
+
}
|
65
99
|
}
|
@@ -16,7 +16,7 @@ import org.slf4j.LoggerFactory;
|
|
16
16
|
import com.google.gson.JsonElement;
|
17
17
|
|
18
18
|
public class KintoneInputColumnVisitor implements ColumnVisitor {
|
19
|
-
private static final String DEFAULT_TIMESTAMP_PATTERN = "%Y-%m-%dT%H:%M:%S
|
19
|
+
private static final String DEFAULT_TIMESTAMP_PATTERN = "%Y-%m-%dT%H:%M:%S%z";
|
20
20
|
private final Logger logger = LoggerFactory.getLogger(KintoneInputColumnVisitor.class);
|
21
21
|
|
22
22
|
private final PageBuilder pageBuilder;
|
@@ -1,23 +1,20 @@
|
|
1
1
|
package org.embulk.input.kintone;
|
2
2
|
|
3
|
-
import
|
4
|
-
import
|
5
|
-
|
3
|
+
import com.cybozu.kintone.client.model.cursor.CreateRecordCursorResponse;
|
4
|
+
import com.cybozu.kintone.client.model.cursor.GetRecordCursorResponse;
|
5
|
+
import com.cybozu.kintone.client.model.record.field.FieldValue;
|
6
|
+
import com.google.common.annotations.VisibleForTesting;
|
6
7
|
import org.embulk.config.ConfigDiff;
|
7
8
|
import org.embulk.config.ConfigSource;
|
8
9
|
import org.embulk.config.TaskReport;
|
9
10
|
import org.embulk.config.TaskSource;
|
10
|
-
import org.embulk.spi
|
11
|
-
import org.embulk.spi.PageBuilder;
|
12
|
-
import org.embulk.spi.InputPlugin;
|
13
|
-
import org.embulk.spi.Schema;
|
14
|
-
import org.embulk.spi.PageOutput;
|
15
|
-
|
16
|
-
import com.cybozu.kintone.client.model.record.field.FieldValue;
|
17
|
-
import com.cybozu.kintone.client.model.record.GetRecordsResponse;
|
11
|
+
import org.embulk.spi.*;
|
18
12
|
import org.slf4j.Logger;
|
19
13
|
import org.slf4j.LoggerFactory;
|
20
14
|
|
15
|
+
import java.util.HashMap;
|
16
|
+
import java.util.List;
|
17
|
+
|
21
18
|
public class KintoneInputPlugin
|
22
19
|
implements InputPlugin {
|
23
20
|
private final Logger logger = LoggerFactory.getLogger(KintoneInputPlugin.class);
|
@@ -54,19 +51,29 @@ public class KintoneInputPlugin
|
|
54
51
|
PluginTask task = taskSource.loadTask(PluginTask.class);
|
55
52
|
|
56
53
|
try {
|
57
|
-
try (PageBuilder pageBuilder =
|
58
|
-
KintoneClient client =
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
54
|
+
try (PageBuilder pageBuilder = getPageBuilder(schema, output)) {
|
55
|
+
KintoneClient client = getKintoneClient();
|
56
|
+
client.validateAuth(task);
|
57
|
+
client.connect(task);
|
58
|
+
|
59
|
+
CreateRecordCursorResponse cursor = client.createCursor(task);
|
60
|
+
GetRecordCursorResponse cursorResponse = new GetRecordCursorResponse();
|
61
|
+
cursorResponse.setNext(true);
|
62
|
+
|
63
|
+
while (cursorResponse.getNext()) {
|
64
|
+
cursorResponse = client.getRecordsByCursor(cursor);
|
65
|
+
for (HashMap<String, FieldValue> record : cursorResponse.getRecords()) {
|
66
|
+
schema.visitColumns(new KintoneInputColumnVisitor(new KintoneAccessor(record), pageBuilder, task));
|
67
|
+
pageBuilder.addRecord();
|
68
|
+
}
|
69
|
+
pageBuilder.flush();
|
64
70
|
}
|
71
|
+
|
65
72
|
pageBuilder.finish();
|
66
73
|
}
|
67
74
|
} catch (Exception e) {
|
68
|
-
|
69
|
-
|
75
|
+
logger.error(e.getMessage());
|
76
|
+
throw e;
|
70
77
|
}
|
71
78
|
return Exec.newTaskReport();
|
72
79
|
}
|
@@ -75,4 +82,16 @@ public class KintoneInputPlugin
|
|
75
82
|
public ConfigDiff guess(ConfigSource config) {
|
76
83
|
return Exec.newConfigDiff();
|
77
84
|
}
|
85
|
+
|
86
|
+
@VisibleForTesting
|
87
|
+
protected PageBuilder getPageBuilder(final Schema schema, final PageOutput output)
|
88
|
+
{
|
89
|
+
return new PageBuilder(Exec.getBufferAllocator(), schema, output);
|
90
|
+
}
|
91
|
+
|
92
|
+
@VisibleForTesting
|
93
|
+
protected KintoneClient getKintoneClient(){
|
94
|
+
return new KintoneClient();
|
95
|
+
}
|
96
|
+
|
78
97
|
}
|
@@ -0,0 +1,17 @@
|
|
1
|
+
package org.embulk.input.kintone;
|
2
|
+
|
3
|
+
import com.cybozu.kintone.client.model.app.form.FieldType;
|
4
|
+
import com.cybozu.kintone.client.model.record.field.FieldValue;
|
5
|
+
|
6
|
+
import java.util.HashMap;
|
7
|
+
|
8
|
+
public class TestHelper {
|
9
|
+
public static HashMap<String, FieldValue> addField(HashMap<String, FieldValue> record, String code, FieldType type,
|
10
|
+
Object value) {
|
11
|
+
FieldValue newField = new FieldValue();
|
12
|
+
newField.setType(type);
|
13
|
+
newField.setValue(value);
|
14
|
+
record.put(code, newField);
|
15
|
+
return record;
|
16
|
+
}
|
17
|
+
}
|
@@ -0,0 +1,100 @@
|
|
1
|
+
package org.embulk.input.kintone;
|
2
|
+
|
3
|
+
|
4
|
+
import com.cybozu.kintone.client.model.app.form.FieldType;
|
5
|
+
import com.cybozu.kintone.client.model.record.field.FieldValue;
|
6
|
+
import com.cybozu.kintone.client.model.record.SubTableValueItem;
|
7
|
+
import com.cybozu.kintone.client.model.member.Member;
|
8
|
+
import org.junit.Test;
|
9
|
+
|
10
|
+
import java.util.ArrayList;
|
11
|
+
import java.util.HashMap;
|
12
|
+
|
13
|
+
import static org.junit.Assert.*;
|
14
|
+
|
15
|
+
|
16
|
+
public class TestKintoneAccessor {
|
17
|
+
private static Member testman1 = new Member("code1", "name1");
|
18
|
+
private static Member testman2 = new Member("code2", "name2");
|
19
|
+
private static Member testgroup1 = new Member("code3", "name3");
|
20
|
+
private static Member testgroup2 = new Member("code4", "name4");
|
21
|
+
private static Member testorg1 = new Member("code5", "name5");
|
22
|
+
private static Member testorg2 = new Member("code6", "name6");
|
23
|
+
private Integer uniqueKey = 1;
|
24
|
+
|
25
|
+
public HashMap<String, FieldValue> createTestRecord() {
|
26
|
+
HashMap<String, FieldValue> testRecord = new HashMap<>();
|
27
|
+
|
28
|
+
TestHelper.addField(testRecord, "文字列__1行", FieldType.SINGLE_LINE_TEXT, "test single text");
|
29
|
+
TestHelper.addField(testRecord, "数値", FieldType.NUMBER, this.uniqueKey);
|
30
|
+
this.uniqueKey += 1;
|
31
|
+
TestHelper.addField(testRecord, "文字列__複数行", FieldType.MULTI_LINE_TEXT, "test multi text");
|
32
|
+
TestHelper.addField(testRecord, "リッチエディター", FieldType.RICH_TEXT, "<div>test rich text<br /></div>");
|
33
|
+
|
34
|
+
ArrayList<String> selectedItemList = new ArrayList<>();
|
35
|
+
selectedItemList.add("sample1");
|
36
|
+
selectedItemList.add("sample2");
|
37
|
+
TestHelper.addField(testRecord, "チェックボックス", FieldType.CHECK_BOX, selectedItemList);
|
38
|
+
TestHelper.addField(testRecord, "ラジオボタン", FieldType.RADIO_BUTTON, "sample2");
|
39
|
+
TestHelper.addField(testRecord, "ドロップダウン", FieldType.DROP_DOWN, "sample3");
|
40
|
+
TestHelper.addField(testRecord, "複数選択", FieldType.MULTI_SELECT, selectedItemList);
|
41
|
+
TestHelper.addField(testRecord, "リンク", FieldType.LINK, "http://cybozu.co.jp/");
|
42
|
+
TestHelper.addField(testRecord, "日付", FieldType.DATE, "2018-01-01");
|
43
|
+
TestHelper.addField(testRecord, "時刻", FieldType.TIME, "12:34");
|
44
|
+
TestHelper.addField(testRecord, "日時", FieldType.DATETIME, "2018-01-02T02:30:00Z");
|
45
|
+
|
46
|
+
ArrayList<Member> userList = new ArrayList<>();
|
47
|
+
userList.add(testman1);
|
48
|
+
userList.add(testman2);
|
49
|
+
TestHelper.addField(testRecord, "ユーザー選択", FieldType.USER_SELECT, userList);
|
50
|
+
ArrayList<Member> groupList = new ArrayList<>();
|
51
|
+
groupList.add(testgroup1);
|
52
|
+
groupList.add(testgroup2);
|
53
|
+
TestHelper.addField(testRecord, "グループ選択", FieldType.GROUP_SELECT, groupList);
|
54
|
+
ArrayList<Member> orgList = new ArrayList<>();
|
55
|
+
orgList.add(testorg1);
|
56
|
+
orgList.add(testorg2);
|
57
|
+
TestHelper.addField(testRecord, "組織選択", FieldType.ORGANIZATION_SELECT, orgList);
|
58
|
+
|
59
|
+
SubTableValueItem tableItem1 = new SubTableValueItem();
|
60
|
+
tableItem1.setID(1);
|
61
|
+
HashMap<String, FieldValue> tableItemValue1 = new HashMap<>();
|
62
|
+
FieldValue fv1 = new FieldValue();
|
63
|
+
fv1.setType(FieldType.SINGLE_LINE_TEXT);
|
64
|
+
fv1.setValue("sample_text1");
|
65
|
+
tableItemValue1.put("sample field1", fv1);
|
66
|
+
tableItem1.setValue(tableItemValue1);
|
67
|
+
ArrayList<SubTableValueItem> subTableRecords = new ArrayList<>();
|
68
|
+
subTableRecords.add(tableItem1);
|
69
|
+
TestHelper.addField(testRecord, "サブテーブル", FieldType.SUBTABLE, subTableRecords);
|
70
|
+
|
71
|
+
return testRecord;
|
72
|
+
}
|
73
|
+
|
74
|
+
@Test
|
75
|
+
public void testAccess() {
|
76
|
+
HashMap<String, FieldValue> testRecord = createTestRecord();
|
77
|
+
KintoneAccessor accessor = new KintoneAccessor(testRecord);
|
78
|
+
String multiValue = "sample1\nsample2";
|
79
|
+
String userSelect = "code1\ncode2";
|
80
|
+
String groupSelect = "code3\ncode4";
|
81
|
+
String orgSelect = "code5\ncode6";
|
82
|
+
String subTableValue = "[{\"id\":1,\"value\":{\"sample field1\":{\"type\":\"SINGLE_LINE_TEXT\",\"value\":\"sample_text1\"}}}]";
|
83
|
+
assertEquals(testRecord.get("文字列__1行").getValue(), accessor.get("文字列__1行"));
|
84
|
+
assertEquals("1", accessor.get("数値"));
|
85
|
+
assertEquals(testRecord.get("文字列__複数行").getValue(), accessor.get("文字列__複数行"));
|
86
|
+
assertEquals(testRecord.get("リッチエディター").getValue(), accessor.get("リッチエディター"));
|
87
|
+
assertEquals(multiValue, accessor.get("チェックボックス"));
|
88
|
+
assertEquals(testRecord.get("ラジオボタン").getValue(), accessor.get("ラジオボタン"));
|
89
|
+
assertEquals(testRecord.get("ドロップダウン").getValue(), accessor.get("ドロップダウン"));
|
90
|
+
assertEquals(multiValue, accessor.get("複数選択"));
|
91
|
+
assertEquals(testRecord.get("リンク").getValue(), accessor.get("リンク"));
|
92
|
+
assertEquals(testRecord.get("日付").getValue(), accessor.get("日付"));
|
93
|
+
assertEquals(testRecord.get("時刻").getValue(), accessor.get("時刻"));
|
94
|
+
assertEquals(testRecord.get("日時").getValue(), accessor.get("日時"));
|
95
|
+
assertEquals(userSelect, accessor.get("ユーザー選択"));
|
96
|
+
assertEquals(groupSelect, accessor.get("グループ選択"));
|
97
|
+
assertEquals(orgSelect, accessor.get("組織選択"));
|
98
|
+
assertEquals(subTableValue, accessor.get("サブテーブル"));
|
99
|
+
}
|
100
|
+
}
|
@@ -0,0 +1,80 @@
|
|
1
|
+
package org.embulk.input.kintone;
|
2
|
+
|
3
|
+
import org.embulk.config.ConfigException;
|
4
|
+
import org.embulk.config.ConfigSource;
|
5
|
+
import org.embulk.spi.InputPlugin;
|
6
|
+
|
7
|
+
import org.embulk.test.TestingEmbulk;
|
8
|
+
import org.junit.Rule;
|
9
|
+
import org.junit.Test;
|
10
|
+
|
11
|
+
import static org.junit.Assert.*;
|
12
|
+
|
13
|
+
public class TestKintoneClient {
|
14
|
+
private ConfigSource config;
|
15
|
+
private KintoneClient client = new KintoneClient();
|
16
|
+
private static final String BASIC_RESOURCE_PATH = "org/embulk/input/kintone/";
|
17
|
+
private static final String SUCCESS_MSG = "Exception should be thrown by this";
|
18
|
+
|
19
|
+
private static ConfigSource loadYamlResource(TestingEmbulk embulk, String fileName) {
|
20
|
+
return embulk.loadYamlResource(BASIC_RESOURCE_PATH + fileName);
|
21
|
+
}
|
22
|
+
|
23
|
+
@Rule
|
24
|
+
public TestingEmbulk embulk = TestingEmbulk.builder()
|
25
|
+
.registerPlugin(InputPlugin.class, "kintone", KintoneInputPlugin.class)
|
26
|
+
.build();
|
27
|
+
|
28
|
+
@Test
|
29
|
+
public void checkClientWithUsernameAndPassword() {
|
30
|
+
config = loadYamlResource(embulk, "base.yml");
|
31
|
+
PluginTask task = config.loadConfig(PluginTask.class);
|
32
|
+
Exception e = assertThrows(Exception.class, ()-> {
|
33
|
+
client.validateAuth(task);
|
34
|
+
throw new Exception(SUCCESS_MSG);
|
35
|
+
});
|
36
|
+
assertEquals(SUCCESS_MSG, e.getMessage());
|
37
|
+
}
|
38
|
+
|
39
|
+
@Test
|
40
|
+
public void checkThrowErrorWithoutAuthInfo() {
|
41
|
+
config = loadYamlResource(embulk, "base.yml");
|
42
|
+
config.remove("username")
|
43
|
+
.remove("password");
|
44
|
+
PluginTask task = config.loadConfig(PluginTask.class);
|
45
|
+
ConfigException e = assertThrows(ConfigException.class, () -> client.validateAuth(task));
|
46
|
+
assertEquals("Username and password or token must be provided", e.getMessage());
|
47
|
+
}
|
48
|
+
|
49
|
+
@Test
|
50
|
+
public void checkClientErrorLackingPassword() {
|
51
|
+
config = loadYamlResource(embulk, "base.yml");
|
52
|
+
config.remove("password");
|
53
|
+
PluginTask task = config.loadConfig(PluginTask.class);
|
54
|
+
ConfigException e = assertThrows(ConfigException.class, () -> client.validateAuth(task));
|
55
|
+
assertEquals("Username and password or token must be provided", e.getMessage());
|
56
|
+
}
|
57
|
+
|
58
|
+
@Test
|
59
|
+
public void checkClientErrorLackingUsername() {
|
60
|
+
config = loadYamlResource(embulk, "base.yml");
|
61
|
+
config.remove("username");
|
62
|
+
PluginTask task = config.loadConfig(PluginTask.class);
|
63
|
+
ConfigException e = assertThrows(ConfigException.class, () -> client.validateAuth(task));
|
64
|
+
assertEquals("Username and password or token must be provided", e.getMessage());
|
65
|
+
}
|
66
|
+
|
67
|
+
@Test
|
68
|
+
public void checkClientWithToken() {
|
69
|
+
config = loadYamlResource(embulk, "base.yml");
|
70
|
+
config.remove("username")
|
71
|
+
.remove("password")
|
72
|
+
.set("token", "token");
|
73
|
+
PluginTask task = config.loadConfig(PluginTask.class);
|
74
|
+
Exception e = assertThrows(Exception.class, ()-> {
|
75
|
+
client.validateAuth(task);
|
76
|
+
throw new Exception(SUCCESS_MSG);
|
77
|
+
});
|
78
|
+
assertEquals(SUCCESS_MSG, e.getMessage());
|
79
|
+
}
|
80
|
+
}
|
@@ -1,5 +1,150 @@
|
|
1
1
|
package org.embulk.input.kintone;
|
2
2
|
|
3
|
-
|
4
|
-
|
3
|
+
import com.cybozu.kintone.client.model.app.form.FieldType;
|
4
|
+
import com.cybozu.kintone.client.model.record.GetRecordsResponse;
|
5
|
+
import com.cybozu.kintone.client.model.record.field.FieldValue;
|
6
|
+
|
7
|
+
import org.embulk.EmbulkTestRuntime;
|
8
|
+
import org.embulk.config.ConfigDiff;
|
9
|
+
import org.embulk.config.ConfigSource;
|
10
|
+
import org.embulk.config.TaskReport;
|
11
|
+
import org.embulk.config.TaskSource;
|
12
|
+
import org.embulk.spi.InputPlugin;
|
13
|
+
import org.embulk.spi.Schema;
|
14
|
+
import org.embulk.spi.TestPageBuilderReader.MockPageOutput;
|
15
|
+
import org.embulk.spi.time.Timestamp;
|
16
|
+
import org.embulk.spi.time.TimestampParser;
|
17
|
+
import org.embulk.spi.util.Pages;
|
18
|
+
import org.embulk.test.TestingEmbulk;
|
19
|
+
|
20
|
+
import org.junit.Before;
|
21
|
+
import org.junit.Rule;
|
22
|
+
import org.junit.Test;
|
23
|
+
|
24
|
+
import java.util.ArrayList;
|
25
|
+
import java.util.HashMap;
|
26
|
+
import java.util.List;
|
27
|
+
import java.util.stream.Collectors;
|
28
|
+
import java.util.stream.IntStream;
|
29
|
+
|
30
|
+
import static org.junit.Assert.*;
|
31
|
+
import static org.mockito.Mockito.*;
|
32
|
+
|
33
|
+
public class TestKintoneInputPlugin {
|
34
|
+
private ConfigSource config;
|
35
|
+
private static final String BASIC_RESOURCE_PATH = "org/embulk/input/kintone/";
|
36
|
+
private KintoneInputPlugin kintoneInputPlugin;
|
37
|
+
private KintoneClient kintoneClient;
|
38
|
+
private MockPageOutput output = new MockPageOutput();
|
39
|
+
private TimestampParser dateParser = TimestampParser.of("%Y-%m-%d", "UTC");
|
40
|
+
private TimestampParser timestampParser = TimestampParser.of("%Y-%m-%dT%H:%M:%S%z", "UTC");
|
41
|
+
|
42
|
+
private static ConfigSource loadYamlResource(TestingEmbulk embulk, String fileName) {
|
43
|
+
return embulk.loadYamlResource(BASIC_RESOURCE_PATH + fileName);
|
44
|
+
}
|
45
|
+
|
46
|
+
@Before
|
47
|
+
public void prepare(){
|
48
|
+
kintoneInputPlugin = spy(new KintoneInputPlugin());
|
49
|
+
kintoneClient = mock(KintoneClient.class);
|
50
|
+
doReturn(kintoneClient).when(kintoneInputPlugin).getKintoneClient();
|
51
|
+
}
|
52
|
+
@Rule
|
53
|
+
public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
|
54
|
+
|
55
|
+
@Rule
|
56
|
+
public TestingEmbulk embulk = TestingEmbulk.builder()
|
57
|
+
.registerPlugin(InputPlugin.class, "kintone", KintoneInputPlugin.class)
|
58
|
+
.build();
|
59
|
+
|
60
|
+
// Comment out for now
|
61
|
+
// @Test
|
62
|
+
public void simpleTest(){
|
63
|
+
config = loadYamlResource(embulk, "base.yml");
|
64
|
+
PluginTask task = config.loadConfig(PluginTask.class);
|
65
|
+
Schema outputSchema = task.getFields().toSchema();
|
66
|
+
GetRecordsResponse response = createSampleData();
|
67
|
+
when(kintoneClient.getResponse(any(PluginTask.class))).thenReturn(response);
|
68
|
+
|
69
|
+
ConfigDiff configDiff = kintoneInputPlugin.transaction(config, new Control());
|
70
|
+
|
71
|
+
assertTrue(configDiff.isEmpty());
|
72
|
+
|
73
|
+
List<Object[]> outputRecords = Pages.toObjects(outputSchema, output.pages);
|
74
|
+
Object[] record1 = outputRecords.get(0);
|
75
|
+
|
76
|
+
Timestamp date1 = dateParser.parse("2020-01-01");
|
77
|
+
Timestamp timestamp1 = timestampParser.parse("2020-01-01T00:00:00Z");
|
78
|
+
|
79
|
+
assertEquals(2, outputRecords.size());
|
80
|
+
|
81
|
+
assertEquals("test single text", record1[0]);
|
82
|
+
assertEquals(1L, record1[1]);
|
83
|
+
assertEquals(1.111, record1[2]);
|
84
|
+
assertEquals(date1, record1[3]);
|
85
|
+
assertEquals(timestamp1, record1[4]);
|
86
|
+
|
87
|
+
Timestamp date2 = dateParser.parse("2020-02-02");
|
88
|
+
Timestamp timestamp2 = timestampParser.parse("2020-02-02T00:00:00Z");
|
89
|
+
|
90
|
+
Object[] record2 = outputRecords.get(1);
|
91
|
+
assertEquals("test single text2", record2[0]);
|
92
|
+
assertEquals(2L, record2[1]);
|
93
|
+
assertEquals(2.222, record2[2]);
|
94
|
+
assertEquals(date2, record2[3]);
|
95
|
+
assertEquals(timestamp2, record2[4]);
|
96
|
+
}
|
97
|
+
|
98
|
+
@Test
|
99
|
+
public void checkDefaultConfigValues() {
|
100
|
+
config = loadYamlResource(embulk, "base.yml");
|
101
|
+
PluginTask task = config.loadConfig(PluginTask.class);
|
102
|
+
assertEquals("dev.cybozu.com", task.getDomain());
|
103
|
+
assertEquals(1, task.getAppId());
|
104
|
+
assertEquals("username", task.getUsername().get());
|
105
|
+
assertEquals("password", task.getPassword().get());
|
106
|
+
assertFalse(task.getToken().isPresent());
|
107
|
+
assertFalse(task.getGuestSpaceId().isPresent());
|
108
|
+
assertFalse(task.getBasicAuthUsername().isPresent());
|
109
|
+
assertFalse(task.getBasicAuthPassword().isPresent());
|
110
|
+
assertFalse(task.getQuery().isPresent());
|
111
|
+
assertNotNull(task.getFields());
|
112
|
+
}
|
113
|
+
|
114
|
+
private GetRecordsResponse createSampleData(){
|
115
|
+
HashMap<String, FieldValue> record1 = new HashMap<>();
|
116
|
+
HashMap<String, FieldValue> record2 = new HashMap<>();
|
117
|
+
ArrayList<HashMap<String, FieldValue>> records = new ArrayList<>();
|
118
|
+
GetRecordsResponse response = new GetRecordsResponse();
|
119
|
+
|
120
|
+
record1 = TestHelper.addField(record1, "foo", FieldType.SINGLE_LINE_TEXT, "test single text");
|
121
|
+
record1 = TestHelper.addField(record1, "bar", FieldType.NUMBER, 1);
|
122
|
+
record1 = TestHelper.addField(record1, "baz", FieldType.NUMBER, 1.111);
|
123
|
+
record1 = TestHelper.addField(record1, "date", FieldType.DATE, "2020-01-01");
|
124
|
+
record1 = TestHelper.addField(record1, "datetime", FieldType.DATE, "2020-01-01T00:00:00Z");
|
125
|
+
records.add(record1);
|
126
|
+
|
127
|
+
record2 = TestHelper.addField(record2, "foo", FieldType.SINGLE_LINE_TEXT, "test single text2");
|
128
|
+
record2 = TestHelper.addField(record2, "bar", FieldType.NUMBER, 2);
|
129
|
+
record2 = TestHelper.addField(record2, "baz", FieldType.NUMBER, 2.222);
|
130
|
+
record2 = TestHelper.addField(record2, "date", FieldType.DATE, "2020-02-02");
|
131
|
+
record2 = TestHelper.addField(record2, "datetime", FieldType.DATE, "2020-02-02T00:00:00Z");
|
132
|
+
records.add(record2);
|
133
|
+
|
134
|
+
response.setRecords(records);
|
135
|
+
response.setTotalCount(2);
|
136
|
+
return response;
|
137
|
+
}
|
138
|
+
|
139
|
+
private class Control implements InputPlugin.Control
|
140
|
+
{
|
141
|
+
@Override
|
142
|
+
public List<TaskReport> run(final TaskSource taskSource, final Schema schema, final int taskCount)
|
143
|
+
{
|
144
|
+
List<TaskReport> reports = IntStream.range(0, taskCount)
|
145
|
+
.mapToObj(i -> kintoneInputPlugin.run(taskSource, schema, i, output))
|
146
|
+
.collect(Collectors.toList());
|
147
|
+
return reports;
|
148
|
+
}
|
149
|
+
}
|
5
150
|
}
|
@@ -0,0 +1,11 @@
|
|
1
|
+
type: kintone
|
2
|
+
domain: dev.cybozu.com
|
3
|
+
app_id: 1
|
4
|
+
username: username
|
5
|
+
password: password
|
6
|
+
fields:
|
7
|
+
- {name: foo, type: string}
|
8
|
+
- {name: bar, type: long}
|
9
|
+
- {name: baz, type: double}
|
10
|
+
- {name: date, type: timestamp, format: '%Y-%m-%d'}
|
11
|
+
- {name: datetime, type: timestamp}
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-input-kintone
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.5
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- giwa
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2021-01-20 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -46,10 +46,11 @@ extensions: []
|
|
46
46
|
extra_rdoc_files: []
|
47
47
|
files:
|
48
48
|
- ".gitignore"
|
49
|
+
- ".travis.yml"
|
49
50
|
- LICENSE.txt
|
50
51
|
- README.md
|
51
52
|
- build.gradle
|
52
|
-
- classpath/embulk-input-kintone-0.1.
|
53
|
+
- classpath/embulk-input-kintone-0.1.5.jar
|
53
54
|
- classpath/gson-2.8.2.jar
|
54
55
|
- classpath/kintone-sdk-0.4.0.jar
|
55
56
|
- config/checkstyle/checkstyle.xml
|
@@ -64,8 +65,12 @@ files:
|
|
64
65
|
- src/main/java/org/embulk/input/kintone/KintoneInputColumnVisitor.java
|
65
66
|
- src/main/java/org/embulk/input/kintone/KintoneInputPlugin.java
|
66
67
|
- src/main/java/org/embulk/input/kintone/PluginTask.java
|
68
|
+
- src/test/java/org/embulk/input/kintone/TestHelper.java
|
69
|
+
- src/test/java/org/embulk/input/kintone/TestKintoneAccessor.java
|
70
|
+
- src/test/java/org/embulk/input/kintone/TestKintoneClient.java
|
67
71
|
- src/test/java/org/embulk/input/kintone/TestKintoneInputPlugin.java
|
68
|
-
|
72
|
+
- src/test/resources/org/embulk/input/kintone/base.yml
|
73
|
+
homepage: https://github.com/trocco-io/embulk-input-kintone
|
69
74
|
licenses:
|
70
75
|
- MIT
|
71
76
|
metadata: {}
|