embulk-input-kintone 0.1.0 → 0.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 50b882dc3ce74fcb86df7ba392da3046c4944e99
-  data.tar.gz: a85a843021b74f4eda4dc0a9241193413bddcd9b
+  metadata.gz: 646c14f2c41e56b23174e35fcbc2147072d49ba5
+  data.tar.gz: 90477839161c398082fb0cceae2c2cb14e1a2bc3
 SHA512:
-  metadata.gz: d8f3927799c2d3507e5dc764d798db24813c95e8bb4f563ba93d75a84cc6ec96ac42fd81f2c4b7161ae87dd306a2658a95c95a25b069152aee88a25b6f7320b4
-  data.tar.gz: 88c09ff176b14da18790495f39e5b5bdadc37d5abbb7f453871ecc54d8f196eed56e1fa6d4b33af7063e5352bcaf4b637aad5d456c67fa3904f912f0cd44fc8e
+  metadata.gz: f0abfe7044ab966a6ad27158bdb56941b6857be9c1bd97714a5c9be87faff45e5c0c88e0f21746965cf7fcfeaad1e63542729ecb5b1ab1655c28c8d7c6946ca4
+  data.tar.gz: da7f3422eccc3a7cd49a3ded84c37fabb02c90489623ec553443ac6c36b9891be2317456b1e1b1a8bf5724efea75d4e08ba755e68fa2e159fd3076b6194befb9

data/.travis.yml

@@ -0,0 +1,7 @@
+language: java
+jdk:
+- oraclejdk8
+- openjdk8
+script:
+- ./gradlew --info check
+dist: trusty

data/README.md

@@ -1,9 +1,13 @@
-# Kintone input plugin for Embulk
+# kintone input plugin for Embulk
+[![Build Status](https://travis-ci.org/trocco-io/embulk-input-kintone.svg?branch=master)](https://travis-ci.org/trocco-io/embulk-input-kintone)
 
 ## Overview
-Kintone input plugin for Embulk loads app records from Kintone.
+kintone input plugin for Embulk loads app records from kintone.
 embulk 0.9 is only supported due to the dependency of kintone-java-sdk 0.4.0, which requires java 8
 
+This plugin uses [cursor API](https://developer.kintone.io/hc/en-us/articles/360000280322). See the limitation on this page.
+e.g. limit, offset are not supported.
+
 * **Plugin type**: input
 * **Resume supported**: no
 * **Cleanup supported**: no
@@ -11,9 +15,7 @@ embulk 0.9 is only supported due to the dependency of kintone-java-sdk 0.4.0, wh
 
 ## Road Map
 - [ ] Guess
-- [ ] Subtable data type support
 - [ ] field name mapping
-- [ ] timestamp_format in fields
 - [ ] handle certification fot authentication
 
 ## Configuration
@@ -27,11 +29,12 @@ embulk 0.9 is only supported due to the dependency of kintone-java-sdk 0.4.0, wh
 - **basic_auth_username**:  Kintone basic auth username Please see Kintone basic auth [here](https://jp.cybozu.help/general/en/admin/list_security/list_ip_basic/basic_auth.html) (string, optional)
 - **basic_auth_password**:  Kintone basic auth password (string, optional)
 - **guest_space_id**: Kintone app belongs to guest space, guest space id is required. (integer, optional)
-- **fields**
+- **fields** (required)
   - **name** the field code of Kintone app record will be retrieved.
-  - **type** Column values are converted to this embulk type. Available values options are: boolean, long, double, string, json, timestamp)
+  - **type** Column values are converted to this embulk type. Available values options are: boolean, long, double, string, json, timestamp) Kintone `SUBTABLE` type is loaded as json text.
+  - **format** Format of the timestamp if type is timestamp. The format for kintone DATETIME is `%Y-%m-%dT%H:%M:%S%z`.
 
-Kintone API has the limitation, therefore this plugin also faces it. See [official documentation](https://developer.kintone.io/hc/en-us/articles/212495188/)
+kintone API has the limitation, therefore this plugin also faces it. See [official documentation](https://developer.kintone.io/hc/en-us/articles/212495188/)
 
 ## Example
 
@@ -59,13 +62,14 @@ in:
   username: user
   password: password
   app_id: 1
-  query: Time > 10:00 and Time < 19:00 and Created_datatime = TODAY() order by $id asc limit 10
+  query: Time > 10:00 and Time < 19:00 and Created_datatime = TODAY() order by $id asc
   fields:
     - {name: $id, type: long}
     - {name: $revision, type: long}
     - {name: Time, type: string}
-    - {name: Created_datatime, type: long}
+    - {name: Created_datatime, type: string}
     - {name: foo, type: string}
+    - {name: datetime, type: timestamp, format: '%Y-%m-%dT%H:%M:%S%z'}
 ```
 
 ## Build
@@ -73,3 +77,9 @@ in:
 ```
 $ ./gradlew gem  # -t to watch change of files and rebuild continuously
 ```
+
+## Development
+```
+$ ./gradew build
+$ ./gradew test
+```

data/build.gradle

@@ -13,7 +13,7 @@ configurations {
     provided
 }
 
-version = "0.1.0"
+version = "0.1.5"
 
 sourceCompatibility = 1.8
 targetCompatibility = 1.8
@@ -22,8 +22,12 @@ dependencies {
     compile  "org.embulk:embulk-core:0.9.12"
     provided "org.embulk:embulk-core:0.9.12"
     compile group: 'com.cybozu.kintone', name: 'kintone-sdk', version: '0.4.0'
-    // compile "YOUR_JAR_DEPENDENCY_GROUP:YOUR_JAR_DEPENDENCY_MODULE:YOUR_JAR_DEPENDENCY_VERSION"
+
     testCompile "junit:junit:4.+"
+    testCompile 'org.embulk:embulk-standards:0.9.12'
+    testCompile 'org.embulk:embulk-test:0.9.12'
+    testCompile "org.mockito:mockito-core:1.+"
+    testCompile "org.embulk:embulk-core:0.9.12:tests"
 }
 
 task classpath(type: Copy, dependsOn: ["jar"]) {
@@ -83,7 +87,7 @@ Gem::Specification.new do |spec|
   spec.description   = %[Loads records from Kintone.]
   spec.email         = ["ugw.gi.world@gmail.com"]
   spec.licenses      = ["MIT"]
-  # TODO set this: spec.homepage      = "https://github.com/ugw.gi.world/embulk-input-kintone"
+  spec.homepage      = "https://github.com/trocco-io/embulk-input-kintone"
 
   spec.files         = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
   spec.test_files    = spec.files.grep(%r"^(test|spec)/")

data/gradle/wrapper/gradle-wrapper.properties

@@ -1,5 +1,6 @@
+#Fri Oct 11 16:04:33 JST 2019
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-4.1-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-4.1-all.zip

data/gradlew.bat

@@ -1,84 +1,84 @@
-@if "%DEBUG%" == "" @echo off
-@rem ##########################################################################
-@rem
-@rem  Gradle startup script for Windows
-@rem
-@rem ##########################################################################
-
-@rem Set local scope for the variables with windows NT shell
-if "%OS%"=="Windows_NT" setlocal
-
-set DIRNAME=%~dp0
-if "%DIRNAME%" == "" set DIRNAME=.
-set APP_BASE_NAME=%~n0
-set APP_HOME=%DIRNAME%
-
-@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
-set DEFAULT_JVM_OPTS=
-
-@rem Find java.exe
-if defined JAVA_HOME goto findJavaFromJavaHome
-
-set JAVA_EXE=java.exe
-%JAVA_EXE% -version >NUL 2>&1
-if "%ERRORLEVEL%" == "0" goto init
-
-echo.
-echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
-echo.
-echo Please set the JAVA_HOME variable in your environment to match the
-echo location of your Java installation.
-
-goto fail
-
-:findJavaFromJavaHome
-set JAVA_HOME=%JAVA_HOME:"=%
-set JAVA_EXE=%JAVA_HOME%/bin/java.exe
-
-if exist "%JAVA_EXE%" goto init
-
-echo.
-echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
-echo.
-echo Please set the JAVA_HOME variable in your environment to match the
-echo location of your Java installation.
-
-goto fail
-
-:init
-@rem Get command-line arguments, handling Windows variants
-
-if not "%OS%" == "Windows_NT" goto win9xME_args
-
-:win9xME_args
-@rem Slurp the command line arguments.
-set CMD_LINE_ARGS=
-set _SKIP=2
-
-:win9xME_args_slurp
-if "x%~1" == "x" goto execute
-
-set CMD_LINE_ARGS=%*
-
-:execute
-@rem Setup the command line
-
-set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
-
-@rem Execute Gradle
-"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
-
-:end
-@rem End local scope for the variables with windows NT shell
-if "%ERRORLEVEL%"=="0" goto mainEnd
-
-:fail
-rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
-rem the _cmd.exe /c_ return code!
-if  not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
-exit /b 1
-
-:mainEnd
-if "%OS%"=="Windows_NT" endlocal
-
-:omega
+@if "%DEBUG%" == "" @echo off
+@rem ##########################################################################
+@rem
+@rem  Gradle startup script for Windows
+@rem
+@rem ##########################################################################
+
+@rem Set local scope for the variables with windows NT shell
+if "%OS%"=="Windows_NT" setlocal
+
+set DIRNAME=%~dp0
+if "%DIRNAME%" == "" set DIRNAME=.
+set APP_BASE_NAME=%~n0
+set APP_HOME=%DIRNAME%
+
+@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+set DEFAULT_JVM_OPTS=
+
+@rem Find java.exe
+if defined JAVA_HOME goto findJavaFromJavaHome
+
+set JAVA_EXE=java.exe
+%JAVA_EXE% -version >NUL 2>&1
+if "%ERRORLEVEL%" == "0" goto init
+
+echo.
+echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
+echo.
+echo Please set the JAVA_HOME variable in your environment to match the
+echo location of your Java installation.
+
+goto fail
+
+:findJavaFromJavaHome
+set JAVA_HOME=%JAVA_HOME:"=%
+set JAVA_EXE=%JAVA_HOME%/bin/java.exe
+
+if exist "%JAVA_EXE%" goto init
+
+echo.
+echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
+echo.
+echo Please set the JAVA_HOME variable in your environment to match the
+echo location of your Java installation.
+
+goto fail
+
+:init
+@rem Get command-line arguments, handling Windows variants
+
+if not "%OS%" == "Windows_NT" goto win9xME_args
+
+:win9xME_args
+@rem Slurp the command line arguments.
+set CMD_LINE_ARGS=
+set _SKIP=2
+
+:win9xME_args_slurp
+if "x%~1" == "x" goto execute
+
+set CMD_LINE_ARGS=%*
+
+:execute
+@rem Setup the command line
+
+set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
+
+@rem Execute Gradle
+"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
+
+:end
+@rem End local scope for the variables with windows NT shell
+if "%ERRORLEVEL%"=="0" goto mainEnd
+
+:fail
+rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
+rem the _cmd.exe /c_ return code!
+if  not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
+exit /b 1
+
+:mainEnd
+if "%OS%"=="Windows_NT" endlocal
+
+:omega

data/src/main/java/org/embulk/input/kintone/KintoneAccessor.java

@@ -5,12 +5,14 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import com.cybozu.kintone.client.model.record.field.FieldValue;
 import com.cybozu.kintone.client.model.member.Member;
+import com.google.gson.Gson;
 
 import java.util.ArrayList;
 import java.util.HashMap;
 
 public class KintoneAccessor {
     private final Logger logger = LoggerFactory.getLogger(KintoneAccessor.class);
+    private final Gson gson = new Gson();
 
     private final HashMap<String, FieldValue> record;
     private final String delimiter = "\n";
@@ -26,12 +28,12 @@ public class KintoneAccessor {
             case GROUP_SELECT:
             case STATUS_ASSIGNEE:
                 ArrayList<Member> members = (ArrayList<Member>) this.record.get(name).getValue();
-                return members.stream().map(m -> m.getCode())
+                return members.stream().map(Member::getCode)
                         .reduce((accum, value) -> accum + this.delimiter + value)
                         .orElse("");
             case SUBTABLE:
-                // TODO: support sub table
-                return "";
+                Object subTableValueItem = this.record.get(name).getValue();
+                return gson.toJson(subTableValueItem);
             case CREATOR:
             case MODIFIER:
                 Member m = (Member) this.record.get(name).getValue();
@@ -45,9 +47,11 @@ public class KintoneAccessor {
                         .orElse("");
             case FILE:
                 ArrayList<FileModel> cbFileList = (ArrayList<FileModel>) this.record.get(name).getValue();
-                return cbFileList.stream().map(f -> f.getFileKey())
+                return cbFileList.stream().map(FileModel::getFileKey)
                         .reduce((accum, value) -> accum + this.delimiter + value)
                         .orElse("");
+            case NUMBER:
+                return String.valueOf(this.record.get(name).getValue());
             default:
                 return (String) this.record.get(name).getValue();
         }

data/src/main/java/org/embulk/input/kintone/KintoneClient.java

@@ -2,11 +2,13 @@ package org.embulk.input.kintone;
 
 import com.cybozu.kintone.client.authentication.Auth;
 import com.cybozu.kintone.client.connection.Connection;
+import com.cybozu.kintone.client.exception.KintoneAPIException;
+import com.cybozu.kintone.client.model.cursor.CreateRecordCursorResponse;
+import com.cybozu.kintone.client.model.cursor.GetRecordCursorResponse;
 import com.cybozu.kintone.client.model.record.GetRecordsResponse;
-import com.cybozu.kintone.client.module.record.Record;
+import com.cybozu.kintone.client.module.recordCursor.RecordCursor;
 import org.embulk.config.ConfigException;
-import org.embulk.spi.*;
-
+import org.embulk.spi.ColumnConfig;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -14,52 +16,84 @@ import java.util.ArrayList;
 
 public class KintoneClient {
     private final Logger logger = LoggerFactory.getLogger(KintoneClient.class);
-    private final PluginTask pluginTask;
-    private ArrayList<String> fields;
+    private static final int FETCH_SIZE = 500;
     private Auth kintoneAuth;
-    private Record kintoneRecordManager;
+    private RecordCursor kintoneRecordManager;
     private Connection con;
 
-    public KintoneClient(final PluginTask pluginTask) {
-        this.pluginTask = pluginTask;
+    public KintoneClient(){
         this.kintoneAuth = new Auth();
+    }
 
-        this.fields = new ArrayList<String>();
-        for (ColumnConfig c : pluginTask.getFields().getColumns()
-        ) {
-            fields.add(c.getName());
-        }
-        this.setAuth();
-        if (pluginTask.getGuestSpaceId().isPresent()) {
-            this.con = new Connection(pluginTask.getDomain(), this.kintoneAuth, pluginTask.getGuestSpaceId().or(-1));
+    public void validateAuth(final PluginTask task) throws ConfigException{
+        if (task.getUsername().isPresent() && task.getPassword().isPresent()) {
+            return;
+        } else if (task.getToken().isPresent()) {
+            return;
         } else {
-            this.con = new Connection(pluginTask.getDomain(), this.kintoneAuth);
+            throw new ConfigException("Username and password or token must be provided");
         }
-        this.kintoneRecordManager = new Record(con);
     }
 
-    private void setAuth() {
-        if (pluginTask.getUsername().isPresent() && pluginTask.getPassword().isPresent()) {
-            this.kintoneAuth.setPasswordAuth(pluginTask.getUsername().get(), pluginTask.getPassword().get());
-        } else if (pluginTask.getToken().isPresent()) {
-            this.kintoneAuth.setApiToken(pluginTask.getToken().get());
+    public void connect(final PluginTask task) {
+        if (task.getUsername().isPresent() && task.getPassword().isPresent()) {
+            this.kintoneAuth.setPasswordAuth(task.getUsername().get(), task.getPassword().get());
+        } else if (task.getToken().isPresent()) {
+            this.kintoneAuth.setApiToken(task.getToken().get());
+        }
+
+        if (task.getBasicAuthUsername().isPresent() && task.getBasicAuthPassword().isPresent()) {
+            this.kintoneAuth.setBasicAuth(task.getBasicAuthUsername().get(),
+                    task.getBasicAuthPassword().get());
+        }
+
+        if (task.getGuestSpaceId().isPresent()) {
+            this.con = new Connection(task.getDomain(), this.kintoneAuth, task.getGuestSpaceId().or(-1));
         } else {
-            throw new ConfigException("Username and password or token must be provided");
+            this.con = new Connection(task.getDomain(), this.kintoneAuth);
         }
+        this.kintoneRecordManager = new RecordCursor(con);
+    }
+
 
-        if (pluginTask.getBasicAuthUsername().isPresent() && pluginTask.getBasicAuthPassword().isPresent()) {
-            this.kintoneAuth.setBasicAuth(pluginTask.getBasicAuthUsername().get(),
-                    pluginTask.getBasicAuthPassword().get());
+    public GetRecordsResponse getResponse(final PluginTask task) {
+        CreateRecordCursorResponse cursor = this.createCursor(task);
+        try {
+            return this.kintoneRecordManager.getAllRecords(cursor.getId());
+        }catch (KintoneAPIException e){
+            this.deleteCursor(cursor);
+            throw new RuntimeException(e);
         }
     }
 
-    public GetRecordsResponse getResponse() {
+    public GetRecordCursorResponse getRecordsByCursor(CreateRecordCursorResponse cursor){
         try {
-            return kintoneRecordManager.getAllRecordsByQuery(
-                    this.pluginTask.getAppId(), this.pluginTask.getQuery().or(""), this.fields);
-        } catch (Exception e) {
+            return this.kintoneRecordManager.getRecords(cursor.getId());
+        }catch (KintoneAPIException e){
+            this.deleteCursor(cursor);
             throw new RuntimeException(e);
         }
     }
 
+    public CreateRecordCursorResponse createCursor(final PluginTask task){
+        ArrayList<String> fields = new ArrayList<>();
+        for (ColumnConfig c : task.getFields().getColumns()
+        ) {
+            fields.add(c.getName());
+        }
+
+        try{
+            return this.kintoneRecordManager.createCursor(task.getAppId(),  fields, task.getQuery().or(""), FETCH_SIZE);
+        }catch (KintoneAPIException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    public void deleteCursor(CreateRecordCursorResponse cursor) {
+        try {
+            this.kintoneRecordManager.deleteCursor(cursor.getId());
+        }catch (KintoneAPIException e){
+            this.logger.error(e.toString());
+        }
+    }
 }

data/src/main/java/org/embulk/input/kintone/KintoneInputColumnVisitor.java

@@ -16,7 +16,7 @@ import org.slf4j.LoggerFactory;
 import com.google.gson.JsonElement;
 
 public class KintoneInputColumnVisitor implements ColumnVisitor {
-    private static final String DEFAULT_TIMESTAMP_PATTERN = "%Y-%m-%dT%H:%M:%S.%L%z";
+    private static final String DEFAULT_TIMESTAMP_PATTERN = "%Y-%m-%dT%H:%M:%S%z";
     private final Logger logger = LoggerFactory.getLogger(KintoneInputColumnVisitor.class);
 
     private final PageBuilder pageBuilder;

data/src/main/java/org/embulk/input/kintone/KintoneInputPlugin.java

@@ -1,23 +1,20 @@
 package org.embulk.input.kintone;
 
-import java.util.HashMap;
-import java.util.List;
-
+import com.cybozu.kintone.client.model.cursor.CreateRecordCursorResponse;
+import com.cybozu.kintone.client.model.cursor.GetRecordCursorResponse;
+import com.cybozu.kintone.client.model.record.field.FieldValue;
+import com.google.common.annotations.VisibleForTesting;
 import org.embulk.config.ConfigDiff;
 import org.embulk.config.ConfigSource;
 import org.embulk.config.TaskReport;
 import org.embulk.config.TaskSource;
-import org.embulk.spi.Exec;
-import org.embulk.spi.PageBuilder;
-import org.embulk.spi.InputPlugin;
-import org.embulk.spi.Schema;
-import org.embulk.spi.PageOutput;
-
-import com.cybozu.kintone.client.model.record.field.FieldValue;
-import com.cybozu.kintone.client.model.record.GetRecordsResponse;
+import org.embulk.spi.*;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.util.HashMap;
+import java.util.List;
+
 public class KintoneInputPlugin
         implements InputPlugin {
     private final Logger logger = LoggerFactory.getLogger(KintoneInputPlugin.class);
@@ -54,19 +51,29 @@ public class KintoneInputPlugin
         PluginTask task = taskSource.loadTask(PluginTask.class);
 
         try {
-            try (PageBuilder pageBuilder = new PageBuilder(Exec.getBufferAllocator(), schema, output)) {
-                KintoneClient client = new KintoneClient(task);
-                // TODO: interface should accept query?
-                GetRecordsResponse response = client.getResponse();
-                for (HashMap<String, FieldValue> record : response.getRecords()) {
-                    schema.visitColumns(new KintoneInputColumnVisitor(new KintoneAccessor(record), pageBuilder, task));
-                    pageBuilder.addRecord();
+            try (PageBuilder pageBuilder = getPageBuilder(schema, output)) {
+                KintoneClient client = getKintoneClient();
+                client.validateAuth(task);
+                client.connect(task);
+
+                CreateRecordCursorResponse cursor = client.createCursor(task);
+                GetRecordCursorResponse cursorResponse = new GetRecordCursorResponse();
+                cursorResponse.setNext(true);
+
+                while (cursorResponse.getNext()) {
+                    cursorResponse = client.getRecordsByCursor(cursor);
+                    for (HashMap<String, FieldValue> record : cursorResponse.getRecords()) {
+                        schema.visitColumns(new KintoneInputColumnVisitor(new KintoneAccessor(record), pageBuilder, task));
+                        pageBuilder.addRecord();
+                    }
+                    pageBuilder.flush();
                 }
+
                 pageBuilder.finish();
             }
         } catch (Exception e) {
-            System.out.println(e.getMessage());
-            System.out.println(e.fillInStackTrace());
+            logger.error(e.getMessage());
+            throw e;
         }
         return Exec.newTaskReport();
     }
@@ -75,4 +82,16 @@ public class KintoneInputPlugin
     public ConfigDiff guess(ConfigSource config) {
         return Exec.newConfigDiff();
     }
+
+    @VisibleForTesting
+    protected PageBuilder getPageBuilder(final Schema schema, final PageOutput output)
+    {
+        return new PageBuilder(Exec.getBufferAllocator(), schema, output);
+    }
+
+    @VisibleForTesting
+    protected KintoneClient getKintoneClient(){
+        return new KintoneClient();
+    }
+
 }

data/src/test/java/org/embulk/input/kintone/TestHelper.java

@@ -0,0 +1,17 @@
+package org.embulk.input.kintone;
+
+import com.cybozu.kintone.client.model.app.form.FieldType;
+import com.cybozu.kintone.client.model.record.field.FieldValue;
+
+import java.util.HashMap;
+
+public class TestHelper {
+    public static HashMap<String, FieldValue> addField(HashMap<String, FieldValue> record, String code, FieldType type,
+                                                 Object value) {
+        FieldValue newField = new FieldValue();
+        newField.setType(type);
+        newField.setValue(value);
+        record.put(code, newField);
+        return record;
+    }
+}

data/src/test/java/org/embulk/input/kintone/TestKintoneAccessor.java

@@ -0,0 +1,100 @@
+package org.embulk.input.kintone;
+
+
+import com.cybozu.kintone.client.model.app.form.FieldType;
+import com.cybozu.kintone.client.model.record.field.FieldValue;
+import com.cybozu.kintone.client.model.record.SubTableValueItem;
+import com.cybozu.kintone.client.model.member.Member;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+
+import static org.junit.Assert.*;
+
+
+public class TestKintoneAccessor {
+    private static Member testman1 = new Member("code1", "name1");
+    private static Member testman2 = new Member("code2", "name2");
+    private static Member testgroup1 = new Member("code3", "name3");
+    private static Member testgroup2 = new Member("code4", "name4");
+    private static Member testorg1 = new Member("code5", "name5");
+    private static Member testorg2 = new Member("code6", "name6");
+    private Integer uniqueKey = 1;
+
+    public HashMap<String, FieldValue> createTestRecord() {
+        HashMap<String, FieldValue> testRecord = new HashMap<>();
+
+        TestHelper.addField(testRecord, "文字列__1行", FieldType.SINGLE_LINE_TEXT, "test single text");
+        TestHelper.addField(testRecord, "数値", FieldType.NUMBER, this.uniqueKey);
+        this.uniqueKey += 1;
+        TestHelper.addField(testRecord, "文字列__複数行", FieldType.MULTI_LINE_TEXT, "test multi text");
+        TestHelper.addField(testRecord, "リッチエディター", FieldType.RICH_TEXT, "<div>test rich text<br /></div>");
+
+        ArrayList<String> selectedItemList = new ArrayList<>();
+        selectedItemList.add("sample1");
+        selectedItemList.add("sample2");
+        TestHelper.addField(testRecord, "チェックボックス", FieldType.CHECK_BOX, selectedItemList);
+        TestHelper.addField(testRecord, "ラジオボタン", FieldType.RADIO_BUTTON, "sample2");
+        TestHelper.addField(testRecord, "ドロップダウン", FieldType.DROP_DOWN, "sample3");
+        TestHelper.addField(testRecord, "複数選択", FieldType.MULTI_SELECT, selectedItemList);
+        TestHelper.addField(testRecord, "リンク", FieldType.LINK, "http://cybozu.co.jp/");
+        TestHelper.addField(testRecord, "日付", FieldType.DATE, "2018-01-01");
+        TestHelper.addField(testRecord, "時刻", FieldType.TIME, "12:34");
+        TestHelper.addField(testRecord, "日時", FieldType.DATETIME, "2018-01-02T02:30:00Z");
+
+        ArrayList<Member> userList = new ArrayList<>();
+        userList.add(testman1);
+        userList.add(testman2);
+        TestHelper.addField(testRecord, "ユーザー選択", FieldType.USER_SELECT, userList);
+        ArrayList<Member> groupList = new ArrayList<>();
+        groupList.add(testgroup1);
+        groupList.add(testgroup2);
+        TestHelper.addField(testRecord, "グループ選択", FieldType.GROUP_SELECT, groupList);
+        ArrayList<Member> orgList = new ArrayList<>();
+        orgList.add(testorg1);
+        orgList.add(testorg2);
+        TestHelper.addField(testRecord, "組織選択", FieldType.ORGANIZATION_SELECT, orgList);
+
+        SubTableValueItem tableItem1 = new SubTableValueItem();
+        tableItem1.setID(1);
+        HashMap<String, FieldValue> tableItemValue1 = new HashMap<>();
+        FieldValue fv1 = new FieldValue();
+        fv1.setType(FieldType.SINGLE_LINE_TEXT);
+        fv1.setValue("sample_text1");
+        tableItemValue1.put("sample field1", fv1);
+        tableItem1.setValue(tableItemValue1);
+        ArrayList<SubTableValueItem> subTableRecords = new ArrayList<>();
+        subTableRecords.add(tableItem1);
+        TestHelper.addField(testRecord, "サブテーブル", FieldType.SUBTABLE, subTableRecords);
+
+        return testRecord;
+    }
+
+    @Test
+    public void testAccess() {
+        HashMap<String, FieldValue> testRecord = createTestRecord();
+        KintoneAccessor accessor = new KintoneAccessor(testRecord);
+        String multiValue = "sample1\nsample2";
+        String userSelect = "code1\ncode2";
+        String groupSelect = "code3\ncode4";
+        String orgSelect = "code5\ncode6";
+        String subTableValue = "[{\"id\":1,\"value\":{\"sample field1\":{\"type\":\"SINGLE_LINE_TEXT\",\"value\":\"sample_text1\"}}}]";
+        assertEquals(testRecord.get("文字列__1行").getValue(), accessor.get("文字列__1行"));
+        assertEquals("1", accessor.get("数値"));
+        assertEquals(testRecord.get("文字列__複数行").getValue(), accessor.get("文字列__複数行"));
+        assertEquals(testRecord.get("リッチエディター").getValue(), accessor.get("リッチエディター"));
+        assertEquals(multiValue, accessor.get("チェックボックス"));
+        assertEquals(testRecord.get("ラジオボタン").getValue(), accessor.get("ラジオボタン"));
+        assertEquals(testRecord.get("ドロップダウン").getValue(), accessor.get("ドロップダウン"));
+        assertEquals(multiValue, accessor.get("複数選択"));
+        assertEquals(testRecord.get("リンク").getValue(), accessor.get("リンク"));
+        assertEquals(testRecord.get("日付").getValue(), accessor.get("日付"));
+        assertEquals(testRecord.get("時刻").getValue(), accessor.get("時刻"));
+        assertEquals(testRecord.get("日時").getValue(), accessor.get("日時"));
+        assertEquals(userSelect, accessor.get("ユーザー選択"));
+        assertEquals(groupSelect, accessor.get("グループ選択"));
+        assertEquals(orgSelect, accessor.get("組織選択"));
+        assertEquals(subTableValue, accessor.get("サブテーブル"));
+    }
+}

data/src/test/java/org/embulk/input/kintone/TestKintoneClient.java

@@ -0,0 +1,80 @@
+package org.embulk.input.kintone;
+
+import org.embulk.config.ConfigException;
+import org.embulk.config.ConfigSource;
+import org.embulk.spi.InputPlugin;
+
+import org.embulk.test.TestingEmbulk;
+import org.junit.Rule;
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+public class TestKintoneClient {
+    private ConfigSource config;
+    private KintoneClient client = new KintoneClient();
+    private static final String BASIC_RESOURCE_PATH = "org/embulk/input/kintone/";
+    private static final String SUCCESS_MSG = "Exception should be thrown by this";
+
+    private static ConfigSource loadYamlResource(TestingEmbulk embulk, String fileName) {
+        return embulk.loadYamlResource(BASIC_RESOURCE_PATH + fileName);
+    }
+
+    @Rule
+    public TestingEmbulk embulk = TestingEmbulk.builder()
+            .registerPlugin(InputPlugin.class, "kintone", KintoneInputPlugin.class)
+            .build();
+
+    @Test
+    public void checkClientWithUsernameAndPassword() {
+        config = loadYamlResource(embulk, "base.yml");
+        PluginTask task = config.loadConfig(PluginTask.class);
+        Exception e = assertThrows(Exception.class, ()-> {
+            client.validateAuth(task);
+            throw new Exception(SUCCESS_MSG);
+        });
+        assertEquals(SUCCESS_MSG, e.getMessage());
+    }
+
+    @Test
+    public void checkThrowErrorWithoutAuthInfo() {
+        config = loadYamlResource(embulk, "base.yml");
+        config.remove("username")
+                .remove("password");
+        PluginTask task = config.loadConfig(PluginTask.class);
+        ConfigException e = assertThrows(ConfigException.class, () -> client.validateAuth(task));
+        assertEquals("Username and password or token must be provided", e.getMessage());
+    }
+
+    @Test
+    public void checkClientErrorLackingPassword() {
+        config = loadYamlResource(embulk, "base.yml");
+        config.remove("password");
+        PluginTask task = config.loadConfig(PluginTask.class);
+        ConfigException e = assertThrows(ConfigException.class, () -> client.validateAuth(task));
+        assertEquals("Username and password or token must be provided", e.getMessage());
+    }
+
+    @Test
+    public void checkClientErrorLackingUsername() {
+        config = loadYamlResource(embulk, "base.yml");
+        config.remove("username");
+        PluginTask task = config.loadConfig(PluginTask.class);
+        ConfigException e = assertThrows(ConfigException.class, () -> client.validateAuth(task));
+        assertEquals("Username and password or token must be provided", e.getMessage());
+    }
+
+    @Test
+    public void checkClientWithToken() {
+        config = loadYamlResource(embulk, "base.yml");
+        config.remove("username")
+                .remove("password")
+                .set("token", "token");
+        PluginTask task = config.loadConfig(PluginTask.class);
+        Exception e = assertThrows(Exception.class, ()-> {
+            client.validateAuth(task);
+            throw new Exception(SUCCESS_MSG);
+        });
+        assertEquals(SUCCESS_MSG, e.getMessage());
+    }
+}

data/src/test/java/org/embulk/input/kintone/TestKintoneInputPlugin.java

@@ -1,5 +1,150 @@
 package org.embulk.input.kintone;
 
-public class TestKintoneInputPlugin
-{
+import com.cybozu.kintone.client.model.app.form.FieldType;
+import com.cybozu.kintone.client.model.record.GetRecordsResponse;
+import com.cybozu.kintone.client.model.record.field.FieldValue;
+
+import org.embulk.EmbulkTestRuntime;
+import org.embulk.config.ConfigDiff;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.TaskReport;
+import org.embulk.config.TaskSource;
+import org.embulk.spi.InputPlugin;
+import org.embulk.spi.Schema;
+import org.embulk.spi.TestPageBuilderReader.MockPageOutput;
+import org.embulk.spi.time.Timestamp;
+import org.embulk.spi.time.TimestampParser;
+import org.embulk.spi.util.Pages;
+import org.embulk.test.TestingEmbulk;
+
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+import static org.junit.Assert.*;
+import static org.mockito.Mockito.*;
+
+public class TestKintoneInputPlugin {
+    private ConfigSource config;
+    private static final String BASIC_RESOURCE_PATH = "org/embulk/input/kintone/";
+    private KintoneInputPlugin kintoneInputPlugin;
+    private KintoneClient kintoneClient;
+    private MockPageOutput output = new MockPageOutput();
+    private TimestampParser dateParser = TimestampParser.of("%Y-%m-%d", "UTC");
+    private TimestampParser timestampParser = TimestampParser.of("%Y-%m-%dT%H:%M:%S%z", "UTC");
+
+    private static ConfigSource loadYamlResource(TestingEmbulk embulk, String fileName) {
+        return embulk.loadYamlResource(BASIC_RESOURCE_PATH + fileName);
+    }
+
+    @Before
+    public void prepare(){
+        kintoneInputPlugin = spy(new KintoneInputPlugin());
+        kintoneClient = mock(KintoneClient.class);
+        doReturn(kintoneClient).when(kintoneInputPlugin).getKintoneClient();
+    }
+    @Rule
+    public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
+
+    @Rule
+    public TestingEmbulk embulk = TestingEmbulk.builder()
+            .registerPlugin(InputPlugin.class, "kintone", KintoneInputPlugin.class)
+            .build();
+
+    // Comment out for now
+    // @Test
+    public void simpleTest(){
+        config = loadYamlResource(embulk, "base.yml");
+        PluginTask task = config.loadConfig(PluginTask.class);
+        Schema outputSchema =  task.getFields().toSchema();
+        GetRecordsResponse response = createSampleData();
+        when(kintoneClient.getResponse(any(PluginTask.class))).thenReturn(response);
+
+        ConfigDiff configDiff = kintoneInputPlugin.transaction(config, new Control());
+
+        assertTrue(configDiff.isEmpty());
+
+        List<Object[]> outputRecords = Pages.toObjects(outputSchema, output.pages);
+        Object[] record1 = outputRecords.get(0);
+
+        Timestamp date1 = dateParser.parse("2020-01-01");
+        Timestamp timestamp1 = timestampParser.parse("2020-01-01T00:00:00Z");
+
+        assertEquals(2, outputRecords.size());
+
+        assertEquals("test single text", record1[0]);
+        assertEquals(1L, record1[1]);
+        assertEquals(1.111, record1[2]);
+        assertEquals(date1, record1[3]);
+        assertEquals(timestamp1, record1[4]);
+
+        Timestamp date2 = dateParser.parse("2020-02-02");
+        Timestamp timestamp2 = timestampParser.parse("2020-02-02T00:00:00Z");
+
+        Object[] record2 = outputRecords.get(1);
+        assertEquals("test single text2", record2[0]);
+        assertEquals(2L, record2[1]);
+        assertEquals(2.222, record2[2]);
+        assertEquals(date2, record2[3]);
+        assertEquals(timestamp2, record2[4]);
+    }
+
+    @Test
+    public void checkDefaultConfigValues() {
+        config = loadYamlResource(embulk, "base.yml");
+        PluginTask task = config.loadConfig(PluginTask.class);
+        assertEquals("dev.cybozu.com", task.getDomain());
+        assertEquals(1, task.getAppId());
+        assertEquals("username", task.getUsername().get());
+        assertEquals("password", task.getPassword().get());
+        assertFalse(task.getToken().isPresent());
+        assertFalse(task.getGuestSpaceId().isPresent());
+        assertFalse(task.getBasicAuthUsername().isPresent());
+        assertFalse(task.getBasicAuthPassword().isPresent());
+        assertFalse(task.getQuery().isPresent());
+        assertNotNull(task.getFields());
+    }
+
+    private GetRecordsResponse createSampleData(){
+        HashMap<String, FieldValue> record1 = new HashMap<>();
+        HashMap<String, FieldValue> record2 = new HashMap<>();
+        ArrayList<HashMap<String, FieldValue>> records = new ArrayList<>();
+        GetRecordsResponse response = new GetRecordsResponse();
+
+        record1 = TestHelper.addField(record1, "foo", FieldType.SINGLE_LINE_TEXT, "test single text");
+        record1 = TestHelper.addField(record1, "bar", FieldType.NUMBER, 1);
+        record1 = TestHelper.addField(record1, "baz", FieldType.NUMBER, 1.111);
+        record1 = TestHelper.addField(record1, "date", FieldType.DATE, "2020-01-01");
+        record1 = TestHelper.addField(record1, "datetime", FieldType.DATE, "2020-01-01T00:00:00Z");
+        records.add(record1);
+
+        record2 = TestHelper.addField(record2, "foo", FieldType.SINGLE_LINE_TEXT, "test single text2");
+        record2 = TestHelper.addField(record2, "bar", FieldType.NUMBER, 2);
+        record2 = TestHelper.addField(record2, "baz", FieldType.NUMBER, 2.222);
+        record2 = TestHelper.addField(record2, "date", FieldType.DATE, "2020-02-02");
+        record2 = TestHelper.addField(record2, "datetime", FieldType.DATE, "2020-02-02T00:00:00Z");
+        records.add(record2);
+
+        response.setRecords(records);
+        response.setTotalCount(2);
+        return response;
+    }
+
+    private class Control implements InputPlugin.Control
+    {
+        @Override
+        public List<TaskReport> run(final TaskSource taskSource, final Schema schema, final int taskCount)
+        {
+            List<TaskReport> reports = IntStream.range(0, taskCount)
+                    .mapToObj(i -> kintoneInputPlugin.run(taskSource, schema, i, output))
+                    .collect(Collectors.toList());
+            return reports;
+        }
+    }
 }

data/src/test/resources/org/embulk/input/kintone/base.yml

@@ -0,0 +1,11 @@
+type: kintone
+domain: dev.cybozu.com
+app_id: 1
+username: username
+password: password
+fields:
+  - {name: foo, type: string}
+  - {name: bar, type: long}
+  - {name: baz, type: double}
+  - {name: date, type: timestamp, format: '%Y-%m-%d'}
+  - {name: datetime, type: timestamp}

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-kintone
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.5
 platform: ruby
 authors:
 - giwa
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-10-11 00:00:00.000000000 Z
+date: 2021-01-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -46,10 +46,11 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".gitignore"
+- ".travis.yml"
 - LICENSE.txt
 - README.md
 - build.gradle
-- classpath/embulk-input-kintone-0.1.0.jar
+- classpath/embulk-input-kintone-0.1.5.jar
 - classpath/gson-2.8.2.jar
 - classpath/kintone-sdk-0.4.0.jar
 - config/checkstyle/checkstyle.xml
@@ -64,8 +65,12 @@ files:
 - src/main/java/org/embulk/input/kintone/KintoneInputColumnVisitor.java
 - src/main/java/org/embulk/input/kintone/KintoneInputPlugin.java
 - src/main/java/org/embulk/input/kintone/PluginTask.java
+- src/test/java/org/embulk/input/kintone/TestHelper.java
+- src/test/java/org/embulk/input/kintone/TestKintoneAccessor.java
+- src/test/java/org/embulk/input/kintone/TestKintoneClient.java
 - src/test/java/org/embulk/input/kintone/TestKintoneInputPlugin.java
-homepage:
+- src/test/resources/org/embulk/input/kintone/base.yml
+homepage: https://github.com/trocco-io/embulk-input-kintone
 licenses:
 - MIT
 metadata: {}