embulk-input-kintone 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 50b882dc3ce74fcb86df7ba392da3046c4944e99
-  data.tar.gz: a85a843021b74f4eda4dc0a9241193413bddcd9b
+  metadata.gz: 68954eaff6b8344ffcddf296dfaf47a73882a315
+  data.tar.gz: '0587da14d8658b81f0f94ed9a2f26475eb2c5595'
 SHA512:
-  metadata.gz: d8f3927799c2d3507e5dc764d798db24813c95e8bb4f563ba93d75a84cc6ec96ac42fd81f2c4b7161ae87dd306a2658a95c95a25b069152aee88a25b6f7320b4
-  data.tar.gz: 88c09ff176b14da18790495f39e5b5bdadc37d5abbb7f453871ecc54d8f196eed56e1fa6d4b33af7063e5352bcaf4b637aad5d456c67fa3904f912f0cd44fc8e
+  metadata.gz: 941e7ed35a81853b42e41da31d78dd0afb3ccf8a27f4f1f6b23e4308caca10f36d705802eaf9d3a21a902f94a4dde3a9548cc519d524b360bfb842a59d2c7337
+  data.tar.gz: 48a62cd9aeccf5d307a971a1da9f9e38a350513569be5a3a5218128b3418b6c4414f3a889450f027b9a3272bc90f8a337144571379d41e175c812e7ccf366ca7

data/README.md

@@ -27,7 +27,7 @@ embulk 0.9 is only supported due to the dependency of kintone-java-sdk 0.4.0, wh
 - **basic_auth_username**:  Kintone basic auth username Please see Kintone basic auth [here](https://jp.cybozu.help/general/en/admin/list_security/list_ip_basic/basic_auth.html) (string, optional)
 - **basic_auth_password**:  Kintone basic auth password (string, optional)
 - **guest_space_id**: Kintone app belongs to guest space, guest space id is required. (integer, optional)
-- **fields**
+- **fields** (required)
   - **name** the field code of Kintone app record will be retrieved.
   - **type** Column values are converted to this embulk type. Available values options are: boolean, long, double, string, json, timestamp)
 
@@ -64,7 +64,7 @@ in:
     - {name: $id, type: long}
     - {name: $revision, type: long}
     - {name: Time, type: string}
-    - {name: Created_datatime, type: long}
+    - {name: Created_datatime, type: string}
     - {name: foo, type: string}
 ```
 
@@ -73,3 +73,9 @@ in:
 ```
 $ ./gradlew gem  # -t to watch change of files and rebuild continuously
 ```
+
+## Development
+```
+$ ./gradew build
+$ ./gradew test
+```

data/build.gradle

@@ -13,7 +13,7 @@ configurations {
     provided
 }
 
-version = "0.1.0"
+version = "0.1.1"
 
 sourceCompatibility = 1.8
 targetCompatibility = 1.8
@@ -22,8 +22,10 @@ dependencies {
     compile  "org.embulk:embulk-core:0.9.12"
     provided "org.embulk:embulk-core:0.9.12"
     compile group: 'com.cybozu.kintone', name: 'kintone-sdk', version: '0.4.0'
-    // compile "YOUR_JAR_DEPENDENCY_GROUP:YOUR_JAR_DEPENDENCY_MODULE:YOUR_JAR_DEPENDENCY_VERSION"
+
     testCompile "junit:junit:4.+"
+    testCompile 'org.embulk:embulk-standards:0.9.12'
+    testCompile 'org.embulk:embulk-test:0.9.12'
 }
 
 task classpath(type: Copy, dependsOn: ["jar"]) {
@@ -83,7 +85,7 @@ Gem::Specification.new do |spec|
   spec.description   = %[Loads records from Kintone.]
   spec.email         = ["ugw.gi.world@gmail.com"]
   spec.licenses      = ["MIT"]
-  # TODO set this: spec.homepage      = "https://github.com/ugw.gi.world/embulk-input-kintone"
+  spec.homepage      = "https://github.com/trocco-io/embulk-input-kintone"
 
   spec.files         = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
   spec.test_files    = spec.files.grep(%r"^(test|spec)/")

data/gradle/wrapper/gradle-wrapper.properties

@@ -1,5 +1,6 @@
+#Fri Oct 11 16:04:33 JST 2019
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-4.1-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-4.1-all.zip

data/src/main/java/org/embulk/input/kintone/KintoneAccessor.java

@@ -26,7 +26,7 @@ public class KintoneAccessor {
             case GROUP_SELECT:
             case STATUS_ASSIGNEE:
                 ArrayList<Member> members = (ArrayList<Member>) this.record.get(name).getValue();
-                return members.stream().map(m -> m.getCode())
+                return members.stream().map(Member::getCode)
                         .reduce((accum, value) -> accum + this.delimiter + value)
                         .orElse("");
             case SUBTABLE:
@@ -45,9 +45,11 @@ public class KintoneAccessor {
                         .orElse("");
             case FILE:
                 ArrayList<FileModel> cbFileList = (ArrayList<FileModel>) this.record.get(name).getValue();
-                return cbFileList.stream().map(f -> f.getFileKey())
+                return cbFileList.stream().map(FileModel::getFileKey)
                         .reduce((accum, value) -> accum + this.delimiter + value)
                         .orElse("");
+            case NUMBER:
+                return String.valueOf(this.record.get(name).getValue());
             default:
                 return (String) this.record.get(name).getValue();
         }

data/src/main/java/org/embulk/input/kintone/KintoneClient.java

@@ -14,49 +14,49 @@ import java.util.ArrayList;
 
 public class KintoneClient {
     private final Logger logger = LoggerFactory.getLogger(KintoneClient.class);
-    private final PluginTask pluginTask;
+    private final PluginTask task;
     private ArrayList<String> fields;
     private Auth kintoneAuth;
     private Record kintoneRecordManager;
     private Connection con;
 
-    public KintoneClient(final PluginTask pluginTask) {
-        this.pluginTask = pluginTask;
+    public KintoneClient(final PluginTask task) {
+        this.task = task;
         this.kintoneAuth = new Auth();
 
-        this.fields = new ArrayList<String>();
-        for (ColumnConfig c : pluginTask.getFields().getColumns()
+        this.fields = new ArrayList<>();
+        for (ColumnConfig c : task.getFields().getColumns()
         ) {
             fields.add(c.getName());
         }
         this.setAuth();
-        if (pluginTask.getGuestSpaceId().isPresent()) {
-            this.con = new Connection(pluginTask.getDomain(), this.kintoneAuth, pluginTask.getGuestSpaceId().or(-1));
+        if (task.getGuestSpaceId().isPresent()) {
+            this.con = new Connection(task.getDomain(), this.kintoneAuth, task.getGuestSpaceId().or(-1));
         } else {
-            this.con = new Connection(pluginTask.getDomain(), this.kintoneAuth);
+            this.con = new Connection(task.getDomain(), this.kintoneAuth);
         }
         this.kintoneRecordManager = new Record(con);
     }
 
     private void setAuth() {
-        if (pluginTask.getUsername().isPresent() && pluginTask.getPassword().isPresent()) {
-            this.kintoneAuth.setPasswordAuth(pluginTask.getUsername().get(), pluginTask.getPassword().get());
-        } else if (pluginTask.getToken().isPresent()) {
-            this.kintoneAuth.setApiToken(pluginTask.getToken().get());
+        if (task.getUsername().isPresent() && task.getPassword().isPresent()) {
+            this.kintoneAuth.setPasswordAuth(task.getUsername().get(), task.getPassword().get());
+        } else if (task.getToken().isPresent()) {
+            this.kintoneAuth.setApiToken(task.getToken().get());
         } else {
             throw new ConfigException("Username and password or token must be provided");
         }
 
-        if (pluginTask.getBasicAuthUsername().isPresent() && pluginTask.getBasicAuthPassword().isPresent()) {
-            this.kintoneAuth.setBasicAuth(pluginTask.getBasicAuthUsername().get(),
-                    pluginTask.getBasicAuthPassword().get());
+        if (task.getBasicAuthUsername().isPresent() && task.getBasicAuthPassword().isPresent()) {
+            this.kintoneAuth.setBasicAuth(task.getBasicAuthUsername().get(),
+                    task.getBasicAuthPassword().get());
         }
     }
 
     public GetRecordsResponse getResponse() {
         try {
             return kintoneRecordManager.getAllRecordsByQuery(
-                    this.pluginTask.getAppId(), this.pluginTask.getQuery().or(""), this.fields);
+                    this.task.getAppId(), this.task.getQuery().or(""), this.fields);
         } catch (Exception e) {
             throw new RuntimeException(e);
         }

data/src/main/java/org/embulk/input/kintone/KintoneInputPlugin.java

@@ -3,10 +3,7 @@ package org.embulk.input.kintone;
 import java.util.HashMap;
 import java.util.List;
 
-import org.embulk.config.ConfigDiff;
-import org.embulk.config.ConfigSource;
-import org.embulk.config.TaskReport;
-import org.embulk.config.TaskSource;
+import org.embulk.config.*;
 import org.embulk.spi.Exec;
 import org.embulk.spi.PageBuilder;
 import org.embulk.spi.InputPlugin;
@@ -56,7 +53,6 @@ public class KintoneInputPlugin
         try {
             try (PageBuilder pageBuilder = new PageBuilder(Exec.getBufferAllocator(), schema, output)) {
                 KintoneClient client = new KintoneClient(task);
-                // TODO: interface should accept query?
                 GetRecordsResponse response = client.getResponse();
                 for (HashMap<String, FieldValue> record : response.getRecords()) {
                     schema.visitColumns(new KintoneInputColumnVisitor(new KintoneAccessor(record), pageBuilder, task));
@@ -65,8 +61,8 @@ public class KintoneInputPlugin
                 pageBuilder.finish();
             }
         } catch (Exception e) {
-            System.out.println(e.getMessage());
-            System.out.println(e.fillInStackTrace());
+            logger.error(e.getMessage());
+            throw e;
         }
         return Exec.newTaskReport();
     }

data/src/test/java/org/embulk/input/kintone/TestKintoneAccessor.java

@@ -0,0 +1,94 @@
+package org.embulk.input.kintone;
+
+
+import com.cybozu.kintone.client.model.app.form.FieldType;
+import com.cybozu.kintone.client.model.record.field.FieldValue;
+import com.cybozu.kintone.client.model.member.Member;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+
+import static org.junit.Assert.*;
+
+
+public class TestKintoneAccessor {
+    private static Member testman1 = new Member("code1", "name1");
+    private static Member testman2 = new Member("code2", "name2");
+    private static Member testgroup1 = new Member("code3", "name3");
+    private static Member testgroup2 = new Member("code4", "name4");
+    private static Member testorg1 = new Member("code5", "name5");
+    private static Member testorg2 = new Member("code6", "name6");
+    private Integer uniqueKey = 1;
+
+    public HashMap<String, FieldValue> createTestRecord() {
+        HashMap<String, FieldValue> testRecord = new HashMap<>();
+
+        testRecord = addField(testRecord, "文字列__1行", FieldType.SINGLE_LINE_TEXT, "test single text");
+        testRecord = addField(testRecord, "数値", FieldType.NUMBER, this.uniqueKey);
+        this.uniqueKey += 1;
+        testRecord = addField(testRecord, "文字列__複数行", FieldType.MULTI_LINE_TEXT, "test multi text");
+        testRecord = addField(testRecord, "リッチエディター", FieldType.RICH_TEXT, "<div>test rich text<br /></div>");
+
+        ArrayList<String> selectedItemList = new ArrayList<>();
+        selectedItemList.add("sample1");
+        selectedItemList.add("sample2");
+        testRecord = addField(testRecord, "チェックボックス", FieldType.CHECK_BOX, selectedItemList);
+        testRecord = addField(testRecord, "ラジオボタン", FieldType.RADIO_BUTTON, "sample2");
+        testRecord = addField(testRecord, "ドロップダウン", FieldType.DROP_DOWN, "sample3");
+        testRecord = addField(testRecord, "複数選択", FieldType.MULTI_SELECT, selectedItemList);
+        testRecord = addField(testRecord, "リンク", FieldType.LINK, "http://cybozu.co.jp/");
+        testRecord = addField(testRecord, "日付", FieldType.DATE, "2018-01-01");
+        testRecord = addField(testRecord, "時刻", FieldType.TIME, "12:34");
+        testRecord = addField(testRecord, "日時", FieldType.DATETIME, "2018-01-02T02:30:00Z");
+
+        ArrayList<Member> userList = new ArrayList<>();
+        userList.add(testman1);
+        userList.add(testman2);
+        addField(testRecord, "ユーザー選択", FieldType.USER_SELECT, userList);
+        ArrayList<Member> groupList = new ArrayList<>();
+        groupList.add(testgroup1);
+        groupList.add(testgroup2);
+        addField(testRecord, "グループ選択", FieldType.GROUP_SELECT, groupList);
+        ArrayList<Member> orgList = new ArrayList<>();
+        orgList.add(testorg1);
+        orgList.add(testorg2);
+        addField(testRecord, "組織選択", FieldType.ORGANIZATION_SELECT, orgList);
+        return testRecord;
+    }
+
+    private HashMap<String, FieldValue> addField(HashMap<String, FieldValue> record, String code, FieldType type,
+                                                 Object value) {
+        FieldValue newField = new FieldValue();
+        newField.setType(type);
+        newField.setValue(value);
+        record.put(code, newField);
+        return record;
+    }
+
+    @Test
+    public void testAccess() {
+        HashMap<String, FieldValue> testRecord = createTestRecord();
+        KintoneAccessor accessor = new KintoneAccessor(testRecord);
+        String multiValue = "sample1\nsample2";
+        String userSelect = "code1\ncode2";
+        String groupSelect = "code3\ncode4";
+        String orgSelect = "code5\ncode6";
+
+        assertEquals(testRecord.get("文字列__1行").getValue(), accessor.get("文字列__1行"));
+        assertEquals("1", accessor.get("数値"));
+        assertEquals(testRecord.get("文字列__複数行").getValue(), accessor.get("文字列__複数行"));
+        assertEquals(testRecord.get("リッチエディター").getValue(), accessor.get("リッチエディター"));
+        assertEquals(multiValue, accessor.get("チェックボックス"));
+        assertEquals(testRecord.get("ラジオボタン").getValue(), accessor.get("ラジオボタン"));
+        assertEquals(testRecord.get("ドロップダウン").getValue(), accessor.get("ドロップダウン"));
+        assertEquals(multiValue, accessor.get("複数選択"));
+        assertEquals(testRecord.get("リンク").getValue(), accessor.get("リンク"));
+        assertEquals(testRecord.get("日付").getValue(), accessor.get("日付"));
+        assertEquals(testRecord.get("時刻").getValue(), accessor.get("時刻"));
+        assertEquals(testRecord.get("日時").getValue(), accessor.get("日時"));
+        assertEquals(userSelect, accessor.get("ユーザー選択"));
+        assertEquals(groupSelect, accessor.get("グループ選択"));
+        assertEquals(orgSelect, accessor.get("組織選択"));
+    }
+}

data/src/test/java/org/embulk/input/kintone/TestKintoneClient.java

@@ -0,0 +1,75 @@
+package org.embulk.input.kintone;
+
+import org.embulk.config.ConfigException;
+import org.embulk.config.ConfigSource;
+import org.embulk.spi.InputPlugin;
+
+import org.embulk.test.TestingEmbulk;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+import static org.junit.Assert.*;
+
+public class TestKintoneClient {
+    private ConfigSource config;
+    private static final String BASIC_RESOURCE_PATH = "org/embulk/input/kintone/";
+
+    private static ConfigSource loadYamlResource(TestingEmbulk embulk, String fileName) {
+        return embulk.loadYamlResource(BASIC_RESOURCE_PATH + fileName);
+    }
+
+    @Rule
+    public TestingEmbulk embulk = TestingEmbulk.builder()
+            .registerPlugin(InputPlugin.class, "kintone", KintoneInputPlugin.class)
+            .build();
+
+    @Test
+    public void checkClientWithUsernameAndPassword() {
+        config = loadYamlResource(embulk, "base.yml");
+        PluginTask task = config.loadConfig(PluginTask.class);
+        KintoneClient client = new KintoneClient(task);
+        assertNotNull(client);
+    }
+
+    @Test
+    public void checkThrowErrorWithoutAuthInfo() {
+        config = loadYamlResource(embulk, "base.yml");
+        config.remove("username")
+                .remove("password");
+        PluginTask task = config.loadConfig(PluginTask.class);
+        ConfigException e = assertThrows(ConfigException.class, () -> new KintoneClient(task));
+        assertEquals("Username and password or token must be provided", e.getMessage());
+    }
+
+    @Test
+    public void checkClientErrorLackingPassword() {
+        config = loadYamlResource(embulk, "base.yml");
+        config.remove("password");
+        PluginTask task = config.loadConfig(PluginTask.class);
+        ConfigException e = assertThrows(ConfigException.class, () -> new KintoneClient(task));
+        assertEquals("Username and password or token must be provided", e.getMessage());
+    }
+
+    @Test
+    public void checkClientErrorLackingUsername() {
+        config = loadYamlResource(embulk, "base.yml");
+        config.remove("username");
+        PluginTask task = config.loadConfig(PluginTask.class);
+        ConfigException e = assertThrows(ConfigException.class, () -> {
+            new KintoneClient(task);
+        });
+        assertEquals("Username and password or token must be provided", e.getMessage());
+    }
+
+    @Test
+    public void checkClientWithToken() {
+        config = loadYamlResource(embulk, "base.yml");
+        config.remove("username")
+                .remove("password")
+                .set("token", "token");
+        PluginTask task = config.loadConfig(PluginTask.class);
+        KintoneClient client = new KintoneClient(task);
+        assertNotNull(client);
+    }
+}

data/src/test/java/org/embulk/input/kintone/TestKintoneInputPlugin.java

@@ -1,5 +1,40 @@
 package org.embulk.input.kintone;
 
-public class TestKintoneInputPlugin
-{
+import org.embulk.config.ConfigSource;
+import org.embulk.spi.InputPlugin;
+
+import org.embulk.test.TestingEmbulk;
+import org.junit.Rule;
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+public class TestKintoneInputPlugin {
+    private ConfigSource config;
+    private static final String BASIC_RESOURCE_PATH = "org/embulk/input/kintone/";
+
+    private static ConfigSource loadYamlResource(TestingEmbulk embulk, String fileName) {
+        return embulk.loadYamlResource(BASIC_RESOURCE_PATH + fileName);
+    }
+
+    @Rule
+    public TestingEmbulk embulk = TestingEmbulk.builder()
+            .registerPlugin(InputPlugin.class, "kintone", KintoneInputPlugin.class)
+            .build();
+
+    @Test
+    public void checkDefaultConfigValues() {
+        config = loadYamlResource(embulk, "base.yml");
+        PluginTask task = config.loadConfig(PluginTask.class);
+        assertEquals("dev.cybozu.com", task.getDomain());
+        assertEquals(1, task.getAppId());
+        assertEquals("username", task.getUsername().get());
+        assertEquals("password", task.getPassword().get());
+        assertFalse(task.getToken().isPresent());
+        assertFalse(task.getGuestSpaceId().isPresent());
+        assertFalse(task.getBasicAuthUsername().isPresent());
+        assertFalse(task.getBasicAuthPassword().isPresent());
+        assertFalse(task.getQuery().isPresent());
+        assertNotNull(task.getFields());
+    }
 }

data/src/test/resources/org/embulk/input/kintone/base.yml

@@ -0,0 +1,9 @@
+type: kintone
+domain: dev.cybozu.com
+app_id: 1
+username: username
+password: password
+fields:
+  - {name: foo, type: string}
+  - {name: bar, type: long}
+  - {name: baz, type: double}

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-kintone
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - giwa
@@ -49,7 +49,7 @@ files:
 - LICENSE.txt
 - README.md
 - build.gradle
-- classpath/embulk-input-kintone-0.1.0.jar
+- classpath/embulk-input-kintone-0.1.1.jar
 - classpath/gson-2.8.2.jar
 - classpath/kintone-sdk-0.4.0.jar
 - config/checkstyle/checkstyle.xml
@@ -64,8 +64,11 @@ files:
 - src/main/java/org/embulk/input/kintone/KintoneInputColumnVisitor.java
 - src/main/java/org/embulk/input/kintone/KintoneInputPlugin.java
 - src/main/java/org/embulk/input/kintone/PluginTask.java
+- src/test/java/org/embulk/input/kintone/TestKintoneAccessor.java
+- src/test/java/org/embulk/input/kintone/TestKintoneClient.java
 - src/test/java/org/embulk/input/kintone/TestKintoneInputPlugin.java
-homepage:
+- src/test/resources/org/embulk/input/kintone/base.yml
+homepage: https://github.com/trocco-io/embulk-input-kintone
 licenses:
 - MIT
 metadata: {}