embulk-input-datastore 0.0.4 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 15a06b07d8765e66681ac6636340eddae2bc84dcc15303d538c89db1619bbb1e
-  data.tar.gz: cdb37b7382f2921fb9fa351dba8f1d1a2abe18951f4f887003c11011c1bfced4
+  metadata.gz: 7073922589f46f02c4b1e1534ccccec56bf56eb9d6316bde1f6f6d7f80b7d1b1
+  data.tar.gz: 6ced669ff8d160252dd8806f801b5e28b8ce7f0a51c65fc969c5a07412897a82
 SHA512:
-  metadata.gz: e35e6c6d76da73a4e76d50b202cb80b157cc3b5308b5b159eb91e1c3c2ca9b1c43c83a9938b507a70fe7341a8d21af00d2f60cac72aa1d8ec810bbfad451f14c
-  data.tar.gz: 37a8123d2607064598eed79b02c714785f9ca93e331b481bd5802384b86ed9b1665f7368c9b81ba3b12853030b1b27da53847a81ad4f2c15919bb8a08054ea15
+  metadata.gz: 891dd771e4683e046c413308ca0c71efbe349ff29ebf9dfc8b7f65ce16442e7d8bd2f010a87c7cbe39d2de2882b63170fe6a0baabfcf8716184018db86152aa1
+  data.tar.gz: beacaf4852d72788a3d77494ee96badb04ff4548e97b3304475a64f62aec856fdbca669bdc0691e45194499ef7a010ddb6b3899c894b9ec79888abff5d5e5830

data/build.gradle

@@ -8,12 +8,26 @@ buildscript {
         classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlin_version"
         classpath "com.github.kamatama41:gradle-embulk-plugin:0.2.0"
         classpath "net.researchgate:gradle-release:2.6.0"
+        classpath 'org.junit.platform:junit-platform-gradle-plugin:1.0.0'
     }
 }
 
 apply plugin: "kotlin"
 apply plugin: "com.github.kamatama41.embulk"
 apply plugin: "net.researchgate.release"
+apply plugin: 'org.junit.platform.gradle.plugin'
+
+junitPlatform {
+    filters {
+        engines {
+            include 'spek'
+        }
+    }
+}
+
+repositories {
+    maven { url "http://dl.bintray.com/jetbrains/spek" }
+}
 
 embulk {
     version = "0.8.18"
@@ -30,6 +44,16 @@ embulk {
 dependencies {
     // To refer KClass::class.java
     compile "org.jetbrains.kotlin:kotlin-reflect:$kotlin_version"
-
     compile 'com.google.cloud:google-cloud-datastore:1.27.0'
+
+    testCompile "org.jetbrains.kotlin:kotlin-reflect:$kotlin_version"
+    testCompile group: 'org.jetbrains.kotlin', name: 'kotlin-test', version: '1.2.50'
+    testCompile group: 'org.jetbrains.kotlin', name: 'kotlin-test-annotations-common', version: '1.2.50'
+    testCompile ('org.jetbrains.spek:spek-api:1.1.5') {
+        exclude group: 'org.jetbrains.kotlin'
+    }
+    testRuntime ('org.jetbrains.spek:spek-junit-platform-engine:1.1.5') {
+        exclude group: 'org.junit.platform'
+        exclude group: 'org.jetbrains.kotlin'
+    }
 }

data/gradle.properties

@@ -1 +1 @@
-version=0.0.4
+version=0.1.0

data/src/main/java/org/embulk/input/datastore/DatastoreInputPlugin.kt

@@ -11,12 +11,14 @@ import org.embulk.spi.*
 import org.embulk.spi.type.Types
 import org.msgpack.value.ValueFactory
 import java.io.FileInputStream
+import java.util.Base64
 
-class DatastoreInputPlugin : InputPlugin {
+class DatastoreInputPlugin(doLogging: Boolean = true) : InputPlugin {
     // number of run() method calls
     private val TASK_COUNT = 1
 
-    private val logger = Exec.getLogger(javaClass)
+    private val logger = if (doLogging) { Exec.getLogger(javaClass) } else { null }
+    private val b64encoder = Base64.getEncoder()
 
     override fun transaction(config: ConfigSource,
                              control: InputPlugin.Control): ConfigDiff {
@@ -59,10 +61,10 @@ class DatastoreInputPlugin : InputPlugin {
 
         datastore.run(query)
                 .forEach { entity ->
-                    logger.debug(entity.toString())
+                    logger?.debug(entity.toString())
 
                     val json = entityToJsonObject(entity)
-                    logger.debug(json)
+                    logger?.debug(json)
 
                     pageBuilder.setJson(col, ValueFactory.newString(json))
                     pageBuilder.addRecord()
@@ -134,7 +136,7 @@ class DatastoreInputPlugin : InputPlugin {
      */
     private fun valueToField(dsValue: Value<Any>): String? {
         return when (dsValue.type) {
-            ValueType.BLOB -> (dsValue.get() as ByteArray).toString()
+            ValueType.BLOB -> "\"${b64encoder.encodeToString((dsValue.get() as Blob).toByteArray())}\""
             ValueType.BOOLEAN -> (dsValue.get() as Boolean).toString()
             ValueType.DOUBLE -> (dsValue.get() as Double).toString()
             ValueType.ENTITY -> entityToJsonObject(dsValue.get() as FullEntity<*>)
@@ -145,7 +147,7 @@ class DatastoreInputPlugin : InputPlugin {
             ValueType.NULL -> "null"
             ValueType.RAW_VALUE -> (dsValue.get() as RawValue).toString()
             ValueType.STRING -> "\"${dsValue.get() as String}\""
-            ValueType.TIMESTAMP -> (dsValue.get() as Timestamp).toString()
+            ValueType.TIMESTAMP -> "\"${dsValue.get() as Timestamp}\""
             else -> null // NOTE: unexpected or unsupported type
         }
     }

data/src/test/java/org/embulk/input/datastore/DatastoreInputPluginSpec.kt

@@ -0,0 +1,116 @@
+package org.embulk.input.datastore
+
+import com.google.cloud.Timestamp
+import com.google.cloud.datastore.*
+import org.jetbrains.spek.api.Spek
+import org.jetbrains.spek.api.dsl.describe
+import org.jetbrains.spek.api.dsl.it
+import org.jetbrains.spek.api.dsl.on
+import kotlin.reflect.full.memberFunctions
+import kotlin.reflect.jvm.isAccessible
+import kotlin.test.assertEquals
+import java.util.Base64
+
+object DatastoreInputPluginSpec : Spek({
+    val b64Encoder = Base64.getEncoder()
+
+    describe("entityToJsonObject()") {
+        val plugin = DatastoreInputPlugin(false)
+
+        val method = plugin::class.memberFunctions.find { it.name == "entityToJsonObject" }
+        val accesibleMethod = method!!.let {
+            it.isAccessible = true
+            it
+        }
+
+        on("single property values") {
+            val testcases = listOf<Pair<FullEntity<IncompleteKey>, String>>(
+                    Pair(
+                            Entity.newBuilder().set("blobProp", Blob.copyFrom("test".toByteArray())).build(),
+                            "{\"blobProp\":\"${b64Encoder.encodeToString("test".toByteArray())}\"}"
+                    ),
+                    Pair(
+                            Entity.newBuilder().set("boolProp", true).build(),
+                            "{\"boolProp\":true}"
+                    ),
+                    Pair(
+                            Entity.newBuilder().set("doubleProp", 1.1).build(),
+                            "{\"doubleProp\":1.1}"
+                    ),
+                    Pair(
+                            Entity.newBuilder().set("entityProp", FullEntity.newBuilder().set("name1", 1).build()).build(),
+                            "{\"entityProp\":{\"name1\":1}}"
+                    ),
+                    Pair(
+                            Entity.newBuilder().set("listProp", 1, 2, 3).build(),
+                            "{\"listProp\":[1, 2, 3]}"
+                    ),
+                    Pair(
+                            Entity.newBuilder().set("nullProp", NullValue.of()).build(),
+                            "{\"nullProp\":null}"
+                    ),
+                    Pair(
+                            Entity.newBuilder().set("longProp", 1).build(),
+                            "{\"longProp\":1}"
+                    ),
+                    Pair(
+                            Entity.newBuilder().set("stringProp", "test").build(),
+                            "{\"stringProp\":\"test\"}"
+                    ),
+                    Pair(
+                            Entity.newBuilder().set("timestampProp", Timestamp.MIN_VALUE).build(),
+                            "{\"timestampProp\":\"0001-01-01T00:00:00Z\"}"
+                    )
+            )
+
+            it("should be converted json value") {
+                testcases.forEach { pair ->
+                    val entity = pair.first
+                    val expected = pair.second
+
+                    val actual = accesibleMethod.call(plugin, entity) as String
+                    assertEquals(expected, actual)
+                }
+            }
+        }
+
+        on("combinational property values") {
+            val testcases = listOf<Pair<FullEntity<IncompleteKey>, String>>(
+                    Pair(
+                            Entity.newBuilder()
+                                    .set("blobProp", Blob.copyFrom("test".toByteArray()))
+                                    .set("boolProp", true)
+                                    .set("doubleProp", 1.1)
+                                    .set("entityProp", FullEntity.newBuilder().set("name1", 1).build())
+                                    .set("listProp", 1, 2, 3)
+                                    .set("nullProp", NullValue.of())
+                                    .set("longProp", 1)
+                                    .set("stringProp", "test")
+                                    .set("timestampProp", Timestamp.MIN_VALUE)
+                                    .build(),
+                            "{" +
+                                    "\"blobProp\":\"${b64Encoder.encodeToString("test".toByteArray())}\", " +
+                                    "\"boolProp\":true, " +
+                                    "\"doubleProp\":1.1, " +
+                                    "\"entityProp\":{\"name1\":1}, " +
+                                    "\"listProp\":[1, 2, 3], " +
+                                    "\"longProp\":1, " +
+                                    "\"nullProp\":null, " +
+                                    "\"stringProp\":\"test\", " +
+                                    "\"timestampProp\":\"0001-01-01T00:00:00Z\"" +
+                                    "}"
+                    )
+            )
+
+            it("should be converted json value") {
+                testcases.forEach { pair ->
+                    val entity = pair.first
+                    val expected = pair.second
+
+                    val actual = accesibleMethod.call(plugin, entity) as String
+                    assertEquals(expected, actual)
+                }
+            }
+        }
+    }
+})

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-datastore
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.1.0
 platform: ruby
 authors:
 - syucream
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-06-22 00:00:00.000000000 Z
+date: 2018-06-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -55,7 +55,7 @@ files:
 - classpath/commons-codec-1.3.jar
 - classpath/commons-logging-1.1.1.jar
 - classpath/datastore-v1-proto-client-1.6.0.jar
-- classpath/embulk-input-datastore-0.0.4.jar
+- classpath/embulk-input-datastore-0.1.0.jar
 - classpath/error_prone_annotations-2.2.0.jar
 - classpath/gax-1.23.0.jar
 - classpath/gax-httpjson-0.40.0.jar
@@ -95,7 +95,7 @@ files:
 - lib/embulk/input/datastore.rb
 - src/main/java/org/embulk/input/datastore/DatastoreInputPlugin.kt
 - src/main/java/org/embulk/input/datastore/PluginTask.kt
-- src/test/java/org/embulk/input/datastore/TestDatastoreInputPlugin.java
+- src/test/java/org/embulk/input/datastore/DatastoreInputPluginSpec.kt
 homepage: https://github.com/syucream/embulk-input-datastore
 licenses:
 - MIT

data/src/test/java/org/embulk/input/datastore/TestDatastoreInputPlugin.java

@@ -1,5 +0,0 @@
-package org.embulk.input.datastore;
-
-public class TestDatastoreInputPlugin
-{
-}