embulk-input-datastore 0.0.3 → 0.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0418d992c45a1a40f72e4810dee870bd580daeb760f95f2725e436c401ffce25
-  data.tar.gz: dddd42cfa92264e2a597258cc70c093c45d5b4f1cbd816264a6fcdb4d54999e9
+  metadata.gz: 15a06b07d8765e66681ac6636340eddae2bc84dcc15303d538c89db1619bbb1e
+  data.tar.gz: cdb37b7382f2921fb9fa351dba8f1d1a2abe18951f4f887003c11011c1bfced4
 SHA512:
-  metadata.gz: 8558927f2a5eafa9c859a911fdd10447c43cd890013e47a9dc7c526d4b31a14d149aa9e8adb056a1eeaca63982233bd9757096f602131f72904fcfb22ae6620f
-  data.tar.gz: a85c26e22745df865733a51f56560f58231db62168e36376264b6fee55ff624e431d1348609cd9281175a598ac68cc82ea66d62cde8e35d339aaff62d2b3328e
+  metadata.gz: e35e6c6d76da73a4e76d50b202cb80b157cc3b5308b5b159eb91e1c3c2ca9b1c43c83a9938b507a70fe7341a8d21af00d2f60cac72aa1d8ec810bbfad451f14c
+  data.tar.gz: 37a8123d2607064598eed79b02c714785f9ca93e331b481bd5802384b86ed9b1665f7368c9b81ba3b12853030b1b27da53847a81ad4f2c15919bb8a08054ea15

data/.gitignore

@@ -9,3 +9,11 @@ build/
 /.metadata/
 .classpath
 .project
+
+# RubyGems
+*.gem
+*.gemspec
+*.iml
+
+# development, debug
+*.json

data/README.md

@@ -39,5 +39,4 @@ $ ./gradlew gem  # -t to watch change of files and rebuild continuously
 
 - Currently this plugin has below limitations:
   - Aggregate fetched properties to 1 'json' type column.
-  - `ENTITY`, `KEY`, `LIST`, `LAT_LNG`, `NULL`, `RAW_VALUE` types are not supported.
 

data/gradle.properties

@@ -1 +1 @@
-version=0.0.3
+version=0.0.4

data/src/main/java/org/embulk/input/datastore/DatastoreInputPlugin.kt

@@ -61,26 +61,10 @@ class DatastoreInputPlugin : InputPlugin {
                 .forEach { entity ->
                     logger.debug(entity.toString())
 
-                    // TODO separate/simplify generating JSON
-                    val pairs = entity.names.flatMap { name ->
-                        val dsValue = entity.getValue<Value<Any>>(name)
-                        val strVal: String? = when (dsValue.type) {
-                            ValueType.BLOB -> (dsValue.get() as ByteArray).toString()
-                            ValueType.BOOLEAN -> (dsValue.get() as Boolean).toString()
-                            ValueType.DOUBLE -> (dsValue.get() as Double).toString()
-                            ValueType.LONG -> (dsValue.get() as Long).toString()
-                            ValueType.STRING -> "\"${dsValue.get() as String}\""
-                            ValueType.TIMESTAMP -> (dsValue.get() as Timestamp).toString()
-                            else -> null // NOTE, TODO: LIST, ENTITY, ... is still unsupported
-                        }
-                        strVal?.let { listOf(Pair<String, String>(name, it)) } ?: listOf()
-                    }
-                    val json = "{" + pairs.map { pair ->
-                        "\"${pair.first}\": ${pair.second}"
-                    }.joinToString() + "}"
-
-                    val msgpackValue = ValueFactory.newString(json)
-                    pageBuilder.setJson(col, msgpackValue)
+                    val json = entityToJsonObject(entity)
+                    logger.debug(json)
+
+                    pageBuilder.setJson(col, ValueFactory.newString(json))
                     pageBuilder.addRecord()
                 }
 
@@ -108,4 +92,61 @@ class DatastoreInputPlugin : InputPlugin {
                 .build()
                 .service
     }
+
+    /**
+     *  Datastore entity -> JSON String
+     *  e.g.) '{"name": "value", ...}'
+     *
+     */
+    private fun entityToJsonObject(entity: FullEntity<*>): String? {
+        val fields = entity.names.flatMap { name ->
+            val dsValue = entity.getValue<Value<Any>>(name)
+            val strVal = valueToField(dsValue)
+            strVal?.let { listOf("\"${name}\":${it}") } ?: listOf()
+        }
+        return "{" + fields.joinToString() + "}"
+    }
+
+    /**
+     *  Datastore property value list -> JSON Array
+     *  e.g.) '[1,2,3]'
+     *
+     */
+    private fun listToJsonArray(values: List<*>): String? {
+        // NOTE: Do unsafe cast because of type erasure...
+        val anyValues = values as? List<Value<Any>>
+
+        anyValues?.let {
+            val fields = it.flatMap { v ->
+                val strVal = valueToField(v)
+                strVal?.let { listOf(it) } ?: listOf()
+            }
+            return "[" + fields.joinToString() + "]"
+        }
+
+        return null
+    }
+
+    /**
+     *  Datastore property value -> JSON field string
+     *  e.g.) '"name":"value"'
+     *
+     */
+    private fun valueToField(dsValue: Value<Any>): String? {
+        return when (dsValue.type) {
+            ValueType.BLOB -> (dsValue.get() as ByteArray).toString()
+            ValueType.BOOLEAN -> (dsValue.get() as Boolean).toString()
+            ValueType.DOUBLE -> (dsValue.get() as Double).toString()
+            ValueType.ENTITY -> entityToJsonObject(dsValue.get() as FullEntity<*>)
+            ValueType.KEY -> (dsValue.get() as Key).toString()
+            ValueType.LAT_LNG -> (dsValue.get() as LatLngValue).toString()
+            ValueType.LIST -> listToJsonArray(dsValue.get() as List<*>)
+            ValueType.LONG -> (dsValue.get() as Long).toString()
+            ValueType.NULL -> "null"
+            ValueType.RAW_VALUE -> (dsValue.get() as RawValue).toString()
+            ValueType.STRING -> "\"${dsValue.get() as String}\""
+            ValueType.TIMESTAMP -> (dsValue.get() as Timestamp).toString()
+            else -> null // NOTE: unexpected or unsupported type
+        }
+    }
 }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-datastore
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.4
 platform: ruby
 authors:
 - syucream
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-06-21 00:00:00.000000000 Z
+date: 2018-06-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -55,7 +55,7 @@ files:
 - classpath/commons-codec-1.3.jar
 - classpath/commons-logging-1.1.1.jar
 - classpath/datastore-v1-proto-client-1.6.0.jar
-- classpath/embulk-input-datastore-0.0.3.jar
+- classpath/embulk-input-datastore-0.0.4.jar
 - classpath/error_prone_annotations-2.2.0.jar
 - classpath/gax-1.23.0.jar
 - classpath/gax-httpjson-0.40.0.jar
@@ -90,8 +90,6 @@ files:
 - classpath/protobuf-java-3.5.1.jar
 - classpath/protobuf-java-util-3.5.1.jar
 - classpath/threetenbp-1.3.3.jar
-- embulk-input-datastore.gemspec
-- embulk-input-datastore.iml
 - examples/datastore2stdout.yaml
 - gradle.properties
 - lib/embulk/input/datastore.rb

data/embulk-input-datastore.gemspec

@@ -1,18 +0,0 @@
-Gem::Specification.new do |spec|
-  spec.name          = "embulk-input-datastore"
-  spec.version       = "0.0.3"
-  spec.authors       = ["syucream"]
-  spec.summary       = %[Datastore input plugin for Embulk]
-  spec.description   = %[Loads records from datastore.]
-  spec.email         = ["syucream1031@gmail.com"]
-  spec.licenses      = ["MIT"]
-  spec.homepage      = "https://github.com/syucream/embulk-input-datastore"
-
-  spec.files         = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
-  spec.test_files    = spec.files.grep(%r"^(test|spec)/")
-  spec.require_paths = ["lib"]
-
-  #spec.add_dependency 'YOUR_GEM_DEPENDENCY', ['~> YOUR_GEM_DEPENDENCY_VERSION']
-  spec.add_development_dependency 'bundler', ['~> 1.0']
-  spec.add_development_dependency 'rake', ['>= 10.0']
-end

data/embulk-input-datastore.iml

@@ -1,13 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<module external.linked.project.id="embulk-input-datastore" external.linked.project.path="$MODULE_DIR$" external.root.project.path="$MODULE_DIR$" external.system.id="GRADLE" external.system.module.group="" external.system.module.version="0.0.3" type="JAVA_MODULE" version="4">
-  <component name="NewModuleRootManager" inherit-compiler-output="true">
-    <exclude-output />
-    <content url="file://$MODULE_DIR$">
-      <excludeFolder url="file://$MODULE_DIR$/.gradle" />
-      <excludeFolder url="file://$MODULE_DIR$/build" />
-      <excludeFolder url="file://$MODULE_DIR$/out" />
-    </content>
-    <orderEntry type="inheritedJdk" />
-    <orderEntry type="sourceFolder" forTests="false" />
-  </component>
-</module>