embulk-input-datastore 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA256:
-  metadata.gz: 7073922589f46f02c4b1e1534ccccec56bf56eb9d6316bde1f6f6d7f80b7d1b1
-  data.tar.gz: 6ced669ff8d160252dd8806f801b5e28b8ce7f0a51c65fc969c5a07412897a82
+SHA1:
+  metadata.gz: 6387915cdeaf057d4c05886846d1b115e61ac244
+  data.tar.gz: a81ab9e7322ccc8896142442ea08a349db13030f
 SHA512:
-  metadata.gz: 891dd771e4683e046c413308ca0c71efbe349ff29ebf9dfc8b7f65ce16442e7d8bd2f010a87c7cbe39d2de2882b63170fe6a0baabfcf8716184018db86152aa1
-  data.tar.gz: beacaf4852d72788a3d77494ee96badb04ff4548e97b3304475a64f62aec856fdbca669bdc0691e45194499ef7a010ddb6b3899c894b9ec79888abff5d5e5830
+  metadata.gz: eb51ca9695d27bf905e4843818793c2aa1e29f04163e0de5a0acd5fd82c1681c04199928e254cb852ebe310ed022f64cde844be977db71ebc5152fd832e1e840
+  data.tar.gz: 326f15894055be12836f6e065572788beb57649aa09ef6773a51cffd6f1d4ffde71d8ff6aafb1349df473fb80a2c1f28281228db0ecaf14851baff46fe3d76b8

data/.gitignore

@@ -9,6 +9,7 @@ build/
 /.metadata/
 .classpath
 .project
+out
 
 # RubyGems
 *.gem

data/examples/datastore2stdout.yaml

@@ -2,7 +2,7 @@ in:
   type: datastore
   project_id: "deep-wares-182115"
   json_keyfile: credential.json
-  gql: "SELECT * FROM myKind"
+  gql: "SELECT * FROM myKind WHERE myProp >= 100 AND myProp < 200"
 
 out:
   type: stdout

data/gradle.properties

@@ -1 +1 @@
-version=0.1.0
+version=0.1.1

data/src/main/java/org/embulk/input/datastore/DatastoreInputPlugin.kt

@@ -53,7 +53,8 @@ class DatastoreInputPlugin(doLogging: Boolean = true) : InputPlugin {
         val pageBuilder = PageBuilder(allocator, schema, output)
 
         val query = Query
-                .newGqlQueryBuilder(Query.ResultType.ENTITY, task.gql)
+                .newGqlQueryBuilder(getGQLResultMode(task.gql), task.gql)
+                .setAllowLiteral(true)
                 .build()
 
         val datastore = createDatastoreClient(task)
@@ -63,8 +64,18 @@ class DatastoreInputPlugin(doLogging: Boolean = true) : InputPlugin {
                 .forEach { entity ->
                     logger?.debug(entity.toString())
 
-                    val json = entityToJsonObject(entity)
-                    logger?.debug(json)
+
+                    val json = when(entity) {
+                        is FullEntity<*> -> entityToJsonObject(entity)
+                        is ProjectionEntity -> entityToJsonObject(entity)
+                        else -> null
+                    }
+
+                    json?.let {
+                        logger?.debug(json)
+                    } ?: run {
+                        logger?.error("Unexpected result type")
+                    }
 
                     pageBuilder.setJson(col, ValueFactory.newString(json))
                     pageBuilder.addRecord()
@@ -100,7 +111,7 @@ class DatastoreInputPlugin(doLogging: Boolean = true) : InputPlugin {
      *  e.g.) '{"name": "value", ...}'
      *
      */
-    private fun entityToJsonObject(entity: FullEntity<*>): String? {
+    private fun entityToJsonObject(entity: BaseEntity<*>): String? {
         val fields = entity.names.flatMap { name ->
             val dsValue = entity.getValue<Value<Any>>(name)
             val strVal = valueToField(dsValue)
@@ -139,7 +150,7 @@ class DatastoreInputPlugin(doLogging: Boolean = true) : InputPlugin {
             ValueType.BLOB -> "\"${b64encoder.encodeToString((dsValue.get() as Blob).toByteArray())}\""
             ValueType.BOOLEAN -> (dsValue.get() as Boolean).toString()
             ValueType.DOUBLE -> (dsValue.get() as Double).toString()
-            ValueType.ENTITY -> entityToJsonObject(dsValue.get() as FullEntity<*>)
+            ValueType.ENTITY -> entityToJsonObject(dsValue.get() as BaseEntity<*>)
             ValueType.KEY -> (dsValue.get() as Key).toString()
             ValueType.LAT_LNG -> (dsValue.get() as LatLngValue).toString()
             ValueType.LIST -> listToJsonArray(dsValue.get() as List<*>)
@@ -151,4 +162,20 @@ class DatastoreInputPlugin(doLogging: Boolean = true) : InputPlugin {
             else -> null // NOTE: unexpected or unsupported type
         }
     }
+
+    /**
+     *  Check the GQL will return FullEntity or ProjectionEntity
+     *  NOTE: GQL accepts '*' at only after 'SELECT'
+     *    https://cloud.google.com/datastore/docs/reference/gql_reference
+     *
+     */
+    private fun getGQLResultMode(gql: String): Query.ResultType<*> {
+        return if (gql.indexOf("*") >= 0) {
+            // Lookup all columns
+            Query.ResultType.ENTITY
+        } else {
+            // Lookup a part of columns
+            Query.ResultType.PROJECTION_ENTITY
+        }
+    }
 }

metadata

@@ -1,41 +1,41 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-datastore
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - syucream
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
 date: 2018-06-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '1.0'
-  type: :development
+  name: bundler
   prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '1.0'
 - !ruby/object:Gem::Dependency
-  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '10.0'
-  type: :development
+  name: rake
   prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '10.0'
 description: Loads records from datastore.
@@ -45,17 +45,23 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- ".gitignore"
+- .gitignore
 - LICENSE
 - Makefile
 - README.md
 - build.gradle
+- examples/datastore2stdout.yaml
+- gradle.properties
+- lib/embulk/input/datastore.rb
+- src/main/java/org/embulk/input/datastore/DatastoreInputPlugin.kt
+- src/main/java/org/embulk/input/datastore/PluginTask.kt
+- src/test/java/org/embulk/input/datastore/DatastoreInputPluginSpec.kt
 - classpath/annotations-13.0.jar
 - classpath/api-common-1.5.0.jar
 - classpath/commons-codec-1.3.jar
 - classpath/commons-logging-1.1.1.jar
 - classpath/datastore-v1-proto-client-1.6.0.jar
-- classpath/embulk-input-datastore-0.1.0.jar
+- classpath/embulk-input-datastore-0.1.1.jar
 - classpath/error_prone_annotations-2.2.0.jar
 - classpath/gax-1.23.0.jar
 - classpath/gax-httpjson-0.40.0.jar
@@ -90,34 +96,28 @@ files:
 - classpath/protobuf-java-3.5.1.jar
 - classpath/protobuf-java-util-3.5.1.jar
 - classpath/threetenbp-1.3.3.jar
-- examples/datastore2stdout.yaml
-- gradle.properties
-- lib/embulk/input/datastore.rb
-- src/main/java/org/embulk/input/datastore/DatastoreInputPlugin.kt
-- src/main/java/org/embulk/input/datastore/PluginTask.kt
-- src/test/java/org/embulk/input/datastore/DatastoreInputPluginSpec.kt
 homepage: https://github.com/syucream/embulk-input-datastore
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.7
-signing_key: 
+rubyforge_project:
+rubygems_version: 2.1.9
+signing_key:
 specification_version: 4
 summary: Datastore input plugin for Embulk
 test_files: []