embulk-input-datastore 0.0.2 → 0.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: a08810cd4edfd6940aebe26cdb15a4471b5d77bb
-  data.tar.gz: 77e8fd4024f7c3e9b7bb8fe444857c9dd907f0de
+SHA256:
+  metadata.gz: 0418d992c45a1a40f72e4810dee870bd580daeb760f95f2725e436c401ffce25
+  data.tar.gz: dddd42cfa92264e2a597258cc70c093c45d5b4f1cbd816264a6fcdb4d54999e9
 SHA512:
-  metadata.gz: b0f57598b143161a2721343ced65cab1b54de29b3d9c30661dc3aaaa4ead810c47816f6da463af3cf43c569c046b61a5de22d6d9f152a32aa8f671325504499a
-  data.tar.gz: a4f63b161e32c4dbb54cd00572b0e3a034aef531bebd30c075e231f91f1c53699ac2bd090b9bd15746e186db2954671a6bbf03c2fc48220139fb92542a618297
+  metadata.gz: 8558927f2a5eafa9c859a911fdd10447c43cd890013e47a9dc7c526d4b31a14d149aa9e8adb056a1eeaca63982233bd9757096f602131f72904fcfb22ae6620f
+  data.tar.gz: a85c26e22745df865733a51f56560f58231db62168e36376264b6fee55ff624e431d1348609cd9281175a598ac68cc82ea66d62cde8e35d339aaff62d2b3328e

data/.gitignore

@@ -1,7 +1,6 @@
 *~
 /pkg/
 /tmp/
-*.gemspec
 .gradle/
 /classpath/
 build/

data/Makefile

@@ -0,0 +1,20 @@
+build:
+	gradle build
+
+embulk_run:
+	gradle embulk_run
+
+publish_gem: build
+	gradle classpath
+	gradle gemPush
+
+.PHONY: start_datastore_emulator
+start_datastore_emulator:
+	gcloud beta emulators datastore start --no-store-on-disk --quiet &
+	sleep 1
+	$(gcloud beta emulators datastore env-init)
+
+.PHONY: stop_datastore_emulator
+stop_datastore_emulator:
+	gcloud beta emulators datastore env-unset
+	ps aux | grep CloudDatastore | grep -v grep | awk '{ print "kill " $2 }' | sh

data/README.md

@@ -1,5 +1,7 @@
 # Datastore input plugin for Embulk
 
+[![Gem Version](https://badge.fury.io/rb/embulk-input-datastore.svg)](https://badge.fury.io/rb/embulk-input-datastore)
+
 A embulk input plugin fetches Cloud Datastore entities.
 
 ## Overview
@@ -11,6 +13,7 @@ A embulk input plugin fetches Cloud Datastore entities.
 
 ## Configuration
 
+- **project_id**: your GCP project_id. (string, required)
 - **json_keyfile**: A path to JSON keyfile. (string, required)
 - **gql**: A GQL fetches to Cloud Datastore (string, required)
 - **json_column_name**: description (string, default: `"record"`)
@@ -20,8 +23,9 @@ A embulk input plugin fetches Cloud Datastore entities.
 ```yaml
 in:
   type: datastore
-  json_keyfile: example1
-  gql: "SELECT * FROM myKind WHERE myProp >= 100 AND myProp < 200"
+  project_id: "your-gcppj-123"
+  json_keyfile: credential.json
+  gql: "SELECT * FROM myKind"
 ```
 
 
@@ -33,5 +37,7 @@ $ ./gradlew gem  # -t to watch change of files and rebuild continuously
 
 ## NOTE
 
-Currently this plugin aggregates fetched entities to 1 'json' type column.
+- Currently this plugin has below limitations:
+  - Aggregate fetched properties to 1 'json' type column.
+  - `ENTITY`, `KEY`, `LIST`, `LAT_LNG`, `NULL`, `RAW_VALUE` types are not supported.
 

data/build.gradle

@@ -11,7 +11,9 @@ buildscript {
     }
 }
 
+apply plugin: "kotlin"
 apply plugin: "com.github.kamatama41.embulk"
+apply plugin: "net.researchgate.release"
 
 embulk {
     version = "0.8.18"
@@ -20,11 +22,14 @@ embulk {
     authors = ["syucream"]
     email = "syucream1031@gmail.com"
     homepage = "https://github.com/syucream/embulk-input-datastore"
+
+    // for debug
+    configYaml = "examples/datastore2stdout.yaml"
 }
 
 dependencies {
     // To refer KClass::class.java
-    implementation "org.jetbrains.kotlin:kotlin-reflect:$kotlin_version"
+    compile "org.jetbrains.kotlin:kotlin-reflect:$kotlin_version"
 
-    implementation 'com.google.cloud:google-cloud-datastore:1.27.0'
+    compile 'com.google.cloud:google-cloud-datastore:1.27.0'
 }

data/embulk-input-datastore.gemspec

@@ -0,0 +1,18 @@
+Gem::Specification.new do |spec|
+  spec.name          = "embulk-input-datastore"
+  spec.version       = "0.0.3"
+  spec.authors       = ["syucream"]
+  spec.summary       = %[Datastore input plugin for Embulk]
+  spec.description   = %[Loads records from datastore.]
+  spec.email         = ["syucream1031@gmail.com"]
+  spec.licenses      = ["MIT"]
+  spec.homepage      = "https://github.com/syucream/embulk-input-datastore"
+
+  spec.files         = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
+  spec.test_files    = spec.files.grep(%r"^(test|spec)/")
+  spec.require_paths = ["lib"]
+
+  #spec.add_dependency 'YOUR_GEM_DEPENDENCY', ['~> YOUR_GEM_DEPENDENCY_VERSION']
+  spec.add_development_dependency 'bundler', ['~> 1.0']
+  spec.add_development_dependency 'rake', ['>= 10.0']
+end

data/embulk-input-datastore.iml

@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<module external.linked.project.id="embulk-input-datastore" external.linked.project.path="$MODULE_DIR$" external.root.project.path="$MODULE_DIR$" external.system.id="GRADLE" external.system.module.group="" external.system.module.version="0.0.1" type="JAVA_MODULE" version="4">
+<module external.linked.project.id="embulk-input-datastore" external.linked.project.path="$MODULE_DIR$" external.root.project.path="$MODULE_DIR$" external.system.id="GRADLE" external.system.module.group="" external.system.module.version="0.0.3" type="JAVA_MODULE" version="4">
   <component name="NewModuleRootManager" inherit-compiler-output="true">
     <exclude-output />
     <content url="file://$MODULE_DIR$">

data/examples/datastore2stdout.yaml

@@ -0,0 +1,8 @@
+in:
+  type: datastore
+  project_id: "deep-wares-182115"
+  json_keyfile: credential.json
+  gql: "SELECT * FROM myKind"
+
+out:
+  type: stdout

data/gradle.properties

@@ -1 +1 @@
-version=0.0.2
+version=0.0.3

data/src/main/java/org/embulk/input/datastore/DatastoreInputPlugin.kt

@@ -1,6 +1,7 @@
 package org.embulk.input.datastore
 
 import com.google.auth.oauth2.GoogleCredentials
+import com.google.cloud.Timestamp
 import com.google.cloud.datastore.*
 import org.embulk.config.TaskReport
 import org.embulk.config.ConfigDiff
@@ -10,16 +11,15 @@ import org.embulk.spi.*
 import org.embulk.spi.type.Types
 import org.msgpack.value.ValueFactory
 import java.io.FileInputStream
-import java.io.IOException
 
 class DatastoreInputPlugin : InputPlugin {
     // number of run() method calls
-    val TASK_COUNT = 1
+    private val TASK_COUNT = 1
 
-    var datastore: Datastore? = null
+    private val logger = Exec.getLogger(javaClass)
 
     override fun transaction(config: ConfigSource,
-                    control: InputPlugin.Control): ConfigDiff {
+                             control: InputPlugin.Control): ConfigDiff {
         val task: PluginTask = config.loadConfig(PluginTask::class.java)
 
         // Support only 1 'json' type column
@@ -27,33 +27,25 @@ class DatastoreInputPlugin : InputPlugin {
                 .add(task.jsonColumnName, Types.JSON)
                 .build()
 
-        // Setup datastore client
-        val cred = GoogleCredentials
-                .fromStream(FileInputStream(task.jsonKeyfile))
-        this.datastore = DatastoreOptions.newBuilder()
-                .setCredentials(cred)
-                .build()
-                .service
-
         return resume(task.dump(), schema, TASK_COUNT, control)
     }
 
     override fun resume(taskSource: TaskSource,
-               schema: Schema, taskCount: Int,
-               control: InputPlugin.Control): ConfigDiff {
+                        schema: Schema, taskCount: Int,
+                        control: InputPlugin.Control): ConfigDiff {
         // XXX Unimplemented
         control.run(taskSource, schema, taskCount)
         return Exec.newConfigDiff()
     }
 
     override fun cleanup(taskSource: TaskSource,
-                schema: Schema, taskCount: Int,
-                successTaskReports: List<TaskReport>) {
+                         schema: Schema, taskCount: Int,
+                         successTaskReports: List<TaskReport>) {
     }
 
     override fun run(taskSource: TaskSource,
-            schema: Schema, taskIndex: Int,
-            output: PageOutput): TaskReport {
+                     schema: Schema, taskIndex: Int,
+                     output: PageOutput): TaskReport {
         val task = taskSource.loadTask(PluginTask::class.java)
         val allocator = task.getBufferAllocator()
         val pageBuilder = PageBuilder(allocator, schema, output)
@@ -62,20 +54,37 @@ class DatastoreInputPlugin : InputPlugin {
                 .newGqlQueryBuilder(Query.ResultType.ENTITY, task.gql)
                 .build()
 
-        datastore?.let {
-            val col = pageBuilder.schema.getColumn(0)
-
-            it.run(query)
-                    .forEach { entity ->
-                        val msgpackValue = ValueFactory.newString(entity.toString())
-                        pageBuilder.setJson(col, msgpackValue)
-                        pageBuilder.addRecord()
+        val datastore = createDatastoreClient(task)
+        val col = pageBuilder.schema.getColumn(0)
+
+        datastore.run(query)
+                .forEach { entity ->
+                    logger.debug(entity.toString())
+
+                    // TODO separate/simplify generating JSON
+                    val pairs = entity.names.flatMap { name ->
+                        val dsValue = entity.getValue<Value<Any>>(name)
+                        val strVal: String? = when (dsValue.type) {
+                            ValueType.BLOB -> (dsValue.get() as ByteArray).toString()
+                            ValueType.BOOLEAN -> (dsValue.get() as Boolean).toString()
+                            ValueType.DOUBLE -> (dsValue.get() as Double).toString()
+                            ValueType.LONG -> (dsValue.get() as Long).toString()
+                            ValueType.STRING -> "\"${dsValue.get() as String}\""
+                            ValueType.TIMESTAMP -> (dsValue.get() as Timestamp).toString()
+                            else -> null // NOTE, TODO: LIST, ENTITY, ... is still unsupported
+                        }
+                        strVal?.let { listOf(Pair<String, String>(name, it)) } ?: listOf()
                     }
+                    val json = "{" + pairs.map { pair ->
+                        "\"${pair.first}\": ${pair.second}"
+                    }.joinToString() + "}"
+
+                    val msgpackValue = ValueFactory.newString(json)
+                    pageBuilder.setJson(col, msgpackValue)
+                    pageBuilder.addRecord()
+                }
 
-            pageBuilder.finish()
-        } ?: run {
-            throw IOException("Datastore client is unavailable.")
-        }
+        pageBuilder.finish()
 
         return Exec.newTaskReport()
     }
@@ -84,4 +93,19 @@ class DatastoreInputPlugin : InputPlugin {
         // XXX Unimplemented
         return Exec.newConfigDiff()
     }
+
+    /**
+     * Setup datastore client
+     *
+     */
+    private fun createDatastoreClient(task: PluginTask): Datastore {
+        val cred = GoogleCredentials
+                .fromStream(FileInputStream(task.jsonKeyfile))
+
+        return DatastoreOptions.newBuilder()
+                .setProjectId(task.projectId)
+                .setCredentials(cred)
+                .build()
+                .service
+    }
 }

data/src/main/java/org/embulk/input/datastore/PluginTask.kt

@@ -7,6 +7,9 @@ import org.embulk.config.Task
 import org.embulk.spi.BufferAllocator
 
 interface PluginTask : Task {
+    @get:Config("project_id")
+    val projectId: String
+
     @get:Config("json_keyfile")
     val jsonKeyfile: String
 

metadata

@@ -1,41 +1,41 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-datastore
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.3
 platform: ruby
 authors:
 - syucream
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-06-19 00:00:00.000000000 Z
+date: 2018-06-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
+  name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
-  name: bundler
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
 - !ruby/object:Gem::Dependency
+  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '10.0'
-  name: rake
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '10.0'
 description: Loads records from datastore.
@@ -45,39 +45,81 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
+- ".gitignore"
 - LICENSE
+- Makefile
 - README.md
 - build.gradle
+- classpath/annotations-13.0.jar
+- classpath/api-common-1.5.0.jar
+- classpath/commons-codec-1.3.jar
+- classpath/commons-logging-1.1.1.jar
+- classpath/datastore-v1-proto-client-1.6.0.jar
+- classpath/embulk-input-datastore-0.0.3.jar
+- classpath/error_prone_annotations-2.2.0.jar
+- classpath/gax-1.23.0.jar
+- classpath/gax-httpjson-0.40.0.jar
+- classpath/google-api-client-1.23.0.jar
+- classpath/google-auth-library-credentials-0.9.0.jar
+- classpath/google-auth-library-oauth2-http-0.9.0.jar
+- classpath/google-cloud-core-1.27.0.jar
+- classpath/google-cloud-core-http-1.27.0.jar
+- classpath/google-cloud-datastore-1.27.0.jar
+- classpath/google-http-client-1.23.0.jar
+- classpath/google-http-client-appengine-1.23.0.jar
+- classpath/google-http-client-jackson-1.23.0.jar
+- classpath/google-http-client-jackson2-1.23.0.jar
+- classpath/google-http-client-protobuf-1.20.0.jar
+- classpath/google-oauth-client-1.23.0.jar
+- classpath/grpc-context-1.10.1.jar
+- classpath/grpc-core-1.10.1.jar
+- classpath/gson-2.7.jar
+- classpath/guava-20.0.jar
+- classpath/httpclient-4.0.1.jar
+- classpath/httpcore-4.0.1.jar
+- classpath/jackson-core-asl-1.9.11.jar
+- classpath/jsr305-3.0.1.jar
+- classpath/kotlin-reflect-1.2.31.jar
+- classpath/kotlin-stdlib-1.2.31.jar
+- classpath/opencensus-api-0.11.1.jar
+- classpath/opencensus-contrib-grpc-metrics-0.11.0.jar
+- classpath/opencensus-contrib-http-util-0.11.1.jar
+- classpath/proto-google-cloud-datastore-v1-0.10.0.jar
+- classpath/proto-google-common-protos-1.9.0.jar
+- classpath/proto-google-iam-v1-0.10.0.jar
+- classpath/protobuf-java-3.5.1.jar
+- classpath/protobuf-java-util-3.5.1.jar
+- classpath/threetenbp-1.3.3.jar
+- embulk-input-datastore.gemspec
 - embulk-input-datastore.iml
+- examples/datastore2stdout.yaml
 - gradle.properties
 - lib/embulk/input/datastore.rb
 - src/main/java/org/embulk/input/datastore/DatastoreInputPlugin.kt
 - src/main/java/org/embulk/input/datastore/PluginTask.kt
 - src/test/java/org/embulk/input/datastore/TestDatastoreInputPlugin.java
-- classpath/embulk-input-datastore-0.0.2.jar
 homepage: https://github.com/syucream/embulk-input-datastore
 licenses:
 - MIT
 metadata: {}
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project:
-rubygems_version: 2.1.9
-signing_key:
+rubyforge_project: 
+rubygems_version: 2.7.7
+signing_key: 
 specification_version: 4
 summary: Datastore input plugin for Embulk
 test_files: []