embulk-input-datastore 0.0.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: a08810cd4edfd6940aebe26cdb15a4471b5d77bb
+  data.tar.gz: 77e8fd4024f7c3e9b7bb8fe444857c9dd907f0de
+SHA512:
+  metadata.gz: b0f57598b143161a2721343ced65cab1b54de29b3d9c30661dc3aaaa4ead810c47816f6da463af3cf43c569c046b61a5de22d6d9f152a32aa8f671325504499a
+  data.tar.gz: a4f63b161e32c4dbb54cd00572b0e3a034aef531bebd30c075e231f91f1c53699ac2bd090b9bd15746e186db2954671a6bbf03c2fc48220139fb92542a618297

data/.gitignore

@@ -0,0 +1,12 @@
+*~
+/pkg/
+/tmp/
+*.gemspec
+.gradle/
+/classpath/
+build/
+.idea
+/.settings/
+/.metadata/
+.classpath
+.project

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2018 Ryo Okubo
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,37 @@
+# Datastore input plugin for Embulk
+
+A embulk input plugin fetches Cloud Datastore entities.
+
+## Overview
+
+* **Plugin type**: input
+* **Resume supported**: no
+* **Cleanup supported**: no
+* **Guess supported**: no
+
+## Configuration
+
+- **json_keyfile**: A path to JSON keyfile. (string, required)
+- **gql**: A GQL fetches to Cloud Datastore (string, required)
+- **json_column_name**: description (string, default: `"record"`)
+
+## Example
+
+```yaml
+in:
+  type: datastore
+  json_keyfile: example1
+  gql: "SELECT * FROM myKind WHERE myProp >= 100 AND myProp < 200"
+```
+
+
+## Build
+
+```
+$ ./gradlew gem  # -t to watch change of files and rebuild continuously
+```
+
+## NOTE
+
+Currently this plugin aggregates fetched entities to 1 'json' type column.
+

data/build.gradle

@@ -0,0 +1,30 @@
+buildscript {
+    ext.kotlin_version = '1.2.31'
+    repositories {
+        jcenter()
+        maven { url 'http://kamatama41.github.com/maven-repository/repository' }
+    }
+    dependencies {
+        classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlin_version"
+        classpath "com.github.kamatama41:gradle-embulk-plugin:0.2.0"
+        classpath "net.researchgate:gradle-release:2.6.0"
+    }
+}
+
+apply plugin: "com.github.kamatama41.embulk"
+
+embulk {
+    version = "0.8.18"
+    category = "input"
+    name = "datastore"
+    authors = ["syucream"]
+    email = "syucream1031@gmail.com"
+    homepage = "https://github.com/syucream/embulk-input-datastore"
+}
+
+dependencies {
+    // To refer KClass::class.java
+    implementation "org.jetbrains.kotlin:kotlin-reflect:$kotlin_version"
+
+    implementation 'com.google.cloud:google-cloud-datastore:1.27.0'
+}

data/embulk-input-datastore.iml

@@ -0,0 +1,13 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module external.linked.project.id="embulk-input-datastore" external.linked.project.path="$MODULE_DIR$" external.root.project.path="$MODULE_DIR$" external.system.id="GRADLE" external.system.module.group="" external.system.module.version="0.0.1" type="JAVA_MODULE" version="4">
+  <component name="NewModuleRootManager" inherit-compiler-output="true">
+    <exclude-output />
+    <content url="file://$MODULE_DIR$">
+      <excludeFolder url="file://$MODULE_DIR$/.gradle" />
+      <excludeFolder url="file://$MODULE_DIR$/build" />
+      <excludeFolder url="file://$MODULE_DIR$/out" />
+    </content>
+    <orderEntry type="inheritedJdk" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+</module>

data/gradle.properties

@@ -0,0 +1 @@
+version=0.0.2

data/lib/embulk/input/datastore.rb

@@ -0,0 +1,3 @@
+Embulk::JavaPlugin.register_input(
+  "datastore", "org.embulk.input.datastore.DatastoreInputPlugin",
+  File.expand_path('../../../../classpath', __FILE__))

data/src/main/java/org/embulk/input/datastore/DatastoreInputPlugin.kt

@@ -0,0 +1,87 @@
+package org.embulk.input.datastore
+
+import com.google.auth.oauth2.GoogleCredentials
+import com.google.cloud.datastore.*
+import org.embulk.config.TaskReport
+import org.embulk.config.ConfigDiff
+import org.embulk.config.ConfigSource
+import org.embulk.config.TaskSource
+import org.embulk.spi.*
+import org.embulk.spi.type.Types
+import org.msgpack.value.ValueFactory
+import java.io.FileInputStream
+import java.io.IOException
+
+class DatastoreInputPlugin : InputPlugin {
+    // number of run() method calls
+    val TASK_COUNT = 1
+
+    var datastore: Datastore? = null
+
+    override fun transaction(config: ConfigSource,
+                    control: InputPlugin.Control): ConfigDiff {
+        val task: PluginTask = config.loadConfig(PluginTask::class.java)
+
+        // Support only 1 'json' type column
+        val schema = Schema.builder()
+                .add(task.jsonColumnName, Types.JSON)
+                .build()
+
+        // Setup datastore client
+        val cred = GoogleCredentials
+                .fromStream(FileInputStream(task.jsonKeyfile))
+        this.datastore = DatastoreOptions.newBuilder()
+                .setCredentials(cred)
+                .build()
+                .service
+
+        return resume(task.dump(), schema, TASK_COUNT, control)
+    }
+
+    override fun resume(taskSource: TaskSource,
+               schema: Schema, taskCount: Int,
+               control: InputPlugin.Control): ConfigDiff {
+        // XXX Unimplemented
+        control.run(taskSource, schema, taskCount)
+        return Exec.newConfigDiff()
+    }
+
+    override fun cleanup(taskSource: TaskSource,
+                schema: Schema, taskCount: Int,
+                successTaskReports: List<TaskReport>) {
+    }
+
+    override fun run(taskSource: TaskSource,
+            schema: Schema, taskIndex: Int,
+            output: PageOutput): TaskReport {
+        val task = taskSource.loadTask(PluginTask::class.java)
+        val allocator = task.getBufferAllocator()
+        val pageBuilder = PageBuilder(allocator, schema, output)
+
+        val query = Query
+                .newGqlQueryBuilder(Query.ResultType.ENTITY, task.gql)
+                .build()
+
+        datastore?.let {
+            val col = pageBuilder.schema.getColumn(0)
+
+            it.run(query)
+                    .forEach { entity ->
+                        val msgpackValue = ValueFactory.newString(entity.toString())
+                        pageBuilder.setJson(col, msgpackValue)
+                        pageBuilder.addRecord()
+                    }
+
+            pageBuilder.finish()
+        } ?: run {
+            throw IOException("Datastore client is unavailable.")
+        }
+
+        return Exec.newTaskReport()
+    }
+
+    override fun guess(config: ConfigSource): ConfigDiff {
+        // XXX Unimplemented
+        return Exec.newConfigDiff()
+    }
+}

data/src/main/java/org/embulk/input/datastore/PluginTask.kt

@@ -0,0 +1,22 @@
+package org.embulk.input.datastore
+
+import org.embulk.config.Config
+import org.embulk.config.ConfigDefault
+import org.embulk.config.ConfigInject
+import org.embulk.config.Task
+import org.embulk.spi.BufferAllocator
+
+interface PluginTask : Task {
+    @get:Config("json_keyfile")
+    val jsonKeyfile: String
+
+    @get:Config("gql")
+    val gql: String
+
+    @get:Config("json_column_name")
+    @get:ConfigDefault("\"record\"")
+    val jsonColumnName: String
+
+    @ConfigInject
+    fun getBufferAllocator(): BufferAllocator
+}

data/src/test/java/org/embulk/input/datastore/TestDatastoreInputPlugin.java

@@ -0,0 +1,5 @@
+package org.embulk.input.datastore;
+
+public class TestDatastoreInputPlugin
+{
+}

metadata

@@ -0,0 +1,83 @@
+--- !ruby/object:Gem::Specification
+name: embulk-input-datastore
+version: !ruby/object:Gem::Version
+  version: 0.0.2
+platform: ruby
+authors:
+- syucream
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2018-06-19 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  name: bundler
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  name: rake
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '10.0'
+description: Loads records from datastore.
+email:
+- syucream1031@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- LICENSE
+- README.md
+- build.gradle
+- embulk-input-datastore.iml
+- gradle.properties
+- lib/embulk/input/datastore.rb
+- src/main/java/org/embulk/input/datastore/DatastoreInputPlugin.kt
+- src/main/java/org/embulk/input/datastore/PluginTask.kt
+- src/test/java/org/embulk/input/datastore/TestDatastoreInputPlugin.java
+- classpath/embulk-input-datastore-0.0.2.jar
+homepage: https://github.com/syucream/embulk-input-datastore
+licenses:
+- MIT
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.1.9
+signing_key:
+specification_version: 4
+summary: Datastore input plugin for Embulk
+test_files: []