embulk-input-datastore 0.1.0 → 0.1.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/.gitignore +1 -0
- data/examples/datastore2stdout.yaml +1 -1
- data/gradle.properties +1 -1
- data/src/main/java/org/embulk/input/datastore/DatastoreInputPlugin.kt +32 -5
- metadata +24 -24
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 6387915cdeaf057d4c05886846d1b115e61ac244
|
4
|
+
data.tar.gz: a81ab9e7322ccc8896142442ea08a349db13030f
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: eb51ca9695d27bf905e4843818793c2aa1e29f04163e0de5a0acd5fd82c1681c04199928e254cb852ebe310ed022f64cde844be977db71ebc5152fd832e1e840
|
7
|
+
data.tar.gz: 326f15894055be12836f6e065572788beb57649aa09ef6773a51cffd6f1d4ffde71d8ff6aafb1349df473fb80a2c1f28281228db0ecaf14851baff46fe3d76b8
|
data/.gitignore
CHANGED
data/gradle.properties
CHANGED
@@ -1 +1 @@
|
|
1
|
-
version=0.1.
|
1
|
+
version=0.1.1
|
@@ -53,7 +53,8 @@ class DatastoreInputPlugin(doLogging: Boolean = true) : InputPlugin {
|
|
53
53
|
val pageBuilder = PageBuilder(allocator, schema, output)
|
54
54
|
|
55
55
|
val query = Query
|
56
|
-
.newGqlQueryBuilder(
|
56
|
+
.newGqlQueryBuilder(getGQLResultMode(task.gql), task.gql)
|
57
|
+
.setAllowLiteral(true)
|
57
58
|
.build()
|
58
59
|
|
59
60
|
val datastore = createDatastoreClient(task)
|
@@ -63,8 +64,18 @@ class DatastoreInputPlugin(doLogging: Boolean = true) : InputPlugin {
|
|
63
64
|
.forEach { entity ->
|
64
65
|
logger?.debug(entity.toString())
|
65
66
|
|
66
|
-
|
67
|
-
|
67
|
+
|
68
|
+
val json = when(entity) {
|
69
|
+
is FullEntity<*> -> entityToJsonObject(entity)
|
70
|
+
is ProjectionEntity -> entityToJsonObject(entity)
|
71
|
+
else -> null
|
72
|
+
}
|
73
|
+
|
74
|
+
json?.let {
|
75
|
+
logger?.debug(json)
|
76
|
+
} ?: run {
|
77
|
+
logger?.error("Unexpected result type")
|
78
|
+
}
|
68
79
|
|
69
80
|
pageBuilder.setJson(col, ValueFactory.newString(json))
|
70
81
|
pageBuilder.addRecord()
|
@@ -100,7 +111,7 @@ class DatastoreInputPlugin(doLogging: Boolean = true) : InputPlugin {
|
|
100
111
|
* e.g.) '{"name": "value", ...}'
|
101
112
|
*
|
102
113
|
*/
|
103
|
-
private fun entityToJsonObject(entity:
|
114
|
+
private fun entityToJsonObject(entity: BaseEntity<*>): String? {
|
104
115
|
val fields = entity.names.flatMap { name ->
|
105
116
|
val dsValue = entity.getValue<Value<Any>>(name)
|
106
117
|
val strVal = valueToField(dsValue)
|
@@ -139,7 +150,7 @@ class DatastoreInputPlugin(doLogging: Boolean = true) : InputPlugin {
|
|
139
150
|
ValueType.BLOB -> "\"${b64encoder.encodeToString((dsValue.get() as Blob).toByteArray())}\""
|
140
151
|
ValueType.BOOLEAN -> (dsValue.get() as Boolean).toString()
|
141
152
|
ValueType.DOUBLE -> (dsValue.get() as Double).toString()
|
142
|
-
ValueType.ENTITY -> entityToJsonObject(dsValue.get() as
|
153
|
+
ValueType.ENTITY -> entityToJsonObject(dsValue.get() as BaseEntity<*>)
|
143
154
|
ValueType.KEY -> (dsValue.get() as Key).toString()
|
144
155
|
ValueType.LAT_LNG -> (dsValue.get() as LatLngValue).toString()
|
145
156
|
ValueType.LIST -> listToJsonArray(dsValue.get() as List<*>)
|
@@ -151,4 +162,20 @@ class DatastoreInputPlugin(doLogging: Boolean = true) : InputPlugin {
|
|
151
162
|
else -> null // NOTE: unexpected or unsupported type
|
152
163
|
}
|
153
164
|
}
|
165
|
+
|
166
|
+
/**
|
167
|
+
* Check the GQL will return FullEntity or ProjectionEntity
|
168
|
+
* NOTE: GQL accepts '*' at only after 'SELECT'
|
169
|
+
* https://cloud.google.com/datastore/docs/reference/gql_reference
|
170
|
+
*
|
171
|
+
*/
|
172
|
+
private fun getGQLResultMode(gql: String): Query.ResultType<*> {
|
173
|
+
return if (gql.indexOf("*") >= 0) {
|
174
|
+
// Lookup all columns
|
175
|
+
Query.ResultType.ENTITY
|
176
|
+
} else {
|
177
|
+
// Lookup a part of columns
|
178
|
+
Query.ResultType.PROJECTION_ENTITY
|
179
|
+
}
|
180
|
+
}
|
154
181
|
}
|
metadata
CHANGED
@@ -1,41 +1,41 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-input-datastore
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- syucream
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
11
|
date: 2018-06-24 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
|
-
name: bundler
|
15
14
|
requirement: !ruby/object:Gem::Requirement
|
16
15
|
requirements:
|
17
|
-
- -
|
16
|
+
- - ~>
|
18
17
|
- !ruby/object:Gem::Version
|
19
18
|
version: '1.0'
|
20
|
-
|
19
|
+
name: bundler
|
21
20
|
prerelease: false
|
21
|
+
type: :development
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
|
-
- -
|
24
|
+
- - ~>
|
25
25
|
- !ruby/object:Gem::Version
|
26
26
|
version: '1.0'
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
|
-
name: rake
|
29
28
|
requirement: !ruby/object:Gem::Requirement
|
30
29
|
requirements:
|
31
|
-
- -
|
30
|
+
- - '>='
|
32
31
|
- !ruby/object:Gem::Version
|
33
32
|
version: '10.0'
|
34
|
-
|
33
|
+
name: rake
|
35
34
|
prerelease: false
|
35
|
+
type: :development
|
36
36
|
version_requirements: !ruby/object:Gem::Requirement
|
37
37
|
requirements:
|
38
|
-
- -
|
38
|
+
- - '>='
|
39
39
|
- !ruby/object:Gem::Version
|
40
40
|
version: '10.0'
|
41
41
|
description: Loads records from datastore.
|
@@ -45,17 +45,23 @@ executables: []
|
|
45
45
|
extensions: []
|
46
46
|
extra_rdoc_files: []
|
47
47
|
files:
|
48
|
-
-
|
48
|
+
- .gitignore
|
49
49
|
- LICENSE
|
50
50
|
- Makefile
|
51
51
|
- README.md
|
52
52
|
- build.gradle
|
53
|
+
- examples/datastore2stdout.yaml
|
54
|
+
- gradle.properties
|
55
|
+
- lib/embulk/input/datastore.rb
|
56
|
+
- src/main/java/org/embulk/input/datastore/DatastoreInputPlugin.kt
|
57
|
+
- src/main/java/org/embulk/input/datastore/PluginTask.kt
|
58
|
+
- src/test/java/org/embulk/input/datastore/DatastoreInputPluginSpec.kt
|
53
59
|
- classpath/annotations-13.0.jar
|
54
60
|
- classpath/api-common-1.5.0.jar
|
55
61
|
- classpath/commons-codec-1.3.jar
|
56
62
|
- classpath/commons-logging-1.1.1.jar
|
57
63
|
- classpath/datastore-v1-proto-client-1.6.0.jar
|
58
|
-
- classpath/embulk-input-datastore-0.1.
|
64
|
+
- classpath/embulk-input-datastore-0.1.1.jar
|
59
65
|
- classpath/error_prone_annotations-2.2.0.jar
|
60
66
|
- classpath/gax-1.23.0.jar
|
61
67
|
- classpath/gax-httpjson-0.40.0.jar
|
@@ -90,34 +96,28 @@ files:
|
|
90
96
|
- classpath/protobuf-java-3.5.1.jar
|
91
97
|
- classpath/protobuf-java-util-3.5.1.jar
|
92
98
|
- classpath/threetenbp-1.3.3.jar
|
93
|
-
- examples/datastore2stdout.yaml
|
94
|
-
- gradle.properties
|
95
|
-
- lib/embulk/input/datastore.rb
|
96
|
-
- src/main/java/org/embulk/input/datastore/DatastoreInputPlugin.kt
|
97
|
-
- src/main/java/org/embulk/input/datastore/PluginTask.kt
|
98
|
-
- src/test/java/org/embulk/input/datastore/DatastoreInputPluginSpec.kt
|
99
99
|
homepage: https://github.com/syucream/embulk-input-datastore
|
100
100
|
licenses:
|
101
101
|
- MIT
|
102
102
|
metadata: {}
|
103
|
-
post_install_message:
|
103
|
+
post_install_message:
|
104
104
|
rdoc_options: []
|
105
105
|
require_paths:
|
106
106
|
- lib
|
107
107
|
required_ruby_version: !ruby/object:Gem::Requirement
|
108
108
|
requirements:
|
109
|
-
- -
|
109
|
+
- - '>='
|
110
110
|
- !ruby/object:Gem::Version
|
111
111
|
version: '0'
|
112
112
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
113
113
|
requirements:
|
114
|
-
- -
|
114
|
+
- - '>='
|
115
115
|
- !ruby/object:Gem::Version
|
116
116
|
version: '0'
|
117
117
|
requirements: []
|
118
|
-
rubyforge_project:
|
119
|
-
rubygems_version: 2.
|
120
|
-
signing_key:
|
118
|
+
rubyforge_project:
|
119
|
+
rubygems_version: 2.1.9
|
120
|
+
signing_key:
|
121
121
|
specification_version: 4
|
122
122
|
summary: Datastore input plugin for Embulk
|
123
123
|
test_files: []
|