embulk-input-datastore 0.0.3 → 0.0.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitignore +8 -0
- data/README.md +0 -1
- data/gradle.properties +1 -1
- data/src/main/java/org/embulk/input/datastore/DatastoreInputPlugin.kt +61 -20
- metadata +3 -5
- data/embulk-input-datastore.gemspec +0 -18
- data/embulk-input-datastore.iml +0 -13
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 15a06b07d8765e66681ac6636340eddae2bc84dcc15303d538c89db1619bbb1e
|
4
|
+
data.tar.gz: cdb37b7382f2921fb9fa351dba8f1d1a2abe18951f4f887003c11011c1bfced4
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: e35e6c6d76da73a4e76d50b202cb80b157cc3b5308b5b159eb91e1c3c2ca9b1c43c83a9938b507a70fe7341a8d21af00d2f60cac72aa1d8ec810bbfad451f14c
|
7
|
+
data.tar.gz: 37a8123d2607064598eed79b02c714785f9ca93e331b481bd5802384b86ed9b1665f7368c9b81ba3b12853030b1b27da53847a81ad4f2c15919bb8a08054ea15
|
data/.gitignore
CHANGED
data/README.md
CHANGED
@@ -39,5 +39,4 @@ $ ./gradlew gem # -t to watch change of files and rebuild continuously
|
|
39
39
|
|
40
40
|
- Currently this plugin has below limitations:
|
41
41
|
- Aggregate fetched properties to 1 'json' type column.
|
42
|
-
- `ENTITY`, `KEY`, `LIST`, `LAT_LNG`, `NULL`, `RAW_VALUE` types are not supported.
|
43
42
|
|
data/gradle.properties
CHANGED
@@ -1 +1 @@
|
|
1
|
-
version=0.0.
|
1
|
+
version=0.0.4
|
@@ -61,26 +61,10 @@ class DatastoreInputPlugin : InputPlugin {
|
|
61
61
|
.forEach { entity ->
|
62
62
|
logger.debug(entity.toString())
|
63
63
|
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
ValueType.BLOB -> (dsValue.get() as ByteArray).toString()
|
69
|
-
ValueType.BOOLEAN -> (dsValue.get() as Boolean).toString()
|
70
|
-
ValueType.DOUBLE -> (dsValue.get() as Double).toString()
|
71
|
-
ValueType.LONG -> (dsValue.get() as Long).toString()
|
72
|
-
ValueType.STRING -> "\"${dsValue.get() as String}\""
|
73
|
-
ValueType.TIMESTAMP -> (dsValue.get() as Timestamp).toString()
|
74
|
-
else -> null // NOTE, TODO: LIST, ENTITY, ... is still unsupported
|
75
|
-
}
|
76
|
-
strVal?.let { listOf(Pair<String, String>(name, it)) } ?: listOf()
|
77
|
-
}
|
78
|
-
val json = "{" + pairs.map { pair ->
|
79
|
-
"\"${pair.first}\": ${pair.second}"
|
80
|
-
}.joinToString() + "}"
|
81
|
-
|
82
|
-
val msgpackValue = ValueFactory.newString(json)
|
83
|
-
pageBuilder.setJson(col, msgpackValue)
|
64
|
+
val json = entityToJsonObject(entity)
|
65
|
+
logger.debug(json)
|
66
|
+
|
67
|
+
pageBuilder.setJson(col, ValueFactory.newString(json))
|
84
68
|
pageBuilder.addRecord()
|
85
69
|
}
|
86
70
|
|
@@ -108,4 +92,61 @@ class DatastoreInputPlugin : InputPlugin {
|
|
108
92
|
.build()
|
109
93
|
.service
|
110
94
|
}
|
95
|
+
|
96
|
+
/**
|
97
|
+
* Datastore entity -> JSON String
|
98
|
+
* e.g.) '{"name": "value", ...}'
|
99
|
+
*
|
100
|
+
*/
|
101
|
+
private fun entityToJsonObject(entity: FullEntity<*>): String? {
|
102
|
+
val fields = entity.names.flatMap { name ->
|
103
|
+
val dsValue = entity.getValue<Value<Any>>(name)
|
104
|
+
val strVal = valueToField(dsValue)
|
105
|
+
strVal?.let { listOf("\"${name}\":${it}") } ?: listOf()
|
106
|
+
}
|
107
|
+
return "{" + fields.joinToString() + "}"
|
108
|
+
}
|
109
|
+
|
110
|
+
/**
|
111
|
+
* Datastore property value list -> JSON Array
|
112
|
+
* e.g.) '[1,2,3]'
|
113
|
+
*
|
114
|
+
*/
|
115
|
+
private fun listToJsonArray(values: List<*>): String? {
|
116
|
+
// NOTE: Do unsafe cast because of type erasure...
|
117
|
+
val anyValues = values as? List<Value<Any>>
|
118
|
+
|
119
|
+
anyValues?.let {
|
120
|
+
val fields = it.flatMap { v ->
|
121
|
+
val strVal = valueToField(v)
|
122
|
+
strVal?.let { listOf(it) } ?: listOf()
|
123
|
+
}
|
124
|
+
return "[" + fields.joinToString() + "]"
|
125
|
+
}
|
126
|
+
|
127
|
+
return null
|
128
|
+
}
|
129
|
+
|
130
|
+
/**
|
131
|
+
* Datastore property value -> JSON field string
|
132
|
+
* e.g.) '"name":"value"'
|
133
|
+
*
|
134
|
+
*/
|
135
|
+
private fun valueToField(dsValue: Value<Any>): String? {
|
136
|
+
return when (dsValue.type) {
|
137
|
+
ValueType.BLOB -> (dsValue.get() as ByteArray).toString()
|
138
|
+
ValueType.BOOLEAN -> (dsValue.get() as Boolean).toString()
|
139
|
+
ValueType.DOUBLE -> (dsValue.get() as Double).toString()
|
140
|
+
ValueType.ENTITY -> entityToJsonObject(dsValue.get() as FullEntity<*>)
|
141
|
+
ValueType.KEY -> (dsValue.get() as Key).toString()
|
142
|
+
ValueType.LAT_LNG -> (dsValue.get() as LatLngValue).toString()
|
143
|
+
ValueType.LIST -> listToJsonArray(dsValue.get() as List<*>)
|
144
|
+
ValueType.LONG -> (dsValue.get() as Long).toString()
|
145
|
+
ValueType.NULL -> "null"
|
146
|
+
ValueType.RAW_VALUE -> (dsValue.get() as RawValue).toString()
|
147
|
+
ValueType.STRING -> "\"${dsValue.get() as String}\""
|
148
|
+
ValueType.TIMESTAMP -> (dsValue.get() as Timestamp).toString()
|
149
|
+
else -> null // NOTE: unexpected or unsupported type
|
150
|
+
}
|
151
|
+
}
|
111
152
|
}
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-input-datastore
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- syucream
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2018-06-
|
11
|
+
date: 2018-06-22 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -55,7 +55,7 @@ files:
|
|
55
55
|
- classpath/commons-codec-1.3.jar
|
56
56
|
- classpath/commons-logging-1.1.1.jar
|
57
57
|
- classpath/datastore-v1-proto-client-1.6.0.jar
|
58
|
-
- classpath/embulk-input-datastore-0.0.
|
58
|
+
- classpath/embulk-input-datastore-0.0.4.jar
|
59
59
|
- classpath/error_prone_annotations-2.2.0.jar
|
60
60
|
- classpath/gax-1.23.0.jar
|
61
61
|
- classpath/gax-httpjson-0.40.0.jar
|
@@ -90,8 +90,6 @@ files:
|
|
90
90
|
- classpath/protobuf-java-3.5.1.jar
|
91
91
|
- classpath/protobuf-java-util-3.5.1.jar
|
92
92
|
- classpath/threetenbp-1.3.3.jar
|
93
|
-
- embulk-input-datastore.gemspec
|
94
|
-
- embulk-input-datastore.iml
|
95
93
|
- examples/datastore2stdout.yaml
|
96
94
|
- gradle.properties
|
97
95
|
- lib/embulk/input/datastore.rb
|
@@ -1,18 +0,0 @@
|
|
1
|
-
Gem::Specification.new do |spec|
|
2
|
-
spec.name = "embulk-input-datastore"
|
3
|
-
spec.version = "0.0.3"
|
4
|
-
spec.authors = ["syucream"]
|
5
|
-
spec.summary = %[Datastore input plugin for Embulk]
|
6
|
-
spec.description = %[Loads records from datastore.]
|
7
|
-
spec.email = ["syucream1031@gmail.com"]
|
8
|
-
spec.licenses = ["MIT"]
|
9
|
-
spec.homepage = "https://github.com/syucream/embulk-input-datastore"
|
10
|
-
|
11
|
-
spec.files = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
|
12
|
-
spec.test_files = spec.files.grep(%r"^(test|spec)/")
|
13
|
-
spec.require_paths = ["lib"]
|
14
|
-
|
15
|
-
#spec.add_dependency 'YOUR_GEM_DEPENDENCY', ['~> YOUR_GEM_DEPENDENCY_VERSION']
|
16
|
-
spec.add_development_dependency 'bundler', ['~> 1.0']
|
17
|
-
spec.add_development_dependency 'rake', ['>= 10.0']
|
18
|
-
end
|
data/embulk-input-datastore.iml
DELETED
@@ -1,13 +0,0 @@
|
|
1
|
-
<?xml version="1.0" encoding="UTF-8"?>
|
2
|
-
<module external.linked.project.id="embulk-input-datastore" external.linked.project.path="$MODULE_DIR$" external.root.project.path="$MODULE_DIR$" external.system.id="GRADLE" external.system.module.group="" external.system.module.version="0.0.3" type="JAVA_MODULE" version="4">
|
3
|
-
<component name="NewModuleRootManager" inherit-compiler-output="true">
|
4
|
-
<exclude-output />
|
5
|
-
<content url="file://$MODULE_DIR$">
|
6
|
-
<excludeFolder url="file://$MODULE_DIR$/.gradle" />
|
7
|
-
<excludeFolder url="file://$MODULE_DIR$/build" />
|
8
|
-
<excludeFolder url="file://$MODULE_DIR$/out" />
|
9
|
-
</content>
|
10
|
-
<orderEntry type="inheritedJdk" />
|
11
|
-
<orderEntry type="sourceFolder" forTests="false" />
|
12
|
-
</component>
|
13
|
-
</module>
|