embulk-input-datastore 0.0.3 → 0.0.4

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 0418d992c45a1a40f72e4810dee870bd580daeb760f95f2725e436c401ffce25
4
- data.tar.gz: dddd42cfa92264e2a597258cc70c093c45d5b4f1cbd816264a6fcdb4d54999e9
3
+ metadata.gz: 15a06b07d8765e66681ac6636340eddae2bc84dcc15303d538c89db1619bbb1e
4
+ data.tar.gz: cdb37b7382f2921fb9fa351dba8f1d1a2abe18951f4f887003c11011c1bfced4
5
5
  SHA512:
6
- metadata.gz: 8558927f2a5eafa9c859a911fdd10447c43cd890013e47a9dc7c526d4b31a14d149aa9e8adb056a1eeaca63982233bd9757096f602131f72904fcfb22ae6620f
7
- data.tar.gz: a85c26e22745df865733a51f56560f58231db62168e36376264b6fee55ff624e431d1348609cd9281175a598ac68cc82ea66d62cde8e35d339aaff62d2b3328e
6
+ metadata.gz: e35e6c6d76da73a4e76d50b202cb80b157cc3b5308b5b159eb91e1c3c2ca9b1c43c83a9938b507a70fe7341a8d21af00d2f60cac72aa1d8ec810bbfad451f14c
7
+ data.tar.gz: 37a8123d2607064598eed79b02c714785f9ca93e331b481bd5802384b86ed9b1665f7368c9b81ba3b12853030b1b27da53847a81ad4f2c15919bb8a08054ea15
data/.gitignore CHANGED
@@ -9,3 +9,11 @@ build/
9
9
  /.metadata/
10
10
  .classpath
11
11
  .project
12
+
13
+ # RubyGems
14
+ *.gem
15
+ *.gemspec
16
+ *.iml
17
+
18
+ # development, debug
19
+ *.json
data/README.md CHANGED
@@ -39,5 +39,4 @@ $ ./gradlew gem # -t to watch change of files and rebuild continuously
39
39
 
40
40
  - Currently this plugin has below limitations:
41
41
  - Aggregate fetched properties to 1 'json' type column.
42
- - `ENTITY`, `KEY`, `LIST`, `LAT_LNG`, `NULL`, `RAW_VALUE` types are not supported.
43
42
 
@@ -1 +1 @@
1
- version=0.0.3
1
+ version=0.0.4
@@ -61,26 +61,10 @@ class DatastoreInputPlugin : InputPlugin {
61
61
  .forEach { entity ->
62
62
  logger.debug(entity.toString())
63
63
 
64
- // TODO separate/simplify generating JSON
65
- val pairs = entity.names.flatMap { name ->
66
- val dsValue = entity.getValue<Value<Any>>(name)
67
- val strVal: String? = when (dsValue.type) {
68
- ValueType.BLOB -> (dsValue.get() as ByteArray).toString()
69
- ValueType.BOOLEAN -> (dsValue.get() as Boolean).toString()
70
- ValueType.DOUBLE -> (dsValue.get() as Double).toString()
71
- ValueType.LONG -> (dsValue.get() as Long).toString()
72
- ValueType.STRING -> "\"${dsValue.get() as String}\""
73
- ValueType.TIMESTAMP -> (dsValue.get() as Timestamp).toString()
74
- else -> null // NOTE, TODO: LIST, ENTITY, ... is still unsupported
75
- }
76
- strVal?.let { listOf(Pair<String, String>(name, it)) } ?: listOf()
77
- }
78
- val json = "{" + pairs.map { pair ->
79
- "\"${pair.first}\": ${pair.second}"
80
- }.joinToString() + "}"
81
-
82
- val msgpackValue = ValueFactory.newString(json)
83
- pageBuilder.setJson(col, msgpackValue)
64
+ val json = entityToJsonObject(entity)
65
+ logger.debug(json)
66
+
67
+ pageBuilder.setJson(col, ValueFactory.newString(json))
84
68
  pageBuilder.addRecord()
85
69
  }
86
70
 
@@ -108,4 +92,61 @@ class DatastoreInputPlugin : InputPlugin {
108
92
  .build()
109
93
  .service
110
94
  }
95
+
96
+ /**
97
+ * Datastore entity -> JSON String
98
+ * e.g.) '{"name": "value", ...}'
99
+ *
100
+ */
101
+ private fun entityToJsonObject(entity: FullEntity<*>): String? {
102
+ val fields = entity.names.flatMap { name ->
103
+ val dsValue = entity.getValue<Value<Any>>(name)
104
+ val strVal = valueToField(dsValue)
105
+ strVal?.let { listOf("\"${name}\":${it}") } ?: listOf()
106
+ }
107
+ return "{" + fields.joinToString() + "}"
108
+ }
109
+
110
+ /**
111
+ * Datastore property value list -> JSON Array
112
+ * e.g.) '[1,2,3]'
113
+ *
114
+ */
115
+ private fun listToJsonArray(values: List<*>): String? {
116
+ // NOTE: Do unsafe cast because of type erasure...
117
+ val anyValues = values as? List<Value<Any>>
118
+
119
+ anyValues?.let {
120
+ val fields = it.flatMap { v ->
121
+ val strVal = valueToField(v)
122
+ strVal?.let { listOf(it) } ?: listOf()
123
+ }
124
+ return "[" + fields.joinToString() + "]"
125
+ }
126
+
127
+ return null
128
+ }
129
+
130
+ /**
131
+ * Datastore property value -> JSON field string
132
+ * e.g.) '"name":"value"'
133
+ *
134
+ */
135
+ private fun valueToField(dsValue: Value<Any>): String? {
136
+ return when (dsValue.type) {
137
+ ValueType.BLOB -> (dsValue.get() as ByteArray).toString()
138
+ ValueType.BOOLEAN -> (dsValue.get() as Boolean).toString()
139
+ ValueType.DOUBLE -> (dsValue.get() as Double).toString()
140
+ ValueType.ENTITY -> entityToJsonObject(dsValue.get() as FullEntity<*>)
141
+ ValueType.KEY -> (dsValue.get() as Key).toString()
142
+ ValueType.LAT_LNG -> (dsValue.get() as LatLngValue).toString()
143
+ ValueType.LIST -> listToJsonArray(dsValue.get() as List<*>)
144
+ ValueType.LONG -> (dsValue.get() as Long).toString()
145
+ ValueType.NULL -> "null"
146
+ ValueType.RAW_VALUE -> (dsValue.get() as RawValue).toString()
147
+ ValueType.STRING -> "\"${dsValue.get() as String}\""
148
+ ValueType.TIMESTAMP -> (dsValue.get() as Timestamp).toString()
149
+ else -> null // NOTE: unexpected or unsupported type
150
+ }
151
+ }
111
152
  }
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: embulk-input-datastore
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.3
4
+ version: 0.0.4
5
5
  platform: ruby
6
6
  authors:
7
7
  - syucream
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2018-06-21 00:00:00.000000000 Z
11
+ date: 2018-06-22 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
@@ -55,7 +55,7 @@ files:
55
55
  - classpath/commons-codec-1.3.jar
56
56
  - classpath/commons-logging-1.1.1.jar
57
57
  - classpath/datastore-v1-proto-client-1.6.0.jar
58
- - classpath/embulk-input-datastore-0.0.3.jar
58
+ - classpath/embulk-input-datastore-0.0.4.jar
59
59
  - classpath/error_prone_annotations-2.2.0.jar
60
60
  - classpath/gax-1.23.0.jar
61
61
  - classpath/gax-httpjson-0.40.0.jar
@@ -90,8 +90,6 @@ files:
90
90
  - classpath/protobuf-java-3.5.1.jar
91
91
  - classpath/protobuf-java-util-3.5.1.jar
92
92
  - classpath/threetenbp-1.3.3.jar
93
- - embulk-input-datastore.gemspec
94
- - embulk-input-datastore.iml
95
93
  - examples/datastore2stdout.yaml
96
94
  - gradle.properties
97
95
  - lib/embulk/input/datastore.rb
@@ -1,18 +0,0 @@
1
- Gem::Specification.new do |spec|
2
- spec.name = "embulk-input-datastore"
3
- spec.version = "0.0.3"
4
- spec.authors = ["syucream"]
5
- spec.summary = %[Datastore input plugin for Embulk]
6
- spec.description = %[Loads records from datastore.]
7
- spec.email = ["syucream1031@gmail.com"]
8
- spec.licenses = ["MIT"]
9
- spec.homepage = "https://github.com/syucream/embulk-input-datastore"
10
-
11
- spec.files = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
12
- spec.test_files = spec.files.grep(%r"^(test|spec)/")
13
- spec.require_paths = ["lib"]
14
-
15
- #spec.add_dependency 'YOUR_GEM_DEPENDENCY', ['~> YOUR_GEM_DEPENDENCY_VERSION']
16
- spec.add_development_dependency 'bundler', ['~> 1.0']
17
- spec.add_development_dependency 'rake', ['>= 10.0']
18
- end
@@ -1,13 +0,0 @@
1
- <?xml version="1.0" encoding="UTF-8"?>
2
- <module external.linked.project.id="embulk-input-datastore" external.linked.project.path="$MODULE_DIR$" external.root.project.path="$MODULE_DIR$" external.system.id="GRADLE" external.system.module.group="" external.system.module.version="0.0.3" type="JAVA_MODULE" version="4">
3
- <component name="NewModuleRootManager" inherit-compiler-output="true">
4
- <exclude-output />
5
- <content url="file://$MODULE_DIR$">
6
- <excludeFolder url="file://$MODULE_DIR$/.gradle" />
7
- <excludeFolder url="file://$MODULE_DIR$/build" />
8
- <excludeFolder url="file://$MODULE_DIR$/out" />
9
- </content>
10
- <orderEntry type="inheritedJdk" />
11
- <orderEntry type="sourceFolder" forTests="false" />
12
- </component>
13
- </module>