embulk-filter-expand_json 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +9 -0
- data/LICENSE.txt +21 -0
- data/README.md +48 -0
- data/build.gradle +75 -0
- data/classpath/asm-1.0.2.jar +0 -0
- data/classpath/asm-3.3.1.jar +0 -0
- data/classpath/embulk-filter-expand_json-0.0.1.jar +0 -0
- data/classpath/json-path-2.0.0.jar +0 -0
- data/classpath/json-smart-2.1.1.jar +0 -0
- data/example/config.yml +35 -0
- data/example/data.tsv +101 -0
- data/gradle/wrapper/gradle-wrapper.jar +0 -0
- data/gradle/wrapper/gradle-wrapper.properties +6 -0
- data/gradlew +164 -0
- data/gradlew.bat +90 -0
- data/lib/embulk/filter/expand_json.rb +3 -0
- data/src/main/java/org/embulk/filter/expand_json/ExpandJsonFilterPlugin.java +284 -0
- data/src/test/java/org/embulk/filter/expand_json/TestExpandJsonFilterPlugin.java +5 -0
- metadata +90 -0
checksums.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
---
|
|
2
|
+
SHA1:
|
|
3
|
+
metadata.gz: 5a19aef396456b82d996d4b61303e1fbc6f69752
|
|
4
|
+
data.tar.gz: cbb34a591fc08f2e10e5375ae61172be713e9f86
|
|
5
|
+
SHA512:
|
|
6
|
+
metadata.gz: b4dd221031f96058833f7a8609b81ba69c645df53eef281dfcbdb10f28055c897c062f839f6daead64bc2a40a8f5775a3101a93683b7779e3d927976db692af3
|
|
7
|
+
data.tar.gz: d6b96f43d2c49a96d312a4c9dc292f80f9d126fb8331a6293f720252f5c911b2e34c7c0f24c4aad9ea7bd51c38acaa7aab32667cd184fa00cd74ecbe0b9e1b20
|
data/.gitignore
ADDED
data/LICENSE.txt
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
|
|
2
|
+
MIT License
|
|
3
|
+
|
|
4
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
|
5
|
+
a copy of this software and associated documentation files (the
|
|
6
|
+
"Software"), to deal in the Software without restriction, including
|
|
7
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
|
8
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
|
9
|
+
permit persons to whom the Software is furnished to do so, subject to
|
|
10
|
+
the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be
|
|
13
|
+
included in all copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
|
16
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
|
17
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
|
18
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
|
19
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
|
20
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
|
21
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
# Expand Json filter plugin for Embulk
|
|
2
|
+
|
|
3
|
+
expand columns having json into multiple columns
|
|
4
|
+
|
|
5
|
+
## Overview
|
|
6
|
+
|
|
7
|
+
* **Plugin type**: filter
|
|
8
|
+
|
|
9
|
+
## Configuration
|
|
10
|
+
|
|
11
|
+
- **json_column_name**: a column name having json to be expanded (string, required)
|
|
12
|
+
- **expanded_columns**: columns expanded into multiple columns (array of hash, required)
|
|
13
|
+
- **name**: name of the column. you can define [JsonPath](http://goessner.net/articles/JsonPath/) style.
|
|
14
|
+
- **type**: type of the column (see below)
|
|
15
|
+
- **format**: format of the timestamp if type is timestamp
|
|
16
|
+
|
|
17
|
+
## Example
|
|
18
|
+
|
|
19
|
+
```yaml
|
|
20
|
+
filters:
|
|
21
|
+
- type: expand_json
|
|
22
|
+
json_column_name: json_payload
|
|
23
|
+
expanded_columns:
|
|
24
|
+
- {name: "$.phone_numbers", type: string}
|
|
25
|
+
- {name: "$.app_id", type: long}
|
|
26
|
+
- {name: "$.point", type: double}
|
|
27
|
+
- {name: "$.created_at", type: timestamp, format: "%Y-%m-%d"}
|
|
28
|
+
- {name: "$.profile.anniversary.et", type: string}
|
|
29
|
+
- {name: "$.profile.anniversary.voluptatem", type: string}
|
|
30
|
+
- {name: "$.profile.like_words[1]", type: string}
|
|
31
|
+
- {name: "$.profile.like_words[2]", type: string}
|
|
32
|
+
- {name: "$.profile.like_words[0]", type: string}
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
## Note
|
|
36
|
+
- If the value evaluated by JsonPath is Array or Hash, the value is written as JSON.
|
|
37
|
+
|
|
38
|
+
## Dependencies
|
|
39
|
+
- https://github.com/jayway/JsonPath
|
|
40
|
+
- use to evaluate [JsonPath](http://goessner.net/articles/JsonPath/)
|
|
41
|
+
- [Apache License Version 2.0](https://github.com/jayway/JsonPath/blob/master/LICENSE)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
## Build
|
|
45
|
+
|
|
46
|
+
```
|
|
47
|
+
$ ./gradlew gem # -t to watch change of files and rebuild continuously
|
|
48
|
+
```
|
data/build.gradle
ADDED
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
plugins {
|
|
2
|
+
id "com.jfrog.bintray" version "1.1"
|
|
3
|
+
id "com.github.jruby-gradle.base" version "0.1.5"
|
|
4
|
+
id "java"
|
|
5
|
+
}
|
|
6
|
+
import com.github.jrubygradle.JRubyExec
|
|
7
|
+
repositories {
|
|
8
|
+
mavenCentral()
|
|
9
|
+
jcenter()
|
|
10
|
+
}
|
|
11
|
+
configurations {
|
|
12
|
+
provided
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
version = "0.0.1"
|
|
16
|
+
sourceCompatibility = 1.7
|
|
17
|
+
targetCompatibility = 1.7
|
|
18
|
+
|
|
19
|
+
dependencies {
|
|
20
|
+
compile "org.embulk:embulk-core:0.7.4"
|
|
21
|
+
provided "org.embulk:embulk-core:0.7.4"
|
|
22
|
+
compile "com.jayway.jsonpath:json-path:2.+"
|
|
23
|
+
testCompile "junit:junit:4.+"
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
task classpath(type: Copy, dependsOn: ["jar"]) {
|
|
27
|
+
doFirst { file("classpath").deleteDir() }
|
|
28
|
+
from (configurations.runtime - configurations.provided + files(jar.archivePath))
|
|
29
|
+
into "classpath"
|
|
30
|
+
}
|
|
31
|
+
clean { delete "classpath" }
|
|
32
|
+
|
|
33
|
+
task gem(type: JRubyExec, dependsOn: ["gemspec", "classpath"]) {
|
|
34
|
+
jrubyArgs "-rrubygems/gem_runner", "-eGem::GemRunner.new.run(ARGV)", "build"
|
|
35
|
+
script "${project.name}.gemspec"
|
|
36
|
+
doLast { ant.move(file: "${project.name}-${project.version}.gem", todir: "pkg") }
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
task gemPush(type: JRubyExec, dependsOn: ["gem"]) {
|
|
40
|
+
jrubyArgs "-rrubygems/gem_runner", "-eGem::GemRunner.new.run(ARGV)", "push"
|
|
41
|
+
script "pkg/${project.name}-${project.version}.gem"
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
task "package"(dependsOn: ["gemspec", "classpath"]) << {
|
|
45
|
+
println "> Build succeeded."
|
|
46
|
+
println "> You can run embulk with '-L ${file(".").absolutePath}' argument."
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
task gemspec {
|
|
50
|
+
ext.gemspecFile = file("${project.name}.gemspec")
|
|
51
|
+
inputs.file "build.gradle"
|
|
52
|
+
outputs.file gemspecFile
|
|
53
|
+
doLast { gemspecFile.write($/
|
|
54
|
+
Gem::Specification.new do |spec|
|
|
55
|
+
spec.name = "${project.name}"
|
|
56
|
+
spec.version = "${project.version}"
|
|
57
|
+
spec.authors = ["Civitaspo"]
|
|
58
|
+
spec.summary = %[Expand Json filter plugin for Embulk]
|
|
59
|
+
spec.description = %[Expand Json]
|
|
60
|
+
spec.email = ["civitaspo@gmail.com"]
|
|
61
|
+
spec.licenses = ["MIT"]
|
|
62
|
+
spec.homepage = "https://github.com/civitaspo/embulk-filter-expand_json"
|
|
63
|
+
|
|
64
|
+
spec.files = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
|
|
65
|
+
spec.test_files = spec.files.grep(%r"^(test|spec)/")
|
|
66
|
+
spec.require_paths = ["lib"]
|
|
67
|
+
|
|
68
|
+
#spec.add_dependency 'YOUR_GEM_DEPENDENCY', ['~> YOUR_GEM_DEPENDENCY_VERSION']
|
|
69
|
+
spec.add_development_dependency 'bundler', ['~> 1.0']
|
|
70
|
+
spec.add_development_dependency 'rake', ['>= 10.0']
|
|
71
|
+
end
|
|
72
|
+
/$)
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
clean { delete "${project.name}.gemspec" }
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
data/example/config.yml
ADDED
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
in:
|
|
2
|
+
type: file
|
|
3
|
+
path_prefix: example/data.tsv
|
|
4
|
+
parser:
|
|
5
|
+
type: csv
|
|
6
|
+
delimiter: "\t"
|
|
7
|
+
charset: UTF-8
|
|
8
|
+
newline: CRLF
|
|
9
|
+
null_string: 'NULL'
|
|
10
|
+
skip_header_lines: 1
|
|
11
|
+
comment_line_marker: '#'
|
|
12
|
+
columns:
|
|
13
|
+
- {name: time, type: timestamp, format: "%Y-%m-%d"}
|
|
14
|
+
- {name: id, type: long}
|
|
15
|
+
- {name: name, type: string}
|
|
16
|
+
- {name: score, type: double}
|
|
17
|
+
- {name: json_payload, type: string}
|
|
18
|
+
|
|
19
|
+
filters:
|
|
20
|
+
- type: expand_json
|
|
21
|
+
json_column_name: json_payload
|
|
22
|
+
expanded_columns:
|
|
23
|
+
- {name: "$.phone_numbers", type: string}
|
|
24
|
+
- {name: "$.app_id", type: long}
|
|
25
|
+
- {name: "$.point", type: double}
|
|
26
|
+
- {name: "$.created_at", type: timestamp, format: "%Y-%m-%d"}
|
|
27
|
+
- {name: "$.profile.anniversary.et", type: string}
|
|
28
|
+
- {name: "$.profile.anniversary", type: string}
|
|
29
|
+
- {name: "$.profile.like_words[1]", type: string}
|
|
30
|
+
- {name: "$.profile.like_words[2]", type: string}
|
|
31
|
+
- {name: "$.profile.like_words", type: string}
|
|
32
|
+
|
|
33
|
+
out:
|
|
34
|
+
type: stdout
|
|
35
|
+
|
data/example/data.tsv
ADDED
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
time id name score json_payload
|
|
2
|
+
2015-10-11 08:06:23 +0900 0 Avis Lind -3256.869635206057 {"phone_numbers":"1-276-220-7263","app_id":0,"point":-1601.6890336884562,"created_at":"2015-10-07 20:23:57 +0900","profile":{"like_words":["maiores","eum","aut"],"anniversary":{"voluptatem":"dolor","et":"ullam"}}}
|
|
3
|
+
2015-10-10 20:10:32 +0900 1 Edythe Bartell 1854.475677760671 {"phone_numbers":"553.980.4072","app_id":9,"point":119.36847814392212,"created_at":"2015-10-06 03:43:52 +0900","profile":{"like_words":["nobis","ad","est"],"anniversary":{"rerum":"inventore","deleniti":"numquam"}}}
|
|
4
|
+
2015-10-09 18:57:11 +0900 2 Mabel Kreiger Sr. -2930.0214667621676 {"phone_numbers":"267-437-9081","app_id":14,"point":-272.41081832483815,"created_at":"2015-10-07 11:53:53 +0900","profile":{"like_words":["itaque","aut","in"],"anniversary":{"eveniet":"in","id":"sit"}}}
|
|
5
|
+
2015-10-11 08:34:20 +0900 3 Jeffry Schowalter 3722.368084343345 {"phone_numbers":"639.217.7325","app_id":0,"point":3928.1109653627186,"created_at":"2015-10-09 08:43:59 +0900","profile":{"like_words":["a","molestiae","iure"],"anniversary":{"non":"harum","dolorem":"provident"}}}
|
|
6
|
+
2015-10-08 02:46:53 +0900 4 Erwin Cole -530.0773717850907 {"phone_numbers":"590.289.2473","app_id":36,"point":-4153.016432643382,"created_at":"2015-10-06 13:15:47 +0900","profile":{"like_words":["quisquam","quasi","a"],"anniversary":{"ducimus":"veritatis","vel":"in"}}}
|
|
7
|
+
2015-10-07 17:27:11 +0900 5 Herman Murazik I 2858.9098009497147 {"phone_numbers":"(196) 116-8976","app_id":0,"point":852.1366833369701,"created_at":"2015-10-09 11:06:36 +0900","profile":{"like_words":["maxime","ad","sunt"],"anniversary":{"molestiae":"architecto","temporibus":"quia"}}}
|
|
8
|
+
2015-10-06 10:33:22 +0900 6 Kaley Fritsch -4170.738141376422 {"phone_numbers":"597.359.0615","app_id":48,"point":-2299.7034880139254,"created_at":"2015-10-07 08:14:01 +0900","profile":{"like_words":["eum","dolor","beatae"],"anniversary":{"fugit":"incidunt","provident":"doloremque"}}}
|
|
9
|
+
2015-10-06 18:52:38 +0900 7 Dr. Everett Gottlieb 3025.8442146041184 {"phone_numbers":"1-108-635-0095","app_id":28,"point":3999.8707818258176,"created_at":"2015-10-11 14:53:52 +0900","profile":{"like_words":["ea","minus","sit"],"anniversary":{"accusamus":"voluptate","omnis":"odio"}}}
|
|
10
|
+
2015-10-10 19:03:47 +0900 8 Freda Lehner -2834.632764837081 {"phone_numbers":"282.908.3908","app_id":48,"point":-2690.3089130787753,"created_at":"2015-10-11 21:20:47 +0900","profile":{"like_words":["voluptas","consequatur","occaecati"],"anniversary":{"delectus":"ipsa","dolorem":"qui"}}}
|
|
11
|
+
2015-10-10 15:37:57 +0900 9 Brennon Zboncak 206.72213107748473 {"phone_numbers":"1-397-425-1652","app_id":63,"point":989.609974671026,"created_at":"2015-10-06 13:00:24 +0900","profile":{"like_words":["facere","optio","veniam"],"anniversary":{"perferendis":"quas","cupiditate":"est"}}}
|
|
12
|
+
2015-10-07 20:18:52 +0900 10 Rudolph Predovic DVM -3884.3771566451423 {"phone_numbers":"1-389-001-8529","app_id":20,"point":-3669.9152918427744,"created_at":"2015-10-06 03:31:58 +0900","profile":{"like_words":["accusantium","qui","iure"],"anniversary":{"eaque":"possimus","et":"cumque"}}}
|
|
13
|
+
2015-10-09 02:05:22 +0900 11 Etha Hirthe 2467.945406584426 {"phone_numbers":"315-155-2339","app_id":22,"point":4428.215301393835,"created_at":"2015-10-07 18:11:24 +0900","profile":{"like_words":["et","ea","ratione"],"anniversary":{"dolorem":"est","sunt":"consequatur"}}}
|
|
14
|
+
2015-10-07 23:52:31 +0900 12 Sallie Ankunding V -3441.0370623067192 {"phone_numbers":"681-141-4345","app_id":60,"point":-1071.645070483149,"created_at":"2015-10-11 00:29:33 +0900","profile":{"like_words":["alias","facere","sint"],"anniversary":{"molestiae":"provident","sapiente":"rerum"}}}
|
|
15
|
+
2015-10-06 03:04:20 +0900 13 Mrs. Miguel Hermann 1521.8619752514212 {"phone_numbers":"329.399.9089","app_id":13,"point":970.1380743174367,"created_at":"2015-10-06 04:11:03 +0900","profile":{"like_words":["aut","voluptatem","nesciunt"],"anniversary":{"soluta":"asperiores","et":"et"}}}
|
|
16
|
+
2015-10-10 10:55:46 +0900 14 Mrs. Magdalen Swift -1484.8405564949508 {"phone_numbers":"(941) 031-6216","app_id":0,"point":-4218.049682365648,"created_at":"2015-10-11 13:00:43 +0900","profile":{"like_words":["voluptatum","ut","saepe"],"anniversary":{"et":"dolores","quas":"aliquid"}}}
|
|
17
|
+
2015-10-06 13:00:38 +0900 15 Toy Homenick 2015.2453965026514 {"phone_numbers":"1-551-393-8752","app_id":105,"point":2651.6556647094835,"created_at":"2015-10-07 17:41:27 +0900","profile":{"like_words":["doloremque","ut","nobis"],"anniversary":{"est":"repudiandae","unde":"libero"}}}
|
|
18
|
+
2015-10-06 00:51:49 +0900 16 Lorena Boehm V -2048.529343446842 {"phone_numbers":"546-693-6731","app_id":48,"point":-1050.3257415942394,"created_at":"2015-10-11 21:47:37 +0900","profile":{"like_words":["perferendis","possimus","ducimus"],"anniversary":{"veniam":"consequatur","similique":"modi"}}}
|
|
19
|
+
2015-10-07 06:41:37 +0900 17 Kenya Beer 4984.333461295299 {"phone_numbers":"1-818-486-1184","app_id":68,"point":3066.8254748764393,"created_at":"2015-10-10 17:38:04 +0900","profile":{"like_words":["voluptatem","vero","ipsam"],"anniversary":{"aut":"est","et":"qui"}}}
|
|
20
|
+
2015-10-10 13:15:14 +0900 18 Mrs. Lizeth Hegmann -3976.8683488150036 {"phone_numbers":"1-319-160-6462","app_id":18,"point":-4500.0980046958375,"created_at":"2015-10-11 01:39:27 +0900","profile":{"like_words":["ipsa","vitae","rerum"],"anniversary":{"et":"dolor","harum":"maxime"}}}
|
|
21
|
+
2015-10-09 05:28:56 +0900 19 Cole Rogahn 813.9059489639924 {"phone_numbers":"(549) 254-5105","app_id":19,"point":548.1637714807997,"created_at":"2015-10-08 07:26:01 +0900","profile":{"like_words":["odio","assumenda","aut"],"anniversary":{"aut":"et","odit":"dignissimos"}}}
|
|
22
|
+
2015-10-09 03:37:12 +0900 20 Mara Wilkinson -830.4013843109051 {"phone_numbers":"1-757-588-7549","app_id":0,"point":-1047.767739409654,"created_at":"2015-10-10 22:34:01 +0900","profile":{"like_words":["et","possimus","maiores"],"anniversary":{"atque":"facilis","neque":"velit"}}}
|
|
23
|
+
2015-10-09 21:31:42 +0900 21 Keenan Stiedemann 3953.0008354444362 {"phone_numbers":"1-874-055-1495","app_id":21,"point":891.1815102146321,"created_at":"2015-10-06 08:06:09 +0900","profile":{"like_words":["quae","suscipit","magni"],"anniversary":{"et":"est","quidem":"beatae"}}}
|
|
24
|
+
2015-10-09 01:24:29 +0900 22 Rhett O'Connell -1551.4589651540823 {"phone_numbers":"372-205-6420","app_id":88,"point":-2007.701257408186,"created_at":"2015-10-07 23:46:19 +0900","profile":{"like_words":["unde","vitae","fugiat"],"anniversary":{"vero":"architecto","temporibus":"omnis"}}}
|
|
25
|
+
2015-10-06 11:52:47 +0900 23 Donavon Kemmer 508.50547593994753 {"phone_numbers":"(571) 832-1878","app_id":138,"point":833.8453806892851,"created_at":"2015-10-07 14:56:33 +0900","profile":{"like_words":["consequuntur","ad","suscipit"],"anniversary":{"dolorem":"ullam","voluptas":"possimus"}}}
|
|
26
|
+
2015-10-06 16:23:49 +0900 24 Miss Courtney Grady -2507.776567206154 {"phone_numbers":"251-829-7488","app_id":144,"point":-4447.923967878997,"created_at":"2015-10-10 11:55:44 +0900","profile":{"like_words":["placeat","architecto","quis"],"anniversary":{"aspernatur":"dignissimos","voluptatem":"ut"}}}
|
|
27
|
+
2015-10-09 06:31:34 +0900 25 Wilton Kovacek 4423.649136977761 {"phone_numbers":"(681) 080-0266","app_id":100,"point":4982.551433162526,"created_at":"2015-10-09 03:11:29 +0900","profile":{"like_words":["architecto","vitae","quaerat"],"anniversary":{"voluptas":"nisi","distinctio":"pariatur"}}}
|
|
28
|
+
2015-10-07 09:01:23 +0900 26 Liam McKenzie DDS -3278.90565357996 {"phone_numbers":"312.196.5744","app_id":104,"point":-629.3130441062467,"created_at":"2015-10-11 02:12:56 +0900","profile":{"like_words":["aut","laudantium","maxime"],"anniversary":{"maxime":"maiores","fugit":"minima"}}}
|
|
29
|
+
2015-10-07 18:22:45 +0900 27 Conrad Hansen 593.543895102616 {"phone_numbers":"169-971-7381","app_id":27,"point":4300.594324371474,"created_at":"2015-10-11 07:46:33 +0900","profile":{"like_words":["voluptas","eveniet","necessitatibus"],"anniversary":{"ut":"similique","tempore":"tempora"}}}
|
|
30
|
+
2015-10-09 22:14:03 +0900 28 Dulce Robel -3084.639911197776 {"phone_numbers":"978.630.7207","app_id":0,"point":-4949.382424690316,"created_at":"2015-10-10 02:20:54 +0900","profile":{"like_words":["quibusdam","maxime","aut"],"anniversary":{"hic":"earum","enim":"consequatur"}}}
|
|
31
|
+
2015-10-07 10:26:01 +0900 29 Carole Sanford 2730.8409556862894 {"phone_numbers":"362.464.7784","app_id":261,"point":2011.1349214830198,"created_at":"2015-10-07 11:20:30 +0900","profile":{"like_words":["distinctio","ipsa","voluptatibus"],"anniversary":{"natus":"officiis","ea":"nulla"}}}
|
|
32
|
+
2015-10-09 21:59:30 +0900 30 Sigrid Herman -810.1022044437887 {"phone_numbers":"(924) 541-4784","app_id":90,"point":-2423.8288326034376,"created_at":"2015-10-10 09:21:38 +0900","profile":{"like_words":["optio","magni","incidunt"],"anniversary":{"consectetur":"cumque","quaerat":"inventore"}}}
|
|
33
|
+
2015-10-11 22:53:45 +0900 31 Rashawn Schneider 4008.851010128666 {"phone_numbers":"173.498.5107","app_id":186,"point":3874.7826562728574,"created_at":"2015-10-06 16:01:11 +0900","profile":{"like_words":["consectetur","rerum","perspiciatis"],"anniversary":{"autem":"id","fugit":"non"}}}
|
|
34
|
+
2015-10-11 08:10:48 +0900 32 Astrid Steuber V -204.8991328158727 {"phone_numbers":"(481) 013-9814","app_id":64,"point":-2263.549947509334,"created_at":"2015-10-06 03:30:09 +0900","profile":{"like_words":["nihil","ut","voluptas"],"anniversary":{"quaerat":"illum","soluta":"minus"}}}
|
|
35
|
+
2015-10-10 14:00:27 +0900 33 Diamond Zemlak 778.1421594249105 {"phone_numbers":"(595) 704-7927","app_id":99,"point":3824.9185606303363,"created_at":"2015-10-09 13:47:39 +0900","profile":{"like_words":["facilis","qui","quis"],"anniversary":{"ea":"numquam","autem":"voluptatum"}}}
|
|
36
|
+
2015-10-10 03:07:36 +0900 34 Mrs. Shannon Beier -2070.751811643287 {"phone_numbers":"532.502.9217","app_id":102,"point":-13.483132626333827,"created_at":"2015-10-06 20:31:09 +0900","profile":{"like_words":["rerum","voluptas","et"],"anniversary":{"omnis":"dolores","ducimus":"atque"}}}
|
|
37
|
+
2015-10-07 04:55:02 +0900 35 Andreanne Koepp 445.5461896463571 {"phone_numbers":"(844) 872-7282","app_id":70,"point":2024.968843940635,"created_at":"2015-10-09 14:37:41 +0900","profile":{"like_words":["enim","qui","debitis"],"anniversary":{"non":"quidem","omnis":"hic"}}}
|
|
38
|
+
2015-10-09 06:09:27 +0900 36 Miss Treva Kshlerin -3488.4838054000684 {"phone_numbers":"829.803.9509","app_id":144,"point":-4712.258490535556,"created_at":"2015-10-07 11:41:49 +0900","profile":{"like_words":["maxime","rem","in"],"anniversary":{"enim":"alias","ut":"placeat"}}}
|
|
39
|
+
2015-10-11 05:04:39 +0900 37 Rex Daniel 2923.4043406285004 {"phone_numbers":"401.582.6263","app_id":0,"point":2040.6041036630954,"created_at":"2015-10-11 16:58:03 +0900","profile":{"like_words":["praesentium","id","culpa"],"anniversary":{"est":"debitis","sequi":"impedit"}}}
|
|
40
|
+
2015-10-10 03:30:04 +0900 38 Mr. Federico Konopelski -2314.3806869025543 {"phone_numbers":"800.586.7396","app_id":114,"point":-2390.1152723977416,"created_at":"2015-10-06 05:28:07 +0900","profile":{"like_words":["adipisci","unde","doloremque"],"anniversary":{"autem":"ex","tenetur":"laborum"}}}
|
|
41
|
+
2015-10-08 16:42:46 +0900 39 Amelia Sanford 1821.125565223592 {"phone_numbers":"(319) 427-8467","app_id":39,"point":4219.58767055414,"created_at":"2015-10-06 04:58:02 +0900","profile":{"like_words":["ducimus","similique","totam"],"anniversary":{"similique":"non","suscipit":"aut"}}}
|
|
42
|
+
2015-10-07 19:00:00 +0900 40 Zackary Stamm V -2937.7888199272 {"phone_numbers":"640.281.5280","app_id":0,"point":-3506.017079901764,"created_at":"2015-10-07 13:56:23 +0900","profile":{"like_words":["reiciendis","veritatis","magni"],"anniversary":{"molestiae":"qui","doloremque":"ea"}}}
|
|
43
|
+
2015-10-09 10:09:57 +0900 41 Arno Corwin V 4151.12660713617 {"phone_numbers":"122.846.7302","app_id":123,"point":3603.5557674915985,"created_at":"2015-10-10 12:45:04 +0900","profile":{"like_words":["perspiciatis","dolorem","similique"],"anniversary":{"necessitatibus":"veritatis","mollitia":"ex"}}}
|
|
44
|
+
2015-10-08 23:32:15 +0900 42 Marcus Wintheiser -1638.369431851208 {"phone_numbers":"804.338.5423","app_id":0,"point":-4114.22951753173,"created_at":"2015-10-06 01:23:22 +0900","profile":{"like_words":["est","atque","inventore"],"anniversary":{"sit":"et","quam":"ad"}}}
|
|
45
|
+
2015-10-09 14:31:12 +0900 43 Madie Lebsack 3229.2300283806267 {"phone_numbers":"1-867-730-9730","app_id":43,"point":1702.921617466058,"created_at":"2015-10-09 03:38:09 +0900","profile":{"like_words":["rerum","eos","totam"],"anniversary":{"ducimus":"voluptatem","beatae":"cupiditate"}}}
|
|
46
|
+
2015-10-10 15:12:16 +0900 44 Dean Bernier -2688.708370133052 {"phone_numbers":"(481) 698-2615","app_id":88,"point":-3584.314291557972,"created_at":"2015-10-09 20:36:48 +0900","profile":{"like_words":["ut","deleniti","eum"],"anniversary":{"explicabo":"labore","odit":"ratione"}}}
|
|
47
|
+
2015-10-08 08:29:38 +0900 45 Tevin Yundt 4277.146563063783 {"phone_numbers":"1-658-587-2827","app_id":45,"point":502.9804730517532,"created_at":"2015-10-09 01:40:38 +0900","profile":{"like_words":["excepturi","libero","ea"],"anniversary":{"amet":"culpa","voluptatibus":"asperiores"}}}
|
|
48
|
+
2015-10-11 17:16:34 +0900 46 Herta Ortiz -2586.2488189026335 {"phone_numbers":"196.022.7761","app_id":138,"point":-907.3082021064893,"created_at":"2015-10-09 21:57:16 +0900","profile":{"like_words":["ex","quod","possimus"],"anniversary":{"voluptatem":"quas","accusamus":"accusamus"}}}
|
|
49
|
+
2015-10-09 06:14:44 +0900 47 Marjory Little IV 2817.1542796692734 {"phone_numbers":"709.190.6695","app_id":141,"point":794.8142102417938,"created_at":"2015-10-07 13:08:57 +0900","profile":{"like_words":["beatae","voluptatem","cumque"],"anniversary":{"laboriosam":"molestiae","aut":"occaecati"}}}
|
|
50
|
+
2015-10-08 18:09:02 +0900 48 Tyrese Cartwright -2616.4545222758784 {"phone_numbers":"(810) 673-9851","app_id":336,"point":-3724.479309186211,"created_at":"2015-10-09 02:43:59 +0900","profile":{"like_words":["id","odio","aut"],"anniversary":{"necessitatibus":"soluta","dignissimos":"impedit"}}}
|
|
51
|
+
2015-10-11 06:09:34 +0900 49 Clarissa Weissnat 3774.246752462179 {"phone_numbers":"258-706-2034","app_id":196,"point":729.5259400859024,"created_at":"2015-10-08 15:42:59 +0900","profile":{"like_words":["magnam","iusto","debitis"],"anniversary":{"assumenda":"et","et":"mollitia"}}}
|
|
52
|
+
2015-10-08 06:45:45 +0900 50 Mariane Price PhD -3731.4091942932796 {"phone_numbers":"549-264-4418","app_id":0,"point":-2212.062843157164,"created_at":"2015-10-10 20:49:30 +0900","profile":{"like_words":["nemo","blanditiis","consequatur"],"anniversary":{"sequi":"provident","quibusdam":"quae"}}}
|
|
53
|
+
2015-10-06 20:46:55 +0900 51 Magdalena Hermiston 170.33149319511773 {"phone_numbers":"295.978.5000","app_id":102,"point":3964.5357515997202,"created_at":"2015-10-08 10:50:54 +0900","profile":{"like_words":["ipsam","vitae","nisi"],"anniversary":{"temporibus":"dolorem","in":"et"}}}
|
|
54
|
+
2015-10-06 14:29:30 +0900 52 Emmitt Grady -3071.6451250949503 {"phone_numbers":"(813) 682-7065","app_id":52,"point":-2283.8875024186896,"created_at":"2015-10-07 07:14:17 +0900","profile":{"like_words":["labore","sunt","modi"],"anniversary":{"ipsum":"praesentium","dolorem":"id"}}}
|
|
55
|
+
2015-10-10 09:51:25 +0900 53 Shawna Marquardt 3829.272519651845 {"phone_numbers":"(119) 598-2279","app_id":0,"point":660.632964450477,"created_at":"2015-10-07 19:18:39 +0900","profile":{"like_words":["quis","dolorem","nihil"],"anniversary":{"perspiciatis":"cumque","et":"at"}}}
|
|
56
|
+
2015-10-10 07:59:06 +0900 54 Nyah Spencer -4975.475925277694 {"phone_numbers":"1-680-025-9126","app_id":54,"point":-853.6538280935893,"created_at":"2015-10-10 22:05:35 +0900","profile":{"like_words":["illo","possimus","dolorum"],"anniversary":{"aut":"nesciunt","autem":"aut"}}}
|
|
57
|
+
2015-10-09 16:44:59 +0900 55 Chloe Boehm 1187.3310695751716 {"phone_numbers":"1-119-603-7565","app_id":165,"point":3202.1011518741584,"created_at":"2015-10-10 07:50:34 +0900","profile":{"like_words":["ut","quam","earum"],"anniversary":{"odit":"praesentium","veritatis":"optio"}}}
|
|
58
|
+
2015-10-11 16:42:16 +0900 56 Elwyn Mills III -1226.6165125104358 {"phone_numbers":"462.846.0237","app_id":168,"point":-1469.7575928467445,"created_at":"2015-10-11 20:01:50 +0900","profile":{"like_words":["quod","eum","deserunt"],"anniversary":{"facilis":"ut","aliquid":"omnis"}}}
|
|
59
|
+
2015-10-08 08:15:12 +0900 57 Lennie Bartoletti 4887.712523033574 {"phone_numbers":"(562) 700-6908","app_id":114,"point":160.15018517562513,"created_at":"2015-10-09 06:54:46 +0900","profile":{"like_words":["nostrum","odit","enim"],"anniversary":{"autem":"soluta","fugit":"dolore"}}}
|
|
60
|
+
2015-10-07 13:14:15 +0900 58 Deontae Barrows -4373.736508737718 {"phone_numbers":"1-545-199-6620","app_id":522,"point":-1080.4664896167392,"created_at":"2015-10-08 08:30:08 +0900","profile":{"like_words":["repellendus","aut","quo"],"anniversary":{"mollitia":"ea","fugit":"pariatur"}}}
|
|
61
|
+
2015-10-07 07:14:24 +0900 59 Nyah Bauch 627.944245639318 {"phone_numbers":"649-754-2638","app_id":118,"point":2050.633129915665,"created_at":"2015-10-09 18:28:14 +0900","profile":{"like_words":["et","dicta","minima"],"anniversary":{"ullam":"modi","nam":"adipisci"}}}
|
|
62
|
+
2015-10-06 06:53:13 +0900 60 Adrienne Leffler -2829.521176150057 {"phone_numbers":"814-630-0851","app_id":180,"point":-3464.7800151417896,"created_at":"2015-10-09 18:09:45 +0900","profile":{"like_words":["labore","ut","deserunt"],"anniversary":{"sed":"quas","voluptatem":"quibusdam"}}}
|
|
63
|
+
2015-10-08 12:48:48 +0900 61 Kayley Keebler 1313.7250089243412 {"phone_numbers":"(807) 953-6933","app_id":305,"point":639.2313363059397,"created_at":"2015-10-07 15:49:52 +0900","profile":{"like_words":["deleniti","labore","dolores"],"anniversary":{"eum":"qui","commodi":"fugiat"}}}
|
|
64
|
+
2015-10-10 06:48:27 +0900 62 Elliot Bogisich V -3822.8090142643587 {"phone_numbers":"929-024-2951","app_id":124,"point":-4779.516962141838,"created_at":"2015-10-10 14:56:08 +0900","profile":{"like_words":["ut","assumenda","voluptas"],"anniversary":{"quo":"ut","eum":"et"}}}
|
|
65
|
+
2015-10-08 00:16:51 +0900 63 Eveline Ernser Sr. 2806.9237318787023 {"phone_numbers":"(502) 877-8577","app_id":189,"point":1892.9853939420839,"created_at":"2015-10-07 05:00:21 +0900","profile":{"like_words":["sit","ipsam","fugiat"],"anniversary":{"vitae":"sit","sint":"quis"}}}
|
|
66
|
+
2015-10-11 08:38:18 +0900 64 Murl Labadie -764.140117568465 {"phone_numbers":"(771) 349-7033","app_id":320,"point":-687.4013382271833,"created_at":"2015-10-10 10:51:23 +0900","profile":{"like_words":["veritatis","aut","labore"],"anniversary":{"et":"consequatur","ut":"soluta"}}}
|
|
67
|
+
2015-10-08 15:05:42 +0900 65 Lacy Cassin 1160.320306512302 {"phone_numbers":"624.888.1685","app_id":325,"point":1587.6076807015297,"created_at":"2015-10-11 19:29:20 +0900","profile":{"like_words":["expedita","a","neque"],"anniversary":{"assumenda":"officia","id":"sed"}}}
|
|
68
|
+
2015-10-10 16:18:19 +0900 66 Maeve Koss -4335.150343119076 {"phone_numbers":"157-152-7236","app_id":330,"point":-4349.117443627459,"created_at":"2015-10-11 06:56:13 +0900","profile":{"like_words":["autem","voluptatem","doloremque"],"anniversary":{"voluptatem":"sit","dolor":"est"}}}
|
|
69
|
+
2015-10-11 08:35:36 +0900 67 Curtis Batz 2736.540517850057 {"phone_numbers":"975.333.7068","app_id":268,"point":3420.202242838706,"created_at":"2015-10-06 13:56:47 +0900","profile":{"like_words":["voluptates","labore","sequi"],"anniversary":{"voluptas":"aspernatur","dolore":"ea"}}}
|
|
70
|
+
2015-10-07 16:42:19 +0900 68 Derek Reichel -3874.881120555271 {"phone_numbers":"973.840.0336","app_id":68,"point":-4717.135607889753,"created_at":"2015-10-10 01:59:17 +0900","profile":{"like_words":["error","quia","molestiae"],"anniversary":{"dolorem":"quis","consectetur":"distinctio"}}}
|
|
71
|
+
2015-10-09 11:19:49 +0900 69 Palma Fisher 496.81518082186113 {"phone_numbers":"1-309-349-3740","app_id":207,"point":3390.1861946268727,"created_at":"2015-10-11 14:29:12 +0900","profile":{"like_words":["ut","ut","voluptates"],"anniversary":{"saepe":"et","enim":"eaque"}}}
|
|
72
|
+
2015-10-08 13:14:09 +0900 70 Clint Reinger III -3038.4356427769153 {"phone_numbers":"856-835-7730","app_id":140,"point":-2835.2053598307175,"created_at":"2015-10-06 14:17:51 +0900","profile":{"like_words":["impedit","ad","animi"],"anniversary":{"odit":"est","vel":"voluptate"}}}
|
|
73
|
+
2015-10-10 17:44:37 +0900 71 Kitty Champlin 4560.4401675187055 {"phone_numbers":"989.638.1296","app_id":71,"point":1253.3143846421438,"created_at":"2015-10-07 21:30:03 +0900","profile":{"like_words":["voluptas","culpa","similique"],"anniversary":{"dolor":"neque","placeat":"est"}}}
|
|
74
|
+
2015-10-07 11:22:37 +0900 72 Mr. Ena Harvey -4200.1942368361515 {"phone_numbers":"317.786.9814","app_id":72,"point":-722.8696017492975,"created_at":"2015-10-08 08:13:59 +0900","profile":{"like_words":["adipisci","enim","voluptas"],"anniversary":{"expedita":"et","voluptatem":"rerum"}}}
|
|
75
|
+
2015-10-09 00:05:19 +0900 73 Dr. Anderson Wolff 3866.7896808354963 {"phone_numbers":"1-343-104-2725","app_id":584,"point":194.1226183271141,"created_at":"2015-10-06 19:39:22 +0900","profile":{"like_words":["quam","dolorum","est"],"anniversary":{"vel":"id","ut":"nulla"}}}
|
|
76
|
+
2015-10-10 11:48:21 +0900 74 Dorris Walsh IV -1961.8325062759172 {"phone_numbers":"(509) 183-0884","app_id":592,"point":-830.865372367256,"created_at":"2015-10-11 16:07:06 +0900","profile":{"like_words":["repellendus","quod","aut"],"anniversary":{"voluptas":"ut","recusandae":"numquam"}}}
|
|
77
|
+
2015-10-09 16:51:12 +0900 75 Justus Kuhic 423.7072330426625 {"phone_numbers":"(774) 762-5269","app_id":450,"point":3579.5815894513685,"created_at":"2015-10-08 09:31:56 +0900","profile":{"like_words":["dolorem","laboriosam","et"],"anniversary":{"quam":"et","ut":"et"}}}
|
|
78
|
+
2015-10-10 04:52:56 +0900 76 Mercedes Schmeler -4992.620748522985 {"phone_numbers":"632.560.4189","app_id":380,"point":-3075.239498270392,"created_at":"2015-10-08 11:30:10 +0900","profile":{"like_words":["molestias","aperiam","sed"],"anniversary":{"quo":"eligendi","fugit":"rerum"}}}
|
|
79
|
+
2015-10-08 15:16:19 +0900 77 Dave Reichel 4100.688594676535 {"phone_numbers":"995.426.5316","app_id":77,"point":2429.988892407077,"created_at":"2015-10-07 01:32:54 +0900","profile":{"like_words":["laborum","nulla","facere"],"anniversary":{"sed":"non","vero":"et"}}}
|
|
80
|
+
2015-10-07 04:39:56 +0900 78 Mossie Bruen -1020.5950461619159 {"phone_numbers":"1-453-224-9306","app_id":468,"point":-3425.2932917507646,"created_at":"2015-10-11 03:29:25 +0900","profile":{"like_words":["et","vel","maxime"],"anniversary":{"voluptas":"pariatur","esse":"aut"}}}
|
|
81
|
+
2015-10-10 21:35:21 +0900 79 Monica Stiedemann 3219.2086780943387 {"phone_numbers":"225.003.3142","app_id":711,"point":1033.4855909658213,"created_at":"2015-10-11 20:28:44 +0900","profile":{"like_words":["eaque","eos","exercitationem"],"anniversary":{"assumenda":"cupiditate","voluptate":"id"}}}
|
|
82
|
+
2015-10-07 06:36:02 +0900 80 Dr. Elyssa Pagac -3301.0154134166187 {"phone_numbers":"760-859-0012","app_id":720,"point":-2910.5746796621092,"created_at":"2015-10-06 20:05:01 +0900","profile":{"like_words":["pariatur","dolor","quasi"],"anniversary":{"aut":"iste","unde":"quia"}}}
|
|
83
|
+
2015-10-07 07:15:44 +0900 81 Donnell Abernathy DDS 4913.704752403618 {"phone_numbers":"1-276-201-1324","app_id":567,"point":2968.2829037436186,"created_at":"2015-10-09 17:40:42 +0900","profile":{"like_words":["et","nemo","et"],"anniversary":{"et":"tempore","a":"autem"}}}
|
|
84
|
+
2015-10-09 06:02:37 +0900 82 Leda Herman -1173.1566575345037 {"phone_numbers":"380-183-9245","app_id":0,"point":-2606.836728126425,"created_at":"2015-10-11 02:54:00 +0900","profile":{"like_words":["animi","quas","aut"],"anniversary":{"temporibus":"amet","ullam":"dolorem"}}}
|
|
85
|
+
2015-10-07 18:30:22 +0900 83 Vanessa Rodriguez 4555.371865371344 {"phone_numbers":"(749) 572-0585","app_id":498,"point":1093.1338706390486,"created_at":"2015-10-08 14:30:48 +0900","profile":{"like_words":["facere","ut","tenetur"],"anniversary":{"aut":"et","est":"eum"}}}
|
|
86
|
+
2015-10-06 16:06:52 +0900 84 Athena Waters DDS -4893.081938542143 {"phone_numbers":"716.422.9287","app_id":0,"point":-1149.3971933280027,"created_at":"2015-10-10 21:18:55 +0900","profile":{"like_words":["dolores","sequi","earum"],"anniversary":{"doloremque":"commodi","possimus":"suscipit"}}}
|
|
87
|
+
2015-10-06 23:17:46 +0900 85 Mrs. Christa Fisher 3707.8687440827152 {"phone_numbers":"(267) 374-5678","app_id":170,"point":1067.7682428341589,"created_at":"2015-10-10 23:22:10 +0900","profile":{"like_words":["eius","quam","commodi"],"anniversary":{"tempora":"ut","ea":"quo"}}}
|
|
88
|
+
2015-10-07 04:26:15 +0900 86 Dannie Kuhic -4867.675930528709 {"phone_numbers":"1-401-313-0962","app_id":688,"point":-3861.479221888725,"created_at":"2015-10-08 23:15:11 +0900","profile":{"like_words":["earum","dolores","distinctio"],"anniversary":{"ipsam":"enim","rerum":"at"}}}
|
|
89
|
+
2015-10-08 09:02:07 +0900 87 Mrs. Ceasar Hammes 3115.1263873476946 {"phone_numbers":"388.667.4018","app_id":261,"point":1296.226795622605,"created_at":"2015-10-07 16:53:07 +0900","profile":{"like_words":["quis","dignissimos","nihil"],"anniversary":{"pariatur":"cum","ratione":"possimus"}}}
|
|
90
|
+
2015-10-10 11:38:53 +0900 88 Retta Cassin -965.9102380984154 {"phone_numbers":"212-958-7048","app_id":528,"point":-1121.2619694929854,"created_at":"2015-10-07 01:04:15 +0900","profile":{"like_words":["sit","inventore","sapiente"],"anniversary":{"eos":"a","eveniet":"odit"}}}
|
|
91
|
+
2015-10-07 16:12:38 +0900 89 Kylee Johnston 3528.170737697339 {"phone_numbers":"619-131-4482","app_id":534,"point":4686.728190447,"created_at":"2015-10-09 20:24:10 +0900","profile":{"like_words":["architecto","doloremque","ducimus"],"anniversary":{"minima":"vel","delectus":"et"}}}
|
|
92
|
+
2015-10-11 22:34:51 +0900 90 Shaniya Schiller -4114.875541321096 {"phone_numbers":"629.766.7294","app_id":180,"point":-1817.7470567027422,"created_at":"2015-10-07 08:15:40 +0900","profile":{"like_words":["rem","voluptates","incidunt"],"anniversary":{"dolore":"omnis","nobis":"velit"}}}
|
|
93
|
+
2015-10-06 22:06:02 +0900 91 Jermain Skiles 4505.159451918198 {"phone_numbers":"191-180-2263","app_id":0,"point":3434.5623432303423,"created_at":"2015-10-07 23:43:10 +0900","profile":{"like_words":["praesentium","facilis","at"],"anniversary":{"enim":"repellat","explicabo":"dolorem"}}}
|
|
94
|
+
2015-10-10 23:52:23 +0900 92 Maximilian Walsh -3075.1129430504398 {"phone_numbers":"655-397-3448","app_id":828,"point":-3084.6459087115227,"created_at":"2015-10-06 02:29:58 +0900","profile":{"like_words":["pariatur","suscipit","harum"],"anniversary":{"nam":"atque","consectetur":"quam"}}}
|
|
95
|
+
2015-10-06 20:17:45 +0900 93 Dejuan Hintz 392.3273912017863 {"phone_numbers":"601-422-0130","app_id":372,"point":1834.1324518267406,"created_at":"2015-10-11 07:01:36 +0900","profile":{"like_words":["modi","autem","consequatur"],"anniversary":{"molestias":"fugit","voluptas":"quibusdam"}}}
|
|
96
|
+
2015-10-08 13:03:57 +0900 94 Clementine Pfannerstill -1541.891110930284 {"phone_numbers":"708.245.3655","app_id":752,"point":-3013.0195755053023,"created_at":"2015-10-11 10:05:07 +0900","profile":{"like_words":["laborum","accusamus","aperiam"],"anniversary":{"ratione":"est","sit":"consequuntur"}}}
|
|
97
|
+
2015-10-06 05:58:37 +0900 95 Callie Sipes 3661.717941738101 {"phone_numbers":"396.116.6442","app_id":665,"point":1214.9686490710287,"created_at":"2015-10-06 09:35:47 +0900","profile":{"like_words":["incidunt","occaecati","quisquam"],"anniversary":{"deleniti":"nisi","dolores":"nemo"}}}
|
|
98
|
+
2015-10-10 19:08:45 +0900 96 Flavio Howe -1458.6278077262718 {"phone_numbers":"697-491-3943","app_id":192,"point":-4555.188389102583,"created_at":"2015-10-10 06:55:15 +0900","profile":{"like_words":["optio","perferendis","ut"],"anniversary":{"autem":"optio","molestiae":"expedita"}}}
|
|
99
|
+
2015-10-09 06:31:31 +0900 97 Allene Funk 3342.439463990594 {"phone_numbers":"(968) 762-0953","app_id":873,"point":3439.9215311345856,"created_at":"2015-10-11 22:35:02 +0900","profile":{"like_words":["sint","quisquam","totam"],"anniversary":{"vitae":"natus","ducimus":"iusto"}}}
|
|
100
|
+
2015-10-10 15:12:33 +0900 98 Jan Kling -435.54543084869965 {"phone_numbers":"(223) 519-9714","app_id":98,"point":-4606.787115867773,"created_at":"2015-10-10 15:15:24 +0900","profile":{"like_words":["ex","natus","inventore"],"anniversary":{"quidem":"asperiores","vel":"libero"}}}
|
|
101
|
+
2015-10-06 09:49:13 +0900 99 Kyra Runolfsson 2945.3144859071417 {"phone_numbers":"656.669.6277","app_id":792,"point":4584.610475193578,"created_at":"2015-10-09 08:26:17 +0900","profile":{"like_words":["officia","sapiente","quia"],"anniversary":{"ut":"occaecati","et":"recusandae"}}}
|
|
Binary file
|
data/gradlew
ADDED
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
|
|
3
|
+
##############################################################################
|
|
4
|
+
##
|
|
5
|
+
## Gradle start up script for UN*X
|
|
6
|
+
##
|
|
7
|
+
##############################################################################
|
|
8
|
+
|
|
9
|
+
# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
|
|
10
|
+
DEFAULT_JVM_OPTS=""
|
|
11
|
+
|
|
12
|
+
APP_NAME="Gradle"
|
|
13
|
+
APP_BASE_NAME=`basename "$0"`
|
|
14
|
+
|
|
15
|
+
# Use the maximum available, or set MAX_FD != -1 to use that value.
|
|
16
|
+
MAX_FD="maximum"
|
|
17
|
+
|
|
18
|
+
warn ( ) {
|
|
19
|
+
echo "$*"
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
die ( ) {
|
|
23
|
+
echo
|
|
24
|
+
echo "$*"
|
|
25
|
+
echo
|
|
26
|
+
exit 1
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
# OS specific support (must be 'true' or 'false').
|
|
30
|
+
cygwin=false
|
|
31
|
+
msys=false
|
|
32
|
+
darwin=false
|
|
33
|
+
case "`uname`" in
|
|
34
|
+
CYGWIN* )
|
|
35
|
+
cygwin=true
|
|
36
|
+
;;
|
|
37
|
+
Darwin* )
|
|
38
|
+
darwin=true
|
|
39
|
+
;;
|
|
40
|
+
MINGW* )
|
|
41
|
+
msys=true
|
|
42
|
+
;;
|
|
43
|
+
esac
|
|
44
|
+
|
|
45
|
+
# For Cygwin, ensure paths are in UNIX format before anything is touched.
|
|
46
|
+
if $cygwin ; then
|
|
47
|
+
[ -n "$JAVA_HOME" ] && JAVA_HOME=`cygpath --unix "$JAVA_HOME"`
|
|
48
|
+
fi
|
|
49
|
+
|
|
50
|
+
# Attempt to set APP_HOME
|
|
51
|
+
# Resolve links: $0 may be a link
|
|
52
|
+
PRG="$0"
|
|
53
|
+
# Need this for relative symlinks.
|
|
54
|
+
while [ -h "$PRG" ] ; do
|
|
55
|
+
ls=`ls -ld "$PRG"`
|
|
56
|
+
link=`expr "$ls" : '.*-> \(.*\)$'`
|
|
57
|
+
if expr "$link" : '/.*' > /dev/null; then
|
|
58
|
+
PRG="$link"
|
|
59
|
+
else
|
|
60
|
+
PRG=`dirname "$PRG"`"/$link"
|
|
61
|
+
fi
|
|
62
|
+
done
|
|
63
|
+
SAVED="`pwd`"
|
|
64
|
+
cd "`dirname \"$PRG\"`/" >&-
|
|
65
|
+
APP_HOME="`pwd -P`"
|
|
66
|
+
cd "$SAVED" >&-
|
|
67
|
+
|
|
68
|
+
CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
|
|
69
|
+
|
|
70
|
+
# Determine the Java command to use to start the JVM.
|
|
71
|
+
if [ -n "$JAVA_HOME" ] ; then
|
|
72
|
+
if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
|
|
73
|
+
# IBM's JDK on AIX uses strange locations for the executables
|
|
74
|
+
JAVACMD="$JAVA_HOME/jre/sh/java"
|
|
75
|
+
else
|
|
76
|
+
JAVACMD="$JAVA_HOME/bin/java"
|
|
77
|
+
fi
|
|
78
|
+
if [ ! -x "$JAVACMD" ] ; then
|
|
79
|
+
die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME
|
|
80
|
+
|
|
81
|
+
Please set the JAVA_HOME variable in your environment to match the
|
|
82
|
+
location of your Java installation."
|
|
83
|
+
fi
|
|
84
|
+
else
|
|
85
|
+
JAVACMD="java"
|
|
86
|
+
which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
|
|
87
|
+
|
|
88
|
+
Please set the JAVA_HOME variable in your environment to match the
|
|
89
|
+
location of your Java installation."
|
|
90
|
+
fi
|
|
91
|
+
|
|
92
|
+
# Increase the maximum file descriptors if we can.
|
|
93
|
+
if [ "$cygwin" = "false" -a "$darwin" = "false" ] ; then
|
|
94
|
+
MAX_FD_LIMIT=`ulimit -H -n`
|
|
95
|
+
if [ $? -eq 0 ] ; then
|
|
96
|
+
if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then
|
|
97
|
+
MAX_FD="$MAX_FD_LIMIT"
|
|
98
|
+
fi
|
|
99
|
+
ulimit -n $MAX_FD
|
|
100
|
+
if [ $? -ne 0 ] ; then
|
|
101
|
+
warn "Could not set maximum file descriptor limit: $MAX_FD"
|
|
102
|
+
fi
|
|
103
|
+
else
|
|
104
|
+
warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT"
|
|
105
|
+
fi
|
|
106
|
+
fi
|
|
107
|
+
|
|
108
|
+
# For Darwin, add options to specify how the application appears in the dock
|
|
109
|
+
if $darwin; then
|
|
110
|
+
GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\""
|
|
111
|
+
fi
|
|
112
|
+
|
|
113
|
+
# For Cygwin, switch paths to Windows format before running java
|
|
114
|
+
if $cygwin ; then
|
|
115
|
+
APP_HOME=`cygpath --path --mixed "$APP_HOME"`
|
|
116
|
+
CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`
|
|
117
|
+
|
|
118
|
+
# We build the pattern for arguments to be converted via cygpath
|
|
119
|
+
ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null`
|
|
120
|
+
SEP=""
|
|
121
|
+
for dir in $ROOTDIRSRAW ; do
|
|
122
|
+
ROOTDIRS="$ROOTDIRS$SEP$dir"
|
|
123
|
+
SEP="|"
|
|
124
|
+
done
|
|
125
|
+
OURCYGPATTERN="(^($ROOTDIRS))"
|
|
126
|
+
# Add a user-defined pattern to the cygpath arguments
|
|
127
|
+
if [ "$GRADLE_CYGPATTERN" != "" ] ; then
|
|
128
|
+
OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)"
|
|
129
|
+
fi
|
|
130
|
+
# Now convert the arguments - kludge to limit ourselves to /bin/sh
|
|
131
|
+
i=0
|
|
132
|
+
for arg in "$@" ; do
|
|
133
|
+
CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -`
|
|
134
|
+
CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option
|
|
135
|
+
|
|
136
|
+
if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition
|
|
137
|
+
eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"`
|
|
138
|
+
else
|
|
139
|
+
eval `echo args$i`="\"$arg\""
|
|
140
|
+
fi
|
|
141
|
+
i=$((i+1))
|
|
142
|
+
done
|
|
143
|
+
case $i in
|
|
144
|
+
(0) set -- ;;
|
|
145
|
+
(1) set -- "$args0" ;;
|
|
146
|
+
(2) set -- "$args0" "$args1" ;;
|
|
147
|
+
(3) set -- "$args0" "$args1" "$args2" ;;
|
|
148
|
+
(4) set -- "$args0" "$args1" "$args2" "$args3" ;;
|
|
149
|
+
(5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;;
|
|
150
|
+
(6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;;
|
|
151
|
+
(7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;;
|
|
152
|
+
(8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;;
|
|
153
|
+
(9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;;
|
|
154
|
+
esac
|
|
155
|
+
fi
|
|
156
|
+
|
|
157
|
+
# Split up the JVM_OPTS And GRADLE_OPTS values into an array, following the shell quoting and substitution rules
|
|
158
|
+
function splitJvmOpts() {
|
|
159
|
+
JVM_OPTS=("$@")
|
|
160
|
+
}
|
|
161
|
+
eval splitJvmOpts $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS
|
|
162
|
+
JVM_OPTS[${#JVM_OPTS[*]}]="-Dorg.gradle.appname=$APP_BASE_NAME"
|
|
163
|
+
|
|
164
|
+
exec "$JAVACMD" "${JVM_OPTS[@]}" -classpath "$CLASSPATH" org.gradle.wrapper.GradleWrapperMain "$@"
|
data/gradlew.bat
ADDED
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
@if "%DEBUG%" == "" @echo off
|
|
2
|
+
@rem ##########################################################################
|
|
3
|
+
@rem
|
|
4
|
+
@rem Gradle startup script for Windows
|
|
5
|
+
@rem
|
|
6
|
+
@rem ##########################################################################
|
|
7
|
+
|
|
8
|
+
@rem Set local scope for the variables with windows NT shell
|
|
9
|
+
if "%OS%"=="Windows_NT" setlocal
|
|
10
|
+
|
|
11
|
+
@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
|
|
12
|
+
set DEFAULT_JVM_OPTS=
|
|
13
|
+
|
|
14
|
+
set DIRNAME=%~dp0
|
|
15
|
+
if "%DIRNAME%" == "" set DIRNAME=.
|
|
16
|
+
set APP_BASE_NAME=%~n0
|
|
17
|
+
set APP_HOME=%DIRNAME%
|
|
18
|
+
|
|
19
|
+
@rem Find java.exe
|
|
20
|
+
if defined JAVA_HOME goto findJavaFromJavaHome
|
|
21
|
+
|
|
22
|
+
set JAVA_EXE=java.exe
|
|
23
|
+
%JAVA_EXE% -version >NUL 2>&1
|
|
24
|
+
if "%ERRORLEVEL%" == "0" goto init
|
|
25
|
+
|
|
26
|
+
echo.
|
|
27
|
+
echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
|
|
28
|
+
echo.
|
|
29
|
+
echo Please set the JAVA_HOME variable in your environment to match the
|
|
30
|
+
echo location of your Java installation.
|
|
31
|
+
|
|
32
|
+
goto fail
|
|
33
|
+
|
|
34
|
+
:findJavaFromJavaHome
|
|
35
|
+
set JAVA_HOME=%JAVA_HOME:"=%
|
|
36
|
+
set JAVA_EXE=%JAVA_HOME%/bin/java.exe
|
|
37
|
+
|
|
38
|
+
if exist "%JAVA_EXE%" goto init
|
|
39
|
+
|
|
40
|
+
echo.
|
|
41
|
+
echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
|
|
42
|
+
echo.
|
|
43
|
+
echo Please set the JAVA_HOME variable in your environment to match the
|
|
44
|
+
echo location of your Java installation.
|
|
45
|
+
|
|
46
|
+
goto fail
|
|
47
|
+
|
|
48
|
+
:init
|
|
49
|
+
@rem Get command-line arguments, handling Windowz variants
|
|
50
|
+
|
|
51
|
+
if not "%OS%" == "Windows_NT" goto win9xME_args
|
|
52
|
+
if "%@eval[2+2]" == "4" goto 4NT_args
|
|
53
|
+
|
|
54
|
+
:win9xME_args
|
|
55
|
+
@rem Slurp the command line arguments.
|
|
56
|
+
set CMD_LINE_ARGS=
|
|
57
|
+
set _SKIP=2
|
|
58
|
+
|
|
59
|
+
:win9xME_args_slurp
|
|
60
|
+
if "x%~1" == "x" goto execute
|
|
61
|
+
|
|
62
|
+
set CMD_LINE_ARGS=%*
|
|
63
|
+
goto execute
|
|
64
|
+
|
|
65
|
+
:4NT_args
|
|
66
|
+
@rem Get arguments from the 4NT Shell from JP Software
|
|
67
|
+
set CMD_LINE_ARGS=%$
|
|
68
|
+
|
|
69
|
+
:execute
|
|
70
|
+
@rem Setup the command line
|
|
71
|
+
|
|
72
|
+
set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
|
|
73
|
+
|
|
74
|
+
@rem Execute Gradle
|
|
75
|
+
"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
|
|
76
|
+
|
|
77
|
+
:end
|
|
78
|
+
@rem End local scope for the variables with windows NT shell
|
|
79
|
+
if "%ERRORLEVEL%"=="0" goto mainEnd
|
|
80
|
+
|
|
81
|
+
:fail
|
|
82
|
+
rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
|
|
83
|
+
rem the _cmd.exe /c_ return code!
|
|
84
|
+
if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
|
|
85
|
+
exit /b 1
|
|
86
|
+
|
|
87
|
+
:mainEnd
|
|
88
|
+
if "%OS%"=="Windows_NT" endlocal
|
|
89
|
+
|
|
90
|
+
:omega
|
|
@@ -0,0 +1,284 @@
|
|
|
1
|
+
package org.embulk.filter.expand_json;
|
|
2
|
+
|
|
3
|
+
import com.fasterxml.jackson.core.JsonProcessingException;
|
|
4
|
+
import com.fasterxml.jackson.databind.ObjectMapper;
|
|
5
|
+
import com.google.common.base.Throwables;
|
|
6
|
+
import com.google.common.collect.ImmutableList;
|
|
7
|
+
import com.google.common.collect.Maps;
|
|
8
|
+
import com.jayway.jsonpath.Configuration;
|
|
9
|
+
import com.jayway.jsonpath.JsonPath;
|
|
10
|
+
import com.jayway.jsonpath.Option;
|
|
11
|
+
import com.jayway.jsonpath.ReadContext;
|
|
12
|
+
import org.embulk.config.Config;
|
|
13
|
+
import org.embulk.config.ConfigDefault;
|
|
14
|
+
import org.embulk.config.ConfigSource;
|
|
15
|
+
import org.embulk.config.Task;
|
|
16
|
+
import org.embulk.config.TaskSource;
|
|
17
|
+
import org.embulk.spi.Column;
|
|
18
|
+
import org.embulk.spi.ColumnConfig;
|
|
19
|
+
import org.embulk.spi.Exec;
|
|
20
|
+
import org.embulk.spi.FilterPlugin;
|
|
21
|
+
import org.embulk.spi.Page;
|
|
22
|
+
import org.embulk.spi.PageBuilder;
|
|
23
|
+
import org.embulk.spi.PageOutput;
|
|
24
|
+
import org.embulk.spi.PageReader;
|
|
25
|
+
import org.embulk.spi.Schema;
|
|
26
|
+
import org.embulk.spi.time.TimestampParser;
|
|
27
|
+
import org.embulk.spi.type.Types;
|
|
28
|
+
import org.joda.time.DateTimeZone;
|
|
29
|
+
import org.jruby.embed.ScriptingContainer;
|
|
30
|
+
import org.slf4j.Logger;
|
|
31
|
+
|
|
32
|
+
import java.util.ArrayList;
|
|
33
|
+
import java.util.HashMap;
|
|
34
|
+
import java.util.List;
|
|
35
|
+
import java.util.Map;
|
|
36
|
+
|
|
37
|
+
public class ExpandJsonFilterPlugin
|
|
38
|
+
implements FilterPlugin
|
|
39
|
+
{
|
|
40
|
+
private final Logger logger = Exec.getLogger(ExpandJsonFilterPlugin.class);
|
|
41
|
+
|
|
42
|
+
public interface PluginTask
|
|
43
|
+
extends Task, TimestampParser.Task
|
|
44
|
+
{
|
|
45
|
+
@Config("json_column_name")
|
|
46
|
+
public String getJsonColumnName();
|
|
47
|
+
|
|
48
|
+
@Config("expanded_columns")
|
|
49
|
+
public List<ColumnConfig> getExpandedColumns();
|
|
50
|
+
|
|
51
|
+
@Config("time_zone")
|
|
52
|
+
@ConfigDefault("\"UTC\"")
|
|
53
|
+
public String getTimeZone();
|
|
54
|
+
|
|
55
|
+
// TODO if needed: add the original column name as the prefix of expanded
|
|
56
|
+
// @Config("add_original_column_name_as_prefix")
|
|
57
|
+
// @ConfigDefault("false")
|
|
58
|
+
// public String getOption2();
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
@Override
|
|
62
|
+
public void transaction(ConfigSource config, Schema inputSchema,
|
|
63
|
+
FilterPlugin.Control control)
|
|
64
|
+
{
|
|
65
|
+
PluginTask task = config.loadConfig(PluginTask.class);
|
|
66
|
+
|
|
67
|
+
Schema outputSchema = buildOutputSchema(inputSchema,
|
|
68
|
+
task.getJsonColumnName(),
|
|
69
|
+
task.getExpandedColumns());
|
|
70
|
+
|
|
71
|
+
control.run(task.dump(), outputSchema);
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
@Override
|
|
75
|
+
public PageOutput open(TaskSource taskSource, final Schema inputSchema,
|
|
76
|
+
final Schema outputSchema, final PageOutput output)
|
|
77
|
+
{
|
|
78
|
+
final PluginTask task = taskSource.loadTask(PluginTask.class);
|
|
79
|
+
|
|
80
|
+
final List<Column> inputColumns = inputSchema.getColumns();
|
|
81
|
+
|
|
82
|
+
final List<Column> inputColumnsExceptExpandedJsonColumn = new ArrayList<>();
|
|
83
|
+
final List<Column> expandedJsonColumns = new ArrayList<>();
|
|
84
|
+
|
|
85
|
+
for (Column column : outputSchema.getColumns()) {
|
|
86
|
+
if (inputColumns.contains(column)) {
|
|
87
|
+
inputColumnsExceptExpandedJsonColumn.add(column);
|
|
88
|
+
}
|
|
89
|
+
else {
|
|
90
|
+
expandedJsonColumns.add(column);
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
Column temporaryJsonColumn = null;
|
|
95
|
+
for (Column column: inputColumns) {
|
|
96
|
+
if (column.getName().contentEquals(task.getJsonColumnName())) {
|
|
97
|
+
temporaryJsonColumn = column;
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
final Column jsonColumn = temporaryJsonColumn;
|
|
101
|
+
|
|
102
|
+
final HashMap<String, TimestampParser> timestampParserMap = buildTimestampParserMap(task.getJRuby(),
|
|
103
|
+
task.getExpandedColumns(),
|
|
104
|
+
task.getTimeZone());
|
|
105
|
+
return new PageOutput()
|
|
106
|
+
{
|
|
107
|
+
private PageReader pageReader = new PageReader(inputSchema);
|
|
108
|
+
|
|
109
|
+
@Override
|
|
110
|
+
public void add(Page page)
|
|
111
|
+
{
|
|
112
|
+
try (PageBuilder pageBuilder = new PageBuilder(Exec.getBufferAllocator(), outputSchema, output)) {
|
|
113
|
+
pageReader.setPage(page);
|
|
114
|
+
|
|
115
|
+
while (pageReader.nextRecord()) {
|
|
116
|
+
setInputColumnsExceptFlattenJsonColumns(pageBuilder, inputColumnsExceptExpandedJsonColumn);
|
|
117
|
+
setExpandedJsonColumns(pageBuilder, jsonColumn, expandedJsonColumns, timestampParserMap);
|
|
118
|
+
pageBuilder.addRecord();
|
|
119
|
+
}
|
|
120
|
+
pageBuilder.finish();
|
|
121
|
+
}
|
|
122
|
+
catch (JsonProcessingException e) {
|
|
123
|
+
logger.error(e.getMessage());
|
|
124
|
+
throw Throwables.propagate(e);
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
@Override
|
|
129
|
+
public void finish()
|
|
130
|
+
{
|
|
131
|
+
output.finish();
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
@Override
|
|
135
|
+
public void close()
|
|
136
|
+
{
|
|
137
|
+
pageReader.close();
|
|
138
|
+
output.close();
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
private void setInputColumnsExceptFlattenJsonColumns(PageBuilder pageBuilder, List<Column> inputColumnsExceptExpandedJsonColumn) {
|
|
142
|
+
for (Column inputColumn: inputColumnsExceptExpandedJsonColumn) {
|
|
143
|
+
if (pageReader.isNull(inputColumn)) {
|
|
144
|
+
pageBuilder.setNull(inputColumn);
|
|
145
|
+
continue;
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
if (Types.STRING.equals(inputColumn.getType())) {
|
|
149
|
+
pageBuilder.setString(inputColumn, pageReader.getString(inputColumn));
|
|
150
|
+
}
|
|
151
|
+
else if (Types.BOOLEAN.equals(inputColumn.getType())) {
|
|
152
|
+
pageBuilder.setBoolean(inputColumn, pageReader.getBoolean(inputColumn));
|
|
153
|
+
}
|
|
154
|
+
else if (Types.DOUBLE.equals(inputColumn.getType())) {
|
|
155
|
+
pageBuilder.setDouble(inputColumn, pageReader.getDouble(inputColumn));
|
|
156
|
+
}
|
|
157
|
+
else if (Types.LONG.equals(inputColumn.getType())) {
|
|
158
|
+
pageBuilder.setLong(inputColumn, pageReader.getLong(inputColumn));
|
|
159
|
+
}
|
|
160
|
+
else if (Types.TIMESTAMP.equals(inputColumn.getType())) {
|
|
161
|
+
pageBuilder.setTimestamp(inputColumn, pageReader.getTimestamp(inputColumn));
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
private void setExpandedJsonColumns(PageBuilder pageBuilder, Column originalJsonColumn, List<Column> expandedJsonColumns, HashMap<String, TimestampParser> timestampParserMap)
|
|
167
|
+
throws JsonProcessingException
|
|
168
|
+
{
|
|
169
|
+
final ReadContext json;
|
|
170
|
+
if (pageReader.isNull(originalJsonColumn)) {
|
|
171
|
+
json = null;
|
|
172
|
+
}
|
|
173
|
+
else {
|
|
174
|
+
String jsonObject = pageReader.getString(originalJsonColumn);
|
|
175
|
+
Configuration conf = Configuration.defaultConfiguration();
|
|
176
|
+
conf = conf.addOptions(Option.DEFAULT_PATH_LEAF_TO_NULL);
|
|
177
|
+
json = JsonPath.using(conf).parse(jsonObject);
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
for (Column expandedJsonColumn: expandedJsonColumns) {
|
|
181
|
+
if (json == null) {
|
|
182
|
+
pageBuilder.setNull(expandedJsonColumn);
|
|
183
|
+
continue;
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
Object value = json.read(expandedJsonColumn.getName());
|
|
187
|
+
final String finalValue = writeJsonPathValueAsString(value);
|
|
188
|
+
if (finalValue == null) {
|
|
189
|
+
pageBuilder.setNull(expandedJsonColumn);
|
|
190
|
+
continue;
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
if (Types.STRING.equals(expandedJsonColumn.getType())) {
|
|
194
|
+
pageBuilder.setString(expandedJsonColumn, finalValue);
|
|
195
|
+
}
|
|
196
|
+
else if (Types.BOOLEAN.equals(expandedJsonColumn.getType())) {
|
|
197
|
+
pageBuilder.setBoolean(expandedJsonColumn, Boolean.parseBoolean(finalValue));
|
|
198
|
+
}
|
|
199
|
+
else if (Types.DOUBLE.equals(expandedJsonColumn.getType())) {
|
|
200
|
+
pageBuilder.setDouble(expandedJsonColumn, Double.parseDouble(finalValue));
|
|
201
|
+
}
|
|
202
|
+
else if (Types.LONG.equals(expandedJsonColumn.getType())) {
|
|
203
|
+
pageBuilder.setLong(expandedJsonColumn, Long.parseLong(finalValue));
|
|
204
|
+
}
|
|
205
|
+
else if (Types.TIMESTAMP.equals(expandedJsonColumn.getType())) {
|
|
206
|
+
TimestampParser parser = timestampParserMap.get(expandedJsonColumn.getName());
|
|
207
|
+
pageBuilder.setTimestamp(expandedJsonColumn, parser.parse(finalValue));
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
private String writeJsonPathValueAsString(Object value)
|
|
213
|
+
throws JsonProcessingException
|
|
214
|
+
{
|
|
215
|
+
if (value == null) {
|
|
216
|
+
return null;
|
|
217
|
+
}
|
|
218
|
+
else if (value instanceof List) {
|
|
219
|
+
return new ObjectMapper().writeValueAsString(value);
|
|
220
|
+
}
|
|
221
|
+
else if (value instanceof Map) {
|
|
222
|
+
return new ObjectMapper().writeValueAsString(value);
|
|
223
|
+
}
|
|
224
|
+
else if (value instanceof String) {
|
|
225
|
+
return (String) value;
|
|
226
|
+
}
|
|
227
|
+
else {
|
|
228
|
+
return String.valueOf(value);
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
};
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
private Schema buildOutputSchema(Schema inputSchema, String jsonColumnName, List<ColumnConfig> expandedColumnConfigs)
|
|
236
|
+
{
|
|
237
|
+
ImmutableList.Builder<Column> builder = ImmutableList.builder();
|
|
238
|
+
|
|
239
|
+
int i = 0; // columns index
|
|
240
|
+
for (Column inputColumn: inputSchema.getColumns()) {
|
|
241
|
+
if (inputColumn.getName().contentEquals(jsonColumnName)) {
|
|
242
|
+
logger.info("removed column: name: {}, type: {}",
|
|
243
|
+
inputColumn.getName(),
|
|
244
|
+
inputColumn.getType());
|
|
245
|
+
for (ColumnConfig expandedColumnConfig: expandedColumnConfigs) {
|
|
246
|
+
logger.info("added column: name: {}, type: {}, options: {}",
|
|
247
|
+
expandedColumnConfig.getName(),
|
|
248
|
+
expandedColumnConfig.getType(),
|
|
249
|
+
expandedColumnConfig.getOption());
|
|
250
|
+
Column outputColumn = new Column(i++,
|
|
251
|
+
expandedColumnConfig.getName(),
|
|
252
|
+
expandedColumnConfig.getType());
|
|
253
|
+
builder.add(outputColumn);
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
else {
|
|
257
|
+
Column outputColumn = new Column(i++,
|
|
258
|
+
inputColumn.getName(),
|
|
259
|
+
inputColumn.getType());
|
|
260
|
+
builder.add(outputColumn);
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
return new Schema(builder.build());
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
private HashMap<String, TimestampParser> buildTimestampParserMap(ScriptingContainer jruby, List<ColumnConfig> expandedColumnConfigs, String timeZone)
|
|
268
|
+
{
|
|
269
|
+
final HashMap<String, TimestampParser> timestampParserMap = Maps.newHashMap();
|
|
270
|
+
for (ColumnConfig expandedColumnConfig: expandedColumnConfigs) {
|
|
271
|
+
if (Types.TIMESTAMP.equals(expandedColumnConfig.getType())) {
|
|
272
|
+
String format = expandedColumnConfig.getOption().get(String.class, "format");
|
|
273
|
+
DateTimeZone timezone = DateTimeZone.forID(timeZone);
|
|
274
|
+
TimestampParser parser = new TimestampParser(jruby, format, timezone);
|
|
275
|
+
|
|
276
|
+
String columnName = expandedColumnConfig.getName();
|
|
277
|
+
|
|
278
|
+
timestampParserMap.put(columnName, parser);
|
|
279
|
+
}
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
return timestampParserMap;
|
|
283
|
+
}
|
|
284
|
+
}
|
metadata
ADDED
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
|
2
|
+
name: embulk-filter-expand_json
|
|
3
|
+
version: !ruby/object:Gem::Version
|
|
4
|
+
version: 0.0.1
|
|
5
|
+
platform: ruby
|
|
6
|
+
authors:
|
|
7
|
+
- Civitaspo
|
|
8
|
+
autorequire:
|
|
9
|
+
bindir: bin
|
|
10
|
+
cert_chain: []
|
|
11
|
+
date: 2015-10-14 00:00:00.000000000 Z
|
|
12
|
+
dependencies:
|
|
13
|
+
- !ruby/object:Gem::Dependency
|
|
14
|
+
name: bundler
|
|
15
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
16
|
+
requirements:
|
|
17
|
+
- - ~>
|
|
18
|
+
- !ruby/object:Gem::Version
|
|
19
|
+
version: '1.0'
|
|
20
|
+
requirement: !ruby/object:Gem::Requirement
|
|
21
|
+
requirements:
|
|
22
|
+
- - ~>
|
|
23
|
+
- !ruby/object:Gem::Version
|
|
24
|
+
version: '1.0'
|
|
25
|
+
prerelease: false
|
|
26
|
+
type: :development
|
|
27
|
+
- !ruby/object:Gem::Dependency
|
|
28
|
+
name: rake
|
|
29
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
30
|
+
requirements:
|
|
31
|
+
- - '>='
|
|
32
|
+
- !ruby/object:Gem::Version
|
|
33
|
+
version: '10.0'
|
|
34
|
+
requirement: !ruby/object:Gem::Requirement
|
|
35
|
+
requirements:
|
|
36
|
+
- - '>='
|
|
37
|
+
- !ruby/object:Gem::Version
|
|
38
|
+
version: '10.0'
|
|
39
|
+
prerelease: false
|
|
40
|
+
type: :development
|
|
41
|
+
description: Expand Json
|
|
42
|
+
email:
|
|
43
|
+
- civitaspo@gmail.com
|
|
44
|
+
executables: []
|
|
45
|
+
extensions: []
|
|
46
|
+
extra_rdoc_files: []
|
|
47
|
+
files:
|
|
48
|
+
- .gitignore
|
|
49
|
+
- LICENSE.txt
|
|
50
|
+
- README.md
|
|
51
|
+
- build.gradle
|
|
52
|
+
- example/config.yml
|
|
53
|
+
- example/data.tsv
|
|
54
|
+
- gradle/wrapper/gradle-wrapper.jar
|
|
55
|
+
- gradle/wrapper/gradle-wrapper.properties
|
|
56
|
+
- gradlew
|
|
57
|
+
- gradlew.bat
|
|
58
|
+
- lib/embulk/filter/expand_json.rb
|
|
59
|
+
- src/main/java/org/embulk/filter/expand_json/ExpandJsonFilterPlugin.java
|
|
60
|
+
- src/test/java/org/embulk/filter/expand_json/TestExpandJsonFilterPlugin.java
|
|
61
|
+
- classpath/asm-1.0.2.jar
|
|
62
|
+
- classpath/asm-3.3.1.jar
|
|
63
|
+
- classpath/embulk-filter-expand_json-0.0.1.jar
|
|
64
|
+
- classpath/json-path-2.0.0.jar
|
|
65
|
+
- classpath/json-smart-2.1.1.jar
|
|
66
|
+
homepage: https://github.com/civitaspo/embulk-filter-expand_json
|
|
67
|
+
licenses:
|
|
68
|
+
- MIT
|
|
69
|
+
metadata: {}
|
|
70
|
+
post_install_message:
|
|
71
|
+
rdoc_options: []
|
|
72
|
+
require_paths:
|
|
73
|
+
- lib
|
|
74
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
|
75
|
+
requirements:
|
|
76
|
+
- - '>='
|
|
77
|
+
- !ruby/object:Gem::Version
|
|
78
|
+
version: '0'
|
|
79
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
80
|
+
requirements:
|
|
81
|
+
- - '>='
|
|
82
|
+
- !ruby/object:Gem::Version
|
|
83
|
+
version: '0'
|
|
84
|
+
requirements: []
|
|
85
|
+
rubyforge_project:
|
|
86
|
+
rubygems_version: 2.1.9
|
|
87
|
+
signing_key:
|
|
88
|
+
specification_version: 4
|
|
89
|
+
summary: Expand Json filter plugin for Embulk
|
|
90
|
+
test_files: []
|