embulk-filter-jsoncolumn 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 3d873e0b016071b32ef851efdd3e3f03d10cbd2e
4
+ data.tar.gz: fb75e43b0de6a87a9e2aaa897e11477fd3afb385
5
+ SHA512:
6
+ metadata.gz: a01db0a66b0254f2ecfa1249acdef85f688774c954ebe518449221d3aae7a585a60cb77af1185b108a991d7740681784a36a2dcf1a6ed268fc7d626434a8f930
7
+ data.tar.gz: fd75212c82d8214d14e7c0e85ac782ebfa05b0def1d3d8bd6238f1dd7f9037cb22f40407e880854e5c0470297dd6b7e6b3cf59c0a81b86c3068d649f40a0da2e
data/.gitignore ADDED
@@ -0,0 +1,5 @@
1
+ *~
2
+ /pkg/
3
+ /tmp/
4
+ /.bundle/
5
+ /Gemfile.lock
data/.ruby-version ADDED
@@ -0,0 +1 @@
1
+ jruby-9.0.4.0
data/Gemfile ADDED
@@ -0,0 +1,2 @@
1
+ source 'https://rubygems.org/'
2
+ gemspec
data/LICENSE.txt ADDED
@@ -0,0 +1,21 @@
1
+
2
+ MIT License
3
+
4
+ Permission is hereby granted, free of charge, to any person obtaining
5
+ a copy of this software and associated documentation files (the
6
+ "Software"), to deal in the Software without restriction, including
7
+ without limitation the rights to use, copy, modify, merge, publish,
8
+ distribute, sublicense, and/or sell copies of the Software, and to
9
+ permit persons to whom the Software is furnished to do so, subject to
10
+ the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be
13
+ included in all copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
16
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
17
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
18
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
19
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
20
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
21
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
data/README.md ADDED
@@ -0,0 +1,78 @@
1
+ # Jsoncolumn filter plugin for Embulk
2
+
3
+ Extract json from input json.
4
+
5
+ ## Overview
6
+
7
+ * **Plugin type**: filter
8
+
9
+ ## Configuration
10
+
11
+ - schema: description (array, default: [], required)
12
+
13
+ ### schema
14
+
15
+ Array of schema definition. Schema name, type must be same as output column.
16
+
17
+ - name: name of schema (string, required)
18
+ - type: type of schema (string, required)
19
+ - path: JsonPath (string, optional)
20
+
21
+ ## Example
22
+
23
+ Sample data.
24
+
25
+ ```json
26
+ "root": {
27
+ "cluster_name": "fuga",
28
+ "nodes": {
29
+ "hoge": {
30
+ "timestamp": 1466645114192,
31
+
32
+ .
33
+ .
34
+ .
35
+ .
36
+
37
+ }
38
+ },
39
+ "status": {
40
+ }
41
+ }
42
+ }
43
+ ```
44
+
45
+ Sample config.
46
+
47
+ ```yaml
48
+ filters:
49
+ - type: jsoncolumn
50
+ schema:
51
+ - {name: cluster_name, type: string, path: "$..cluster_name"}
52
+ - {name: nodes, type: string, path: "$..nodes"}
53
+ ```
54
+
55
+ Result.
56
+
57
+ ```json
58
+ {
59
+ "cluster_name": "fuga",
60
+ "nodes": {
61
+ "hoge": {
62
+ "timestamp": 1466645114192,
63
+
64
+ .
65
+ .
66
+ .
67
+ .
68
+
69
+ }
70
+ }
71
+ }
72
+ ```
73
+
74
+ ## Build
75
+
76
+ ```
77
+ $ rake
78
+ ```
data/Rakefile ADDED
@@ -0,0 +1,3 @@
1
+ require "bundler/gem_tasks"
2
+
3
+ task default: :build
@@ -0,0 +1,21 @@
1
+
2
+ Gem::Specification.new do |spec|
3
+ spec.name = "embulk-filter-jsoncolumn"
4
+ spec.version = "0.1.0"
5
+ spec.authors = ["dokuma"]
6
+ spec.summary = "Jsoncolumn filter plugin for Embulk"
7
+ spec.description = "Jsoncolumn"
8
+ spec.email = ["dokuma.h@gmail.com"]
9
+ spec.licenses = ["MIT"]
10
+ spec.homepage = "https://github.com/dokuma/embulk-filter-jsoncolumn"
11
+
12
+ spec.files = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
13
+ spec.test_files = spec.files.grep(%r{^(test|spec)/})
14
+ spec.require_paths = ["lib"]
15
+ spec.has_rdoc = false
16
+
17
+ spec.add_development_dependency "jsonpath", "~> 0.5"
18
+ spec.add_development_dependency 'embulk', ['>= 0.8.9']
19
+ spec.add_development_dependency 'bundler', ['>= 1.10.6']
20
+ spec.add_development_dependency 'rake', ['>= 10.0']
21
+ end
@@ -0,0 +1,93 @@
1
+ require "jsonpath"
2
+
3
+ module Embulk
4
+ module Filter
5
+
6
+ class Jsoncolumn < FilterPlugin
7
+ Plugin.register_filter("jsoncolumn", self)
8
+
9
+ def self.transaction(config, in_schema, &control)
10
+ # configuration code:
11
+ task = {
12
+ 'schema' => config.param("schema", :array),
13
+ 'columns' =>
14
+ config.param('schema', :array, :default => []).inject({}){|a, col|
15
+ a[col['name']] = col['type'].to_sym
16
+ a
17
+ }
18
+ }
19
+
20
+ columns = task['columns'].map.with_index{|(name, type), i|
21
+ Column.new(i, name, type)
22
+ }
23
+
24
+ #out_columns = in_schema + columns
25
+ out_columns = columns
26
+
27
+ yield(task, out_columns)
28
+ end
29
+
30
+ def init
31
+ # initialization code:
32
+ end
33
+
34
+ def close
35
+ end
36
+
37
+ def add(page)
38
+ # filtering code:
39
+ page.each do |records|
40
+ records.each do |record|
41
+ r = JSON.parse(record)
42
+ page_builder.add(make_record(task['schema'], r))
43
+ end
44
+ end
45
+ end
46
+
47
+ def make_record(schema, e)
48
+ schema.map do |c|
49
+ name = c["name"]
50
+ path = c["path"]
51
+ val = path.nil? ? e[name] : find_by_path(e, path)
52
+
53
+ puts "PATH: #{path}"
54
+ puts "VAL: #{val}"
55
+
56
+ v = val.nil? ? "" : val
57
+ type = c["type"]
58
+ case type
59
+ when "string"
60
+ v
61
+ when "long"
62
+ v.to_i
63
+ when "double"
64
+ v.to_f
65
+ when "boolean"
66
+ if v.nil?
67
+ nil
68
+ elsif v.kind_of?(String)
69
+ ["yes", "true", "1"].include?(v.downcase)
70
+ elsif v.kind_of?(Numeric)
71
+ !v.zero?
72
+ else
73
+ !!v
74
+ end
75
+ when "timestamp"
76
+ v.empty? ? nil : Time.strptime(v, c["format"])
77
+ else
78
+ raise "Unsupported type #{type}"
79
+ end
80
+ end
81
+ end
82
+
83
+ def find_by_path(e, path)
84
+ JsonPath.on(e, path).first
85
+ end
86
+
87
+ def finish
88
+ page_builder.finish
89
+ end
90
+ end
91
+
92
+ end
93
+ end
metadata ADDED
@@ -0,0 +1,108 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: embulk-filter-jsoncolumn
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.1.0
5
+ platform: ruby
6
+ authors:
7
+ - dokuma
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2016-06-29 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: jsonpath
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - "~>"
18
+ - !ruby/object:Gem::Version
19
+ version: '0.5'
20
+ type: :development
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - "~>"
25
+ - !ruby/object:Gem::Version
26
+ version: '0.5'
27
+ - !ruby/object:Gem::Dependency
28
+ name: embulk
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - ">="
32
+ - !ruby/object:Gem::Version
33
+ version: 0.8.9
34
+ type: :development
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - ">="
39
+ - !ruby/object:Gem::Version
40
+ version: 0.8.9
41
+ - !ruby/object:Gem::Dependency
42
+ name: bundler
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - ">="
46
+ - !ruby/object:Gem::Version
47
+ version: 1.10.6
48
+ type: :development
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - ">="
53
+ - !ruby/object:Gem::Version
54
+ version: 1.10.6
55
+ - !ruby/object:Gem::Dependency
56
+ name: rake
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - ">="
60
+ - !ruby/object:Gem::Version
61
+ version: '10.0'
62
+ type: :development
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - ">="
67
+ - !ruby/object:Gem::Version
68
+ version: '10.0'
69
+ description: Jsoncolumn
70
+ email:
71
+ - dokuma.h@gmail.com
72
+ executables: []
73
+ extensions: []
74
+ extra_rdoc_files: []
75
+ files:
76
+ - ".gitignore"
77
+ - ".ruby-version"
78
+ - Gemfile
79
+ - LICENSE.txt
80
+ - README.md
81
+ - Rakefile
82
+ - embulk-filter-jsoncolumn.gemspec
83
+ - lib/embulk/filter/jsoncolumn.rb
84
+ homepage: https://github.com/dokuma/embulk-filter-jsoncolumn
85
+ licenses:
86
+ - MIT
87
+ metadata: {}
88
+ post_install_message:
89
+ rdoc_options: []
90
+ require_paths:
91
+ - lib
92
+ required_ruby_version: !ruby/object:Gem::Requirement
93
+ requirements:
94
+ - - ">="
95
+ - !ruby/object:Gem::Version
96
+ version: '0'
97
+ required_rubygems_version: !ruby/object:Gem::Requirement
98
+ requirements:
99
+ - - ">="
100
+ - !ruby/object:Gem::Version
101
+ version: '0'
102
+ requirements: []
103
+ rubyforge_project:
104
+ rubygems_version: 2.5.1
105
+ signing_key:
106
+ specification_version: 4
107
+ summary: Jsoncolumn filter plugin for Embulk
108
+ test_files: []