data_collector 0.6.0 → 0.10.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (6) hide show
  1. checksums.yaml +4 -4
  2. data/data_collector.gemspec +28 -28
  3. data/lib/data_collector/output.rb +21 -1
  4. data/lib/data_collector/rules_ng.rb +29 -25
  5. data/lib/data_collector/version.rb +1 -1
  6. metadata +37 -37
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 78a054b825d99611c8f5070120c11b478975152433b3c76b9a63a6694b7c8cd0
4
- data.tar.gz: f11143808a8e1647a791b0e12e1c5855dc1589de37f7ea50c0e8789f88538774
3
+ metadata.gz: e0d6b86c3c19e523dcd9782fd55523592cc00ec99a665b68b4844bbf6eb40d43
4
+ data.tar.gz: 17c10292b608fd84cf0dc62c3770f2194219112a9482f5fe2bd8e4133f2b8743
5
5
  SHA512:
6
- metadata.gz: 0b3396b16e5500add7dbb8e2883e5bfd8e14ba898ce7291ed876ed1a423a59a2dc9bba42e1c96150a7119284398c58c18daf8ea7dc82844e3cee55d9ea17fe7f
7
- data.tar.gz: a7f7821eb19e50918321b8d1b47caa4ab9d04d40b2918a502f86512fbe25b8c35b4bee0705eb1044ade83cdb7f0c0f45a4023363e14c8153283ed9bdfec32f1a
6
+ metadata.gz: 81c6c31acb3a6f30a82cc19c4d12feb7ce2cb92e57e6dd4c3e3feb680de722f08e3786a76534b35df5cd16c778607aeb66b944fda4b000f433a856437f7452c3
7
+ data.tar.gz: 4809213a2a5f51883730fb422e471460631f19a9ec0a736c7a5ed3f2c0b3bdc7680b4befe94101cd95c4d01a00093a22b784e1029bfdd2cb80c45558e4eb2fde
data/data_collector.gemspec CHANGED
@@ -1,30 +1,30 @@
1
1
 
2
- lib = File.expand_path("../lib", __FILE__)
2
+ lib = File.expand_path('../lib', __FILE__)
3
3
  $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
- require "data_collector/version"
4
+ require 'data_collector/version'
5
5
 
6
6
  Gem::Specification.new do |spec|
7
- spec.name = "data_collector"
7
+ spec.name = 'data_collector'
8
8
  spec.version = DataCollector::VERSION
9
- spec.authors = ["Mehmet Celik"]
10
- spec.email = ["mehmet@celik.be"]
9
+ spec.authors = ['Mehmet Celik']
10
+ spec.email = ['mehmet@celik.be']
11
11
 
12
12
  spec.summary = %q{ETL helper library}
13
13
  spec.description = %q{INPUT, FILTER, OUTPUT data with RULES and code}
14
- spec.homepage = "https://github.com/mehmetc/data_collector"
15
- spec.license = "MIT"
14
+ spec.homepage = 'https://github.com/mehmetc/data_collector'
15
+ spec.license = 'MIT'
16
16
 
17
17
  # Prevent pushing this gem to RubyGems.org. To allow pushes either set the 'allowed_push_host'
18
18
  # to allow pushing to a single host or delete this section to allow pushing to any host.
19
19
  if spec.respond_to?(:metadata)
20
20
  #spec.metadata["allowed_push_host"] = "https://github.com/mehmetc/data_collect'"
21
21
 
22
- spec.metadata["homepage_uri"] = spec.homepage
23
- spec.metadata["source_code_uri"] = "https://github.com/mehmetc/data_collect"
24
- spec.metadata["changelog_uri"] = "https://github.com/mehmetc/data_collect"
22
+ spec.metadata['homepage_uri'] = spec.homepage
23
+ spec.metadata['source_code_uri'] = 'https://github.com/mehmetc/data_collect'
24
+ spec.metadata['changelog_uri'] = 'https://github.com/mehmetc/data_collect'
25
25
  else
26
- raise "RubyGems 2.0 or newer is required to protect against " \
27
- "public gem pushes."
26
+ raise 'RubyGems 2.0 or newer is required to protect against ' \
27
+ 'public gem pushes.'
28
28
  end
29
29
 
30
30
  # Specify which files should be added to the gem when it is released.
@@ -32,21 +32,21 @@ Gem::Specification.new do |spec|
32
32
  spec.files = Dir.chdir(File.expand_path('..', __FILE__)) do
33
33
  `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
34
34
  end
35
- spec.bindir = "exe"
35
+ spec.bindir = 'exe'
36
36
  spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
37
- spec.require_paths = ["lib"]
38
-
39
- spec.add_runtime_dependency "nokogiri", "~> 1.10"
40
- spec.add_runtime_dependency "json", "~> 2.2"
41
- spec.add_runtime_dependency "jsonpath", "~> 1.0"
42
- spec.add_runtime_dependency "nori", "~> 2.6"
43
- spec.add_runtime_dependency "http", "~> 4.1"
44
- spec.add_runtime_dependency "mime-types", "~> 3.2"
45
- spec.add_runtime_dependency "minitar", "= 0.9"
46
- spec.add_runtime_dependency "activesupport", "~> 5.2"
47
- spec.add_runtime_dependency "json-ld", "~> 3.1"
48
-
49
- spec.add_development_dependency "bundler", "~> 2.0"
50
- spec.add_development_dependency "rake", ">= 12.3"
51
- spec.add_development_dependency "minitest", "~> 5.0"
37
+ spec.require_paths = ['lib']
38
+
39
+ spec.add_runtime_dependency 'activesupport', '~> 6.1'
40
+ spec.add_runtime_dependency 'http', '~> 5.0'
41
+ spec.add_runtime_dependency 'json', '~> 2.5'
42
+ spec.add_runtime_dependency 'json-ld', '~> 3.1'
43
+ spec.add_runtime_dependency 'jsonpath', '~> 1.1'
44
+ spec.add_runtime_dependency 'mime-types', '~> 3.3'
45
+ spec.add_runtime_dependency 'minitar', '= 0.9'
46
+ spec.add_runtime_dependency 'nokogiri', '~> 1.12'
47
+ spec.add_runtime_dependency 'nori', '~> 2.6'
48
+
49
+ spec.add_development_dependency 'bundler', '~> 2.0'
50
+ spec.add_development_dependency 'minitest', '~> 5.14'
51
+ spec.add_development_dependency 'rake', '~> 13.0'
52
52
  end
data/lib/data_collector/output.rb CHANGED
@@ -32,7 +32,11 @@ module DataCollector
32
32
  unless v.nil?
33
33
  if data.has_key?(k)
34
34
  if data[k].is_a?(Array) then
35
- data[k] << v
35
+ if v.is_a?(Array)
36
+ data[k] += v
37
+ else
38
+ data[k] << v
39
+ end
36
40
  else
37
41
  t = data[k]
38
42
  data[k] = Array.new([t, v])
@@ -58,6 +62,22 @@ module DataCollector
58
62
  end
59
63
  end
60
64
 
65
+ def key?(key)
66
+ @data.key?(key)
67
+ end
68
+
69
+ def has_key?(key)
70
+ @data.key?(key)
71
+ end
72
+
73
+ def include?(key)
74
+ @data.key?(key)
75
+ end
76
+
77
+ def keys
78
+ @data.keys
79
+ end
80
+
61
81
  def raw
62
82
  @data
63
83
  end
data/lib/data_collector/rules_ng.rb CHANGED
@@ -18,7 +18,7 @@ module DataCollector
18
18
 
19
19
  def apply_rule(tag, rule, input_data, output_data, options = {})
20
20
  rule_filter = rule
21
- rule_payload = ""
21
+ rule_payload = ''
22
22
 
23
23
  case rule
24
24
  when Array
@@ -28,7 +28,7 @@ module DataCollector
28
28
  return output_data
29
29
  when String
30
30
  rule_filter = rule
31
- rule_payload = ""
31
+ rule_payload = ''
32
32
  else
33
33
  rule_filter = rule.keys.first
34
34
  rule_payload = rule.values.first
@@ -40,14 +40,11 @@ module DataCollector
40
40
  data = rule_payload
41
41
  else
42
42
  data = rule_payload.select { |s| s.is_a?(String) }
43
- rule_payload = rule_payload.delete_if { |s| s.is_a?(String) }
44
- rule_payload = "@" if rule_payload.empty?
45
- # if rule_payload.size == 1
46
- # rule_payload = rule_payload.first
47
- # end
43
+ rule_payload = rule_payload.reject { |s| s.is_a?(String) }
44
+ rule_payload = '@' if rule_payload.empty?
48
45
  end
49
- when /json_path\:/
50
- data = json_path_filter(rule_filter.gsub(/^json_path\:/), input_data)
46
+ when /json_path:/
47
+ data = json_path_filter(rule_filter.gsub(/^json_path:/), input_data)
51
48
  else
52
49
  data = json_path_filter(rule_filter, input_data)
53
50
  end
@@ -62,24 +59,27 @@ module DataCollector
62
59
 
63
60
  def apply_filtered_data_on_payload(input_data, payload, options = {})
64
61
  return nil if input_data.nil?
62
+
65
63
  output_data = nil
66
64
  case payload.class.name
67
65
  when 'Proc'
68
66
  data = input_data.is_a?(Array) ? input_data : [input_data]
69
- if options && options.empty?
70
- output_data = data.map { |d| payload.call(d) }
71
- else
72
- output_data = data.map { |d| payload.call(d, options) }
73
- end
67
+ output_data = if options.empty?
68
+ data.map { |d| payload.call(d) }
69
+ else
70
+ data.map { |d| payload.call(d, options) }
71
+ end
74
72
  when 'Hash'
75
73
  input_data = [input_data] unless input_data.is_a?(Array)
76
- output_data = input_data.map do |m|
77
- if payload.key?('suffix')
78
- "#{m}#{payload['suffix']}"
79
- else
80
- payload[m]
74
+ if input_data.is_a?(Array)
75
+ output_data = input_data.map do |m|
76
+ if payload.key?('suffix')
77
+ "#{m}#{payload['suffix']}"
78
+ else
79
+ payload[m]
80
+ end
81
81
  end
82
- end if input_data.is_a?(Array)
82
+ end
83
83
  when 'Array'
84
84
  output_data = input_data
85
85
  payload.each do |p|
@@ -90,8 +90,13 @@ module DataCollector
90
90
  end
91
91
 
92
92
  output_data.compact! if output_data.is_a?(Array)
93
- output_data.flatten! if output_data.is_a?(Array)# || output_data.is_a?(Hash)
94
- output_data = output_data.first if output_data.is_a?(Array) && output_data.size == 1
93
+ output_data.flatten! if output_data.is_a?(Array)
94
+ if output_data.is_a?(Array) &&
95
+ output_data.size == 1 &&
96
+ (output_data.first.is_a?(Array) || output_data.first.is_a?(Hash))
97
+ output_data = output_data.first
98
+ end
99
+
95
100
  output_data
96
101
  end
97
102
 
@@ -99,9 +104,8 @@ module DataCollector
99
104
  data = nil
100
105
  return data if input_data.nil? || input_data.empty?
101
106
  return input_data if input_data.is_a?(String)
102
- Core::filter(input_data, filter)
103
- end
104
-
105
107
 
108
+ Core.filter(input_data, filter)
109
+ end
106
110
  end
107
111
  end
data/lib/data_collector/version.rb CHANGED
@@ -1,4 +1,4 @@
1
1
  # encoding: utf-8
2
2
  module DataCollector
3
- VERSION = "0.6.0"
3
+ VERSION = "0.10.0"
4
4
  end
metadata CHANGED
@@ -1,99 +1,99 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: data_collector
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.6.0
4
+ version: 0.10.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Mehmet Celik
8
- autorequire:
8
+ autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2020-04-06 00:00:00.000000000 Z
11
+ date: 2021-10-06 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
- name: nokogiri
14
+ name: activesupport
15
15
  requirement: !ruby/object:Gem::Requirement
16
16
  requirements:
17
17
  - - "~>"
18
18
  - !ruby/object:Gem::Version
19
- version: '1.10'
19
+ version: '6.1'
20
20
  type: :runtime
21
21
  prerelease: false
22
22
  version_requirements: !ruby/object:Gem::Requirement
23
23
  requirements:
24
24
  - - "~>"
25
25
  - !ruby/object:Gem::Version
26
- version: '1.10'
26
+ version: '6.1'
27
27
  - !ruby/object:Gem::Dependency
28
- name: json
28
+ name: http
29
29
  requirement: !ruby/object:Gem::Requirement
30
30
  requirements:
31
31
  - - "~>"
32
32
  - !ruby/object:Gem::Version
33
- version: '2.2'
33
+ version: '5.0'
34
34
  type: :runtime
35
35
  prerelease: false
36
36
  version_requirements: !ruby/object:Gem::Requirement
37
37
  requirements:
38
38
  - - "~>"
39
39
  - !ruby/object:Gem::Version
40
- version: '2.2'
40
+ version: '5.0'
41
41
  - !ruby/object:Gem::Dependency
42
- name: jsonpath
42
+ name: json
43
43
  requirement: !ruby/object:Gem::Requirement
44
44
  requirements:
45
45
  - - "~>"
46
46
  - !ruby/object:Gem::Version
47
- version: '1.0'
47
+ version: '2.5'
48
48
  type: :runtime
49
49
  prerelease: false
50
50
  version_requirements: !ruby/object:Gem::Requirement
51
51
  requirements:
52
52
  - - "~>"
53
53
  - !ruby/object:Gem::Version
54
- version: '1.0'
54
+ version: '2.5'
55
55
  - !ruby/object:Gem::Dependency
56
- name: nori
56
+ name: json-ld
57
57
  requirement: !ruby/object:Gem::Requirement
58
58
  requirements:
59
59
  - - "~>"
60
60
  - !ruby/object:Gem::Version
61
- version: '2.6'
61
+ version: '3.1'
62
62
  type: :runtime
63
63
  prerelease: false
64
64
  version_requirements: !ruby/object:Gem::Requirement
65
65
  requirements:
66
66
  - - "~>"
67
67
  - !ruby/object:Gem::Version
68
- version: '2.6'
68
+ version: '3.1'
69
69
  - !ruby/object:Gem::Dependency
70
- name: http
70
+ name: jsonpath
71
71
  requirement: !ruby/object:Gem::Requirement
72
72
  requirements:
73
73
  - - "~>"
74
74
  - !ruby/object:Gem::Version
75
- version: '4.1'
75
+ version: '1.1'
76
76
  type: :runtime
77
77
  prerelease: false
78
78
  version_requirements: !ruby/object:Gem::Requirement
79
79
  requirements:
80
80
  - - "~>"
81
81
  - !ruby/object:Gem::Version
82
- version: '4.1'
82
+ version: '1.1'
83
83
  - !ruby/object:Gem::Dependency
84
84
  name: mime-types
85
85
  requirement: !ruby/object:Gem::Requirement
86
86
  requirements:
87
87
  - - "~>"
88
88
  - !ruby/object:Gem::Version
89
- version: '3.2'
89
+ version: '3.3'
90
90
  type: :runtime
91
91
  prerelease: false
92
92
  version_requirements: !ruby/object:Gem::Requirement
93
93
  requirements:
94
94
  - - "~>"
95
95
  - !ruby/object:Gem::Version
96
- version: '3.2'
96
+ version: '3.3'
97
97
  - !ruby/object:Gem::Dependency
98
98
  name: minitar
99
99
  requirement: !ruby/object:Gem::Requirement
@@ -109,33 +109,33 @@ dependencies:
109
109
  - !ruby/object:Gem::Version
110
110
  version: '0.9'
111
111
  - !ruby/object:Gem::Dependency
112
- name: activesupport
112
+ name: nokogiri
113
113
  requirement: !ruby/object:Gem::Requirement
114
114
  requirements:
115
115
  - - "~>"
116
116
  - !ruby/object:Gem::Version
117
- version: '5.2'
117
+ version: '1.12'
118
118
  type: :runtime
119
119
  prerelease: false
120
120
  version_requirements: !ruby/object:Gem::Requirement
121
121
  requirements:
122
122
  - - "~>"
123
123
  - !ruby/object:Gem::Version
124
- version: '5.2'
124
+ version: '1.12'
125
125
  - !ruby/object:Gem::Dependency
126
- name: json-ld
126
+ name: nori
127
127
  requirement: !ruby/object:Gem::Requirement
128
128
  requirements:
129
129
  - - "~>"
130
130
  - !ruby/object:Gem::Version
131
- version: '3.1'
131
+ version: '2.6'
132
132
  type: :runtime
133
133
  prerelease: false
134
134
  version_requirements: !ruby/object:Gem::Requirement
135
135
  requirements:
136
136
  - - "~>"
137
137
  - !ruby/object:Gem::Version
138
- version: '3.1'
138
+ version: '2.6'
139
139
  - !ruby/object:Gem::Dependency
140
140
  name: bundler
141
141
  requirement: !ruby/object:Gem::Requirement
@@ -151,33 +151,33 @@ dependencies:
151
151
  - !ruby/object:Gem::Version
152
152
  version: '2.0'
153
153
  - !ruby/object:Gem::Dependency
154
- name: rake
154
+ name: minitest
155
155
  requirement: !ruby/object:Gem::Requirement
156
156
  requirements:
157
- - - ">="
157
+ - - "~>"
158
158
  - !ruby/object:Gem::Version
159
- version: '12.3'
159
+ version: '5.14'
160
160
  type: :development
161
161
  prerelease: false
162
162
  version_requirements: !ruby/object:Gem::Requirement
163
163
  requirements:
164
- - - ">="
164
+ - - "~>"
165
165
  - !ruby/object:Gem::Version
166
- version: '12.3'
166
+ version: '5.14'
167
167
  - !ruby/object:Gem::Dependency
168
- name: minitest
168
+ name: rake
169
169
  requirement: !ruby/object:Gem::Requirement
170
170
  requirements:
171
171
  - - "~>"
172
172
  - !ruby/object:Gem::Version
173
- version: '5.0'
173
+ version: '13.0'
174
174
  type: :development
175
175
  prerelease: false
176
176
  version_requirements: !ruby/object:Gem::Requirement
177
177
  requirements:
178
178
  - - "~>"
179
179
  - !ruby/object:Gem::Version
180
- version: '5.0'
180
+ version: '13.0'
181
181
  description: INPUT, FILTER, OUTPUT data with RULES and code
182
182
  email:
183
183
  - mehmet@celik.be
@@ -211,7 +211,7 @@ metadata:
211
211
  homepage_uri: https://github.com/mehmetc/data_collector
212
212
  source_code_uri: https://github.com/mehmetc/data_collect
213
213
  changelog_uri: https://github.com/mehmetc/data_collect
214
- post_install_message:
214
+ post_install_message:
215
215
  rdoc_options: []
216
216
  require_paths:
217
217
  - lib
@@ -226,8 +226,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
226
226
  - !ruby/object:Gem::Version
227
227
  version: '0'
228
228
  requirements: []
229
- rubygems_version: 3.0.6
230
- signing_key:
229
+ rubygems_version: 3.1.4
230
+ signing_key:
231
231
  specification_version: 4
232
232
  summary: ETL helper library
233
233
  test_files: []