data_collector 0.2.0 → 0.2.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +32 -0
- data/lib/data_collector/core.rb +27 -0
- data/lib/data_collector/rules.rb +1 -1
- data/lib/data_collector/version.rb +1 -1
- metadata +1 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: b38644344f6bcfa74d68a2281b83c803879334fbd0f281fde4de8f92b180c5ec
|
|
4
|
+
data.tar.gz: 582650f6f0237f4a45c45ab137b6eac0110117b77a7b917eae448d17285ae999
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 550f42750545164f6e7f47347144023be9c1bb58ff661886e1a5ef51c2804fcea8d75b87c65ce29b6a288a3e24bcbfce26f5987c17424cdb1023fda510675b49
|
|
7
|
+
data.tar.gz: ee0d60db7824fda4688c0c41aed8f8e2884aeb5d9e0b4178c53685a4fcc44e3ef93f848f3b597d6c7b8fdb85a64ffe155f03e9e6bc59fcf54fc109b662a5d892
|
data/README.md
CHANGED
|
@@ -73,6 +73,38 @@ filter data from a hash using [JsonPath](http://goessner.net/articles/JsonPath/i
|
|
|
73
73
|
filtered_data = filter(data, "$..metadata.record")
|
|
74
74
|
```
|
|
75
75
|
|
|
76
|
+
#### rules
|
|
77
|
+
|
|
78
|
+
rules allows you to define a simple structure to run against a JSONPath filter
|
|
79
|
+
|
|
80
|
+
A rule is made up of a Hash the key is the map key field its value is a Hash with a JSONPath filter and options to apply a convert method on the filtered results.
|
|
81
|
+
Available convert methods are: time, map, each, call, suffix
|
|
82
|
+
- time: Parses a given time/date string into a Time object
|
|
83
|
+
- map: applies a mapping to a filter
|
|
84
|
+
- suffix: adds a suffix to a result
|
|
85
|
+
- call: executes a lambda on the filter
|
|
86
|
+
- each: runs a lambda on each row of a filter
|
|
87
|
+
|
|
88
|
+
example:
|
|
89
|
+
```ruby
|
|
90
|
+
my_rules = {
|
|
91
|
+
'identifier' => {"filter" => '$..id'},
|
|
92
|
+
'language' => {'filter' => '$..lang',
|
|
93
|
+
'options' => {'convert' => 'map',
|
|
94
|
+
'map' => {'nl' => 'dut', 'fr' => 'fre', 'de' => 'ger', 'en' => 'eng'}
|
|
95
|
+
}
|
|
96
|
+
},
|
|
97
|
+
'subject' => {'filter' => '$..keywords',
|
|
98
|
+
options' => {'convert' => 'each',
|
|
99
|
+
'lambda' => lambda {|d| d.split(',')}
|
|
100
|
+
}
|
|
101
|
+
},
|
|
102
|
+
'creationdate' => {'filter' => '$..published_date', 'convert' => 'time'}
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
rules.run(my_rules, record, output)
|
|
106
|
+
```
|
|
107
|
+
|
|
76
108
|
#### config
|
|
77
109
|
config is an object that points to "config.yml" you can read and/or store data to this object.
|
|
78
110
|
|
data/lib/data_collector/core.rb
CHANGED
|
@@ -47,6 +47,33 @@ module DataCollector
|
|
|
47
47
|
@output ||= Output.new
|
|
48
48
|
end
|
|
49
49
|
|
|
50
|
+
#You can apply rules to input
|
|
51
|
+
# A rule is made up of a Hash the key is the map key field its value is a Hash with a JSONPath filter and
|
|
52
|
+
# options to apply a convert method on the filtered results.
|
|
53
|
+
#
|
|
54
|
+
# available convert methods are: time, map, each, call, suffix
|
|
55
|
+
# - time: Parses a given time/date string into a Time object
|
|
56
|
+
# - map: applies a mapping to a filter
|
|
57
|
+
# - suffix: adds a suffix to a result
|
|
58
|
+
# - call: executes a lambda on the filter
|
|
59
|
+
# - each: runs a lambda on each row of a filter
|
|
60
|
+
#
|
|
61
|
+
# example:
|
|
62
|
+
# my_rules = {
|
|
63
|
+
# 'identifier' => {"filter" => '$..id'},
|
|
64
|
+
# 'language' => {'filter' => '$..lang',
|
|
65
|
+
# 'options' => {'convert' => 'map',
|
|
66
|
+
# 'map' => {'nl' => 'dut', 'fr' => 'fre', 'de' => 'ger', 'en' => 'eng'}
|
|
67
|
+
# }
|
|
68
|
+
# },
|
|
69
|
+
# 'subject' => {'filter' => '$..keywords',
|
|
70
|
+
# options' => {'convert' => 'each',
|
|
71
|
+
# 'lambda' => lambda {|d| d.split(',')}
|
|
72
|
+
# }
|
|
73
|
+
# },
|
|
74
|
+
# 'creationdate' => {'filter' => '$..published_date', 'convert' => 'time'}
|
|
75
|
+
# }
|
|
76
|
+
# rules.run(my_rules, input, output)
|
|
50
77
|
def rules
|
|
51
78
|
@rules ||= Rules.new
|
|
52
79
|
end
|
data/lib/data_collector/rules.rb
CHANGED
|
@@ -25,7 +25,7 @@ module DataCollector
|
|
|
25
25
|
|
|
26
26
|
private
|
|
27
27
|
def apply_rule(map_to_key, rule, from_record, to_record)
|
|
28
|
-
if
|
|
28
|
+
if rule.has_key?('options') && rule['options'].has_key?('convert') && rule['options']['convert'].eql?('each')
|
|
29
29
|
result = get_value_for(map_to_key, rule['filter'], from_record, rule['options'])
|
|
30
30
|
|
|
31
31
|
if result.is_a?(Array)
|