data_collector 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 54fed64f173cd683fa66d514e65c74d42bb992965fdb85ca15df823e97e4ded1
-  data.tar.gz: 18170f3fd1660fef26448961ab5e5af22c42af405750f54873e6cb650718f3b2
+  metadata.gz: 5fb3cc5c42f536a8bdc99421eb6f46972c6c97597f5c0af63e96463f320837ef
+  data.tar.gz: d377133bf042985f6d68b2bb2960d9c4ddae169a39d4602df4a3f4003d6b6434
 SHA512:
-  metadata.gz: ddb5f0e7b5619cbf29ba7a396381ac0bdba315ac94aef9b6d210f4184f47e028f47e4d96608268c83dcbb8c3d291df3bc23493c76ea11a57f33802d1c32e4903
-  data.tar.gz: 805a7b7775abab148ed13be3e70bd7dd16e8de4f9335b7b109e38b99e4298008ea05e093df835bf519e886de9b134ed43a50a6dde462d5c2f2623bfa9a591213
+  metadata.gz: cd48af243e1615e8cafaf8ed1ab4d9019661e3205286fbd32c9ec4bb3e2faa7e07025a62b378062f10dfc0f1775acb401ad4cb901851cbbaf4cb78a2e7c6474a
+  data.tar.gz: c92eee26040f89ab0b691ba6bf4ad9792222e545e82bb9ebea8b4a4941831615a74d9f389e58d6feccf89ba42c6dd5a02c55bfed996fc670cd1d9600d5023476

data/README.md

@@ -1,8 +1,13 @@
 # DataCollector
 Convenience module to Extract, Transform and Load your data.
+You have 3 main objects that you can use for ETL => INPUT, OUTPUT and FILTER
+and support objects like CONFIG, LOG, RULES and the new RULES_NG
+
+Including the DataCollector::Core module into your application gives you access to these objects.
+ 
 
 #### input    
-Read input from an URI
+Read input from an URI. This URI can have a http, https or file scheme
 
 **Public methods**
 ```ruby
@@ -26,7 +31,7 @@ example:
 Inputs can be JSON, XML or CSV or XML in a TAR.GZ file
 
  ### output 
-Output is an object you can store data that needs to be written to an output stream.  
+Output is an object you can store key/value pairs that needs to be written to an output stream.  
 ```ruby  
     output[:name] = 'John'
     output[:last_name] = 'Doe'
@@ -88,15 +93,17 @@ filter data from a hash using [JSONPath](http://goessner.net/articles/JsonPath/i
 ```
 
 #### rules
+    See newer rules_ng object
 Allows you to define a simple lambda structure to run against a JSONPath filter
 
 A rule is made up of a Hash the key is the map key field its value is a Hash with a JSONPath filter and options to apply a convert method on the filtered results.
-Available convert methods are: time, map, each, call, suffix
+Available convert methods are: time, map, each, call, suffix, text
   - time: parses a given time/date string into a Time object
   - map: applies a mapping to a filter
   - suffix: adds a suffix to a result
   - call: executes a lambda on the filter
   - each: runs a lambda on each row of a filter
+  - text: passthrough method. Returns value unchanged
 
  example:
 ```ruby 
@@ -118,6 +125,19 @@ Available convert methods are: time, map, each, call, suffix
 rules.run(my_rules, record, output)
 ```    
 
+#### rules_ng
+!!! not compatible with RULES object
+
+TODO: work in progress see test for examples on how to use
+
+```
+RULE_SET
+    RULES*
+        FILTERS*
+            LAMBDA*
+            SUFFIX
+```
+
 #### config
 config is an object that points to "config.yml" you can read and/or store data to this object.
 

data/data_collector.gemspec

@@ -47,6 +47,6 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency "json-ld", "~> 3.1"
 
   spec.add_development_dependency "bundler", "~> 2.0"
-  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rake", ">= 12.3"
   spec.add_development_dependency "minitest", "~> 5.0"
 end

data/lib/data_collector/core.rb

@@ -5,6 +5,7 @@ require 'logger'
 require_relative 'input'
 require_relative 'output'
 require_relative 'rules'
+require_relative 'rules_ng'
 require_relative 'config_file'
 
 module DataCollector
@@ -79,6 +80,11 @@ module DataCollector
       @rules ||= Rules.new
     end
 
+    # New rules runner
+    def rules_ng
+      @rules_ng ||= RulesNg.new
+    end
+
     # evaluator http://jsonpath.com/
     # uitleg http://goessner.net/articles/JsonPath/index.html
     def filter(data, filter_path)

data/lib/data_collector/rules.rb

@@ -6,31 +6,32 @@ module DataCollector
       @logger = Logger.new(STDOUT)
     end
 
-    def run(rule_map, from_record, to_record)
+    def run(rule_map, from_record, to_record, options = {})
       rule_map.each do |map_to_key, rule|
         if rule.is_a?(Array)
           rule.each do |sub_rule|
-            apply_rule(map_to_key, sub_rule, from_record, to_record)
+            apply_rule(map_to_key, sub_rule, from_record, to_record, options)
           end
         else
-          apply_rule(map_to_key, rule, from_record, to_record)
+          apply_rule(map_to_key, rule, from_record, to_record, options)
         end
       end
 
       to_record.each do |element|
         element = element.delete_if do |k, v|
-          v != false && (v.nil? || v.empty?)
+          v != false && (v.nil?)
         end
       end
     end
 
     private
 
-    def apply_rule(map_to_key, rule, from_record, to_record)
+    def apply_rule(map_to_key, rule, from_record, to_record, options = {})
       if rule.has_key?('text')
-        to_record << { map_to_key.to_sym => rule['text'] }
+        suffix = (rule && rule.key?('options') && rule['options'].key?('suffix')) ? rule['options']['suffix'] : ''
+        to_record << { map_to_key.to_sym => add_suffix(rule['text'], suffix) }
       elsif rule.has_key?('options') && rule['options'].has_key?('convert') && rule['options']['convert'].eql?('each')
-        result = get_value_for(map_to_key, rule['filter'], from_record, rule['options'])
+        result = get_value_for(map_to_key, rule['filter'], from_record, rule['options'], options)
 
         if result.is_a?(Array)
           result.each do |m|
@@ -40,14 +41,14 @@ module DataCollector
           to_record << {map_to_key.to_sym => result}
         end
       else
-        result = get_value_for(map_to_key, rule['filter'], from_record, rule['options'])
+        result = get_value_for(map_to_key, rule['filter'], from_record, rule['options'], options)
         return if result && result.empty?
 
         to_record << {map_to_key.to_sym => result}
       end
     end
 
-    def get_value_for(tag_key, filter_path, record, options = {})
+    def get_value_for(tag_key, filter_path, record, rule_options = {}, options = {})
       data = nil
       if record
         if filter_path.is_a?(Array) && !record.is_a?(Array)
@@ -56,34 +57,47 @@ module DataCollector
 
         data = Core::filter(record, filter_path)
 
-        if data && options
-          if options.key?('convert')
-            case options['convert']
+        if data && rule_options
+          if rule_options.key?('convert')
+            case rule_options['convert']
             when 'time'
-              data = Time.parse(data).strftime('%Y-%m-%d')
+              result = []
+              data = [data] unless data.is_a?(Array)
+              data.each do |d|
+                result << Time.parse(d)
+              end
+              data = result
             when 'map'
               if data.is_a?(Array)
                 data = data.map do |r|
-                  return options['map'][r] if options['map'].key?(r)
+                  rule_options['map'][r] if rule_options['map'].key?(r)
                 end
 
                 data.compact!
-                data.flatten! if options.key?('flatten') && options['flatten']
+                data.flatten! if rule_options.key?('flatten') && rule_options['flatten']
               else
-                return options['map'][data] if options['map'].key?(data)
+                return rule_options['map'][data] if rule_options['map'].key?(data)
               end
             when 'each'
               data = [data] unless data.is_a?(Array)
-              data = data.map { |d| options['lambda'].call(d) }
-              data.flatten! if options.key?('flatten') && options['flatten']
+              if options.empty?
+                data = data.map { |d| rule_options['lambda'].call(d) }
+              else
+                data = data.map { |d| rule_options['lambda'].call(d, options) }
+              end
+              data.flatten! if rule_options.key?('flatten') && rule_options['flatten']
             when 'call'
-              return options['lambda'].call(data)
+              if options.empty?
+                data = rule_options['lambda'].call(data)
+              else
+                data = rule_options['lambda'].call(data, options)
+              end
+              return data
             end
           end
 
-          if options.key?('suffix')
-            data = data.to_s
-            data += options['suffix']
+          if rule_options.key?('suffix')
+            data = add_suffix(data, rule_options['suffix'])
           end
 
         end
@@ -93,5 +107,24 @@ module DataCollector
       return data
     end
 
+    def add_suffix(data, suffix)
+      case data.class.name
+      when 'Array'
+        result = []
+        data.each do |d|
+          result <<  add_suffix(d, suffix)
+        end
+        data = result
+      when 'Hash'
+        data.each do |k, v|
+          data[k] = add_suffix(v, suffix)
+        end
+      else
+        data = data.to_s
+        data += suffix
+      end
+      data
+    end
+
   end
 end

data/lib/data_collector/rules_ng.rb

@@ -0,0 +1,88 @@
+require 'logger'
+
+module DataCollector
+  class RulesNg
+    def initialize(logger = Logger.new(STDOUT))
+      @logger = logger
+    end
+
+    def run(rules, input_data, output_data, options = {})
+      rules.each do |tag, rule|
+        apply_rule(tag, rule, input_data, output_data, options)
+      end
+    end
+
+    private
+    def apply_rule(tag, rule, input_data, output_data, options = {})
+      if rule.is_a?(Array)
+        rule.each do |sub_rule|
+           apply_rule(tag, sub_rule, input_data, output_data, options)
+        end
+        return output_data
+      end
+
+      rule_filter = rule.keys.first
+      rule_payload = rule.values.first
+      case rule_filter
+      when 'text'
+        if rule_payload.is_a?(String)
+          data = rule_payload
+        else
+          data = rule_payload.select{|s| s.is_a?(String)}
+          rule_payload = rule_payload.delete_if{|s| s.is_a?(String)}
+          if rule_payload.size == 1
+            rule_payload = rule_payload.first
+          end
+        end
+      when /json_path\:/
+        data = json_path_filter(rule_filter.gsub(/^json_path\:/), input_data)
+      else
+        data = json_path_filter(rule_filter, input_data)
+      end
+
+      data = apply_filtered_data_on_payload(data, rule_payload, options)
+
+      output_data << {tag.to_sym => data}
+    end
+
+    def apply_filtered_data_on_payload(input_data, payload, options = {})
+      output_data = nil
+      case payload.class.name
+      when 'Proc'
+        if options && options.empty?
+          output_data = payload.call(input_data)
+        else
+          output_data = payload.call(input_data, options)
+        end
+      when 'Hash'
+        input_data = [input_data] unless input_data.is_a?(Array)
+        output_data = input_data.map do |m|
+          if payload.key?('suffix')
+            "#{m}#{payload['suffix']}"
+          else
+            payload[m]
+          end
+        end if input_data.is_a?(Array)
+      when 'Array'
+        output_data = input_data
+        payload.each do |p|
+          output_data = apply_filtered_data_on_payload(output_data, p, options)
+        end
+      else
+        output_data = input_data
+      end
+
+      output_data.compact! if output_data.is_a?(Array)
+      output_data
+    end
+
+    def json_path_filter(filter, input_data)
+      data = nil
+      return data if input_data.nil? || input_data.empty?
+      return input_data if input_data.is_a?(String)
+      Core::filter(input_data, filter)
+    end
+
+
+  end
+end

data/lib/data_collector/version.rb

@@ -1,4 +1,4 @@
 # encoding: utf-8
 module DataCollector
-  VERSION = "0.3.0"
+  VERSION = "0.4.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: data_collector
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Mehmet Celik
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-02-11 00:00:00.000000000 Z
+date: 2020-03-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
@@ -154,16 +154,16 @@ dependencies:
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '12.3'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '12.3'
 - !ruby/object:Gem::Dependency
   name: minitest
   requirement: !ruby/object:Gem::Requirement
@@ -201,6 +201,7 @@ files:
 - lib/data_collector/input.rb
 - lib/data_collector/output.rb
 - lib/data_collector/rules.rb
+- lib/data_collector/rules_ng.rb
 - lib/data_collector/runner.rb
 - lib/data_collector/version.rb
 homepage: https://github.com/mehmetc/data_collector
@@ -225,7 +226,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.6
+rubygems_version: 3.0.2
 signing_key: 
 specification_version: 4
 summary: ETL helper library