data_collector 0.4.0 → 0.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5fb3cc5c42f536a8bdc99421eb6f46972c6c97597f5c0af63e96463f320837ef
-  data.tar.gz: d377133bf042985f6d68b2bb2960d9c4ddae169a39d4602df4a3f4003d6b6434
+  metadata.gz: bfc42cf6807d68d0bf356494918fb3309ad19de5b037e9fac5f374c82a267ef6
+  data.tar.gz: a13d553f5003943755cc3e14dbdfd3fbf22a6ee00625ace045e468dba7ebf160
 SHA512:
-  metadata.gz: cd48af243e1615e8cafaf8ed1ab4d9019661e3205286fbd32c9ec4bb3e2faa7e07025a62b378062f10dfc0f1775acb401ad4cb901851cbbaf4cb78a2e7c6474a
-  data.tar.gz: c92eee26040f89ab0b691ba6bf4ad9792222e545e82bb9ebea8b4a4941831615a74d9f389e58d6feccf89ba42c6dd5a02c55bfed996fc670cd1d9600d5023476
+  metadata.gz: 6ec4eab7ef70145e63a84c814683e5faf026ab37d51e687f24b456a91d66e1bbb22955ec2de70f59eb6c877f1c133c1b62344dc7382abaa859f36c85d82a0fa3
+  data.tar.gz: d1f88a5d04f68c699837032f2f74fd30ebfcad7a492dc0009c211e9462a6c8f711141809dc1f01ad1b38de604353a490008d0470b767278dcd088ef2d6f37ef9

data/README.md

@@ -1,10 +1,15 @@
 # DataCollector
-Convenience module to Extract, Transform and Load your data.
-You have 3 main objects that you can use for ETL => INPUT, OUTPUT and FILTER
-and support objects like CONFIG, LOG, RULES and the new RULES_NG
+Convenience module to Extract, Transform and Load your data.  
+You have main objects that help you to 'INPUT', 'OUTPUT' and 'FILTER' data. The basic ETL components.
+Support objects like CONFIG, LOG, RULES and the new RULES_NG just to make life easier.
 
 Including the DataCollector::Core module into your application gives you access to these objects.
  
+The RULES and RULES_NG objects work in a very simple concept. Rules exist of 3 components:
+ - a destination tag
+ - a jsonpath filter to get the data 
+ - a lambda to execute on every filter hit 
+
 
 #### input    
 Read input from an URI. This URI can have a http, https or file scheme
@@ -92,7 +97,7 @@ filter data from a hash using [JSONPath](http://goessner.net/articles/JsonPath/i
     filtered_data = filter(data, "$..metadata.record")
 ```
 
-#### rules
+#### rules (depricated)
     See newer rules_ng object
 Allows you to define a simple lambda structure to run against a JSONPath filter
 
@@ -138,6 +143,90 @@ RULE_SET
             SUFFIX
 ```
 
+##### Examples
+
+Here you find different rule combination that are possible
+
+``` ruby
+ RULE_SETS = {
+      'rs_only_filter' => {
+          'only_filter' => "$.title"
+      },
+      'rs_only_text' => {
+          'plain_text_tag' => {
+              'text' => 'hello world'
+          }
+      },
+      'rs_text_with_suffix' => {
+          'text_tag_with_suffix' => {
+              'text' => ['hello_world', {'suffix' => '-suffix'}]
+          }
+      },
+      'rs_map_with_json_filter' => {
+          'language' => {
+              '@' => {'nl' => 'dut', 'fr' => 'fre', 'de' => 'ger', 'en' => 'eng'}
+          }
+      },
+      'rs_hash_with_json_filter' => {
+          'multiple_of_2' => {
+              '@' => lambda { |d| d.to_i * 2 }
+          }
+      },
+      'rs_hash_with_multiple_json_filter' => {
+          'multiple_of' => [
+              {'@' => lambda { |d| d.to_i * 2 }},
+              {'@' => lambda { |d| d.to_i * 3 }}
+          ]
+      },
+      'rs_hash_with_json_filter_and_suffix' => {
+          'multiple_of_with_suffix' => {
+              '@' => [lambda {|d| d.to_i*2}, 'suffix' => '-multiple_of_2']
+          }
+      },
+      'rs_hash_with_json_filter_and_multiple_lambdas' => {
+          'multiple_lambdas' => {
+              '@' => [lambda {|d| d.to_i*2}, lambda {|d| Math.sqrt(d.to_i) }]
+          }
+      },
+      'rs_hash_with_json_filter_and_option' => {
+          'subjects' => {
+              '$..subject' => [
+                  lambda {|d,o|
+                    {
+                        doc_id: o['id'],
+                        subject: d
+                    }
+                  }
+              ]
+          }
+      }
+```
+
+Here is an example on how to call last RULESET "rs_hash_with_json_filter_and_option".
+ 
+***rules_ng.run*** can have 4 parameters. First 3 are mandatory. The last one ***options*** can hold data static to a rule set.
+
+```ruby
+    include DataCollector::Core
+    output.clear
+    data = {'subject' => ['water', 'thermodynamics']}
+
+    rules_ng.run(RULE_SETS['rs_hash_with_json_filter_and_option'], data, output, {'id' => 1})
+
+```
+
+Results in: 
+```json
+  {
+   "subjects":[
+                {"doc_id":1,"subject":"water"},
+                {"doc_id":1,"subject":"thermodynamics"}
+              ]
+  }
+```
+
+
+
 #### config
 config is an object that points to "config.yml" you can read and/or store data to this object.
 

data/lib/data_collector/output.rb

@@ -32,7 +32,11 @@ module DataCollector
       unless v.nil?
         if data.has_key?(k)
           if data[k].is_a?(Array) then
-            data[k] << v
+            if v.is_a?(Array)
+              data[k] += v
+            else
+              data[k] << v
+            end
           else
             t = data[k]
             data[k] = Array.new([t, v])
@@ -58,6 +62,22 @@ module DataCollector
       end
     end
 
+    def key?(key)
+      @data.key?(key)
+    end
+
+    def has_key?(key)
+      @data.key?(key)
+    end
+
+    def include?(key)
+      @data.key?(key)
+    end
+
+    def keys
+      @data.keys
+    end
+
     def raw
       @data
     end

data/lib/data_collector/rules_ng.rb

@@ -10,29 +10,38 @@ module DataCollector
       rules.each do |tag, rule|
         apply_rule(tag, rule, input_data, output_data, options)
       end
+
+      output_data
     end
 
     private
+
     def apply_rule(tag, rule, input_data, output_data, options = {})
-      if rule.is_a?(Array)
+      rule_filter = rule
+      rule_payload = ""
+
+      case rule
+      when Array
         rule.each do |sub_rule|
-           apply_rule(tag, sub_rule, input_data, output_data, options)
+          apply_rule(tag, sub_rule, input_data, output_data, options)
         end
         return output_data
+      when String
+        rule_filter = rule
+        rule_payload = ""
+      else
+        rule_filter = rule.keys.first
+        rule_payload = rule.values.first
       end
 
-      rule_filter = rule.keys.first
-      rule_payload = rule.values.first
       case rule_filter
       when 'text'
         if rule_payload.is_a?(String)
           data = rule_payload
         else
-          data = rule_payload.select{|s| s.is_a?(String)}
-          rule_payload = rule_payload.delete_if{|s| s.is_a?(String)}
-          if rule_payload.size == 1
-            rule_payload = rule_payload.first
-          end
+          data = rule_payload.select { |s| s.is_a?(String) }
+          rule_payload = rule_payload.reject { |s| s.is_a?(String) }
+          rule_payload = "@" if rule_payload.empty?
         end
       when /json_path\:/
         data = json_path_filter(rule_filter.gsub(/^json_path\:/), input_data)
@@ -42,17 +51,22 @@ module DataCollector
 
       data = apply_filtered_data_on_payload(data, rule_payload, options)
 
-      output_data << {tag.to_sym => data}
+      output_data << {tag.to_sym => data} unless data.nil? || (data.is_a?(Array) && data.empty?)
+    rescue StandardError => e
+      puts "error running rule '#{tag}'\n\t#{e.message}"
+      puts e.backtrace.join("\n")
     end
 
     def apply_filtered_data_on_payload(input_data, payload, options = {})
+      return nil if input_data.nil?
       output_data = nil
       case payload.class.name
       when 'Proc'
+        data = input_data.is_a?(Array) ? input_data : [input_data]
         if options && options.empty?
-          output_data = payload.call(input_data)
+          output_data = data.map { |d| payload.call(d) }
         else
-          output_data = payload.call(input_data, options)
+          output_data = data.map { |d| payload.call(d, options) }
         end
       when 'Hash'
         input_data = [input_data] unless input_data.is_a?(Array)
@@ -69,10 +83,13 @@ module DataCollector
           output_data = apply_filtered_data_on_payload(output_data, p, options)
         end
       else
-        output_data = input_data
+        output_data = [input_data]
       end
 
       output_data.compact! if output_data.is_a?(Array)
+      output_data.flatten! if output_data.is_a?(Array)# || output_data.is_a?(Hash)
+      output_data = output_data.first if output_data.is_a?(Array) && output_data.size == 1 && (output_data.first.is_a?(Array) || output_data.first.is_a?(Hash))
+
       output_data
     end
 

data/lib/data_collector/version.rb

@@ -1,4 +1,4 @@
 # encoding: utf-8
 module DataCollector
-  VERSION = "0.4.0"
+  VERSION = "0.9.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: data_collector
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.9.0
 platform: ruby
 authors:
 - Mehmet Celik
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-03-20 00:00:00.000000000 Z
+date: 2020-06-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
@@ -226,7 +226,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.2
+rubygems_version: 3.0.6
 signing_key: 
 specification_version: 4
 summary: ETL helper library