data_collector 0.5.0 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f8b064bcc0483a98014157586cb18473d240486f3e4c1550b9eeba0cc026af6b
-  data.tar.gz: 503c959ca4d3ac06e65e41f0f721aed795b27871d220d676eeb6bc7b81a169d1
+  metadata.gz: 78a054b825d99611c8f5070120c11b478975152433b3c76b9a63a6694b7c8cd0
+  data.tar.gz: f11143808a8e1647a791b0e12e1c5855dc1589de37f7ea50c0e8789f88538774
 SHA512:
-  metadata.gz: 34b2b890765598022e97ee914d6159bd6741ac6d2f711189c81144e880a6590b74183355417d32c3cf39197aad0be914c5d6c6c0089379beac3e14c4bf0b1b01
-  data.tar.gz: 860aa1bab4a0a0c580840f12bf42f77b76598a7433b033211d200647117c91754b49c76540cb52bf58a96184498293c8c0e73659481a7a938277dd53cbd9f7e1
+  metadata.gz: 0b3396b16e5500add7dbb8e2883e5bfd8e14ba898ce7291ed876ed1a423a59a2dc9bba42e1c96150a7119284398c58c18daf8ea7dc82844e3cee55d9ea17fe7f
+  data.tar.gz: a7f7821eb19e50918321b8d1b47caa4ab9d04d40b2918a502f86512fbe25b8c35b4bee0705eb1044ade83cdb7f0c0f45a4023363e14c8153283ed9bdfec32f1a

data/README.md

@@ -1,10 +1,15 @@
 # DataCollector
-Convenience module to Extract, Transform and Load your data.
-You have 3 main objects that you can use for ETL => INPUT, OUTPUT and FILTER
-and support objects like CONFIG, LOG, RULES and the new RULES_NG
+Convenience module to Extract, Transform and Load your data.  
+You have main objects that help you to 'INPUT', 'OUTPUT' and 'FILTER' data. The basic ETL components.
+Support objects like CONFIG, LOG, RULES and the new RULES_NG just to make life easier.
 
 Including the DataCollector::Core module into your application gives you access to these objects.
  
+The RULES and RULES_NG objects work in a very simple concept. Rules exist of 3 components:
+ - a destination tag
+ - a jsonpath filter to get the data 
+ - a lambda to execute on every filter hit 
+
 
 #### input    
 Read input from an URI. This URI can have a http, https or file scheme
@@ -92,7 +97,7 @@ filter data from a hash using [JSONPath](http://goessner.net/articles/JsonPath/i
     filtered_data = filter(data, "$..metadata.record")
 ```
 
-#### rules
+#### rules (depricated)
     See newer rules_ng object
 Allows you to define a simple lambda structure to run against a JSONPath filter
 
@@ -138,6 +143,90 @@ RULE_SET
             SUFFIX
 ```
 
+##### Examples
+
+Here you find different rule combination that are possible
+
+``` ruby
+ RULE_SETS = {
+      'rs_only_filter' => {
+          'only_filter' => "$.title"
+      },
+      'rs_only_text' => {
+          'plain_text_tag' => {
+              'text' => 'hello world'
+          }
+      },
+      'rs_text_with_suffix' => {
+          'text_tag_with_suffix' => {
+              'text' => ['hello_world', {'suffix' => '-suffix'}]
+          }
+      },
+      'rs_map_with_json_filter' => {
+          'language' => {
+              '@' => {'nl' => 'dut', 'fr' => 'fre', 'de' => 'ger', 'en' => 'eng'}
+          }
+      },
+      'rs_hash_with_json_filter' => {
+          'multiple_of_2' => {
+              '@' => lambda { |d| d.to_i * 2 }
+          }
+      },
+      'rs_hash_with_multiple_json_filter' => {
+          'multiple_of' => [
+              {'@' => lambda { |d| d.to_i * 2 }},
+              {'@' => lambda { |d| d.to_i * 3 }}
+          ]
+      },
+      'rs_hash_with_json_filter_and_suffix' => {
+          'multiple_of_with_suffix' => {
+              '@' => [lambda {|d| d.to_i*2}, 'suffix' => '-multiple_of_2']
+          }
+      },
+      'rs_hash_with_json_filter_and_multiple_lambdas' => {
+          'multiple_lambdas' => {
+              '@' => [lambda {|d| d.to_i*2}, lambda {|d| Math.sqrt(d.to_i) }]
+          }
+      },
+      'rs_hash_with_json_filter_and_option' => {
+          'subjects' => {
+              '$..subject' => [
+                  lambda {|d,o|
+                    {
+                        doc_id: o['id'],
+                        subject: d
+                    }
+                  }
+              ]
+          }
+      }
+```
+
+Here is an example on how to call last RULESET "rs_hash_with_json_filter_and_option".
+ 
+***rules_ng.run*** can have 4 parameters. First 3 are mandatory. The last one ***options*** can hold data static to a rule set.
+
+```ruby
+    include DataCollector::Core
+    output.clear
+    data = {'subject' => ['water', 'thermodynamics']}
+
+    rules_ng.run(RULE_SETS['rs_hash_with_json_filter_and_option'], data, output, {'id' => 1})
+
+```
+
+Results in: 
+```json
+  {
+   "subjects":[
+                {"doc_id":1,"subject":"water"},
+                {"doc_id":1,"subject":"thermodynamics"}
+              ]
+  }
+```
+
+
+
 #### config
 config is an object that points to "config.yml" you can read and/or store data to this object.
 

data/lib/data_collector/rules_ng.rb

@@ -41,6 +41,7 @@ module DataCollector
         else
           data = rule_payload.select { |s| s.is_a?(String) }
           rule_payload = rule_payload.delete_if { |s| s.is_a?(String) }
+          rule_payload = "@" if rule_payload.empty?
           # if rule_payload.size == 1
           #   rule_payload = rule_payload.first
           # end
@@ -88,9 +89,9 @@ module DataCollector
         output_data = [input_data]
       end
 
-      output_data = output_data.first if output_data.is_a?(Array) && output_data.size == 1
       output_data.compact! if output_data.is_a?(Array)
       output_data.flatten! if output_data.is_a?(Array)# || output_data.is_a?(Hash)
+      output_data = output_data.first if output_data.is_a?(Array) && output_data.size == 1
       output_data
     end
 

data/lib/data_collector/version.rb

@@ -1,4 +1,4 @@
 # encoding: utf-8
 module DataCollector
-  VERSION = "0.5.0"
+  VERSION = "0.6.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: data_collector
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.6.0
 platform: ruby
 authors:
 - Mehmet Celik
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-04-01 00:00:00.000000000 Z
+date: 2020-04-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri