data_collector 0.12.0 → 0.13.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 00ab3a14bc19fd6570cc8061cd39077cbc238816d547385a1babb93f2174dbb8
-  data.tar.gz: 3293a317720d0f72581bad60621572c4fca4caa06fc29f12cbaac9eee6e3e559
+  metadata.gz: 9ada83aba097793e130f92865b86f6fbf9b8e43e23b73e788f8683397d3cf7d3
+  data.tar.gz: 03706e1fc9f239ef7f265b826916b08f98af250bb537214acfef9aa71d5bb1f1
 SHA512:
-  metadata.gz: 2fae16f848aaf7153d9fdce64cfcf2059a2712155fcdb28f0f46cb70d18419f15745ee90fb87b7d0d81dfcf9ebd48badb5aea201ef8d7108a3b2eea9f3479610
-  data.tar.gz: c224c1440dfc879fed651cb1984d363315c1cb45e9f1ee594171d2c5a846f50f167c415a1a83994d509f1757752cb90fa0eba659a3d48c2afdd5cf21fe1c1b28
+  metadata.gz: 9ccbf0850f4e259996eba02f86bd20ac4355020ef2df429b185adea71c9619c6c6a784d967b2e78e131c65f54d7a9369b88f5379fbf2d0f36e112d969b948841
+  data.tar.gz: d6379863103a592300727dea957ce63b5bd51bdf46d3f0f5b1cdd28402172aa8258188ec770892203b20e2688da801458eee20bc026f1c60be094d653259b815

data/README.md

@@ -99,18 +99,18 @@ filter data from a hash using [JSONPath](http://goessner.net/articles/JsonPath/i
 
 #### rules (depricated)
     See newer rules_ng object
-Allows you to define a simple lambda structure to run against a JSONPath filter
-
-A rule is made up of a Hash the key is the map key field its value is a Hash with a JSONPath filter and options to apply a convert method on the filtered results.
-Available convert methods are: time, map, each, call, suffix, text
-  - time: parses a given time/date string into a Time object
-  - map: applies a mapping to a filter
-  - suffix: adds a suffix to a result
-  - call: executes a lambda on the filter
-  - each: runs a lambda on each row of a filter
-  - text: passthrough method. Returns value unchanged
-
- example:
+~~Allows you to define a simple lambda structure to run against a JSONPath filter~~
+
+~~A rule is made up of a Hash the key is the map key field its value is a Hash with a JSONPath filter and options to apply a convert method on the filtered results.~~
+~~Available convert methods are: time, map, each, call, suffix, text~~  
+~~- time: parses a given time/date string into a Time object~~  
+~~- map: applies a mapping to a filter~~  
+~~- suffix: adds a suffix to a result~~  
+~~- call: executes a lambda on the filter~~  
+~~- each: runs a lambda on each row of a filter~~  
+~~- text: passthrough method. Returns value unchanged~~  
+
+~~example:~~
 ```ruby 
  my_rules = {
    'identifier' => {"filter" => '$..id'},
@@ -204,7 +204,11 @@ Here you find different rule combination that are possible
 
 Here is an example on how to call last RULESET "rs_hash_with_json_filter_and_option".
  
-***rules_ng.run*** can have 4 parameters. First 3 are mandatory. The last one ***options*** can hold data static to a rule set.
+***rules_ng.run*** can have 4 parameters. First 3 are mandatory. The last one ***options*** can hold data static to a rule set or engine directives.
+
+List of engine directives:
+  - _no_array_with_one_element: defaults to false. if the result is an array with 1 element just return the element. 
+
 
 ```ruby
     include DataCollector::Core

data/data_collector.gemspec

@@ -49,4 +49,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'bundler', '~> 2.3'
   spec.add_development_dependency 'minitest', '~> 5.16'
   spec.add_development_dependency 'rake', '~> 13.0'
+  spec.add_development_dependency 'webmock', '~> 3.18'
 end

data/lib/data_collector/input.rb

@@ -61,17 +61,33 @@ module DataCollector
 
     def from_https(uri, options = {})
       data = nil
+      HTTP.default_options = HTTP::Options.new(features: { logging: { logger: @logger } })
       http = HTTP
 
-      if options.keys.include?(:user) && options.keys.include?(:password)
+      #http.use(logging: {logger: @logger})
+
+      if options.key?(:user) && options.key?(:password)
+        @logger.debug "Set Basic_auth"
         user = options[:user]
         password = options[:password]
         http = HTTP.basic_auth(user: user, pass: password)
-      else
-        @logger.warn ("User or Password parameter not found")
+      elsif options.key?(:bearer_token)
+        @logger.debug  "Set authorization bearer token"
+        bearer = options[:bearer_token]
+        bearer = "Bearer #{bearer}" unless bearer =~ /^Bearer /i
+        http = HTTP.auth(bearer)
       end
 
-      http_response = http.get(escape_uri(uri))
+      if options.key?(:verify_ssl) && uri.scheme.eql?('https')
+        @logger.warn "Disabling SSL verification. "
+        #shouldn't use this but we all do ...
+        ctx = OpenSSL::SSL::SSLContext.new
+        ctx.verify_mode = OpenSSL::SSL::VERIFY_NONE
+
+        http_response = http.get(escape_uri(uri), ssl_context: ctx)
+      else
+        http_response = http.get(escape_uri(uri))
+      end
 
       case http_response.code
       when 200
@@ -167,6 +183,7 @@ module DataCollector
       file_type = if headers.include?('Content-Type')
                     headers['Content-Type'].split(';').first
                   else
+                    @logger.debug  "No Header content-type available"
                     MIME::Types.of(filename_from(headers)).first.content_type
                   end
 

data/lib/data_collector/output.rb

@@ -82,6 +82,11 @@ module DataCollector
       @data
     end
 
+    def crush
+      data = @data
+      @data = deep_compact(data)
+    end
+
     def clear
       @data = {}
       #GC.start(full_mark: true, immediate_sweep: true)
@@ -216,7 +221,27 @@ module DataCollector
       out
     end
 
-
+    def deep_compact( data )
+      if data.is_a?(Hash)
+        #puts " - Hash - #{data}"
+        data.compact!
+        data.each { |k, v| data[k] = deep_compact(v) }
+        data.compact!
+        data
+      elsif data.is_a?(Array)
+        #puts " - Array - #{data}"
+        data.each { |v| deep_compact(v) }
+        data.empty? ? nil : data
+        #puts " - Array size- #{data.size}"
+        data.size == 1 ? data[0] : data
+      elsif data.is_a?(String)
+        #puts " - String - #{data}"
+        data.blank? ? nil : data
+      else
+        data
+      end
+    end
+  
     private
 
     def tar_file(tar_file_name)

data/lib/data_collector/rules_ng.rb

@@ -97,6 +97,11 @@ module DataCollector
         output_data = output_data.first
       end
 
+      if options.key?('_no_array_with_one_element') && options['_no_array_with_one_element'] &&
+        output_data.is_a?(Array) && output_data.size == 1
+        output_data = output_data.first
+      end
+
       output_data
     end
 

data/lib/data_collector/version.rb

@@ -1,4 +1,4 @@
 # encoding: utf-8
 module DataCollector
-  VERSION = "0.12.0"
+  VERSION = "0.13.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: data_collector
 version: !ruby/object:Gem::Version
-  version: 0.12.0
+  version: 0.13.0
 platform: ruby
 authors:
 - Mehmet Celik
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-08-01 00:00:00.000000000 Z
+date: 2023-02-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -178,6 +178,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '13.0'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.18'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.18'
 description: INPUT, FILTER, OUTPUT data with RULES and code
 email:
 - mehmet@celik.be
@@ -226,7 +240,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
+rubygems_version: 3.1.6
 signing_key:
 specification_version: 4
 summary: ETL helper library