data_collector 0.25.0 → 0.27.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8065c0d1b54cf1c39be5cfcb70a4fae1f33b02021182c75538ed03e73393d35a
-  data.tar.gz: 879054ad24b178f08bfea7914c5fbc593c32c611bd4e3ad833869a3e5b36d5b1
+  metadata.gz: b001584d7cb8f525f22370a79a4921104358f9de512d81da6236f65a590220bc
+  data.tar.gz: 773bf24b624e28e056a82f0c665e603a119830c9cc872d1938be02ab7251ad03
 SHA512:
-  metadata.gz: 390ac889c52055cfd8f5326c6e7c1549faee6b8c41af4535b9fc5d3038701f62c441caaf41895d7ad64b4941e609fdee32ac745255b9cb2fbc0981d787b00847
-  data.tar.gz: dbc57c97f30e5659ccfebba0850e99eee24ecada4a5e9a136ae6b036f19b587a4d66460e509c32d36be13cd4d745ed89f9a73e11a32b1b15fe800036c1836bea
+  metadata.gz: 8d0a62ddf8408c72e6277080f7105bd8d12f39c2f30e4a44910d890932a50ef9399d096d98b990b87db422a6af7557821e45045a049c64ebbdd90d8cd811062c
+  data.tar.gz: 53e2290744797a8763ccd0491a02c9dd1ee4e73919f95c20d66bb03889cca18f31cb4090728f1ef3acee6b76ef9e062f4196d17b90b5bf0ec3d2132e74543c2a

data/README.md

@@ -8,6 +8,7 @@ include DataCollector::Core
 ```
 Every object can be used on its own.
 
+### DataCollector Objects
 #### Pipeline
 Allows you to create a simple pipeline of operations to process data. With a data pipeline, you can collect, process, and transform data, and then transfer it to various systems and applications.
 
@@ -20,6 +21,7 @@ executed in the [ISO8601 duration format](https://www.digi.com/resources/documen
      - schedule: [ISO8601 duration format](https://www.digi.com/resources/documentation/digidocs//90001488-13/reference/r_iso_8601_duration_format.htm)
      - cron: in cron format ex. '1 12 * * *' intervals are not supported
      - uri: a directory/file to watch
+     - xml_typecast: true/false -> convert convert string values to TrueClass, FalseClass, Time, Date, and DateTime
  - .run: start the pipeline. blocking if a schedule is supplied
  - .stop: stop the pipeline
  - .pause: pause the pipeline. Restart using .run
@@ -74,12 +76,14 @@ For a push input component, a listener is created with a processing logic block
 A push happens when new data is created in a directory, message queue, ...
 
 ```ruby
-  from_uri(source, options = {:raw, :content_type})
+  from_uri(source, options = {:raw, :content_type, :headers, :cookies})
 ```
 - source: an uri with a scheme of http, https, file, amqp
 - options:
     - raw: _boolean_ do not parse
     - content_type: _string_ force a content_type if the 'Content-Type' returned by the http server is incorrect 
+    - headers: request headers
+    - cookies: session cookies etc.
 
 ###### example:
 ```ruby  

data/lib/data_collector/input/rpc.rb

@@ -55,7 +55,7 @@ module DataCollector
           parse_uri
           server.rabbitmq_url = @bunny_uri.to_s
           server.rabbitmq_exchange = @bunny_channel
-          server.logger = DataCollector::Core.logger
+          #server.logger = DataCollector::Core.logger
         end
       end
 

data/lib/data_collector/input.rb

@@ -31,7 +31,7 @@ module DataCollector
       raise DataCollector::Error 'from_uri expects a scheme like file:// of https://' unless source =~ /:\/\//
 
       scheme, path = source.split('://')
-      source="#{scheme}://#{URI.encode_uri_component(path)}"
+      source="#{scheme}://#{URI.encode_www_form_component(path)}"
       uri = URI(source)
       begin
         data = nil
@@ -41,7 +41,7 @@ module DataCollector
         when 'https'
           data = from_https(uri, options)
         when 'file'
-          absolute_path = File.absolute_path("#{URI.decode_uri_component("#{uri.host}#{uri.path}")}")
+          absolute_path = File.absolute_path("#{URI.decode_www_form_component("#{uri.host}#{uri.path}")}")
           if File.directory?(absolute_path)
             #raise DataCollector::Error, "#{uri.host}/#{uri.path} not found" unless File.exist?("#{uri.host}/#{uri.path}")
             return from_dir(uri, options)
@@ -80,7 +80,7 @@ module DataCollector
     end
 
     def from_https(uri, options = {})
-      uri = URI.decode_uri_component("#{uri.to_s}")
+      uri = URI.decode_www_form_component("#{uri.to_s}")
       data = nil
       if options.with_indifferent_access.include?(:logging) && options.with_indifferent_access[:logging]
         HTTP.default_options = HTTP::Options.new(features: { logging: { logger: @logger } })
@@ -102,6 +102,16 @@ module DataCollector
         http = HTTP.auth(bearer)
       end
 
+      if options.key?(:cookies) 
+        @logger.debug  "Set cookies"
+        http = http.cookies( options[:cookies] )
+      end
+
+      if options.key?(:headers) 
+        @logger.debug  "Set http headers"
+        http = http.headers( options[:headers] )
+      end
+          
       if options.key?(:verify_ssl) && uri.scheme.eql?('https')
         @logger.warn "Disabling SSL verification. "
         #shouldn't use this but we all do ...
@@ -109,10 +119,11 @@ module DataCollector
         ctx.verify_mode = OpenSSL::SSL::VERIFY_NONE
 
         http_response = http.follow.get(escape_uri(uri), ssl_context: ctx)
+
       else
         http_response = http.follow.get(escape_uri(uri))
       end
-
+ 
       case http_response.code
       when 200..299
         @raw = data = http_response.body.to_s
@@ -130,15 +141,15 @@ module DataCollector
           when 'application/json'
             data = JSON.parse(data)
           when 'application/atom+xml'
-            data = xml_to_hash(data)
+            data = xml_to_hash(data, options)
           when 'text/csv'
             data = csv_to_hash(data)
           when 'application/xml'
-            data = xml_to_hash(data)
+            data = xml_to_hash(data, options)
           when 'text/xml'
-            data = xml_to_hash(data)
+            data = xml_to_hash(data, options)
           else
-            data = xml_to_hash(data)
+            data = xml_to_hash(data, options)
           end
         end
 
@@ -171,14 +182,14 @@ module DataCollector
         when '.json'
           data = JSON.parse(data)
         when '.xml'
-          data = xml_to_hash(data)
+          data = xml_to_hash(data, options)
         when '.gz'
           Minitar.open(Zlib::GzipReader.new(File.open("#{absolute_path}", 'rb'))) do |i|
             i.each do |entry|
               data = entry.read
             end
           end
-          data = xml_to_hash(data)
+          data = xml_to_hash(data, options)
         when '.csv'
           data = csv_to_hash(data)
         else
@@ -201,10 +212,11 @@ module DataCollector
       DataCollector::Input::Rpc.new(uri, options)
     end
 
-    def xml_to_hash(data)
+    def xml_to_hash(data, options = {})
       #gsub('<\/', '< /') outherwise wrong XML-parsing (see records lirias1729192 )
       data = data.gsub /</, '< /'
-      nori = Nori.new(parser: :nokogiri, strip_namespaces: true, convert_tags_to: lambda { |tag| tag.gsub(/^@/, '_') })
+      xml_typecast = options.with_indifferent_access.key?('xml_typecast') ? options.with_indifferent_access['xml_typecast'] : true
+      nori = Nori.new(parser: :nokogiri, advanced_typecasting: xml_typecast, strip_namespaces: true, convert_tags_to: lambda { |tag| tag.gsub(/^@/, '_') })
       nori.parse(data)
       #JSON.parse(nori.parse(data).to_json)
     end
@@ -235,7 +247,7 @@ module DataCollector
     end
 
     def normalize_uri(uri)
-      "#{URI.decode_uri_component(uri.host)}#{URI.decode_uri_component(uri.path)}"
+      "#{URI.decode_www_form_component(uri.host)}#{URI.decode_www_form_component(uri.path)}"
     end
   end
 end

data/lib/data_collector/version.rb

@@ -1,4 +1,4 @@
 # encoding: utf-8
 module DataCollector
-  VERSION = "0.25.0"
+  VERSION = "0.27.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: data_collector
 version: !ruby/object:Gem::Version
-  version: 0.25.0
+  version: 0.27.0
 platform: ruby
 authors:
 - Mehmet Celik
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-07-31 00:00:00.000000000 Z
+date: 2023-08-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport