data_collector 0.26.0 → 0.28.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 057dac92f5a83458b244a962ef0518fa093365a38c2388582de0556ac7b884ca
-  data.tar.gz: 9aebb908d52dda01dc3efce7dcda3e40c8b4f45e718cef94052c9d37fca604eb
+  metadata.gz: 71b519b75704a9bb6e46601c8195ab69608f19fae12ecde7c6c57869b8fca450
+  data.tar.gz: aa4a0697ffb9f6bc21c45f2ecdfa13c9deca7c8b69f2e8159f4901ab6be8d646
 SHA512:
-  metadata.gz: aa91e78ec3aef0010553886539284af4aafb3ed7014ef35e6f7094377bf9ad2efb7f6025bc7f8715d4558d8a5a1242325761a12a6e696590828fa1752c579198
-  data.tar.gz: 7309cc34f3d022d212ab9870558909ab84be3c6550160ce822808f98edb259c96669899247a7faefec6365068ac5aa68a6d5b43f279591156f47c9f02d0e182c
+  metadata.gz: 44dbd4e953260ef2c66b31f8a2914f4378bbeb969c670207bfa2272c4c2d5096a8ec5b3352b913093fb601b6d6c15ebc14d029a936e88c65ddf3d6eabe32e4d2
+  data.tar.gz: ee74ae98b99ab8b88e81d1c2c5437d860c314683e517858dc739924049c55ce8a4614cbfa40a97ac8bcd619cd678b6121e2158f45813370ee2c36de185a7244e

data/README.md

@@ -8,6 +8,7 @@ include DataCollector::Core
 ```
 Every object can be used on its own.
 
+### DataCollector Objects
 #### Pipeline
 Allows you to create a simple pipeline of operations to process data. With a data pipeline, you can collect, process, and transform data, and then transfer it to various systems and applications.
 
@@ -20,6 +21,7 @@ executed in the [ISO8601 duration format](https://www.digi.com/resources/documen
      - schedule: [ISO8601 duration format](https://www.digi.com/resources/documentation/digidocs//90001488-13/reference/r_iso_8601_duration_format.htm)
      - cron: in cron format ex. '1 12 * * *' intervals are not supported
      - uri: a directory/file to watch
+     - xml_typecast: true/false -> convert convert string values to TrueClass, FalseClass, Time, Date, and DateTime
  - .run: start the pipeline. blocking if a schedule is supplied
  - .stop: stop the pipeline
  - .pause: pause the pipeline. Restart using .run
@@ -74,12 +76,14 @@ For a push input component, a listener is created with a processing logic block
 A push happens when new data is created in a directory, message queue, ...
 
 ```ruby
-  from_uri(source, options = {:raw, :content_type})
+  from_uri(source, options = {:raw, :content_type, :headers, :cookies})
 ```
 - source: an uri with a scheme of http, https, file, amqp
 - options:
     - raw: _boolean_ do not parse
     - content_type: _string_ force a content_type if the 'Content-Type' returned by the http server is incorrect 
+    - headers: request headers
+    - cookies: session cookies etc.
 
 ###### example:
 ```ruby  

data/lib/data_collector/input.rb

@@ -66,7 +66,13 @@ module DataCollector
         else
           data
         end
-      rescue => e
+      rescue DataCollector::InputError => e
+        @logger.info(e.message)
+        raise e
+      rescue DataCollector::Error => e
+        @logger.info(e.message)
+        nil
+      rescue StandardError => e
         @logger.info(e.message)
         puts e.backtrace.join("\n")
         nil
@@ -141,28 +147,28 @@ module DataCollector
           when 'application/json'
             data = JSON.parse(data)
           when 'application/atom+xml'
-            data = xml_to_hash(data)
+            data = xml_to_hash(data, options)
           when 'text/csv'
             data = csv_to_hash(data)
           when 'application/xml'
-            data = xml_to_hash(data)
+            data = xml_to_hash(data, options)
           when 'text/xml'
-            data = xml_to_hash(data)
+            data = xml_to_hash(data, options)
           else
-            data = xml_to_hash(data)
+            data = xml_to_hash(data, options)
           end
         end
 
         raise '206 Partial Content' if http_response.code ==206
 
       when 401
-        raise 'Unauthorized'
+        raise DataCollector::InputError, 'Unauthorized'
       when 403
-        raise 'Forbidden'
+        raise DataCollector::InputError, 'Forbidden'
       when 404
-        raise 'Not found'
+        raise DataCollector::InputError, 'Not found'
       else
-        raise "Unable to process received status code = #{http_response.code}"
+        raise DataCollector::InputError, "Unable to process received status code = #{http_response.code}"
       end
 
       #[data, http_response.code]
@@ -182,14 +188,14 @@ module DataCollector
         when '.json'
           data = JSON.parse(data)
         when '.xml'
-          data = xml_to_hash(data)
+          data = xml_to_hash(data, options)
         when '.gz'
           Minitar.open(Zlib::GzipReader.new(File.open("#{absolute_path}", 'rb'))) do |i|
             i.each do |entry|
               data = entry.read
             end
           end
-          data = xml_to_hash(data)
+          data = xml_to_hash(data, options)
         when '.csv'
           data = csv_to_hash(data)
         else
@@ -212,10 +218,11 @@ module DataCollector
       DataCollector::Input::Rpc.new(uri, options)
     end
 
-    def xml_to_hash(data)
+    def xml_to_hash(data, options = {})
       #gsub('<\/', '< /') outherwise wrong XML-parsing (see records lirias1729192 )
       data = data.gsub /</, '< /'
-      nori = Nori.new(parser: :nokogiri, strip_namespaces: true, convert_tags_to: lambda { |tag| tag.gsub(/^@/, '_') })
+      xml_typecast = options.with_indifferent_access.key?('xml_typecast') ? options.with_indifferent_access['xml_typecast'] : true
+      nori = Nori.new(parser: :nokogiri, advanced_typecasting: xml_typecast, strip_namespaces: true, convert_tags_to: lambda { |tag| tag.gsub(/^@/, '_') })
       nori.parse(data)
       #JSON.parse(nori.parse(data).to_json)
     end

data/lib/data_collector/version.rb

@@ -1,4 +1,4 @@
 # encoding: utf-8
 module DataCollector
-  VERSION = "0.26.0"
+  VERSION = "0.28.0"
 end

data/lib/data_collector.rb

@@ -9,4 +9,6 @@ require 'data_collector/ext/xml_utility_node'
 
 module DataCollector
   class Error < StandardError; end
+
+  class InputError < StandardError; end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: data_collector
 version: !ruby/object:Gem::Version
-  version: 0.26.0
+  version: 0.28.0
 platform: ruby
 authors:
 - Mehmet Celik
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-08-02 00:00:00.000000000 Z
+date: 2023-08-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport