data_collector 0.28.0 → 0.30.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 71b519b75704a9bb6e46601c8195ab69608f19fae12ecde7c6c57869b8fca450
-  data.tar.gz: aa4a0697ffb9f6bc21c45f2ecdfa13c9deca7c8b69f2e8159f4901ab6be8d646
+  metadata.gz: 47a88bcc23a8fdc3922f22b3cb984eed3ea7b725d139e72b1f6f669d70ebb8e6
+  data.tar.gz: 62a4c5775dca4cdd6c6776c6d8628a717d1eb021a314b663ac7f0b6c95e317ac
 SHA512:
-  metadata.gz: 44dbd4e953260ef2c66b31f8a2914f4378bbeb969c670207bfa2272c4c2d5096a8ec5b3352b913093fb601b6d6c15ebc14d029a936e88c65ddf3d6eabe32e4d2
-  data.tar.gz: ee74ae98b99ab8b88e81d1c2c5437d860c314683e517858dc739924049c55ce8a4614cbfa40a97ac8bcd619cd678b6121e2158f45813370ee2c36de185a7244e
+  metadata.gz: 95cff72022a43f18c9495cfe6b1f3a66a1f086e196dcb03528152c5a3c4dc34b8e551d9f6fb2d576e758a9c61b34693632990342488b2406fcd3b44a003e4381
+  data.tar.gz: 71abb93a64fbb419b5eb4f5e8e0ba6221f69b289fe8b133bbc5b208446e7b4b132d4199f769775170d69f2e5fb1051836ce9ce68222ca4a199eb0a0854978826

data/lib/data_collector/input.rb

@@ -1,4 +1,4 @@
-#encoding: UTF-8
+# encoding: UTF-8
 require 'http'
 require 'open-uri'
 require 'nokogiri'
@@ -16,7 +16,7 @@ require_relative 'input/dir'
 require_relative 'input/queue'
 require_relative 'input/rpc'
 
-#require_relative 'ext/xml_utility_node'
+# require_relative 'ext/xml_utility_node'
 module DataCollector
   class Input
     attr_reader :raw
@@ -25,13 +25,14 @@ module DataCollector
       @logger = Logger.new(STDOUT)
     end
 
-    def from_uri(source, options = {})
+    def from_uri(source, options = {}, &block)
+      block_consumed = false
       source = CGI.unescapeHTML(source)
       @logger.info("Reading #{source}")
-      raise DataCollector::Error 'from_uri expects a scheme like file:// of https://' unless source =~ /:\/\//
+      raise DataCollector::Error, 'from_uri expects a scheme like file:// of https://' unless source =~ /:\/\//
 
       scheme, path = source.split('://')
-      source="#{scheme}://#{URI.encode_www_form_component(path)}"
+      source = "#{scheme}://#{URI.encode_www_form_component(path)}"
       uri = URI(source)
       begin
         data = nil
@@ -43,11 +44,14 @@ module DataCollector
         when 'file'
           absolute_path = File.absolute_path("#{URI.decode_www_form_component("#{uri.host}#{uri.path}")}")
           if File.directory?(absolute_path)
-            #raise DataCollector::Error, "#{uri.host}/#{uri.path} not found" unless File.exist?("#{uri.host}/#{uri.path}")
             return from_dir(uri, options)
           else
-            #            raise DataCollector::Error, "#{uri.host}/#{uri.path} not found" unless File.exist?("#{uri.host}/#{uri.path}")
-            data = from_file(uri, options)
+            if block_given?
+              data = from_file(uri, options, &block)
+              block_consumed = true if data.is_a?(TrueClass)
+            else
+              data = from_file(uri, options)
+            end
           end
         when /amqp/
           if uri.scheme =~ /^rpc/
@@ -61,19 +65,19 @@ module DataCollector
 
         data = data.nil? ? 'no data found' : data
 
-        if block_given?
+        if block_given? && !block_consumed
           yield data
         else
           data
         end
       rescue DataCollector::InputError => e
-        @logger.info(e.message)
+        @logger.error(e.message)
         raise e
       rescue DataCollector::Error => e
-        @logger.info(e.message)
+        @logger.error(e.message)
         nil
       rescue StandardError => e
-        @logger.info(e.message)
+        @logger.error(e.message)
         puts e.backtrace.join("\n")
         nil
       end
@@ -94,7 +98,7 @@ module DataCollector
 
       http = HTTP
 
-      #http.use(logging: {logger: @logger})
+      # http.use(logging: {logger: @logger})
 
       if options.key?(:user) && options.key?(:password)
         @logger.debug "Set Basic_auth"
@@ -102,34 +106,33 @@ module DataCollector
         password = options[:password]
         http = HTTP.basic_auth(user: user, pass: password)
       elsif options.key?(:bearer_token)
-        @logger.debug  "Set authorization bearer token"
+        @logger.debug "Set authorization bearer token"
         bearer = options[:bearer_token]
         bearer = "Bearer #{bearer}" unless bearer =~ /^Bearer /i
         http = HTTP.auth(bearer)
       end
 
-      if options.key?(:cookies) 
-        @logger.debug  "Set cookies"
-        http = http.cookies( options[:cookies] )
+      if options.key?(:cookies)
+        @logger.debug "Set cookies"
+        http = http.cookies(options[:cookies])
       end
 
-      if options.key?(:headers) 
-        @logger.debug  "Set http headers"
-        http = http.headers( options[:headers] )
+      if options.key?(:headers)
+        @logger.debug "Set http headers"
+        http = http.headers(options[:headers])
       end
-          
+
       if options.key?(:verify_ssl) && uri.scheme.eql?('https')
         @logger.warn "Disabling SSL verification. "
-        #shouldn't use this but we all do ...
+        # shouldn't use this but we all do ...
         ctx = OpenSSL::SSL::SSLContext.new
         ctx.verify_mode = OpenSSL::SSL::VERIFY_NONE
 
         http_response = http.follow.get(escape_uri(uri), ssl_context: ctx)
-
       else
         http_response = http.follow.get(escape_uri(uri))
       end
- 
+
       case http_response.code
       when 200..299
         @raw = data = http_response.body.to_s
@@ -159,7 +162,7 @@ module DataCollector
           end
         end
 
-        raise '206 Partial Content' if http_response.code ==206
+        raise '206 Partial Content' if http_response.code == 206
 
       when 401
         raise DataCollector::InputError, 'Unauthorized'
@@ -175,7 +178,7 @@ module DataCollector
       data
     end
 
-    def from_file(uri, options = {})
+    def from_file(uri, options = {}, &block)
       data = nil
       uri = normalize_uri(uri)
       absolute_path = File.absolute_path(uri)
@@ -190,12 +193,29 @@ module DataCollector
         when '.xml'
           data = xml_to_hash(data, options)
         when '.gz'
+          tar_data = []
           Minitar.open(Zlib::GzipReader.new(File.open("#{absolute_path}", 'rb'))) do |i|
             i.each do |entry|
-              data = entry.read
-            end
-          end
-          data = xml_to_hash(data, options)
+              next unless entry.typeflag.eql?('0')
+              if block_given?
+                data = xml_to_hash(entry.read, options)
+                yield data
+
+                data = true
+              else
+                tar_data << entry.read
+
+                if tar_data.length == 1
+                  data = xml_to_hash(tar_data.first, options)
+                else
+                  data = []
+                  tar_data.each do |d|
+                    data << xml_to_hash(d, options)
+                  end
+                end
+              end #block
+            end #entry
+          end #tar
         when '.csv'
           data = csv_to_hash(data)
         else
@@ -219,12 +239,15 @@ module DataCollector
     end
 
     def xml_to_hash(data, options = {})
-      #gsub('&lt;\/', '&lt; /') outherwise wrong XML-parsing (see records lirias1729192 )
+      # gsub('&lt;\/', '&lt; /') outherwise wrong XML-parsing (see records lirias1729192 )
+      return unless data.is_a?(String)
+      data.force_encoding('UTF-8')
+      data = data.encode("UTF-8", invalid: :replace, replace: "")
       data = data.gsub /&lt;/, '&lt; /'
+
       xml_typecast = options.with_indifferent_access.key?('xml_typecast') ? options.with_indifferent_access['xml_typecast'] : true
       nori = Nori.new(parser: :nokogiri, advanced_typecasting: xml_typecast, strip_namespaces: true, convert_tags_to: lambda { |tag| tag.gsub(/^@/, '_') })
       nori.parse(data)
-      #JSON.parse(nori.parse(data).to_json)
     end
 
     def csv_to_hash(data)
@@ -245,7 +268,7 @@ module DataCollector
       file_type = if headers.include?('Content-Type')
                     headers['Content-Type'].split(';').first
                   else
-                    @logger.debug  "No Header content-type available"
+                    @logger.debug "No Header content-type available"
                     MIME::Types.of(filename_from(headers)).first.content_type
                   end
 

data/lib/data_collector/version.rb

@@ -1,4 +1,4 @@
 # encoding: utf-8
 module DataCollector
-  VERSION = "0.28.0"
+  VERSION = "0.30.0"
 end

data/lib/data_collector.rb

@@ -8,7 +8,9 @@ require 'data_collector/pipeline'
 require 'data_collector/ext/xml_utility_node'
 
 module DataCollector
-  class Error < StandardError; end
+  class Error < StandardError
+  end
 
-  class InputError < StandardError; end
+  class InputError < Error
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: data_collector
 version: !ruby/object:Gem::Version
-  version: 0.28.0
+  version: 0.30.0
 platform: ruby
 authors:
 - Mehmet Celik
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-08-07 00:00:00.000000000 Z
+date: 2023-08-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport