data_collector 0.29.0 → 0.31.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d09248a6f7a15b7c6ad783502506e9cdc3ba5010070a810b9bc8c803b15fc862
-  data.tar.gz: d47019eddd26810162229555997d957f35823e45b105036cbd1e4f90d361be81
+  metadata.gz: ec0485c6ec0c9c429f8dcef45b7357874f3f6de8449902f54f38a1fb36e2548a
+  data.tar.gz: 57820299fe1718fa07e561dc933c14d04aa2fbfe5ad264dfafa2db861c162aed
 SHA512:
-  metadata.gz: 8f508edbc7ad18d413bf6deac82c0848ab04955edd347b2a749b0ecf8baf3caef4052fdcd8b424d6b17e82b8da18e4f9cc8b40160d1b22566241986a4d5c6e7c
-  data.tar.gz: 4c339eb2adc53e2afe615a9b1ebca1649c0466c51366babd574bf7bdfbb4ef0c8df121308561514c69dc6fcd31cd6c4fa6e8d689ab0cff0e9bcff9554f96f85a
+  metadata.gz: aea723ca8aee53554b06259ad463b7bc460b0363bfce702dd3d7292519c185660fd86270a3d9b6c84db49133ed74d57eebcf27fcf974184465506967b96de56b
+  data.tar.gz: ffc8d3e780aab83742fc8fefdabf3f10fbdcbed5fdcddce1a69c81754e84f951e7b7ccd896037f91f82f7da23d8fd650c67d6982281ab80f23e5bb54ec0c9537

data/data_collector.gemspec

@@ -36,18 +36,18 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ['lib']
 
-  spec.add_runtime_dependency 'activesupport', '~> 6.1'
+  spec.add_runtime_dependency 'activesupport', '~> 7.0'
   spec.add_runtime_dependency 'http', '~> 5.1'
   spec.add_runtime_dependency 'json', '~> 2.6'
-  spec.add_runtime_dependency 'json-ld', '~> 3.2'
+  spec.add_runtime_dependency 'json-ld', '~> 3.3'
   spec.add_runtime_dependency 'jsonpath', '~> 1.1'
-  spec.add_runtime_dependency 'mime-types', '~> 3.4'
+  spec.add_runtime_dependency 'mime-types', '~> 3.5'
   spec.add_runtime_dependency 'minitar', '= 0.9'
   spec.add_runtime_dependency 'nokogiri', '~> 1.15'
   spec.add_runtime_dependency 'nori', '~> 2.6'
   spec.add_runtime_dependency 'iso8601', '~> 0.13'
   spec.add_runtime_dependency 'listen', '~> 3.8'
-  spec.add_runtime_dependency 'bunny', '~> 2.20'
+  spec.add_runtime_dependency 'bunny', '~> 2.22'
   spec.add_runtime_dependency 'bunny_burrow', '~> 1.5'
   spec.add_runtime_dependency 'builder', '~> 3.2'
   spec.add_runtime_dependency 'parse-cron', '~> 0.1'

data/lib/data_collector/input.rb

@@ -1,4 +1,4 @@
-#encoding: UTF-8
+# encoding: UTF-8
 require 'http'
 require 'open-uri'
 require 'nokogiri'
@@ -16,7 +16,7 @@ require_relative 'input/dir'
 require_relative 'input/queue'
 require_relative 'input/rpc'
 
-#require_relative 'ext/xml_utility_node'
+# require_relative 'ext/xml_utility_node'
 module DataCollector
   class Input
     attr_reader :raw
@@ -25,13 +25,14 @@ module DataCollector
       @logger = Logger.new(STDOUT)
     end
 
-    def from_uri(source, options = {})
+    def from_uri(source, options = {}, &block)
+      block_consumed = false
       source = CGI.unescapeHTML(source)
       @logger.info("Reading #{source}")
       raise DataCollector::Error, 'from_uri expects a scheme like file:// of https://' unless source =~ /:\/\//
 
       scheme, path = source.split('://')
-      source="#{scheme}://#{URI.encode_www_form_component(path)}"
+      source = "#{scheme}://#{URI.encode_www_form_component(path)}"
       uri = URI(source)
       begin
         data = nil
@@ -43,11 +44,14 @@ module DataCollector
         when 'file'
           absolute_path = File.absolute_path("#{URI.decode_www_form_component("#{uri.host}#{uri.path}")}")
           if File.directory?(absolute_path)
-            #raise DataCollector::Error, "#{uri.host}/#{uri.path} not found" unless File.exist?("#{uri.host}/#{uri.path}")
             return from_dir(uri, options)
           else
-            #            raise DataCollector::Error, "#{uri.host}/#{uri.path} not found" unless File.exist?("#{uri.host}/#{uri.path}")
-            data = from_file(uri, options)
+            if block_given?
+              data = from_file(uri, options, &block)
+              block_consumed = true if data.is_a?(TrueClass)
+            else
+              data = from_file(uri, options)
+            end
           end
         when /amqp/
           if uri.scheme =~ /^rpc/
@@ -61,7 +65,7 @@ module DataCollector
 
         data = data.nil? ? 'no data found' : data
 
-        if block_given?
+        if block_given? && !block_consumed
           yield data
         else
           data
@@ -94,7 +98,7 @@ module DataCollector
 
       http = HTTP
 
-      #http.use(logging: {logger: @logger})
+      # http.use(logging: {logger: @logger})
 
       if options.key?(:user) && options.key?(:password)
         @logger.debug "Set Basic_auth"
@@ -102,34 +106,33 @@ module DataCollector
         password = options[:password]
         http = HTTP.basic_auth(user: user, pass: password)
       elsif options.key?(:bearer_token)
-        @logger.debug  "Set authorization bearer token"
+        @logger.debug "Set authorization bearer token"
         bearer = options[:bearer_token]
         bearer = "Bearer #{bearer}" unless bearer =~ /^Bearer /i
         http = HTTP.auth(bearer)
       end
 
-      if options.key?(:cookies) 
-        @logger.debug  "Set cookies"
-        http = http.cookies( options[:cookies] )
+      if options.key?(:cookies)
+        @logger.debug "Set cookies"
+        http = http.cookies(options[:cookies])
       end
 
-      if options.key?(:headers) 
-        @logger.debug  "Set http headers"
-        http = http.headers( options[:headers] )
+      if options.key?(:headers)
+        @logger.debug "Set http headers"
+        http = http.headers(options[:headers])
       end
-          
+
       if options.key?(:verify_ssl) && uri.scheme.eql?('https')
         @logger.warn "Disabling SSL verification. "
-        #shouldn't use this but we all do ...
+        # shouldn't use this but we all do ...
         ctx = OpenSSL::SSL::SSLContext.new
         ctx.verify_mode = OpenSSL::SSL::VERIFY_NONE
 
         http_response = http.follow.get(escape_uri(uri), ssl_context: ctx)
-
       else
         http_response = http.follow.get(escape_uri(uri))
       end
- 
+
       case http_response.code
       when 200..299
         @raw = data = http_response.body.to_s
@@ -159,7 +162,7 @@ module DataCollector
           end
         end
 
-        raise '206 Partial Content' if http_response.code ==206
+        raise '206 Partial Content' if http_response.code == 206
 
       when 401
         raise DataCollector::InputError, 'Unauthorized'
@@ -175,7 +178,7 @@ module DataCollector
       data
     end
 
-    def from_file(uri, options = {})
+    def from_file(uri, options = {}, &block)
       data = nil
       uri = normalize_uri(uri)
       absolute_path = File.absolute_path(uri)
@@ -190,12 +193,29 @@ module DataCollector
         when '.xml'
           data = xml_to_hash(data, options)
         when '.gz'
+          tar_data = []
           Minitar.open(Zlib::GzipReader.new(File.open("#{absolute_path}", 'rb'))) do |i|
             i.each do |entry|
-              data = entry.read
-            end
-          end
-          data = xml_to_hash(data, options)
+              next unless entry.typeflag.eql?('0')
+              if block_given?
+                data = xml_to_hash(entry.read, options)
+                yield data
+
+                data = true
+              else
+                tar_data << entry.read
+
+                if tar_data.length == 1
+                  data = xml_to_hash(tar_data.first, options)
+                else
+                  data = []
+                  tar_data.each do |d|
+                    data << xml_to_hash(d, options)
+                  end
+                end
+              end #block
+            end #entry
+          end #tar
         when '.csv'
           data = csv_to_hash(data)
         else
@@ -219,9 +239,12 @@ module DataCollector
     end
 
     def xml_to_hash(data, options = {})
-      #gsub('&lt;\/', '&lt; /') outherwise wrong XML-parsing (see records lirias1729192 )
-      data.force_encoding('UTF-8') #encode("UTF-8", invalid: :replace, replace: "")
+      # gsub('&lt;\/', '&lt; /') outherwise wrong XML-parsing (see records lirias1729192 )
+      return unless data.is_a?(String)
+      data.force_encoding('UTF-8')
+      data = data.encode("UTF-8", invalid: :replace, replace: "")
       data = data.gsub /&lt;/, '&lt; /'
+
       xml_typecast = options.with_indifferent_access.key?('xml_typecast') ? options.with_indifferent_access['xml_typecast'] : true
       nori = Nori.new(parser: :nokogiri, advanced_typecasting: xml_typecast, strip_namespaces: true, convert_tags_to: lambda { |tag| tag.gsub(/^@/, '_') })
       nori.parse(data)
@@ -245,7 +268,7 @@ module DataCollector
       file_type = if headers.include?('Content-Type')
                     headers['Content-Type'].split(';').first
                   else
-                    @logger.debug  "No Header content-type available"
+                    @logger.debug "No Header content-type available"
                     MIME::Types.of(filename_from(headers)).first.content_type
                   end
 

data/lib/data_collector/output.rb

@@ -6,6 +6,8 @@ require 'minitar'
 require 'zlib'
 require 'cgi'
 require 'active_support/core_ext/hash'
+require "active_support/isolated_execution_state"
+require 'active_support/xml_mini'
 require 'fileutils'
 require_relative './output/rpc'
 

data/lib/data_collector/version.rb

@@ -1,4 +1,4 @@
 # encoding: utf-8
 module DataCollector
-  VERSION = "0.29.0"
+  VERSION = "0.31.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: data_collector
 version: !ruby/object:Gem::Version
-  version: 0.29.0
+  version: 0.31.0
 platform: ruby
 authors:
 - Mehmet Celik
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-08-08 00:00:00.000000000 Z
+date: 2023-09-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '6.1'
+        version: '7.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '6.1'
+        version: '7.0'
 - !ruby/object:Gem::Dependency
   name: http
   requirement: !ruby/object:Gem::Requirement
@@ -58,14 +58,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.2'
+        version: '3.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.2'
+        version: '3.3'
 - !ruby/object:Gem::Dependency
   name: jsonpath
   requirement: !ruby/object:Gem::Requirement
@@ -86,14 +86,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.4'
+        version: '3.5'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.4'
+        version: '3.5'
 - !ruby/object:Gem::Dependency
   name: minitar
   requirement: !ruby/object:Gem::Requirement
@@ -170,14 +170,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.20'
+        version: '2.22'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.20'
+        version: '2.22'
 - !ruby/object:Gem::Dependency
   name: bunny_burrow
   requirement: !ruby/object:Gem::Requirement
@@ -333,7 +333,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.13
+rubygems_version: 3.4.19
 signing_key:
 specification_version: 4
 summary: ETL helper library