data_collector 0.60.0 → 0.62.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a929b89fe8361f9efa13b102aea2b7792ee4c524948978cb0da4521797d45489
-  data.tar.gz: 0ab8859d19d8ba21ac72648fa845d18aeb5ef8af058dca4a5f2b417ec5826997
+  metadata.gz: '0923a758b6a621afd2d2fb4aa85f580cb757ee3ca1150e7175827a9bf617e07b'
+  data.tar.gz: 1a83fecc0244088447747f4c196ffcfddc6cb70e5ab8f40232a0401ac9912fc3
 SHA512:
-  metadata.gz: cc2112dc0b352d19c3950c0a05c96905eb9646f687a305e12af8e930262271800e5db41d22578e2246c663ddcc901df7b439e781b61a8dd09873a8cb8d442e41
-  data.tar.gz: d18ce58fe0f3d6544f054c8a389fb30700d76c60e597ce8b973e2f72c34a9bc21c6ee62cebf606b2d1de9b0249e91d5d9bc9afff4e97c12386d0b2eaf3c0a59c
+  metadata.gz: d6d75c0bd763bbf2c107ebe7e5bc2c5c813454c6ad496eb234d38c79fae01525969e5d8b6abd601cf3103f8363f3f7965957cb54103eb10aaa80b65f80aa3073
+  data.tar.gz: '029d8bbf6819f90b955f373d8ee8e795fca2b130fc1900b487ef692a1b56cf7f7a8d02b222637bf83361354f1e795713c731eba90fb446f2906a758986c99af1'

data/data_collector.gemspec

@@ -14,6 +14,8 @@ Gem::Specification.new do |spec|
   spec.homepage      = 'https://github.com/mehmetc/data_collector'
   spec.license       = 'MIT'
 
+  spec.required_ruby_version = ">= 3.4"
+
   # Prevent pushing this gem to RubyGems.org. To allow pushes either set the 'allowed_push_host'
   # to allow pushing to a single host or delete this section to allow pushing to any host.
   if spec.respond_to?(:metadata)

data/lib/data_collector/config_file.rb

@@ -10,7 +10,7 @@ module DataCollector
     @mtime = nil
 
     def self.version
-      '0.0.3'
+      '0.0.4'
     end
 
     def self.name
@@ -59,7 +59,8 @@ module DataCollector
 
       ftime = File.exist?("#{@config_file_path}/#{@config_file_name}") ? File.mtime("#{@config_file_path}/#{@config_file_name}") : nil
       if @config.nil? || @config.empty? || @mtime != ftime
-        config = YAML::load_file("#{@config_file_path}/#{@config_file_name}", aliases: true, permitted_classes: [Time, Symbol])
+        # config = YAML::load_file("#{@config_file_path}/#{@config_file_name}", aliases: true, permitted_classes: [Time, Symbol])
+        config = interpret_yaml_with_expressions("#{@config_file_path}/#{@config_file_name}", ENV)
         @config = process(config)
       end
     end
@@ -92,5 +93,36 @@ module DataCollector
     private_class_method :init
     private_class_method :discover_config_file_path
     private_class_method :process
+
+    private
+    def self.interpret_yaml_with_expressions(yaml_file, variables = {})
+      # Read the YAML file content
+      content = File.read(yaml_file)
+
+      # Process any :key: "${expression}" patterns
+      processed_content = content.gsub(/:key:\s*"?\$\{([^}]+)\}"?/) do |match|
+        expression = $1.strip
+
+        # Evaluate the expression using the provided variables
+        if variables.key?(expression)
+          # Replace with the variable value
+          match.gsub("${#{expression}}", variables[expression].to_s)
+        elsif expression.include?('.')
+          # Handle dot notation for nested variables
+          parts = expression.split('.')
+          value = variables
+          parts.each do |part|
+            value = value[part.to_sym] if value.is_a?(Hash) && value.key?(part.to_sym)
+          end
+          value.to_s
+        else
+          # Keep the original if we can't evaluate
+          match
+        end
+      end
+
+      # Parse the processed YAML content
+      YAML.load(processed_content, aliases: true, permitted_classes: [Time, Symbol])
+    end
   end
 end

data/lib/data_collector/core.rb

@@ -138,7 +138,9 @@ module DataCollector
     module_function :error
 
     def logger(*destinations)
-      @logger ||= begin
+      @logger if @logger && (@log_destination.eql?(destinations.flatten) || destinations.empty? || destinations.nil?)
+      @log_destination = destinations.flatten
+      @logger =  begin
                     destinations = STDOUT if destinations.nil? || destinations.empty?
                     Logger.new(ProxyLogger.new(destinations))
                   rescue StandardError => e

data/lib/data_collector/input.rb

@@ -32,6 +32,8 @@ module DataCollector
       data = nil
       if source.is_a?(StringIO)
         data = from_stringio(source, options)
+      elsif source.respond_to?(:read)
+        data = from_tempfile(source, options)
       else
         source = CGI.unescapeHTML(source)
         @logger.info("Reading #{source}")
@@ -153,7 +155,21 @@ module DataCollector
 
       case http_response.code
       when 200..299
-        @raw = data = http_response.body.to_s
+
+        if http_response.code == 206
+          @logger.debug "HTTP response 206 Partial Content"
+          data = http_response.body.readpartial
+          loop do
+            partial_data = http_response.body.readpartial
+            if partial_data.nil? || partial_data.empty?
+              break
+            end
+            data = data + partial_data.to_s
+          end
+          @raw = data
+        else
+          @raw = data = http_response.body.to_s
+        end
 
         file_type = options.with_indifferent_access.has_key?(:content_type) ? options.with_indifferent_access[:content_type] : file_type_from(http_response.headers)
 
@@ -185,8 +201,6 @@ module DataCollector
           end
         end
 
-        raise '206 Partial Content' if http_response.code == 206
-
       when 401
         raise DataCollector::InputError, 'Unauthorized'
       when 403
@@ -210,6 +224,17 @@ module DataCollector
       data = JSON.parse(graph.dump(:jsonld, validate: false, standard_prefixes: true))
     end
 
+    def from_tempfile(tempfile, options = {}, &block)
+      #file = Tempfile.new(["dc_", ".#{preferred_extension}"])
+      begin
+        tempfile.rewind
+        #file.write(temp_input_file.read)
+        #file.close
+        from_file(URI("file://#{tempfile.path}"), options)
+        #ensure
+        #file.unlink
+      end
+    end
     def from_stringio(sio, options = {}, &block)
       raise DataCollector::InputError, "No IO input" unless sio.is_a?(StringIO)
       raise DataCollector::InputError, "content_type option not supplied" unless options.key?(:content_type)
@@ -232,11 +257,15 @@ module DataCollector
       uri = normalize_uri(uri)
       absolute_path = File.absolute_path(uri)
       file_type = MIME::Types.type_for(uri).first.to_s
-      options['file_type'] = options[:content_type] || file_type
+      file_type = File.extname(absolute_path) if file_type.empty?
+      options['file_type'] = MIME::Types[(options[:content_type] || file_type)].first.preferred_extension
+
+      options['file_extention'] = ".#{options['file_type']}"
       raise DataCollector::Error, "#{uri.to_s} not found" unless File.exist?("#{absolute_path}")
       unless options.has_key?('raw') && options['raw'] == true
         @raw = data = File.read("#{absolute_path}")
-        case File.extname(absolute_path)
+
+        case options['file_extention']
         when '.jsonld'
           data = JSON.parse(data)
         when '.json'

data/lib/data_collector/version.rb

@@ -1,4 +1,4 @@
 # encoding: utf-8
 module DataCollector
-  VERSION = "0.60.0"
+  VERSION = "0.62.0"
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: data_collector
 version: !ruby/object:Gem::Version
-  version: 0.60.0
+  version: 0.62.0
 platform: ruby
 authors:
 - Mehmet Celik
 bindir: exe
 cert_chain: []
-date: 2025-02-24 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -400,14 +400,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: '3.4'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.2
+rubygems_version: 3.6.8
 specification_version: 4
 summary: ETL helper library
 test_files: []