data_collector 0.44.0 → 0.46.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c289298a8de94ccad3a239ed6d05bf33db68c36072015f8bdb2d2a857809bd0f
-  data.tar.gz: 1f30741dd80b5cabbf9062869260d319c9db647facec1de41fa8921640967600
+  metadata.gz: 72db9fd94027f6a4a9d3112c070e4c52ed8e824676e10923d7bcf76fe5015d14
+  data.tar.gz: 6e1e4b8827ae44abe027568682426d4a9cc610f06cb99e9afdfe63bd0b8e5295
 SHA512:
-  metadata.gz: ac4f4d7d8906f11ea0840cf416058a665ac2139fd6d3cd0057986624c702e1774fa752c6984ae08c0bc9754c0bfdf9c8fe553635d96ec14cbcc67641e4f95792
-  data.tar.gz: af893f7320eaa88ff7026f3a2a04e74cface4cd1858db450d6204baa73751b9d6e6c6b743f979319f5e277a7b213ec0ef2989180ae99d1c54921ac07bf2d1817
+  metadata.gz: 867c15eddb233301ddd0090138989f8230424ae159f7ba2aba8669d0d985c29262eb9bb4029283dd2dd130c6534621c6db1e8f946d6563c60e2c2f3e719c48bd
+  data.tar.gz: fc370007a9ec72e5d748c51e4cb191334dacc79639245cbc395d7bebe0b90fd77574466e889177e49e041580998b7749d4441d9e3ca815307f2fad8adf43204b

data/data_collector.gemspec

@@ -36,14 +36,14 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ['lib']
 
-  spec.add_runtime_dependency 'activesupport', '~> 7.0'
-  spec.add_runtime_dependency 'http', '~> 5.1'
-  spec.add_runtime_dependency 'json', '~> 2.6'
+  spec.add_runtime_dependency 'activesupport', '~> 7.1'
+  spec.add_runtime_dependency 'http', '~> 5.2'
+  spec.add_runtime_dependency 'json', '~> 2.7'
   spec.add_runtime_dependency 'jsonpath', '~> 1.1'
   spec.add_runtime_dependency 'mime-types', '~> 3.5'
   spec.add_runtime_dependency 'minitar', '= 0.9'
-  spec.add_runtime_dependency 'nokogiri', '~> 1.15'
-  spec.add_runtime_dependency 'nori', '~> 2.6'
+  spec.add_runtime_dependency 'nokogiri', '~> 1.16'
+  spec.add_runtime_dependency 'nori', '~> 2.7'
   spec.add_runtime_dependency 'iso8601', '~> 0.13'
   spec.add_runtime_dependency 'listen', '~> 3.8'
   spec.add_runtime_dependency 'bunny', '~> 2.22'
@@ -52,10 +52,12 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency 'parse-cron', '~> 0.1'
   spec.add_runtime_dependency 'linkeddata', '~> 3.3'
   spec.add_runtime_dependency 'connection_pool', '~> 2.4'
+  #spec.add_runtime_dependency 'grpc', '~> 1.61'
 
 
   spec.add_development_dependency 'bundler', '~> 2.3'
-  spec.add_development_dependency 'minitest', '~> 5.18'
-  spec.add_development_dependency 'rake', '~> 13.0'
-  spec.add_development_dependency 'webmock', '~> 3.18'
+  spec.add_development_dependency 'minitest', '~> 5.22'
+  spec.add_development_dependency 'rake', '~> 13.1'
+  spec.add_development_dependency 'webmock', '~> 3.22'
+  #spec.add_development_dependency 'grpc-tools', '~> 1.61'
 end

data/lib/data_collector/config_file.rb

@@ -37,7 +37,7 @@ module DataCollector
     def self.[]=(key, value)
       init
       @config[key] = value
-      File.open("#{path}/config.yml", 'w') do |f|
+      File.open("#{path}/#{@config_file_name}", 'w') do |f|
         f.puts @config.to_yaml
       end
     end
@@ -54,11 +54,11 @@ module DataCollector
 
     def self.init
       discover_config_file_path
-      raise Errno::ENOENT, "#{@config_file_path}/config.yml Not Found. Set path to config.yml" unless File.exist?("#{@config_file_path}/config.yml")
+      raise Errno::ENOENT, "#{@config_file_path}/#{@config_file_name} Not Found. Set path to #{@config_file_name}" unless File.exist?("#{@config_file_path}/#{@config_file_name}")
 
-      ftime = File.exist?("#{@config_file_path}/config.yml") ? File.mtime("#{@config_file_path}/config.yml") : nil
+      ftime = File.exist?("#{@config_file_path}/#{@config_file_name}") ? File.mtime("#{@config_file_path}/#{@config_file_name}") : nil
       if @config.empty? || @mtime != ftime
-        config = YAML::load_file("#{@config_file_path}/config.yml")
+        config = YAML::load_file("#{@config_file_path}/#{@config_file_name}", permitted_classes: [Time, Symbol])
         @config = process(config)
       end
     end
@@ -67,9 +67,9 @@ module DataCollector
       if @config_file_path.nil? || @config_file_path.empty?
         if ENV.key?('CONFIG_FILE_PATH')
           @config_file_path = ENV['CONFIG_FILE_PATH']
-        elsif File.exist?('config.yml')
+        elsif File.exist?(@config_file_name)
           @config_file_path = '.'
-        elsif File.exist?("config/config.yml")
+        elsif File.exist?("config/#{@config_file_name}")
           @config_file_path = 'config'
         end
       end

data/lib/data_collector/input.rb

@@ -140,6 +140,11 @@ module DataCollector
         http_query_options[:body] = options[:body]
 
         http_response = http.follow.post(escape_uri(uri), http_query_options)
+      elsif options.key?(:method) && options[:method].downcase.eql?('put')
+        raise DataCollector::InputError, "No body found, a PUT request needs a body" unless options.key?(:body)
+        http_query_options[:body] = options[:body]
+
+        http_response = http.follow.put(escape_uri(uri), http_query_options)
       else
         http_response = http.follow.get(escape_uri(uri), http_query_options)
       end
@@ -172,7 +177,7 @@ module DataCollector
             data = html_to_hash(data, options)
           when 'text/turtle'
             graph = RDF::Graph.new do |graph|
-              RDF::Turtle::Reader.new(data) {|reader| graph << reader}
+              RDF::Turtle::Reader.new(data) { |reader| graph << reader }
             end
             data = JSON.parse(graph.dump(:jsonld, validate: false, standard_prefixes: true))
           when /^image/
@@ -238,9 +243,9 @@ module DataCollector
                     data << xml_to_hash(d, options)
                   end
                 end
-              end #block
-            end #entry
-          end #tar
+              end # block
+            end # entry
+          end # tar
         when '.csv'
           data = csv_to_hash(data, options)
         when '.jpg', '.png', '.gif'
@@ -267,7 +272,7 @@ module DataCollector
 
     def image_to_data(data, options = {})
       file_type = options['file_type']
-      "data:#{file_type};#{Base64.encode64(data)}"
+      "data:#{file_type};base64,#{Base64.encode64(data)}"
     end
 
     def xml_to_hash(data, options = {})
@@ -290,7 +295,7 @@ module DataCollector
 
     def csv_to_hash(data, options = {})
       csv_option_keys = options.keys & CSV::DEFAULT_OPTIONS.keys
-      all_cvs_options = {headers: true, header_converters: [:downcase, :symbol]}
+      all_cvs_options = { headers: true, header_converters: [:downcase, :symbol] }
 
       csv_option_keys.each do |k|
         all_cvs_options[k] = options[k]

data/lib/data_collector/output.rb

@@ -288,8 +288,8 @@ module DataCollector
     end
 
     def to_https(uri, options)
-
-      raise 'TODO'
+      input = Input.new
+      input.from_uri(uri.to_s, options)
     end
 
     def to_file(uri, options)

data/lib/data_collector/proto/datacollector.proto

@@ -0,0 +1,13 @@
+syntax = "proto3";
+package data_collector;
+
+option java_multiple_files = true;
+option java_outer_classname = "DataCollectorProto";
+option java_package = "be.celik";
+option ruby_package = "data_collector";
+
+import "google/protobuf/struct.proto";
+
+message MessageStruct {
+  google.protobuf.Struct data = 1;
+}

data/lib/data_collector/version.rb

@@ -1,4 +1,4 @@
 # encoding: utf-8
 module DataCollector
-  VERSION = "0.44.0"
+  VERSION = "0.46.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: data_collector
 version: !ruby/object:Gem::Version
-  version: 0.44.0
+  version: 0.46.0
 platform: ruby
 authors:
 - Mehmet Celik
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-01-08 00:00:00.000000000 Z
+date: 2024-02-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -16,42 +16,42 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '7.0'
+        version: '7.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '7.0'
+        version: '7.1'
 - !ruby/object:Gem::Dependency
   name: http
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.1'
+        version: '5.2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.1'
+        version: '5.2'
 - !ruby/object:Gem::Dependency
   name: json
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.6'
+        version: '2.7'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.6'
+        version: '2.7'
 - !ruby/object:Gem::Dependency
   name: jsonpath
   requirement: !ruby/object:Gem::Requirement
@@ -100,28 +100,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.15'
+        version: '1.16'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.15'
+        version: '1.16'
 - !ruby/object:Gem::Dependency
   name: nori
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.6'
+        version: '2.7'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.6'
+        version: '2.7'
 - !ruby/object:Gem::Dependency
   name: iso8601
   requirement: !ruby/object:Gem::Requirement
@@ -254,42 +254,42 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.18'
+        version: '5.22'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.18'
+        version: '5.22'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '13.0'
+        version: '13.1'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '13.0'
+        version: '13.1'
 - !ruby/object:Gem::Dependency
   name: webmock
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.18'
+        version: '3.22'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.18'
+        version: '3.22'
 description: INPUT, FILTER, OUTPUT data with RULES and code
 email:
 - mehmet@celik.be
@@ -322,6 +322,7 @@ files:
 - lib/data_collector/output/generic.rb
 - lib/data_collector/output/rpc.rb
 - lib/data_collector/pipeline.rb
+- lib/data_collector/proto/datacollector.proto
 - lib/data_collector/rules.rb
 - lib/data_collector/rules.rb.depricated
 - lib/data_collector/rules_ng.rb
@@ -350,7 +351,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.21
+rubygems_version: 3.5.6
 signing_key:
 specification_version: 4
 summary: ETL helper library