data_collector 0.24.0 → 0.26.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7745e79eb3836ab3c469cc5da39f395d42c144940b98115c96058fb01f8a629c
-  data.tar.gz: 77aeb246a6a23477d07195c020091999825af21c9639c0a9679017daefeea9e9
+  metadata.gz: 057dac92f5a83458b244a962ef0518fa093365a38c2388582de0556ac7b884ca
+  data.tar.gz: 9aebb908d52dda01dc3efce7dcda3e40c8b4f45e718cef94052c9d37fca604eb
 SHA512:
-  metadata.gz: e48596ac6e5fc14be89c2aadc50416558fa4a08594d28cccee630c1157dc365a556999c82d303e4669c94e1db88d6b0cf3f044730d0057586220a6c3172b72a6
-  data.tar.gz: 2112dfa9191e8aa948317a16d581a9ea487327030cb013b43937757c144c33ca4fc975faee91fd6731d8f043ed0cefd5ecc28a562351c5de9b96196167871e36
+  metadata.gz: aa91e78ec3aef0010553886539284af4aafb3ed7014ef35e6f7094377bf9ad2efb7f6025bc7f8715d4558d8a5a1242325761a12a6e696590828fa1752c579198
+  data.tar.gz: 7309cc34f3d022d212ab9870558909ab84be3c6550160ce822808f98edb259c96669899247a7faefec6365068ac5aa68a6d5b43f279591156f47c9f02d0e182c

data/README.md

@@ -15,6 +15,11 @@ You can set a schedule for pipelines that are triggered by new data, specifying
 executed in the [ISO8601 duration format](https://www.digi.com/resources/documentation/digidocs//90001488-13/reference/r_iso_8601_duration_format.htm). The processing logic is then executed.   
 ###### methods:
  - .new(options): options can be schedule in [ISO8601 duration format](https://www.digi.com/resources/documentation/digidocs//90001488-13/reference/r_iso_8601_duration_format.htm)  and name
+   - options:
+     - name: pipeline name
+     - schedule: [ISO8601 duration format](https://www.digi.com/resources/documentation/digidocs//90001488-13/reference/r_iso_8601_duration_format.htm)
+     - cron: in cron format ex. '1 12 * * *' intervals are not supported
+     - uri: a directory/file to watch
  - .run: start the pipeline. blocking if a schedule is supplied
  - .stop: stop the pipeline
  - .pause: pause the pipeline. Restart using .run
@@ -37,6 +42,19 @@ end
 pipeline.run
 ```
 
+```ruby
+#create a pipeline scheduled to run every morning at 06:00 am
+pipeline = Pipeline.new(schedule: '0 6 * * *')
+
+pipeline.on_message do |input, output|
+  data = input.from_uri("https://dummyjson.com/comments?limit=10")
+  # process data
+end
+
+pipeline.run
+```
+
+
 ```ruby
 #create a pipeline to listen and process files in a directory
 extract = DataCollector::Pipeline.new(name: 'extract', uri: 'file://./data/in')

data/data_collector.gemspec

@@ -50,6 +50,7 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency 'bunny', '~> 2.20'
   spec.add_runtime_dependency 'bunny_burrow', '~> 1.5'
   spec.add_runtime_dependency 'builder', '~> 3.2'
+  spec.add_runtime_dependency 'parse-cron', '~> 0.1'
 
   spec.add_development_dependency 'bundler', '~> 2.3'
   spec.add_development_dependency 'minitest', '~> 5.18'

data/lib/data_collector/input/rpc.rb

@@ -55,7 +55,7 @@ module DataCollector
           parse_uri
           server.rabbitmq_url = @bunny_uri.to_s
           server.rabbitmq_exchange = @bunny_channel
-          server.logger = DataCollector::Core.logger
+          #server.logger = DataCollector::Core.logger
         end
       end
 

data/lib/data_collector/input.rb

@@ -31,7 +31,7 @@ module DataCollector
       raise DataCollector::Error 'from_uri expects a scheme like file:// of https://' unless source =~ /:\/\//
 
       scheme, path = source.split('://')
-      source="#{scheme}://#{URI.encode_uri_component(path)}"
+      source="#{scheme}://#{URI.encode_www_form_component(path)}"
       uri = URI(source)
       begin
         data = nil
@@ -41,7 +41,7 @@ module DataCollector
         when 'https'
           data = from_https(uri, options)
         when 'file'
-          absolute_path = File.absolute_path("#{URI.decode_uri_component("#{uri.host}#{uri.path}")}")
+          absolute_path = File.absolute_path("#{URI.decode_www_form_component("#{uri.host}#{uri.path}")}")
           if File.directory?(absolute_path)
             #raise DataCollector::Error, "#{uri.host}/#{uri.path} not found" unless File.exist?("#{uri.host}/#{uri.path}")
             return from_dir(uri, options)
@@ -80,7 +80,7 @@ module DataCollector
     end
 
     def from_https(uri, options = {})
-      uri = URI.decode_uri_component("#{uri.to_s}")
+      uri = URI.decode_www_form_component("#{uri.to_s}")
       data = nil
       if options.with_indifferent_access.include?(:logging) && options.with_indifferent_access[:logging]
         HTTP.default_options = HTTP::Options.new(features: { logging: { logger: @logger } })
@@ -102,6 +102,16 @@ module DataCollector
         http = HTTP.auth(bearer)
       end
 
+      if options.key?(:cookies) 
+        @logger.debug  "Set cookies"
+        http = http.cookies( options[:cookies] )
+      end
+
+      if options.key?(:headers) 
+        @logger.debug  "Set http headers"
+        http = http.headers( options[:headers] )
+      end
+          
       if options.key?(:verify_ssl) && uri.scheme.eql?('https')
         @logger.warn "Disabling SSL verification. "
         #shouldn't use this but we all do ...
@@ -109,10 +119,11 @@ module DataCollector
         ctx.verify_mode = OpenSSL::SSL::VERIFY_NONE
 
         http_response = http.follow.get(escape_uri(uri), ssl_context: ctx)
+
       else
         http_response = http.follow.get(escape_uri(uri))
       end
-
+ 
       case http_response.code
       when 200..299
         @raw = data = http_response.body.to_s
@@ -235,7 +246,7 @@ module DataCollector
     end
 
     def normalize_uri(uri)
-      "#{URI.decode_uri_component(uri.host)}#{URI.decode_uri_component(uri.path)}"
+      "#{URI.decode_www_form_component(uri.host)}#{URI.decode_www_form_component(uri.path)}"
     end
   end
 end

data/lib/data_collector/pipeline.rb

@@ -1,4 +1,5 @@
 require 'iso8601'
+require 'parse-cron'
 
 module DataCollector
   class Pipeline
@@ -12,6 +13,7 @@ module DataCollector
       @run_count = 0
 
       @schedule = options[:schedule] || {}
+      @cron = options[:cron || '']
       @name = options[:name] || "pipeline-#{Time.now.to_i}-#{rand(10000)}"
       @options = options
       @listeners = []
@@ -43,6 +45,20 @@ module DataCollector
 
           DataCollector::Core.log("PIPELINE running in #{interval.size} seconds")
           sleep interval.size
+          handle_on_message(@input, @output) unless paused?
+        end
+      elsif @cron && !@cron.empty?
+        cron_parser = CronParser.new(@cron)
+        while running?
+          @run_count += 1
+          start_time = ISO8601::DateTime.new(Time.now.to_datetime.to_s)
+          next_run = cron_parser.next(start_time.to_time)
+
+          interval = ISO8601::TimeInterval.from_datetimes(start_time,  ISO8601::DateTime.new(next_run.to_datetime.to_s))
+
+          DataCollector::Core.log("PIPELINE running at #{next_run.to_datetime.strftime('%Y-%m-%dT%H:%M:%S')} or in #{interval.size} seconds")
+          sleep interval.size
+
           handle_on_message(@input, @output) unless paused?
         end
       else # run once

data/lib/data_collector/version.rb

@@ -1,4 +1,4 @@
 # encoding: utf-8
 module DataCollector
-  VERSION = "0.24.0"
+  VERSION = "0.26.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: data_collector
 version: !ruby/object:Gem::Version
-  version: 0.24.0
+  version: 0.26.0
 platform: ruby
 authors:
 - Mehmet Celik
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-07-24 00:00:00.000000000 Z
+date: 2023-08-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -206,6 +206,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.2'
+- !ruby/object:Gem::Dependency
+  name: parse-cron
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -319,7 +333,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.10
+rubygems_version: 3.4.13
 signing_key:
 specification_version: 4
 summary: ETL helper library