data_collector 0.23.0 → 0.25.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1a6db4367b12a41101620cc818a1995868aab20b65157de8c19ea60c68aa90a0
-  data.tar.gz: 39a2cbcd665f85d71bda741a131c54d4f2871ae83ae63b86c9eed9f0d84e405a
+  metadata.gz: 8065c0d1b54cf1c39be5cfcb70a4fae1f33b02021182c75538ed03e73393d35a
+  data.tar.gz: 879054ad24b178f08bfea7914c5fbc593c32c611bd4e3ad833869a3e5b36d5b1
 SHA512:
-  metadata.gz: 5d7e6e243f9693603c4502b2f1bbe2fb64b8cf95718be257066940d6fafed24a8352ded805f285b617e25f5fbd703bf6e812dbd9f2b85d5c22cc143e7cdfdc1e
-  data.tar.gz: 73b0c03ef6ccb7855cfc1b6f424c543cc00fc79e182ccc2525c068f3b8d97535e4eaaaa66b0fd59225049d6a0dfb3dc55861319638dfb4c6df489ede6c1f9077
+  metadata.gz: 390ac889c52055cfd8f5326c6e7c1549faee6b8c41af4535b9fc5d3038701f62c441caaf41895d7ad64b4941e609fdee32ac745255b9cb2fbc0981d787b00847
+  data.tar.gz: dbc57c97f30e5659ccfebba0850e99eee24ecada4a5e9a136ae6b036f19b587a4d66460e509c32d36be13cd4d745ed89f9a73e11a32b1b15fe800036c1836bea

data/README.md

@@ -15,6 +15,11 @@ You can set a schedule for pipelines that are triggered by new data, specifying
 executed in the [ISO8601 duration format](https://www.digi.com/resources/documentation/digidocs//90001488-13/reference/r_iso_8601_duration_format.htm). The processing logic is then executed.   
 ###### methods:
  - .new(options): options can be schedule in [ISO8601 duration format](https://www.digi.com/resources/documentation/digidocs//90001488-13/reference/r_iso_8601_duration_format.htm)  and name
+   - options:
+     - name: pipeline name
+     - schedule: [ISO8601 duration format](https://www.digi.com/resources/documentation/digidocs//90001488-13/reference/r_iso_8601_duration_format.htm)
+     - cron: in cron format ex. '1 12 * * *' intervals are not supported
+     - uri: a directory/file to watch
  - .run: start the pipeline. blocking if a schedule is supplied
  - .stop: stop the pipeline
  - .pause: pause the pipeline. Restart using .run
@@ -37,6 +42,19 @@ end
 pipeline.run
 ```
 
+```ruby
+#create a pipeline scheduled to run every morning at 06:00 am
+pipeline = Pipeline.new(schedule: '0 6 * * *')
+
+pipeline.on_message do |input, output|
+  data = input.from_uri("https://dummyjson.com/comments?limit=10")
+  # process data
+end
+
+pipeline.run
+```
+
+
 ```ruby
 #create a pipeline to listen and process files in a directory
 extract = DataCollector::Pipeline.new(name: 'extract', uri: 'file://./data/in')

data/data_collector.gemspec

@@ -50,6 +50,7 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency 'bunny', '~> 2.20'
   spec.add_runtime_dependency 'bunny_burrow', '~> 1.5'
   spec.add_runtime_dependency 'builder', '~> 3.2'
+  spec.add_runtime_dependency 'parse-cron', '~> 0.1'
 
   spec.add_development_dependency 'bundler', '~> 2.3'
   spec.add_development_dependency 'minitest', '~> 5.18'

data/lib/data_collector/input/dir.rb

@@ -26,7 +26,7 @@ module DataCollector
       private
 
       def create_listener
-        absolute_path = File.absolute_path("#{URI.decode_uri_component(@uri.to_s)}")
+        absolute_path = File.absolute_path("#{@uri.host}#{@uri.path}")
         raise DataCollector::Error, "#{@uri.to_s} not found" unless File.exist?(absolute_path)
 
         @listener ||= Listen.to(absolute_path, @options) do |modified, added, _|

data/lib/data_collector/input/generic.rb

@@ -4,7 +4,7 @@ module DataCollector
   class Input
     class Generic
       def initialize(uri, options = {})
-        @uri = URI(URI.decode_uri_component(uri.to_s))
+        @uri = URI(URI.decode_uri_component(uri.to_s)) #"#{uri.scheme}://#{URI.decode_uri_component(uri.host)}#{URI.decode_uri_component(uri.path)}"
         @options = options
         @running = false
 

data/lib/data_collector/input.rb

@@ -160,7 +160,8 @@ module DataCollector
 
     def from_file(uri, options = {})
       data = nil
-      absolute_path = File.absolute_path("#{URI.decode_uri_component(uri.to_s)}")
+      uri = normalize_uri(uri)
+      absolute_path = File.absolute_path(uri)
       raise DataCollector::Error, "#{uri.to_s} not found" unless File.exist?("#{absolute_path}")
       unless options.has_key?('raw') && options['raw'] == true
         @raw = data = File.read("#{absolute_path}")
@@ -233,5 +234,8 @@ module DataCollector
       return file_type
     end
 
+    def normalize_uri(uri)
+      "#{URI.decode_uri_component(uri.host)}#{URI.decode_uri_component(uri.path)}"
+    end
   end
 end

data/lib/data_collector/pipeline.rb

@@ -1,4 +1,5 @@
 require 'iso8601'
+require 'parse-cron'
 
 module DataCollector
   class Pipeline
@@ -12,6 +13,7 @@ module DataCollector
       @run_count = 0
 
       @schedule = options[:schedule] || {}
+      @cron = options[:cron || '']
       @name = options[:name] || "pipeline-#{Time.now.to_i}-#{rand(10000)}"
       @options = options
       @listeners = []
@@ -43,6 +45,20 @@ module DataCollector
 
           DataCollector::Core.log("PIPELINE running in #{interval.size} seconds")
           sleep interval.size
+          handle_on_message(@input, @output) unless paused?
+        end
+      elsif @cron && !@cron.empty?
+        cron_parser = CronParser.new(@cron)
+        while running?
+          @run_count += 1
+          start_time = ISO8601::DateTime.new(Time.now.to_datetime.to_s)
+          next_run = cron_parser.next(start_time.to_time)
+
+          interval = ISO8601::TimeInterval.from_datetimes(start_time,  ISO8601::DateTime.new(next_run.to_datetime.to_s))
+
+          DataCollector::Core.log("PIPELINE running at #{next_run.to_datetime.strftime('%Y-%m-%dT%H:%M:%S')} or in #{interval.size} seconds")
+          sleep interval.size
+
           handle_on_message(@input, @output) unless paused?
         end
       else # run once

data/lib/data_collector/version.rb

@@ -1,4 +1,4 @@
 # encoding: utf-8
 module DataCollector
-  VERSION = "0.23.0"
+  VERSION = "0.25.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: data_collector
 version: !ruby/object:Gem::Version
-  version: 0.23.0
+  version: 0.25.0
 platform: ruby
 authors:
 - Mehmet Celik
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-07-24 00:00:00.000000000 Z
+date: 2023-07-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -206,6 +206,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.2'
+- !ruby/object:Gem::Dependency
+  name: parse-cron
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -319,7 +333,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.10
+rubygems_version: 3.4.13
 signing_key:
 specification_version: 4
 summary: ETL helper library