data_collector 0.24.0 → 0.25.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7745e79eb3836ab3c469cc5da39f395d42c144940b98115c96058fb01f8a629c
-  data.tar.gz: 77aeb246a6a23477d07195c020091999825af21c9639c0a9679017daefeea9e9
+  metadata.gz: 8065c0d1b54cf1c39be5cfcb70a4fae1f33b02021182c75538ed03e73393d35a
+  data.tar.gz: 879054ad24b178f08bfea7914c5fbc593c32c611bd4e3ad833869a3e5b36d5b1
 SHA512:
-  metadata.gz: e48596ac6e5fc14be89c2aadc50416558fa4a08594d28cccee630c1157dc365a556999c82d303e4669c94e1db88d6b0cf3f044730d0057586220a6c3172b72a6
-  data.tar.gz: 2112dfa9191e8aa948317a16d581a9ea487327030cb013b43937757c144c33ca4fc975faee91fd6731d8f043ed0cefd5ecc28a562351c5de9b96196167871e36
+  metadata.gz: 390ac889c52055cfd8f5326c6e7c1549faee6b8c41af4535b9fc5d3038701f62c441caaf41895d7ad64b4941e609fdee32ac745255b9cb2fbc0981d787b00847
+  data.tar.gz: dbc57c97f30e5659ccfebba0850e99eee24ecada4a5e9a136ae6b036f19b587a4d66460e509c32d36be13cd4d745ed89f9a73e11a32b1b15fe800036c1836bea

data/README.md

@@ -15,6 +15,11 @@ You can set a schedule for pipelines that are triggered by new data, specifying
 executed in the [ISO8601 duration format](https://www.digi.com/resources/documentation/digidocs//90001488-13/reference/r_iso_8601_duration_format.htm). The processing logic is then executed.   
 ###### methods:
  - .new(options): options can be schedule in [ISO8601 duration format](https://www.digi.com/resources/documentation/digidocs//90001488-13/reference/r_iso_8601_duration_format.htm)  and name
+   - options:
+     - name: pipeline name
+     - schedule: [ISO8601 duration format](https://www.digi.com/resources/documentation/digidocs//90001488-13/reference/r_iso_8601_duration_format.htm)
+     - cron: in cron format ex. '1 12 * * *' intervals are not supported
+     - uri: a directory/file to watch
  - .run: start the pipeline. blocking if a schedule is supplied
  - .stop: stop the pipeline
  - .pause: pause the pipeline. Restart using .run
@@ -37,6 +42,19 @@ end
 pipeline.run
 ```
 
+```ruby
+#create a pipeline scheduled to run every morning at 06:00 am
+pipeline = Pipeline.new(schedule: '0 6 * * *')
+
+pipeline.on_message do |input, output|
+  data = input.from_uri("https://dummyjson.com/comments?limit=10")
+  # process data
+end
+
+pipeline.run
+```
+
+
 ```ruby
 #create a pipeline to listen and process files in a directory
 extract = DataCollector::Pipeline.new(name: 'extract', uri: 'file://./data/in')

data/data_collector.gemspec

@@ -50,6 +50,7 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency 'bunny', '~> 2.20'
   spec.add_runtime_dependency 'bunny_burrow', '~> 1.5'
   spec.add_runtime_dependency 'builder', '~> 3.2'
+  spec.add_runtime_dependency 'parse-cron', '~> 0.1'
 
   spec.add_development_dependency 'bundler', '~> 2.3'
   spec.add_development_dependency 'minitest', '~> 5.18'

data/lib/data_collector/pipeline.rb

@@ -1,4 +1,5 @@
 require 'iso8601'
+require 'parse-cron'
 
 module DataCollector
   class Pipeline
@@ -12,6 +13,7 @@ module DataCollector
       @run_count = 0
 
       @schedule = options[:schedule] || {}
+      @cron = options[:cron || '']
       @name = options[:name] || "pipeline-#{Time.now.to_i}-#{rand(10000)}"
       @options = options
       @listeners = []
@@ -43,6 +45,20 @@ module DataCollector
 
           DataCollector::Core.log("PIPELINE running in #{interval.size} seconds")
           sleep interval.size
+          handle_on_message(@input, @output) unless paused?
+        end
+      elsif @cron && !@cron.empty?
+        cron_parser = CronParser.new(@cron)
+        while running?
+          @run_count += 1
+          start_time = ISO8601::DateTime.new(Time.now.to_datetime.to_s)
+          next_run = cron_parser.next(start_time.to_time)
+
+          interval = ISO8601::TimeInterval.from_datetimes(start_time,  ISO8601::DateTime.new(next_run.to_datetime.to_s))
+
+          DataCollector::Core.log("PIPELINE running at #{next_run.to_datetime.strftime('%Y-%m-%dT%H:%M:%S')} or in #{interval.size} seconds")
+          sleep interval.size
+
           handle_on_message(@input, @output) unless paused?
         end
       else # run once

data/lib/data_collector/version.rb

@@ -1,4 +1,4 @@
 # encoding: utf-8
 module DataCollector
-  VERSION = "0.24.0"
+  VERSION = "0.25.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: data_collector
 version: !ruby/object:Gem::Version
-  version: 0.24.0
+  version: 0.25.0
 platform: ruby
 authors:
 - Mehmet Celik
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-07-24 00:00:00.000000000 Z
+date: 2023-07-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -206,6 +206,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.2'
+- !ruby/object:Gem::Dependency
+  name: parse-cron
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -319,7 +333,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.10
+rubygems_version: 3.4.13
 signing_key:
 specification_version: 4
 summary: ETL helper library