dwf 0.1.9 → 0.1.13

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d37bac690df59157001462d6017a8a63d3956a280689de789d9a99163a476832
-  data.tar.gz: 935d76e99ff5ca109f00be80c01950c20272969f556fc70d0e1ef5c026c024e4
+  metadata.gz: 131525481d493d4765a33fa9b5d34524f74f221b05399b1a39b2cc47d439aa65
+  data.tar.gz: 4483fe6f6e98e45579d65362c2ba3f0298303522d103647c0a3d04085580af16
 SHA512:
-  metadata.gz: c105d178cd44f12577ee9eeed5065768a23adc508da7d686e50c46d26e4eed9c160dc71b10942ab8041e38f6c732384742f40fa9d3076e0a924fbdfb1cd62845
-  data.tar.gz: ad9c6f1041f092847364e91408887be311435be62e0502ab54f2b408c92fc598a879792eabb00c81b05d5c97fb37ea4a19136765d54b0d4efe9dbdb9ad2f64b5
+  metadata.gz: 9a5e357a59d77faa86eaa54552f19b2ad06b82af8a72bc80a2c66fd8ba56584391c08878c0e0b16cf32df6e6dc564d01bf60b0c16b2bcafe7649b6522643db48
+  data.tar.gz: 056e815c2beed08ca7e8fccc0c22afb291c849f9195b1bd5d7469d22096b62a37a8a457a21eeaa949f436e034586217e3aacf4e7d6cf44e52f325c88126c8b6c

data/.github/workflows/build_gem.yaml

@@ -1,13 +1,9 @@
 name: Ruby Gem
 
 on:
-  # Manually publish
   workflow_dispatch:
-  # Alternatively, publish whenever changes are merged to the `main` branch.
   push:
     branches: [ master ]
-    paths:
-      - 'dwf.gemspec'
 
 jobs:
   build:

data/.github/workflows/test.yaml

@@ -13,12 +13,16 @@ jobs:
 
     runs-on: ubuntu-latest
 
+    strategy:
+      matrix:
+        ruby-version: ["2.5", "2.6", "2.7", "3.0"]
+
     steps:
       - uses: actions/checkout@v2
       - name: Set up Ruby
         uses: ruby/setup-ruby@477b21f02be01bcb8030d50f37cfec92bfa615b6
         with:
-          ruby-version: 3.0.0
+          ruby-version: ${{ matrix.ruby-version }}
       - name: Install dependencies
         run: bundle install
       - name: Run tests

data/.gitignore

@@ -4,3 +4,4 @@ Gemfile.lock
 dwf-*.gem
 :w
 :W
+coverage/

data/CHANGELOG.md

@@ -1,5 +1,132 @@
 # Changelog
 All notable changes to this project will be documented in this file.
+
+## 0.1.12
+### Added
+#### Dynamic workflows
+There might be a case when you have to contruct the workflow dynamically depend on the input
+As an example, let's write a workflow which puts from 1 to 100 into the terminal parallely . Additionally after finish all job, it will puts the finshed word into the terminal
+```ruby
+class FirstMainItem < Dwf::Item
+  def perform
+    puts "#{self.class.name}: running #{params}"
+  end
+end
+
+SecondMainItem = Class.new(FirstMainItem)
+
+class TestWf < Dwf::Workflow
+  def configure
+    items = (1..100).to_a.map do |number|
+      run FirstMainItem, params: number
+    end
+    run SecondMainItem, after: items, params: "finished"
+  end
+end
+
+```
+We can achieve that because run method returns the id of the created job, which we can use for chaining dependencies.
+Now, when we create the workflow like this:
+```ruby
+wf = TestWf.create
+# wf.callback_type = Dwf::Workflow::SK_BATCH
+wf.start!
+```
+
+## 0.1.12
+### Added
+#### Subworkflow for all callback types
+same with `0.1.11`
+## 0.1.11
+### Added
+#### Subworkflow - Only support sidekiq pro
+There might be a case when you want to reuse a workflow in another workflow
+
+As an example, let's write a workflow which contain another workflow, expected that the SubWorkflow workflow execute after `SecondItem` and the `ThirdItem` execute after `SubWorkflow`
+
+```ruby
+gem 'dwf', '~> 0.1.11'
+```
+
+### Setup
+```ruby
+class FirstItem < Dwf::Item
+  def perform
+    puts "Main flow: #{self.class.name} running"
+    puts "Main flow: #{self.class.name} finish"
+  end
+end
+
+SecondItem = Class.new(FirstItem)
+ThirtItem = Class.new(FirstItem)
+
+class FirstSubItem < Dwf::Item
+  def perform
+    puts "Sub flow: #{self.class.name} running"
+    puts "Sub flow: #{self.class.name} finish"
+  end
+end
+
+SecondSubItem = Class.new(FirstSubItem)
+
+class SubWorkflow < Dwf::Workflow
+  def configure
+    run FirstSubItem
+    run SecondSubItem, after: FirstSubItem
+  end
+end
+
+
+class TestWf < Dwf::Workflow
+  def configure
+    run FirstItem
+    run SecondItem, after: FirstItem
+    run SubWorkflow, after: SecondItem
+    run ThirtItem, after: SubWorkflow
+  end
+end
+
+wf = TestWf.create
+wf.start!
+```
+
+### Result
+```
+Main flow: FirstItem running
+Main flow: FirstItem finish
+Main flow: SecondItem running
+Main flow: SecondItem finish
+Sub flow: FirstSubItem running
+Sub flow: FirstSubItem finish
+Sub flow: SecondSubItem running
+Sub flow: SecondSubItem finish
+Main flow: ThirtItem running
+Main flow: ThirtItem finish
+```
+
+## 0.1.10
+### Added
+- Allow to use argument within workflow and update the defining callback way
+```
+class TestWf < Dwf::Workflow
+  def configure(arguments)
+    run A
+    run B, after: A, params: argument
+    run C, after: A, params: argument
+  end
+end
+
+wf = TestWf.create(arguments)
+wf.callback_type = Dwf::Workflow::SK_BATCH
+
+```
+- Support `find` workflow and `reload` workflow
+```
+wf = TestWf.create
+Dwf::Workflow.find(wf.id)
+wf.reload
+```
+
 ## 0.1.9
 ### Added
 ### Fixed

data/README.md

@@ -1,25 +1,48 @@
-# DSL playground
-[Gush](https://github.com/chaps-io/gush) cloned without [ActiveJob](https://guides.rubyonrails.org/active_job_basics.html) but requried [Sidekiq](https://github.com/mperham/sidekiq). This project is for researching DSL purpose
+# DWF
+Distributed workflow runner following [Gush](https://github.com/chaps-io/gush) interface using [Sidekiq](https://github.com/mperham/sidekiq) and [Redis](https://redis.io/). This project is for researching DSL purpose
 
 # Installation
 ## 1. Add `dwf` to Gemfile
 ```ruby
-gem 'dwf', '~> 0.1.9'
+gem 'dwf', '~> 0.1.12'
 ```
-## 2. Execute flow
+## 2. Execute flow example
 ### Declare jobs
 
 ```ruby
 require 'dwf'
 
-class A < Dwf::Item
+class FirstItem < Dwf::Item
   def perform
-    puts "#{self.class.name} Working"
-    sleep 2
-    puts params
-    puts "#{self.class.name} Finished"
+    puts "#{self.class.name}: running"
+    puts "#{self.class.name}: finish"
   end
 end
+
+class SecondItem < Dwf::Item
+  def perform
+    puts "#{self.class.name}: running"
+    output('Send to ThirdItem')
+    puts "#{self.class.name} finish"
+  end
+end
+
+class ThirdItem < Dwf::Item
+  def perform
+    puts "#{self.class.name}: running"
+    puts "#{self.class.name}: finish"
+  end
+end
+
+class FourthItem < Dwf::Item
+  def perform
+    puts "#{self.class.name}: running"
+    puts "payloads from incoming: #{payloads.inspect}"
+    puts "#{self.class.name}: finish"
+  end
+end
+
+FifthItem = Class.new(FirstItem)
 ```
 
 ### Declare flow
@@ -28,20 +51,23 @@ require 'dwf'
 
 class TestWf < Dwf::Workflow
   def configure
-    run A
-    run B, after: A
-    run C, after: A
-    run E, after: [B, C], params: 'E say hello'
-    run D, after: [E], params: 'D say hello'
-    run F, params: 'F say hello'
+    run FirstItem
+    run SecondItem, after: FirstItem
+    run ThirdItem, after: FirstItem
+    run FourthItem, after: [ThirdItem, SecondItem]
+    run FifthItem, after: FourthItem
   end
 end
 ```
-
+### Start background worker process
+```
+bundle exec sidekiq -q dwf
+```
 
 ### Execute flow
 ```ruby
-wf = TestWf.create(callback_type: Dwf::Workflow::SK_BATCH)
+wf = TestWf.create
+wf.callback_type = Dwf::Workflow::SK_BATCH
 wf.start!
 ```
 
@@ -54,21 +80,16 @@ By default `dwf` will use `Dwf::Workflow::BUILD_IN` callback.
 
 ### Output
 ```
-A Working
-F Working
-A Finished
-F say hello
-F Finished
-C Working
-B Working
-C Finished
-B Finished
-E Working
-E say hello
-E Finished
-D Working
-D say hello
-D Finished
+FirstItem: running
+FirstItem: finish
+SecondItem: running
+SecondItem finish
+ThirdItem: running
+ThirdItem: finish
+FourthItem: running
+FourthItem: finish
+FifthItem: running
+FifthItem: finish
 ```
 
 # Config redis and default queue
@@ -83,8 +104,8 @@ Dwf.config do |config|
   config.namespace = 'dwf'
 end
 ```
-
-# Pinelining
+# Advanced features
+## Pipelining
 You can pass jobs result to next nodes
 
 ```ruby
@@ -117,6 +138,96 @@ end
   }
 ]
 ```
+## Sub workflow
+There might be a case when you want to reuse a workflow in another workflow
+
+As an example, let's write a workflow which contain another workflow, expected that the SubWorkflow workflow execute after `SecondItem` and the `ThirdItem` execute after `SubWorkflow`
+
+### Setup
+```ruby
+class FirstItem < Dwf::Item
+  def perform
+    puts "Main flow: #{self.class.name} running"
+    puts "Main flow: #{self.class.name} finish"
+  end
+end
+
+SecondItem = Class.new(FirstItem)
+ThirtItem = Class.new(FirstItem)
+
+class FirstSubItem < Dwf::Item
+  def perform
+    puts "Sub flow: #{self.class.name} running"
+    puts "Sub flow: #{self.class.name} finish"
+  end
+end
+
+SecondSubItem = Class.new(FirstSubItem)
+
+class SubWorkflow < Dwf::Workflow
+  def configure
+    run FirstSubItem
+    run SecondSubItem, after: FirstSubItem
+  end
+end
+
+
+class TestWf < Dwf::Workflow
+  def configure
+    run FirstItem
+    run SecondItem, after: FirstItem
+    run SubWorkflow, after: SecondItem
+    run ThirtItem, after: SubWorkflow
+  end
+end
+
+wf = TestWf.create
+wf.start!
+```
+
+### Result
+```
+Main flow: FirstItem running
+Main flow: FirstItem finish
+Main flow: SecondItem running
+Main flow: SecondItem finish
+Sub flow: FirstSubItem running
+Sub flow: FirstSubItem finish
+Sub flow: SecondSubItem running
+Sub flow: SecondSubItem finish
+Main flow: ThirtItem running
+Main flow: ThirtItem finish
+```
+
+## Dynamic workflows
+There might be a case when you have to contruct the workflow dynamically depend on the input
+As an example, let's write a workflow which puts from 1 to 100 into the terminal parallelly . Additionally after finish all job, it will puts the finshed word into the terminal
+```ruby
+class FirstMainItem < Dwf::Item
+  def perform
+    puts "#{self.class.name}: running #{params}"
+  end
+end
+
+SecondMainItem = Class.new(FirstMainItem)
+
+class TestWf < Dwf::Workflow
+  def configure
+    items = (1..100).to_a.map do |number|
+      run FirstMainItem, params: number
+    end
+    run SecondMainItem, after: items, params: "finished"
+  end
+end
+
+```
+We can achieve that because run method returns the id of the created job, which we can use for chaining dependencies.
+Now, when we create the workflow like this:
+```ruby
+wf = TestWf.create
+# wf.callback_type = Dwf::Workflow::SK_BATCH
+wf.start!
+```
 
 # Todo
 - [x] Make it work
@@ -124,9 +235,12 @@ end
 - [x] Support with build-in callback
 - [x] Add github workflow
 - [x] Redis configurable
-- [x] Pinelining
-- [ ] [WIP] Test
+- [x] Pipelining
+- [X] Test
+- [x] Sub workflow
 - [ ] Support [Resque](https://github.com/resque/resque)
+- [ ] Key value store plugable
+  - [ ] research https://github.com/moneta-rb/moneta
 
 # References
 - https://github.com/chaps-io/gush

data/dwf.gemspec

@@ -3,14 +3,15 @@
 
 lib = File.expand_path('../lib', __FILE__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require_relative 'lib/dwf/version'
 
 Gem::Specification.new do |spec|
-  spec.name          = "dwf"
-  spec.version       = '0.1.9'
-  spec.authors       = ["dthtien"]
-  spec.email         = ["tiendt2311@gmail.com"]
+  spec.name          = 'dwf'
+  spec.version       = Dwf::VERSION
+  spec.authors       = ['dthtien']
+  spec.email         = ['tiendt2311@gmail.com']
 
-  spec.summary       = 'Gush cloned without ActiveJob but requried Sidekiq. This project is for researching DSL purpose'
+  spec.summary       = 'Distributed workflow runner following Gush interface using Sidekiq and Redis'
   spec.description   = 'Workflow'
   spec.homepage      = 'https://github.com/dthtien/wf'
   spec.license       = "MIT"
@@ -25,8 +26,10 @@ Gem::Specification.new do |spec|
   # guide at: https://bundler.io/guides/creating_gem.html
 
   spec.add_development_dependency 'byebug', '~> 11.1.3'
+  spec.add_development_dependency 'mock_redis', '~> 0.27.2'
   spec.add_dependency 'redis', '~> 4.2.0'
+  spec.add_dependency 'redis-mutex', '~> 4.0.2'
   spec.add_development_dependency 'rspec', '~> 3.2'
-  spec.add_development_dependency 'mock_redis', '~> 0.27.2'
   spec.add_dependency 'sidekiq', '~> 6.2.0'
+  spec.add_development_dependency 'simplecov'
 end

data/lib/dwf/callback.rb

@@ -9,8 +9,8 @@ module Dwf
       previous_job_names = options['names']
       workflow_id = options['workflow_id']
       processing_job_names = previous_job_names.map do |job_name|
-        job = client.find_job(workflow_id, job_name)
-        job.outgoing
+        node = client.find_node(job_name, workflow_id)
+        node.outgoing
       end.flatten.uniq
       return if processing_job_names.empty?
 
@@ -19,7 +19,7 @@ module Dwf
     end
 
     def start(job)
-      job.outgoing.any? ? start_with_batch(job) : job.perform_async
+      job.outgoing.any? ? start_with_batch(job) : job.persist_and_perform_async!
     end
 
     private
@@ -40,11 +40,14 @@ module Dwf
       batch.on(
         :success,
         'Dwf::Callback#process_next_step',
-        names: jobs.map(&:klass),
+        names: jobs.map(&:name),
         workflow_id: workflow_id
       )
       batch.jobs do
-        jobs.each { |job| job.persist_and_perform_async! if job.ready_to_start? }
+        jobs.each do |job|
+          job.reload
+          job.persist_and_perform_async! if job.ready_to_start?
+        end
       end
     end
 
@@ -61,25 +64,24 @@ module Dwf
 
     def fetch_jobs(processing_job_names, workflow_id)
       processing_job_names.map do |job_name|
-        client.find_job(workflow_id, job_name)
+        client.find_node(job_name, workflow_id)
       end.compact
     end
 
-    def with_lock(workflow_id, job_name)
-      client.check_or_lock(workflow_id, job_name)
-      yield
-      client.release_lock(workflow_id, job_name)
+    def with_lock(workflow_id, job_name, &block)
+      client.check_or_lock(workflow_id, job_name, &block)
     end
 
-    def start_with_batch(job)
+    def start_with_batch(node)
       batch = Sidekiq::Batch.new
+      workflow_id = node.is_a?(Dwf::Workflow) ? node.parent_id : node.workflow_id
       batch.on(
         :success,
         'Dwf::Callback#process_next_step',
-        names: [job.name],
-        workflow_id: job.workflow_id
+        names: [node.name],
+        workflow_id: workflow_id
       )
-      batch.jobs { job.perform_async }
+      batch.jobs { node.persist_and_perform_async! }
     end
 
     def client

data/lib/dwf/client.rb

@@ -1,3 +1,6 @@
+require_relative 'errors'
+require 'redis-mutex'
+
 module Dwf
   class Client
     attr_reader :config
@@ -20,18 +23,50 @@ module Dwf
       Dwf::Item.from_hash(Dwf::Utils.symbolize_keys(data))
     end
 
+    def find_node(name, workflow_id)
+      if Utils.workflow_name?(name)
+        if name.include?('|')
+          _, id = name.split('|')
+        else
+          id = workflow_id(name, workflow_id)
+        end
+        find_workflow(id)
+      else
+        find_job(workflow_id, name)
+      end
+    end
+
+    def find_workflow(id)
+      key = redis.keys("dwf.workflows.#{id}*").first
+      data = redis.get(key)
+      raise WorkflowNotFound, "Workflow with given id doesn't exist" if data.nil?
+
+      hash = JSON.parse(data)
+      hash = Dwf::Utils.symbolize_keys(hash)
+      nodes = parse_nodes(id)
+      workflow_from_hash(hash, nodes)
+    end
+
+    def find_sub_workflow(name, parent_id)
+      find_workflow(workflow_id(name, parent_id))
+    end
+
+    def sub_workflows(id)
+      keys = redis.keys("dwf.workflows.*.*.#{id}")
+      keys.map do |key|
+        id = key.split('.')[2]
+
+        find_workflow(id)
+      end
+    end
+
     def persist_job(job)
       redis.hset("dwf.jobs.#{job.workflow_id}.#{job.klass}", job.id, job.as_json)
     end
 
-    def check_or_lock(workflow_id, job_name)
+    def check_or_lock(workflow_id, job_name, &block)
       key = "wf_enqueue_outgoing_jobs_#{workflow_id}-#{job_name}"
-
-      if key_exists?(key)
-        sleep 2
-      else
-        set(key, 'running')
-      end
+      RedisMutex.with_lock(key, sleep: 0.3, block: 2, &block)
     end
 
     def release_lock(workflow_id, job_name)
@@ -39,7 +74,10 @@ module Dwf
     end
 
     def persist_workflow(workflow)
-      redis.set("dwf.workflows.#{workflow.id}", workflow.as_json)
+      key = [
+        'dwf', 'workflows', workflow.id, workflow.class.name, workflow.parent_id
+      ].compact.join('.')
+      redis.set(key, workflow.as_json)
     end
 
     def build_job_id(workflow_id, job_klass)
@@ -84,6 +122,13 @@ module Dwf
 
     private
 
+    def workflow_id(name, parent_id)
+      key = redis.keys("dwf.workflows.*.#{name}.#{parent_id}").first
+      return if key.nil?
+
+      key.split('.')[2]
+    end
+
     def find_job_by_klass_and_id(workflow_id, job_name)
       job_klass, job_id = job_name.split('|')
 
@@ -99,8 +144,36 @@ module Dwf
       job
     end
 
+    def parse_nodes(id)
+      keys = redis.scan_each(match: "dwf.jobs.#{id}.*")
+
+      items = keys.map do |key|
+        redis.hvals(key).map do |json|
+          node = Dwf::Utils.symbolize_keys JSON.parse(json)
+          Dwf::Item.from_hash(node)
+        end
+      end.flatten
+      workflows = sub_workflows(id)
+      items + workflows
+    end
+
+    def workflow_from_hash(hash, jobs = [])
+      flow = Module.const_get(hash[:klass]).new(*hash[:arguments])
+      flow.jobs = []
+      flow.outgoing = hash.fetch(:outgoing, [])
+      flow.parent_id = hash[:parent_id]
+      flow.incoming = hash.fetch(:incoming, [])
+      flow.stopped = hash.fetch(:stopped, false)
+      flow.callback_type = hash.fetch(:callback_type, Workflow::BUILD_IN)
+      flow.id = hash[:id]
+      flow.jobs = jobs
+      flow
+    end
+
     def redis
-      @redis ||= Redis.new(config.redis_opts)
+      @redis ||= Redis.new(config.redis_opts).tap do |instance|
+        RedisClassy.redis = instance
+      end
     end
   end
 end

data/lib/dwf/concerns/checkable.rb

@@ -0,0 +1,29 @@
+module Dwf
+  module Concerns
+    module Checkable
+      def no_dependencies?
+        incoming.empty?
+      end
+
+      def leaf?
+        outgoing.empty?
+      end
+
+      def ready_to_start?
+        !running? && !enqueued? && !finished? && !failed? && parents_succeeded?
+      end
+
+      def succeeded?
+        finished? && !failed?
+      end
+
+      def running?
+        started? && !finished?
+      end
+
+      def started?
+        !!started_at
+      end
+    end
+  end
+end

data/lib/dwf/errors.rb

@@ -0,0 +1,3 @@
+module Dwf
+  class WorkflowNotFound < StandardError; end
+end