dwf 0.1.8 → 0.1.12

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 395f410168239441a56db548cccc50d729433bedcc9fd0d4c3ae15fea09c1019
-  data.tar.gz: 5a944cc497760afe6b92f38772d594d56f15837ab51495609dba24f7578af309
+  metadata.gz: 4d6ec140cc691fd53ceb5d495add6e3a76ce6a7b6daf2bfbde982e6e40a19987
+  data.tar.gz: 66911fc7ec1a54e38aaaca160ee455b2463f1888ecbbf981c64a17e9d790c1a2
 SHA512:
-  metadata.gz: 0123f65c0242c905c93e0998365a43b4193432c6e5b74af4a33422e7b489424294a70c68cf4e42feb627ecc038429e0dba6dc54af80acd1dc7628c2b56ff532c
-  data.tar.gz: 674a7144dc944d878cfddb549254985dc0a136fd84069df184a44c8a375f6ccc47e9be3d6b445f6bfcc83af8f52ed6b23fa9687f23714e49b30e24a6bbdea1de
+  metadata.gz: 240a6603023b3edc79b69d1793e8805e86ec298f2927df7bd3a1342ad241fb3a0b619a6aa50266348003c20f751fdd58dc91a68dead319916ca351a09bee5f0f
+  data.tar.gz: 4af1937daa29fab5f2fd818f22a64bea90d35074a0ca70109e4a36cbaa754fae199799167385f78579a854152563c1e8074c6ff91ad931e544b20c67bf7de054

data/.github/workflows/build_gem.yaml

@@ -1,13 +1,9 @@
 name: Ruby Gem
 
 on:
-  # Manually publish
   workflow_dispatch:
-  # Alternatively, publish whenever changes are merged to the `main` branch.
   push:
     branches: [ master ]
-    paths:
-      - 'dwf.gemspec'
 
 jobs:
   build:

data/.gitignore

@@ -4,3 +4,4 @@ Gemfile.lock
 dwf-*.gem
 :w
 :W
+coverage/

data/CHANGELOG.md

@@ -1,5 +1,100 @@
 # Changelog
 All notable changes to this project will be documented in this file.
+## 0.1.11
+### Added
+#### Subworkflow - Only support sidekiq pro
+There might be a case when you want to reuse a workflow in another workflow
+
+As an example, let's write a workflow which contain another workflow, expected that the SubWorkflow workflow execute after `SecondItem` and the `ThirdItem` execute after `SubWorkflow`
+
+```ruby
+gem 'dwf', '~> 0.1.11'
+```
+
+### Setup
+```ruby
+class FirstItem < Dwf::Item
+  def perform
+    puts "Main flow: #{self.class.name} running"
+    puts "Main flow: #{self.class.name} finish"
+  end
+end
+
+SecondItem = Class.new(FirstItem)
+ThirtItem = Class.new(FirstItem)
+
+class FirstSubItem < Dwf::Item
+  def perform
+    puts "Sub flow: #{self.class.name} running"
+    puts "Sub flow: #{self.class.name} finish"
+  end
+end
+
+SecondSubItem = Class.new(FirstSubItem)
+
+class SubWorkflow < Dwf::Workflow
+  def configure
+    run FirstSubItem
+    run SecondSubItem, after: FirstSubItem
+  end
+end
+
+
+class TestWf < Dwf::Workflow
+  def configure
+    run FirstItem
+    run SecondItem, after: FirstItem
+    run SubWorkflow, after: SecondItem
+    run ThirtItem, after: SubWorkflow
+  end
+end
+
+wf = TestWf.create
+wf.start!
+```
+
+### Result
+```
+Main flow: FirstItem running
+Main flow: FirstItem finish
+Main flow: SecondItem running
+Main flow: SecondItem finish
+Sub flow: FirstSubItem running
+Sub flow: FirstSubItem finish
+Sub flow: SecondSubItem running
+Sub flow: SecondSubItem finish
+Main flow: ThirtItem running
+Main flow: ThirtItem finish
+```
+
+## 0.1.10
+### Added
+- Allow to use argument within workflow and update the defining callback way
+```
+class TestWf < Dwf::Workflow
+  def configure(arguments)
+    run A
+    run B, after: A, params: argument
+    run C, after: A, params: argument
+  end
+end
+
+wf = TestWf.create(arguments)
+wf.callback_type = Dwf::Workflow::SK_BATCH
+
+```
+- Support `find` workflow and `reload` workflow
+```
+wf = TestWf.create
+Dwf::Workflow.find(wf.id)
+wf.reload
+```
+
+## 0.1.9
+### Added
+### Fixed
+- fix incorrect argument at configuration
+
 ## 0.1.8
 ### Added
 - add pinlining feature

data/README.md

@@ -1,25 +1,48 @@
-# DSL playground
-[Gush](https://github.com/chaps-io/gush) cloned without [ActiveJob](https://guides.rubyonrails.org/active_job_basics.html) but requried [Sidekiq](https://github.com/mperham/sidekiq). This project is for researching DSL purpose
+# DWF
+Distributed workflow runner following [Gush](https://github.com/chaps-io/gush) interface using [Sidekiq](https://github.com/mperham/sidekiq) and [Redis](https://redis.io/). This project is for researching DSL purpose
 
 # Installation
 ## 1. Add `dwf` to Gemfile
 ```ruby
-gem 'dwf', '~> 0.1.6'
+gem 'dwf', '~> 0.1.12'
 ```
-## 2. Execute flow
+## 2. Execute flow example
 ### Declare jobs
 
 ```ruby
 require 'dwf'
 
-class A < Dwf::Item
+class FirstItem < Dwf::Item
   def perform
-    puts "#{self.class.name} Working"
-    sleep 2
-    puts params
-    puts "#{self.class.name} Finished"
+    puts "#{self.class.name}: running"
+    puts "#{self.class.name}: finish"
   end
 end
+
+class SecondItem < Dwf::Item
+  def perform
+    puts "#{self.class.name}: running"
+    output('Send to ThirdItem')
+    puts "#{self.class.name} finish"
+  end
+end
+
+class ThirdItem < Dwf::Item
+  def perform
+    puts "#{self.class.name}: running"
+    puts "#{self.class.name}: finish"
+  end
+end
+
+class FourthItem < Dwf::Item
+  def perform
+    puts "#{self.class.name}: running"
+    puts "payloads from incoming: #{payloads.inspect}"
+    puts "#{self.class.name}: finish"
+  end
+end
+
+FifthItem = Class.new(FirstItem)
 ```
 
 ### Declare flow
@@ -28,20 +51,23 @@ require 'dwf'
 
 class TestWf < Dwf::Workflow
   def configure
-    run A
-    run B, after: A
-    run C, after: A
-    run E, after: [B, C], params: 'E say hello'
-    run D, after: [E], params: 'D say hello'
-    run F, params: 'F say hello'
+    run FirstItem
+    run SecondItem, after: FirstItem
+    run ThirdItem, after: FirstItem
+    run FourthItem, after: [ThirdItem, SecondItem]
+    run FifthItem, after: FourthItem
   end
 end
 ```
-
+### Start background worker process
+```
+bundle exec sidekiq -q dwf
+```
 
 ### Execute flow
 ```ruby
-wf = TestWf.create(callback_type: Dwf::Workflow::SK_BATCH)
+wf = TestWf.create
+wf.callback_type = Dwf::Workflow::SK_BATCH
 wf.start!
 ```
 
@@ -54,36 +80,32 @@ By default `dwf` will use `Dwf::Workflow::BUILD_IN` callback.
 
 ### Output
 ```
-A Working
-F Working
-A Finished
-F say hello
-F Finished
-C Working
-B Working
-C Finished
-B Finished
-E Working
-E say hello
-E Finished
-D Working
-D say hello
-D Finished
+FirstItem: running
+FirstItem: finish
+SecondItem: running
+SecondItem finish
+ThirdItem: running
+ThirdItem: finish
+FourthItem: running
+FourthItem: finish
+FifthItem: running
+FifthItem: finish
 ```
 
 # Config redis and default queue
+`dwf` uses redis as the key value stograge through [redis-rb](https://github.com/redis/redis-rb), So you can pass redis configuration by `redis_opts`
 ```ruby
 Dwf.config do |config|
   SENTINELS = [
     { host: "127.0.0.1", port: 26380 },
     { host: "127.0.0.1", port: 26381 }
   ]
-  config.opts = { host: 'mymaster', sentinels: SENTINELS, role: :master }
+  config.redis_opts = { host: 'mymaster', sentinels: SENTINELS, role: :master }
   config.namespace = 'dwf'
 end
 ```
-
-# Pinelining
+# Advanced features
+## Pipelining
 You can pass jobs result to next nodes
 
 ```ruby
@@ -116,6 +138,66 @@ end
   }
 ]
 ```
+## Sub workflow
+There might be a case when you want to reuse a workflow in another workflow
+
+As an example, let's write a workflow which contain another workflow, expected that the SubWorkflow workflow execute after `SecondItem` and the `ThirdItem` execute after `SubWorkflow`
+
+### Setup
+```ruby
+class FirstItem < Dwf::Item
+  def perform
+    puts "Main flow: #{self.class.name} running"
+    puts "Main flow: #{self.class.name} finish"
+  end
+end
+
+SecondItem = Class.new(FirstItem)
+ThirtItem = Class.new(FirstItem)
+
+class FirstSubItem < Dwf::Item
+  def perform
+    puts "Sub flow: #{self.class.name} running"
+    puts "Sub flow: #{self.class.name} finish"
+  end
+end
+
+SecondSubItem = Class.new(FirstSubItem)
+
+class SubWorkflow < Dwf::Workflow
+  def configure
+    run FirstSubItem
+    run SecondSubItem, after: FirstSubItem
+  end
+end
+
+
+class TestWf < Dwf::Workflow
+  def configure
+    run FirstItem
+    run SecondItem, after: FirstItem
+    run SubWorkflow, after: SecondItem
+    run ThirtItem, after: SubWorkflow
+  end
+end
+
+wf = TestWf.create
+wf.start!
+```
+
+### Result
+```
+Main flow: FirstItem running
+Main flow: FirstItem finish
+Main flow: SecondItem running
+Main flow: SecondItem finish
+Sub flow: FirstSubItem running
+Sub flow: FirstSubItem finish
+Sub flow: SecondSubItem running
+Sub flow: SecondSubItem finish
+Main flow: ThirtItem running
+Main flow: ThirtItem finish
+```
 
 # Todo
 - [x] Make it work
@@ -123,9 +205,12 @@ end
 - [x] Support with build-in callback
 - [x] Add github workflow
 - [x] Redis configurable
-- [ ] [WIP] Test
-- [ ] Transfer output through each node
+- [x] Pipelining
+- [X] Test
+- [x] Sub workflow
 - [ ] Support [Resque](https://github.com/resque/resque)
+- [ ] Key value store plugable
+  - [ ] research https://github.com/moneta-rb/moneta
 
 # References
 - https://github.com/chaps-io/gush

data/dwf.gemspec

@@ -3,14 +3,15 @@
 
 lib = File.expand_path('../lib', __FILE__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require_relative 'lib/dwf/version'
 
 Gem::Specification.new do |spec|
-  spec.name          = "dwf"
-  spec.version       = '0.1.8'
-  spec.authors       = ["dthtien"]
-  spec.email         = ["tiendt2311@gmail.com"]
+  spec.name          = 'dwf'
+  spec.version       = Dwf::VERSION
+  spec.authors       = ['dthtien']
+  spec.email         = ['tiendt2311@gmail.com']
 
-  spec.summary       = 'Gush cloned without ActiveJob but requried Sidekiq. This project is for researching DSL purpose'
+  spec.summary       = 'Distributed workflow runner following Gush interface using Sidekiq and Redis'
   spec.description   = 'Workflow'
   spec.homepage      = 'https://github.com/dthtien/wf'
   spec.license       = "MIT"
@@ -25,8 +26,9 @@ Gem::Specification.new do |spec|
   # guide at: https://bundler.io/guides/creating_gem.html
 
   spec.add_development_dependency 'byebug', '~> 11.1.3'
+  spec.add_development_dependency 'mock_redis', '~> 0.27.2'
   spec.add_dependency 'redis', '~> 4.2.0'
   spec.add_development_dependency 'rspec', '~> 3.2'
-  spec.add_development_dependency 'mock_redis', '~> 0.27.2'
   spec.add_dependency 'sidekiq', '~> 6.2.0'
+  spec.add_development_dependency 'simplecov'
 end

data/lib/dwf/callback.rb

@@ -9,8 +9,8 @@ module Dwf
       previous_job_names = options['names']
       workflow_id = options['workflow_id']
       processing_job_names = previous_job_names.map do |job_name|
-        job = client.find_job(workflow_id, job_name)
-        job.outgoing
+        node = client.find_node(job_name, workflow_id)
+        node.outgoing
       end.flatten.uniq
       return if processing_job_names.empty?
 
@@ -19,7 +19,7 @@ module Dwf
     end
 
     def start(job)
-      job.outgoing.any? ? start_with_batch(job) : job.perform_async
+      job.outgoing.any? ? start_with_batch(job) : job.persist_and_perform_async!
     end
 
     private
@@ -40,11 +40,13 @@ module Dwf
       batch.on(
         :success,
         'Dwf::Callback#process_next_step',
-        names: jobs.map(&:klass),
+        names: jobs.map(&:name),
         workflow_id: workflow_id
       )
       batch.jobs do
-        jobs.each { |job| job.persist_and_perform_async! if job.ready_to_start? }
+        jobs.each do |job|
+          job.persist_and_perform_async! if job.ready_to_start?
+        end
       end
     end
 
@@ -61,7 +63,7 @@ module Dwf
 
     def fetch_jobs(processing_job_names, workflow_id)
       processing_job_names.map do |job_name|
-        client.find_job(workflow_id, job_name)
+        client.find_node(job_name, workflow_id)
       end.compact
     end
 
@@ -71,15 +73,16 @@ module Dwf
       client.release_lock(workflow_id, job_name)
     end
 
-    def start_with_batch(job)
+    def start_with_batch(node)
       batch = Sidekiq::Batch.new
+      workflow_id = node.is_a?(Dwf::Workflow) ? node.parent_id : node.workflow_id
       batch.on(
         :success,
         'Dwf::Callback#process_next_step',
-        names: [job.name],
-        workflow_id: job.workflow_id
+        names: [node.name],
+        workflow_id: workflow_id
       )
-      batch.jobs { job.perform_async }
+      batch.jobs { node.persist_and_perform_async! }
     end
 
     def client

data/lib/dwf/client.rb

@@ -1,3 +1,5 @@
+require_relative 'errors'
+
 module Dwf
   class Client
     attr_reader :config
@@ -20,6 +22,43 @@ module Dwf
       Dwf::Item.from_hash(Dwf::Utils.symbolize_keys(data))
     end
 
+    def find_node(name, workflow_id)
+      if Utils.workflow_name?(name)
+        if name.include?('|')
+          _, id = name.split('|')
+        else
+          id = workflow_id(name, workflow_id)
+        end
+        find_workflow(id)
+      else
+        find_job(workflow_id, name)
+      end
+    end
+
+    def find_workflow(id)
+      key = redis.keys("dwf.workflows.#{id}*").first
+      data = redis.get(key)
+      raise WorkflowNotFound, "Workflow with given id doesn't exist" if data.nil?
+
+      hash = JSON.parse(data)
+      hash = Dwf::Utils.symbolize_keys(hash)
+      nodes = parse_nodes(id)
+      workflow_from_hash(hash, nodes)
+    end
+
+    def find_sub_workflow(name, parent_id)
+      find_workflow(workflow_id(name, parent_id))
+    end
+
+    def sub_workflows(id)
+      keys = redis.keys("dwf.workflows.*.*.#{id}")
+      keys.map do |key|
+        id = key.split('.')[2]
+
+        find_workflow(id)
+      end
+    end
+
     def persist_job(job)
       redis.hset("dwf.jobs.#{job.workflow_id}.#{job.klass}", job.id, job.as_json)
     end
@@ -39,7 +78,10 @@ module Dwf
     end
 
     def persist_workflow(workflow)
-      redis.set("dwf.workflows.#{workflow.id}", workflow.as_json)
+      key = [
+        'dwf', 'workflows', workflow.id, workflow.class.name, workflow.parent_id
+      ].compact.join('.')
+      redis.set(key, workflow.as_json)
     end
 
     def build_job_id(workflow_id, job_klass)
@@ -84,6 +126,13 @@ module Dwf
 
     private
 
+    def workflow_id(name, parent_id)
+      key = redis.keys("dwf.workflows.*.#{name}.#{parent_id}").first
+      return if key.nil?
+
+      key.split('.')[2]
+    end
+
     def find_job_by_klass_and_id(workflow_id, job_name)
       job_klass, job_id = job_name.split('|')
 
@@ -99,6 +148,32 @@ module Dwf
       job
     end
 
+    def parse_nodes(id)
+      keys = redis.scan_each(match: "dwf.jobs.#{id}.*")
+
+      items = keys.map do |key|
+        redis.hvals(key).map do |json|
+          node = Dwf::Utils.symbolize_keys JSON.parse(json)
+          Dwf::Item.from_hash(node)
+        end
+      end.flatten
+      workflows = sub_workflows(id)
+      items + workflows
+    end
+
+    def workflow_from_hash(hash, jobs = [])
+      flow = Module.const_get(hash[:klass]).new(*hash[:arguments])
+      flow.jobs = []
+      flow.outgoing = hash.fetch(:outgoing, [])
+      flow.parent_id = hash[:parent_id]
+      flow.incoming = hash.fetch(:incoming, [])
+      flow.stopped = hash.fetch(:stopped, false)
+      flow.callback_type = hash.fetch(:callback_type, Workflow::BUILD_IN)
+      flow.id = hash[:id]
+      flow.jobs = jobs
+      flow
+    end
+
     def redis
       @redis ||= Redis.new(config.redis_opts)
     end

data/lib/dwf/concerns/checkable.rb

@@ -0,0 +1,29 @@
+module Dwf
+  module Concerns
+    module Checkable
+      def no_dependencies?
+        incoming.empty?
+      end
+
+      def leaf?
+        outgoing.empty?
+      end
+
+      def ready_to_start?
+        !running? && !enqueued? && !finished? && !failed? && parents_succeeded?
+      end
+
+      def succeeded?
+        finished? && !failed?
+      end
+
+      def running?
+        started? && !finished?
+      end
+
+      def started?
+        !!started_at
+      end
+    end
+  end
+end

data/lib/dwf/configuration.rb

@@ -9,7 +9,7 @@ module Dwf
 
     def initialize(hash = {})
       @namespace = hash.fetch(:namespace, NAMESPACE)
-      @redis_opts = hash.fetch(:redis_url, REDIS_OPTS)
+      @redis_opts = hash.fetch(:redis_opts, REDIS_OPTS)
     end
   end
 end

data/lib/dwf/errors.rb

@@ -0,0 +1,3 @@
+module Dwf
+  class WorkflowNotFound < StandardError; end
+end