dwf 0.1.10 → 0.1.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fba0bf3a5195787db6540ddfe2431a265b09b78859f333ed9295b4e4677d0a25
-  data.tar.gz: 66488a6107dec45933fc1631e713db3fb43a6e31dbd5d5a351a79a6bbb31f1f7
+  metadata.gz: e545ff4c6f4b56071875f642f1262fee7762fabb63d86c816a0c943cb8e34657
+  data.tar.gz: dfe832f98724b81b8ff1198320cb66013869cca6ba2b2f89d47c4caf0b2108f9
 SHA512:
-  metadata.gz: 88c01767c12ae11ff5f2aaeaf68b08a8c04a54bb909c51babd0f02cd915b1c8d790131c197034bbce46a8ab4d8e0dfa40a0f7afe4f1353bff65d952654438454
-  data.tar.gz: 799827548d718f4ae763c00c4519f636f448eb0aba40e6d5a60f83443cf2f479b3d8fbfd536744e11fc9092d22be3a18a7bb764b25cafcb27eccef3168ab0170
+  metadata.gz: 7e77ca9abe614aa1da026818f1a49028bcc3878bd9bd5838e8cdfdf245ab09d124ab4300bd077eed1c3e8b07e46b321c1dd59d18face658fda3f6c636e28954f
+  data.tar.gz: 6a6ff8756a13fcec73966c41ded7b808cae24a73e1cec2bd301dacfdbfde6022812c7f27c6dacc12a5bb469aae241e59316eb544b2ca7ac1f0530e1c5f5ffa2c

data/.gitignore

@@ -4,3 +4,4 @@ Gemfile.lock
 dwf-*.gem
 :w
 :W
+coverage/

data/CHANGELOG.md

@@ -1,5 +1,72 @@
 # Changelog
 All notable changes to this project will be documented in this file.
+## 0.1.11
+### Added
+#### Subworkflow - Only support sidekiq pro
+There might be a case when you want to reuse a workflow in another workflow
+
+As an example, let's write a workflow which contain another workflow, expected that the SubWorkflow workflow execute after `SecondItem` and the `ThirdItem` execute after `SubWorkflow`
+
+```ruby
+gem 'dwf', '~> 0.1.11'
+```
+
+### Setup
+```ruby
+class FirstItem < Dwf::Item
+  def perform
+    puts "Main flow: #{self.class.name} running"
+    puts "Main flow: #{self.class.name} finish"
+  end
+end
+
+SecondItem = Class.new(FirstItem)
+ThirtItem = Class.new(FirstItem)
+
+class FirstSubItem < Dwf::Item
+  def perform
+    puts "Sub flow: #{self.class.name} running"
+    puts "Sub flow: #{self.class.name} finish"
+  end
+end
+
+SecondSubItem = Class.new(FirstSubItem)
+
+class SubWorkflow < Dwf::Workflow
+  def configure
+    run FirstSubItem
+    run SecondSubItem, after: FirstSubItem
+  end
+end
+
+
+class TestWf < Dwf::Workflow
+  def configure
+    run FirstItem
+    run SecondItem, after: FirstItem
+    run SubWorkflow, after: SecondItem
+    run ThirtItem, after: SubWorkflow
+  end
+end
+
+wf = TestWf.create
+wf.start!
+```
+
+### Result
+```
+Main flow: FirstItem running
+Main flow: FirstItem finish
+Main flow: SecondItem running
+Main flow: SecondItem finish
+Sub flow: FirstSubItem running
+Sub flow: FirstSubItem finish
+Sub flow: SecondSubItem running
+Sub flow: SecondSubItem finish
+Main flow: ThirtItem running
+Main flow: ThirtItem finish
+```
+
 ## 0.1.10
 ### Added
 - Allow to use argument within workflow and update the defining callback way

data/README.md

@@ -4,22 +4,45 @@
 # Installation
 ## 1. Add `dwf` to Gemfile
 ```ruby
-gem 'dwf', '~> 0.1.9'
+gem 'dwf', '~> 0.1.10'
 ```
-## 2. Execute flow
+## 2. Execute flow example
 ### Declare jobs
 
 ```ruby
 require 'dwf'
 
-class A < Dwf::Item
+class FirstItem < Dwf::Item
   def perform
-    puts "#{self.class.name} Working"
-    sleep 2
-    puts params
-    puts "#{self.class.name} Finished"
+    puts "#{self.class.name}: running"
+    puts "#{self.class.name}: finish"
   end
 end
+
+class SecondItem < Dwf::Item
+  def perform
+    puts "#{self.class.name}: running"
+    output('Send to ThirdItem')
+    puts "#{self.class.name} finish"
+  end
+end
+
+class ThirdItem < Dwf::Item
+  def perform
+    puts "#{self.class.name}: running"
+    puts "#{self.class.name}: finish"
+  end
+end
+
+class FourthItem < Dwf::Item
+  def perform
+    puts "#{self.class.name}: running"
+    puts "payloads from incoming: #{payloads.inspect}"
+    puts "#{self.class.name}: finish"
+  end
+end
+
+FifthItem = Class.new(FirstItem)
 ```
 
 ### Declare flow
@@ -28,16 +51,18 @@ require 'dwf'
 
 class TestWf < Dwf::Workflow
   def configure
-    run A
-    run B, after: A
-    run C, after: A
-    run E, after: [B, C], params: 'E say hello'
-    run D, after: [E], params: 'D say hello'
-    run F, params: 'F say hello'
+    run FirstItem
+    run SecondItem, after: FirstItem
+    run ThirdItem, after: FirstItem
+    run FourthItem, after: [ThirdItem, SecondItem]
+    run FifthItem, after: FourthItem
   end
 end
 ```
-
+### Start background worker process
+```
+bundle exec sidekiq -q dwf
+```
 
 ### Execute flow
 ```ruby
@@ -55,21 +80,16 @@ By default `dwf` will use `Dwf::Workflow::BUILD_IN` callback.
 
 ### Output
 ```
-A Working
-F Working
-A Finished
-F say hello
-F Finished
-C Working
-B Working
-C Finished
-B Finished
-E Working
-E say hello
-E Finished
-D Working
-D say hello
-D Finished
+FirstItem: running
+FirstItem: finish
+SecondItem: running
+SecondItem finish
+ThirdItem: running
+ThirdItem: finish
+FourthItem: running
+FourthItem: finish
+FifthItem: running
+FifthItem: finish
 ```
 
 # Config redis and default queue
@@ -84,8 +104,8 @@ Dwf.config do |config|
   config.namespace = 'dwf'
 end
 ```
-
-# Pinelining
+# Advanced features
+## Pipelining
 You can pass jobs result to next nodes
 
 ```ruby
@@ -118,6 +138,70 @@ end
   }
 ]
 ```
+## Subworkflow - Only support sidekiq pro
+There might be a case when you want to reuse a workflow in another workflow
+
+As an example, let's write a workflow which contain another workflow, expected that the SubWorkflow workflow execute after `SecondItem` and the `ThirdItem` execute after `SubWorkflow`
+
+```ruby
+gem 'dwf', '~> 0.1.11'
+```
+
+### Setup
+```ruby
+class FirstItem < Dwf::Item
+  def perform
+    puts "Main flow: #{self.class.name} running"
+    puts "Main flow: #{self.class.name} finish"
+  end
+end
+
+SecondItem = Class.new(FirstItem)
+ThirtItem = Class.new(FirstItem)
+
+class FirstSubItem < Dwf::Item
+  def perform
+    puts "Sub flow: #{self.class.name} running"
+    puts "Sub flow: #{self.class.name} finish"
+  end
+end
+
+SecondSubItem = Class.new(FirstSubItem)
+
+class SubWorkflow < Dwf::Workflow
+  def configure
+    run FirstSubItem
+    run SecondSubItem, after: FirstSubItem
+  end
+end
+
+
+class TestWf < Dwf::Workflow
+  def configure
+    run FirstItem
+    run SecondItem, after: FirstItem
+    run SubWorkflow, after: SecondItem
+    run ThirtItem, after: SubWorkflow
+  end
+end
+
+wf = TestWf.create
+wf.start!
+```
+
+### Result
+```
+Main flow: FirstItem running
+Main flow: FirstItem finish
+Main flow: SecondItem running
+Main flow: SecondItem finish
+Sub flow: FirstSubItem running
+Sub flow: FirstSubItem finish
+Sub flow: SecondSubItem running
+Sub flow: SecondSubItem finish
+Main flow: ThirtItem running
+Main flow: ThirtItem finish
+```
 
 # Todo
 - [x] Make it work
@@ -125,9 +209,9 @@ end
 - [x] Support with build-in callback
 - [x] Add github workflow
 - [x] Redis configurable
-- [x] Pinelining
+- [x] Pipelining
 - [X] Test
-- [ ] Consistent item name
+- [ ] WIP - subworkflow
 - [ ] Support [Resque](https://github.com/resque/resque)
 - [ ] Key value store plugable
   - [ ] research https://github.com/moneta-rb/moneta

data/dwf.gemspec

@@ -3,10 +3,11 @@
 
 lib = File.expand_path('../lib', __FILE__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require_relative 'lib/dwf/version'
 
 Gem::Specification.new do |spec|
   spec.name          = "dwf"
-  spec.version       = '0.1.10'
+  spec.version       = Dwf::VERSION
   spec.authors       = ["dthtien"]
   spec.email         = ["tiendt2311@gmail.com"]
 
@@ -25,8 +26,9 @@ Gem::Specification.new do |spec|
   # guide at: https://bundler.io/guides/creating_gem.html
 
   spec.add_development_dependency 'byebug', '~> 11.1.3'
+  spec.add_development_dependency 'mock_redis', '~> 0.27.2'
   spec.add_dependency 'redis', '~> 4.2.0'
   spec.add_development_dependency 'rspec', '~> 3.2'
-  spec.add_development_dependency 'mock_redis', '~> 0.27.2'
   spec.add_dependency 'sidekiq', '~> 6.2.0'
+  spec.add_development_dependency 'simplecov'
 end

data/lib/dwf/callback.rb

@@ -9,8 +9,8 @@ module Dwf
       previous_job_names = options['names']
       workflow_id = options['workflow_id']
       processing_job_names = previous_job_names.map do |job_name|
-        job = client.find_job(workflow_id, job_name)
-        job.outgoing
+        node = client.find_node(job_name, workflow_id)
+        node.outgoing
       end.flatten.uniq
       return if processing_job_names.empty?
 
@@ -19,7 +19,7 @@ module Dwf
     end
 
     def start(job)
-      job.outgoing.any? ? start_with_batch(job) : job.perform_async
+      job.outgoing.any? ? start_with_batch(job) : job.persist_and_perform_async!
     end
 
     private
@@ -40,11 +40,13 @@ module Dwf
       batch.on(
         :success,
         'Dwf::Callback#process_next_step',
-        names: jobs.map(&:klass),
+        names: jobs.map(&:name),
         workflow_id: workflow_id
       )
       batch.jobs do
-        jobs.each { |job| job.persist_and_perform_async! if job.ready_to_start? }
+        jobs.each do |job|
+          job.persist_and_perform_async! if job.ready_to_start?
+        end
       end
     end
 
@@ -61,7 +63,7 @@ module Dwf
 
     def fetch_jobs(processing_job_names, workflow_id)
       processing_job_names.map do |job_name|
-        client.find_job(workflow_id, job_name)
+        client.find_node(job_name, workflow_id)
       end.compact
     end
 
@@ -71,15 +73,16 @@ module Dwf
       client.release_lock(workflow_id, job_name)
     end
 
-    def start_with_batch(job)
+    def start_with_batch(node)
       batch = Sidekiq::Batch.new
+      workflow_id = node.is_a?(Dwf::Workflow) ? node.parent_id : node.workflow_id
       batch.on(
         :success,
         'Dwf::Callback#process_next_step',
-        names: [job.name],
-        workflow_id: job.workflow_id
+        names: [node.name],
+        workflow_id: workflow_id
       )
-      batch.jobs { job.perform_async }
+      batch.jobs { node.persist_and_perform_async! }
     end
 
     def client

data/lib/dwf/client.rb

@@ -22,8 +22,22 @@ module Dwf
       Dwf::Item.from_hash(Dwf::Utils.symbolize_keys(data))
     end
 
+    def find_node(name, workflow_id)
+      if Utils.workflow_name?(name)
+        if name.include?('|')
+          _, id = name.split('|')
+        else
+          id = workflow_id(name, workflow_id)
+        end
+        find_workflow(id)
+      else
+        find_job(workflow_id, name)
+      end
+    end
+
     def find_workflow(id)
-      data = redis.get("dwf.workflows.#{id}")
+      key = redis.keys("dwf.workflows.#{id}*").first
+      data = redis.get(key)
       raise WorkflowNotFound, "Workflow with given id doesn't exist" if data.nil?
 
       hash = JSON.parse(data)
@@ -32,6 +46,19 @@ module Dwf
       workflow_from_hash(hash, nodes)
     end
 
+    def find_sub_workflow(name, parent_id)
+      find_workflow(workflow_id(name, parent_id))
+    end
+
+    def sub_workflows(id)
+      keys = redis.keys("dwf.workflows.*.*.#{id}")
+      keys.map do |key|
+        id = key.split('.')[2]
+
+        find_workflow(id)
+      end
+    end
+
     def persist_job(job)
       redis.hset("dwf.jobs.#{job.workflow_id}.#{job.klass}", job.id, job.as_json)
     end
@@ -51,7 +78,10 @@ module Dwf
     end
 
     def persist_workflow(workflow)
-      redis.set("dwf.workflows.#{workflow.id}", workflow.as_json)
+      key = [
+        'dwf', 'workflows', workflow.id, workflow.class.name, workflow.parent_id
+      ].compact.join('.')
+      redis.set(key, workflow.as_json)
     end
 
     def build_job_id(workflow_id, job_klass)
@@ -96,6 +126,13 @@ module Dwf
 
     private
 
+    def workflow_id(name, parent_id)
+      key = redis.keys("dwf.workflows.*.#{name}.#{parent_id}").first
+      return if key.nil?
+
+      key.split('.')[2]
+    end
+
     def find_job_by_klass_and_id(workflow_id, job_name)
       job_klass, job_id = job_name.split('|')
 
@@ -114,22 +151,26 @@ module Dwf
     def parse_nodes(id)
       keys = redis.scan_each(match: "dwf.jobs.#{id}.*")
 
-      keys.map do |key|
+      items = keys.map do |key|
         redis.hvals(key).map do |json|
-          Dwf::Utils.symbolize_keys JSON.parse(json)
+          node = Dwf::Utils.symbolize_keys JSON.parse(json)
+          Dwf::Item.from_hash(node)
         end
       end.flatten
+      workflows = sub_workflows(id)
+      items + workflows
     end
 
-    def workflow_from_hash(hash, nodes = [])
+    def workflow_from_hash(hash, jobs = [])
       flow = Module.const_get(hash[:klass]).new(*hash[:arguments])
       flow.jobs = []
+      flow.outgoing = hash.fetch(:outgoing, [])
+      flow.parent_id = hash[:parent_id]
+      flow.incoming = hash.fetch(:incoming, [])
       flow.stopped = hash.fetch(:stopped, false)
+      flow.callback_type = hash.fetch(:callback_type, Workflow::BUILD_IN)
       flow.id = hash[:id]
-      flow.jobs = nodes.map do |node|
-        Dwf::Item.from_hash(node)
-      end
-
+      flow.jobs = jobs
       flow
     end
 

data/lib/dwf/concerns/checkable.rb

@@ -0,0 +1,29 @@
+module Dwf
+  module Concerns
+    module Checkable
+      def no_dependencies?
+        incoming.empty?
+      end
+
+      def leaf?
+        outgoing.empty?
+      end
+
+      def ready_to_start?
+        !running? && !enqueued? && !finished? && !failed? && parents_succeeded?
+      end
+
+      def succeeded?
+        finished? && !failed?
+      end
+
+      def running?
+        started? && !finished?
+      end
+
+      def started?
+        !!started_at
+      end
+    end
+  end
+end

data/lib/dwf/errors.rb

@@ -1,3 +1,5 @@
 module Dwf
-    class WorkflowNotFound < StandardError; end
+  class WorkflowNotFound < StandardError; end
+
+  class UnsupportCallback < StandardError; end
 end

data/lib/dwf/item.rb

@@ -1,12 +1,16 @@
 # frozen_string_literal: true
 
 require_relative 'client'
+require_relative 'concerns/checkable'
 
 module Dwf
   class Item
+    include Concerns::Checkable
+
     attr_reader :workflow_id, :id, :params, :queue, :klass, :started_at,
-      :enqueued_at, :finished_at, :failed_at, :callback_type, :output_payload
-    attr_accessor :incoming, :outgoing
+                :enqueued_at, :finished_at, :failed_at, :output_payload
+    attr_writer :payloads
+    attr_accessor :incoming, :outgoing, :callback_type
 
     def initialize(options = {})
       assign_attributes(options)
@@ -16,9 +20,17 @@ module Dwf
       Module.const_get(hash[:klass]).new(hash)
     end
 
+    def start_initial!
+      cb_build_in? ? persist_and_perform_async! : start_batch!
+    end
+
+    def start_batch!
+      enqueue_and_persist!
+      Dwf::Callback.new.start(self)
+    end
+
     def persist_and_perform_async!
-      enqueue!
-      persist!
+      enqueue_and_persist!
       perform_async
     end
 
@@ -35,7 +47,7 @@ module Dwf
 
     def perform_async
       Dwf::Worker.set(queue: queue || client.config.namespace)
-                 .perform_async(workflow_id, name)
+        .perform_async(workflow_id, name)
     end
 
     def name
@@ -51,20 +63,11 @@ module Dwf
     end
 
     def parents_succeeded?
-      incoming.all? do |name|
-        client.find_job(workflow_id, name).succeeded?
-      end
+      incoming.all? { |name| client.find_node(name, workflow_id).succeeded? }
     end
 
     def payloads
-      incoming.map do |job_name|
-        job = client.find_job(workflow_id, job_name)
-        {
-          id: job.name,
-          class: job.klass.to_s,
-          output: job.output_payload
-        }
-      end
+      @payloads ||= build_payloads
     end
 
     def enqueue!
@@ -109,22 +112,6 @@ module Dwf
       !failed_at.nil?
     end
 
-    def succeeded?
-      finished? && !failed?
-    end
-
-    def started?
-      !started_at.nil?
-    end
-
-    def running?
-      started? && !finished?
-    end
-
-    def ready_to_start?
-      !running? && !enqueued? && !finished? && !failed? && parents_succeeded?
-    end
-
     def current_timestamp
       Time.now.to_i
     end
@@ -152,7 +139,8 @@ module Dwf
         params: params,
         workflow_id: workflow_id,
         callback_type: callback_type,
-        output_payload: output_payload
+        output_payload: output_payload,
+        payloads: @payloads
       }
     end
 
@@ -166,6 +154,11 @@ module Dwf
 
     private
 
+    def enqueue_and_persist!
+      enqueue!
+      persist!
+    end
+
     def client
       @client ||= Dwf::Client.new
     end
@@ -184,6 +177,21 @@ module Dwf
       @started_at = options[:started_at]
       @callback_type = options[:callback_type]
       @output_payload = options[:output_payload]
+      @payloads = options[:payloads]
+    end
+
+    def build_payloads
+      data = incoming.map do |job_name|
+        node = client.find_node(job_name, workflow_id)
+        next if node.output_payload.nil?
+
+        {
+          id: node.name,
+          class: node.klass.to_s,
+          output: node.output_payload
+        }
+      end.compact
+      data.empty? ? nil : data
     end
   end
 end

data/lib/dwf/utils.rb

@@ -34,5 +34,11 @@ module Dwf
         obj
       end
     end
+
+    def self.workflow_name?(name)
+      node_name = name.include?('|') ? name.split('|').first : name
+
+      Module.const_get(node_name) <= Workflow
+    end
   end
 end

data/lib/dwf/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Dwf
-  VERSION = '0.1.8'
+  VERSION = '0.1.11'
 end