dwf 0.1.7 → 0.1.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fb0f6f6f30b7ed1ba19120a4a1f77e450940e45433153de51f47df9f108d0fae
-  data.tar.gz: e0e86fdd169b7cbcdc811cf62f87b3043fbbd708bdf0f41fb006b392e4ab784b
+  metadata.gz: e545ff4c6f4b56071875f642f1262fee7762fabb63d86c816a0c943cb8e34657
+  data.tar.gz: dfe832f98724b81b8ff1198320cb66013869cca6ba2b2f89d47c4caf0b2108f9
 SHA512:
-  metadata.gz: 4a3770d4d7f63b7ede02d59e1ce43cb2634b39cbd9e037c920e4f816acb7b393de0f9030fda735e8ea0873f3679da120ff405f725e87c60c1b9eb6a7fbf355e3
-  data.tar.gz: 469e25cdf11441c59c80ff47197dcabcf40e0b086ab06babb6b39a0597483517c479ee9ea65767819b69afebf83db6dcdb5926a9839a2e40d02c03cf39a6eb57
+  metadata.gz: 7e77ca9abe614aa1da026818f1a49028bcc3878bd9bd5838e8cdfdf245ab09d124ab4300bd077eed1c3e8b07e46b321c1dd59d18face658fda3f6c636e28954f
+  data.tar.gz: 6a6ff8756a13fcec73966c41ded7b808cae24a73e1cec2bd301dacfdbfde6022812c7f27c6dacc12a5bb469aae241e59316eb544b2ca7ac1f0530e1c5f5ffa2c

data/.gitignore

@@ -4,3 +4,4 @@ Gemfile.lock
 dwf-*.gem
 :w
 :W
+coverage/

data/CHANGELOG.md

@@ -1,5 +1,142 @@
 # Changelog
 All notable changes to this project will be documented in this file.
+## 0.1.11
+### Added
+#### Subworkflow - Only support sidekiq pro
+There might be a case when you want to reuse a workflow in another workflow
+
+As an example, let's write a workflow which contain another workflow, expected that the SubWorkflow workflow execute after `SecondItem` and the `ThirdItem` execute after `SubWorkflow`
+
+```ruby
+gem 'dwf', '~> 0.1.11'
+```
+
+### Setup
+```ruby
+class FirstItem < Dwf::Item
+  def perform
+    puts "Main flow: #{self.class.name} running"
+    puts "Main flow: #{self.class.name} finish"
+  end
+end
+
+SecondItem = Class.new(FirstItem)
+ThirtItem = Class.new(FirstItem)
+
+class FirstSubItem < Dwf::Item
+  def perform
+    puts "Sub flow: #{self.class.name} running"
+    puts "Sub flow: #{self.class.name} finish"
+  end
+end
+
+SecondSubItem = Class.new(FirstSubItem)
+
+class SubWorkflow < Dwf::Workflow
+  def configure
+    run FirstSubItem
+    run SecondSubItem, after: FirstSubItem
+  end
+end
+
+
+class TestWf < Dwf::Workflow
+  def configure
+    run FirstItem
+    run SecondItem, after: FirstItem
+    run SubWorkflow, after: SecondItem
+    run ThirtItem, after: SubWorkflow
+  end
+end
+
+wf = TestWf.create
+wf.start!
+```
+
+### Result
+```
+Main flow: FirstItem running
+Main flow: FirstItem finish
+Main flow: SecondItem running
+Main flow: SecondItem finish
+Sub flow: FirstSubItem running
+Sub flow: FirstSubItem finish
+Sub flow: SecondSubItem running
+Sub flow: SecondSubItem finish
+Main flow: ThirtItem running
+Main flow: ThirtItem finish
+```
+
+## 0.1.10
+### Added
+- Allow to use argument within workflow and update the defining callback way
+```
+class TestWf < Dwf::Workflow
+  def configure(arguments)
+    run A
+    run B, after: A, params: argument
+    run C, after: A, params: argument
+  end
+end
+
+wf = TestWf.create(arguments)
+wf.callback_type = Dwf::Workflow::SK_BATCH
+
+```
+- Support `find` workflow and `reload` workflow
+```
+wf = TestWf.create
+Dwf::Workflow.find(wf.id)
+wf.reload
+```
+
+## 0.1.9
+### Added
+### Fixed
+- fix incorrect argument at configuration
+
+## 0.1.8
+### Added
+- add pinlining feature
+
+```ruby
+class SendOutput < Dwf::Item
+  def perform
+    output('it works')
+  end
+end
+
+```
+
+`output` method used to output data from the job to add outgoing jobs
+
+```ruby
+class ReceiveOutput < Dwf::Item
+  def perform
+    message = payloads.first[:output] # it works
+  end
+end
+```
+
+`payloads` is an array that containing outputs from incoming jobs
+
+```
+[
+  {
+    id: "SendOutput|1849a3f9-5fce-401e-a73a-91fc1048356",
+    class: "SendOutput",
+    output: 'it works'
+  }
+]
+```
+
+```ruby
+Dwf.config do |config|
+  config.opts = { url 'redis://127.0.0.1:6379' }
+  config.namespace = 'dwf'
+end
+```
+
 ## 0.1.7
 ### Added
 - Allow to config redis and queue

data/README.md

@@ -4,22 +4,45 @@
 # Installation
 ## 1. Add `dwf` to Gemfile
 ```ruby
-gem 'dwf', '~> 0.1.6'
+gem 'dwf', '~> 0.1.10'
 ```
-## 2. Execute flow
+## 2. Execute flow example
 ### Declare jobs
 
 ```ruby
 require 'dwf'
 
-class A < Dwf::Item
+class FirstItem < Dwf::Item
   def perform
-    puts "#{self.class.name} Working"
-    sleep 2
-    puts params
-    puts "#{self.class.name} Finished"
+    puts "#{self.class.name}: running"
+    puts "#{self.class.name}: finish"
   end
 end
+
+class SecondItem < Dwf::Item
+  def perform
+    puts "#{self.class.name}: running"
+    output('Send to ThirdItem')
+    puts "#{self.class.name} finish"
+  end
+end
+
+class ThirdItem < Dwf::Item
+  def perform
+    puts "#{self.class.name}: running"
+    puts "#{self.class.name}: finish"
+  end
+end
+
+class FourthItem < Dwf::Item
+  def perform
+    puts "#{self.class.name}: running"
+    puts "payloads from incoming: #{payloads.inspect}"
+    puts "#{self.class.name}: finish"
+  end
+end
+
+FifthItem = Class.new(FirstItem)
 ```
 
 ### Declare flow
@@ -28,20 +51,23 @@ require 'dwf'
 
 class TestWf < Dwf::Workflow
   def configure
-    run A
-    run B, after: A
-    run C, after: A
-    run E, after: [B, C], params: 'E say hello'
-    run D, after: [E], params: 'D say hello'
-    run F, params: 'F say hello'
+    run FirstItem
+    run SecondItem, after: FirstItem
+    run ThirdItem, after: FirstItem
+    run FourthItem, after: [ThirdItem, SecondItem]
+    run FifthItem, after: FourthItem
   end
 end
 ```
-
+### Start background worker process
+```
+bundle exec sidekiq -q dwf
+```
 
 ### Execute flow
 ```ruby
-wf = TestWf.create(callback_type: Dwf::Workflow::SK_BATCH)
+wf = TestWf.create
+wf.callback_type = Dwf::Workflow::SK_BATCH
 wf.start!
 ```
 
@@ -54,35 +80,128 @@ By default `dwf` will use `Dwf::Workflow::BUILD_IN` callback.
 
 ### Output
 ```
-A Working
-F Working
-A Finished
-F say hello
-F Finished
-C Working
-B Working
-C Finished
-B Finished
-E Working
-E say hello
-E Finished
-D Working
-D say hello
-D Finished
+FirstItem: running
+FirstItem: finish
+SecondItem: running
+SecondItem finish
+ThirdItem: running
+ThirdItem: finish
+FourthItem: running
+FourthItem: finish
+FifthItem: running
+FifthItem: finish
 ```
 
 # Config redis and default queue
+`dwf` uses redis as the key value stograge through [redis-rb](https://github.com/redis/redis-rb), So you can pass redis configuration by `redis_opts`
 ```ruby
 Dwf.config do |config|
   SENTINELS = [
     { host: "127.0.0.1", port: 26380 },
     { host: "127.0.0.1", port: 26381 }
   ]
-  config.opts = { host: 'mymaster', sentinels: SENTINELS, role: :master }
+  config.redis_opts = { host: 'mymaster', sentinels: SENTINELS, role: :master }
   config.namespace = 'dwf'
 end
 ```
+# Advanced features
+## Pipelining
+You can pass jobs result to next nodes
+
+```ruby
+class SendOutput < Dwf::Item
+  def perform
+    output('it works')
+  end
+end
+
+```
+
+`output` method used to output data from the job to add outgoing jobs
+
+```ruby
+class ReceiveOutput < Dwf::Item
+  def perform
+    message = payloads.first[:output] # it works
+  end
+end
+```
+
+`payloads` is an array that containing outputs from incoming jobs
+
+```ruby
+[
+  {
+    id: "SendOutput|1849a3f9-5fce-401e-a73a-91fc1048356",
+    class: "SendOutput",
+    output: 'it works'
+  }
+]
+```
+## Subworkflow - Only support sidekiq pro
+There might be a case when you want to reuse a workflow in another workflow
+
+As an example, let's write a workflow which contain another workflow, expected that the SubWorkflow workflow execute after `SecondItem` and the `ThirdItem` execute after `SubWorkflow`
+
+```ruby
+gem 'dwf', '~> 0.1.11'
+```
+
+### Setup
+```ruby
+class FirstItem < Dwf::Item
+  def perform
+    puts "Main flow: #{self.class.name} running"
+    puts "Main flow: #{self.class.name} finish"
+  end
+end
+
+SecondItem = Class.new(FirstItem)
+ThirtItem = Class.new(FirstItem)
+
+class FirstSubItem < Dwf::Item
+  def perform
+    puts "Sub flow: #{self.class.name} running"
+    puts "Sub flow: #{self.class.name} finish"
+  end
+end
+
+SecondSubItem = Class.new(FirstSubItem)
+
+class SubWorkflow < Dwf::Workflow
+  def configure
+    run FirstSubItem
+    run SecondSubItem, after: FirstSubItem
+  end
+end
+
+
+class TestWf < Dwf::Workflow
+  def configure
+    run FirstItem
+    run SecondItem, after: FirstItem
+    run SubWorkflow, after: SecondItem
+    run ThirtItem, after: SubWorkflow
+  end
+end
+
+wf = TestWf.create
+wf.start!
+```
 
+### Result
+```
+Main flow: FirstItem running
+Main flow: FirstItem finish
+Main flow: SecondItem running
+Main flow: SecondItem finish
+Sub flow: FirstSubItem running
+Sub flow: FirstSubItem finish
+Sub flow: SecondSubItem running
+Sub flow: SecondSubItem finish
+Main flow: ThirtItem running
+Main flow: ThirtItem finish
+```
 
 # Todo
 - [x] Make it work
@@ -90,9 +209,12 @@ end
 - [x] Support with build-in callback
 - [x] Add github workflow
 - [x] Redis configurable
-- [ ] [WIP] Test
-- [ ] Transfer output through each node
+- [x] Pipelining
+- [X] Test
+- [ ] WIP - subworkflow
 - [ ] Support [Resque](https://github.com/resque/resque)
+- [ ] Key value store plugable
+  - [ ] research https://github.com/moneta-rb/moneta
 
 # References
 - https://github.com/chaps-io/gush

data/dwf.gemspec

@@ -26,8 +26,9 @@ Gem::Specification.new do |spec|
   # guide at: https://bundler.io/guides/creating_gem.html
 
   spec.add_development_dependency 'byebug', '~> 11.1.3'
+  spec.add_development_dependency 'mock_redis', '~> 0.27.2'
   spec.add_dependency 'redis', '~> 4.2.0'
   spec.add_development_dependency 'rspec', '~> 3.2'
-  spec.add_development_dependency 'mock_redis', '~> 0.27.2'
   spec.add_dependency 'sidekiq', '~> 6.2.0'
+  spec.add_development_dependency 'simplecov'
 end

data/lib/dwf/callback.rb

@@ -9,8 +9,8 @@ module Dwf
       previous_job_names = options['names']
       workflow_id = options['workflow_id']
       processing_job_names = previous_job_names.map do |job_name|
-        job = client.find_job(workflow_id, job_name)
-        job.outgoing
+        node = client.find_node(job_name, workflow_id)
+        node.outgoing
       end.flatten.uniq
       return if processing_job_names.empty?
 
@@ -19,7 +19,7 @@ module Dwf
     end
 
     def start(job)
-      job.outgoing.any? ? start_with_batch(job) : job.perform_async
+      job.outgoing.any? ? start_with_batch(job) : job.persist_and_perform_async!
     end
 
     private
@@ -40,11 +40,13 @@ module Dwf
       batch.on(
         :success,
         'Dwf::Callback#process_next_step',
-        names: jobs.map(&:klass),
+        names: jobs.map(&:name),
         workflow_id: workflow_id
       )
       batch.jobs do
-        jobs.each { |job| job.persist_and_perform_async! if job.ready_to_start? }
+        jobs.each do |job|
+          job.persist_and_perform_async! if job.ready_to_start?
+        end
       end
     end
 
@@ -61,7 +63,7 @@ module Dwf
 
     def fetch_jobs(processing_job_names, workflow_id)
       processing_job_names.map do |job_name|
-        client.find_job(workflow_id, job_name)
+        client.find_node(job_name, workflow_id)
       end.compact
     end
 
@@ -71,15 +73,16 @@ module Dwf
       client.release_lock(workflow_id, job_name)
     end
 
-    def start_with_batch(job)
+    def start_with_batch(node)
       batch = Sidekiq::Batch.new
+      workflow_id = node.is_a?(Dwf::Workflow) ? node.parent_id : node.workflow_id
       batch.on(
         :success,
         'Dwf::Callback#process_next_step',
-        names: [job.name],
-        workflow_id: job.workflow_id
+        names: [node.name],
+        workflow_id: workflow_id
       )
-      batch.jobs { job.perform_async }
+      batch.jobs { node.persist_and_perform_async! }
     end
 
     def client

data/lib/dwf/client.rb

@@ -1,3 +1,5 @@
+require_relative 'errors'
+
 module Dwf
   class Client
     attr_reader :config
@@ -20,6 +22,43 @@ module Dwf
       Dwf::Item.from_hash(Dwf::Utils.symbolize_keys(data))
     end
 
+    def find_node(name, workflow_id)
+      if Utils.workflow_name?(name)
+        if name.include?('|')
+          _, id = name.split('|')
+        else
+          id = workflow_id(name, workflow_id)
+        end
+        find_workflow(id)
+      else
+        find_job(workflow_id, name)
+      end
+    end
+
+    def find_workflow(id)
+      key = redis.keys("dwf.workflows.#{id}*").first
+      data = redis.get(key)
+      raise WorkflowNotFound, "Workflow with given id doesn't exist" if data.nil?
+
+      hash = JSON.parse(data)
+      hash = Dwf::Utils.symbolize_keys(hash)
+      nodes = parse_nodes(id)
+      workflow_from_hash(hash, nodes)
+    end
+
+    def find_sub_workflow(name, parent_id)
+      find_workflow(workflow_id(name, parent_id))
+    end
+
+    def sub_workflows(id)
+      keys = redis.keys("dwf.workflows.*.*.#{id}")
+      keys.map do |key|
+        id = key.split('.')[2]
+
+        find_workflow(id)
+      end
+    end
+
     def persist_job(job)
       redis.hset("dwf.jobs.#{job.workflow_id}.#{job.klass}", job.id, job.as_json)
     end
@@ -39,7 +78,10 @@ module Dwf
     end
 
     def persist_workflow(workflow)
-      redis.set("dwf.workflows.#{workflow.id}", workflow.as_json)
+      key = [
+        'dwf', 'workflows', workflow.id, workflow.class.name, workflow.parent_id
+      ].compact.join('.')
+      redis.set(key, workflow.as_json)
     end
 
     def build_job_id(workflow_id, job_klass)
@@ -84,6 +126,13 @@ module Dwf
 
     private
 
+    def workflow_id(name, parent_id)
+      key = redis.keys("dwf.workflows.*.#{name}.#{parent_id}").first
+      return if key.nil?
+
+      key.split('.')[2]
+    end
+
     def find_job_by_klass_and_id(workflow_id, job_name)
       job_klass, job_id = job_name.split('|')
 
@@ -99,6 +148,32 @@ module Dwf
       job
     end
 
+    def parse_nodes(id)
+      keys = redis.scan_each(match: "dwf.jobs.#{id}.*")
+
+      items = keys.map do |key|
+        redis.hvals(key).map do |json|
+          node = Dwf::Utils.symbolize_keys JSON.parse(json)
+          Dwf::Item.from_hash(node)
+        end
+      end.flatten
+      workflows = sub_workflows(id)
+      items + workflows
+    end
+
+    def workflow_from_hash(hash, jobs = [])
+      flow = Module.const_get(hash[:klass]).new(*hash[:arguments])
+      flow.jobs = []
+      flow.outgoing = hash.fetch(:outgoing, [])
+      flow.parent_id = hash[:parent_id]
+      flow.incoming = hash.fetch(:incoming, [])
+      flow.stopped = hash.fetch(:stopped, false)
+      flow.callback_type = hash.fetch(:callback_type, Workflow::BUILD_IN)
+      flow.id = hash[:id]
+      flow.jobs = jobs
+      flow
+    end
+
     def redis
       @redis ||= Redis.new(config.redis_opts)
     end

data/lib/dwf/concerns/checkable.rb

@@ -0,0 +1,29 @@
+module Dwf
+  module Concerns
+    module Checkable
+      def no_dependencies?
+        incoming.empty?
+      end
+
+      def leaf?
+        outgoing.empty?
+      end
+
+      def ready_to_start?
+        !running? && !enqueued? && !finished? && !failed? && parents_succeeded?
+      end
+
+      def succeeded?
+        finished? && !failed?
+      end
+
+      def running?
+        started? && !finished?
+      end
+
+      def started?
+        !!started_at
+      end
+    end
+  end
+end

data/lib/dwf/configuration.rb

@@ -9,7 +9,7 @@ module Dwf
 
     def initialize(hash = {})
       @namespace = hash.fetch(:namespace, NAMESPACE)
-      @redis_opts = hash.fetch(:redis_url, REDIS_OPTS)
+      @redis_opts = hash.fetch(:redis_opts, REDIS_OPTS)
     end
   end
 end

data/lib/dwf/errors.rb

@@ -0,0 +1,5 @@
+module Dwf
+  class WorkflowNotFound < StandardError; end
+
+  class UnsupportCallback < StandardError; end
+end