dwf 0.1.10 → 0.1.11

data/lib/dwf/workflow.rb

@@ -2,16 +2,20 @@
 
 require_relative 'client'
 require_relative 'worker'
+require_relative 'concerns/checkable'
 require_relative 'callback'
 
 module Dwf
   class Workflow
+    include Concerns::Checkable
+
     CALLBACK_TYPES = [
       BUILD_IN = 'build-in',
       SK_BATCH = 'sk-batch'
     ].freeze
-    attr_accessor :jobs, :callback_type, :stopped, :id
-    attr_reader :dependencies, :started_at, :finished_at, :persisted, :arguments
+    attr_accessor :jobs, :stopped, :id, :incoming, :outgoing, :parent_id
+    attr_reader :dependencies, :started_at, :finished_at, :persisted, :arguments, :klass,
+                :callback_type
 
     class << self
       def create(*args)
@@ -31,8 +35,12 @@ module Dwf
       @jobs = []
       @persisted = false
       @stopped = false
-      @arguments = args
+      @arguments = *args
+      @parent_id = nil
+      @klass = self.class
       @callback_type = BUILD_IN
+      @incoming = []
+      @outgoing = []
 
       setup
     end
@@ -44,16 +52,42 @@ module Dwf
       true
     end
 
+    def name
+      "#{self.class.name}|#{id}"
+    end
+
+    def sub_workflow?
+      !parent_id.nil?
+    end
+
+    def callback_type=(type = BUILD_IN)
+      @callback_type = type
+      jobs.each { |job| job.callback_type = type }
+    end
+
     alias save persist!
 
     def start!
+      raise UnsupportCallback, 'Sub workflow only works with Sidekiq batch callback' if invalid_callback?
+
       mark_as_started
       persist!
       initial_jobs.each do |job|
-        cb_build_in? ? job.persist_and_perform_async! : Dwf::Callback.new.start(job)
+        job.payloads = payloads if sub_workflow?
+        job.start_initial!
       end
     end
 
+    def payloads
+      @payloads ||= build_payloads
+    end
+
+    def start_initial!
+      cb_build_in? ? start! : Callback.new.start(self)
+    end
+
+    alias persist_and_perform_async! start!
+
     def reload
       flow = self.class.find(id)
       self.stopped = flow.stopped
@@ -73,14 +107,7 @@ module Dwf
     def configure(*arguments); end
 
     def run(klass, options = {})
-      node = klass.new(
-        workflow_id: id,
-        id: client.build_job_id(id, klass.to_s),
-        params: options.fetch(:params, {}),
-        queue: options[:queue],
-        callback_type: callback_type
-      )
-
+      node = build_node(klass, options)
       jobs << node
 
       build_dependencies_structure(node, options)
@@ -112,7 +139,10 @@ module Dwf
         stopped: stopped,
         started_at: started_at,
         finished_at: finished_at,
-        callback_type: callback_type
+        callback_type: callback_type,
+        incoming: incoming,
+        outgoing: outgoing,
+        parent_id: parent_id
       }
     end
 
@@ -124,13 +154,7 @@ module Dwf
       jobs.all?(&:finished?)
     end
 
-    def started?
-      !!started_at
-    end
-
-    def running?
-      started? && !finished?
-    end
+    alias enqueued? started?
 
     def failed?
       jobs.any?(&:failed?)
@@ -140,6 +164,10 @@ module Dwf
       stopped
     end
 
+    def parents_succeeded?
+      incoming.all? { |name| client.find_node(name, parent_id).succeeded? }
+    end
+
     def status
       return :failed if failed?
       return :running if running?
@@ -157,8 +185,39 @@ module Dwf
       @stopped = false
     end
 
+    def leaf_nodes
+      jobs.select(&:leaf?)
+    end
+
+    def output_payload
+      leaf_nodes.map do |node|
+        data = node.output_payload
+        next if data.nil?
+
+        data
+      end.compact
+    end
+
     private
 
+    def build_node(klass, options)
+      if klass < Dwf::Workflow
+        node = options[:params].nil? ? klass.new : klass.new(options[:params])
+        node.parent_id = id
+        node.callback_type = SK_BATCH
+        node.save
+        node
+      else
+        klass.new(
+          workflow_id: id,
+          id: client.build_job_id(id, klass.to_s),
+          params: options.fetch(:params, {}),
+          queue: options[:queue],
+          callback_type: callback_type
+        )
+      end
+    end
+
     def initial_jobs
       jobs.select(&:no_dependencies?)
     end
@@ -168,16 +227,52 @@ module Dwf
       resolve_dependencies
     end
 
+    def find_node(node_name)
+      if Utils.workflow_name?(node_name)
+        find_subworkflow(node_name)
+      else
+        find_job(node_name)
+      end
+    end
+
+    def find_subworkflow(node_name)
+      fname, = node_name.split('|')
+      jobs.find { |j| j.klass.name == fname }
+    end
+
     def resolve_dependencies
       @dependencies.each do |dependency|
-        from = find_job(dependency[:from])
-        to   = find_job(dependency[:to])
+        from = find_node(dependency[:from])
+        to   = find_node(dependency[:to])
 
-        to.incoming << dependency[:from]
-        from.outgoing << dependency[:to]
+        to.incoming << from.name
+        from.outgoing << to.name
       end
     end
 
+    def invalid_callback?
+      cb_build_in? && jobs.any? { |job| job.class < Workflow }
+    end
+
+    def build_payloads
+      return unless sub_workflow?
+
+      data = incoming.map do |job_name|
+        next if Utils.workflow_name?(job_name)
+
+        node = client.find_node(job_name, parent_id)
+        next if node.output_payload.nil?
+
+        {
+          id: node.name,
+          class: node.klass.to_s,
+          output: node.output_payload
+        }
+      end.compact
+
+      data.empty? ? nil : data
+    end
+
     def build_dependencies_structure(node, options)
       deps_after = [*options[:after]]
 

data/spec/dwf/client_spec.rb

@@ -2,6 +2,8 @@
 
 require 'spec_helper'
 require 'mock_redis'
+FirstWorkflow = Class.new(Dwf::Workflow)
+SecondWorkflow = Class.new(Dwf::Workflow)
 
 describe Dwf::Client, client: true do
   let(:client) { described_class.new }
@@ -30,12 +32,12 @@ describe Dwf::Client, client: true do
     end
 
     context 'find by item name' do
-      it {
+      it do
         item = client.find_job(workflow_id, job.name)
         expect(item.workflow_id).to eq workflow_id
         expect(item.id).to eq id
         expect(item.name).to eq job.name
-      }
+      end
     end
   end
 
@@ -62,6 +64,47 @@ describe Dwf::Client, client: true do
     end
   end
 
+  describe '#find_node' do
+    context 'find job' do
+      let!(:job) do
+        j = Dwf::Item.new(workflow_id: workflow_id, id: id)
+        j.persist!
+        j
+      end
+
+      it do
+        item = client.find_node(Dwf::Item.name, workflow_id)
+        expect(item.workflow_id).to eq workflow_id
+        expect(item.id).to eq id
+        expect(item.name).to eq job.name
+      end
+    end
+
+    context 'find_workflow' do
+      let!(:wf1) { FirstWorkflow.create }
+      let!(:wf2) do
+        wf = SecondWorkflow.new
+        wf.parent_id = wf1.id
+        wf.save
+        wf
+      end
+
+      context 'find with class name and parent id' do
+        it do
+          wf = client.find_node(wf2.class.name, wf1.id)
+          expect(wf).to be_kind_of(SecondWorkflow)
+        end
+      end
+
+      context 'find with name and parent id' do
+        it do
+          wf = client.find_node(wf2.name, wf1.id)
+          expect(wf).to be_kind_of(SecondWorkflow)
+        end
+      end
+    end
+  end
+
   describe '#persist_job' do
     let!(:job) { Dwf::Item.new(workflow_id: workflow_id, id: id) }
 
@@ -76,13 +119,44 @@ describe Dwf::Client, client: true do
     end
   end
 
+  describe '#find_sub_workflow' do
+    let!(:wf1) { FirstWorkflow.create }
+    let!(:wf2) do
+      wf = SecondWorkflow.new
+      wf.parent_id = wf1.id
+      wf.save
+      wf
+    end
+
+    it do
+      wf = client.find_sub_workflow(wf2.class.name, wf1.id)
+      expect(wf).to be_kind_of(SecondWorkflow)
+    end
+  end
+
+  describe '#sub_workflows' do
+    let!(:wf1) { FirstWorkflow.create }
+    let!(:wf2) do
+      wf = SecondWorkflow.new
+      wf.parent_id = wf1.id
+      wf.save
+      wf
+    end
+
+    it do
+      wfs = client.sub_workflows(wf1.id)
+      expect(wfs).not_to be_empty
+      expect(wfs.first).to be_kind_of(SecondWorkflow)
+    end
+  end
+
   describe '#persist_workflow' do
     let(:workflow) { Dwf::Workflow.new }
 
     it do
       expect(redis.exists?("dwf.workflows.#{workflow.id}")).to be_falsy
       client.persist_workflow(workflow)
-      expect(redis.exists?("dwf.workflows.#{workflow.id}")).to be_truthy
+      expect(redis.keys("dwf.workflows.#{workflow.id}*").any?).to be_truthy
     end
   end
 

data/spec/dwf/item_spec.rb

@@ -20,7 +20,8 @@ describe Dwf::Item, item: true do
       klass: 'Dwf::Item',
       started_at: started_at,
       finished_at: finished_at,
-      callback_type: Dwf::Workflow::BUILD_IN
+      callback_type: Dwf::Workflow::BUILD_IN,
+      payloads: nil
     }
   end
   let!(:item) { described_class.new(options) }
@@ -89,7 +90,7 @@ describe Dwf::Item, item: true do
     before do
       allow(Dwf::Client).to receive(:new).and_return client_double
       allow(client_double)
-        .to receive(:find_job).and_return a_item
+        .to receive(:find_node).and_return a_item
     end
 
     context 'parent jobs already finished' do
@@ -98,8 +99,8 @@ describe Dwf::Item, item: true do
       it do
         expect(item.parents_succeeded?).to be_truthy
         expect(client_double)
-          .to have_received(:find_job)
-          .with(workflow_id, incoming.first)
+          .to have_received(:find_node)
+          .with(incoming.first, workflow_id)
       end
     end
 
@@ -109,8 +110,8 @@ describe Dwf::Item, item: true do
       it do
         expect(item.parents_succeeded?).to be_falsy
         expect(client_double)
-          .to have_received(:find_job)
-          .with(workflow_id, incoming.first)
+          .to have_received(:find_node)
+          .with(incoming.first, workflow_id)
       end
     end
   end
@@ -189,8 +190,9 @@ describe Dwf::Item, item: true do
   end
 
   describe '#payloads' do
-    let(:incoming) { ["A|#{SecureRandom.uuid}"] }
-    let(:client_double) { double(find_job: nil) }
+    let(:incoming) { ["Dwf::Item|#{SecureRandom.uuid}", "Dwf::Workflow|#{workflow_id}"] }
+    let(:client_double) { double(find_job: nil, build_workflow_id: workflow_id) }
+    let(:workflow) { Dwf::Workflow.new }
     let!(:a_item) do
       described_class.new(
         workflow_id: SecureRandom.uuid,
@@ -203,7 +205,11 @@ describe Dwf::Item, item: true do
     before do
       allow(Dwf::Client).to receive(:new).and_return client_double
       allow(client_double)
-        .to receive(:find_job).and_return a_item
+        .to receive(:find_node)
+        .with(incoming.first, workflow_id).and_return a_item
+      allow(client_double)
+        .to receive(:find_node)
+        .with(incoming.last, workflow_id).and_return workflow
     end
 
     it do
@@ -212,9 +218,42 @@ describe Dwf::Item, item: true do
           class: a_item.class.name,
           id: a_item.name,
           output: 1
+        },
+        {
+          class: workflow.class.name,
+          id: workflow.name,
+          output: []
         }
       ]
-      expect(item.payloads).to eq expected_payload
+      expect(item.payloads).to match_array expected_payload
+    end
+  end
+
+  describe '#start_batch!' do
+    let(:callback_double) { double(start: nil) }
+    let(:client_double) { double(persist_job: nil) }
+
+    before do
+      allow(Dwf::Client).to receive(:new).and_return client_double
+      allow(Dwf::Callback).to receive(:new).and_return callback_double
+      item.start_batch!
+    end
+
+    it do
+      expect(callback_double).to have_received(:start).with(item)
+      expect(item.enqueued_at).not_to be_nil
+    end
+  end
+
+  describe '#leaf?' do
+    context 'when item has outgoing item' do
+      let(:outgoing) { ["Dwf::Item|#{SecureRandom.uuid}"] }
+      it { expect(item.leaf?).to be_falsy }
+    end
+
+    context 'when item does not have outgoing item' do
+      let(:outgoing) { [] }
+      it { expect(item.leaf?).to be_truthy }
     end
   end
 end

data/spec/dwf/utils_spec.rb

@@ -20,4 +20,13 @@ describe Dwf::Utils, utils: true do
 
     it { expect(described_class.symbolize_keys(hash)).to eq expected }
   end
+
+  describe "#workflow_name?" do
+    FirstWorkflow = Class.new(Dwf::Workflow)
+    FirstJob = Class.new(Dwf::Item)
+
+    it { expect(described_class.workflow_name?(Dwf::Workflow.name)).to be_truthy }
+    it { expect(described_class.workflow_name?(FirstWorkflow.name)).to be_truthy }
+    it { expect(described_class.workflow_name?(FirstJob.name)).to be_falsy }
+  end
 end