dwf 0.1.9 → 0.1.13

data/lib/dwf/item.rb

@@ -1,12 +1,16 @@
 # frozen_string_literal: true
 
 require_relative 'client'
+require_relative 'concerns/checkable'
 
 module Dwf
   class Item
+    include Concerns::Checkable
+
     attr_reader :workflow_id, :id, :params, :queue, :klass, :started_at,
-      :enqueued_at, :finished_at, :failed_at, :callback_type, :output_payload
-    attr_accessor :incoming, :outgoing
+                :enqueued_at, :finished_at, :failed_at, :output_payload
+    attr_writer :payloads
+    attr_accessor :incoming, :outgoing, :callback_type
 
     def initialize(options = {})
       assign_attributes(options)
@@ -16,9 +20,17 @@ module Dwf
       Module.const_get(hash[:klass]).new(hash)
     end
 
+    def start_initial!
+      cb_build_in? ? persist_and_perform_async! : start_batch!
+    end
+
+    def start_batch!
+      enqueue_and_persist!
+      Dwf::Callback.new.start(self)
+    end
+
     def persist_and_perform_async!
-      enqueue!
-      persist!
+      enqueue_and_persist!
       perform_async
     end
 
@@ -28,14 +40,18 @@ module Dwf
       callback_type == Dwf::Workflow::BUILD_IN
     end
 
+    def workflow
+      @workflow ||= client.find_workflow(workflow_id)
+    end
+
     def reload
       item = client.find_job(workflow_id, name)
       assign_attributes(item.to_hash)
     end
 
-    def perform_async
-      Dwf::Worker.set(queue: queue || client.config.namespace)
-                 .perform_async(workflow_id, name)
+    def perform_async(options = {})
+      Dwf::Worker.set(options.merge(queue: queue || client.config.namespace))
+        .perform_async(workflow_id, name)
     end
 
     def name
@@ -51,20 +67,11 @@ module Dwf
     end
 
     def parents_succeeded?
-      incoming.all? do |name|
-        client.find_job(workflow_id, name).succeeded?
-      end
+      incoming.all? { |name| client.find_node(name, workflow_id).succeeded? }
     end
 
     def payloads
-      incoming.map do |job_name|
-        job = client.find_job(workflow_id, job_name)
-        {
-          id: job.name,
-          class: job.klass.to_s,
-          output: job.output_payload
-        }
-      end
+      @payloads ||= build_payloads
     end
 
     def enqueue!
@@ -109,33 +116,21 @@ module Dwf
       !failed_at.nil?
     end
 
-    def succeeded?
-      finished? && !failed?
-    end
-
-    def started?
-      !started_at.nil?
-    end
-
-    def running?
-      started? && !finished?
-    end
-
-    def ready_to_start?
-      !running? && !enqueued? && !finished? && !failed? && parents_succeeded?
-    end
-
     def current_timestamp
       Time.now.to_i
     end
 
     def enqueue_outgoing_jobs
+      return workflow.enqueue_outgoing_jobs if leaf?
+
       outgoing.each do |job_name|
-        client.check_or_lock(workflow_id, job_name)
-        out = client.find_job(workflow_id, job_name)
-        out.persist_and_perform_async! if out.ready_to_start?
-        client.release_lock(workflow_id, job_name)
+        client.check_or_lock(workflow_id, job_name) do
+          out = client.find_node(job_name, workflow_id)
+          out.persist_and_perform_async! if out.ready_to_start?
+        end
       end
+    rescue RedisMutex::LockError
+      perform_async(wait: 2)
     end
 
     def to_hash
@@ -152,7 +147,8 @@ module Dwf
         params: params,
         workflow_id: workflow_id,
         callback_type: callback_type,
-        output_payload: output_payload
+        output_payload: output_payload,
+        payloads: @payloads
       }
     end
 
@@ -166,6 +162,11 @@ module Dwf
 
     private
 
+    def enqueue_and_persist!
+      enqueue!
+      persist!
+    end
+
     def client
       @client ||= Dwf::Client.new
     end
@@ -184,6 +185,21 @@ module Dwf
       @started_at = options[:started_at]
       @callback_type = options[:callback_type]
       @output_payload = options[:output_payload]
+      @payloads = options[:payloads]
+    end
+
+    def build_payloads
+      data = incoming.map do |job_name|
+        node = client.find_node(job_name, workflow_id)
+        next if node.output_payload.nil?
+
+        {
+          id: node.name,
+          class: node.klass.to_s,
+          output: node.output_payload
+        }
+      end.compact
+      data.empty? ? nil : data
     end
   end
 end

data/lib/dwf/utils.rb

@@ -34,5 +34,11 @@ module Dwf
         obj
       end
     end
+
+    def self.workflow_name?(name)
+      node_name = name.include?('|') ? name.split('|').first : name
+
+      Module.const_get(node_name) <= Workflow
+    end
   end
 end

data/lib/dwf/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Dwf
-  VERSION = '0.1.8'
+  VERSION = '0.1.13'
 end

data/lib/dwf/workflow.rb

@@ -2,15 +2,18 @@
 
 require_relative 'client'
 require_relative 'worker'
-require_relative 'callback'
+require_relative 'concerns/checkable'
 
 module Dwf
   class Workflow
+    include Concerns::Checkable
+
     CALLBACK_TYPES = [
       BUILD_IN = 'build-in',
       SK_BATCH = 'sk-batch'
     ].freeze
-    attr_reader :dependencies, :jobs, :started_at, :finished_at, :persisted, :stopped,
+    attr_accessor :jobs, :stopped, :id, :incoming, :outgoing, :parent_id
+    attr_reader :dependencies, :started_at, :finished_at, :persisted, :arguments, :klass,
                 :callback_type
 
     class << self
@@ -19,51 +22,89 @@ module Dwf
         flow.save
         flow
       end
+
+      def find(id)
+        Dwf::Client.new.find_workflow(id)
+      end
     end
 
-    def initialize(options = {})
+    def initialize(*args)
       @dependencies = []
-      @id = id
+      @id = build_id
       @jobs = []
       @persisted = false
       @stopped = false
-      @callback_type = options[:callback_type] || BUILD_IN
+      @arguments = *args
+      @parent_id = nil
+      @klass = self.class
+      @callback_type = BUILD_IN
+      @incoming = []
+      @outgoing = []
 
       setup
     end
 
+    def persist!
+      client.persist_workflow(self)
+      jobs.each(&:persist!)
+      mark_as_persisted
+      true
+    end
+
+    def name
+      "#{self.class.name}|#{id}"
+    end
+
+    def sub_workflow?
+      !parent_id.nil?
+    end
+
+    def callback_type=(type = BUILD_IN)
+      @callback_type = type
+      jobs.each { |job| job.callback_type = type }
+    end
+
+    alias save persist!
+
     def start!
+      mark_as_started
+      persist!
       initial_jobs.each do |job|
-        cb_build_in? ? job.persist_and_perform_async! : Dwf::Callback.new.start(job)
+        job.payloads = payloads if sub_workflow?
+        job.start_initial!
       end
     end
 
-    def save
-      client.persist_workflow(self)
-      jobs.each(&:persist!)
-      mark_as_persisted
-      true
+    def payloads
+      @payloads ||= build_payloads
+    end
+
+    def start_initial!
+      cb_build_in? ? start! : Callback.new.start(self)
+    end
+
+    alias persist_and_perform_async! start!
+
+    def reload
+      flow = self.class.find(id)
+      self.stopped = flow.stopped
+      self.jobs = flow.jobs
+
+      self
     end
 
     def cb_build_in?
       callback_type == BUILD_IN
     end
 
-    def id
-      @id ||= client.build_workflow_id
+    def build_id
+      client.build_workflow_id
     end
 
-    def configure; end
+    def configure(*arguments); end
 
     def run(klass, options = {})
-      node = klass.new(
-        workflow_id: id,
-        id: client.build_job_id(id, klass.to_s),
-        params: options.fetch(:params, {}),
-        queue: options[:queue],
-        callback_type: callback_type
-      )
-
+      node = build_node(klass, options)
       jobs << node
 
       build_dependencies_structure(node, options)
@@ -95,7 +136,10 @@ module Dwf
         stopped: stopped,
         started_at: started_at,
         finished_at: finished_at,
-        callback_type: callback_type
+        callback_type: callback_type,
+        incoming: incoming,
+        outgoing: outgoing,
+        parent_id: parent_id
       }
     end
 
@@ -107,13 +151,7 @@ module Dwf
       jobs.all?(&:finished?)
     end
 
-    def started?
-      !!started_at
-    end
-
-    def running?
-      started? && !finished?
-    end
+    alias enqueued? started?
 
     def failed?
       jobs.any?(&:failed?)
@@ -123,6 +161,10 @@ module Dwf
       stopped
     end
 
+    def parents_succeeded?
+      incoming.all? { |name| client.find_node(name, parent_id).succeeded? }
+    end
+
     def status
       return :failed if failed?
       return :running if running?
@@ -132,6 +174,17 @@ module Dwf
       :running
     end
 
+    def enqueue_outgoing_jobs
+      return unless sub_workflow?
+
+      outgoing.each do |job_name|
+        client.check_or_lock(parent_id, job_name) do
+          node = client.find_node(job_name, parent_id)
+          node.persist_and_perform_async! if node.ready_to_start?
+        end
+      end
+    end
+
     def mark_as_persisted
       @persisted = true
     end
@@ -140,28 +193,94 @@ module Dwf
       @stopped = false
     end
 
+    def leaf_nodes
+      jobs.select(&:leaf?)
+    end
+
+    def output_payload
+      leaf_nodes.map do |node|
+        data = node.output_payload
+        next if data.nil?
+
+        data
+      end.compact
+    end
 
     private
 
+    def build_node(klass, options)
+      if klass < Dwf::Workflow
+        node = options[:params].nil? ? klass.new : klass.new(options[:params])
+        node.parent_id = id
+        node.callback_type = callback_type
+        node.save
+        node
+      else
+        klass.new(
+          workflow_id: id,
+          id: client.build_job_id(id, klass.to_s),
+          params: options.fetch(:params, {}),
+          queue: options[:queue],
+          callback_type: callback_type
+        )
+      end
+    end
+
     def initial_jobs
       jobs.select(&:no_dependencies?)
     end
 
     def setup
-      configure
+      configure(*arguments)
       resolve_dependencies
     end
 
+    def find_node(node_name)
+      if Utils.workflow_name?(node_name)
+        find_subworkflow(node_name)
+      else
+        find_job(node_name)
+      end
+    end
+
+    def find_subworkflow(node_name)
+      fname, = node_name.split('|')
+      jobs.find { |j| j.klass.name == fname }
+    end
+
     def resolve_dependencies
       @dependencies.each do |dependency|
-        from = find_job(dependency[:from])
-        to   = find_job(dependency[:to])
+        from = find_node(dependency[:from])
+        to   = find_node(dependency[:to])
 
-        to.incoming << dependency[:from]
-        from.outgoing << dependency[:to]
+        to.incoming << from.name
+        from.outgoing << to.name
       end
     end
 
+    def invalid_callback?
+      cb_build_in? && jobs.any? { |job| job.class < Workflow }
+    end
+
+    def build_payloads
+      return unless sub_workflow?
+
+      data = incoming.map do |job_name|
+        next if Utils.workflow_name?(job_name)
+
+        node = client.find_node(job_name, parent_id)
+        next if node.output_payload.nil?
+
+        {
+          id: node.name,
+          class: node.klass.to_s,
+          output: node.output_payload
+        }
+      end.compact
+
+      data.empty? ? nil : data
+    end
+
     def build_dependencies_structure(node, options)
       deps_after = [*options[:after]]
 

data/spec/dwf/client_spec.rb

@@ -2,6 +2,8 @@
 
 require 'spec_helper'
 require 'mock_redis'
+FirstWorkflow = Class.new(Dwf::Workflow)
+SecondWorkflow = Class.new(Dwf::Workflow)
 
 describe Dwf::Client, client: true do
   let(:client) { described_class.new }
@@ -30,12 +32,76 @@ describe Dwf::Client, client: true do
     end
 
     context 'find by item name' do
-      it {
+      it do
         item = client.find_job(workflow_id, job.name)
         expect(item.workflow_id).to eq workflow_id
         expect(item.id).to eq id
         expect(item.name).to eq job.name
-      }
+      end
+    end
+  end
+
+  describe '#find_workflow' do
+    before do
+      wf = Dwf::Workflow.new
+      wf.id = workflow_id
+      wf.save
+      j = Dwf::Item.new(id: id, workflow_id: workflow_id)
+      j.persist!
+    end
+
+    it do
+      wf = client.find_workflow(workflow_id)
+
+      expect(wf).not_to be_nil
+      expect(wf.jobs.first).to be_kind_of(Dwf::Item)
+    end
+
+    it do
+      expect do
+        client.find_workflow(SecureRandom.uuid)
+      end.to raise_error Dwf::WorkflowNotFound
+    end
+  end
+
+  describe '#find_node' do
+    context 'find job' do
+      let!(:job) do
+        j = Dwf::Item.new(workflow_id: workflow_id, id: id)
+        j.persist!
+        j
+      end
+
+      it do
+        item = client.find_node(Dwf::Item.name, workflow_id)
+        expect(item.workflow_id).to eq workflow_id
+        expect(item.id).to eq id
+        expect(item.name).to eq job.name
+      end
+    end
+
+    context 'find_workflow' do
+      let!(:wf1) { FirstWorkflow.create }
+      let!(:wf2) do
+        wf = SecondWorkflow.new
+        wf.parent_id = wf1.id
+        wf.save
+        wf
+      end
+
+      context 'find with class name and parent id' do
+        it do
+          wf = client.find_node(wf2.class.name, wf1.id)
+          expect(wf).to be_kind_of(SecondWorkflow)
+        end
+      end
+
+      context 'find with name and parent id' do
+        it do
+          wf = client.find_node(wf2.name, wf1.id)
+          expect(wf).to be_kind_of(SecondWorkflow)
+        end
+      end
     end
   end
 
@@ -53,13 +119,44 @@ describe Dwf::Client, client: true do
     end
   end
 
+  describe '#find_sub_workflow' do
+    let!(:wf1) { FirstWorkflow.create }
+    let!(:wf2) do
+      wf = SecondWorkflow.new
+      wf.parent_id = wf1.id
+      wf.save
+      wf
+    end
+
+    it do
+      wf = client.find_sub_workflow(wf2.class.name, wf1.id)
+      expect(wf).to be_kind_of(SecondWorkflow)
+    end
+  end
+
+  describe '#sub_workflows' do
+    let!(:wf1) { FirstWorkflow.create }
+    let!(:wf2) do
+      wf = SecondWorkflow.new
+      wf.parent_id = wf1.id
+      wf.save
+      wf
+    end
+
+    it do
+      wfs = client.sub_workflows(wf1.id)
+      expect(wfs).not_to be_empty
+      expect(wfs.first).to be_kind_of(SecondWorkflow)
+    end
+  end
+
   describe '#persist_workflow' do
     let(:workflow) { Dwf::Workflow.new }
 
     it do
       expect(redis.exists?("dwf.workflows.#{workflow.id}")).to be_falsy
       client.persist_workflow(workflow)
-      expect(redis.exists?("dwf.workflows.#{workflow.id}")).to be_truthy
+      expect(redis.keys("dwf.workflows.#{workflow.id}*").any?).to be_truthy
     end
   end
 
@@ -73,24 +170,18 @@ describe Dwf::Client, client: true do
 
       before do
         allow(client).to receive(:set)
-        redis.set("wf_enqueue_outgoing_jobs_#{workflow_id}-#{job_name}", 'running')
-        client.check_or_lock(workflow_id, job_name)
-      end
-
-      it { expect(client).not_to have_received(:set) }
-    end
-
-    context 'job is not running' do
-      let(:job_name) { 'ahihi' }
-
-      before do
-        allow(redis).to receive(:set)
-        client.check_or_lock(workflow_id, job_name)
+        allow(RedisMutex).to receive(:with_lock)
+        client.check_or_lock(workflow_id, job_name) {}
       end
 
       it do
-        expect(redis).to have_received(:set)
-          .with("wf_enqueue_outgoing_jobs_#{workflow_id}-#{job_name}", 'running')
+        expect(RedisMutex)
+          .to have_received(:with_lock)
+          .with(
+            "wf_enqueue_outgoing_jobs_#{workflow_id}-#{job_name}",
+            sleep: 0.3,
+            block: 2
+          )
       end
     end
   end