rworkflow 0.6.1

data/lib/rworkflow/lifecycle.rb

@@ -0,0 +1,102 @@
+module Rworkflow
+  class Lifecycle
+    attr_reader :states
+    attr_accessor :initial, :default, :state_class, :state_options
+
+    CARDINALITY_ALL_STARTED = :all_started # Indicates a cardinality equal to the jobs pushed at the start of the workflow
+
+    DEFAULT_STATE_OPTIONS = {
+      cardinality: State::DEFAULT_CARDINALITY,
+      priority: State::DEFAULT_PRIORITY,
+      policy: State::STATE_POLICY_NO_WAIT
+    }
+
+    def initialize(state_class: State, state_options: {})
+      @state_options = DEFAULT_STATE_OPTIONS.merge(state_options)
+      @state_class = state_class
+      @states = {}
+      @default = nil
+      yield(self) if block_given?
+    end
+
+    def state(name, options = {})
+      options = @state_options.merge(options)
+      new_state = @state_class.new(**options)
+
+      yield(new_state) if block_given?
+
+      @states[name] = new_state
+    end
+
+    def transition(from, name)
+      from_state = @states[from]
+      fail(StateError, from) if from_state.nil?
+
+      return from_state.perform(name, @default)
+    end
+
+    def concat!(from, name, lifecycle, &state_merge_handler)
+      state_merge_handler ||= lambda do |_, original_state, concat_state|
+        original_state.merge(concat_state)
+      end
+
+      @states.merge!(lifecycle.states, &state_merge_handler)
+
+      next_state = lifecycle.initial
+      @states[from].transition(name, next_state)
+      return self
+    end
+
+    def rename_state(old_state_name, new_state_name)
+      old_state = @states[old_state_name]
+      @states[new_state_name] = old_state
+      @states.delete(old_state)
+
+      @initial = new_state_name if @initial == old_state_name
+    end
+
+    def to_h
+      return {
+        initial: @initial,
+        default: @default,
+        state_class: @state_class,
+        state_options: @state_options,
+        states: Hash[@states.map { |name, state| [name, state.to_h] }]
+      }
+    end
+
+    def to_graph
+      states = @states || []
+      return digraph do
+        states.each do |from, state|
+          state.transitions.each do |transition, to|
+            edge(from.to_s, to.to_s).label(transition.to_s)
+          end
+        end
+      end
+    end
+
+    def serialize
+      return self.class.serialize(self)
+    end
+
+    class << self
+      def serialize(lifecycle)
+        return lifecycle.to_h
+      end
+
+      def unserialize(hash)
+        return self.new do |lf|
+          lf.initial = hash[:initial]
+          lf.default = hash[:default]
+          lf.state_options = hash[:state_options]
+          lf.state_class = hash[:state_class]
+
+          hash[:states].each do |name, state_hash|
+            lf.states[name] = lf.state_class.unserialize(state_hash)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/rworkflow/minitest/test.rb

@@ -0,0 +1,53 @@
+module Rworkflow
+  module Minitest
+    # Include in your test classes to add functionality for worker and workflow tests
+    module Test
+      def setup
+        super
+        rworkflow_setup
+      end
+
+      def teardown
+        super
+        rworkflow_teardown
+      end
+
+      def rworkflow_setup
+      end
+      protected :rworkflow_setup
+
+      def rworkflow_teardown
+      end
+      protected :rworkflow_teardown
+
+      # @params [Class] the worker class to instantiate
+      # @params [Hash] options hash
+      # @option [Class] :flow workflow class to instantiate; defaults to SidekiqFlow
+      # @option [Class] :name the state name
+      def rworkflow_worker(worker_class, flow: ::SidekiqFlow, name: nil, meta: {})
+        name ||= worker_class.name
+        worker = worker_class.new
+        workflow = flow.new(name)
+        meta.each { |key, value| workflow.set(key, value) }
+
+        worker.instance_variable_set(:@workflow, workflow)
+        worker.instance_variable_set(:@state_name, name)
+
+        workflow.extend(WorkerUnitTestFlow)
+        if defined?(flexmock)
+          flexmock(workflow.class).should_receive(:terminal?).and_return(true)
+        end
+
+        yield(workflow) if block_given?
+
+        return worker, workflow
+      end
+    end
+
+    module WorkerUnitTestFlow
+      def transition(_, name, objects)
+        push(objects, name)
+      end
+    end
+  end
+end

data/lib/rworkflow/minitest/worker.rb

@@ -0,0 +1,17 @@
+module Rworkflow
+  # Disable pushing back indefinitely
+  class Worker
+    def initialize(*args)
+      super
+      @__pushed_back = []
+    end
+
+    def pushed_back
+      return @__pushed_back
+    end
+
+    def push_back(objects)
+      @__pushed_back.concat(objects)
+    end
+  end
+end

data/lib/rworkflow/minitest.rb

@@ -0,0 +1,8 @@
+require 'rworkflow/minitest/worker'
+require 'rworkflow/minitest/test'
+
+if defined?(Minitest)
+  class Minitest::Test
+    include Rworkflow::Minitest::Test
+  end
+end

data/lib/rworkflow/sidekiq_flow.rb

@@ -0,0 +1,186 @@
+module Rworkflow
+  class SidekiqFlow < Flow
+
+    STATE_POLICY_GATED = :gated
+    MAX_EXPECTED_DURATION = 4.hours
+    PRIORITIES = [:critical, :high, nil, :low]
+
+    def initialize(id)
+      super(id)
+      @open_gates = RedisRds::Set.new("#{@redis_key}__open_gates")
+    end
+
+    def cleanup
+      super()
+      @open_gates.delete()
+    end
+
+    def push(objects, name)
+      pushed = 0
+      pushed = super(objects, name)
+    ensure
+      create_jobs(name, pushed) if pushed > 0
+      return pushed
+    end
+
+    def expected_duration
+      return MAX_EXPECTED_DURATION
+    end
+
+    def paused?
+      return @flow_data.get(:paused).to_i > 0
+    end
+
+    def status
+      return (paused?) ? 'Paused' : super()
+    end
+
+    def pause
+      return if self.finished?
+      @flow_data.incr(:paused)
+    rescue StandardError => e
+      Rails.logger.error("Error pausing flow #{self.id}: #{e.message}")
+    end
+
+    # for now assumes
+    def continue
+      return if self.finished? || !self.valid? || !self.paused?
+      if @flow_data.decr(:paused) == 0
+        workers = Hash[get_counters.select { |name, _| !self.class.terminal?(name) && name != :processing }]
+
+        # enqueue jobs
+        workers.each { |worker, num_objects| create_jobs(worker, num_objects) }
+      end
+    rescue StandardError => e
+      Rails.logger.error("Error continuing flow #{self.id}: #{e.message}")
+    end
+
+    def create_jobs(state_name, num_objects)
+      return if paused? || num_objects < 1 || self.class.terminal?(state_name) || gated?(state_name)
+      state = @lifecycle.states[state_name]
+      worker_class = if state.respond_to?(:worker_class)
+        state.worker_class
+      else
+        begin
+          state_name.constantize
+        rescue NameError => _
+          Rails.logger.error("Trying to push to a non existent worker class #{state_name} in workflow #{@id}")
+          nil
+        end
+      end
+
+      if !worker_class.nil?
+        cardinality = get_state_cardinality(state_name)
+
+        if state.policy == State::STATE_POLICY_WAIT
+          amount = ((num_objects + get_state_list(state_name).size) / cardinality.to_f).floor
+        else
+          amount = (num_objects / cardinality.to_f).ceil
+        end
+
+        state_priority = self.priority || state.priority
+        amount.times { worker_class.enqueue_job_with_priority(state_priority, @id, state_name) }
+      end
+    end
+
+    def priority
+      return @priority ||= begin self.get(:priority) end
+    end
+
+    def gated?(state_name)
+      state = @lifecycle.states[state_name]
+      return state.policy == STATE_POLICY_GATED && !@open_gates.include?(state_name)
+    end
+
+    def open_gate(state_name)
+      @open_gates.add(state_name)
+      num_objects = count(state_name)
+      create_jobs(state_name, num_objects)
+    end
+
+    def close_gate(state_name)
+      @open_gates.remove(state_name)
+    end
+
+    class << self
+      def create(lifecycle, name = '', options)
+        workflow = super(lifecycle, name, options)
+        workflow.set(:priority, options[:priority]) unless options[:priority].nil?
+
+        return workflow
+      end
+
+      def get_manual_priority
+        return :high
+      end
+
+      def cleanup_broken_flows
+        broken = []
+        flows = self.all
+        flows.each do |flow|
+          if flow.valid?
+            if flow.finished? && !flow.public?
+              broken << [flow, 'finished']
+            elsif !flow.started? && flow.created_at < 1.day.ago
+              broken << [flow, 'never started']
+            end
+          else
+            broken << [flow, 'invalid']
+          end
+        end
+
+        broken.each do |flow_pair|
+          flow_pair.first.cleanup
+          puts "Cleaned up #{flow_pair.second} flow #{flow_pair.first.id}"
+        end
+        puts ">>> Cleaned up #{broken.size} broken flows <<<"
+      end
+
+      def enqueue_missing_jobs
+        queued_flow_map = build_flow_map
+        running_flows = self.all.select { |f| f.valid? && !f.finished? && !f.paused? }
+        running_flows.each do |flow|
+          state_map = queued_flow_map.fetch(flow.id, {})
+          create_missing_jobs(flow, state_map)
+        end
+      end
+
+      def build_flow_map
+        flow_map = {}
+        queues = SidekiqHelper.get_queue_sizes.keys
+        queues.each do |queue_name|
+          queue = Sidekiq::Queue.new(queue_name)
+          queue.each do |job|
+            klass = begin
+              job.klass.constantize
+            rescue NameError => _
+              nil
+            end
+
+            if !klass.nil? && klass <= Rworkflow::Worker
+              id = job.args.first
+              state_name = job.args.second
+              state_map = flow_map.fetch(id, {})
+              state_map[state_name] = state_map.fetch(state_name, 0) + 1
+              flow_map[id] = state_map
+            end
+          end
+        end
+        return flow_map
+      end
+
+      def create_missing_jobs(flow, state_map)
+        counters = flow.get_counters
+        counters.each do |state, num_objects|
+          next if flow.class.terminal?(state) || state == :processing
+          enqueued = state_map.fetch(state, 0) * flow.get_state_cardinality(state)
+          missing = num_objects - enqueued
+          if missing > 0
+            flow.create_jobs(state, missing)
+            puts "Created #{missing} missing jobs for state #{state} in flow #{flow.id}"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/rworkflow/sidekiq_helper.rb

@@ -0,0 +1,84 @@
+require 'sidekiq/api'
+
+module Rworkflow
+  module SidekiqHelper
+    def self.included(klass)
+      klass.send :extend,  ClassMethods
+      klass.send :include, InstanceMethods
+    end
+
+    module ClassMethods
+      # Mix-in methods
+      def enqueue_job(*params)
+        enqueue_job_with_priority(nil, *params)
+      end
+
+      def enqueue_job_with_priority(priority, *params)
+        if should_perform_job_async?
+          self.perform_with_priority(priority, *params)
+        else
+          inline_perform(params)
+        end
+      end
+
+      def enqueue_job_at(at_time, *params)
+        if should_perform_job_async?
+          self.perform_at(at_time, *params)
+        else
+          inline_perform(params)
+        end
+      end
+
+      def enqueue_job_in(time_diff, *params)
+        if should_perform_job_async?
+          self.perform_in(time_diff, *params)
+        else
+          inline_perform(params)
+        end
+      end
+
+      def should_perform_job_async?
+        return Rails.env.production?
+      end
+
+      def inline_perform(params)
+        worker = self.new
+        args = JSON.parse(params.to_json)
+        jid = Digest::MD5.hexdigest((Time.now.to_f * 1000).to_i.to_s)
+        worker.jid = jid
+        worker.perform(*args)
+      end
+    end
+
+    module InstanceMethods
+    end
+
+    # Static methods
+    class << self
+      def configure_server host, port, db
+        Sidekiq.configure_server do |config|
+          config.redis = {:url => "redis://#{host}:#{port}/#{db}", :namespace => 'sidekiq'}
+          config.server_middleware do |chain|
+            chain.add SidekiqServerMiddleware
+          end
+        end
+      end
+
+      def configure_client host, port, db
+        Sidekiq.configure_client do |config|
+          config.redis = {:url => "redis://#{host}:#{port}/#{db}", :namespace => 'sidekiq'}
+        end
+      end
+
+      def get_queue_sizes
+        stats = Sidekiq::Stats.new
+        return stats.queues
+      end
+
+      def get_queue_sizes_sum
+        stats = Sidekiq::Stats.new
+        return stats.enqueued
+      end
+    end
+  end
+end

data/lib/rworkflow/sidekiq_lifecycle.rb

@@ -0,0 +1,8 @@
+module Rworkflow
+  class SidekiqLifecycle < Lifecycle
+    def initialize(**options)
+      options[:state_class] ||= SidekiqState
+      super(**options)
+    end
+  end
+end

data/lib/rworkflow/sidekiq_state.rb

@@ -0,0 +1,42 @@
+module Rworkflow
+  class SidekiqState < State
+    attr_accessor :worker_class
+
+    def initialize(worker: nil, **options)
+      super(**options)
+      @worker_class = worker
+    end
+
+    def merge!(state)
+      super
+      @worker_class = state.worker_class if state.respond_to?(:worker_class)
+    end
+
+    def clone
+      cloned = super
+      cloned.worker_class = @worker_class
+
+      return cloned
+    end
+
+    def ==(state)
+      return super && state.worker_class == @worker_class
+    end
+
+    def to_h
+      h = super
+      h[:worker_class] = @worker_class
+
+      return h
+    end
+
+    class << self
+      def unserialize(state_hash)
+        state = super(state_hash)
+        state.worker_class = state_hash[:worker_class]
+
+        return state
+      end
+    end
+  end
+end

data/lib/rworkflow/state.rb

@@ -0,0 +1,104 @@
+module Rworkflow
+  class State
+    DEFAULT_CARDINALITY = 1
+    DEFAULT_PRIORITY = nil
+
+    # To be refactored into Policy objects
+    STATE_POLICY_WAIT = :wait
+    STATE_POLICY_NO_WAIT = :no_wait
+
+    attr_accessor :cardinality, :priority, :policy
+    attr_reader :transitions
+
+    def initialize(cardinality: DEFAULT_CARDINALITY, priority: DEFAULT_PRIORITY, policy: STATE_POLICY_NO_WAIT, **_)
+      @cardinality = cardinality
+      @priority = priority
+      @policy = policy
+
+      @transitions = {}
+    end
+
+    def transition(name, to)
+      @transitions[name] = to
+    end
+
+    def perform(name, default = nil)
+      to_state = @transitions[name] || default
+      raise TransitionError.new(name) if to_state.nil?
+      return to_state
+    end
+
+    # Default rule: new state overwrites old state when applicable
+    def merge!(state)
+      @cardinality = state.cardinality
+      @priority = state.priority
+      @policy = state.policy
+
+      @transitions.merge!(state.transitions) do |_, _, transition|
+        transition
+      end
+
+      return self
+    end
+
+    def merge(state)
+      return self.clone.merge!(state)
+    end
+
+    def clone
+      cloned = self.class.new(cardinality: @cardinality, priority: @priority, policy: @policy)
+      @transitions.each { |from, to| cloned.transition(from, to) }
+      return cloned
+    end
+
+    def ==(state)
+      return @cardinality == state.cardinality &&
+        @priority == state.priority &&
+        @policy == state.policy &&
+        @transitions == state.transitions
+    end
+
+    def to_h
+      return {
+        transitions: @transitions,
+        cardinality: @cardinality,
+        priority: @priority,
+        policy: @policy
+      }
+    end
+
+    def to_graph
+      transitions = @transitions # need to capture for block, as digraph rebinds context
+
+      return digraph do
+        transitions.each do |transition, to|
+          edge('self', to.to_s).label(transition.to_s)
+        end
+      end
+    end
+
+    def inspect
+      return "[ Cardinality: #{@cardinality} ; Policy: #{@policy} ; Priority: #{@priority} ] -> #{to_graph.to_s}"
+    end
+
+    def serialize
+      return self.class.serialize(self)
+    end
+
+    class << self
+      def serialize(state)
+        return state.to_h
+      end
+
+      def unserialize(state_hash)
+        state = self.new(**state_hash)
+
+        state_hash[:transitions].each do |from, to|
+          state.transition(from, to)
+        end
+
+        return state
+      end
+    end
+  end
+end

data/lib/rworkflow/state_error.rb

@@ -0,0 +1,13 @@
+module Rworkflow
+  class StateError < StandardError
+    attr_reader :name
+
+    def initialize(name)
+      @name = name
+    end
+
+    def message
+      return "#{@name} state does not exist"
+    end
+  end
+end

data/lib/rworkflow/transition_error.rb

@@ -0,0 +1,9 @@
+require 'rworkflow/state_error'
+
+module Rworkflow
+  class TransitionError < StateError
+    def message
+      return "#{@name} transition does not exist"
+    end
+  end
+end

data/lib/rworkflow/version.rb

@@ -0,0 +1,3 @@
+module Rworkflow
+  VERSION = '0.6.1'.freeze
+end

data/lib/rworkflow/worker.rb

@@ -0,0 +1,62 @@
+require 'sidekiq'
+
+module Rworkflow
+  class Worker
+    include Sidekiq::Worker
+    include SidekiqHelper
+
+    sidekiq_options queue: :mysql
+
+    def perform(id, state_name)
+      @workflow = self.class.load_workflow(id)
+      @state_name = state_name
+      if !@workflow.nil?
+        if !@workflow.paused?
+          @workflow.fetch(self.jid, state_name) do |objects|
+            if objects.present?
+              Rails.logger.debug("Starting #{self.class}::process() (flow #{id})")
+              process(objects)
+              Rails.logger.debug("Finished #{self.class}::process() (flow #{id})")
+            else
+              Rails.logger.debug("No objects to process for #{self.class}")
+            end
+          end
+        end
+      end
+    rescue Exception => e
+      Rails.logger.error("Exception produced on #{@state_name} for flow #{id} on perform: #{e.message}\n#{e.backtrace}")
+      raise e
+    end
+
+    def transition(to_state, objects)
+      @workflow.transition(@state_name, to_state, objects)
+      Rails.logger.debug("State #{@state_name} transitioned #{Array.wrap(objects).size} objects to state #{to_state} (flow #{@workflow.id})")
+    end
+
+    def push_back(objects)
+      @workflow.push(objects, @state_name)
+      Rails.logger.debug("State #{@state_name} pushed back #{Array.wrap(objects).size} objects (flow #{@workflow.id})")
+    end
+
+    def process(_objects)
+      raise NotImplementedError
+    end
+
+    class << self
+      def generate_lifecycle(&block)
+        return Rworkflow::Lifecycle.new do |lc|
+          lc.state(self.class.name, worker: self.class, &block)
+          lc.initial = self.class.name
+        end
+      end
+
+      def load_workflow(id)
+        workflow = Flow.load(id)
+        return workflow if !workflow.nil? && workflow.valid?
+
+        Rails.logger.warn("Worker #{self.name} tried to load non existent workflow #{id}")
+        return nil
+      end
+    end
+  end
+end