distribot 0.1.1

data/lib/distribot/flow.rb

@@ -0,0 +1,200 @@
+
+module Distribot
+  class NotRunningError < StandardError; end
+  class NotPausedError < StandardError; end
+
+  # rubocop:disable ClassLength
+  class Flow
+    attr_accessor :id, :phases, :consumer, :finished_callback_queue, :created_at, :data
+
+    def initialize(attrs = {})
+      self.id = attrs[:id]
+      self.created_at = attrs[:created_at] unless attrs[:created_at].nil?
+      self.phases = []
+      (attrs[:phases] || []).each do |options|
+        add_phase(options)
+      end
+      self.data = attrs[:data]
+    end
+
+    def self.active
+      redis.smembers('distribot.flows.active').map do |id|
+        self.find(id)
+      end
+    end
+
+    def self.create!(attrs = {})
+      new(attrs).save!
+    end
+
+    # rubocop:disable Metrics/AbcSize
+    def save!(&block)
+      fail StandardError, 'Cannot re-save a flow' if id
+      self.id = SecureRandom.uuid
+      record_id = redis_id + ':definition'
+      self.created_at = Time.now.to_f
+
+      # Actually save the record:
+      redis.set record_id, serialize
+
+      # Transition into the first phase:
+      add_transition to: current_phase, timestamp: Time.now.utc.to_f
+
+      # Add our id to the list of active flows:
+      redis.sadd 'distribot.flows.active', id
+      redis.incr('distribot.flows.running')
+
+      # Announce our arrival to the rest of the system:
+      Distribot.publish! 'distribot.flow.created', flow_id: id
+
+      wait_for_flow_to_finish(block) if block_given?
+      self
+    end
+
+    def self.find(id)
+      redis_id = Distribot.redis_id('flow', id)
+      raw_json = redis.get("#{redis_id}:definition") || return
+      new(
+        JSON.parse(raw_json, symbolize_names: true)
+      )
+    end
+
+    def add_phase(options = {})
+      phases << Phase.new(options)
+    end
+
+    def phase(name)
+      phases.find { |x| x.name == name }
+    end
+
+    def pause!
+      fail NotRunningError, 'Cannot pause unless running' unless running?
+      add_transition(
+        from: current_phase,
+        to: 'paused',
+        timestamp: Time.now.utc.to_f
+      )
+    end
+
+    def resume!
+      fail NotPausedError, 'Cannot resume unless paused' unless paused?
+
+      # Find the last transition before we were paused:
+      prev_phase = transitions.reverse.find { |x| x.to != 'paused' }
+      # Back to where we once belonged
+      add_transition(
+        from: 'paused', to: prev_phase.to, timestamp: Time.now.utc.to_f
+      )
+    end
+
+    def paused?
+      current_phase == 'paused'
+    end
+
+    def cancel!
+      fail NotRunningError, 'Cannot cancel unless running' unless running?
+      add_transition(
+        from: current_phase, to: 'canceled', timestamp: Time.now.utc.to_f
+      )
+      redis.decr 'distribot.flows.running'
+      redis.srem 'distribot.flows.active', id
+    end
+
+    def canceled?
+      current_phase == 'canceled'
+    end
+
+    def running?
+      ! (paused? || canceled? || finished?)
+    end
+
+    def redis_id
+      @redis_id ||= Distribot.redis_id('flow', id)
+    end
+
+    def transition_to!(phase)
+      previous_transition = transitions.last
+      prev = previous_transition ? previous_transition[:to] : nil
+      add_transition(from: prev, to: phase, timestamp: Time.now.utc.to_f)
+      Distribot.publish!(
+        'distribot.flow.phase.started',
+        flow_id: id,
+        phase: phase
+      )
+    end
+
+    def add_transition(item)
+      redis.sadd(redis_id + ':transitions', item.to_json)
+    end
+
+    def transitions
+      redis.smembers(redis_id + ':transitions').map do |item|
+        OpenStruct.new JSON.parse(item, symbolize_names: true)
+      end.sort_by(&:timestamp)
+    end
+
+    def current_phase
+      latest_transition = transitions.last
+      if latest_transition
+        latest_transition.to
+      else
+        phases.find(&:is_initial).name
+      end
+    end
+
+    def next_phase
+      current = current_phase
+      phases.find { |x| x.name == current }.transitions_to
+    end
+
+    def finished?
+      phase(transitions.last.to).is_final
+    end
+
+    def stubbornly(task, &block)
+      loop do
+        begin
+          return block.call
+        rescue NoMethodError => e
+          warn "Error during #{task}: #{e} --- #{e.backtrace.join("\n")}"
+          sleep 1
+        end
+      end
+    end
+
+    private
+
+    def wait_for_flow_to_finish(block)
+      Thread.new do
+        loop do
+          sleep 1
+          if finished? || canceled?
+            block.call(flow_id: id)
+            break
+          end
+        end
+      end
+    end
+
+    def self.redis
+      Distribot.redis
+    end
+
+    def redis
+      self.class.redis
+    end
+
+    def serialize
+      to_hash.to_json
+    end
+
+    def to_hash
+      {
+        id: id,
+        created_at: created_at,
+        phases: phases.map(&:to_hash),
+        data: self.data
+      }
+    end
+  end
+end

data/lib/distribot/flow_created_handler.rb

@@ -0,0 +1,12 @@
+
+module Distribot
+  class FlowCreatedHandler
+    include Distribot::Handler
+    subscribe_to 'distribot.flow.created', handler: :callback
+
+    def callback(message)
+      flow = Distribot::Flow.find(message[:flow_id])
+      flow.transition_to! flow.next_phase
+    end
+  end
+end

data/lib/distribot/flow_finished_handler.rb

@@ -0,0 +1,12 @@
+
+module Distribot
+  class FlowFinishedHandler
+    include Distribot::Handler
+    subscribe_to 'distribot.flow.finished', handler: :callback
+
+    def callback(message)
+      Distribot.redis.decr('distribot.flows.running')
+      Distribot.redis.srem 'distribot.flows.active', message[:flow_id]
+    end
+  end
+end

data/lib/distribot/handler.rb

@@ -0,0 +1,40 @@
+
+module Distribot
+  module Handler
+    attr_accessor :queue_name, :consumers
+
+    def self.included(base)
+      base.extend ClassMethods
+    end
+
+    def initialize
+      self.consumers = []
+      self.queue_name = self.class.queue
+      handler = self.class.handler
+      Distribot.subscribe(queue_name, self.class.subscribe_args) do |message|
+        send(handler, message)
+      end
+    end
+
+    def self.handler_for(klass)
+      klass.handler
+    end
+
+    def self.queue_for(klass)
+      klass.queue
+    end
+
+    module ClassMethods
+      class << self
+        attr_accessor :queue, :handler, :subscribe_args
+      end
+      attr_reader :handler, :queue, :subscribe_args
+
+      def subscribe_to(queue_name, handler_args)
+        @queue = queue_name
+        @handler = handler_args.delete :handler
+        @subscribe_args = handler_args
+      end
+    end
+  end
+end

data/lib/distribot/handler_finished_handler.rb

@@ -0,0 +1,29 @@
+
+module Distribot
+  class HandlerFinishedHandler
+    include Distribot::Handler
+    subscribe_to 'distribot.flow.handler.finished', handler: :callback
+
+    def callback(message)
+      flow = Distribot::Flow.find(message[:flow_id])
+      phase = flow.phase(message[:phase])
+
+      Distribot.publish!(
+        'distribot.flow.phase.finished',
+        flow_id: flow.id,
+        phase: phase.name
+      ) if self.all_phase_handler_tasks_are_complete?(flow, phase)
+    end
+
+    def all_phase_handler_tasks_are_complete?(flow, phase)
+      redis = Distribot.redis
+      name = phase.name
+      phase.handlers
+        .map { |h| "distribot.flow.#{flow.id}.#{name}.#{h}.tasks" }
+        .map { |task_counter_key| redis.get(task_counter_key) }
+        .reject(&:nil?)
+        .select { |val| val.to_i > 0 }
+        .empty?
+    end
+  end
+end

data/lib/distribot/phase.rb

@@ -0,0 +1,46 @@
+
+module Distribot
+  class Phase
+    attr_accessor :id,
+                  :name,
+                  :is_initial,
+                  :is_final,
+                  :transitions_to,
+                  :on_error_transition_to,
+                  :handlers
+
+    def initialize(attrs = {})
+      attrs.each do |key, val|
+        next if key.to_s == 'handlers'
+        public_send("#{key}=", val)
+      end
+      self.name = name
+      self.handlers = []
+      initialize_handlers(attrs[:handlers] || [])
+    end
+
+    def to_hash
+      {
+        id: id,
+        name: name,
+        is_initial: is_initial || false,
+        is_final: is_final || false,
+        transitions_to: transitions_to,
+        on_error_transition_to: on_error_transition_to,
+        handlers: handlers
+      }
+    end
+
+    private
+
+    def initialize_handlers(handler_args)
+      handler_args.each do |handler|
+        if handler.is_a? Hash
+          handlers.push(PhaseHandler.new handler)
+        else
+          handlers.push(PhaseHandler.new name: handler)
+        end
+      end
+    end
+  end
+end

data/lib/distribot/phase_finished_handler.rb

@@ -0,0 +1,19 @@
+
+module Distribot
+  class PhaseFinishedHandler
+    include Distribot::Handler
+    subscribe_to 'distribot.flow.phase.finished', handler: :callback
+
+    def callback(message)
+      flow = Distribot::Flow.find(message[:flow_id])
+      return unless flow.current_phase == message[:phase]
+      if flow.next_phase
+        flow.transition_to! flow.next_phase
+      else
+        Distribot.publish!(
+          'distribot.flow.finished', flow_id: flow.id
+        )
+      end
+    end
+  end
+end

data/lib/distribot/phase_handler.rb

@@ -0,0 +1,15 @@
+
+module Distribot
+  class PhaseHandler
+    attr_accessor :name, :version
+    def initialize(attrs = {})
+      attrs.each do |key, val|
+        public_send("#{key}=", val)
+      end
+    end
+
+    def to_s
+      name
+    end
+  end
+end

data/lib/distribot/phase_started_handler.rb

@@ -0,0 +1,69 @@
+
+module Distribot
+  require 'semantic'
+  class PhaseStartedHandler
+    include Distribot::Handler
+    subscribe_to 'distribot.flow.phase.started', handler: :callback
+
+    def callback(message)
+      flow = Distribot::Flow.find(message[:flow_id])
+      phase = flow.phase(message[:phase])
+      if phase.handlers.empty?
+        Distribot.publish!(
+          'distribot.flow.phase.finished',
+          flow_id: flow.id,
+          phase: phase.name
+        )
+      else
+        handler_versions = phase.handlers.map do |handler|
+          version = best_version(handler)
+          unless version && !version.blank?
+            fail "Cannot find a good #{handler} version #{handler.version}"
+          end
+          {
+            handler.to_s => version
+          }
+        end.reduce({}, :merge)
+        phase.handlers.each do |handler|
+          init_handler(flow, phase, handler, handler_versions[handler.to_s])
+        end
+      end
+    end
+
+    # rubocop:disable Metrics/LineLength
+    def init_handler(flow, phase, handler, version)
+      Distribot.publish!(
+        "distribot.flow.handler.#{handler}.#{version}.enumerate",
+        flow_id: flow.id,
+        phase: phase.name,
+        task_queue: "distribot.flow.handler.#{handler}.#{version}.tasks",
+        task_counter: "distribot.flow.#{flow.id}.#{phase.name}.#{handler}.finished",
+        finished_queue: 'distribot.flow.task.finished'
+      )
+    end
+
+    def best_version(handler)
+      if handler.version
+        wanted_version = Gem::Dependency.new(handler.to_s, handler.version)
+        # Figure out the highest acceptable version of the handler we can assign work to:
+        handler_versions(handler.to_s)
+          .reverse
+          .find { |x| wanted_version.match?(handler.to_s, x.to_s) }
+          .to_s
+      else
+        # Find the highest version for this queue:
+        handler_versions(handler.to_s).last
+      end
+    end
+
+    def handler_versions(handler)
+      queue_prefix = "distribot.flow.handler.#{handler}."
+      Distribot.connector.queues
+        .select { |x| x.start_with? queue_prefix }
+        .reject { |x| x.end_with? '.enumerate' }
+        .map { |x| x.gsub(/^#{queue_prefix}/, '').gsub(/\.tasks$/, '') }
+        .map { |x| Semantic::Version.new x }
+        .sort
+    end
+  end
+end

data/lib/distribot/task_finished_handler.rb

@@ -0,0 +1,37 @@
+
+module Distribot
+  class TaskFinishedHandler
+    include Distribot::Handler
+    subscribe_to 'distribot.flow.task.finished', handler: :callback
+
+    def callback(message)
+      task_counter_key = task_counter(message)
+      current_value = Distribot.redis.get(task_counter_key) || return
+      return unless current_value.to_i == 0
+      Distribot.redis.del(task_counter_key)
+      announce_handler_has_finished(message)
+    end
+
+    def announce_handler_has_finished(message)
+      Distribot.publish!(
+        'distribot.flow.handler.finished',
+        flow_id: message[:flow_id],
+        phase: message[:phase],
+        handler: message[:handler],
+        task_queue: message[:task_queue]
+      )
+    end
+
+    def task_counter(message)
+      # i.e. - distribot.flow.flowId.phaseName.handlerName.finished
+      [
+        'distribot',
+        'flow',
+        message[:flow_id],
+        message[:phase],
+        message[:handler].to_s,
+        'finished'
+      ].join('.')
+    end
+  end
+end

data/lib/distribot/worker.rb

@@ -0,0 +1,148 @@
+
+module Distribot
+  class FlowCanceledError < StandardError; end
+  class FlowPausedError < StandardError; end
+  module Worker
+    def self.included(base)
+      base.extend(ClassMethods)
+    end
+
+    module ClassMethods
+      class << self
+        attr_accessor :version, :enumerator, :process_tasks_with, :processor
+      end
+
+      attr_reader :processor, :enumerator
+
+      def enumerate_with(callback)
+        @enumerator = callback
+      end
+
+      def process_tasks_with(callback)
+        @processor = callback
+      end
+
+      # Does both setting/getting:
+      def version(val = nil)
+        @version ||= '0.0.0'
+        @version = val unless val.nil?
+        @version
+      end
+
+      def enumeration_queue
+        self.version ||= '0.0.0'
+        "distribot.flow.handler.#{self}.#{self.version}.enumerate"
+      end
+
+      def task_queue
+        self.version ||= '0.0.0'
+        "distribot.flow.handler.#{self}.#{self.version}.tasks"
+      end
+    end
+
+    attr_accessor :task_consumers
+
+    def run
+      prepare_for_enumeration
+      subscribe_to_task_queue
+      self
+    end
+
+    def logger
+      Distribot.logger
+    end
+
+    def prepare_for_enumeration
+      logger.tagged("handler:#{self.class}") do
+        pool = Concurrent::FixedThreadPool.new(5)
+        Distribot.subscribe(self.class.enumeration_queue, solo: true) do |message|
+          pool.post do
+            logger.tagged(message.map { |k, v| [k, v].join(':') }) do
+              context = OpenStruct.new(message)
+              trycatch do
+                tasks = enumerate_tasks(message)
+                announce_tasks(context, message, tasks)
+              end
+            end
+          end
+        end
+      end
+    end
+
+    def subscribe_to_task_queue
+      logger.tagged("handler:#{self.class}") do
+        subscribe_args = { reenqueue_on_failure: true, solo: true }
+        pool = Concurrent::FixedThreadPool.new(500)
+        Distribot.subscribe(self.class.task_queue, subscribe_args) do |task|
+          pool.post do
+            logger_tags = task.map { |k, v| [k, v].join(':') }
+            logger.tagged(logger_tags) do
+              handle_task_execution(task)
+            end
+          end
+        end
+      end
+    end
+
+    def handle_task_execution(task)
+      context = OpenStruct.new(
+        flow_id: task[:flow_id],
+        phase: task[:phase],
+        finished_queue: 'distribot.flow.task.finished'
+      )
+      trycatch do
+        process_single_task(context, task)
+      end
+    end
+
+    def process_single_task(context, task)
+      inspect_task!(context)
+      # Your code is called right here:
+      send(self.class.processor, context, task)
+      task_counter_key = "distribot.flow.#{context.flow_id}.#{context.phase}.#{self.class}.finished"
+      Distribot.redis.decr(task_counter_key)
+      publish_args = {
+        flow_id: context.flow_id,
+        phase: context.phase,
+        handler: self.class.to_s
+      }
+      Distribot.publish!(context.finished_queue, publish_args)
+    end
+
+    def enumerate_tasks(message)
+      trycatch do
+        context = OpenStruct.new(message)
+        flow = Distribot::Flow.find(context.flow_id)
+        fail FlowCanceledError if flow.canceled?
+        send(self.class.enumerator, context)
+      end
+    end
+
+    private
+
+    def announce_tasks(context, message, tasks)
+      task_counter = message[:task_counter]
+      Distribot.redis.incrby task_counter, tasks.count
+      Distribot.redis.incrby "#{task_counter}.total", tasks.count
+      tasks.each do |task|
+        task.merge!(flow_id: context.flow_id, phase: context.phase)
+        Distribot.publish! message[:task_queue], task
+      end
+    end
+
+    def inspect_task!(context)
+      flow = Distribot::Flow.find(context.flow_id)
+      fail FlowCanceledError if flow.canceled?
+      fail FlowPausedError if flow.paused?
+    end
+
+    def trycatch(&block)
+      # Put the try/catch logic here to reduce boilerplate code:
+      block.call
+    rescue StandardError => e
+      logger.error "ERROR: #{e} --- #{e.backtrace.join("\n")}"
+      warn "ERROR: #{e} --- #{e.backtrace.join("\n")}"
+      raise e
+    end
+  end
+end