durable_flow 0.1.0

data/lib/durable_flow/models/workflow_log.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module DurableFlow
+  class WorkflowLog < ApplicationRecord
+    include Live::Broadcastable
+
+    self.table_name = "durable_flow_workflow_logs"
+
+    belongs_to :workflow_run, class_name: "DurableFlow::WorkflowRun"
+    belongs_to :workflow_step, class_name: "DurableFlow::WorkflowStep", optional: true
+
+    LEVELS = WorkflowLogger::LEVELS
+
+    validates :level, inclusion: { in: LEVELS }
+    validates :message, presence: true
+
+    scope :ordered, -> { order(:created_at, :id) }
+
+    def data_value
+      Serializer.load(data)
+    end
+
+    def live_snapshot
+      {
+        id: id,
+        workflow_run_id: workflow_run_id,
+        workflow_step_id: workflow_step_id,
+        level: level,
+        message: message,
+        data: data,
+        created_at: created_at,
+        updated_at: updated_at,
+      }
+    end
+  end
+end

data/lib/durable_flow/models/workflow_run.rb

@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+module DurableFlow
+  class WorkflowRun < ApplicationRecord
+    include Live::Broadcastable
+
+    self.table_name = "durable_flow_workflow_runs"
+
+    TERMINAL_STATUSES = %w[completed failed].freeze
+
+    has_many :workflow_steps, class_name: "DurableFlow::WorkflowStep", dependent: :delete_all
+    has_many :workflow_waits, class_name: "DurableFlow::WorkflowWait", dependent: :delete_all
+    has_many :workflow_logs, class_name: "DurableFlow::WorkflowLog", dependent: :delete_all
+
+    scope :active, -> { where.not(status: TERMINAL_STATUSES) }
+
+    def acquire_execution_lock!(owner:, ttl:)
+      now = Time.current
+      lock_expires_at = now + ttl
+
+      updated = self.class
+        .where(id: id)
+        .active
+        .where("execution_locked_by IS NULL OR execution_lock_expires_at <= ?", now)
+        .update_all(
+          execution_locked_by: owner,
+          execution_locked_at: now,
+          execution_lock_expires_at: lock_expires_at,
+          updated_at: now,
+        )
+
+      reload if updated == 1
+      updated == 1
+    end
+
+    def release_execution_lock!(owner:)
+      self.class
+        .where(id: id, execution_locked_by: owner)
+        .update_all(
+          execution_locked_by: nil,
+          execution_locked_at: nil,
+          execution_lock_expires_at: nil,
+          updated_at: Time.current,
+        )
+    end
+
+    def refresh_execution_lock!(owner:, ttl:)
+      now = Time.current
+
+      updated = self.class
+        .where(id: id, execution_locked_by: owner)
+        .update_all(
+          execution_lock_expires_at: now + ttl,
+          updated_at: now,
+        )
+
+      reload if updated == 1
+      updated == 1
+    end
+
+    def execution_locked?
+      execution_locked_by.present? && execution_lock_expires_at.present? && execution_lock_expires_at > Time.current
+    end
+
+    def completed?
+      status == "completed"
+    end
+
+    def failed?
+      status == "failed"
+    end
+
+    def terminal?
+      completed? || failed?
+    end
+
+    def timeline
+      WorkflowTimeline.new(self)
+    end
+
+    def live_snapshot
+      {
+        id: id,
+        run_id: run_id,
+        job_id: job_id,
+        workflow_class: workflow_class,
+        status: status,
+        queue_name: queue_name,
+        priority: priority,
+        started_at: started_at,
+        interrupted_at: interrupted_at,
+        completed_at: completed_at,
+        failed_at: failed_at,
+        execution_locked: execution_locked?,
+        execution_lock_expires_at: execution_lock_expires_at,
+        created_at: created_at,
+        updated_at: updated_at,
+      }
+    end
+  end
+end

data/lib/durable_flow/models/workflow_step.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module DurableFlow
+  class WorkflowStep < ApplicationRecord
+    include Live::Broadcastable
+
+    self.table_name = "durable_flow_workflow_steps"
+
+    belongs_to :workflow_run, class_name: "DurableFlow::WorkflowRun"
+    has_many :workflow_logs, class_name: "DurableFlow::WorkflowLog", dependent: :nullify
+
+    def succeeded?
+      status == "succeeded"
+    end
+
+    def result_value
+      Serializer.load(result)
+    end
+
+    def complete!(value)
+      update!(
+        status: "succeeded",
+        result: Serializer.dump(value),
+        completed_at: Time.current,
+      )
+    end
+
+    def metadata_hash
+      metadata.presence || {}
+    end
+
+    def live_snapshot
+      {
+        id: id,
+        workflow_run_id: workflow_run_id,
+        name: name,
+        status: status,
+        attempts: attempts,
+        result: result,
+        metadata: metadata,
+        started_at: started_at,
+        completed_at: completed_at,
+        created_at: created_at,
+        updated_at: updated_at,
+      }
+    end
+  end
+end

data/lib/durable_flow/models/workflow_wait.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module DurableFlow
+  class WorkflowWait < ApplicationRecord
+    include Live::Broadcastable
+
+    self.table_name = "durable_flow_workflow_waits"
+
+    belongs_to :workflow_run, class_name: "DurableFlow::WorkflowRun"
+    belongs_to :workflow_step, class_name: "DurableFlow::WorkflowStep"
+    belongs_to :workflow_event, class_name: "DurableFlow::WorkflowEvent", optional: true
+
+    scope :pending, -> { where(status: "pending") }
+
+    def match_value
+      Serializer.load(self.match)
+    end
+
+    def matches_event?(event)
+      event.name == event_name && event.matches_payload?(match_value)
+    end
+
+    def live_snapshot
+      {
+        id: id,
+        workflow_run_id: workflow_run_id,
+        workflow_step_id: workflow_step_id,
+        workflow_event_id: workflow_event_id,
+        event_name: event_name,
+        status: status,
+        match: self.match,
+        timeout_at: timeout_at,
+        created_at: created_at,
+        updated_at: updated_at,
+      }
+    end
+  end
+end

data/lib/durable_flow/railtie.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module DurableFlow
+  class Railtie < ::Rails::Railtie
+    initializer "durable_flow.subscribe_to_events", after: :load_config_initializers do
+      ActiveSupport.on_load(:active_record) do
+        DurableFlow.subscribe_to_rails_events!
+      end
+    end
+  end
+end

data/lib/durable_flow/schema.rb

@@ -0,0 +1,144 @@
+# frozen_string_literal: true
+
+module DurableFlow
+  module Schema
+    module_function
+
+    def define(connection = ActiveRecord::Base.connection)
+      create_workflow_runs(connection)
+      create_workflow_steps(connection)
+      create_workflow_events(connection)
+      create_workflow_waits(connection)
+      create_workflow_logs(connection)
+    end
+
+    def create_workflow_runs(connection)
+      if connection.data_source_exists?(:durable_flow_workflow_runs)
+        ensure_workflow_run_lock_columns(connection)
+        return
+      end
+
+      connection.create_table :durable_flow_workflow_runs do |t|
+        t.string :run_id, null: false
+        t.string :job_id, null: false
+        t.string :workflow_class, null: false
+        t.string :status, null: false, default: "enqueued"
+        t.string :queue_name
+        t.integer :priority
+        t.json :arguments
+        t.json :serialized_job
+        t.json :last_error
+        t.datetime :started_at
+        t.datetime :interrupted_at
+        t.datetime :completed_at
+        t.datetime :failed_at
+        t.string :execution_locked_by
+        t.datetime :execution_locked_at
+        t.datetime :execution_lock_expires_at
+        t.timestamps
+      end
+
+      connection.add_index :durable_flow_workflow_runs, :run_id, unique: true
+      connection.add_index :durable_flow_workflow_runs, [ :workflow_class, :status ], name: "idx_durable_flow_runs_on_class_status"
+      connection.add_index :durable_flow_workflow_runs, :execution_lock_expires_at, name: "idx_durable_flow_runs_on_lock_expiry"
+    end
+
+    def ensure_workflow_run_lock_columns(connection)
+      unless connection.column_exists?(:durable_flow_workflow_runs, :execution_locked_by)
+        connection.add_column :durable_flow_workflow_runs, :execution_locked_by, :string
+      end
+
+      unless connection.column_exists?(:durable_flow_workflow_runs, :execution_locked_at)
+        connection.add_column :durable_flow_workflow_runs, :execution_locked_at, :datetime
+      end
+
+      unless connection.column_exists?(:durable_flow_workflow_runs, :execution_lock_expires_at)
+        connection.add_column :durable_flow_workflow_runs, :execution_lock_expires_at, :datetime
+      end
+
+      unless connection.index_exists?(:durable_flow_workflow_runs, :execution_lock_expires_at, name: "idx_durable_flow_runs_on_lock_expiry")
+        connection.add_index :durable_flow_workflow_runs, :execution_lock_expires_at, name: "idx_durable_flow_runs_on_lock_expiry"
+      end
+    end
+
+    def create_workflow_steps(connection)
+      return if connection.data_source_exists?(:durable_flow_workflow_steps)
+
+      connection.create_table :durable_flow_workflow_steps do |t|
+        t.references :workflow_run, null: false, index: false
+        t.string :name, null: false
+        t.string :status, null: false, default: "pending"
+        t.integer :attempts, null: false, default: 0
+        t.json :result
+        t.json :metadata
+        t.datetime :started_at
+        t.datetime :completed_at
+        t.timestamps
+      end
+
+      connection.add_index :durable_flow_workflow_steps,
+        [ :workflow_run_id, :name ],
+        unique: true,
+        name: "idx_durable_flow_steps_on_run_and_name"
+    end
+
+    def create_workflow_events(connection)
+      return if connection.data_source_exists?(:durable_flow_workflow_events)
+
+      connection.create_table :durable_flow_workflow_events do |t|
+        t.string :name, null: false
+        t.json :payload
+        t.json :tags
+        t.json :context
+        t.json :source_location
+        t.datetime :occurred_at, null: false
+        t.timestamps
+      end
+
+      connection.add_index :durable_flow_workflow_events, [ :name, :occurred_at ], name: "idx_durable_flow_events_on_name_time"
+    end
+
+    def create_workflow_waits(connection)
+      return if connection.data_source_exists?(:durable_flow_workflow_waits)
+
+      connection.create_table :durable_flow_workflow_waits do |t|
+        t.references :workflow_run, null: false, index: false
+        t.references :workflow_step, null: false, index: false
+        t.references :workflow_event, index: false
+        t.string :event_name, null: false
+        t.string :status, null: false, default: "pending"
+        t.json :match
+        t.datetime :timeout_at
+        t.timestamps
+      end
+
+      connection.add_index :durable_flow_workflow_waits,
+        [ :workflow_run_id, :workflow_step_id ],
+        unique: true,
+        name: "idx_durable_flow_waits_on_run_and_step"
+      connection.add_index :durable_flow_workflow_waits,
+        [ :event_name, :status ],
+        name: "idx_durable_flow_waits_on_event_status"
+    end
+
+    def create_workflow_logs(connection)
+      return if connection.data_source_exists?(:durable_flow_workflow_logs)
+
+      connection.create_table :durable_flow_workflow_logs do |t|
+        t.references :workflow_run, null: false, index: false
+        t.references :workflow_step, index: false
+        t.string :level, null: false
+        t.string :message, null: false
+        t.json :data
+        t.timestamps
+      end
+
+      connection.add_index :durable_flow_workflow_logs,
+        [ :workflow_run_id, :created_at ],
+        name: "idx_durable_flow_logs_on_run_time"
+      connection.add_index :durable_flow_workflow_logs,
+        [ :workflow_step_id, :created_at ],
+        name: "idx_durable_flow_logs_on_step_time"
+    end
+  end
+end

data/lib/durable_flow/serializer.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module DurableFlow
+  module Serializer
+    module_function
+
+    def dump(value)
+      ActiveJob::Arguments.serialize([ value ]).first
+    end
+
+    def load(value)
+      ActiveJob::Arguments.deserialize([ value ]).first
+    end
+  end
+end

data/lib/durable_flow/step_proxy.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module DurableFlow
+  class StepProxy
+    def initialize(workflow)
+      @workflow = workflow
+    end
+
+    def sleep(name, duration = nil, **options)
+      @workflow.sleep_step(name, duration, until_time: options[:until] || options[:until_time])
+    end
+
+    def wait_for_event(name, event: nil, timeout: nil, match: {})
+      @workflow.wait_for_event_step(name, event_name: event || name, timeout: timeout, match: match)
+    end
+
+    def wait_for_workflow(name, workflow_or_run_id, timeout: nil)
+      run_id = workflow_or_run_id.respond_to?(:job_id) ? workflow_or_run_id.job_id : workflow_or_run_id.to_s
+      wait_for_event(name, event: DurableFlow::WORKFLOW_COMPLETED_EVENT, timeout: timeout, match: { run_id: run_id })
+    end
+  end
+end

data/lib/durable_flow/test_helper.rb

@@ -0,0 +1,217 @@
+# frozen_string_literal: true
+
+require "active_job/test_helper"
+require "active_support/concern"
+require "active_support/testing/time_helpers"
+require "durable_flow"
+require "time"
+
+module DurableFlow
+  module TestHelper
+    extend ActiveSupport::Concern
+
+    included do
+      include ActiveJob::TestHelper
+      include ActiveSupport::Testing::TimeHelpers
+    end
+
+    def clear_durable_flow!(clear_jobs: true, reset_live: true)
+      DurableFlow.reset_live_broadcasters! if reset_live
+      send(:clear_enqueued_jobs) if clear_jobs && respond_to?(:clear_enqueued_jobs, true)
+      send(:clear_performed_jobs) if clear_jobs && respond_to?(:clear_performed_jobs, true)
+
+      WorkflowWait.delete_all
+      WorkflowEvent.delete_all
+      WorkflowLog.delete_all
+      WorkflowStep.delete_all
+      WorkflowRun.delete_all
+    end
+
+    def durable_flow_run(run_id)
+      WorkflowRun.find_by!(run_id: run_id.to_s)
+    end
+
+    def durable_flow_run_for(workflow_class)
+      WorkflowRun.where(workflow_class: workflow_class_name(workflow_class)).order(:created_at, :id).last ||
+        flunk("Expected a DurableFlow run for #{workflow_class_name(workflow_class)}")
+    end
+
+    def durable_flow_timeline_for(workflow_or_run)
+      durable_flow_resolve_run(workflow_or_run).timeline
+    end
+
+    def durable_flow_step(workflow_or_run, name)
+      run = durable_flow_resolve_run(workflow_or_run)
+      run.workflow_steps.find_by!(name: name.to_s)
+    end
+
+    def perform_durable_flow_jobs(**options, &block)
+      raise "Include ActiveJob::TestHelper to perform DurableFlow jobs" unless respond_to?(:perform_enqueued_jobs)
+
+      perform_enqueued_jobs(**options, &block)
+    end
+
+    def notify_workflow_event(name, **payload)
+      payload.empty? ? DurableFlow.notify(name) : DurableFlow.notify(name, payload)
+    end
+
+    def resume_workflows_for(name, **payload)
+      notify_workflow_event(name, **payload)
+      perform_durable_flow_jobs(at: Time.current)
+    end
+
+    def assert_workflow_completed(workflow_or_run)
+      assert_workflow_status(workflow_or_run, "completed")
+    end
+
+    def assert_workflow_failed(workflow_or_run, error: nil)
+      run = assert_workflow_status(workflow_or_run, "failed")
+
+      if error
+        expected_class = error.respond_to?(:name) ? error.name : error.to_s
+        assert_equal expected_class, run.last_error&.fetch("class", nil)
+      end
+
+      run
+    end
+
+    def assert_workflow_sleeping(workflow_or_run, step: nil)
+      run = assert_workflow_status(workflow_or_run, "sleeping")
+      sleep_step = step ? durable_flow_step(run, step) : run.workflow_steps.find_by!(status: "sleeping")
+
+      assert_equal "sleeping", sleep_step.status
+      assert sleep_step.metadata_hash["wake_at"].present?, "Expected sleep step #{sleep_step.name.inspect} to store wake_at metadata"
+      sleep_step
+    end
+
+    def assert_workflow_waiting_for(workflow_or_run, event_name, match: nil)
+      run = assert_workflow_status(workflow_or_run, "waiting")
+      wait = run.workflow_waits.find_by!(event_name: event_name.to_s)
+
+      assert_equal "pending", wait.status
+      assert_equal match, wait.match_value if match
+      wait
+    end
+
+    def assert_step_succeeded(workflow_or_run, name)
+      step = durable_flow_step(workflow_or_run, name)
+
+      assert_equal "succeeded", step.status
+      step
+    end
+
+    def assert_step_result(workflow_or_run, name, expected)
+      step = assert_step_succeeded(workflow_or_run, name)
+
+      assert_equal expected, step.result_value
+      step
+    end
+
+    def assert_step_attempts(workflow_or_run, name, expected)
+      step = durable_flow_step(workflow_or_run, name)
+
+      assert_equal expected, step.attempts
+      step
+    end
+
+    def assert_workflow_log(workflow_or_run, level: nil, message: nil, data: nil)
+      run = durable_flow_resolve_run(workflow_or_run)
+      assert_log_in(run.workflow_logs.ordered, level: level, message: message, data: data)
+    end
+
+    def assert_step_log(workflow_or_run, step_name, level: nil, message: nil, data: nil)
+      step = durable_flow_step(workflow_or_run, step_name)
+      assert_log_in(step.workflow_logs.ordered, level: level, message: message, data: data)
+    end
+
+    def travel_to_next_workflow_wake(workflow_or_run = nil)
+      wake_at = next_workflow_wake_at(workflow_or_run)
+
+      assert wake_at, "Expected a sleeping DurableFlow step with wake_at metadata"
+      travel_to wake_at
+      wake_at
+    end
+
+    def next_workflow_wake_at(workflow_or_run = nil)
+      scope = WorkflowStep.where(status: "sleeping")
+      scope = scope.where(workflow_run: durable_flow_resolve_run(workflow_or_run)) if workflow_or_run
+
+      scope.filter_map { |step| parse_workflow_wake_at(step.metadata_hash["wake_at"]) }.min
+    end
+
+    def capture_durable_flow_changes
+      changes = []
+      subscriber = DurableFlow.on_change { |change| changes << change }
+
+      yield changes
+      changes
+    ensure
+      DurableFlow.unsubscribe_from_changes(subscriber) if subscriber
+    end
+
+    def assert_durable_flow_change(changes, type, **payload)
+      change = changes.find do |candidate|
+        candidate.type == type && payload.all? { |key, value| durable_flow_change_value(candidate, key) == value }
+      end
+
+      assert change, "Expected DurableFlow change #{type.inspect} with #{payload.inspect}, got #{changes.map(&:payload).inspect}"
+      change
+    end
+
+    private
+      def assert_workflow_status(workflow_or_run, status)
+        run = durable_flow_resolve_run(workflow_or_run).reload
+
+        assert_equal status, run.status
+        run
+      end
+
+      def assert_log_in(scope, level:, message:, data:)
+        logs = scope.to_a
+        log = logs.find do |candidate|
+          (level.nil? || candidate.level == level.to_s) &&
+            (message.nil? || durable_flow_message_matches?(candidate.message, message)) &&
+            (data.nil? || candidate.data_value == data)
+        end
+
+        assert log, "Expected workflow log with #{ { level: level, message: message, data: data }.compact.inspect }, got #{logs.map { |entry| [ entry.level, entry.message, entry.data_value ] }.inspect}"
+        log
+      end
+
+      def durable_flow_resolve_run(workflow_or_run)
+        case workflow_or_run
+        when WorkflowRun
+          workflow_or_run
+        when String
+          durable_flow_run(workflow_or_run)
+        else
+          durable_flow_run_for(workflow_or_run)
+        end
+      end
+
+      def durable_flow_message_matches?(actual, expected)
+        expected.is_a?(Regexp) ? expected.match?(actual) : actual == expected.to_s
+      end
+
+      def durable_flow_change_value(change, key)
+        if change.snapshot.key?(key)
+          change.snapshot[key]
+        elsif change.payload.key?(key)
+          change.payload[key]
+        elsif change.respond_to?(key)
+          change.public_send(key)
+        end
+      end
+
+      def parse_workflow_wake_at(value)
+        return if value.blank?
+        return value if value.is_a?(Time)
+
+        Time.iso8601(value.to_s)
+      end
+
+      def workflow_class_name(workflow_class)
+        workflow_class.respond_to?(:name) ? workflow_class.name : workflow_class.to_s
+      end
+  end
+end

data/lib/durable_flow/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module DurableFlow
+  VERSION = "0.1.0"
+end