active_durable 0.5.0

data/lib/active_durable/flow.rb

@@ -0,0 +1,277 @@
+# frozen_string_literal: true
+
+module ActiveDurable
+  # The object a recipe receives. Every call that touches the outside world goes through it, so it can
+  # be checkpointed in the notebook and skipped on replay.
+  #
+  #   Durable.define :checkout do |flow, order_id:|
+  #     flow.transaction :reserve_stock, undo: -> { ... } do ... end
+  #     flow.step :charge, undo: ->(charge, ticket) { ... } do |ticket| ... end
+  #     flow.pivot(:ship) { |ticket| ... }
+  #     flow.step(:email) { ... }
+  #   end
+  class Flow
+    UndoEntry = Struct.new(:name, :kind, :result, :undo)
+    STEP_OPTIONS = %i[retry undo_on_failure].freeze
+
+    attr_reader :undo_stack
+
+    def initialize(runner, compensating:)
+      @runner = runner
+      @notebook = runner.notebook
+      @execution = runner.execution
+      @compensating = compensating
+      @pivoted = false
+      @position = 0
+      @seen = {}
+      @undo_stack = []
+    end
+
+    def execution_id
+      @execution.id
+    end
+
+    # The ticket (idempotency key) a step receives. It is the same every time the step runs.
+    def ticket_for(name)
+      "#{execution_id}:#{name}"
+    end
+
+    def compensating?
+      @compensating
+    end
+
+    def compensating!
+      @compensating = true
+    end
+
+    def pivoted?
+      @pivoted
+    end
+
+    # A step that talks to the outside world. It runs at most until it is recorded; pass the ticket to the
+    # service as its idempotency key so a repeat after a crash is recognised.
+    def step(name, undo: nil, **options, &block)
+      run_step(name, "step", undo, options, block)
+    end
+
+    # A step that only touches your own database. It runs in the same transaction that records it,
+    # so it happens exactly once.
+    def transaction(name, undo: nil, **options, &block)
+      run_step(name, "transaction", undo, options, block)
+    end
+
+    # The point of no return. Before it, failures are compensated; after it, steps are retried.
+    def pivot(name, **options, &block)
+      run_step(name, "pivot", nil, options, block)
+    end
+
+    # Rejects the saga for a business reason: no retries, straight to compensation.
+    def abort!(message)
+      raise Abort, message
+    end
+
+    # Waits without holding a worker: the wake-up time is written down and the execution is released.
+    def sleep(name, duration)
+      name, position = visit!(name, "sleep")
+      entry = @notebook[name]
+      if entry.nil?
+        raise StopForward, name if compensating?
+
+        wake_at = now + duration
+        @notebook.complete!(name, kind: "sleep", position: position,
+                                  result: { "wake_at" => wake_at.utc.iso8601(6) })
+        @runner.suspend!(wake_at, "sleeping")
+      end
+
+      wake_at = Time.iso8601(entry.result.fetch("wake_at"))
+      return nil if now >= wake_at
+      raise StopForward, name if compensating?
+
+      @runner.suspend!(wake_at, "sleeping")
+    end
+
+    # Waits for Durable.signal(execution_id, name, payload) and returns the payload.
+    def wait_for(name, timeout: nil)
+      name, position = visit!(name, "wait")
+      entry = @notebook[name]
+      return entry.result.deep_dup if entry&.completed?
+      raise StepFailed.new(name, entry.error&.fetch("message", nil)) if entry&.failed?
+      raise StopForward, name if compensating?
+
+      signal = SignalRecord.next_for(execution_id, name)
+      return consume_signal(signal, name, position).deep_dup if signal
+
+      if entry.nil?
+        deadline = timeout && (now + timeout)
+        @notebook.wait!(name, kind: "wait", position: position, wake_at: deadline)
+        @runner.suspend!(deadline, "waiting")
+      end
+
+      if entry.wake_at && now >= entry.wake_at
+        error = WaitTimeout.new("no :#{name} signal arrived before #{entry.wake_at.utc.iso8601}")
+        @notebook.fail!(name, kind: "wait", position: position, attempts: 1,
+                              error: ActiveDurable.dump_error(error, step: name))
+        raise StepFailed.new(name, error)
+      end
+
+      @runner.suspend!(entry.wake_at, "waiting")
+    end
+
+    # Called when the recipe returns: every step the notebook knows about must have been reached.
+    def finish!
+      return if compensating?
+
+      missing = @notebook.forward_entries.reject { |entry| @seen.key?(entry.name) }
+      return if missing.empty?
+
+      names = missing.map { |entry| ":#{entry.name}" }.join(", ")
+      raise RecipeChanged, "execution #{execution_id} recorded #{names}, but the recipe no longer reaches " \
+                           "#{missing.size == 1 ? "it" : "them"}. #{RECIPE_CHANGED_HINT}"
+    end
+
+    RECIPE_CHANGED_HINT = "Either the recipe changed while this execution was in flight, or code outside a " \
+                          "step read data that changed between runs. Keep reads that decide the path inside " \
+                          "steps, or define a new recipe version."
+
+    private
+
+    def now
+      ActiveDurable.now
+    end
+
+    def config
+      ActiveDurable.config
+    end
+
+    def run_step(name, kind, undo, options, block)
+      raise InvalidRecipe, "flow.#{kind} :#{name} needs a block" unless block
+
+      unknown = options.keys - STEP_OPTIONS
+      raise InvalidRecipe, "unknown option(s) for flow.#{kind}: #{unknown.join(", ")}" if unknown.any?
+
+      name, position = visit!(name, kind)
+      check_undo!(name, kind, undo, options)
+
+      entry = @notebook[name]
+      return remember(name, kind, entry.result, undo) if entry&.completed?
+
+      if entry&.failed?
+        remember_failure(name, kind, undo, options)
+        raise StepFailed.new(name, entry.error&.fetch("message", nil))
+      end
+      raise StopForward, name if compensating?
+
+      @runner.suspend!(entry.wake_at, "sleeping") if entry&.retrying? && entry.wake_at && entry.wake_at > now
+
+      result = execute(name, kind, position, entry, undo, options, block)
+      remember(name, kind, result, undo)
+    end
+
+    def execute(name, kind, position, entry, undo, options, block)
+      ticket = ticket_for(name)
+      ActiveDurable.crash_point(:before_step, name)
+      result = ActiveDurable.instrument("step", execution_id: execution_id, step: name, kind: kind) do
+        if kind == "transaction"
+          @notebook.transaction { record_result(name, kind, position, block.call(ticket)) }
+        else
+          record_result(name, kind, position, block.call(ticket))
+        end
+      end
+      ActiveDurable.crash_point(:after_record, name)
+      result
+    rescue NotSerializable, InvalidRecipe
+      raise
+    rescue StandardError => e
+      handle_failure(name, kind, position, entry, undo, options, e)
+    end
+
+    def record_result(name, kind, position, value)
+      result = Serializer.normalize(value, "the result of :#{name}")
+      ActiveDurable.crash_point(:after_call, name)
+      @notebook.complete!(name, kind: kind, position: position, result: result)
+      result
+    end
+
+    def handle_failure(name, kind, position, entry, undo, options, error)
+      attempts = (entry&.attempts || 0) + 1
+      default = @pivoted ? config.after_pivot_attempts : config.step_attempts
+      policy = RetryPolicy.build(options[:retry], default_attempts: default)
+      dumped = ActiveDurable.dump_error(error, step: name)
+
+      if error.is_a?(Abort) || attempts >= policy.attempts
+        @notebook.fail!(name, kind: kind, position: position, attempts: attempts, error: dumped)
+        remember_failure(name, kind, undo, options)
+        raise StepFailed.new(name, error)
+      end
+
+      wake_at = now + policy.delay(attempts)
+      @notebook.retry!(name, kind: kind, position: position, attempts: attempts, wake_at: wake_at, error: dumped)
+      @runner.suspend!(wake_at, "sleeping")
+    end
+
+    def remember(name, kind, result, undo)
+      @undo_stack << UndoEntry.new(name, kind, result, undo) if undo
+      @pivoted = true if kind == "pivot"
+      result.deep_dup
+    end
+
+    # A failed step whose outcome may be unknown (a timeout after the charge went through) can ask for
+    # its own undo too. It receives nil as the result, plus the ticket to look the outcome up.
+    def remember_failure(name, kind, undo, options)
+      @undo_stack << UndoEntry.new(name, kind, nil, undo) if undo && options[:undo_on_failure]
+    end
+
+    def check_undo!(name, kind, undo, options)
+      if options[:undo_on_failure]
+        raise InvalidRecipe, "undo_on_failure: on :#{name} needs an undo:" if undo.nil?
+        if kind == "transaction"
+          raise InvalidRecipe, "undo_on_failure: makes no sense on flow.transaction :#{name}: a failed " \
+                               "transaction was rolled back"
+        end
+      end
+      return if undo.nil?
+      raise InvalidRecipe, "undo: for :#{name} must respond to #call" unless undo.respond_to?(:call)
+      return unless @pivoted
+
+      raise InvalidRecipe, ":#{name} comes after the point of no return (flow.pivot), so it cannot declare " \
+                           "undo:. Steps after the pivot are retried, never undone."
+    end
+
+    def visit!(name, kind)
+      name = name.to_s
+      raise InvalidRecipe, "step names cannot be blank" if name.empty?
+      raise InvalidRecipe, "step names cannot end in ':undo' (#{name})" if name.end_with?(":undo")
+      if @seen.key?(name)
+        raise DuplicateStepName, "the recipe uses the step name :#{name} twice. Each step needs its own name: " \
+                                 "it is the step's key in the notebook."
+      end
+
+      @position += 1
+      @seen[name] = @position
+      check_recipe!(name, kind, @position)
+      [name, @position]
+    end
+
+    def check_recipe!(name, kind, position)
+      recorded = @notebook.at_position(position)
+      entry = @notebook[name]
+      return if recorded.nil? && entry.nil?
+      return if recorded && recorded.name == name && recorded.kind == kind
+
+      expected = recorded ? ":#{recorded.name} (#{recorded.kind})" : "nothing"
+      raise RecipeChanged, "execution #{execution_id} recorded #{expected} at position #{position}, but the " \
+                           "recipe reached :#{name} (#{kind}). #{RECIPE_CHANGED_HINT}"
+    end
+
+    def consume_signal(signal, name, position)
+      payload = signal.payload
+      @notebook.transaction do
+        taken = SignalRecord.where(id: signal.id, consumed_at: nil).update_all(consumed_at: now)
+        raise LeaseLost, "signal #{signal.id} for #{execution_id} was already consumed" if taken.zero?
+
+        @notebook.complete!(name, kind: "wait", position: position, result: payload)
+      end
+      payload
+    end
+  end
+end

data/lib/active_durable/flow_parallel.rb

@@ -0,0 +1,200 @@
+# frozen_string_literal: true
+
+module ActiveDurable
+  class Flow # rubocop:disable Style/Documentation -- documented in flow.rb
+    # flow.parallel: several steps at the same time, each one checkpointed on its own.
+    #
+    # Branches run in threads (at most config.parallel_concurrency at once), which suits steps that wait on the
+    # network. Each branch is a notebook entry named "<parallel>/<branch>", with its own ticket and retries. After
+    # a crash only the unfinished branches run again. If a branch runs out of attempts, the saga compensates the
+    # branches that completed (last finished, first undone) and every step before the parallel block.
+    module Parallel
+      def parallel(name, &block)
+        raise InvalidRecipe, "flow.parallel :#{name} needs a block" unless block
+
+        name, position = visit!(name, "parallel")
+        group = ParallelGroup.new(name)
+        block.call(group)
+        branches = group.branches
+        raise InvalidRecipe, "flow.parallel :#{name} declared no branches" if branches.empty?
+
+        prepare_branches!(name, branches)
+        entry = @notebook[name]
+        return finish_recorded_parallel(name, entry, branches) if entry&.completed? || entry&.failed?
+        raise StopForward, name if compensating?
+
+        run_parallel(name, position, branches)
+      end
+
+      private
+
+      def prepare_branches!(name, branches)
+        branches.each do |branch|
+          check_undo!(branch.full_name, branch.kind, branch.undo, branch.options)
+          raise DuplicateStepName, "the step name :#{branch.full_name} is used twice" if @seen.key?(branch.full_name)
+
+          @seen[branch.full_name] = nil
+        end
+
+        recorded = @notebook.branches_of(name).map(&:name)
+        missing = recorded - branches.map(&:full_name)
+        return if missing.empty?
+
+        raise RecipeChanged, "flow.parallel :#{name} recorded the branches #{missing.join(", ")}, which the recipe " \
+                             "no longer declares. #{RECIPE_CHANGED_HINT}"
+      end
+
+      def finish_recorded_parallel(name, entry, branches)
+        if entry.completed?
+          unexpected = branches.map(&:name) - entry.result.keys
+          if unexpected.any?
+            raise RecipeChanged, "flow.parallel :#{name} already completed without the branches " \
+                                 "#{unexpected.join(", ")}. #{RECIPE_CHANGED_HINT}"
+          end
+
+          remember_branches(name, branches)
+          return entry.result.deep_dup
+        end
+
+        remember_branches(name, branches, include_failed: true)
+        raise StepFailed.new(name, entry.error&.fetch("message", nil))
+      end
+
+      def run_parallel(name, position, branches)
+        outcomes = branch_outcomes(branches)
+        failed = outcomes.select { |_, (state, _)| state == :failed }
+        if failed.any?
+          full_name, (_, error) = failed.first
+          remember_branches(name, branches, include_failed: true)
+          @notebook.fail!(name, kind: "parallel", position: position, attempts: 1,
+                                error: ActiveDurable.dump_error(error, step: full_name))
+          raise StepFailed.new(full_name, error)
+        end
+
+        wakes = outcomes.values.filter_map { |state, value| value if state == :retry }
+        @runner.suspend!(wakes.min, "sleeping") if wakes.any?
+
+        results = branches.to_h { |branch| [branch.name, outcomes.fetch(branch.full_name).last] }
+        @notebook.complete!(name, kind: "parallel", position: position, result: results)
+        remember_branches(name, branches)
+        results.deep_dup
+      end
+
+      # { full_name => [:completed, result] | [:retry, wake_at] | [:failed, error] }
+      def branch_outcomes(branches)
+        outcomes = {}
+        runnable = []
+        branches.each do |branch|
+          entry = @notebook[branch.full_name]
+          if entry&.completed? then outcomes[branch.full_name] = [:completed, entry.result]
+          elsif entry&.failed? then outcomes[branch.full_name] = [:failed, StepFailed.new(branch.full_name)]
+          elsif entry&.retrying? && entry.wake_at && entry.wake_at > now
+            outcomes[branch.full_name] = [:retry, entry.wake_at]
+          else
+            runnable << branch
+          end
+        end
+        runnable.each_slice(config.parallel_concurrency) { |slice| outcomes.merge!(run_threads(slice)) }
+        outcomes
+      end
+
+      def run_threads(slice)
+        wrappers = ActiveDurable.branch_wrappers.map { |wrapper| [wrapper, wrapper.capture] }
+        threads = slice.map do |branch|
+          Thread.new do
+            Thread.current.report_on_exception = false
+            in_branch_context(wrappers) { [branch.full_name, execute_branch(branch)] }
+          end
+        end
+        values = join_all(threads)
+        crash = values.find { |value| value.is_a?(Exception) }
+        raise crash if crash
+
+        values.to_h
+      end
+
+      # Waits for every thread, even if one of them blew up, so nothing keeps writing behind our back.
+      def join_all(threads)
+        join = lambda do
+          threads.map do |thread|
+            thread.value
+          rescue Exception => e # rubocop:disable Lint/RescueException
+            e
+          end
+        end
+        if defined?(ActiveSupport::Dependencies) && ActiveSupport::Dependencies.respond_to?(:interlock)
+          ActiveSupport::Dependencies.interlock.permit_concurrent_loads(&join)
+        else
+          join.call
+        end
+      end
+
+      def in_branch_context(wrappers, &block)
+        run = lambda do
+          Record.connection_pool.with_connection do
+            wrappers.reverse.inject(block) { |inner, (wrapper, captured)| -> { wrapper.wrap(captured, &inner) } }.call
+          end
+        end
+        if defined?(Rails) && Rails.respond_to?(:application) && Rails.application
+          Rails.application.executor.wrap(&run)
+        else
+          run.call
+        end
+      end
+
+      def execute_branch(branch)
+        entry = @notebook[branch.full_name]
+        ticket = ticket_for(branch.full_name)
+        ActiveDurable.crash_point(:before_step, branch.full_name)
+        result = ActiveDurable.instrument("step", execution_id: execution_id, step: branch.full_name,
+                                                  kind: branch.kind) do
+          if branch.kind == "transaction"
+            @notebook.transaction { record_result(branch.full_name, branch.kind, nil, branch.block.call(ticket)) }
+          else
+            record_result(branch.full_name, branch.kind, nil, branch.block.call(ticket))
+          end
+        end
+        ActiveDurable.crash_point(:after_record, branch.full_name)
+        [:completed, result]
+      rescue NotSerializable, InvalidRecipe
+        raise
+      rescue StandardError => e
+        branch_failure(branch, entry, e)
+      end
+
+      def branch_failure(branch, entry, error)
+        attempts = (entry&.attempts || 0) + 1
+        default = @pivoted ? config.after_pivot_attempts : config.step_attempts
+        policy = RetryPolicy.build(branch.options[:retry], default_attempts: default)
+        dumped = ActiveDurable.dump_error(error, step: branch.full_name)
+        if error.is_a?(Abort) || attempts >= policy.attempts
+          @notebook.fail!(branch.full_name, kind: branch.kind, position: nil, attempts: attempts, error: dumped)
+          return [:failed, error]
+        end
+
+        wake_at = now + policy.delay(attempts)
+        @notebook.retry!(branch.full_name, kind: branch.kind, position: nil, attempts: attempts, wake_at: wake_at,
+                                           error: dumped)
+        [:retry, wake_at]
+      end
+
+      # Registers the undos of the branches that completed, in the order they finished. With include_failed,
+      # also those of branches that failed or were still retrying and asked for undo_on_failure.
+      def remember_branches(name, branches, include_failed: false)
+        by_name = branches.index_by(&:full_name)
+        @notebook.branches_of(name).each do |entry|
+          branch = by_name[entry.name]
+          next unless branch&.undo
+
+          if entry.completed?
+            @undo_stack << UndoEntry.new(entry.name, branch.kind, entry.result, branch.undo)
+          elsif include_failed && branch.options[:undo_on_failure] && (entry.failed? || entry.retrying?)
+            @undo_stack << UndoEntry.new(entry.name, branch.kind, nil, branch.undo)
+          end
+        end
+      end
+    end
+
+    include Parallel
+  end
+end

data/lib/active_durable/lease.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module ActiveDurable
+  # Who is allowed to run an execution right now.
+  #
+  # A worker claims an execution with a conditional UPDATE that only succeeds when nobody holds a live
+  # lease. The claim writes a fresh random token. Every later write (notebook entries, status changes)
+  # is conditional on that token still being there, so a worker whose lease expired and was taken over
+  # cannot write anything else: its writes raise LeaseLost and it stops.
+  class Lease
+    attr_reader :execution_id, :token
+
+    def self.claim(execution_id)
+      now = ActiveDurable.now
+      token = SecureRandom.uuid
+      claimed = Execution.where(id: execution_id, status: Execution::ACTIVE)
+                         .where("locked_until IS NULL OR locked_until < ?", now)
+                         .update_all(lease_token: token, locked_until: now + duration, status: "running",
+                                     updated_at: now)
+      claimed == 1 ? new(execution_id, token) : nil
+    end
+
+    def self.duration
+      ActiveDurable.config.lease_duration.to_f
+    end
+
+    def initialize(execution_id, token)
+      @execution_id = execution_id
+      @token = token
+    end
+
+    # Extends the lease (and optionally changes other columns). Raises LeaseLost if it is no longer ours.
+    def renew!(attributes = {})
+      update!(attributes.merge(locked_until: ActiveDurable.now + self.class.duration))
+    end
+
+    # Gives the execution back, usually with a new status.
+    def release!(attributes = {})
+      update!(attributes.merge(locked_until: nil))
+    end
+
+    private
+
+    def update!(attributes)
+      rows = Execution.where(id: execution_id, lease_token: token)
+                      .update_all(attributes.merge(updated_at: ActiveDurable.now))
+      raise LeaseLost, "lost the lease on #{execution_id}: another worker took it over" if rows.zero?
+    end
+  end
+end

data/lib/active_durable/notebook.rb

@@ -0,0 +1,100 @@
+# frozen_string_literal: true
+
+module ActiveDurable
+  # The notebook of one execution: every step that ran, what it returned and its state.
+  # Loaded once per run; every write is fenced by the lease.
+  class Notebook
+    def initialize(execution, lease)
+      @execution = execution
+      @lease = lease
+      @mutex = Mutex.new # parallel branches write from several threads
+      @entries = Step.where(execution_id: execution.id).order(:id).index_by(&:name)
+      @by_position = {}
+      @entries.each_value { |entry| @by_position[entry.position] = entry if entry.position }
+    end
+
+    def [](name)
+      @mutex.synchronize { @entries[name] }
+    end
+
+    # Completed branch entries of a flow.parallel, in the order they finished.
+    def branches_of(parallel_name)
+      prefix = "#{parallel_name}/"
+      @mutex.synchronize { @entries.values.select { |entry| entry.name.start_with?(prefix) && !entry.undo? } }
+            .sort_by { |entry| [entry.updated_at, entry.id] }
+    end
+
+    def at_position(position)
+      @by_position[position]
+    end
+
+    def forward_entries
+      @mutex.synchronize { @entries.values.reject(&:undo?) }
+    end
+
+    def complete!(name, kind:, position:, result:)
+      write!(name, kind: kind, position: position, status: "completed", result: result, error: nil, wake_at: nil)
+    end
+
+    def retry!(name, kind:, position:, attempts:, wake_at:, error:)
+      write!(name, kind: kind, position: position, status: "retrying", attempts: attempts, wake_at: wake_at,
+                   error: error)
+    end
+
+    def fail!(name, kind:, position:, attempts:, error:)
+      write!(name, kind: kind, position: position, status: "failed", attempts: attempts, error: error,
+                   wake_at: nil)
+    end
+
+    def wait!(name, kind:, position:, wake_at:)
+      write!(name, kind: kind, position: position, status: "waiting", wake_at: wake_at)
+    end
+
+    # Runs the block in a database transaction and remembers the notebook writes made inside it only once it
+    # commits. flow.transaction steps and their undos use it: a step that rolled back must not look completed.
+    def transaction(&)
+      pending = []
+      Thread.current[pending_key] = pending
+      result = Record.transaction(&)
+      pending.each { |name, entry| remember(name, entry) }
+      result
+    ensure
+      Thread.current[pending_key] = nil
+    end
+
+    private
+
+    # The database write happens outside the mutex, so a slow transaction in one branch never blocks
+    # another branch while it holds the lock (SQLite would deadlock). Existing entries are updated with
+    # update_all and read back, so the object other code holds never changes before the write commits.
+    def write!(name, **attributes)
+      now = ActiveDurable.now
+      current = self[name]
+      entry = Record.transaction do
+        @lease.renew!
+        if current
+          Step.where(id: current.id).update_all(attributes.merge(updated_at: now))
+          Step.find(current.id)
+        else
+          Step.create!(execution_id: @execution.id, name: name, created_at: now, updated_at: now, **attributes)
+        end
+      end
+      pending = Thread.current[pending_key]
+      pending ? pending << [name, entry] : remember(name, entry)
+      entry
+    rescue ActiveRecord::RecordNotUnique
+      raise LeaseLost, "another worker already wrote :#{name} for #{@execution.id}"
+    end
+
+    def remember(name, entry)
+      @mutex.synchronize do
+        @entries[name] = entry
+        @by_position[entry.position] = entry if entry.position
+      end
+    end
+
+    def pending_key
+      @pending_key ||= :"active_durable_notebook_#{object_id}"
+    end
+  end
+end