stepper_motor 0.1.0

data/lib/stepper_motor/journey.rb

@@ -0,0 +1,274 @@
+# frozen_string_literal: true
+
+# StepperMotor is a module for building multi-step flows where steps are sequential and only
+# ever progress forward. The building block of StepperMotor is StepperMotor::Journey
+module StepperMotor
+  # A Journey is the main building block of StepperMotor. You create a journey to guide a particular model
+  # ("hero") through a sequence of steps. Any of your model can be the hero and have multiple Journeys. To create
+  # your own Journey, subclass the `StepperMotor::Journey` class and define your steps. For example, a drip mail
+  # campaign can look like this:
+  #
+  #
+  #     class ResubscribeCampaign < StepperMotor::Journey
+  #       step do
+  #         ReinviteMailer.with(recipient: hero).deliver_later
+  #       end
+  #
+  #       step, wait: 3.days do
+  #         cancel! if hero.active?
+  #         ReinviteMailer.with(recipient: hero).deliver_later
+  #       end
+  #
+  #       step, wait: 3.days do
+  #         cancel! if hero.active?
+  #         ReinviteMailer.with(recipient: hero).deliver_later
+  #       end
+  #
+  #       step, wait: 3.days do
+  #         cancel! if hero.active?
+  #         hero.close_account!
+  #       end
+  #     end
+  #
+  # Creating a record for the Journey (just using `create!`) will instantly send your hero on their way:
+  #
+  #     ResubscribeCampaign.create!(hero: current_account)
+  #
+  # To stop the journey forcibly, delete it from your database - or call `cancel!` within any of the steps.
+  class Journey < ActiveRecord::Base
+    self.table_name = "stepper_motor_journeys"
+
+    # @return [Array] the step definitions defined so far
+    class_attribute :step_definitions, default: []
+
+    belongs_to :hero, polymorphic: true, optional: true
+
+    STATES = %w[ready performing canceled finished]
+    enum state: STATES.zip(STATES).to_h, _default: "ready"
+
+    # Allows querying for journeys for this specific hero. This uses a scope for convenience as the hero
+    # is referenced using it's global ID (same ID that ActiveJob uses for serialization)
+    scope :for_hero, ->(hero) {
+      where(hero: hero)
+    }
+
+    after_create do |journey|
+      journey.step_definitions.any? ? journey.set_next_step_and_enqueue(journey.step_definitions.first) : journey.finished!
+    end
+
+    # Defines a step in the journey.
+    # Steps are stacked top to bottom and get performed in sequence.
+    def self.step(name = nil, wait: nil, after: nil, &blk)
+      wait = if wait && after
+        raise StepConfigurationError, "Either wait: or after: can be specified, but not both"
+      elsif !wait && !after
+        0
+      elsif after
+        accumulated = step_definitions.map(&:wait).sum
+        after - accumulated
+      else
+        wait
+      end
+      raise StepConfigurationError, "wait: cannot be negative, but computed was #{wait}s" if wait.negative?
+      name ||= "step_%d" % (step_definitions.length + 1)
+      name = name.to_s
+
+      known_step_names = step_definitions.map(&:name)
+      raise StepConfigurationError, "Step named #{name.inspect} already defined" if known_step_names.include?(name)
+
+      # Create the step definition
+      step_definition = StepperMotor::Step.new(name: name, wait: wait, seq: step_definitions.length, &blk)
+
+      # As per Rails docs: you need to be aware when using class_attribute with mutable structures
+      # as Array or Hash. In such cases, you don’t want to do changes in place. Instead use setters.
+      # See https://apidock.com/rails/v7.1.3.2/Class/class_attribute
+      self.step_definitions = step_definitions + [step_definition]
+    end
+
+    # Returns the `Step` object for a named step. This is used when performing a step, but can also
+    # be useful in other contexts.
+    #
+    # @param by_step_name[Symbol,String] the name of the step to find
+    # @return [StepperMotor::Step?]
+    def self.lookup_step_definition(by_step_name)
+      step_definitions.find { |d| d.name.to_s == by_step_name.to_s }
+    end
+
+    # Alias for the class attribute, for brevity
+    #
+    # @see Journey.step_definitions
+    def step_definitions
+      self.class.step_definitions
+    end
+
+    # Alias for the class method, for brevity
+    #
+    # @see Journey.lookup_step_definition
+    def lookup_step_definition(by_step_name)
+      self.class.lookup_step_definition(by_step_name)
+    end
+
+    # Is a convenient way to end a hero's journey. Imagine you enter a step where you are inviting a user
+    # to rejoin the platform, and are just about to send them an email - but they have already joined. You
+    # can therefore cancel their journey. Canceling bails you out of the `step`-defined block and sets the journey record to the `canceled` state.
+    #
+    # Calling `cancel!` will abort the execution of the current step.
+    def cancel!
+      canceled!
+      throw :abort_step
+    end
+
+    # Inside a step it is possible to ask StepperMotor to retry to start the step at a later point in time. Maybe now is an inconvenient moment
+    # (are you about to send a push notification at 3AM perhaps?). The `wait:` parameter specifies how long to defer reattempting the step for.
+    # Reattempting will resume the step from the beginning, so the step should be idempotent.
+    #
+    # Calling `reattempt!` will abort the execution of the current step.
+    def reattempt!(wait: nil)
+      # The default `wait` is the one for the step definition
+      @reattempt_after = wait || @current_step_definition.wait || 0
+      throw :abort_step
+    end
+
+    # Performs the next step in the journey. Will check whether any other process has performed the step already
+    # and whether the record is unchanged, and will then lock it and set the state to 'performimg'.
+    #
+    # After setting the state, it will determine the next step to perform, and perform it. Depending on the outcome of
+    # the step another `PerformStepJob` may get enqueued. If the journey ends here, the journey record will set its state
+    # to 'finished'.
+    #
+    # @return [void]
+    def perform_next_step!
+      # Make sure we can't start running the same step of the same journey twice
+      next_step_name_before_locking = next_step_name
+      with_lock do
+        # Make sure no other worker has snatched this journey and made steps instead of us
+        return unless ready? && next_step_name == next_step_name_before_locking
+        performing!
+        after_locking_for_step(next_step_name)
+      end
+      current_step_name = next_step_name
+
+      if current_step_name
+        logger.debug { "preparing to perform step #{current_step_name}" }
+      else
+        logger.debug { "no next step - finishing journey" }
+        # If there is no step set - just terminate the journey
+        return finished! unless current_step_name
+      end
+
+      before_step_starts(current_step_name)
+
+      # Recover the step definition
+      @current_step_definition = lookup_step_definition(current_step_name)
+
+      unless @current_step_definition
+        logger.debug { "no definition for #{current_step_name} - finishing journey" }
+        return finished!
+      end
+
+      # Is we tried to run the step but it is not yet time to do so,
+      # enqueue a new job to perform it and stop
+      if next_step_to_be_performed_at > Time.current
+        logger.warn { "tried to perform #{current_step_name} prematurely" }
+        schedule!
+        return ready!
+      end
+
+      # Perform the actual step
+      increment!(:steps_entered)
+      logger.debug { "entering step #{current_step_name}" }
+
+      catch(:abort_step) do
+        instance_exec(&@current_step_definition)
+      end
+
+      # By the end of the step the Journey must either be untouched or saved
+      if changed?
+        raise StepperMotor::JourneyNotPersisted, <<~MSG
+          #{self} had its attributes changed but was not saved inside step #{current_step_name.inspect}
+          this means that the subsequent execution (which may be done asynchronously) is likely to see
+          a stale Journey, and will execute incorrectly. If you mutate the Journey inside
+          of a step, make sure to call `save!` or use methods that save in-place
+          (such as `increment!`).
+        MSG
+      end
+
+      increment!(:steps_completed)
+      logger.debug { "completed #{current_step_name} without exceptions" }
+
+      if canceled?
+        # The step aborted the journey, nothing to do
+        logger.info { "has been canceled inside #{current_step_name}" }
+      elsif @reattempt_after
+        # The step asked the actions to be attempted at a later time
+        logger.info { "will reattempt #{current_step_name} in #{@reattempt_after} seconds" }
+        update!(previous_step_name: current_step_name, next_step_name: current_step_name, next_step_to_be_performed_at: Time.current + @reattempt_after)
+        schedule!
+        ready!
+      elsif finished?
+        logger.info { "was marked finished inside the step" }
+        update!(previous_step_name: current_step_name, next_step_name: nil)
+      elsif (next_step_definition = step_definitions[@current_step_definition.seq + 1])
+        logger.info { "will continue to #{next_step_definition.name}" }
+        set_next_step_and_enqueue(next_step_definition)
+        ready!
+      else
+        # The hero's journey is complete
+        logger.info { "journey completed" }
+        finished!
+        update!(previous_step_name: current_step_name, next_step_name: nil)
+      end
+    ensure
+      # The instance variables must not be present if `perform_next_step!` gets called
+      # on this same object again. This will be the case if the steps are performed inline
+      # and not via background jobs (which reload the model)
+      @reattempt_after = nil
+      @current_step_definition = nil
+      after_step_completes(current_step_name) if current_step_name
+    end
+
+    # @return [ActiveSupport::Duration]
+    def time_remaining_until_final_step
+      current_step_seq = @current_step_definition&.seq || -1
+      subsequent_steps = step_definitions.select { |definition| definition.seq > current_step_seq }
+      seconds_remaining = subsequent_steps.map { |definition| definition.wait.to_f }.sum
+      seconds_remaining.seconds # Convert to ActiveSupport::Duration
+    end
+
+    def set_next_step_and_enqueue(next_step_definition)
+      wait = next_step_definition.wait
+      update!(previous_step_name: next_step_name, next_step_name: next_step_definition.name, next_step_to_be_performed_at: Time.current + wait)
+      schedule!
+    end
+
+    def logger
+      if (logger_from_parent = super)
+        tag = [self.class.to_s, to_param].join(":")
+        tag << " at " << @current_step_definition.name if @current_step_definition
+        logger_from_parent.tagged(tag)
+      else
+        # Furnish a "null logger"
+        ActiveSupport::Logger.new(nil)
+      end
+    end
+
+    def after_locking_for_step(step_name)
+    end
+
+    def before_step_starts(step_name)
+    end
+
+    def after_step_completes(step_name)
+    end
+
+    def schedule!
+      StepperMotor.scheduler.schedule(self)
+    end
+
+    def to_global_id
+      # This gets included into ActiveModel during Rails bootstrap,
+      # for now do this manually
+      GlobalID.create(self, app: "stepper-motor")
+    end
+  end
+end

data/lib/stepper_motor/perform_step_job.rb

@@ -0,0 +1,14 @@
+require "active_job"
+
+class StepperMotor::PerformStepJob < ActiveJob::Base
+  def perform(journey_gid)
+    # Pass the GlobalID instead of the record itself, so that we can rescue the non-existing record
+    # exception here as opposed to the job deserialization
+    journey = begin
+      GlobalID::Locator.locate(journey_gid)
+    rescue ActiveRecord::RecordNotFound
+      return # The journey has been canceled and destroyed previously or elsewhere
+    end
+    journey.perform_next_step!
+  end
+end

data/lib/stepper_motor/railtie.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module StepperMotor
+  UNINITIALISED_DATABASE_EXCEPTIONS = [
+    ActiveRecord::NoDatabaseError,
+    ActiveRecord::StatementInvalid,
+    ActiveRecord::ConnectionNotEstablished
+  ]
+
+  class Railtie < Rails::Railtie
+    rake_tasks do
+      task preload: :setup do
+        if defined?(Rails) && Rails.respond_to?(:application)
+          if Rails.application.config.eager_load
+            ActiveSupport.run_load_hooks(:before_eager_load, Rails.application)
+            Rails.application.config.eager_load_namespaces.each(&:eager_load!)
+          end
+        end
+      end
+    end
+
+    generators do
+      require "generators/install_generator"
+    end
+
+    # The `to_prepare` block which is executed once in production
+    # and before each request in development.
+    config.to_prepare do
+      if defined?(Rails) && Rails.respond_to?(:application)
+        _config_from_rails = Rails.application.config.try(:gouda)
+        # if config_from_rails
+        #  StepperMotor.config.scheduling_mode = config_from_rails[:scheduling_mode]
+        # end
+      else
+        # Set default configuration
+      end
+
+      begin
+        # Perform any tasks which touch the database here
+      rescue *StepperMotor::UNINITIALISED_DATABASE_EXCEPTIONS
+        # Do nothing. On a freshly checked-out Rails app, running even unrelated Rails tasks
+        # (such as asset compilation) - or, more importantly, initial db:create -
+        # will cause a NoDatabaseError, as this is a chicken-and-egg problem. That error
+        # is safe to ignore in this instance - we should let the outer task proceed,
+        # because if there is no database we should allow it to get created.
+      end
+    end
+  end
+end

data/lib/stepper_motor/reap_hung_journeys_job.rb

@@ -0,0 +1,12 @@
+# The purpose of this job is to find journeys which have, for whatever reason, remained in the
+# `performing` state for far longer than the journey is supposed to. At the moment it assumes
+# any journey that stayed in `performing` for longer than 1 hour has hung. Add this job to your
+# cron table and perform it regularly.
+class StepperMotor::ReapHungJourneysJob < ActiveJob::Base
+  def perform
+    StepperMotor::Journey.where("state = 'performing' AND updated_at < ?", 1.hour.ago).find_each do |hung_journey|
+      hung_journey.update!(state: "ready")
+      StepperMotor.scheduler.schedule(hung_journey)
+    end
+  end
+end

data/lib/stepper_motor/step.rb

@@ -0,0 +1,17 @@
+# Describes a step in a journey. These objects get stored inside the `step_definitions`
+# array of the Journey subclass. When the step gets performed, the block passed to the
+# constructor will be instance_exec'd with the Journey model being the context
+class StepperMotor::Step
+  attr_reader :name, :wait, :seq
+  def initialize(name:, seq:, wait: 0, &step_block)
+    @step_block = step_block
+    @name = name.to_s
+    @wait = wait
+    @seq = seq
+  end
+
+  # Makes the Step object itself callable
+  def to_proc
+    @step_block
+  end
+end

data/lib/stepper_motor/test_helper.rb

@@ -0,0 +1,34 @@
+module StepperMotor::TestHelper
+  # Allows running a given Journey to completion, skipping across the waiting periods.
+  # This is useful to evaluate all side effects of a Journey. The helper will ensure
+  # that the number of steps performed is equal to the number of steps defined - this way
+  # it will not enter into an endless loop. If, after completing all the steps, the journey
+  # has neither canceled nor finished, an exception will be raised.
+  #
+  # @param journey[StepperMotor::Journey] the journey to speedrun
+  # @return void
+  def speedrun_journey(journey)
+    journey.save!
+    n_steps = journey.step_definitions.length
+    n_steps.times do
+      journey.reload
+      break if journey.canceled? || journey.finished?
+      journey.update(next_step_to_be_performed_at: Time.current)
+      journey.perform_next_step!
+    end
+    journey.reload
+    journey_did_complete = journey.canceled? || journey.finished?
+    raise "Journey #{journey} did not finish or cancel after performing #{n_steps} steps" unless journey_did_complete
+  end
+
+  # Performs the named step of the journey without waiting for the time to perform the step.
+  #
+  # @param journey[StepperMotor::Journey] the journey to speedrun
+  # @param step_name[Symbol] the name of the step to run
+  # @return void
+  def immediately_perform_single_step(journey, step_name)
+    journey.save!
+    journey.update!(next_step_name: step_name, next_step_to_be_performed_at: Time.current)
+    journey.perform_next_step!
+  end
+end

data/lib/stepper_motor/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module StepperMotor
+  VERSION = "0.1.0"
+end

data/lib/stepper_motor.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+require_relative "stepper_motor/version"
+require "active_support"
+
+module StepperMotor
+  class Error < StandardError; end
+
+  class JourneyNotPersisted < Error; end
+
+  class StepConfigurationError < ArgumentError; end
+
+  autoload :Journey, File.dirname(__FILE__) + "/stepper_motor/journey.rb"
+  autoload :Step, File.dirname(__FILE__) + "/stepper_motor/step.rb"
+  autoload :PerformStepJob, File.dirname(__FILE__) + "/stepper_motor/perform_step_job.rb"
+  autoload :InstallGenerator, File.dirname(__FILE__) + "/generators/install_generator.rb"
+  autoload :ForwardScheduler, File.dirname(__FILE__) + "/stepper_motor/forward_scheduler.rb"
+  autoload :CyclicScheduler, File.dirname(__FILE__) + "/stepper_motor/cyclic_scheduler.rb"
+  autoload :TestHelper, File.dirname(__FILE__) + "/stepper_motor/test_helper.rb"
+
+  mattr_accessor :scheduler, default: ForwardScheduler.new
+end

data/sig/stepper_motor.rbs

@@ -0,0 +1,4 @@
+module StepperMotor
+  VERSION: String
+  # See the writing guide of rbs: https://github.com/ruby/rbs#guides
+end

data/spec/helpers/side_effects.rb

@@ -0,0 +1,76 @@
+module SideEffects
+  module SpecHelper
+    def self.included(into)
+      into.before(:each) { SideEffects.clear! }
+      into.after(:each) { SideEffects.clear! }
+      super
+    end
+  end
+
+  def self.produced?(name)
+    Thread.current[:side_effects].to_h.key?(name.to_s)
+  end
+
+  def self.none?
+    Thread.current[:side_effects].to_h.empty?
+  end
+
+  def self.names
+    Thread.current[:side_effects].to_h.keys.map(&:to_s)
+  end
+
+  def self.clear!
+    Thread.current[:side_effects] = {}
+  end
+
+  def self.touch!(name)
+    Thread.current[:side_effects][name.to_s] = true
+  end
+
+  RSpec::Matchers.define :have_produced_side_effects_named do |*side_effect_names|
+    match(notify_expectation_failures: true) do |actual|
+      SideEffects.clear!
+      actual.call
+      side_effect_names.each do |side_effect_name|
+        expect(SideEffects).to be_produced(side_effect_name), "The side effect named #{side_effect_name.inspect} should have been produced, but wasn't"
+      end
+      true
+    end
+
+    def supports_block_expectations?
+      true
+    end
+  end
+
+  RSpec::Matchers.define :not_have_produced_side_effects_named do |*side_effect_names|
+    match(notify_expectation_failures: true) do |actual|
+      expect(side_effect_names).not_to be_empty
+
+      SideEffects.clear!
+      actual.call
+
+      side_effect_names.each do |side_effect_name|
+        expect(SideEffects).not_to be_produced(side_effect_name), "The side effect named #{side_effect_name.inspect} should not have been produced, but was"
+      end
+
+      true
+    end
+
+    def supports_block_expectations?
+      true
+    end
+  end
+
+  RSpec::Matchers.define :not_have_produced_any_side_effects do
+    match(notify_expectation_failures: true) do |actual|
+      SideEffects.clear!
+      actual.call
+      expect(SideEffects).to be_none
+      true
+    end
+
+    def supports_block_expectations?
+      true
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+require "stepper_motor"
+require "active_support/testing/time_helpers"
+require "active_job"
+require "active_record"
+require "globalid"
+require_relative "helpers/side_effects"
+
+module StepperMotorRailtieTestHelpers
+  def establish_test_connection
+    ActiveRecord::Base.establish_connection(adapter: "sqlite3", database: fake_app_root + "/db.sqlite3")
+    StepperMotor::InstallGenerator.source_root(File.dirname(__FILE__) + "/../../lib")
+  end
+
+  def fake_app_root
+    File.dirname(__FILE__) + "/app"
+  end
+
+  def run_generator
+    generator = StepperMotor::InstallGenerator.new
+    generator.destination_root = fake_app_root
+    generator.create_migration_file
+  end
+
+  def run_migrations
+    # Before running the migrations we need to require the migration files, since there is no
+    # "full" Rails environment available
+    Dir.glob(fake_app_root + "/db/migrate/*.rb").sort.each do |migration_file_path|
+      require migration_file_path
+    end
+
+    ActiveRecord::Migrator.migrations_paths = [File.join(fake_app_root + "/db/migrate")]
+    ActiveRecord::Tasks::DatabaseTasks.root = fake_app_root
+    ActiveRecord::Tasks::DatabaseTasks.migrate
+  end
+end
+
+RSpec.configure do |config|
+  # Enable flags like --only-failures and --next-failure
+  config.example_status_persistence_file_path = ".rspec_status"
+
+  # Disable RSpec exposing methods globally on `Module` and `main`
+  config.disable_monkey_patching!
+
+  config.expect_with :rspec do |c|
+    c.syntax = :expect
+  end
+
+  config.include ActiveSupport::Testing::TimeHelpers
+  config.include StepperMotorRailtieTestHelpers
+  config.include SideEffects::SpecHelper
+end

data/spec/stepper_motor/cyclic_scheduler_spec.rb

@@ -0,0 +1,67 @@
+require_relative "../spec_helper"
+
+RSpec.describe "StepperMotor::CyclicScheduler" do
+  include ActiveJob::TestHelper
+
+  class FarFutureJourney < StepperMotor::Journey
+    step :do_thing, wait: 40.minutes do
+      raise "We do not test this so it should never run"
+    end
+  end
+
+  before do
+    @previous_scheduler = StepperMotor.scheduler
+    establish_test_connection
+    run_generator
+    run_migrations
+    StepperMotor::Journey.delete_all
+  end
+
+  after do
+    StepperMotor.scheduler = @previous_scheduler
+  end
+
+  it "does not schedule a journey which is too far in the future" do
+    scheduler = StepperMotor::CyclicScheduler.new(cycle_duration: 30.seconds)
+    StepperMotor.scheduler = scheduler
+
+    expect(scheduler).to receive(:schedule).with(instance_of(FarFutureJourney)).once.and_call_original
+    _journey = FarFutureJourney.create!
+
+    expect(scheduler).not_to receive(:schedule)
+    scheduler.run_scheduling_cycle
+  end
+
+  it "only schedules journeys which are within its execution window" do
+    scheduler = StepperMotor::CyclicScheduler.new(cycle_duration: 40.minutes)
+    StepperMotor.scheduler = scheduler
+
+    expect(scheduler).to receive(:schedule).with(instance_of(FarFutureJourney)).once.and_call_original
+    journey = FarFutureJourney.create!
+
+    expect(scheduler).to receive(:schedule).with(journey).and_call_original
+    scheduler.run_scheduling_cycle
+  end
+
+  it "also schedules journeys which had to run in the past" do
+    scheduler = StepperMotor::CyclicScheduler.new(cycle_duration: 10.seconds)
+    StepperMotor.scheduler = scheduler
+
+    expect(scheduler).to receive(:schedule).with(instance_of(FarFutureJourney)).once.and_call_original
+    journey = FarFutureJourney.create!
+    journey.update!(next_step_to_be_performed_at: 10.minutes.ago)
+
+    expect(scheduler).to receive(:schedule).with(journey).and_call_original
+    scheduler.run_scheduling_cycle
+  end
+
+  it "performs the scheduling job" do
+    scheduler = StepperMotor::CyclicScheduler.new(cycle_duration: 10.seconds)
+    StepperMotor.scheduler = scheduler
+    job_class = StepperMotor::CyclicScheduler::RunSchedulingCycleJob
+    expect(scheduler).to receive(:run_scheduling_cycle).and_call_original
+    job_class.perform_now
+  end
+
+  it "does not perform the job if the configured scheduler is not the CyclicScheduler"
+end

data/spec/stepper_motor/generator_spec.rb

@@ -0,0 +1,14 @@
+require_relative "../spec_helper"
+
+RSpec.describe "StepperMotor::InstallGenerator" do
+  it "is able to set up a test database" do
+    expect {
+      establish_test_connection
+      run_generator
+      run_migrations
+    }.not_to raise_error
+    expect(ActiveRecord::Base.connection.tables).to include("stepper_motor_journeys")
+  ensure
+    FileUtils.rm_rf(fake_app_root)
+  end
+end