qube_rails 0.2.0

data/app/models/qube_rails/connection.rb

@@ -0,0 +1,303 @@
+# frozen_string_literal: true
+
+require "erb"
+require "time"
+
+module QubeRails
+  # One QuickBooks Desktop connection: a company file on a customer's computer, reached through
+  # the Web Connector. Its owner is the account (or shop, or whatever your app's customer is)
+  # that connected it, and an owner can have any number: two company files, or a new file after
+  # a reconnect.
+  #
+  # qube_id is QuBe Sync's id for it, nil until QuBe Sync answers the create. status is
+  # "pending" until the setup ends, then "connected", "setup_cancelled" or "stalled" (with
+  # stall_reason), kept up to date by its own webhooks (QubeRails::ConnectionEvents). revision
+  # is the last of those applied.
+  #
+  # The QuickBooks ids your app learns are kept against it, by list family (link, qb_id_for,
+  # find_linked), since a ListID is unique only within one list family of one company file.
+  class Connection < ApplicationRecord
+    self.table_name = "qube_connections"
+
+    # The ones that have never connected, newest first: what Connect sends back through setup
+    # rather than making another.
+    scope :never_connected, -> { where(status: %w[pending setup_cancelled]).order(created_at: :desc, id: :desc) }
+
+    STATUSES = %w[pending connected setup_cancelled stalled].freeze
+
+    belongs_to :owner, polymorphic: true
+    has_many :runs, class_name: "QubeRails::Run", inverse_of: :connection
+    has_many :queued_requests, class_name: "QubeRails::QueuedRequest", inverse_of: :connection
+    has_many :quickbooks_records, class_name: "QubeRails::QuickbooksRecord", inverse_of: :connection
+
+    validates :status, inclusion: { in: STATUSES }
+
+    # The connections owner has, whatever its key's type.
+    # The row, with its runs (and their notifications), its requests and its QuickBooks links: a connection
+    # QuBe Sync no longer has. ConnectionsController#destroy deletes QuBe Sync's first.
+    def disconnect!
+      transaction do
+        QubeRails.config.connection_events_class.new(self).disconnected
+        runs.find_each(&:destroy!)
+        QubeRails::QueuedRequest.where(connection_id: id).delete_all # its deliveries cascade
+        QubeRails::QuickbooksRecord.where(connection_id: id).delete_all
+        destroy!
+      end
+    end
+
+    # When its last completed run of workflow started (nil before one): a sync's next since.
+    def last_synced_at(workflow)
+      QubeRails::SyncMark.find_by(connection_id: id, workflow: workflow.to_s)&.started_at
+    end
+
+    def self.owned_by(owner)
+      type, id = key_for(owner)
+      where(owner_type: type, owner_id: id)
+    end
+
+    # Where its events go: QUBE_WEBHOOK_URL/connections/<id>.
+    def webhook_url
+      "#{QubeRails.config.webhook_url}/connections/#{ERB::Util.url_encode(id.to_s)}"
+    end
+
+    def connected?
+      status == "connected"
+    end
+
+    # Stalled because the Web Connector isn't checking in (the QuickBooks computer is off, or
+    # the Web Connector closed), rather than because it can't open QuickBooks.
+    def not_checking_in?
+      status == "stalled" && stall_reason == "not_checking_in"
+    end
+
+    # Clock difference allowed between this app and QuBe Sync.
+    CLOCK_SKEW_SECONDS = 5 * 60
+
+    # For a row without a qube_id: QuBe Sync's connection made for it, when a create landed but
+    # its id was never saved (it didn't commit, or the process died between the call and the
+    # save). Saves and returns the id, or nil, so a second connection is never made for one row.
+    #
+    # It lists the app's connections (GET /api/v2/connections, which isn't paged) and adopts one
+    # only when exactly one has this row's webhook URL, is unclaimed (it has never connected, or
+    # it's simulated: a simulated connection connects as it's made), was made at or after this row
+    # (its created_at), and has the row's name when the row has one. So a row id used again (MySQL 5.7 after a restart, a restored database) never takes
+    # an old connection. A connection with no created_at (a server from before v2 sent it) is
+    # never adopted. The listing only runs for a row without an id, which is rare.
+    def adopt_qube_connection
+      return qube_id unless qube_id.to_s.empty?
+
+      listed = Array(QubeRails.client.list_connections).select { |made| made.is_a?(Hash) }
+      if listed.any? { |made| made["created_at"].to_s.empty? && same_url?(made) }
+        QubeRails.logger.warn("QuBe Sync: connection #{id}: QuBe Sync's connections carry no created_at, so " \
+                              "none is adopted; a new one is made instead")
+      end
+      candidates = listed.select { |made| made_for_this_row?(made) }
+      if candidates.size > 1
+        QubeRails.logger.warn("QuBe Sync: connection #{id} has no qube_id and #{candidates.size} of QuBe Sync's " \
+                              "connections could be its; adopted none")
+        return nil
+      end
+      found = candidates.first
+      return nil unless found
+
+      QubeRails.logger.info("QuBe Sync: connection #{id} adopted QuBe Sync's connection #{found["id"]}, " \
+                            "made for it before its id was saved")
+      update!(qube_id: found["id"])
+      qube_id
+    end
+
+    # Keeps record's QuickBooks id for kind ("invoice", "customer", "item_service"...) on this
+    # connection, with its EditSequence and its name (FullName, or RefNumber), and returns the
+    # row. A record has one id per kind, so linking it again replaces its id. An id already linked
+    # in that kind's family (QuickbooksRecord.family_for) to another of your records moves to this
+    # one, and that's logged: the same QuickBooks record can stand for one of yours at a time. The
+    # same record and id linked with another kind of the family (an item converted from service
+    # to non-inventory) changes the row's kind.
+    def link(record, kind:, qb_id:, edit_sequence: nil, name: nil)
+      kind = kind.to_s
+      qb_id = qb_id.to_s
+      raise ArgumentError, "link needs a qb_id" if qb_id.empty?
+
+      family = QuickbooksRecord.family_for(kind)
+      type, id = self.class.key_for(record)
+      attempts = 0
+      begin
+        # A savepoint, so a unique-key race inside a caller's transaction (the job's row lock)
+        # can be retried.
+        transaction(requires_new: true) do
+          taken = quickbooks_records.lock.find_by(list: family, qb_id: qb_id)
+          mine = quickbooks_records.lock.find_by(kind: kind, record_type: type, record_id: id)
+          if replaced?(mine, by: taken)
+            QubeRails.logger.info("QuBe Sync: connection #{self.id}: #{type} #{id}'s #{kind} " \
+                                  "#{mine.qb_id} replaced by #{qb_id}")
+            mine.destroy!
+          end
+          if moved?(taken, to: [type, id])
+            QubeRails.logger.warn("QuBe Sync: connection #{self.id}: #{kind} #{qb_id} moved from " \
+                                  "#{taken.record_type} #{taken.record_id} to #{type} #{id}")
+          end
+          row = taken || mine || quickbooks_records.build
+          row.assign_attributes(kind: kind, qb_id: qb_id, record_type: type, record_id: id)
+          row.edit_sequence = edit_sequence unless edit_sequence.nil?
+          row.name = name unless name.nil?
+          row.save!
+          row
+        end
+      rescue ActiveRecord::RecordNotUnique
+        # Another worker linked the same record or id at the same moment: once more, now it's
+        # there to find.
+        attempts += 1
+        retry if attempts < 2
+        raise
+      end
+    end
+
+    # record's QuickBooks id for kind on this connection, or nil: its link of that kind, else,
+    # outside the name family, its one link in that kind's family (an item linked as
+    # item_service, asked for as item or item_non_inventory). In the name family each kind is its
+    # own QuickBooks record (a customer and a vendor are two names), so a customer's id is never
+    # handed out as a vendor's. Raises ArgumentError when the family has several and none is that
+    # kind.
+    def qb_id_for(record, kind:)
+      type, id = self.class.key_for(record)
+      mine = quickbooks_records.where(record_type: type, record_id: id)
+      exact = mine.where(kind: kind.to_s).limit(1).pluck(:qb_id).first
+      family = QuickbooksRecord.family_for(kind)
+      return exact if exact || family == "name"
+
+      row = only_one(mine.where(list: family), "#{type} #{id}'s #{kind}")
+      row && row.qb_id
+    end
+
+    # Where record stands with QuickBooks on this connection for operation (its SLUG): a
+    # QubeRails::SyncStatus, worked out from its runs and its link, never stored.
+    def sync_status(record, operation:)
+      SyncStatus.for(self, record, operation: operation)
+    end
+
+    # Your records in scope (a model or a relation) with a QuickBooks link on this connection, or
+    # without one: for list pages and filters, one query each. They join on
+    # qube_quickbooks_records' record index, by your table's primary key as text.
+    def in_quickbooks(scope)
+      scope.where(linked_here(scope))
+    end
+
+    def not_in_quickbooks(scope)
+      scope.where.not(linked_here(scope))
+    end
+
+    # Your records in scope whose run of operation on this connection is waiting for your user's
+    # answer, or still on its way (starting or running): their latest run, since a start returns
+    # the run that hasn't ended rather than starting another.
+    def needing_answer(scope, operation:)
+      scope.where(run_here(scope, operation, %w[awaiting_input]))
+    end
+
+    def sending(scope, operation:)
+      scope.where(run_here(scope, operation, %w[starting running]))
+    end
+
+    # Your record linked to that QuickBooks id, or nil.
+    def find_linked(qb_id, kind: nil)
+      row = quickbooks_record(qb_id, kind: kind)
+      row && row.record
+    end
+
+    # The row for that QuickBooks id (with its kind, edit_sequence and name), or nil. With a kind,
+    # it's looked up in that kind's family; without one, in every family, and an id found in more
+    # than one (the same counter value is a customer and an account) raises ArgumentError, so the
+    # caller says which kind it means.
+    def quickbooks_record(qb_id, kind: nil)
+      rows = quickbooks_records.where(qb_id: qb_id.to_s)
+      return rows.find_by(list: QuickbooksRecord.family_for(kind)) if kind
+
+      only_one(rows, "QuickBooks id #{qb_id}")
+    end
+
+    private
+
+    # The record's own link is a different row from the one holding this qb_id, so it gives way.
+    def replaced?(mine, by:)
+      !mine.nil? && !by.nil? && by.id != mine.id
+    end
+
+    # This qb_id was linked to another of your records.
+    def moved?(taken, to:)
+      !taken.nil? && !taken.record_id.nil? && [taken.record_type, taken.record_id] != to
+    end
+
+    def same_url?(made)
+      made["webhook_url"].to_s.chomp("/") == webhook_url
+    end
+
+    def made_for_this_row?(made)
+      return false if made["id"].to_s.empty?
+      return false unless same_url?(made)
+      return false unless unclaimed?(made)
+      return false unless name.to_s.empty? || made["name"] == name
+
+      made_at = parse_time(made["created_at"])
+      return false unless made_at && created_at
+
+      made_at >= created_at - CLOCK_SKEW_SECONDS
+    end
+
+    # Nobody can have set it up: it has never connected, or it's simulated and connected itself
+    # when it was made. Either way, only the lost answer to the create carried its setup link.
+    def unclaimed?(made)
+      made["last_connected_at"].nil? || made["type"] == "simulated"
+    end
+
+    def parse_time(value)
+      Time.iso8601(value.to_s)
+    rescue ArgumentError
+      nil
+    end
+
+    # EXISTS a link on this connection for each row of scope's table.
+    def linked_here(scope)
+      model = scope.all.klass
+      QuickbooksRecord.where(connection_id: id, record_type: model.base_class.name)
+                      .where("qube_quickbooks_records.record_id = #{id_as_text(model, QuickbooksRecord)}")
+                      .arel.exists
+    end
+
+    # EXISTS a run of operation on this connection, in one of states, for each row of scope's
+    # table.
+    def run_here(scope, operation, states)
+      model = scope.all.klass
+      Run.where(connection_id: id, operation: operation.to_s, record_type: model.base_class.name,
+                state: states)
+         .where("qube_runs.record_id = #{id_as_text(model, Run)}")
+         .arel.exists
+    end
+
+    # The app's primary key as the text the engine keeps ids as (integer or UUID alike), to
+    # compare with engine_model's record_id. Elsewhere a VARCHAR, which compares without padding.
+    # MySQL casts only to CHAR, in the connection's collation, which can differ from the
+    # column's (an "illegal mix of collations"), so it's given the column's.
+    def id_as_text(model, engine_model)
+      db = model.connection
+      column = "#{db.quote_table_name(model.table_name)}.#{db.quote_column_name(model.primary_key)}"
+      return "CAST(#{column} AS VARCHAR(255))" unless mysql?(db)
+
+      collation = engine_model.columns_hash["record_id"].collation
+      return "CAST(#{column} AS CHAR)" unless collation
+
+      "CONVERT(CAST(#{column} AS CHAR) USING #{collation.split("_").first}) COLLATE #{collation}"
+    end
+
+    def mysql?(db)
+      db.adapter_name.match?(/mysql|trilogy/i)
+    end
+
+    def only_one(scope, what)
+      rows = scope.limit(2).to_a
+      if rows.size > 1
+        kinds = scope.pluck(:kind).sort.join(", ")
+        raise ArgumentError, "QuBe Sync: #{what} is linked as more than one kind (#{kinds}): pass kind:"
+      end
+      rows.first
+    end
+  end
+end

data/app/models/qube_rails/notification.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module QubeRails
+  # A notification a run's handler has taken, by its delivery_id: QubeRails::DatabaseDedupe
+  # writes it in the same transaction as the handler, so a redelivery is skipped and a handler
+  # that raised runs again. Deleted with its run.
+  class Notification < ApplicationRecord
+    self.table_name = "qube_notifications"
+
+    belongs_to :run, class_name: "QubeRails::Run"
+  end
+end

data/app/models/qube_rails/queued_request.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+require "erb"
+
+module QubeRails
+  # One plain v2 request (QubeRails::Request.queue) for one of your records, on one connection.
+  # Written before it's sent; its webhook URL is QUBE_WEBHOOK_URL/requests/<id>.
+  #
+  # state: "queued", "retryable", "response_received", "error", "timed_out", "discarded", or
+  # "failed" (QuBe Sync refused the queue, or params raised). ended_at is set once it's over:
+  # every page answered, or a final error.
+  class QueuedRequest < ApplicationRecord
+    self.table_name = "qube_requests"
+
+    belongs_to :connection, class_name: "QubeRails::Connection", inverse_of: :queued_requests
+    belongs_to :record, polymorphic: true
+    has_many :deliveries, class_name: "QubeRails::RequestDelivery", foreign_key: :request_id,
+                          inverse_of: :queued_request, dependent: :delete_all
+
+    JSONCoder.serialize(self, :params, :error)
+
+    # Set by Request.queue when it returned this row rather than queueing another. Not saved.
+    attr_writer :reused
+
+    def reused?
+      @reused == true
+    end
+
+    def self.latest_for(record, operation)
+      type, id = key_for(record)
+      where(record_type: type, record_id: id, operation: operation.to_s)
+        .order(created_at: :desc, id: :desc).first
+    end
+
+    # Ended requests older than older_than (at least a day, as Run.prunable).
+    def self.prunable(older_than:)
+      cutoff = [older_than, 1.day].max.ago
+      where.not(ended_at: nil).where("ended_at < ?", cutoff)
+    end
+
+    def ended?
+      !ended_at.nil?
+    end
+
+    # Every page answered.
+    def done?
+      ended? && state == "response_received"
+    end
+
+    def webhook_url
+      "#{QubeRails.config.webhook_url}/requests/#{ERB::Util.url_encode(id.to_s)}"
+    end
+
+    # Sends a queue QuBe Sync never confirmed again: same params and Idempotency-Key.
+    def retry_queue!
+      request_class = QubeRails.config.operation_class(operation_slug)
+      raise UnknownOperation, "QuBe Sync: no operation has the slug #{operation_slug.inspect}" unless request_class
+
+      request_class.retry_queue(self)
+    end
+
+    def operation_slug
+      self[:operation]
+    end
+  end
+end

data/app/models/qube_rails/quickbooks_record.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module QubeRails
+  # A QuickBooks record your app knows on one connection: its kind ("customer", "item_service",
+  # "invoice"...), its list family (list), its ListID or TxnID (qb_id), its EditSequence and its
+  # name, and which of your records it stands for, if any. Written through Connection#link.
+  #
+  # QuickBooks makes a ListID unique within a list family, not within one kind (verified on
+  # QuickBooks Enterprise 24.0): customers, vendors, employees and other names share one counter,
+  # every item type shares another, and each other list (accounts, classes, terms...) has its
+  # own, while the same counter value turns up in several families. So a row is unique by
+  # connection, family and qb_id, and by connection, kind and your record (one contact can be
+  # both a customer and a vendor, as two QuickBooks names).
+  class QuickbooksRecord < ApplicationRecord
+    self.table_name = "qube_quickbooks_records"
+
+    NAME_KINDS = %w[customer vendor employee other_name].freeze
+
+    belongs_to :connection, class_name: "QubeRails::Connection", inverse_of: :quickbooks_records
+    belongs_to :record, polymorphic: true, optional: true
+
+    before_validation { self.list = self.class.family_for(kind) }
+
+    # The list family a kind's ids are unique within: "name" for customers, vendors, employees
+    # and other names; "item" for "item" and every item_* kind; for anything else (an account, a
+    # class, a transaction, a kind this version doesn't know) the kind itself. TxnIDs are taken as
+    # unique per transaction type until QuickBooks shows otherwise.
+    def self.family_for(kind)
+      kind = kind.to_s
+      return "name" if NAME_KINDS.include?(kind)
+      return "item" if kind == "item" || kind.start_with?("item_")
+
+      kind
+    end
+  end
+end

data/app/models/qube_rails/request_delivery.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module QubeRails
+  # A request webhook the handler has taken, by page_request_id:state, written in the handler's
+  # transaction. Deleted with its request.
+  class RequestDelivery < ApplicationRecord
+    self.table_name = "qube_request_deliveries"
+
+    belongs_to :queued_request, class_name: "QubeRails::QueuedRequest", foreign_key: :request_id,
+                                inverse_of: :deliveries
+  end
+end

data/app/models/qube_rails/run.rb

@@ -0,0 +1,187 @@
+# frozen_string_literal: true
+
+require "erb"
+
+module QubeRails
+  # One workflow run, of one operation, for one of your records, on one connection. Written
+  # before the run starts (Operation.start), so a webhook that arrives first finds it; its
+  # webhook URL is QUBE_WEBHOOK_URL/runs/<id>.
+  #
+  # reference is new for every run and sent with it, and every event is checked against it and
+  # the connection's qube_id. revision is the last event applied; state ("starting", then the
+  # run's own: "running", "awaiting_input", "completed", "failed", "cancelled") and outcome are
+  # the run's as of that event. question and links are the question it's asking, if any, for the
+  # question partial. input is what the run was started with. A start QuBe Sync refused (a 4xx),
+  # or whose input raised, leaves state "failed"; one that got no answer (a timeout, a 5xx, a
+  # dropped connection) stays "starting", for retry_start!.
+  class Run < ApplicationRecord
+    self.table_name = "qube_runs"
+
+    belongs_to :connection, class_name: "QubeRails::Connection", inverse_of: :runs
+    belongs_to :record, polymorphic: true
+    has_many :notifications, class_name: "QubeRails::Notification", dependent: :delete_all
+
+    JSONCoder.serialize(self, :input, :error, :question, :links)
+
+    ENDED_STATES = %w[completed failed cancelled].freeze
+
+    # Set by Operation.start when it returned this run, one that hadn't ended, rather than starting
+    # another: the record's run already on its way. Not saved.
+    attr_writer :reused
+
+    def reused?
+      @reused == true
+    end
+
+    # record's newest run of operation, or nil: the one whose question the partial shows.
+    def self.latest_for(record, operation)
+      type, id = key_for(record)
+      where(record_type: type, record_id: id, operation: operation.to_s)
+        .order(created_at: :desc, id: :desc).first
+    end
+
+    # Sends a start that didn't get an answer again: same reference (the Idempotency-Key),
+    # workflow, input and webhook URL, so QuBe Sync answers with the run the first one started,
+    # if it did, instead of starting another. See Operation.retry_start.
+    def retry_start!
+      operation = QubeRails.config.operation_class(operation_slug)
+      raise UnknownOperation, "QuBe Sync: no operation has the slug #{operation_slug.inspect}" unless operation
+
+      operation.retry_start(self)
+    end
+
+    def webhook_url
+      "#{QubeRails.config.webhook_url}/runs/#{ERB::Util.url_encode(id.to_s)}"
+    end
+
+    def humanize(word)
+      text = word.to_s.tr("_", " ")
+      text.empty? ? text : text[0].upcase + text[1..-1]
+    end
+    private :humanize
+
+    def operation_slug
+      self[:operation]
+    end
+
+    # Whether it has ended: completed, failed or cancelled.
+    # Ended runs older than older_than (at least a day: a late webhook for a deleted run is
+    # retried for hours). A run that failed before it started has no ended_at.
+    def self.prunable(older_than:)
+      cutoff = [older_than, 1.day].max.ago
+      where(state: ENDED_STATES).where("COALESCE(ended_at, updated_at) < ?", cutoff)
+    end
+
+    def ended?
+      ENDED_STATES.include?(state)
+    end
+
+    def failed?
+      state == "failed"
+    end
+
+    # Cancelled by your app (state "cancelled"), or a chart that ended on its own "cancelled"
+    # outcome (the app chose to stop): neither is a success.
+    def cancelled?
+      state == "cancelled" || (state == "completed" && outcome == "cancelled")
+    end
+
+    # How long a start may go unconfirmed before the status says so and offers to send it again.
+    UNCONFIRMED_AFTER_SECONDS = 60
+
+    # How long a confirmed run may go without news before the status offers to have QuBe Sync
+    # send its latest event again (QubeRails::RunsController#redeliver).
+    QUIET_AFTER_SECONDS = 10 * 60
+
+    # A run QuBe Sync confirmed that hasn't been heard from for QUIET_AFTER_SECONDS: running,
+    # or waiting on a question this app never received. Its webhook may have been lost (the app
+    # was down longer than QuBe Sync's retries), so the status offers to send it again.
+    def looks_stuck?
+      quiet_since?(Time.now - QUIET_AFTER_SECONDS) && (state == "running" || question_missing?)
+    end
+
+    # A start QuBe Sync hasn't confirmed for a minute: send it again with retry_start!.
+    def quiet_since?(time)
+      !qube_run_id.to_s.empty? && !ended? && updated_at && updated_at < time
+    end
+    private :quiet_since?
+
+    def question_missing?
+      state == "awaiting_input" && question.nil?
+    end
+    private :question_missing?
+
+    def unconfirmed?
+      state == "starting" && created_at && created_at < Time.now - UNCONFIRMED_AFTER_SECONDS
+    end
+
+    # Where it stands, in a few words for a person: "Added to QuickBooks", "Done", "Couldn't
+    # sync", "Cancelled", "Needs your answer", "Sending to QuickBooks", "Still waiting for
+    # QuickBooks" (a start unconfirmed for a minute) or "Syncing with QuickBooks". Why it failed
+    # or was cancelled is reason_text. The words are I18n's, under qube_rails.status.
+    def summary_text
+      return QubeRails.t("status.cancelled") if cancelled?
+
+      case state
+      when "starting"
+        QubeRails.t(unconfirmed? ? "status.unconfirmed" : "status.starting")
+      when "completed"
+        case outcome.to_s
+        when "added" then QubeRails.t("status.added")
+        when "", "completed" then QubeRails.t("status.done")
+        else QubeRails.t("status.done_with", outcome: humanize(outcome))
+        end
+      when "failed" then QubeRails.t("status.failed")
+      when "awaiting_input" then QubeRails.t("status.needs_answer")
+      else QubeRails.t("status.running")
+      end
+    end
+
+    # Why it failed (its error message) or was cancelled (the reason it carried), as a sentence,
+    # or nil.
+    def reason_text
+      text = if cancelled? then cancel_reason.to_s.strip
+             elsif state == "failed" then error_message.to_s.strip
+             end
+      return nil if text.nil? || text.empty?
+
+      text = text[0].upcase + text[1..]
+      text.match?(/[.!?]\z/) ? text : "#{text}."
+    end
+
+    # Why it was cancelled: the reason it carried, else what a cancelled outcome means.
+    def cancel_reason_text
+      reason = cancel_reason.to_s.strip
+      reason.empty? ? QubeRails.t("status.cancel_reason") : reason
+    end
+
+    # A failed run's message: its error's own (a sentence, or the message of a code, message and
+    # detail), else its outcome in words, or nil.
+    def error_message
+      text =
+        case error
+        when String then error
+        when Hash then error["message"] || error["detail"]
+        end
+      text = text.to_s.strip
+      return text unless text.empty?
+      return QubeRails.t("status.not_started") if outcome.nil? && ended_at.nil?
+
+      outcome.to_s.empty? || outcome == "failed" ? nil : humanize(outcome)
+    end
+
+    # The error's detail, when it carries one apart from its message.
+    def error_detail
+      return nil unless error.is_a?(Hash)
+
+      detail = error["detail"]
+      detail = detail.to_s.strip
+      detail.empty? || detail == error_message ? nil : detail
+    end
+
+    # The question it's asking, as its webhook's prompt had it, or nil.
+    def open_question
+      question.is_a?(Hash) ? question : nil
+    end
+  end
+end

data/app/models/qube_rails/sync_mark.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module QubeRails
+  # When a connection's last completed run of a workflow started (Connection#last_synced_at).
+  class SyncMark < ApplicationRecord
+    self.table_name = "qube_sync_marks"
+
+    belongs_to :connection, class_name: "QubeRails::Connection"
+
+    # Moves the mark to started_at, never back: runs can complete out of order.
+    def self.advance(connection, workflow, started_at)
+      mark = find_or_create_by!(connection_id: connection.id, workflow: workflow) do |made|
+        made.started_at = started_at
+      end
+      where(id: mark.id).where("started_at < ?", started_at)
+                        .update_all(started_at: started_at, updated_at: Time.current)
+    rescue ActiveRecord::RecordNotUnique
+      retry
+    end
+  end
+end

data/app/views/qube_rails/_error.html.erb

@@ -0,0 +1,13 @@
+<%# A message from one of the engine's routes, shown once. Without a message local it takes
+    flash[:qube_rails_error], for a page without the run partials:
+      render "qube_rails/error"
+    The question and run status pieces pass theirs (message: nil shows nothing). %>
+<% if local_assigns.key?(:message) %>
+<%   message = local_assigns[:message] %>
+<% else %>
+<%   message = flash[:qube_rails_error] %>
+<%   flash.delete(:qube_rails_error) %>
+<% end %>
+<% if message.present? %>
+<p class="<%= qube_classes(:error, "qube-error") %>" role="alert"><%= message %></p>
+<% end %>

data/app/views/qube_rails/_question.html.erb

@@ -0,0 +1,21 @@
+<%# The question a QuBe Sync run is asking about record, with a form per choice, or nothing
+    when it isn't asking. Render it on the record's page, with its operation's slug:
+      render "qube_rails/question", record: @invoice, operation: "create_invoice"
+    It builds a QubeRails::QuestionView from record's latest run of that operation and renders
+    the qube_rails/question/* pieces; override a piece by putting a file at its path in your
+    app (README, "Styling"). Its search boxes and rows table need the behaviour script
+    (qube_rails_assets in your layout). %>
+<% unless QubeRails.config.operation_class(operation) %>
+<%   raise ArgumentError, "QuBe Sync: no operation has the slug #{operation.inspect} (config.operations)" %>
+<% end %>
+<% run = QubeRails::Run.latest_for(record, operation) %>
+<%# Refreshed when the run's next event comes, in an app with turbo-rails (QubeRails.broadcast_refresh). %>
+<%= turbo_stream_from(record, :qube_rails) if respond_to?(:turbo_stream_from) %>
+<% error = flash[:qube_rails_error] %>
+<% flash.delete(:qube_rails_error) # shown once, by whichever partial comes first %>
+<% question = QubeRails::QuestionView.from_run(run, qube_rails, error: error) %>
+<% if question %>
+<%= render "qube_rails/question/question", question: question %>
+<% else %>
+<%= render "qube_rails/error", message: error %>
+<% end %>