clickwrap 0.0.0 → 0.1.0

data/lib/clickwrap/integrity/attestation_reconciler.rb

@@ -0,0 +1,114 @@
+# frozen_string_literal: true
+
+module Clickwrap
+  module Integrity
+    # Finds committed events for which a configured external integrity adapter
+    # never left an attestation row, then asks only for those missing records.
+    # This closes the recoverable part of the after-commit crash window: the
+    # required evidence remains committed, and an operator can safely discover
+    # and retry the optional external work later.
+    #
+    # No local database can prove whether a process died immediately before or
+    # immediately after a provider accepted a request. Adapters should therefore
+    # use the event digest (timestamps) or exact chain snapshot (anchors) as their
+    # idempotency key. A retry may otherwise create a second valid provider
+    # record; Clickwrap preserves both and never calls that exactly-once delivery.
+    class AttestationReconciler
+      Outcome = Data.define(:event_id, :kind, :state, :attestation_id) do
+        def recorded? = attestation_id.present?
+
+        def to_h
+          {
+            "event_id" => event_id,
+            "kind" => kind,
+            "state" => state,
+            "attestation_id" => attestation_id
+          }.compact
+        end
+      end
+
+      Result = Data.define(:outcomes) do
+        def attempted = outcomes.length
+        def recorded = outcomes.count(&:recorded?)
+        def not_recorded = attempted - recorded
+
+        def counts
+          { "attempted" => attempted, "recorded" => recorded, "not_recorded" => not_recorded }
+        end
+
+        def clean? = not_recorded.zero?
+        def to_h = { "counts" => counts, "outcomes" => outcomes.map(&:to_h) }
+      end
+
+      class << self
+        def missing_counts(scope: Event.all)
+          configured_kinds.to_h do |kind|
+            eligible = eligible_scope(scope, kind)
+            recorded = IntegrityAttestation.where(kind: kind).select(:event_id)
+            [kind, eligible.where.not(id: recorded).count]
+          end
+        end
+
+        def configured_kinds
+          kinds = []
+          kinds << "third_party_timestamp" if Clickwrap.config.timestamp_receipts_with
+          kinds << "event_anchor" if Clickwrap.config.anchor_event_history_with
+          kinds
+        end
+
+        def eligible_scope(scope, kind)
+          kind == "event_anchor" ? scope.where.not(chain_scope: nil) : scope
+        end
+      end
+
+      def initialize(scope: Event.all, retry_failed_attestations: false)
+        @scope = scope
+        @retry_failed_attestations = retry_failed_attestations
+      end
+
+      attr_reader :scope, :retry_failed_attestations
+
+      def call
+        outcomes = []
+
+        each_event do |event|
+          configured_kinds_for(event).each do |kind|
+            next unless attempt_needed?(event, kind)
+
+            attestation = attest(event, kind)
+            outcomes << Outcome.new(
+              event_id: event.id,
+              kind: kind,
+              state: attestation&.state || "not_recorded",
+              attestation_id: attestation&.id
+            )
+          end
+        end
+
+        Result.new(outcomes: outcomes.freeze)
+      end
+
+      private
+
+      def each_event(&)
+        scope.respond_to?(:find_each) ? scope.find_each(&) : scope.each(&)
+      end
+
+      def configured_kinds_for(event)
+        self.class.configured_kinds.reject do |kind|
+          kind == "event_anchor" && event.chain_scope.blank?
+        end
+      end
+
+      def attempt_needed?(event, kind)
+        latest = IntegrityAttestation.where(event_id: event.id, kind: kind).order(:attempted_at, :id).last
+        latest.nil? || (retry_failed_attestations && latest.state == "failed")
+      end
+
+      def attest(event, kind)
+        attestor = Attestor.new(event)
+        kind == "event_anchor" ? attestor.anchor_event : attestor.timestamp_event
+      end
+    end
+  end
+end

data/lib/clickwrap/integrity/attestor.rb

@@ -0,0 +1,221 @@
+# frozen_string_literal: true
+
+module Clickwrap
+  module Integrity
+    # Invokes optional external adapters only after the required event commit.
+    # An outage can never roll back the protected action; every ordinary result
+    # is persisted, and an exception is reported through the host's existing
+    # after-commit failure hook.
+    class Attestor
+      ChainSnapshot = Data.define(:chain_scope, :sequence, :last_event_id, :last_event_digest) do
+        def checkpoint_digest = last_event_digest
+      end
+
+      def self.attest_after_commit(event) = new(event).attest_after_commit
+
+      def initialize(event)
+        @event = event
+      end
+
+      attr_reader :event
+
+      def attest_after_commit
+        timestamp_event if Clickwrap.config.timestamp_receipts_with
+        anchor_event if Clickwrap.config.anchor_event_history_with && event.chain_scope.present?
+      end
+
+      def timestamp_event
+        adapter = Clickwrap.config.timestamp_receipts_with
+        attempted_at = Clickwrap.now
+        token = adapter.timestamp(event.event_digest)
+        token_body = normalized_result(token)
+        token_digest = token_body["digest"]
+
+        unless token_digest.to_s == event.event_digest.to_s
+          return record!(
+            kind: "third_party_timestamp", state: "failed", adapter: adapter,
+            attempted_at: attempted_at, provider_result: token_body,
+            verification: { "checked" => false, "verified" => false,
+                            "detail" => "The provider result named a different digest." }
+          )
+        end
+
+        verification = if token_body["issued"] == true
+                         normalized_result(adapter.verify(token_body["token"], event.event_digest))
+                       else
+                         { "checked" => false, "verified" => false,
+                           "detail" => token_body["detail"] }
+                       end
+
+        record!(
+          kind: "third_party_timestamp",
+          state: attestation_state(token_body["issued"], verification),
+          adapter: adapter,
+          attempted_at: attempted_at,
+          provider_result: token_body,
+          verification: verification,
+          provider_reference: token_body["token_reference"],
+          provider_reported_at: parse_time(token_body["provider_reported_time"])
+        )
+      rescue StandardError => error
+        record_exception(kind: "third_party_timestamp", adapter: adapter,
+                         attempted_at: attempted_at, error: error)
+      end
+
+      def anchor_event
+        adapter = Clickwrap.config.anchor_event_history_with
+        attempted_at = Clickwrap.now
+        snapshot = ChainSnapshot.new(
+          chain_scope: event.chain_scope,
+          sequence: event.chain_sequence,
+          last_event_id: event.id,
+          last_event_digest: event.event_digest
+        )
+        publication = normalized_result(adapter.anchor(snapshot))
+        verification = if publication["anchored"] == true
+                         # Verification receives the exact publication result
+                         # as well as the expected chain head. An adapter that
+                         # merely re-reads "some current head" cannot prove the
+                         # reference stored beside this attestation is the one
+                         # it actually checked.
+                         normalized_result(adapter.verify(publication, snapshot))
+                       else
+                         { "checked" => false, "verified" => false,
+                           "detail" => publication["detail"] }
+                       end
+
+        record!(
+          kind: "event_anchor",
+          state: attestation_state(publication["anchored"], verification),
+          adapter: adapter,
+          attempted_at: attempted_at,
+          provider_result: publication,
+          verification: verification,
+          provider_reference: publication["reference"],
+          provider_reported_at: parse_time(publication["published_at"]),
+          chain_scope: event.chain_scope,
+          chain_sequence: event.chain_sequence
+        )
+      rescue StandardError => error
+        record_exception(kind: "event_anchor", adapter: adapter,
+                         attempted_at: attempted_at, error: error,
+                         chain_scope: event.chain_scope, chain_sequence: event.chain_sequence)
+      end
+
+      private
+
+      def record!(kind:, state:, adapter:, attempted_at:, provider_result:, verification:,
+                  provider_reference: nil, provider_reported_at: nil, chain_scope: nil, chain_sequence: nil)
+        IntegrityAttestation.create!(
+          event_id: event.id,
+          kind: kind,
+          state: state,
+          provider_name: provider_name(adapter, provider_result),
+          subject_digest: event.event_digest,
+          chain_scope: chain_scope,
+          chain_sequence: chain_sequence,
+          provider_reference: provider_reference,
+          provider_result: provider_result,
+          verification: verification,
+          adapter_capabilities: safe_capabilities(adapter),
+          attempted_at: attempted_at,
+          provider_reported_at: provider_reported_at,
+          created_at: Clickwrap.now
+        )
+      end
+
+      def attestation_state(issued, verification)
+        return "unavailable" unless issued == true
+        return "verified" if verification["checked"] == true && verification["verified"] == true
+
+        "issued_unverified"
+      end
+
+      def provider_name(adapter, body)
+        body["provider_name"].presence ||
+          (adapter.respond_to?(:provider_name) ? adapter.provider_name.to_s : adapter.class.name)
+      end
+
+      def normalized_result(value)
+        body = value.respond_to?(:to_h) ? value.to_h : value
+        unless body.respond_to?(:to_h)
+          raise ConfigurationError,
+                "An integrity adapter must return a Hash-like result, got #{value.class}."
+        end
+
+        body.to_h.each_with_object({}) do |(key, nested), result|
+          result[key.to_s] = normalize_value(nested)
+        end
+      end
+
+      def normalize_value(value)
+        case value
+        when Hash then value.to_h { |key, nested| [key.to_s, normalize_value(nested)] }
+        when Array then value.map { |nested| normalize_value(nested) }
+        when Time, ActiveSupport::TimeWithZone then Receipt.format_time(value)
+        else value
+        end
+      end
+
+      def parse_time(value)
+        return value if value.is_a?(Time)
+        return nil if value.blank?
+
+        Time.parse(value.to_s).utc
+      rescue ArgumentError
+        nil
+      end
+
+      def safe_capabilities(adapter)
+        normalized_result(adapter.capabilities)
+      rescue StandardError => error
+        {
+          "unavailable" => true,
+          "error_class" => error.class.name,
+          "detail" => "The adapter's capabilities could not be read."
+        }
+      end
+
+      def record_exception(kind:, adapter:, attempted_at:, error:, chain_scope: nil, chain_sequence: nil)
+        adapter_error = error
+        IntegrityAttestation.create!(
+          event_id: event.id,
+          kind: kind,
+          state: "failed",
+          provider_name: safe_provider_name(adapter),
+          subject_digest: event.event_digest,
+          chain_scope: chain_scope,
+          chain_sequence: chain_sequence,
+          provider_result: {
+            "error_class" => adapter_error.class.name,
+            "detail" => adapter_error.message.to_s.slice(0, 1_000)
+          },
+          verification: {
+            "checked" => false,
+            "verified" => false,
+            "detail" => "The adapter raised before this attestation could be verified."
+          },
+          adapter_capabilities: safe_capabilities(adapter),
+          attempted_at: attempted_at,
+          created_at: Clickwrap.now
+        )
+      rescue StandardError => error
+        report_failure(error)
+      ensure
+        report_failure(adapter_error)
+      end
+
+      def safe_provider_name(adapter)
+        value = adapter.provider_name if adapter.respond_to?(:provider_name)
+        value.to_s.presence || adapter.class.name.presence || "unknown_integrity_adapter"
+      rescue StandardError
+        adapter.class.name.presence || "unknown_integrity_adapter"
+      end
+
+      def report_failure(error)
+        Clickwrap.report_after_commit_failure(error, event)
+        nil
+      end
+    end
+  end
+end

data/lib/clickwrap/integrity/chain.rb

@@ -0,0 +1,313 @@
+# frozen_string_literal: true
+
+module Clickwrap
+  module Integrity
+    # Walks the optional event chains and reports what still links up.
+    #
+    #   result = Clickwrap::Integrity::Chain.verify
+    #   result.success?     # => true
+    #   result.counts       # => {"checked" => 41_882, "verified" => 41_882, "breaks" => 0}
+    #   result.first_break  # => nil
+    #
+    # ============================================================================
+    # WHAT A CHAIN DETECTS, EXACTLY. Each event carries the digest of the one
+    # before it, so an event that is later rewritten or removed stops linking up
+    # with its successors, and this walk finds the first place that happens. That
+    # is a real and useful property: ordinary corruption, a well-meant
+    # `update_column`, a restored partial backup, and a row edited by hand all
+    # show up here.
+    #
+    # What it does NOT do is stop, or detect, a privileged actor who rewrites an
+    # event AND every digest that follows it. Whoever can write the events table
+    # can usually write this table too, and a chain whose head lives in the same
+    # database as the chain cannot say otherwise. That is precisely the gap the
+    # optional independent anchor adapter addresses, and even then the claim is
+    # only ever as strong as the anchor. The chain makes rewriting history
+    # detectable for as long as the head remains trustworthy — no more than that,
+    # and this class never says more than that.
+    # ============================================================================
+    class Chain
+      BATCH_SIZE = 1_000
+
+      # Every way a walk can stop lining up, as a stable symbol so a monitor can
+      # branch on it without matching English.
+      REASONS = %i[
+        digest_does_not_match
+        previous_digest_does_not_link
+        sequence_gap
+        earlier_events_missing
+        chain_head_missing
+        chain_tail_missing
+        chain_head_event_mismatch
+        chain_head_digest_mismatch
+      ].freeze
+
+      Break = Data.define(:chain_scope, :chain_sequence, :event_id, :reason, :detail) do
+        def to_h
+          {
+            "chain_scope" => chain_scope,
+            "chain_sequence" => chain_sequence,
+            "event_id" => event_id,
+            "reason" => reason.to_s,
+            "detail" => detail
+          }.compact
+        end
+
+        def to_s = "#{chain_scope}##{chain_sequence} #{event_id}: #{detail}"
+      end
+
+      Result = Data.define(
+        :chaining_enabled,
+        :checked,
+        :verified,
+        :documented_dispositions,
+        :scopes,
+        :breaks,
+        :started_mid_chain
+      ) do
+        def success? = breaks.empty?
+        def first_break = breaks.first
+
+        def counts
+          {
+            "checked" => checked,
+            "verified" => verified,
+            "documented_dispositions" => documented_dispositions,
+            "breaks" => breaks.length,
+            "scopes" => scopes.length
+          }
+        end
+
+        def to_h
+          {
+            "chaining_enabled" => chaining_enabled,
+            "counts" => counts,
+            "scopes" => scopes,
+            "started_mid_chain" => started_mid_chain,
+            "first_break" => first_break&.to_h,
+            "breaks" => breaks.map(&:to_h),
+            "detects" => "An event rewritten or removed after it was written, for as long as the " \
+                         "chain head remains trustworthy. Not a rewrite of the events and their " \
+                         "digests together by a privileged actor."
+          }
+        end
+      end
+
+      # `from:` and `to:` accept either a chain sequence number or a time. A
+      # sequence is the natural way to re-check one span of a chain; a time is
+      # the natural way to run "everything since last night" from cron, and
+      # refusing one of them would just make an operator convert by hand.
+      def self.verify(scope: nil, from: nil, to: nil) = new(scope: scope, from: from, to: to).verify
+
+      def initialize(scope: nil, from: nil, to: nil)
+        @scope = scope&.to_s
+        @from = from
+        @to = to
+        @breaks = []
+        @checked = 0
+        @verified = 0
+        @documented_dispositions = 0
+        @started_mid_chain = []
+      end
+
+      attr_reader :scope, :from, :to
+
+      def verify
+        scopes = chain_scopes
+        scopes.each { |chain_scope| walk(chain_scope) }
+
+        Result.new(
+          chaining_enabled: !Clickwrap.config.chain_event_history_with.nil?,
+          checked: @checked,
+          verified: @verified,
+          documented_dispositions: @documented_dispositions,
+          scopes: scopes,
+          breaks: @breaks,
+          started_mid_chain: @started_mid_chain
+        )
+      end
+
+      private
+
+      def chain_scopes
+        return [scope] if scope
+
+        event_scopes = Event.where.not(chain_scope: nil).distinct.pluck(:chain_scope)
+        head_scopes = ChainHead.distinct.pluck(:chain_scope)
+        (event_scopes + head_scopes).compact.uniq.sort
+      end
+
+      def walk(chain_scope)
+        previous = nil
+
+        each_event(chain_scope) do |event|
+          @checked += 1
+
+          if previous.nil?
+            check_first(chain_scope, event)
+          else
+            check_link(chain_scope, event, previous)
+          end
+
+          check_digest(chain_scope, event)
+          previous = event
+        end
+
+        check_durable_head(chain_scope, previous) if full_chain_walk?
+      end
+
+      def check_durable_head(chain_scope, last_event)
+        head = ChainHead.find_by(chain_scope: chain_scope)
+
+        unless head
+          add_head_break(
+            chain_scope,
+            last_event,
+            :chain_head_missing,
+            "Events name this chain scope, but its durable chain-head row is missing. The walk " \
+            "cannot establish whether the newest recorded events are still present."
+          )
+          return
+        end
+
+        if last_event.nil?
+          return if head.chain_sequence.to_i.zero? && head.last_event_id.blank? && head.last_event_digest.blank?
+
+          add_head_break(
+            chain_scope,
+            nil,
+            :chain_tail_missing,
+            "The durable head records sequence #{head.chain_sequence} and event #{head.last_event_id}, " \
+            "but this scope has no event rows. Its recorded tail is missing."
+          )
+          return
+        end
+
+        if last_event.chain_sequence != head.chain_sequence
+          add_head_break(
+            chain_scope,
+            last_event,
+            :chain_tail_missing,
+            "The newest event row is sequence #{last_event.chain_sequence}, while the durable " \
+            "head records sequence #{head.chain_sequence}. One or more newest events are missing."
+          )
+        end
+
+        if last_event.id.to_s != head.last_event_id.to_s
+          add_head_break(
+            chain_scope,
+            last_event,
+            :chain_head_event_mismatch,
+            "The newest event row is #{last_event.id}, while the durable head names " \
+            "#{head.last_event_id}. The head and chain tail do not describe the same event."
+          )
+        end
+
+        return if Digest.secure_compare?(last_event.event_digest.to_s, head.last_event_digest.to_s)
+
+        add_head_break(
+          chain_scope,
+          last_event,
+          :chain_head_digest_mismatch,
+          "The newest event digest does not match the digest retained by the durable chain head."
+        )
+      end
+
+      # Keyset pagination on the (chain_scope, chain_sequence) index. `find_each`
+      # would order by primary key, and the primary key here is a ULID: close to
+      # chain order, but "close to" is not the property a chain walk can rely on.
+      def each_event(chain_scope, &block)
+        cursor = nil
+
+        loop do
+          relation = bounded(Event.where(chain_scope: chain_scope)).order(:chain_sequence).limit(BATCH_SIZE)
+          relation = relation.where(chain_sequence: (cursor + 1)..) if cursor
+          batch = relation.to_a
+          break if batch.empty?
+
+          batch.each(&block)
+          cursor = batch.last.chain_sequence
+          break if cursor.nil?
+        end
+      end
+
+      def bounded(relation)
+        relation = apply_bound(relation, from, :from)
+        apply_bound(relation, to, :to)
+      end
+
+      def apply_bound(relation, value, side)
+        return relation if value.nil?
+
+        column = value.is_a?(Integer) ? :chain_sequence : :recorded_at_by_server
+        range = side == :from ? (value..) : (..value)
+
+        relation.where(column => range)
+      end
+
+      # The first event of a walk is the one link that cannot be checked against
+      # a predecessor this walk has seen. Sequence 1 with no previous digest is a
+      # genuine chain start; anything else is either a deliberately bounded run
+      # (which is reported, not hidden) or a chain missing its beginning.
+      def check_first(chain_scope, event)
+        return if event.chain_sequence == 1 && event.previous_event_digest.blank?
+
+        if bounded_run?
+          @started_mid_chain << { "chain_scope" => chain_scope, "chain_sequence" => event.chain_sequence }
+          return
+        end
+
+        add_break(chain_scope, event, :earlier_events_missing,
+                  "The walk starts at sequence #{event.chain_sequence}, so the events before it are " \
+                  "no longer in the table and nothing can check the link into this one.")
+      end
+
+      def check_link(chain_scope, event, previous)
+        unless event.chain_sequence == previous.chain_sequence + 1
+          add_break(chain_scope, event, :sequence_gap,
+                    "Sequence jumps from #{previous.chain_sequence} to #{event.chain_sequence}, so " \
+                    "at least one event between them is gone.")
+        end
+
+        return if Digest.secure_compare?(event.previous_event_digest.to_s, previous.event_digest.to_s)
+
+        add_break(chain_scope, event, :previous_digest_does_not_link,
+                  "This event records a different predecessor digest than event #{previous.id} " \
+                  "actually has, so the two no longer form a chain.")
+      end
+
+      def check_digest(chain_scope, event)
+        case event.digest_integrity_status
+        when :verified
+          @verified += 1
+        when :documented_core_disposition
+          @documented_dispositions += 1
+        else
+          add_break(chain_scope, event, :digest_does_not_match,
+                    "Recomputing the canonical body of this event produces a different digest than " \
+                    "the one stored with it, and no valid disposition event accounts for the missing " \
+                    "payload. Its meaningful bytes changed after it was written or its disposition " \
+                    "marker is unexplained. That does not, on its own, say who changed it or when.")
+        end
+      end
+
+      def add_break(chain_scope, event, reason, detail)
+        @breaks << Break.new(chain_scope: chain_scope, chain_sequence: event.chain_sequence,
+                             event_id: event.id, reason: reason, detail: detail)
+      end
+
+      def add_head_break(chain_scope, event, reason, detail)
+        @breaks << Break.new(
+          chain_scope: chain_scope,
+          chain_sequence: event&.chain_sequence,
+          event_id: event&.id,
+          reason: reason,
+          detail: detail
+        )
+      end
+
+      def bounded_run? = !from.nil?
+      def full_chain_walk? = from.nil? && to.nil?
+    end
+  end
+end