railwatch 0.2.2 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c0487618698d9dc329892ddabb4762ead8838ce593052b9fcb9fe684163dca15
-  data.tar.gz: f0704d125a03f31aee2a0a3b60f40413335703e6eb20296e6153885d7ae9b6fb
+  metadata.gz: ace6fc321d145d2e579b808a8ce2813b23bc85060d320f39dde2586da8c7db72
+  data.tar.gz: 545154019cd93d2d3e605da2b5edf7318265d5c9b43c6e086b514da06a0fe43d
 SHA512:
-  metadata.gz: fefe712b31d5b060d1f17856da5e7e05ce1303204ecc8c57b4a8ea59a1f2ea0e9bf427ac66b36201450841802ea7327bf17e9a139e5c448fb0deff851ae5c33d
-  data.tar.gz: 41119e84bb12b026516e3cd48dc7fd36835f32a88574e3ad25e8244dd5396925dccf9205ef41171943997adb57062e579784b76b9b3b6e7b63dedd69c1aede52
+  metadata.gz: 20299db14a1865dcff87eddb613254b2fe83fc7ffc67f55afff8dd4b08e5c4593ee11b6082a4d57a9a7777045c4d5e3fa5df34c5c103bca008fde2bbe2663475
+  data.tar.gz: f1105cf854e7b671aeb72b268959e49207101d09afb97f40ac90715d61739562fb2d494f2c21c443f09a86c49f5e2019af998848394c9d368b2ad0c2e602e0aa

data/CHANGELOG.md

@@ -1,5 +1,35 @@
 # Changelog
 
+## 0.3.0 (2026-09-19)
+
+- An embedded install can now also mirror its telemetry to Railwatch Cloud,
+  or to any receiver speaking the same protocol. It is off unless you ask
+  for it: `c.export_enabled = true` (or `RAILWATCH_EXPORT_ENABLED=true`),
+  reusing the token and ingest URL you already have. A token being present
+  is not consent -- an embedded install that has one configured still sends
+  nothing.
+
+  Records are held in a durable queue in your own telemetry database,
+  admitted in the same transaction as the rows they mirror, and sent by one
+  leased thread. A delivery keeps the exact bytes it will send until the
+  receiver acknowledges it, so a retry is the same delivery rather than a
+  second one; the receiver recognises repeats and answers with the original
+  counts. Local capture never waits on the network, and a queue that cannot
+  drain sheds rather than growing without limit. `railwatch:export:status`
+  shows what is queued; `railwatch:doctor` reports export health, and fails
+  if you asked for mirroring and it cannot work.
+
+  Embedded and cloud installs are otherwise unchanged: same install, same
+  records, same dashboard. See docs/embedded.md.
+
+- Ingest acknowledgements are read more carefully. An all-zero
+  acknowledgement with no reason no longer counts as a successful delivery
+  for a non-empty batch -- nothing legitimate answers a 500-record batch
+  that way, but a proxy error page does, and those were being treated as
+  stored. A paused or over-quota environment still takes the batch rather
+  than burning the retry ladder, and now says so rather than looking like
+  storage.
+
 ## 0.2.2 (2026-09-18)
 
 - README: describe embedded mode. The gem has had two destinations since

data/app/models/railwatch/ingest/batch.rb

@@ -38,6 +38,7 @@ module Railwatch
         @session_buckets = Set.new
         @bucket_cache = {}
         @exception_ids = []
+        @export = nil
       end
 
       # The ledger row for a batch id that has already been written, or nil.
@@ -48,6 +49,10 @@ module Railwatch
       end
 
       def write!
+        # Prepared before mapping and outside the transaction: mirroring sends
+        # what we were given, not what we kept, and encoding is not something
+        # to do while holding the write lock.
+        prepare_export
         map_all
         log_truncations
         accepted = 0
@@ -65,7 +70,8 @@ module Railwatch
                                                        dropped_by_client: @dropped_by_client, backpressure_factor: @backpressure_factor,
                                                        bytes: @bytes, gem_version: @gem_version,
                                                        counts_by_type: @counts, rejections: @rejections.first(20),
-                                                       batch_id: @batch_id, followups: followups)
+                                                       batch_id: @batch_id, followups: followups,
+                                                       **export_columns)
             end
           end
         end
@@ -76,6 +82,9 @@ module Railwatch
           @environment.count_events!(accepted)
         end
         enqueue_followups
+        # After the transaction: the sender must never find a delivery that
+        # has not committed yet.
+        Export::Sender.wake! if @queued
         broadcast_live
         Result.new(accepted: accepted, rejected: @rejections.size, rejections: @rejections)
       end
@@ -94,6 +103,46 @@ module Railwatch
 
       private
 
+      # Mirroring, when this install has been told to. Off is the whole of the
+      # cost: no encode, no query, no row, one boolean.
+      def prepare_export
+        return unless @embedded && Railwatch.config.export?
+
+        @export = Export::Outbox.new(Railwatch.config, @environment)
+        @selections = Export::Policy.fetch(Railwatch.config.export_policy).prepare(
+          records: @records, encoder: Transport::WireEncoder.new(batch_bytes: Railwatch.config.batch_bytes),
+          source_batch_id: @batch_id || SecureRandom.uuid,
+          metadata: { "dropped" => @dropped_by_client, "backpressure_factor" => @backpressure_factor.to_s }
+        )
+      rescue StandardError => e
+        # A batch must still be stored when mirroring cannot be prepared.
+        Railwatch.debug { "export preparation failed: #{e.class}: #{e.message}" }
+        @export = nil
+        @export_error = "shed_encoding"
+      end
+
+      # Runs inside the batch's own transaction, so the rows and the intent to
+      # mirror them commit together or not at all.
+      # In a savepoint, so a failure here sheds the mirroring and nothing
+      # else. Without it an export table that is missing -- a gem upgraded
+      # without db:prepare, with export already on -- would roll back the
+      # batch that carried it, and every batch after, until nothing was being
+      # recorded at all. Monitoring must degrade to local-only rather than
+      # stop.
+      def export_columns
+        return { export_disposition: @export_error } if @export_error
+        return {} unless @export
+
+        admission = TelemetryRecord.transaction(requires_new: true) do
+          @export.enqueue!(@selections, now: @received_at)
+        end
+        @queued = admission.disposition == "queued"
+        { export_disposition: admission.disposition, export_record_count: admission.record_count }
+      rescue StandardError => e
+        Railwatch.debug { "export admission failed: #{e.class}: #{e.message}" }
+        { export_disposition: "shed_error" }
+      end
+
       ROLLED_UP = %w[request job_attempt scheduled_task command channel_action query outgoing_request cache_event mail visit notification span llm_call].freeze
       MAX_PAST_AGE = 30.days
       MAX_FUTURE_AGE = 1.hour

data/app/models/railwatch/telemetry/export_delivery.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Railwatch
+  module Telemetry
+    # One immutable delivery waiting to be sent, or the record of one that no
+    # longer is. The body is the exact bytes that will go on the wire, kept
+    # until the delivery reaches a terminal state and then freed: a retry has
+    # to be the same delivery, and the receiver recognises it by those bytes.
+    class ExportDelivery < TelemetryRecord
+      DISPOSITIONS = %w[acked rejected expired discarded].freeze
+
+      belongs_to :export_destination
+
+      scope :live, -> { where.not(state: "done") }
+      scope :due, ->(now = Time.current) { live.where(state: "pending").where(next_attempt_at: ..now) }
+      scope :overdue, ->(now = Time.current) { live.where(expires_at: ...now) }
+      scope :oldest_first, -> { order(:id) }
+
+      def live? = state != "done"
+
+      def sending? = state == "sending"
+
+      # True when this claim is still the one allowed to finish the delivery.
+      # A sender that stalled past its lease may wake and complete its request
+      # anyway; the receiver's receipt makes that harmless, but it must not be
+      # able to overwrite what the new owner has since recorded.
+      def held_by?(token, generation)
+        sending? && claim_token == token && claim_generation == generation
+      end
+    end
+  end
+end

data/app/models/railwatch/telemetry/export_destination.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+module Railwatch
+  module Telemetry
+    # Where this database mirrors its telemetry, and everything durable about
+    # that relationship: who we are to the receiver, which token we were bound
+    # with, whether it is currently taking deliveries, and who holds the lease
+    # to send them.
+    class ExportDestination < TelemetryRecord
+      STATES = %w[ready deferred unauthorized inactive].freeze
+      COUNTERS = %w[acked rejected expired discarded shed].freeze
+
+      has_many :export_deliveries, dependent: :delete_all
+
+      validates :state, inclusion: { in: STATES }
+
+      def self.digest(value) = Digest::SHA256.hexdigest(value.to_s)
+
+      # The binding for this url and token. A different token for the same url
+      # is a different binding decision, not a silent rebind: queued bytes
+      # were admitted under the old credential and must not follow the new one
+      # to whatever tenant it belongs to.
+      def self.bind!(url:, token:, now: Time.current, retried: false)
+        row = find_or_initialize_by(url_sha256: digest(url))
+        row.url = url
+        row.producer_id ||= SecureRandom.uuid
+        fingerprint = digest(token)
+        if row.persisted? && row.credential_sha256 != fingerprint
+          row.update!(credential_sha256: fingerprint, state: "unauthorized", reason: "credential_changed",
+                      retry_at: nil)
+        else
+          row.credential_sha256 = fingerprint
+          row.created_at ||= now
+          row.save!
+        end
+        row
+      rescue ActiveRecord::RecordNotUnique
+        # Two processes binding for the first time at once. One row wins; the
+        # loser wants that row, not an error.
+        raise if retried
+
+        bind!(url: url, token: token, now: now, retried: true)
+      end
+
+      # A pause the receiver asked for ends when it said it would. Only a
+      # credential problem needs a person: everything else is a delay, and a
+      # delay that never ends is an outage we caused ourselves.
+      # Needs a person before anything can move again: a credential that was
+      # changed, or a destination taken out of service. Queueing into one
+      # just fills it with work that rebind! will throw away.
+      def blocked? = %w[unauthorized inactive].include?(state)
+
+      def sendable?(now: Time.current)
+        return false unless %w[ready deferred].include?(state)
+
+        retry_at.nil? || retry_at <= now
+      end
+
+      def bump!(counter, by = 1)
+        return unless COUNTERS.include?(counter.to_s)
+
+        self.counters = counters.merge(counter.to_s => counters.fetch(counter.to_s, 0) + by)
+      end
+    end
+  end
+end

data/db/railwatch_telemetry_migrate/20260919000000_create_export_queue.rb

@@ -0,0 +1,108 @@
+# frozen_string_literal: true
+
+# The durable queue an embedded install uses to mirror its telemetry to a
+# remote receiver.
+#
+# Why the bodies are stored rather than re-derived from the rows next door:
+# the rows are not a reversible copy of what was sent. Mapping renames and
+# drops fields, caps strings, swaps a query's SQL for a shape reference and
+# coalesces people; and ingest rejects records the receiver may well accept,
+# so sending only what we kept would not be the same telemetry. A delivery
+# therefore keeps the exact bytes it will send, and frees them the moment it
+# reaches a terminal state -- the cost tracks what is unacknowledged, not what
+# is stored.
+#
+# Nothing here is written unless export is explicitly enabled.
+class CreateExportQueue < ActiveRecord::Migration[8.1]
+  def change
+    # One row per destination this database has ever been pointed at.
+    create_table :export_destinations do |t|
+      t.string :url, null: false
+      t.string :url_sha256, limit: 64, null: false
+      # Permanent for this database's lineage: it is how the receiver tells
+      # our deliveries from another installation's.
+      t.string :producer_id, limit: 36, null: false
+      # A fingerprint, never the token. If the token changes, queued bytes
+      # must not follow it to whatever tenant the new one belongs to.
+      t.string :credential_sha256, limit: 64, null: false
+
+      t.string :state, limit: 16, null: false, default: "ready"
+      t.datetime :retry_at, precision: 6
+      t.string :reason, limit: 64
+
+      t.string :lease_owner, limit: 36
+      t.bigint :lease_generation, null: false, default: 0
+      t.datetime :lease_expires_at, precision: 6
+
+      # Live totals, maintained in the same transaction as the rows they
+      # describe, so admission can be decided without counting the table.
+      t.bigint :queued_bytes, null: false, default: 0
+      t.bigint :queued_deliveries, null: false, default: 0
+      # Lifetime accounting: acked, rejected, expired, discarded, shed.
+      t.json :counters, null: false, default: {}
+
+      t.timestamps
+    end
+
+    add_index :export_destinations, :url_sha256, unique: true
+    add_index :export_destinations, :producer_id, unique: true
+
+    create_table :export_deliveries do |t|
+      t.references :export_destination, null: false, foreign_key: true
+      # A UUIDv7: its embedded time is how the receiver ages it out, so a
+      # delivery cannot be made young again by relabelling it.
+      t.string :delivery_id, limit: 36, null: false
+      # What this delivery is a delivery OF. One per source batch today; a
+      # future policy that selects across batches supplies its own key.
+      t.string :selection_key, limit: 160, null: false
+
+      t.binary :body
+      t.string :body_sha256, limit: 64, null: false
+      t.string :metadata_sha256, limit: 64, null: false
+      t.bigint :body_bytes, null: false
+      t.bigint :ndjson_bytes, null: false
+      t.integer :record_count, null: false
+      # Everything about the delivery that is not its body: version, drop
+      # counts, backpressure. Digested into metadata_sha256 so the same id
+      # arriving with different counts is a conflict, not an update.
+      t.json :wire_metadata, null: false, default: {}
+
+      t.string :state, limit: 8, null: false, default: "pending"
+      # Set only when done: acked, rejected, expired, discarded.
+      t.string :disposition, limit: 16
+      t.datetime :enqueued_at, null: false, precision: 6
+      t.datetime :expires_at, null: false, precision: 6
+      t.datetime :next_attempt_at, null: false, precision: 6
+      t.bigint :attempts, null: false, default: 0
+
+      t.string :claim_token, limit: 36
+      t.bigint :claim_generation
+      t.datetime :claim_expires_at, precision: 6
+
+      t.integer :last_status
+      t.string :last_reason, limit: 64
+      t.json :ack
+      t.datetime :finished_at, precision: 6
+
+      t.timestamps
+    end
+
+    add_index :export_deliveries, %i[export_destination_id delivery_id], unique: true,
+      name: "index_export_deliveries_on_destination_and_delivery"
+    # One delivery per selection: a batch replayed into the same transaction
+    # cannot enqueue itself twice.
+    add_index :export_deliveries, %i[export_destination_id selection_key], unique: true,
+      name: "index_export_deliveries_on_destination_and_selection"
+    add_index :export_deliveries, %i[export_destination_id id], where: "state <> 'done'",
+      name: "index_export_deliveries_live"
+    add_index :export_deliveries, :expires_at, where: "body IS NOT NULL",
+      name: "index_export_deliveries_expiring"
+    add_index :export_deliveries, :finished_at, where: "state = 'done'",
+      name: "index_export_deliveries_finished"
+
+    # Why a batch did or did not enqueue anything. Null on every row an
+    # install without export ever writes.
+    add_column :ingest_batches, :export_disposition, :string, limit: 24
+    add_column :ingest_batches, :export_record_count, :bigint
+  end
+end

data/docs/embedded.md

@@ -14,6 +14,36 @@ place for many apps, point the gem at Railwatch Cloud instead
 ([Getting started](getting-started.md)); the two are switchable with
 one setting.
 
+## Three ways to run it
+
+Same gem, same install, same records. The only question is where they end
+up:
+
+| | Records live | Dashboard |
+|---|---|---|
+| **Embedded** | your app's SQLite files | `/railwatch` in your app |
+| **Cloud** | Railwatch Cloud | the hosted one |
+| **Both** | your app's files, *and* Railwatch Cloud | either |
+
+Embedded is `c.transport = :local`, which is what `--local` writes.
+Cloud is the default. "Both" is embedded plus one more line:
+
+```ruby
+c.export_enabled = true   # or RAILWATCH_EXPORT_ENABLED=true
+```
+
+It reuses the token and ingest URL you already have, so an install that
+was pointed at the cloud and moved to embedded needs nothing else to send
+to both. Everything captured locally is mirrored — the same records the
+same install would have sent had you chosen the cloud — so the hosted
+dashboard is as complete as it would be either way.
+
+It is off unless you set that flag. A token being present is not consent:
+an embedded install that has one configured still sends nothing.
+`railwatch:doctor` says nothing about export until you ask for it, and
+fails loudly if you ask for it and it cannot work. `railwatch:export:status`
+shows what is queued.
+
 ## Install
 
 ```sh

data/lib/railwatch/configuration.rb

@@ -65,6 +65,8 @@ module Railwatch
     # "they left the default".
     DEFAULT_BASE_CONTROLLER = "ActionController::Base"
 
+    attr_accessor :export_enabled, :export_policy, :export_url, :export_token, :export_max_bytes,
+                  :export_max_deliveries, :export_max_age
     attr_accessor :enabled, :token, :ingest_url, :allow_http, :server, :environment, :transport,
                   :issue_prefix, :repository_url, :retention_days, :dashboard_user, :writer_socket,
                   :http_basic_auth_enabled, :http_basic_auth_user, :http_basic_auth_password, :base_controller_class,
@@ -103,6 +105,16 @@ module Railwatch
       @repository_url = ENV["RAILWATCH_REPOSITORY_URL"]
       @retention_days = env_int("RAILWATCH_RETENTION_DAYS", 7)
       @dashboard_user = nil
+      # Mirroring an embedded install's telemetry to a remote receiver. Off
+      # unless asked for: an embedded install's promise is that nothing leaves
+      # the machine, and a token being present is not consent.
+      @export_enabled = env_bool("RAILWATCH_EXPORT_ENABLED", false)
+      @export_policy = ENV.fetch("RAILWATCH_EXPORT_POLICY", "everything").to_sym
+      @export_url = ENV["RAILWATCH_EXPORT_URL"]
+      @export_token = ENV["RAILWATCH_EXPORT_TOKEN"]
+      @export_max_bytes = env_int("RAILWATCH_EXPORT_MAX_BYTES", 256 * 1024 * 1024)
+      @export_max_deliveries = env_int("RAILWATCH_EXPORT_MAX_DELIVERIES", 100_000)
+      @export_max_age = env_int("RAILWATCH_EXPORT_MAX_AGE_SECONDS", 86_400)
       # Dashboard access, the way Mission Control Jobs does it: HTTP Basic
       # authentication is on and CLOSED by default. With no user and password
       # configured every dashboard request is 401, so an install that forgot
@@ -300,6 +312,46 @@ module Railwatch
     # anything else ships it to ingest_url over HTTPS.
     def local? = transport.to_s == "local"
 
+    # The receiver admits an unseen delivery for seven days; queueing one for
+    # longer cannot help.
+    MAX_EXPORT_AGE = 7 * 24 * 60 * 60
+
+    # Mirroring is a thing an embedded install opts into; it is meaningless
+    # for an install that is already sending everything over HTTP.
+    def export? = export_enabled && local? && export_problem.nil?
+
+    # Why export is configured but unusable, or nil when it is fine. The
+    # doctor reports this; nothing silently half-enables.
+    def export_problem
+      return nil unless export_enabled
+      return "export needs transport :local; an :http install already sends everything" unless local?
+      return "RAILWATCH_EXPORT_POLICY #{export_policy} is not implemented" unless export_policy.to_s == "everything"
+      return "no export token: set RAILWATCH_EXPORT_TOKEN or RAILWATCH_TOKEN" if resolved_export_token.to_s.empty?
+      return "no export url: set RAILWATCH_EXPORT_URL or RAILWATCH_INGEST_URL" if resolved_export_url.to_s.empty?
+      return "export url must be HTTPS (or set RAILWATCH_ALLOW_HTTP=true)" unless url_allowed?(resolved_export_url)
+      # Past this a receiver stops recognising a delivery's id, so holding one
+      # any longer just means discovering later that it can never be sent.
+      if export_max_age > MAX_EXPORT_AGE
+        return "RAILWATCH_EXPORT_MAX_AGE_SECONDS cannot exceed #{MAX_EXPORT_AGE} (the receiver stops recognising a delivery past that)"
+      end
+
+      nil
+    end
+
+    # The receiver, and the credential we are bound to it with. Both fall back
+    # to the ordinary ingest settings so switching an install from embedded to
+    # cloud needs no second set of values.
+    def resolved_export_url
+      url = export_url.presence || (ingest_url.presence && URI.join(ingest_url, "/ingest").to_s)
+      url&.sub(%r{/\z}, "")
+    rescue URI::Error
+      # BadURIError (a relative ingest_url) is not an InvalidURIError, and
+      # letting it out of here takes the doctor down with it.
+      nil
+    end
+
+    def resolved_export_token = export_token.presence || token
+
     # Absolute path of the writer socket, or nil when the writer is off.
     def writer_socket_path
       path = writer_socket.to_s
@@ -373,7 +425,15 @@ module Railwatch
     end
 
     def ingest_url_allowed?
-      uri = URI.parse(ingest_url.to_s)
+      url_allowed?(ingest_url)
+    end
+
+    # The same policy, applied to whatever URL is actually about to be
+    # requested. A transport pointed at an explicit endpoint must be judged on
+    # that endpoint: approving it because some other configured URL happens to
+    # be HTTPS would put the token on the wire in plaintext.
+    def url_allowed?(url)
+      uri = url.is_a?(URI::Generic) ? url : URI.parse(url.to_s)
       return true if uri.scheme == "https"
       return false unless uri.scheme == "http"
 

data/lib/railwatch/engine.rb

@@ -216,6 +216,17 @@ module Railwatch
       at_exit { Railwatch::Maintenance.stop! }
     end
 
+    # Draining the export queue. Same shape and ordering as the others:
+    # stopped before the reporter's final flush, so its last claim is
+    # released rather than left to expire. Sender.start! is a no-op unless
+    # export is configured and usable.
+    initializer "railwatch.export" do
+      next unless Railwatch.enabled?
+
+      Railwatch::Export::Sender.start!
+      at_exit { Railwatch::Export::Sender.stop! }
+    end
+
     # lib/tasks/railwatch_tasks.rake is picked up by Rails::Engine's default
     # lib/tasks convention; the rake_tasks block above only installs the
     # Rake::Task patch.

data/lib/railwatch/export/client.rb

@@ -0,0 +1,117 @@
+# frozen_string_literal: true
+
+module Railwatch
+  module Export
+    # Speaks the receipt protocol: names the delivery so the receiver can
+    # recognise a repeat of it, and turns the answer into something the queue
+    # can act on.
+    #
+    # It makes one attempt and has no opinion about when to try again. That
+    # belongs to the queue, which is the thing with durable storage.
+    class Client
+      Outcome = Struct.new(:disposition, :status, :reason, :retry_after_at, :ack, keyword_init: true)
+
+      # The queue is bound to a fingerprint of the export credential, so the
+      # request has to be made with that credential: authenticating as someone
+      # else would deliver these bytes to a tenant that never admitted them.
+      def initialize(config)
+        @config = config
+        @transports = {}
+      end
+
+      # Not `send`: shadowing Object#send on a class makes it impossible to
+      # reach the real one, and reads as a coincidence rather than a verb.
+      def deliver(claim, producer_id:)
+        result = transport_for(claim).deliver_encoded(
+          body: claim.body, expected_count: claim.record_count, batch_id: claim.delivery_id,
+          dropped: claim.metadata.fetch("dropped", 0).to_i,
+          dropped_bytes: claim.metadata.fetch("dropped_bytes", 0).to_i,
+          # Stored with the delivery, not read from the reporter now: this is
+          # what the batch was carrying when it was queued.
+          backpressure_factor: claim.metadata.fetch("backpressure_factor", 1.0).to_f,
+          gem_version: claim.metadata.fetch("version", Railwatch::VERSION),
+          headers: headers(claim, producer_id)
+        )
+        interpret(result)
+      end
+
+      def reset_after_fork!
+        @transports = {}
+        self
+      end
+
+      # Drops any latch the transports have picked up. After a credential
+      # problem is fixed, the next send must be an actual send.
+      def reset!
+        @transports = {}
+        self
+      end
+
+      private
+
+      # Keyed on the destination this delivery is FOR, not on whatever the
+      # configuration says right now. A process reconfigured mid-flight would
+      # otherwise post bytes admitted for one receiver to another one -- and,
+      # if they belong to different tenants, hand a customer's telemetry to
+      # somebody else while marking it delivered.
+      def transport_for(claim)
+        key = [ claim.url, claim.token_digest ]
+        @transports[key] ||= begin
+          credentialed = @config.dup
+          credentialed.token = @config.resolved_export_token
+          Transport::Http.new(credentialed, endpoint: claim.url)
+        end
+      end
+
+      def headers(claim, producer_id)
+        {
+          "X-Railwatch-Producer-Id" => producer_id,
+          "X-Railwatch-Body-SHA256" => Digest::SHA256.hexdigest(claim.body),
+          "X-Railwatch-Policy" => claim.metadata.fetch("policy", Policy::Everything::VERSION)
+        }
+      end
+
+      # The receiver's answer, reduced to what the queue needs to decide.
+      # Anything it cannot read is treated as "not stored": keeping bytes we
+      # might not need costs a retry, discarding bytes that never arrived
+      # costs the telemetry.
+      STORED = %w[committed already_committed].freeze
+      REJECTING_STATUSES = [ 400, 409, 410, 413, 422 ].freeze
+
+      def interpret(result)
+        return Outcome.new(disposition: :stored, status: result.status, ack: result.to_h) if stored?(result)
+        return Outcome.new(disposition: :rejected, status: result.status, reason: reason_for(result)) if rejected?(result)
+
+        Outcome.new(disposition: :deferred, status: result.status, reason: reason_for(result),
+                    retry_after_at: result.retry_after_at)
+      end
+
+      # A 200 that stored the records, or a duplicate the receiver already
+      # holds. When the receiver names what it did, we believe the name and
+      # not the arithmetic: counts that happen to add up are not a receipt.
+      def stored?(result)
+        return false unless result.ok && !result.deferred?
+        return STORED.include?(result.ack_disposition) if result.ack_disposition
+
+        true
+      end
+
+      # Permanently unacceptable TO THE RECEIVER: sending it again cannot
+      # change the answer. 409 is a delivery id reused for different bytes,
+      # 410 one too old to be recognised.
+      #
+      # Deliberately not our own refusals. `:permanent` from the transport
+      # means we declined to send -- a latched credential failure, a
+      # configuration we will not use -- and throwing the bytes away because
+      # of something on this side would destroy telemetry that was never
+      # offered to anyone.
+      def rejected?(result)
+        REJECTING_STATUSES.include?(result.status)
+      end
+
+      def reason_for(result)
+        (result.reason || result.error || result.status).to_s[0, 64]
+      end
+    end
+  end
+end

data/lib/railwatch/export/lease.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+module Railwatch
+  module Export
+    # Which process is allowed to send, right now.
+    #
+    # Every eligible process runs a sender; the lease decides which one does
+    # anything. It is held for a short time and renewed, so a process that
+    # dies holding it blocks the queue for seconds rather than forever.
+    #
+    # The generation is a fence. A holder that stalls past its expiry may wake
+    # and finish a request it had already started -- the receipt at the other
+    # end makes that harmless -- but it must not then be able to overwrite
+    # what the new holder has since recorded. Every write it attempts carries
+    # the generation it was granted, and a stale one matches nothing.
+    module Lease
+      TTL = 30
+      # Renewed well inside the TTL: a renewal that has to wait on the write
+      # lock still has room to land before the lease it is extending lapses.
+      RENEW_EVERY = 10
+
+      module_function
+
+      # Takes the lease, or extends it if we already hold it. Returns the
+      # generation we hold it under, or nil if someone else has it.
+      def acquire(destination_id, owner:, now: Time.current)
+        taken = Telemetry::ExportDestination
+          .where(id: destination_id)
+          .where("lease_owner IS NULL OR lease_owner = ? OR lease_expires_at < ?", owner, now)
+          .update_all([
+            "lease_owner = ?, lease_expires_at = ?, lease_generation = lease_generation + 1, updated_at = ?",
+            owner, now + TTL, now
+          ])
+        return nil if taken.zero?
+
+        Telemetry::ExportDestination.where(id: destination_id).pick(:lease_generation)
+      end
+
+      # Extends a lease we still hold. Cannot revive an expired one: taking it
+      # again is acquire's job, and that increments the generation so anything
+      # in flight under the old one is fenced out.
+      def renew(destination_id, owner:, generation:, now: Time.current)
+        Telemetry::ExportDestination
+          .where(id: destination_id, lease_owner: owner, lease_generation: generation)
+          .where(lease_expires_at: now..)
+          .update_all([ "lease_expires_at = ?, updated_at = ?", now + TTL, now ]) == 1
+      end
+
+      def release(destination_id, owner:, generation:, now: Time.current)
+        Telemetry::ExportDestination
+          .where(id: destination_id, lease_owner: owner, lease_generation: generation)
+          .update_all([ "lease_owner = NULL, lease_expires_at = NULL, updated_at = ?", now ]) == 1
+      end
+    end
+  end
+end