railwatch 0.2.2 → 0.3.0

data/lib/railwatch/transport/http.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "net/http"
+require "time"
 require "openssl"
 require "zlib"
 require "json"
@@ -15,20 +16,34 @@ module Railwatch
       RETRYABLE_STATUSES = [ 402, 408, 429 ].freeze
       UNAUTHORIZED_STATUS = 401
 
-      Result = Struct.new(:ok, :status, :accepted, :rejected, :rejections, :error, :retryable_error, keyword_init: true) do
+      # `reason` and `retry_after_at` are what the receiver said about this
+      # delivery beyond its counts: why it was not stored, and when to come
+      # back. They are carried rather than discarded so a caller with durable
+      # storage can wait instead of guessing.
+      Result = Struct.new(:ok, :status, :accepted, :rejected, :rejections, :error, :retryable_error,
+                          :reason, :retry_after_at, :disposition, :ack_disposition, keyword_init: true) do
         def retryable?
+          # A permanent failure says so outright: without this, its absent
+          # status would read as "no response yet", which is retryable.
+          return false if disposition == :permanent
+
           !ok && (retryable_error || Http.retryable_status?(status))
         end
+
+        # The receiver took the batch off our hands without storing it: a
+        # paused or over-quota environment. Not a failure, and not storage.
+        def deferred? = disposition == :deferred
       end
 
       def self.retryable_status?(status)
         status.nil? || RETRYABLE_STATUSES.include?(status) || (500..599).cover?(status)
       end
 
-      def initialize(config)
+      def initialize(config, endpoint: nil)
         @config = config
-        @uri = URI.join(config.ingest_url, "/ingest")
+        @uri = endpoint ? URI.parse(endpoint) : URI.join(config.ingest_url, "/ingest")
         @unauthorized = false
+        @encoder = WireEncoder.new(batch_bytes: config.batch_bytes)
       end
 
       def unauthorized?
@@ -43,12 +58,16 @@ module Railwatch
       end
 
       def deliver(records, dropped: 0, dropped_bytes: 0, backpressure_factor: 1.0, batch_id: SecureRandom.uuid)
-        unless @config.ingest_url_allowed?
+        unless destination_allowed?
           return Result.new(ok: false, error: "plain HTTP ingest is disabled; use HTTPS or set RAILWATCH_ALLOW_HTTP=true")
         end
         return Result.new(ok: false, status: UNAUTHORIZED_STATUS, error: "unauthorized, flushing stopped") if @unauthorized
 
-        body, sent, over_cap, over_cap_bytes = encode(records)
+        encoded = @encoder.encode(records)
+        body = encoded.body
+        sent = encoded.sent
+        over_cap = encoded.over_cap
+        over_cap_bytes = encoded.over_cap_bytes
         if over_cap.positive?
           # Not a delivery failure: a batch this large will be exactly as
           # large on every retry, so raising a retryable error here would burn
@@ -73,8 +92,33 @@ module Railwatch
         end
       end
 
+      # Sends bytes that were encoded earlier and stored. Exactly one attempt:
+      # the caller owns a durable queue and its own retry schedule, and
+      # multiplying two ladders together would turn one backoff into sixty-four.
+      def deliver_encoded(body:, expected_count:, batch_id:, headers: {}, dropped: 0, dropped_bytes: 0,
+                          backpressure_factor: 1.0, gem_version: Railwatch::VERSION)
+        unless destination_allowed?
+          return permanent("plain HTTP ingest is disabled; use HTTPS or set RAILWATCH_ALLOW_HTTP=true")
+        end
+        # The same latch deliver honours. A caller with its own queue would
+        # otherwise keep presenting a token the receiver has already refused.
+        return permanent("unauthorized, flushing stopped", status: UNAUTHORIZED_STATUS) if @unauthorized
+
+        response = post(body, dropped, dropped_bytes, backpressure_factor, batch_id,
+                        headers: headers, gem_version: gem_version)
+        result = parse(response, expected_count: expected_count)
+        apply_status_policy(result)
+        result
+      rescue ArgumentError, URI::Error, TypeError, NoMethodError => e
+        # Bad input or bad configuration, not a bad network. Retrying the same
+        # stored bytes cannot fix it, and a durable queue would retry forever.
+        permanent("#{e.class}: #{e.message}")
+      rescue StandardError => e
+        Result.new(ok: false, error: "#{e.class}: #{e.message}", retryable_error: true)
+      end
+
       def ping
-        return false unless @config.ingest_url_allowed?
+        return false unless destination_allowed?
 
         response = request(Net::HTTP::Get.new(URI.join(@config.ingest_url, "/ingest/ping")))
         response.is_a?(Net::HTTPSuccess)
@@ -84,37 +128,19 @@ module Railwatch
 
       private
 
-      # The one serialization of the batch, so it is also where its exact
-      # uncompressed size is known. Records past config.batch_bytes are left
-      # out and reported back to the caller rather than growing the request
-      # without limit. Returns [body, records written, records left out,
-      # bytes left out].
-      def encode(records)
-        io = StringIO.new
-        gz = Zlib::GzipWriter.new(io)
-        bytes = 0
-        sent = 0
-        over_cap = 0
-        over_cap_bytes = 0
-        records.each do |record|
-          json = JSON.generate(record)
-          size = json.bytesize + 1
-          if bytes + size > @config.batch_bytes
-            over_cap += 1
-            over_cap_bytes += size
-            next
-          end
-          gz.write(json)
-          gz.write("\n")
-          bytes += size
-          sent += 1
-        end
-        gz.close
-        [ io.string, sent, over_cap, over_cap_bytes ]
+      def destination_allowed? = @config.url_allowed?(@uri)
+
+      # A failure the caller must not retry: nothing about repeating it can
+      # change the outcome. `retryable?` treats a nil status as transient, so
+      # these say so explicitly.
+      def permanent(error, status: nil)
+        Result.new(ok: false, status: status, error: error, retryable_error: false, disposition: :permanent)
       end
 
-      def post(body, dropped, dropped_bytes, backpressure_factor, batch_id)
+      def post(body, dropped, dropped_bytes, backpressure_factor, batch_id, headers: {},
+               gem_version: Railwatch::VERSION)
         req = Net::HTTP::Post.new(@uri)
+        headers.each { |name, value| req[name] = value.to_s }
         req["Content-Type"] = "application/x-ndjson"
         req["Content-Encoding"] = "gzip"
         req["X-Railwatch-Dropped"] = dropped.to_s if dropped.positive?
@@ -122,7 +148,11 @@ module Railwatch
         if backpressure_factor > 1.0
           req["X-Railwatch-Backpressure-Factor"] = backpressure_factor.to_s
         end
-        req["X-Railwatch-Version"] = Railwatch::VERSION
+        # The version this payload was built by, which for a stored delivery is
+        # not the version running now. The receiver digests this header, so
+        # sending today's value would turn an upgrade into a conflict and
+        # destroy a delivery it had already accepted.
+        req["X-Railwatch-Version"] = gem_version
         req["X-Railwatch-Batch-Id"] = batch_id
         req.body = body
         request(req)
@@ -135,7 +165,10 @@ module Railwatch
           use_ssl: @uri.scheme == "https",
           open_timeout: @config.connect_timeout,
           read_timeout: @config.timeout,
-          write_timeout: @config.timeout
+          write_timeout: @config.timeout,
+          # POST is not in Net::HTTP's idempotent retry set, but say so:
+          # a caller with its own queue must be able to trust "one attempt".
+          max_retries: 0
         }
         # Net::HTTP currently defaults HTTPS clients to VERIFY_PEER. Set it
         # explicitly so a Ruby default change cannot silently weaken ingest.
@@ -145,16 +178,59 @@ module Railwatch
         end
       end
 
+      # A receiver in trouble can answer with something enormous -- a proxy
+      # error page, a stack trace. Only ever look at the first slice of it.
+      MAX_RESPONSE_BYTES = 64 * 1024
+      # The longest delay we will take from a receiver, in either spelling.
+      MAX_RETRY_AFTER = 86_400
+
       def parse(response, expected_count:)
         if response.is_a?(Net::HTTPSuccess)
           parse_acknowledgement(response, expected_count)
         else
-          Result.new(ok: false, status: response.code.to_i, error: response.body.to_s[0, 200])
+          Result.new(ok: false, status: response.code.to_i, error: summarize(response.body),
+                     retry_after_at: retry_after_at(response))
+        end
+      end
+
+      # Bounds what we keep and log, not what Net::HTTP already read off the
+      # socket -- it buffers the whole response before we ever see it.
+      def summarize(body) = body.to_s.byteslice(0, 200).to_s.scrub
+
+      # Seconds, or an HTTP date. Anything else is not a delay we can trust,
+      # so the caller falls back to its own backoff rather than a guess.
+      def retry_after_at(response)
+        raw = response["Retry-After"].to_s.strip
+        return nil if raw.empty?
+
+        now = Time.now
+        if raw.match?(/\A\d{1,7}\z/)
+          seconds = raw.to_i
+          seconds <= MAX_RETRY_AFTER ? now + seconds : nil
+        else
+          parsed = begin
+            Time.httpdate(raw)
+          rescue ArgumentError
+            nil
+          end
+          return nil unless parsed
+          # One second of slack for whole-second HTTP-date precision, and the
+          # same ceiling the numeric form gets: a far-future date must not
+          # park a delivery for years.
+          parsed.between?(now - 1, now + MAX_RETRY_AFTER) ? parsed : nil
         end
       end
 
       def parse_acknowledgement(response, expected_count)
-        data = JSON.parse(response.body)
+        body = response.body.to_s
+        # Refused whole rather than parsed in part: truncating first would let
+        # a padded prefix parse as a complete document, and would reject a
+        # large but valid acknowledgement as malformed JSON.
+        if body.bytesize > MAX_RESPONSE_BYTES
+          return invalid_acknowledgement(response, "acknowledgement larger than #{MAX_RESPONSE_BYTES} bytes")
+        end
+
+        data = JSON.parse(body)
         return invalid_acknowledgement(response, "response must be a JSON object") unless data.is_a?(Hash)
 
         accepted = data["accepted"]
@@ -162,7 +238,8 @@ module Railwatch
         unless accepted.is_a?(Integer) && accepted >= 0 && rejected.is_a?(Integer) && rejected >= 0
           return invalid_acknowledgement(response, "accepted and rejected must be non-negative integers")
         end
-        unless drained?(data, accepted, rejected) || accepted + rejected == expected_count
+        reason = data["reason"].is_a?(String) ? data["reason"][0, 64] : nil
+        if reason.nil? && accepted + rejected != expected_count
           return invalid_acknowledgement(response,
                                          "accepted + rejected was #{accepted + rejected}, expected #{expected_count}")
         end
@@ -173,27 +250,24 @@ module Railwatch
         end
 
         Result.new(ok: true, status: response.code.to_i, accepted: accepted, rejected: rejected,
-                   rejections: Array(rejections).first(10))
-      rescue JSON::ParserError => error
-        invalid_acknowledgement(response, "invalid JSON (#{error.message})")
-      end
-
-      # Ingest can take a whole batch off our hands without storing any of it:
-      # a paused or over-quota environment answers 200 with
-      # {"accepted":0,"rejected":0,"reason":"paused"}. That batch IS delivered
-      # -- the platform decided its fate -- so retrying it would burn eight
-      # attempts and drop the records anyway. Any acknowledgement carrying a
-      # `reason`, and any all-zero acknowledgement, drains the batch.
-      def drained?(data, accepted, rejected)
-        data.key?("reason") || (accepted.zero? && rejected.zero?)
+                   rejections: Array(rejections).first(10), reason: reason,
+                   retry_after_at: retry_after_at(response),
+                   disposition: reason ? :deferred : :stored,
+                   ack_disposition: data["disposition"].is_a?(String) ? data["disposition"][0, 32] : nil)
+      rescue JSON::ParserError
+        # The parser's message quotes the document, which may be a proxy page
+        # echoing the request. Say what happened, not what it contained.
+        invalid_acknowledgement(response, "invalid JSON")
       end
 
       # A proxy-generated 2xx page or a contract mismatch cannot acknowledge
       # the submitted records. Keep the batch for Reporter retry instead of
       # silently treating it as delivered.
       def invalid_acknowledgement(response, detail)
+        # Keep the delay even though we could not read the rest: a receiver
+        # asking for room still means it, whatever state its body was in.
         Result.new(ok: false, status: response.code.to_i, error: "invalid ingest acknowledgement: #{detail}",
-                   retryable_error: true)
+                   retryable_error: true, retry_after_at: retry_after_at(response))
       end
 
       def apply_status_policy(result)

data/lib/railwatch/transport/wire_encoder.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+require "zlib"
+require "stringio"
+require "json"
+require "digest"
+
+module Railwatch
+  module Transport
+    # Turns records into the wire body: gzipped NDJSON, one record per line.
+    #
+    # Lifted out of Transport::Http so a durable queue can hold the encoded
+    # bytes and send them later without re-encoding. A retry has to be the
+    # same delivery, which means the same bytes, which means encoding is
+    # something you do once and keep -- not something you redo per attempt.
+    class WireEncoder
+      # What one encode produced. `over_cap` records did not fit and were
+      # left out; the caller counts them as dropped rather than growing the
+      # request without limit, since they would not fit on a retry either.
+      Encoded = Struct.new(:body, :sha256, :sent, :over_cap, :over_cap_bytes, :uncompressed_bytes,
+                           keyword_init: true)
+
+      def initialize(batch_bytes:)
+        @batch_bytes = batch_bytes
+      end
+
+      def encode(records)
+        io = StringIO.new
+        # mtime 0 so encoding the same records twice on this runtime gives the
+        # same bytes, rather than differing by the second they were encoded.
+        # It is not a portability guarantee -- zlib builds may deflate
+        # identical input differently -- which is why a stored delivery keeps
+        # its bytes and its digest rather than re-deriving them later.
+        gz = Zlib::GzipWriter.new(io, Zlib::DEFAULT_COMPRESSION, Zlib::DEFAULT_STRATEGY)
+        gz.mtime = 0
+        bytes = 0
+        sent = 0
+        over_cap = 0
+        over_cap_bytes = 0
+        records.each do |record|
+          json = JSON.generate(record)
+          size = json.bytesize + 1
+          if bytes + size > @batch_bytes
+            over_cap += 1
+            over_cap_bytes += size
+            next
+          end
+          gz.write(json)
+          gz.write("\n")
+          bytes += size
+          sent += 1
+        end
+        gz.close
+        body = io.string.b
+        Encoded.new(body: body, sha256: Digest::SHA256.hexdigest(body), sent: sent, over_cap: over_cap,
+                    over_cap_bytes: over_cap_bytes, uncompressed_bytes: bytes)
+      end
+    end
+  end
+end

data/lib/railwatch/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Railwatch
-  VERSION = "0.2.2"
+  VERSION = "0.3.0"
 end

data/lib/railwatch.rb

@@ -20,6 +20,12 @@ require "railwatch/execution"
 require "railwatch/current"
 require "railwatch/record"
 require "railwatch/buffer"
+require "railwatch/transport/wire_encoder"
+require "railwatch/export/policy"
+require "railwatch/export/lease"
+require "railwatch/export/outbox"
+require "railwatch/export/client"
+require "railwatch/export/sender"
 require "railwatch/transport/http"
 require "railwatch/transport/local"
 require "railwatch/transport/socket"
@@ -103,6 +109,7 @@ module Railwatch
       Subscribers::Users.restart_after_fork!
       Subscribers::ProcessInfo.restart_after_fork!
       Health.restart_after_fork!
+      Export::Sender.restart_after_fork!
       Sessions.restart_after_fork!
       Maintenance.restart_after_fork!
     end

data/lib/tasks/railwatch_tasks.rake

@@ -95,6 +95,19 @@ namespace :railwatch do
       # the two databases the engine writes to and the jobs that derive
       # rollups and issues from them.
       check.call(true, "transport", "local (telemetry stays in this app; dashboard at the engine mount)")
+      # Mirroring is opt-in, so silence means "not asked for". Asked for and
+      # not working is the case worth failing on: the operator believes their
+      # telemetry is leaving the box and it is not.
+      if config.export_enabled
+        problem = config.export_problem
+        check.call(problem.nil?, "export", problem || "mirroring to #{config.resolved_export_url}", fatal: true)
+        if problem.nil?
+          status = Railwatch::Export::Outbox.new(config, Railwatch::Environment.current)
+                                            .then { |outbox| Railwatch::Environment.current.with_telemetry { outbox.status } }
+          check.call(status[:state] == "ready", "export destination",
+                     status[:state] == "ready" ? "#{status[:queued_deliveries]} queued (#{status[:queued_bytes]} bytes), producer #{status[:producer_id]}" : "#{status[:state]}: #{status[:reason]} (bin/rails railwatch:export:rebind to clear)")
+        end
+      end
       %w[railwatch railwatch_telemetry].each do |name|
         configured = ActiveRecord::Base.configurations.configs_for(env_name: Rails.env, name: name)
         check.call(!configured.nil?, "#{name} database",
@@ -284,6 +297,29 @@ namespace :railwatch do
     puts "\nRailwatch is wired up."
   end
 
+  namespace :export do
+    desc "Show what the export queue is holding and whether it can send"
+    task status: :environment do
+      env = Railwatch::Environment.current
+      status = env.with_telemetry { Railwatch::Export::Outbox.new(Railwatch.config, env).status }
+      status.each { |key, value| puts "#{key}: #{value}" }
+    end
+
+    desc "Clear a credential block, abandoning work admitted under the old token"
+    task rebind: :environment do
+      env = Railwatch::Environment.current
+      discarded = env.with_telemetry { Railwatch::Export::Outbox.new(Railwatch.config, env).rebind! }
+      puts discarded ? "rebound; #{discarded} queued deliveries abandoned" : "export is not configured"
+    end
+
+    desc "Abandon everything queued for export without contacting the receiver"
+    task discard: :environment do
+      env = Railwatch::Environment.current
+      count = env.with_telemetry { Railwatch::Export::Outbox.new(Railwatch.config, env).discard_all! }
+      puts "#{count} queued deliveries abandoned"
+    end
+  end
+
   desc "Print where to create an ingest token for this app's Railwatch platform"
   task token: :environment do
     base = Railwatch::Endpoints.new(Railwatch.config)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: railwatch
 version: !ruby/object:Gem::Version
-  version: 0.2.2
+  version: 0.3.0
 platform: ruby
 authors:
 - Cole Robertson
@@ -173,6 +173,8 @@ files:
 - app/models/railwatch/telemetry/enqueued_job.rb
 - app/models/railwatch/telemetry/exception.rb
 - app/models/railwatch/telemetry/execution.rb
+- app/models/railwatch/telemetry/export_delivery.rb
+- app/models/railwatch/telemetry/export_destination.rb
 - app/models/railwatch/telemetry/health_sample.rb
 - app/models/railwatch/telemetry/ingest_batch.rb
 - app/models/railwatch/telemetry/llm_call.rb
@@ -230,6 +232,7 @@ files:
 - db/railwatch_telemetry_migrate/20260915200000_add_explained_index_to_queries.rb
 - db/railwatch_telemetry_migrate/20260915210000_add_slowest_index_to_queries.rb
 - db/railwatch_telemetry_migrate/20260917010000_add_batch_ledger_to_ingest_batches.rb
+- db/railwatch_telemetry_migrate/20260919000000_create_export_queue.rb
 - docs/ai-and-mcp.md
 - docs/configuration.md
 - docs/embedded.md
@@ -263,6 +266,11 @@ files:
 - lib/railwatch/embedded.rb
 - lib/railwatch/engine.rb
 - lib/railwatch/execution.rb
+- lib/railwatch/export/client.rb
+- lib/railwatch/export/lease.rb
+- lib/railwatch/export/outbox.rb
+- lib/railwatch/export/policy.rb
+- lib/railwatch/export/sender.rb
 - lib/railwatch/faraday.rb
 - lib/railwatch/health.rb
 - lib/railwatch/ingest_request_body_limit.rb
@@ -308,6 +316,7 @@ files:
 - lib/railwatch/transport/http.rb
 - lib/railwatch/transport/local.rb
 - lib/railwatch/transport/socket.rb
+- lib/railwatch/transport/wire_encoder.rb
 - lib/railwatch/version.rb
 - lib/railwatch/writer.rb
 - lib/tasks/railwatch_tasks.rake