railwatch 0.1.4 → 0.2.0

data/lib/railwatch/json_compat.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+require "active_support/json"
+
+module Railwatch
+  # Rails 8.1, up to and including 8.1.3.1, calls `JSON.parse` with a
+  # positional options hash. json 3 made those options keyword arguments
+  # (rails/rails#58784), so on that pair every signed cookie, every JSON
+  # column, and this gem's own SQLite migrations raise ArgumentError. Ruby
+  # 3.4.10 ships json 3.0.2 as a default gem, so a fresh machine meets it
+  # without choosing to. The fix is merged on Rails' 8-1-stable branch and
+  # unreleased as of 8.1.3.1.
+  #
+  # Detected by behaviour, not by version numbers: the pair is asked to
+  # decode a document once. That way the check is right about combinations
+  # nobody has enumerated (a patched Rails, a backport, a future json that
+  # restores the old signature), and it goes quiet by itself the day the
+  # host upgrades, with no release of this gem required.
+  #
+  # Railwatch does not pin `json` for the host. The breakage is the host
+  # application's either way -- its sessions are already failing -- so the
+  # gem reports it and the installer offers the pin, rather than quietly
+  # constraining everybody's bundle for a bug that is not ours and is on its
+  # way out.
+  module JsonCompat
+    PIN = %(gem "json", "< 3")
+    ISSUE = "rails/rails#58784"
+
+    module_function
+
+    def broken?
+      return @broken unless @broken.nil?
+
+      @broken = begin
+        ActiveSupport::JSON.decode("{}")
+        false
+      rescue ArgumentError
+        true
+      rescue StandardError
+        # Anything else is not this bug.
+        false
+      end
+    end
+
+    # One line, usable from the installer, the doctor and a boot log.
+    def advice
+      "json #{json_version} cannot be decoded by Rails #{rails_version} (#{ISSUE}): signed cookies, JSON " \
+        "columns and Railwatch's own migrations all raise ArgumentError on this pair. Add #{PIN} to your " \
+        "Gemfile until Rails ships the fix, then remove it."
+    end
+
+    def json_version = defined?(::JSON::VERSION) ? ::JSON::VERSION : "unknown"
+    def rails_version = defined?(::Rails) && ::Rails.respond_to?(:version) ? ::Rails.version : "unknown"
+
+    # Test hook.
+    def reset!
+      @broken = nil
+    end
+  end
+end

data/lib/railwatch/maintenance.rb

@@ -0,0 +1,183 @@
+# frozen_string_literal: true
+
+# TASKS is built from duration literals at load time; the engine has these
+# loaded already, but a writer process required ahead of the app must not
+# depend on that.
+require "active_support/core_ext/numeric/time"
+
+module Railwatch
+  # The embedded install's own clock. One background thread per web/worker
+  # process wakes every TICK seconds and runs whichever maintenance task is
+  # due: release-health rollups, threshold and anomaly scans, missed
+  # scheduled tasks, auto-resolve, pruning. Every process runs the clock;
+  # one process at a time runs a given task, claimed through a lease row in
+  # the railwatch database (MaintenanceTask), so a Puma cluster and a Solid
+  # Queue worker on the same host do not all prune at once.
+  #
+  # Same thread shape as Railwatch::Health: parked on a ConditionVariable,
+  # started from the engine, re-armed in every forked child. Nothing here
+  # goes through Active Job -- the bodies call the job classes' perform
+  # directly -- so an embedded install needs no worker, no recurring.yml,
+  # and never writes the host's queue adapter.
+  #
+  # A task must never be visible to the app it is maintaining: each body runs
+  # inside Railwatch.ignore and rescues everything.
+  module Maintenance
+    ROLES = %w[web worker writer].freeze
+    TICK = 30
+    FOLLOWUP_BATCHES_PER_TICK = 200
+
+    # name => [interval, lease, body]. The lease is how long a claim is held
+    # by a process that never releases it (crashed mid-task); it is a ceiling
+    # on the work, not an estimate of it.
+    TASKS = {
+      # A batch's exceptions are grouped into issues right after it commits;
+      # a process that dies in between leaves the work recorded on the
+      # batch's ledger row. Bounded per tick so a long outage drains over a
+      # few ticks rather than one long one.
+      "drain_followups" => [ 1.minute, 5.minutes, lambda { |env|
+        env.with_telemetry do
+          Telemetry::IngestBatch.with_pending_followups.limit(FOLLOWUP_BATCHES_PER_TICK).each do |batch|
+            batch.drain_followups!(env)
+          end
+        end
+      } ],
+      "release_health" => [ 1.minute, 5.minutes, lambda { |env|
+        now = Time.current
+        [ now.beginning_of_hour, (now - 1.hour).beginning_of_hour ].each do |bucket|
+          ReleaseHealthRollupJob.new.perform(env, bucket)
+        end
+      } ],
+      # Embedded batches fold themselves into the current hour as they land
+      # (Ingest::RollupAbsorber), so the reconciler only ever needs to catch
+      # rows that arrived after their hour closed.
+      "rollup_reconcile" => [ 1.hour, 10.minutes, lambda { |env|
+        RollupJob.new.perform(env, (Time.current - 1.hour).beginning_of_hour)
+      } ],
+      "performance_scan" => [ 5.minutes, 10.minutes, lambda { |env|
+        DetectPerformanceIssuesJob.new.perform(env)
+      } ],
+      "anomaly_scan" => [ 5.minutes, 10.minutes, lambda { |env|
+        DetectAnomaliesJob.new.perform(env) if AnomalyRule.where(enabled: true).exists?
+      } ],
+      "scheduled_tasks" => [ 10.minutes, 10.minutes, lambda { |env|
+        CheckScheduledTasksJob.new.perform(env)
+      } ],
+      "auto_resolve" => [ 24.hours, 10.minutes, lambda { |_env|
+        AutoResolveIssuesJob.new.perform
+      } ],
+      # PASSIVE, not TRUNCATE: this runs inside a Puma worker, and a
+      # truncating checkpoint blocks every reader and writer on the file.
+      "prune" => [ 24.hours, 60.minutes, lambda { |env|
+        PruneTelemetryJob.new.perform(env, checkpoint: "PASSIVE")
+        OptimizeTelemetryJob.new.perform(env)
+        FollowupReceipt.prune!
+      } ]
+    }.freeze
+
+    @mutex = Mutex.new
+    @wakeup = ConditionVariable.new
+    @thread = nil
+    @pid = nil
+    @stopping = false
+
+    module_function
+
+    # Idempotent; Railwatch.restart_after_fork! calls it again in every forked
+    # child.
+    def start!
+      return unless Railwatch.enabled? && Railwatch.config.local?
+      return if defined?(Rails) && Rails.env.test?
+      return unless ROLES.include?(Subscribers::ProcessInfo.role)
+      # With a writer process listening, it is the one clock. The lease table
+      # would keep two clocks honest, but there is no reason to run a second.
+      return if !Writer.running? && Writer.listening?
+      return if @thread&.alive? && @pid == Process.pid
+
+      @mutex.synchronize do
+        return if @thread&.alive? && @pid == Process.pid
+
+        @pid = Process.pid
+        @stopping = false
+        @thread = Thread.new { run }
+        @thread.name = "railwatch-maintenance"
+        @thread.abort_on_exception = false
+        @thread.report_on_exception = false
+      end
+    end
+
+    # A forked child inherits a dead thread and may inherit a mutex held by a
+    # vanished parent thread; every primitive is replaced before start!.
+    def restart_after_fork!
+      @mutex = Mutex.new
+      @wakeup = ConditionVariable.new
+      @thread = nil
+      @pid = nil
+      @stopping = false
+      start!
+    end
+
+    def stop!
+      return unless @thread
+
+      @stopping = true
+      @mutex.synchronize { @wakeup.signal }
+      @thread.join(1)
+      @thread = nil
+    end
+
+    def run
+      until @stopping
+        @mutex.synchronize { @wakeup.wait(@mutex, TICK) unless @stopping }
+        tick unless @stopping
+      end
+    end
+
+    # Runs every task that is due and unclaimed. Returns the names it ran.
+    # Public so a spec, or an operator in a console, can drive the clock by
+    # hand. One failing task is reported and does not stop the others.
+    #
+    # A web worker's clock starts at boot, before the Puma plugin has forked
+    # the writer, so the start-time check in start! cannot see it. Checked
+    # again on every tick: once a writer is listening this process's clock
+    # stands down and stays down (the lease table would keep both honest,
+    # but there is no reason to run a second one).
+    def tick(now: Time.current)
+      return [] if !Writer.running? && Writer.listening?
+
+      ran = []
+      Rails.application.executor.wrap do
+        env = Environment.current
+        TASKS.each do |name, (every, lease, body)|
+          token = MaintenanceTask.claim(name, every: every, lease: lease, owner: owner, now: now) or next
+
+          succeeded = false
+          begin
+            Railwatch.ignore { body.call(env) }
+            succeeded = true
+            ran << name
+          rescue StandardError => e
+            # Not Rails.error: this thread has no execution for Railwatch.ignore
+            # to pause, so a report there would be captured by Railwatch's own
+            # subscriber and opened as an application issue about Railwatch.
+            Railwatch.debug { "maintenance #{name} failed: #{e.class}: #{e.message}" }
+            Railwatch.notify_unrecoverable(TaskError.new("maintenance task #{name} failed: #{e.class}: #{e.message}"))
+          ensure
+            MaintenanceTask.release(name, token: token, ran_at: now, succeeded: succeeded)
+          end
+        end
+      end
+      ran
+    rescue StandardError => e
+      # The lease table not being migrated yet is the usual way to get here.
+      Railwatch.debug { "maintenance tick failed: #{e.class}: #{e.message}" }
+      ran
+    end
+
+    def owner
+      "#{Railwatch.config.server}:#{Process.pid}"
+    end
+
+    class TaskError < StandardError; end
+  end
+end

data/lib/railwatch/patches/runner_command.rb

@@ -107,13 +107,33 @@ module Railwatch
       # Expanded so a relative path is judged by where it actually resolves.
       # An argument File.expand_path refuses (a "~nobody/x.rb") is matched
       # as-is rather than assumed interactive: when in doubt, report.
+      #
+      # A scratch directory that contains the application itself is not a
+      # scratch directory for the application's own files: an app checked out
+      # under /tmp (a CI runner, a throwaway worktree) must not have its
+      # script/ classified as a shell session because of where the checkout
+      # lives. A scratch path inside the app (an operator's own
+      # `Rails.root/tmp/`) is left alone by this rule and still matches.
       def self.scratch?(argument)
         path = begin
           File.expand_path(argument)
         rescue StandardError
           argument
         end
-        Railwatch.config.interactive_runner_paths.any? { |directory| path.start_with?(directory) }
+        root = application_root
+        Railwatch.config.interactive_runner_paths.any? do |directory|
+          next false if root && "#{root}/".start_with?(directory) && path.start_with?("#{root}/")
+
+          path.start_with?(directory)
+        end
+      end
+
+      def self.application_root
+        return nil unless defined?(Rails) && Rails.respond_to?(:root) && Rails.root
+
+        Rails.root.to_s
+      rescue StandardError
+        nil
       end
     end
   end

data/lib/railwatch/record.rb

@@ -54,32 +54,57 @@ module Railwatch
     # Counting stops as soon as `limit` is exceeded: past that the only fact
     # the caller uses is "too big", so there is no reason to keep walking.
     def buffered_bytes(value, limit:, depth: 0)
-      return limit + 1 if depth > MAX_SIZING_DEPTH
+      bytes = weigh(value, limit, depth)
+      bytes > limit ? limit + 1 : bytes
+    end
 
-      bytes = case value
+    # The walk itself. Positional arguments, and scalars weighed inline in
+    # the container loops rather than through a call per value: a record is
+    # thirty-odd scalars under one Hash, so the common case is one call and
+    # one loop, and only a nested header or payload hash recurses. Halves the
+    # per-record cost against the one-method-per-value version (measured:
+    # 6.2 to 2.8 us for a query record).
+    def weigh(value, limit, depth)
+      case value
       when String then 40 + value.bytesize
       when Hash then hash_bytes(value, limit, depth)
       when Array then array_bytes(value, limit, depth)
-      when Symbol then 16
       else 16
       end
-      bytes > limit ? limit + 1 : bytes
     end
 
     def hash_bytes(hash, limit, depth)
+      # Children are weighed inline below, so the bound is checked for them
+      # here: a container whose contents would sit past MAX_SIZING_DEPTH is
+      # over the limit by definition, which is what a per-value depth check
+      # produced before the contents were inlined.
+      return limit + 1 if depth > MAX_SIZING_DEPTH || (depth == MAX_SIZING_DEPTH && !hash.empty?)
+
       bytes = 80 + (hash.size * 40)
-      hash.each do |key, value|
-        bytes += buffered_bytes(key, limit: limit, depth: depth + 1)
-        bytes += buffered_bytes(value, limit: limit, depth: depth + 1)
+      hash.each_pair do |key, value|
+        bytes += key.is_a?(Symbol) ? 16 : weigh(key, limit, depth + 1)
+        bytes += case value
+        when String then 40 + value.bytesize
+        when Hash then hash_bytes(value, limit, depth + 1)
+        when Array then array_bytes(value, limit, depth + 1)
+        else 16
+        end
         break if bytes > limit
       end
       bytes
     end
 
     def array_bytes(array, limit, depth)
+      return limit + 1 if depth > MAX_SIZING_DEPTH || (depth == MAX_SIZING_DEPTH && !array.empty?)
+
       bytes = 40 + (array.size * 8)
       array.each do |value|
-        bytes += buffered_bytes(value, limit: limit, depth: depth + 1)
+        bytes += case value
+        when String then 40 + value.bytesize
+        when Hash then hash_bytes(value, limit, depth + 1)
+        when Array then array_bytes(value, limit, depth + 1)
+        else 16
+        end
         break if bytes > limit
       end
       bytes

data/lib/railwatch/reporter.rb

@@ -187,6 +187,12 @@ module Railwatch
     end
 
     def forked_transport
+      # A child may be a different kind of process from its parent: the
+      # writer forked from a Puma master must write SQLite itself, not hand
+      # batches back to the socket it is about to serve.
+      reselected = Railwatch.local_transport if @config.local?
+      return reselected if reselected && reselected.class != @transport.class
+
       transport = @transport.dup
       transport.reset_after_fork! if transport.respond_to?(:reset_after_fork!)
       transport
@@ -327,6 +333,16 @@ module Railwatch
           @retry_attempt = 0
           @retry_at = nil
           Railwatch.debug { "gave up on a batch of #{batch.records.size} records after #{MAX_RETRY_ATTEMPTS} retries (#{result.error || result.status}); dropped and counted" }
+          # Losing a batch is not a debug-level event: with an ingest (or an
+          # embedded writer) that never comes back this is the only place the
+          # loss is ever reported, and the dropped counter it leaves behind
+          # rides on the NEXT successful delivery, which may never happen.
+          Railwatch.notify_unrecoverable(
+            DeliveryError.new("Railwatch dropped #{batch.records.size} records after #{MAX_RETRY_ATTEMPTS} failed delivery attempts: " \
+                              "#{result.error || result.status}",
+                              status: result.status, records: batch.records.size, bytes: batch.bytes,
+                              dropped: batch.dropped, dropped_bytes: batch.dropped_bytes)
+          )
           next
         end
         @retry_batch = batch

data/lib/railwatch/subscribers/process_info.rb

@@ -89,7 +89,8 @@ module Railwatch
       # $PROGRAM_NAME after boot and would otherwise turn the supervisor,
       # dispatcher, and scheduler into "web" on every health sample.
       def role
-        if defined?(::SolidQueue) && ($PROGRAM_NAME.include?("jobs") || $PROGRAM_NAME.start_with?("solid-queue-") || ARGV.first.to_s.start_with?("solid_queue:")) then "worker"
+        if defined?(Railwatch::Writer) && Railwatch::Writer.running? then "writer"
+        elsif defined?(::SolidQueue) && ($PROGRAM_NAME.include?("jobs") || $PROGRAM_NAME.start_with?("solid-queue-") || ARGV.first.to_s.start_with?("solid_queue:")) then "worker"
         elsif defined?(::Rails::Console) then "console"
         elsif $PROGRAM_NAME.end_with?("rake") then "command"
         elsif defined?(::Puma) then "web"

data/lib/railwatch/transport/local.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+module Railwatch
+  module Transport
+    # Writes each batch straight into the engine's telemetry database instead
+    # of POSTing it. Same Reporter, same buffer, same backpressure; the only
+    # difference from Transport::Http is that deliver ends in Ingest::Batch
+    # rather than Net::HTTP. Runs on the reporter thread, never on a request.
+    class Local
+      Result = Struct.new(:ok, :status, :accepted, :rejected, :rejections, :error, :retryable_error, keyword_init: true) do
+        def retryable? = retryable_error ? true : false
+      end
+
+      def initialize(config)
+        @config = config
+      end
+
+      def deliver(records, dropped: 0, dropped_bytes: 0, backpressure_factor: 1.0, batch_id: nil)
+        wire = stringify(records)
+        # The reporter thread is not a request or a job: nothing has set up
+        # an ExecutionContext for it. Query log tags, Rails.error.report and
+        # connection checkin all assume one, so run the write inside the
+        # executor the same way a job would.
+        #
+        # The rescue is INSIDE the wrap. The executor reports any exception
+        # that escapes its block to Rails.error before re-raising it, and
+        # Railwatch subscribes to Rails.error: a write that failed (the
+        # telemetry database not migrated yet, a locked file) would otherwise
+        # be captured as one of the application's own exceptions and opened
+        # as an issue about Railwatch, by Railwatch.
+        Rails.application.executor.wrap do
+          write(wire, records.size, dropped: dropped, backpressure_factor: backpressure_factor, batch_id: batch_id)
+        rescue StandardError => e
+          Railwatch.debug { "local ingest failed: #{e.class}: #{e.message}" }
+          # With a batch id the reporter can safely retry: the ledger says
+          # whether the write committed, and the unique execution_id index
+          # makes a replay of a half-visible batch a no-op. Without one (an
+          # older caller) the batch is consumed, since a retry could
+          # double-insert. A malformed record never gets here: the mapper
+          # counts it as rejected and the rest of the batch is written.
+          Result.new(ok: false, error: "#{e.class}: #{e.message}", retryable_error: !batch_id.nil?)
+        end
+      end
+
+      def ping = true
+      def unauthorized? = false
+      def reset_after_fork! = self
+
+      private
+
+      def write(wire, count, dropped:, backpressure_factor:, batch_id:)
+        environment = Environment.current
+        # The same contract the HTTP path gets from X-Railwatch-Batch-Id: a
+        # batch the reporter retries after a failure is written once. The
+        # ledger row is created inside the batch transaction, so its presence
+        # is the commit.
+        if (ledger = environment.with_telemetry { Ingest::Batch.committed(batch_id) })
+          return Result.new(ok: true, status: 200, accepted: ledger.accepted, rejected: ledger.rejected, rejections: [])
+        end
+
+        result = Ingest::Batch.new(environment, wire, dropped_by_client: dropped, backpressure_factor: backpressure_factor,
+                                   gem_version: Railwatch::VERSION, embedded: true, batch_id: batch_id).write!
+        Result.new(ok: true, status: 200, accepted: result.accepted, rejected: result.rejected,
+                   rejections: result.rejections.first(10))
+      end
+
+      # The gem builds symbol-keyed records, nested hashes included (stages,
+      # counters, headers); the mapper reads string keys throughout and
+      # rejects a symbol-keyed structure. A JSON round trip is the deep
+      # stringify. Measured against the alternatives on a 500-record batch:
+      # 1.9 ms and 11k allocations here versus 4.8 ms and 25k for
+      # deep_transform_keys, which also leaves Symbol values as Symbols.
+      def stringify(records)
+        JSON.parse(JSON.generate(records))
+      end
+    end
+  end
+end

data/lib/railwatch/transport/socket.rb

@@ -0,0 +1,183 @@
+# frozen_string_literal: true
+
+require "socket"
+require "zlib"
+require "json"
+
+module Railwatch
+  module Transport
+    # Hands each batch to the embedded writer process (Railwatch::Writer) over
+    # a Unix socket instead of writing it into SQLite on this thread. The
+    # workers' Ruby threads then never map a record, hold SQLite's write lock,
+    # or run a t-digest; all of that happens in one process whose GVL no
+    # request shares. Same Reporter, same buffer, same batch id, same ledger:
+    # a batch the writer already committed replays as a no-op there.
+    #
+    # One request per connection, length-prefixed, gzip JSON both ways:
+    #   > [u32 length][gzip(JSON {batch_id, records, dropped, dropped_bytes, backpressure_factor})]
+    #   < [u32 length][JSON {ok, status, accepted, rejected, rejections, error, retryable_error}]
+    #
+    # Whether a writer is EXPECTED decides what a missing one means. Under
+    # the Puma plugin (which sets Writer.expected! in the master before it
+    # forks the workers) a socket that is absent or refusing is a writer
+    # that is starting or restarting: the batch is retained and retried on
+    # the reporter's backoff ladder, which is bounded -- after
+    # Reporter::MAX_RETRY_ATTEMPTS the batch is counted as dropped and
+    # reported through on_unrecoverable, rather than held for ever. Anywhere else
+    # (a plain `rails server`, a runner, a Solid Queue worker, the suite) no
+    # writer will ever appear, so the first miss switches this transport to
+    # an in-process Transport::Local for the rest of the process's life and
+    # embedded mode works exactly as it did before the writer existed. A
+    # refusing socket in that case is a stale inode from a dead writer and
+    # falls back the same way rather than dropping batches forever.
+    class Socket
+      Result = Local::Result
+      MAX_REPLY_BYTES = 1 << 20
+      # How long an in-process fallback lasts before the socket is tried
+      # again. The fallback is provisional on purpose: a process that missed
+      # the writer once (a bare `puma` whose plugin could not mark one
+      # expected, a worker forked before the writer bound) would otherwise
+      # write its own batches for the rest of its life.
+      FALLBACK_RECHECK = 30
+      # How long a process that EXPECTS a writer keeps retaining batches for
+      # one that never answers before it writes them itself. A writer that is
+      # restarting is back in seconds; one that cannot bind at all (an
+      # unwritable socket directory, a fork that keeps failing) would
+      # otherwise retain until the reporter's retry cap and then drop the
+      # batch. Writing it here instead keeps the records.
+      WRITER_GRACE = 60
+
+      def initialize(config, path: nil, expected: nil)
+        @config = config
+        @path = path || config.writer_socket_path
+        @expected = expected
+        # A path the kernel cannot bind (over 108 bytes on Linux) can never
+        # have a writer behind it; do not spend a batch finding out, and do
+        # not keep re-checking it either.
+        @unusable_path = !Writer.usable_path?(@path)
+        @fallback = @unusable_path ? Local.new(config) : nil
+        @fallback_at = Clock.monotonic if @fallback
+        Railwatch.debug { "writer socket path #{@path.inspect} is too long; writing batches in-process" } if @fallback
+      end
+
+      attr_reader :path
+
+      def fallback? = !@fallback.nil?
+
+      # Read when a batch misses, not when the transport is built: under
+      # `rails server` the app boots (and the reporter picks this transport)
+      # before Puma evaluates config/puma.rb, so the plugin's Writer.expected!
+      # lands after construction, and the workers inherit a copy of this
+      # object at fork. A flag captured here at construction was always false
+      # in every worker, and the first flush before the writer was listening
+      # fell back to in-process writes for the life of the worker.
+      def expected? = @expected.nil? ? Writer.expected? : @expected
+
+      def deliver(records, dropped: 0, dropped_bytes: 0, backpressure_factor: 1.0, batch_id: nil)
+        return @fallback.deliver(records, dropped: dropped, dropped_bytes: dropped_bytes,
+                                 backpressure_factor: backpressure_factor, batch_id: batch_id) if falling_back?
+
+        payload = Zlib.gzip(JSON.generate(batch_id: batch_id, records: records, dropped: dropped,
+                                          dropped_bytes: dropped_bytes, backpressure_factor: backpressure_factor))
+        reply = exchange(payload)
+        Result.new(**reply.slice("ok", "status", "accepted", "rejected", "rejections", "error", "retryable_error").transform_keys(&:to_sym))
+      rescue Errno::ENOENT, Errno::ECONNREFUSED, Errno::ENOTSOCK => e
+        @missing_since ||= Clock.monotonic
+        if expected? && Clock.monotonic - @missing_since < WRITER_GRACE
+          Railwatch.debug { "writer not answering at #{@path} (#{e.class}); retaining the batch" }
+          return Result.new(ok: false, error: "writer unavailable: #{e.message}", retryable_error: true)
+        end
+
+        Railwatch.debug do
+          if expected?
+            "no writer at #{@path} after #{WRITER_GRACE}s (#{e.class}); writing batches in-process, re-checking every #{FALLBACK_RECHECK}s"
+          else
+            "no writer at #{@path} (#{e.class}) and none expected; writing batches in-process, re-checking every #{FALLBACK_RECHECK}s"
+          end
+        end
+        @fallback = Local.new(@config)
+        @fallback_at = Clock.monotonic
+        deliver(records, dropped: dropped, dropped_bytes: dropped_bytes, backpressure_factor: backpressure_factor, batch_id: batch_id)
+      rescue SystemCallError, IOError, Zlib::Error, JSON::ParserError, Timeout::Error => e
+        Railwatch.debug { "writer delivery failed: #{e.class}: #{e.message}" }
+        Result.new(ok: false, error: "#{e.class}: #{e.message}", retryable_error: true)
+      end
+
+      # Whether this batch goes in-process. A fallback for an unbindable path
+      # is permanent; any other one is re-examined every FALLBACK_RECHECK
+      # seconds, so a writer that turns up later takes the work back.
+      def falling_back?
+        return false if @fallback.nil?
+        return true if @unusable_path || Clock.monotonic - @fallback_at < FALLBACK_RECHECK
+
+        if Writer.listening?(@path)
+          Railwatch.debug { "writer is answering at #{@path} again; handing batches back to it" }
+          @fallback = nil
+          @missing_since = nil
+          false
+        else
+          @fallback_at = Clock.monotonic
+          true
+        end
+      end
+
+      def ping
+        return true if @fallback
+
+        UNIXSocket.new(@path).close
+        true
+      rescue SystemCallError, ArgumentError
+        false
+      end
+
+      def unauthorized? = false
+      def reset_after_fork! = self
+
+      private
+
+      # Every read and write is under one deadline of config.timeout, the
+      # same budget the HTTP transport gives a POST; a writer that accepts but
+      # stops reading cannot hold the reporter thread past it.
+      def exchange(payload)
+        UNIXSocket.open(@path) do |sock|
+          deadline = Clock.monotonic + @config.timeout
+          write_exactly(sock, [ payload.bytesize ].pack("N") + payload, deadline)
+          header = read_exactly(sock, 4, deadline)
+          length = header.unpack1("N")
+          raise IOError, "writer reply too large (#{length} bytes)" if length > MAX_REPLY_BYTES
+
+          JSON.parse(read_exactly(sock, length, deadline))
+        end
+      end
+
+      def write_exactly(sock, data, deadline)
+        offset = 0
+        while offset < data.bytesize
+          remaining = deadline - Clock.monotonic
+          raise Timeout::Error, "writer did not accept the batch within #{@config.timeout}s" if remaining <= 0
+          raise IOError, "writer closed the connection" unless sock.wait_writable(remaining)
+
+          written = sock.write_nonblock(data.byteslice(offset..), exception: false)
+          offset += written if written.is_a?(Integer)
+        end
+      end
+
+      def read_exactly(sock, count, deadline)
+        buffer = +""
+        while buffer.bytesize < count
+          remaining = deadline - Clock.monotonic
+          raise Timeout::Error, "writer did not answer within #{@config.timeout}s" if remaining <= 0
+          raise IOError, "writer closed the connection" unless sock.wait_readable(remaining)
+
+          chunk = sock.read_nonblock(count - buffer.bytesize, exception: false)
+          case chunk
+          when :wait_readable then next
+          when nil then raise IOError, "writer closed the connection"
+          else buffer << chunk
+          end
+        end
+        buffer
+      end
+    end
+  end
+end

data/lib/railwatch/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Railwatch
-  VERSION = "0.1.4"
+  VERSION = "0.2.0"
 end