railwatch 0.9.4 → 1.0.0

data/lib/railwatch/maintenance.rb

@@ -1,194 +0,0 @@
-# frozen_string_literal: true
-
-# TASKS is built from duration literals at load time; the engine has these
-# loaded already, but a writer process required ahead of the app must not
-# depend on that.
-require "active_support/core_ext/numeric/time"
-
-module Railwatch
-  # The embedded install's own clock. One background thread per web/worker
-  # process wakes every TICK seconds and runs whichever maintenance task is
-  # due: release-health rollups, threshold and anomaly scans, missed
-  # scheduled tasks, auto-resolve, pruning. Every process runs the clock;
-  # one process at a time runs a given task, claimed through a lease row in
-  # the railwatch database (MaintenanceTask), so a Puma cluster and a Solid
-  # Queue worker on the same host do not all prune at once.
-  #
-  # Same thread shape as Railwatch::Health: parked on a ConditionVariable,
-  # started from the engine, re-armed in every forked child. Nothing here
-  # goes through Active Job -- the bodies call the job classes' perform
-  # directly -- so an embedded install needs no worker, no recurring.yml,
-  # and never writes the host's queue adapter.
-  #
-  # A task must never be visible to the app it is maintaining: each body runs
-  # inside Railwatch.ignore and rescues everything.
-  module Maintenance
-    ROLES = %w[web worker writer].freeze
-    TICK = 30
-    FOLLOWUP_BATCHES_PER_TICK = 200
-
-    # name => [interval, lease, body]. The lease is how long a claim is held
-    # by a process that never releases it (crashed mid-task); it is a ceiling
-    # on the work, not an estimate of it.
-    TASKS = {
-      # A batch's exceptions are grouped into issues right after it commits;
-      # a process that dies in between leaves the work recorded on the
-      # batch's ledger row. Bounded per tick so a long outage drains over a
-      # few ticks rather than one long one.
-      "drain_followups" => [ 1.minute, 5.minutes, lambda { |env|
-        env.with_telemetry do
-          Telemetry::IngestBatch.with_pending_followups.limit(FOLLOWUP_BATCHES_PER_TICK).each do |batch|
-            batch.drain_followups!(env)
-          end
-        end
-      } ],
-      "release_health" => [ 1.minute, 5.minutes, lambda { |env|
-        now = Time.current
-        [ now.beginning_of_hour, (now - 1.hour).beginning_of_hour ].each do |bucket|
-          ReleaseHealthRollupJob.new.perform(env, bucket)
-        end
-      } ],
-      # Embedded batches fold themselves into the current hour as they land
-      # (Ingest::RollupAbsorber), so the reconciler only ever needs to catch
-      # rows that arrived after their hour closed.
-      "rollup_reconcile" => [ 1.hour, 10.minutes, lambda { |env|
-        RollupJob.new.perform(env, (Time.current - 1.hour).beginning_of_hour)
-      } ],
-      "performance_scan" => [ 5.minutes, 10.minutes, lambda { |env|
-        DetectPerformanceIssuesJob.new.perform(env)
-      } ],
-      "anomaly_scan" => [ 5.minutes, 10.minutes, lambda { |env|
-        DetectAnomaliesJob.new.perform(env) if AnomalyRule.where(enabled: true).exists?
-      } ],
-      "scheduled_tasks" => [ 10.minutes, 10.minutes, lambda { |env|
-        CheckScheduledTasksJob.new.perform(env)
-      } ],
-      "auto_resolve" => [ 24.hours, 10.minutes, lambda { |_env|
-        AutoResolveIssuesJob.new.perform
-      } ],
-      # Deliveries the export queue has stopped waiting for, and the history
-      # of ones it finished. Without this the age limit is advice rather than
-      # a limit: a delivery whose receiver never recovers keeps its bytes,
-      # and the queue eventually fills and sheds live telemetry instead.
-      "export_expiry" => [ 5.minutes, 10.minutes, lambda { |env|
-        next unless Railwatch.config.export?
-
-        outbox = Export::Outbox.new(Railwatch.config, env)
-        outbox.expire!
-        outbox.prune!
-      } ],
-      # PASSIVE, not TRUNCATE: this runs inside a Puma worker, and a
-      # truncating checkpoint blocks every reader and writer on the file.
-      "prune" => [ 24.hours, 60.minutes, lambda { |env|
-        PruneTelemetryJob.new.perform(env, checkpoint: "PASSIVE")
-        OptimizeTelemetryJob.new.perform(env)
-        FollowupReceipt.prune!
-      } ]
-    }.freeze
-
-    @mutex = Mutex.new
-    @wakeup = ConditionVariable.new
-    @thread = nil
-    @pid = nil
-    @stopping = false
-
-    module_function
-
-    # Idempotent; Railwatch.restart_after_fork! calls it again in every forked
-    # child.
-    def start!
-      return unless Railwatch.enabled? && Railwatch.config.local?
-      return if defined?(Rails) && Rails.env.test?
-      return unless ROLES.include?(Subscribers::ProcessInfo.role)
-      # With a writer process listening, it is the one clock. The lease table
-      # would keep two clocks honest, but there is no reason to run a second.
-      return if !Writer.running? && Writer.listening?
-      return if @thread&.alive? && @pid == Process.pid
-
-      @mutex.synchronize do
-        return if @thread&.alive? && @pid == Process.pid
-
-        @pid = Process.pid
-        @stopping = false
-        @thread = Thread.new { run }
-        @thread.name = "railwatch-maintenance"
-        @thread.abort_on_exception = false
-        @thread.report_on_exception = false
-      end
-    end
-
-    # A forked child inherits a dead thread and may inherit a mutex held by a
-    # vanished parent thread; every primitive is replaced before start!.
-    def restart_after_fork!
-      @mutex = Mutex.new
-      @wakeup = ConditionVariable.new
-      @thread = nil
-      @pid = nil
-      @stopping = false
-      start!
-    end
-
-    def stop!
-      return unless @thread
-
-      @stopping = true
-      @mutex.synchronize { @wakeup.signal }
-      @thread.join(1)
-      @thread = nil
-    end
-
-    def run
-      until @stopping
-        @mutex.synchronize { @wakeup.wait(@mutex, TICK) unless @stopping }
-        tick unless @stopping
-      end
-    end
-
-    # Runs every task that is due and unclaimed. Returns the names it ran.
-    # Public so a spec, or an operator in a console, can drive the clock by
-    # hand. One failing task is reported and does not stop the others.
-    #
-    # A web worker's clock starts at boot, before the Puma plugin has forked
-    # the writer, so the start-time check in start! cannot see it. Checked
-    # again on every tick: once a writer is listening this process's clock
-    # stands down and stays down (the lease table would keep both honest,
-    # but there is no reason to run a second one).
-    def tick(now: Time.current)
-      return [] if !Writer.running? && Writer.listening?
-
-      ran = []
-      Rails.application.executor.wrap do
-        env = Environment.current
-        TASKS.each do |name, (every, lease, body)|
-          token = MaintenanceTask.claim(name, every: every, lease: lease, owner: owner, now: now) or next
-
-          succeeded = false
-          begin
-            Railwatch.ignore { body.call(env) }
-            succeeded = true
-            ran << name
-          rescue StandardError => e
-            # Not Rails.error: this thread has no execution for Railwatch.ignore
-            # to pause, so a report there would be captured by Railwatch's own
-            # subscriber and opened as an application issue about Railwatch.
-            Railwatch.debug { "maintenance #{name} failed: #{e.class}: #{e.message}" }
-            Railwatch.notify_unrecoverable(TaskError.new("maintenance task #{name} failed: #{e.class}: #{e.message}"))
-          ensure
-            MaintenanceTask.release(name, token: token, ran_at: now, succeeded: succeeded)
-          end
-        end
-      end
-      ran
-    rescue StandardError => e
-      # The lease table not being migrated yet is the usual way to get here.
-      Railwatch.debug { "maintenance tick failed: #{e.class}: #{e.message}" }
-      ran
-    end
-
-    def owner
-      "#{Railwatch.config.server}:#{Process.pid}"
-    end
-
-    class TaskError < StandardError; end
-  end
-end

data/lib/railwatch/patches/migration_busy_timeout.rb

@@ -1,73 +0,0 @@
-# frozen_string_literal: true
-
-module Railwatch
-  module Patches
-    # Gives the migration connection to a Railwatch database a long SQLite
-    # busy timeout, for the migration only.
-    #
-    # A deploy migrates while the previous release is still serving, and on
-    # an embedded install that release's writer process is committing
-    # telemetry to the same file the whole time. Each of its transactions
-    # takes SQLite's single write lock: about a second on a warm 15 GB file
-    # (the export sender's claim scans the destination's whole delivery
-    # history), several seconds while a deploy's image build has the disk.
-    # The migration connection waited only the database's configured
-    # `timeout` -- 5 s in the generated database.yml -- and then raised
-    # SQLite3::BusyException, which db:prepare reports as a failed migration
-    # and the container entrypoint as a failed boot. That is how a deploy of
-    # rebulk-system crash-looped eight times without one migration running.
-    #
-    # The configured timeout is the right one for the app's own requests,
-    # which should fail fast rather than queue behind ingest. A migration is
-    # the opposite: it runs once, before the server binds, and waiting is the
-    # only way through. So only the connection Active Record migrates with is
-    # changed, only while it migrates, and only for a database whose
-    # migrations_paths include this gem's. The host's own databases keep
-    # their timeouts.
-    module MigrationBusyTimeout
-      def migrate(*, **)
-        pool = migration_connection_pool
-        raw = Railwatch::Patches::MigrationBusyTimeout.lengthen(pool)
-        super
-      ensure
-        Railwatch::Patches::MigrationBusyTimeout.restore(pool, raw) if raw
-      end
-
-      class << self
-        # The raw SQLite connection whose timeout was lengthened, or nil when
-        # this is not a Railwatch database (or not SQLite).
-        def lengthen(pool)
-          return unless railwatch_database?(pool.db_config)
-
-          raw = pool.lease_connection.raw_connection
-          return unless raw.respond_to?(:busy_handler_timeout=)
-
-          raw.busy_handler_timeout = (Railwatch.config.migration_busy_timeout.to_f * 1000).to_i
-          raw
-        rescue StandardError => e
-          Railwatch.debug { "migration busy timeout not applied: #{e.class}: #{e.message}" }
-          nil
-        end
-
-        # Back to what database.yml says, so a connection that outlives the
-        # migration (db:migrate in a long-lived process, a spec) behaves as
-        # configured afterwards.
-        def restore(pool, raw)
-          timeout = pool.db_config.configuration_hash[:timeout]
-          if timeout
-            raw.busy_handler_timeout = Integer(timeout)
-          else
-            raw.busy_handler(nil)
-          end
-        rescue StandardError => e
-          Railwatch.debug { "migration busy timeout not restored: #{e.class}: #{e.message}" }
-        end
-
-        def railwatch_database?(db_config)
-          ours = %i[railwatch railwatch_telemetry].map { |name| File.expand_path(Railwatch.migrations_path(name)) }
-          Array(db_config.migrations_paths).any? { |path| ours.include?(File.expand_path(path.to_s, Rails.root.to_s)) }
-        end
-      end
-    end
-  end
-end

data/lib/railwatch/transport/local.rb

@@ -1,78 +0,0 @@
-# frozen_string_literal: true
-
-module Railwatch
-  module Transport
-    # Writes each batch straight into the engine's telemetry database instead
-    # of POSTing it. Same Reporter, same buffer, same backpressure; the only
-    # difference from Transport::Http is that deliver ends in Ingest::Batch
-    # rather than Net::HTTP. Runs on the reporter thread, never on a request.
-    class Local
-      Result = Struct.new(:ok, :status, :accepted, :rejected, :rejections, :error, :retryable_error, keyword_init: true) do
-        def retryable? = retryable_error ? true : false
-      end
-
-      def initialize(config)
-        @config = config
-      end
-
-      def deliver(records, dropped: 0, dropped_bytes: 0, backpressure_factor: 1.0, batch_id: nil)
-        wire = stringify(records)
-        # The reporter thread is not a request or a job: nothing has set up
-        # an ExecutionContext for it. Query log tags, Rails.error.report and
-        # connection checkin all assume one, so run the write inside the
-        # executor the same way a job would.
-        #
-        # The rescue is INSIDE the wrap. The executor reports any exception
-        # that escapes its block to Rails.error before re-raising it, and
-        # Railwatch subscribes to Rails.error: a write that failed (the
-        # telemetry database not migrated yet, a locked file) would otherwise
-        # be captured as one of the application's own exceptions and opened
-        # as an issue about Railwatch, by Railwatch.
-        Rails.application.executor.wrap do
-          write(wire, records.size, dropped: dropped, backpressure_factor: backpressure_factor, batch_id: batch_id)
-        rescue StandardError => e
-          Railwatch.debug { "local ingest failed: #{e.class}: #{e.message}" }
-          # With a batch id the reporter can safely retry: the ledger says
-          # whether the write committed, and the unique execution_id index
-          # makes a replay of a half-visible batch a no-op. Without one (an
-          # older caller) the batch is consumed, since a retry could
-          # double-insert. A malformed record never gets here: the mapper
-          # counts it as rejected and the rest of the batch is written.
-          Result.new(ok: false, error: "#{e.class}: #{e.message}", retryable_error: !batch_id.nil?)
-        end
-      end
-
-      def ping = true
-      def unauthorized? = false
-      def reset_after_fork! = self
-
-      private
-
-      def write(wire, count, dropped:, backpressure_factor:, batch_id:)
-        environment = Environment.current
-        # The same contract the HTTP path gets from X-Railwatch-Batch-Id: a
-        # batch the reporter retries after a failure is written once. The
-        # ledger row is created inside the batch transaction, so its presence
-        # is the commit.
-        if (ledger = environment.with_telemetry { Ingest::Batch.committed(batch_id) })
-          return Result.new(ok: true, status: 200, accepted: ledger.accepted, rejected: ledger.rejected, rejections: [])
-        end
-
-        result = Ingest::Batch.new(environment, wire, dropped_by_client: dropped, backpressure_factor: backpressure_factor,
-                                   gem_version: Railwatch::VERSION, embedded: true, batch_id: batch_id).write!
-        Result.new(ok: true, status: 200, accepted: result.accepted, rejected: result.rejected,
-                   rejections: result.rejections.first(10))
-      end
-
-      # The gem builds symbol-keyed records, nested hashes included (stages,
-      # counters, headers); the mapper reads string keys throughout and
-      # rejects a symbol-keyed structure. A JSON round trip is the deep
-      # stringify. Measured against the alternatives on a 500-record batch:
-      # 1.9 ms and 11k allocations here versus 4.8 ms and 25k for
-      # deep_transform_keys, which also leaves Symbol values as Symbols.
-      def stringify(records)
-        JSON.parse(JSON.generate(records))
-      end
-    end
-  end
-end

data/lib/railwatch/transport/socket.rb

@@ -1,183 +0,0 @@
-# frozen_string_literal: true
-
-require "socket"
-require "zlib"
-require "json"
-
-module Railwatch
-  module Transport
-    # Hands each batch to the embedded writer process (Railwatch::Writer) over
-    # a Unix socket instead of writing it into SQLite on this thread. The
-    # workers' Ruby threads then never map a record, hold SQLite's write lock,
-    # or run a t-digest; all of that happens in one process whose GVL no
-    # request shares. Same Reporter, same buffer, same batch id, same ledger:
-    # a batch the writer already committed replays as a no-op there.
-    #
-    # One request per connection, length-prefixed, gzip JSON both ways:
-    #   > [u32 length][gzip(JSON {batch_id, records, dropped, dropped_bytes, backpressure_factor})]
-    #   < [u32 length][JSON {ok, status, accepted, rejected, rejections, error, retryable_error}]
-    #
-    # Whether a writer is EXPECTED decides what a missing one means. Under
-    # the Puma plugin (which sets Writer.expected! in the master before it
-    # forks the workers) a socket that is absent or refusing is a writer
-    # that is starting or restarting: the batch is retained and retried on
-    # the reporter's backoff ladder, which is bounded -- after
-    # Reporter::MAX_RETRY_ATTEMPTS the batch is counted as dropped and
-    # reported through on_unrecoverable, rather than held for ever. Anywhere else
-    # (a plain `rails server`, a runner, a Solid Queue worker, the suite) no
-    # writer will ever appear, so the first miss switches this transport to
-    # an in-process Transport::Local for the rest of the process's life and
-    # embedded mode works exactly as it did before the writer existed. A
-    # refusing socket in that case is a stale inode from a dead writer and
-    # falls back the same way rather than dropping batches forever.
-    class Socket
-      Result = Local::Result
-      MAX_REPLY_BYTES = 1 << 20
-      # How long an in-process fallback lasts before the socket is tried
-      # again. The fallback is provisional on purpose: a process that missed
-      # the writer once (a bare `puma` whose plugin could not mark one
-      # expected, a worker forked before the writer bound) would otherwise
-      # write its own batches for the rest of its life.
-      FALLBACK_RECHECK = 30
-      # How long a process that EXPECTS a writer keeps retaining batches for
-      # one that never answers before it writes them itself. A writer that is
-      # restarting is back in seconds; one that cannot bind at all (an
-      # unwritable socket directory, a fork that keeps failing) would
-      # otherwise retain until the reporter's retry cap and then drop the
-      # batch. Writing it here instead keeps the records.
-      WRITER_GRACE = 60
-
-      def initialize(config, path: nil, expected: nil)
-        @config = config
-        @path = path || config.writer_socket_path
-        @expected = expected
-        # A path the kernel cannot bind (over 108 bytes on Linux) can never
-        # have a writer behind it; do not spend a batch finding out, and do
-        # not keep re-checking it either.
-        @unusable_path = !Writer.usable_path?(@path)
-        @fallback = @unusable_path ? Local.new(config) : nil
-        @fallback_at = Clock.monotonic if @fallback
-        Railwatch.debug { "writer socket path #{@path.inspect} is too long; writing batches in-process" } if @fallback
-      end
-
-      attr_reader :path
-
-      def fallback? = !@fallback.nil?
-
-      # Read when a batch misses, not when the transport is built: under
-      # `rails server` the app boots (and the reporter picks this transport)
-      # before Puma evaluates config/puma.rb, so the plugin's Writer.expected!
-      # lands after construction, and the workers inherit a copy of this
-      # object at fork. A flag captured here at construction was always false
-      # in every worker, and the first flush before the writer was listening
-      # fell back to in-process writes for the life of the worker.
-      def expected? = @expected.nil? ? Writer.expected? : @expected
-
-      def deliver(records, dropped: 0, dropped_bytes: 0, backpressure_factor: 1.0, batch_id: nil)
-        return @fallback.deliver(records, dropped: dropped, dropped_bytes: dropped_bytes,
-                                 backpressure_factor: backpressure_factor, batch_id: batch_id) if falling_back?
-
-        payload = Zlib.gzip(JSON.generate(batch_id: batch_id, records: records, dropped: dropped,
-                                          dropped_bytes: dropped_bytes, backpressure_factor: backpressure_factor))
-        reply = exchange(payload)
-        Result.new(**reply.slice("ok", "status", "accepted", "rejected", "rejections", "error", "retryable_error").transform_keys(&:to_sym))
-      rescue Errno::ENOENT, Errno::ECONNREFUSED, Errno::ENOTSOCK => e
-        @missing_since ||= Clock.monotonic
-        if expected? && Clock.monotonic - @missing_since < WRITER_GRACE
-          Railwatch.debug { "writer not answering at #{@path} (#{e.class}); retaining the batch" }
-          return Result.new(ok: false, error: "writer unavailable: #{e.message}", retryable_error: true)
-        end
-
-        Railwatch.debug do
-          if expected?
-            "no writer at #{@path} after #{WRITER_GRACE}s (#{e.class}); writing batches in-process, re-checking every #{FALLBACK_RECHECK}s"
-          else
-            "no writer at #{@path} (#{e.class}) and none expected; writing batches in-process, re-checking every #{FALLBACK_RECHECK}s"
-          end
-        end
-        @fallback = Local.new(@config)
-        @fallback_at = Clock.monotonic
-        deliver(records, dropped: dropped, dropped_bytes: dropped_bytes, backpressure_factor: backpressure_factor, batch_id: batch_id)
-      rescue SystemCallError, IOError, Zlib::Error, JSON::ParserError, Timeout::Error => e
-        Railwatch.debug { "writer delivery failed: #{e.class}: #{e.message}" }
-        Result.new(ok: false, error: "#{e.class}: #{e.message}", retryable_error: true)
-      end
-
-      # Whether this batch goes in-process. A fallback for an unbindable path
-      # is permanent; any other one is re-examined every FALLBACK_RECHECK
-      # seconds, so a writer that turns up later takes the work back.
-      def falling_back?
-        return false if @fallback.nil?
-        return true if @unusable_path || Clock.monotonic - @fallback_at < FALLBACK_RECHECK
-
-        if Writer.listening?(@path)
-          Railwatch.debug { "writer is answering at #{@path} again; handing batches back to it" }
-          @fallback = nil
-          @missing_since = nil
-          false
-        else
-          @fallback_at = Clock.monotonic
-          true
-        end
-      end
-
-      def ping
-        return true if @fallback
-
-        UNIXSocket.new(@path).close
-        true
-      rescue SystemCallError, ArgumentError
-        false
-      end
-
-      def unauthorized? = false
-      def reset_after_fork! = self
-
-      private
-
-      # Every read and write is under one deadline of config.timeout, the
-      # same budget the HTTP transport gives a POST; a writer that accepts but
-      # stops reading cannot hold the reporter thread past it.
-      def exchange(payload)
-        UNIXSocket.open(@path) do |sock|
-          deadline = Clock.monotonic + @config.timeout
-          write_exactly(sock, [ payload.bytesize ].pack("N") + payload, deadline)
-          header = read_exactly(sock, 4, deadline)
-          length = header.unpack1("N")
-          raise IOError, "writer reply too large (#{length} bytes)" if length > MAX_REPLY_BYTES
-
-          JSON.parse(read_exactly(sock, length, deadline))
-        end
-      end
-
-      def write_exactly(sock, data, deadline)
-        offset = 0
-        while offset < data.bytesize
-          remaining = deadline - Clock.monotonic
-          raise Timeout::Error, "writer did not accept the batch within #{@config.timeout}s" if remaining <= 0
-          raise IOError, "writer closed the connection" unless sock.wait_writable(remaining)
-
-          written = sock.write_nonblock(data.byteslice(offset..), exception: false)
-          offset += written if written.is_a?(Integer)
-        end
-      end
-
-      def read_exactly(sock, count, deadline)
-        buffer = +""
-        while buffer.bytesize < count
-          remaining = deadline - Clock.monotonic
-          raise Timeout::Error, "writer did not answer within #{@config.timeout}s" if remaining <= 0
-          raise IOError, "writer closed the connection" unless sock.wait_readable(remaining)
-
-          chunk = sock.read_nonblock(count - buffer.bytesize, exception: false)
-          case chunk
-          when :wait_readable then next
-          when nil then raise IOError, "writer closed the connection"
-          else buffer << chunk
-          end
-        end
-        buffer
-      end
-    end
-  end
-end