railwatch 0.1.3 → 0.2.0.pre1

data/lib/railwatch/subscribers/llm.rb

@@ -67,7 +67,13 @@ module Railwatch
           streaming: p[:streaming] == true,
           message_count: p[:message_count],
           tool_count: Array(p[:tools]).size,
+          tools: tool_names(p),
           cost_nanos: cost_nanos(p),
+          cost_reported: cost_reported(p),
+          finish_reason: finish_reason(p),
+          provider_request_id: provider_request_id(p),
+          params: params(operation, p),
+          **attachments(p),
           **tokens(p),
           **workflow(p),
           **outcome(p),
@@ -103,6 +109,8 @@ module Railwatch
           provider: p[:provider].to_s,
           model: p[:model].to_s,
           tool_name: tool_name[0, 255],
+          tool_call_id: p[:tool_call_id]&.to_s&.slice(0, 128),
+          params: ({ result_class: p[:result_class].to_s[0, 128] } if p[:result_class]),
           duration: micros(event),
           **workflow(p),
           **outcome(p),
@@ -110,6 +118,145 @@ module Railwatch
           completion: content(p[:result_content]))
       end
 
+      # Why the model stopped. :max_tokens means the answer was cut off --
+      # a truncated extraction reads exactly like a complete one without
+      # this, which is the failure most worth being able to see.
+      def finish_reason(payload)
+        response = payload[:response]
+        return nil unless response.respond_to?(:finish_reason)
+
+        response.finish_reason&.to_s&.slice(0, 32)
+      end
+
+      # Message#raw is the Faraday response (protocol.rb hands it in), so the
+      # provider's own request id is in its headers. It is what a provider
+      # support ticket asks for, and the only key that joins our record to
+      # theirs.
+      REQUEST_ID_HEADERS = %w[request-id x-request-id x-amzn-requestid].freeze
+
+      def provider_request_id(payload)
+        raw = payload[:response]
+        raw = raw.raw if raw.respond_to?(:raw)
+        headers = raw.respond_to?(:headers) ? raw.headers : nil
+        return nil unless headers.respond_to?(:[])
+
+        REQUEST_ID_HEADERS.each do |name|
+          value = headers[name]
+          return value.to_s[0, 128] if value.present?
+        end
+        nil
+      end
+
+      # Which tools the model could reach on this call. tool_count alone says
+      # how many; retracing needs which.
+      def tool_names(payload)
+        names = Array(payload[:tools]).map(&:to_s)
+        names.empty? ? nil : names.first(50).join(",")[0, 1024]
+      end
+
+      # Whether the provider priced the call itself, or we estimated it from
+      # the registry. The difference matters when a total is queried against
+      # an invoice.
+      def cost_reported(payload)
+        tokens = payload[:tokens]
+        return nil unless tokens.respond_to?(:reported_cost)
+        # No cost means no provenance to report. false would claim the
+        # registry priced it, which is the same false certainty cost_nanos
+        # avoids by being nil rather than zero.
+        return nil if cost_nanos(payload).nil?
+
+        !tokens.reported_cost.nil?
+      end
+
+      # What the call carried besides text. Images and PDFs are most of the
+      # input tokens on a document-reading call, and without this an
+      # expensive scan is indistinguishable from an expensive prompt.
+      # Only the last user turn is measured: earlier turns were counted by
+      # the calls that sent them, and walking the whole history would both
+      # double-count and cost O(messages) on every call.
+      def attachments(payload)
+        message = last_user_message(payload)
+        list = message.respond_to?(:attachments) ? Array(message.attachments) : []
+        return {} if list.empty?
+
+        types = list.filter_map { |a| a.type.to_s if a.respond_to?(:type) }.tally
+          .sort_by { |_, n| -n }.map { |type, n| n > 1 ? "#{type}x#{n}" : type }.join(",")
+        { attachments: list.size, attachment_types: types[0, 128],
+          attachment_names: content(list.filter_map { |a| a.filename if a.respond_to?(:filename) }.join(", ")) }
+      end
+
+      # The knobs that change what a call costs and what it returns, so a
+      # surprising result can be reproduced with the settings that produced
+      # it. Provider options go through the app's own parameter filter: they
+      # are request configuration, but an app can put anything in them.
+      COMMON_PARAMS = %i[temperature max_output_tokens tool_choice tool_call_limit
+                         thinking caching citations dimensions task_type size count
+                         voice format language pages document_count top_n].freeze
+
+      def params(operation, payload)
+        out = {}
+        COMMON_PARAMS.each do |key|
+          value = payload[key]
+          next if value.nil?
+          # false is kept, not dropped: `caching` defaults to nil, so
+          # caching: false is a deliberate choice, and reproducing a call
+          # needs the settings it ran with. RubyLLM does not distinguish a
+          # boolean that was set from one that defaulted, so record both
+          # rather than guess which mattered.
+          out[key] = value.is_a?(Numeric) || [ true, false ].include?(value) ? value : value.to_s[0, 128]
+        end
+        out[:schema] = true if payload[:schema]
+        out[:server_tools] = Array(payload[:server_tools]).map(&:to_s).first(20) if payload[:server_tools].present?
+        if (usage = payload[:tokens]).respond_to?(:server_tool_use) && usage.server_tool_use.present?
+          out[:server_tool_use] = usage.server_tool_use
+        end
+        if (options = payload[:provider_options]).is_a?(Hash) && !options.empty?
+          out[:provider_options] = provider_options(options)
+        end
+        out[:operation] = operation unless out.empty?
+        out.empty? ? nil : out
+      end
+
+      # Two filters, because one is not enough here. The app's parameter
+      # filter defaults to password-shaped names only, and provider_options
+      # is the one place in this payload where a per-request credential
+      # plausibly lives -- an api_key passed per call sails straight through
+      # a password filter. The redactor's credential-name matcher (the same
+      # one that catches X-Api-Key on a header) closes that.
+      def provider_options(options)
+        redact_credentials(Railwatch.redactor.params(options.transform_keys(&:to_s)))
+      end
+
+      # Recursive, because provider options nest: extra_headers carrying an
+      # authorization value is a hash inside the hash, and a top-level scan
+      # walks straight past it.
+      def redact_credentials(value, depth = 0)
+        return value if depth > 4
+
+        case value
+        when Hash
+          value.each_with_object({}) do |(key, item), out|
+            # The matcher is written for header names, which are hyphenated;
+            # provider options are Ruby-ish and use underscores, so api_key
+            # would sail past a pattern expecting api-key.
+            out[key] = if Railwatch.redactor.redact_header?(key.to_s.tr("_", "-"))
+              Redactor::FILTERED
+            else
+              redact_credentials(item, depth + 1)
+            end
+          end
+        when Array then value.map { |item| redact_credentials(item, depth + 1) }
+        else value
+        end
+      end
+
+      def last_user_message(payload)
+        messages = payload[:input_messages]
+        return nil unless messages.respond_to?(:reverse_each)
+
+        messages.reverse_each.find { |m| m.respond_to?(:role) && m.role.to_s == "user" }
+      end
+
       # 2.0 sends a RubyLLM::Tokens; 1.16 sends bare counts on the event.
       def tokens(payload)
         counts = payload[:tokens]
@@ -161,11 +308,9 @@ module Railwatch
       # The last thing the app asked, which is the half of a conversation
       # worth seeing next to a cost. Earlier turns are the app's own records.
       def prompt_text(payload)
-        messages = payload[:input_messages]
-        return payload[:input] || payload[:prompt] || payload[:query] unless messages.respond_to?(:reverse_each)
+        return payload[:input] || payload[:prompt] || payload[:query] unless payload[:input_messages].respond_to?(:reverse_each)
 
-        last = messages.reverse_each.find { |m| m.respond_to?(:role) && m.role.to_s == "user" }
-        message_text(last)
+        message_text(last_user_message(payload))
       end
 
       def message_text(message)

data/lib/railwatch/subscribers/process_info.rb

@@ -89,7 +89,8 @@ module Railwatch
       # $PROGRAM_NAME after boot and would otherwise turn the supervisor,
       # dispatcher, and scheduler into "web" on every health sample.
       def role
-        if defined?(::SolidQueue) && ($PROGRAM_NAME.include?("jobs") || $PROGRAM_NAME.start_with?("solid-queue-") || ARGV.first.to_s.start_with?("solid_queue:")) then "worker"
+        if defined?(Railwatch::Writer) && Railwatch::Writer.running? then "writer"
+        elsif defined?(::SolidQueue) && ($PROGRAM_NAME.include?("jobs") || $PROGRAM_NAME.start_with?("solid-queue-") || ARGV.first.to_s.start_with?("solid_queue:")) then "worker"
         elsif defined?(::Rails::Console) then "console"
         elsif $PROGRAM_NAME.end_with?("rake") then "command"
         elsif defined?(::Puma) then "web"

data/lib/railwatch/transport/local.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+module Railwatch
+  module Transport
+    # Writes each batch straight into the engine's telemetry database instead
+    # of POSTing it. Same Reporter, same buffer, same backpressure; the only
+    # difference from Transport::Http is that deliver ends in Ingest::Batch
+    # rather than Net::HTTP. Runs on the reporter thread, never on a request.
+    class Local
+      Result = Struct.new(:ok, :status, :accepted, :rejected, :rejections, :error, :retryable_error, keyword_init: true) do
+        def retryable? = retryable_error ? true : false
+      end
+
+      def initialize(config)
+        @config = config
+      end
+
+      def deliver(records, dropped: 0, dropped_bytes: 0, backpressure_factor: 1.0, batch_id: nil)
+        wire = stringify(records)
+        # The reporter thread is not a request or a job: nothing has set up
+        # an ExecutionContext for it. Query log tags, Rails.error.report and
+        # connection checkin all assume one, so run the write inside the
+        # executor the same way a job would.
+        #
+        # The rescue is INSIDE the wrap. The executor reports any exception
+        # that escapes its block to Rails.error before re-raising it, and
+        # Railwatch subscribes to Rails.error: a write that failed (the
+        # telemetry database not migrated yet, a locked file) would otherwise
+        # be captured as one of the application's own exceptions and opened
+        # as an issue about Railwatch, by Railwatch.
+        Rails.application.executor.wrap do
+          write(wire, records.size, dropped: dropped, backpressure_factor: backpressure_factor, batch_id: batch_id)
+        rescue StandardError => e
+          Railwatch.debug { "local ingest failed: #{e.class}: #{e.message}" }
+          # With a batch id the reporter can safely retry: the ledger says
+          # whether the write committed, and the unique execution_id index
+          # makes a replay of a half-visible batch a no-op. Without one (an
+          # older caller) the batch is consumed, since a retry could
+          # double-insert. A malformed record never gets here: the mapper
+          # counts it as rejected and the rest of the batch is written.
+          Result.new(ok: false, error: "#{e.class}: #{e.message}", retryable_error: !batch_id.nil?)
+        end
+      end
+
+      def ping = true
+      def unauthorized? = false
+      def reset_after_fork! = self
+
+      private
+
+      def write(wire, count, dropped:, backpressure_factor:, batch_id:)
+        environment = Environment.current
+        # The same contract the HTTP path gets from X-Railwatch-Batch-Id: a
+        # batch the reporter retries after a failure is written once. The
+        # ledger row is created inside the batch transaction, so its presence
+        # is the commit.
+        if (ledger = environment.with_telemetry { Ingest::Batch.committed(batch_id) })
+          return Result.new(ok: true, status: 200, accepted: ledger.accepted, rejected: ledger.rejected, rejections: [])
+        end
+
+        result = Ingest::Batch.new(environment, wire, dropped_by_client: dropped, backpressure_factor: backpressure_factor,
+                                   gem_version: Railwatch::VERSION, embedded: true, batch_id: batch_id).write!
+        Result.new(ok: true, status: 200, accepted: result.accepted, rejected: result.rejected,
+                   rejections: result.rejections.first(10))
+      end
+
+      # The gem builds symbol-keyed records, nested hashes included (stages,
+      # counters, headers); the mapper reads string keys throughout and
+      # rejects a symbol-keyed structure. A JSON round trip is the deep
+      # stringify. Measured against the alternatives on a 500-record batch:
+      # 1.9 ms and 11k allocations here versus 4.8 ms and 25k for
+      # deep_transform_keys, which also leaves Symbol values as Symbols.
+      def stringify(records)
+        JSON.parse(JSON.generate(records))
+      end
+    end
+  end
+end

data/lib/railwatch/transport/socket.rb

@@ -0,0 +1,183 @@
+# frozen_string_literal: true
+
+require "socket"
+require "zlib"
+require "json"
+
+module Railwatch
+  module Transport
+    # Hands each batch to the embedded writer process (Railwatch::Writer) over
+    # a Unix socket instead of writing it into SQLite on this thread. The
+    # workers' Ruby threads then never map a record, hold SQLite's write lock,
+    # or run a t-digest; all of that happens in one process whose GVL no
+    # request shares. Same Reporter, same buffer, same batch id, same ledger:
+    # a batch the writer already committed replays as a no-op there.
+    #
+    # One request per connection, length-prefixed, gzip JSON both ways:
+    #   > [u32 length][gzip(JSON {batch_id, records, dropped, dropped_bytes, backpressure_factor})]
+    #   < [u32 length][JSON {ok, status, accepted, rejected, rejections, error, retryable_error}]
+    #
+    # Whether a writer is EXPECTED decides what a missing one means. Under
+    # the Puma plugin (which sets Writer.expected! in the master before it
+    # forks the workers) a socket that is absent or refusing is a writer
+    # that is starting or restarting: the batch is retained and retried on
+    # the reporter's backoff ladder, which is bounded -- after
+    # Reporter::MAX_RETRY_ATTEMPTS the batch is counted as dropped and
+    # reported through on_unrecoverable, rather than held for ever. Anywhere else
+    # (a plain `rails server`, a runner, a Solid Queue worker, the suite) no
+    # writer will ever appear, so the first miss switches this transport to
+    # an in-process Transport::Local for the rest of the process's life and
+    # embedded mode works exactly as it did before the writer existed. A
+    # refusing socket in that case is a stale inode from a dead writer and
+    # falls back the same way rather than dropping batches forever.
+    class Socket
+      Result = Local::Result
+      MAX_REPLY_BYTES = 1 << 20
+      # How long an in-process fallback lasts before the socket is tried
+      # again. The fallback is provisional on purpose: a process that missed
+      # the writer once (a bare `puma` whose plugin could not mark one
+      # expected, a worker forked before the writer bound) would otherwise
+      # write its own batches for the rest of its life.
+      FALLBACK_RECHECK = 30
+      # How long a process that EXPECTS a writer keeps retaining batches for
+      # one that never answers before it writes them itself. A writer that is
+      # restarting is back in seconds; one that cannot bind at all (an
+      # unwritable socket directory, a fork that keeps failing) would
+      # otherwise retain until the reporter's retry cap and then drop the
+      # batch. Writing it here instead keeps the records.
+      WRITER_GRACE = 60
+
+      def initialize(config, path: nil, expected: nil)
+        @config = config
+        @path = path || config.writer_socket_path
+        @expected = expected
+        # A path the kernel cannot bind (over 108 bytes on Linux) can never
+        # have a writer behind it; do not spend a batch finding out, and do
+        # not keep re-checking it either.
+        @unusable_path = !Writer.usable_path?(@path)
+        @fallback = @unusable_path ? Local.new(config) : nil
+        @fallback_at = Clock.monotonic if @fallback
+        Railwatch.debug { "writer socket path #{@path.inspect} is too long; writing batches in-process" } if @fallback
+      end
+
+      attr_reader :path
+
+      def fallback? = !@fallback.nil?
+
+      # Read when a batch misses, not when the transport is built: under
+      # `rails server` the app boots (and the reporter picks this transport)
+      # before Puma evaluates config/puma.rb, so the plugin's Writer.expected!
+      # lands after construction, and the workers inherit a copy of this
+      # object at fork. A flag captured here at construction was always false
+      # in every worker, and the first flush before the writer was listening
+      # fell back to in-process writes for the life of the worker.
+      def expected? = @expected.nil? ? Writer.expected? : @expected
+
+      def deliver(records, dropped: 0, dropped_bytes: 0, backpressure_factor: 1.0, batch_id: nil)
+        return @fallback.deliver(records, dropped: dropped, dropped_bytes: dropped_bytes,
+                                 backpressure_factor: backpressure_factor, batch_id: batch_id) if falling_back?
+
+        payload = Zlib.gzip(JSON.generate(batch_id: batch_id, records: records, dropped: dropped,
+                                          dropped_bytes: dropped_bytes, backpressure_factor: backpressure_factor))
+        reply = exchange(payload)
+        Result.new(**reply.slice("ok", "status", "accepted", "rejected", "rejections", "error", "retryable_error").transform_keys(&:to_sym))
+      rescue Errno::ENOENT, Errno::ECONNREFUSED, Errno::ENOTSOCK => e
+        @missing_since ||= Clock.monotonic
+        if expected? && Clock.monotonic - @missing_since < WRITER_GRACE
+          Railwatch.debug { "writer not answering at #{@path} (#{e.class}); retaining the batch" }
+          return Result.new(ok: false, error: "writer unavailable: #{e.message}", retryable_error: true)
+        end
+
+        Railwatch.debug do
+          if expected?
+            "no writer at #{@path} after #{WRITER_GRACE}s (#{e.class}); writing batches in-process, re-checking every #{FALLBACK_RECHECK}s"
+          else
+            "no writer at #{@path} (#{e.class}) and none expected; writing batches in-process, re-checking every #{FALLBACK_RECHECK}s"
+          end
+        end
+        @fallback = Local.new(@config)
+        @fallback_at = Clock.monotonic
+        deliver(records, dropped: dropped, dropped_bytes: dropped_bytes, backpressure_factor: backpressure_factor, batch_id: batch_id)
+      rescue SystemCallError, IOError, Zlib::Error, JSON::ParserError, Timeout::Error => e
+        Railwatch.debug { "writer delivery failed: #{e.class}: #{e.message}" }
+        Result.new(ok: false, error: "#{e.class}: #{e.message}", retryable_error: true)
+      end
+
+      # Whether this batch goes in-process. A fallback for an unbindable path
+      # is permanent; any other one is re-examined every FALLBACK_RECHECK
+      # seconds, so a writer that turns up later takes the work back.
+      def falling_back?
+        return false if @fallback.nil?
+        return true if @unusable_path || Clock.monotonic - @fallback_at < FALLBACK_RECHECK
+
+        if Writer.listening?(@path)
+          Railwatch.debug { "writer is answering at #{@path} again; handing batches back to it" }
+          @fallback = nil
+          @missing_since = nil
+          false
+        else
+          @fallback_at = Clock.monotonic
+          true
+        end
+      end
+
+      def ping
+        return true if @fallback
+
+        UNIXSocket.new(@path).close
+        true
+      rescue SystemCallError, ArgumentError
+        false
+      end
+
+      def unauthorized? = false
+      def reset_after_fork! = self
+
+      private
+
+      # Every read and write is under one deadline of config.timeout, the
+      # same budget the HTTP transport gives a POST; a writer that accepts but
+      # stops reading cannot hold the reporter thread past it.
+      def exchange(payload)
+        UNIXSocket.open(@path) do |sock|
+          deadline = Clock.monotonic + @config.timeout
+          write_exactly(sock, [ payload.bytesize ].pack("N") + payload, deadline)
+          header = read_exactly(sock, 4, deadline)
+          length = header.unpack1("N")
+          raise IOError, "writer reply too large (#{length} bytes)" if length > MAX_REPLY_BYTES
+
+          JSON.parse(read_exactly(sock, length, deadline))
+        end
+      end
+
+      def write_exactly(sock, data, deadline)
+        offset = 0
+        while offset < data.bytesize
+          remaining = deadline - Clock.monotonic
+          raise Timeout::Error, "writer did not accept the batch within #{@config.timeout}s" if remaining <= 0
+          raise IOError, "writer closed the connection" unless sock.wait_writable(remaining)
+
+          written = sock.write_nonblock(data.byteslice(offset..), exception: false)
+          offset += written if written.is_a?(Integer)
+        end
+      end
+
+      def read_exactly(sock, count, deadline)
+        buffer = +""
+        while buffer.bytesize < count
+          remaining = deadline - Clock.monotonic
+          raise Timeout::Error, "writer did not answer within #{@config.timeout}s" if remaining <= 0
+          raise IOError, "writer closed the connection" unless sock.wait_readable(remaining)
+
+          chunk = sock.read_nonblock(count - buffer.bytesize, exception: false)
+          case chunk
+          when :wait_readable then next
+          when nil then raise IOError, "writer closed the connection"
+          else buffer << chunk
+          end
+        end
+        buffer
+      end
+    end
+  end
+end

data/lib/railwatch/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Railwatch
-  VERSION = "0.1.3"
+  VERSION = "0.2.0.pre1"
 end