restless-sdk 0.1.0

data/lib/restless/text.rb

@@ -0,0 +1,171 @@
+# frozen_string_literal: true
+
+module Restless
+  # Shared text primitives: CONTRACT.md section 2.
+  #
+  # Everything in here exists because the obvious Ruby spelling silently
+  # disagrees with the reference implementation. Read the comments before
+  # replacing any of it with a one-liner.
+  module Text
+    # PRIM-002. The whitespace set, enumerated.
+    #
+    # Written as code points rather than literal characters because most of
+    # them are invisible and several are indistinguishable from a plain space
+    # in an editor. This is the JavaScript `\s` set. Ruby's `\s` is ASCII-only
+    # and NARROWER (no NBSP, no Zs category, no LS/PS/ZWNBSP), so using it
+    # would split an auth-scheme header at a different place and collapse a
+    # different set of runs in `normalize_message`.
+    WS_CODEPOINTS = [
+      0x0009, # TAB
+      0x000A, # LF
+      0x000B, # VT
+      0x000C, # FF
+      0x000D, # CR
+      0x0020, # SPACE
+      0x00A0, # NBSP
+      0x1680, # OGHAM SPACE MARK
+      0x2000, 0x2001, 0x2002, 0x2003, 0x2004, 0x2005,
+      0x2006, 0x2007, 0x2008, 0x2009, 0x200A,
+      0x2028, # LINE SEPARATOR
+      0x2029, # PARAGRAPH SEPARATOR
+      0x202F, # NARROW NO-BREAK SPACE
+      0x205F, # MEDIUM MATHEMATICAL SPACE
+      0x3000, # IDEOGRAPHIC SPACE
+      0xFEFF  # ZERO WIDTH NO-BREAK SPACE
+      # Deliberately NOT U+180E, which left the Zs category in Unicode 6.3.
+    ].freeze
+
+    WS_CHARS = WS_CODEPOINTS.map { |cp| [cp].pack("U") }.freeze
+    WS_SET = WS_CHARS.each_with_object({}) { |c, h| h[c] = true }.freeze
+
+    # The same set as a regex character-class body, for interpolation.
+    WS_CLASS = "\\u0009\\u000A\\u000B\\u000C\\u000D\\u0020" \
+               "\\u00A0\\u1680\\u2000-\\u200A\\u2028\\u2029" \
+               "\\u202F\\u205F\\u3000\\uFEFF"
+
+    # PRIM-001. WORD is exactly [A-Za-z0-9_]. Never `\w`: ASCII-only in Ruby
+    # today, which happens to be right, but spelling it out is what keeps it
+    # right.
+    WORD_CLASS = "A-Za-z0-9_"
+
+    REPLACEMENT_CHAR = [0xFFFD].pack("U")
+
+    module_function
+
+    # PRIM-010. Code points, not UTF-16 code units and not bytes.
+    def code_points(str)
+      str.chars
+    end
+
+    # PRIM-011. UTF-8 bytes.
+    def utf8_length(str)
+      to_utf8(str).bytesize
+    end
+
+    # PRIM-013 / SAFETY-001. Get a valid UTF-8 String out of anything,
+    # without raising.
+    #
+    # Ruby cannot represent an unpaired surrogate at all (see PRIM-035 and
+    # CONFORMANCE.md), so the surrogate substitution the contract describes is
+    # structurally impossible here. What IS possible is a String tagged UTF-8
+    # that holds invalid bytes -- Rack hands those out routinely -- so scrub.
+    #
+    # BINARY is DECODED, never transcoded. `IO#read(n)` always returns
+    # ASCII-8BIT and the Rack spec requires `rack.input` to be opened in binary
+    # mode, so every captured request body arrives here tagged BINARY. Ruby
+    # transcodes BINARY to UTF-8 byte by byte and has no mapping for anything
+    # above 0x7F, so `encode` turns each byte of a multi-byte character into a
+    # separate U+FFFD: a body of `{"toKen":...}` with a U+212A KELVIN SIGN came
+    # out as `{"to���en":...}`. That destroys every non-ASCII
+    # body the SDK captures, and it is a redaction bypass on top of the fold
+    # bug in Redact.normalize_name, because a mangled key no longer matches the
+    # denylist. Bytes tagged BINARY are UTF-8 bytes; decode them like
+    # `bytes_to_utf8` does, which is Node's `Buffer#toString("utf8")`.
+    #
+    # A String tagged with a REAL other encoding (ISO-8859-1 and friends) is
+    # still transcoded, because there the tag is information rather than the
+    # absence of one.
+    def to_utf8(str)
+      return "" if str.nil?
+
+      s = str.is_a?(String) ? str : str.to_s
+      if s.encoding == Encoding::UTF_8
+        s.valid_encoding? ? s : s.scrub(REPLACEMENT_CHAR)
+      elsif s.encoding == Encoding::ASCII_8BIT
+        s.dup.force_encoding(Encoding::UTF_8).scrub(REPLACEMENT_CHAR)
+      else
+        s.encode(Encoding::UTF_8, invalid: :replace, undef: :replace,
+                                  replace: REPLACEMENT_CHAR)
+      end
+    rescue StandardError
+      ""
+    end
+
+    # Decode raw bytes as UTF-8, replacing anything ill-formed with U+FFFD.
+    #
+    # Byte-for-byte identical to Node's `Buffer.from(bytes).toString("utf8")`,
+    # including the maximal-subpart rule (a truncated 3-byte sequence yields
+    # ONE U+FFFD, an overlong pair yields two). Verified differentially.
+    def bytes_to_utf8(bytes)
+      bytes.pack("C*").force_encoding(Encoding::UTF_8).scrub(REPLACEMENT_CHAR)
+    end
+
+    # PRIM-020. Unicode FULL, locale-independent lowercase mapping.
+    #
+    # `String#downcase` is exactly that, and the two traps the contract calls
+    # out both come out right: U+0130 becomes `i` + U+0307 (two code points,
+    # matching JavaScript and Python, unlike Go's SIMPLE mapping), and U+212A
+    # KELVIN SIGN becomes a plain `k`. It is not `casefold`, and it applies no
+    # locale tailoring unless one is asked for, so no Turkish dotless-i.
+    #
+    # This is the ONLY lowercase in the SDK. Section 4 name normalization used
+    # to have its own ASCII-only fold for names; that fold was
+    # a redaction bypass (see Redact.normalize_name) and is gone.
+    #
+    # What `String#downcase` does not do is the Final_Sigma contextual rule
+    # that `String.prototype.toLowerCase` applies: a word-final capital sigma
+    # lowercases to U+03C2 in the reference and to U+03C3 here. PRIM-021
+    # records that as a real difference between the languages and explicitly
+    # does NOT require it, because nothing in this contract can observe it.
+    # Greek is outside WORD, so FP-020 step 6 replaces both sigma forms with a
+    # space before either can reach a token (`"ΑΣ failed"` and `"ΑΣΑ failed"`
+    # both normalize to `failed` either way), and the REDACT-010 denylists are
+    # ASCII. A hand-rolled \p{Cased} / \p{Case_Ignorable} implementation used
+    # to live here; it was removed rather than carried as code no vector,
+    # fuzzer or caller can distinguish from this one-liner.
+    def full_lower(str)
+      str.downcase
+    end
+
+    def ws?(char)
+      WS_SET.key?(char)
+    end
+
+    # `String.prototype.trim` semantics: strip exactly the PRIM-002 set.
+    #
+    # Ruby's `String#strip` removes ASCII whitespace plus NUL and leaves NBSP
+    # and friends in place, which is a different set in both directions.
+    def ws_trim(str)
+      chars = str.chars
+      first = 0
+      first += 1 while first < chars.length && ws?(chars[first])
+      last = chars.length - 1
+      last -= 1 while last >= first && ws?(chars[last])
+      return "" if last < first
+
+      chars[first..last].join
+    end
+
+    # PRIM-040. `YYYY-MM-DDTHH:MM:SS.sssZ`, exactly three fractional digits
+    # and a literal Z.
+    #
+    # Hand-built: `Time#iso8601` needs `require "time"` plus an explicit digit
+    # count, and emits `+00:00` rather than `Z` unless the receiver is already
+    # UTC. The ingest parses this field permissively and silently falls back to
+    # server time when it cannot, so a wrong format loses real request timing
+    # with no error anywhere in the system.
+    def iso8601_millis(time)
+      time.utc.strftime("%Y-%m-%dT%H:%M:%S.%LZ")
+    end
+  end
+end

data/lib/restless/uploader.rb

@@ -0,0 +1,319 @@
+# frozen_string_literal: true
+
+require "json"
+require "net/http"
+require "uri"
+
+require_relative "env"
+require_relative "har"
+require_relative "version"
+
+module Restless
+  # CONTRACT.md sections 8 (wire format) and 9 (batching).
+  #
+  # Never raises to callers. Upload failures go to stderr under the debug flag
+  # and are otherwise swallowed: observability must not break the request path
+  # (SAFETY-004, SAFETY-008).
+  class Uploader
+    BATCH_SIZE = 10          # BATCH-001
+    FLUSH_INTERVAL_MS = 5000 # BATCH-002
+    MAX_QUEUE = 1000         # BATCH-004
+    HTTP_TIMEOUT_S = 10
+
+    attr_reader :base_url, :request_id_prefix
+
+    def initialize(api_key:, base_url:, request_id_prefix: nil, on_response: nil,
+                   transport: nil)
+      @api_key = api_key.to_s
+      @base_url = base_url.to_s
+      @request_id_prefix = request_id_prefix
+      @on_response = on_response
+      # Test hook. Anything responding to
+      # `call(url, headers, body) -> [status, body]`.
+      @transport = transport
+
+      @queue = []
+      @mutex = Mutex.new
+      @timer = nil
+      @inflight = []
+
+      warn_if_insecure
+    end
+
+    def api_key?
+      !@api_key.empty?
+    end
+
+    # WIRE-006. The project key and every captured header would otherwise ship
+    # in the clear with no signal anywhere.
+    def warn_if_insecure
+      uri = URI.parse(@base_url)
+      return unless uri.scheme == "http"
+      return if %w[localhost 127.0.0.1].include?(uri.host)
+
+      warn("[restless-sdk] RESTLESS_BASE_URL=#{@base_url} is plain HTTP -- your API " \
+           "key and every captured header will be transmitted unencrypted. " \
+           "Use https:// or localhost.")
+    rescue StandardError
+      nil
+    end
+
+    def push(captured)
+      # BATCH-008.
+      return if Env.test_run? && !Env.setup_mode?
+
+      batch = nil
+      @mutex.synchronize do
+        if @queue.length >= MAX_QUEUE
+          # Drop the OLDEST. The newest entries are the ones an operator is
+          # actively debugging.
+          @queue.shift
+          Env.debug_log("queue at #{MAX_QUEUE} -- dropping oldest captured request")
+        end
+        @queue << captured
+
+        if flush_immediately? || @queue.length >= BATCH_SIZE
+          batch = take_batch
+        else
+          start_timer
+        end
+      end
+
+      upload_async(batch) if batch
+      nil
+    end
+
+    # BATCH-005, BATCH-006, BATCH-007. Synchronous: resolves when the attempt
+    # completes.
+    def flush
+      batch = @mutex.synchronize { take_batch }
+      upload(batch) unless batch.nil? || batch.empty?
+      join_inflight
+      nil
+    end
+
+    def options
+      {
+        base_url: @base_url,
+        request_id_prefix: @request_id_prefix,
+        has_api_key: api_key?
+      }
+    end
+
+    private
+
+    # BATCH-003. Flush every push when the app is not in production, or when
+    # the ingest is on localhost. Both keep the customer dev loop and
+    # self-hosted setups low-latency.
+    def flush_immediately?
+      return true unless Env.production?
+      return true if Env.localhost?(@base_url)
+
+      false
+    end
+
+    # Caller holds @mutex.
+    def take_batch
+      cancel_timer
+      return nil if @queue.empty?
+
+      batch = @queue
+      @queue = []
+      batch
+    end
+
+    # Caller holds @mutex.
+    def start_timer
+      return if @timer
+
+      @timer = Thread.new do
+        sleep(FLUSH_INTERVAL_MS / 1000.0)
+        begin
+          batch = @mutex.synchronize do
+            @timer = nil
+            @queue.empty? ? nil : take_batch_unsafe
+          end
+          upload(batch) if batch
+        rescue StandardError => e
+          Env.debug_log("flush timer failed: #{e.class}: #{e.message}")
+        end
+      end
+      @timer.abort_on_exception = false
+    end
+
+    # Like take_batch but without cancelling the timer, which the timer thread
+    # has already cleared.
+    def take_batch_unsafe
+      batch = @queue
+      @queue = []
+      batch
+    end
+
+    # Caller holds @mutex.
+    def cancel_timer
+      timer = @timer
+      @timer = nil
+      timer.kill if timer && timer != Thread.current
+    end
+
+    # SAFETY-008. The capture path performs no blocking I/O; uploads are
+    # asynchronous and fire-and-forget.
+    def upload_async(batch)
+      thread = Thread.new { upload(batch) }
+      thread.abort_on_exception = false
+      @mutex.synchronize do
+        @inflight.reject! { |t| !t.alive? }
+        @inflight << thread
+      end
+    end
+
+    def join_inflight
+      threads = @mutex.synchronize { @inflight.dup }
+      threads.each { |t| t.join(HTTP_TIMEOUT_S) }
+      @mutex.synchronize { @inflight.reject! { |t| !t.alive? } }
+    end
+
+    def upload(batch)
+      return if batch.nil? || batch.empty?
+
+      # BATCH-007. No key: drop the batch, never retry or accumulate.
+      unless api_key?
+        Env.debug_log("no API key -- dropping batch of #{batch.length}")
+        return
+      end
+
+      fingerprints = distinct_fingerprints(batch)
+      payload = batch.map { |captured| entry_for(captured) }
+      url = "#{@base_url}/v1/request" # WIRE-001
+
+      Env.debug_log("uploading #{batch.length} entr#{batch.length == 1 ? 'y' : 'ies'} to #{url}")
+
+      headers = {
+        "Content-Type" => "application/json",           # WIRE-002
+        "Authorization" => "Bearer #{@api_key}",        # WIRE-003
+        "X-Restless-Spec-Version" => SPEC_VERSION,      # META-002
+        "User-Agent" => "#{SDK_NAME}/#{VERSION}"
+      }
+
+      # WIRE-004: a JSON ARRAY, even for a single capture.
+      status, body = send_request(url, headers, JSON.generate(payload))
+
+      # WIRE-024: a non-2xx is never retried and never raises. The batch is
+      # dropped.
+      unless status && status >= 200 && status < 300
+        Env.debug_log("upload failed: #{status.inspect}")
+        return
+      end
+
+      handle_response(body, fingerprints)
+    rescue StandardError => e
+      # SAFETY-004.
+      Env.debug_log("upload error: #{e.class}: #{e.message}")
+      nil
+    end
+
+    def send_request(url, headers, body)
+      return @transport.call(url, headers, body) if @transport
+
+      uri = URI.parse(url)
+      http = Net::HTTP.new(uri.host, uri.port)
+      http.use_ssl = uri.scheme == "https"
+      http.open_timeout = HTTP_TIMEOUT_S
+      http.read_timeout = HTTP_TIMEOUT_S
+      request = Net::HTTP::Post.new(uri.request_uri)
+      headers.each { |k, v| request[k] = v }
+      request.body = body
+      response = http.request(request)
+      [response.code.to_i, response.body]
+    end
+
+    # WIRE-020..023.
+    def handle_response(raw_body, batch_fingerprints)
+      return if @on_response.nil?
+
+      parsed = begin
+        JSON.parse(raw_body.to_s)
+      rescue StandardError
+        # WIRE-020: a non-JSON or unparseable body is ignored without error.
+        nil
+      end
+      return unless parsed.is_a?(Hash)
+
+      @on_response.call(parsed, batch_fingerprints)
+    end
+
+    # CACHE-012 feeds off this: every distinct fingerprint key in the batch.
+    def distinct_fingerprints(batch)
+      seen = {}
+      out = []
+      batch.each do |captured|
+        fingerprint = captured["errorFingerprint"]
+        next unless fingerprint.is_a?(Hash)
+
+        key = fingerprint["key"]
+        next if !key.is_a?(String) || key.empty? || seen[key]
+
+        seen[key] = true
+        out << key
+      end
+      out
+    end
+
+    # WIRE-010..019.
+    def entry_for(captured)
+      har = Har.to_har_entry(captured)
+      user = captured["user"] || {}
+      owner = user["owner"] || {}
+      owner_id = owner["id"]
+      masked_key = user["apiKey"]
+
+      # WIRE-013: `emails` is always an ARRAY. A single-string `email` from
+      # enrichment is wrapped; an absent one becomes [].
+      raw_email = owner["email"]
+      emails = if raw_email.is_a?(Array)
+                 raw_email
+               elsif raw_email.nil? || raw_email == ""
+                 []
+               else
+                 [raw_email]
+               end
+
+      entry = {
+        # WIRE-011: the RAW uuid, with no display prefix applied.
+        "_id" => captured["requestId"]
+      }
+      entry["routePattern"] = captured["routePattern"] if captured["routePattern"]
+      # WIRE-017: absent for successful responses.
+      entry["errorFingerprint"] = captured["errorFingerprint"] if captured["errorFingerprint"]
+      # WIRE-012: owner id, else the masked end-user key, else "anonymous".
+      entry["group"] = {
+        "id" => (owner_id && !owner_id.empty? ? owner_id : nil) || masked_key || "anonymous",
+        "label" => owner["label"] || "",
+        "emails" => emails
+      }
+      # WIRE-014: the individual caller, indexed independently of the group.
+      entry["apiKey"] = masked_key if masked_key
+      # WIRE-015/WIRE-019: the wire name is `projectId`, the user-facing name
+      # is `owner`. The mismatch is deliberate and coupled to the ingest's
+      # storage schema.
+      entry["projectId"] = owner_id if owner_id
+
+      # SETUP-005: unknown top-level fields on the setup result ride along.
+      (user["extra"] || {}).each do |key, value|
+        entry[key] = value unless entry.key?(key)
+      end
+
+      # WIRE-018: reserved constants.
+      entry["clientIPAddress"] = "127.0.0.1"
+      entry["development"] = false
+      entry["request"] = {
+        "log" => {
+          "version" => "1.2",
+          "creator" => { "name" => SDK_NAME, "version" => VERSION }, # WIRE-016
+          "entries" => [har]
+        }
+      }
+      entry
+    end
+  end
+end

data/lib/restless/version.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Restless
+  # Identity of this SDK, and the contract version it implements.
+  #
+  # META-001: the spec version an SDK implements must be recorded in a
+  # machine-readable form alongside its conformance level.
+
+  VERSION = "0.1.0"
+
+  # WIRE-016: distinct per implementation, so the ingest can attribute a
+  # payload to a language.
+  SDK_NAME = "restless-sdk-ruby"
+
+  # The spec/CONTRACT.md version this SDK is verified against.
+  #
+  # REDACT-010 requires full Unicode name folding
+  # to require full Unicode lowercase. Keep this in step with
+  # spec/VECTORS_VERSION.
+  SPEC_VERSION = "1.0.0"
+
+  # CONTRACT.md 1.1: L1 is the pure functions, L2 adds batching, caches,
+  # injection and the safety guarantees.
+  CONFORMANCE_LEVEL = "L2"
+end

data/lib/restless.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+require_relative "restless/version"
+require_relative "restless/text"
+require_relative "restless/env"
+require_relative "restless/mask"
+require_relative "restless/redact"
+require_relative "restless/fingerprint"
+require_relative "restless/stack_frames"
+require_relative "restless/har"
+require_relative "restless/request_id"
+require_relative "restless/injection"
+require_relative "restless/caches"
+require_relative "restless/settings"
+require_relative "restless/uploader"
+require_relative "restless/capture"
+require_relative "restless/client"
+# Depends on nothing outside the stdlib -- a Rack middleware is just an object
+# with a `call(env)` -- so it is loaded eagerly rather than hidden behind a
+# lazy require.
+require_relative "restless/rack"
+
+# Capture your API traffic and send it to Restless.
+#
+# This SDK implements version 1.0.0 of the Restless SDK Contract at level L2.
+# See CONFORMANCE.md.
+#
+#     client = Restless.new(ENV["RESTLESS_KEY"])
+#
+#     client.setup do |request|
+#       {
+#         api_key: client.mask(request.header("Authorization")),
+#         owner: {
+#           id: workspace_id_for(request),
+#           enrich: ->(id) { { label: Workspace.find(id).name } },
+#         },
+#       }
+#     end
+#
+#     use client.rack
+module Restless
+  class << self
+    # Construct a client. See Restless::Client#initialize.
+    def new(api_key = nil, **options)
+      Client.new(api_key, **options)
+    end
+
+    # MASK-001, available without a client for scripts and tests.
+    def mask(api_key)
+      Mask.mask(api_key)
+    end
+
+    def new_request_id
+      RequestId.new_request_id
+    end
+  end
+end

data/spec/VECTORS_VERSION

@@ -0,0 +1 @@
+1.0.0