restless-sdk 0.1.0 → 0.2.0

data/lib/restless/rack.rb

@@ -210,21 +210,45 @@ module Restless
     def finish(env, setup, our_id, started_at, clock, response,
                request_body:, stack_frame:, inject: true)
       status, headers, body = response
-      headers = normalize_headers(headers)
+      route = route_pattern(env)
+
+      # SAMPLE-032. One decision per request, here: Rack hands over the status
+      # and headers together, before the server has written a byte, so the
+      # status is final and none of our headers exist yet. That holds for a
+      # streaming body and for a hijack too.
+      outcome = @engine.sample(env["REQUEST_METHOD"], route, status.to_i)
+
+      # SAMPLE-033. Dropped: no log, so no id header, no injected headers and
+      # no `debug` object pointing at one. The response goes back exactly as
+      # the app returned it, body unread.
+      return response if outcome == :drop
 
-      body_text, body = read_response_body(body, headers)
+      # SAMPLE-034. A stub never ships a body, and is always a success, so
+      # there is nothing to inject into it either: leave the body unread.
+      stub = outcome == :stub
+
+      headers = normalize_headers(headers)
+      # Rack's partial hijack hands the server a callable in the headers. It
+      # is not a header: keep it out of the capture and give it back intact.
+      hijack = headers.delete("rack.hijack")
+
+      body_text, body = if stub || hijack
+                          [nil, body]
+                        else
+                          read_response_body(body, headers)
+                        end
       duration = ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - clock) * 1000).round
 
       captured = {
         "requestId" => our_id,
         "startedAt" => Text.iso8601_millis(started_at),
         "duration" => duration,
-        "routePattern" => route_pattern(env),
+        "routePattern" => route,
         "request" => {
           "method" => env["REQUEST_METHOD"],
           "url" => Middleware.full_url(env),
           "headers" => request_headers(env),
-          "body" => request_body
+          "body" => stub ? nil : request_body
         },
         "response" => {
           "status" => status.to_i,
@@ -234,6 +258,7 @@ module Restless
         "user" => setup
       }
       captured.delete("routePattern") if captured["routePattern"].nil?
+      captured["sampled"] = true if stub # WIRE-030
 
       options = @engine.uploader.options
       # REQID-010, REQID-011.
@@ -242,6 +267,18 @@ module Restless
                        options[:request_id_prefix], options[:has_api_key]
                      ))
 
+      # INJECT-002. The headers ship on every status; only the body work below
+      # is 4xx/5xx, and fingerprinting a success would be wasted.
+      if inject && status.to_i < 400
+        headers.merge!(safely({}) do
+          Injection.headers(
+            request_id: our_id,
+            prefix: options[:request_id_prefix],
+            portal_url: @engine.portal_url
+          )
+        end)
+      end
+
       if inject && status.to_i >= 400
         # INJECT-009: the fingerprint is computed against the customer's RAW
         # response, snapshotted before any injected header or body field.
@@ -257,18 +294,18 @@ module Restless
           Injection.build(
             status: status.to_i,
             request_id: our_id,
-            base_url: options[:base_url],
             prefix: options[:request_id_prefix],
             recovery: recovery,
             method: env["REQUEST_METHOD"],
             path: captured["routePattern"],
-            docs_url: @engine.docs_url
+            portal_url: @engine.portal_url
           )
         end
 
         if injection
           headers.merge!(injection[:headers]) # INJECT-002
-          rewritten = Injection.apply_body(body_text, headers["content-type"],
+          rewritten = injection[:debug] &&
+                      Injection.apply_body(body_text, headers["content-type"],
                                            injection[:debug]) # INJECT-003
           if rewritten && !rewritten.equal?(body_text) && rewritten != body_text
             body = [rewritten]
@@ -281,6 +318,8 @@ module Restless
 
       safely(nil) { @engine.record(captured, stack_frame: stack_frame) }
 
+      # After `record`, which copies the headers it keeps.
+      headers["rack.hijack"] = hijack if hijack
       [status, headers, body]
     rescue StandardError => e
       Env.debug_log("middleware finish failed: #{e.class}: #{e.message}")
@@ -293,7 +332,14 @@ module Restless
     def normalize_headers(headers)
       out = {}
       (headers || {}).each do |name, value|
-        out[name.to_s.downcase] = value.is_a?(Array) ? value.join(", ") : value.to_s
+        name = name.to_s.downcase
+        out[name] = if name == "rack.hijack"
+                      value # a callable, not a header value
+                    elsif value.is_a?(Array)
+                      value.join(", ")
+                    else
+                      value.to_s
+                    end
       end
       out
     end

data/lib/restless/sampling.rb

@@ -0,0 +1,225 @@
+# frozen_string_literal: true
+
+require "digest"
+
+module Restless
+  # CONTRACT.md section 12.1 (`SAMPLE`). Per-endpoint control over how much
+  # of each request is sent.
+  #
+  # Everything here is pure. The middleware draws `u` once per request and
+  # hands it to `decide`; the tests and the conformance driver pass a fixed
+  # one.
+  #
+  # A mode is "on", "off", "errors" or a Float rate already quantized to a
+  # multiple of 0.0001 (SAMPLE-002). Outcomes are :full, :stub and :drop.
+  module Sampling
+    # A merged, validated configuration. `routes` maps normalized keys
+    # (`"GET /pets/{id}"`) to modes, in insertion order.
+    Config = Struct.new(:default, :routes) do
+      # SAMPLE-041. All "on" with no routes is exactly the behaviour without
+      # sampling, and has nothing to report.
+      def default?
+        default == "on" && routes.empty?
+      end
+    end
+
+    STRING_MODES = %w[on off errors].freeze
+
+    # SAMPLE-002.
+    RATE_STEPS = 10_000
+
+    # SAMPLE-010. Printable ASCII only, so every SDK sorts and hashes keys
+    # byte for byte the same. `\A...\z`, never `^...$` (PRIM-005).
+    PATH_RE = %r{\A/[\x21-\x7e]*\z}.freeze
+    METHOD_RE = /\A(?:[A-Za-z]+|\*)\z/.freeze
+    # SAMPLE-011. WORD is ASCII (PRIM-001), so no `\w`.
+    PARAM_RE = /:([A-Za-z0-9_]+)/.freeze
+
+    module_function
+
+    # SAMPLE-001..003. The normalized mode, or nil when `value` is not a
+    # valid one (the caller decides what invalid falls back to).
+    #
+    # Symbols (`:off`) are accepted as the Ruby spelling of the string modes.
+    # They can only come from code: JSON has no symbols.
+    def normalize_mode(value)
+      value = value.to_s if value.is_a?(Symbol)
+      return value if value.is_a?(String) && STRING_MODES.include?(value)
+      # `true` is not Numeric in Ruby, unlike in some languages. Complex is
+      # Numeric but not real.
+      return nil unless value.is_a?(Numeric) && value.real?
+
+      rate = value.to_f
+      return nil unless rate.finite? && rate >= 0 && rate <= 1
+
+      steps = quantize(rate)
+      # A rate that rounds to 1 IS "on"; one spelling, so the dashboard and
+      # the hash never see two.
+      steps >= RATE_STEPS ? "on" : steps / RATE_STEPS.to_f
+    end
+
+    # SAMPLE-002. floor(r * 10000 + 0.5) in double precision. Not `round`,
+    # which would make the rounding mode a per-language question.
+    def quantize(rate)
+      (rate.to_f * RATE_STEPS + 0.5).floor
+    end
+
+    # SAMPLE-011. The normalized path, or nil when it is not a valid one.
+    def normalize_path(path)
+      path = path.to_s
+      # Normalizing only rewrites ASCII, so a non-ASCII path can never become
+      # valid. Checking first also keeps a string with invalid bytes away
+      # from `gsub`, which would raise on it.
+      return nil unless path.ascii_only?
+
+      normalized = path.gsub(PARAM_RE, '{\1}')
+      normalized = normalized[0..-2] while normalized.length > 1 && normalized.end_with?("/")
+      PATH_RE.match?(normalized) ? normalized : nil
+    end
+
+    # SAMPLE-010. `"get  /pets/:id/"` becomes `"GET /pets/{id}"`. Nil when
+    # the key is not valid.
+    def normalize_route_key(key)
+      key = key.to_s
+      # As in normalize_path: never valid, and `sub` would raise on bad bytes.
+      return nil unless key.ascii_only?
+
+      # Only U+0020 is trimmed: a tab is not a separator (SAMPLE-010).
+      trimmed = key.sub(/\A +/, "").sub(/ +\z/, "")
+      space = trimmed.index(" ")
+      return nil if space.nil?
+
+      method = trimmed[0, space]
+      path = normalize_path(trimmed[space..-1].sub(/\A +/, ""))
+      return nil if path.nil? || !METHOD_RE.match?(method)
+
+      "#{method.upcase(:ascii)} #{path}"
+    end
+
+    # SAMPLE-013, SAMPLE-014. Merge the settings-file and code configs and
+    # validate every value. Never raises: a bad mode degrades to "on", a bad
+    # key is ignored, and either is reported in `warnings` for the debug log.
+    #
+    # Returns `[Config, warnings]`.
+    def normalize(settings = nil, code = nil)
+      warnings = []
+      config = Config.new("on", {})
+
+      [["settings", settings], ["code", code]].each do |source, raw|
+        next if raw.nil?
+        unless raw.is_a?(Hash)
+          warnings << "sampling (#{source}): expected a Hash, ignored"
+          next
+        end
+
+        if member?(raw, "default")
+          value = member(raw, "default")
+          mode = normalize_mode(value)
+          warnings << "sampling default (#{source}): invalid mode #{value.inspect}, using \"on\"" if mode.nil?
+          config.default = mode || "on"
+        end
+
+        routes = member(raw, "routes")
+        next if routes.nil?
+        unless routes.is_a?(Hash)
+          warnings << "sampling routes (#{source}): expected a Hash, ignored"
+          next
+        end
+
+        routes.each do |raw_key, raw_mode|
+          key = raw_key.is_a?(String) || raw_key.is_a?(Symbol) ? normalize_route_key(raw_key.to_s) : nil
+          if key.nil?
+            warnings << "sampling routes (#{source}): invalid route key #{raw_key.inspect}, ignored"
+            next
+          end
+
+          mode = normalize_mode(raw_mode)
+          if mode.nil?
+            warnings << "sampling routes[#{raw_key.inspect}] (#{source}): invalid mode " \
+                        "#{raw_mode.inspect}, using \"on\""
+          end
+          # Re-insert so a later duplicate takes the later position as well as
+          # the value. Nothing contract-visible depends on the position (the
+          # hash and the report sort), but insertion order stays honest.
+          config.routes.delete(key)
+          config.routes[key] = mode || "on"
+        end
+      end
+
+      [config, warnings]
+    end
+
+    # SAMPLE-020..022. The mode for one request. `route` is the matched
+    # pattern, nil when the framework reported none.
+    def resolve_mode(config, method, route)
+      if route.is_a?(String) && !route.empty?
+        path = normalize_path(route)
+        if path
+          exact = config.routes["#{method.to_s.upcase(:ascii)} #{path}"]
+          return exact unless exact.nil?
+
+          any = config.routes["* #{path}"]
+          return any unless any.nil?
+        end
+      end
+      config.default
+    end
+
+    # SAMPLE-030, SAMPLE-031. `u` is uniform in [0, 1), drawn once per
+    # request.
+    def decide(mode, status, u)
+      return :drop if mode == "off"
+
+      error = status >= 400
+      return(error ? :full : :drop) if mode == "errors"
+      return :full if !mode.is_a?(Numeric) || error
+
+      u < mode ? :full : :stub
+    end
+
+    # SAMPLE-040. The canonical text the hash is computed over, or nil for
+    # the default configuration (SAMPLE-041).
+    def canonical(config)
+      return nil if config.default?
+
+      # Keys are printable ASCII (SAMPLE-010) and String#<=> compares bytes,
+      # so this is the byte order the contract asks for.
+      out = +"default\t#{token(config.default)}\n"
+      config.routes.keys.sort.each do |key|
+        out << "#{key}\t#{token(config.routes[key])}\n"
+      end
+      out
+    end
+
+    # SAMPLE-040. Lowercase hex SHA-256 of the canonical form, or nil.
+    def digest(config)
+      text = canonical(config)
+      text && Digest::SHA256.hexdigest(text)
+    end
+
+    # SAMPLE-050. The routes in sorted key order, as the report carries them.
+    def sorted_routes(config)
+      out = {}
+      config.routes.keys.sort.each { |key| out[key] = config.routes[key] }
+      out
+    end
+
+    # `0.1` -> "0.1000", built from the integer step count, never from float
+    # formatting.
+    def token(mode)
+      return mode unless mode.is_a?(Numeric)
+
+      format("0.%04d", quantize(mode))
+    end
+
+    # `default:` and `routes:` read naturally as symbols in Ruby code, and
+    # arrive as strings from the settings file.
+    def member?(hash, name)
+      hash.key?(name.to_sym) || hash.key?(name)
+    end
+
+    def member(hash, name)
+      hash.key?(name.to_sym) ? hash[name.to_sym] : hash[name]
+    end
+  end
+end

data/lib/restless/settings.rb

@@ -4,8 +4,8 @@ require "json"
 
 module Restless
   # CONTRACT.md section 12. `.restless/settings.json` is created and owned by
-  # the `api` CLI (`npx api setup`); the SDK consumes exactly two fields of it
-  # at runtime (CONFIG-015).
+  # the `restless` CLI (`npx restless init`); the SDK consumes exactly three fields
+  # of it at runtime (CONFIG-015).
   module Settings
     class ConfigError < StandardError; end
 
@@ -35,7 +35,7 @@ module Restless
       end
 
       # CONFIG-013, CONFIG-014. Returns {id:, name:, request_id_prefix:,
-      # redact:} or nil.
+      # redact:, sampling:} or nil.
       def resolve_api(settings, name = nil)
         return nil if settings.nil?
 
@@ -68,7 +68,10 @@ module Restless
           id: api["id"],
           name: api["name"],
           request_id_prefix: api["requestIdPrefix"],
-          redact: api["redact"].is_a?(Hash) ? api["redact"] : nil
+          redact: api["redact"].is_a?(Hash) ? api["redact"] : nil,
+          # Raw: Sampling.normalize validates it, and warns about a bad one
+          # under DEBUG rather than dropping it silently here.
+          sampling: api["sampling"]
         }
       end
 

data/lib/restless/uploader.rb

@@ -22,9 +22,13 @@ module Restless
 
     attr_reader :base_url, :request_id_prefix
 
+    # `sampling_hash` is the SAMPLE-040 hash of a non-default sampling config,
+    # stamped on every upload as `x-restless-sampling` (SAMPLE-052). Nil when
+    # sampling is not configured, and then no header is sent at all.
     def initialize(api_key:, base_url:, request_id_prefix: nil, on_response: nil,
-                   transport: nil)
+                   sampling_hash: nil, transport: nil)
       @api_key = api_key.to_s
+      @sampling_hash = sampling_hash
       @base_url = base_url.to_s
       @request_id_prefix = request_id_prefix
       @on_response = on_response
@@ -92,6 +96,27 @@ module Restless
       nil
     end
 
+    # SAMPLE-050, SAMPLE-051. POST the sampling report on a background
+    # thread, then yield (on that thread) however it went. Same gates as an
+    # upload: nothing under a test runner or without a project key. Never
+    # raises, never retries.
+    def send_sampling_report_async(report, &on_done)
+      thread = Thread.new do
+        begin
+          send_sampling_report(report)
+        ensure
+          begin
+            on_done.call if on_done
+          rescue StandardError => e
+            Env.debug_log("sampling report callback failed: #{e.class}: #{e.message}")
+          end
+        end
+      end
+      thread.abort_on_exception = false
+      track(thread)
+      nil
+    end
+
     def options
       {
         base_url: @base_url,
@@ -161,12 +186,36 @@ module Restless
     def upload_async(batch)
       thread = Thread.new { upload(batch) }
       thread.abort_on_exception = false
+      track(thread)
+    end
+
+    # `flush` joins these, so it also waits for an in-flight sampling report.
+    def track(thread)
       @mutex.synchronize do
         @inflight.reject! { |t| !t.alive? }
         @inflight << thread
       end
     end
 
+    def send_sampling_report(report)
+      return if Env.test_run? && !Env.setup_mode?
+      return unless api_key?
+
+      url = "#{@base_url}/v1/sampling"
+      headers = {
+        "Content-Type" => "application/json",
+        "Authorization" => "Bearer #{@api_key}",
+        "X-Restless-Spec-Version" => SPEC_VERSION,
+        "User-Agent" => "#{SDK_NAME}/#{VERSION}"
+      }
+      status, = send_request(url, headers, JSON.generate(report))
+      Env.debug_log("sampling report: #{status.inspect}")
+    rescue StandardError => e
+      # SAFETY-004.
+      Env.debug_log("sampling report error: #{e.class}: #{e.message}")
+      nil
+    end
+
     def join_inflight
       threads = @mutex.synchronize { @inflight.dup }
       threads.each { |t| t.join(HTTP_TIMEOUT_S) }
@@ -194,6 +243,7 @@ module Restless
         "X-Restless-Spec-Version" => SPEC_VERSION,      # META-002
         "User-Agent" => "#{SDK_NAME}/#{VERSION}"
       }
+      headers["x-restless-sampling"] = @sampling_hash if @sampling_hash # SAMPLE-052
 
       # WIRE-004: a JSON ARRAY, even for a single capture.
       status, body = send_request(url, headers, JSON.generate(payload))
@@ -285,6 +335,8 @@ module Restless
       entry["routePattern"] = captured["routePattern"] if captured["routePattern"]
       # WIRE-017: absent for successful responses.
       entry["errorFingerprint"] = captured["errorFingerprint"] if captured["errorFingerprint"]
+      # WIRE-030: present, and true, only on a stub. Never `false`.
+      entry["sampled"] = true if captured["sampled"]
       # WIRE-012: owner id, else the masked end-user key, else "anonymous".
       entry["group"] = {
         "id" => (owner_id && !owner_id.empty? ? owner_id : nil) || masked_key || "anonymous",

data/lib/restless/version.rb

@@ -6,7 +6,7 @@ module Restless
   # META-001: the spec version an SDK implements must be recorded in a
   # machine-readable form alongside its conformance level.
 
-  VERSION = "0.1.0"
+  VERSION = "0.2.0"
 
   # WIRE-016: distinct per implementation, so the ingest can attribute a
   # payload to a language.
@@ -17,7 +17,7 @@ module Restless
   # REDACT-010 requires full Unicode name folding
   # to require full Unicode lowercase. Keep this in step with
   # spec/VECTORS_VERSION.
-  SPEC_VERSION = "1.0.0"
+  SPEC_VERSION = "1.1.0"
 
   # CONTRACT.md 1.1: L1 is the pure functions, L2 adds batching, caches,
   # injection and the safety guarantees.

data/lib/restless.rb

@@ -8,6 +8,7 @@ require_relative "restless/redact"
 require_relative "restless/fingerprint"
 require_relative "restless/stack_frames"
 require_relative "restless/har"
+require_relative "restless/sampling"
 require_relative "restless/request_id"
 require_relative "restless/injection"
 require_relative "restless/caches"
@@ -22,7 +23,7 @@ require_relative "restless/rack"
 
 # Capture your API traffic and send it to Restless.
 #
-# This SDK implements version 1.0.0 of the Restless SDK Contract at level L2.
+# This SDK implements version 1.1.0 of the Restless SDK Contract at level L2.
 # See CONFORMANCE.md.
 #
 #     client = Restless.new(ENV["RESTLESS_KEY"])

data/spec/VECTORS_VERSION

@@ -1 +1 @@
-1.0.0
+1.1.0