k9o 1.4.0 → 1.5.0

data/lib/k9o/http_transport.rb

@@ -29,12 +29,13 @@ module K9o
   # Per-instance HTTP config bound on K9oClient.
   class HttpClientConfig
     attr_reader :base_url, :api_key, :org_slug, :project_slug, :default_headers, :http_client, :retry_config,
-                :region_mutex
+                :region_mutex, :expected_key_refs
     # Cached regional origin (see K9o::Region); nil until a definite answer.
     attr_accessor :regional_origin
 
     def initialize(base_url:, api_key: nil, org_slug: nil, project_slug: nil, default_headers: {}, http_client: nil,
-                   retry_config: nil)
+                   retry_config: nil, expected_key_refs: nil)
+      @expected_key_refs = expected_key_refs
       @base_url = base_url
       @api_key = api_key
       @org_slug = org_slug
@@ -49,6 +50,8 @@ module K9o
 
   # HTTP transport using Net::HTTP stdlib.
   module HttpTransport
+    # Seconds to wait for a response, as in the Python SDK (120 s read). A long call, such as Rampart `classify` on a long input, can run about 20 s.
+    DEFAULT_READ_TIMEOUT = 120
     RETRYABLE_METHODS = %w[GET HEAD OPTIONS DELETE].freeze
     RETRYABLE_STATUS = [502, 503, 504, 429].freeze
 
@@ -68,14 +71,32 @@ module K9o
     #
     # `authenticated: false` sends NO credential: no api_key bearer, and any
     # authorization / cookie header from `default_headers` or `headers:` is dropped.
+    #
+    # `origin:` sends the request to that origin instead of the client's regional base. Only the end-user credential
+    # flows set it, after the rule in K9o::Locate.
+    #
+    # `bare: true` is for the few public feeds that answer `{ ...fields, meta }` without a `data` wrapper (the Rampart
+    # key feed): the fields come back as `data`.
+    #
+    # `binary: true` is for download routes: a 2xx answer comes back as a K9o::Download (bytes and content type),
+    # not parsed; an error answer is still the JSON error envelope.
     def request(config:, method:, path:, body: nil, query: nil, headers: nil, authenticated: true, multipart: nil,
-                raw_body: nil)
+                raw_body: nil, bare: false, binary: false, origin: nil)
       retry_with_backoff(method, config.retry_config || RetryConfig.default) do
         request_once(config: config, method: method, path: path, body: body, query: query, headers: headers,
-                     authenticated: authenticated, multipart: multipart, raw_body: raw_body)
+                     authenticated: authenticated, multipart: multipart, raw_body: raw_body, bare: bare,
+                     binary: binary, origin: origin)
       end
     end
 
+    # A response header, or nil (also for response doubles that have no header table).
+    def header_of(response, name)
+      value = response[name]
+      value.is_a?(String) ? value : nil
+    rescue NameError, IndexError, TypeError
+      nil
+    end
+
     # Run a block with retry-with-backoff. Extracted so the retry policy is
     # unit-testable without a live HTTP round-trip.
     def retry_with_backoff(method, retry_config)
@@ -118,8 +139,8 @@ module K9o
 
     # Single-attempt request. `request` wraps this in retry-with-backoff.
     def request_once(config:, method:, path:, body: nil, query: nil, headers: nil, authenticated: true, multipart: nil,
-                     raw_body: nil)
-      url = "#{Region.resolve_origin(config).sub(%r{/\z}, '')}#{path}"
+                     raw_body: nil, bare: false, binary: false, origin: nil)
+      url = "#{(origin || Region.resolve_origin(config)).sub(%r{/\z}, '')}#{path}"
       if query && !query.empty?
         url += '?' + URI.encode_www_form(query)
       end
@@ -127,7 +148,7 @@ module K9o
       uri = URI.parse(url)
       http = config.http_client || ::Net::HTTP.new(uri.host, uri.port)
       http.use_ssl = (uri.scheme == 'https')
-      http.read_timeout = 30
+      http.read_timeout = DEFAULT_READ_TIMEOUT
 
       req_class = case method.to_s.upcase
                   when 'GET' then ::Net::HTTP::Get
@@ -179,6 +200,10 @@ module K9o
 
       return Envelope.new(data: nil, meta: Meta.new) if status == 204
 
+      if binary && status >= 200 && status < 300
+        return Download.new(body: (response.body || '').b, content_type: header_of(response, 'content-type'))
+      end
+
       begin
         parsed = JSON.parse(response.body || '')
       rescue JSON::ParserError => e
@@ -186,11 +211,17 @@ module K9o
           code: 'K9O_INTERNAL_ERROR',
           message: "Failed to parse JSON response from #{url}: #{e.message}",
           context: { url: url, status: status },
-          status: status
+          status: status,
+          request_id: KError.request_id_from_header(header_of(response, 'x-k9o-request-id'))
         )
       end
 
       if status >= 200 && status < 300
+        if bare && parsed.is_a?(Hash)
+          return Envelope.new(data: parsed.reject { |key, _| key == 'meta' },
+                              meta: Meta.from_wire(parsed['meta'].is_a?(Hash) ? parsed['meta'] : {}))
+        end
+
         unless parsed.is_a?(Hash) && parsed.key?('data') && parsed.key?('meta')
           raise KError.new(
             code: 'K9O_INTERNAL_ERROR',
@@ -203,14 +234,16 @@ module K9o
       end
 
       if parsed.is_a?(Hash) && parsed['code'].is_a?(String) && parsed['message'].is_a?(String)
-        raise KError.from_wire(parsed, status)
+        raise KError.from_wire(parsed, status, retry_after_header: header_of(response, 'retry-after'),
+                                           request_id_header: header_of(response, 'x-k9o-request-id'))
       end
 
       raise KError.new(
         code: 'K9O_INTERNAL_ERROR',
         message: "Upstream returned #{status} with non-KError body",
         context: { url: url, status: status },
-        status: status
+        status: status,
+        request_id: KError.request_id_from_header(header_of(response, 'x-k9o-request-id'))
       )
     end
 
@@ -222,7 +255,7 @@ module K9o
     #
     # The Ruby async-iteration idiom is Enumerator — caller
     # writes `client.compute.llms.invoke_stream(...).each { |chunk| ... }`.
-    def stream_sse(config:, method:, path:, body: nil, query: nil)
+    def stream_sse(config:, method:, path:, body: nil, query: nil, headers: nil, idle_timeout: nil, raw: false)
       url = "#{Region.resolve_origin(config).sub(%r{/\z}, '')}#{path}"
       if query && !query.empty?
         url += '?' + URI.encode_www_form(query)
@@ -232,7 +265,8 @@ module K9o
       Enumerator.new do |yielder|
         http = config.http_client || ::Net::HTTP.new(uri.host, uri.port)
         http.use_ssl = (uri.scheme == 'https')
-        http.read_timeout = 300
+        # The read timeout is a per-read idle limit, not a total limit on the stream.
+        http.read_timeout = idle_timeout || 300
 
         req_class = case method.to_s.upcase
                     when 'GET' then ::Net::HTTP::Get
@@ -241,6 +275,7 @@ module K9o
                     end
         req = req_class.new(uri.request_uri)
         req['accept'] = 'text/event-stream'
+        headers&.each { |k, v| req[k] = v }
         config.default_headers.each { |k, v| req[k] = v }
         req['x-k9o-project-slug'] = config.project_slug if config.project_slug
         req['x-k9o-org-slug'] = config.org_slug if config.org_slug
@@ -262,20 +297,34 @@ module K9o
                 nil
               end
               if parsed.is_a?(Hash) && parsed['code'].is_a?(String) && parsed['message'].is_a?(String)
-                raise KError.from_wire(parsed, status)
+                raise KError.from_wire(parsed, status, retry_after_header: header_of(response, 'retry-after'),
+                                           request_id_header: header_of(response, 'x-k9o-request-id'))
               end
 
               raise KError.new(
                 code: 'K9O_INTERNAL_ERROR',
                 message: "Upstream returned #{status} opening stream",
                 context: { url: url, status: status },
-                status: status
+                status: status,
+                request_id: KError.request_id_from_header(header_of(response, 'x-k9o-request-id'))
               )
             end
 
             buffer = +''
             data_lines = []
+            event_name = nil
+            event_id = nil
+            retry_ms = nil
             flush = lambda do
+              if raw
+                # Raw mode yields every frame as { event:, id:, data:, retry: } (comment lines are dropped).
+                had = !data_lines.empty? || event_name || event_id || retry_ms
+                frame = { event: event_name, id: event_id, data: data_lines.empty? ? nil : data_lines.join("\n"), retry: retry_ms }
+                data_lines = []
+                event_name = event_id = retry_ms = nil
+                yielder << frame if had
+                next true
+              end
               return true if data_lines.empty?
 
               payload = data_lines.join("\n")
@@ -302,6 +351,14 @@ module K9o
                   rest = line[5..]
                   rest = rest[1..] if rest.start_with?(' ')
                   data_lines << rest
+                elsif raw && line.start_with?('event:', 'id:', 'retry:')
+                  field, rest = line.split(':', 2)
+                  rest = rest[1..] if rest.start_with?(' ')
+                  case field
+                  when 'event' then event_name = rest
+                  when 'id' then event_id = rest
+                  else retry_ms = Integer(rest, exception: false)
+                  end
                 end
               end
             end

data/lib/k9o/jcs.rb

@@ -0,0 +1,117 @@
+# frozen_string_literal: true
+
+module K9o
+  # The platform's canonical JSON form (the Kilimanjaro JCS code-point profile), implemented here with no dependency.
+  #
+  #   K9o::Jcs.canonicalize({ "b" => 2, "a" => [1, 3] })   # => '{"a":[1,3],"b":2}'
+  #
+  # Object members are sorted by UTF-8 byte order (the same as Unicode code-point order), there is no whitespace,
+  # strings are escaped minimally (+"+ and +\+, the short escapes for backspace, tab, newline, form feed and carriage
+  # return, every other control character as lowercase +\u00xx+; +/+ and every other character are written as they
+  # are), and numbers are integers only: base 10, no exponent, +-0+ written +0+. A fractional number, NaN, an infinity
+  # or an integer whose absolute value is above 2^53 - 1 raises, so a value that cannot be hashed identically
+  # everywhere never gets a hash.
+  module Jcs
+    # Raised for a value that has no canonical form. +kind+ is :non_integer_number, :numeric_ceiling_exceeded or
+    # :unsupported_value.
+    class Error < ArgumentError
+      attr_reader :kind
+
+      def initialize(message, kind = :unsupported_value)
+        super(message)
+        @kind = kind
+      end
+    end
+
+    ESCAPES = {
+      '"' => '\\"', '\\' => '\\\\', "\b" => '\\b', "\t" => '\\t', "\n" => '\\n', "\f" => '\\f', "\r" => '\\r'
+    }.freeze
+    MAX_SAFE_INTEGER = 9_007_199_254_740_991
+
+    module_function
+
+    # The canonical form of +value+ as a UTF-8 String.
+    def canonicalize(value)
+      out = +''
+      write(value, out)
+      out.force_encoding(Encoding::UTF_8)
+    end
+
+    # The canonical form as bytes (UTF-8), the preimage to hash.
+    def canonical_bytes(value)
+      canonicalize(value).b
+    end
+
+    def write(value, out)
+      case value
+      when nil then out << 'null'
+      when true then out << 'true'
+      when false then out << 'false'
+      when Integer then out << number(value)
+      when Float then out << number(value)
+      when String then write_string(value, out)
+      when Symbol then write_string(value.to_s, out)
+      when Array
+        out << '['
+        value.each_with_index do |item, index|
+          out << ',' if index.positive?
+          write(item, out)
+        end
+        out << ']'
+      when Hash then write_object(value, out)
+      else
+        raise Error, "#{value.class} has no JSON form"
+      end
+    end
+
+    def write_object(hash, out)
+      members = hash.map { |key, val| [key.is_a?(Symbol) ? key.to_s : key, val] }
+      members.each do |key, _|
+        raise Error, 'object keys must be strings' unless key.is_a?(String)
+      end
+      keys = members.map(&:first)
+      raise Error, 'object has duplicate keys' unless keys.uniq.size == keys.size
+
+      sorted = members.sort_by { |key, _| key.b }
+      out << '{'
+      sorted.each_with_index do |(key, val), index|
+        out << ',' if index.positive?
+        write_string(key, out)
+        out << ':'
+        write(val, out)
+      end
+      out << '}'
+    end
+
+    def write_string(string, out)
+      text = string.encoding == Encoding::UTF_8 ? string : string.encode(Encoding::UTF_8)
+      raise Error, 'string is not valid UTF-8' unless text.valid_encoding?
+
+      out << '"'
+      text.each_char do |ch|
+        if (esc = ESCAPES[ch])
+          out << esc
+        elsif ch.ord < 0x20
+          out << format('\\u%04x', ch.ord)
+        else
+          out << ch
+        end
+      end
+      out << '"'
+    end
+
+    # Integers only, within +/-(2^53 - 1). A float that is a whole number is written as that integer (a JSON reader
+    # that parses +3.0+ sees the integer 3).
+    def number(num)
+      if num.is_a?(Float)
+        raise Error.new('NaN and Infinity have no canonical form', :non_integer_number) unless num.finite?
+        raise Error.new('a fractional number has no canonical form', :non_integer_number) unless num == num.floor
+
+        num = num.to_i
+      end
+      raise Error.new("integer #{num} is above 2^53 - 1", :numeric_ceiling_exceeded) if num.abs > MAX_SAFE_INTEGER
+
+      num.to_s
+    end
+  end
+end

data/lib/k9o/kerror.rb

@@ -8,8 +8,12 @@ module K9o
   # Code-narrowing: callers rescue K9o::KError and switch on e.code.
   class KError < StandardError
     attr_reader :code, :context, :recommendation, :status, :request_id
+    # Seconds the gateway asks the caller to wait before retrying, or nil.
+    # Informational: the SDK's own retry behaviour does not read it.
+    attr_reader :retry_after_sec
 
-    def initialize(code:, message:, context: {}, recommendation: nil, status: nil, request_id: nil)
+    def initialize(code:, message:, context: {}, recommendation: nil, status: nil, request_id: nil,
+                   retry_after_sec: nil)
       super(message)
       @code = code
       @message_text = message  # avoid infinite recursion in to_s (Exception#message calls to_s)
@@ -17,13 +21,21 @@ module K9o
       @recommendation = recommendation
       @status = status
       @request_id = request_id
+      @retry_after_sec = retry_after_sec
+    end
+
+    # The step-up challenge a 401 K9O_STEP_UP_REQUIRED carries (context.stepUp) as a K9o::Keys::StepUpChallenge
+    # (+scope_set_root+, +nonce+, +min_tier+, +methods+, +max_age_sec+, +single_use+, +rp_id+), or nil.
+    def step_up
+      Keys::StepUp.challenge_from(@context)
     end
 
     def with_recommendation(rec)
       KError.new(
         code: @code, message: @message_text,
         context: @context, recommendation: rec,
-        status: @status, request_id: @request_id
+        status: @status, request_id: @request_id,
+        retry_after_sec: @retry_after_sec
       )
     end
 
@@ -38,15 +50,55 @@ module K9o
     end
 
     # Rehydrate a wire KError payload into a typed K9o::KError.
-    def self.from_wire(wire, status)
+    # +retry_after_header+ is the response's Retry-After header (whole seconds); it is used when the body's
+    # recommendation carries no +retryAfterSec+. +request_id_header+ is the response's x-k9o-request-id header;
+    # it is used when the body carries no request id.
+    def self.from_wire(wire, status, retry_after_header: nil, request_id_header: nil)
       KError.new(
         code: wire['code'] || 'K9O_INTERNAL_ERROR',
         message: wire['message'] || '',
         context: wire['context'] || {},
-        recommendation: wire['recommendation'],
+        recommendation: recommendation_text(wire['recommendation']),
         status: status,
-        request_id: wire['requestId'] || wire['request_id']
+        request_id: wire['requestId'] || wire['request_id'] || request_id_from_header(request_id_header),
+        retry_after_sec: retry_after(wire['recommendation']) || retry_after_from_header(retry_after_header)
       )
     end
+
+    # The gateway's request id from the x-k9o-request-id header value, or nil when absent or blank.
+    def self.request_id_from_header(value)
+      return nil unless value.is_a?(String) && !value.strip.empty?
+
+      value.strip
+    end
+
+    # Whole seconds from a Retry-After header value; an HTTP date or anything else gives nil.
+    def self.retry_after_from_header(value)
+      return nil unless value.is_a?(String) && value.strip.match?(/\A\d+\z/)
+
+      value.strip.to_i
+    end
+
+    # The gateway sends `recommendation` as null, a string, or an object such
+    # as {"action": "..."} or {"retryAfterSec": 30, "note": "..."}. A string is
+    # kept; an object gives its `action`, else its `note`, when a string;
+    # anything else gives nil.
+    def self.recommendation_text(raw)
+      return raw if raw.is_a?(String)
+      return nil unless raw.is_a?(Hash)
+
+      [raw['action'], raw['note']].find { |v| v.is_a?(String) }
+    end
+
+    # `retryAfterSec` from an object recommendation when a finite number >= 0.
+    def self.retry_after(raw)
+      return nil unless raw.is_a?(Hash)
+
+      n = raw['retryAfterSec']
+      return nil unless n.is_a?(Numeric) && !n.is_a?(Complex) && n >= 0
+      return nil if n.respond_to?(:finite?) && !n.finite?
+
+      n
+    end
   end
 end

data/lib/k9o/keys.rb

@@ -0,0 +1,131 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module K9o
+  # client.keys: the project's encryption keys. Metadata only: no key material is ever returned.
+  #
+  #   list(status: nil)            GET  /@<slug>/vault/keys           data: { "items" => [key] } (not paged)
+  #   get(key_id)                  GET  /@<slug>/vault/keys/<id>      data: key
+  #   rotate(key_id, step_up: nil) POST /@<slug>/vault/keys/<id>/rotate
+  #
+  # A key is { "id", "version", "status", "custodyMode", "createdAt", "rotatedAt", "byokRef", "hyokRef" }.
+  # Reading needs PROJECT_ADMIN or PROJECT_AUDITOR; rotating needs PROJECT_ADMIN and a fresh step-up proof.
+  # A person's credential only: service, workload, extension and end-user tokens are refused (403).
+  class Keys
+    STATUSES = %w[active all].freeze
+    STEP_UP_HEADER = 'x-k9o-step-up-proof'
+
+    def initialize(config)
+      @config = config
+    end
+
+    # The project's keys. Only the active key by default; +status:+ "all" adds the rotating and revoked versions.
+    # The list is not paged: +data+ is { "items" => [key] } and nothing else.
+    def list(status: nil)
+      query = {}
+      query['status'] = ApprovalChecks.enum!(status, STATUSES, 'status') unless status.nil?
+      envelope = call('GET', '', query: query)
+      data = envelope.data
+      return envelope unless data.is_a?(Hash) && data.key?('nextCursor')
+
+      Envelope.new(data: data.reject { |k, _| k == 'nextCursor' }, meta: envelope.meta)
+    end
+
+    def get(key_id)
+      call('GET', "/#{ApprovalChecks.id_segment(key_id, 'key')}")
+    end
+
+    # Rotate the active key: every data key is re-wrapped under a new version and the old version is revoked.
+    # +data+ is { "keyId", "previousVersion", "newVersion", "status" => "rotated" }.
+    #
+    # Rotation needs a fresh step-up proof. The first request answers 401 K9O_STEP_UP_REQUIRED with a challenge
+    # (K9o::KError#step_up, a K9o::Keys::StepUpChallenge). Pass +step_up:+ a callable and the SDK hands it the
+    # challenge, the HTTP method and the URL being called, expects { method:, proof: } back (+method+ is one of the
+    # challenge's +methods+; +proof+ is that method's own proof, for DPoP bound to this method and URL) and sends the
+    # same request again with the +x-k9o-step-up-proof+ header: base64url(JSON { scopeSetRoot, nonce, method, proof }).
+    # It retries exactly once. Without a hook the K9O_STEP_UP_REQUIRED error is raised and carries the challenge.
+    # An error from your hook, a second 401 or any other failure of the retry is raised as it is. The proof is never
+    # logged. Nothing in the platform mints the proof for you.
+    #
+    # Other errors (all K9o::KError): 409 K9O_STATE_CONFLICT with context reason "rotation_in_flight" or
+    # "key_not_active"; 429 K9O_RATE_LIMITED with reason "rotation_rate_cap" (at most 3 rotations per rolling 24 hours;
+    # +retry_after_sec+ says when the next one is possible) or "key_service_busy". A refused rotation does not use up
+    # a proof. +headers:+ adds request headers. The call is never retried otherwise.
+    def rotate(key_id, step_up: nil, headers: nil)
+      tail = "/#{ApprovalChecks.id_segment(key_id, 'key')}/rotate"
+      unless step_up.nil? || step_up.respond_to?(:call)
+        raise ApprovalChecks.invalid('step_up must respond to call(challenge, method, url)')
+      end
+
+      begin
+        call('POST', tail, headers: headers)
+      rescue KError => e
+        raise unless step_up && e.code == 'K9O_STEP_UP_REQUIRED' && e.status == 401
+
+        challenge = e.step_up
+        raise unless challenge
+
+        method = 'POST'
+        url = "#{Region.resolve_origin(@config).sub(%r{/\z}, '')}#{UrlGrammar.project_path(slug_for_keys, "vault/keys#{tail}")}"
+        answer = step_up.call(challenge, method, url)
+        proof_header = StepUp.header(challenge, answer)
+        call('POST', tail, headers: (headers || {}).merge(STEP_UP_HEADER => proof_header))
+      end
+    end
+
+    # The step-up challenge from a 401 K9O_STEP_UP_REQUIRED (context.stepUp).
+    StepUpChallenge = Struct.new(:scope_set_root, :nonce, :min_tier, :methods, :max_age_sec, :single_use, :rp_id,
+                                 keyword_init: true)
+
+    module StepUp
+      module_function
+
+      # The challenge in an error's context, or nil.
+      def challenge_from(context)
+        raw = context.is_a?(Hash) ? (context['stepUp'] || context[:stepUp]) : nil
+        return nil unless raw.is_a?(Hash)
+
+        root = raw['scopeSetRoot']
+        nonce = raw['nonce']
+        return nil unless root.is_a?(String) && nonce.is_a?(String)
+
+        StepUpChallenge.new(scope_set_root: root, nonce: nonce, min_tier: raw['minTier'],
+                            methods: Array(raw['methods']), max_age_sec: raw['maxAgeSec'],
+                            single_use: raw['singleUse'], rp_id: raw['rpId'])
+      end
+
+      # base64url (no padding) of JSON { scopeSetRoot, nonce, method, proof }, from the hook's answer.
+      def header(challenge, answer)
+        unless answer.is_a?(Hash)
+          raise ApprovalChecks.invalid('the step_up hook must return { method:, proof: }')
+        end
+
+        method = answer[:method] || answer['method']
+        proof = answer[:proof] || answer['proof']
+        unless method.is_a?(String) && !method.empty? && proof.is_a?(String) && !proof.empty?
+          raise ApprovalChecks.invalid('the step_up hook must return a method and a proof, both non-empty strings')
+        end
+        unless challenge.methods.empty? || challenge.methods.include?(method)
+          raise ApprovalChecks.invalid("step-up method must be one of: #{challenge.methods.join(', ')}")
+        end
+
+        json = JSON.generate('scopeSetRoot' => challenge.scope_set_root, 'nonce' => challenge.nonce,
+                             'method' => method, 'proof' => proof)
+        [json].pack('m0').tr('+/', '-_').delete('=')
+      end
+    end
+
+    private
+
+    def slug_for_keys
+      Studio.require_project_slug(@config, 'keys')
+    end
+
+    def call(method, tail, query: nil, headers: nil)
+      slug = slug_for_keys
+      HttpTransport.request(config: @config, method: method, path: UrlGrammar.project_path(slug, "vault/keys#{tail}"),
+                            query: query, headers: headers)
+    end
+  end
+end