calmlens 0.1.1 → 0.17.1

data/lib/calmlens/session.rb

@@ -10,12 +10,8 @@ module CalmLens
   # The resources share one session per client. It never prints the API key.
   # @api private
   class Session
-    MAX_JSON_BYTES = 32 * 1024 * 1024
-    MAX_ERROR_BYTES = 1024 * 1024
-    MAX_RETRY_DELAY = 60
-    MAX_BACKOFF = 8
-    ENSURE_LENSES_MAX_PER_REQUEST = 100
-    USER_AGENT = "calmlens-ruby/#{VERSION} (ruby #{RUBY_VERSION})".freeze
+    RETRYABLE_STATUSES = [429, 502, 503, 504].freeze
+    USER_AGENT = "calmlens-ruby/#{VERSION} (ruby/#{RUBY_VERSION})".freeze
 
     # Raised through the transport so an exception from the caller's streaming block is
     # re-raised unchanged instead of being mistaken for a network failure.
@@ -29,9 +25,12 @@ module CalmLens
     end
 
     attr_reader :project_id, :base_url, :cdn_base_url
+    # Leaves the upload size checks the API also makes (part and image sizes) to the API, so an
+    # offline stand-in answers them like the API does. {MockServerClient} sets it.
+    attr_accessor :defer_upload_checks
 
     def initialize(api_key:, project_id:, base_url:, cdn_base_url:, timeout:, open_timeout:, max_retries:,
-                   retry_base_delay:, transport:)
+                   retry_base_delay:, max_response_bytes:, upload_timeout:, transport:)
       @api_key = api_key
       @project_id = project_id
       @base_url = base_url
@@ -40,6 +39,8 @@ module CalmLens
       @open_timeout = open_timeout
       @max_retries = max_retries
       @retry_base_delay = retry_base_delay
+      @max_response_bytes = max_response_bytes
+      @upload_timeout = upload_timeout
       @transport = transport
       @lenses = {}
       @lens_lock = Mutex.new
@@ -47,17 +48,32 @@ module CalmLens
 
     def inspect = "#<#{self.class.name} project_id=#{@project_id.inspect} api_key=[REDACTED]>"
 
+    # Seconds on a monotonic clock, for decision deadlines. {MockServerClient} replaces it.
+    def monotonic = @monotonic ? @monotonic.call : Process.clock_gettime(Process::CLOCK_MONOTONIC)
+
+    # Waits between retries and decision checks. {MockServerClient} replaces it.
+    def pause(seconds) = @pause ? @pause.call(seconds) : sleep(seconds)
+
+    # Replaces the clock and wait used between retries and decision checks, so a mock's waits take no
+    # real time.
+    def use_clock(monotonic:, pause:)
+      @monotonic = monotonic
+      @pause = pause
+    end
+
     def close
       @transport.close if @transport.respond_to?(:close)
     end
 
     def project(path) = "/projects/#{Internal.segment(@project_id)}/#{path}"
 
-    # Sends an API request and returns the parsed JSON body.
+    # Sends an API request and returns the parsed JSON body (nil for an empty 204 or DELETE response).
     #
     # @param idempotent [Boolean] send an `Idempotency-Key` (generated unless the caller gave one),
     #   reused across retries so the API runs the request at most once.
-    def request(method, path, request_options:, query: {}, body: nil, raw: nil, headers: {}, idempotent: false)
+    # @param upload [Boolean] a binary upload: `upload_timeout`, when set, limits each attempt.
+    def request(method, path, request_options:, query: {}, body: nil, raw: nil, headers: {}, idempotent: false,
+                upload: false)
       url = @base_url + path
       params = Internal.query(query)
       url += "?#{URI.encode_www_form(params)}" unless params.empty?
@@ -66,20 +82,25 @@ module CalmLens
         raw = JSON.generate(body)
         headers['Content-Type'] = 'application/json'
       end
-      if raw.is_a?(UploadBody)
+      raw = UploadBody.new(raw) if raw && !raw.is_a?(UploadBody)
+      if raw&.size
         headers['Content-Length'] = raw.size.to_s
       elsif raw
-        raw = UploadBody.new(raw)
+        headers['Transfer-Encoding'] = 'chunked'
       end
       headers['Idempotency-Key'] = request_options[:idempotency_key] || SecureRandom.uuid if idempotent || request_options[:idempotency_key]
-      execute(method, url, headers, raw, request_options) { |response| parse(response) }
+      limit = request_options.fetch(:max_response_bytes, @max_response_bytes)
+      deadline = request_options.fetch(:upload_timeout, @upload_timeout) if upload
+      execute(method, url, headers, raw, request_options, deadline: deadline) { |response| parse(method, response, limit) }
     end
 
-    # Sends a CDN request. With a block, yields the streaming {Response} and returns the block's
+    # Sends a CDN request, retrying like an API read, and returns the last response rather than
+    # raising an error status. With a block, yields the streaming {Response} and returns the block's
     # value; otherwise returns the response with its body read.
     def cdn(method, url, headers, request_options, &block)
-      execute(method, url, headers, nil, request_options, stream: !block.nil?) do |response|
-        next response.tap(&:body) unless block
+      limit = request_options.fetch(:max_response_bytes, nil)
+      execute(method, url, headers, nil, request_options, allow_http_errors: true) do |response|
+        next response.tap { response.body(limit: limit) } unless block
 
         streaming = Response.new(status: response.status, headers: response.headers) do |&sink|
           response.each_chunk do |chunk|
@@ -99,17 +120,17 @@ module CalmLens
     end
 
     # Resolves lens definitions, sending only definitions this client has not saved before, at most
-    # ENSURE_LENSES_MAX_PER_REQUEST per request, in order.
+    # ServerClient::ENSURE_LENSES_MAX_PER_REQUEST per request, in order. Calls are serialized, so concurrent calls
+    # for the same definitions send them once.
     def ensure_lenses(lenses, request_options)
       if !lenses.is_a?(Array) || lenses.empty? || lenses.any? { |lens| !lens.is_a?(Hash) || lens['identifier'].to_s.empty? }
         raise Internal.invalid('At least one lens with an identifier is required')
       end
 
       @lens_lock.synchronize do
-        keys = lenses.map { |lens| JSON.generate(lens.sort.to_h) }
+        keys = lenses.map { |lens| lens_key(lens) }
         missing = keys.zip(lenses).to_h.reject { |key, _| @lenses.key?(key) }
-        resolved = @lenses.dup
-        missing.each_slice(ENSURE_LENSES_MAX_PER_REQUEST) do |slice|
+        missing.each_slice(ServerClient::ENSURE_LENSES_MAX_PER_REQUEST) do |slice|
           chunk = slice.to_h
           returned = request('PUT', project('lenses'), body: { 'lenses' => chunk.values },
                                                        request_options: request_options)
@@ -118,56 +139,113 @@ module CalmLens
             raise InvalidResponseError.new(SdkErrorCodes::INVALID_RESPONSE, 'Unexpected lens response')
           end
 
-          resolved.merge!(chunk.keys.zip(returned).to_h)
           # A new version of a lens replaces the cached definition with the same identifier.
           chunk.keys.zip(returned).each do |key, lens|
             @lenses.delete_if { |_, saved| saved['identifier'] == lens['identifier'] }
             @lenses[key] = lens
           end
         end
-        JSON.parse(JSON.generate(keys.map { |key| resolved.fetch(key) }))
-      end
-    end
+        keys.map do |key|
+          raise InvalidResponseError.new(SdkErrorCodes::INVALID_RESPONSE, 'CalmLens did not return a lens') unless @lenses.key?(key)
 
-    # Whether the saved lenses line up, one for one and in order, with the definitions that were sent.
-    def lenses_match?(returned, sent)
-      returned.is_a?(Array) && returned.size == sent.size &&
-        returned.zip(sent).all? { |saved, definition| saved.is_a?(Hash) && saved['identifier'] == definition['identifier'] }
+          JSON.parse(JSON.generate(@lenses[key]))
+        end
+      end
     end
 
     def headers_for(extra)
-      extra.reject { |name, _| %w[x-api-key user-agent].include?(name.downcase) }
+      extra.reject { |name, _| %w[x-api-key user-agent authorization].include?(name.downcase) }
            .merge('X-Api-Key' => @api_key, 'User-Agent' => USER_AGENT)
     end
 
     private
 
-    def execute(method, url, headers, body, request_options, stream: false, &handler)
+    # The fields that identify a lens definition, as canonical JSON: an unset description or
+    # transforms list is the same definition as an explicit null.
+    def lens_key(lens)
+      fields = { 'identifier' => lens['identifier'], 'name' => lens['name'], 'description' => lens['description'],
+                 'type' => lens['type'], 'transforms' => lens['transforms'] }
+      %w[active version].each { |name| fields[name] = lens[name] if lens.key?(name) }
+      JSON.generate(canonical(fields))
+    end
+
+    def canonical(value)
+      case value
+      when Hash then value.sort_by { |key, _| key.to_s }.to_h { |key, item| [key.to_s, canonical(item)] }
+      when Array then value.map { |item| canonical(item) }
+      else value
+      end
+    end
+
+    # Whether the saved lenses line up, one for one and in order, with the definitions that were sent.
+    def lenses_match?(returned, sent)
+      returned.is_a?(Array) && returned.size == sent.size &&
+        returned.zip(sent).all? { |saved, definition| saved.is_a?(Hash) && saved['identifier'] == definition['identifier'] }
+    end
+
+    # Sends a request, retrying transport failures, timeouts, 429, 502–504 and
+    # `idempotency/request-in-progress` when it is safe to repeat: reads, and requests with an
+    # `Idempotency-Key` whose body can be sent again. Redirects are never followed.
+    def execute(method, url, headers, body, request_options, allow_http_errors: false, deadline: nil, &handler)
       retries = request_options.fetch(:max_retries, @max_retries)
-      retry_allowed = %w[GET HEAD].include?(method) || headers.key?('Idempotency-Key')
+      retryable = retries.positive? && (%w[GET HEAD].include?(method) || headers.key?('Idempotency-Key')) &&
+                  (body.nil? || body.replayable?)
       request = Transport::Request.new(method: method, url: url, headers: headers_for(headers), body: body&.content,
                                        open_timeout: request_options.fetch(:open_timeout, @open_timeout),
                                        timeout: request_options.fetch(:timeout, @timeout))
+      limit = request_options.fetch(:max_response_bytes, @max_response_bytes)
       attempt = 0
-      begin
+      loop do
         body&.rewind if attempt.positive?
-        delivered = false
-        send_request(request) do |response|
-          delivered = true if stream
-          handler.call(response)
+        context = { again: retryable && attempt < retries, attempt: attempt, limit: limit, deadline: deadline,
+                    allow_http_errors: allow_http_errors }
+        outcome, value = send_attempt(request, context, &handler)
+        if outcome == :error
+          delay = http_delay(value.retry_after, attempt)
+          raise value unless context[:again] && delay <= ServerClient::RETRY_AFTER_MAX && value.retryable?
+
+          outcome = :retry
+          value = delay
         end
-      rescue Error => e
-        raise unless retry_allowed && !delivered && attempt < retries && e.retryable? && (body.nil? || body.replayable?)
-
-        delay = retry_delay(attempt, e.retry_after)
-        raise if delay.nil?
+        return value if outcome == :ok
 
-        sleep(delay)
+        pause(value)
         attempt += 1
-        retry
       end
     end
 
+    # One attempt: `[:ok, result]`, `[:error, error]` for an error response, or `[:retry, seconds]`.
+    # A response returned to the caller (or its streaming block) is never retried.
+    def send_attempt(request, context, &handler)
+      delivered = false
+      within(context[:deadline]) do
+        send_request(request) do |response|
+          raise redirect_error(response) if redirect?(response)
+
+          if context[:allow_http_errors] && !response.success?
+            delay = http_delay(Error.parse_retry_after(response.headers['retry-after']), context[:attempt])
+            retry_status = RETRYABLE_STATUSES.include?(response.status) && delay <= ServerClient::RETRY_AFTER_MAX
+            next [:retry, delay] if context[:again] && retry_status
+          end
+          next [:error, http_error(response, context[:limit])] unless response.success? || context[:allow_http_errors]
+
+          delivered = true
+          [:ok, handler.call(response)]
+        end
+      end
+    rescue Error => e
+      raise unless context[:again] && !delivered && e.retryable?
+
+      [:retry, backoff(context[:attempt])]
+    end
+
+    # Runs one attempt within an upload deadline, when there is one.
+    def within(deadline, &)
+      deadline ? ::Timeout.timeout(deadline, &) : yield
+    rescue ::Timeout::Error => e
+      raise TimeoutError.new(SdkErrorCodes::TIMEOUT, 'Request timed out'), cause: e
+    end
+
     def send_request(request, &handler)
       yielded = false
       result = nil
@@ -185,9 +263,9 @@ module CalmLens
     rescue Error
       raise
     rescue ::Timeout::Error => e
-      raise TimeoutError.new(SdkErrorCodes::TIMEOUT, e.message), cause: e
+      raise TimeoutError.new(SdkErrorCodes::TIMEOUT, 'Request timed out'), cause: e
     rescue *Transport::ERRORS => e
-      raise ConnectionError.new(SdkErrorCodes::TRANSPORT_ERROR, e.message), cause: e
+      raise ConnectionError.new(SdkErrorCodes::TRANSPORT_ERROR, "Request failed: #{e.message}"), cause: e
     rescue ArgumentError, URI::Error => e
       raise InvalidArgumentError.new(SdkErrorCodes::INVALID_ARGUMENT, e.message), cause: e
     end
@@ -201,45 +279,69 @@ module CalmLens
       Response.new(status: status, headers: headers, body: value.body.to_s)
     end
 
-    def retry_delay(attempt, retry_after)
-      return (retry_after <= MAX_RETRY_DELAY ? retry_after : nil) if retry_after
+    # A redirect the SDK must not follow: it would send the API key to another origin.
+    def redirect?(response) = response.status.between?(300, 399) && response.headers.key?('location')
 
-      rand * [@retry_base_delay * (2**attempt), MAX_BACKOFF].min
+    def redirect_error(response)
+      Error.build(SdkErrorCodes::HTTP_ERROR, "Unexpected redirect (HTTP #{response.status})", status: response.status,
+                                                                                              headers: response.headers)
     end
 
-    def parse(response)
-      text = response.body(limit: response.success? ? MAX_JSON_BYTES : MAX_ERROR_BYTES).dup.force_encoding(Encoding::UTF_8)
-      begin
-        data = text.empty? ? nil : JSON.parse(text)
-      rescue JSON::ParserError
-        raise invalid_body(response) if response.success?
+    # Exponential backoff with jitter: half the delay is fixed, the other half random.
+    def backoff(attempt)
+      delay = [@retry_base_delay * (2**attempt), ServerClient::RETRY_MAX_DELAY].min
+      ((delay / 2.0) + (rand * delay / 2.0)).round(3)
+    end
 
-        data = nil
-      end
-      return data if response.success?
+    def http_delay(retry_after, attempt) = retry_after.nil? ? backoff(attempt) : retry_after
 
-      raise api_error(response, data)
-    rescue InvalidResponseError
-      raise if response.success?
+    def read(response, limit)
+      declared = response.content_length
+      raise too_large(response, limit) if declared && declared > limit
 
-      raise api_error(response, nil)
+      response.body(limit: limit)
+    rescue InvalidResponseError => e
+      raise e unless e.code == SdkErrorCodes::RESPONSE_TOO_LARGE
+
+      raise too_large(response, limit)
     end
 
-    def invalid_body(response)
-      InvalidResponseError.new(SdkErrorCodes::INVALID_RESPONSE, "Invalid JSON response (HTTP #{response.status})",
-                               headers: response.headers)
+    def too_large(response, limit)
+      InvalidResponseError.new(SdkErrorCodes::RESPONSE_TOO_LARGE, "Response body exceeds #{limit} bytes",
+                               status: response.status, headers: response.headers)
     end
 
-    def api_error(response, data)
+    def parse(method, response, limit)
+      text = read(response, limit).dup.force_encoding(Encoding::UTF_8)
+      return nil if text.empty? && (response.status == 204 || method == 'DELETE')
+
+      JSON.parse(text)
+    rescue JSON::ParserError, EncodingError
+      raise InvalidResponseError.new(SdkErrorCodes::INVALID_RESPONSE, "Invalid JSON response (HTTP #{response.status})",
+                                     status: response.status, headers: response.headers)
+    end
+
+    # The error for a non-2xx response: the API's code and details, or `sdk/http-error`.
+    def http_error(response, limit)
+      text = read(response, limit).dup.force_encoding(Encoding::UTF_8)
+      data = begin
+        JSON.parse(text)
+      rescue JSON::ParserError, EncodingError
+        nil
+      end
       unless data.is_a?(Hash) && data['code'].is_a?(String)
         return Error.build(SdkErrorCodes::HTTP_ERROR, "HTTP #{response.status}", status: response.status,
                                                                                  headers: response.headers)
       end
 
-      details = (data['details'].is_a?(Hash) ? data['details'] : {}).merge('network' => true)
-      details['traceId'] = data['traceId'] if data.key?('traceId')
-      Error.build(data['code'], data['message'].is_a?(String) ? data['message'] : nil,
-                  status: response.status, details: details, headers: response.headers)
+      # The X-Request-Id header, or the body's `requestId` when a proxy dropped the header.
+      request_id = response.headers['x-request-id'] || (data['requestId'] if data['requestId'].is_a?(String))
+      message = data['message'].is_a?(String) ? data['message'] : nil
+      details = data['details'].is_a?(Hash) ? data['details'] : nil
+      Error.build(data['code'], message, status: response.status, details: details, headers: response.headers,
+                                         request_id: request_id)
+    rescue InvalidResponseError => e
+      e
     end
   end
   private_constant :Session

data/lib/calmlens/testing/classification.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+# Generated by packages/sdk-contract/mockClassification.ts from packages/shared/src/ModerationCategories.ts and ModerationPolicyDimensions.ts; do not edit.
+module CalmLens
+  module MockClassification
+    # The parent categories, in the API's order.
+    CATEGORIES = ["sexual", "violence", "self-harm", "harassment", "hate", "criminal-activity", "personal-information", "profanity"].freeze
+
+    # Each category's qualifiers.
+    QUALIFIERS = {
+      "sexual" => ["sexual-pornographic-media", "sexual-nudity", "sexual-activity", "sexual-explicit-language", "sexual-assault-media", "sexual-non-consensual", "sexual-intimate-image-abuse", "sexual-extreme", "sexual-minors", "sexual-solicitation", "sexual-paid", "sexual-adult-promotion", "sexual-revealing-clothing", "sexual-suggestive", "sexual-non-erotic-nudity", "sexual-text", "sexual-historical", "sexual-documentary", "sexual-educational", "sexual-discussion", "sexual-fictional", "sexual-support-prevention"].freeze,
+      "violence" => ["violence-high-impact", "violence-attack", "violence-threat", "violence-preparation", "violence-encouragement", "violence-wish", "violence-coded-incitement", "violence-extremism", "violence-injury", "violence-violent-death", "violence-natural-death", "violence-graphic", "violence-decapitation-act", "violence-gore", "violence-weapon", "violence-sport", "violence-text", "violence-historical", "violence-documentary", "violence-educational", "violence-discussion", "violence-fictional", "violence-fantasy", "violence-support-prevention"].freeze,
+      "self-harm" => ["self-harm-age-restricted", "self-harm-thoughts", "self-harm-preparation", "self-harm-act", "self-harm-methods", "self-harm-encouragement", "self-harm-historical", "self-harm-documentary", "self-harm-educational", "self-harm-fictional", "self-harm-support-prevention"].freeze,
+      "harassment" => ["harassment-abuse", "harassment-mild", "harassment-bullying", "harassment-public-target", "harassment-threat", "harassment-sexual", "harassment-encouragement", "harassment-historical", "harassment-documentary", "harassment-educational", "harassment-fictional", "harassment-support-prevention"].freeze,
+      "hate" => ["hate-nazi-symbols-public-display", "hate-nazi-insignia-public-display", "hate-nazi-genocide-public-peace", "hate-nazi-genocide-denial-or-approval", "hate-adjudicated-nazi-crimes-denial", "hate-abuse", "hate-slurs", "hate-dehumanization", "hate-inferiority", "hate-exclusion", "hate-symbol", "hate-hostility", "hate-threat", "hate-personal", "hate-historical", "hate-documentary", "hate-educational", "hate-fictional", "hate-counterspeech", "hate-support-prevention"].freeze,
+      "criminal-activity" => ["criminal-activity-demonstration", "criminal-activity-preparation", "criminal-activity-request", "criminal-activity-instructions", "criminal-activity-promotion", "criminal-activity-coded-incitement", "criminal-activity-substances", "criminal-activity-grooming", "criminal-activity-csam", "criminal-activity-alcohol", "criminal-activity-tobacco", "criminal-activity-gambling", "criminal-activity-fraud", "criminal-activity-sexual-solicitation", "criminal-activity-graffiti", "criminal-activity-historical", "criminal-activity-documentary", "criminal-activity-educational", "criminal-activity-discussion", "criminal-activity-fictional", "criminal-activity-support-prevention"].freeze,
+      "personal-information" => ["personal-information-contact", "personal-information-email", "personal-information-phone", "personal-information-government-id", "personal-information-financial", "personal-information-location", "personal-information-identifiers", "personal-information-credentials", "personal-information-request", "personal-information-self-disclosure", "personal-information-third-party", "personal-information-historical", "personal-information-documentary", "personal-information-educational", "personal-information-fictional", "personal-information-support-prevention"].freeze,
+      "profanity" => ["profanity-mild", "profanity-strong", "profanity-slurs", "profanity-obfuscated", "profanity-historical", "profanity-documentary", "profanity-educational", "profanity-fictional", "profanity-support-prevention"].freeze,
+    }.freeze
+  end
+end