calmlens 0.21.0 → 0.23.0

data/lib/calmlens/testing/fake_api.rb

@@ -43,8 +43,7 @@ module CalmLens
     # semantic search.
     # @api private
     class FakeAPI
-      JSON_QUERY = %w[wait isFinal sizeInBytes totalSizeInBytes partNumber pageSize processingOptions metadata
-                      tags context].freeze
+      JSON_QUERY = %w[wait isFinal sizeInBytes totalSizeInBytes partNumber pageSize].freeze
       # Keyword search fields, in the order matches are reported, and where each value sits in the asset.
       SEARCH_FIELDS = { 'name' => %w[name], 'id' => %w[id], 'externalId' => %w[externalId],
                         'description' => %w[description], 'tags' => %w[tags], 'type' => %w[type],
@@ -112,6 +111,35 @@ module CalmLens
       # The mock API key's `{ 'actions', 'folderId', 'includeSubfolders' }` grants.
       attr_accessor :grants
 
+      # Splits an upload form as the API reads it: exactly an `options` part, then a `file` part, then
+      # the closing boundary.
+      # @return [Array(String, String), nil] the options part's text and the file's bytes, or nil
+      #   when the body isn't such a form.
+      def self.upload_form(body, content_type)
+        boundary = content_type.to_s.match(%r{\Amultipart/form-data\s*;\s*boundary=(?:"([^"]+)"|([^;\s]+))}i)
+        return nil unless boundary
+
+        framed = "\r\n".b + body.to_s.b
+        delimiter = "\r\n--#{boundary[1] || boundary[2]}".b
+        return nil unless framed.start_with?(delimiter)
+
+        parts = []
+        at = 0
+        until framed.byteslice(at + delimiter.bytesize, 2) == '--'
+          start = at + delimiter.bytesize
+          header_end = framed.index("\r\n\r\n".b, start)
+          following = header_end && framed.index(delimiter, header_end + 4)
+          return nil unless following
+
+          name = framed.byteslice(start, header_end - start)[/content-disposition:\s*form-data\s*;[^\r\n]*?\bname="([^"]*)"/i, 1]
+          parts << [name, framed.byteslice(header_end + 4, following - header_end - 4)]
+          at = following
+        end
+        return nil unless parts.map(&:first) == %w[options file]
+
+        [parts[0][1].force_encoding(Encoding::UTF_8), parts[1][1]]
+      end
+
       def initialize(project_id:, base_url:, cdn_base_url:, now:, create_id:, moderate:, api_key:, grants:)
         @project_id = project_id
         @api_key = api_key
@@ -127,6 +155,12 @@ module CalmLens
         # reset `baseline`) and the one the project moderates with.
         @policies = {}
         @project_policy_id = nil
+        # The project's configuration: upload presets by key, the settings it has set, its metadata
+        # fields in display order and the revision the settings and fields share.
+        @upload_presets = {}
+        @project_settings = {}
+        @metadata_fields = []
+        @settings_revision = 1
         @request_count = 0
         # Each asset's latest `assetSequence`, which the API keeps on the asset row.
         @asset_event_sequences = Hash.new(0)
@@ -238,6 +272,9 @@ module CalmLens
         # Like the API: a queued re-run no longer replaces a reviewed decision.
         @reruns.delete(item['id'])
         apply_outcome(item, decision, 'requested')
+        # Like the API: the human review queue's decision settles its review, one through the API
+        # leaves it waiting.
+        item['classification']['moderation']['pendingReview'] = nil unless by_api_key
         review = { 'id' => @create_id.call('rev'), 'decision' => decision, 'note' => note,
                    'decidedBy' => { 'type' => 'api-key', 'id' => nil }, 'reviewedAt' => @now.call, 'applied' => true }
         actor = by_api_key ? API_KEY_ACTOR : REVIEWER_ACTOR
@@ -357,19 +394,25 @@ module CalmLens
         json = content_type == 'application/json' && body && upload.nil? ? JSON.parse(body) : {}
         # HTTP stacks trim header values, so the API sees the key without surrounding whitespace.
         key = header(headers, 'Idempotency-Key')&.to_s&.gsub(/\A[ \t\r\n]+|[ \t\r\n]+\z/, '')
-        return dispatch(method, parts, query, json, body, upload) unless key && method == 'POST'
+        request = { query: query, json: json, body: body, upload: upload, content_type: content_type }
+        return dispatch(method, parts, request) unless key && method == 'POST'
 
         fail!(ErrorCodes::INVALID_IDEMPOTENCY_KEY) unless key.match?(IDEMPOTENCY_KEY)
         # Like the API: a successful response is saved and replayed for the same key and request;
-        # a failed one frees the key.
-        fingerprint = [method, relative, content_type, Digest::SHA256.hexdigest(body.to_s)]
+        # a failed one frees the key. An upload form counts by its options and file, not its boundary.
+        form = upload && FakeAPI.upload_form(body, content_type)
+        fingerprint = if form
+                        [method, relative, 'multipart/form-data', form[0], Digest::SHA256.hexdigest(form[1])]
+                      else
+                        [method, relative, content_type, Digest::SHA256.hexdigest(body.to_s)]
+                      end
         saved = @state.idempotency[key]
         if saved
           fail!(ErrorCodes::IDEMPOTENCY_KEY_REUSED) unless saved[:fingerprint] == fingerprint
           return Response.new(status: saved[:status], body: saved[:body],
                               headers: saved[:headers].merge('idempotent-replayed' => 'true', 'x-request-id' => request_id))
         end
-        response = dispatch(method, parts, query, json, body, upload)
+        response = dispatch(method, parts, request)
         if response.success?
           @state.idempotency[key] = { fingerprint: fingerprint, status: response.status, body: response.body,
                                       headers: response.headers }
@@ -387,15 +430,17 @@ module CalmLens
         fail!(ErrorCodes::INVALID_UPLOAD_PAYLOAD, "#{key} is not valid JSON")
       end
 
-      def dispatch(method, parts, query, json, body, upload)
+      def dispatch(method, parts, request)
+        query, json, body = request.values_at(:query, :json, :body)
         data = case parts.first
-               when 'projects' then project(method, parts, query, json, body, upload)
+               when 'projects' then project(method, parts, request)
                when 'assets' then asset(method, parts, query, json, body)
                when 'folders' then folder(method, parts, json)
                when 'hash-lists' then hash_list(method, parts, query, json)
                when 'hash-list-entries' then remove_hash_list_entry(method, parts)
                when 'policies' then policies(method, parts, query, json)
                when 'lenses' then lens(method, parts, json)
+               when 'upload-presets' then upload_preset(method, parts)
                else route_not_found!(method, parts)
                end
         data.is_a?(Response) ? data : respond(200, deep_copy(data))
@@ -403,29 +448,40 @@ module CalmLens
 
       def deep_copy(value) = JSON.parse(JSON.generate(value))
 
-      # `upload` is the X-CalmLens-Upload header, which picks how POST .../assets reads the
-      # request: absent is a JSON body, 'content' raw file content and 'part' a raw first part.
-      def project(method, parts, query, json, body, upload)
+      def project(method, parts, request)
         fail!(ErrorCodes::API_KEY_PROJECT_MISMATCH) unless parts[1] == @project_id
-        case [method, parts[2..], upload]
-        in ['GET', ['events'], _] then events(query)
-        in ['PUT', ['lenses'], _] then lenses(json.fetch('lenses', []))
-        in ['GET', ['lenses'], _] then page(@state.lenses.values, query, JSON.generate(['lenses', @project_id]))
-        in ['POST', ['lenses'], _] then create_lens(json)
-        in ['GET', ['folders'], _] then folder_page(query)
-        in ['POST', ['folders'], _] then create_folder(json)
-        in ['GET', ['hash-lists'], _] then page(project_hash_lists, query, JSON.generate([@project_id]))
-        in ['POST', ['hash-lists'], _] then create_hash_list(json)
-        in ['GET', ['policy'], _] then show_policy(find_policy(ensure_policies))
-        in ['PUT', ['policy'], _] then show_policy(assign_policy(json))
-        in ['GET', ['assets'], _] then list(query)
-        in ['GET', ['assets', 'search'], _] then search(query)
-        in ['GET', ['assets', String => asset_id, 'timeline'], _] then timeline(asset_id, query)
-        in ['POST', ['assets', 'batch'], _] then batch(json)
-        in ['POST', ['signed-asset-urls'], _] then sign_asset_url(json)
-        in ['POST', ['assets'], nil] then create_asset(json, query['wait'] == false, url: json['url'])
-        in ['POST', ['assets'], 'content'] then create_asset(query, query['wait'] == false, data: body)
-        in ['POST', ['assets'], 'part'] then start_multipart(query, body)
+        query, json = request.values_at(:query, :json)
+        case [method, parts[2..]]
+        in ['GET', ['events']] then events(query)
+        in ['PUT', ['lenses']] then lenses(json.fetch('lenses', []))
+        in ['GET', ['lenses']] then page(@state.lenses.values, query, JSON.generate(['lenses', @project_id]))
+        in ['POST', ['lenses']] then create_lens(json)
+        in ['GET', ['folders']] then folder_page(query)
+        in ['POST', ['folders']] then create_folder(json)
+        in ['GET', ['hash-lists']] then page(project_hash_lists, query, JSON.generate([@project_id]))
+        in ['POST', ['hash-lists']] then create_hash_list(json)
+        in ['GET', ['policy']] then show_policy(find_policy(ensure_policies))
+        in ['PUT', ['policy']] then show_policy(assign_policy(json))
+        in ['GET', ['assets']] then list(query)
+        in ['GET', ['assets', 'search']] then search(query)
+        in ['GET', ['assets', String => asset_id, 'timeline']] then timeline(asset_id, query)
+        in ['POST', ['assets', 'batch']] then batch(json)
+        in ['POST', ['signed-asset-urls']] then sign_asset_url(json)
+        in ['POST', ['assets']] then create_upload(request)
+        else configuration(method, parts, query, json)
+        end
+      end
+
+      # A project's upload presets, settings and metadata fields.
+      def configuration(method, parts, query, json)
+        case [method, parts[2..]]
+        in ['GET', ['upload-presets']] then upload_preset_page(query)
+        in ['PUT', ['upload-presets']] then ensure_upload_presets(json)
+        in ['GET', ['settings']] then show_project_settings
+        in ['PUT', ['settings']] then update_project_settings(json)
+        in ['GET', ['metadata-fields']] then show_metadata_fields
+        in ['PUT', ['metadata-fields']] then ensure_metadata_fields(json)
+        in ['DELETE', ['metadata-fields', String => field_id]] then delete_metadata_field(field_id, query)
         else route_not_found!(method, parts)
         end
       end
@@ -560,6 +616,7 @@ module CalmLens
         old = @state.lenses[definition['identifier']]
         return new_lens(definition) unless old
 
+        definition = active_by_default(definition)
         same = %w[name type active].all? { |key| old[key] == definition[key] } &&
                old['description'] == definition['description'] && canonical(old['transform']) == canonical(definition['transform'])
         return old if same
@@ -569,10 +626,13 @@ module CalmLens
 
       def new_lens(definition)
         now = @now.call
-        lens = { 'id' => @create_id.call('lens'), 'projectId' => @project_id }.merge(definition)
+        lens = { 'id' => @create_id.call('lens'), 'projectId' => @project_id }.merge(active_by_default(definition))
         @state.lenses[definition['identifier']] = lens.merge('createdAt' => now, 'updatedAt' => now)
       end
 
+      # Like the API, a lens is served unless its definition says otherwise.
+      def active_by_default(definition) = definition['active'].nil? ? definition.merge('active' => true) : definition
+
       # POST /projects/:id/lenses: like the API, the identifier must be new to the project.
       def create_lens(json)
         identifier = json['identifier']
@@ -606,6 +666,129 @@ module CalmLens
         end
       end
 
+      # A page of the project's upload presets, oldest first unless `sort` says otherwise.
+      def upload_preset_page(query)
+        page(@upload_presets.values, query, JSON.generate(['upload-presets', @project_id]),
+             default: { 'field' => 'createdAt', 'direction' => 'asc' })
+      end
+
+      # Like the API: every preset's revision and settings are checked before any is saved, and the
+      # stored presets come back in the order sent.
+      def ensure_upload_presets(json)
+        definitions = json['uploadPresets']
+        invalid!('validation/too-small', 'uploadPresets') unless definitions.is_a?(Array) && !definitions.empty?
+
+        plans = definitions.each_with_index.map do |definition, index|
+          %w[key name].each do |field|
+            value = definition[field]
+            invalid!('validation/invalid-type', "uploadPresets.#{index}.#{field}") unless value.is_a?(String) && !value.empty?
+          end
+          existing = @upload_presets[definition['key']]
+          stored = existing ? existing['revision'] : 0
+          if definition.key?('revision') && definition['revision'] != stored
+            fail!(ErrorCodes::UPLOAD_PRESET_REVISION_CONFLICT, details: { 'key' => definition['key'], 'revision' => stored })
+          end
+          values = upload_preset_values(definition)
+          limited = ![nil, 0].include?(values['dailyUploadLimit'])
+          fail!(ErrorCodes::UPLOAD_PRESET_DAILY_LIMIT_REQUIRED) if values['access'] == 'anyone' && !limited
+
+          [existing, values]
+        end
+        { 'uploadPresets' => plans.map { |existing, values| save_upload_preset(existing, values) } }
+      end
+
+      # The stored values a definition sets: fields left out take their defaults.
+      def upload_preset_values(definition)
+        given = ->(field, default = nil) { definition[field].nil? ? default : definition[field] }
+        list = ->(field) { definition[field].is_a?(Array) && !definition[field].empty? ? definition[field].uniq : nil }
+        { 'key' => definition['key'], 'name' => definition['name'], 'access' => given.call('access', 'token'),
+          'enabled' => given.call('enabled', true), 'allowedContentTypes' => list.call('allowedContentTypes'),
+          'maxSizeInBytes' => given.call('maxSizeInBytes'), 'folderId' => given.call('folderId'),
+          'policyId' => given.call('policyId'), 'visibility' => given.call('visibility', 'private'),
+          'processingOptions' => given.call('processingOptions'), 'revealDecision' => given.call('revealDecision', false),
+          'dailyUploadLimit' => given.call('dailyUploadLimit'), 'allowedOrigins' => list.call('allowedOrigins') }
+      end
+
+      # Saves a preset's values under its key; an unchanged preset keeps its revision.
+      def save_upload_preset(existing, values)
+        return existing if existing && existing.slice(*values.keys) == values
+
+        now = @now.call
+        @upload_presets[values['key']] =
+          { 'id' => existing ? existing['id'] : @create_id.call('preset'), 'projectId' => @project_id, 'createdByUserId' => nil,
+            **values, 'revision' => (existing ? existing['revision'] : 0) + 1,
+            'createdAt' => existing ? existing['createdAt'] : now, 'updatedAt' => now }
+      end
+
+      # DELETE /upload-presets/:id.
+      def upload_preset(method, parts)
+        route_not_found!(method, parts) unless method == 'DELETE' && parts.size == 2
+        preset = @upload_presets.values.find { |item| item['id'] == parts[1] }
+        fail!(ErrorCodes::UPLOAD_PRESET_NOT_FOUND) unless preset
+        @upload_presets.delete(preset['key'])
+        {}
+      end
+
+      def show_project_settings
+        settings = { 'optOutJurisdictions' => [], 'blockUnmoderatedAssets' => true, 'storeSensitiveMetadata' => false,
+                     'streamingEnabled' => false, 'aiMediaSearchEnabled' => false, 'remoteUrlRule' => nil,
+                     'websiteSubmissionsPerMinute' => nil }.merge(@project_settings)
+        settings['humanReview'] = { 'allowedSamplePercent' => 0, 'restrictedSamplePercent' => 0, 'uncertaintyMarginPercent' => 0,
+                                    'reviewMissingEvidence' => false }.merge(@project_settings['humanReview'] || {})
+        { 'projectId' => @project_id, 'revision' => @settings_revision, 'settings' => settings }
+      end
+
+      # A revision to check, from a body or the query; none refuses nothing.
+      def check_settings_revision(revision)
+        return if revision.nil? || revision.to_s == @settings_revision.to_s
+
+        fail!(ErrorCodes::PROJECT_SETTINGS_REVISION_CONFLICT)
+      end
+
+      # Replaces the settings: those left out return to their defaults. A real change bumps the revision.
+      def update_project_settings(json)
+        check_settings_revision(json['revision'])
+        invalid!('validation/invalid-type', 'settings') unless json['settings'].is_a?(Hash)
+
+        before = show_project_settings['settings']
+        previous = @project_settings
+        @project_settings = json['settings']
+        if show_project_settings['settings'] == before
+          @project_settings = previous
+        else
+          @settings_revision += 1
+        end
+        show_project_settings
+      end
+
+      def show_metadata_fields = { 'projectId' => @project_id, 'revision' => @settings_revision, 'fields' => @metadata_fields }
+
+      # Creates or replaces fields by ID, keeping the others in order. A real change bumps the revision.
+      def ensure_metadata_fields(json)
+        check_settings_revision(json['revision'])
+        fields = json['fields']
+        invalid!('validation/too-small', 'fields') unless fields.is_a?(Array) && !fields.empty?
+
+        by_id = fields.to_h { |field| [field['id'], field] }
+        known = @metadata_fields.map { |field| field['id'] }
+        merged = @metadata_fields.map { |field| by_id.fetch(field['id'], field) } + fields.reject { |field| known.include?(field['id']) }
+        unless merged == @metadata_fields
+          @metadata_fields = merged
+          @settings_revision += 1
+        end
+        show_metadata_fields
+      end
+
+      def delete_metadata_field(field_id, query)
+        kept = @metadata_fields.reject { |field| field['id'] == field_id }
+        fail!(ErrorCodes::METADATA_FIELD_NOT_FOUND, details: { 'fieldId' => field_id }) if kept.size == @metadata_fields.size
+
+        check_settings_revision(query['revision'])
+        @metadata_fields = kept
+        @settings_revision += 1
+        show_metadata_fields
+      end
+
       # Keyword search like the API: every whitespace-separated token (case-insensitive, at most 8
       # distinct) must appear in some field. Results rank by the API's score, then newest first.
       def search(query)
@@ -663,7 +846,7 @@ module CalmLens
         keep = processing.is_a?(Hash) && processing['keepContentAfterProcessing'] == true
         context = options['context'].is_a?(Hash) ? ModerationContext.select_items(options['context'].fetch('items', [])) : []
         # Fields the API returns as null when they are not set.
-        asset = { 'id' => @create_id.call('asset'), 'projectId' => @project_id, 'parentId' => nil,
+        asset = { 'id' => @create_id.call('asset'), 'projectId' => @project_id, 'parentId' => nil, 'parentPages' => [],
                   'folderId' => options['folderId'], 'uploadPresetId' => options['uploadPresetId'],
                   'createdAt' => now, 'updatedAt' => now, 'name' => options['name'], 'description' => options['description'],
                   'externalId' => options['externalId'], 'externalUrl' => url || options['externalUrl'],
@@ -698,7 +881,8 @@ module CalmLens
                                 # Like the API, moderation's results are null until it decides, and a
                                 # feature's result is null until it has run.
                                 'moderation' => { 'status' => 'pending', 'manualReviews' => [], 'automatedStatus' => nil,
-                                                  'source' => nil, 'rule' => nil, 'explanationComplete' => nil,
+                                                  'policy' => nil, 'pendingReview' => nil, 'source' => nil, 'rule' => nil,
+                                                  'explanationComplete' => nil,
                                                   'categories' => nil, 'flaggedCategories' => [], 'regions' => [],
                                                   'raw' => nil, 'timeline' => nil },
                                 'ocr' => nil, 'linkDetection' => nil }
@@ -729,13 +913,122 @@ module CalmLens
         fail!(ErrorCodes::INVALID_UPLOAD_PAYLOAD, 'Multipart upload parts must be at least 5MiB except the final part.')
       end
 
+      # `POST .../assets` as the API reads it. Asset fields in the query are refused; then the
+      # `X-CalmLens-Upload` header picks the body: absent is JSON (text, a first text part or a URL),
+      # `content` an upload form with a whole file and `part` one with a first part.
+      def create_upload(request)
+        query, json, body, upload = request.values_at(:query, :json, :body, :upload)
+        in_query = Assets::UPLOAD_OPTION_KEYS.select { |key| query.key?(key) }
+        unless in_query.empty?
+          fail!(ErrorCodes::UPLOAD_OPTIONS_IN_QUERY, "Send the asset's fields in the request body, not in the query string",
+                details: { 'fields' => in_query })
+        end
+        queue = query['wait'] == false
+        return create_from_json(json, queue) if upload.nil?
+
+        route_not_found!('POST', ['assets']) unless %w[content part].include?(upload)
+
+        options, bytes = form_fields(body, request[:content_type])
+        fields = options.merge('contentType' => query['contentType'])
+        return create_asset(apply_upload_preset(fields, bytes), queue, data: bytes) if upload == 'content'
+
+        start_multipart(apply_upload_preset(fields.merge(query.slice('isFinal', 'totalSizeInBytes')), bytes), bytes, queue)
+      end
+
+      # An API-key upload through a preset, as the API applies it: the preset gives the folder and
+      # visibility, which the upload can't change, and the processing options unless the upload sets
+      # its own, and refuses types and sizes it doesn't accept.
+      def apply_upload_preset(fields, data = nil)
+        return fields unless fields['uploadPresetId']
+
+        preset = @upload_presets.values.find { |item| item['id'] == fields['uploadPresetId'] }
+        fail!(ErrorCodes::UPLOAD_PRESET_NOT_FOUND) unless preset
+        fail!(ErrorCodes::UPLOAD_PRESET_DISABLED) unless preset['enabled']
+        check_preset_fields(preset, fields)
+        check_preset_limits(preset, fields, data)
+        fields.merge('folderId' => preset['folderId'] || fields['folderId'], 'visibility' => preset['visibility'],
+                     'processingOptions' => fields['processingOptions'] || deep_copy(preset['processingOptions']))
+      end
+
+      def check_preset_fields(preset, fields)
+        if preset['folderId'] && fields['folderId'] && fields['folderId'] != preset['folderId']
+          fail!(ErrorCodes::UPLOAD_PRESET_FIELD_NOT_ALLOWED, details: { 'field' => 'folderId' })
+        end
+        return unless fields['visibility'] && fields['visibility'] != preset['visibility']
+
+        fail!(ErrorCodes::UPLOAD_PRESET_FIELD_NOT_ALLOWED, details: { 'field' => 'visibility' })
+      end
+
+      def check_preset_limits(preset, fields, data)
+        allowed = preset['allowedContentTypes']
+        type = fields['contentType']
+        if allowed && type && allowed.none? { |pattern| preset_accepts?(pattern, fields['name'], type) }
+          fail!(ErrorCodes::UNSUPPORTED_UPLOAD_TYPE,
+                details: { 'source' => 'preset', 'contentType' => type, 'allowedContentTypes' => allowed })
+        end
+        limit = preset['maxSizeInBytes']
+        size = fields['totalSizeInBytes'] || data&.bytesize
+        fail!(ErrorCodes::UPLOAD_TOO_LARGE, details: { 'limitBytes' => limit, 'source' => 'preset' }) if limit && size && size > limit
+      end
+
+      # Whether an upload matches one entry of a preset's list, such as `image/*`, `text/plain` or `.pdf`.
+      def preset_accepts?(pattern, name, type)
+        pattern = pattern.strip.downcase
+        return true if ['*', '*/*'].include?(pattern)
+        return type.downcase.start_with?(pattern.delete_suffix('*')) if pattern.end_with?('/*')
+        return name.to_s.downcase.end_with?(pattern) if pattern.start_with?('.')
+
+        type.downcase == pattern
+      end
+
+      # A JSON create: text as `content` or a first `part` (`{ plainText, type }`), or a URL.
+      def create_from_json(json, queue)
+        key = %w[content part].find { |name| json.key?(name) }
+        return create_asset(apply_upload_preset(json), queue, url: json['url']) unless key
+
+        text = json[key]
+        invalid!('validation/invalid-type', key) unless text.is_a?(Hash) && text['plainText'].is_a?(String) && text['type'].is_a?(String)
+        fields = json.except('content', 'part', 'sizeInBytes', 'totalSizeInBytes').merge('contentType' => text['type'])
+        data = text['plainText'].encode(Encoding::UTF_8).b
+        return create_asset(apply_upload_preset(fields, data), queue, data: data) if key == 'content'
+
+        part = fields.merge('isFinal' => text['isFinal'] == true, 'totalSizeInBytes' => json['totalSizeInBytes'])
+        start_multipart(apply_upload_preset(part, data), data, queue)
+      end
+
+      # The asset fields in an upload form's options part, and the file's bytes. Like the API, a raw
+      # upload must be `multipart/form-data` (`request/unsupported-content-type`, 415), holding an
+      # `options` JSON object of known fields with a name, then a `file` part.
+      def form_fields(body, content_type)
+        mime = content_type.to_s.split(';').first.to_s.strip.downcase
+        unless mime == 'multipart/form-data'
+          fail!(ErrorCodes::UNSUPPORTED_CONTENT_TYPE, 'Send a raw upload as multipart/form-data: an options part, then a file part.',
+                details: { 'contentType' => mime.empty? ? nil : mime })
+        end
+        form = FakeAPI.upload_form(body, content_type)
+        fail!(ErrorCodes::INVALID_UPLOAD_PAYLOAD, 'The upload form must hold an options part, then a file part') unless form
+
+        options = begin
+          JSON.parse(form[0])
+        rescue JSON::ParserError
+          nil
+        end
+        unless options.is_a?(Hash)
+          fail!(ErrorCodes::INVALID_UPLOAD_PAYLOAD, "The options part must be a JSON object with the asset's fields")
+        end
+        unknown = options.keys.find { |name| !Assets::UPLOAD_OPTION_KEYS.include?(name) }
+        invalid!('validation/unknown-field', unknown) if unknown
+        invalid!('validation/invalid-type', 'name') unless options['name'].is_a?(String)
+        [options, form[1]]
+      end
+
       # In the API's order: a declared total, then a single final part as the whole upload, then the
       # part itself. Whether to wait and the declared total are saved with an open upload.
-      def start_multipart(query, data)
-        check_upload(query)
-        total = query['totalSizeInBytes']
-        type = kind(query['contentType'].to_s, nil)
-        final = query['isFinal'] == true
+      def start_multipart(fields, data, queue)
+        check_upload(fields)
+        total = fields['totalSizeInBytes']
+        type = kind(fields['contentType'].to_s, nil)
+        final = fields['isFinal'] == true
         size = data.to_s.bytesize
         code = (total && image_size_error(type, total, true)) ||
                (final ? total_size_error(total, size, true) || image_size_error(type, size, true) : nil)
@@ -743,8 +1036,8 @@ module CalmLens
         check_part_size(size, final)
         fail!(total_size_error(total, size, false)) if !final && total_size_error(total, size, false)
 
-        asset = create_asset(query, query['wait'] == false, data: data, open: !final)
-        @state.uploads[asset['id']] = { 'queue' => query['wait'] == false, 'total' => total } unless final
+        asset = create_asset(fields, queue, data: data, open: !final)
+        @state.uploads[asset['id']] = { 'queue' => queue, 'total' => total } unless final
         asset
       end
 
@@ -784,7 +1077,7 @@ module CalmLens
       # faces in an image, so a decided image reports that the features its upload ran found nothing:
       # OCR and person and face detection when turned on, link detection unless turned off.
       def decided(asset, decision)
-        apply_outcome(asset, decision.status, 'policy', decision.report)
+        apply_outcome(asset, decision.status, 'policy', decision.report(@now.call))
         classification = asset['classification']
         return asset unless classification['status'] == 'completed' && asset['type'] == 'image'
 
@@ -1507,9 +1800,9 @@ module CalmLens
         grant, item = resolve_asset(raw_segments[2])
         return grant if grant.is_a?(Response)
         return asset_not_found unless item && RESOURCES.include?(resource)
-        if resource == 'original' && grant && !grant[:original]
-          return error(ErrorCodes::SIGNED_URL_ORIGINAL_NOT_GRANTED, "This signed asset URL doesn't grant the original file", status: 403)
-        end
+
+        refused = refuse_viewer(item, resource, grant, headers)
+        return refused if refused
 
         params = URI.decode_www_form(query.to_s).to_h
         # A grant authorises a private asset without the API key.
@@ -1521,6 +1814,18 @@ module CalmLens
         deliver(method, item, resource, params, selected ? options || {} : nil, authorised)
       end
 
+      # What a grant doesn't cover, or, like the CDN, a private asset for anyone without a grant or the
+      # project's API key: it answers as an unknown asset does, whatever its state, except for its
+      # public placeholder and its info, which has its own rule. An error response, or nil.
+      def refuse_viewer(item, resource, grant, headers)
+        if resource == 'original' && grant && !grant[:original]
+          return error(ErrorCodes::SIGNED_URL_ORIGINAL_NOT_GRANTED, "This signed asset URL doesn't grant the original file", status: 403)
+        end
+        return if grant || item['visibility'] == 'public' || header(headers, 'X-Api-Key') == @api_key
+
+        asset_not_found unless %w[placeholder info].include?(resource)
+      end
+
       # Whether the path after the resource selects a lens or transform, and its options; nil when the
       # CDN refuses it. Like the CDN, the adaptive stream's files sit beneath `/hls/` (its master
       # playlist is `/hls/index.m3u8`). Otherwise the selector after the resource is read
@@ -1559,7 +1864,7 @@ module CalmLens
       # A private asset's info needs a grant or an API key with `assets:download`.
       def info(method, item, authorised, params)
         restricted = item['visibility'] != 'public'
-        return error('asset/not-available', 'Asset is not available', status: 404) if restricted && !authorised
+        return asset_not_found if restricted && !authorised
 
         claims = JSON.parse(params.fetch('viewerClaims', '{}'))
         body = { 'id' => item['id'], 'projectId' => item['projectId'], 'type' => item['type'], 'externalUrl' => item['externalUrl'],

data/lib/calmlens/testing/mock_decision.rb

@@ -5,11 +5,13 @@ require_relative 'classification'
 module CalmLens
   # A moderation decision for {MockServerClient}'s `moderate:` option or
   # {MockServerClient#complete_processing}, started with {.of}: flag the categories it found, with
-  # their scores and qualifiers, and mark the categories it couldn't assess. Each call checks its
-  # names and scores and raises `sdk/invalid-argument` for one CalmLens doesn't have.
+  # their scores and qualifiers, mark the categories it couldn't assess, and leave it waiting for
+  # human review. Each call checks its names and scores and raises `sdk/invalid-argument` for one
+  # CalmLens doesn't have.
   #
   # @example
   #   CalmLens::MockDecision.of('blocked').flag('violence', 0.9, 'violence-threat' => 0.9).not_evaluated('profanity')
+  #   CalmLens::MockDecision.of('limited').flag('violence', 0.5).pending_review('uncertain', ['violence'])
   class MockDecision
     # Where the mock's moderation leaves an upload: `pending` (still processing), `failed`
     # (processing failed, retryable), or a decision with that moderation status (`purged` also
@@ -19,6 +21,9 @@ module CalmLens
     STATUS_ACTIONS = { 'allowed' => 'allow', 'limited' => 'limit', 'age-restricted' => 'age-restrict', 'blocked' => 'block',
                        'purged' => 'purge' }.freeze
     private_constant :STATUS_ACTIONS
+    # Why a decision waits for human review.
+    PENDING_REVIEW_REASONS = %w[sampled uncertain].freeze
+    private_constant :PENDING_REVIEW_REASONS
 
     # @return [String]
     attr_reader :status
@@ -81,17 +86,40 @@ module CalmLens
       self
     end
 
+    # Leaves the decision waiting for human review, as `classification.moderation.pendingReview`
+    # reports: `sampled`, or `uncertain` with the `uncertain_categories` that made it so.
+    # {MockServerClient#decide} clears it; {Assets#decide} doesn't, as with the API.
+    # @param reason [String] `sampled` or `uncertain`
+    # @param uncertain_categories [Array<String>]
+    # @return [self]
+    # @raise [InvalidArgumentError] for a `pending` or `failed` decision, another reason, a `sampled`
+    #   reason with uncertain categories, or an unknown category
+    def pending_review(reason, uncertain_categories = [])
+      raise Internal.invalid("a #{@status} decision can't wait for review") if undecided?
+
+      reason = reason.to_s
+      raise Internal.invalid("reason must be one of #{PENDING_REVIEW_REASONS.join(', ')}") unless PENDING_REVIEW_REASONS.include?(reason)
+      raise Internal.invalid('uncertain_categories must be an Array') unless uncertain_categories.is_a?(Array)
+      raise Internal.invalid('a sampled review has no uncertain categories') if reason == 'sampled' && !uncertain_categories.empty?
+
+      @pending_review = { 'reason' => reason, 'uncertainCategories' => uncertain_categories.map { |category| known_category(category) } }
+      self
+    end
+
     # The report a decided asset gets, as the API sends it: every category (unflagged ones
     # evaluated with score 0), the policy's actions, the automated status and `flaggedCategories`.
+    # @param requested_at [Integer] when a pending review joined the queue, in epoch milliseconds
     # @return [Hash{String => Object}, nil] `categories`, `score` and `policy`; nil for `pending` and `failed`
-    def report
+    def report(requested_at = 0)
       return if undecided?
 
       evaluated = (MockClassification::CATEGORIES - @not_evaluated).map { |category| [category, @flags.dig(category, :score) || 0] }
       { 'raw' => { 'categories' => MockClassification::CATEGORIES.to_h { |category| [category, category_report(category)] },
                    'hashMatches' => [] },
-        'categories' => evaluated.to_h { |category, value| [category, { 'score' => value, 'action' => action(value) }] },
-        'explanationComplete' => true, 'source' => 'policy', 'rule' => nil, 'automatedStatus' => @status,
+        'categories' => evaluated.to_h { |category, value| [category, policy_result(value)] },
+        # The mock keeps no policies, so no policy is named.
+        'explanationComplete' => true, 'policy' => nil, 'source' => 'policy', 'rule' => nil, 'automatedStatus' => @status,
+        'pendingReview' => @pending_review && { 'requestedAt' => requested_at, **@pending_review },
         'regions' => [], 'linkSafety' => nil, 'flaggedCategories' => evaluated.reject { |_, value| action(value) == 'allow' }.map(&:first) }
     end
 
@@ -101,9 +129,15 @@ module CalmLens
 
     def undecided? = %w[pending failed].include?(@status)
 
-    def check_category(category)
+    def known_category(category)
       category = category.to_s
       raise Internal.invalid("#{category} is not a CalmLens category") unless MockClassification::CATEGORIES.include?(category)
+
+      category
+    end
+
+    def check_category(category)
+      category = known_category(category)
       raise Internal.invalid("a #{@status} decision has no findings") if undecided?
       raise Internal.invalid("#{category} is already in this decision") if @flags.key?(category) || @not_evaluated.include?(category)
 
@@ -114,6 +148,13 @@ module CalmLens
 
     def action(score) = @status != 'allowed' && score >= 0.5 ? STATUS_ACTIONS[@status] : 'allow'
 
+    # A category's policy result. The mock keeps no policies: the category's setting is the
+    # decision's own action.
+    def policy_result(score)
+      { 'score' => score, 'action' => action(score), 'source' => { 'kind' => score >= 0.5 ? 'default' : 'below-threshold' },
+        'settings' => { 'action' => STATUS_ACTIONS[@status], 'allowances' => [], 'restrictions' => [] } }
+    end
+
     # The API's likelihood for a score (ClassificationCategories.ts, scoreToLikelihood).
     def likelihood(score)
       if score >= 0.8 then 'very-likely'