calmlens 0.21.0 → 0.22.0

data/lib/calmlens/testing/fake_api.rb

@@ -43,8 +43,7 @@ module CalmLens
     # semantic search.
     # @api private
     class FakeAPI
-      JSON_QUERY = %w[wait isFinal sizeInBytes totalSizeInBytes partNumber pageSize processingOptions metadata
-                      tags context].freeze
+      JSON_QUERY = %w[wait isFinal sizeInBytes totalSizeInBytes partNumber pageSize].freeze
       # Keyword search fields, in the order matches are reported, and where each value sits in the asset.
       SEARCH_FIELDS = { 'name' => %w[name], 'id' => %w[id], 'externalId' => %w[externalId],
                         'description' => %w[description], 'tags' => %w[tags], 'type' => %w[type],
@@ -82,7 +81,7 @@ module CalmLens
       # from its pixels. The `placeholder` resource serves it as a PNG, like the CDN.
       THUMBHASH = '1QcSHQRnh493V4dIh4eXh1h4kJUI'
       # Removing or withholding content removes its rendered file, adaptive stream and preview, as the API does.
-      NO_RENDERING = { 'rendered' => nil, 'hls' => nil, 'preview' => nil }.freeze
+      NO_RENDERING = { 'rendered' => nil, 'hls' => nil, 'hlsStatus' => nil, 'preview' => nil }.freeze
       # The size of the waveform the API makes as an audio asset's preview.
       WAVEFORM_SIZE = { 'width' => 1200, 'height' => 300 }.freeze
       # Audio and video formats browsers play as uploaded, so the API serves them as stored (Asset.ts).
@@ -112,6 +111,35 @@ module CalmLens
       # The mock API key's `{ 'actions', 'folderId', 'includeSubfolders' }` grants.
       attr_accessor :grants
 
+      # Splits an upload form as the API reads it: exactly an `options` part, then a `file` part, then
+      # the closing boundary.
+      # @return [Array(String, String), nil] the options part's text and the file's bytes, or nil
+      #   when the body isn't such a form.
+      def self.upload_form(body, content_type)
+        boundary = content_type.to_s.match(%r{\Amultipart/form-data\s*;\s*boundary=(?:"([^"]+)"|([^;\s]+))}i)
+        return nil unless boundary
+
+        framed = "\r\n".b + body.to_s.b
+        delimiter = "\r\n--#{boundary[1] || boundary[2]}".b
+        return nil unless framed.start_with?(delimiter)
+
+        parts = []
+        at = 0
+        until framed.byteslice(at + delimiter.bytesize, 2) == '--'
+          start = at + delimiter.bytesize
+          header_end = framed.index("\r\n\r\n".b, start)
+          following = header_end && framed.index(delimiter, header_end + 4)
+          return nil unless following
+
+          name = framed.byteslice(start, header_end - start)[/content-disposition:\s*form-data\s*;[^\r\n]*?\bname="([^"]*)"/i, 1]
+          parts << [name, framed.byteslice(header_end + 4, following - header_end - 4)]
+          at = following
+        end
+        return nil unless parts.map(&:first) == %w[options file]
+
+        [parts[0][1].force_encoding(Encoding::UTF_8), parts[1][1]]
+      end
+
       def initialize(project_id:, base_url:, cdn_base_url:, now:, create_id:, moderate:, api_key:, grants:)
         @project_id = project_id
         @api_key = api_key
@@ -127,6 +155,12 @@ module CalmLens
         # reset `baseline`) and the one the project moderates with.
         @policies = {}
         @project_policy_id = nil
+        # The project's configuration: upload presets by key, the settings it has set, its metadata
+        # fields in display order and the revision the settings and fields share.
+        @upload_presets = {}
+        @project_settings = {}
+        @metadata_fields = []
+        @settings_revision = 1
         @request_count = 0
         # Each asset's latest `assetSequence`, which the API keeps on the asset row.
         @asset_event_sequences = Hash.new(0)
@@ -357,19 +391,25 @@ module CalmLens
         json = content_type == 'application/json' && body && upload.nil? ? JSON.parse(body) : {}
         # HTTP stacks trim header values, so the API sees the key without surrounding whitespace.
         key = header(headers, 'Idempotency-Key')&.to_s&.gsub(/\A[ \t\r\n]+|[ \t\r\n]+\z/, '')
-        return dispatch(method, parts, query, json, body, upload) unless key && method == 'POST'
+        request = { query: query, json: json, body: body, upload: upload, content_type: content_type }
+        return dispatch(method, parts, request) unless key && method == 'POST'
 
         fail!(ErrorCodes::INVALID_IDEMPOTENCY_KEY) unless key.match?(IDEMPOTENCY_KEY)
         # Like the API: a successful response is saved and replayed for the same key and request;
-        # a failed one frees the key.
-        fingerprint = [method, relative, content_type, Digest::SHA256.hexdigest(body.to_s)]
+        # a failed one frees the key. An upload form counts by its options and file, not its boundary.
+        form = upload && FakeAPI.upload_form(body, content_type)
+        fingerprint = if form
+                        [method, relative, 'multipart/form-data', form[0], Digest::SHA256.hexdigest(form[1])]
+                      else
+                        [method, relative, content_type, Digest::SHA256.hexdigest(body.to_s)]
+                      end
         saved = @state.idempotency[key]
         if saved
           fail!(ErrorCodes::IDEMPOTENCY_KEY_REUSED) unless saved[:fingerprint] == fingerprint
           return Response.new(status: saved[:status], body: saved[:body],
                               headers: saved[:headers].merge('idempotent-replayed' => 'true', 'x-request-id' => request_id))
         end
-        response = dispatch(method, parts, query, json, body, upload)
+        response = dispatch(method, parts, request)
         if response.success?
           @state.idempotency[key] = { fingerprint: fingerprint, status: response.status, body: response.body,
                                       headers: response.headers }
@@ -387,15 +427,17 @@ module CalmLens
         fail!(ErrorCodes::INVALID_UPLOAD_PAYLOAD, "#{key} is not valid JSON")
       end
 
-      def dispatch(method, parts, query, json, body, upload)
+      def dispatch(method, parts, request)
+        query, json, body = request.values_at(:query, :json, :body)
         data = case parts.first
-               when 'projects' then project(method, parts, query, json, body, upload)
+               when 'projects' then project(method, parts, request)
                when 'assets' then asset(method, parts, query, json, body)
                when 'folders' then folder(method, parts, json)
                when 'hash-lists' then hash_list(method, parts, query, json)
                when 'hash-list-entries' then remove_hash_list_entry(method, parts)
                when 'policies' then policies(method, parts, query, json)
                when 'lenses' then lens(method, parts, json)
+               when 'upload-presets' then upload_preset(method, parts)
                else route_not_found!(method, parts)
                end
         data.is_a?(Response) ? data : respond(200, deep_copy(data))
@@ -403,29 +445,40 @@ module CalmLens
 
       def deep_copy(value) = JSON.parse(JSON.generate(value))
 
-      # `upload` is the X-CalmLens-Upload header, which picks how POST .../assets reads the
-      # request: absent is a JSON body, 'content' raw file content and 'part' a raw first part.
-      def project(method, parts, query, json, body, upload)
+      def project(method, parts, request)
         fail!(ErrorCodes::API_KEY_PROJECT_MISMATCH) unless parts[1] == @project_id
-        case [method, parts[2..], upload]
-        in ['GET', ['events'], _] then events(query)
-        in ['PUT', ['lenses'], _] then lenses(json.fetch('lenses', []))
-        in ['GET', ['lenses'], _] then page(@state.lenses.values, query, JSON.generate(['lenses', @project_id]))
-        in ['POST', ['lenses'], _] then create_lens(json)
-        in ['GET', ['folders'], _] then folder_page(query)
-        in ['POST', ['folders'], _] then create_folder(json)
-        in ['GET', ['hash-lists'], _] then page(project_hash_lists, query, JSON.generate([@project_id]))
-        in ['POST', ['hash-lists'], _] then create_hash_list(json)
-        in ['GET', ['policy'], _] then show_policy(find_policy(ensure_policies))
-        in ['PUT', ['policy'], _] then show_policy(assign_policy(json))
-        in ['GET', ['assets'], _] then list(query)
-        in ['GET', ['assets', 'search'], _] then search(query)
-        in ['GET', ['assets', String => asset_id, 'timeline'], _] then timeline(asset_id, query)
-        in ['POST', ['assets', 'batch'], _] then batch(json)
-        in ['POST', ['signed-asset-urls'], _] then sign_asset_url(json)
-        in ['POST', ['assets'], nil] then create_asset(json, query['wait'] == false, url: json['url'])
-        in ['POST', ['assets'], 'content'] then create_asset(query, query['wait'] == false, data: body)
-        in ['POST', ['assets'], 'part'] then start_multipart(query, body)
+        query, json = request.values_at(:query, :json)
+        case [method, parts[2..]]
+        in ['GET', ['events']] then events(query)
+        in ['PUT', ['lenses']] then lenses(json.fetch('lenses', []))
+        in ['GET', ['lenses']] then page(@state.lenses.values, query, JSON.generate(['lenses', @project_id]))
+        in ['POST', ['lenses']] then create_lens(json)
+        in ['GET', ['folders']] then folder_page(query)
+        in ['POST', ['folders']] then create_folder(json)
+        in ['GET', ['hash-lists']] then page(project_hash_lists, query, JSON.generate([@project_id]))
+        in ['POST', ['hash-lists']] then create_hash_list(json)
+        in ['GET', ['policy']] then show_policy(find_policy(ensure_policies))
+        in ['PUT', ['policy']] then show_policy(assign_policy(json))
+        in ['GET', ['assets']] then list(query)
+        in ['GET', ['assets', 'search']] then search(query)
+        in ['GET', ['assets', String => asset_id, 'timeline']] then timeline(asset_id, query)
+        in ['POST', ['assets', 'batch']] then batch(json)
+        in ['POST', ['signed-asset-urls']] then sign_asset_url(json)
+        in ['POST', ['assets']] then create_upload(request)
+        else configuration(method, parts, query, json)
+        end
+      end
+
+      # A project's upload presets, settings and metadata fields.
+      def configuration(method, parts, query, json)
+        case [method, parts[2..]]
+        in ['GET', ['upload-presets']] then upload_preset_page(query)
+        in ['PUT', ['upload-presets']] then ensure_upload_presets(json)
+        in ['GET', ['settings']] then show_project_settings
+        in ['PUT', ['settings']] then update_project_settings(json)
+        in ['GET', ['metadata-fields']] then show_metadata_fields
+        in ['PUT', ['metadata-fields']] then ensure_metadata_fields(json)
+        in ['DELETE', ['metadata-fields', String => field_id]] then delete_metadata_field(field_id, query)
         else route_not_found!(method, parts)
         end
       end
@@ -606,6 +659,129 @@ module CalmLens
         end
       end
 
+      # A page of the project's upload presets, oldest first unless `sort` says otherwise.
+      def upload_preset_page(query)
+        page(@upload_presets.values, query, JSON.generate(['upload-presets', @project_id]),
+             default: { 'field' => 'createdAt', 'direction' => 'asc' })
+      end
+
+      # Like the API: every preset's revision and settings are checked before any is saved, and the
+      # stored presets come back in the order sent.
+      def ensure_upload_presets(json)
+        definitions = json['uploadPresets']
+        invalid!('validation/too-small', 'uploadPresets') unless definitions.is_a?(Array) && !definitions.empty?
+
+        plans = definitions.each_with_index.map do |definition, index|
+          %w[key name].each do |field|
+            value = definition[field]
+            invalid!('validation/invalid-type', "uploadPresets.#{index}.#{field}") unless value.is_a?(String) && !value.empty?
+          end
+          existing = @upload_presets[definition['key']]
+          stored = existing ? existing['revision'] : 0
+          if definition.key?('revision') && definition['revision'] != stored
+            fail!(ErrorCodes::UPLOAD_PRESET_REVISION_CONFLICT, details: { 'key' => definition['key'], 'revision' => stored })
+          end
+          values = upload_preset_values(definition)
+          limited = ![nil, 0].include?(values['dailyUploadLimit'])
+          fail!(ErrorCodes::UPLOAD_PRESET_DAILY_LIMIT_REQUIRED) if values['access'] == 'anyone' && !limited
+
+          [existing, values]
+        end
+        { 'uploadPresets' => plans.map { |existing, values| save_upload_preset(existing, values) } }
+      end
+
+      # The stored values a definition sets: fields left out take their defaults.
+      def upload_preset_values(definition)
+        given = ->(field, default = nil) { definition[field].nil? ? default : definition[field] }
+        list = ->(field) { definition[field].is_a?(Array) && !definition[field].empty? ? definition[field].uniq : nil }
+        { 'key' => definition['key'], 'name' => definition['name'], 'access' => given.call('access', 'token'),
+          'enabled' => given.call('enabled', true), 'allowedContentTypes' => list.call('allowedContentTypes'),
+          'maxSizeInBytes' => given.call('maxSizeInBytes'), 'folderId' => given.call('folderId'),
+          'policyId' => given.call('policyId'), 'visibility' => given.call('visibility', 'private'),
+          'processingOptions' => given.call('processingOptions'), 'revealDecision' => given.call('revealDecision', false),
+          'dailyUploadLimit' => given.call('dailyUploadLimit'), 'allowedOrigins' => list.call('allowedOrigins') }
+      end
+
+      # Saves a preset's values under its key; an unchanged preset keeps its revision.
+      def save_upload_preset(existing, values)
+        return existing if existing && existing.slice(*values.keys) == values
+
+        now = @now.call
+        @upload_presets[values['key']] =
+          { 'id' => existing ? existing['id'] : @create_id.call('preset'), 'projectId' => @project_id, 'createdByUserId' => nil,
+            **values, 'revision' => (existing ? existing['revision'] : 0) + 1,
+            'createdAt' => existing ? existing['createdAt'] : now, 'updatedAt' => now }
+      end
+
+      # DELETE /upload-presets/:id.
+      def upload_preset(method, parts)
+        route_not_found!(method, parts) unless method == 'DELETE' && parts.size == 2
+        preset = @upload_presets.values.find { |item| item['id'] == parts[1] }
+        fail!(ErrorCodes::UPLOAD_PRESET_NOT_FOUND) unless preset
+        @upload_presets.delete(preset['key'])
+        {}
+      end
+
+      def show_project_settings
+        settings = { 'optOutJurisdictions' => [], 'blockUnmoderatedAssets' => true, 'storeSensitiveMetadata' => false,
+                     'streamingEnabled' => false, 'aiMediaSearchEnabled' => false, 'remoteUrlRule' => nil,
+                     'websiteSubmissionsPerMinute' => nil }.merge(@project_settings)
+        settings['humanReview'] = { 'allowedSamplePercent' => 0, 'restrictedSamplePercent' => 0, 'uncertaintyMarginPercent' => 0,
+                                    'reviewMissingEvidence' => false }.merge(@project_settings['humanReview'] || {})
+        { 'projectId' => @project_id, 'revision' => @settings_revision, 'settings' => settings }
+      end
+
+      # A revision to check, from a body or the query; none refuses nothing.
+      def check_settings_revision(revision)
+        return if revision.nil? || revision.to_s == @settings_revision.to_s
+
+        fail!(ErrorCodes::PROJECT_SETTINGS_REVISION_CONFLICT)
+      end
+
+      # Replaces the settings: those left out return to their defaults. A real change bumps the revision.
+      def update_project_settings(json)
+        check_settings_revision(json['revision'])
+        invalid!('validation/invalid-type', 'settings') unless json['settings'].is_a?(Hash)
+
+        before = show_project_settings['settings']
+        previous = @project_settings
+        @project_settings = json['settings']
+        if show_project_settings['settings'] == before
+          @project_settings = previous
+        else
+          @settings_revision += 1
+        end
+        show_project_settings
+      end
+
+      def show_metadata_fields = { 'projectId' => @project_id, 'revision' => @settings_revision, 'fields' => @metadata_fields }
+
+      # Creates or replaces fields by ID, keeping the others in order. A real change bumps the revision.
+      def ensure_metadata_fields(json)
+        check_settings_revision(json['revision'])
+        fields = json['fields']
+        invalid!('validation/too-small', 'fields') unless fields.is_a?(Array) && !fields.empty?
+
+        by_id = fields.to_h { |field| [field['id'], field] }
+        known = @metadata_fields.map { |field| field['id'] }
+        merged = @metadata_fields.map { |field| by_id.fetch(field['id'], field) } + fields.reject { |field| known.include?(field['id']) }
+        unless merged == @metadata_fields
+          @metadata_fields = merged
+          @settings_revision += 1
+        end
+        show_metadata_fields
+      end
+
+      def delete_metadata_field(field_id, query)
+        kept = @metadata_fields.reject { |field| field['id'] == field_id }
+        fail!(ErrorCodes::METADATA_FIELD_NOT_FOUND, details: { 'fieldId' => field_id }) if kept.size == @metadata_fields.size
+
+        check_settings_revision(query['revision'])
+        @metadata_fields = kept
+        @settings_revision += 1
+        show_metadata_fields
+      end
+
       # Keyword search like the API: every whitespace-separated token (case-insensitive, at most 8
       # distinct) must appear in some field. Results rank by the API's score, then newest first.
       def search(query)
@@ -663,7 +839,7 @@ module CalmLens
         keep = processing.is_a?(Hash) && processing['keepContentAfterProcessing'] == true
         context = options['context'].is_a?(Hash) ? ModerationContext.select_items(options['context'].fetch('items', [])) : []
         # Fields the API returns as null when they are not set.
-        asset = { 'id' => @create_id.call('asset'), 'projectId' => @project_id, 'parentId' => nil,
+        asset = { 'id' => @create_id.call('asset'), 'projectId' => @project_id, 'parentId' => nil, 'parentPages' => [],
                   'folderId' => options['folderId'], 'uploadPresetId' => options['uploadPresetId'],
                   'createdAt' => now, 'updatedAt' => now, 'name' => options['name'], 'description' => options['description'],
                   'externalId' => options['externalId'], 'externalUrl' => url || options['externalUrl'],
@@ -698,7 +874,8 @@ module CalmLens
                                 # Like the API, moderation's results are null until it decides, and a
                                 # feature's result is null until it has run.
                                 'moderation' => { 'status' => 'pending', 'manualReviews' => [], 'automatedStatus' => nil,
-                                                  'source' => nil, 'rule' => nil, 'explanationComplete' => nil,
+                                                  'policy' => nil, 'pendingReview' => nil, 'source' => nil, 'rule' => nil,
+                                                  'explanationComplete' => nil,
                                                   'categories' => nil, 'flaggedCategories' => [], 'regions' => [],
                                                   'raw' => nil, 'timeline' => nil },
                                 'ocr' => nil, 'linkDetection' => nil }
@@ -729,13 +906,75 @@ module CalmLens
         fail!(ErrorCodes::INVALID_UPLOAD_PAYLOAD, 'Multipart upload parts must be at least 5MiB except the final part.')
       end
 
+      # `POST .../assets` as the API reads it. Asset fields in the query are refused; then the
+      # `X-CalmLens-Upload` header picks the body: absent is JSON (text, a first text part or a URL),
+      # `content` an upload form with a whole file and `part` one with a first part.
+      def create_upload(request)
+        query, json, body, upload = request.values_at(:query, :json, :body, :upload)
+        in_query = Assets::UPLOAD_OPTION_KEYS.select { |key| query.key?(key) }
+        unless in_query.empty?
+          fail!(ErrorCodes::UPLOAD_OPTIONS_IN_QUERY, "Send the asset's fields in the request body, not in the query string",
+                details: { 'fields' => in_query })
+        end
+        queue = query['wait'] == false
+        return create_from_json(json, queue) if upload.nil?
+
+        route_not_found!('POST', ['assets']) unless %w[content part].include?(upload)
+
+        options, bytes = form_fields(body, request[:content_type])
+        fields = options.merge('contentType' => query['contentType'])
+        return create_asset(fields, queue, data: bytes) if upload == 'content'
+
+        start_multipart(fields.merge(query.slice('isFinal', 'totalSizeInBytes')), bytes, queue)
+      end
+
+      # A JSON create: text as `content` or a first `part` (`{ plainText, type }`), or a URL.
+      def create_from_json(json, queue)
+        key = %w[content part].find { |name| json.key?(name) }
+        return create_asset(json, queue, url: json['url']) unless key
+
+        text = json[key]
+        invalid!('validation/invalid-type', key) unless text.is_a?(Hash) && text['plainText'].is_a?(String) && text['type'].is_a?(String)
+        fields = json.except('content', 'part', 'sizeInBytes', 'totalSizeInBytes').merge('contentType' => text['type'])
+        data = text['plainText'].encode(Encoding::UTF_8).b
+        return create_asset(fields, queue, data: data) if key == 'content'
+
+        start_multipart(fields.merge('isFinal' => text['isFinal'] == true, 'totalSizeInBytes' => json['totalSizeInBytes']), data, queue)
+      end
+
+      # The asset fields in an upload form's options part, and the file's bytes. Like the API, a raw
+      # upload must be `multipart/form-data` (`request/unsupported-content-type`, 415), holding an
+      # `options` JSON object of known fields with a name, then a `file` part.
+      def form_fields(body, content_type)
+        mime = content_type.to_s.split(';').first.to_s.strip.downcase
+        unless mime == 'multipart/form-data'
+          fail!(ErrorCodes::UNSUPPORTED_CONTENT_TYPE, 'Send a raw upload as multipart/form-data: an options part, then a file part.',
+                details: { 'contentType' => mime.empty? ? nil : mime })
+        end
+        form = FakeAPI.upload_form(body, content_type)
+        fail!(ErrorCodes::INVALID_UPLOAD_PAYLOAD, 'The upload form must hold an options part, then a file part') unless form
+
+        options = begin
+          JSON.parse(form[0])
+        rescue JSON::ParserError
+          nil
+        end
+        unless options.is_a?(Hash)
+          fail!(ErrorCodes::INVALID_UPLOAD_PAYLOAD, "The options part must be a JSON object with the asset's fields")
+        end
+        unknown = options.keys.find { |name| !Assets::UPLOAD_OPTION_KEYS.include?(name) }
+        invalid!('validation/unknown-field', unknown) if unknown
+        invalid!('validation/invalid-type', 'name') unless options['name'].is_a?(String)
+        [options, form[1]]
+      end
+
       # In the API's order: a declared total, then a single final part as the whole upload, then the
       # part itself. Whether to wait and the declared total are saved with an open upload.
-      def start_multipart(query, data)
-        check_upload(query)
-        total = query['totalSizeInBytes']
-        type = kind(query['contentType'].to_s, nil)
-        final = query['isFinal'] == true
+      def start_multipart(fields, data, queue)
+        check_upload(fields)
+        total = fields['totalSizeInBytes']
+        type = kind(fields['contentType'].to_s, nil)
+        final = fields['isFinal'] == true
         size = data.to_s.bytesize
         code = (total && image_size_error(type, total, true)) ||
                (final ? total_size_error(total, size, true) || image_size_error(type, size, true) : nil)
@@ -743,8 +982,8 @@ module CalmLens
         check_part_size(size, final)
         fail!(total_size_error(total, size, false)) if !final && total_size_error(total, size, false)
 
-        asset = create_asset(query, query['wait'] == false, data: data, open: !final)
-        @state.uploads[asset['id']] = { 'queue' => query['wait'] == false, 'total' => total } unless final
+        asset = create_asset(fields, queue, data: data, open: !final)
+        @state.uploads[asset['id']] = { 'queue' => queue, 'total' => total } unless final
         asset
       end
 
@@ -897,7 +1136,7 @@ module CalmLens
       # play the upload), audio with a waveform preview and video with no poster; and every preview has
       # its ThumbHash. Text, spreadsheets and other files have neither. The mock decodes nothing, so
       # sizes are unknown, audio has no peaks, and audio and video never get an adaptive stream.
-      def rendering(asset, processed) = { **rendered_file(asset, processed), 'hls' => nil }
+      def rendering(asset, processed) = { **rendered_file(asset, processed), 'hls' => nil, 'hlsStatus' => nil }
 
       # The rendered file and preview of {#rendering}.
       def rendered_file(asset, processed)
@@ -1559,7 +1798,7 @@ module CalmLens
       # A private asset's info needs a grant or an API key with `assets:download`.
       def info(method, item, authorised, params)
         restricted = item['visibility'] != 'public'
-        return error('asset/not-available', 'Asset is not available', status: 404) if restricted && !authorised
+        return asset_not_found if restricted && !authorised
 
         claims = JSON.parse(params.fetch('viewerClaims', '{}'))
         body = { 'id' => item['id'], 'projectId' => item['projectId'], 'type' => item['type'], 'externalUrl' => item['externalUrl'],

data/lib/calmlens/testing/mock_decision.rb

@@ -91,7 +91,8 @@ module CalmLens
       { 'raw' => { 'categories' => MockClassification::CATEGORIES.to_h { |category| [category, category_report(category)] },
                    'hashMatches' => [] },
         'categories' => evaluated.to_h { |category, value| [category, { 'score' => value, 'action' => action(value) }] },
-        'explanationComplete' => true, 'source' => 'policy', 'rule' => nil, 'automatedStatus' => @status,
+        # The mock keeps no policies, so no policy is named.
+        'explanationComplete' => true, 'policy' => nil, 'source' => 'policy', 'rule' => nil, 'automatedStatus' => @status,
         'regions' => [], 'linkSafety' => nil, 'flaggedCategories' => evaluated.reject { |_, value| action(value) == 'allow' }.map(&:first) }
     end
 

data/lib/calmlens/testing/test_inputs.rb

@@ -8,8 +8,7 @@ require_relative 'test_image_grid'
 module CalmLens
   # Signed test inputs: text codes, test images and media phrases that a project's moderation
   # recognises as a declared scenario when they are signed with its secret test input key (shown in
-  # the portal's project settings to those who can read them) and the project has "Recognise test
-  # inputs" turned on, so CalmLens returns that scenario instead of classifying them. The formats
+  # the portal's project settings to those who can read them), so CalmLens returns that scenario instead of classifying them. The formats
   # match the JavaScript SDK's `calmlens/test-inputs` byte for byte.
   #
   # A scenario is a Hash with the API's keys, as Strings or Symbols (snake_case keys are converted

data/lib/calmlens/testing.rb

@@ -47,12 +47,26 @@ module CalmLens
     #
     # `path` is relative to `base_url` for API calls (`/projects/<id>/assets`, without `/v1`) and to
     # `cdn_base_url` for CDN requests, with each segment decoded. `query` holds the decoded query
-    # string. `body` is the bytes sent: JSON text, or the uploaded content (streams are read), or
-    # nil. `status` is the HTTP status the mock answered with, or nil when the request failed
-    # without a response.
+    # string; an upload's holds only transport values (`wait`, `contentType`, sizes and part
+    # numbers). `body` is the bytes sent (streams are read), or nil: JSON text, including a text
+    # upload (`{name, ..., content: {plainText, type}}`), an upload form, or a later part's bytes.
+    # `status` is the HTTP status the mock answered with, or nil when the request failed without a
+    # response.
     Request = Struct.new(:method, :url, :path, :query, :headers, :body, :status, keyword_init: true) do
       # @return [Object] the body parsed as JSON.
       def json = body && JSON.parse(body)
+
+      # An upload form's asset fields and file: `{'options' => Hash, 'file' => String}`, the file as
+      # UTF-8 text for a `text/*` content type and binary otherwise; nil for any other body.
+      # @return [Hash, nil]
+      def form
+        type = headers.find { |name, _| name.to_s.casecmp?('Content-Type') }&.last
+        options, file = Testing::FakeAPI.upload_form(body, type)
+        return nil unless options
+
+        text = query['contentType'].to_s.downcase.start_with?('text/')
+        { 'options' => JSON.parse(options), 'file' => text ? file.dup.force_encoding(Encoding::UTF_8) : file }
+      end
     end
 
     # @return [Testing::State] the stored assets, folders, content and events.
@@ -70,6 +84,9 @@ module CalmLens
                    policies: %i[list retrieve retrieve_assigned create update reset upgrade set_automatic_updates delete
                                 assign],
                    lenses: %i[list create retrieve update delete],
+                   upload_presets: %i[list ensure delete],
+                   project_settings: %i[get update],
+                   metadata_fields: %i[list ensure delete],
                    text: %i[classify], events: %i[list],
                    cdn: %i[serve sign_asset_url get_asset_info] }.freeze
     private_constant :OPERATIONS

data/lib/calmlens/upload_body.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'securerandom'
+
 module CalmLens
   # An upload payload: a binary String, or an IO streamed with an exact byte count (or, when its
   # length is unknown, until it ends, sent with chunked transfer encoding).
@@ -98,6 +100,20 @@ module CalmLens
       UploadBody.new(@source.slice(offset, length))
     end
 
+    # This body as the `file` part of an upload form, after an `options` part holding
+    # `options_json`: `multipart/form-data` with a random boundary. A stream stays a stream, read
+    # as the form is sent, and the form can be sent again whenever this body can.
+    # @return [Array(UploadBody, String)] the form and its Content-Type.
+    def to_form(options_json, type)
+      boundary = "calmlens-#{SecureRandom.uuid}"
+      head = "--#{boundary}\r\nContent-Disposition: form-data; name=\"options\"\r\nContent-Type: application/json\r\n\r\n".b +
+             options_json.b + "\r\n--#{boundary}\r\nContent-Disposition: form-data; name=\"file\"\r\n".b +
+             "Content-Type: #{type.delete("\r\n")}\r\n\r\n".b
+      tail = "\r\n--#{boundary}--\r\n".b
+      source = @source.is_a?(String) ? head + @source + tail : FormReader.new(head, @source, tail)
+      [UploadBody.new(source), "multipart/form-data; boundary=#{boundary}"]
+    end
+
     # The bytes, read fully. Used only by the offline mock.
     def read_all
       return @source if @source.is_a?(String)
@@ -111,9 +127,12 @@ module CalmLens
     class IOReader
       attr_reader :size
 
-      def initialize(io, size)
+      # @param bounded [Boolean] the stream goes on after `size` bytes (a slice of a larger file), so
+      #   reading stops there without checking for more.
+      def initialize(io, size, bounded: false)
         @io = io
         @size = size
+        @bounded = bounded
         @remaining = size
         @start = (io.pos if io.respond_to?(:pos) && io.respond_to?(:seek))
       rescue IOError, SystemCallError
@@ -125,7 +144,7 @@ module CalmLens
       # A reader of `length` bytes from `offset` of this rewindable stream, positioned there.
       def slice(offset, length)
         @io.seek(@start + offset)
-        IOReader.new(@io, length)
+        IOReader.new(@io, length, bounded: true)
       end
 
       def rewind
@@ -138,6 +157,8 @@ module CalmLens
         return read_unsized(length, buffer) if @size.nil?
 
         if @remaining.zero?
+          return length.nil? ? ''.b : buffer&.clear && nil if @bounded
+
           extra = @io.read(1)
           unless extra.nil? || extra.empty?
             raise Internal.invalid('The stream holds more bytes than sizeInBytes',
@@ -169,6 +190,68 @@ module CalmLens
         read(length, buffer) || raise(EOFError)
       end
     end
+
+    # An upload form around a streamed file: the form's head, the file's bytes as an {IOReader}
+    # reads them, then its closing boundary.
+    class FormReader
+      def initialize(head, file, tail)
+        @head = head
+        @file = file
+        @tail = tail
+        @size = file.size && (head.bytesize + file.size + tail.bytesize)
+        rewind_pieces
+      end
+
+      attr_reader :size
+
+      def rewindable? = @file.rewindable?
+
+      def rewind
+        @file.rewind
+        rewind_pieces
+      end
+
+      # Reads like IO#read: everything left without a length, else up to `length` bytes, nil at the end.
+      def read(length = nil, buffer = nil)
+        return (buffer ? buffer.replace(''.b) : ''.b) if length&.zero?
+
+        output = ''.b
+        until @pieces.empty? || (length && output.bytesize >= length)
+          chunk = read_piece(length && (length - output.bytesize))
+          chunk.nil? || chunk.empty? ? next_piece : output << chunk
+        end
+        return buffer&.clear && nil if output.empty? && length
+
+        buffer ? buffer.replace(output) : output
+      end
+
+      def readpartial(length, buffer = nil)
+        read(length, buffer) || raise(EOFError)
+      end
+
+      private
+
+      def rewind_pieces
+        @pieces = %i[head file tail]
+        @offset = 0
+      end
+
+      def next_piece
+        @pieces.shift
+        @offset = 0
+      end
+
+      # Up to `length` bytes (all, without one) of the current piece; empty or nil once it is used up.
+      def read_piece(length)
+        piece = @pieces.first
+        return @file.read(length) if piece == :file
+
+        text = piece == :head ? @head : @tail
+        chunk = text.byteslice(@offset, length || text.bytesize).to_s
+        @offset += chunk.bytesize
+        chunk
+      end
+    end
   end
   private_constant :UploadBody
 end

data/lib/calmlens/version.rb

@@ -2,5 +2,5 @@
 
 module CalmLens
   # The gem version. The gemspec and the User-Agent header read it from here.
-  VERSION = '0.21.0'
+  VERSION = '0.22.0'
 end

data/lib/calmlens.rb

@@ -29,5 +29,6 @@ require_relative 'calmlens/upload_tokens'
 require_relative 'calmlens/events'
 require_relative 'calmlens/cdn'
 require_relative 'calmlens/lenses'
+require_relative 'calmlens/project_configuration'
 require_relative 'calmlens/server_client'
 require_relative 'calmlens/webhooks'