calmlens 0.21.0 → 0.23.0

data/lib/calmlens/testing/test_inputs.rb

@@ -8,8 +8,7 @@ require_relative 'test_image_grid'
 module CalmLens
   # Signed test inputs: text codes, test images and media phrases that a project's moderation
   # recognises as a declared scenario when they are signed with its secret test input key (shown in
-  # the portal's project settings to those who can read them) and the project has "Recognise test
-  # inputs" turned on, so CalmLens returns that scenario instead of classifying them. The formats
+  # the portal's project settings to those who can read them), so CalmLens returns that scenario instead of classifying them. The formats
   # match the JavaScript SDK's `calmlens/test-inputs` byte for byte.
   #
   # A scenario is a Hash with the API's keys, as Strings or Symbols (snake_case keys are converted
@@ -57,7 +56,7 @@ module CalmLens
     person = { 'box' => [2500, 1500, 5000, 8500] }
     face = ->(age = nil) { { 'box' => [3500, 2000, 3000, 3500], **(age ? { 'age' => age } : {}) } }
     # Ready-made scenarios (deeply frozen): a clean input, a limited one, adult images and text,
-    # regional delivery, content restricted in Germany, mixed instances in one category, and test
+    # regional delivery, delivery blocked in Germany, mixed instances in one category, and test
     # images with one person and face, nobody, an adult's face, a minor's face and a group of three.
     PRESETS = deep_freeze.call(
       clean: { 'instances' => [] },
@@ -67,7 +66,7 @@ module CalmLens
       regional_delivery: { 'instances' => [], 'regionalActions' => [{ 'region' => 'DE', 'action' => 'block' },
                                                                     { 'region' => 'GB', 'action' => 'age-restrict' },
                                                                     { 'region' => 'FR', 'action' => 'limit' }] },
-      germany: one.call('hate', 'hate-symbol', 'hate-nazi-symbols-public-display'),
+      germany: { 'instances' => [], 'regionalActions' => [{ 'region' => 'DE', 'action' => 'block' }] },
       mixed: { 'instances' => [{ 'category' => 'harassment', 'qualifiers' => %w[harassment-abuse harassment-documentary] },
                                { 'category' => 'harassment', 'qualifiers' => ['harassment-threat'] }] },
       one_face: { 'instances' => [], 'detections' => { 'people' => [person], 'faces' => [face.call] } },
@@ -98,7 +97,8 @@ module CalmLens
       preset = case signal
                when 'harassment.insult' then PRESETS[:limited]
                when 'sexual.adult' then PRESETS[kind == 'image' ? :adult_image : :adult_text]
-               when 'hate.regional-symbol' then PRESETS[:germany]
+               when 'hate.regional-symbol'
+                 { 'instances' => [{ 'category' => 'hate', 'qualifiers' => %w[hate-symbol hate-nazi-symbols-public-display] }] }
                when 'sexual.involving-minors' then { 'instances' => [{ 'category' => 'sexual', 'qualifiers' => ['sexual-minors'] }] }
                when 'violence.graphic'
                  { 'instances' => [{ 'category' => 'violence', 'qualifiers' => %w[violence-injury violence-graphic] }] }

data/lib/calmlens/testing.rb

@@ -27,8 +27,10 @@ module CalmLens
   # grant (the default key has no grants), or the URL is signed: {CDN#sign_asset_url} returns
   # `{cdn_base_url}/assets/{project_id}/sig_…`, a readable grant that only the mock's CDN accepts,
   # which serves `original` only when signed with `original: true`. {#fetch_cdn} loads such a URL
-  # without the API key, as a browser would. {#decide} applies a reviewer's decision, and
-  # {Assets#wait_for_decision} takes no real time: a pending asset times out at once.
+  # without the API key, as a browser would; without the key or a grant, a private asset answers
+  # as an unknown one does, `asset/not-found` (404) whatever its state, and only its placeholder is
+  # public. {#decide} applies a reviewer's decision, and {Assets#wait_for_decision} takes no real
+  # time: a pending asset times out at once.
   # Filters join comparisons with `;` (and), which binds tighter than `,` (or), and group them with
   # `( … )`; list cursors resume after the last item returned. {Assets#search} is the API's keyword
   # search: every query word (case-insensitive, up to 8) must appear in a searchable field, and
@@ -47,12 +49,26 @@ module CalmLens
     #
     # `path` is relative to `base_url` for API calls (`/projects/<id>/assets`, without `/v1`) and to
     # `cdn_base_url` for CDN requests, with each segment decoded. `query` holds the decoded query
-    # string. `body` is the bytes sent: JSON text, or the uploaded content (streams are read), or
-    # nil. `status` is the HTTP status the mock answered with, or nil when the request failed
-    # without a response.
+    # string; an upload's holds only transport values (`wait`, `contentType`, sizes and part
+    # numbers). `body` is the bytes sent (streams are read), or nil: JSON text, including a text
+    # upload (`{name, ..., content: {plainText, type}}`), an upload form, or a later part's bytes.
+    # `status` is the HTTP status the mock answered with, or nil when the request failed without a
+    # response.
     Request = Struct.new(:method, :url, :path, :query, :headers, :body, :status, keyword_init: true) do
       # @return [Object] the body parsed as JSON.
       def json = body && JSON.parse(body)
+
+      # An upload form's asset fields and file: `{'options' => Hash, 'file' => String}`, the file as
+      # UTF-8 text for a `text/*` content type and binary otherwise; nil for any other body.
+      # @return [Hash, nil]
+      def form
+        type = headers.find { |name, _| name.to_s.casecmp?('Content-Type') }&.last
+        options, file = Testing::FakeAPI.upload_form(body, type)
+        return nil unless options
+
+        text = query['contentType'].to_s.downcase.start_with?('text/')
+        { 'options' => JSON.parse(options), 'file' => text ? file.dup.force_encoding(Encoding::UTF_8) : file }
+      end
     end
 
     # @return [Testing::State] the stored assets, folders, content and events.
@@ -64,12 +80,15 @@ module CalmLens
     REQUEST_FAILURES = %w[transport-error timeout].freeze
     # The methods {#fail_next_request} can target, by resource. Iterators are their list calls.
     OPERATIONS = { assets: %i[search list retrieve list_children list_timeline update create classify upload_part move
-                              retry explain wait_for_decision remove_content delete batch],
+                              retry explain decide wait_for_decision remove_content delete batch],
                    folders: %i[list retrieve create update delete move],
                    hash_lists: %i[list retrieve create update delete list_entries add_entry remove_entry],
                    policies: %i[list retrieve retrieve_assigned create update reset upgrade set_automatic_updates delete
                                 assign],
                    lenses: %i[list create retrieve update delete],
+                   upload_presets: %i[list ensure delete],
+                   project_settings: %i[get update],
+                   metadata_fields: %i[list ensure delete],
                    text: %i[classify], events: %i[list],
                    cdn: %i[serve sign_asset_url get_asset_info] }.freeze
     private_constant :OPERATIONS
@@ -183,8 +202,9 @@ module CalmLens
 
     # Applies a reviewer's decision to a stored asset, as the portal or API would: the moderation
     # status changes (`purged` also removes the content), the review joins `manualReviews` with
-    # `decidedBy` the mock API key (`{ 'type' => 'api-key', 'id' => nil }`) and `applied: true`, and an
-    # `asset.<decision>` event whose `actor` is a `user` joins the events feed.
+    # `decidedBy` the mock API key (`{ 'type' => 'api-key', 'id' => nil }`) and `applied: true`, an
+    # `asset.<decision>` event whose `actor` is a `user` joins the events feed, and a waiting
+    # `pendingReview` is settled ({Assets#decide} leaves it waiting, as with the API).
     # @param asset_id [String]
     # @param decision [String] `allowed`, `limited`, `age-restricted`, `blocked` or `purged`
     # @return [Hash] the asset
@@ -254,7 +274,8 @@ module CalmLens
     # Fetches a URL of the mock's CDN as a browser would, with a GET and without the API key: a signed
     # asset URL from {CDN#sign_asset_url} (with a resource appended), or a public one from
     # {CDN#get_asset_url}. The request is recorded like any other. Error statuses are returned, not
-    # raised.
+    # raised. A private asset without a grant answers `asset/not-found` (404) whatever its state,
+    # except for its public placeholder.
     # @param url [String]
     # @return [Response] with its body read
     def fetch_cdn(url)

data/lib/calmlens/transform_schema.rb

@@ -46,20 +46,6 @@ module CalmLens
           "blur": {
             "type": "number"
           },
-          "redact": {
-            "type": "object",
-            "properties": {
-              "text": {
-                "type": "boolean"
-              },
-              "personalInformation": {
-                "type": "boolean"
-              },
-              "faces": {
-                "type": "boolean"
-              }
-            }
-          },
           "border": {
             "type": "object",
             "properties": {

data/lib/calmlens/transport.rb

@@ -19,7 +19,10 @@ module CalmLens
     # @!attribute open_timeout [Numeric] seconds allowed to open a connection.
     # @!attribute timeout [Numeric] seconds allowed for each read or write; not a whole-transfer deadline.
     Request = Struct.new(:method, :url, :headers, :body, :open_timeout, :timeout, keyword_init: true) do
+      # Struct's own `to_s` and `pretty_print` would show the headers, which hold the API key.
       def inspect = "#<#{self.class.name} #{method} #{url}>"
+      def to_s = inspect
+      def pretty_print(printer) = printer.text(inspect)
     end
 
     # Exceptions that mean the connection failed before a complete response arrived.

data/lib/calmlens/upload_body.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'securerandom'
+
 module CalmLens
   # An upload payload: a binary String, or an IO streamed with an exact byte count (or, when its
   # length is unknown, until it ends, sent with chunked transfer encoding).
@@ -98,6 +100,20 @@ module CalmLens
       UploadBody.new(@source.slice(offset, length))
     end
 
+    # This body as the `file` part of an upload form, after an `options` part holding
+    # `options_json`: `multipart/form-data` with a random boundary. A stream stays a stream, read
+    # as the form is sent, and the form can be sent again whenever this body can.
+    # @return [Array(UploadBody, String)] the form and its Content-Type.
+    def to_form(options_json, type)
+      boundary = "calmlens-#{SecureRandom.uuid}"
+      head = "--#{boundary}\r\nContent-Disposition: form-data; name=\"options\"\r\nContent-Type: application/json\r\n\r\n".b +
+             options_json.b + "\r\n--#{boundary}\r\nContent-Disposition: form-data; name=\"file\"\r\n".b +
+             "Content-Type: #{type.delete("\r\n")}\r\n\r\n".b
+      tail = "\r\n--#{boundary}--\r\n".b
+      source = @source.is_a?(String) ? head + @source + tail : FormReader.new(head, @source, tail)
+      [UploadBody.new(source), "multipart/form-data; boundary=#{boundary}"]
+    end
+
     # The bytes, read fully. Used only by the offline mock.
     def read_all
       return @source if @source.is_a?(String)
@@ -111,9 +127,12 @@ module CalmLens
     class IOReader
       attr_reader :size
 
-      def initialize(io, size)
+      # @param bounded [Boolean] the stream goes on after `size` bytes (a slice of a larger file), so
+      #   reading stops there without checking for more.
+      def initialize(io, size, bounded: false)
         @io = io
         @size = size
+        @bounded = bounded
         @remaining = size
         @start = (io.pos if io.respond_to?(:pos) && io.respond_to?(:seek))
       rescue IOError, SystemCallError
@@ -125,7 +144,7 @@ module CalmLens
       # A reader of `length` bytes from `offset` of this rewindable stream, positioned there.
       def slice(offset, length)
         @io.seek(@start + offset)
-        IOReader.new(@io, length)
+        IOReader.new(@io, length, bounded: true)
       end
 
       def rewind
@@ -138,6 +157,8 @@ module CalmLens
         return read_unsized(length, buffer) if @size.nil?
 
         if @remaining.zero?
+          return length.nil? ? ''.b : buffer&.clear && nil if @bounded
+
           extra = @io.read(1)
           unless extra.nil? || extra.empty?
             raise Internal.invalid('The stream holds more bytes than sizeInBytes',
@@ -169,6 +190,68 @@ module CalmLens
         read(length, buffer) || raise(EOFError)
       end
     end
+
+    # An upload form around a streamed file: the form's head, the file's bytes as an {IOReader}
+    # reads them, then its closing boundary.
+    class FormReader
+      def initialize(head, file, tail)
+        @head = head
+        @file = file
+        @tail = tail
+        @size = file.size && (head.bytesize + file.size + tail.bytesize)
+        rewind_pieces
+      end
+
+      attr_reader :size
+
+      def rewindable? = @file.rewindable?
+
+      def rewind
+        @file.rewind
+        rewind_pieces
+      end
+
+      # Reads like IO#read: everything left without a length, else up to `length` bytes, nil at the end.
+      def read(length = nil, buffer = nil)
+        return (buffer ? buffer.replace(''.b) : ''.b) if length&.zero?
+
+        output = ''.b
+        until @pieces.empty? || (length && output.bytesize >= length)
+          chunk = read_piece(length && (length - output.bytesize))
+          chunk.nil? || chunk.empty? ? next_piece : output << chunk
+        end
+        return buffer&.clear && nil if output.empty? && length
+
+        buffer ? buffer.replace(output) : output
+      end
+
+      def readpartial(length, buffer = nil)
+        read(length, buffer) || raise(EOFError)
+      end
+
+      private
+
+      def rewind_pieces
+        @pieces = %i[head file tail]
+        @offset = 0
+      end
+
+      def next_piece
+        @pieces.shift
+        @offset = 0
+      end
+
+      # Up to `length` bytes (all, without one) of the current piece; empty or nil once it is used up.
+      def read_piece(length)
+        piece = @pieces.first
+        return @file.read(length) if piece == :file
+
+        text = piece == :head ? @head : @tail
+        chunk = text.byteslice(@offset, length || text.bytesize).to_s
+        @offset += chunk.bytesize
+        chunk
+      end
+    end
   end
   private_constant :UploadBody
 end

data/lib/calmlens/version.rb

@@ -2,5 +2,5 @@
 
 module CalmLens
   # The gem version. The gemspec and the User-Agent header read it from here.
-  VERSION = '0.21.0'
+  VERSION = '0.23.0'
 end

data/lib/calmlens.rb

@@ -29,5 +29,6 @@ require_relative 'calmlens/upload_tokens'
 require_relative 'calmlens/events'
 require_relative 'calmlens/cdn'
 require_relative 'calmlens/lenses'
+require_relative 'calmlens/project_configuration'
 require_relative 'calmlens/server_client'
 require_relative 'calmlens/webhooks'

data/sig/calmlens.rbs

@@ -31,6 +31,7 @@ module CalmLens
                      ?headers: Hash[String, String], ?request_id: String?) -> void
     # Whole seconds from the `Retry-After` header.
     def retry_after: () -> Integer?
+    def local?: () -> bool
     def not_found?: () -> bool
     def retryable?: () -> bool
     def self.build: (String code, ?String? message, ?status: Integer?, ?details: Hash[String, json_value]?,
@@ -122,6 +123,9 @@ module CalmLens
     attr_reader events: Events
     attr_reader cdn: CDN
     attr_reader lenses: Lenses
+    attr_reader upload_presets: UploadPresets
+    attr_reader project_settings: ProjectSettings
+    attr_reader metadata_fields: MetadataFields
     attr_reader project_id: String
     attr_reader base_url: String
     attr_reader cdn_base_url: String
@@ -237,6 +241,25 @@ module CalmLens
     def delete: (lens_id_options options) -> nil
   end
 
+  class UploadPresets < Resource
+    def list: (?list_upload_presets_options options) -> upload_preset_page
+    def each: (?list_upload_presets_options options) { (upload_preset) -> void } -> nil
+            | (?list_upload_presets_options options) -> Enumerator[upload_preset, nil]
+    def ensure: (ensure_upload_presets_options options) -> ensure_upload_presets_response
+    def delete: (upload_preset_id_options options) -> nil
+  end
+
+  class ProjectSettings < Resource
+    def get: (?request_only_options options) -> project_settings_resource
+    def update: (update_project_settings_options options) -> project_settings_resource
+  end
+
+  class MetadataFields < Resource
+    def list: (?request_only_options options) -> metadata_fields_resource
+    def ensure: (ensure_metadata_fields_options options) -> metadata_fields_resource
+    def delete: (delete_metadata_field_options options) -> nil
+  end
+
   class UploadTokens < Resource
     TOKEN_PREFIX: String
     SIGNING_CONTEXT: String
@@ -379,6 +402,8 @@ module CalmLens
       # The HTTP status the mock answered with; nil when the request failed without a response.
       attr_accessor status: Integer?
       def json: () -> json_value
+      # An upload form's `options` (parsed) and `file` (its bytes); nil for any other body.
+      def form: () -> { 'options' => Hash[String, json_value], 'file' => String }?
     end
 
     interface _Clock
@@ -445,8 +470,11 @@ module CalmLens
     # Flags `category` with a score from 0 to 1 and optional qualifier scores.
     def flag: (String | Symbol category, Numeric score, ?Hash[String | Symbol, Numeric] qualifiers) -> self
     def not_evaluated: (String | Symbol category) -> self
+    # Leaves the decision waiting for human review: `sampled`, or `uncertain` with the categories that made it so.
+    def pending_review: ("sampled" | "uncertain" | :sampled | :uncertain reason, ?Array[String | Symbol] uncertain_categories) -> self
     # The `categories`, `score` and `policy` a decided asset gets; nil for `pending` and `failed`.
-    def report: () -> Hash[String, json_value]?
+    # `requested_at` is when a pending review joined the queue, in epoch milliseconds.
+    def report: (?Integer requested_at) -> Hash[String, json_value]?
   end
 
   # What MockServerClient's `moderate:` callback receives.

data/sig/error_codes.rbs

@@ -37,12 +37,14 @@ module CalmLens
     PERMISSION_DENIED: String
     UPLOAD_TOO_LARGE: String
     DOCUMENT_TOO_MANY_ITEMS: String
+    MEDIA_DURATION_LIMIT: String
     INVALID_IMAGE: String
     IMAGE_TOO_LARGE: String
     FREE_UPLOADS_EXHAUSTED: String
     UPLOADS_PAUSED: String
     UNSUPPORTED_UPLOAD_TYPE: String
     INVALID_UPLOAD_PAYLOAD: String
+    UPLOAD_OPTIONS_IN_QUERY: String
     USE_UPLOAD_PART: String
     UPLOAD_NOT_OPEN: String
     INVALID_UPLOAD_PART_NUMBER: String
@@ -99,6 +101,14 @@ module CalmLens
     UPLOAD_PRESET_DAILY_LIMIT_REACHED: String
     UPLOAD_PRESET_ORIGIN_NOT_ALLOWED: String
     UPLOAD_PRESET_FIELD_NOT_ALLOWED: String
+    UPLOAD_PRESET_REVISION_CONFLICT: String
+    UPLOAD_PRESET_KEY_TAKEN: String
+    UPLOAD_PRESET_DAILY_LIMIT_REQUIRED: String
+    UPLOAD_PRESET_INVALID_FOLDER: String
+    UPLOAD_PRESET_INVALID_POLICY: String
+    PROJECT_SETTINGS_REVISION_CONFLICT: String
+    SEARCH_ESTIMATE_CHANGED: String
+    METADATA_FIELD_NOT_FOUND: String
     UPLOAD_TOKEN_INVALID: String
     UPLOAD_TOKEN_EXPIRED: String
     UPLOAD_TOKEN_USED: String

data/sig/qualifiers.rbs

@@ -57,6 +57,7 @@ module CalmLens
     ?"sexual-educational" => qualifier_result,
     ?"sexual-discussion" => qualifier_result,
     ?"sexual-fictional" => qualifier_result,
+    ?"sexual-artistic" => qualifier_result,
     ?"sexual-support-prevention" => qualifier_result,
     ?"violence-attack" => qualifier_result,
     ?"violence-threat" => qualifier_result,