calmlens 0.20.0 → 0.22.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -6,6 +6,7 @@ require 'securerandom'
6
6
  require 'uri'
7
7
  require_relative 'placeholder'
8
8
  require_relative 'mock_decision'
9
+ require_relative 'rsql'
9
10
 
10
11
  module CalmLens
11
12
  module Testing
@@ -36,35 +37,68 @@ module CalmLens
36
37
  :hash_list_entries, keyword_init: true)
37
38
 
38
39
  # A stateful stand-in for the CalmLens API and CDN. It stores assets and folders, applies
39
- # filters and ordering, serves stored bytes, and answers errors with the API's codes, statuses,
40
+ # filters and sorting, serves stored bytes, and answers errors with the API's codes, statuses,
40
41
  # details and body shape, checking requests in the API's order. It moderates through the
41
42
  # `moderate` callback rather than classifying content, and does not run transforms or index for
42
43
  # semantic search.
43
44
  # @api private
44
45
  class FakeAPI
45
- JSON_QUERY = %w[queue isFinal sizeInBytes totalSizeInBytes partNumber pageSize limit processingOptions metadata
46
- context].freeze
46
+ JSON_QUERY = %w[wait isFinal sizeInBytes totalSizeInBytes partNumber pageSize].freeze
47
47
  # Keyword search fields, in the order matches are reported, and where each value sits in the asset.
48
48
  SEARCH_FIELDS = { 'name' => %w[name], 'id' => %w[id], 'externalId' => %w[externalId],
49
49
  'description' => %w[description], 'tags' => %w[tags], 'type' => %w[type],
50
50
  'content.format' => %w[content format], 'content.extension' => %w[content extension],
51
51
  'classification.moderation.status' => %w[classification moderation status] }.freeze
52
52
  MAX_SEARCH_TOKENS = 8
53
- MAX_SEARCH_LIMIT = 20
53
+ MAX_SEARCH_PAGE_SIZE = 20
54
54
  MAX_SEARCH_QUERY_LENGTH = 200
55
55
  # Every API cursor list returns this many items unless `pageSize` says otherwise.
56
56
  DEFAULT_PAGE_SIZE = 50
57
57
  MAX_PAGE_SIZE = 100
58
58
  REVIEW_DECISIONS = %w[allowed limited age-restricted blocked purged].freeze
59
+ # The moderation status each `asset.<status>` event announces.
60
+ ANNOUNCED_STATUSES = REVIEW_DECISIONS.to_h { |status| ["asset.#{status}", status] }.freeze
59
61
  MAX_DECISION_NOTE_LENGTH = 1499
60
- RESOURCES = %w[original rendered placeholder].freeze
62
+ # The actor of events the mock's API key causes: mock keys have no ID.
63
+ API_KEY_ACTOR = { 'type' => 'api-key', 'id' => nil }.freeze
64
+ # The actor of a reviewer's decision through {MockServerClient#decide}.
65
+ REVIEWER_ACTOR = { 'type' => 'user', 'id' => nil }.freeze
66
+ # The actor of the pipeline's processing events.
67
+ AUTOMATED_ACTOR = { 'type' => 'automated', 'id' => nil }.freeze
68
+ # An event type's shape (`resource.action`), as the mock checks `types` without the catalog.
69
+ EVENT_TYPE_PATTERN = /\A[a-z][a-z-]*\.[a-z][a-z-]*\z/
70
+ # The resources an asset's CDN path names.
71
+ RESOURCES = %w[original rendered preview hls embed placeholder info].freeze
72
+ # The page the mock serves at `/embed`, standing in for the CDN's embed page.
73
+ EMBED_HTML = '<!doctype html><html lang="en"><head><meta charset="utf-8"><title>Embed</title></head>' \
74
+ '<body></body></html>'
75
+
61
76
  MIN_PART = 5 * 1024 * 1024
62
77
  MAX_MULTIPART_IMAGE = 20 * 1024 * 1024
63
78
  # Like the API: an upload has at most this many parts.
64
79
  MAX_PARTS = 10_000
65
- # The ThumbHash the mock gives every classified image, standing in for one computed from its
66
- # pixels. The `placeholder` resource serves it as a PNG, like the CDN.
80
+ # The ThumbHash the mock gives every processed asset's preview, standing in for one computed
81
+ # from its pixels. The `placeholder` resource serves it as a PNG, like the CDN.
67
82
  THUMBHASH = '1QcSHQRnh493V4dIh4eXh1h4kJUI'
83
+ # Removing or withholding content removes its rendered file, adaptive stream and preview, as the API does.
84
+ NO_RENDERING = { 'rendered' => nil, 'hls' => nil, 'hlsStatus' => nil, 'preview' => nil }.freeze
85
+ # The size of the waveform the API makes as an audio asset's preview.
86
+ WAVEFORM_SIZE = { 'width' => 1200, 'height' => 300 }.freeze
87
+ # Audio and video formats browsers play as uploaded, so the API serves them as stored (Asset.ts).
88
+ BROWSER_SAFE_MEDIA_FORMATS = %w[video/mp4 audio/mpeg audio/aac audio/mp4 audio/x-m4a audio/ogg].freeze
89
+ # Spreadsheets, which have no rendered file or preview (Asset.ts).
90
+ SPREADSHEET_MIME_TYPES = %w[application/vnd.openxmlformats-officedocument.spreadsheetml.sheet
91
+ application/vnd.ms-excel.sheet.macroenabled.12 application/vnd.ms-excel
92
+ application/vnd.oasis.opendocument.spreadsheet].freeze
93
+ # The settings of the mock's presets: every category with its default action (`allow` for
94
+ # personal information and profanity, `block` otherwise) and no rules. The mock holds no
95
+ # built-in preset settings, and uploads are never decided by a policy.
96
+ PRESET_SETTINGS = {
97
+ 'categorySettings' => MockClassification::CATEGORIES.to_h do |category|
98
+ [category, { 'action' => %w[personal-information profanity].include?(category) ? 'allow' : 'block',
99
+ 'allowances' => [], 'restrictions' => [] }]
100
+ end
101
+ }.freeze
68
102
  # The built-in presets the mock knows, by display name.
69
103
  PRESET_NAMES = { 'general-community' => 'General', 'children-platform' => 'Kids',
70
104
  'education-support' => 'Learning and support', 'wholesome-community' => 'Teen-safe' }.freeze
@@ -77,8 +111,38 @@ module CalmLens
77
111
  # The mock API key's `{ 'actions', 'folderId', 'includeSubfolders' }` grants.
78
112
  attr_accessor :grants
79
113
 
80
- def initialize(project_id:, base_url:, cdn_base_url:, now:, create_id:, moderate:, grants:)
114
+ # Splits an upload form as the API reads it: exactly an `options` part, then a `file` part, then
115
+ # the closing boundary.
116
+ # @return [Array(String, String), nil] the options part's text and the file's bytes, or nil
117
+ # when the body isn't such a form.
118
+ def self.upload_form(body, content_type)
119
+ boundary = content_type.to_s.match(%r{\Amultipart/form-data\s*;\s*boundary=(?:"([^"]+)"|([^;\s]+))}i)
120
+ return nil unless boundary
121
+
122
+ framed = "\r\n".b + body.to_s.b
123
+ delimiter = "\r\n--#{boundary[1] || boundary[2]}".b
124
+ return nil unless framed.start_with?(delimiter)
125
+
126
+ parts = []
127
+ at = 0
128
+ until framed.byteslice(at + delimiter.bytesize, 2) == '--'
129
+ start = at + delimiter.bytesize
130
+ header_end = framed.index("\r\n\r\n".b, start)
131
+ following = header_end && framed.index(delimiter, header_end + 4)
132
+ return nil unless following
133
+
134
+ name = framed.byteslice(start, header_end - start)[/content-disposition:\s*form-data\s*;[^\r\n]*?\bname="([^"]*)"/i, 1]
135
+ parts << [name, framed.byteslice(header_end + 4, following - header_end - 4)]
136
+ at = following
137
+ end
138
+ return nil unless parts.map(&:first) == %w[options file]
139
+
140
+ [parts[0][1].force_encoding(Encoding::UTF_8), parts[1][1]]
141
+ end
142
+
143
+ def initialize(project_id:, base_url:, cdn_base_url:, now:, create_id:, moderate:, api_key:, grants:)
81
144
  @project_id = project_id
145
+ @api_key = api_key
82
146
  @base_url = base_url
83
147
  @cdn_base_url = cdn_base_url
84
148
  @now = now
@@ -91,23 +155,48 @@ module CalmLens
91
155
  # reset `baseline`) and the one the project moderates with.
92
156
  @policies = {}
93
157
  @project_policy_id = nil
158
+ # The project's configuration: upload presets by key, the settings it has set, its metadata
159
+ # fields in display order and the revision the settings and fields share.
160
+ @upload_presets = {}
161
+ @project_settings = {}
162
+ @metadata_fields = []
163
+ @settings_revision = 1
94
164
  @request_count = 0
165
+ # Each asset's latest `assetSequence`, which the API keeps on the asset row.
166
+ @asset_event_sequences = Hash.new(0)
167
+ # The processing options each asset was uploaded with, which the API keeps but doesn't return.
168
+ @processing_options = {}
169
+ # Decided assets whose re-run (`assets.explain` with `rerun: true`) is still queued: they keep
170
+ # their decision until `complete_processing` decides them.
171
+ @reruns = {}
95
172
  end
96
173
 
97
- # Adds an event to the feed, filling the fields the API always sets.
174
+ # Adds an event to the feed, filling the fields the API always sets. Its `actor` is the mock's
175
+ # API key unless the event names another.
98
176
  def add_event(event)
99
177
  time = @now.call
100
- record = { 'id' => @create_id.call('evt'), 'projectId' => @project_id, 'severity' => 'low', 'description' => nil,
101
- 'assetId' => nil, 'asset' => nil, 'facts' => [], 'createdAt' => time }.merge(event)
178
+ record = { 'id' => @create_id.call('evt'), 'projectId' => @project_id, 'actor' => deep_copy(API_KEY_ACTOR),
179
+ 'severity' => 'low', 'description' => nil,
180
+ 'assetId' => nil, 'assetSequence' => nil, 'asset' => nil, 'facts' => [],
181
+ 'createdAt' => time }.merge(event)
102
182
  record['title'] ||= record['type']
183
+ # Like the API: each event about an asset takes the asset's next number; others have none.
184
+ asset_id = record['assetId']
185
+ record['assetSequence'] = asset_id ? (@asset_event_sequences[asset_id] += 1) : nil
103
186
  @state.events << record
104
187
  record
105
188
  end
106
189
 
107
- # Adds an event like {#add_event}; one about a stored asset carries the asset as it is now.
190
+ # Adds an event like {#add_event}; one about a stored asset carries the asset as it is now,
191
+ # with the moderation status an `asset.<status>` event announces.
108
192
  def enqueue_event(event)
109
193
  item = own_asset(event['assetId'])
110
- event = event.merge('asset' => snapshot(item)) if item && !event.key?('asset')
194
+ if item && !event.key?('asset')
195
+ asset = snapshot(item)
196
+ announced = ANNOUNCED_STATUSES[event['type']]
197
+ asset['classification']['moderation']['status'] = announced if announced
198
+ event = event.merge('asset' => asset)
199
+ end
111
200
  add_event(event)
112
201
  end
113
202
 
@@ -121,25 +210,29 @@ module CalmLens
121
210
  item = own_asset(asset_id)
122
211
  return unless item
123
212
 
124
- record_decision(item, decision, nil, nil)
213
+ record_decision(item, decision, nil, false)
125
214
  deep_copy(item)
126
215
  end
127
216
 
128
217
  # Finishes processing an asset as the pipeline would, with `decision` or else what `moderate`
129
- # answers for the `complete` trigger, and returns it; nil when there is no such asset.
218
+ # answers for the `complete` trigger, and returns it; nil when there is no such asset. A decided
219
+ # asset whose re-run is still queued counts as processing.
130
220
  def complete_processing(asset_id, decision)
131
221
  item = own_asset(asset_id)
132
222
  return unless item
133
- raise Internal.invalid("Asset #{asset_id} isn't processing") unless item['classification']['status'] == 'processing'
134
223
 
135
- moderate_stored(item, 'complete', decision.nil? ? nil : MockDecision.from(decision))
224
+ rerun = @reruns.key?(asset_id)
225
+ raise Internal.invalid("Asset #{asset_id} isn't processing") unless rerun || item['classification']['status'] == 'processing'
226
+
227
+ settled = decision.nil? ? nil : MockDecision.from(decision)
228
+ rerun ? settle_rerun(item, 'complete', settled) : moderate_stored(item, 'complete', settled)
136
229
  deep_copy(item)
137
230
  end
138
231
 
139
232
  # @return [Response]
140
233
  def handle(method, url, headers, body)
141
234
  if url.start_with?("#{@cdn_base_url}/")
142
- serve(method, url.delete_prefix(@cdn_base_url))
235
+ serve(method, url.delete_prefix(@cdn_base_url), headers)
143
236
  else
144
237
  api(method, url.delete_prefix(@base_url), headers, body)
145
238
  end
@@ -172,29 +265,29 @@ module CalmLens
172
265
  end
173
266
 
174
267
  # Applies a reviewer's decision like `decideAssetManually` in the API: the status and the review,
175
- # then (for `purged`) the content removal and its event, then `asset.<decision>`.
176
- def record_decision(item, decision, note, reviewer_name)
268
+ # then (for `purged`) the content removal and its event, then `asset.<decision>`. Either way the
269
+ # review is the mock API key's, which has no ID. A decision through the API (`by_api_key`) has
270
+ # the API key as its events' actor; the test hook's is a reviewer's (a `user`).
271
+ def record_decision(item, decision, note, by_api_key)
272
+ # Like the API: a queued re-run no longer replaces a reviewed decision.
273
+ @reruns.delete(item['id'])
177
274
  apply_outcome(item, decision, 'requested')
178
275
  review = { 'id' => @create_id.call('rev'), 'decision' => decision, 'note' => note,
179
- 'reviewedAt' => @now.call, 'appliedToAsset' => true }
180
- reviewer_facts = []
181
- if reviewer_name
182
- review.merge!('reviewerName' => reviewer_name, 'reviewerEmail' => nil)
183
- reviewer_facts = [{ 'key' => 'reviewer-user-id', 'value' => nil }, { 'key' => 'reviewer-name', 'value' => reviewer_name }]
184
- end
276
+ 'decidedBy' => { 'type' => 'api-key', 'id' => nil }, 'reviewedAt' => @now.call, 'applied' => true }
277
+ actor = by_api_key ? API_KEY_ACTOR : REVIEWER_ACTOR
185
278
  item['classification']['moderation']['manualReviews'] << review
186
- lifecycle_event('asset.content-removed', item) if decision == 'purged'
279
+ lifecycle_event('asset.content-removed', item, actor: actor) if decision == 'purged'
187
280
  # Like the event catalog: only allowing and limiting are low severity.
188
281
  add_event('type' => "asset.#{decision}", 'severity' => %w[allowed limited].include?(decision) ? 'low' : 'high',
189
282
  'title' => "asset.#{decision}", 'assetId' => item['id'], 'asset' => snapshot(item),
190
- 'facts' => [{ 'key' => 'decided-by', 'value' => 'reviewer' }, { 'key' => 'asset-type', 'value' => item['type'] },
283
+ 'actor' => deep_copy(actor),
284
+ 'facts' => [{ 'key' => 'asset-type', 'value' => item['type'] },
191
285
  { 'key' => 'asset-name', 'value' => item['name'], 'url' => "https://calmlens.com/go/asset/#{item['id']}" },
192
- *reviewer_facts, { 'key' => 'note', 'value' => note }])
286
+ { 'key' => 'note', 'value' => note }])
193
287
  end
194
288
 
195
- # `POST /assets/:id/decision` as the API validates and applies it. The mock's key is unnamed, so the
196
- # review and its event name the reviewer `An API key`. An asset still uploading or processing is
197
- # refused with `asset/processing`.
289
+ # `POST /assets/:id/decision` as the API validates and applies it, with the API key as its
290
+ # events' actor. An asset still uploading or processing is refused with `asset/processing`.
198
291
  def decide_request(item, json)
199
292
  decision = json['decision']
200
293
  invalid!('validation/invalid-union', 'decision') unless REVIEW_DECISIONS.include?(decision)
@@ -211,7 +304,7 @@ module CalmLens
211
304
  fail!(ErrorCodes::ASSET_DECISION_UNCHANGED) if status == decision
212
305
  fail!(ErrorCodes::ASSET_CONTENT_PURGED) if status == 'purged' && decision != 'blocked'
213
306
 
214
- record_decision(item, decision, note, 'An API key')
307
+ record_decision(item, decision, note, true)
215
308
  item
216
309
  end
217
310
 
@@ -219,26 +312,34 @@ module CalmLens
219
312
  def snapshot(item)
220
313
  { 'id' => item['id'], 'externalId' => item['externalId'], 'uploadPresetId' => item['uploadPresetId'],
221
314
  'upload' => { 'status' => item['upload']['status'] },
222
- 'classification' => { 'status' => item['classification']['status'],
315
+ 'classification' => { 'status' => item['classification']['status'], 'testInput' => item['classification']['testInput'],
223
316
  'moderation' => { 'status' => item['classification']['moderation']['status'] } },
224
317
  'content' => { 'status' => item['content']['status'] } }
225
318
  end
226
319
 
227
- # The `asset.deleted`, `asset.content-removed` and `asset.retried` events the API records.
228
- def lifecycle_event(type, item)
320
+ # The `asset.deleted`, `asset.content-removed` and `asset.retried` events the API records, with
321
+ # the `batch-id` fact of the batch that caused them, if any.
322
+ def lifecycle_event(type, item, batch_id: nil, actor: API_KEY_ACTOR)
229
323
  name = { 'key' => 'asset-name', 'value' => item['name'] }
230
324
  # A deleted asset has no portal page to link to.
231
325
  name['url'] = "https://calmlens.com/go/asset/#{item['id']}" unless type == 'asset.deleted'
326
+ facts = [name, { 'key' => 'asset-type', 'value' => item['type'] }]
327
+ facts << { 'key' => 'batch-id', 'value' => batch_id } if batch_id
232
328
  add_event('type' => type, 'severity' => 'low', 'assetId' => item['id'], 'asset' => snapshot(item),
233
- 'facts' => [name, { 'key' => 'asset-type', 'value' => item['type'] }])
329
+ 'actor' => deep_copy(actor), 'facts' => facts)
234
330
  end
235
331
 
236
332
  def fail!(code, message = nil, details: nil) = raise(Failure.new(code, message, details: details))
237
333
 
238
334
  def route_not_found!(method, parts) = fail!(ErrorCodes::ROUTE_NOT_FOUND, "No route for #{method} /#{parts.join('/')}")
239
335
 
240
- # A request the API's schema refuses: `validation/<issue>` with HTTP 400 and the field.
241
- def invalid!(code, path) = raise(Failure.new(code, "Invalid #{path}", details: { 'path' => path }, status: 400))
336
+ # A request the API's schema refuses: `validation/invalid-request` with HTTP 400, listing the
337
+ # field and its issue code (such as `validation/too-big`) in `details.issues`.
338
+ def invalid!(code, path)
339
+ message = "Invalid #{path}"
340
+ raise Failure.new(ErrorCodes::INVALID_REQUEST, message,
341
+ details: { 'issues' => [{ 'path' => path, 'message' => message, 'code' => code }] }, status: 400)
342
+ end
242
343
 
243
344
  # Like the API's schema (METADATA_ITEM_SCHEMA): metadata is nil or a list of `{id, value}`
244
345
  # Hashes with a String `id` and any JSON `value`.
@@ -253,32 +354,23 @@ module CalmLens
253
354
  end
254
355
  end
255
356
 
256
- # Like the API's schema, checked before the asset is looked up: asset updates take no null
257
- # name, and signed URLs no null field.
357
+ # Like the API's schema, checked before the asset is looked up: asset updates take no null name.
258
358
  def check_asset_body(method, action, json)
259
- case [method, action]
260
- in ['PATCH', nil]
261
- check_metadata(json['metadata'])
262
- invalid!('validation/invalid-type', 'name') if json.key?('name') && json['name'].nil?
263
- in ['POST', 'signed-url']
264
- %w[viewerClaims resource lens transform expiresInSeconds].each do |key|
265
- next unless json.key?(key) && json[key].nil?
359
+ return unless method == 'PATCH' && action.nil?
266
360
 
267
- invalid!(key == 'resource' ? 'validation/invalid-value' : 'validation/invalid-type', key)
268
- end
269
- else nil
270
- end
361
+ check_metadata(json['metadata'])
362
+ invalid!('validation/invalid-type', 'name') if json.key?('name') && json['name'].nil?
271
363
  end
272
364
 
273
365
  # Like the API: an upload's metadata is checked, then its MIME type, which must be one the
274
- # API can moderate (`upload/invalid-mime-type` with `details.mimeType`).
366
+ # API can moderate (`upload/unsupported-type` with `details.mimeType` and `details.source`).
275
367
  def check_upload(options)
276
368
  check_metadata(options['metadata'])
277
369
  type = options['contentType']
278
370
  return if type.nil? || Internal.accepted_upload_mime_type?(type)
279
371
 
280
- fail!(ErrorCodes::INVALID_UPLOAD_MIME_TYPE, "#{type} is not a valid upload file type",
281
- details: { 'mimeType' => type })
372
+ fail!(ErrorCodes::UNSUPPORTED_UPLOAD_TYPE, "#{type} is not a valid upload file type",
373
+ details: { 'mimeType' => type, 'source' => 'calmlens' })
282
374
  end
283
375
 
284
376
  # Like the API's query schema: an integer from 1 to `max`.
@@ -299,19 +391,25 @@ module CalmLens
299
391
  json = content_type == 'application/json' && body && upload.nil? ? JSON.parse(body) : {}
300
392
  # HTTP stacks trim header values, so the API sees the key without surrounding whitespace.
301
393
  key = header(headers, 'Idempotency-Key')&.to_s&.gsub(/\A[ \t\r\n]+|[ \t\r\n]+\z/, '')
302
- return dispatch(method, parts, query, json, body, upload) unless key && method == 'POST'
394
+ request = { query: query, json: json, body: body, upload: upload, content_type: content_type }
395
+ return dispatch(method, parts, request) unless key && method == 'POST'
303
396
 
304
397
  fail!(ErrorCodes::INVALID_IDEMPOTENCY_KEY) unless key.match?(IDEMPOTENCY_KEY)
305
398
  # Like the API: a successful response is saved and replayed for the same key and request;
306
- # a failed one frees the key.
307
- fingerprint = [method, relative, content_type, Digest::SHA256.hexdigest(body.to_s)]
399
+ # a failed one frees the key. An upload form counts by its options and file, not its boundary.
400
+ form = upload && FakeAPI.upload_form(body, content_type)
401
+ fingerprint = if form
402
+ [method, relative, 'multipart/form-data', form[0], Digest::SHA256.hexdigest(form[1])]
403
+ else
404
+ [method, relative, content_type, Digest::SHA256.hexdigest(body.to_s)]
405
+ end
308
406
  saved = @state.idempotency[key]
309
407
  if saved
310
408
  fail!(ErrorCodes::IDEMPOTENCY_KEY_REUSED) unless saved[:fingerprint] == fingerprint
311
409
  return Response.new(status: saved[:status], body: saved[:body],
312
410
  headers: saved[:headers].merge('idempotent-replayed' => 'true', 'x-request-id' => request_id))
313
411
  end
314
- response = dispatch(method, parts, query, json, body, upload)
412
+ response = dispatch(method, parts, request)
315
413
  if response.success?
316
414
  @state.idempotency[key] = { fingerprint: fingerprint, status: response.status, body: response.body,
317
415
  headers: response.headers }
@@ -329,13 +427,17 @@ module CalmLens
329
427
  fail!(ErrorCodes::INVALID_UPLOAD_PAYLOAD, "#{key} is not valid JSON")
330
428
  end
331
429
 
332
- def dispatch(method, parts, query, json, body, upload)
430
+ def dispatch(method, parts, request)
431
+ query, json, body = request.values_at(:query, :json, :body)
333
432
  data = case parts.first
334
- when 'projects' then project(method, parts, query, json, body, upload)
433
+ when 'projects' then project(method, parts, request)
335
434
  when 'assets' then asset(method, parts, query, json, body)
336
435
  when 'folders' then folder(method, parts, json)
337
436
  when 'hash-lists' then hash_list(method, parts, query, json)
338
437
  when 'hash-list-entries' then remove_hash_list_entry(method, parts)
438
+ when 'policies' then policies(method, parts, query, json)
439
+ when 'lenses' then lens(method, parts, json)
440
+ when 'upload-presets' then upload_preset(method, parts)
339
441
  else route_not_found!(method, parts)
340
442
  end
341
443
  data.is_a?(Response) ? data : respond(200, deep_copy(data))
@@ -343,27 +445,40 @@ module CalmLens
343
445
 
344
446
  def deep_copy(value) = JSON.parse(JSON.generate(value))
345
447
 
346
- # `upload` is the X-CalmLens-Upload header, which picks how POST .../assets reads the
347
- # request: absent is a JSON body, 'content' raw file content and 'part' a raw first part.
348
- def project(method, parts, query, json, body, upload)
448
+ def project(method, parts, request)
349
449
  fail!(ErrorCodes::API_KEY_PROJECT_MISMATCH) unless parts[1] == @project_id
350
- case [method, parts[2..], upload]
351
- in ['GET', ['events'], _] then events(query)
352
- in ['PUT', ['lenses'], _] then lenses(json.fetch('lenses', []))
353
- in ['GET', ['folders'], _] then folder_page(query)
354
- in ['POST', ['folders'], _] then create_folder(json)
355
- in ['GET', ['hash-lists'], _] then page(project_hash_lists, query, JSON.generate([@project_id]))
356
- in ['POST', ['hash-lists'], _] then create_hash_list(json)
357
- in ['GET', ['policies'], _] then policy_list
358
- in ['POST', ['policies'], _] then create_policy(json)
359
- in ['PUT', ['policy'], _] then show_policy(assign_policy(json))
360
- in [_, ['policies', String => policy_id, *rest], _] then policy(method, policy_id, rest, json)
361
- in ['GET', ['assets'], _] then list(query)
362
- in ['GET', ['assets', 'search'], _] then search(query)
363
- in ['POST', ['assets', 'batch'], _] then batch(json)
364
- in ['POST', ['assets'], nil] then create_asset(json, query['queue'] == true, url: json['url'])
365
- in ['POST', ['assets'], 'content'] then create_asset(query, query['queue'] == true, data: body)
366
- in ['POST', ['assets'], 'part'] then start_multipart(query, body)
450
+ query, json = request.values_at(:query, :json)
451
+ case [method, parts[2..]]
452
+ in ['GET', ['events']] then events(query)
453
+ in ['PUT', ['lenses']] then lenses(json.fetch('lenses', []))
454
+ in ['GET', ['lenses']] then page(@state.lenses.values, query, JSON.generate(['lenses', @project_id]))
455
+ in ['POST', ['lenses']] then create_lens(json)
456
+ in ['GET', ['folders']] then folder_page(query)
457
+ in ['POST', ['folders']] then create_folder(json)
458
+ in ['GET', ['hash-lists']] then page(project_hash_lists, query, JSON.generate([@project_id]))
459
+ in ['POST', ['hash-lists']] then create_hash_list(json)
460
+ in ['GET', ['policy']] then show_policy(find_policy(ensure_policies))
461
+ in ['PUT', ['policy']] then show_policy(assign_policy(json))
462
+ in ['GET', ['assets']] then list(query)
463
+ in ['GET', ['assets', 'search']] then search(query)
464
+ in ['GET', ['assets', String => asset_id, 'timeline']] then timeline(asset_id, query)
465
+ in ['POST', ['assets', 'batch']] then batch(json)
466
+ in ['POST', ['signed-asset-urls']] then sign_asset_url(json)
467
+ in ['POST', ['assets']] then create_upload(request)
468
+ else configuration(method, parts, query, json)
469
+ end
470
+ end
471
+
472
+ # A project's upload presets, settings and metadata fields.
473
+ def configuration(method, parts, query, json)
474
+ case [method, parts[2..]]
475
+ in ['GET', ['upload-presets']] then upload_preset_page(query)
476
+ in ['PUT', ['upload-presets']] then ensure_upload_presets(json)
477
+ in ['GET', ['settings']] then show_project_settings
478
+ in ['PUT', ['settings']] then update_project_settings(json)
479
+ in ['GET', ['metadata-fields']] then show_metadata_fields
480
+ in ['PUT', ['metadata-fields']] then ensure_metadata_fields(json)
481
+ in ['DELETE', ['metadata-fields', String => field_id]] then delete_metadata_field(field_id, query)
367
482
  else route_not_found!(method, parts)
368
483
  end
369
484
  end
@@ -379,7 +494,8 @@ module CalmLens
379
494
  # (`asset/not-found`) before anything happens. The operation then runs at once, in request
380
495
  # order, rather than in the background: `delete` deletes each asset with its stored child
381
496
  # assets and records `asset.deleted` with the asset as it was; `remove-content` removes the
382
- # content like DELETE /assets/:id/content. Mock assets carry no safety holds.
497
+ # content like DELETE /assets/:id/content. Each event carries the batch's `batch-id` fact. Mock
498
+ # assets carry no safety holds.
383
499
  def batch(json)
384
500
  ids = json['assetIds']
385
501
  invalid!('validation/invalid-type', 'assetIds') unless ids.is_a?(Array)
@@ -394,15 +510,16 @@ module CalmLens
394
510
  fail!(ErrorCodes::ASSET_NOT_FOUND) unless item && item['projectId'] == @project_id
395
511
  item
396
512
  end
513
+ batch_id = @create_id.call('batch')
397
514
  items.each do |item|
398
515
  if json['operation'] == 'delete'
399
516
  delete_tree(item)
400
- lifecycle_event('asset.deleted', item)
517
+ lifecycle_event('asset.deleted', item, batch_id: batch_id)
401
518
  else
402
- delete_content(item)
519
+ remove_content(item, batch_id)
403
520
  end
404
521
  end
405
- respond(202, { 'queued' => true })
522
+ respond(202, { 'batchId' => batch_id, 'status' => 'queued', 'assetCount' => items.size })
406
523
  end
407
524
 
408
525
  # Deletes an asset, its stored child assets and their content.
@@ -417,26 +534,26 @@ module CalmLens
417
534
  deleting.each do |id|
418
535
  @state.assets.delete(id)
419
536
  @state.content.delete(id)
537
+ @processing_options.delete(id)
420
538
  end
421
539
  end
422
540
 
423
541
  def list(query)
424
- items = project_assets.reject { |item| item['parentId'] }
425
- items = items.select { |item| item['folderId'] == query['folderId'] } if query.key?('folderId')
426
- page(RSQL.filter(items, query['filter']), query, JSON.generate([@project_id, query['filter'], query['folderId']]))
542
+ # Like the API: the query schema is checked before the filter.
543
+ page_query(query)
544
+ page(RSQL.filter(project_assets, query['filter']), query, JSON.generate([@project_id, query['filter']]))
427
545
  end
428
546
 
429
547
  # Like the API: one keyset page of a parent's child folders, optionally matching `search`, by
430
- # `name` ascending unless `ordering` (one entry, `name` or `createdAt`) says otherwise, with an
548
+ # `name` ascending unless `sort` (one field, `name` or `createdAt`) says otherwise, with an
431
549
  # `id` tie-breaker, and the ancestors of the parent and the listed folders. Names compare by
432
550
  # code point, so case matters.
433
551
  def folder_page(query)
434
552
  found = project_folders.select do |f|
435
553
  f['parentId'] == query['parentId'] && (!query['search'] || f['name'].downcase.include?(query['search'].downcase))
436
554
  end
437
- invalid!('validation/too-big', 'ordering') if ordering(query, nil).size > 1
438
555
  scope = JSON.generate([@project_id, query['parentId'], query['search']])
439
- result = page(found, query, scope, fields: %w[name createdAt], default: { 'field' => 'name', 'direction' => 'asc' })
556
+ result = page(found, query, scope, fields: %w[name createdAt], max: 1, default: { 'field' => 'name', 'direction' => 'asc' })
440
557
  page = result['items']
441
558
  ids = ([query['parentId']] + page.map { |f| f['id'] }).compact.uniq
442
559
  queue = ids.dup
@@ -453,46 +570,85 @@ module CalmLens
453
570
  def project_folders = @state.folders.values.select { |item| item['projectId'] == @project_id }
454
571
 
455
572
  # Like the API: without a cursor, no events and a `nextCursor` for now; with one, up to `pageSize`
456
- # events added since, oldest first. `nextCursor` is never nil. The mock's key holds every permission,
457
- # so no event type is excluded.
573
+ # events added since, oldest first, of the `types` (comma-separated) and `assetId` asked for.
574
+ # `nextCursor` is never nil, and a cursor belongs to the filters it was issued with. The mock's
575
+ # key holds every permission, so no event type is excluded.
458
576
  def events(query)
459
577
  size = check_range(query.fetch('pageSize', DEFAULT_PAGE_SIZE), 'pageSize', MAX_PAGE_SIZE)
578
+ types = query.key?('types') ? query['types'].to_s.split(',', -1) : []
579
+ if query.key?('types') && (types.empty? || types.any? { |type| !type.match?(EVENT_TYPE_PATTERN) })
580
+ invalid!('validation/invalid-value', 'types')
581
+ end
582
+ asset_id = query['assetId']
583
+ invalid!('validation/invalid-format', 'assetId') if asset_id && !asset_id.match?(ASSET_ID_PATTERN)
584
+ filter = [JSON.generate([types.uniq.sort, asset_id])].pack('m0')
585
+ matches = ->(event) { (types.empty? || types.include?(event['type'])) && (asset_id.nil? || event['assetId'] == asset_id) }
460
586
  feed = @state.events
461
- return { 'items' => [], 'nextCursor' => "mock_#{feed.size}", 'hasMore' => false, 'excludedTypes' => [] } unless query['cursor']
587
+ unless query['cursor']
588
+ return { 'items' => [], 'nextCursor' => "mock_#{feed.size}~#{filter}", 'hasMore' => false, 'excludedTypes' => [] }
589
+ end
462
590
 
463
- match = query['cursor'].to_s.match(/\Amock_(\d+)\z/)
464
- position = match && Integer(match[1], 10)
591
+ match = query['cursor'].to_s.match(/\Amock_(\d+)~(.*)\z/)
592
+ position = match && match[2] == filter ? Integer(match[1], 10) : nil
465
593
  fail!(ErrorCodes::INVALID_CURSOR) unless position && position <= feed.size
466
594
 
467
- taken = feed[position, size]
468
- following = position + taken.size
469
- { 'items' => taken, 'nextCursor' => "mock_#{following}", 'hasMore' => following < feed.size, 'excludedTypes' => [] }
595
+ taken = []
596
+ following = position
597
+ while following < feed.size && taken.size < size
598
+ event = feed[following]
599
+ following += 1
600
+ taken << event if matches.call(event)
601
+ end
602
+ { 'items' => taken, 'nextCursor' => "mock_#{following}~#{filter}", 'hasMore' => feed.drop(following).any?(&matches),
603
+ 'excludedTypes' => [] }
470
604
  end
471
605
 
472
- # Saves lenses in order, like the API: an unchanged definition returns the stored lens, a
473
- # change needs a greater version, and lenses before a conflict stay saved.
606
+ # Saves lenses in order, like the API: a new identifier creates a lens, an unchanged
607
+ # definition returns the stored lens and a changed one updates it in place.
474
608
  def lenses(definitions)
475
609
  { 'lenses' => definitions.map { |definition| save_lens(definition) } }
476
610
  end
477
611
 
478
612
  def save_lens(definition)
479
- now = @now.call
480
613
  old = @state.lenses[definition['identifier']]
481
- unless old
482
- lens = { 'id' => @create_id.call('lens'), 'projectId' => @project_id }.merge(definition)
483
- return @state.lenses[definition['identifier']] = lens.merge('createdAt' => now, 'updatedAt' => now)
484
- end
614
+ return new_lens(definition) unless old
485
615
 
486
616
  same = %w[name type active].all? { |key| old[key] == definition[key] } &&
487
617
  old['description'] == definition['description'] && canonical(old['transform']) == canonical(definition['transform'])
488
- return old if same && old['version'] == definition['version']
489
- if definition['version'].to_i > old['version'].to_i
490
- return @state.lenses[old['identifier']] =
491
- old.merge(definition).merge('updatedAt' => now)
492
- end
618
+ return old if same
493
619
 
494
- fail!(ErrorCodes::LENS_VERSION_CONFLICT, details: { 'currentVersion' => old['version'], 'minimumVersion' => old['version'] + 1,
495
- 'requestedVersion' => definition['version'] })
620
+ @state.lenses[old['identifier']] = old.merge(definition).merge('updatedAt' => @now.call)
621
+ end
622
+
623
+ def new_lens(definition)
624
+ now = @now.call
625
+ lens = { 'id' => @create_id.call('lens'), 'projectId' => @project_id }.merge(definition)
626
+ @state.lenses[definition['identifier']] = lens.merge('createdAt' => now, 'updatedAt' => now)
627
+ end
628
+
629
+ # POST /projects/:id/lenses: like the API, the identifier must be new to the project.
630
+ def create_lens(json)
631
+ identifier = json['identifier']
632
+ invalid!('validation/invalid-type', 'identifier') unless identifier.is_a?(String) && !identifier.empty?
633
+ fail!(ErrorCodes::LENS_IDENTIFIER_TAKEN, details: { 'identifier' => identifier }) if @state.lenses.key?(identifier)
634
+
635
+ # Like the API, a new lens is 201 Created.
636
+ respond(201, deep_copy(new_lens(json)))
637
+ end
638
+
639
+ # GET, PATCH and DELETE /lenses/:id. An update keeps the identifier and changes the rest of
640
+ # the definition, as ensuring the changed definition would.
641
+ def lens(method, parts, json)
642
+ item = @state.lenses.values.find { |lens| lens['id'] == parts[1] && lens['projectId'] == @project_id }
643
+ fail!(ErrorCodes::LENS_NOT_FOUND) unless item && parts.size == 2
644
+ case method
645
+ when 'GET' then item
646
+ when 'PATCH' then save_lens(item.merge(json.except('identifier', 'id', 'projectId', 'createdAt', 'updatedAt')))
647
+ when 'DELETE'
648
+ @state.lenses.delete(item['identifier'])
649
+ {}
650
+ else route_not_found!(method, parts)
651
+ end
496
652
  end
497
653
 
498
654
  def canonical(value)
@@ -503,13 +659,136 @@ module CalmLens
503
659
  end
504
660
  end
505
661
 
662
+ # A page of the project's upload presets, oldest first unless `sort` says otherwise.
663
+ def upload_preset_page(query)
664
+ page(@upload_presets.values, query, JSON.generate(['upload-presets', @project_id]),
665
+ default: { 'field' => 'createdAt', 'direction' => 'asc' })
666
+ end
667
+
668
+ # Like the API: every preset's revision and settings are checked before any is saved, and the
669
+ # stored presets come back in the order sent.
670
+ def ensure_upload_presets(json)
671
+ definitions = json['uploadPresets']
672
+ invalid!('validation/too-small', 'uploadPresets') unless definitions.is_a?(Array) && !definitions.empty?
673
+
674
+ plans = definitions.each_with_index.map do |definition, index|
675
+ %w[key name].each do |field|
676
+ value = definition[field]
677
+ invalid!('validation/invalid-type', "uploadPresets.#{index}.#{field}") unless value.is_a?(String) && !value.empty?
678
+ end
679
+ existing = @upload_presets[definition['key']]
680
+ stored = existing ? existing['revision'] : 0
681
+ if definition.key?('revision') && definition['revision'] != stored
682
+ fail!(ErrorCodes::UPLOAD_PRESET_REVISION_CONFLICT, details: { 'key' => definition['key'], 'revision' => stored })
683
+ end
684
+ values = upload_preset_values(definition)
685
+ limited = ![nil, 0].include?(values['dailyUploadLimit'])
686
+ fail!(ErrorCodes::UPLOAD_PRESET_DAILY_LIMIT_REQUIRED) if values['access'] == 'anyone' && !limited
687
+
688
+ [existing, values]
689
+ end
690
+ { 'uploadPresets' => plans.map { |existing, values| save_upload_preset(existing, values) } }
691
+ end
692
+
693
+ # The stored values a definition sets: fields left out take their defaults.
694
+ def upload_preset_values(definition)
695
+ given = ->(field, default = nil) { definition[field].nil? ? default : definition[field] }
696
+ list = ->(field) { definition[field].is_a?(Array) && !definition[field].empty? ? definition[field].uniq : nil }
697
+ { 'key' => definition['key'], 'name' => definition['name'], 'access' => given.call('access', 'token'),
698
+ 'enabled' => given.call('enabled', true), 'allowedContentTypes' => list.call('allowedContentTypes'),
699
+ 'maxSizeInBytes' => given.call('maxSizeInBytes'), 'folderId' => given.call('folderId'),
700
+ 'policyId' => given.call('policyId'), 'visibility' => given.call('visibility', 'private'),
701
+ 'processingOptions' => given.call('processingOptions'), 'revealDecision' => given.call('revealDecision', false),
702
+ 'dailyUploadLimit' => given.call('dailyUploadLimit'), 'allowedOrigins' => list.call('allowedOrigins') }
703
+ end
704
+
705
+ # Saves a preset's values under its key; an unchanged preset keeps its revision.
706
+ def save_upload_preset(existing, values)
707
+ return existing if existing && existing.slice(*values.keys) == values
708
+
709
+ now = @now.call
710
+ @upload_presets[values['key']] =
711
+ { 'id' => existing ? existing['id'] : @create_id.call('preset'), 'projectId' => @project_id, 'createdByUserId' => nil,
712
+ **values, 'revision' => (existing ? existing['revision'] : 0) + 1,
713
+ 'createdAt' => existing ? existing['createdAt'] : now, 'updatedAt' => now }
714
+ end
715
+
716
+ # DELETE /upload-presets/:id.
717
+ def upload_preset(method, parts)
718
+ route_not_found!(method, parts) unless method == 'DELETE' && parts.size == 2
719
+ preset = @upload_presets.values.find { |item| item['id'] == parts[1] }
720
+ fail!(ErrorCodes::UPLOAD_PRESET_NOT_FOUND) unless preset
721
+ @upload_presets.delete(preset['key'])
722
+ {}
723
+ end
724
+
725
+ def show_project_settings
726
+ settings = { 'optOutJurisdictions' => [], 'blockUnmoderatedAssets' => true, 'storeSensitiveMetadata' => false,
727
+ 'streamingEnabled' => false, 'aiMediaSearchEnabled' => false, 'remoteUrlRule' => nil,
728
+ 'websiteSubmissionsPerMinute' => nil }.merge(@project_settings)
729
+ settings['humanReview'] = { 'allowedSamplePercent' => 0, 'restrictedSamplePercent' => 0, 'uncertaintyMarginPercent' => 0,
730
+ 'reviewMissingEvidence' => false }.merge(@project_settings['humanReview'] || {})
731
+ { 'projectId' => @project_id, 'revision' => @settings_revision, 'settings' => settings }
732
+ end
733
+
734
+ # A revision to check, from a body or the query; none refuses nothing.
735
+ def check_settings_revision(revision)
736
+ return if revision.nil? || revision.to_s == @settings_revision.to_s
737
+
738
+ fail!(ErrorCodes::PROJECT_SETTINGS_REVISION_CONFLICT)
739
+ end
740
+
741
+ # Replaces the settings: those left out return to their defaults. A real change bumps the revision.
742
+ def update_project_settings(json)
743
+ check_settings_revision(json['revision'])
744
+ invalid!('validation/invalid-type', 'settings') unless json['settings'].is_a?(Hash)
745
+
746
+ before = show_project_settings['settings']
747
+ previous = @project_settings
748
+ @project_settings = json['settings']
749
+ if show_project_settings['settings'] == before
750
+ @project_settings = previous
751
+ else
752
+ @settings_revision += 1
753
+ end
754
+ show_project_settings
755
+ end
756
+
757
+ def show_metadata_fields = { 'projectId' => @project_id, 'revision' => @settings_revision, 'fields' => @metadata_fields }
758
+
759
+ # Creates or replaces fields by ID, keeping the others in order. A real change bumps the revision.
760
+ def ensure_metadata_fields(json)
761
+ check_settings_revision(json['revision'])
762
+ fields = json['fields']
763
+ invalid!('validation/too-small', 'fields') unless fields.is_a?(Array) && !fields.empty?
764
+
765
+ by_id = fields.to_h { |field| [field['id'], field] }
766
+ known = @metadata_fields.map { |field| field['id'] }
767
+ merged = @metadata_fields.map { |field| by_id.fetch(field['id'], field) } + fields.reject { |field| known.include?(field['id']) }
768
+ unless merged == @metadata_fields
769
+ @metadata_fields = merged
770
+ @settings_revision += 1
771
+ end
772
+ show_metadata_fields
773
+ end
774
+
775
+ def delete_metadata_field(field_id, query)
776
+ kept = @metadata_fields.reject { |field| field['id'] == field_id }
777
+ fail!(ErrorCodes::METADATA_FIELD_NOT_FOUND, details: { 'fieldId' => field_id }) if kept.size == @metadata_fields.size
778
+
779
+ check_settings_revision(query['revision'])
780
+ @metadata_fields = kept
781
+ @settings_revision += 1
782
+ show_metadata_fields
783
+ end
784
+
506
785
  # Keyword search like the API: every whitespace-separated token (case-insensitive, at most 8
507
786
  # distinct) must appear in some field. Results rank by the API's score, then newest first.
508
787
  def search(query)
509
788
  text = query['query'].to_s.strip
510
789
  invalid!('validation/too-small', 'query') if text.empty?
511
790
  invalid!('validation/too-big', 'query') if text.length > MAX_SEARCH_QUERY_LENGTH
512
- limit = check_range(query.fetch('limit', 10), 'limit', MAX_SEARCH_LIMIT)
791
+ limit = check_range(query.fetch('pageSize', 10), 'pageSize', MAX_SEARCH_PAGE_SIZE)
513
792
  normalized = text.downcase
514
793
  tokens = normalized.split(/\s+/).uniq.first(MAX_SEARCH_TOKENS)
515
794
  items = project_assets.filter_map do |item|
@@ -560,17 +839,21 @@ module CalmLens
560
839
  keep = processing.is_a?(Hash) && processing['keepContentAfterProcessing'] == true
561
840
  context = options['context'].is_a?(Hash) ? ModerationContext.select_items(options['context'].fetch('items', [])) : []
562
841
  # Fields the API returns as null when they are not set.
563
- asset = { 'id' => @create_id.call('asset'), 'projectId' => @project_id, 'parentId' => nil,
842
+ asset = { 'id' => @create_id.call('asset'), 'projectId' => @project_id, 'parentId' => nil, 'parentPages' => [],
564
843
  'folderId' => options['folderId'], 'uploadPresetId' => options['uploadPresetId'],
565
844
  'createdAt' => now, 'updatedAt' => now, 'name' => options['name'], 'description' => options['description'],
566
845
  'externalId' => options['externalId'], 'externalUrl' => url || options['externalUrl'],
567
- 'type' => kind(mime, url), 'metadata' => options['metadata'], 'authorId' => options['authorId'],
846
+ 'type' => kind(mime, url), 'metadata' => options['metadata'] || [], 'authorId' => options['authorId'],
568
847
  # Like the API, only the posts CalmLens uses are kept, and only while the content is held.
569
- 'context' => context.empty? ? nil : { 'items' => context }, 'tags' => nil,
570
- 'processingOptions' => processing, 'visibility' => options.fetch('visibility', 'private'),
848
+ 'context' => context.empty? ? nil : { 'items' => context }, 'tags' => options['tags'] || [],
849
+ 'visibility' => options.fetch('visibility', 'private'),
571
850
  **sections(now: now, open: open, keep: keep, format: mime, size: data&.bytesize) }
572
851
  @state.assets[asset['id']] = asset
573
- @state.content[asset['id']] = { 'type' => mime, 'parts' => { 1 => data } } if data
852
+ @processing_options[asset['id']] = processing
853
+ if data
854
+ @state.content[asset['id']] = { 'type' => mime, 'parts' => { 1 => data } }
855
+ asset['content'].merge!(rendering(asset, false)) unless open
856
+ end
574
857
  open ? asset : upload_completed(asset, queue, data)
575
858
  end
576
859
 
@@ -578,20 +861,30 @@ module CalmLens
578
861
  {
579
862
  'content' => { 'status' => keep ? 'stored' : 'temporary', 'removalReason' => nil, 'updatedAt' => now,
580
863
  'safetyHold' => nil, 'format' => format.empty? ? nil : format, 'extension' => nil,
581
- 'sizeInBytes' => size, 'checksums' => nil, 'image' => nil, 'extractedText' => nil,
582
- 'fileMetadata' => nil },
864
+ # The mock decodes nothing, so the file's own details are unknown, and like
865
+ # the API, the rendered file and preview come with processing.
866
+ 'sizeInBytes' => size, 'checksums' => nil, 'width' => nil, 'height' => nil,
867
+ 'durationSeconds' => nil, 'pageCount' => nil, 'perceptualHash' => nil,
868
+ 'perceptualHashQuality' => nil, 'metadata' => nil, 'extractedText' => nil, **NO_RENDERING },
583
869
  # Like the API, only an open upload counts the parts received.
584
870
  'upload' => { 'status' => open ? 'uploading' : 'completed', 'updatedAt' => now, 'partsReceived' => open ? 1 : nil },
585
- 'classification' => { 'status' => open ? 'pending' : 'processing', 'failureReason' => nil, 'updatedAt' => now,
586
- 'completedAt' => nil,
587
- 'moderation' => { 'status' => 'pending', 'manualReviews' => [],
588
- 'flaggedCategories' => [], 'regions' => [] } }
871
+ 'classification' => { 'status' => open ? 'queued' : 'processing', 'failureReason' => nil, 'updatedAt' => now,
872
+ # The mock never reads test inputs.
873
+ 'completedAt' => nil, 'progress' => nil, 'testInput' => false,
874
+ # Like the API, moderation's results are null until it decides, and a
875
+ # feature's result is null until it has run.
876
+ 'moderation' => { 'status' => 'pending', 'manualReviews' => [], 'automatedStatus' => nil,
877
+ 'policy' => nil, 'pendingReview' => nil, 'source' => nil, 'rule' => nil,
878
+ 'explanationComplete' => nil,
879
+ 'categories' => nil, 'flaggedCategories' => [], 'regions' => [],
880
+ 'raw' => nil, 'timeline' => nil },
881
+ 'ocr' => nil, 'linkDetection' => nil }
589
882
  }
590
883
  end
591
884
 
592
885
  # Like the API: image uploads in parts total at most 20 MiB, at any size below that.
593
886
  def image_size_error(type, size, _final)
594
- ErrorCodes::MULTIPART_IMAGE_TOO_LARGE if type == 'image' && size > MAX_MULTIPART_IMAGE
887
+ ErrorCodes::UPLOAD_TOO_LARGE if type == 'image' && size > MAX_MULTIPART_IMAGE
595
888
  end
596
889
 
597
890
  # Like the API: the parts may not overshoot a declared total, and must reach it at the end.
@@ -601,28 +894,96 @@ module CalmLens
601
894
  ErrorCodes::UPLOAD_SIZE_MISMATCH if received > total || (final && received != total)
602
895
  end
603
896
 
897
+ # The failure for a size error code, with the limit `upload/too-large` carries.
898
+ def size_failure!(code)
899
+ details = code == ErrorCodes::UPLOAD_TOO_LARGE ? { 'limitBytes' => MAX_MULTIPART_IMAGE, 'source' => 'plan' } : nil
900
+ fail!(code, details: details)
901
+ end
902
+
604
903
  def check_part_size(size, final)
605
904
  return if final || size >= MIN_PART
606
905
 
607
906
  fail!(ErrorCodes::INVALID_UPLOAD_PAYLOAD, 'Multipart upload parts must be at least 5MiB except the final part.')
608
907
  end
609
908
 
909
+ # `POST .../assets` as the API reads it. Asset fields in the query are refused; then the
910
+ # `X-CalmLens-Upload` header picks the body: absent is JSON (text, a first text part or a URL),
911
+ # `content` an upload form with a whole file and `part` one with a first part.
912
+ def create_upload(request)
913
+ query, json, body, upload = request.values_at(:query, :json, :body, :upload)
914
+ in_query = Assets::UPLOAD_OPTION_KEYS.select { |key| query.key?(key) }
915
+ unless in_query.empty?
916
+ fail!(ErrorCodes::UPLOAD_OPTIONS_IN_QUERY, "Send the asset's fields in the request body, not in the query string",
917
+ details: { 'fields' => in_query })
918
+ end
919
+ queue = query['wait'] == false
920
+ return create_from_json(json, queue) if upload.nil?
921
+
922
+ route_not_found!('POST', ['assets']) unless %w[content part].include?(upload)
923
+
924
+ options, bytes = form_fields(body, request[:content_type])
925
+ fields = options.merge('contentType' => query['contentType'])
926
+ return create_asset(fields, queue, data: bytes) if upload == 'content'
927
+
928
+ start_multipart(fields.merge(query.slice('isFinal', 'totalSizeInBytes')), bytes, queue)
929
+ end
930
+
931
+ # A JSON create: text as `content` or a first `part` (`{ plainText, type }`), or a URL.
932
+ def create_from_json(json, queue)
933
+ key = %w[content part].find { |name| json.key?(name) }
934
+ return create_asset(json, queue, url: json['url']) unless key
935
+
936
+ text = json[key]
937
+ invalid!('validation/invalid-type', key) unless text.is_a?(Hash) && text['plainText'].is_a?(String) && text['type'].is_a?(String)
938
+ fields = json.except('content', 'part', 'sizeInBytes', 'totalSizeInBytes').merge('contentType' => text['type'])
939
+ data = text['plainText'].encode(Encoding::UTF_8).b
940
+ return create_asset(fields, queue, data: data) if key == 'content'
941
+
942
+ start_multipart(fields.merge('isFinal' => text['isFinal'] == true, 'totalSizeInBytes' => json['totalSizeInBytes']), data, queue)
943
+ end
944
+
945
+ # The asset fields in an upload form's options part, and the file's bytes. Like the API, a raw
946
+ # upload must be `multipart/form-data` (`request/unsupported-content-type`, 415), holding an
947
+ # `options` JSON object of known fields with a name, then a `file` part.
948
+ def form_fields(body, content_type)
949
+ mime = content_type.to_s.split(';').first.to_s.strip.downcase
950
+ unless mime == 'multipart/form-data'
951
+ fail!(ErrorCodes::UNSUPPORTED_CONTENT_TYPE, 'Send a raw upload as multipart/form-data: an options part, then a file part.',
952
+ details: { 'contentType' => mime.empty? ? nil : mime })
953
+ end
954
+ form = FakeAPI.upload_form(body, content_type)
955
+ fail!(ErrorCodes::INVALID_UPLOAD_PAYLOAD, 'The upload form must hold an options part, then a file part') unless form
956
+
957
+ options = begin
958
+ JSON.parse(form[0])
959
+ rescue JSON::ParserError
960
+ nil
961
+ end
962
+ unless options.is_a?(Hash)
963
+ fail!(ErrorCodes::INVALID_UPLOAD_PAYLOAD, "The options part must be a JSON object with the asset's fields")
964
+ end
965
+ unknown = options.keys.find { |name| !Assets::UPLOAD_OPTION_KEYS.include?(name) }
966
+ invalid!('validation/unknown-field', unknown) if unknown
967
+ invalid!('validation/invalid-type', 'name') unless options['name'].is_a?(String)
968
+ [options, form[1]]
969
+ end
970
+
610
971
  # In the API's order: a declared total, then a single final part as the whole upload, then the
611
- # part itself. The queue mode and declared total are saved with an open upload.
612
- def start_multipart(query, data)
613
- check_upload(query)
614
- total = query['totalSizeInBytes']
615
- type = kind(query['contentType'].to_s, nil)
616
- final = query['isFinal'] == true
972
+ # part itself. Whether to wait and the declared total are saved with an open upload.
973
+ def start_multipart(fields, data, queue)
974
+ check_upload(fields)
975
+ total = fields['totalSizeInBytes']
976
+ type = kind(fields['contentType'].to_s, nil)
977
+ final = fields['isFinal'] == true
617
978
  size = data.to_s.bytesize
618
979
  code = (total && image_size_error(type, total, true)) ||
619
980
  (final ? total_size_error(total, size, true) || image_size_error(type, size, true) : nil)
620
- fail!(code) if code
981
+ size_failure!(code) if code
621
982
  check_part_size(size, final)
622
983
  fail!(total_size_error(total, size, false)) if !final && total_size_error(total, size, false)
623
984
 
624
- asset = create_asset(query, query['queue'] == true, data: data, open: !final)
625
- @state.uploads[asset['id']] = { 'queue' => query['queue'] == true, 'total' => total } unless final
985
+ asset = create_asset(fields, queue, data: data, open: !final)
986
+ @state.uploads[asset['id']] = { 'queue' => queue, 'total' => total } unless final
626
987
  asset
627
988
  end
628
989
 
@@ -658,8 +1019,26 @@ module CalmLens
658
1019
  # Without `moderate`, queued work stays pending until completed, which allows it.
659
1020
  def default_moderation(trigger, queued) = queued && trigger != 'complete' ? 'pending' : 'allowed'
660
1021
 
661
- # Applies `decision` to `asset` with its report.
662
- def decided(asset, decision) = apply_outcome(asset, decision.status, 'policy', decision.report)
1022
+ # Applies `decision` to `asset` with its report. The mock reads no text, QR codes, people or
1023
+ # faces in an image, so a decided image reports that the features its upload ran found nothing:
1024
+ # OCR and person and face detection when turned on, link detection unless turned off.
1025
+ def decided(asset, decision)
1026
+ apply_outcome(asset, decision.status, 'policy', decision.report)
1027
+ classification = asset['classification']
1028
+ return asset unless classification['status'] == 'completed' && asset['type'] == 'image'
1029
+
1030
+ processing = @processing_options[asset['id']]
1031
+ features = processing.is_a?(Hash) && processing['features'].is_a?(Hash) ? processing['features'] : {}
1032
+ setting = ->(name) { features[name].is_a?(Hash) ? features[name]['enabled'] : nil }
1033
+ classification['ocr'] = { 'status' => 'evaluated', 'textRegions' => [] } if setting.call('ocr') == true
1034
+ unless setting.call('linkDetection') == false
1035
+ classification['linkDetection'] = { 'status' => 'evaluated', 'qrCodes' => [], 'links' => [] }
1036
+ end
1037
+ { 'personDetection' => 'people', 'faceDetection' => 'faces' }.each do |feature, list|
1038
+ classification[feature] = { 'status' => 'evaluated', list => [] } if setting.call(feature) == true
1039
+ end
1040
+ asset
1041
+ end
663
1042
 
664
1043
  # Asks `moderate` (unless `decision` is given) and settles a processing asset: a decision and
665
1044
  # its events, or nothing for `pending`.
@@ -681,6 +1060,7 @@ module CalmLens
681
1060
 
682
1061
  if decision.status == 'failed'
683
1062
  return add_event('type' => 'asset.failed', 'severity' => 'high', 'assetId' => item['id'], 'asset' => snapshot(item),
1063
+ 'actor' => deep_copy(AUTOMATED_ACTOR),
684
1064
  'facts' => [asset_name_fact(item), { 'key' => 'asset-type', 'value' => item['type'] },
685
1065
  { 'key' => 'previous-status', 'value' => 'processing' },
686
1066
  { 'key' => 'failure-reason',
@@ -689,40 +1069,38 @@ module CalmLens
689
1069
 
690
1070
  flagged = decision.report['flaggedCategories']
691
1071
  add_event('type' => 'asset.classified', 'severity' => 'low', 'assetId' => item['id'], 'asset' => snapshot(item),
1072
+ 'actor' => deep_copy(AUTOMATED_ACTOR),
692
1073
  'facts' => [{ 'key' => 'flagged-categories', 'value' => flagged },
693
1074
  { 'key' => 'moderation-status', 'value' => decision.status },
694
1075
  { 'key' => 'asset-type', 'value' => item['type'] }, asset_name_fact(item)])
695
1076
  # Like the event catalog: only allowing and limiting are low severity.
696
1077
  add_event('type' => "asset.#{decision.status}", 'severity' => %w[allowed limited].include?(decision.status) ? 'low' : 'high',
697
- 'assetId' => item['id'], 'asset' => snapshot(item),
698
- 'facts' => [{ 'key' => 'decided-by', 'value' => 'automated' }, { 'key' => 'flagged-categories', 'value' => flagged },
1078
+ 'assetId' => item['id'], 'asset' => snapshot(item), 'actor' => deep_copy(AUTOMATED_ACTOR),
1079
+ 'facts' => [{ 'key' => 'flagged-categories', 'value' => flagged },
699
1080
  { 'key' => 'asset-type', 'value' => item['type'] }, asset_name_fact(item)])
700
1081
  end
701
1082
 
702
1083
  # Moves an asset to an outcome like the pipeline would: a decision finishes processing, `purged`
703
1084
  # deletes the content (with `purge_reason`, `policy` for the pipeline), and finished processing
704
- # removes content kept only temporarily. A classified image gets a ThumbHash for its
705
- # placeholder, which goes with its content. `report` is a decision's categories, score and policy.
1085
+ # removes content kept only temporarily. A decided asset gets its rendered file and preview
1086
+ # (with a ThumbHash for its placeholder), which go with its content. `report` is a decision's
1087
+ # categories, score and policy.
706
1088
  def apply_outcome(asset, outcome, purge_reason = 'policy', report = nil)
707
1089
  processing, moderation = AssetRecord.statuses(outcome)
708
1090
  now = @now.call
709
1091
  decided = processing == 'completed'
710
1092
  purge = moderation == 'purged' && asset['content']['status'] != 'removed'
711
1093
  remove = purge || (decided && asset['content']['status'] == 'temporary')
712
- image = asset['content']['image']
713
- if decided && asset['type'] == 'image' && asset['content']['status'] != 'removed' && !purge
714
- image = { 'width' => nil, 'height' => nil, 'perceptualHash' => nil }.merge(image || {})
715
- .merge('thumbhash' => THUMBHASH,
716
- 'averageColor' => image&.fetch('averageColor',
717
- nil) || '#7f7f7f')
718
- elsif moderation == 'purged'
719
- image = without_placeholder(image)
720
- end
721
- asset['content']['image'] = image
1094
+ # Like the API, an asset whose content is held has its rendered file and preview, completed by
1095
+ # its decision, and removing content (when processing finishes or by purging) removes them
1096
+ # with it, so no placeholder outlives the content. A safety hold withholds them.
1097
+ content = asset['content']
1098
+ held = content['status'] != 'removed' && !purge && !content['safetyHold']
1099
+ content.merge!(held ? rendering(asset, decided) : NO_RENDERING)
722
1100
  if remove
723
1101
  forget_content(asset)
724
- asset['content'].merge!('status' => 'removed', 'removalReason' => purge ? purge_reason : 'retention', 'updatedAt' => now,
725
- 'extractedText' => nil, 'fileMetadata' => nil, 'image' => without_placeholder(image))
1102
+ content.merge!('status' => 'removed', 'removalReason' => purge ? purge_reason : 'retention', 'updatedAt' => now,
1103
+ 'extractedText' => nil, 'metadata' => nil, **NO_RENDERING)
726
1104
  end
727
1105
  asset['upload'].merge!('status' => 'completed', 'updatedAt' => now)
728
1106
  update_classification(asset['classification'], processing, moderation, now)
@@ -744,8 +1122,49 @@ module CalmLens
744
1122
  item['context'] = nil
745
1123
  end
746
1124
 
747
- # Removing content removes the placeholder and average colour with it.
748
- def without_placeholder(image) = image&.merge('thumbhash' => nil, 'averageColor' => nil)
1125
+ # Whether browsers play an audio or video upload's format as it is, so it is served as stored.
1126
+ def browser_safe_media?(format) = BROWSER_SAFE_MEDIA_FORMATS.include?(mime_type(format))
1127
+
1128
+ # A MIME type without parameters, in lower case.
1129
+ def mime_type(format) = format.to_s.downcase.split(';', 2).first.to_s.strip
1130
+
1131
+ # The rendered file and preview the API reports for an asset with its content held (Asset.ts,
1132
+ # getAssetRendered and getAssetPreview). Once uploaded, an image renders as itself with a preview
1133
+ # that has no ThumbHash yet, and a PDF and browser-safe audio and video render as uploaded. Once
1134
+ # processed, a website renders as its screenshot; a document or presentation as a PDF, with its
1135
+ # first page as the preview; video and audio as a browser-safe file (MP4 or M4A unless browsers
1136
+ # play the upload), audio with a waveform preview and video with no poster; and every preview has
1137
+ # its ThumbHash. Text, spreadsheets and other files have neither. The mock decodes nothing, so
1138
+ # sizes are unknown, audio has no peaks, and audio and video never get an adaptive stream.
1139
+ def rendering(asset, processed) = { **rendered_file(asset, processed), 'hls' => nil, 'hlsStatus' => nil }
1140
+
1141
+ # The rendered file and preview of {#rendering}.
1142
+ def rendered_file(asset, processed)
1143
+ format = asset['content']['format']
1144
+ mime = mime_type(format)
1145
+ safe = browser_safe_media?(format)
1146
+ media = %w[video audio].include?(asset['type'])
1147
+ return NO_RENDERING.dup unless processed || asset['type'] == 'image' || mime == 'application/pdf' || (media && safe)
1148
+
1149
+ unsized = { 'width' => nil, 'height' => nil }
1150
+ preview = lambda do |size = unsized|
1151
+ size.merge('thumbhash' => processed ? THUMBHASH : nil, 'averageColor' => processed ? '#7f7f7f' : nil)
1152
+ end
1153
+ case asset['type']
1154
+ when 'image' then { 'rendered' => { 'kind' => 'image', 'format' => format || 'image/png', **unsized }, 'preview' => preview.call }
1155
+ when 'website' then { 'rendered' => { 'kind' => 'image', 'format' => 'image/png', **unsized }, 'preview' => preview.call }
1156
+ when 'multimedia-document'
1157
+ return NO_RENDERING.dup if SPREADSHEET_MIME_TYPES.include?(mime)
1158
+
1159
+ { 'rendered' => { 'kind' => 'pdf', 'format' => 'application/pdf', **unsized }, 'preview' => processed ? preview.call : nil }
1160
+ when 'video' then { 'rendered' => { 'kind' => 'video', 'format' => safe ? format : 'video/mp4', **unsized }, 'preview' => nil }
1161
+ when 'audio'
1162
+ rendered = { 'kind' => 'audio', 'format' => safe ? format : 'audio/mp4', **unsized }
1163
+ rendered['peaks'] = [] if processed
1164
+ { 'rendered' => rendered, 'preview' => processed ? preview.call(WAVEFORM_SIZE.dup) : nil }
1165
+ else NO_RENDERING.dup
1166
+ end
1167
+ end
749
1168
 
750
1169
  def asset(method, parts, query, json, body)
751
1170
  id = parts[1]
@@ -758,9 +1177,10 @@ module CalmLens
758
1177
  in ['DELETE', nil]
759
1178
  @state.assets.delete(id)
760
1179
  @state.content.delete(id)
1180
+ @processing_options.delete(id)
761
1181
  lifecycle_event('asset.deleted', item)
762
1182
  nil
763
- in ['DELETE', 'content'] then delete_content(item)
1183
+ in ['DELETE', 'content'] then remove_content(item)
764
1184
  in ['GET', 'children'] then page(project_assets.select { |child| child['parentId'] == id }, query, id)
765
1185
  in ['POST', 'retry'] then retry_asset(item)
766
1186
  in ['POST', 'explain'] then explain(item, json['rerun'])
@@ -768,7 +1188,6 @@ module CalmLens
768
1188
  in ['POST', 'move']
769
1189
  check_folder(json['folderId']) if json['folderId']
770
1190
  touch(item.merge!('folderId' => json['folderId']))
771
- in ['POST', 'signed-url'] then signed_url(id, json)
772
1191
  in ['POST', 'part'] then part(item, query, body)
773
1192
  else route_not_found!(method, parts)
774
1193
  end
@@ -787,21 +1206,21 @@ module CalmLens
787
1206
  touch(item.merge!(json.except('id', 'projectId', 'upload', 'classification', 'content')))
788
1207
  end
789
1208
 
790
- # Like the API: the record and report stay; stored content, extracted text and the placeholder
791
- # go. Content already removed keeps its reason, and an `asset.content-removed` event is
792
- # recorded either way.
793
- def delete_content(item)
1209
+ # Like the API: the record and report stay; stored content, extracted text, metadata, the
1210
+ # rendered file and the preview go. Content already removed keeps its reason, and an `asset.content-removed` event is
1211
+ # recorded either way, with the `batch-id` fact of the batch that removed it, if any.
1212
+ def remove_content(item, batch_id = nil)
794
1213
  forget_content(item)
795
1214
  content = item['content']
796
1215
  if content['status'] == 'removed'
797
- content['image'] = without_placeholder(content['image']) if content.dig('image', 'thumbhash')
1216
+ content.merge!(NO_RENDERING)
798
1217
  else
799
1218
  now = @now.call
800
1219
  content.merge!('status' => 'removed', 'removalReason' => 'requested', 'updatedAt' => now, 'extractedText' => nil,
801
- 'fileMetadata' => nil, 'image' => without_placeholder(content['image']))
1220
+ 'metadata' => nil, **NO_RENDERING)
802
1221
  item['updatedAt'] = now
803
1222
  end
804
- lifecycle_event('asset.content-removed', item)
1223
+ lifecycle_event('asset.content-removed', item, batch_id: batch_id)
805
1224
  item
806
1225
  end
807
1226
 
@@ -830,16 +1249,61 @@ module CalmLens
830
1249
  fail!(ErrorCodes::ASSET_REVIEWED) unless moderation.fetch('manualReviews', []).empty?
831
1250
  fail!(ErrorCodes::ASSET_CONTENT_UNAVAILABLE) if item['content']['status'] == 'removed'
832
1251
 
833
- # Like the API: back to processing, to be moderated again under the current policy.
834
- apply_outcome(item, 'pending')
835
- moderate_stored(item, 'rerun')
1252
+ # Like the API: the request is recorded, and the asset returns as it was. It keeps its decision
1253
+ # while it's moderated again under the current policy.
1254
+ add_event('type' => 'asset.rerun-requested', 'severity' => 'low', 'assetId' => item['id'],
1255
+ 'asset' => snapshot(item), 'actor' => deep_copy(API_KEY_ACTOR),
1256
+ 'facts' => [asset_name_fact(item), { 'key' => 'asset-type', 'value' => item['type'] },
1257
+ { 'key' => 'moderation-status', 'value' => moderation['status'] }])
1258
+ response = deep_copy(item)
1259
+ settle_rerun(item, 'rerun')
1260
+ response
836
1261
  end
837
1262
 
838
- def signed_url(id, json)
839
- expires = json.fetch('expiresInSeconds', 900)
840
- params = URI.encode_www_form(json.slice('lens', 'transform').merge('expires' => expires))
841
- { 'url' => "#{@cdn_base_url}/signed/#{URI.encode_www_form_component(id)}/#{json.fetch('resource', 'rendered')}?#{params}",
842
- 'expiresAt' => @now.call + (expires * 1000) }
1263
+ # Applies a re-run's decision to a decided asset like the pipeline: `pending` stays queued for
1264
+ # `complete_processing`, `failed` keeps the asset's decision, and any other replaces it with the
1265
+ # pipeline's events.
1266
+ def settle_rerun(item, trigger, decision = nil)
1267
+ decision ||= decision_for(item, trigger, true)
1268
+ @reruns.delete(item['id'])
1269
+ if decision.status == 'pending'
1270
+ @reruns[item['id']] = true
1271
+ elsif decision.status != 'failed'
1272
+ moderate_stored(item, trigger, decision)
1273
+ end
1274
+ end
1275
+
1276
+ # Signs an asset URL like the API (`POST /projects/:projectId/signed-asset-urls`): a grant to one
1277
+ # asset, by ID or external ID, until it expires, serving the original only when asked. Only the
1278
+ # mock's CDN accepts the grant.
1279
+ def sign_asset_url(json)
1280
+ # Like the API's schema, none of these fields takes null.
1281
+ %w[assetId externalId expiresIn original].each do |key|
1282
+ invalid!('validation/invalid-type', key) if json.key?(key) && json[key].nil?
1283
+ end
1284
+ if json.key?('assetId') == json.key?('externalId')
1285
+ message = 'Send either assetId or externalId'
1286
+ raise Failure.new(ErrorCodes::INVALID_REQUEST, message,
1287
+ details: { 'issues' => [{ 'path' => '', 'message' => message, 'code' => 'custom' }] }, status: 400)
1288
+ end
1289
+ expires = json.fetch('expiresIn', CDN::DEFAULT_SIGNED_URL_EXPIRES_IN)
1290
+ invalid!('validation/invalid-type', 'expiresIn') unless expires.is_a?(Integer) || (expires.is_a?(Float) && expires == expires.floor)
1291
+ invalid!('validation/too-small', 'expiresIn') if expires < 1
1292
+ invalid!('validation/too-big', 'expiresIn') if expires > CDN::MAX_SIGNED_URL_EXPIRES_IN
1293
+ original = json.fetch('original', false)
1294
+ invalid!('validation/invalid-type', 'original') unless [true, false].include?(original)
1295
+
1296
+ item = if json.key?('externalId')
1297
+ project_assets.find { |asset| asset['externalId'] == json['externalId'] }
1298
+ else
1299
+ own_asset(json['assetId'].to_s)
1300
+ end
1301
+ fail!(ErrorCodes::ASSET_NOT_FOUND) unless item
1302
+ fail!(ErrorCodes::ASSET_HELD) if item['content']['safetyHold']
1303
+
1304
+ expires_at = @now.call + (expires.to_i * 1000)
1305
+ { 'url' => "#{@cdn_base_url}/assets/#{Internal.component(@project_id)}/#{MockGrant.encode(item['id'], original, expires_at)}",
1306
+ 'expiresAt' => expires_at }
843
1307
  end
844
1308
 
845
1309
  # In the API's order: the upload must be open, parts arrive in order, image totals and the
@@ -857,7 +1321,7 @@ module CalmLens
857
1321
  received = item['content']['sizeInBytes'].to_i + size
858
1322
  upload = @state.uploads[item['id']] || {}
859
1323
  code = image_size_error(item['type'], received, final) || total_size_error(upload['total'], received, final)
860
- fail!(code) if code
1324
+ size_failure!(code) if code
861
1325
  check_part_size(size, final)
862
1326
 
863
1327
  stored = (@state.content[item['id']] ||= { 'type' => item['content']['format'].to_s, 'parts' => {} })
@@ -870,7 +1334,8 @@ module CalmLens
870
1334
  return item unless final
871
1335
 
872
1336
  @state.uploads.delete(item['id'])
873
- queue = query.key?('queue') ? query['queue'] == true : upload.fetch('queue', false)
1337
+ # `wait=false` queues the final part; without `wait`, the upload's saved choice applies.
1338
+ queue = query.key?('wait') ? query['wait'] == false : upload.fetch('queue', false)
874
1339
  apply_outcome(item, 'pending')
875
1340
  upload_completed(item, queue)
876
1341
  end
@@ -933,28 +1398,37 @@ module CalmLens
933
1398
  end
934
1399
 
935
1400
  # Policies: the project starts on the organisation's own General preset policy, following
936
- # automatic updates. The mock holds no built-in preset settings: a preset's settings are
937
- # `{ 'categorySettings' => {} }` at version 1, and uploads are never decided by a policy.
938
- def new_policy(name, settings, description: '', automatic_updates: false, preset: nil)
1401
+ # automatic updates (mode `live`). The mock's presets have one release, version 1, with
1402
+ # PRESET_SETTINGS. A stored policy is the API's shape without `projectIds`, with its preset's
1403
+ # `id`, `version` and `own`, whether it follows its preset (`followsPreset`) and its reset
1404
+ # `baseline`.
1405
+ def new_policy(name, settings, description: '', follows_preset: false, preset: nil)
939
1406
  now = @now.call
940
1407
  item = { 'id' => @create_id.call('pol'), 'name' => name, 'description' => description, 'settings' => settings,
941
- 'revision' => 1, 'automaticUpdates' => automatic_updates, 'preset' => preset,
1408
+ 'revision' => 1, 'followsPreset' => follows_preset, 'preset' => preset,
942
1409
  'baseline' => deep_copy(settings), 'createdAt' => now, 'updatedAt' => now }
943
1410
  @policies[item['id']] = item
944
1411
  end
945
1412
 
1413
+ # The project's policy, after creating the organisation's own General policy on first use.
946
1414
  def ensure_policies
947
- return if @project_policy_id
1415
+ return @project_policy_id if @project_policy_id
948
1416
 
949
1417
  preset = { 'id' => 'general-community', 'version' => 1, 'own' => true }
950
- @project_policy_id = new_policy(PRESET_NAMES['general-community'], { 'categorySettings' => {} },
951
- automatic_updates: true, preset: preset)['id']
1418
+ @project_policy_id = new_policy(PRESET_NAMES['general-community'], deep_copy(PRESET_SETTINGS),
1419
+ follows_preset: true, preset: preset)['id']
952
1420
  end
953
1421
 
954
- # A stored policy as the API shows it.
1422
+ # A stored policy as the API shows it: an own preset policy is `live` while it follows its
1423
+ # preset and `pinned` otherwise; any other preset policy is a `copy`.
955
1424
  def show_policy(item)
956
- shown = item.except('baseline')
957
- shown['preset'] = item['preset'].merge('latestVersion' => 1) if item['preset']
1425
+ shown = item.except('baseline', 'followsPreset')
1426
+ if (preset = item['preset'])
1427
+ mode = if preset['own'] then item['followsPreset'] ? 'live' : 'pinned'
1428
+ else 'copy'
1429
+ end
1430
+ shown['preset'] = { 'id' => preset['id'], 'version' => preset['version'], 'latestVersion' => 1, 'mode' => mode }
1431
+ end
958
1432
  shown.merge('projectIds' => item['id'] == @project_policy_id ? [@project_id] : [])
959
1433
  end
960
1434
 
@@ -963,9 +1437,39 @@ module CalmLens
963
1437
  @policies[id] || fail!(ErrorCodes::POLICY_NOT_FOUND)
964
1438
  end
965
1439
 
966
- def policy_list
1440
+ # `/policies` and `/policies/:id[/action]`, the API key's organisation's policies.
1441
+ def policies(method, parts, query, json)
1442
+ _, id, *rest = parts
1443
+ if id.nil?
1444
+ return policy_page(query) if method == 'GET'
1445
+ return create_policy(json) if method == 'POST'
1446
+
1447
+ route_not_found!(method, parts)
1448
+ end
1449
+ policy(method, id, rest, json)
1450
+ end
1451
+
1452
+ # A page of the organisation's policies, oldest first unless `sort` says otherwise.
1453
+ def policy_page(query)
967
1454
  ensure_policies
968
- { 'policies' => @policies.values.map { |item| show_policy(item) }, 'projectPolicyId' => @project_policy_id }
1455
+ page(@policies.values.map { |item| show_policy(item) }, query, JSON.generate(['policies', @project_id]),
1456
+ default: { 'field' => 'createdAt', 'direction' => 'asc' })
1457
+ end
1458
+
1459
+ # Refuses settings the API refuses (MODERATION_POLICY_SETTINGS_SCHEMA): every category, each
1460
+ # with an `action` and its `allowances` and `restrictions` lists, and no other category.
1461
+ def check_policy_settings(settings)
1462
+ categories = settings.is_a?(Hash) ? settings['categorySettings'] : nil
1463
+ invalid!('validation/invalid-type', 'settings.categorySettings') unless categories.is_a?(Hash)
1464
+ MockClassification::CATEGORIES.each do |category|
1465
+ path = "settings.categorySettings.#{category}"
1466
+ value = categories[category]
1467
+ invalid!('validation/invalid-type', path) unless value.is_a?(Hash)
1468
+ %w[allowances restrictions].each { |key| invalid!('validation/invalid-type', "#{path}.#{key}") unless value[key].is_a?(Array) }
1469
+ invalid!('validation/invalid-value', "#{path}.action") unless value['action'].is_a?(String)
1470
+ end
1471
+ unknown = categories.keys.find { |key| !MockClassification::CATEGORIES.include?(key) }
1472
+ invalid!('validation/unknown-field', "settings.categorySettings.#{unknown}") if unknown
969
1473
  end
970
1474
 
971
1475
  def create_policy(json)
@@ -975,10 +1479,11 @@ module CalmLens
975
1479
  if settings.nil? == preset_id.nil? || (preset_id && !PRESET_NAMES.key?(preset_id)) || (preset_id.nil? && !named)
976
1480
  fail!(ErrorCodes::INVALID_VALUE, 'Send settings with a name, or presetId')
977
1481
  end
1482
+ check_policy_settings(settings) unless preset_id
978
1483
  description = json['description'].is_a?(String) ? json['description'] : ''
979
1484
  item = if preset_id
980
1485
  origin = { 'id' => preset_id, 'version' => 1, 'own' => false }
981
- new_policy(named ? name.strip : PRESET_NAMES[preset_id], { 'categorySettings' => {} },
1486
+ new_policy(named ? name.strip : PRESET_NAMES[preset_id], deep_copy(PRESET_SETTINGS),
982
1487
  description: description, preset: origin)
983
1488
  else
984
1489
  new_policy(name.strip, settings, description: description)
@@ -1010,26 +1515,29 @@ module CalmLens
1010
1515
  fail!(ErrorCodes::POLICY_IN_USE) if item['id'] == @project_policy_id
1011
1516
  @policies.delete(id)
1012
1517
  {}
1013
- in ['POST', ['preset']] then apply_policy_preset(item, json)
1518
+ in ['POST', ['reset']] then restore_policy(item, 'reset', json)
1519
+ in ['POST', ['upgrade']] then restore_policy(item, 'upgrade', json)
1014
1520
  in ['PUT', ['automatic-updates']] then set_policy_automatic_updates(item, json)
1015
1521
  else route_not_found!(method, ['policies', id, *rest])
1016
1522
  end
1017
1523
  end
1018
1524
 
1019
1525
  def update_policy(item, json)
1526
+ check_policy_settings(json['settings']) if json.key?('settings')
1020
1527
  values = %w[name description].select { |key| json[key].is_a?(String) }
1021
1528
  .to_h { |key| [key, key == 'name' ? json[key].strip : json[key]] }
1022
1529
  values['settings'] = json['settings'] if json.key?('settings')
1023
- # Saving a policy that follows its preset pins it as a copy.
1024
- save_policy(item, json['revision'], values.merge('automaticUpdates' => false))
1530
+ # Saving a `live` policy pins it.
1531
+ save_policy(item, json['revision'], values.merge('followsPreset' => false))
1025
1532
  end
1026
1533
 
1027
- def apply_policy_preset(item, json)
1534
+ # Resets a `pinned` or `copy` policy to its preset release, or upgrades it to the latest.
1535
+ def restore_policy(item, operation, json)
1028
1536
  preset = item['preset']
1029
- fail!(ErrorCodes::POLICY_NO_PRESET) if preset.nil? || item['automaticUpdates']
1537
+ fail!(ErrorCodes::POLICY_NO_PRESET) if preset.nil? || item['followsPreset']
1030
1538
  fail!(ErrorCodes::POLICY_PRESET_VERSION_CONFLICT) unless json['presetVersion'] == preset['version']
1031
- # The mock's presets have one release, so an update never has a newer one to apply.
1032
- fail!(ErrorCodes::POLICY_PRESET_UPDATE_UNAVAILABLE) if json['operation'] == 'update'
1539
+ # The mock's presets have one release, so an upgrade never has a newer one to apply.
1540
+ fail!(ErrorCodes::POLICY_PRESET_UPDATE_UNAVAILABLE) if operation == 'upgrade'
1033
1541
 
1034
1542
  save_policy(item, json['revision'], 'settings' => deep_copy(item['baseline']))
1035
1543
  end
@@ -1037,11 +1545,11 @@ module CalmLens
1037
1545
  def set_policy_automatic_updates(item, json)
1038
1546
  enabled = json['enabled'] == true
1039
1547
  preset = item['preset']
1040
- fail!(ErrorCodes::POLICY_AUTOMATIC_UPDATES_UNAVAILABLE) if preset.nil? || !preset['own'] || item['automaticUpdates'] == enabled
1548
+ fail!(ErrorCodes::POLICY_AUTOMATIC_UPDATES_UNAVAILABLE) if preset.nil? || !preset['own'] || item['followsPreset'] == enabled
1041
1549
  fail!(ErrorCodes::POLICY_REVISION_CONFLICT) unless json['revision'] == item['revision']
1042
1550
  fail!(ErrorCodes::POLICY_PRESET_VERSION_CONFLICT) unless json['presetVersion'] == 1
1043
1551
 
1044
- values = { 'automaticUpdates' => enabled }
1552
+ values = { 'followsPreset' => enabled }
1045
1553
  values.merge!('name' => PRESET_NAMES[preset['id']], 'description' => '', 'settings' => deep_copy(item['baseline'])) if enabled
1046
1554
  save_policy(item, json['revision'], values)
1047
1555
  end
@@ -1103,10 +1611,10 @@ module CalmLens
1103
1611
  if asset_id
1104
1612
  asset = @state.assets[asset_id]
1105
1613
  fail!(ErrorCodes::ASSET_NOT_FOUND) unless asset && asset['projectId'] == @project_id
1106
- image = asset.dig('content', 'image')
1107
- fail!(ErrorCodes::HASH_LIST_ASSET_NOT_HASHED) unless image.is_a?(Hash) && image['perceptualHash']
1108
- hash = image['perceptualHash']
1109
- quality = image['perceptualHashQuality']
1614
+ content = asset['content']
1615
+ fail!(ErrorCodes::HASH_LIST_ASSET_NOT_HASHED) unless content['perceptualHash']
1616
+ hash = content['perceptualHash']
1617
+ quality = content['perceptualHashQuality']
1110
1618
  end
1111
1619
  fail!(ErrorCodes::INVALID_UPLOAD_PAYLOAD, 'Use a PDQ hash of 64 hexadecimal characters') unless hash.match?(/\A\h{64}\z/)
1112
1620
 
@@ -1115,7 +1623,7 @@ module CalmLens
1115
1623
 
1116
1624
  entry = { 'id' => @create_id.call('hle'), 'projectId' => @project_id, 'hashListId' => list['id'], 'hash' => hash,
1117
1625
  'quality' => quality, 'label' => json['label'].is_a?(String) ? json['label'] : nil,
1118
- 'sourceAssetId' => asset_id, 'createdAt' => @now.call }
1626
+ 'assetId' => asset_id, 'createdAt' => @now.call }
1119
1627
  @state.hash_list_entries[entry['id']] = entry
1120
1628
  respond(201, deep_copy(entry))
1121
1629
  end
@@ -1129,17 +1637,43 @@ module CalmLens
1129
1637
  {}
1130
1638
  end
1131
1639
 
1132
- # One keyset page like the API's: ordered by `fields` (`createdAt` descending by default; an `id`
1133
- # tie-breaker is added), starting after the cursor's values. The cursor is tied to `scope` (the
1134
- # filter, folder or parent) and the ordering, so it only continues the query it came from.
1135
- def page(items, query, scope, fields: %w[createdAt id], default: { 'field' => 'createdAt', 'direction' => 'desc' })
1640
+ # An asset's findings timeline, as the API lists it (publicGetAssetTimeline.ts): refused while a
1641
+ # safety hold is on. The mock analyses no audio or video, so it keeps no segments: every
1642
+ # timeline is empty, and no cursor continues one.
1643
+ def timeline(id, query)
1644
+ item = own_asset(id)
1645
+ fail!(ErrorCodes::ASSET_NOT_FOUND) unless item
1646
+ check_range(query.fetch('pageSize', DEFAULT_PAGE_SIZE), 'pageSize', MAX_PAGE_SIZE)
1647
+ fail!(ErrorCodes::ASSET_HELD, details: { 'hold' => 'safety' }) if item['content']['safetyHold']
1648
+ fail!(ErrorCodes::INVALID_CURSOR) if query.key?('cursor')
1649
+
1650
+ { 'items' => [], 'nextCursor' => nil, 'hasMore' => false }
1651
+ end
1652
+
1653
+ # The page size and sort, refused like the API's query schema: `sort` is comma-separated
1654
+ # fields from `fields`, each descending when prefixed with `-`, none repeated and at most `max`
1655
+ # (`validation/invalid-value` at `sort` otherwise). Without one, `default` applies.
1656
+ def page_query(query, fields = %w[createdAt id], default = { 'field' => 'createdAt', 'direction' => 'desc' }, max: 2)
1136
1657
  size = check_range(query.fetch('pageSize', DEFAULT_PAGE_SIZE), 'pageSize', MAX_PAGE_SIZE)
1137
- ordering = ordering(query, default)
1138
- # Like the API's query schema: each entry's field and direction are enum values.
1139
- ordering.each_with_index do |rule, index|
1140
- invalid!('validation/invalid-value', "ordering.#{index}.field") unless fields.include?(rule['field'])
1141
- invalid!('validation/invalid-value', "ordering.#{index}.direction") unless %w[asc desc].include?(rule['direction'])
1658
+ [size, query.key?('sort') ? parse_sort(query['sort'], fields, max) : [default.dup]]
1659
+ end
1660
+
1661
+ def parse_sort(value, fields, max)
1662
+ sort = value.to_s.split(',', -1).map(&:strip).map do |part|
1663
+ field = part.delete_prefix('-')
1664
+ invalid!('validation/invalid-value', 'sort') unless part.match?(/\A-?[A-Za-z][A-Za-z0-9.]*\z/) && fields.include?(field)
1665
+ { 'field' => field, 'direction' => part.start_with?('-') ? 'desc' : 'asc' }
1142
1666
  end
1667
+ fields_sent = sort.map { |rule| rule['field'] }
1668
+ invalid!('validation/invalid-value', 'sort') if sort.empty? || fields_sent.uniq.size != fields_sent.size || sort.size > max
1669
+ sort
1670
+ end
1671
+
1672
+ # One keyset page like the API's: sorted by `sort` (`createdAt` descending by default; an `id`
1673
+ # tie-breaker is added), starting after the cursor's values. The cursor is tied to `scope` (the
1674
+ # filter, folder or parent) and the sort, so it only continues the query it came from.
1675
+ def page(items, query, scope, fields: %w[createdAt id], max: 2, default: { 'field' => 'createdAt', 'direction' => 'desc' })
1676
+ size, ordering = page_query(query, fields, default, max: max)
1143
1677
  ordering << { 'field' => 'id', 'direction' => ordering.first['direction'] } unless ordering.any? { |o| o['field'] == 'id' }
1144
1678
  signature = JSON.generate([scope, ordering])
1145
1679
  sorted = items.sort { |a, b| compare_items(a, b, ordering) }
@@ -1149,12 +1683,8 @@ module CalmLens
1149
1683
  end
1150
1684
  selected = sorted.first(size)
1151
1685
  more = sorted.size > size
1152
- cursor = if more
1153
- [JSON.generate({ 'scope' => signature, 'values' => ordering.map do |o|
1154
- selected.last[o['field']]
1155
- end })].pack('m0')
1156
- end
1157
- { 'items' => selected, 'pageSize' => size, 'hasMore' => more, 'ordering' => ordering, 'nextCursor' => cursor }
1686
+ cursor = ([JSON.generate({ 'scope' => signature, 'values' => ordering.map { |o| selected.last[o['field']] } })].pack('m0') if more)
1687
+ { 'items' => selected, 'hasMore' => more, 'nextCursor' => cursor }
1158
1688
  end
1159
1689
 
1160
1690
  def cursor_values(cursor, signature, ordering)
@@ -1177,15 +1707,6 @@ module CalmLens
1177
1707
  false
1178
1708
  end
1179
1709
 
1180
- def ordering(query, default)
1181
- entries = query.keys.filter_map { |key| key.match(/\Aordering\[(\d+)\]\[(field|direction)\]\z/) }
1182
- return [default&.dup].compact if entries.empty?
1183
-
1184
- entries.group_by { |match| Integer(match[1], 10) }.sort.map do |_, matches|
1185
- matches.to_h { |match| [match[2], query[match[0]]] }
1186
- end
1187
- end
1188
-
1189
1710
  def compare_items(left, right, ordering)
1190
1711
  ordering.each do |rule|
1191
1712
  result = compare_values(left[rule['field']], right[rule['field']])
@@ -1204,79 +1725,200 @@ module CalmLens
1204
1725
  left.to_s <=> right.to_s
1205
1726
  end
1206
1727
 
1207
- # Answers like the CDN: a placeholder is the asset's ThumbHash as a PNG, public whatever
1208
- # the asset's visibility or moderation; other resources pass delivery checks first, which refuse
1209
- # everyone (content still uploading, blocked or purged, failed processing, removed content),
1210
- # then an age-restricted asset needs `viewerClaims.ageConfirmed=true` whatever the key, and a
1211
- # private asset needs an API key with `assets:download` for its folder.
1212
- def serve(method, relative)
1728
+ # Answers like the CDN (serveAssetResource.ts): `original` is the upload; `rendered` the image,
1729
+ # PDF, video or audio file `content.rendered` describes; `preview` the still `content.preview`
1730
+ # describes; `hls` the adaptive stream's files, which the mock never prepares; `embed` a small
1731
+ # page standing in for the embed page of any asset; and `placeholder` the preview's ThumbHash as a PNG, public
1732
+ # whatever the asset's visibility or moderation. A safety hold refuses every resource. Other resources pass delivery checks first,
1733
+ # which refuse everyone (content still uploading, blocked or purged, failed processing, removed
1734
+ # content), then an age-restricted asset needs `viewerClaims.ageConfirmed=true` whatever the key,
1735
+ # and a private asset needs an API key with `assets:download` for its folder. The mock converts
1736
+ # nothing: an upload stands in for its rendered file, an image for its own preview, and the
1737
+ # placeholder PNG for a screenshot, page or waveform.
1738
+ def serve(method, relative, headers)
1213
1739
  path, query = relative.split('#', 2).first.split('?', 2)
1214
1740
  raw_segments = path.split('/').reject(&:empty?)
1215
- _, project, identifier, resource = raw_segments.map { |part| URI.decode_www_form_component(part) }
1216
- # Like the CDN, the selector after the resource is read first, and lenses and transforms
1217
- # apply only to the rendered resource.
1218
- begin
1219
- lens, options = TransformRules.parse_selector(raw_segments.drop(4).join('/'))
1220
- rescue ArgumentError
1221
- return invalid_transform
1222
- end
1223
- return invalid_transform if resource == 'original' && (lens || options)
1224
-
1225
- item = project == @project_id && project_assets.find { |asset| [asset['id'], asset['externalId']].include?(identifier) }
1226
- return error('asset/not-found', 'Asset not found', status: 404) unless item && RESOURCES.include?(resource)
1741
+ root, project, _, resource = raw_segments.map { |part| URI.decode_www_form_component(part) }
1742
+ selected, options = selector(resource, raw_segments.drop(4))
1743
+ return invalid_transform if selected.nil?
1744
+ return asset_not_found unless root == 'assets' && project == @project_id && raw_segments[2]
1745
+
1746
+ grant, item = resolve_asset(raw_segments[2])
1747
+ return grant if grant.is_a?(Response)
1748
+ return asset_not_found unless item && RESOURCES.include?(resource)
1749
+ if resource == 'original' && grant && !grant[:original]
1750
+ return error(ErrorCodes::SIGNED_URL_ORIGINAL_NOT_GRANTED, "This signed asset URL doesn't grant the original file", status: 403)
1751
+ end
1752
+
1753
+ params = URI.decode_www_form(query.to_s).to_h
1754
+ # A grant authorises a private asset without the API key.
1755
+ authorised = !grant.nil? || can_download?(item['folderId'], headers)
1756
+ return info(method, item, authorised, params) if resource == 'info'
1757
+ return not_available if item['content']['safetyHold']
1227
1758
  return placeholder(method, item) if resource == 'placeholder'
1228
1759
 
1229
- deliver(method, item, URI.decode_www_form(query.to_s).to_h, options)
1760
+ deliver(method, item, resource, params, selected ? options || {} : nil, authorised)
1761
+ end
1762
+
1763
+ # Whether the path after the resource selects a lens or transform, and its options; nil when the
1764
+ # CDN refuses it. Like the CDN, the adaptive stream's files sit beneath `/hls/` (its master
1765
+ # playlist is `/hls/index.m3u8`). Otherwise the selector after the resource is read
1766
+ # (AssetTransformParser.ts); the original, the embed page and the info take none.
1767
+ def selector(resource, path)
1768
+ lens, options = TransformRules.parse_selector(resource == 'hls' ? '' : path.join('/'))
1769
+ selected = !(lens || options).nil?
1770
+ %w[original embed info].include?(resource) && selected ? nil : [selected, options]
1771
+ rescue ArgumentError
1772
+ nil
1773
+ end
1774
+
1775
+ # Like the CDN (CdnUtils.ts), the asset segment is `ext:` and the URL-encoded external ID, a
1776
+ # grant (`sig_…`) or the asset ID. Returns `[grant, asset]`, or an error response for a grant
1777
+ # the mock didn't sign or that has expired.
1778
+ def resolve_asset(segment)
1779
+ if segment.start_with?('sig_')
1780
+ grant = MockGrant.decode(segment)
1781
+ return error(ErrorCodes::SIGNED_URL_INVALID, "The signed asset URL's grant is invalid", status: 403) unless grant
1782
+ return error(ErrorCodes::SIGNED_URL_EXPIRED, 'The signed URL has expired', status: 403) if grant[:expires_at] <= @now.call
1783
+
1784
+ [grant, project_assets.find { |asset| asset['id'] == grant[:asset_id] }]
1785
+ elsif segment.start_with?(CDN::EXTERNAL_ID_PREFIX)
1786
+ external = MockGrant.percent_decode(segment.delete_prefix(CDN::EXTERNAL_ID_PREFIX))
1787
+ [nil, external && !external.empty? ? project_assets.find { |asset| asset['externalId'] == external } : nil]
1788
+ else
1789
+ id = MockGrant.percent_decode(segment)
1790
+ [nil, project_assets.find { |asset| asset['id'] == id }]
1791
+ end
1792
+ end
1793
+
1794
+ def asset_not_found = error(ErrorCodes::ASSET_NOT_FOUND, 'Asset not found', status: 404)
1795
+
1796
+ # The display info at `/info`, like the CDN's (AssetInfo.ts): the allowlisted fields and whether
1797
+ # the viewer the claims describe can see the content, decided as the other resources decide it.
1798
+ # A private asset's info needs a grant or an API key with `assets:download`.
1799
+ def info(method, item, authorised, params)
1800
+ restricted = item['visibility'] != 'public'
1801
+ return asset_not_found if restricted && !authorised
1802
+
1803
+ claims = JSON.parse(params.fetch('viewerClaims', '{}'))
1804
+ body = { 'id' => item['id'], 'projectId' => item['projectId'], 'type' => item['type'], 'externalUrl' => item['externalUrl'],
1805
+ 'content' => item['content'].slice(*%w[format width height durationSeconds pageCount rendered preview hls]),
1806
+ 'availability' => availability(item, claims.is_a?(Hash) && claims['ageConfirmed'] == true) }
1807
+ cdn_response(method, JSON.generate(body), { 'content-type' => 'application/json',
1808
+ 'cache-control' => "#{restricted ? 'private' : 'public'}, max-age=60",
1809
+ 'cache-tag' => "asset-#{item['id']}" })
1810
+ rescue JSON::ParserError
1811
+ error(ErrorCodes::INVALID_VIEWER_CLAIMS, 'Invalid viewer claims', status: 400)
1230
1812
  end
1231
1813
 
1232
- # Serves a found asset's content after the delivery, age and access checks, then the
1233
- # transform options' schema check.
1234
- def deliver(method, item, params, options)
1814
+ # Whether a viewer can see an asset's content now, decided as the other resources decide it.
1815
+ def availability(item, age_confirmed)
1235
1816
  moderation = item['classification']['moderation']['status']
1236
- stored = @state.content[item['id']]
1237
- if stored.nil? || item['upload']['status'] == 'uploading' || %w[blocked purged].include?(moderation) ||
1817
+ if item['content']['safetyHold'] || item['upload']['status'] == 'uploading' || %w[blocked purged].include?(moderation) ||
1238
1818
  item['classification']['status'] == 'failed' || item['content']['status'] == 'removed'
1239
- return not_available
1819
+ 'unavailable'
1820
+ elsif moderation == 'age-restricted' && !age_confirmed
1821
+ 'age-restricted'
1822
+ else
1823
+ 'available'
1240
1824
  end
1825
+ end
1241
1826
 
1242
- if moderation == 'age-restricted' && !viewer_age_confirmed?(params)
1243
- return error('viewer/age-confirmation-required', 'Age confirmation required',
1244
- status: 403)
1245
- end
1827
+ # Serves a found asset's resource after the delivery, age and access checks, then whether it
1828
+ # has the resource and takes the selector, then the transform options' schema check.
1829
+ # `selector` is the selector's options (empty for a lens alone), or nil without one.
1830
+ def deliver(method, item, resource, params, selector, authorised)
1831
+ stored = @state.content[item['id']]
1832
+ refused = refuse_delivery(item, resource, stored, params, authorised)
1833
+ return refused if refused
1246
1834
 
1247
1835
  restricted = item['visibility'] != 'public'
1248
- if restricted && !can_download?(item['folderId'])
1249
- return error('asset/api-key-missing-permission',
1250
- 'This API key needs assets:download permission for this asset’s folder.', status: 403)
1836
+ return embed(method, item, restricted, params) if resource == 'embed'
1837
+
1838
+ # Then whether the asset has the resource, and whether it is a still image that takes a lens
1839
+ # or transform: the preview always is, the rendered file only when it is an image.
1840
+ rendered, preview = item['content'].values_at('rendered', 'preview')
1841
+ return no_rendered_version if (resource == 'rendered' && !rendered) || (resource == 'preview' && !preview)
1842
+ # The mock prepares no stream, so it has no HLS files.
1843
+ return no_rendered_version if resource == 'hls'
1844
+
1845
+ still = resource == 'preview' || (resource == 'rendered' && rendered['kind'] == 'image')
1846
+ return invalid_transform if !still && selector
1847
+
1848
+ refused = selector ? TransformRules.check(selector).flatten : []
1849
+ return error(ErrorCodes::INVALID_TRANSFORM_OPTIONS, "Invalid transform options: #{refused.join(', ')}", status: 400) if refused.any?
1850
+
1851
+ bytes, type = resource_body(item, resource, stored)
1852
+ return not_available unless bytes
1853
+
1854
+ immutable = still || (resource == 'original' && item['type'] == 'image')
1855
+ cdn_response(method, bytes, viewer_delivery_headers(item, type, restricted, immutable, params))
1856
+ end
1857
+
1858
+ # The delivery checks, which refuse everyone, then age and access, as an error response, or nil.
1859
+ # Only the original needs the upload still stored.
1860
+ def refuse_delivery(item, resource, stored, params, authorised)
1861
+ moderation = item['classification']['moderation']['status']
1862
+ if (resource == 'original' && stored.nil?) || item['upload']['status'] == 'uploading' ||
1863
+ %w[blocked purged].include?(moderation) || item['classification']['status'] == 'failed' ||
1864
+ item['content']['status'] == 'removed'
1865
+ return not_available
1251
1866
  end
1867
+ if moderation == 'age-restricted' && !viewer_age_confirmed?(params)
1868
+ return error('viewer/age-confirmation-required', 'Age confirmation required', status: 403)
1869
+ end
1870
+ return if item['visibility'] == 'public' || authorised
1871
+
1872
+ error(ErrorCodes::PERMISSION_DENIED, 'This API key needs assets:download permission for this asset’s folder.', status: 403)
1873
+ end
1252
1874
 
1253
- refused = options ? TransformRules.check(options).flatten : []
1254
- return error('asset/invalid-transform-options', "Invalid transform options: #{refused.join(', ')}", status: 400) if refused.any?
1875
+ # A resource's bytes and Content-Type: the upload, which stands in for its own rendered file
1876
+ # and an image's preview, or the placeholder PNG for a still the mock doesn't make. Nil bytes
1877
+ # when the upload is no longer stored.
1878
+ def resource_body(item, resource, stored)
1879
+ rendered = item['content']['rendered']
1880
+ serves_upload = resource == 'original' || (resource == 'rendered' && rendered['kind'] != 'image') || item['type'] == 'image'
1881
+ return [Placeholder.png(item.dig('content', 'preview', 'thumbhash') || THUMBHASH), 'image/png'] unless serves_upload
1882
+ return [nil, nil] unless stored
1255
1883
 
1256
- type = stored['type'].to_s.empty? ? 'application/octet-stream' : stored['type']
1257
- cdn_response(method, stored['parts'].sort.map(&:last).join.b,
1258
- viewer_delivery_headers(item, type, restricted, params))
1884
+ type = if resource == 'rendered' then rendered['format']
1885
+ elsif stored['type'].to_s.empty? then 'application/octet-stream'
1886
+ else stored['type']
1887
+ end
1888
+ [stored['parts'].sort.map(&:last).join.b, type]
1259
1889
  end
1260
1890
 
1261
1891
  def viewer_age_confirmed?(params)
1262
1892
  JSON.parse(params.fetch('viewerClaims', '{}'))['ageConfirmed'] == true
1263
1893
  end
1264
1894
 
1265
- def viewer_delivery_headers(item, type, restricted, params)
1266
- headers = delivery_headers(item, type, restricted)
1895
+ def viewer_delivery_headers(item, type, restricted, immutable, params)
1896
+ headers = delivery_headers(item, type, restricted, immutable)
1267
1897
  headers['Cache-Control'] = 'private, no-store' if params.key?('viewerClaims')
1268
1898
  headers
1269
1899
  end
1270
1900
 
1271
- def invalid_transform = error('asset/invalid-transform', 'Invalid transform selector', status: 400)
1901
+ # The embed page, which shows any asset that may be delivered, as its viewer for the asset's
1902
+ # type or a notice that the type has none; cached like other non-image content.
1903
+ def embed(method, item, restricted, params)
1904
+ cache = if params.key?('viewerClaims') then 'private, no-store'
1905
+ else "#{restricted ? 'private' : 'public'}, max-age=3600"
1906
+ end
1907
+ cdn_response(method, EMBED_HTML, { 'content-type' => 'text/html; charset=utf-8', 'cache-control' => cache,
1908
+ 'cache-tag' => "asset-#{item['id']}", 'x-content-type-options' => 'nosniff' })
1909
+ end
1910
+
1911
+ def invalid_transform = error(ErrorCodes::INVALID_TRANSFORM, 'Invalid transform selector', status: 400)
1912
+
1913
+ def no_rendered_version = error('asset/no-rendered-version', 'This asset has no such resource', status: 404)
1272
1914
 
1273
1915
  def not_available = error('asset/not-available', 'Asset is not available', status: 403)
1274
1916
 
1275
- # Like the CDN's placeholder handler: an image's ThumbHash, public whatever the asset's
1917
+ # Like the CDN's placeholder handler: the preview's ThumbHash, public whatever the asset's
1276
1918
  # visibility or moderation; refused for purged assets and assets without one (not yet
1277
- # classified, or content removed).
1919
+ # processed, without a preview, or content removed).
1278
1920
  def placeholder(method, item)
1279
- thumbhash = item.dig('content', 'image', 'thumbhash')
1921
+ thumbhash = item.dig('content', 'preview', 'thumbhash')
1280
1922
  return not_available if thumbhash.nil? || item['classification']['moderation']['status'] == 'purged'
1281
1923
 
1282
1924
  cdn_response(method, Placeholder.png(thumbhash),
@@ -1284,14 +1926,13 @@ module CalmLens
1284
1926
  'cache-tag' => "asset-#{item['id']}", 'etag' => "\"#{thumbhash}\"")
1285
1927
  end
1286
1928
 
1287
- # Like the CDN: an image's bytes are immutable for a year, other content is kept for an hour,
1288
- # by shared caches only when delivery is public. Active content is served as a download.
1289
- def delivery_headers(item, type, restricted)
1290
- image = item['type'] == 'image'
1291
- headers = { 'content-type' => type,
1292
- 'cache-control' => "#{restricted ? 'private' : 'public'}, #{image ? 'max-age=31536000, immutable' : 'max-age=3600'}",
1929
+ # Like the CDN: a still image's bytes are immutable for a year, other content is kept for an
1930
+ # hour, by shared caches only when delivery is public. Active content is served as a download.
1931
+ def delivery_headers(item, type, restricted, immutable)
1932
+ lifetime = immutable ? 'max-age=31536000, immutable' : 'max-age=3600'
1933
+ headers = { 'content-type' => type, 'cache-control' => "#{restricted ? 'private' : 'public'}, #{lifetime}",
1293
1934
  'cache-tag' => "asset-#{item['id']}" }
1294
- headers.merge!('etag' => "\"#{item['id']}\"", 'expires' => Time.at((@now.call / 1000) + 31_536_000).utc.httpdate) if image
1935
+ headers.merge!('etag' => "\"#{item['id']}\"", 'expires' => Time.at((@now.call / 1000) + 31_536_000).utc.httpdate) if immutable
1295
1936
  headers.merge!('x-content-type-options' => 'nosniff', 'content-security-policy' => "sandbox; default-src 'none'")
1296
1937
  mime = type.downcase.split(';', 2).first.to_s.strip
1297
1938
  headers['content-disposition'] = 'attachment' if ACTIVE_TYPES.include?(mime) || mime.end_with?('+xml')
@@ -1303,8 +1944,11 @@ module CalmLens
1303
1944
  headers: headers.merge('content-length' => bytes.bytesize.to_s, 'x-request-id' => request_id))
1304
1945
  end
1305
1946
 
1306
- # Whether the mock API key's grants give `assets:download` for `folder_id`.
1307
- def can_download?(folder_id)
1947
+ # Whether the request carries the mock API key and its grants give `assets:download` for
1948
+ # `folder_id`.
1949
+ def can_download?(folder_id, headers)
1950
+ return false unless header(headers, 'X-Api-Key') == @api_key
1951
+
1308
1952
  @grants.any? do |grant|
1309
1953
  next false unless Array(grant['actions']).include?('assets:download')
1310
1954
  next true if grant['folderId'].nil?
@@ -1327,6 +1971,35 @@ module CalmLens
1327
1971
  end
1328
1972
  end
1329
1973
 
1974
+ # The mock's grant, the asset segment of a URL {CDN#sign_asset_url} signs on the mock: `sig_` and
1975
+ # the lowercase hex of the UTF-8 text `{assetId}.{1|0}.{expiresAt}`, its asset, original access and
1976
+ # expiry. Every SDK's mock writes it the same way. It is readable, not signed, as the mock has no
1977
+ # secret to keep.
1978
+ # @api private
1979
+ module MockGrant
1980
+ module_function
1981
+
1982
+ def encode(asset_id, original, expires_at)
1983
+ "sig_#{"#{asset_id}.#{original ? 1 : 0}.#{expires_at}".unpack1('H*')}"
1984
+ end
1985
+
1986
+ # `{asset_id:, original:, expires_at:}`, or nil for a segment the mock didn't sign.
1987
+ def decode(segment)
1988
+ hex = segment.delete_prefix('sig_')
1989
+ return unless hex.match?(/\A(?:[0-9a-f]{2})+\z/)
1990
+
1991
+ text = [hex].pack('H*').force_encoding(Encoding::UTF_8)
1992
+ match = text.valid_encoding? && /\A(.+)\.([01])\.(\d+)\z/m.match(text)
1993
+ match && { asset_id: match[1], original: match[2] == '1', expires_at: Integer(match[3], 10) }
1994
+ end
1995
+
1996
+ # Percent-decodes a path segment like decodeURIComponent; nil when it isn't valid UTF-8.
1997
+ def percent_decode(text)
1998
+ decoded = text.b.gsub(/%(\h\h)/) { Regexp.last_match(1).hex.chr }.force_encoding(Encoding::UTF_8)
1999
+ decoded if decoded.valid_encoding?
2000
+ end
2001
+ end
2002
+
1330
2003
  # The processing and moderation status of each outcome.
1331
2004
  # @api private
1332
2005
  module AssetRecord
@@ -1345,125 +2018,5 @@ module CalmLens
1345
2018
  end
1346
2019
  end
1347
2020
  end
1348
-
1349
- # The RSQL subset the fake API understands: `==` and `!=` (with `*` wildcards), `=in=` and
1350
- # `=out=` with a `(value,value)` list, and `=gt=`, `=ge=`, `=lt=`, `=le=`, with quoted values,
1351
- # combined with `;` (and), which binds tighter than `,` (or), and grouped with `( … )`.
1352
- # Anything else raises instead of being ignored, so a test never passes on an unfiltered list.
1353
- # @api private
1354
- module RSQL
1355
- module_function
1356
-
1357
- def filter(items, expression)
1358
- return items if expression.nil? || expression.to_s.strip.empty?
1359
-
1360
- test = parse(expression.to_s)
1361
- items.select { |item| test.call(item) }
1362
- end
1363
-
1364
- # Compiles `,` (or) of `;` (and) terms, where a term is a comparison or a `( … )` group.
1365
- def parse(expression)
1366
- alternatives = split(expression, ',').map do |alternative|
1367
- split(alternative, ';').map do |term|
1368
- text = term.strip
1369
- raise unsupported(expression) if text.empty?
1370
-
1371
- group = unwrap(text)
1372
- group ? parse(group) : comparison(text)
1373
- end
1374
- end
1375
- ->(item) { alternatives.any? { |terms| terms.all? { |test| test.call(item) } } }
1376
- end
1377
-
1378
- # The expression inside `( … )` when the parentheses enclose all of `text`, else nil.
1379
- def unwrap(text)
1380
- return unless text.start_with?('(') && text.end_with?(')')
1381
-
1382
- depth = 0
1383
- quote = nil
1384
- text.each_char.with_index do |char, index|
1385
- if quote
1386
- quote = nil if char == quote
1387
- elsif ['"', "'"].include?(char)
1388
- quote = char
1389
- elsif char == '('
1390
- depth += 1
1391
- elsif char == ')'
1392
- depth -= 1
1393
- return index == text.length - 1 ? text[1..-2] : nil if depth.zero?
1394
- end
1395
- end
1396
- nil
1397
- end
1398
-
1399
- def unsupported(term)
1400
- InvalidArgumentError.new(SdkErrorCodes::INVALID_ARGUMENT,
1401
- 'MockServerClient filters support ==, !=, =in=, =out=, =gt=, =ge=, =lt= and ' \
1402
- "=le=, joined by ; and , and grouped with parentheses (got #{term.inspect})")
1403
- end
1404
-
1405
- # Splits on `separator` outside quotes and parentheses (groups and `=in=(...)` lists).
1406
- def split(text, separator)
1407
- parts = ['']
1408
- quote = nil
1409
- depth = 0
1410
- text.each_char do |char|
1411
- if quote
1412
- quote = nil if char == quote
1413
- elsif ['"', "'"].include?(char)
1414
- quote = char
1415
- elsif char == '('
1416
- depth += 1
1417
- elsif char == ')'
1418
- depth -= 1
1419
- elsif char == separator && depth.zero?
1420
- parts << ''
1421
- next
1422
- end
1423
- parts[-1] += char
1424
- end
1425
- parts
1426
- end
1427
-
1428
- def comparison(term)
1429
- match = term.strip.match(/\A([A-Za-z][\w.]*)(==|!=|=in=|=out=|=gt=|=ge=|=lt=|=le=)(.+)\z/m)
1430
- raise unsupported(term) unless match
1431
-
1432
- field, operator, raw = match.captures
1433
- values = list(operator, raw.strip)
1434
- lambda do |item|
1435
- actual = field.split('.').reduce(item) { |object, key| object.is_a?(Hash) ? object[key] : nil }
1436
- entries = Array(actual.nil? ? 'null' : actual).map(&:to_s)
1437
- case operator
1438
- when '==', '=in=' then values.any? { |value| matches?(entries, value) }
1439
- when '!=', '=out=' then values.none? { |value| matches?(entries, value) }
1440
- else entries.any? { |entry| ordered?(entry, operator, values.first) }
1441
- end
1442
- end
1443
- end
1444
-
1445
- def list(operator, raw)
1446
- return [unquote(raw)] unless %w[=in= =out=].include?(operator)
1447
-
1448
- unless raw.start_with?('(') && raw.end_with?(')')
1449
- raise InvalidArgumentError.new(SdkErrorCodes::INVALID_ARGUMENT, "#{operator} takes a (value,value) list")
1450
- end
1451
-
1452
- split(raw[1..-2], ',').map { |value| unquote(value.strip) }
1453
- end
1454
-
1455
- def unquote(value) = value.match?(/\A(["']).*\1\z/m) ? value[1..-2] : value
1456
-
1457
- def matches?(entries, value)
1458
- pattern = Regexp.new("\\A#{Regexp.escape(value).gsub('\*', '.*')}\\z", Regexp::MULTILINE)
1459
- entries.any? { |entry| pattern.match?(entry) }
1460
- end
1461
-
1462
- # Numbers compare as numbers and everything else (such as ISO dates) as text.
1463
- def ordered?(entry, operator, value)
1464
- left, right = [entry, value].all? { |text| text.match?(/\A-?\d+(\.\d+)?\z/) } ? [entry.to_f, value.to_f] : [entry, value]
1465
- { '=gt=' => left > right, '=ge=' => left >= right, '=lt=' => left < right, '=le=' => left <= right }.fetch(operator)
1466
- end
1467
- end
1468
2021
  end
1469
2022
  end