calmlens 0.19.0 → 0.21.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -6,6 +6,7 @@ require 'securerandom'
6
6
  require 'uri'
7
7
  require_relative 'placeholder'
8
8
  require_relative 'mock_decision'
9
+ require_relative 'rsql'
9
10
 
10
11
  module CalmLens
11
12
  module Testing
@@ -36,35 +37,69 @@ module CalmLens
36
37
  :hash_list_entries, keyword_init: true)
37
38
 
38
39
  # A stateful stand-in for the CalmLens API and CDN. It stores assets and folders, applies
39
- # filters and ordering, serves stored bytes, and answers errors with the API's codes, statuses,
40
+ # filters and sorting, serves stored bytes, and answers errors with the API's codes, statuses,
40
41
  # details and body shape, checking requests in the API's order. It moderates through the
41
42
  # `moderate` callback rather than classifying content, and does not run transforms or index for
42
43
  # semantic search.
43
44
  # @api private
44
45
  class FakeAPI
45
- JSON_QUERY = %w[queue isFinal sizeInBytes totalSizeInBytes partNumber pageSize limit processingOptions metadata
46
- context].freeze
46
+ JSON_QUERY = %w[wait isFinal sizeInBytes totalSizeInBytes partNumber pageSize processingOptions metadata
47
+ tags context].freeze
47
48
  # Keyword search fields, in the order matches are reported, and where each value sits in the asset.
48
49
  SEARCH_FIELDS = { 'name' => %w[name], 'id' => %w[id], 'externalId' => %w[externalId],
49
50
  'description' => %w[description], 'tags' => %w[tags], 'type' => %w[type],
50
51
  'content.format' => %w[content format], 'content.extension' => %w[content extension],
51
52
  'classification.moderation.status' => %w[classification moderation status] }.freeze
52
53
  MAX_SEARCH_TOKENS = 8
53
- MAX_SEARCH_LIMIT = 20
54
+ MAX_SEARCH_PAGE_SIZE = 20
54
55
  MAX_SEARCH_QUERY_LENGTH = 200
55
56
  # Every API cursor list returns this many items unless `pageSize` says otherwise.
56
57
  DEFAULT_PAGE_SIZE = 50
57
58
  MAX_PAGE_SIZE = 100
58
59
  REVIEW_DECISIONS = %w[allowed limited age-restricted blocked purged].freeze
60
+ # The moderation status each `asset.<status>` event announces.
61
+ ANNOUNCED_STATUSES = REVIEW_DECISIONS.to_h { |status| ["asset.#{status}", status] }.freeze
59
62
  MAX_DECISION_NOTE_LENGTH = 1499
60
- RESOURCES = %w[original rendered placeholder].freeze
63
+ # The actor of events the mock's API key causes: mock keys have no ID.
64
+ API_KEY_ACTOR = { 'type' => 'api-key', 'id' => nil }.freeze
65
+ # The actor of a reviewer's decision through {MockServerClient#decide}.
66
+ REVIEWER_ACTOR = { 'type' => 'user', 'id' => nil }.freeze
67
+ # The actor of the pipeline's processing events.
68
+ AUTOMATED_ACTOR = { 'type' => 'automated', 'id' => nil }.freeze
69
+ # An event type's shape (`resource.action`), as the mock checks `types` without the catalog.
70
+ EVENT_TYPE_PATTERN = /\A[a-z][a-z-]*\.[a-z][a-z-]*\z/
71
+ # The resources an asset's CDN path names.
72
+ RESOURCES = %w[original rendered preview hls embed placeholder info].freeze
73
+ # The page the mock serves at `/embed`, standing in for the CDN's embed page.
74
+ EMBED_HTML = '<!doctype html><html lang="en"><head><meta charset="utf-8"><title>Embed</title></head>' \
75
+ '<body></body></html>'
76
+
61
77
  MIN_PART = 5 * 1024 * 1024
62
78
  MAX_MULTIPART_IMAGE = 20 * 1024 * 1024
63
79
  # Like the API: an upload has at most this many parts.
64
80
  MAX_PARTS = 10_000
65
- # The ThumbHash the mock gives every classified image, standing in for one computed from its
66
- # pixels. The `placeholder` resource serves it as a PNG, like the CDN.
81
+ # The ThumbHash the mock gives every processed asset's preview, standing in for one computed
82
+ # from its pixels. The `placeholder` resource serves it as a PNG, like the CDN.
67
83
  THUMBHASH = '1QcSHQRnh493V4dIh4eXh1h4kJUI'
84
+ # Removing or withholding content removes its rendered file, adaptive stream and preview, as the API does.
85
+ NO_RENDERING = { 'rendered' => nil, 'hls' => nil, 'preview' => nil }.freeze
86
+ # The size of the waveform the API makes as an audio asset's preview.
87
+ WAVEFORM_SIZE = { 'width' => 1200, 'height' => 300 }.freeze
88
+ # Audio and video formats browsers play as uploaded, so the API serves them as stored (Asset.ts).
89
+ BROWSER_SAFE_MEDIA_FORMATS = %w[video/mp4 audio/mpeg audio/aac audio/mp4 audio/x-m4a audio/ogg].freeze
90
+ # Spreadsheets, which have no rendered file or preview (Asset.ts).
91
+ SPREADSHEET_MIME_TYPES = %w[application/vnd.openxmlformats-officedocument.spreadsheetml.sheet
92
+ application/vnd.ms-excel.sheet.macroenabled.12 application/vnd.ms-excel
93
+ application/vnd.oasis.opendocument.spreadsheet].freeze
94
+ # The settings of the mock's presets: every category with its default action (`allow` for
95
+ # personal information and profanity, `block` otherwise) and no rules. The mock holds no
96
+ # built-in preset settings, and uploads are never decided by a policy.
97
+ PRESET_SETTINGS = {
98
+ 'categorySettings' => MockClassification::CATEGORIES.to_h do |category|
99
+ [category, { 'action' => %w[personal-information profanity].include?(category) ? 'allow' : 'block',
100
+ 'allowances' => [], 'restrictions' => [] }]
101
+ end
102
+ }.freeze
68
103
  # The built-in presets the mock knows, by display name.
69
104
  PRESET_NAMES = { 'general-community' => 'General', 'children-platform' => 'Kids',
70
105
  'education-support' => 'Learning and support', 'wholesome-community' => 'Teen-safe' }.freeze
@@ -77,8 +112,9 @@ module CalmLens
77
112
  # The mock API key's `{ 'actions', 'folderId', 'includeSubfolders' }` grants.
78
113
  attr_accessor :grants
79
114
 
80
- def initialize(project_id:, base_url:, cdn_base_url:, now:, create_id:, moderate:, grants:)
115
+ def initialize(project_id:, base_url:, cdn_base_url:, now:, create_id:, moderate:, api_key:, grants:)
81
116
  @project_id = project_id
117
+ @api_key = api_key
82
118
  @base_url = base_url
83
119
  @cdn_base_url = cdn_base_url
84
120
  @now = now
@@ -92,22 +128,41 @@ module CalmLens
92
128
  @policies = {}
93
129
  @project_policy_id = nil
94
130
  @request_count = 0
131
+ # Each asset's latest `assetSequence`, which the API keeps on the asset row.
132
+ @asset_event_sequences = Hash.new(0)
133
+ # The processing options each asset was uploaded with, which the API keeps but doesn't return.
134
+ @processing_options = {}
135
+ # Decided assets whose re-run (`assets.explain` with `rerun: true`) is still queued: they keep
136
+ # their decision until `complete_processing` decides them.
137
+ @reruns = {}
95
138
  end
96
139
 
97
- # Adds an event to the feed, filling the fields the API always sets.
140
+ # Adds an event to the feed, filling the fields the API always sets. Its `actor` is the mock's
141
+ # API key unless the event names another.
98
142
  def add_event(event)
99
143
  time = @now.call
100
- record = { 'id' => @create_id.call('evt'), 'projectId' => @project_id, 'severity' => 'low', 'description' => nil,
101
- 'assetId' => nil, 'asset' => nil, 'facts' => [], 'createdAt' => time }.merge(event)
144
+ record = { 'id' => @create_id.call('evt'), 'projectId' => @project_id, 'actor' => deep_copy(API_KEY_ACTOR),
145
+ 'severity' => 'low', 'description' => nil,
146
+ 'assetId' => nil, 'assetSequence' => nil, 'asset' => nil, 'facts' => [],
147
+ 'createdAt' => time }.merge(event)
102
148
  record['title'] ||= record['type']
149
+ # Like the API: each event about an asset takes the asset's next number; others have none.
150
+ asset_id = record['assetId']
151
+ record['assetSequence'] = asset_id ? (@asset_event_sequences[asset_id] += 1) : nil
103
152
  @state.events << record
104
153
  record
105
154
  end
106
155
 
107
- # Adds an event like {#add_event}; one about a stored asset carries the asset as it is now.
156
+ # Adds an event like {#add_event}; one about a stored asset carries the asset as it is now,
157
+ # with the moderation status an `asset.<status>` event announces.
108
158
  def enqueue_event(event)
109
159
  item = own_asset(event['assetId'])
110
- event = event.merge('asset' => snapshot(item)) if item && !event.key?('asset')
160
+ if item && !event.key?('asset')
161
+ asset = snapshot(item)
162
+ announced = ANNOUNCED_STATUSES[event['type']]
163
+ asset['classification']['moderation']['status'] = announced if announced
164
+ event = event.merge('asset' => asset)
165
+ end
111
166
  add_event(event)
112
167
  end
113
168
 
@@ -121,25 +176,29 @@ module CalmLens
121
176
  item = own_asset(asset_id)
122
177
  return unless item
123
178
 
124
- record_decision(item, decision, nil, nil)
179
+ record_decision(item, decision, nil, false)
125
180
  deep_copy(item)
126
181
  end
127
182
 
128
183
  # Finishes processing an asset as the pipeline would, with `decision` or else what `moderate`
129
- # answers for the `complete` trigger, and returns it; nil when there is no such asset.
184
+ # answers for the `complete` trigger, and returns it; nil when there is no such asset. A decided
185
+ # asset whose re-run is still queued counts as processing.
130
186
  def complete_processing(asset_id, decision)
131
187
  item = own_asset(asset_id)
132
188
  return unless item
133
- raise Internal.invalid("Asset #{asset_id} isn't processing") unless item['classification']['status'] == 'processing'
134
189
 
135
- moderate_stored(item, 'complete', decision.nil? ? nil : MockDecision.from(decision))
190
+ rerun = @reruns.key?(asset_id)
191
+ raise Internal.invalid("Asset #{asset_id} isn't processing") unless rerun || item['classification']['status'] == 'processing'
192
+
193
+ settled = decision.nil? ? nil : MockDecision.from(decision)
194
+ rerun ? settle_rerun(item, 'complete', settled) : moderate_stored(item, 'complete', settled)
136
195
  deep_copy(item)
137
196
  end
138
197
 
139
198
  # @return [Response]
140
199
  def handle(method, url, headers, body)
141
200
  if url.start_with?("#{@cdn_base_url}/")
142
- serve(method, url.delete_prefix(@cdn_base_url))
201
+ serve(method, url.delete_prefix(@cdn_base_url), headers)
143
202
  else
144
203
  api(method, url.delete_prefix(@base_url), headers, body)
145
204
  end
@@ -172,29 +231,29 @@ module CalmLens
172
231
  end
173
232
 
174
233
  # Applies a reviewer's decision like `decideAssetManually` in the API: the status and the review,
175
- # then (for `purged`) the content removal and its event, then `asset.<decision>`.
176
- def record_decision(item, decision, note, reviewer_name)
234
+ # then (for `purged`) the content removal and its event, then `asset.<decision>`. Either way the
235
+ # review is the mock API key's, which has no ID. A decision through the API (`by_api_key`) has
236
+ # the API key as its events' actor; the test hook's is a reviewer's (a `user`).
237
+ def record_decision(item, decision, note, by_api_key)
238
+ # Like the API: a queued re-run no longer replaces a reviewed decision.
239
+ @reruns.delete(item['id'])
177
240
  apply_outcome(item, decision, 'requested')
178
241
  review = { 'id' => @create_id.call('rev'), 'decision' => decision, 'note' => note,
179
- 'reviewedAt' => @now.call, 'appliedToAsset' => true }
180
- reviewer_facts = []
181
- if reviewer_name
182
- review.merge!('reviewerName' => reviewer_name, 'reviewerEmail' => nil)
183
- reviewer_facts = [{ 'key' => 'reviewer-user-id', 'value' => nil }, { 'key' => 'reviewer-name', 'value' => reviewer_name }]
184
- end
242
+ 'decidedBy' => { 'type' => 'api-key', 'id' => nil }, 'reviewedAt' => @now.call, 'applied' => true }
243
+ actor = by_api_key ? API_KEY_ACTOR : REVIEWER_ACTOR
185
244
  item['classification']['moderation']['manualReviews'] << review
186
- lifecycle_event('asset.content-removed', item) if decision == 'purged'
245
+ lifecycle_event('asset.content-removed', item, actor: actor) if decision == 'purged'
187
246
  # Like the event catalog: only allowing and limiting are low severity.
188
247
  add_event('type' => "asset.#{decision}", 'severity' => %w[allowed limited].include?(decision) ? 'low' : 'high',
189
248
  'title' => "asset.#{decision}", 'assetId' => item['id'], 'asset' => snapshot(item),
190
- 'facts' => [{ 'key' => 'decided-by', 'value' => 'reviewer' }, { 'key' => 'asset-type', 'value' => item['type'] },
249
+ 'actor' => deep_copy(actor),
250
+ 'facts' => [{ 'key' => 'asset-type', 'value' => item['type'] },
191
251
  { 'key' => 'asset-name', 'value' => item['name'], 'url' => "https://calmlens.com/go/asset/#{item['id']}" },
192
- *reviewer_facts, { 'key' => 'note', 'value' => note }])
252
+ { 'key' => 'note', 'value' => note }])
193
253
  end
194
254
 
195
- # `POST /assets/:id/decision` as the API validates and applies it. The mock's key is unnamed, so the
196
- # review and its event name the reviewer `An API key`. An asset still uploading or processing is
197
- # refused with `asset/processing`.
255
+ # `POST /assets/:id/decision` as the API validates and applies it, with the API key as its
256
+ # events' actor. An asset still uploading or processing is refused with `asset/processing`.
198
257
  def decide_request(item, json)
199
258
  decision = json['decision']
200
259
  invalid!('validation/invalid-union', 'decision') unless REVIEW_DECISIONS.include?(decision)
@@ -211,7 +270,7 @@ module CalmLens
211
270
  fail!(ErrorCodes::ASSET_DECISION_UNCHANGED) if status == decision
212
271
  fail!(ErrorCodes::ASSET_CONTENT_PURGED) if status == 'purged' && decision != 'blocked'
213
272
 
214
- record_decision(item, decision, note, 'An API key')
273
+ record_decision(item, decision, note, true)
215
274
  item
216
275
  end
217
276
 
@@ -219,26 +278,66 @@ module CalmLens
219
278
  def snapshot(item)
220
279
  { 'id' => item['id'], 'externalId' => item['externalId'], 'uploadPresetId' => item['uploadPresetId'],
221
280
  'upload' => { 'status' => item['upload']['status'] },
222
- 'classification' => { 'status' => item['classification']['status'],
281
+ 'classification' => { 'status' => item['classification']['status'], 'testInput' => item['classification']['testInput'],
223
282
  'moderation' => { 'status' => item['classification']['moderation']['status'] } },
224
283
  'content' => { 'status' => item['content']['status'] } }
225
284
  end
226
285
 
227
- # The `asset.deleted`, `asset.content-removed` and `asset.retried` events the API records.
228
- def lifecycle_event(type, item)
286
+ # The `asset.deleted`, `asset.content-removed` and `asset.retried` events the API records, with
287
+ # the `batch-id` fact of the batch that caused them, if any.
288
+ def lifecycle_event(type, item, batch_id: nil, actor: API_KEY_ACTOR)
229
289
  name = { 'key' => 'asset-name', 'value' => item['name'] }
230
290
  # A deleted asset has no portal page to link to.
231
291
  name['url'] = "https://calmlens.com/go/asset/#{item['id']}" unless type == 'asset.deleted'
292
+ facts = [name, { 'key' => 'asset-type', 'value' => item['type'] }]
293
+ facts << { 'key' => 'batch-id', 'value' => batch_id } if batch_id
232
294
  add_event('type' => type, 'severity' => 'low', 'assetId' => item['id'], 'asset' => snapshot(item),
233
- 'facts' => [name, { 'key' => 'asset-type', 'value' => item['type'] }])
295
+ 'actor' => deep_copy(actor), 'facts' => facts)
234
296
  end
235
297
 
236
298
  def fail!(code, message = nil, details: nil) = raise(Failure.new(code, message, details: details))
237
299
 
238
300
  def route_not_found!(method, parts) = fail!(ErrorCodes::ROUTE_NOT_FOUND, "No route for #{method} /#{parts.join('/')}")
239
301
 
240
- # A request the API's schema refuses: `validation/<issue>` with HTTP 400 and the field.
241
- def invalid!(code, path) = raise(Failure.new(code, "Invalid #{path}", details: { 'path' => path }, status: 400))
302
+ # A request the API's schema refuses: `validation/invalid-request` with HTTP 400, listing the
303
+ # field and its issue code (such as `validation/too-big`) in `details.issues`.
304
+ def invalid!(code, path)
305
+ message = "Invalid #{path}"
306
+ raise Failure.new(ErrorCodes::INVALID_REQUEST, message,
307
+ details: { 'issues' => [{ 'path' => path, 'message' => message, 'code' => code }] }, status: 400)
308
+ end
309
+
310
+ # Like the API's schema (METADATA_ITEM_SCHEMA): metadata is nil or a list of `{id, value}`
311
+ # Hashes with a String `id` and any JSON `value`.
312
+ def check_metadata(metadata)
313
+ return if metadata.nil?
314
+
315
+ invalid!('validation/invalid-type', 'metadata') unless metadata.is_a?(Array)
316
+ metadata.each_with_index do |item, index|
317
+ invalid!('validation/invalid-type', "metadata.#{index}") unless item.is_a?(Hash)
318
+ invalid!('validation/invalid-type', "metadata.#{index}.id") unless item['id'].is_a?(String)
319
+ invalid!('validation/invalid-value', "metadata.#{index}.value") unless item.key?('value')
320
+ end
321
+ end
322
+
323
+ # Like the API's schema, checked before the asset is looked up: asset updates take no null name.
324
+ def check_asset_body(method, action, json)
325
+ return unless method == 'PATCH' && action.nil?
326
+
327
+ check_metadata(json['metadata'])
328
+ invalid!('validation/invalid-type', 'name') if json.key?('name') && json['name'].nil?
329
+ end
330
+
331
+ # Like the API: an upload's metadata is checked, then its MIME type, which must be one the
332
+ # API can moderate (`upload/unsupported-type` with `details.mimeType` and `details.source`).
333
+ def check_upload(options)
334
+ check_metadata(options['metadata'])
335
+ type = options['contentType']
336
+ return if type.nil? || Internal.accepted_upload_mime_type?(type)
337
+
338
+ fail!(ErrorCodes::UNSUPPORTED_UPLOAD_TYPE, "#{type} is not a valid upload file type",
339
+ details: { 'mimeType' => type, 'source' => 'calmlens' })
340
+ end
242
341
 
243
342
  # Like the API's query schema: an integer from 1 to `max`.
244
343
  def check_range(value, path, max)
@@ -253,10 +352,11 @@ module CalmLens
253
352
  parts = uri.path.split('/').reject(&:empty?).map { |part| URI.decode_www_form_component(part) }
254
353
  query = URI.decode_www_form(uri.query.to_s).to_h.to_h { |key, value| [key, query_value(key, value)] }
255
354
  content_type = header(headers, 'Content-Type')
256
- json = content_type == 'application/json' && body ? JSON.parse(body) : {}
355
+ upload = header(headers, 'X-CalmLens-Upload')
356
+ # A raw upload's body is the file itself, whatever its type.
357
+ json = content_type == 'application/json' && body && upload.nil? ? JSON.parse(body) : {}
257
358
  # HTTP stacks trim header values, so the API sees the key without surrounding whitespace.
258
359
  key = header(headers, 'Idempotency-Key')&.to_s&.gsub(/\A[ \t\r\n]+|[ \t\r\n]+\z/, '')
259
- upload = header(headers, 'X-CalmLens-Upload')
260
360
  return dispatch(method, parts, query, json, body, upload) unless key && method == 'POST'
261
361
 
262
362
  fail!(ErrorCodes::INVALID_IDEMPOTENCY_KEY) unless key.match?(IDEMPOTENCY_KEY)
@@ -294,6 +394,8 @@ module CalmLens
294
394
  when 'folders' then folder(method, parts, json)
295
395
  when 'hash-lists' then hash_list(method, parts, query, json)
296
396
  when 'hash-list-entries' then remove_hash_list_entry(method, parts)
397
+ when 'policies' then policies(method, parts, query, json)
398
+ when 'lenses' then lens(method, parts, json)
297
399
  else route_not_found!(method, parts)
298
400
  end
299
401
  data.is_a?(Response) ? data : respond(200, deep_copy(data))
@@ -308,19 +410,21 @@ module CalmLens
308
410
  case [method, parts[2..], upload]
309
411
  in ['GET', ['events'], _] then events(query)
310
412
  in ['PUT', ['lenses'], _] then lenses(json.fetch('lenses', []))
413
+ in ['GET', ['lenses'], _] then page(@state.lenses.values, query, JSON.generate(['lenses', @project_id]))
414
+ in ['POST', ['lenses'], _] then create_lens(json)
311
415
  in ['GET', ['folders'], _] then folder_page(query)
312
416
  in ['POST', ['folders'], _] then create_folder(json)
313
417
  in ['GET', ['hash-lists'], _] then page(project_hash_lists, query, JSON.generate([@project_id]))
314
418
  in ['POST', ['hash-lists'], _] then create_hash_list(json)
315
- in ['GET', ['policies'], _] then policy_list
316
- in ['POST', ['policies'], _] then create_policy(json)
419
+ in ['GET', ['policy'], _] then show_policy(find_policy(ensure_policies))
317
420
  in ['PUT', ['policy'], _] then show_policy(assign_policy(json))
318
- in [_, ['policies', String => policy_id, *rest], _] then policy(method, policy_id, rest, json)
319
421
  in ['GET', ['assets'], _] then list(query)
320
422
  in ['GET', ['assets', 'search'], _] then search(query)
423
+ in ['GET', ['assets', String => asset_id, 'timeline'], _] then timeline(asset_id, query)
321
424
  in ['POST', ['assets', 'batch'], _] then batch(json)
322
- in ['POST', ['assets'], nil] then create_asset(json, query['queue'] == true, url: json['url'])
323
- in ['POST', ['assets'], 'content'] then create_asset(query, query['queue'] == true, data: body)
425
+ in ['POST', ['signed-asset-urls'], _] then sign_asset_url(json)
426
+ in ['POST', ['assets'], nil] then create_asset(json, query['wait'] == false, url: json['url'])
427
+ in ['POST', ['assets'], 'content'] then create_asset(query, query['wait'] == false, data: body)
324
428
  in ['POST', ['assets'], 'part'] then start_multipart(query, body)
325
429
  else route_not_found!(method, parts)
326
430
  end
@@ -337,7 +441,8 @@ module CalmLens
337
441
  # (`asset/not-found`) before anything happens. The operation then runs at once, in request
338
442
  # order, rather than in the background: `delete` deletes each asset with its stored child
339
443
  # assets and records `asset.deleted` with the asset as it was; `remove-content` removes the
340
- # content like DELETE /assets/:id/content. Mock assets carry no safety holds.
444
+ # content like DELETE /assets/:id/content. Each event carries the batch's `batch-id` fact. Mock
445
+ # assets carry no safety holds.
341
446
  def batch(json)
342
447
  ids = json['assetIds']
343
448
  invalid!('validation/invalid-type', 'assetIds') unless ids.is_a?(Array)
@@ -352,15 +457,16 @@ module CalmLens
352
457
  fail!(ErrorCodes::ASSET_NOT_FOUND) unless item && item['projectId'] == @project_id
353
458
  item
354
459
  end
460
+ batch_id = @create_id.call('batch')
355
461
  items.each do |item|
356
462
  if json['operation'] == 'delete'
357
463
  delete_tree(item)
358
- lifecycle_event('asset.deleted', item)
464
+ lifecycle_event('asset.deleted', item, batch_id: batch_id)
359
465
  else
360
- delete_content(item)
466
+ remove_content(item, batch_id)
361
467
  end
362
468
  end
363
- respond(202, { 'queued' => true })
469
+ respond(202, { 'batchId' => batch_id, 'status' => 'queued', 'assetCount' => items.size })
364
470
  end
365
471
 
366
472
  # Deletes an asset, its stored child assets and their content.
@@ -375,26 +481,26 @@ module CalmLens
375
481
  deleting.each do |id|
376
482
  @state.assets.delete(id)
377
483
  @state.content.delete(id)
484
+ @processing_options.delete(id)
378
485
  end
379
486
  end
380
487
 
381
488
  def list(query)
382
- items = project_assets.reject { |item| item['parentId'] }
383
- items = items.select { |item| item['folderId'] == query['folderId'] } if query.key?('folderId')
384
- page(RSQL.filter(items, query['filter']), query, JSON.generate([@project_id, query['filter'], query['folderId']]))
489
+ # Like the API: the query schema is checked before the filter.
490
+ page_query(query)
491
+ page(RSQL.filter(project_assets, query['filter']), query, JSON.generate([@project_id, query['filter']]))
385
492
  end
386
493
 
387
494
  # Like the API: one keyset page of a parent's child folders, optionally matching `search`, by
388
- # `name` ascending unless `ordering` (one entry, `name` or `createdAt`) says otherwise, with an
495
+ # `name` ascending unless `sort` (one field, `name` or `createdAt`) says otherwise, with an
389
496
  # `id` tie-breaker, and the ancestors of the parent and the listed folders. Names compare by
390
497
  # code point, so case matters.
391
498
  def folder_page(query)
392
499
  found = project_folders.select do |f|
393
500
  f['parentId'] == query['parentId'] && (!query['search'] || f['name'].downcase.include?(query['search'].downcase))
394
501
  end
395
- invalid!('validation/too-big', 'ordering') if ordering(query, nil).size > 1
396
502
  scope = JSON.generate([@project_id, query['parentId'], query['search']])
397
- result = page(found, query, scope, fields: %w[name createdAt], default: { 'field' => 'name', 'direction' => 'asc' })
503
+ result = page(found, query, scope, fields: %w[name createdAt], max: 1, default: { 'field' => 'name', 'direction' => 'asc' })
398
504
  page = result['items']
399
505
  ids = ([query['parentId']] + page.map { |f| f['id'] }).compact.uniq
400
506
  queue = ids.dup
@@ -411,46 +517,85 @@ module CalmLens
411
517
  def project_folders = @state.folders.values.select { |item| item['projectId'] == @project_id }
412
518
 
413
519
  # Like the API: without a cursor, no events and a `nextCursor` for now; with one, up to `pageSize`
414
- # events added since, oldest first. `nextCursor` is never nil. The mock's key holds every permission,
415
- # so no event type is excluded.
520
+ # events added since, oldest first, of the `types` (comma-separated) and `assetId` asked for.
521
+ # `nextCursor` is never nil, and a cursor belongs to the filters it was issued with. The mock's
522
+ # key holds every permission, so no event type is excluded.
416
523
  def events(query)
417
524
  size = check_range(query.fetch('pageSize', DEFAULT_PAGE_SIZE), 'pageSize', MAX_PAGE_SIZE)
525
+ types = query.key?('types') ? query['types'].to_s.split(',', -1) : []
526
+ if query.key?('types') && (types.empty? || types.any? { |type| !type.match?(EVENT_TYPE_PATTERN) })
527
+ invalid!('validation/invalid-value', 'types')
528
+ end
529
+ asset_id = query['assetId']
530
+ invalid!('validation/invalid-format', 'assetId') if asset_id && !asset_id.match?(ASSET_ID_PATTERN)
531
+ filter = [JSON.generate([types.uniq.sort, asset_id])].pack('m0')
532
+ matches = ->(event) { (types.empty? || types.include?(event['type'])) && (asset_id.nil? || event['assetId'] == asset_id) }
418
533
  feed = @state.events
419
- return { 'items' => [], 'nextCursor' => "mock_#{feed.size}", 'hasMore' => false, 'excludedTypes' => [] } unless query['cursor']
534
+ unless query['cursor']
535
+ return { 'items' => [], 'nextCursor' => "mock_#{feed.size}~#{filter}", 'hasMore' => false, 'excludedTypes' => [] }
536
+ end
420
537
 
421
- match = query['cursor'].to_s.match(/\Amock_(\d+)\z/)
422
- position = match && Integer(match[1], 10)
538
+ match = query['cursor'].to_s.match(/\Amock_(\d+)~(.*)\z/)
539
+ position = match && match[2] == filter ? Integer(match[1], 10) : nil
423
540
  fail!(ErrorCodes::INVALID_CURSOR) unless position && position <= feed.size
424
541
 
425
- taken = feed[position, size]
426
- following = position + taken.size
427
- { 'items' => taken, 'nextCursor' => "mock_#{following}", 'hasMore' => following < feed.size, 'excludedTypes' => [] }
542
+ taken = []
543
+ following = position
544
+ while following < feed.size && taken.size < size
545
+ event = feed[following]
546
+ following += 1
547
+ taken << event if matches.call(event)
548
+ end
549
+ { 'items' => taken, 'nextCursor' => "mock_#{following}~#{filter}", 'hasMore' => feed.drop(following).any?(&matches),
550
+ 'excludedTypes' => [] }
428
551
  end
429
552
 
430
- # Saves lenses in order, like the API: an unchanged definition returns the stored lens, a
431
- # change needs a greater version, and lenses before a conflict stay saved.
553
+ # Saves lenses in order, like the API: a new identifier creates a lens, an unchanged
554
+ # definition returns the stored lens and a changed one updates it in place.
432
555
  def lenses(definitions)
433
556
  { 'lenses' => definitions.map { |definition| save_lens(definition) } }
434
557
  end
435
558
 
436
559
  def save_lens(definition)
437
- now = @now.call
438
560
  old = @state.lenses[definition['identifier']]
439
- unless old
440
- lens = { 'id' => @create_id.call('lens'), 'projectId' => @project_id }.merge(definition)
441
- return @state.lenses[definition['identifier']] = lens.merge('createdAt' => now, 'updatedAt' => now)
442
- end
561
+ return new_lens(definition) unless old
443
562
 
444
563
  same = %w[name type active].all? { |key| old[key] == definition[key] } &&
445
564
  old['description'] == definition['description'] && canonical(old['transform']) == canonical(definition['transform'])
446
- return old if same && old['version'] == definition['version']
447
- if definition['version'].to_i > old['version'].to_i
448
- return @state.lenses[old['identifier']] =
449
- old.merge(definition).merge('updatedAt' => now)
450
- end
565
+ return old if same
566
+
567
+ @state.lenses[old['identifier']] = old.merge(definition).merge('updatedAt' => @now.call)
568
+ end
451
569
 
452
- fail!(ErrorCodes::LENS_VERSION_CONFLICT, details: { 'currentVersion' => old['version'], 'minimumVersion' => old['version'] + 1,
453
- 'requestedVersion' => definition['version'] })
570
+ def new_lens(definition)
571
+ now = @now.call
572
+ lens = { 'id' => @create_id.call('lens'), 'projectId' => @project_id }.merge(definition)
573
+ @state.lenses[definition['identifier']] = lens.merge('createdAt' => now, 'updatedAt' => now)
574
+ end
575
+
576
+ # POST /projects/:id/lenses: like the API, the identifier must be new to the project.
577
+ def create_lens(json)
578
+ identifier = json['identifier']
579
+ invalid!('validation/invalid-type', 'identifier') unless identifier.is_a?(String) && !identifier.empty?
580
+ fail!(ErrorCodes::LENS_IDENTIFIER_TAKEN, details: { 'identifier' => identifier }) if @state.lenses.key?(identifier)
581
+
582
+ # Like the API, a new lens is 201 Created.
583
+ respond(201, deep_copy(new_lens(json)))
584
+ end
585
+
586
+ # GET, PATCH and DELETE /lenses/:id. An update keeps the identifier and changes the rest of
587
+ # the definition, as ensuring the changed definition would.
588
+ def lens(method, parts, json)
589
+ item = @state.lenses.values.find { |lens| lens['id'] == parts[1] && lens['projectId'] == @project_id }
590
+ fail!(ErrorCodes::LENS_NOT_FOUND) unless item && parts.size == 2
591
+ case method
592
+ when 'GET' then item
593
+ when 'PATCH' then save_lens(item.merge(json.except('identifier', 'id', 'projectId', 'createdAt', 'updatedAt')))
594
+ when 'DELETE'
595
+ @state.lenses.delete(item['identifier'])
596
+ {}
597
+ else route_not_found!(method, parts)
598
+ end
454
599
  end
455
600
 
456
601
  def canonical(value)
@@ -467,7 +612,7 @@ module CalmLens
467
612
  text = query['query'].to_s.strip
468
613
  invalid!('validation/too-small', 'query') if text.empty?
469
614
  invalid!('validation/too-big', 'query') if text.length > MAX_SEARCH_QUERY_LENGTH
470
- limit = check_range(query.fetch('limit', 10), 'limit', MAX_SEARCH_LIMIT)
615
+ limit = check_range(query.fetch('pageSize', 10), 'pageSize', MAX_SEARCH_PAGE_SIZE)
471
616
  normalized = text.downcase
472
617
  tokens = normalized.split(/\s+/).uniq.first(MAX_SEARCH_TOKENS)
473
618
  items = project_assets.filter_map do |item|
@@ -507,6 +652,7 @@ module CalmLens
507
652
  end
508
653
 
509
654
  def create_asset(options, queue, url: nil, data: nil, open: false)
655
+ check_upload(options)
510
656
  check_folder(options['folderId']) if options['folderId']
511
657
  if options['externalId'] && project_assets.any? { |item| item['externalId'] == options['externalId'] }
512
658
  fail!(ErrorCodes::DUPLICATE_EXTERNAL_ID)
@@ -521,13 +667,17 @@ module CalmLens
521
667
  'folderId' => options['folderId'], 'uploadPresetId' => options['uploadPresetId'],
522
668
  'createdAt' => now, 'updatedAt' => now, 'name' => options['name'], 'description' => options['description'],
523
669
  'externalId' => options['externalId'], 'externalUrl' => url || options['externalUrl'],
524
- 'type' => kind(mime, url), 'metadata' => options['metadata'], 'authorId' => options['authorId'],
670
+ 'type' => kind(mime, url), 'metadata' => options['metadata'] || [], 'authorId' => options['authorId'],
525
671
  # Like the API, only the posts CalmLens uses are kept, and only while the content is held.
526
- 'context' => context.empty? ? nil : { 'items' => context }, 'tags' => nil,
527
- 'processingOptions' => processing, 'visibility' => options.fetch('visibility', 'private'),
672
+ 'context' => context.empty? ? nil : { 'items' => context }, 'tags' => options['tags'] || [],
673
+ 'visibility' => options.fetch('visibility', 'private'),
528
674
  **sections(now: now, open: open, keep: keep, format: mime, size: data&.bytesize) }
529
675
  @state.assets[asset['id']] = asset
530
- @state.content[asset['id']] = { 'type' => mime, 'parts' => { 1 => data } } if data
676
+ @processing_options[asset['id']] = processing
677
+ if data
678
+ @state.content[asset['id']] = { 'type' => mime, 'parts' => { 1 => data } }
679
+ asset['content'].merge!(rendering(asset, false)) unless open
680
+ end
531
681
  open ? asset : upload_completed(asset, queue, data)
532
682
  end
533
683
 
@@ -535,20 +685,29 @@ module CalmLens
535
685
  {
536
686
  'content' => { 'status' => keep ? 'stored' : 'temporary', 'removalReason' => nil, 'updatedAt' => now,
537
687
  'safetyHold' => nil, 'format' => format.empty? ? nil : format, 'extension' => nil,
538
- 'sizeInBytes' => size, 'checksums' => nil, 'image' => nil, 'extractedText' => nil,
539
- 'fileMetadata' => nil },
688
+ # The mock decodes nothing, so the file's own details are unknown, and like
689
+ # the API, the rendered file and preview come with processing.
690
+ 'sizeInBytes' => size, 'checksums' => nil, 'width' => nil, 'height' => nil,
691
+ 'durationSeconds' => nil, 'pageCount' => nil, 'perceptualHash' => nil,
692
+ 'perceptualHashQuality' => nil, 'metadata' => nil, 'extractedText' => nil, **NO_RENDERING },
540
693
  # Like the API, only an open upload counts the parts received.
541
694
  'upload' => { 'status' => open ? 'uploading' : 'completed', 'updatedAt' => now, 'partsReceived' => open ? 1 : nil },
542
- 'classification' => { 'status' => open ? 'pending' : 'processing', 'failureReason' => nil, 'updatedAt' => now,
543
- 'completedAt' => nil,
544
- 'moderation' => { 'status' => 'pending', 'manualReviews' => [],
545
- 'flaggedCategories' => [], 'regions' => [] } }
695
+ 'classification' => { 'status' => open ? 'queued' : 'processing', 'failureReason' => nil, 'updatedAt' => now,
696
+ # The mock never reads test inputs.
697
+ 'completedAt' => nil, 'progress' => nil, 'testInput' => false,
698
+ # Like the API, moderation's results are null until it decides, and a
699
+ # feature's result is null until it has run.
700
+ 'moderation' => { 'status' => 'pending', 'manualReviews' => [], 'automatedStatus' => nil,
701
+ 'source' => nil, 'rule' => nil, 'explanationComplete' => nil,
702
+ 'categories' => nil, 'flaggedCategories' => [], 'regions' => [],
703
+ 'raw' => nil, 'timeline' => nil },
704
+ 'ocr' => nil, 'linkDetection' => nil }
546
705
  }
547
706
  end
548
707
 
549
708
  # Like the API: image uploads in parts total at most 20 MiB, at any size below that.
550
709
  def image_size_error(type, size, _final)
551
- ErrorCodes::MULTIPART_IMAGE_TOO_LARGE if type == 'image' && size > MAX_MULTIPART_IMAGE
710
+ ErrorCodes::UPLOAD_TOO_LARGE if type == 'image' && size > MAX_MULTIPART_IMAGE
552
711
  end
553
712
 
554
713
  # Like the API: the parts may not overshoot a declared total, and must reach it at the end.
@@ -558,6 +717,12 @@ module CalmLens
558
717
  ErrorCodes::UPLOAD_SIZE_MISMATCH if received > total || (final && received != total)
559
718
  end
560
719
 
720
+ # The failure for a size error code, with the limit `upload/too-large` carries.
721
+ def size_failure!(code)
722
+ details = code == ErrorCodes::UPLOAD_TOO_LARGE ? { 'limitBytes' => MAX_MULTIPART_IMAGE, 'source' => 'plan' } : nil
723
+ fail!(code, details: details)
724
+ end
725
+
561
726
  def check_part_size(size, final)
562
727
  return if final || size >= MIN_PART
563
728
 
@@ -565,20 +730,21 @@ module CalmLens
565
730
  end
566
731
 
567
732
  # In the API's order: a declared total, then a single final part as the whole upload, then the
568
- # part itself. The queue mode and declared total are saved with an open upload.
733
+ # part itself. Whether to wait and the declared total are saved with an open upload.
569
734
  def start_multipart(query, data)
735
+ check_upload(query)
570
736
  total = query['totalSizeInBytes']
571
737
  type = kind(query['contentType'].to_s, nil)
572
738
  final = query['isFinal'] == true
573
739
  size = data.to_s.bytesize
574
740
  code = (total && image_size_error(type, total, true)) ||
575
741
  (final ? total_size_error(total, size, true) || image_size_error(type, size, true) : nil)
576
- fail!(code) if code
742
+ size_failure!(code) if code
577
743
  check_part_size(size, final)
578
744
  fail!(total_size_error(total, size, false)) if !final && total_size_error(total, size, false)
579
745
 
580
- asset = create_asset(query, query['queue'] == true, data: data, open: !final)
581
- @state.uploads[asset['id']] = { 'queue' => query['queue'] == true, 'total' => total } unless final
746
+ asset = create_asset(query, query['wait'] == false, data: data, open: !final)
747
+ @state.uploads[asset['id']] = { 'queue' => query['wait'] == false, 'total' => total } unless final
582
748
  asset
583
749
  end
584
750
 
@@ -614,8 +780,26 @@ module CalmLens
614
780
  # Without `moderate`, queued work stays pending until completed, which allows it.
615
781
  def default_moderation(trigger, queued) = queued && trigger != 'complete' ? 'pending' : 'allowed'
616
782
 
617
- # Applies `decision` to `asset` with its report.
618
- def decided(asset, decision) = apply_outcome(asset, decision.status, 'policy', decision.report)
783
+ # Applies `decision` to `asset` with its report. The mock reads no text, QR codes, people or
784
+ # faces in an image, so a decided image reports that the features its upload ran found nothing:
785
+ # OCR and person and face detection when turned on, link detection unless turned off.
786
+ def decided(asset, decision)
787
+ apply_outcome(asset, decision.status, 'policy', decision.report)
788
+ classification = asset['classification']
789
+ return asset unless classification['status'] == 'completed' && asset['type'] == 'image'
790
+
791
+ processing = @processing_options[asset['id']]
792
+ features = processing.is_a?(Hash) && processing['features'].is_a?(Hash) ? processing['features'] : {}
793
+ setting = ->(name) { features[name].is_a?(Hash) ? features[name]['enabled'] : nil }
794
+ classification['ocr'] = { 'status' => 'evaluated', 'textRegions' => [] } if setting.call('ocr') == true
795
+ unless setting.call('linkDetection') == false
796
+ classification['linkDetection'] = { 'status' => 'evaluated', 'qrCodes' => [], 'links' => [] }
797
+ end
798
+ { 'personDetection' => 'people', 'faceDetection' => 'faces' }.each do |feature, list|
799
+ classification[feature] = { 'status' => 'evaluated', list => [] } if setting.call(feature) == true
800
+ end
801
+ asset
802
+ end
619
803
 
620
804
  # Asks `moderate` (unless `decision` is given) and settles a processing asset: a decision and
621
805
  # its events, or nothing for `pending`.
@@ -637,6 +821,7 @@ module CalmLens
637
821
 
638
822
  if decision.status == 'failed'
639
823
  return add_event('type' => 'asset.failed', 'severity' => 'high', 'assetId' => item['id'], 'asset' => snapshot(item),
824
+ 'actor' => deep_copy(AUTOMATED_ACTOR),
640
825
  'facts' => [asset_name_fact(item), { 'key' => 'asset-type', 'value' => item['type'] },
641
826
  { 'key' => 'previous-status', 'value' => 'processing' },
642
827
  { 'key' => 'failure-reason',
@@ -645,40 +830,38 @@ module CalmLens
645
830
 
646
831
  flagged = decision.report['flaggedCategories']
647
832
  add_event('type' => 'asset.classified', 'severity' => 'low', 'assetId' => item['id'], 'asset' => snapshot(item),
833
+ 'actor' => deep_copy(AUTOMATED_ACTOR),
648
834
  'facts' => [{ 'key' => 'flagged-categories', 'value' => flagged },
649
835
  { 'key' => 'moderation-status', 'value' => decision.status },
650
836
  { 'key' => 'asset-type', 'value' => item['type'] }, asset_name_fact(item)])
651
837
  # Like the event catalog: only allowing and limiting are low severity.
652
838
  add_event('type' => "asset.#{decision.status}", 'severity' => %w[allowed limited].include?(decision.status) ? 'low' : 'high',
653
- 'assetId' => item['id'], 'asset' => snapshot(item),
654
- 'facts' => [{ 'key' => 'decided-by', 'value' => 'automated' }, { 'key' => 'flagged-categories', 'value' => flagged },
839
+ 'assetId' => item['id'], 'asset' => snapshot(item), 'actor' => deep_copy(AUTOMATED_ACTOR),
840
+ 'facts' => [{ 'key' => 'flagged-categories', 'value' => flagged },
655
841
  { 'key' => 'asset-type', 'value' => item['type'] }, asset_name_fact(item)])
656
842
  end
657
843
 
658
844
  # Moves an asset to an outcome like the pipeline would: a decision finishes processing, `purged`
659
845
  # deletes the content (with `purge_reason`, `policy` for the pipeline), and finished processing
660
- # removes content kept only temporarily. A classified image gets a ThumbHash for its
661
- # placeholder, which goes with its content. `report` is a decision's categories, score and policy.
846
+ # removes content kept only temporarily. A decided asset gets its rendered file and preview
847
+ # (with a ThumbHash for its placeholder), which go with its content. `report` is a decision's
848
+ # categories, score and policy.
662
849
  def apply_outcome(asset, outcome, purge_reason = 'policy', report = nil)
663
850
  processing, moderation = AssetRecord.statuses(outcome)
664
851
  now = @now.call
665
852
  decided = processing == 'completed'
666
853
  purge = moderation == 'purged' && asset['content']['status'] != 'removed'
667
854
  remove = purge || (decided && asset['content']['status'] == 'temporary')
668
- image = asset['content']['image']
669
- if decided && asset['type'] == 'image' && asset['content']['status'] != 'removed' && !purge
670
- image = { 'width' => nil, 'height' => nil, 'perceptualHash' => nil }.merge(image || {})
671
- .merge('thumbhash' => THUMBHASH,
672
- 'averageColor' => image&.fetch('averageColor',
673
- nil) || '#7f7f7f')
674
- elsif moderation == 'purged'
675
- image = without_placeholder(image)
676
- end
677
- asset['content']['image'] = image
855
+ # Like the API, an asset whose content is held has its rendered file and preview, completed by
856
+ # its decision, and removing content (when processing finishes or by purging) removes them
857
+ # with it, so no placeholder outlives the content. A safety hold withholds them.
858
+ content = asset['content']
859
+ held = content['status'] != 'removed' && !purge && !content['safetyHold']
860
+ content.merge!(held ? rendering(asset, decided) : NO_RENDERING)
678
861
  if remove
679
862
  forget_content(asset)
680
- asset['content'].merge!('status' => 'removed', 'removalReason' => purge ? purge_reason : 'retention', 'updatedAt' => now,
681
- 'extractedText' => nil, 'fileMetadata' => nil, 'image' => without_placeholder(image))
863
+ content.merge!('status' => 'removed', 'removalReason' => purge ? purge_reason : 'retention', 'updatedAt' => now,
864
+ 'extractedText' => nil, 'metadata' => nil, **NO_RENDERING)
682
865
  end
683
866
  asset['upload'].merge!('status' => 'completed', 'updatedAt' => now)
684
867
  update_classification(asset['classification'], processing, moderation, now)
@@ -700,11 +883,53 @@ module CalmLens
700
883
  item['context'] = nil
701
884
  end
702
885
 
703
- # Removing content removes the placeholder and average colour with it.
704
- def without_placeholder(image) = image&.merge('thumbhash' => nil, 'averageColor' => nil)
886
+ # Whether browsers play an audio or video upload's format as it is, so it is served as stored.
887
+ def browser_safe_media?(format) = BROWSER_SAFE_MEDIA_FORMATS.include?(mime_type(format))
888
+
889
+ # A MIME type without parameters, in lower case.
890
+ def mime_type(format) = format.to_s.downcase.split(';', 2).first.to_s.strip
891
+
892
+ # The rendered file and preview the API reports for an asset with its content held (Asset.ts,
893
+ # getAssetRendered and getAssetPreview). Once uploaded, an image renders as itself with a preview
894
+ # that has no ThumbHash yet, and a PDF and browser-safe audio and video render as uploaded. Once
895
+ # processed, a website renders as its screenshot; a document or presentation as a PDF, with its
896
+ # first page as the preview; video and audio as a browser-safe file (MP4 or M4A unless browsers
897
+ # play the upload), audio with a waveform preview and video with no poster; and every preview has
898
+ # its ThumbHash. Text, spreadsheets and other files have neither. The mock decodes nothing, so
899
+ # sizes are unknown, audio has no peaks, and audio and video never get an adaptive stream.
900
+ def rendering(asset, processed) = { **rendered_file(asset, processed), 'hls' => nil }
901
+
902
+ # The rendered file and preview of {#rendering}.
903
+ def rendered_file(asset, processed)
904
+ format = asset['content']['format']
905
+ mime = mime_type(format)
906
+ safe = browser_safe_media?(format)
907
+ media = %w[video audio].include?(asset['type'])
908
+ return NO_RENDERING.dup unless processed || asset['type'] == 'image' || mime == 'application/pdf' || (media && safe)
909
+
910
+ unsized = { 'width' => nil, 'height' => nil }
911
+ preview = lambda do |size = unsized|
912
+ size.merge('thumbhash' => processed ? THUMBHASH : nil, 'averageColor' => processed ? '#7f7f7f' : nil)
913
+ end
914
+ case asset['type']
915
+ when 'image' then { 'rendered' => { 'kind' => 'image', 'format' => format || 'image/png', **unsized }, 'preview' => preview.call }
916
+ when 'website' then { 'rendered' => { 'kind' => 'image', 'format' => 'image/png', **unsized }, 'preview' => preview.call }
917
+ when 'multimedia-document'
918
+ return NO_RENDERING.dup if SPREADSHEET_MIME_TYPES.include?(mime)
919
+
920
+ { 'rendered' => { 'kind' => 'pdf', 'format' => 'application/pdf', **unsized }, 'preview' => processed ? preview.call : nil }
921
+ when 'video' then { 'rendered' => { 'kind' => 'video', 'format' => safe ? format : 'video/mp4', **unsized }, 'preview' => nil }
922
+ when 'audio'
923
+ rendered = { 'kind' => 'audio', 'format' => safe ? format : 'audio/mp4', **unsized }
924
+ rendered['peaks'] = [] if processed
925
+ { 'rendered' => rendered, 'preview' => processed ? preview.call(WAVEFORM_SIZE.dup) : nil }
926
+ else NO_RENDERING.dup
927
+ end
928
+ end
705
929
 
706
930
  def asset(method, parts, query, json, body)
707
931
  id = parts[1]
932
+ check_asset_body(method, parts[2], json)
708
933
  item = @state.assets[id]
709
934
  fail!(ErrorCodes::ASSET_NOT_FOUND) unless item && item['projectId'] == @project_id
710
935
  case [method, parts[2]]
@@ -713,9 +938,10 @@ module CalmLens
713
938
  in ['DELETE', nil]
714
939
  @state.assets.delete(id)
715
940
  @state.content.delete(id)
941
+ @processing_options.delete(id)
716
942
  lifecycle_event('asset.deleted', item)
717
943
  nil
718
- in ['DELETE', 'content'] then delete_content(item)
944
+ in ['DELETE', 'content'] then remove_content(item)
719
945
  in ['GET', 'children'] then page(project_assets.select { |child| child['parentId'] == id }, query, id)
720
946
  in ['POST', 'retry'] then retry_asset(item)
721
947
  in ['POST', 'explain'] then explain(item, json['rerun'])
@@ -723,7 +949,6 @@ module CalmLens
723
949
  in ['POST', 'move']
724
950
  check_folder(json['folderId']) if json['folderId']
725
951
  touch(item.merge!('folderId' => json['folderId']))
726
- in ['POST', 'signed-url'] then signed_url(id, json)
727
952
  in ['POST', 'part'] then part(item, query, body)
728
953
  else route_not_found!(method, parts)
729
954
  end
@@ -742,21 +967,21 @@ module CalmLens
742
967
  touch(item.merge!(json.except('id', 'projectId', 'upload', 'classification', 'content')))
743
968
  end
744
969
 
745
- # Like the API: the record and report stay; stored content, extracted text and the placeholder
746
- # go. Content already removed keeps its reason, and an `asset.content-removed` event is
747
- # recorded either way.
748
- def delete_content(item)
970
+ # Like the API: the record and report stay; stored content, extracted text, metadata, the
971
+ # rendered file and the preview go. Content already removed keeps its reason, and an `asset.content-removed` event is
972
+ # recorded either way, with the `batch-id` fact of the batch that removed it, if any.
973
+ def remove_content(item, batch_id = nil)
749
974
  forget_content(item)
750
975
  content = item['content']
751
976
  if content['status'] == 'removed'
752
- content['image'] = without_placeholder(content['image']) if content.dig('image', 'thumbhash')
977
+ content.merge!(NO_RENDERING)
753
978
  else
754
979
  now = @now.call
755
980
  content.merge!('status' => 'removed', 'removalReason' => 'requested', 'updatedAt' => now, 'extractedText' => nil,
756
- 'fileMetadata' => nil, 'image' => without_placeholder(content['image']))
981
+ 'metadata' => nil, **NO_RENDERING)
757
982
  item['updatedAt'] = now
758
983
  end
759
- lifecycle_event('asset.content-removed', item)
984
+ lifecycle_event('asset.content-removed', item, batch_id: batch_id)
760
985
  item
761
986
  end
762
987
 
@@ -785,16 +1010,61 @@ module CalmLens
785
1010
  fail!(ErrorCodes::ASSET_REVIEWED) unless moderation.fetch('manualReviews', []).empty?
786
1011
  fail!(ErrorCodes::ASSET_CONTENT_UNAVAILABLE) if item['content']['status'] == 'removed'
787
1012
 
788
- # Like the API: back to processing, to be moderated again under the current policy.
789
- apply_outcome(item, 'pending')
790
- moderate_stored(item, 'rerun')
1013
+ # Like the API: the request is recorded, and the asset returns as it was. It keeps its decision
1014
+ # while it's moderated again under the current policy.
1015
+ add_event('type' => 'asset.rerun-requested', 'severity' => 'low', 'assetId' => item['id'],
1016
+ 'asset' => snapshot(item), 'actor' => deep_copy(API_KEY_ACTOR),
1017
+ 'facts' => [asset_name_fact(item), { 'key' => 'asset-type', 'value' => item['type'] },
1018
+ { 'key' => 'moderation-status', 'value' => moderation['status'] }])
1019
+ response = deep_copy(item)
1020
+ settle_rerun(item, 'rerun')
1021
+ response
791
1022
  end
792
1023
 
793
- def signed_url(id, json)
794
- expires = json.fetch('expiresInSeconds', 900)
795
- params = URI.encode_www_form(json.slice('lens', 'transform').merge('expires' => expires))
796
- { 'url' => "#{@cdn_base_url}/signed/#{URI.encode_www_form_component(id)}/#{json.fetch('resource', 'rendered')}?#{params}",
797
- 'expiresAt' => @now.call + (expires * 1000) }
1024
+ # Applies a re-run's decision to a decided asset like the pipeline: `pending` stays queued for
1025
+ # `complete_processing`, `failed` keeps the asset's decision, and any other replaces it with the
1026
+ # pipeline's events.
1027
+ def settle_rerun(item, trigger, decision = nil)
1028
+ decision ||= decision_for(item, trigger, true)
1029
+ @reruns.delete(item['id'])
1030
+ if decision.status == 'pending'
1031
+ @reruns[item['id']] = true
1032
+ elsif decision.status != 'failed'
1033
+ moderate_stored(item, trigger, decision)
1034
+ end
1035
+ end
1036
+
1037
+ # Signs an asset URL like the API (`POST /projects/:projectId/signed-asset-urls`): a grant to one
1038
+ # asset, by ID or external ID, until it expires, serving the original only when asked. Only the
1039
+ # mock's CDN accepts the grant.
1040
+ def sign_asset_url(json)
1041
+ # Like the API's schema, none of these fields takes null.
1042
+ %w[assetId externalId expiresIn original].each do |key|
1043
+ invalid!('validation/invalid-type', key) if json.key?(key) && json[key].nil?
1044
+ end
1045
+ if json.key?('assetId') == json.key?('externalId')
1046
+ message = 'Send either assetId or externalId'
1047
+ raise Failure.new(ErrorCodes::INVALID_REQUEST, message,
1048
+ details: { 'issues' => [{ 'path' => '', 'message' => message, 'code' => 'custom' }] }, status: 400)
1049
+ end
1050
+ expires = json.fetch('expiresIn', CDN::DEFAULT_SIGNED_URL_EXPIRES_IN)
1051
+ invalid!('validation/invalid-type', 'expiresIn') unless expires.is_a?(Integer) || (expires.is_a?(Float) && expires == expires.floor)
1052
+ invalid!('validation/too-small', 'expiresIn') if expires < 1
1053
+ invalid!('validation/too-big', 'expiresIn') if expires > CDN::MAX_SIGNED_URL_EXPIRES_IN
1054
+ original = json.fetch('original', false)
1055
+ invalid!('validation/invalid-type', 'original') unless [true, false].include?(original)
1056
+
1057
+ item = if json.key?('externalId')
1058
+ project_assets.find { |asset| asset['externalId'] == json['externalId'] }
1059
+ else
1060
+ own_asset(json['assetId'].to_s)
1061
+ end
1062
+ fail!(ErrorCodes::ASSET_NOT_FOUND) unless item
1063
+ fail!(ErrorCodes::ASSET_HELD) if item['content']['safetyHold']
1064
+
1065
+ expires_at = @now.call + (expires.to_i * 1000)
1066
+ { 'url' => "#{@cdn_base_url}/assets/#{Internal.component(@project_id)}/#{MockGrant.encode(item['id'], original, expires_at)}",
1067
+ 'expiresAt' => expires_at }
798
1068
  end
799
1069
 
800
1070
  # In the API's order: the upload must be open, parts arrive in order, image totals and the
@@ -812,7 +1082,7 @@ module CalmLens
812
1082
  received = item['content']['sizeInBytes'].to_i + size
813
1083
  upload = @state.uploads[item['id']] || {}
814
1084
  code = image_size_error(item['type'], received, final) || total_size_error(upload['total'], received, final)
815
- fail!(code) if code
1085
+ size_failure!(code) if code
816
1086
  check_part_size(size, final)
817
1087
 
818
1088
  stored = (@state.content[item['id']] ||= { 'type' => item['content']['format'].to_s, 'parts' => {} })
@@ -825,7 +1095,8 @@ module CalmLens
825
1095
  return item unless final
826
1096
 
827
1097
  @state.uploads.delete(item['id'])
828
- queue = query.key?('queue') ? query['queue'] == true : upload.fetch('queue', false)
1098
+ # `wait=false` queues the final part; without `wait`, the upload's saved choice applies.
1099
+ queue = query.key?('wait') ? query['wait'] == false : upload.fetch('queue', false)
829
1100
  apply_outcome(item, 'pending')
830
1101
  upload_completed(item, queue)
831
1102
  end
@@ -887,29 +1158,38 @@ module CalmLens
887
1158
  touch(item.merge!('parentId' => parent_id))
888
1159
  end
889
1160
 
890
- # Policies: the project starts on its General preset policy, following automatic updates. The
891
- # mock holds no built-in preset settings: a preset's settings are `{ 'categorySettings' => {} }`
892
- # at version 1, and uploads are never decided by a policy.
893
- def new_policy(name, settings, description: '', automatic_updates: false, preset: nil)
1161
+ # Policies: the project starts on the organisation's own General preset policy, following
1162
+ # automatic updates (mode `live`). The mock's presets have one release, version 1, with
1163
+ # PRESET_SETTINGS. A stored policy is the API's shape without `projectIds`, with its preset's
1164
+ # `id`, `version` and `own`, whether it follows its preset (`followsPreset`) and its reset
1165
+ # `baseline`.
1166
+ def new_policy(name, settings, description: '', follows_preset: false, preset: nil)
894
1167
  now = @now.call
895
1168
  item = { 'id' => @create_id.call('pol'), 'name' => name, 'description' => description, 'settings' => settings,
896
- 'revision' => 1, 'automaticUpdates' => automatic_updates, 'preset' => preset,
1169
+ 'revision' => 1, 'followsPreset' => follows_preset, 'preset' => preset,
897
1170
  'baseline' => deep_copy(settings), 'createdAt' => now, 'updatedAt' => now }
898
1171
  @policies[item['id']] = item
899
1172
  end
900
1173
 
1174
+ # The project's policy, after creating the organisation's own General policy on first use.
901
1175
  def ensure_policies
902
- return if @project_policy_id
1176
+ return @project_policy_id if @project_policy_id
903
1177
 
904
- preset = { 'id' => 'general-community', 'version' => 1, 'projectId' => @project_id }
905
- @project_policy_id = new_policy(PRESET_NAMES['general-community'], { 'categorySettings' => {} },
906
- automatic_updates: true, preset: preset)['id']
1178
+ preset = { 'id' => 'general-community', 'version' => 1, 'own' => true }
1179
+ @project_policy_id = new_policy(PRESET_NAMES['general-community'], deep_copy(PRESET_SETTINGS),
1180
+ follows_preset: true, preset: preset)['id']
907
1181
  end
908
1182
 
909
- # A stored policy as the API shows it.
1183
+ # A stored policy as the API shows it: an own preset policy is `live` while it follows its
1184
+ # preset and `pinned` otherwise; any other preset policy is a `copy`.
910
1185
  def show_policy(item)
911
- shown = item.except('baseline')
912
- shown['preset'] = item['preset'].merge('latestVersion' => 1) if item['preset']
1186
+ shown = item.except('baseline', 'followsPreset')
1187
+ if (preset = item['preset'])
1188
+ mode = if preset['own'] then item['followsPreset'] ? 'live' : 'pinned'
1189
+ else 'copy'
1190
+ end
1191
+ shown['preset'] = { 'id' => preset['id'], 'version' => preset['version'], 'latestVersion' => 1, 'mode' => mode }
1192
+ end
913
1193
  shown.merge('projectIds' => item['id'] == @project_policy_id ? [@project_id] : [])
914
1194
  end
915
1195
 
@@ -918,9 +1198,39 @@ module CalmLens
918
1198
  @policies[id] || fail!(ErrorCodes::POLICY_NOT_FOUND)
919
1199
  end
920
1200
 
921
- def policy_list
1201
+ # `/policies` and `/policies/:id[/action]`, the API key's organisation's policies.
1202
+ def policies(method, parts, query, json)
1203
+ _, id, *rest = parts
1204
+ if id.nil?
1205
+ return policy_page(query) if method == 'GET'
1206
+ return create_policy(json) if method == 'POST'
1207
+
1208
+ route_not_found!(method, parts)
1209
+ end
1210
+ policy(method, id, rest, json)
1211
+ end
1212
+
1213
+ # A page of the organisation's policies, oldest first unless `sort` says otherwise.
1214
+ def policy_page(query)
922
1215
  ensure_policies
923
- { 'policies' => @policies.values.map { |item| show_policy(item) }, 'projectPolicyId' => @project_policy_id }
1216
+ page(@policies.values.map { |item| show_policy(item) }, query, JSON.generate(['policies', @project_id]),
1217
+ default: { 'field' => 'createdAt', 'direction' => 'asc' })
1218
+ end
1219
+
1220
+ # Refuses settings the API refuses (MODERATION_POLICY_SETTINGS_SCHEMA): every category, each
1221
+ # with an `action` and its `allowances` and `restrictions` lists, and no other category.
1222
+ def check_policy_settings(settings)
1223
+ categories = settings.is_a?(Hash) ? settings['categorySettings'] : nil
1224
+ invalid!('validation/invalid-type', 'settings.categorySettings') unless categories.is_a?(Hash)
1225
+ MockClassification::CATEGORIES.each do |category|
1226
+ path = "settings.categorySettings.#{category}"
1227
+ value = categories[category]
1228
+ invalid!('validation/invalid-type', path) unless value.is_a?(Hash)
1229
+ %w[allowances restrictions].each { |key| invalid!('validation/invalid-type', "#{path}.#{key}") unless value[key].is_a?(Array) }
1230
+ invalid!('validation/invalid-value', "#{path}.action") unless value['action'].is_a?(String)
1231
+ end
1232
+ unknown = categories.keys.find { |key| !MockClassification::CATEGORIES.include?(key) }
1233
+ invalid!('validation/unknown-field', "settings.categorySettings.#{unknown}") if unknown
924
1234
  end
925
1235
 
926
1236
  def create_policy(json)
@@ -930,10 +1240,11 @@ module CalmLens
930
1240
  if settings.nil? == preset_id.nil? || (preset_id && !PRESET_NAMES.key?(preset_id)) || (preset_id.nil? && !named)
931
1241
  fail!(ErrorCodes::INVALID_VALUE, 'Send settings with a name, or presetId')
932
1242
  end
1243
+ check_policy_settings(settings) unless preset_id
933
1244
  description = json['description'].is_a?(String) ? json['description'] : ''
934
1245
  item = if preset_id
935
- origin = { 'id' => preset_id, 'version' => 1, 'projectId' => nil }
936
- new_policy(named ? name.strip : PRESET_NAMES[preset_id], { 'categorySettings' => {} },
1246
+ origin = { 'id' => preset_id, 'version' => 1, 'own' => false }
1247
+ new_policy(named ? name.strip : PRESET_NAMES[preset_id], deep_copy(PRESET_SETTINGS),
937
1248
  description: description, preset: origin)
938
1249
  else
939
1250
  new_policy(name.strip, settings, description: description)
@@ -965,26 +1276,29 @@ module CalmLens
965
1276
  fail!(ErrorCodes::POLICY_IN_USE) if item['id'] == @project_policy_id
966
1277
  @policies.delete(id)
967
1278
  {}
968
- in ['POST', ['preset']] then apply_policy_preset(item, json)
1279
+ in ['POST', ['reset']] then restore_policy(item, 'reset', json)
1280
+ in ['POST', ['upgrade']] then restore_policy(item, 'upgrade', json)
969
1281
  in ['PUT', ['automatic-updates']] then set_policy_automatic_updates(item, json)
970
1282
  else route_not_found!(method, ['policies', id, *rest])
971
1283
  end
972
1284
  end
973
1285
 
974
1286
  def update_policy(item, json)
1287
+ check_policy_settings(json['settings']) if json.key?('settings')
975
1288
  values = %w[name description].select { |key| json[key].is_a?(String) }
976
1289
  .to_h { |key| [key, key == 'name' ? json[key].strip : json[key]] }
977
1290
  values['settings'] = json['settings'] if json.key?('settings')
978
- # Saving a policy that follows its preset pins it as a copy.
979
- save_policy(item, json['revision'], values.merge('automaticUpdates' => false))
1291
+ # Saving a `live` policy pins it.
1292
+ save_policy(item, json['revision'], values.merge('followsPreset' => false))
980
1293
  end
981
1294
 
982
- def apply_policy_preset(item, json)
1295
+ # Resets a `pinned` or `copy` policy to its preset release, or upgrades it to the latest.
1296
+ def restore_policy(item, operation, json)
983
1297
  preset = item['preset']
984
- fail!(ErrorCodes::POLICY_NO_PRESET) if preset.nil? || item['automaticUpdates']
1298
+ fail!(ErrorCodes::POLICY_NO_PRESET) if preset.nil? || item['followsPreset']
985
1299
  fail!(ErrorCodes::POLICY_PRESET_VERSION_CONFLICT) unless json['presetVersion'] == preset['version']
986
- # The mock's presets have one release, so an update never has a newer one to apply.
987
- fail!(ErrorCodes::POLICY_PRESET_UPDATE_UNAVAILABLE) if json['operation'] == 'update'
1300
+ # The mock's presets have one release, so an upgrade never has a newer one to apply.
1301
+ fail!(ErrorCodes::POLICY_PRESET_UPDATE_UNAVAILABLE) if operation == 'upgrade'
988
1302
 
989
1303
  save_policy(item, json['revision'], 'settings' => deep_copy(item['baseline']))
990
1304
  end
@@ -992,13 +1306,11 @@ module CalmLens
992
1306
  def set_policy_automatic_updates(item, json)
993
1307
  enabled = json['enabled'] == true
994
1308
  preset = item['preset']
995
- if preset.nil? || preset['projectId'] != @project_id || item['automaticUpdates'] == enabled
996
- fail!(ErrorCodes::POLICY_AUTOMATIC_UPDATES_UNAVAILABLE)
997
- end
1309
+ fail!(ErrorCodes::POLICY_AUTOMATIC_UPDATES_UNAVAILABLE) if preset.nil? || !preset['own'] || item['followsPreset'] == enabled
998
1310
  fail!(ErrorCodes::POLICY_REVISION_CONFLICT) unless json['revision'] == item['revision']
999
1311
  fail!(ErrorCodes::POLICY_PRESET_VERSION_CONFLICT) unless json['presetVersion'] == 1
1000
1312
 
1001
- values = { 'automaticUpdates' => enabled }
1313
+ values = { 'followsPreset' => enabled }
1002
1314
  values.merge!('name' => PRESET_NAMES[preset['id']], 'description' => '', 'settings' => deep_copy(item['baseline'])) if enabled
1003
1315
  save_policy(item, json['revision'], values)
1004
1316
  end
@@ -1060,10 +1372,10 @@ module CalmLens
1060
1372
  if asset_id
1061
1373
  asset = @state.assets[asset_id]
1062
1374
  fail!(ErrorCodes::ASSET_NOT_FOUND) unless asset && asset['projectId'] == @project_id
1063
- image = asset.dig('content', 'image')
1064
- fail!(ErrorCodes::HASH_LIST_ASSET_NOT_HASHED) unless image.is_a?(Hash) && image['perceptualHash']
1065
- hash = image['perceptualHash']
1066
- quality = image['perceptualHashQuality']
1375
+ content = asset['content']
1376
+ fail!(ErrorCodes::HASH_LIST_ASSET_NOT_HASHED) unless content['perceptualHash']
1377
+ hash = content['perceptualHash']
1378
+ quality = content['perceptualHashQuality']
1067
1379
  end
1068
1380
  fail!(ErrorCodes::INVALID_UPLOAD_PAYLOAD, 'Use a PDQ hash of 64 hexadecimal characters') unless hash.match?(/\A\h{64}\z/)
1069
1381
 
@@ -1072,7 +1384,7 @@ module CalmLens
1072
1384
 
1073
1385
  entry = { 'id' => @create_id.call('hle'), 'projectId' => @project_id, 'hashListId' => list['id'], 'hash' => hash,
1074
1386
  'quality' => quality, 'label' => json['label'].is_a?(String) ? json['label'] : nil,
1075
- 'sourceAssetId' => asset_id, 'createdAt' => @now.call }
1387
+ 'assetId' => asset_id, 'createdAt' => @now.call }
1076
1388
  @state.hash_list_entries[entry['id']] = entry
1077
1389
  respond(201, deep_copy(entry))
1078
1390
  end
@@ -1086,17 +1398,43 @@ module CalmLens
1086
1398
  {}
1087
1399
  end
1088
1400
 
1089
- # One keyset page like the API's: ordered by `fields` (`createdAt` descending by default; an `id`
1090
- # tie-breaker is added), starting after the cursor's values. The cursor is tied to `scope` (the
1091
- # filter, folder or parent) and the ordering, so it only continues the query it came from.
1092
- def page(items, query, scope, fields: %w[createdAt id], default: { 'field' => 'createdAt', 'direction' => 'desc' })
1401
+ # An asset's findings timeline, as the API lists it (publicGetAssetTimeline.ts): refused while a
1402
+ # safety hold is on. The mock analyses no audio or video, so it keeps no segments: every
1403
+ # timeline is empty, and no cursor continues one.
1404
+ def timeline(id, query)
1405
+ item = own_asset(id)
1406
+ fail!(ErrorCodes::ASSET_NOT_FOUND) unless item
1407
+ check_range(query.fetch('pageSize', DEFAULT_PAGE_SIZE), 'pageSize', MAX_PAGE_SIZE)
1408
+ fail!(ErrorCodes::ASSET_HELD, details: { 'hold' => 'safety' }) if item['content']['safetyHold']
1409
+ fail!(ErrorCodes::INVALID_CURSOR) if query.key?('cursor')
1410
+
1411
+ { 'items' => [], 'nextCursor' => nil, 'hasMore' => false }
1412
+ end
1413
+
1414
+ # The page size and sort, refused like the API's query schema: `sort` is comma-separated
1415
+ # fields from `fields`, each descending when prefixed with `-`, none repeated and at most `max`
1416
+ # (`validation/invalid-value` at `sort` otherwise). Without one, `default` applies.
1417
+ def page_query(query, fields = %w[createdAt id], default = { 'field' => 'createdAt', 'direction' => 'desc' }, max: 2)
1093
1418
  size = check_range(query.fetch('pageSize', DEFAULT_PAGE_SIZE), 'pageSize', MAX_PAGE_SIZE)
1094
- ordering = ordering(query, default)
1095
- # Like the API's query schema: each entry's field and direction are enum values.
1096
- ordering.each_with_index do |rule, index|
1097
- invalid!('validation/invalid-value', "ordering.#{index}.field") unless fields.include?(rule['field'])
1098
- invalid!('validation/invalid-value', "ordering.#{index}.direction") unless %w[asc desc].include?(rule['direction'])
1419
+ [size, query.key?('sort') ? parse_sort(query['sort'], fields, max) : [default.dup]]
1420
+ end
1421
+
1422
+ def parse_sort(value, fields, max)
1423
+ sort = value.to_s.split(',', -1).map(&:strip).map do |part|
1424
+ field = part.delete_prefix('-')
1425
+ invalid!('validation/invalid-value', 'sort') unless part.match?(/\A-?[A-Za-z][A-Za-z0-9.]*\z/) && fields.include?(field)
1426
+ { 'field' => field, 'direction' => part.start_with?('-') ? 'desc' : 'asc' }
1099
1427
  end
1428
+ fields_sent = sort.map { |rule| rule['field'] }
1429
+ invalid!('validation/invalid-value', 'sort') if sort.empty? || fields_sent.uniq.size != fields_sent.size || sort.size > max
1430
+ sort
1431
+ end
1432
+
1433
+ # One keyset page like the API's: sorted by `sort` (`createdAt` descending by default; an `id`
1434
+ # tie-breaker is added), starting after the cursor's values. The cursor is tied to `scope` (the
1435
+ # filter, folder or parent) and the sort, so it only continues the query it came from.
1436
+ def page(items, query, scope, fields: %w[createdAt id], max: 2, default: { 'field' => 'createdAt', 'direction' => 'desc' })
1437
+ size, ordering = page_query(query, fields, default, max: max)
1100
1438
  ordering << { 'field' => 'id', 'direction' => ordering.first['direction'] } unless ordering.any? { |o| o['field'] == 'id' }
1101
1439
  signature = JSON.generate([scope, ordering])
1102
1440
  sorted = items.sort { |a, b| compare_items(a, b, ordering) }
@@ -1106,12 +1444,8 @@ module CalmLens
1106
1444
  end
1107
1445
  selected = sorted.first(size)
1108
1446
  more = sorted.size > size
1109
- cursor = if more
1110
- [JSON.generate({ 'scope' => signature, 'values' => ordering.map do |o|
1111
- selected.last[o['field']]
1112
- end })].pack('m0')
1113
- end
1114
- { 'items' => selected, 'pageSize' => size, 'hasMore' => more, 'ordering' => ordering, 'nextCursor' => cursor }
1447
+ cursor = ([JSON.generate({ 'scope' => signature, 'values' => ordering.map { |o| selected.last[o['field']] } })].pack('m0') if more)
1448
+ { 'items' => selected, 'hasMore' => more, 'nextCursor' => cursor }
1115
1449
  end
1116
1450
 
1117
1451
  def cursor_values(cursor, signature, ordering)
@@ -1134,15 +1468,6 @@ module CalmLens
1134
1468
  false
1135
1469
  end
1136
1470
 
1137
- def ordering(query, default)
1138
- entries = query.keys.filter_map { |key| key.match(/\Aordering\[(\d+)\]\[(field|direction)\]\z/) }
1139
- return [default&.dup].compact if entries.empty?
1140
-
1141
- entries.group_by { |match| Integer(match[1], 10) }.sort.map do |_, matches|
1142
- matches.to_h { |match| [match[2], query[match[0]]] }
1143
- end
1144
- end
1145
-
1146
1471
  def compare_items(left, right, ordering)
1147
1472
  ordering.each do |rule|
1148
1473
  result = compare_values(left[rule['field']], right[rule['field']])
@@ -1161,62 +1486,200 @@ module CalmLens
1161
1486
  left.to_s <=> right.to_s
1162
1487
  end
1163
1488
 
1164
- # Answers like the CDN: a placeholder is the asset's ThumbHash as a PNG, public whatever
1165
- # the asset's visibility or moderation; other resources pass delivery checks first, which refuse
1166
- # everyone (content still uploading, blocked or purged, failed processing, removed content),
1167
- # then an age-restricted asset needs `viewerClaims.ageConfirmed=true` whatever the key, and a
1168
- # private asset needs an API key with `assets:download` for its folder.
1169
- def serve(method, relative)
1489
+ # Answers like the CDN (serveAssetResource.ts): `original` is the upload; `rendered` the image,
1490
+ # PDF, video or audio file `content.rendered` describes; `preview` the still `content.preview`
1491
+ # describes; `hls` the adaptive stream's files, which the mock never prepares; `embed` a small
1492
+ # page standing in for the embed page of any asset; and `placeholder` the preview's ThumbHash as a PNG, public
1493
+ # whatever the asset's visibility or moderation. A safety hold refuses every resource. Other resources pass delivery checks first,
1494
+ # which refuse everyone (content still uploading, blocked or purged, failed processing, removed
1495
+ # content), then an age-restricted asset needs `viewerClaims.ageConfirmed=true` whatever the key,
1496
+ # and a private asset needs an API key with `assets:download` for its folder. The mock converts
1497
+ # nothing: an upload stands in for its rendered file, an image for its own preview, and the
1498
+ # placeholder PNG for a screenshot, page or waveform.
1499
+ def serve(method, relative, headers)
1170
1500
  path, query = relative.split('#', 2).first.split('?', 2)
1171
- segments = path.split('/').reject(&:empty?).map { |part| URI.decode_www_form_component(part) }
1172
- _, project, identifier, resource = segments
1173
- item = project == @project_id && project_assets.find { |asset| [asset['id'], asset['externalId']].include?(identifier) }
1174
- return error('asset/not-found', 'Asset not found', status: 404) unless item && RESOURCES.include?(resource)
1501
+ raw_segments = path.split('/').reject(&:empty?)
1502
+ root, project, _, resource = raw_segments.map { |part| URI.decode_www_form_component(part) }
1503
+ selected, options = selector(resource, raw_segments.drop(4))
1504
+ return invalid_transform if selected.nil?
1505
+ return asset_not_found unless root == 'assets' && project == @project_id && raw_segments[2]
1506
+
1507
+ grant, item = resolve_asset(raw_segments[2])
1508
+ return grant if grant.is_a?(Response)
1509
+ return asset_not_found unless item && RESOURCES.include?(resource)
1510
+ if resource == 'original' && grant && !grant[:original]
1511
+ return error(ErrorCodes::SIGNED_URL_ORIGINAL_NOT_GRANTED, "This signed asset URL doesn't grant the original file", status: 403)
1512
+ end
1175
1513
 
1514
+ params = URI.decode_www_form(query.to_s).to_h
1515
+ # A grant authorises a private asset without the API key.
1516
+ authorised = !grant.nil? || can_download?(item['folderId'], headers)
1517
+ return info(method, item, authorised, params) if resource == 'info'
1518
+ return not_available if item['content']['safetyHold']
1176
1519
  return placeholder(method, item) if resource == 'placeholder'
1177
1520
 
1521
+ deliver(method, item, resource, params, selected ? options || {} : nil, authorised)
1522
+ end
1523
+
1524
+ # Whether the path after the resource selects a lens or transform, and its options; nil when the
1525
+ # CDN refuses it. Like the CDN, the adaptive stream's files sit beneath `/hls/` (its master
1526
+ # playlist is `/hls/index.m3u8`). Otherwise the selector after the resource is read
1527
+ # (AssetTransformParser.ts); the original, the embed page and the info take none.
1528
+ def selector(resource, path)
1529
+ lens, options = TransformRules.parse_selector(resource == 'hls' ? '' : path.join('/'))
1530
+ selected = !(lens || options).nil?
1531
+ %w[original embed info].include?(resource) && selected ? nil : [selected, options]
1532
+ rescue ArgumentError
1533
+ nil
1534
+ end
1535
+
1536
+ # Like the CDN (CdnUtils.ts), the asset segment is `ext:` and the URL-encoded external ID, a
1537
+ # grant (`sig_…`) or the asset ID. Returns `[grant, asset]`, or an error response for a grant
1538
+ # the mock didn't sign or that has expired.
1539
+ def resolve_asset(segment)
1540
+ if segment.start_with?('sig_')
1541
+ grant = MockGrant.decode(segment)
1542
+ return error(ErrorCodes::SIGNED_URL_INVALID, "The signed asset URL's grant is invalid", status: 403) unless grant
1543
+ return error(ErrorCodes::SIGNED_URL_EXPIRED, 'The signed URL has expired', status: 403) if grant[:expires_at] <= @now.call
1544
+
1545
+ [grant, project_assets.find { |asset| asset['id'] == grant[:asset_id] }]
1546
+ elsif segment.start_with?(CDN::EXTERNAL_ID_PREFIX)
1547
+ external = MockGrant.percent_decode(segment.delete_prefix(CDN::EXTERNAL_ID_PREFIX))
1548
+ [nil, external && !external.empty? ? project_assets.find { |asset| asset['externalId'] == external } : nil]
1549
+ else
1550
+ id = MockGrant.percent_decode(segment)
1551
+ [nil, project_assets.find { |asset| asset['id'] == id }]
1552
+ end
1553
+ end
1554
+
1555
+ def asset_not_found = error(ErrorCodes::ASSET_NOT_FOUND, 'Asset not found', status: 404)
1556
+
1557
+ # The display info at `/info`, like the CDN's (AssetInfo.ts): the allowlisted fields and whether
1558
+ # the viewer the claims describe can see the content, decided as the other resources decide it.
1559
+ # A private asset's info needs a grant or an API key with `assets:download`.
1560
+ def info(method, item, authorised, params)
1561
+ restricted = item['visibility'] != 'public'
1562
+ return error('asset/not-available', 'Asset is not available', status: 404) if restricted && !authorised
1563
+
1564
+ claims = JSON.parse(params.fetch('viewerClaims', '{}'))
1565
+ body = { 'id' => item['id'], 'projectId' => item['projectId'], 'type' => item['type'], 'externalUrl' => item['externalUrl'],
1566
+ 'content' => item['content'].slice(*%w[format width height durationSeconds pageCount rendered preview hls]),
1567
+ 'availability' => availability(item, claims.is_a?(Hash) && claims['ageConfirmed'] == true) }
1568
+ cdn_response(method, JSON.generate(body), { 'content-type' => 'application/json',
1569
+ 'cache-control' => "#{restricted ? 'private' : 'public'}, max-age=60",
1570
+ 'cache-tag' => "asset-#{item['id']}" })
1571
+ rescue JSON::ParserError
1572
+ error(ErrorCodes::INVALID_VIEWER_CLAIMS, 'Invalid viewer claims', status: 400)
1573
+ end
1574
+
1575
+ # Whether a viewer can see an asset's content now, decided as the other resources decide it.
1576
+ def availability(item, age_confirmed)
1178
1577
  moderation = item['classification']['moderation']['status']
1179
- stored = @state.content[item['id']]
1180
- if stored.nil? || item['upload']['status'] == 'uploading' || %w[blocked purged].include?(moderation) ||
1578
+ if item['content']['safetyHold'] || item['upload']['status'] == 'uploading' || %w[blocked purged].include?(moderation) ||
1181
1579
  item['classification']['status'] == 'failed' || item['content']['status'] == 'removed'
1182
- return not_available
1580
+ 'unavailable'
1581
+ elsif moderation == 'age-restricted' && !age_confirmed
1582
+ 'age-restricted'
1583
+ else
1584
+ 'available'
1183
1585
  end
1586
+ end
1184
1587
 
1185
- params = URI.decode_www_form(query.to_s).to_h
1186
- opted_in = viewer_age_confirmed?(params)
1187
- if moderation == 'age-restricted' && !opted_in
1188
- return error('viewer/age-confirmation-required', 'Age confirmation required',
1189
- status: 403)
1190
- end
1588
+ # Serves a found asset's resource after the delivery, age and access checks, then whether it
1589
+ # has the resource and takes the selector, then the transform options' schema check.
1590
+ # `selector` is the selector's options (empty for a lens alone), or nil without one.
1591
+ def deliver(method, item, resource, params, selector, authorised)
1592
+ stored = @state.content[item['id']]
1593
+ refused = refuse_delivery(item, resource, stored, params, authorised)
1594
+ return refused if refused
1191
1595
 
1192
1596
  restricted = item['visibility'] != 'public'
1193
- if restricted && !can_download?(item['folderId'])
1194
- return error('asset/api-key-missing-permission',
1195
- 'This API key needs assets:download permission for this asset’s folder.', status: 403)
1597
+ return embed(method, item, restricted, params) if resource == 'embed'
1598
+
1599
+ # Then whether the asset has the resource, and whether it is a still image that takes a lens
1600
+ # or transform: the preview always is, the rendered file only when it is an image.
1601
+ rendered, preview = item['content'].values_at('rendered', 'preview')
1602
+ return no_rendered_version if (resource == 'rendered' && !rendered) || (resource == 'preview' && !preview)
1603
+ # The mock prepares no stream, so it has no HLS files.
1604
+ return no_rendered_version if resource == 'hls'
1605
+
1606
+ still = resource == 'preview' || (resource == 'rendered' && rendered['kind'] == 'image')
1607
+ return invalid_transform if !still && selector
1608
+
1609
+ refused = selector ? TransformRules.check(selector).flatten : []
1610
+ return error(ErrorCodes::INVALID_TRANSFORM_OPTIONS, "Invalid transform options: #{refused.join(', ')}", status: 400) if refused.any?
1611
+
1612
+ bytes, type = resource_body(item, resource, stored)
1613
+ return not_available unless bytes
1614
+
1615
+ immutable = still || (resource == 'original' && item['type'] == 'image')
1616
+ cdn_response(method, bytes, viewer_delivery_headers(item, type, restricted, immutable, params))
1617
+ end
1618
+
1619
+ # The delivery checks, which refuse everyone, then age and access, as an error response, or nil.
1620
+ # Only the original needs the upload still stored.
1621
+ def refuse_delivery(item, resource, stored, params, authorised)
1622
+ moderation = item['classification']['moderation']['status']
1623
+ if (resource == 'original' && stored.nil?) || item['upload']['status'] == 'uploading' ||
1624
+ %w[blocked purged].include?(moderation) || item['classification']['status'] == 'failed' ||
1625
+ item['content']['status'] == 'removed'
1626
+ return not_available
1627
+ end
1628
+ if moderation == 'age-restricted' && !viewer_age_confirmed?(params)
1629
+ return error('viewer/age-confirmation-required', 'Age confirmation required', status: 403)
1196
1630
  end
1631
+ return if item['visibility'] == 'public' || authorised
1197
1632
 
1198
- type = stored['type'].to_s.empty? ? 'application/octet-stream' : stored['type']
1199
- cdn_response(method, stored['parts'].sort.map(&:last).join.b,
1200
- viewer_delivery_headers(item, type, restricted, params))
1633
+ error(ErrorCodes::PERMISSION_DENIED, 'This API key needs assets:download permission for this asset’s folder.', status: 403)
1634
+ end
1635
+
1636
+ # A resource's bytes and Content-Type: the upload, which stands in for its own rendered file
1637
+ # and an image's preview, or the placeholder PNG for a still the mock doesn't make. Nil bytes
1638
+ # when the upload is no longer stored.
1639
+ def resource_body(item, resource, stored)
1640
+ rendered = item['content']['rendered']
1641
+ serves_upload = resource == 'original' || (resource == 'rendered' && rendered['kind'] != 'image') || item['type'] == 'image'
1642
+ return [Placeholder.png(item.dig('content', 'preview', 'thumbhash') || THUMBHASH), 'image/png'] unless serves_upload
1643
+ return [nil, nil] unless stored
1644
+
1645
+ type = if resource == 'rendered' then rendered['format']
1646
+ elsif stored['type'].to_s.empty? then 'application/octet-stream'
1647
+ else stored['type']
1648
+ end
1649
+ [stored['parts'].sort.map(&:last).join.b, type]
1201
1650
  end
1202
1651
 
1203
1652
  def viewer_age_confirmed?(params)
1204
1653
  JSON.parse(params.fetch('viewerClaims', '{}'))['ageConfirmed'] == true
1205
1654
  end
1206
1655
 
1207
- def viewer_delivery_headers(item, type, restricted, params)
1208
- headers = delivery_headers(item, type, restricted)
1656
+ def viewer_delivery_headers(item, type, restricted, immutable, params)
1657
+ headers = delivery_headers(item, type, restricted, immutable)
1209
1658
  headers['Cache-Control'] = 'private, no-store' if params.key?('viewerClaims')
1210
1659
  headers
1211
1660
  end
1212
1661
 
1662
+ # The embed page, which shows any asset that may be delivered, as its viewer for the asset's
1663
+ # type or a notice that the type has none; cached like other non-image content.
1664
+ def embed(method, item, restricted, params)
1665
+ cache = if params.key?('viewerClaims') then 'private, no-store'
1666
+ else "#{restricted ? 'private' : 'public'}, max-age=3600"
1667
+ end
1668
+ cdn_response(method, EMBED_HTML, { 'content-type' => 'text/html; charset=utf-8', 'cache-control' => cache,
1669
+ 'cache-tag' => "asset-#{item['id']}", 'x-content-type-options' => 'nosniff' })
1670
+ end
1671
+
1672
+ def invalid_transform = error(ErrorCodes::INVALID_TRANSFORM, 'Invalid transform selector', status: 400)
1673
+
1674
+ def no_rendered_version = error('asset/no-rendered-version', 'This asset has no such resource', status: 404)
1675
+
1213
1676
  def not_available = error('asset/not-available', 'Asset is not available', status: 403)
1214
1677
 
1215
- # Like the CDN's placeholder handler: an image's ThumbHash, public whatever the asset's
1678
+ # Like the CDN's placeholder handler: the preview's ThumbHash, public whatever the asset's
1216
1679
  # visibility or moderation; refused for purged assets and assets without one (not yet
1217
- # classified, or content removed).
1680
+ # processed, without a preview, or content removed).
1218
1681
  def placeholder(method, item)
1219
- thumbhash = item.dig('content', 'image', 'thumbhash')
1682
+ thumbhash = item.dig('content', 'preview', 'thumbhash')
1220
1683
  return not_available if thumbhash.nil? || item['classification']['moderation']['status'] == 'purged'
1221
1684
 
1222
1685
  cdn_response(method, Placeholder.png(thumbhash),
@@ -1224,14 +1687,13 @@ module CalmLens
1224
1687
  'cache-tag' => "asset-#{item['id']}", 'etag' => "\"#{thumbhash}\"")
1225
1688
  end
1226
1689
 
1227
- # Like the CDN: an image's bytes are immutable for a year, other content is kept for an hour,
1228
- # by shared caches only when delivery is public. Active content is served as a download.
1229
- def delivery_headers(item, type, restricted)
1230
- image = item['type'] == 'image'
1231
- headers = { 'content-type' => type,
1232
- 'cache-control' => "#{restricted ? 'private' : 'public'}, #{image ? 'max-age=31536000, immutable' : 'max-age=3600'}",
1690
+ # Like the CDN: a still image's bytes are immutable for a year, other content is kept for an
1691
+ # hour, by shared caches only when delivery is public. Active content is served as a download.
1692
+ def delivery_headers(item, type, restricted, immutable)
1693
+ lifetime = immutable ? 'max-age=31536000, immutable' : 'max-age=3600'
1694
+ headers = { 'content-type' => type, 'cache-control' => "#{restricted ? 'private' : 'public'}, #{lifetime}",
1233
1695
  'cache-tag' => "asset-#{item['id']}" }
1234
- headers.merge!('etag' => "\"#{item['id']}\"", 'expires' => Time.at((@now.call / 1000) + 31_536_000).utc.httpdate) if image
1696
+ headers.merge!('etag' => "\"#{item['id']}\"", 'expires' => Time.at((@now.call / 1000) + 31_536_000).utc.httpdate) if immutable
1235
1697
  headers.merge!('x-content-type-options' => 'nosniff', 'content-security-policy' => "sandbox; default-src 'none'")
1236
1698
  mime = type.downcase.split(';', 2).first.to_s.strip
1237
1699
  headers['content-disposition'] = 'attachment' if ACTIVE_TYPES.include?(mime) || mime.end_with?('+xml')
@@ -1243,8 +1705,11 @@ module CalmLens
1243
1705
  headers: headers.merge('content-length' => bytes.bytesize.to_s, 'x-request-id' => request_id))
1244
1706
  end
1245
1707
 
1246
- # Whether the mock API key's grants give `assets:download` for `folder_id`.
1247
- def can_download?(folder_id)
1708
+ # Whether the request carries the mock API key and its grants give `assets:download` for
1709
+ # `folder_id`.
1710
+ def can_download?(folder_id, headers)
1711
+ return false unless header(headers, 'X-Api-Key') == @api_key
1712
+
1248
1713
  @grants.any? do |grant|
1249
1714
  next false unless Array(grant['actions']).include?('assets:download')
1250
1715
  next true if grant['folderId'].nil?
@@ -1267,6 +1732,35 @@ module CalmLens
1267
1732
  end
1268
1733
  end
1269
1734
 
1735
+ # The mock's grant, the asset segment of a URL {CDN#sign_asset_url} signs on the mock: `sig_` and
1736
+ # the lowercase hex of the UTF-8 text `{assetId}.{1|0}.{expiresAt}`, its asset, original access and
1737
+ # expiry. Every SDK's mock writes it the same way. It is readable, not signed, as the mock has no
1738
+ # secret to keep.
1739
+ # @api private
1740
+ module MockGrant
1741
+ module_function
1742
+
1743
+ def encode(asset_id, original, expires_at)
1744
+ "sig_#{"#{asset_id}.#{original ? 1 : 0}.#{expires_at}".unpack1('H*')}"
1745
+ end
1746
+
1747
+ # `{asset_id:, original:, expires_at:}`, or nil for a segment the mock didn't sign.
1748
+ def decode(segment)
1749
+ hex = segment.delete_prefix('sig_')
1750
+ return unless hex.match?(/\A(?:[0-9a-f]{2})+\z/)
1751
+
1752
+ text = [hex].pack('H*').force_encoding(Encoding::UTF_8)
1753
+ match = text.valid_encoding? && /\A(.+)\.([01])\.(\d+)\z/m.match(text)
1754
+ match && { asset_id: match[1], original: match[2] == '1', expires_at: Integer(match[3], 10) }
1755
+ end
1756
+
1757
+ # Percent-decodes a path segment like decodeURIComponent; nil when it isn't valid UTF-8.
1758
+ def percent_decode(text)
1759
+ decoded = text.b.gsub(/%(\h\h)/) { Regexp.last_match(1).hex.chr }.force_encoding(Encoding::UTF_8)
1760
+ decoded if decoded.valid_encoding?
1761
+ end
1762
+ end
1763
+
1270
1764
  # The processing and moderation status of each outcome.
1271
1765
  # @api private
1272
1766
  module AssetRecord
@@ -1285,125 +1779,5 @@ module CalmLens
1285
1779
  end
1286
1780
  end
1287
1781
  end
1288
-
1289
- # The RSQL subset the fake API understands: `==` and `!=` (with `*` wildcards), `=in=` and
1290
- # `=out=` with a `(value,value)` list, and `=gt=`, `=ge=`, `=lt=`, `=le=`, with quoted values,
1291
- # combined with `;` (and), which binds tighter than `,` (or), and grouped with `( … )`.
1292
- # Anything else raises instead of being ignored, so a test never passes on an unfiltered list.
1293
- # @api private
1294
- module RSQL
1295
- module_function
1296
-
1297
- def filter(items, expression)
1298
- return items if expression.nil? || expression.to_s.strip.empty?
1299
-
1300
- test = parse(expression.to_s)
1301
- items.select { |item| test.call(item) }
1302
- end
1303
-
1304
- # Compiles `,` (or) of `;` (and) terms, where a term is a comparison or a `( … )` group.
1305
- def parse(expression)
1306
- alternatives = split(expression, ',').map do |alternative|
1307
- split(alternative, ';').map do |term|
1308
- text = term.strip
1309
- raise unsupported(expression) if text.empty?
1310
-
1311
- group = unwrap(text)
1312
- group ? parse(group) : comparison(text)
1313
- end
1314
- end
1315
- ->(item) { alternatives.any? { |terms| terms.all? { |test| test.call(item) } } }
1316
- end
1317
-
1318
- # The expression inside `( … )` when the parentheses enclose all of `text`, else nil.
1319
- def unwrap(text)
1320
- return unless text.start_with?('(') && text.end_with?(')')
1321
-
1322
- depth = 0
1323
- quote = nil
1324
- text.each_char.with_index do |char, index|
1325
- if quote
1326
- quote = nil if char == quote
1327
- elsif ['"', "'"].include?(char)
1328
- quote = char
1329
- elsif char == '('
1330
- depth += 1
1331
- elsif char == ')'
1332
- depth -= 1
1333
- return index == text.length - 1 ? text[1..-2] : nil if depth.zero?
1334
- end
1335
- end
1336
- nil
1337
- end
1338
-
1339
- def unsupported(term)
1340
- InvalidArgumentError.new(SdkErrorCodes::INVALID_ARGUMENT,
1341
- 'MockServerClient filters support ==, !=, =in=, =out=, =gt=, =ge=, =lt= and ' \
1342
- "=le=, joined by ; and , and grouped with parentheses (got #{term.inspect})")
1343
- end
1344
-
1345
- # Splits on `separator` outside quotes and parentheses (groups and `=in=(...)` lists).
1346
- def split(text, separator)
1347
- parts = ['']
1348
- quote = nil
1349
- depth = 0
1350
- text.each_char do |char|
1351
- if quote
1352
- quote = nil if char == quote
1353
- elsif ['"', "'"].include?(char)
1354
- quote = char
1355
- elsif char == '('
1356
- depth += 1
1357
- elsif char == ')'
1358
- depth -= 1
1359
- elsif char == separator && depth.zero?
1360
- parts << ''
1361
- next
1362
- end
1363
- parts[-1] += char
1364
- end
1365
- parts
1366
- end
1367
-
1368
- def comparison(term)
1369
- match = term.strip.match(/\A([A-Za-z][\w.]*)(==|!=|=in=|=out=|=gt=|=ge=|=lt=|=le=)(.+)\z/m)
1370
- raise unsupported(term) unless match
1371
-
1372
- field, operator, raw = match.captures
1373
- values = list(operator, raw.strip)
1374
- lambda do |item|
1375
- actual = field.split('.').reduce(item) { |object, key| object.is_a?(Hash) ? object[key] : nil }
1376
- entries = Array(actual.nil? ? 'null' : actual).map(&:to_s)
1377
- case operator
1378
- when '==', '=in=' then values.any? { |value| matches?(entries, value) }
1379
- when '!=', '=out=' then values.none? { |value| matches?(entries, value) }
1380
- else entries.any? { |entry| ordered?(entry, operator, values.first) }
1381
- end
1382
- end
1383
- end
1384
-
1385
- def list(operator, raw)
1386
- return [unquote(raw)] unless %w[=in= =out=].include?(operator)
1387
-
1388
- unless raw.start_with?('(') && raw.end_with?(')')
1389
- raise InvalidArgumentError.new(SdkErrorCodes::INVALID_ARGUMENT, "#{operator} takes a (value,value) list")
1390
- end
1391
-
1392
- split(raw[1..-2], ',').map { |value| unquote(value.strip) }
1393
- end
1394
-
1395
- def unquote(value) = value.match?(/\A(["']).*\1\z/m) ? value[1..-2] : value
1396
-
1397
- def matches?(entries, value)
1398
- pattern = Regexp.new("\\A#{Regexp.escape(value).gsub('\*', '.*')}\\z", Regexp::MULTILINE)
1399
- entries.any? { |entry| pattern.match?(entry) }
1400
- end
1401
-
1402
- # Numbers compare as numbers and everything else (such as ISO dates) as text.
1403
- def ordered?(entry, operator, value)
1404
- left, right = [entry, value].all? { |text| text.match?(/\A-?\d+(\.\d+)?\z/) } ? [entry.to_f, value.to_f] : [entry, value]
1405
- { '=gt=' => left > right, '=ge=' => left >= right, '=lt=' => left < right, '=le=' => left <= right }.fetch(operator)
1406
- end
1407
- end
1408
1782
  end
1409
1783
  end