calmlens 0.20.0 → 0.21.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -6,6 +6,7 @@ require 'securerandom'
6
6
  require 'uri'
7
7
  require_relative 'placeholder'
8
8
  require_relative 'mock_decision'
9
+ require_relative 'rsql'
9
10
 
10
11
  module CalmLens
11
12
  module Testing
@@ -36,35 +37,69 @@ module CalmLens
36
37
  :hash_list_entries, keyword_init: true)
37
38
 
38
39
  # A stateful stand-in for the CalmLens API and CDN. It stores assets and folders, applies
39
- # filters and ordering, serves stored bytes, and answers errors with the API's codes, statuses,
40
+ # filters and sorting, serves stored bytes, and answers errors with the API's codes, statuses,
40
41
  # details and body shape, checking requests in the API's order. It moderates through the
41
42
  # `moderate` callback rather than classifying content, and does not run transforms or index for
42
43
  # semantic search.
43
44
  # @api private
44
45
  class FakeAPI
45
- JSON_QUERY = %w[queue isFinal sizeInBytes totalSizeInBytes partNumber pageSize limit processingOptions metadata
46
- context].freeze
46
+ JSON_QUERY = %w[wait isFinal sizeInBytes totalSizeInBytes partNumber pageSize processingOptions metadata
47
+ tags context].freeze
47
48
  # Keyword search fields, in the order matches are reported, and where each value sits in the asset.
48
49
  SEARCH_FIELDS = { 'name' => %w[name], 'id' => %w[id], 'externalId' => %w[externalId],
49
50
  'description' => %w[description], 'tags' => %w[tags], 'type' => %w[type],
50
51
  'content.format' => %w[content format], 'content.extension' => %w[content extension],
51
52
  'classification.moderation.status' => %w[classification moderation status] }.freeze
52
53
  MAX_SEARCH_TOKENS = 8
53
- MAX_SEARCH_LIMIT = 20
54
+ MAX_SEARCH_PAGE_SIZE = 20
54
55
  MAX_SEARCH_QUERY_LENGTH = 200
55
56
  # Every API cursor list returns this many items unless `pageSize` says otherwise.
56
57
  DEFAULT_PAGE_SIZE = 50
57
58
  MAX_PAGE_SIZE = 100
58
59
  REVIEW_DECISIONS = %w[allowed limited age-restricted blocked purged].freeze
60
+ # The moderation status each `asset.<status>` event announces.
61
+ ANNOUNCED_STATUSES = REVIEW_DECISIONS.to_h { |status| ["asset.#{status}", status] }.freeze
59
62
  MAX_DECISION_NOTE_LENGTH = 1499
60
- RESOURCES = %w[original rendered placeholder].freeze
63
+ # The actor of events the mock's API key causes: mock keys have no ID.
64
+ API_KEY_ACTOR = { 'type' => 'api-key', 'id' => nil }.freeze
65
+ # The actor of a reviewer's decision through {MockServerClient#decide}.
66
+ REVIEWER_ACTOR = { 'type' => 'user', 'id' => nil }.freeze
67
+ # The actor of the pipeline's processing events.
68
+ AUTOMATED_ACTOR = { 'type' => 'automated', 'id' => nil }.freeze
69
+ # An event type's shape (`resource.action`), as the mock checks `types` without the catalog.
70
+ EVENT_TYPE_PATTERN = /\A[a-z][a-z-]*\.[a-z][a-z-]*\z/
71
+ # The resources an asset's CDN path names.
72
+ RESOURCES = %w[original rendered preview hls embed placeholder info].freeze
73
+ # The page the mock serves at `/embed`, standing in for the CDN's embed page.
74
+ EMBED_HTML = '<!doctype html><html lang="en"><head><meta charset="utf-8"><title>Embed</title></head>' \
75
+ '<body></body></html>'
76
+
61
77
  MIN_PART = 5 * 1024 * 1024
62
78
  MAX_MULTIPART_IMAGE = 20 * 1024 * 1024
63
79
  # Like the API: an upload has at most this many parts.
64
80
  MAX_PARTS = 10_000
65
- # The ThumbHash the mock gives every classified image, standing in for one computed from its
66
- # pixels. The `placeholder` resource serves it as a PNG, like the CDN.
81
+ # The ThumbHash the mock gives every processed asset's preview, standing in for one computed
82
+ # from its pixels. The `placeholder` resource serves it as a PNG, like the CDN.
67
83
  THUMBHASH = '1QcSHQRnh493V4dIh4eXh1h4kJUI'
84
+ # Removing or withholding content removes its rendered file, adaptive stream and preview, as the API does.
85
+ NO_RENDERING = { 'rendered' => nil, 'hls' => nil, 'preview' => nil }.freeze
86
+ # The size of the waveform the API makes as an audio asset's preview.
87
+ WAVEFORM_SIZE = { 'width' => 1200, 'height' => 300 }.freeze
88
+ # Audio and video formats browsers play as uploaded, so the API serves them as stored (Asset.ts).
89
+ BROWSER_SAFE_MEDIA_FORMATS = %w[video/mp4 audio/mpeg audio/aac audio/mp4 audio/x-m4a audio/ogg].freeze
90
+ # Spreadsheets, which have no rendered file or preview (Asset.ts).
91
+ SPREADSHEET_MIME_TYPES = %w[application/vnd.openxmlformats-officedocument.spreadsheetml.sheet
92
+ application/vnd.ms-excel.sheet.macroenabled.12 application/vnd.ms-excel
93
+ application/vnd.oasis.opendocument.spreadsheet].freeze
94
+ # The settings of the mock's presets: every category with its default action (`allow` for
95
+ # personal information and profanity, `block` otherwise) and no rules. The mock holds no
96
+ # built-in preset settings, and uploads are never decided by a policy.
97
+ PRESET_SETTINGS = {
98
+ 'categorySettings' => MockClassification::CATEGORIES.to_h do |category|
99
+ [category, { 'action' => %w[personal-information profanity].include?(category) ? 'allow' : 'block',
100
+ 'allowances' => [], 'restrictions' => [] }]
101
+ end
102
+ }.freeze
68
103
  # The built-in presets the mock knows, by display name.
69
104
  PRESET_NAMES = { 'general-community' => 'General', 'children-platform' => 'Kids',
70
105
  'education-support' => 'Learning and support', 'wholesome-community' => 'Teen-safe' }.freeze
@@ -77,8 +112,9 @@ module CalmLens
77
112
  # The mock API key's `{ 'actions', 'folderId', 'includeSubfolders' }` grants.
78
113
  attr_accessor :grants
79
114
 
80
- def initialize(project_id:, base_url:, cdn_base_url:, now:, create_id:, moderate:, grants:)
115
+ def initialize(project_id:, base_url:, cdn_base_url:, now:, create_id:, moderate:, api_key:, grants:)
81
116
  @project_id = project_id
117
+ @api_key = api_key
82
118
  @base_url = base_url
83
119
  @cdn_base_url = cdn_base_url
84
120
  @now = now
@@ -92,22 +128,41 @@ module CalmLens
92
128
  @policies = {}
93
129
  @project_policy_id = nil
94
130
  @request_count = 0
131
+ # Each asset's latest `assetSequence`, which the API keeps on the asset row.
132
+ @asset_event_sequences = Hash.new(0)
133
+ # The processing options each asset was uploaded with, which the API keeps but doesn't return.
134
+ @processing_options = {}
135
+ # Decided assets whose re-run (`assets.explain` with `rerun: true`) is still queued: they keep
136
+ # their decision until `complete_processing` decides them.
137
+ @reruns = {}
95
138
  end
96
139
 
97
- # Adds an event to the feed, filling the fields the API always sets.
140
+ # Adds an event to the feed, filling the fields the API always sets. Its `actor` is the mock's
141
+ # API key unless the event names another.
98
142
  def add_event(event)
99
143
  time = @now.call
100
- record = { 'id' => @create_id.call('evt'), 'projectId' => @project_id, 'severity' => 'low', 'description' => nil,
101
- 'assetId' => nil, 'asset' => nil, 'facts' => [], 'createdAt' => time }.merge(event)
144
+ record = { 'id' => @create_id.call('evt'), 'projectId' => @project_id, 'actor' => deep_copy(API_KEY_ACTOR),
145
+ 'severity' => 'low', 'description' => nil,
146
+ 'assetId' => nil, 'assetSequence' => nil, 'asset' => nil, 'facts' => [],
147
+ 'createdAt' => time }.merge(event)
102
148
  record['title'] ||= record['type']
149
+ # Like the API: each event about an asset takes the asset's next number; others have none.
150
+ asset_id = record['assetId']
151
+ record['assetSequence'] = asset_id ? (@asset_event_sequences[asset_id] += 1) : nil
103
152
  @state.events << record
104
153
  record
105
154
  end
106
155
 
107
- # Adds an event like {#add_event}; one about a stored asset carries the asset as it is now.
156
+ # Adds an event like {#add_event}; one about a stored asset carries the asset as it is now,
157
+ # with the moderation status an `asset.<status>` event announces.
108
158
  def enqueue_event(event)
109
159
  item = own_asset(event['assetId'])
110
- event = event.merge('asset' => snapshot(item)) if item && !event.key?('asset')
160
+ if item && !event.key?('asset')
161
+ asset = snapshot(item)
162
+ announced = ANNOUNCED_STATUSES[event['type']]
163
+ asset['classification']['moderation']['status'] = announced if announced
164
+ event = event.merge('asset' => asset)
165
+ end
111
166
  add_event(event)
112
167
  end
113
168
 
@@ -121,25 +176,29 @@ module CalmLens
121
176
  item = own_asset(asset_id)
122
177
  return unless item
123
178
 
124
- record_decision(item, decision, nil, nil)
179
+ record_decision(item, decision, nil, false)
125
180
  deep_copy(item)
126
181
  end
127
182
 
128
183
  # Finishes processing an asset as the pipeline would, with `decision` or else what `moderate`
129
- # answers for the `complete` trigger, and returns it; nil when there is no such asset.
184
+ # answers for the `complete` trigger, and returns it; nil when there is no such asset. A decided
185
+ # asset whose re-run is still queued counts as processing.
130
186
  def complete_processing(asset_id, decision)
131
187
  item = own_asset(asset_id)
132
188
  return unless item
133
- raise Internal.invalid("Asset #{asset_id} isn't processing") unless item['classification']['status'] == 'processing'
134
189
 
135
- moderate_stored(item, 'complete', decision.nil? ? nil : MockDecision.from(decision))
190
+ rerun = @reruns.key?(asset_id)
191
+ raise Internal.invalid("Asset #{asset_id} isn't processing") unless rerun || item['classification']['status'] == 'processing'
192
+
193
+ settled = decision.nil? ? nil : MockDecision.from(decision)
194
+ rerun ? settle_rerun(item, 'complete', settled) : moderate_stored(item, 'complete', settled)
136
195
  deep_copy(item)
137
196
  end
138
197
 
139
198
  # @return [Response]
140
199
  def handle(method, url, headers, body)
141
200
  if url.start_with?("#{@cdn_base_url}/")
142
- serve(method, url.delete_prefix(@cdn_base_url))
201
+ serve(method, url.delete_prefix(@cdn_base_url), headers)
143
202
  else
144
203
  api(method, url.delete_prefix(@base_url), headers, body)
145
204
  end
@@ -172,29 +231,29 @@ module CalmLens
172
231
  end
173
232
 
174
233
  # Applies a reviewer's decision like `decideAssetManually` in the API: the status and the review,
175
- # then (for `purged`) the content removal and its event, then `asset.<decision>`.
176
- def record_decision(item, decision, note, reviewer_name)
234
+ # then (for `purged`) the content removal and its event, then `asset.<decision>`. Either way the
235
+ # review is the mock API key's, which has no ID. A decision through the API (`by_api_key`) has
236
+ # the API key as its events' actor; the test hook's is a reviewer's (a `user`).
237
+ def record_decision(item, decision, note, by_api_key)
238
+ # Like the API: a queued re-run no longer replaces a reviewed decision.
239
+ @reruns.delete(item['id'])
177
240
  apply_outcome(item, decision, 'requested')
178
241
  review = { 'id' => @create_id.call('rev'), 'decision' => decision, 'note' => note,
179
- 'reviewedAt' => @now.call, 'appliedToAsset' => true }
180
- reviewer_facts = []
181
- if reviewer_name
182
- review.merge!('reviewerName' => reviewer_name, 'reviewerEmail' => nil)
183
- reviewer_facts = [{ 'key' => 'reviewer-user-id', 'value' => nil }, { 'key' => 'reviewer-name', 'value' => reviewer_name }]
184
- end
242
+ 'decidedBy' => { 'type' => 'api-key', 'id' => nil }, 'reviewedAt' => @now.call, 'applied' => true }
243
+ actor = by_api_key ? API_KEY_ACTOR : REVIEWER_ACTOR
185
244
  item['classification']['moderation']['manualReviews'] << review
186
- lifecycle_event('asset.content-removed', item) if decision == 'purged'
245
+ lifecycle_event('asset.content-removed', item, actor: actor) if decision == 'purged'
187
246
  # Like the event catalog: only allowing and limiting are low severity.
188
247
  add_event('type' => "asset.#{decision}", 'severity' => %w[allowed limited].include?(decision) ? 'low' : 'high',
189
248
  'title' => "asset.#{decision}", 'assetId' => item['id'], 'asset' => snapshot(item),
190
- 'facts' => [{ 'key' => 'decided-by', 'value' => 'reviewer' }, { 'key' => 'asset-type', 'value' => item['type'] },
249
+ 'actor' => deep_copy(actor),
250
+ 'facts' => [{ 'key' => 'asset-type', 'value' => item['type'] },
191
251
  { 'key' => 'asset-name', 'value' => item['name'], 'url' => "https://calmlens.com/go/asset/#{item['id']}" },
192
- *reviewer_facts, { 'key' => 'note', 'value' => note }])
252
+ { 'key' => 'note', 'value' => note }])
193
253
  end
194
254
 
195
- # `POST /assets/:id/decision` as the API validates and applies it. The mock's key is unnamed, so the
196
- # review and its event name the reviewer `An API key`. An asset still uploading or processing is
197
- # refused with `asset/processing`.
255
+ # `POST /assets/:id/decision` as the API validates and applies it, with the API key as its
256
+ # events' actor. An asset still uploading or processing is refused with `asset/processing`.
198
257
  def decide_request(item, json)
199
258
  decision = json['decision']
200
259
  invalid!('validation/invalid-union', 'decision') unless REVIEW_DECISIONS.include?(decision)
@@ -211,7 +270,7 @@ module CalmLens
211
270
  fail!(ErrorCodes::ASSET_DECISION_UNCHANGED) if status == decision
212
271
  fail!(ErrorCodes::ASSET_CONTENT_PURGED) if status == 'purged' && decision != 'blocked'
213
272
 
214
- record_decision(item, decision, note, 'An API key')
273
+ record_decision(item, decision, note, true)
215
274
  item
216
275
  end
217
276
 
@@ -219,26 +278,34 @@ module CalmLens
219
278
  def snapshot(item)
220
279
  { 'id' => item['id'], 'externalId' => item['externalId'], 'uploadPresetId' => item['uploadPresetId'],
221
280
  'upload' => { 'status' => item['upload']['status'] },
222
- 'classification' => { 'status' => item['classification']['status'],
281
+ 'classification' => { 'status' => item['classification']['status'], 'testInput' => item['classification']['testInput'],
223
282
  'moderation' => { 'status' => item['classification']['moderation']['status'] } },
224
283
  'content' => { 'status' => item['content']['status'] } }
225
284
  end
226
285
 
227
- # The `asset.deleted`, `asset.content-removed` and `asset.retried` events the API records.
228
- def lifecycle_event(type, item)
286
+ # The `asset.deleted`, `asset.content-removed` and `asset.retried` events the API records, with
287
+ # the `batch-id` fact of the batch that caused them, if any.
288
+ def lifecycle_event(type, item, batch_id: nil, actor: API_KEY_ACTOR)
229
289
  name = { 'key' => 'asset-name', 'value' => item['name'] }
230
290
  # A deleted asset has no portal page to link to.
231
291
  name['url'] = "https://calmlens.com/go/asset/#{item['id']}" unless type == 'asset.deleted'
292
+ facts = [name, { 'key' => 'asset-type', 'value' => item['type'] }]
293
+ facts << { 'key' => 'batch-id', 'value' => batch_id } if batch_id
232
294
  add_event('type' => type, 'severity' => 'low', 'assetId' => item['id'], 'asset' => snapshot(item),
233
- 'facts' => [name, { 'key' => 'asset-type', 'value' => item['type'] }])
295
+ 'actor' => deep_copy(actor), 'facts' => facts)
234
296
  end
235
297
 
236
298
  def fail!(code, message = nil, details: nil) = raise(Failure.new(code, message, details: details))
237
299
 
238
300
  def route_not_found!(method, parts) = fail!(ErrorCodes::ROUTE_NOT_FOUND, "No route for #{method} /#{parts.join('/')}")
239
301
 
240
- # A request the API's schema refuses: `validation/<issue>` with HTTP 400 and the field.
241
- def invalid!(code, path) = raise(Failure.new(code, "Invalid #{path}", details: { 'path' => path }, status: 400))
302
+ # A request the API's schema refuses: `validation/invalid-request` with HTTP 400, listing the
303
+ # field and its issue code (such as `validation/too-big`) in `details.issues`.
304
+ def invalid!(code, path)
305
+ message = "Invalid #{path}"
306
+ raise Failure.new(ErrorCodes::INVALID_REQUEST, message,
307
+ details: { 'issues' => [{ 'path' => path, 'message' => message, 'code' => code }] }, status: 400)
308
+ end
242
309
 
243
310
  # Like the API's schema (METADATA_ITEM_SCHEMA): metadata is nil or a list of `{id, value}`
244
311
  # Hashes with a String `id` and any JSON `value`.
@@ -253,32 +320,23 @@ module CalmLens
253
320
  end
254
321
  end
255
322
 
256
- # Like the API's schema, checked before the asset is looked up: asset updates take no null
257
- # name, and signed URLs no null field.
323
+ # Like the API's schema, checked before the asset is looked up: asset updates take no null name.
258
324
  def check_asset_body(method, action, json)
259
- case [method, action]
260
- in ['PATCH', nil]
261
- check_metadata(json['metadata'])
262
- invalid!('validation/invalid-type', 'name') if json.key?('name') && json['name'].nil?
263
- in ['POST', 'signed-url']
264
- %w[viewerClaims resource lens transform expiresInSeconds].each do |key|
265
- next unless json.key?(key) && json[key].nil?
325
+ return unless method == 'PATCH' && action.nil?
266
326
 
267
- invalid!(key == 'resource' ? 'validation/invalid-value' : 'validation/invalid-type', key)
268
- end
269
- else nil
270
- end
327
+ check_metadata(json['metadata'])
328
+ invalid!('validation/invalid-type', 'name') if json.key?('name') && json['name'].nil?
271
329
  end
272
330
 
273
331
  # Like the API: an upload's metadata is checked, then its MIME type, which must be one the
274
- # API can moderate (`upload/invalid-mime-type` with `details.mimeType`).
332
+ # API can moderate (`upload/unsupported-type` with `details.mimeType` and `details.source`).
275
333
  def check_upload(options)
276
334
  check_metadata(options['metadata'])
277
335
  type = options['contentType']
278
336
  return if type.nil? || Internal.accepted_upload_mime_type?(type)
279
337
 
280
- fail!(ErrorCodes::INVALID_UPLOAD_MIME_TYPE, "#{type} is not a valid upload file type",
281
- details: { 'mimeType' => type })
338
+ fail!(ErrorCodes::UNSUPPORTED_UPLOAD_TYPE, "#{type} is not a valid upload file type",
339
+ details: { 'mimeType' => type, 'source' => 'calmlens' })
282
340
  end
283
341
 
284
342
  # Like the API's query schema: an integer from 1 to `max`.
@@ -336,6 +394,8 @@ module CalmLens
336
394
  when 'folders' then folder(method, parts, json)
337
395
  when 'hash-lists' then hash_list(method, parts, query, json)
338
396
  when 'hash-list-entries' then remove_hash_list_entry(method, parts)
397
+ when 'policies' then policies(method, parts, query, json)
398
+ when 'lenses' then lens(method, parts, json)
339
399
  else route_not_found!(method, parts)
340
400
  end
341
401
  data.is_a?(Response) ? data : respond(200, deep_copy(data))
@@ -350,19 +410,21 @@ module CalmLens
350
410
  case [method, parts[2..], upload]
351
411
  in ['GET', ['events'], _] then events(query)
352
412
  in ['PUT', ['lenses'], _] then lenses(json.fetch('lenses', []))
413
+ in ['GET', ['lenses'], _] then page(@state.lenses.values, query, JSON.generate(['lenses', @project_id]))
414
+ in ['POST', ['lenses'], _] then create_lens(json)
353
415
  in ['GET', ['folders'], _] then folder_page(query)
354
416
  in ['POST', ['folders'], _] then create_folder(json)
355
417
  in ['GET', ['hash-lists'], _] then page(project_hash_lists, query, JSON.generate([@project_id]))
356
418
  in ['POST', ['hash-lists'], _] then create_hash_list(json)
357
- in ['GET', ['policies'], _] then policy_list
358
- in ['POST', ['policies'], _] then create_policy(json)
419
+ in ['GET', ['policy'], _] then show_policy(find_policy(ensure_policies))
359
420
  in ['PUT', ['policy'], _] then show_policy(assign_policy(json))
360
- in [_, ['policies', String => policy_id, *rest], _] then policy(method, policy_id, rest, json)
361
421
  in ['GET', ['assets'], _] then list(query)
362
422
  in ['GET', ['assets', 'search'], _] then search(query)
423
+ in ['GET', ['assets', String => asset_id, 'timeline'], _] then timeline(asset_id, query)
363
424
  in ['POST', ['assets', 'batch'], _] then batch(json)
364
- in ['POST', ['assets'], nil] then create_asset(json, query['queue'] == true, url: json['url'])
365
- in ['POST', ['assets'], 'content'] then create_asset(query, query['queue'] == true, data: body)
425
+ in ['POST', ['signed-asset-urls'], _] then sign_asset_url(json)
426
+ in ['POST', ['assets'], nil] then create_asset(json, query['wait'] == false, url: json['url'])
427
+ in ['POST', ['assets'], 'content'] then create_asset(query, query['wait'] == false, data: body)
366
428
  in ['POST', ['assets'], 'part'] then start_multipart(query, body)
367
429
  else route_not_found!(method, parts)
368
430
  end
@@ -379,7 +441,8 @@ module CalmLens
379
441
  # (`asset/not-found`) before anything happens. The operation then runs at once, in request
380
442
  # order, rather than in the background: `delete` deletes each asset with its stored child
381
443
  # assets and records `asset.deleted` with the asset as it was; `remove-content` removes the
382
- # content like DELETE /assets/:id/content. Mock assets carry no safety holds.
444
+ # content like DELETE /assets/:id/content. Each event carries the batch's `batch-id` fact. Mock
445
+ # assets carry no safety holds.
383
446
  def batch(json)
384
447
  ids = json['assetIds']
385
448
  invalid!('validation/invalid-type', 'assetIds') unless ids.is_a?(Array)
@@ -394,15 +457,16 @@ module CalmLens
394
457
  fail!(ErrorCodes::ASSET_NOT_FOUND) unless item && item['projectId'] == @project_id
395
458
  item
396
459
  end
460
+ batch_id = @create_id.call('batch')
397
461
  items.each do |item|
398
462
  if json['operation'] == 'delete'
399
463
  delete_tree(item)
400
- lifecycle_event('asset.deleted', item)
464
+ lifecycle_event('asset.deleted', item, batch_id: batch_id)
401
465
  else
402
- delete_content(item)
466
+ remove_content(item, batch_id)
403
467
  end
404
468
  end
405
- respond(202, { 'queued' => true })
469
+ respond(202, { 'batchId' => batch_id, 'status' => 'queued', 'assetCount' => items.size })
406
470
  end
407
471
 
408
472
  # Deletes an asset, its stored child assets and their content.
@@ -417,26 +481,26 @@ module CalmLens
417
481
  deleting.each do |id|
418
482
  @state.assets.delete(id)
419
483
  @state.content.delete(id)
484
+ @processing_options.delete(id)
420
485
  end
421
486
  end
422
487
 
423
488
  def list(query)
424
- items = project_assets.reject { |item| item['parentId'] }
425
- items = items.select { |item| item['folderId'] == query['folderId'] } if query.key?('folderId')
426
- page(RSQL.filter(items, query['filter']), query, JSON.generate([@project_id, query['filter'], query['folderId']]))
489
+ # Like the API: the query schema is checked before the filter.
490
+ page_query(query)
491
+ page(RSQL.filter(project_assets, query['filter']), query, JSON.generate([@project_id, query['filter']]))
427
492
  end
428
493
 
429
494
  # Like the API: one keyset page of a parent's child folders, optionally matching `search`, by
430
- # `name` ascending unless `ordering` (one entry, `name` or `createdAt`) says otherwise, with an
495
+ # `name` ascending unless `sort` (one field, `name` or `createdAt`) says otherwise, with an
431
496
  # `id` tie-breaker, and the ancestors of the parent and the listed folders. Names compare by
432
497
  # code point, so case matters.
433
498
  def folder_page(query)
434
499
  found = project_folders.select do |f|
435
500
  f['parentId'] == query['parentId'] && (!query['search'] || f['name'].downcase.include?(query['search'].downcase))
436
501
  end
437
- invalid!('validation/too-big', 'ordering') if ordering(query, nil).size > 1
438
502
  scope = JSON.generate([@project_id, query['parentId'], query['search']])
439
- result = page(found, query, scope, fields: %w[name createdAt], default: { 'field' => 'name', 'direction' => 'asc' })
503
+ result = page(found, query, scope, fields: %w[name createdAt], max: 1, default: { 'field' => 'name', 'direction' => 'asc' })
440
504
  page = result['items']
441
505
  ids = ([query['parentId']] + page.map { |f| f['id'] }).compact.uniq
442
506
  queue = ids.dup
@@ -453,46 +517,85 @@ module CalmLens
453
517
  def project_folders = @state.folders.values.select { |item| item['projectId'] == @project_id }
454
518
 
455
519
  # Like the API: without a cursor, no events and a `nextCursor` for now; with one, up to `pageSize`
456
- # events added since, oldest first. `nextCursor` is never nil. The mock's key holds every permission,
457
- # so no event type is excluded.
520
+ # events added since, oldest first, of the `types` (comma-separated) and `assetId` asked for.
521
+ # `nextCursor` is never nil, and a cursor belongs to the filters it was issued with. The mock's
522
+ # key holds every permission, so no event type is excluded.
458
523
  def events(query)
459
524
  size = check_range(query.fetch('pageSize', DEFAULT_PAGE_SIZE), 'pageSize', MAX_PAGE_SIZE)
525
+ types = query.key?('types') ? query['types'].to_s.split(',', -1) : []
526
+ if query.key?('types') && (types.empty? || types.any? { |type| !type.match?(EVENT_TYPE_PATTERN) })
527
+ invalid!('validation/invalid-value', 'types')
528
+ end
529
+ asset_id = query['assetId']
530
+ invalid!('validation/invalid-format', 'assetId') if asset_id && !asset_id.match?(ASSET_ID_PATTERN)
531
+ filter = [JSON.generate([types.uniq.sort, asset_id])].pack('m0')
532
+ matches = ->(event) { (types.empty? || types.include?(event['type'])) && (asset_id.nil? || event['assetId'] == asset_id) }
460
533
  feed = @state.events
461
- return { 'items' => [], 'nextCursor' => "mock_#{feed.size}", 'hasMore' => false, 'excludedTypes' => [] } unless query['cursor']
534
+ unless query['cursor']
535
+ return { 'items' => [], 'nextCursor' => "mock_#{feed.size}~#{filter}", 'hasMore' => false, 'excludedTypes' => [] }
536
+ end
462
537
 
463
- match = query['cursor'].to_s.match(/\Amock_(\d+)\z/)
464
- position = match && Integer(match[1], 10)
538
+ match = query['cursor'].to_s.match(/\Amock_(\d+)~(.*)\z/)
539
+ position = match && match[2] == filter ? Integer(match[1], 10) : nil
465
540
  fail!(ErrorCodes::INVALID_CURSOR) unless position && position <= feed.size
466
541
 
467
- taken = feed[position, size]
468
- following = position + taken.size
469
- { 'items' => taken, 'nextCursor' => "mock_#{following}", 'hasMore' => following < feed.size, 'excludedTypes' => [] }
542
+ taken = []
543
+ following = position
544
+ while following < feed.size && taken.size < size
545
+ event = feed[following]
546
+ following += 1
547
+ taken << event if matches.call(event)
548
+ end
549
+ { 'items' => taken, 'nextCursor' => "mock_#{following}~#{filter}", 'hasMore' => feed.drop(following).any?(&matches),
550
+ 'excludedTypes' => [] }
470
551
  end
471
552
 
472
- # Saves lenses in order, like the API: an unchanged definition returns the stored lens, a
473
- # change needs a greater version, and lenses before a conflict stay saved.
553
+ # Saves lenses in order, like the API: a new identifier creates a lens, an unchanged
554
+ # definition returns the stored lens and a changed one updates it in place.
474
555
  def lenses(definitions)
475
556
  { 'lenses' => definitions.map { |definition| save_lens(definition) } }
476
557
  end
477
558
 
478
559
  def save_lens(definition)
479
- now = @now.call
480
560
  old = @state.lenses[definition['identifier']]
481
- unless old
482
- lens = { 'id' => @create_id.call('lens'), 'projectId' => @project_id }.merge(definition)
483
- return @state.lenses[definition['identifier']] = lens.merge('createdAt' => now, 'updatedAt' => now)
484
- end
561
+ return new_lens(definition) unless old
485
562
 
486
563
  same = %w[name type active].all? { |key| old[key] == definition[key] } &&
487
564
  old['description'] == definition['description'] && canonical(old['transform']) == canonical(definition['transform'])
488
- return old if same && old['version'] == definition['version']
489
- if definition['version'].to_i > old['version'].to_i
490
- return @state.lenses[old['identifier']] =
491
- old.merge(definition).merge('updatedAt' => now)
492
- end
565
+ return old if same
493
566
 
494
- fail!(ErrorCodes::LENS_VERSION_CONFLICT, details: { 'currentVersion' => old['version'], 'minimumVersion' => old['version'] + 1,
495
- 'requestedVersion' => definition['version'] })
567
+ @state.lenses[old['identifier']] = old.merge(definition).merge('updatedAt' => @now.call)
568
+ end
569
+
570
+ def new_lens(definition)
571
+ now = @now.call
572
+ lens = { 'id' => @create_id.call('lens'), 'projectId' => @project_id }.merge(definition)
573
+ @state.lenses[definition['identifier']] = lens.merge('createdAt' => now, 'updatedAt' => now)
574
+ end
575
+
576
+ # POST /projects/:id/lenses: like the API, the identifier must be new to the project.
577
+ def create_lens(json)
578
+ identifier = json['identifier']
579
+ invalid!('validation/invalid-type', 'identifier') unless identifier.is_a?(String) && !identifier.empty?
580
+ fail!(ErrorCodes::LENS_IDENTIFIER_TAKEN, details: { 'identifier' => identifier }) if @state.lenses.key?(identifier)
581
+
582
+ # Like the API, a new lens is 201 Created.
583
+ respond(201, deep_copy(new_lens(json)))
584
+ end
585
+
586
+ # GET, PATCH and DELETE /lenses/:id. An update keeps the identifier and changes the rest of
587
+ # the definition, as ensuring the changed definition would.
588
+ def lens(method, parts, json)
589
+ item = @state.lenses.values.find { |lens| lens['id'] == parts[1] && lens['projectId'] == @project_id }
590
+ fail!(ErrorCodes::LENS_NOT_FOUND) unless item && parts.size == 2
591
+ case method
592
+ when 'GET' then item
593
+ when 'PATCH' then save_lens(item.merge(json.except('identifier', 'id', 'projectId', 'createdAt', 'updatedAt')))
594
+ when 'DELETE'
595
+ @state.lenses.delete(item['identifier'])
596
+ {}
597
+ else route_not_found!(method, parts)
598
+ end
496
599
  end
497
600
 
498
601
  def canonical(value)
@@ -509,7 +612,7 @@ module CalmLens
509
612
  text = query['query'].to_s.strip
510
613
  invalid!('validation/too-small', 'query') if text.empty?
511
614
  invalid!('validation/too-big', 'query') if text.length > MAX_SEARCH_QUERY_LENGTH
512
- limit = check_range(query.fetch('limit', 10), 'limit', MAX_SEARCH_LIMIT)
615
+ limit = check_range(query.fetch('pageSize', 10), 'pageSize', MAX_SEARCH_PAGE_SIZE)
513
616
  normalized = text.downcase
514
617
  tokens = normalized.split(/\s+/).uniq.first(MAX_SEARCH_TOKENS)
515
618
  items = project_assets.filter_map do |item|
@@ -564,13 +667,17 @@ module CalmLens
564
667
  'folderId' => options['folderId'], 'uploadPresetId' => options['uploadPresetId'],
565
668
  'createdAt' => now, 'updatedAt' => now, 'name' => options['name'], 'description' => options['description'],
566
669
  'externalId' => options['externalId'], 'externalUrl' => url || options['externalUrl'],
567
- 'type' => kind(mime, url), 'metadata' => options['metadata'], 'authorId' => options['authorId'],
670
+ 'type' => kind(mime, url), 'metadata' => options['metadata'] || [], 'authorId' => options['authorId'],
568
671
  # Like the API, only the posts CalmLens uses are kept, and only while the content is held.
569
- 'context' => context.empty? ? nil : { 'items' => context }, 'tags' => nil,
570
- 'processingOptions' => processing, 'visibility' => options.fetch('visibility', 'private'),
672
+ 'context' => context.empty? ? nil : { 'items' => context }, 'tags' => options['tags'] || [],
673
+ 'visibility' => options.fetch('visibility', 'private'),
571
674
  **sections(now: now, open: open, keep: keep, format: mime, size: data&.bytesize) }
572
675
  @state.assets[asset['id']] = asset
573
- @state.content[asset['id']] = { 'type' => mime, 'parts' => { 1 => data } } if data
676
+ @processing_options[asset['id']] = processing
677
+ if data
678
+ @state.content[asset['id']] = { 'type' => mime, 'parts' => { 1 => data } }
679
+ asset['content'].merge!(rendering(asset, false)) unless open
680
+ end
574
681
  open ? asset : upload_completed(asset, queue, data)
575
682
  end
576
683
 
@@ -578,20 +685,29 @@ module CalmLens
578
685
  {
579
686
  'content' => { 'status' => keep ? 'stored' : 'temporary', 'removalReason' => nil, 'updatedAt' => now,
580
687
  'safetyHold' => nil, 'format' => format.empty? ? nil : format, 'extension' => nil,
581
- 'sizeInBytes' => size, 'checksums' => nil, 'image' => nil, 'extractedText' => nil,
582
- 'fileMetadata' => nil },
688
+ # The mock decodes nothing, so the file's own details are unknown, and like
689
+ # the API, the rendered file and preview come with processing.
690
+ 'sizeInBytes' => size, 'checksums' => nil, 'width' => nil, 'height' => nil,
691
+ 'durationSeconds' => nil, 'pageCount' => nil, 'perceptualHash' => nil,
692
+ 'perceptualHashQuality' => nil, 'metadata' => nil, 'extractedText' => nil, **NO_RENDERING },
583
693
  # Like the API, only an open upload counts the parts received.
584
694
  'upload' => { 'status' => open ? 'uploading' : 'completed', 'updatedAt' => now, 'partsReceived' => open ? 1 : nil },
585
- 'classification' => { 'status' => open ? 'pending' : 'processing', 'failureReason' => nil, 'updatedAt' => now,
586
- 'completedAt' => nil,
587
- 'moderation' => { 'status' => 'pending', 'manualReviews' => [],
588
- 'flaggedCategories' => [], 'regions' => [] } }
695
+ 'classification' => { 'status' => open ? 'queued' : 'processing', 'failureReason' => nil, 'updatedAt' => now,
696
+ # The mock never reads test inputs.
697
+ 'completedAt' => nil, 'progress' => nil, 'testInput' => false,
698
+ # Like the API, moderation's results are null until it decides, and a
699
+ # feature's result is null until it has run.
700
+ 'moderation' => { 'status' => 'pending', 'manualReviews' => [], 'automatedStatus' => nil,
701
+ 'source' => nil, 'rule' => nil, 'explanationComplete' => nil,
702
+ 'categories' => nil, 'flaggedCategories' => [], 'regions' => [],
703
+ 'raw' => nil, 'timeline' => nil },
704
+ 'ocr' => nil, 'linkDetection' => nil }
589
705
  }
590
706
  end
591
707
 
592
708
  # Like the API: image uploads in parts total at most 20 MiB, at any size below that.
593
709
  def image_size_error(type, size, _final)
594
- ErrorCodes::MULTIPART_IMAGE_TOO_LARGE if type == 'image' && size > MAX_MULTIPART_IMAGE
710
+ ErrorCodes::UPLOAD_TOO_LARGE if type == 'image' && size > MAX_MULTIPART_IMAGE
595
711
  end
596
712
 
597
713
  # Like the API: the parts may not overshoot a declared total, and must reach it at the end.
@@ -601,6 +717,12 @@ module CalmLens
601
717
  ErrorCodes::UPLOAD_SIZE_MISMATCH if received > total || (final && received != total)
602
718
  end
603
719
 
720
+ # The failure for a size error code, with the limit `upload/too-large` carries.
721
+ def size_failure!(code)
722
+ details = code == ErrorCodes::UPLOAD_TOO_LARGE ? { 'limitBytes' => MAX_MULTIPART_IMAGE, 'source' => 'plan' } : nil
723
+ fail!(code, details: details)
724
+ end
725
+
604
726
  def check_part_size(size, final)
605
727
  return if final || size >= MIN_PART
606
728
 
@@ -608,7 +730,7 @@ module CalmLens
608
730
  end
609
731
 
610
732
  # In the API's order: a declared total, then a single final part as the whole upload, then the
611
- # part itself. The queue mode and declared total are saved with an open upload.
733
+ # part itself. Whether to wait and the declared total are saved with an open upload.
612
734
  def start_multipart(query, data)
613
735
  check_upload(query)
614
736
  total = query['totalSizeInBytes']
@@ -617,12 +739,12 @@ module CalmLens
617
739
  size = data.to_s.bytesize
618
740
  code = (total && image_size_error(type, total, true)) ||
619
741
  (final ? total_size_error(total, size, true) || image_size_error(type, size, true) : nil)
620
- fail!(code) if code
742
+ size_failure!(code) if code
621
743
  check_part_size(size, final)
622
744
  fail!(total_size_error(total, size, false)) if !final && total_size_error(total, size, false)
623
745
 
624
- asset = create_asset(query, query['queue'] == true, data: data, open: !final)
625
- @state.uploads[asset['id']] = { 'queue' => query['queue'] == true, 'total' => total } unless final
746
+ asset = create_asset(query, query['wait'] == false, data: data, open: !final)
747
+ @state.uploads[asset['id']] = { 'queue' => query['wait'] == false, 'total' => total } unless final
626
748
  asset
627
749
  end
628
750
 
@@ -658,8 +780,26 @@ module CalmLens
658
780
  # Without `moderate`, queued work stays pending until completed, which allows it.
659
781
  def default_moderation(trigger, queued) = queued && trigger != 'complete' ? 'pending' : 'allowed'
660
782
 
661
- # Applies `decision` to `asset` with its report.
662
- def decided(asset, decision) = apply_outcome(asset, decision.status, 'policy', decision.report)
783
+ # Applies `decision` to `asset` with its report. The mock reads no text, QR codes, people or
784
+ # faces in an image, so a decided image reports that the features its upload ran found nothing:
785
+ # OCR and person and face detection when turned on, link detection unless turned off.
786
+ def decided(asset, decision)
787
+ apply_outcome(asset, decision.status, 'policy', decision.report)
788
+ classification = asset['classification']
789
+ return asset unless classification['status'] == 'completed' && asset['type'] == 'image'
790
+
791
+ processing = @processing_options[asset['id']]
792
+ features = processing.is_a?(Hash) && processing['features'].is_a?(Hash) ? processing['features'] : {}
793
+ setting = ->(name) { features[name].is_a?(Hash) ? features[name]['enabled'] : nil }
794
+ classification['ocr'] = { 'status' => 'evaluated', 'textRegions' => [] } if setting.call('ocr') == true
795
+ unless setting.call('linkDetection') == false
796
+ classification['linkDetection'] = { 'status' => 'evaluated', 'qrCodes' => [], 'links' => [] }
797
+ end
798
+ { 'personDetection' => 'people', 'faceDetection' => 'faces' }.each do |feature, list|
799
+ classification[feature] = { 'status' => 'evaluated', list => [] } if setting.call(feature) == true
800
+ end
801
+ asset
802
+ end
663
803
 
664
804
  # Asks `moderate` (unless `decision` is given) and settles a processing asset: a decision and
665
805
  # its events, or nothing for `pending`.
@@ -681,6 +821,7 @@ module CalmLens
681
821
 
682
822
  if decision.status == 'failed'
683
823
  return add_event('type' => 'asset.failed', 'severity' => 'high', 'assetId' => item['id'], 'asset' => snapshot(item),
824
+ 'actor' => deep_copy(AUTOMATED_ACTOR),
684
825
  'facts' => [asset_name_fact(item), { 'key' => 'asset-type', 'value' => item['type'] },
685
826
  { 'key' => 'previous-status', 'value' => 'processing' },
686
827
  { 'key' => 'failure-reason',
@@ -689,40 +830,38 @@ module CalmLens
689
830
 
690
831
  flagged = decision.report['flaggedCategories']
691
832
  add_event('type' => 'asset.classified', 'severity' => 'low', 'assetId' => item['id'], 'asset' => snapshot(item),
833
+ 'actor' => deep_copy(AUTOMATED_ACTOR),
692
834
  'facts' => [{ 'key' => 'flagged-categories', 'value' => flagged },
693
835
  { 'key' => 'moderation-status', 'value' => decision.status },
694
836
  { 'key' => 'asset-type', 'value' => item['type'] }, asset_name_fact(item)])
695
837
  # Like the event catalog: only allowing and limiting are low severity.
696
838
  add_event('type' => "asset.#{decision.status}", 'severity' => %w[allowed limited].include?(decision.status) ? 'low' : 'high',
697
- 'assetId' => item['id'], 'asset' => snapshot(item),
698
- 'facts' => [{ 'key' => 'decided-by', 'value' => 'automated' }, { 'key' => 'flagged-categories', 'value' => flagged },
839
+ 'assetId' => item['id'], 'asset' => snapshot(item), 'actor' => deep_copy(AUTOMATED_ACTOR),
840
+ 'facts' => [{ 'key' => 'flagged-categories', 'value' => flagged },
699
841
  { 'key' => 'asset-type', 'value' => item['type'] }, asset_name_fact(item)])
700
842
  end
701
843
 
702
844
  # Moves an asset to an outcome like the pipeline would: a decision finishes processing, `purged`
703
845
  # deletes the content (with `purge_reason`, `policy` for the pipeline), and finished processing
704
- # removes content kept only temporarily. A classified image gets a ThumbHash for its
705
- # placeholder, which goes with its content. `report` is a decision's categories, score and policy.
846
+ # removes content kept only temporarily. A decided asset gets its rendered file and preview
847
+ # (with a ThumbHash for its placeholder), which go with its content. `report` is a decision's
848
+ # categories, score and policy.
706
849
  def apply_outcome(asset, outcome, purge_reason = 'policy', report = nil)
707
850
  processing, moderation = AssetRecord.statuses(outcome)
708
851
  now = @now.call
709
852
  decided = processing == 'completed'
710
853
  purge = moderation == 'purged' && asset['content']['status'] != 'removed'
711
854
  remove = purge || (decided && asset['content']['status'] == 'temporary')
712
- image = asset['content']['image']
713
- if decided && asset['type'] == 'image' && asset['content']['status'] != 'removed' && !purge
714
- image = { 'width' => nil, 'height' => nil, 'perceptualHash' => nil }.merge(image || {})
715
- .merge('thumbhash' => THUMBHASH,
716
- 'averageColor' => image&.fetch('averageColor',
717
- nil) || '#7f7f7f')
718
- elsif moderation == 'purged'
719
- image = without_placeholder(image)
720
- end
721
- asset['content']['image'] = image
855
+ # Like the API, an asset whose content is held has its rendered file and preview, completed by
856
+ # its decision, and removing content (when processing finishes or by purging) removes them
857
+ # with it, so no placeholder outlives the content. A safety hold withholds them.
858
+ content = asset['content']
859
+ held = content['status'] != 'removed' && !purge && !content['safetyHold']
860
+ content.merge!(held ? rendering(asset, decided) : NO_RENDERING)
722
861
  if remove
723
862
  forget_content(asset)
724
- asset['content'].merge!('status' => 'removed', 'removalReason' => purge ? purge_reason : 'retention', 'updatedAt' => now,
725
- 'extractedText' => nil, 'fileMetadata' => nil, 'image' => without_placeholder(image))
863
+ content.merge!('status' => 'removed', 'removalReason' => purge ? purge_reason : 'retention', 'updatedAt' => now,
864
+ 'extractedText' => nil, 'metadata' => nil, **NO_RENDERING)
726
865
  end
727
866
  asset['upload'].merge!('status' => 'completed', 'updatedAt' => now)
728
867
  update_classification(asset['classification'], processing, moderation, now)
@@ -744,8 +883,49 @@ module CalmLens
744
883
  item['context'] = nil
745
884
  end
746
885
 
747
- # Removing content removes the placeholder and average colour with it.
748
- def without_placeholder(image) = image&.merge('thumbhash' => nil, 'averageColor' => nil)
886
+ # Whether browsers play an audio or video upload's format as it is, so it is served as stored.
887
+ def browser_safe_media?(format) = BROWSER_SAFE_MEDIA_FORMATS.include?(mime_type(format))
888
+
889
+ # A MIME type without parameters, in lower case.
890
+ def mime_type(format) = format.to_s.downcase.split(';', 2).first.to_s.strip
891
+
892
+ # The rendered file and preview the API reports for an asset with its content held (Asset.ts,
893
+ # getAssetRendered and getAssetPreview). Once uploaded, an image renders as itself with a preview
894
+ # that has no ThumbHash yet, and a PDF and browser-safe audio and video render as uploaded. Once
895
+ # processed, a website renders as its screenshot; a document or presentation as a PDF, with its
896
+ # first page as the preview; video and audio as a browser-safe file (MP4 or M4A unless browsers
897
+ # play the upload), audio with a waveform preview and video with no poster; and every preview has
898
+ # its ThumbHash. Text, spreadsheets and other files have neither. The mock decodes nothing, so
899
+ # sizes are unknown, audio has no peaks, and audio and video never get an adaptive stream.
900
+ def rendering(asset, processed) = { **rendered_file(asset, processed), 'hls' => nil }
901
+
902
+ # The rendered file and preview of {#rendering}.
903
+ def rendered_file(asset, processed)
904
+ format = asset['content']['format']
905
+ mime = mime_type(format)
906
+ safe = browser_safe_media?(format)
907
+ media = %w[video audio].include?(asset['type'])
908
+ return NO_RENDERING.dup unless processed || asset['type'] == 'image' || mime == 'application/pdf' || (media && safe)
909
+
910
+ unsized = { 'width' => nil, 'height' => nil }
911
+ preview = lambda do |size = unsized|
912
+ size.merge('thumbhash' => processed ? THUMBHASH : nil, 'averageColor' => processed ? '#7f7f7f' : nil)
913
+ end
914
+ case asset['type']
915
+ when 'image' then { 'rendered' => { 'kind' => 'image', 'format' => format || 'image/png', **unsized }, 'preview' => preview.call }
916
+ when 'website' then { 'rendered' => { 'kind' => 'image', 'format' => 'image/png', **unsized }, 'preview' => preview.call }
917
+ when 'multimedia-document'
918
+ return NO_RENDERING.dup if SPREADSHEET_MIME_TYPES.include?(mime)
919
+
920
+ { 'rendered' => { 'kind' => 'pdf', 'format' => 'application/pdf', **unsized }, 'preview' => processed ? preview.call : nil }
921
+ when 'video' then { 'rendered' => { 'kind' => 'video', 'format' => safe ? format : 'video/mp4', **unsized }, 'preview' => nil }
922
+ when 'audio'
923
+ rendered = { 'kind' => 'audio', 'format' => safe ? format : 'audio/mp4', **unsized }
924
+ rendered['peaks'] = [] if processed
925
+ { 'rendered' => rendered, 'preview' => processed ? preview.call(WAVEFORM_SIZE.dup) : nil }
926
+ else NO_RENDERING.dup
927
+ end
928
+ end
749
929
 
750
930
  def asset(method, parts, query, json, body)
751
931
  id = parts[1]
@@ -758,9 +938,10 @@ module CalmLens
758
938
  in ['DELETE', nil]
759
939
  @state.assets.delete(id)
760
940
  @state.content.delete(id)
941
+ @processing_options.delete(id)
761
942
  lifecycle_event('asset.deleted', item)
762
943
  nil
763
- in ['DELETE', 'content'] then delete_content(item)
944
+ in ['DELETE', 'content'] then remove_content(item)
764
945
  in ['GET', 'children'] then page(project_assets.select { |child| child['parentId'] == id }, query, id)
765
946
  in ['POST', 'retry'] then retry_asset(item)
766
947
  in ['POST', 'explain'] then explain(item, json['rerun'])
@@ -768,7 +949,6 @@ module CalmLens
768
949
  in ['POST', 'move']
769
950
  check_folder(json['folderId']) if json['folderId']
770
951
  touch(item.merge!('folderId' => json['folderId']))
771
- in ['POST', 'signed-url'] then signed_url(id, json)
772
952
  in ['POST', 'part'] then part(item, query, body)
773
953
  else route_not_found!(method, parts)
774
954
  end
@@ -787,21 +967,21 @@ module CalmLens
787
967
  touch(item.merge!(json.except('id', 'projectId', 'upload', 'classification', 'content')))
788
968
  end
789
969
 
790
- # Like the API: the record and report stay; stored content, extracted text and the placeholder
791
- # go. Content already removed keeps its reason, and an `asset.content-removed` event is
792
- # recorded either way.
793
- def delete_content(item)
970
+ # Like the API: the record and report stay; stored content, extracted text, metadata, the
971
+ # rendered file and the preview go. Content already removed keeps its reason, and an `asset.content-removed` event is
972
+ # recorded either way, with the `batch-id` fact of the batch that removed it, if any.
973
+ def remove_content(item, batch_id = nil)
794
974
  forget_content(item)
795
975
  content = item['content']
796
976
  if content['status'] == 'removed'
797
- content['image'] = without_placeholder(content['image']) if content.dig('image', 'thumbhash')
977
+ content.merge!(NO_RENDERING)
798
978
  else
799
979
  now = @now.call
800
980
  content.merge!('status' => 'removed', 'removalReason' => 'requested', 'updatedAt' => now, 'extractedText' => nil,
801
- 'fileMetadata' => nil, 'image' => without_placeholder(content['image']))
981
+ 'metadata' => nil, **NO_RENDERING)
802
982
  item['updatedAt'] = now
803
983
  end
804
- lifecycle_event('asset.content-removed', item)
984
+ lifecycle_event('asset.content-removed', item, batch_id: batch_id)
805
985
  item
806
986
  end
807
987
 
@@ -830,16 +1010,61 @@ module CalmLens
830
1010
  fail!(ErrorCodes::ASSET_REVIEWED) unless moderation.fetch('manualReviews', []).empty?
831
1011
  fail!(ErrorCodes::ASSET_CONTENT_UNAVAILABLE) if item['content']['status'] == 'removed'
832
1012
 
833
- # Like the API: back to processing, to be moderated again under the current policy.
834
- apply_outcome(item, 'pending')
835
- moderate_stored(item, 'rerun')
1013
+ # Like the API: the request is recorded, and the asset returns as it was. It keeps its decision
1014
+ # while it's moderated again under the current policy.
1015
+ add_event('type' => 'asset.rerun-requested', 'severity' => 'low', 'assetId' => item['id'],
1016
+ 'asset' => snapshot(item), 'actor' => deep_copy(API_KEY_ACTOR),
1017
+ 'facts' => [asset_name_fact(item), { 'key' => 'asset-type', 'value' => item['type'] },
1018
+ { 'key' => 'moderation-status', 'value' => moderation['status'] }])
1019
+ response = deep_copy(item)
1020
+ settle_rerun(item, 'rerun')
1021
+ response
836
1022
  end
837
1023
 
838
- def signed_url(id, json)
839
- expires = json.fetch('expiresInSeconds', 900)
840
- params = URI.encode_www_form(json.slice('lens', 'transform').merge('expires' => expires))
841
- { 'url' => "#{@cdn_base_url}/signed/#{URI.encode_www_form_component(id)}/#{json.fetch('resource', 'rendered')}?#{params}",
842
- 'expiresAt' => @now.call + (expires * 1000) }
1024
+ # Applies a re-run's decision to a decided asset like the pipeline: `pending` stays queued for
1025
+ # `complete_processing`, `failed` keeps the asset's decision, and any other replaces it with the
1026
+ # pipeline's events.
1027
+ def settle_rerun(item, trigger, decision = nil)
1028
+ decision ||= decision_for(item, trigger, true)
1029
+ @reruns.delete(item['id'])
1030
+ if decision.status == 'pending'
1031
+ @reruns[item['id']] = true
1032
+ elsif decision.status != 'failed'
1033
+ moderate_stored(item, trigger, decision)
1034
+ end
1035
+ end
1036
+
1037
+ # Signs an asset URL like the API (`POST /projects/:projectId/signed-asset-urls`): a grant to one
1038
+ # asset, by ID or external ID, until it expires, serving the original only when asked. Only the
1039
+ # mock's CDN accepts the grant.
1040
+ def sign_asset_url(json)
1041
+ # Like the API's schema, none of these fields takes null.
1042
+ %w[assetId externalId expiresIn original].each do |key|
1043
+ invalid!('validation/invalid-type', key) if json.key?(key) && json[key].nil?
1044
+ end
1045
+ if json.key?('assetId') == json.key?('externalId')
1046
+ message = 'Send either assetId or externalId'
1047
+ raise Failure.new(ErrorCodes::INVALID_REQUEST, message,
1048
+ details: { 'issues' => [{ 'path' => '', 'message' => message, 'code' => 'custom' }] }, status: 400)
1049
+ end
1050
+ expires = json.fetch('expiresIn', CDN::DEFAULT_SIGNED_URL_EXPIRES_IN)
1051
+ invalid!('validation/invalid-type', 'expiresIn') unless expires.is_a?(Integer) || (expires.is_a?(Float) && expires == expires.floor)
1052
+ invalid!('validation/too-small', 'expiresIn') if expires < 1
1053
+ invalid!('validation/too-big', 'expiresIn') if expires > CDN::MAX_SIGNED_URL_EXPIRES_IN
1054
+ original = json.fetch('original', false)
1055
+ invalid!('validation/invalid-type', 'original') unless [true, false].include?(original)
1056
+
1057
+ item = if json.key?('externalId')
1058
+ project_assets.find { |asset| asset['externalId'] == json['externalId'] }
1059
+ else
1060
+ own_asset(json['assetId'].to_s)
1061
+ end
1062
+ fail!(ErrorCodes::ASSET_NOT_FOUND) unless item
1063
+ fail!(ErrorCodes::ASSET_HELD) if item['content']['safetyHold']
1064
+
1065
+ expires_at = @now.call + (expires.to_i * 1000)
1066
+ { 'url' => "#{@cdn_base_url}/assets/#{Internal.component(@project_id)}/#{MockGrant.encode(item['id'], original, expires_at)}",
1067
+ 'expiresAt' => expires_at }
843
1068
  end
844
1069
 
845
1070
  # In the API's order: the upload must be open, parts arrive in order, image totals and the
@@ -857,7 +1082,7 @@ module CalmLens
857
1082
  received = item['content']['sizeInBytes'].to_i + size
858
1083
  upload = @state.uploads[item['id']] || {}
859
1084
  code = image_size_error(item['type'], received, final) || total_size_error(upload['total'], received, final)
860
- fail!(code) if code
1085
+ size_failure!(code) if code
861
1086
  check_part_size(size, final)
862
1087
 
863
1088
  stored = (@state.content[item['id']] ||= { 'type' => item['content']['format'].to_s, 'parts' => {} })
@@ -870,7 +1095,8 @@ module CalmLens
870
1095
  return item unless final
871
1096
 
872
1097
  @state.uploads.delete(item['id'])
873
- queue = query.key?('queue') ? query['queue'] == true : upload.fetch('queue', false)
1098
+ # `wait=false` queues the final part; without `wait`, the upload's saved choice applies.
1099
+ queue = query.key?('wait') ? query['wait'] == false : upload.fetch('queue', false)
874
1100
  apply_outcome(item, 'pending')
875
1101
  upload_completed(item, queue)
876
1102
  end
@@ -933,28 +1159,37 @@ module CalmLens
933
1159
  end
934
1160
 
935
1161
  # Policies: the project starts on the organisation's own General preset policy, following
936
- # automatic updates. The mock holds no built-in preset settings: a preset's settings are
937
- # `{ 'categorySettings' => {} }` at version 1, and uploads are never decided by a policy.
938
- def new_policy(name, settings, description: '', automatic_updates: false, preset: nil)
1162
+ # automatic updates (mode `live`). The mock's presets have one release, version 1, with
1163
+ # PRESET_SETTINGS. A stored policy is the API's shape without `projectIds`, with its preset's
1164
+ # `id`, `version` and `own`, whether it follows its preset (`followsPreset`) and its reset
1165
+ # `baseline`.
1166
+ def new_policy(name, settings, description: '', follows_preset: false, preset: nil)
939
1167
  now = @now.call
940
1168
  item = { 'id' => @create_id.call('pol'), 'name' => name, 'description' => description, 'settings' => settings,
941
- 'revision' => 1, 'automaticUpdates' => automatic_updates, 'preset' => preset,
1169
+ 'revision' => 1, 'followsPreset' => follows_preset, 'preset' => preset,
942
1170
  'baseline' => deep_copy(settings), 'createdAt' => now, 'updatedAt' => now }
943
1171
  @policies[item['id']] = item
944
1172
  end
945
1173
 
1174
+ # The project's policy, after creating the organisation's own General policy on first use.
946
1175
  def ensure_policies
947
- return if @project_policy_id
1176
+ return @project_policy_id if @project_policy_id
948
1177
 
949
1178
  preset = { 'id' => 'general-community', 'version' => 1, 'own' => true }
950
- @project_policy_id = new_policy(PRESET_NAMES['general-community'], { 'categorySettings' => {} },
951
- automatic_updates: true, preset: preset)['id']
1179
+ @project_policy_id = new_policy(PRESET_NAMES['general-community'], deep_copy(PRESET_SETTINGS),
1180
+ follows_preset: true, preset: preset)['id']
952
1181
  end
953
1182
 
954
- # A stored policy as the API shows it.
1183
+ # A stored policy as the API shows it: an own preset policy is `live` while it follows its
1184
+ # preset and `pinned` otherwise; any other preset policy is a `copy`.
955
1185
  def show_policy(item)
956
- shown = item.except('baseline')
957
- shown['preset'] = item['preset'].merge('latestVersion' => 1) if item['preset']
1186
+ shown = item.except('baseline', 'followsPreset')
1187
+ if (preset = item['preset'])
1188
+ mode = if preset['own'] then item['followsPreset'] ? 'live' : 'pinned'
1189
+ else 'copy'
1190
+ end
1191
+ shown['preset'] = { 'id' => preset['id'], 'version' => preset['version'], 'latestVersion' => 1, 'mode' => mode }
1192
+ end
958
1193
  shown.merge('projectIds' => item['id'] == @project_policy_id ? [@project_id] : [])
959
1194
  end
960
1195
 
@@ -963,9 +1198,39 @@ module CalmLens
963
1198
  @policies[id] || fail!(ErrorCodes::POLICY_NOT_FOUND)
964
1199
  end
965
1200
 
966
- def policy_list
1201
+ # `/policies` and `/policies/:id[/action]`, the API key's organisation's policies.
1202
+ def policies(method, parts, query, json)
1203
+ _, id, *rest = parts
1204
+ if id.nil?
1205
+ return policy_page(query) if method == 'GET'
1206
+ return create_policy(json) if method == 'POST'
1207
+
1208
+ route_not_found!(method, parts)
1209
+ end
1210
+ policy(method, id, rest, json)
1211
+ end
1212
+
1213
+ # A page of the organisation's policies, oldest first unless `sort` says otherwise.
1214
+ def policy_page(query)
967
1215
  ensure_policies
968
- { 'policies' => @policies.values.map { |item| show_policy(item) }, 'projectPolicyId' => @project_policy_id }
1216
+ page(@policies.values.map { |item| show_policy(item) }, query, JSON.generate(['policies', @project_id]),
1217
+ default: { 'field' => 'createdAt', 'direction' => 'asc' })
1218
+ end
1219
+
1220
+ # Refuses settings the API refuses (MODERATION_POLICY_SETTINGS_SCHEMA): every category, each
1221
+ # with an `action` and its `allowances` and `restrictions` lists, and no other category.
1222
+ def check_policy_settings(settings)
1223
+ categories = settings.is_a?(Hash) ? settings['categorySettings'] : nil
1224
+ invalid!('validation/invalid-type', 'settings.categorySettings') unless categories.is_a?(Hash)
1225
+ MockClassification::CATEGORIES.each do |category|
1226
+ path = "settings.categorySettings.#{category}"
1227
+ value = categories[category]
1228
+ invalid!('validation/invalid-type', path) unless value.is_a?(Hash)
1229
+ %w[allowances restrictions].each { |key| invalid!('validation/invalid-type', "#{path}.#{key}") unless value[key].is_a?(Array) }
1230
+ invalid!('validation/invalid-value', "#{path}.action") unless value['action'].is_a?(String)
1231
+ end
1232
+ unknown = categories.keys.find { |key| !MockClassification::CATEGORIES.include?(key) }
1233
+ invalid!('validation/unknown-field', "settings.categorySettings.#{unknown}") if unknown
969
1234
  end
970
1235
 
971
1236
  def create_policy(json)
@@ -975,10 +1240,11 @@ module CalmLens
975
1240
  if settings.nil? == preset_id.nil? || (preset_id && !PRESET_NAMES.key?(preset_id)) || (preset_id.nil? && !named)
976
1241
  fail!(ErrorCodes::INVALID_VALUE, 'Send settings with a name, or presetId')
977
1242
  end
1243
+ check_policy_settings(settings) unless preset_id
978
1244
  description = json['description'].is_a?(String) ? json['description'] : ''
979
1245
  item = if preset_id
980
1246
  origin = { 'id' => preset_id, 'version' => 1, 'own' => false }
981
- new_policy(named ? name.strip : PRESET_NAMES[preset_id], { 'categorySettings' => {} },
1247
+ new_policy(named ? name.strip : PRESET_NAMES[preset_id], deep_copy(PRESET_SETTINGS),
982
1248
  description: description, preset: origin)
983
1249
  else
984
1250
  new_policy(name.strip, settings, description: description)
@@ -1010,26 +1276,29 @@ module CalmLens
1010
1276
  fail!(ErrorCodes::POLICY_IN_USE) if item['id'] == @project_policy_id
1011
1277
  @policies.delete(id)
1012
1278
  {}
1013
- in ['POST', ['preset']] then apply_policy_preset(item, json)
1279
+ in ['POST', ['reset']] then restore_policy(item, 'reset', json)
1280
+ in ['POST', ['upgrade']] then restore_policy(item, 'upgrade', json)
1014
1281
  in ['PUT', ['automatic-updates']] then set_policy_automatic_updates(item, json)
1015
1282
  else route_not_found!(method, ['policies', id, *rest])
1016
1283
  end
1017
1284
  end
1018
1285
 
1019
1286
  def update_policy(item, json)
1287
+ check_policy_settings(json['settings']) if json.key?('settings')
1020
1288
  values = %w[name description].select { |key| json[key].is_a?(String) }
1021
1289
  .to_h { |key| [key, key == 'name' ? json[key].strip : json[key]] }
1022
1290
  values['settings'] = json['settings'] if json.key?('settings')
1023
- # Saving a policy that follows its preset pins it as a copy.
1024
- save_policy(item, json['revision'], values.merge('automaticUpdates' => false))
1291
+ # Saving a `live` policy pins it.
1292
+ save_policy(item, json['revision'], values.merge('followsPreset' => false))
1025
1293
  end
1026
1294
 
1027
- def apply_policy_preset(item, json)
1295
+ # Resets a `pinned` or `copy` policy to its preset release, or upgrades it to the latest.
1296
+ def restore_policy(item, operation, json)
1028
1297
  preset = item['preset']
1029
- fail!(ErrorCodes::POLICY_NO_PRESET) if preset.nil? || item['automaticUpdates']
1298
+ fail!(ErrorCodes::POLICY_NO_PRESET) if preset.nil? || item['followsPreset']
1030
1299
  fail!(ErrorCodes::POLICY_PRESET_VERSION_CONFLICT) unless json['presetVersion'] == preset['version']
1031
- # The mock's presets have one release, so an update never has a newer one to apply.
1032
- fail!(ErrorCodes::POLICY_PRESET_UPDATE_UNAVAILABLE) if json['operation'] == 'update'
1300
+ # The mock's presets have one release, so an upgrade never has a newer one to apply.
1301
+ fail!(ErrorCodes::POLICY_PRESET_UPDATE_UNAVAILABLE) if operation == 'upgrade'
1033
1302
 
1034
1303
  save_policy(item, json['revision'], 'settings' => deep_copy(item['baseline']))
1035
1304
  end
@@ -1037,11 +1306,11 @@ module CalmLens
1037
1306
  def set_policy_automatic_updates(item, json)
1038
1307
  enabled = json['enabled'] == true
1039
1308
  preset = item['preset']
1040
- fail!(ErrorCodes::POLICY_AUTOMATIC_UPDATES_UNAVAILABLE) if preset.nil? || !preset['own'] || item['automaticUpdates'] == enabled
1309
+ fail!(ErrorCodes::POLICY_AUTOMATIC_UPDATES_UNAVAILABLE) if preset.nil? || !preset['own'] || item['followsPreset'] == enabled
1041
1310
  fail!(ErrorCodes::POLICY_REVISION_CONFLICT) unless json['revision'] == item['revision']
1042
1311
  fail!(ErrorCodes::POLICY_PRESET_VERSION_CONFLICT) unless json['presetVersion'] == 1
1043
1312
 
1044
- values = { 'automaticUpdates' => enabled }
1313
+ values = { 'followsPreset' => enabled }
1045
1314
  values.merge!('name' => PRESET_NAMES[preset['id']], 'description' => '', 'settings' => deep_copy(item['baseline'])) if enabled
1046
1315
  save_policy(item, json['revision'], values)
1047
1316
  end
@@ -1103,10 +1372,10 @@ module CalmLens
1103
1372
  if asset_id
1104
1373
  asset = @state.assets[asset_id]
1105
1374
  fail!(ErrorCodes::ASSET_NOT_FOUND) unless asset && asset['projectId'] == @project_id
1106
- image = asset.dig('content', 'image')
1107
- fail!(ErrorCodes::HASH_LIST_ASSET_NOT_HASHED) unless image.is_a?(Hash) && image['perceptualHash']
1108
- hash = image['perceptualHash']
1109
- quality = image['perceptualHashQuality']
1375
+ content = asset['content']
1376
+ fail!(ErrorCodes::HASH_LIST_ASSET_NOT_HASHED) unless content['perceptualHash']
1377
+ hash = content['perceptualHash']
1378
+ quality = content['perceptualHashQuality']
1110
1379
  end
1111
1380
  fail!(ErrorCodes::INVALID_UPLOAD_PAYLOAD, 'Use a PDQ hash of 64 hexadecimal characters') unless hash.match?(/\A\h{64}\z/)
1112
1381
 
@@ -1115,7 +1384,7 @@ module CalmLens
1115
1384
 
1116
1385
  entry = { 'id' => @create_id.call('hle'), 'projectId' => @project_id, 'hashListId' => list['id'], 'hash' => hash,
1117
1386
  'quality' => quality, 'label' => json['label'].is_a?(String) ? json['label'] : nil,
1118
- 'sourceAssetId' => asset_id, 'createdAt' => @now.call }
1387
+ 'assetId' => asset_id, 'createdAt' => @now.call }
1119
1388
  @state.hash_list_entries[entry['id']] = entry
1120
1389
  respond(201, deep_copy(entry))
1121
1390
  end
@@ -1129,17 +1398,43 @@ module CalmLens
1129
1398
  {}
1130
1399
  end
1131
1400
 
1132
- # One keyset page like the API's: ordered by `fields` (`createdAt` descending by default; an `id`
1133
- # tie-breaker is added), starting after the cursor's values. The cursor is tied to `scope` (the
1134
- # filter, folder or parent) and the ordering, so it only continues the query it came from.
1135
- def page(items, query, scope, fields: %w[createdAt id], default: { 'field' => 'createdAt', 'direction' => 'desc' })
1401
+ # An asset's findings timeline, as the API lists it (publicGetAssetTimeline.ts): refused while a
1402
+ # safety hold is on. The mock analyses no audio or video, so it keeps no segments: every
1403
+ # timeline is empty, and no cursor continues one.
1404
+ def timeline(id, query)
1405
+ item = own_asset(id)
1406
+ fail!(ErrorCodes::ASSET_NOT_FOUND) unless item
1407
+ check_range(query.fetch('pageSize', DEFAULT_PAGE_SIZE), 'pageSize', MAX_PAGE_SIZE)
1408
+ fail!(ErrorCodes::ASSET_HELD, details: { 'hold' => 'safety' }) if item['content']['safetyHold']
1409
+ fail!(ErrorCodes::INVALID_CURSOR) if query.key?('cursor')
1410
+
1411
+ { 'items' => [], 'nextCursor' => nil, 'hasMore' => false }
1412
+ end
1413
+
1414
+ # The page size and sort, refused like the API's query schema: `sort` is comma-separated
1415
+ # fields from `fields`, each descending when prefixed with `-`, none repeated and at most `max`
1416
+ # (`validation/invalid-value` at `sort` otherwise). Without one, `default` applies.
1417
+ def page_query(query, fields = %w[createdAt id], default = { 'field' => 'createdAt', 'direction' => 'desc' }, max: 2)
1136
1418
  size = check_range(query.fetch('pageSize', DEFAULT_PAGE_SIZE), 'pageSize', MAX_PAGE_SIZE)
1137
- ordering = ordering(query, default)
1138
- # Like the API's query schema: each entry's field and direction are enum values.
1139
- ordering.each_with_index do |rule, index|
1140
- invalid!('validation/invalid-value', "ordering.#{index}.field") unless fields.include?(rule['field'])
1141
- invalid!('validation/invalid-value', "ordering.#{index}.direction") unless %w[asc desc].include?(rule['direction'])
1419
+ [size, query.key?('sort') ? parse_sort(query['sort'], fields, max) : [default.dup]]
1420
+ end
1421
+
1422
+ def parse_sort(value, fields, max)
1423
+ sort = value.to_s.split(',', -1).map(&:strip).map do |part|
1424
+ field = part.delete_prefix('-')
1425
+ invalid!('validation/invalid-value', 'sort') unless part.match?(/\A-?[A-Za-z][A-Za-z0-9.]*\z/) && fields.include?(field)
1426
+ { 'field' => field, 'direction' => part.start_with?('-') ? 'desc' : 'asc' }
1142
1427
  end
1428
+ fields_sent = sort.map { |rule| rule['field'] }
1429
+ invalid!('validation/invalid-value', 'sort') if sort.empty? || fields_sent.uniq.size != fields_sent.size || sort.size > max
1430
+ sort
1431
+ end
1432
+
1433
+ # One keyset page like the API's: sorted by `sort` (`createdAt` descending by default; an `id`
1434
+ # tie-breaker is added), starting after the cursor's values. The cursor is tied to `scope` (the
1435
+ # filter, folder or parent) and the sort, so it only continues the query it came from.
1436
+ def page(items, query, scope, fields: %w[createdAt id], max: 2, default: { 'field' => 'createdAt', 'direction' => 'desc' })
1437
+ size, ordering = page_query(query, fields, default, max: max)
1143
1438
  ordering << { 'field' => 'id', 'direction' => ordering.first['direction'] } unless ordering.any? { |o| o['field'] == 'id' }
1144
1439
  signature = JSON.generate([scope, ordering])
1145
1440
  sorted = items.sort { |a, b| compare_items(a, b, ordering) }
@@ -1149,12 +1444,8 @@ module CalmLens
1149
1444
  end
1150
1445
  selected = sorted.first(size)
1151
1446
  more = sorted.size > size
1152
- cursor = if more
1153
- [JSON.generate({ 'scope' => signature, 'values' => ordering.map do |o|
1154
- selected.last[o['field']]
1155
- end })].pack('m0')
1156
- end
1157
- { 'items' => selected, 'pageSize' => size, 'hasMore' => more, 'ordering' => ordering, 'nextCursor' => cursor }
1447
+ cursor = ([JSON.generate({ 'scope' => signature, 'values' => ordering.map { |o| selected.last[o['field']] } })].pack('m0') if more)
1448
+ { 'items' => selected, 'hasMore' => more, 'nextCursor' => cursor }
1158
1449
  end
1159
1450
 
1160
1451
  def cursor_values(cursor, signature, ordering)
@@ -1177,15 +1468,6 @@ module CalmLens
1177
1468
  false
1178
1469
  end
1179
1470
 
1180
- def ordering(query, default)
1181
- entries = query.keys.filter_map { |key| key.match(/\Aordering\[(\d+)\]\[(field|direction)\]\z/) }
1182
- return [default&.dup].compact if entries.empty?
1183
-
1184
- entries.group_by { |match| Integer(match[1], 10) }.sort.map do |_, matches|
1185
- matches.to_h { |match| [match[2], query[match[0]]] }
1186
- end
1187
- end
1188
-
1189
1471
  def compare_items(left, right, ordering)
1190
1472
  ordering.each do |rule|
1191
1473
  result = compare_values(left[rule['field']], right[rule['field']])
@@ -1204,79 +1486,200 @@ module CalmLens
1204
1486
  left.to_s <=> right.to_s
1205
1487
  end
1206
1488
 
1207
- # Answers like the CDN: a placeholder is the asset's ThumbHash as a PNG, public whatever
1208
- # the asset's visibility or moderation; other resources pass delivery checks first, which refuse
1209
- # everyone (content still uploading, blocked or purged, failed processing, removed content),
1210
- # then an age-restricted asset needs `viewerClaims.ageConfirmed=true` whatever the key, and a
1211
- # private asset needs an API key with `assets:download` for its folder.
1212
- def serve(method, relative)
1489
+ # Answers like the CDN (serveAssetResource.ts): `original` is the upload; `rendered` the image,
1490
+ # PDF, video or audio file `content.rendered` describes; `preview` the still `content.preview`
1491
+ # describes; `hls` the adaptive stream's files, which the mock never prepares; `embed` a small
1492
+ # page standing in for the embed page of any asset; and `placeholder` the preview's ThumbHash as a PNG, public
1493
+ # whatever the asset's visibility or moderation. A safety hold refuses every resource. Other resources pass delivery checks first,
1494
+ # which refuse everyone (content still uploading, blocked or purged, failed processing, removed
1495
+ # content), then an age-restricted asset needs `viewerClaims.ageConfirmed=true` whatever the key,
1496
+ # and a private asset needs an API key with `assets:download` for its folder. The mock converts
1497
+ # nothing: an upload stands in for its rendered file, an image for its own preview, and the
1498
+ # placeholder PNG for a screenshot, page or waveform.
1499
+ def serve(method, relative, headers)
1213
1500
  path, query = relative.split('#', 2).first.split('?', 2)
1214
1501
  raw_segments = path.split('/').reject(&:empty?)
1215
- _, project, identifier, resource = raw_segments.map { |part| URI.decode_www_form_component(part) }
1216
- # Like the CDN, the selector after the resource is read first, and lenses and transforms
1217
- # apply only to the rendered resource.
1218
- begin
1219
- lens, options = TransformRules.parse_selector(raw_segments.drop(4).join('/'))
1220
- rescue ArgumentError
1221
- return invalid_transform
1502
+ root, project, _, resource = raw_segments.map { |part| URI.decode_www_form_component(part) }
1503
+ selected, options = selector(resource, raw_segments.drop(4))
1504
+ return invalid_transform if selected.nil?
1505
+ return asset_not_found unless root == 'assets' && project == @project_id && raw_segments[2]
1506
+
1507
+ grant, item = resolve_asset(raw_segments[2])
1508
+ return grant if grant.is_a?(Response)
1509
+ return asset_not_found unless item && RESOURCES.include?(resource)
1510
+ if resource == 'original' && grant && !grant[:original]
1511
+ return error(ErrorCodes::SIGNED_URL_ORIGINAL_NOT_GRANTED, "This signed asset URL doesn't grant the original file", status: 403)
1222
1512
  end
1223
- return invalid_transform if resource == 'original' && (lens || options)
1224
1513
 
1225
- item = project == @project_id && project_assets.find { |asset| [asset['id'], asset['externalId']].include?(identifier) }
1226
- return error('asset/not-found', 'Asset not found', status: 404) unless item && RESOURCES.include?(resource)
1514
+ params = URI.decode_www_form(query.to_s).to_h
1515
+ # A grant authorises a private asset without the API key.
1516
+ authorised = !grant.nil? || can_download?(item['folderId'], headers)
1517
+ return info(method, item, authorised, params) if resource == 'info'
1518
+ return not_available if item['content']['safetyHold']
1227
1519
  return placeholder(method, item) if resource == 'placeholder'
1228
1520
 
1229
- deliver(method, item, URI.decode_www_form(query.to_s).to_h, options)
1521
+ deliver(method, item, resource, params, selected ? options || {} : nil, authorised)
1230
1522
  end
1231
1523
 
1232
- # Serves a found asset's content after the delivery, age and access checks, then the
1233
- # transform options' schema check.
1234
- def deliver(method, item, params, options)
1524
+ # Whether the path after the resource selects a lens or transform, and its options; nil when the
1525
+ # CDN refuses it. Like the CDN, the adaptive stream's files sit beneath `/hls/` (its master
1526
+ # playlist is `/hls/index.m3u8`). Otherwise the selector after the resource is read
1527
+ # (AssetTransformParser.ts); the original, the embed page and the info take none.
1528
+ def selector(resource, path)
1529
+ lens, options = TransformRules.parse_selector(resource == 'hls' ? '' : path.join('/'))
1530
+ selected = !(lens || options).nil?
1531
+ %w[original embed info].include?(resource) && selected ? nil : [selected, options]
1532
+ rescue ArgumentError
1533
+ nil
1534
+ end
1535
+
1536
+ # Like the CDN (CdnUtils.ts), the asset segment is `ext:` and the URL-encoded external ID, a
1537
+ # grant (`sig_…`) or the asset ID. Returns `[grant, asset]`, or an error response for a grant
1538
+ # the mock didn't sign or that has expired.
1539
+ def resolve_asset(segment)
1540
+ if segment.start_with?('sig_')
1541
+ grant = MockGrant.decode(segment)
1542
+ return error(ErrorCodes::SIGNED_URL_INVALID, "The signed asset URL's grant is invalid", status: 403) unless grant
1543
+ return error(ErrorCodes::SIGNED_URL_EXPIRED, 'The signed URL has expired', status: 403) if grant[:expires_at] <= @now.call
1544
+
1545
+ [grant, project_assets.find { |asset| asset['id'] == grant[:asset_id] }]
1546
+ elsif segment.start_with?(CDN::EXTERNAL_ID_PREFIX)
1547
+ external = MockGrant.percent_decode(segment.delete_prefix(CDN::EXTERNAL_ID_PREFIX))
1548
+ [nil, external && !external.empty? ? project_assets.find { |asset| asset['externalId'] == external } : nil]
1549
+ else
1550
+ id = MockGrant.percent_decode(segment)
1551
+ [nil, project_assets.find { |asset| asset['id'] == id }]
1552
+ end
1553
+ end
1554
+
1555
+ def asset_not_found = error(ErrorCodes::ASSET_NOT_FOUND, 'Asset not found', status: 404)
1556
+
1557
+ # The display info at `/info`, like the CDN's (AssetInfo.ts): the allowlisted fields and whether
1558
+ # the viewer the claims describe can see the content, decided as the other resources decide it.
1559
+ # A private asset's info needs a grant or an API key with `assets:download`.
1560
+ def info(method, item, authorised, params)
1561
+ restricted = item['visibility'] != 'public'
1562
+ return error('asset/not-available', 'Asset is not available', status: 404) if restricted && !authorised
1563
+
1564
+ claims = JSON.parse(params.fetch('viewerClaims', '{}'))
1565
+ body = { 'id' => item['id'], 'projectId' => item['projectId'], 'type' => item['type'], 'externalUrl' => item['externalUrl'],
1566
+ 'content' => item['content'].slice(*%w[format width height durationSeconds pageCount rendered preview hls]),
1567
+ 'availability' => availability(item, claims.is_a?(Hash) && claims['ageConfirmed'] == true) }
1568
+ cdn_response(method, JSON.generate(body), { 'content-type' => 'application/json',
1569
+ 'cache-control' => "#{restricted ? 'private' : 'public'}, max-age=60",
1570
+ 'cache-tag' => "asset-#{item['id']}" })
1571
+ rescue JSON::ParserError
1572
+ error(ErrorCodes::INVALID_VIEWER_CLAIMS, 'Invalid viewer claims', status: 400)
1573
+ end
1574
+
1575
+ # Whether a viewer can see an asset's content now, decided as the other resources decide it.
1576
+ def availability(item, age_confirmed)
1235
1577
  moderation = item['classification']['moderation']['status']
1236
- stored = @state.content[item['id']]
1237
- if stored.nil? || item['upload']['status'] == 'uploading' || %w[blocked purged].include?(moderation) ||
1578
+ if item['content']['safetyHold'] || item['upload']['status'] == 'uploading' || %w[blocked purged].include?(moderation) ||
1238
1579
  item['classification']['status'] == 'failed' || item['content']['status'] == 'removed'
1239
- return not_available
1580
+ 'unavailable'
1581
+ elsif moderation == 'age-restricted' && !age_confirmed
1582
+ 'age-restricted'
1583
+ else
1584
+ 'available'
1240
1585
  end
1586
+ end
1241
1587
 
1242
- if moderation == 'age-restricted' && !viewer_age_confirmed?(params)
1243
- return error('viewer/age-confirmation-required', 'Age confirmation required',
1244
- status: 403)
1245
- end
1588
+ # Serves a found asset's resource after the delivery, age and access checks, then whether it
1589
+ # has the resource and takes the selector, then the transform options' schema check.
1590
+ # `selector` is the selector's options (empty for a lens alone), or nil without one.
1591
+ def deliver(method, item, resource, params, selector, authorised)
1592
+ stored = @state.content[item['id']]
1593
+ refused = refuse_delivery(item, resource, stored, params, authorised)
1594
+ return refused if refused
1246
1595
 
1247
1596
  restricted = item['visibility'] != 'public'
1248
- if restricted && !can_download?(item['folderId'])
1249
- return error('asset/api-key-missing-permission',
1250
- 'This API key needs assets:download permission for this asset’s folder.', status: 403)
1597
+ return embed(method, item, restricted, params) if resource == 'embed'
1598
+
1599
+ # Then whether the asset has the resource, and whether it is a still image that takes a lens
1600
+ # or transform: the preview always is, the rendered file only when it is an image.
1601
+ rendered, preview = item['content'].values_at('rendered', 'preview')
1602
+ return no_rendered_version if (resource == 'rendered' && !rendered) || (resource == 'preview' && !preview)
1603
+ # The mock prepares no stream, so it has no HLS files.
1604
+ return no_rendered_version if resource == 'hls'
1605
+
1606
+ still = resource == 'preview' || (resource == 'rendered' && rendered['kind'] == 'image')
1607
+ return invalid_transform if !still && selector
1608
+
1609
+ refused = selector ? TransformRules.check(selector).flatten : []
1610
+ return error(ErrorCodes::INVALID_TRANSFORM_OPTIONS, "Invalid transform options: #{refused.join(', ')}", status: 400) if refused.any?
1611
+
1612
+ bytes, type = resource_body(item, resource, stored)
1613
+ return not_available unless bytes
1614
+
1615
+ immutable = still || (resource == 'original' && item['type'] == 'image')
1616
+ cdn_response(method, bytes, viewer_delivery_headers(item, type, restricted, immutable, params))
1617
+ end
1618
+
1619
+ # The delivery checks, which refuse everyone, then age and access, as an error response, or nil.
1620
+ # Only the original needs the upload still stored.
1621
+ def refuse_delivery(item, resource, stored, params, authorised)
1622
+ moderation = item['classification']['moderation']['status']
1623
+ if (resource == 'original' && stored.nil?) || item['upload']['status'] == 'uploading' ||
1624
+ %w[blocked purged].include?(moderation) || item['classification']['status'] == 'failed' ||
1625
+ item['content']['status'] == 'removed'
1626
+ return not_available
1251
1627
  end
1628
+ if moderation == 'age-restricted' && !viewer_age_confirmed?(params)
1629
+ return error('viewer/age-confirmation-required', 'Age confirmation required', status: 403)
1630
+ end
1631
+ return if item['visibility'] == 'public' || authorised
1632
+
1633
+ error(ErrorCodes::PERMISSION_DENIED, 'This API key needs assets:download permission for this asset’s folder.', status: 403)
1634
+ end
1252
1635
 
1253
- refused = options ? TransformRules.check(options).flatten : []
1254
- return error('asset/invalid-transform-options', "Invalid transform options: #{refused.join(', ')}", status: 400) if refused.any?
1636
+ # A resource's bytes and Content-Type: the upload, which stands in for its own rendered file
1637
+ # and an image's preview, or the placeholder PNG for a still the mock doesn't make. Nil bytes
1638
+ # when the upload is no longer stored.
1639
+ def resource_body(item, resource, stored)
1640
+ rendered = item['content']['rendered']
1641
+ serves_upload = resource == 'original' || (resource == 'rendered' && rendered['kind'] != 'image') || item['type'] == 'image'
1642
+ return [Placeholder.png(item.dig('content', 'preview', 'thumbhash') || THUMBHASH), 'image/png'] unless serves_upload
1643
+ return [nil, nil] unless stored
1255
1644
 
1256
- type = stored['type'].to_s.empty? ? 'application/octet-stream' : stored['type']
1257
- cdn_response(method, stored['parts'].sort.map(&:last).join.b,
1258
- viewer_delivery_headers(item, type, restricted, params))
1645
+ type = if resource == 'rendered' then rendered['format']
1646
+ elsif stored['type'].to_s.empty? then 'application/octet-stream'
1647
+ else stored['type']
1648
+ end
1649
+ [stored['parts'].sort.map(&:last).join.b, type]
1259
1650
  end
1260
1651
 
1261
1652
  def viewer_age_confirmed?(params)
1262
1653
  JSON.parse(params.fetch('viewerClaims', '{}'))['ageConfirmed'] == true
1263
1654
  end
1264
1655
 
1265
- def viewer_delivery_headers(item, type, restricted, params)
1266
- headers = delivery_headers(item, type, restricted)
1656
+ def viewer_delivery_headers(item, type, restricted, immutable, params)
1657
+ headers = delivery_headers(item, type, restricted, immutable)
1267
1658
  headers['Cache-Control'] = 'private, no-store' if params.key?('viewerClaims')
1268
1659
  headers
1269
1660
  end
1270
1661
 
1271
- def invalid_transform = error('asset/invalid-transform', 'Invalid transform selector', status: 400)
1662
+ # The embed page, which shows any asset that may be delivered, as its viewer for the asset's
1663
+ # type or a notice that the type has none; cached like other non-image content.
1664
+ def embed(method, item, restricted, params)
1665
+ cache = if params.key?('viewerClaims') then 'private, no-store'
1666
+ else "#{restricted ? 'private' : 'public'}, max-age=3600"
1667
+ end
1668
+ cdn_response(method, EMBED_HTML, { 'content-type' => 'text/html; charset=utf-8', 'cache-control' => cache,
1669
+ 'cache-tag' => "asset-#{item['id']}", 'x-content-type-options' => 'nosniff' })
1670
+ end
1671
+
1672
+ def invalid_transform = error(ErrorCodes::INVALID_TRANSFORM, 'Invalid transform selector', status: 400)
1673
+
1674
+ def no_rendered_version = error('asset/no-rendered-version', 'This asset has no such resource', status: 404)
1272
1675
 
1273
1676
  def not_available = error('asset/not-available', 'Asset is not available', status: 403)
1274
1677
 
1275
- # Like the CDN's placeholder handler: an image's ThumbHash, public whatever the asset's
1678
+ # Like the CDN's placeholder handler: the preview's ThumbHash, public whatever the asset's
1276
1679
  # visibility or moderation; refused for purged assets and assets without one (not yet
1277
- # classified, or content removed).
1680
+ # processed, without a preview, or content removed).
1278
1681
  def placeholder(method, item)
1279
- thumbhash = item.dig('content', 'image', 'thumbhash')
1682
+ thumbhash = item.dig('content', 'preview', 'thumbhash')
1280
1683
  return not_available if thumbhash.nil? || item['classification']['moderation']['status'] == 'purged'
1281
1684
 
1282
1685
  cdn_response(method, Placeholder.png(thumbhash),
@@ -1284,14 +1687,13 @@ module CalmLens
1284
1687
  'cache-tag' => "asset-#{item['id']}", 'etag' => "\"#{thumbhash}\"")
1285
1688
  end
1286
1689
 
1287
- # Like the CDN: an image's bytes are immutable for a year, other content is kept for an hour,
1288
- # by shared caches only when delivery is public. Active content is served as a download.
1289
- def delivery_headers(item, type, restricted)
1290
- image = item['type'] == 'image'
1291
- headers = { 'content-type' => type,
1292
- 'cache-control' => "#{restricted ? 'private' : 'public'}, #{image ? 'max-age=31536000, immutable' : 'max-age=3600'}",
1690
+ # Like the CDN: a still image's bytes are immutable for a year, other content is kept for an
1691
+ # hour, by shared caches only when delivery is public. Active content is served as a download.
1692
+ def delivery_headers(item, type, restricted, immutable)
1693
+ lifetime = immutable ? 'max-age=31536000, immutable' : 'max-age=3600'
1694
+ headers = { 'content-type' => type, 'cache-control' => "#{restricted ? 'private' : 'public'}, #{lifetime}",
1293
1695
  'cache-tag' => "asset-#{item['id']}" }
1294
- headers.merge!('etag' => "\"#{item['id']}\"", 'expires' => Time.at((@now.call / 1000) + 31_536_000).utc.httpdate) if image
1696
+ headers.merge!('etag' => "\"#{item['id']}\"", 'expires' => Time.at((@now.call / 1000) + 31_536_000).utc.httpdate) if immutable
1295
1697
  headers.merge!('x-content-type-options' => 'nosniff', 'content-security-policy' => "sandbox; default-src 'none'")
1296
1698
  mime = type.downcase.split(';', 2).first.to_s.strip
1297
1699
  headers['content-disposition'] = 'attachment' if ACTIVE_TYPES.include?(mime) || mime.end_with?('+xml')
@@ -1303,8 +1705,11 @@ module CalmLens
1303
1705
  headers: headers.merge('content-length' => bytes.bytesize.to_s, 'x-request-id' => request_id))
1304
1706
  end
1305
1707
 
1306
- # Whether the mock API key's grants give `assets:download` for `folder_id`.
1307
- def can_download?(folder_id)
1708
+ # Whether the request carries the mock API key and its grants give `assets:download` for
1709
+ # `folder_id`.
1710
+ def can_download?(folder_id, headers)
1711
+ return false unless header(headers, 'X-Api-Key') == @api_key
1712
+
1308
1713
  @grants.any? do |grant|
1309
1714
  next false unless Array(grant['actions']).include?('assets:download')
1310
1715
  next true if grant['folderId'].nil?
@@ -1327,6 +1732,35 @@ module CalmLens
1327
1732
  end
1328
1733
  end
1329
1734
 
1735
+ # The mock's grant, the asset segment of a URL {CDN#sign_asset_url} signs on the mock: `sig_` and
1736
+ # the lowercase hex of the UTF-8 text `{assetId}.{1|0}.{expiresAt}`, its asset, original access and
1737
+ # expiry. Every SDK's mock writes it the same way. It is readable, not signed, as the mock has no
1738
+ # secret to keep.
1739
+ # @api private
1740
+ module MockGrant
1741
+ module_function
1742
+
1743
+ def encode(asset_id, original, expires_at)
1744
+ "sig_#{"#{asset_id}.#{original ? 1 : 0}.#{expires_at}".unpack1('H*')}"
1745
+ end
1746
+
1747
+ # `{asset_id:, original:, expires_at:}`, or nil for a segment the mock didn't sign.
1748
+ def decode(segment)
1749
+ hex = segment.delete_prefix('sig_')
1750
+ return unless hex.match?(/\A(?:[0-9a-f]{2})+\z/)
1751
+
1752
+ text = [hex].pack('H*').force_encoding(Encoding::UTF_8)
1753
+ match = text.valid_encoding? && /\A(.+)\.([01])\.(\d+)\z/m.match(text)
1754
+ match && { asset_id: match[1], original: match[2] == '1', expires_at: Integer(match[3], 10) }
1755
+ end
1756
+
1757
+ # Percent-decodes a path segment like decodeURIComponent; nil when it isn't valid UTF-8.
1758
+ def percent_decode(text)
1759
+ decoded = text.b.gsub(/%(\h\h)/) { Regexp.last_match(1).hex.chr }.force_encoding(Encoding::UTF_8)
1760
+ decoded if decoded.valid_encoding?
1761
+ end
1762
+ end
1763
+
1330
1764
  # The processing and moderation status of each outcome.
1331
1765
  # @api private
1332
1766
  module AssetRecord
@@ -1345,125 +1779,5 @@ module CalmLens
1345
1779
  end
1346
1780
  end
1347
1781
  end
1348
-
1349
- # The RSQL subset the fake API understands: `==` and `!=` (with `*` wildcards), `=in=` and
1350
- # `=out=` with a `(value,value)` list, and `=gt=`, `=ge=`, `=lt=`, `=le=`, with quoted values,
1351
- # combined with `;` (and), which binds tighter than `,` (or), and grouped with `( … )`.
1352
- # Anything else raises instead of being ignored, so a test never passes on an unfiltered list.
1353
- # @api private
1354
- module RSQL
1355
- module_function
1356
-
1357
- def filter(items, expression)
1358
- return items if expression.nil? || expression.to_s.strip.empty?
1359
-
1360
- test = parse(expression.to_s)
1361
- items.select { |item| test.call(item) }
1362
- end
1363
-
1364
- # Compiles `,` (or) of `;` (and) terms, where a term is a comparison or a `( … )` group.
1365
- def parse(expression)
1366
- alternatives = split(expression, ',').map do |alternative|
1367
- split(alternative, ';').map do |term|
1368
- text = term.strip
1369
- raise unsupported(expression) if text.empty?
1370
-
1371
- group = unwrap(text)
1372
- group ? parse(group) : comparison(text)
1373
- end
1374
- end
1375
- ->(item) { alternatives.any? { |terms| terms.all? { |test| test.call(item) } } }
1376
- end
1377
-
1378
- # The expression inside `( … )` when the parentheses enclose all of `text`, else nil.
1379
- def unwrap(text)
1380
- return unless text.start_with?('(') && text.end_with?(')')
1381
-
1382
- depth = 0
1383
- quote = nil
1384
- text.each_char.with_index do |char, index|
1385
- if quote
1386
- quote = nil if char == quote
1387
- elsif ['"', "'"].include?(char)
1388
- quote = char
1389
- elsif char == '('
1390
- depth += 1
1391
- elsif char == ')'
1392
- depth -= 1
1393
- return index == text.length - 1 ? text[1..-2] : nil if depth.zero?
1394
- end
1395
- end
1396
- nil
1397
- end
1398
-
1399
- def unsupported(term)
1400
- InvalidArgumentError.new(SdkErrorCodes::INVALID_ARGUMENT,
1401
- 'MockServerClient filters support ==, !=, =in=, =out=, =gt=, =ge=, =lt= and ' \
1402
- "=le=, joined by ; and , and grouped with parentheses (got #{term.inspect})")
1403
- end
1404
-
1405
- # Splits on `separator` outside quotes and parentheses (groups and `=in=(...)` lists).
1406
- def split(text, separator)
1407
- parts = ['']
1408
- quote = nil
1409
- depth = 0
1410
- text.each_char do |char|
1411
- if quote
1412
- quote = nil if char == quote
1413
- elsif ['"', "'"].include?(char)
1414
- quote = char
1415
- elsif char == '('
1416
- depth += 1
1417
- elsif char == ')'
1418
- depth -= 1
1419
- elsif char == separator && depth.zero?
1420
- parts << ''
1421
- next
1422
- end
1423
- parts[-1] += char
1424
- end
1425
- parts
1426
- end
1427
-
1428
- def comparison(term)
1429
- match = term.strip.match(/\A([A-Za-z][\w.]*)(==|!=|=in=|=out=|=gt=|=ge=|=lt=|=le=)(.+)\z/m)
1430
- raise unsupported(term) unless match
1431
-
1432
- field, operator, raw = match.captures
1433
- values = list(operator, raw.strip)
1434
- lambda do |item|
1435
- actual = field.split('.').reduce(item) { |object, key| object.is_a?(Hash) ? object[key] : nil }
1436
- entries = Array(actual.nil? ? 'null' : actual).map(&:to_s)
1437
- case operator
1438
- when '==', '=in=' then values.any? { |value| matches?(entries, value) }
1439
- when '!=', '=out=' then values.none? { |value| matches?(entries, value) }
1440
- else entries.any? { |entry| ordered?(entry, operator, values.first) }
1441
- end
1442
- end
1443
- end
1444
-
1445
- def list(operator, raw)
1446
- return [unquote(raw)] unless %w[=in= =out=].include?(operator)
1447
-
1448
- unless raw.start_with?('(') && raw.end_with?(')')
1449
- raise InvalidArgumentError.new(SdkErrorCodes::INVALID_ARGUMENT, "#{operator} takes a (value,value) list")
1450
- end
1451
-
1452
- split(raw[1..-2], ',').map { |value| unquote(value.strip) }
1453
- end
1454
-
1455
- def unquote(value) = value.match?(/\A(["']).*\1\z/m) ? value[1..-2] : value
1456
-
1457
- def matches?(entries, value)
1458
- pattern = Regexp.new("\\A#{Regexp.escape(value).gsub('\*', '.*')}\\z", Regexp::MULTILINE)
1459
- entries.any? { |entry| pattern.match?(entry) }
1460
- end
1461
-
1462
- # Numbers compare as numbers and everything else (such as ISO dates) as text.
1463
- def ordered?(entry, operator, value)
1464
- left, right = [entry, value].all? { |text| text.match?(/\A-?\d+(\.\d+)?\z/) } ? [entry.to_f, value.to_f] : [entry, value]
1465
- { '=gt=' => left > right, '=ge=' => left >= right, '=lt=' => left < right, '=le=' => left <= right }.fetch(operator)
1466
- end
1467
- end
1468
1782
  end
1469
1783
  end