calmlens 0.19.0 → 0.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +3 -1
- data/lib/calmlens/assets.rb +110 -100
- data/lib/calmlens/cdn.rb +197 -41
- data/lib/calmlens/error_codes.rb +80 -84
- data/lib/calmlens/errors.rb +2 -2
- data/lib/calmlens/events.rb +50 -1
- data/lib/calmlens/folders.rb +14 -9
- data/lib/calmlens/hash_lists.rb +29 -9
- data/lib/calmlens/internal.rb +19 -19
- data/lib/calmlens/lenses.rb +80 -6
- data/lib/calmlens/option_keys.rb +64 -20
- data/lib/calmlens/policies.rb +50 -18
- data/lib/calmlens/resource.rb +29 -5
- data/lib/calmlens/server_client.rb +8 -6
- data/lib/calmlens/session.rb +29 -12
- data/lib/calmlens/testing/asset_filters.rb +39 -0
- data/lib/calmlens/testing/classification.rb +8 -2
- data/lib/calmlens/testing/fake_api.rb +722 -348
- data/lib/calmlens/testing/mock_decision.rb +5 -4
- data/lib/calmlens/testing/rsql.rb +245 -0
- data/lib/calmlens/testing/test_input_detections.rb +61 -0
- data/lib/calmlens/testing/test_input_scenario.rb +6 -2
- data/lib/calmlens/testing/test_inputs.rb +33 -11
- data/lib/calmlens/testing.rb +40 -13
- data/lib/calmlens/transform_rules.rb +120 -0
- data/lib/calmlens/transform_schema.rb +236 -0
- data/lib/calmlens/transforms.rb +16 -0
- data/lib/calmlens/upload_mime_types.rb +51 -0
- data/lib/calmlens/upload_tokens.rb +3 -3
- data/lib/calmlens/version.rb +1 -1
- data/lib/calmlens/webhooks.rb +58 -13
- data/lib/calmlens.rb +3 -0
- data/sig/calmlens.rbs +89 -14
- data/sig/error_codes.rbs +18 -20
- data/sig/qualifiers.rbs +2 -0
- data/sig/types.rbs +326 -185
- metadata +8 -2
|
@@ -6,6 +6,7 @@ require 'securerandom'
|
|
|
6
6
|
require 'uri'
|
|
7
7
|
require_relative 'placeholder'
|
|
8
8
|
require_relative 'mock_decision'
|
|
9
|
+
require_relative 'rsql'
|
|
9
10
|
|
|
10
11
|
module CalmLens
|
|
11
12
|
module Testing
|
|
@@ -36,35 +37,69 @@ module CalmLens
|
|
|
36
37
|
:hash_list_entries, keyword_init: true)
|
|
37
38
|
|
|
38
39
|
# A stateful stand-in for the CalmLens API and CDN. It stores assets and folders, applies
|
|
39
|
-
# filters and
|
|
40
|
+
# filters and sorting, serves stored bytes, and answers errors with the API's codes, statuses,
|
|
40
41
|
# details and body shape, checking requests in the API's order. It moderates through the
|
|
41
42
|
# `moderate` callback rather than classifying content, and does not run transforms or index for
|
|
42
43
|
# semantic search.
|
|
43
44
|
# @api private
|
|
44
45
|
class FakeAPI
|
|
45
|
-
JSON_QUERY = %w[
|
|
46
|
-
context].freeze
|
|
46
|
+
JSON_QUERY = %w[wait isFinal sizeInBytes totalSizeInBytes partNumber pageSize processingOptions metadata
|
|
47
|
+
tags context].freeze
|
|
47
48
|
# Keyword search fields, in the order matches are reported, and where each value sits in the asset.
|
|
48
49
|
SEARCH_FIELDS = { 'name' => %w[name], 'id' => %w[id], 'externalId' => %w[externalId],
|
|
49
50
|
'description' => %w[description], 'tags' => %w[tags], 'type' => %w[type],
|
|
50
51
|
'content.format' => %w[content format], 'content.extension' => %w[content extension],
|
|
51
52
|
'classification.moderation.status' => %w[classification moderation status] }.freeze
|
|
52
53
|
MAX_SEARCH_TOKENS = 8
|
|
53
|
-
|
|
54
|
+
MAX_SEARCH_PAGE_SIZE = 20
|
|
54
55
|
MAX_SEARCH_QUERY_LENGTH = 200
|
|
55
56
|
# Every API cursor list returns this many items unless `pageSize` says otherwise.
|
|
56
57
|
DEFAULT_PAGE_SIZE = 50
|
|
57
58
|
MAX_PAGE_SIZE = 100
|
|
58
59
|
REVIEW_DECISIONS = %w[allowed limited age-restricted blocked purged].freeze
|
|
60
|
+
# The moderation status each `asset.<status>` event announces.
|
|
61
|
+
ANNOUNCED_STATUSES = REVIEW_DECISIONS.to_h { |status| ["asset.#{status}", status] }.freeze
|
|
59
62
|
MAX_DECISION_NOTE_LENGTH = 1499
|
|
60
|
-
|
|
63
|
+
# The actor of events the mock's API key causes: mock keys have no ID.
|
|
64
|
+
API_KEY_ACTOR = { 'type' => 'api-key', 'id' => nil }.freeze
|
|
65
|
+
# The actor of a reviewer's decision through {MockServerClient#decide}.
|
|
66
|
+
REVIEWER_ACTOR = { 'type' => 'user', 'id' => nil }.freeze
|
|
67
|
+
# The actor of the pipeline's processing events.
|
|
68
|
+
AUTOMATED_ACTOR = { 'type' => 'automated', 'id' => nil }.freeze
|
|
69
|
+
# An event type's shape (`resource.action`), as the mock checks `types` without the catalog.
|
|
70
|
+
EVENT_TYPE_PATTERN = /\A[a-z][a-z-]*\.[a-z][a-z-]*\z/
|
|
71
|
+
# The resources an asset's CDN path names.
|
|
72
|
+
RESOURCES = %w[original rendered preview hls embed placeholder info].freeze
|
|
73
|
+
# The page the mock serves at `/embed`, standing in for the CDN's embed page.
|
|
74
|
+
EMBED_HTML = '<!doctype html><html lang="en"><head><meta charset="utf-8"><title>Embed</title></head>' \
|
|
75
|
+
'<body></body></html>'
|
|
76
|
+
|
|
61
77
|
MIN_PART = 5 * 1024 * 1024
|
|
62
78
|
MAX_MULTIPART_IMAGE = 20 * 1024 * 1024
|
|
63
79
|
# Like the API: an upload has at most this many parts.
|
|
64
80
|
MAX_PARTS = 10_000
|
|
65
|
-
# The ThumbHash the mock gives every
|
|
66
|
-
# pixels. The `placeholder` resource serves it as a PNG, like the CDN.
|
|
81
|
+
# The ThumbHash the mock gives every processed asset's preview, standing in for one computed
|
|
82
|
+
# from its pixels. The `placeholder` resource serves it as a PNG, like the CDN.
|
|
67
83
|
THUMBHASH = '1QcSHQRnh493V4dIh4eXh1h4kJUI'
|
|
84
|
+
# Removing or withholding content removes its rendered file, adaptive stream and preview, as the API does.
|
|
85
|
+
NO_RENDERING = { 'rendered' => nil, 'hls' => nil, 'preview' => nil }.freeze
|
|
86
|
+
# The size of the waveform the API makes as an audio asset's preview.
|
|
87
|
+
WAVEFORM_SIZE = { 'width' => 1200, 'height' => 300 }.freeze
|
|
88
|
+
# Audio and video formats browsers play as uploaded, so the API serves them as stored (Asset.ts).
|
|
89
|
+
BROWSER_SAFE_MEDIA_FORMATS = %w[video/mp4 audio/mpeg audio/aac audio/mp4 audio/x-m4a audio/ogg].freeze
|
|
90
|
+
# Spreadsheets, which have no rendered file or preview (Asset.ts).
|
|
91
|
+
SPREADSHEET_MIME_TYPES = %w[application/vnd.openxmlformats-officedocument.spreadsheetml.sheet
|
|
92
|
+
application/vnd.ms-excel.sheet.macroenabled.12 application/vnd.ms-excel
|
|
93
|
+
application/vnd.oasis.opendocument.spreadsheet].freeze
|
|
94
|
+
# The settings of the mock's presets: every category with its default action (`allow` for
|
|
95
|
+
# personal information and profanity, `block` otherwise) and no rules. The mock holds no
|
|
96
|
+
# built-in preset settings, and uploads are never decided by a policy.
|
|
97
|
+
PRESET_SETTINGS = {
|
|
98
|
+
'categorySettings' => MockClassification::CATEGORIES.to_h do |category|
|
|
99
|
+
[category, { 'action' => %w[personal-information profanity].include?(category) ? 'allow' : 'block',
|
|
100
|
+
'allowances' => [], 'restrictions' => [] }]
|
|
101
|
+
end
|
|
102
|
+
}.freeze
|
|
68
103
|
# The built-in presets the mock knows, by display name.
|
|
69
104
|
PRESET_NAMES = { 'general-community' => 'General', 'children-platform' => 'Kids',
|
|
70
105
|
'education-support' => 'Learning and support', 'wholesome-community' => 'Teen-safe' }.freeze
|
|
@@ -77,8 +112,9 @@ module CalmLens
|
|
|
77
112
|
# The mock API key's `{ 'actions', 'folderId', 'includeSubfolders' }` grants.
|
|
78
113
|
attr_accessor :grants
|
|
79
114
|
|
|
80
|
-
def initialize(project_id:, base_url:, cdn_base_url:, now:, create_id:, moderate:, grants:)
|
|
115
|
+
def initialize(project_id:, base_url:, cdn_base_url:, now:, create_id:, moderate:, api_key:, grants:)
|
|
81
116
|
@project_id = project_id
|
|
117
|
+
@api_key = api_key
|
|
82
118
|
@base_url = base_url
|
|
83
119
|
@cdn_base_url = cdn_base_url
|
|
84
120
|
@now = now
|
|
@@ -92,22 +128,41 @@ module CalmLens
|
|
|
92
128
|
@policies = {}
|
|
93
129
|
@project_policy_id = nil
|
|
94
130
|
@request_count = 0
|
|
131
|
+
# Each asset's latest `assetSequence`, which the API keeps on the asset row.
|
|
132
|
+
@asset_event_sequences = Hash.new(0)
|
|
133
|
+
# The processing options each asset was uploaded with, which the API keeps but doesn't return.
|
|
134
|
+
@processing_options = {}
|
|
135
|
+
# Decided assets whose re-run (`assets.explain` with `rerun: true`) is still queued: they keep
|
|
136
|
+
# their decision until `complete_processing` decides them.
|
|
137
|
+
@reruns = {}
|
|
95
138
|
end
|
|
96
139
|
|
|
97
|
-
# Adds an event to the feed, filling the fields the API always sets.
|
|
140
|
+
# Adds an event to the feed, filling the fields the API always sets. Its `actor` is the mock's
|
|
141
|
+
# API key unless the event names another.
|
|
98
142
|
def add_event(event)
|
|
99
143
|
time = @now.call
|
|
100
|
-
record = { 'id' => @create_id.call('evt'), 'projectId' => @project_id, '
|
|
101
|
-
'
|
|
144
|
+
record = { 'id' => @create_id.call('evt'), 'projectId' => @project_id, 'actor' => deep_copy(API_KEY_ACTOR),
|
|
145
|
+
'severity' => 'low', 'description' => nil,
|
|
146
|
+
'assetId' => nil, 'assetSequence' => nil, 'asset' => nil, 'facts' => [],
|
|
147
|
+
'createdAt' => time }.merge(event)
|
|
102
148
|
record['title'] ||= record['type']
|
|
149
|
+
# Like the API: each event about an asset takes the asset's next number; others have none.
|
|
150
|
+
asset_id = record['assetId']
|
|
151
|
+
record['assetSequence'] = asset_id ? (@asset_event_sequences[asset_id] += 1) : nil
|
|
103
152
|
@state.events << record
|
|
104
153
|
record
|
|
105
154
|
end
|
|
106
155
|
|
|
107
|
-
# Adds an event like {#add_event}; one about a stored asset carries the asset as it is now
|
|
156
|
+
# Adds an event like {#add_event}; one about a stored asset carries the asset as it is now,
|
|
157
|
+
# with the moderation status an `asset.<status>` event announces.
|
|
108
158
|
def enqueue_event(event)
|
|
109
159
|
item = own_asset(event['assetId'])
|
|
110
|
-
|
|
160
|
+
if item && !event.key?('asset')
|
|
161
|
+
asset = snapshot(item)
|
|
162
|
+
announced = ANNOUNCED_STATUSES[event['type']]
|
|
163
|
+
asset['classification']['moderation']['status'] = announced if announced
|
|
164
|
+
event = event.merge('asset' => asset)
|
|
165
|
+
end
|
|
111
166
|
add_event(event)
|
|
112
167
|
end
|
|
113
168
|
|
|
@@ -121,25 +176,29 @@ module CalmLens
|
|
|
121
176
|
item = own_asset(asset_id)
|
|
122
177
|
return unless item
|
|
123
178
|
|
|
124
|
-
record_decision(item, decision, nil,
|
|
179
|
+
record_decision(item, decision, nil, false)
|
|
125
180
|
deep_copy(item)
|
|
126
181
|
end
|
|
127
182
|
|
|
128
183
|
# Finishes processing an asset as the pipeline would, with `decision` or else what `moderate`
|
|
129
|
-
# answers for the `complete` trigger, and returns it; nil when there is no such asset.
|
|
184
|
+
# answers for the `complete` trigger, and returns it; nil when there is no such asset. A decided
|
|
185
|
+
# asset whose re-run is still queued counts as processing.
|
|
130
186
|
def complete_processing(asset_id, decision)
|
|
131
187
|
item = own_asset(asset_id)
|
|
132
188
|
return unless item
|
|
133
|
-
raise Internal.invalid("Asset #{asset_id} isn't processing") unless item['classification']['status'] == 'processing'
|
|
134
189
|
|
|
135
|
-
|
|
190
|
+
rerun = @reruns.key?(asset_id)
|
|
191
|
+
raise Internal.invalid("Asset #{asset_id} isn't processing") unless rerun || item['classification']['status'] == 'processing'
|
|
192
|
+
|
|
193
|
+
settled = decision.nil? ? nil : MockDecision.from(decision)
|
|
194
|
+
rerun ? settle_rerun(item, 'complete', settled) : moderate_stored(item, 'complete', settled)
|
|
136
195
|
deep_copy(item)
|
|
137
196
|
end
|
|
138
197
|
|
|
139
198
|
# @return [Response]
|
|
140
199
|
def handle(method, url, headers, body)
|
|
141
200
|
if url.start_with?("#{@cdn_base_url}/")
|
|
142
|
-
serve(method, url.delete_prefix(@cdn_base_url))
|
|
201
|
+
serve(method, url.delete_prefix(@cdn_base_url), headers)
|
|
143
202
|
else
|
|
144
203
|
api(method, url.delete_prefix(@base_url), headers, body)
|
|
145
204
|
end
|
|
@@ -172,29 +231,29 @@ module CalmLens
|
|
|
172
231
|
end
|
|
173
232
|
|
|
174
233
|
# Applies a reviewer's decision like `decideAssetManually` in the API: the status and the review,
|
|
175
|
-
# then (for `purged`) the content removal and its event, then `asset.<decision>`.
|
|
176
|
-
|
|
234
|
+
# then (for `purged`) the content removal and its event, then `asset.<decision>`. Either way the
|
|
235
|
+
# review is the mock API key's, which has no ID. A decision through the API (`by_api_key`) has
|
|
236
|
+
# the API key as its events' actor; the test hook's is a reviewer's (a `user`).
|
|
237
|
+
def record_decision(item, decision, note, by_api_key)
|
|
238
|
+
# Like the API: a queued re-run no longer replaces a reviewed decision.
|
|
239
|
+
@reruns.delete(item['id'])
|
|
177
240
|
apply_outcome(item, decision, 'requested')
|
|
178
241
|
review = { 'id' => @create_id.call('rev'), 'decision' => decision, 'note' => note,
|
|
179
|
-
'reviewedAt' => @now.call, '
|
|
180
|
-
|
|
181
|
-
if reviewer_name
|
|
182
|
-
review.merge!('reviewerName' => reviewer_name, 'reviewerEmail' => nil)
|
|
183
|
-
reviewer_facts = [{ 'key' => 'reviewer-user-id', 'value' => nil }, { 'key' => 'reviewer-name', 'value' => reviewer_name }]
|
|
184
|
-
end
|
|
242
|
+
'decidedBy' => { 'type' => 'api-key', 'id' => nil }, 'reviewedAt' => @now.call, 'applied' => true }
|
|
243
|
+
actor = by_api_key ? API_KEY_ACTOR : REVIEWER_ACTOR
|
|
185
244
|
item['classification']['moderation']['manualReviews'] << review
|
|
186
|
-
lifecycle_event('asset.content-removed', item) if decision == 'purged'
|
|
245
|
+
lifecycle_event('asset.content-removed', item, actor: actor) if decision == 'purged'
|
|
187
246
|
# Like the event catalog: only allowing and limiting are low severity.
|
|
188
247
|
add_event('type' => "asset.#{decision}", 'severity' => %w[allowed limited].include?(decision) ? 'low' : 'high',
|
|
189
248
|
'title' => "asset.#{decision}", 'assetId' => item['id'], 'asset' => snapshot(item),
|
|
190
|
-
'
|
|
249
|
+
'actor' => deep_copy(actor),
|
|
250
|
+
'facts' => [{ 'key' => 'asset-type', 'value' => item['type'] },
|
|
191
251
|
{ 'key' => 'asset-name', 'value' => item['name'], 'url' => "https://calmlens.com/go/asset/#{item['id']}" },
|
|
192
|
-
|
|
252
|
+
{ 'key' => 'note', 'value' => note }])
|
|
193
253
|
end
|
|
194
254
|
|
|
195
|
-
# `POST /assets/:id/decision` as the API validates and applies it
|
|
196
|
-
#
|
|
197
|
-
# refused with `asset/processing`.
|
|
255
|
+
# `POST /assets/:id/decision` as the API validates and applies it, with the API key as its
|
|
256
|
+
# events' actor. An asset still uploading or processing is refused with `asset/processing`.
|
|
198
257
|
def decide_request(item, json)
|
|
199
258
|
decision = json['decision']
|
|
200
259
|
invalid!('validation/invalid-union', 'decision') unless REVIEW_DECISIONS.include?(decision)
|
|
@@ -211,7 +270,7 @@ module CalmLens
|
|
|
211
270
|
fail!(ErrorCodes::ASSET_DECISION_UNCHANGED) if status == decision
|
|
212
271
|
fail!(ErrorCodes::ASSET_CONTENT_PURGED) if status == 'purged' && decision != 'blocked'
|
|
213
272
|
|
|
214
|
-
record_decision(item, decision, note,
|
|
273
|
+
record_decision(item, decision, note, true)
|
|
215
274
|
item
|
|
216
275
|
end
|
|
217
276
|
|
|
@@ -219,26 +278,66 @@ module CalmLens
|
|
|
219
278
|
def snapshot(item)
|
|
220
279
|
{ 'id' => item['id'], 'externalId' => item['externalId'], 'uploadPresetId' => item['uploadPresetId'],
|
|
221
280
|
'upload' => { 'status' => item['upload']['status'] },
|
|
222
|
-
'classification' => { 'status' => item['classification']['status'],
|
|
281
|
+
'classification' => { 'status' => item['classification']['status'], 'testInput' => item['classification']['testInput'],
|
|
223
282
|
'moderation' => { 'status' => item['classification']['moderation']['status'] } },
|
|
224
283
|
'content' => { 'status' => item['content']['status'] } }
|
|
225
284
|
end
|
|
226
285
|
|
|
227
|
-
# The `asset.deleted`, `asset.content-removed` and `asset.retried` events the API records
|
|
228
|
-
|
|
286
|
+
# The `asset.deleted`, `asset.content-removed` and `asset.retried` events the API records, with
|
|
287
|
+
# the `batch-id` fact of the batch that caused them, if any.
|
|
288
|
+
def lifecycle_event(type, item, batch_id: nil, actor: API_KEY_ACTOR)
|
|
229
289
|
name = { 'key' => 'asset-name', 'value' => item['name'] }
|
|
230
290
|
# A deleted asset has no portal page to link to.
|
|
231
291
|
name['url'] = "https://calmlens.com/go/asset/#{item['id']}" unless type == 'asset.deleted'
|
|
292
|
+
facts = [name, { 'key' => 'asset-type', 'value' => item['type'] }]
|
|
293
|
+
facts << { 'key' => 'batch-id', 'value' => batch_id } if batch_id
|
|
232
294
|
add_event('type' => type, 'severity' => 'low', 'assetId' => item['id'], 'asset' => snapshot(item),
|
|
233
|
-
'
|
|
295
|
+
'actor' => deep_copy(actor), 'facts' => facts)
|
|
234
296
|
end
|
|
235
297
|
|
|
236
298
|
def fail!(code, message = nil, details: nil) = raise(Failure.new(code, message, details: details))
|
|
237
299
|
|
|
238
300
|
def route_not_found!(method, parts) = fail!(ErrorCodes::ROUTE_NOT_FOUND, "No route for #{method} /#{parts.join('/')}")
|
|
239
301
|
|
|
240
|
-
# A request the API's schema refuses: `validation
|
|
241
|
-
|
|
302
|
+
# A request the API's schema refuses: `validation/invalid-request` with HTTP 400, listing the
|
|
303
|
+
# field and its issue code (such as `validation/too-big`) in `details.issues`.
|
|
304
|
+
def invalid!(code, path)
|
|
305
|
+
message = "Invalid #{path}"
|
|
306
|
+
raise Failure.new(ErrorCodes::INVALID_REQUEST, message,
|
|
307
|
+
details: { 'issues' => [{ 'path' => path, 'message' => message, 'code' => code }] }, status: 400)
|
|
308
|
+
end
|
|
309
|
+
|
|
310
|
+
# Like the API's schema (METADATA_ITEM_SCHEMA): metadata is nil or a list of `{id, value}`
|
|
311
|
+
# Hashes with a String `id` and any JSON `value`.
|
|
312
|
+
def check_metadata(metadata)
|
|
313
|
+
return if metadata.nil?
|
|
314
|
+
|
|
315
|
+
invalid!('validation/invalid-type', 'metadata') unless metadata.is_a?(Array)
|
|
316
|
+
metadata.each_with_index do |item, index|
|
|
317
|
+
invalid!('validation/invalid-type', "metadata.#{index}") unless item.is_a?(Hash)
|
|
318
|
+
invalid!('validation/invalid-type', "metadata.#{index}.id") unless item['id'].is_a?(String)
|
|
319
|
+
invalid!('validation/invalid-value', "metadata.#{index}.value") unless item.key?('value')
|
|
320
|
+
end
|
|
321
|
+
end
|
|
322
|
+
|
|
323
|
+
# Like the API's schema, checked before the asset is looked up: asset updates take no null name.
|
|
324
|
+
def check_asset_body(method, action, json)
|
|
325
|
+
return unless method == 'PATCH' && action.nil?
|
|
326
|
+
|
|
327
|
+
check_metadata(json['metadata'])
|
|
328
|
+
invalid!('validation/invalid-type', 'name') if json.key?('name') && json['name'].nil?
|
|
329
|
+
end
|
|
330
|
+
|
|
331
|
+
# Like the API: an upload's metadata is checked, then its MIME type, which must be one the
|
|
332
|
+
# API can moderate (`upload/unsupported-type` with `details.mimeType` and `details.source`).
|
|
333
|
+
def check_upload(options)
|
|
334
|
+
check_metadata(options['metadata'])
|
|
335
|
+
type = options['contentType']
|
|
336
|
+
return if type.nil? || Internal.accepted_upload_mime_type?(type)
|
|
337
|
+
|
|
338
|
+
fail!(ErrorCodes::UNSUPPORTED_UPLOAD_TYPE, "#{type} is not a valid upload file type",
|
|
339
|
+
details: { 'mimeType' => type, 'source' => 'calmlens' })
|
|
340
|
+
end
|
|
242
341
|
|
|
243
342
|
# Like the API's query schema: an integer from 1 to `max`.
|
|
244
343
|
def check_range(value, path, max)
|
|
@@ -253,10 +352,11 @@ module CalmLens
|
|
|
253
352
|
parts = uri.path.split('/').reject(&:empty?).map { |part| URI.decode_www_form_component(part) }
|
|
254
353
|
query = URI.decode_www_form(uri.query.to_s).to_h.to_h { |key, value| [key, query_value(key, value)] }
|
|
255
354
|
content_type = header(headers, 'Content-Type')
|
|
256
|
-
|
|
355
|
+
upload = header(headers, 'X-CalmLens-Upload')
|
|
356
|
+
# A raw upload's body is the file itself, whatever its type.
|
|
357
|
+
json = content_type == 'application/json' && body && upload.nil? ? JSON.parse(body) : {}
|
|
257
358
|
# HTTP stacks trim header values, so the API sees the key without surrounding whitespace.
|
|
258
359
|
key = header(headers, 'Idempotency-Key')&.to_s&.gsub(/\A[ \t\r\n]+|[ \t\r\n]+\z/, '')
|
|
259
|
-
upload = header(headers, 'X-CalmLens-Upload')
|
|
260
360
|
return dispatch(method, parts, query, json, body, upload) unless key && method == 'POST'
|
|
261
361
|
|
|
262
362
|
fail!(ErrorCodes::INVALID_IDEMPOTENCY_KEY) unless key.match?(IDEMPOTENCY_KEY)
|
|
@@ -294,6 +394,8 @@ module CalmLens
|
|
|
294
394
|
when 'folders' then folder(method, parts, json)
|
|
295
395
|
when 'hash-lists' then hash_list(method, parts, query, json)
|
|
296
396
|
when 'hash-list-entries' then remove_hash_list_entry(method, parts)
|
|
397
|
+
when 'policies' then policies(method, parts, query, json)
|
|
398
|
+
when 'lenses' then lens(method, parts, json)
|
|
297
399
|
else route_not_found!(method, parts)
|
|
298
400
|
end
|
|
299
401
|
data.is_a?(Response) ? data : respond(200, deep_copy(data))
|
|
@@ -308,19 +410,21 @@ module CalmLens
|
|
|
308
410
|
case [method, parts[2..], upload]
|
|
309
411
|
in ['GET', ['events'], _] then events(query)
|
|
310
412
|
in ['PUT', ['lenses'], _] then lenses(json.fetch('lenses', []))
|
|
413
|
+
in ['GET', ['lenses'], _] then page(@state.lenses.values, query, JSON.generate(['lenses', @project_id]))
|
|
414
|
+
in ['POST', ['lenses'], _] then create_lens(json)
|
|
311
415
|
in ['GET', ['folders'], _] then folder_page(query)
|
|
312
416
|
in ['POST', ['folders'], _] then create_folder(json)
|
|
313
417
|
in ['GET', ['hash-lists'], _] then page(project_hash_lists, query, JSON.generate([@project_id]))
|
|
314
418
|
in ['POST', ['hash-lists'], _] then create_hash_list(json)
|
|
315
|
-
in ['GET', ['
|
|
316
|
-
in ['POST', ['policies'], _] then create_policy(json)
|
|
419
|
+
in ['GET', ['policy'], _] then show_policy(find_policy(ensure_policies))
|
|
317
420
|
in ['PUT', ['policy'], _] then show_policy(assign_policy(json))
|
|
318
|
-
in [_, ['policies', String => policy_id, *rest], _] then policy(method, policy_id, rest, json)
|
|
319
421
|
in ['GET', ['assets'], _] then list(query)
|
|
320
422
|
in ['GET', ['assets', 'search'], _] then search(query)
|
|
423
|
+
in ['GET', ['assets', String => asset_id, 'timeline'], _] then timeline(asset_id, query)
|
|
321
424
|
in ['POST', ['assets', 'batch'], _] then batch(json)
|
|
322
|
-
in ['POST', ['
|
|
323
|
-
in ['POST', ['assets'],
|
|
425
|
+
in ['POST', ['signed-asset-urls'], _] then sign_asset_url(json)
|
|
426
|
+
in ['POST', ['assets'], nil] then create_asset(json, query['wait'] == false, url: json['url'])
|
|
427
|
+
in ['POST', ['assets'], 'content'] then create_asset(query, query['wait'] == false, data: body)
|
|
324
428
|
in ['POST', ['assets'], 'part'] then start_multipart(query, body)
|
|
325
429
|
else route_not_found!(method, parts)
|
|
326
430
|
end
|
|
@@ -337,7 +441,8 @@ module CalmLens
|
|
|
337
441
|
# (`asset/not-found`) before anything happens. The operation then runs at once, in request
|
|
338
442
|
# order, rather than in the background: `delete` deletes each asset with its stored child
|
|
339
443
|
# assets and records `asset.deleted` with the asset as it was; `remove-content` removes the
|
|
340
|
-
# content like DELETE /assets/:id/content.
|
|
444
|
+
# content like DELETE /assets/:id/content. Each event carries the batch's `batch-id` fact. Mock
|
|
445
|
+
# assets carry no safety holds.
|
|
341
446
|
def batch(json)
|
|
342
447
|
ids = json['assetIds']
|
|
343
448
|
invalid!('validation/invalid-type', 'assetIds') unless ids.is_a?(Array)
|
|
@@ -352,15 +457,16 @@ module CalmLens
|
|
|
352
457
|
fail!(ErrorCodes::ASSET_NOT_FOUND) unless item && item['projectId'] == @project_id
|
|
353
458
|
item
|
|
354
459
|
end
|
|
460
|
+
batch_id = @create_id.call('batch')
|
|
355
461
|
items.each do |item|
|
|
356
462
|
if json['operation'] == 'delete'
|
|
357
463
|
delete_tree(item)
|
|
358
|
-
lifecycle_event('asset.deleted', item)
|
|
464
|
+
lifecycle_event('asset.deleted', item, batch_id: batch_id)
|
|
359
465
|
else
|
|
360
|
-
|
|
466
|
+
remove_content(item, batch_id)
|
|
361
467
|
end
|
|
362
468
|
end
|
|
363
|
-
respond(202, { 'queued' =>
|
|
469
|
+
respond(202, { 'batchId' => batch_id, 'status' => 'queued', 'assetCount' => items.size })
|
|
364
470
|
end
|
|
365
471
|
|
|
366
472
|
# Deletes an asset, its stored child assets and their content.
|
|
@@ -375,26 +481,26 @@ module CalmLens
|
|
|
375
481
|
deleting.each do |id|
|
|
376
482
|
@state.assets.delete(id)
|
|
377
483
|
@state.content.delete(id)
|
|
484
|
+
@processing_options.delete(id)
|
|
378
485
|
end
|
|
379
486
|
end
|
|
380
487
|
|
|
381
488
|
def list(query)
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
page(RSQL.filter(
|
|
489
|
+
# Like the API: the query schema is checked before the filter.
|
|
490
|
+
page_query(query)
|
|
491
|
+
page(RSQL.filter(project_assets, query['filter']), query, JSON.generate([@project_id, query['filter']]))
|
|
385
492
|
end
|
|
386
493
|
|
|
387
494
|
# Like the API: one keyset page of a parent's child folders, optionally matching `search`, by
|
|
388
|
-
# `name` ascending unless `
|
|
495
|
+
# `name` ascending unless `sort` (one field, `name` or `createdAt`) says otherwise, with an
|
|
389
496
|
# `id` tie-breaker, and the ancestors of the parent and the listed folders. Names compare by
|
|
390
497
|
# code point, so case matters.
|
|
391
498
|
def folder_page(query)
|
|
392
499
|
found = project_folders.select do |f|
|
|
393
500
|
f['parentId'] == query['parentId'] && (!query['search'] || f['name'].downcase.include?(query['search'].downcase))
|
|
394
501
|
end
|
|
395
|
-
invalid!('validation/too-big', 'ordering') if ordering(query, nil).size > 1
|
|
396
502
|
scope = JSON.generate([@project_id, query['parentId'], query['search']])
|
|
397
|
-
result = page(found, query, scope, fields: %w[name createdAt], default: { 'field' => 'name', 'direction' => 'asc' })
|
|
503
|
+
result = page(found, query, scope, fields: %w[name createdAt], max: 1, default: { 'field' => 'name', 'direction' => 'asc' })
|
|
398
504
|
page = result['items']
|
|
399
505
|
ids = ([query['parentId']] + page.map { |f| f['id'] }).compact.uniq
|
|
400
506
|
queue = ids.dup
|
|
@@ -411,46 +517,85 @@ module CalmLens
|
|
|
411
517
|
def project_folders = @state.folders.values.select { |item| item['projectId'] == @project_id }
|
|
412
518
|
|
|
413
519
|
# Like the API: without a cursor, no events and a `nextCursor` for now; with one, up to `pageSize`
|
|
414
|
-
# events added since, oldest first
|
|
415
|
-
#
|
|
520
|
+
# events added since, oldest first, of the `types` (comma-separated) and `assetId` asked for.
|
|
521
|
+
# `nextCursor` is never nil, and a cursor belongs to the filters it was issued with. The mock's
|
|
522
|
+
# key holds every permission, so no event type is excluded.
|
|
416
523
|
def events(query)
|
|
417
524
|
size = check_range(query.fetch('pageSize', DEFAULT_PAGE_SIZE), 'pageSize', MAX_PAGE_SIZE)
|
|
525
|
+
types = query.key?('types') ? query['types'].to_s.split(',', -1) : []
|
|
526
|
+
if query.key?('types') && (types.empty? || types.any? { |type| !type.match?(EVENT_TYPE_PATTERN) })
|
|
527
|
+
invalid!('validation/invalid-value', 'types')
|
|
528
|
+
end
|
|
529
|
+
asset_id = query['assetId']
|
|
530
|
+
invalid!('validation/invalid-format', 'assetId') if asset_id && !asset_id.match?(ASSET_ID_PATTERN)
|
|
531
|
+
filter = [JSON.generate([types.uniq.sort, asset_id])].pack('m0')
|
|
532
|
+
matches = ->(event) { (types.empty? || types.include?(event['type'])) && (asset_id.nil? || event['assetId'] == asset_id) }
|
|
418
533
|
feed = @state.events
|
|
419
|
-
|
|
534
|
+
unless query['cursor']
|
|
535
|
+
return { 'items' => [], 'nextCursor' => "mock_#{feed.size}~#{filter}", 'hasMore' => false, 'excludedTypes' => [] }
|
|
536
|
+
end
|
|
420
537
|
|
|
421
|
-
match = query['cursor'].to_s.match(/\Amock_(\d+)\z/)
|
|
422
|
-
position = match && Integer(match[1], 10)
|
|
538
|
+
match = query['cursor'].to_s.match(/\Amock_(\d+)~(.*)\z/)
|
|
539
|
+
position = match && match[2] == filter ? Integer(match[1], 10) : nil
|
|
423
540
|
fail!(ErrorCodes::INVALID_CURSOR) unless position && position <= feed.size
|
|
424
541
|
|
|
425
|
-
taken =
|
|
426
|
-
following = position
|
|
427
|
-
|
|
542
|
+
taken = []
|
|
543
|
+
following = position
|
|
544
|
+
while following < feed.size && taken.size < size
|
|
545
|
+
event = feed[following]
|
|
546
|
+
following += 1
|
|
547
|
+
taken << event if matches.call(event)
|
|
548
|
+
end
|
|
549
|
+
{ 'items' => taken, 'nextCursor' => "mock_#{following}~#{filter}", 'hasMore' => feed.drop(following).any?(&matches),
|
|
550
|
+
'excludedTypes' => [] }
|
|
428
551
|
end
|
|
429
552
|
|
|
430
|
-
# Saves lenses in order, like the API:
|
|
431
|
-
#
|
|
553
|
+
# Saves lenses in order, like the API: a new identifier creates a lens, an unchanged
|
|
554
|
+
# definition returns the stored lens and a changed one updates it in place.
|
|
432
555
|
def lenses(definitions)
|
|
433
556
|
{ 'lenses' => definitions.map { |definition| save_lens(definition) } }
|
|
434
557
|
end
|
|
435
558
|
|
|
436
559
|
def save_lens(definition)
|
|
437
|
-
now = @now.call
|
|
438
560
|
old = @state.lenses[definition['identifier']]
|
|
439
|
-
unless old
|
|
440
|
-
lens = { 'id' => @create_id.call('lens'), 'projectId' => @project_id }.merge(definition)
|
|
441
|
-
return @state.lenses[definition['identifier']] = lens.merge('createdAt' => now, 'updatedAt' => now)
|
|
442
|
-
end
|
|
561
|
+
return new_lens(definition) unless old
|
|
443
562
|
|
|
444
563
|
same = %w[name type active].all? { |key| old[key] == definition[key] } &&
|
|
445
564
|
old['description'] == definition['description'] && canonical(old['transform']) == canonical(definition['transform'])
|
|
446
|
-
return old if same
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
end
|
|
565
|
+
return old if same
|
|
566
|
+
|
|
567
|
+
@state.lenses[old['identifier']] = old.merge(definition).merge('updatedAt' => @now.call)
|
|
568
|
+
end
|
|
451
569
|
|
|
452
|
-
|
|
453
|
-
|
|
570
|
+
def new_lens(definition)
|
|
571
|
+
now = @now.call
|
|
572
|
+
lens = { 'id' => @create_id.call('lens'), 'projectId' => @project_id }.merge(definition)
|
|
573
|
+
@state.lenses[definition['identifier']] = lens.merge('createdAt' => now, 'updatedAt' => now)
|
|
574
|
+
end
|
|
575
|
+
|
|
576
|
+
# POST /projects/:id/lenses: like the API, the identifier must be new to the project.
|
|
577
|
+
def create_lens(json)
|
|
578
|
+
identifier = json['identifier']
|
|
579
|
+
invalid!('validation/invalid-type', 'identifier') unless identifier.is_a?(String) && !identifier.empty?
|
|
580
|
+
fail!(ErrorCodes::LENS_IDENTIFIER_TAKEN, details: { 'identifier' => identifier }) if @state.lenses.key?(identifier)
|
|
581
|
+
|
|
582
|
+
# Like the API, a new lens is 201 Created.
|
|
583
|
+
respond(201, deep_copy(new_lens(json)))
|
|
584
|
+
end
|
|
585
|
+
|
|
586
|
+
# GET, PATCH and DELETE /lenses/:id. An update keeps the identifier and changes the rest of
|
|
587
|
+
# the definition, as ensuring the changed definition would.
|
|
588
|
+
def lens(method, parts, json)
|
|
589
|
+
item = @state.lenses.values.find { |lens| lens['id'] == parts[1] && lens['projectId'] == @project_id }
|
|
590
|
+
fail!(ErrorCodes::LENS_NOT_FOUND) unless item && parts.size == 2
|
|
591
|
+
case method
|
|
592
|
+
when 'GET' then item
|
|
593
|
+
when 'PATCH' then save_lens(item.merge(json.except('identifier', 'id', 'projectId', 'createdAt', 'updatedAt')))
|
|
594
|
+
when 'DELETE'
|
|
595
|
+
@state.lenses.delete(item['identifier'])
|
|
596
|
+
{}
|
|
597
|
+
else route_not_found!(method, parts)
|
|
598
|
+
end
|
|
454
599
|
end
|
|
455
600
|
|
|
456
601
|
def canonical(value)
|
|
@@ -467,7 +612,7 @@ module CalmLens
|
|
|
467
612
|
text = query['query'].to_s.strip
|
|
468
613
|
invalid!('validation/too-small', 'query') if text.empty?
|
|
469
614
|
invalid!('validation/too-big', 'query') if text.length > MAX_SEARCH_QUERY_LENGTH
|
|
470
|
-
limit = check_range(query.fetch('
|
|
615
|
+
limit = check_range(query.fetch('pageSize', 10), 'pageSize', MAX_SEARCH_PAGE_SIZE)
|
|
471
616
|
normalized = text.downcase
|
|
472
617
|
tokens = normalized.split(/\s+/).uniq.first(MAX_SEARCH_TOKENS)
|
|
473
618
|
items = project_assets.filter_map do |item|
|
|
@@ -507,6 +652,7 @@ module CalmLens
|
|
|
507
652
|
end
|
|
508
653
|
|
|
509
654
|
def create_asset(options, queue, url: nil, data: nil, open: false)
|
|
655
|
+
check_upload(options)
|
|
510
656
|
check_folder(options['folderId']) if options['folderId']
|
|
511
657
|
if options['externalId'] && project_assets.any? { |item| item['externalId'] == options['externalId'] }
|
|
512
658
|
fail!(ErrorCodes::DUPLICATE_EXTERNAL_ID)
|
|
@@ -521,13 +667,17 @@ module CalmLens
|
|
|
521
667
|
'folderId' => options['folderId'], 'uploadPresetId' => options['uploadPresetId'],
|
|
522
668
|
'createdAt' => now, 'updatedAt' => now, 'name' => options['name'], 'description' => options['description'],
|
|
523
669
|
'externalId' => options['externalId'], 'externalUrl' => url || options['externalUrl'],
|
|
524
|
-
'type' => kind(mime, url), 'metadata' => options['metadata'], 'authorId' => options['authorId'],
|
|
670
|
+
'type' => kind(mime, url), 'metadata' => options['metadata'] || [], 'authorId' => options['authorId'],
|
|
525
671
|
# Like the API, only the posts CalmLens uses are kept, and only while the content is held.
|
|
526
|
-
'context' => context.empty? ? nil : { 'items' => context }, 'tags' =>
|
|
527
|
-
'
|
|
672
|
+
'context' => context.empty? ? nil : { 'items' => context }, 'tags' => options['tags'] || [],
|
|
673
|
+
'visibility' => options.fetch('visibility', 'private'),
|
|
528
674
|
**sections(now: now, open: open, keep: keep, format: mime, size: data&.bytesize) }
|
|
529
675
|
@state.assets[asset['id']] = asset
|
|
530
|
-
@
|
|
676
|
+
@processing_options[asset['id']] = processing
|
|
677
|
+
if data
|
|
678
|
+
@state.content[asset['id']] = { 'type' => mime, 'parts' => { 1 => data } }
|
|
679
|
+
asset['content'].merge!(rendering(asset, false)) unless open
|
|
680
|
+
end
|
|
531
681
|
open ? asset : upload_completed(asset, queue, data)
|
|
532
682
|
end
|
|
533
683
|
|
|
@@ -535,20 +685,29 @@ module CalmLens
|
|
|
535
685
|
{
|
|
536
686
|
'content' => { 'status' => keep ? 'stored' : 'temporary', 'removalReason' => nil, 'updatedAt' => now,
|
|
537
687
|
'safetyHold' => nil, 'format' => format.empty? ? nil : format, 'extension' => nil,
|
|
538
|
-
|
|
539
|
-
|
|
688
|
+
# The mock decodes nothing, so the file's own details are unknown, and like
|
|
689
|
+
# the API, the rendered file and preview come with processing.
|
|
690
|
+
'sizeInBytes' => size, 'checksums' => nil, 'width' => nil, 'height' => nil,
|
|
691
|
+
'durationSeconds' => nil, 'pageCount' => nil, 'perceptualHash' => nil,
|
|
692
|
+
'perceptualHashQuality' => nil, 'metadata' => nil, 'extractedText' => nil, **NO_RENDERING },
|
|
540
693
|
# Like the API, only an open upload counts the parts received.
|
|
541
694
|
'upload' => { 'status' => open ? 'uploading' : 'completed', 'updatedAt' => now, 'partsReceived' => open ? 1 : nil },
|
|
542
|
-
'classification' => { 'status' => open ? '
|
|
543
|
-
|
|
544
|
-
'
|
|
545
|
-
|
|
695
|
+
'classification' => { 'status' => open ? 'queued' : 'processing', 'failureReason' => nil, 'updatedAt' => now,
|
|
696
|
+
# The mock never reads test inputs.
|
|
697
|
+
'completedAt' => nil, 'progress' => nil, 'testInput' => false,
|
|
698
|
+
# Like the API, moderation's results are null until it decides, and a
|
|
699
|
+
# feature's result is null until it has run.
|
|
700
|
+
'moderation' => { 'status' => 'pending', 'manualReviews' => [], 'automatedStatus' => nil,
|
|
701
|
+
'source' => nil, 'rule' => nil, 'explanationComplete' => nil,
|
|
702
|
+
'categories' => nil, 'flaggedCategories' => [], 'regions' => [],
|
|
703
|
+
'raw' => nil, 'timeline' => nil },
|
|
704
|
+
'ocr' => nil, 'linkDetection' => nil }
|
|
546
705
|
}
|
|
547
706
|
end
|
|
548
707
|
|
|
549
708
|
# Like the API: image uploads in parts total at most 20 MiB, at any size below that.
|
|
550
709
|
def image_size_error(type, size, _final)
|
|
551
|
-
ErrorCodes::
|
|
710
|
+
ErrorCodes::UPLOAD_TOO_LARGE if type == 'image' && size > MAX_MULTIPART_IMAGE
|
|
552
711
|
end
|
|
553
712
|
|
|
554
713
|
# Like the API: the parts may not overshoot a declared total, and must reach it at the end.
|
|
@@ -558,6 +717,12 @@ module CalmLens
|
|
|
558
717
|
ErrorCodes::UPLOAD_SIZE_MISMATCH if received > total || (final && received != total)
|
|
559
718
|
end
|
|
560
719
|
|
|
720
|
+
# The failure for a size error code, with the limit `upload/too-large` carries.
|
|
721
|
+
def size_failure!(code)
|
|
722
|
+
details = code == ErrorCodes::UPLOAD_TOO_LARGE ? { 'limitBytes' => MAX_MULTIPART_IMAGE, 'source' => 'plan' } : nil
|
|
723
|
+
fail!(code, details: details)
|
|
724
|
+
end
|
|
725
|
+
|
|
561
726
|
def check_part_size(size, final)
|
|
562
727
|
return if final || size >= MIN_PART
|
|
563
728
|
|
|
@@ -565,20 +730,21 @@ module CalmLens
|
|
|
565
730
|
end
|
|
566
731
|
|
|
567
732
|
# In the API's order: a declared total, then a single final part as the whole upload, then the
|
|
568
|
-
# part itself.
|
|
733
|
+
# part itself. Whether to wait and the declared total are saved with an open upload.
|
|
569
734
|
def start_multipart(query, data)
|
|
735
|
+
check_upload(query)
|
|
570
736
|
total = query['totalSizeInBytes']
|
|
571
737
|
type = kind(query['contentType'].to_s, nil)
|
|
572
738
|
final = query['isFinal'] == true
|
|
573
739
|
size = data.to_s.bytesize
|
|
574
740
|
code = (total && image_size_error(type, total, true)) ||
|
|
575
741
|
(final ? total_size_error(total, size, true) || image_size_error(type, size, true) : nil)
|
|
576
|
-
|
|
742
|
+
size_failure!(code) if code
|
|
577
743
|
check_part_size(size, final)
|
|
578
744
|
fail!(total_size_error(total, size, false)) if !final && total_size_error(total, size, false)
|
|
579
745
|
|
|
580
|
-
asset = create_asset(query, query['
|
|
581
|
-
@state.uploads[asset['id']] = { 'queue' => query['
|
|
746
|
+
asset = create_asset(query, query['wait'] == false, data: data, open: !final)
|
|
747
|
+
@state.uploads[asset['id']] = { 'queue' => query['wait'] == false, 'total' => total } unless final
|
|
582
748
|
asset
|
|
583
749
|
end
|
|
584
750
|
|
|
@@ -614,8 +780,26 @@ module CalmLens
|
|
|
614
780
|
# Without `moderate`, queued work stays pending until completed, which allows it.
|
|
615
781
|
def default_moderation(trigger, queued) = queued && trigger != 'complete' ? 'pending' : 'allowed'
|
|
616
782
|
|
|
617
|
-
# Applies `decision` to `asset` with its report.
|
|
618
|
-
|
|
783
|
+
# Applies `decision` to `asset` with its report. The mock reads no text, QR codes, people or
|
|
784
|
+
# faces in an image, so a decided image reports that the features its upload ran found nothing:
|
|
785
|
+
# OCR and person and face detection when turned on, link detection unless turned off.
|
|
786
|
+
def decided(asset, decision)
|
|
787
|
+
apply_outcome(asset, decision.status, 'policy', decision.report)
|
|
788
|
+
classification = asset['classification']
|
|
789
|
+
return asset unless classification['status'] == 'completed' && asset['type'] == 'image'
|
|
790
|
+
|
|
791
|
+
processing = @processing_options[asset['id']]
|
|
792
|
+
features = processing.is_a?(Hash) && processing['features'].is_a?(Hash) ? processing['features'] : {}
|
|
793
|
+
setting = ->(name) { features[name].is_a?(Hash) ? features[name]['enabled'] : nil }
|
|
794
|
+
classification['ocr'] = { 'status' => 'evaluated', 'textRegions' => [] } if setting.call('ocr') == true
|
|
795
|
+
unless setting.call('linkDetection') == false
|
|
796
|
+
classification['linkDetection'] = { 'status' => 'evaluated', 'qrCodes' => [], 'links' => [] }
|
|
797
|
+
end
|
|
798
|
+
{ 'personDetection' => 'people', 'faceDetection' => 'faces' }.each do |feature, list|
|
|
799
|
+
classification[feature] = { 'status' => 'evaluated', list => [] } if setting.call(feature) == true
|
|
800
|
+
end
|
|
801
|
+
asset
|
|
802
|
+
end
|
|
619
803
|
|
|
620
804
|
# Asks `moderate` (unless `decision` is given) and settles a processing asset: a decision and
|
|
621
805
|
# its events, or nothing for `pending`.
|
|
@@ -637,6 +821,7 @@ module CalmLens
|
|
|
637
821
|
|
|
638
822
|
if decision.status == 'failed'
|
|
639
823
|
return add_event('type' => 'asset.failed', 'severity' => 'high', 'assetId' => item['id'], 'asset' => snapshot(item),
|
|
824
|
+
'actor' => deep_copy(AUTOMATED_ACTOR),
|
|
640
825
|
'facts' => [asset_name_fact(item), { 'key' => 'asset-type', 'value' => item['type'] },
|
|
641
826
|
{ 'key' => 'previous-status', 'value' => 'processing' },
|
|
642
827
|
{ 'key' => 'failure-reason',
|
|
@@ -645,40 +830,38 @@ module CalmLens
|
|
|
645
830
|
|
|
646
831
|
flagged = decision.report['flaggedCategories']
|
|
647
832
|
add_event('type' => 'asset.classified', 'severity' => 'low', 'assetId' => item['id'], 'asset' => snapshot(item),
|
|
833
|
+
'actor' => deep_copy(AUTOMATED_ACTOR),
|
|
648
834
|
'facts' => [{ 'key' => 'flagged-categories', 'value' => flagged },
|
|
649
835
|
{ 'key' => 'moderation-status', 'value' => decision.status },
|
|
650
836
|
{ 'key' => 'asset-type', 'value' => item['type'] }, asset_name_fact(item)])
|
|
651
837
|
# Like the event catalog: only allowing and limiting are low severity.
|
|
652
838
|
add_event('type' => "asset.#{decision.status}", 'severity' => %w[allowed limited].include?(decision.status) ? 'low' : 'high',
|
|
653
|
-
'assetId' => item['id'], 'asset' => snapshot(item),
|
|
654
|
-
'facts' => [{ 'key' => '
|
|
839
|
+
'assetId' => item['id'], 'asset' => snapshot(item), 'actor' => deep_copy(AUTOMATED_ACTOR),
|
|
840
|
+
'facts' => [{ 'key' => 'flagged-categories', 'value' => flagged },
|
|
655
841
|
{ 'key' => 'asset-type', 'value' => item['type'] }, asset_name_fact(item)])
|
|
656
842
|
end
|
|
657
843
|
|
|
658
844
|
# Moves an asset to an outcome like the pipeline would: a decision finishes processing, `purged`
|
|
659
845
|
# deletes the content (with `purge_reason`, `policy` for the pipeline), and finished processing
|
|
660
|
-
# removes content kept only temporarily. A
|
|
661
|
-
# placeholder, which
|
|
846
|
+
# removes content kept only temporarily. A decided asset gets its rendered file and preview
|
|
847
|
+
# (with a ThumbHash for its placeholder), which go with its content. `report` is a decision's
|
|
848
|
+
# categories, score and policy.
|
|
662
849
|
def apply_outcome(asset, outcome, purge_reason = 'policy', report = nil)
|
|
663
850
|
processing, moderation = AssetRecord.statuses(outcome)
|
|
664
851
|
now = @now.call
|
|
665
852
|
decided = processing == 'completed'
|
|
666
853
|
purge = moderation == 'purged' && asset['content']['status'] != 'removed'
|
|
667
854
|
remove = purge || (decided && asset['content']['status'] == 'temporary')
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
elsif moderation == 'purged'
|
|
675
|
-
image = without_placeholder(image)
|
|
676
|
-
end
|
|
677
|
-
asset['content']['image'] = image
|
|
855
|
+
# Like the API, an asset whose content is held has its rendered file and preview, completed by
|
|
856
|
+
# its decision, and removing content (when processing finishes or by purging) removes them
|
|
857
|
+
# with it, so no placeholder outlives the content. A safety hold withholds them.
|
|
858
|
+
content = asset['content']
|
|
859
|
+
held = content['status'] != 'removed' && !purge && !content['safetyHold']
|
|
860
|
+
content.merge!(held ? rendering(asset, decided) : NO_RENDERING)
|
|
678
861
|
if remove
|
|
679
862
|
forget_content(asset)
|
|
680
|
-
|
|
681
|
-
|
|
863
|
+
content.merge!('status' => 'removed', 'removalReason' => purge ? purge_reason : 'retention', 'updatedAt' => now,
|
|
864
|
+
'extractedText' => nil, 'metadata' => nil, **NO_RENDERING)
|
|
682
865
|
end
|
|
683
866
|
asset['upload'].merge!('status' => 'completed', 'updatedAt' => now)
|
|
684
867
|
update_classification(asset['classification'], processing, moderation, now)
|
|
@@ -700,11 +883,53 @@ module CalmLens
|
|
|
700
883
|
item['context'] = nil
|
|
701
884
|
end
|
|
702
885
|
|
|
703
|
-
#
|
|
704
|
-
def
|
|
886
|
+
# Whether browsers play an audio or video upload's format as it is, so it is served as stored.
|
|
887
|
+
def browser_safe_media?(format) = BROWSER_SAFE_MEDIA_FORMATS.include?(mime_type(format))
|
|
888
|
+
|
|
889
|
+
# A MIME type without parameters, in lower case.
|
|
890
|
+
def mime_type(format) = format.to_s.downcase.split(';', 2).first.to_s.strip
|
|
891
|
+
|
|
892
|
+
# The rendered file and preview the API reports for an asset with its content held (Asset.ts,
|
|
893
|
+
# getAssetRendered and getAssetPreview). Once uploaded, an image renders as itself with a preview
|
|
894
|
+
# that has no ThumbHash yet, and a PDF and browser-safe audio and video render as uploaded. Once
|
|
895
|
+
# processed, a website renders as its screenshot; a document or presentation as a PDF, with its
|
|
896
|
+
# first page as the preview; video and audio as a browser-safe file (MP4 or M4A unless browsers
|
|
897
|
+
# play the upload), audio with a waveform preview and video with no poster; and every preview has
|
|
898
|
+
# its ThumbHash. Text, spreadsheets and other files have neither. The mock decodes nothing, so
|
|
899
|
+
# sizes are unknown, audio has no peaks, and audio and video never get an adaptive stream.
|
|
900
|
+
def rendering(asset, processed) = { **rendered_file(asset, processed), 'hls' => nil }
|
|
901
|
+
|
|
902
|
+
# The rendered file and preview of {#rendering}.
|
|
903
|
+
def rendered_file(asset, processed)
|
|
904
|
+
format = asset['content']['format']
|
|
905
|
+
mime = mime_type(format)
|
|
906
|
+
safe = browser_safe_media?(format)
|
|
907
|
+
media = %w[video audio].include?(asset['type'])
|
|
908
|
+
return NO_RENDERING.dup unless processed || asset['type'] == 'image' || mime == 'application/pdf' || (media && safe)
|
|
909
|
+
|
|
910
|
+
unsized = { 'width' => nil, 'height' => nil }
|
|
911
|
+
preview = lambda do |size = unsized|
|
|
912
|
+
size.merge('thumbhash' => processed ? THUMBHASH : nil, 'averageColor' => processed ? '#7f7f7f' : nil)
|
|
913
|
+
end
|
|
914
|
+
case asset['type']
|
|
915
|
+
when 'image' then { 'rendered' => { 'kind' => 'image', 'format' => format || 'image/png', **unsized }, 'preview' => preview.call }
|
|
916
|
+
when 'website' then { 'rendered' => { 'kind' => 'image', 'format' => 'image/png', **unsized }, 'preview' => preview.call }
|
|
917
|
+
when 'multimedia-document'
|
|
918
|
+
return NO_RENDERING.dup if SPREADSHEET_MIME_TYPES.include?(mime)
|
|
919
|
+
|
|
920
|
+
{ 'rendered' => { 'kind' => 'pdf', 'format' => 'application/pdf', **unsized }, 'preview' => processed ? preview.call : nil }
|
|
921
|
+
when 'video' then { 'rendered' => { 'kind' => 'video', 'format' => safe ? format : 'video/mp4', **unsized }, 'preview' => nil }
|
|
922
|
+
when 'audio'
|
|
923
|
+
rendered = { 'kind' => 'audio', 'format' => safe ? format : 'audio/mp4', **unsized }
|
|
924
|
+
rendered['peaks'] = [] if processed
|
|
925
|
+
{ 'rendered' => rendered, 'preview' => processed ? preview.call(WAVEFORM_SIZE.dup) : nil }
|
|
926
|
+
else NO_RENDERING.dup
|
|
927
|
+
end
|
|
928
|
+
end
|
|
705
929
|
|
|
706
930
|
def asset(method, parts, query, json, body)
|
|
707
931
|
id = parts[1]
|
|
932
|
+
check_asset_body(method, parts[2], json)
|
|
708
933
|
item = @state.assets[id]
|
|
709
934
|
fail!(ErrorCodes::ASSET_NOT_FOUND) unless item && item['projectId'] == @project_id
|
|
710
935
|
case [method, parts[2]]
|
|
@@ -713,9 +938,10 @@ module CalmLens
|
|
|
713
938
|
in ['DELETE', nil]
|
|
714
939
|
@state.assets.delete(id)
|
|
715
940
|
@state.content.delete(id)
|
|
941
|
+
@processing_options.delete(id)
|
|
716
942
|
lifecycle_event('asset.deleted', item)
|
|
717
943
|
nil
|
|
718
|
-
in ['DELETE', 'content'] then
|
|
944
|
+
in ['DELETE', 'content'] then remove_content(item)
|
|
719
945
|
in ['GET', 'children'] then page(project_assets.select { |child| child['parentId'] == id }, query, id)
|
|
720
946
|
in ['POST', 'retry'] then retry_asset(item)
|
|
721
947
|
in ['POST', 'explain'] then explain(item, json['rerun'])
|
|
@@ -723,7 +949,6 @@ module CalmLens
|
|
|
723
949
|
in ['POST', 'move']
|
|
724
950
|
check_folder(json['folderId']) if json['folderId']
|
|
725
951
|
touch(item.merge!('folderId' => json['folderId']))
|
|
726
|
-
in ['POST', 'signed-url'] then signed_url(id, json)
|
|
727
952
|
in ['POST', 'part'] then part(item, query, body)
|
|
728
953
|
else route_not_found!(method, parts)
|
|
729
954
|
end
|
|
@@ -742,21 +967,21 @@ module CalmLens
|
|
|
742
967
|
touch(item.merge!(json.except('id', 'projectId', 'upload', 'classification', 'content')))
|
|
743
968
|
end
|
|
744
969
|
|
|
745
|
-
# Like the API: the record and report stay; stored content, extracted text
|
|
746
|
-
# go. Content already removed keeps its reason, and an `asset.content-removed` event is
|
|
747
|
-
# recorded either way.
|
|
748
|
-
def
|
|
970
|
+
# Like the API: the record and report stay; stored content, extracted text, metadata, the
|
|
971
|
+
# rendered file and the preview go. Content already removed keeps its reason, and an `asset.content-removed` event is
|
|
972
|
+
# recorded either way, with the `batch-id` fact of the batch that removed it, if any.
|
|
973
|
+
def remove_content(item, batch_id = nil)
|
|
749
974
|
forget_content(item)
|
|
750
975
|
content = item['content']
|
|
751
976
|
if content['status'] == 'removed'
|
|
752
|
-
content
|
|
977
|
+
content.merge!(NO_RENDERING)
|
|
753
978
|
else
|
|
754
979
|
now = @now.call
|
|
755
980
|
content.merge!('status' => 'removed', 'removalReason' => 'requested', 'updatedAt' => now, 'extractedText' => nil,
|
|
756
|
-
'
|
|
981
|
+
'metadata' => nil, **NO_RENDERING)
|
|
757
982
|
item['updatedAt'] = now
|
|
758
983
|
end
|
|
759
|
-
lifecycle_event('asset.content-removed', item)
|
|
984
|
+
lifecycle_event('asset.content-removed', item, batch_id: batch_id)
|
|
760
985
|
item
|
|
761
986
|
end
|
|
762
987
|
|
|
@@ -785,16 +1010,61 @@ module CalmLens
|
|
|
785
1010
|
fail!(ErrorCodes::ASSET_REVIEWED) unless moderation.fetch('manualReviews', []).empty?
|
|
786
1011
|
fail!(ErrorCodes::ASSET_CONTENT_UNAVAILABLE) if item['content']['status'] == 'removed'
|
|
787
1012
|
|
|
788
|
-
# Like the API:
|
|
789
|
-
|
|
790
|
-
|
|
1013
|
+
# Like the API: the request is recorded, and the asset returns as it was. It keeps its decision
|
|
1014
|
+
# while it's moderated again under the current policy.
|
|
1015
|
+
add_event('type' => 'asset.rerun-requested', 'severity' => 'low', 'assetId' => item['id'],
|
|
1016
|
+
'asset' => snapshot(item), 'actor' => deep_copy(API_KEY_ACTOR),
|
|
1017
|
+
'facts' => [asset_name_fact(item), { 'key' => 'asset-type', 'value' => item['type'] },
|
|
1018
|
+
{ 'key' => 'moderation-status', 'value' => moderation['status'] }])
|
|
1019
|
+
response = deep_copy(item)
|
|
1020
|
+
settle_rerun(item, 'rerun')
|
|
1021
|
+
response
|
|
791
1022
|
end
|
|
792
1023
|
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
1024
|
+
# Applies a re-run's decision to a decided asset like the pipeline: `pending` stays queued for
|
|
1025
|
+
# `complete_processing`, `failed` keeps the asset's decision, and any other replaces it with the
|
|
1026
|
+
# pipeline's events.
|
|
1027
|
+
def settle_rerun(item, trigger, decision = nil)
|
|
1028
|
+
decision ||= decision_for(item, trigger, true)
|
|
1029
|
+
@reruns.delete(item['id'])
|
|
1030
|
+
if decision.status == 'pending'
|
|
1031
|
+
@reruns[item['id']] = true
|
|
1032
|
+
elsif decision.status != 'failed'
|
|
1033
|
+
moderate_stored(item, trigger, decision)
|
|
1034
|
+
end
|
|
1035
|
+
end
|
|
1036
|
+
|
|
1037
|
+
# Signs an asset URL like the API (`POST /projects/:projectId/signed-asset-urls`): a grant to one
|
|
1038
|
+
# asset, by ID or external ID, until it expires, serving the original only when asked. Only the
|
|
1039
|
+
# mock's CDN accepts the grant.
|
|
1040
|
+
def sign_asset_url(json)
|
|
1041
|
+
# Like the API's schema, none of these fields takes null.
|
|
1042
|
+
%w[assetId externalId expiresIn original].each do |key|
|
|
1043
|
+
invalid!('validation/invalid-type', key) if json.key?(key) && json[key].nil?
|
|
1044
|
+
end
|
|
1045
|
+
if json.key?('assetId') == json.key?('externalId')
|
|
1046
|
+
message = 'Send either assetId or externalId'
|
|
1047
|
+
raise Failure.new(ErrorCodes::INVALID_REQUEST, message,
|
|
1048
|
+
details: { 'issues' => [{ 'path' => '', 'message' => message, 'code' => 'custom' }] }, status: 400)
|
|
1049
|
+
end
|
|
1050
|
+
expires = json.fetch('expiresIn', CDN::DEFAULT_SIGNED_URL_EXPIRES_IN)
|
|
1051
|
+
invalid!('validation/invalid-type', 'expiresIn') unless expires.is_a?(Integer) || (expires.is_a?(Float) && expires == expires.floor)
|
|
1052
|
+
invalid!('validation/too-small', 'expiresIn') if expires < 1
|
|
1053
|
+
invalid!('validation/too-big', 'expiresIn') if expires > CDN::MAX_SIGNED_URL_EXPIRES_IN
|
|
1054
|
+
original = json.fetch('original', false)
|
|
1055
|
+
invalid!('validation/invalid-type', 'original') unless [true, false].include?(original)
|
|
1056
|
+
|
|
1057
|
+
item = if json.key?('externalId')
|
|
1058
|
+
project_assets.find { |asset| asset['externalId'] == json['externalId'] }
|
|
1059
|
+
else
|
|
1060
|
+
own_asset(json['assetId'].to_s)
|
|
1061
|
+
end
|
|
1062
|
+
fail!(ErrorCodes::ASSET_NOT_FOUND) unless item
|
|
1063
|
+
fail!(ErrorCodes::ASSET_HELD) if item['content']['safetyHold']
|
|
1064
|
+
|
|
1065
|
+
expires_at = @now.call + (expires.to_i * 1000)
|
|
1066
|
+
{ 'url' => "#{@cdn_base_url}/assets/#{Internal.component(@project_id)}/#{MockGrant.encode(item['id'], original, expires_at)}",
|
|
1067
|
+
'expiresAt' => expires_at }
|
|
798
1068
|
end
|
|
799
1069
|
|
|
800
1070
|
# In the API's order: the upload must be open, parts arrive in order, image totals and the
|
|
@@ -812,7 +1082,7 @@ module CalmLens
|
|
|
812
1082
|
received = item['content']['sizeInBytes'].to_i + size
|
|
813
1083
|
upload = @state.uploads[item['id']] || {}
|
|
814
1084
|
code = image_size_error(item['type'], received, final) || total_size_error(upload['total'], received, final)
|
|
815
|
-
|
|
1085
|
+
size_failure!(code) if code
|
|
816
1086
|
check_part_size(size, final)
|
|
817
1087
|
|
|
818
1088
|
stored = (@state.content[item['id']] ||= { 'type' => item['content']['format'].to_s, 'parts' => {} })
|
|
@@ -825,7 +1095,8 @@ module CalmLens
|
|
|
825
1095
|
return item unless final
|
|
826
1096
|
|
|
827
1097
|
@state.uploads.delete(item['id'])
|
|
828
|
-
|
|
1098
|
+
# `wait=false` queues the final part; without `wait`, the upload's saved choice applies.
|
|
1099
|
+
queue = query.key?('wait') ? query['wait'] == false : upload.fetch('queue', false)
|
|
829
1100
|
apply_outcome(item, 'pending')
|
|
830
1101
|
upload_completed(item, queue)
|
|
831
1102
|
end
|
|
@@ -887,29 +1158,38 @@ module CalmLens
|
|
|
887
1158
|
touch(item.merge!('parentId' => parent_id))
|
|
888
1159
|
end
|
|
889
1160
|
|
|
890
|
-
# Policies: the project starts on
|
|
891
|
-
#
|
|
892
|
-
#
|
|
893
|
-
|
|
1161
|
+
# Policies: the project starts on the organisation's own General preset policy, following
|
|
1162
|
+
# automatic updates (mode `live`). The mock's presets have one release, version 1, with
|
|
1163
|
+
# PRESET_SETTINGS. A stored policy is the API's shape without `projectIds`, with its preset's
|
|
1164
|
+
# `id`, `version` and `own`, whether it follows its preset (`followsPreset`) and its reset
|
|
1165
|
+
# `baseline`.
|
|
1166
|
+
def new_policy(name, settings, description: '', follows_preset: false, preset: nil)
|
|
894
1167
|
now = @now.call
|
|
895
1168
|
item = { 'id' => @create_id.call('pol'), 'name' => name, 'description' => description, 'settings' => settings,
|
|
896
|
-
'revision' => 1, '
|
|
1169
|
+
'revision' => 1, 'followsPreset' => follows_preset, 'preset' => preset,
|
|
897
1170
|
'baseline' => deep_copy(settings), 'createdAt' => now, 'updatedAt' => now }
|
|
898
1171
|
@policies[item['id']] = item
|
|
899
1172
|
end
|
|
900
1173
|
|
|
1174
|
+
# The project's policy, after creating the organisation's own General policy on first use.
|
|
901
1175
|
def ensure_policies
|
|
902
|
-
return if @project_policy_id
|
|
1176
|
+
return @project_policy_id if @project_policy_id
|
|
903
1177
|
|
|
904
|
-
preset = { 'id' => 'general-community', 'version' => 1, '
|
|
905
|
-
@project_policy_id = new_policy(PRESET_NAMES['general-community'],
|
|
906
|
-
|
|
1178
|
+
preset = { 'id' => 'general-community', 'version' => 1, 'own' => true }
|
|
1179
|
+
@project_policy_id = new_policy(PRESET_NAMES['general-community'], deep_copy(PRESET_SETTINGS),
|
|
1180
|
+
follows_preset: true, preset: preset)['id']
|
|
907
1181
|
end
|
|
908
1182
|
|
|
909
|
-
# A stored policy as the API shows it
|
|
1183
|
+
# A stored policy as the API shows it: an own preset policy is `live` while it follows its
|
|
1184
|
+
# preset and `pinned` otherwise; any other preset policy is a `copy`.
|
|
910
1185
|
def show_policy(item)
|
|
911
|
-
shown = item.except('baseline')
|
|
912
|
-
|
|
1186
|
+
shown = item.except('baseline', 'followsPreset')
|
|
1187
|
+
if (preset = item['preset'])
|
|
1188
|
+
mode = if preset['own'] then item['followsPreset'] ? 'live' : 'pinned'
|
|
1189
|
+
else 'copy'
|
|
1190
|
+
end
|
|
1191
|
+
shown['preset'] = { 'id' => preset['id'], 'version' => preset['version'], 'latestVersion' => 1, 'mode' => mode }
|
|
1192
|
+
end
|
|
913
1193
|
shown.merge('projectIds' => item['id'] == @project_policy_id ? [@project_id] : [])
|
|
914
1194
|
end
|
|
915
1195
|
|
|
@@ -918,9 +1198,39 @@ module CalmLens
|
|
|
918
1198
|
@policies[id] || fail!(ErrorCodes::POLICY_NOT_FOUND)
|
|
919
1199
|
end
|
|
920
1200
|
|
|
921
|
-
|
|
1201
|
+
# `/policies` and `/policies/:id[/action]`, the API key's organisation's policies.
|
|
1202
|
+
def policies(method, parts, query, json)
|
|
1203
|
+
_, id, *rest = parts
|
|
1204
|
+
if id.nil?
|
|
1205
|
+
return policy_page(query) if method == 'GET'
|
|
1206
|
+
return create_policy(json) if method == 'POST'
|
|
1207
|
+
|
|
1208
|
+
route_not_found!(method, parts)
|
|
1209
|
+
end
|
|
1210
|
+
policy(method, id, rest, json)
|
|
1211
|
+
end
|
|
1212
|
+
|
|
1213
|
+
# A page of the organisation's policies, oldest first unless `sort` says otherwise.
|
|
1214
|
+
def policy_page(query)
|
|
922
1215
|
ensure_policies
|
|
923
|
-
|
|
1216
|
+
page(@policies.values.map { |item| show_policy(item) }, query, JSON.generate(['policies', @project_id]),
|
|
1217
|
+
default: { 'field' => 'createdAt', 'direction' => 'asc' })
|
|
1218
|
+
end
|
|
1219
|
+
|
|
1220
|
+
# Refuses settings the API refuses (MODERATION_POLICY_SETTINGS_SCHEMA): every category, each
|
|
1221
|
+
# with an `action` and its `allowances` and `restrictions` lists, and no other category.
|
|
1222
|
+
def check_policy_settings(settings)
|
|
1223
|
+
categories = settings.is_a?(Hash) ? settings['categorySettings'] : nil
|
|
1224
|
+
invalid!('validation/invalid-type', 'settings.categorySettings') unless categories.is_a?(Hash)
|
|
1225
|
+
MockClassification::CATEGORIES.each do |category|
|
|
1226
|
+
path = "settings.categorySettings.#{category}"
|
|
1227
|
+
value = categories[category]
|
|
1228
|
+
invalid!('validation/invalid-type', path) unless value.is_a?(Hash)
|
|
1229
|
+
%w[allowances restrictions].each { |key| invalid!('validation/invalid-type', "#{path}.#{key}") unless value[key].is_a?(Array) }
|
|
1230
|
+
invalid!('validation/invalid-value', "#{path}.action") unless value['action'].is_a?(String)
|
|
1231
|
+
end
|
|
1232
|
+
unknown = categories.keys.find { |key| !MockClassification::CATEGORIES.include?(key) }
|
|
1233
|
+
invalid!('validation/unknown-field', "settings.categorySettings.#{unknown}") if unknown
|
|
924
1234
|
end
|
|
925
1235
|
|
|
926
1236
|
def create_policy(json)
|
|
@@ -930,10 +1240,11 @@ module CalmLens
|
|
|
930
1240
|
if settings.nil? == preset_id.nil? || (preset_id && !PRESET_NAMES.key?(preset_id)) || (preset_id.nil? && !named)
|
|
931
1241
|
fail!(ErrorCodes::INVALID_VALUE, 'Send settings with a name, or presetId')
|
|
932
1242
|
end
|
|
1243
|
+
check_policy_settings(settings) unless preset_id
|
|
933
1244
|
description = json['description'].is_a?(String) ? json['description'] : ''
|
|
934
1245
|
item = if preset_id
|
|
935
|
-
origin = { 'id' => preset_id, 'version' => 1, '
|
|
936
|
-
new_policy(named ? name.strip : PRESET_NAMES[preset_id],
|
|
1246
|
+
origin = { 'id' => preset_id, 'version' => 1, 'own' => false }
|
|
1247
|
+
new_policy(named ? name.strip : PRESET_NAMES[preset_id], deep_copy(PRESET_SETTINGS),
|
|
937
1248
|
description: description, preset: origin)
|
|
938
1249
|
else
|
|
939
1250
|
new_policy(name.strip, settings, description: description)
|
|
@@ -965,26 +1276,29 @@ module CalmLens
|
|
|
965
1276
|
fail!(ErrorCodes::POLICY_IN_USE) if item['id'] == @project_policy_id
|
|
966
1277
|
@policies.delete(id)
|
|
967
1278
|
{}
|
|
968
|
-
in ['POST', ['
|
|
1279
|
+
in ['POST', ['reset']] then restore_policy(item, 'reset', json)
|
|
1280
|
+
in ['POST', ['upgrade']] then restore_policy(item, 'upgrade', json)
|
|
969
1281
|
in ['PUT', ['automatic-updates']] then set_policy_automatic_updates(item, json)
|
|
970
1282
|
else route_not_found!(method, ['policies', id, *rest])
|
|
971
1283
|
end
|
|
972
1284
|
end
|
|
973
1285
|
|
|
974
1286
|
def update_policy(item, json)
|
|
1287
|
+
check_policy_settings(json['settings']) if json.key?('settings')
|
|
975
1288
|
values = %w[name description].select { |key| json[key].is_a?(String) }
|
|
976
1289
|
.to_h { |key| [key, key == 'name' ? json[key].strip : json[key]] }
|
|
977
1290
|
values['settings'] = json['settings'] if json.key?('settings')
|
|
978
|
-
# Saving a policy
|
|
979
|
-
save_policy(item, json['revision'], values.merge('
|
|
1291
|
+
# Saving a `live` policy pins it.
|
|
1292
|
+
save_policy(item, json['revision'], values.merge('followsPreset' => false))
|
|
980
1293
|
end
|
|
981
1294
|
|
|
982
|
-
|
|
1295
|
+
# Resets a `pinned` or `copy` policy to its preset release, or upgrades it to the latest.
|
|
1296
|
+
def restore_policy(item, operation, json)
|
|
983
1297
|
preset = item['preset']
|
|
984
|
-
fail!(ErrorCodes::POLICY_NO_PRESET) if preset.nil? || item['
|
|
1298
|
+
fail!(ErrorCodes::POLICY_NO_PRESET) if preset.nil? || item['followsPreset']
|
|
985
1299
|
fail!(ErrorCodes::POLICY_PRESET_VERSION_CONFLICT) unless json['presetVersion'] == preset['version']
|
|
986
|
-
# The mock's presets have one release, so an
|
|
987
|
-
fail!(ErrorCodes::POLICY_PRESET_UPDATE_UNAVAILABLE) if
|
|
1300
|
+
# The mock's presets have one release, so an upgrade never has a newer one to apply.
|
|
1301
|
+
fail!(ErrorCodes::POLICY_PRESET_UPDATE_UNAVAILABLE) if operation == 'upgrade'
|
|
988
1302
|
|
|
989
1303
|
save_policy(item, json['revision'], 'settings' => deep_copy(item['baseline']))
|
|
990
1304
|
end
|
|
@@ -992,13 +1306,11 @@ module CalmLens
|
|
|
992
1306
|
def set_policy_automatic_updates(item, json)
|
|
993
1307
|
enabled = json['enabled'] == true
|
|
994
1308
|
preset = item['preset']
|
|
995
|
-
if preset.nil? || preset['
|
|
996
|
-
fail!(ErrorCodes::POLICY_AUTOMATIC_UPDATES_UNAVAILABLE)
|
|
997
|
-
end
|
|
1309
|
+
fail!(ErrorCodes::POLICY_AUTOMATIC_UPDATES_UNAVAILABLE) if preset.nil? || !preset['own'] || item['followsPreset'] == enabled
|
|
998
1310
|
fail!(ErrorCodes::POLICY_REVISION_CONFLICT) unless json['revision'] == item['revision']
|
|
999
1311
|
fail!(ErrorCodes::POLICY_PRESET_VERSION_CONFLICT) unless json['presetVersion'] == 1
|
|
1000
1312
|
|
|
1001
|
-
values = { '
|
|
1313
|
+
values = { 'followsPreset' => enabled }
|
|
1002
1314
|
values.merge!('name' => PRESET_NAMES[preset['id']], 'description' => '', 'settings' => deep_copy(item['baseline'])) if enabled
|
|
1003
1315
|
save_policy(item, json['revision'], values)
|
|
1004
1316
|
end
|
|
@@ -1060,10 +1372,10 @@ module CalmLens
|
|
|
1060
1372
|
if asset_id
|
|
1061
1373
|
asset = @state.assets[asset_id]
|
|
1062
1374
|
fail!(ErrorCodes::ASSET_NOT_FOUND) unless asset && asset['projectId'] == @project_id
|
|
1063
|
-
|
|
1064
|
-
fail!(ErrorCodes::HASH_LIST_ASSET_NOT_HASHED) unless
|
|
1065
|
-
hash =
|
|
1066
|
-
quality =
|
|
1375
|
+
content = asset['content']
|
|
1376
|
+
fail!(ErrorCodes::HASH_LIST_ASSET_NOT_HASHED) unless content['perceptualHash']
|
|
1377
|
+
hash = content['perceptualHash']
|
|
1378
|
+
quality = content['perceptualHashQuality']
|
|
1067
1379
|
end
|
|
1068
1380
|
fail!(ErrorCodes::INVALID_UPLOAD_PAYLOAD, 'Use a PDQ hash of 64 hexadecimal characters') unless hash.match?(/\A\h{64}\z/)
|
|
1069
1381
|
|
|
@@ -1072,7 +1384,7 @@ module CalmLens
|
|
|
1072
1384
|
|
|
1073
1385
|
entry = { 'id' => @create_id.call('hle'), 'projectId' => @project_id, 'hashListId' => list['id'], 'hash' => hash,
|
|
1074
1386
|
'quality' => quality, 'label' => json['label'].is_a?(String) ? json['label'] : nil,
|
|
1075
|
-
'
|
|
1387
|
+
'assetId' => asset_id, 'createdAt' => @now.call }
|
|
1076
1388
|
@state.hash_list_entries[entry['id']] = entry
|
|
1077
1389
|
respond(201, deep_copy(entry))
|
|
1078
1390
|
end
|
|
@@ -1086,17 +1398,43 @@ module CalmLens
|
|
|
1086
1398
|
{}
|
|
1087
1399
|
end
|
|
1088
1400
|
|
|
1089
|
-
#
|
|
1090
|
-
#
|
|
1091
|
-
#
|
|
1092
|
-
def
|
|
1401
|
+
# An asset's findings timeline, as the API lists it (publicGetAssetTimeline.ts): refused while a
|
|
1402
|
+
# safety hold is on. The mock analyses no audio or video, so it keeps no segments: every
|
|
1403
|
+
# timeline is empty, and no cursor continues one.
|
|
1404
|
+
def timeline(id, query)
|
|
1405
|
+
item = own_asset(id)
|
|
1406
|
+
fail!(ErrorCodes::ASSET_NOT_FOUND) unless item
|
|
1407
|
+
check_range(query.fetch('pageSize', DEFAULT_PAGE_SIZE), 'pageSize', MAX_PAGE_SIZE)
|
|
1408
|
+
fail!(ErrorCodes::ASSET_HELD, details: { 'hold' => 'safety' }) if item['content']['safetyHold']
|
|
1409
|
+
fail!(ErrorCodes::INVALID_CURSOR) if query.key?('cursor')
|
|
1410
|
+
|
|
1411
|
+
{ 'items' => [], 'nextCursor' => nil, 'hasMore' => false }
|
|
1412
|
+
end
|
|
1413
|
+
|
|
1414
|
+
# The page size and sort, refused like the API's query schema: `sort` is comma-separated
|
|
1415
|
+
# fields from `fields`, each descending when prefixed with `-`, none repeated and at most `max`
|
|
1416
|
+
# (`validation/invalid-value` at `sort` otherwise). Without one, `default` applies.
|
|
1417
|
+
def page_query(query, fields = %w[createdAt id], default = { 'field' => 'createdAt', 'direction' => 'desc' }, max: 2)
|
|
1093
1418
|
size = check_range(query.fetch('pageSize', DEFAULT_PAGE_SIZE), 'pageSize', MAX_PAGE_SIZE)
|
|
1094
|
-
|
|
1095
|
-
|
|
1096
|
-
|
|
1097
|
-
|
|
1098
|
-
|
|
1419
|
+
[size, query.key?('sort') ? parse_sort(query['sort'], fields, max) : [default.dup]]
|
|
1420
|
+
end
|
|
1421
|
+
|
|
1422
|
+
def parse_sort(value, fields, max)
|
|
1423
|
+
sort = value.to_s.split(',', -1).map(&:strip).map do |part|
|
|
1424
|
+
field = part.delete_prefix('-')
|
|
1425
|
+
invalid!('validation/invalid-value', 'sort') unless part.match?(/\A-?[A-Za-z][A-Za-z0-9.]*\z/) && fields.include?(field)
|
|
1426
|
+
{ 'field' => field, 'direction' => part.start_with?('-') ? 'desc' : 'asc' }
|
|
1099
1427
|
end
|
|
1428
|
+
fields_sent = sort.map { |rule| rule['field'] }
|
|
1429
|
+
invalid!('validation/invalid-value', 'sort') if sort.empty? || fields_sent.uniq.size != fields_sent.size || sort.size > max
|
|
1430
|
+
sort
|
|
1431
|
+
end
|
|
1432
|
+
|
|
1433
|
+
# One keyset page like the API's: sorted by `sort` (`createdAt` descending by default; an `id`
|
|
1434
|
+
# tie-breaker is added), starting after the cursor's values. The cursor is tied to `scope` (the
|
|
1435
|
+
# filter, folder or parent) and the sort, so it only continues the query it came from.
|
|
1436
|
+
def page(items, query, scope, fields: %w[createdAt id], max: 2, default: { 'field' => 'createdAt', 'direction' => 'desc' })
|
|
1437
|
+
size, ordering = page_query(query, fields, default, max: max)
|
|
1100
1438
|
ordering << { 'field' => 'id', 'direction' => ordering.first['direction'] } unless ordering.any? { |o| o['field'] == 'id' }
|
|
1101
1439
|
signature = JSON.generate([scope, ordering])
|
|
1102
1440
|
sorted = items.sort { |a, b| compare_items(a, b, ordering) }
|
|
@@ -1106,12 +1444,8 @@ module CalmLens
|
|
|
1106
1444
|
end
|
|
1107
1445
|
selected = sorted.first(size)
|
|
1108
1446
|
more = sorted.size > size
|
|
1109
|
-
cursor = if more
|
|
1110
|
-
|
|
1111
|
-
selected.last[o['field']]
|
|
1112
|
-
end })].pack('m0')
|
|
1113
|
-
end
|
|
1114
|
-
{ 'items' => selected, 'pageSize' => size, 'hasMore' => more, 'ordering' => ordering, 'nextCursor' => cursor }
|
|
1447
|
+
cursor = ([JSON.generate({ 'scope' => signature, 'values' => ordering.map { |o| selected.last[o['field']] } })].pack('m0') if more)
|
|
1448
|
+
{ 'items' => selected, 'hasMore' => more, 'nextCursor' => cursor }
|
|
1115
1449
|
end
|
|
1116
1450
|
|
|
1117
1451
|
def cursor_values(cursor, signature, ordering)
|
|
@@ -1134,15 +1468,6 @@ module CalmLens
|
|
|
1134
1468
|
false
|
|
1135
1469
|
end
|
|
1136
1470
|
|
|
1137
|
-
def ordering(query, default)
|
|
1138
|
-
entries = query.keys.filter_map { |key| key.match(/\Aordering\[(\d+)\]\[(field|direction)\]\z/) }
|
|
1139
|
-
return [default&.dup].compact if entries.empty?
|
|
1140
|
-
|
|
1141
|
-
entries.group_by { |match| Integer(match[1], 10) }.sort.map do |_, matches|
|
|
1142
|
-
matches.to_h { |match| [match[2], query[match[0]]] }
|
|
1143
|
-
end
|
|
1144
|
-
end
|
|
1145
|
-
|
|
1146
1471
|
def compare_items(left, right, ordering)
|
|
1147
1472
|
ordering.each do |rule|
|
|
1148
1473
|
result = compare_values(left[rule['field']], right[rule['field']])
|
|
@@ -1161,62 +1486,200 @@ module CalmLens
|
|
|
1161
1486
|
left.to_s <=> right.to_s
|
|
1162
1487
|
end
|
|
1163
1488
|
|
|
1164
|
-
# Answers like the CDN:
|
|
1165
|
-
#
|
|
1166
|
-
#
|
|
1167
|
-
#
|
|
1168
|
-
#
|
|
1169
|
-
|
|
1489
|
+
# Answers like the CDN (serveAssetResource.ts): `original` is the upload; `rendered` the image,
|
|
1490
|
+
# PDF, video or audio file `content.rendered` describes; `preview` the still `content.preview`
|
|
1491
|
+
# describes; `hls` the adaptive stream's files, which the mock never prepares; `embed` a small
|
|
1492
|
+
# page standing in for the embed page of any asset; and `placeholder` the preview's ThumbHash as a PNG, public
|
|
1493
|
+
# whatever the asset's visibility or moderation. A safety hold refuses every resource. Other resources pass delivery checks first,
|
|
1494
|
+
# which refuse everyone (content still uploading, blocked or purged, failed processing, removed
|
|
1495
|
+
# content), then an age-restricted asset needs `viewerClaims.ageConfirmed=true` whatever the key,
|
|
1496
|
+
# and a private asset needs an API key with `assets:download` for its folder. The mock converts
|
|
1497
|
+
# nothing: an upload stands in for its rendered file, an image for its own preview, and the
|
|
1498
|
+
# placeholder PNG for a screenshot, page or waveform.
|
|
1499
|
+
def serve(method, relative, headers)
|
|
1170
1500
|
path, query = relative.split('#', 2).first.split('?', 2)
|
|
1171
|
-
|
|
1172
|
-
|
|
1173
|
-
|
|
1174
|
-
return
|
|
1501
|
+
raw_segments = path.split('/').reject(&:empty?)
|
|
1502
|
+
root, project, _, resource = raw_segments.map { |part| URI.decode_www_form_component(part) }
|
|
1503
|
+
selected, options = selector(resource, raw_segments.drop(4))
|
|
1504
|
+
return invalid_transform if selected.nil?
|
|
1505
|
+
return asset_not_found unless root == 'assets' && project == @project_id && raw_segments[2]
|
|
1506
|
+
|
|
1507
|
+
grant, item = resolve_asset(raw_segments[2])
|
|
1508
|
+
return grant if grant.is_a?(Response)
|
|
1509
|
+
return asset_not_found unless item && RESOURCES.include?(resource)
|
|
1510
|
+
if resource == 'original' && grant && !grant[:original]
|
|
1511
|
+
return error(ErrorCodes::SIGNED_URL_ORIGINAL_NOT_GRANTED, "This signed asset URL doesn't grant the original file", status: 403)
|
|
1512
|
+
end
|
|
1175
1513
|
|
|
1514
|
+
params = URI.decode_www_form(query.to_s).to_h
|
|
1515
|
+
# A grant authorises a private asset without the API key.
|
|
1516
|
+
authorised = !grant.nil? || can_download?(item['folderId'], headers)
|
|
1517
|
+
return info(method, item, authorised, params) if resource == 'info'
|
|
1518
|
+
return not_available if item['content']['safetyHold']
|
|
1176
1519
|
return placeholder(method, item) if resource == 'placeholder'
|
|
1177
1520
|
|
|
1521
|
+
deliver(method, item, resource, params, selected ? options || {} : nil, authorised)
|
|
1522
|
+
end
|
|
1523
|
+
|
|
1524
|
+
# Whether the path after the resource selects a lens or transform, and its options; nil when the
|
|
1525
|
+
# CDN refuses it. Like the CDN, the adaptive stream's files sit beneath `/hls/` (its master
|
|
1526
|
+
# playlist is `/hls/index.m3u8`). Otherwise the selector after the resource is read
|
|
1527
|
+
# (AssetTransformParser.ts); the original, the embed page and the info take none.
|
|
1528
|
+
def selector(resource, path)
|
|
1529
|
+
lens, options = TransformRules.parse_selector(resource == 'hls' ? '' : path.join('/'))
|
|
1530
|
+
selected = !(lens || options).nil?
|
|
1531
|
+
%w[original embed info].include?(resource) && selected ? nil : [selected, options]
|
|
1532
|
+
rescue ArgumentError
|
|
1533
|
+
nil
|
|
1534
|
+
end
|
|
1535
|
+
|
|
1536
|
+
# Like the CDN (CdnUtils.ts), the asset segment is `ext:` and the URL-encoded external ID, a
|
|
1537
|
+
# grant (`sig_…`) or the asset ID. Returns `[grant, asset]`, or an error response for a grant
|
|
1538
|
+
# the mock didn't sign or that has expired.
|
|
1539
|
+
def resolve_asset(segment)
|
|
1540
|
+
if segment.start_with?('sig_')
|
|
1541
|
+
grant = MockGrant.decode(segment)
|
|
1542
|
+
return error(ErrorCodes::SIGNED_URL_INVALID, "The signed asset URL's grant is invalid", status: 403) unless grant
|
|
1543
|
+
return error(ErrorCodes::SIGNED_URL_EXPIRED, 'The signed URL has expired', status: 403) if grant[:expires_at] <= @now.call
|
|
1544
|
+
|
|
1545
|
+
[grant, project_assets.find { |asset| asset['id'] == grant[:asset_id] }]
|
|
1546
|
+
elsif segment.start_with?(CDN::EXTERNAL_ID_PREFIX)
|
|
1547
|
+
external = MockGrant.percent_decode(segment.delete_prefix(CDN::EXTERNAL_ID_PREFIX))
|
|
1548
|
+
[nil, external && !external.empty? ? project_assets.find { |asset| asset['externalId'] == external } : nil]
|
|
1549
|
+
else
|
|
1550
|
+
id = MockGrant.percent_decode(segment)
|
|
1551
|
+
[nil, project_assets.find { |asset| asset['id'] == id }]
|
|
1552
|
+
end
|
|
1553
|
+
end
|
|
1554
|
+
|
|
1555
|
+
def asset_not_found = error(ErrorCodes::ASSET_NOT_FOUND, 'Asset not found', status: 404)
|
|
1556
|
+
|
|
1557
|
+
# The display info at `/info`, like the CDN's (AssetInfo.ts): the allowlisted fields and whether
|
|
1558
|
+
# the viewer the claims describe can see the content, decided as the other resources decide it.
|
|
1559
|
+
# A private asset's info needs a grant or an API key with `assets:download`.
|
|
1560
|
+
def info(method, item, authorised, params)
|
|
1561
|
+
restricted = item['visibility'] != 'public'
|
|
1562
|
+
return error('asset/not-available', 'Asset is not available', status: 404) if restricted && !authorised
|
|
1563
|
+
|
|
1564
|
+
claims = JSON.parse(params.fetch('viewerClaims', '{}'))
|
|
1565
|
+
body = { 'id' => item['id'], 'projectId' => item['projectId'], 'type' => item['type'], 'externalUrl' => item['externalUrl'],
|
|
1566
|
+
'content' => item['content'].slice(*%w[format width height durationSeconds pageCount rendered preview hls]),
|
|
1567
|
+
'availability' => availability(item, claims.is_a?(Hash) && claims['ageConfirmed'] == true) }
|
|
1568
|
+
cdn_response(method, JSON.generate(body), { 'content-type' => 'application/json',
|
|
1569
|
+
'cache-control' => "#{restricted ? 'private' : 'public'}, max-age=60",
|
|
1570
|
+
'cache-tag' => "asset-#{item['id']}" })
|
|
1571
|
+
rescue JSON::ParserError
|
|
1572
|
+
error(ErrorCodes::INVALID_VIEWER_CLAIMS, 'Invalid viewer claims', status: 400)
|
|
1573
|
+
end
|
|
1574
|
+
|
|
1575
|
+
# Whether a viewer can see an asset's content now, decided as the other resources decide it.
|
|
1576
|
+
def availability(item, age_confirmed)
|
|
1178
1577
|
moderation = item['classification']['moderation']['status']
|
|
1179
|
-
|
|
1180
|
-
if stored.nil? || item['upload']['status'] == 'uploading' || %w[blocked purged].include?(moderation) ||
|
|
1578
|
+
if item['content']['safetyHold'] || item['upload']['status'] == 'uploading' || %w[blocked purged].include?(moderation) ||
|
|
1181
1579
|
item['classification']['status'] == 'failed' || item['content']['status'] == 'removed'
|
|
1182
|
-
|
|
1580
|
+
'unavailable'
|
|
1581
|
+
elsif moderation == 'age-restricted' && !age_confirmed
|
|
1582
|
+
'age-restricted'
|
|
1583
|
+
else
|
|
1584
|
+
'available'
|
|
1183
1585
|
end
|
|
1586
|
+
end
|
|
1184
1587
|
|
|
1185
|
-
|
|
1186
|
-
|
|
1187
|
-
|
|
1188
|
-
|
|
1189
|
-
|
|
1190
|
-
|
|
1588
|
+
# Serves a found asset's resource after the delivery, age and access checks, then whether it
|
|
1589
|
+
# has the resource and takes the selector, then the transform options' schema check.
|
|
1590
|
+
# `selector` is the selector's options (empty for a lens alone), or nil without one.
|
|
1591
|
+
def deliver(method, item, resource, params, selector, authorised)
|
|
1592
|
+
stored = @state.content[item['id']]
|
|
1593
|
+
refused = refuse_delivery(item, resource, stored, params, authorised)
|
|
1594
|
+
return refused if refused
|
|
1191
1595
|
|
|
1192
1596
|
restricted = item['visibility'] != 'public'
|
|
1193
|
-
|
|
1194
|
-
|
|
1195
|
-
|
|
1597
|
+
return embed(method, item, restricted, params) if resource == 'embed'
|
|
1598
|
+
|
|
1599
|
+
# Then whether the asset has the resource, and whether it is a still image that takes a lens
|
|
1600
|
+
# or transform: the preview always is, the rendered file only when it is an image.
|
|
1601
|
+
rendered, preview = item['content'].values_at('rendered', 'preview')
|
|
1602
|
+
return no_rendered_version if (resource == 'rendered' && !rendered) || (resource == 'preview' && !preview)
|
|
1603
|
+
# The mock prepares no stream, so it has no HLS files.
|
|
1604
|
+
return no_rendered_version if resource == 'hls'
|
|
1605
|
+
|
|
1606
|
+
still = resource == 'preview' || (resource == 'rendered' && rendered['kind'] == 'image')
|
|
1607
|
+
return invalid_transform if !still && selector
|
|
1608
|
+
|
|
1609
|
+
refused = selector ? TransformRules.check(selector).flatten : []
|
|
1610
|
+
return error(ErrorCodes::INVALID_TRANSFORM_OPTIONS, "Invalid transform options: #{refused.join(', ')}", status: 400) if refused.any?
|
|
1611
|
+
|
|
1612
|
+
bytes, type = resource_body(item, resource, stored)
|
|
1613
|
+
return not_available unless bytes
|
|
1614
|
+
|
|
1615
|
+
immutable = still || (resource == 'original' && item['type'] == 'image')
|
|
1616
|
+
cdn_response(method, bytes, viewer_delivery_headers(item, type, restricted, immutable, params))
|
|
1617
|
+
end
|
|
1618
|
+
|
|
1619
|
+
# The delivery checks, which refuse everyone, then age and access, as an error response, or nil.
|
|
1620
|
+
# Only the original needs the upload still stored.
|
|
1621
|
+
def refuse_delivery(item, resource, stored, params, authorised)
|
|
1622
|
+
moderation = item['classification']['moderation']['status']
|
|
1623
|
+
if (resource == 'original' && stored.nil?) || item['upload']['status'] == 'uploading' ||
|
|
1624
|
+
%w[blocked purged].include?(moderation) || item['classification']['status'] == 'failed' ||
|
|
1625
|
+
item['content']['status'] == 'removed'
|
|
1626
|
+
return not_available
|
|
1627
|
+
end
|
|
1628
|
+
if moderation == 'age-restricted' && !viewer_age_confirmed?(params)
|
|
1629
|
+
return error('viewer/age-confirmation-required', 'Age confirmation required', status: 403)
|
|
1196
1630
|
end
|
|
1631
|
+
return if item['visibility'] == 'public' || authorised
|
|
1197
1632
|
|
|
1198
|
-
|
|
1199
|
-
|
|
1200
|
-
|
|
1633
|
+
error(ErrorCodes::PERMISSION_DENIED, 'This API key needs assets:download permission for this asset’s folder.', status: 403)
|
|
1634
|
+
end
|
|
1635
|
+
|
|
1636
|
+
# A resource's bytes and Content-Type: the upload, which stands in for its own rendered file
|
|
1637
|
+
# and an image's preview, or the placeholder PNG for a still the mock doesn't make. Nil bytes
|
|
1638
|
+
# when the upload is no longer stored.
|
|
1639
|
+
def resource_body(item, resource, stored)
|
|
1640
|
+
rendered = item['content']['rendered']
|
|
1641
|
+
serves_upload = resource == 'original' || (resource == 'rendered' && rendered['kind'] != 'image') || item['type'] == 'image'
|
|
1642
|
+
return [Placeholder.png(item.dig('content', 'preview', 'thumbhash') || THUMBHASH), 'image/png'] unless serves_upload
|
|
1643
|
+
return [nil, nil] unless stored
|
|
1644
|
+
|
|
1645
|
+
type = if resource == 'rendered' then rendered['format']
|
|
1646
|
+
elsif stored['type'].to_s.empty? then 'application/octet-stream'
|
|
1647
|
+
else stored['type']
|
|
1648
|
+
end
|
|
1649
|
+
[stored['parts'].sort.map(&:last).join.b, type]
|
|
1201
1650
|
end
|
|
1202
1651
|
|
|
1203
1652
|
def viewer_age_confirmed?(params)
|
|
1204
1653
|
JSON.parse(params.fetch('viewerClaims', '{}'))['ageConfirmed'] == true
|
|
1205
1654
|
end
|
|
1206
1655
|
|
|
1207
|
-
def viewer_delivery_headers(item, type, restricted, params)
|
|
1208
|
-
headers = delivery_headers(item, type, restricted)
|
|
1656
|
+
def viewer_delivery_headers(item, type, restricted, immutable, params)
|
|
1657
|
+
headers = delivery_headers(item, type, restricted, immutable)
|
|
1209
1658
|
headers['Cache-Control'] = 'private, no-store' if params.key?('viewerClaims')
|
|
1210
1659
|
headers
|
|
1211
1660
|
end
|
|
1212
1661
|
|
|
1662
|
+
# The embed page, which shows any asset that may be delivered, as its viewer for the asset's
|
|
1663
|
+
# type or a notice that the type has none; cached like other non-image content.
|
|
1664
|
+
def embed(method, item, restricted, params)
|
|
1665
|
+
cache = if params.key?('viewerClaims') then 'private, no-store'
|
|
1666
|
+
else "#{restricted ? 'private' : 'public'}, max-age=3600"
|
|
1667
|
+
end
|
|
1668
|
+
cdn_response(method, EMBED_HTML, { 'content-type' => 'text/html; charset=utf-8', 'cache-control' => cache,
|
|
1669
|
+
'cache-tag' => "asset-#{item['id']}", 'x-content-type-options' => 'nosniff' })
|
|
1670
|
+
end
|
|
1671
|
+
|
|
1672
|
+
def invalid_transform = error(ErrorCodes::INVALID_TRANSFORM, 'Invalid transform selector', status: 400)
|
|
1673
|
+
|
|
1674
|
+
def no_rendered_version = error('asset/no-rendered-version', 'This asset has no such resource', status: 404)
|
|
1675
|
+
|
|
1213
1676
|
def not_available = error('asset/not-available', 'Asset is not available', status: 403)
|
|
1214
1677
|
|
|
1215
|
-
# Like the CDN's placeholder handler:
|
|
1678
|
+
# Like the CDN's placeholder handler: the preview's ThumbHash, public whatever the asset's
|
|
1216
1679
|
# visibility or moderation; refused for purged assets and assets without one (not yet
|
|
1217
|
-
#
|
|
1680
|
+
# processed, without a preview, or content removed).
|
|
1218
1681
|
def placeholder(method, item)
|
|
1219
|
-
thumbhash = item.dig('content', '
|
|
1682
|
+
thumbhash = item.dig('content', 'preview', 'thumbhash')
|
|
1220
1683
|
return not_available if thumbhash.nil? || item['classification']['moderation']['status'] == 'purged'
|
|
1221
1684
|
|
|
1222
1685
|
cdn_response(method, Placeholder.png(thumbhash),
|
|
@@ -1224,14 +1687,13 @@ module CalmLens
|
|
|
1224
1687
|
'cache-tag' => "asset-#{item['id']}", 'etag' => "\"#{thumbhash}\"")
|
|
1225
1688
|
end
|
|
1226
1689
|
|
|
1227
|
-
# Like the CDN:
|
|
1228
|
-
# by shared caches only when delivery is public. Active content is served as a download.
|
|
1229
|
-
def delivery_headers(item, type, restricted)
|
|
1230
|
-
|
|
1231
|
-
headers = { 'content-type' => type,
|
|
1232
|
-
'cache-control' => "#{restricted ? 'private' : 'public'}, #{image ? 'max-age=31536000, immutable' : 'max-age=3600'}",
|
|
1690
|
+
# Like the CDN: a still image's bytes are immutable for a year, other content is kept for an
|
|
1691
|
+
# hour, by shared caches only when delivery is public. Active content is served as a download.
|
|
1692
|
+
def delivery_headers(item, type, restricted, immutable)
|
|
1693
|
+
lifetime = immutable ? 'max-age=31536000, immutable' : 'max-age=3600'
|
|
1694
|
+
headers = { 'content-type' => type, 'cache-control' => "#{restricted ? 'private' : 'public'}, #{lifetime}",
|
|
1233
1695
|
'cache-tag' => "asset-#{item['id']}" }
|
|
1234
|
-
headers.merge!('etag' => "\"#{item['id']}\"", 'expires' => Time.at((@now.call / 1000) + 31_536_000).utc.httpdate) if
|
|
1696
|
+
headers.merge!('etag' => "\"#{item['id']}\"", 'expires' => Time.at((@now.call / 1000) + 31_536_000).utc.httpdate) if immutable
|
|
1235
1697
|
headers.merge!('x-content-type-options' => 'nosniff', 'content-security-policy' => "sandbox; default-src 'none'")
|
|
1236
1698
|
mime = type.downcase.split(';', 2).first.to_s.strip
|
|
1237
1699
|
headers['content-disposition'] = 'attachment' if ACTIVE_TYPES.include?(mime) || mime.end_with?('+xml')
|
|
@@ -1243,8 +1705,11 @@ module CalmLens
|
|
|
1243
1705
|
headers: headers.merge('content-length' => bytes.bytesize.to_s, 'x-request-id' => request_id))
|
|
1244
1706
|
end
|
|
1245
1707
|
|
|
1246
|
-
# Whether the mock API key
|
|
1247
|
-
|
|
1708
|
+
# Whether the request carries the mock API key and its grants give `assets:download` for
|
|
1709
|
+
# `folder_id`.
|
|
1710
|
+
def can_download?(folder_id, headers)
|
|
1711
|
+
return false unless header(headers, 'X-Api-Key') == @api_key
|
|
1712
|
+
|
|
1248
1713
|
@grants.any? do |grant|
|
|
1249
1714
|
next false unless Array(grant['actions']).include?('assets:download')
|
|
1250
1715
|
next true if grant['folderId'].nil?
|
|
@@ -1267,6 +1732,35 @@ module CalmLens
|
|
|
1267
1732
|
end
|
|
1268
1733
|
end
|
|
1269
1734
|
|
|
1735
|
+
# The mock's grant, the asset segment of a URL {CDN#sign_asset_url} signs on the mock: `sig_` and
|
|
1736
|
+
# the lowercase hex of the UTF-8 text `{assetId}.{1|0}.{expiresAt}`, its asset, original access and
|
|
1737
|
+
# expiry. Every SDK's mock writes it the same way. It is readable, not signed, as the mock has no
|
|
1738
|
+
# secret to keep.
|
|
1739
|
+
# @api private
|
|
1740
|
+
module MockGrant
|
|
1741
|
+
module_function
|
|
1742
|
+
|
|
1743
|
+
def encode(asset_id, original, expires_at)
|
|
1744
|
+
"sig_#{"#{asset_id}.#{original ? 1 : 0}.#{expires_at}".unpack1('H*')}"
|
|
1745
|
+
end
|
|
1746
|
+
|
|
1747
|
+
# `{asset_id:, original:, expires_at:}`, or nil for a segment the mock didn't sign.
|
|
1748
|
+
def decode(segment)
|
|
1749
|
+
hex = segment.delete_prefix('sig_')
|
|
1750
|
+
return unless hex.match?(/\A(?:[0-9a-f]{2})+\z/)
|
|
1751
|
+
|
|
1752
|
+
text = [hex].pack('H*').force_encoding(Encoding::UTF_8)
|
|
1753
|
+
match = text.valid_encoding? && /\A(.+)\.([01])\.(\d+)\z/m.match(text)
|
|
1754
|
+
match && { asset_id: match[1], original: match[2] == '1', expires_at: Integer(match[3], 10) }
|
|
1755
|
+
end
|
|
1756
|
+
|
|
1757
|
+
# Percent-decodes a path segment like decodeURIComponent; nil when it isn't valid UTF-8.
|
|
1758
|
+
def percent_decode(text)
|
|
1759
|
+
decoded = text.b.gsub(/%(\h\h)/) { Regexp.last_match(1).hex.chr }.force_encoding(Encoding::UTF_8)
|
|
1760
|
+
decoded if decoded.valid_encoding?
|
|
1761
|
+
end
|
|
1762
|
+
end
|
|
1763
|
+
|
|
1270
1764
|
# The processing and moderation status of each outcome.
|
|
1271
1765
|
# @api private
|
|
1272
1766
|
module AssetRecord
|
|
@@ -1285,125 +1779,5 @@ module CalmLens
|
|
|
1285
1779
|
end
|
|
1286
1780
|
end
|
|
1287
1781
|
end
|
|
1288
|
-
|
|
1289
|
-
# The RSQL subset the fake API understands: `==` and `!=` (with `*` wildcards), `=in=` and
|
|
1290
|
-
# `=out=` with a `(value,value)` list, and `=gt=`, `=ge=`, `=lt=`, `=le=`, with quoted values,
|
|
1291
|
-
# combined with `;` (and), which binds tighter than `,` (or), and grouped with `( … )`.
|
|
1292
|
-
# Anything else raises instead of being ignored, so a test never passes on an unfiltered list.
|
|
1293
|
-
# @api private
|
|
1294
|
-
module RSQL
|
|
1295
|
-
module_function
|
|
1296
|
-
|
|
1297
|
-
def filter(items, expression)
|
|
1298
|
-
return items if expression.nil? || expression.to_s.strip.empty?
|
|
1299
|
-
|
|
1300
|
-
test = parse(expression.to_s)
|
|
1301
|
-
items.select { |item| test.call(item) }
|
|
1302
|
-
end
|
|
1303
|
-
|
|
1304
|
-
# Compiles `,` (or) of `;` (and) terms, where a term is a comparison or a `( … )` group.
|
|
1305
|
-
def parse(expression)
|
|
1306
|
-
alternatives = split(expression, ',').map do |alternative|
|
|
1307
|
-
split(alternative, ';').map do |term|
|
|
1308
|
-
text = term.strip
|
|
1309
|
-
raise unsupported(expression) if text.empty?
|
|
1310
|
-
|
|
1311
|
-
group = unwrap(text)
|
|
1312
|
-
group ? parse(group) : comparison(text)
|
|
1313
|
-
end
|
|
1314
|
-
end
|
|
1315
|
-
->(item) { alternatives.any? { |terms| terms.all? { |test| test.call(item) } } }
|
|
1316
|
-
end
|
|
1317
|
-
|
|
1318
|
-
# The expression inside `( … )` when the parentheses enclose all of `text`, else nil.
|
|
1319
|
-
def unwrap(text)
|
|
1320
|
-
return unless text.start_with?('(') && text.end_with?(')')
|
|
1321
|
-
|
|
1322
|
-
depth = 0
|
|
1323
|
-
quote = nil
|
|
1324
|
-
text.each_char.with_index do |char, index|
|
|
1325
|
-
if quote
|
|
1326
|
-
quote = nil if char == quote
|
|
1327
|
-
elsif ['"', "'"].include?(char)
|
|
1328
|
-
quote = char
|
|
1329
|
-
elsif char == '('
|
|
1330
|
-
depth += 1
|
|
1331
|
-
elsif char == ')'
|
|
1332
|
-
depth -= 1
|
|
1333
|
-
return index == text.length - 1 ? text[1..-2] : nil if depth.zero?
|
|
1334
|
-
end
|
|
1335
|
-
end
|
|
1336
|
-
nil
|
|
1337
|
-
end
|
|
1338
|
-
|
|
1339
|
-
def unsupported(term)
|
|
1340
|
-
InvalidArgumentError.new(SdkErrorCodes::INVALID_ARGUMENT,
|
|
1341
|
-
'MockServerClient filters support ==, !=, =in=, =out=, =gt=, =ge=, =lt= and ' \
|
|
1342
|
-
"=le=, joined by ; and , and grouped with parentheses (got #{term.inspect})")
|
|
1343
|
-
end
|
|
1344
|
-
|
|
1345
|
-
# Splits on `separator` outside quotes and parentheses (groups and `=in=(...)` lists).
|
|
1346
|
-
def split(text, separator)
|
|
1347
|
-
parts = ['']
|
|
1348
|
-
quote = nil
|
|
1349
|
-
depth = 0
|
|
1350
|
-
text.each_char do |char|
|
|
1351
|
-
if quote
|
|
1352
|
-
quote = nil if char == quote
|
|
1353
|
-
elsif ['"', "'"].include?(char)
|
|
1354
|
-
quote = char
|
|
1355
|
-
elsif char == '('
|
|
1356
|
-
depth += 1
|
|
1357
|
-
elsif char == ')'
|
|
1358
|
-
depth -= 1
|
|
1359
|
-
elsif char == separator && depth.zero?
|
|
1360
|
-
parts << ''
|
|
1361
|
-
next
|
|
1362
|
-
end
|
|
1363
|
-
parts[-1] += char
|
|
1364
|
-
end
|
|
1365
|
-
parts
|
|
1366
|
-
end
|
|
1367
|
-
|
|
1368
|
-
def comparison(term)
|
|
1369
|
-
match = term.strip.match(/\A([A-Za-z][\w.]*)(==|!=|=in=|=out=|=gt=|=ge=|=lt=|=le=)(.+)\z/m)
|
|
1370
|
-
raise unsupported(term) unless match
|
|
1371
|
-
|
|
1372
|
-
field, operator, raw = match.captures
|
|
1373
|
-
values = list(operator, raw.strip)
|
|
1374
|
-
lambda do |item|
|
|
1375
|
-
actual = field.split('.').reduce(item) { |object, key| object.is_a?(Hash) ? object[key] : nil }
|
|
1376
|
-
entries = Array(actual.nil? ? 'null' : actual).map(&:to_s)
|
|
1377
|
-
case operator
|
|
1378
|
-
when '==', '=in=' then values.any? { |value| matches?(entries, value) }
|
|
1379
|
-
when '!=', '=out=' then values.none? { |value| matches?(entries, value) }
|
|
1380
|
-
else entries.any? { |entry| ordered?(entry, operator, values.first) }
|
|
1381
|
-
end
|
|
1382
|
-
end
|
|
1383
|
-
end
|
|
1384
|
-
|
|
1385
|
-
def list(operator, raw)
|
|
1386
|
-
return [unquote(raw)] unless %w[=in= =out=].include?(operator)
|
|
1387
|
-
|
|
1388
|
-
unless raw.start_with?('(') && raw.end_with?(')')
|
|
1389
|
-
raise InvalidArgumentError.new(SdkErrorCodes::INVALID_ARGUMENT, "#{operator} takes a (value,value) list")
|
|
1390
|
-
end
|
|
1391
|
-
|
|
1392
|
-
split(raw[1..-2], ',').map { |value| unquote(value.strip) }
|
|
1393
|
-
end
|
|
1394
|
-
|
|
1395
|
-
def unquote(value) = value.match?(/\A(["']).*\1\z/m) ? value[1..-2] : value
|
|
1396
|
-
|
|
1397
|
-
def matches?(entries, value)
|
|
1398
|
-
pattern = Regexp.new("\\A#{Regexp.escape(value).gsub('\*', '.*')}\\z", Regexp::MULTILINE)
|
|
1399
|
-
entries.any? { |entry| pattern.match?(entry) }
|
|
1400
|
-
end
|
|
1401
|
-
|
|
1402
|
-
# Numbers compare as numbers and everything else (such as ISO dates) as text.
|
|
1403
|
-
def ordered?(entry, operator, value)
|
|
1404
|
-
left, right = [entry, value].all? { |text| text.match?(/\A-?\d+(\.\d+)?\z/) } ? [entry.to_f, value.to_f] : [entry, value]
|
|
1405
|
-
{ '=gt=' => left > right, '=ge=' => left >= right, '=lt=' => left < right, '=le=' => left <= right }.fetch(operator)
|
|
1406
|
-
end
|
|
1407
|
-
end
|
|
1408
1782
|
end
|
|
1409
1783
|
end
|