calmlens 0.20.0 → 0.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +3 -1
- data/lib/calmlens/assets.rb +107 -98
- data/lib/calmlens/cdn.rb +197 -41
- data/lib/calmlens/error_codes.rb +80 -84
- data/lib/calmlens/errors.rb +2 -2
- data/lib/calmlens/events.rb +50 -1
- data/lib/calmlens/folders.rb +14 -9
- data/lib/calmlens/hash_lists.rb +29 -9
- data/lib/calmlens/internal.rb +9 -19
- data/lib/calmlens/lenses.rb +79 -6
- data/lib/calmlens/option_keys.rb +46 -25
- data/lib/calmlens/policies.rb +50 -18
- data/lib/calmlens/resource.rb +24 -6
- data/lib/calmlens/server_client.rb +8 -6
- data/lib/calmlens/session.rb +29 -12
- data/lib/calmlens/testing/asset_filters.rb +39 -0
- data/lib/calmlens/testing/classification.rb +8 -2
- data/lib/calmlens/testing/fake_api.rb +682 -368
- data/lib/calmlens/testing/mock_decision.rb +5 -4
- data/lib/calmlens/testing/rsql.rb +245 -0
- data/lib/calmlens/testing/test_input_detections.rb +61 -0
- data/lib/calmlens/testing/test_input_scenario.rb +6 -2
- data/lib/calmlens/testing/test_inputs.rb +31 -10
- data/lib/calmlens/testing.rb +36 -12
- data/lib/calmlens/transform_rules.rb +1 -1
- data/lib/calmlens/upload_mime_types.rb +9 -1
- data/lib/calmlens/upload_tokens.rb +3 -3
- data/lib/calmlens/version.rb +1 -1
- data/lib/calmlens/webhooks.rb +58 -13
- data/sig/calmlens.rbs +87 -13
- data/sig/error_codes.rbs +18 -20
- data/sig/qualifiers.rbs +2 -0
- data/sig/types.rbs +326 -187
- metadata +5 -2
|
@@ -6,6 +6,7 @@ require 'securerandom'
|
|
|
6
6
|
require 'uri'
|
|
7
7
|
require_relative 'placeholder'
|
|
8
8
|
require_relative 'mock_decision'
|
|
9
|
+
require_relative 'rsql'
|
|
9
10
|
|
|
10
11
|
module CalmLens
|
|
11
12
|
module Testing
|
|
@@ -36,35 +37,69 @@ module CalmLens
|
|
|
36
37
|
:hash_list_entries, keyword_init: true)
|
|
37
38
|
|
|
38
39
|
# A stateful stand-in for the CalmLens API and CDN. It stores assets and folders, applies
|
|
39
|
-
# filters and
|
|
40
|
+
# filters and sorting, serves stored bytes, and answers errors with the API's codes, statuses,
|
|
40
41
|
# details and body shape, checking requests in the API's order. It moderates through the
|
|
41
42
|
# `moderate` callback rather than classifying content, and does not run transforms or index for
|
|
42
43
|
# semantic search.
|
|
43
44
|
# @api private
|
|
44
45
|
class FakeAPI
|
|
45
|
-
JSON_QUERY = %w[
|
|
46
|
-
context].freeze
|
|
46
|
+
JSON_QUERY = %w[wait isFinal sizeInBytes totalSizeInBytes partNumber pageSize processingOptions metadata
|
|
47
|
+
tags context].freeze
|
|
47
48
|
# Keyword search fields, in the order matches are reported, and where each value sits in the asset.
|
|
48
49
|
SEARCH_FIELDS = { 'name' => %w[name], 'id' => %w[id], 'externalId' => %w[externalId],
|
|
49
50
|
'description' => %w[description], 'tags' => %w[tags], 'type' => %w[type],
|
|
50
51
|
'content.format' => %w[content format], 'content.extension' => %w[content extension],
|
|
51
52
|
'classification.moderation.status' => %w[classification moderation status] }.freeze
|
|
52
53
|
MAX_SEARCH_TOKENS = 8
|
|
53
|
-
|
|
54
|
+
MAX_SEARCH_PAGE_SIZE = 20
|
|
54
55
|
MAX_SEARCH_QUERY_LENGTH = 200
|
|
55
56
|
# Every API cursor list returns this many items unless `pageSize` says otherwise.
|
|
56
57
|
DEFAULT_PAGE_SIZE = 50
|
|
57
58
|
MAX_PAGE_SIZE = 100
|
|
58
59
|
REVIEW_DECISIONS = %w[allowed limited age-restricted blocked purged].freeze
|
|
60
|
+
# The moderation status each `asset.<status>` event announces.
|
|
61
|
+
ANNOUNCED_STATUSES = REVIEW_DECISIONS.to_h { |status| ["asset.#{status}", status] }.freeze
|
|
59
62
|
MAX_DECISION_NOTE_LENGTH = 1499
|
|
60
|
-
|
|
63
|
+
# The actor of events the mock's API key causes: mock keys have no ID.
|
|
64
|
+
API_KEY_ACTOR = { 'type' => 'api-key', 'id' => nil }.freeze
|
|
65
|
+
# The actor of a reviewer's decision through {MockServerClient#decide}.
|
|
66
|
+
REVIEWER_ACTOR = { 'type' => 'user', 'id' => nil }.freeze
|
|
67
|
+
# The actor of the pipeline's processing events.
|
|
68
|
+
AUTOMATED_ACTOR = { 'type' => 'automated', 'id' => nil }.freeze
|
|
69
|
+
# An event type's shape (`resource.action`), as the mock checks `types` without the catalog.
|
|
70
|
+
EVENT_TYPE_PATTERN = /\A[a-z][a-z-]*\.[a-z][a-z-]*\z/
|
|
71
|
+
# The resources an asset's CDN path names.
|
|
72
|
+
RESOURCES = %w[original rendered preview hls embed placeholder info].freeze
|
|
73
|
+
# The page the mock serves at `/embed`, standing in for the CDN's embed page.
|
|
74
|
+
EMBED_HTML = '<!doctype html><html lang="en"><head><meta charset="utf-8"><title>Embed</title></head>' \
|
|
75
|
+
'<body></body></html>'
|
|
76
|
+
|
|
61
77
|
MIN_PART = 5 * 1024 * 1024
|
|
62
78
|
MAX_MULTIPART_IMAGE = 20 * 1024 * 1024
|
|
63
79
|
# Like the API: an upload has at most this many parts.
|
|
64
80
|
MAX_PARTS = 10_000
|
|
65
|
-
# The ThumbHash the mock gives every
|
|
66
|
-
# pixels. The `placeholder` resource serves it as a PNG, like the CDN.
|
|
81
|
+
# The ThumbHash the mock gives every processed asset's preview, standing in for one computed
|
|
82
|
+
# from its pixels. The `placeholder` resource serves it as a PNG, like the CDN.
|
|
67
83
|
THUMBHASH = '1QcSHQRnh493V4dIh4eXh1h4kJUI'
|
|
84
|
+
# Removing or withholding content removes its rendered file, adaptive stream and preview, as the API does.
|
|
85
|
+
NO_RENDERING = { 'rendered' => nil, 'hls' => nil, 'preview' => nil }.freeze
|
|
86
|
+
# The size of the waveform the API makes as an audio asset's preview.
|
|
87
|
+
WAVEFORM_SIZE = { 'width' => 1200, 'height' => 300 }.freeze
|
|
88
|
+
# Audio and video formats browsers play as uploaded, so the API serves them as stored (Asset.ts).
|
|
89
|
+
BROWSER_SAFE_MEDIA_FORMATS = %w[video/mp4 audio/mpeg audio/aac audio/mp4 audio/x-m4a audio/ogg].freeze
|
|
90
|
+
# Spreadsheets, which have no rendered file or preview (Asset.ts).
|
|
91
|
+
SPREADSHEET_MIME_TYPES = %w[application/vnd.openxmlformats-officedocument.spreadsheetml.sheet
|
|
92
|
+
application/vnd.ms-excel.sheet.macroenabled.12 application/vnd.ms-excel
|
|
93
|
+
application/vnd.oasis.opendocument.spreadsheet].freeze
|
|
94
|
+
# The settings of the mock's presets: every category with its default action (`allow` for
|
|
95
|
+
# personal information and profanity, `block` otherwise) and no rules. The mock holds no
|
|
96
|
+
# built-in preset settings, and uploads are never decided by a policy.
|
|
97
|
+
PRESET_SETTINGS = {
|
|
98
|
+
'categorySettings' => MockClassification::CATEGORIES.to_h do |category|
|
|
99
|
+
[category, { 'action' => %w[personal-information profanity].include?(category) ? 'allow' : 'block',
|
|
100
|
+
'allowances' => [], 'restrictions' => [] }]
|
|
101
|
+
end
|
|
102
|
+
}.freeze
|
|
68
103
|
# The built-in presets the mock knows, by display name.
|
|
69
104
|
PRESET_NAMES = { 'general-community' => 'General', 'children-platform' => 'Kids',
|
|
70
105
|
'education-support' => 'Learning and support', 'wholesome-community' => 'Teen-safe' }.freeze
|
|
@@ -77,8 +112,9 @@ module CalmLens
|
|
|
77
112
|
# The mock API key's `{ 'actions', 'folderId', 'includeSubfolders' }` grants.
|
|
78
113
|
attr_accessor :grants
|
|
79
114
|
|
|
80
|
-
def initialize(project_id:, base_url:, cdn_base_url:, now:, create_id:, moderate:, grants:)
|
|
115
|
+
def initialize(project_id:, base_url:, cdn_base_url:, now:, create_id:, moderate:, api_key:, grants:)
|
|
81
116
|
@project_id = project_id
|
|
117
|
+
@api_key = api_key
|
|
82
118
|
@base_url = base_url
|
|
83
119
|
@cdn_base_url = cdn_base_url
|
|
84
120
|
@now = now
|
|
@@ -92,22 +128,41 @@ module CalmLens
|
|
|
92
128
|
@policies = {}
|
|
93
129
|
@project_policy_id = nil
|
|
94
130
|
@request_count = 0
|
|
131
|
+
# Each asset's latest `assetSequence`, which the API keeps on the asset row.
|
|
132
|
+
@asset_event_sequences = Hash.new(0)
|
|
133
|
+
# The processing options each asset was uploaded with, which the API keeps but doesn't return.
|
|
134
|
+
@processing_options = {}
|
|
135
|
+
# Decided assets whose re-run (`assets.explain` with `rerun: true`) is still queued: they keep
|
|
136
|
+
# their decision until `complete_processing` decides them.
|
|
137
|
+
@reruns = {}
|
|
95
138
|
end
|
|
96
139
|
|
|
97
|
-
# Adds an event to the feed, filling the fields the API always sets.
|
|
140
|
+
# Adds an event to the feed, filling the fields the API always sets. Its `actor` is the mock's
|
|
141
|
+
# API key unless the event names another.
|
|
98
142
|
def add_event(event)
|
|
99
143
|
time = @now.call
|
|
100
|
-
record = { 'id' => @create_id.call('evt'), 'projectId' => @project_id, '
|
|
101
|
-
'
|
|
144
|
+
record = { 'id' => @create_id.call('evt'), 'projectId' => @project_id, 'actor' => deep_copy(API_KEY_ACTOR),
|
|
145
|
+
'severity' => 'low', 'description' => nil,
|
|
146
|
+
'assetId' => nil, 'assetSequence' => nil, 'asset' => nil, 'facts' => [],
|
|
147
|
+
'createdAt' => time }.merge(event)
|
|
102
148
|
record['title'] ||= record['type']
|
|
149
|
+
# Like the API: each event about an asset takes the asset's next number; others have none.
|
|
150
|
+
asset_id = record['assetId']
|
|
151
|
+
record['assetSequence'] = asset_id ? (@asset_event_sequences[asset_id] += 1) : nil
|
|
103
152
|
@state.events << record
|
|
104
153
|
record
|
|
105
154
|
end
|
|
106
155
|
|
|
107
|
-
# Adds an event like {#add_event}; one about a stored asset carries the asset as it is now
|
|
156
|
+
# Adds an event like {#add_event}; one about a stored asset carries the asset as it is now,
|
|
157
|
+
# with the moderation status an `asset.<status>` event announces.
|
|
108
158
|
def enqueue_event(event)
|
|
109
159
|
item = own_asset(event['assetId'])
|
|
110
|
-
|
|
160
|
+
if item && !event.key?('asset')
|
|
161
|
+
asset = snapshot(item)
|
|
162
|
+
announced = ANNOUNCED_STATUSES[event['type']]
|
|
163
|
+
asset['classification']['moderation']['status'] = announced if announced
|
|
164
|
+
event = event.merge('asset' => asset)
|
|
165
|
+
end
|
|
111
166
|
add_event(event)
|
|
112
167
|
end
|
|
113
168
|
|
|
@@ -121,25 +176,29 @@ module CalmLens
|
|
|
121
176
|
item = own_asset(asset_id)
|
|
122
177
|
return unless item
|
|
123
178
|
|
|
124
|
-
record_decision(item, decision, nil,
|
|
179
|
+
record_decision(item, decision, nil, false)
|
|
125
180
|
deep_copy(item)
|
|
126
181
|
end
|
|
127
182
|
|
|
128
183
|
# Finishes processing an asset as the pipeline would, with `decision` or else what `moderate`
|
|
129
|
-
# answers for the `complete` trigger, and returns it; nil when there is no such asset.
|
|
184
|
+
# answers for the `complete` trigger, and returns it; nil when there is no such asset. A decided
|
|
185
|
+
# asset whose re-run is still queued counts as processing.
|
|
130
186
|
def complete_processing(asset_id, decision)
|
|
131
187
|
item = own_asset(asset_id)
|
|
132
188
|
return unless item
|
|
133
|
-
raise Internal.invalid("Asset #{asset_id} isn't processing") unless item['classification']['status'] == 'processing'
|
|
134
189
|
|
|
135
|
-
|
|
190
|
+
rerun = @reruns.key?(asset_id)
|
|
191
|
+
raise Internal.invalid("Asset #{asset_id} isn't processing") unless rerun || item['classification']['status'] == 'processing'
|
|
192
|
+
|
|
193
|
+
settled = decision.nil? ? nil : MockDecision.from(decision)
|
|
194
|
+
rerun ? settle_rerun(item, 'complete', settled) : moderate_stored(item, 'complete', settled)
|
|
136
195
|
deep_copy(item)
|
|
137
196
|
end
|
|
138
197
|
|
|
139
198
|
# @return [Response]
|
|
140
199
|
def handle(method, url, headers, body)
|
|
141
200
|
if url.start_with?("#{@cdn_base_url}/")
|
|
142
|
-
serve(method, url.delete_prefix(@cdn_base_url))
|
|
201
|
+
serve(method, url.delete_prefix(@cdn_base_url), headers)
|
|
143
202
|
else
|
|
144
203
|
api(method, url.delete_prefix(@base_url), headers, body)
|
|
145
204
|
end
|
|
@@ -172,29 +231,29 @@ module CalmLens
|
|
|
172
231
|
end
|
|
173
232
|
|
|
174
233
|
# Applies a reviewer's decision like `decideAssetManually` in the API: the status and the review,
|
|
175
|
-
# then (for `purged`) the content removal and its event, then `asset.<decision>`.
|
|
176
|
-
|
|
234
|
+
# then (for `purged`) the content removal and its event, then `asset.<decision>`. Either way the
|
|
235
|
+
# review is the mock API key's, which has no ID. A decision through the API (`by_api_key`) has
|
|
236
|
+
# the API key as its events' actor; the test hook's is a reviewer's (a `user`).
|
|
237
|
+
def record_decision(item, decision, note, by_api_key)
|
|
238
|
+
# Like the API: a queued re-run no longer replaces a reviewed decision.
|
|
239
|
+
@reruns.delete(item['id'])
|
|
177
240
|
apply_outcome(item, decision, 'requested')
|
|
178
241
|
review = { 'id' => @create_id.call('rev'), 'decision' => decision, 'note' => note,
|
|
179
|
-
'reviewedAt' => @now.call, '
|
|
180
|
-
|
|
181
|
-
if reviewer_name
|
|
182
|
-
review.merge!('reviewerName' => reviewer_name, 'reviewerEmail' => nil)
|
|
183
|
-
reviewer_facts = [{ 'key' => 'reviewer-user-id', 'value' => nil }, { 'key' => 'reviewer-name', 'value' => reviewer_name }]
|
|
184
|
-
end
|
|
242
|
+
'decidedBy' => { 'type' => 'api-key', 'id' => nil }, 'reviewedAt' => @now.call, 'applied' => true }
|
|
243
|
+
actor = by_api_key ? API_KEY_ACTOR : REVIEWER_ACTOR
|
|
185
244
|
item['classification']['moderation']['manualReviews'] << review
|
|
186
|
-
lifecycle_event('asset.content-removed', item) if decision == 'purged'
|
|
245
|
+
lifecycle_event('asset.content-removed', item, actor: actor) if decision == 'purged'
|
|
187
246
|
# Like the event catalog: only allowing and limiting are low severity.
|
|
188
247
|
add_event('type' => "asset.#{decision}", 'severity' => %w[allowed limited].include?(decision) ? 'low' : 'high',
|
|
189
248
|
'title' => "asset.#{decision}", 'assetId' => item['id'], 'asset' => snapshot(item),
|
|
190
|
-
'
|
|
249
|
+
'actor' => deep_copy(actor),
|
|
250
|
+
'facts' => [{ 'key' => 'asset-type', 'value' => item['type'] },
|
|
191
251
|
{ 'key' => 'asset-name', 'value' => item['name'], 'url' => "https://calmlens.com/go/asset/#{item['id']}" },
|
|
192
|
-
|
|
252
|
+
{ 'key' => 'note', 'value' => note }])
|
|
193
253
|
end
|
|
194
254
|
|
|
195
|
-
# `POST /assets/:id/decision` as the API validates and applies it
|
|
196
|
-
#
|
|
197
|
-
# refused with `asset/processing`.
|
|
255
|
+
# `POST /assets/:id/decision` as the API validates and applies it, with the API key as its
|
|
256
|
+
# events' actor. An asset still uploading or processing is refused with `asset/processing`.
|
|
198
257
|
def decide_request(item, json)
|
|
199
258
|
decision = json['decision']
|
|
200
259
|
invalid!('validation/invalid-union', 'decision') unless REVIEW_DECISIONS.include?(decision)
|
|
@@ -211,7 +270,7 @@ module CalmLens
|
|
|
211
270
|
fail!(ErrorCodes::ASSET_DECISION_UNCHANGED) if status == decision
|
|
212
271
|
fail!(ErrorCodes::ASSET_CONTENT_PURGED) if status == 'purged' && decision != 'blocked'
|
|
213
272
|
|
|
214
|
-
record_decision(item, decision, note,
|
|
273
|
+
record_decision(item, decision, note, true)
|
|
215
274
|
item
|
|
216
275
|
end
|
|
217
276
|
|
|
@@ -219,26 +278,34 @@ module CalmLens
|
|
|
219
278
|
def snapshot(item)
|
|
220
279
|
{ 'id' => item['id'], 'externalId' => item['externalId'], 'uploadPresetId' => item['uploadPresetId'],
|
|
221
280
|
'upload' => { 'status' => item['upload']['status'] },
|
|
222
|
-
'classification' => { 'status' => item['classification']['status'],
|
|
281
|
+
'classification' => { 'status' => item['classification']['status'], 'testInput' => item['classification']['testInput'],
|
|
223
282
|
'moderation' => { 'status' => item['classification']['moderation']['status'] } },
|
|
224
283
|
'content' => { 'status' => item['content']['status'] } }
|
|
225
284
|
end
|
|
226
285
|
|
|
227
|
-
# The `asset.deleted`, `asset.content-removed` and `asset.retried` events the API records
|
|
228
|
-
|
|
286
|
+
# The `asset.deleted`, `asset.content-removed` and `asset.retried` events the API records, with
|
|
287
|
+
# the `batch-id` fact of the batch that caused them, if any.
|
|
288
|
+
def lifecycle_event(type, item, batch_id: nil, actor: API_KEY_ACTOR)
|
|
229
289
|
name = { 'key' => 'asset-name', 'value' => item['name'] }
|
|
230
290
|
# A deleted asset has no portal page to link to.
|
|
231
291
|
name['url'] = "https://calmlens.com/go/asset/#{item['id']}" unless type == 'asset.deleted'
|
|
292
|
+
facts = [name, { 'key' => 'asset-type', 'value' => item['type'] }]
|
|
293
|
+
facts << { 'key' => 'batch-id', 'value' => batch_id } if batch_id
|
|
232
294
|
add_event('type' => type, 'severity' => 'low', 'assetId' => item['id'], 'asset' => snapshot(item),
|
|
233
|
-
'
|
|
295
|
+
'actor' => deep_copy(actor), 'facts' => facts)
|
|
234
296
|
end
|
|
235
297
|
|
|
236
298
|
def fail!(code, message = nil, details: nil) = raise(Failure.new(code, message, details: details))
|
|
237
299
|
|
|
238
300
|
def route_not_found!(method, parts) = fail!(ErrorCodes::ROUTE_NOT_FOUND, "No route for #{method} /#{parts.join('/')}")
|
|
239
301
|
|
|
240
|
-
# A request the API's schema refuses: `validation
|
|
241
|
-
|
|
302
|
+
# A request the API's schema refuses: `validation/invalid-request` with HTTP 400, listing the
|
|
303
|
+
# field and its issue code (such as `validation/too-big`) in `details.issues`.
|
|
304
|
+
def invalid!(code, path)
|
|
305
|
+
message = "Invalid #{path}"
|
|
306
|
+
raise Failure.new(ErrorCodes::INVALID_REQUEST, message,
|
|
307
|
+
details: { 'issues' => [{ 'path' => path, 'message' => message, 'code' => code }] }, status: 400)
|
|
308
|
+
end
|
|
242
309
|
|
|
243
310
|
# Like the API's schema (METADATA_ITEM_SCHEMA): metadata is nil or a list of `{id, value}`
|
|
244
311
|
# Hashes with a String `id` and any JSON `value`.
|
|
@@ -253,32 +320,23 @@ module CalmLens
|
|
|
253
320
|
end
|
|
254
321
|
end
|
|
255
322
|
|
|
256
|
-
# Like the API's schema, checked before the asset is looked up: asset updates take no null
|
|
257
|
-
# name, and signed URLs no null field.
|
|
323
|
+
# Like the API's schema, checked before the asset is looked up: asset updates take no null name.
|
|
258
324
|
def check_asset_body(method, action, json)
|
|
259
|
-
|
|
260
|
-
in ['PATCH', nil]
|
|
261
|
-
check_metadata(json['metadata'])
|
|
262
|
-
invalid!('validation/invalid-type', 'name') if json.key?('name') && json['name'].nil?
|
|
263
|
-
in ['POST', 'signed-url']
|
|
264
|
-
%w[viewerClaims resource lens transform expiresInSeconds].each do |key|
|
|
265
|
-
next unless json.key?(key) && json[key].nil?
|
|
325
|
+
return unless method == 'PATCH' && action.nil?
|
|
266
326
|
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
else nil
|
|
270
|
-
end
|
|
327
|
+
check_metadata(json['metadata'])
|
|
328
|
+
invalid!('validation/invalid-type', 'name') if json.key?('name') && json['name'].nil?
|
|
271
329
|
end
|
|
272
330
|
|
|
273
331
|
# Like the API: an upload's metadata is checked, then its MIME type, which must be one the
|
|
274
|
-
# API can moderate (`upload/
|
|
332
|
+
# API can moderate (`upload/unsupported-type` with `details.mimeType` and `details.source`).
|
|
275
333
|
def check_upload(options)
|
|
276
334
|
check_metadata(options['metadata'])
|
|
277
335
|
type = options['contentType']
|
|
278
336
|
return if type.nil? || Internal.accepted_upload_mime_type?(type)
|
|
279
337
|
|
|
280
|
-
fail!(ErrorCodes::
|
|
281
|
-
details: { 'mimeType' => type })
|
|
338
|
+
fail!(ErrorCodes::UNSUPPORTED_UPLOAD_TYPE, "#{type} is not a valid upload file type",
|
|
339
|
+
details: { 'mimeType' => type, 'source' => 'calmlens' })
|
|
282
340
|
end
|
|
283
341
|
|
|
284
342
|
# Like the API's query schema: an integer from 1 to `max`.
|
|
@@ -336,6 +394,8 @@ module CalmLens
|
|
|
336
394
|
when 'folders' then folder(method, parts, json)
|
|
337
395
|
when 'hash-lists' then hash_list(method, parts, query, json)
|
|
338
396
|
when 'hash-list-entries' then remove_hash_list_entry(method, parts)
|
|
397
|
+
when 'policies' then policies(method, parts, query, json)
|
|
398
|
+
when 'lenses' then lens(method, parts, json)
|
|
339
399
|
else route_not_found!(method, parts)
|
|
340
400
|
end
|
|
341
401
|
data.is_a?(Response) ? data : respond(200, deep_copy(data))
|
|
@@ -350,19 +410,21 @@ module CalmLens
|
|
|
350
410
|
case [method, parts[2..], upload]
|
|
351
411
|
in ['GET', ['events'], _] then events(query)
|
|
352
412
|
in ['PUT', ['lenses'], _] then lenses(json.fetch('lenses', []))
|
|
413
|
+
in ['GET', ['lenses'], _] then page(@state.lenses.values, query, JSON.generate(['lenses', @project_id]))
|
|
414
|
+
in ['POST', ['lenses'], _] then create_lens(json)
|
|
353
415
|
in ['GET', ['folders'], _] then folder_page(query)
|
|
354
416
|
in ['POST', ['folders'], _] then create_folder(json)
|
|
355
417
|
in ['GET', ['hash-lists'], _] then page(project_hash_lists, query, JSON.generate([@project_id]))
|
|
356
418
|
in ['POST', ['hash-lists'], _] then create_hash_list(json)
|
|
357
|
-
in ['GET', ['
|
|
358
|
-
in ['POST', ['policies'], _] then create_policy(json)
|
|
419
|
+
in ['GET', ['policy'], _] then show_policy(find_policy(ensure_policies))
|
|
359
420
|
in ['PUT', ['policy'], _] then show_policy(assign_policy(json))
|
|
360
|
-
in [_, ['policies', String => policy_id, *rest], _] then policy(method, policy_id, rest, json)
|
|
361
421
|
in ['GET', ['assets'], _] then list(query)
|
|
362
422
|
in ['GET', ['assets', 'search'], _] then search(query)
|
|
423
|
+
in ['GET', ['assets', String => asset_id, 'timeline'], _] then timeline(asset_id, query)
|
|
363
424
|
in ['POST', ['assets', 'batch'], _] then batch(json)
|
|
364
|
-
in ['POST', ['
|
|
365
|
-
in ['POST', ['assets'],
|
|
425
|
+
in ['POST', ['signed-asset-urls'], _] then sign_asset_url(json)
|
|
426
|
+
in ['POST', ['assets'], nil] then create_asset(json, query['wait'] == false, url: json['url'])
|
|
427
|
+
in ['POST', ['assets'], 'content'] then create_asset(query, query['wait'] == false, data: body)
|
|
366
428
|
in ['POST', ['assets'], 'part'] then start_multipart(query, body)
|
|
367
429
|
else route_not_found!(method, parts)
|
|
368
430
|
end
|
|
@@ -379,7 +441,8 @@ module CalmLens
|
|
|
379
441
|
# (`asset/not-found`) before anything happens. The operation then runs at once, in request
|
|
380
442
|
# order, rather than in the background: `delete` deletes each asset with its stored child
|
|
381
443
|
# assets and records `asset.deleted` with the asset as it was; `remove-content` removes the
|
|
382
|
-
# content like DELETE /assets/:id/content.
|
|
444
|
+
# content like DELETE /assets/:id/content. Each event carries the batch's `batch-id` fact. Mock
|
|
445
|
+
# assets carry no safety holds.
|
|
383
446
|
def batch(json)
|
|
384
447
|
ids = json['assetIds']
|
|
385
448
|
invalid!('validation/invalid-type', 'assetIds') unless ids.is_a?(Array)
|
|
@@ -394,15 +457,16 @@ module CalmLens
|
|
|
394
457
|
fail!(ErrorCodes::ASSET_NOT_FOUND) unless item && item['projectId'] == @project_id
|
|
395
458
|
item
|
|
396
459
|
end
|
|
460
|
+
batch_id = @create_id.call('batch')
|
|
397
461
|
items.each do |item|
|
|
398
462
|
if json['operation'] == 'delete'
|
|
399
463
|
delete_tree(item)
|
|
400
|
-
lifecycle_event('asset.deleted', item)
|
|
464
|
+
lifecycle_event('asset.deleted', item, batch_id: batch_id)
|
|
401
465
|
else
|
|
402
|
-
|
|
466
|
+
remove_content(item, batch_id)
|
|
403
467
|
end
|
|
404
468
|
end
|
|
405
|
-
respond(202, { 'queued' =>
|
|
469
|
+
respond(202, { 'batchId' => batch_id, 'status' => 'queued', 'assetCount' => items.size })
|
|
406
470
|
end
|
|
407
471
|
|
|
408
472
|
# Deletes an asset, its stored child assets and their content.
|
|
@@ -417,26 +481,26 @@ module CalmLens
|
|
|
417
481
|
deleting.each do |id|
|
|
418
482
|
@state.assets.delete(id)
|
|
419
483
|
@state.content.delete(id)
|
|
484
|
+
@processing_options.delete(id)
|
|
420
485
|
end
|
|
421
486
|
end
|
|
422
487
|
|
|
423
488
|
def list(query)
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
page(RSQL.filter(
|
|
489
|
+
# Like the API: the query schema is checked before the filter.
|
|
490
|
+
page_query(query)
|
|
491
|
+
page(RSQL.filter(project_assets, query['filter']), query, JSON.generate([@project_id, query['filter']]))
|
|
427
492
|
end
|
|
428
493
|
|
|
429
494
|
# Like the API: one keyset page of a parent's child folders, optionally matching `search`, by
|
|
430
|
-
# `name` ascending unless `
|
|
495
|
+
# `name` ascending unless `sort` (one field, `name` or `createdAt`) says otherwise, with an
|
|
431
496
|
# `id` tie-breaker, and the ancestors of the parent and the listed folders. Names compare by
|
|
432
497
|
# code point, so case matters.
|
|
433
498
|
def folder_page(query)
|
|
434
499
|
found = project_folders.select do |f|
|
|
435
500
|
f['parentId'] == query['parentId'] && (!query['search'] || f['name'].downcase.include?(query['search'].downcase))
|
|
436
501
|
end
|
|
437
|
-
invalid!('validation/too-big', 'ordering') if ordering(query, nil).size > 1
|
|
438
502
|
scope = JSON.generate([@project_id, query['parentId'], query['search']])
|
|
439
|
-
result = page(found, query, scope, fields: %w[name createdAt], default: { 'field' => 'name', 'direction' => 'asc' })
|
|
503
|
+
result = page(found, query, scope, fields: %w[name createdAt], max: 1, default: { 'field' => 'name', 'direction' => 'asc' })
|
|
440
504
|
page = result['items']
|
|
441
505
|
ids = ([query['parentId']] + page.map { |f| f['id'] }).compact.uniq
|
|
442
506
|
queue = ids.dup
|
|
@@ -453,46 +517,85 @@ module CalmLens
|
|
|
453
517
|
def project_folders = @state.folders.values.select { |item| item['projectId'] == @project_id }
|
|
454
518
|
|
|
455
519
|
# Like the API: without a cursor, no events and a `nextCursor` for now; with one, up to `pageSize`
|
|
456
|
-
# events added since, oldest first
|
|
457
|
-
#
|
|
520
|
+
# events added since, oldest first, of the `types` (comma-separated) and `assetId` asked for.
|
|
521
|
+
# `nextCursor` is never nil, and a cursor belongs to the filters it was issued with. The mock's
|
|
522
|
+
# key holds every permission, so no event type is excluded.
|
|
458
523
|
def events(query)
|
|
459
524
|
size = check_range(query.fetch('pageSize', DEFAULT_PAGE_SIZE), 'pageSize', MAX_PAGE_SIZE)
|
|
525
|
+
types = query.key?('types') ? query['types'].to_s.split(',', -1) : []
|
|
526
|
+
if query.key?('types') && (types.empty? || types.any? { |type| !type.match?(EVENT_TYPE_PATTERN) })
|
|
527
|
+
invalid!('validation/invalid-value', 'types')
|
|
528
|
+
end
|
|
529
|
+
asset_id = query['assetId']
|
|
530
|
+
invalid!('validation/invalid-format', 'assetId') if asset_id && !asset_id.match?(ASSET_ID_PATTERN)
|
|
531
|
+
filter = [JSON.generate([types.uniq.sort, asset_id])].pack('m0')
|
|
532
|
+
matches = ->(event) { (types.empty? || types.include?(event['type'])) && (asset_id.nil? || event['assetId'] == asset_id) }
|
|
460
533
|
feed = @state.events
|
|
461
|
-
|
|
534
|
+
unless query['cursor']
|
|
535
|
+
return { 'items' => [], 'nextCursor' => "mock_#{feed.size}~#{filter}", 'hasMore' => false, 'excludedTypes' => [] }
|
|
536
|
+
end
|
|
462
537
|
|
|
463
|
-
match = query['cursor'].to_s.match(/\Amock_(\d+)\z/)
|
|
464
|
-
position = match && Integer(match[1], 10)
|
|
538
|
+
match = query['cursor'].to_s.match(/\Amock_(\d+)~(.*)\z/)
|
|
539
|
+
position = match && match[2] == filter ? Integer(match[1], 10) : nil
|
|
465
540
|
fail!(ErrorCodes::INVALID_CURSOR) unless position && position <= feed.size
|
|
466
541
|
|
|
467
|
-
taken =
|
|
468
|
-
following = position
|
|
469
|
-
|
|
542
|
+
taken = []
|
|
543
|
+
following = position
|
|
544
|
+
while following < feed.size && taken.size < size
|
|
545
|
+
event = feed[following]
|
|
546
|
+
following += 1
|
|
547
|
+
taken << event if matches.call(event)
|
|
548
|
+
end
|
|
549
|
+
{ 'items' => taken, 'nextCursor' => "mock_#{following}~#{filter}", 'hasMore' => feed.drop(following).any?(&matches),
|
|
550
|
+
'excludedTypes' => [] }
|
|
470
551
|
end
|
|
471
552
|
|
|
472
|
-
# Saves lenses in order, like the API:
|
|
473
|
-
#
|
|
553
|
+
# Saves lenses in order, like the API: a new identifier creates a lens, an unchanged
|
|
554
|
+
# definition returns the stored lens and a changed one updates it in place.
|
|
474
555
|
def lenses(definitions)
|
|
475
556
|
{ 'lenses' => definitions.map { |definition| save_lens(definition) } }
|
|
476
557
|
end
|
|
477
558
|
|
|
478
559
|
def save_lens(definition)
|
|
479
|
-
now = @now.call
|
|
480
560
|
old = @state.lenses[definition['identifier']]
|
|
481
|
-
unless old
|
|
482
|
-
lens = { 'id' => @create_id.call('lens'), 'projectId' => @project_id }.merge(definition)
|
|
483
|
-
return @state.lenses[definition['identifier']] = lens.merge('createdAt' => now, 'updatedAt' => now)
|
|
484
|
-
end
|
|
561
|
+
return new_lens(definition) unless old
|
|
485
562
|
|
|
486
563
|
same = %w[name type active].all? { |key| old[key] == definition[key] } &&
|
|
487
564
|
old['description'] == definition['description'] && canonical(old['transform']) == canonical(definition['transform'])
|
|
488
|
-
return old if same
|
|
489
|
-
if definition['version'].to_i > old['version'].to_i
|
|
490
|
-
return @state.lenses[old['identifier']] =
|
|
491
|
-
old.merge(definition).merge('updatedAt' => now)
|
|
492
|
-
end
|
|
565
|
+
return old if same
|
|
493
566
|
|
|
494
|
-
|
|
495
|
-
|
|
567
|
+
@state.lenses[old['identifier']] = old.merge(definition).merge('updatedAt' => @now.call)
|
|
568
|
+
end
|
|
569
|
+
|
|
570
|
+
def new_lens(definition)
|
|
571
|
+
now = @now.call
|
|
572
|
+
lens = { 'id' => @create_id.call('lens'), 'projectId' => @project_id }.merge(definition)
|
|
573
|
+
@state.lenses[definition['identifier']] = lens.merge('createdAt' => now, 'updatedAt' => now)
|
|
574
|
+
end
|
|
575
|
+
|
|
576
|
+
# POST /projects/:id/lenses: like the API, the identifier must be new to the project.
|
|
577
|
+
def create_lens(json)
|
|
578
|
+
identifier = json['identifier']
|
|
579
|
+
invalid!('validation/invalid-type', 'identifier') unless identifier.is_a?(String) && !identifier.empty?
|
|
580
|
+
fail!(ErrorCodes::LENS_IDENTIFIER_TAKEN, details: { 'identifier' => identifier }) if @state.lenses.key?(identifier)
|
|
581
|
+
|
|
582
|
+
# Like the API, a new lens is 201 Created.
|
|
583
|
+
respond(201, deep_copy(new_lens(json)))
|
|
584
|
+
end
|
|
585
|
+
|
|
586
|
+
# GET, PATCH and DELETE /lenses/:id. An update keeps the identifier and changes the rest of
|
|
587
|
+
# the definition, as ensuring the changed definition would.
|
|
588
|
+
def lens(method, parts, json)
|
|
589
|
+
item = @state.lenses.values.find { |lens| lens['id'] == parts[1] && lens['projectId'] == @project_id }
|
|
590
|
+
fail!(ErrorCodes::LENS_NOT_FOUND) unless item && parts.size == 2
|
|
591
|
+
case method
|
|
592
|
+
when 'GET' then item
|
|
593
|
+
when 'PATCH' then save_lens(item.merge(json.except('identifier', 'id', 'projectId', 'createdAt', 'updatedAt')))
|
|
594
|
+
when 'DELETE'
|
|
595
|
+
@state.lenses.delete(item['identifier'])
|
|
596
|
+
{}
|
|
597
|
+
else route_not_found!(method, parts)
|
|
598
|
+
end
|
|
496
599
|
end
|
|
497
600
|
|
|
498
601
|
def canonical(value)
|
|
@@ -509,7 +612,7 @@ module CalmLens
|
|
|
509
612
|
text = query['query'].to_s.strip
|
|
510
613
|
invalid!('validation/too-small', 'query') if text.empty?
|
|
511
614
|
invalid!('validation/too-big', 'query') if text.length > MAX_SEARCH_QUERY_LENGTH
|
|
512
|
-
limit = check_range(query.fetch('
|
|
615
|
+
limit = check_range(query.fetch('pageSize', 10), 'pageSize', MAX_SEARCH_PAGE_SIZE)
|
|
513
616
|
normalized = text.downcase
|
|
514
617
|
tokens = normalized.split(/\s+/).uniq.first(MAX_SEARCH_TOKENS)
|
|
515
618
|
items = project_assets.filter_map do |item|
|
|
@@ -564,13 +667,17 @@ module CalmLens
|
|
|
564
667
|
'folderId' => options['folderId'], 'uploadPresetId' => options['uploadPresetId'],
|
|
565
668
|
'createdAt' => now, 'updatedAt' => now, 'name' => options['name'], 'description' => options['description'],
|
|
566
669
|
'externalId' => options['externalId'], 'externalUrl' => url || options['externalUrl'],
|
|
567
|
-
'type' => kind(mime, url), 'metadata' => options['metadata'], 'authorId' => options['authorId'],
|
|
670
|
+
'type' => kind(mime, url), 'metadata' => options['metadata'] || [], 'authorId' => options['authorId'],
|
|
568
671
|
# Like the API, only the posts CalmLens uses are kept, and only while the content is held.
|
|
569
|
-
'context' => context.empty? ? nil : { 'items' => context }, 'tags' =>
|
|
570
|
-
'
|
|
672
|
+
'context' => context.empty? ? nil : { 'items' => context }, 'tags' => options['tags'] || [],
|
|
673
|
+
'visibility' => options.fetch('visibility', 'private'),
|
|
571
674
|
**sections(now: now, open: open, keep: keep, format: mime, size: data&.bytesize) }
|
|
572
675
|
@state.assets[asset['id']] = asset
|
|
573
|
-
@
|
|
676
|
+
@processing_options[asset['id']] = processing
|
|
677
|
+
if data
|
|
678
|
+
@state.content[asset['id']] = { 'type' => mime, 'parts' => { 1 => data } }
|
|
679
|
+
asset['content'].merge!(rendering(asset, false)) unless open
|
|
680
|
+
end
|
|
574
681
|
open ? asset : upload_completed(asset, queue, data)
|
|
575
682
|
end
|
|
576
683
|
|
|
@@ -578,20 +685,29 @@ module CalmLens
|
|
|
578
685
|
{
|
|
579
686
|
'content' => { 'status' => keep ? 'stored' : 'temporary', 'removalReason' => nil, 'updatedAt' => now,
|
|
580
687
|
'safetyHold' => nil, 'format' => format.empty? ? nil : format, 'extension' => nil,
|
|
581
|
-
|
|
582
|
-
|
|
688
|
+
# The mock decodes nothing, so the file's own details are unknown, and like
|
|
689
|
+
# the API, the rendered file and preview come with processing.
|
|
690
|
+
'sizeInBytes' => size, 'checksums' => nil, 'width' => nil, 'height' => nil,
|
|
691
|
+
'durationSeconds' => nil, 'pageCount' => nil, 'perceptualHash' => nil,
|
|
692
|
+
'perceptualHashQuality' => nil, 'metadata' => nil, 'extractedText' => nil, **NO_RENDERING },
|
|
583
693
|
# Like the API, only an open upload counts the parts received.
|
|
584
694
|
'upload' => { 'status' => open ? 'uploading' : 'completed', 'updatedAt' => now, 'partsReceived' => open ? 1 : nil },
|
|
585
|
-
'classification' => { 'status' => open ? '
|
|
586
|
-
|
|
587
|
-
'
|
|
588
|
-
|
|
695
|
+
'classification' => { 'status' => open ? 'queued' : 'processing', 'failureReason' => nil, 'updatedAt' => now,
|
|
696
|
+
# The mock never reads test inputs.
|
|
697
|
+
'completedAt' => nil, 'progress' => nil, 'testInput' => false,
|
|
698
|
+
# Like the API, moderation's results are null until it decides, and a
|
|
699
|
+
# feature's result is null until it has run.
|
|
700
|
+
'moderation' => { 'status' => 'pending', 'manualReviews' => [], 'automatedStatus' => nil,
|
|
701
|
+
'source' => nil, 'rule' => nil, 'explanationComplete' => nil,
|
|
702
|
+
'categories' => nil, 'flaggedCategories' => [], 'regions' => [],
|
|
703
|
+
'raw' => nil, 'timeline' => nil },
|
|
704
|
+
'ocr' => nil, 'linkDetection' => nil }
|
|
589
705
|
}
|
|
590
706
|
end
|
|
591
707
|
|
|
592
708
|
# Like the API: image uploads in parts total at most 20 MiB, at any size below that.
|
|
593
709
|
def image_size_error(type, size, _final)
|
|
594
|
-
ErrorCodes::
|
|
710
|
+
ErrorCodes::UPLOAD_TOO_LARGE if type == 'image' && size > MAX_MULTIPART_IMAGE
|
|
595
711
|
end
|
|
596
712
|
|
|
597
713
|
# Like the API: the parts may not overshoot a declared total, and must reach it at the end.
|
|
@@ -601,6 +717,12 @@ module CalmLens
|
|
|
601
717
|
ErrorCodes::UPLOAD_SIZE_MISMATCH if received > total || (final && received != total)
|
|
602
718
|
end
|
|
603
719
|
|
|
720
|
+
# The failure for a size error code, with the limit `upload/too-large` carries.
|
|
721
|
+
def size_failure!(code)
|
|
722
|
+
details = code == ErrorCodes::UPLOAD_TOO_LARGE ? { 'limitBytes' => MAX_MULTIPART_IMAGE, 'source' => 'plan' } : nil
|
|
723
|
+
fail!(code, details: details)
|
|
724
|
+
end
|
|
725
|
+
|
|
604
726
|
def check_part_size(size, final)
|
|
605
727
|
return if final || size >= MIN_PART
|
|
606
728
|
|
|
@@ -608,7 +730,7 @@ module CalmLens
|
|
|
608
730
|
end
|
|
609
731
|
|
|
610
732
|
# In the API's order: a declared total, then a single final part as the whole upload, then the
|
|
611
|
-
# part itself.
|
|
733
|
+
# part itself. Whether to wait and the declared total are saved with an open upload.
|
|
612
734
|
def start_multipart(query, data)
|
|
613
735
|
check_upload(query)
|
|
614
736
|
total = query['totalSizeInBytes']
|
|
@@ -617,12 +739,12 @@ module CalmLens
|
|
|
617
739
|
size = data.to_s.bytesize
|
|
618
740
|
code = (total && image_size_error(type, total, true)) ||
|
|
619
741
|
(final ? total_size_error(total, size, true) || image_size_error(type, size, true) : nil)
|
|
620
|
-
|
|
742
|
+
size_failure!(code) if code
|
|
621
743
|
check_part_size(size, final)
|
|
622
744
|
fail!(total_size_error(total, size, false)) if !final && total_size_error(total, size, false)
|
|
623
745
|
|
|
624
|
-
asset = create_asset(query, query['
|
|
625
|
-
@state.uploads[asset['id']] = { 'queue' => query['
|
|
746
|
+
asset = create_asset(query, query['wait'] == false, data: data, open: !final)
|
|
747
|
+
@state.uploads[asset['id']] = { 'queue' => query['wait'] == false, 'total' => total } unless final
|
|
626
748
|
asset
|
|
627
749
|
end
|
|
628
750
|
|
|
@@ -658,8 +780,26 @@ module CalmLens
|
|
|
658
780
|
# Without `moderate`, queued work stays pending until completed, which allows it.
|
|
659
781
|
def default_moderation(trigger, queued) = queued && trigger != 'complete' ? 'pending' : 'allowed'
|
|
660
782
|
|
|
661
|
-
# Applies `decision` to `asset` with its report.
|
|
662
|
-
|
|
783
|
+
# Applies `decision` to `asset` with its report. The mock reads no text, QR codes, people or
|
|
784
|
+
# faces in an image, so a decided image reports that the features its upload ran found nothing:
|
|
785
|
+
# OCR and person and face detection when turned on, link detection unless turned off.
|
|
786
|
+
def decided(asset, decision)
|
|
787
|
+
apply_outcome(asset, decision.status, 'policy', decision.report)
|
|
788
|
+
classification = asset['classification']
|
|
789
|
+
return asset unless classification['status'] == 'completed' && asset['type'] == 'image'
|
|
790
|
+
|
|
791
|
+
processing = @processing_options[asset['id']]
|
|
792
|
+
features = processing.is_a?(Hash) && processing['features'].is_a?(Hash) ? processing['features'] : {}
|
|
793
|
+
setting = ->(name) { features[name].is_a?(Hash) ? features[name]['enabled'] : nil }
|
|
794
|
+
classification['ocr'] = { 'status' => 'evaluated', 'textRegions' => [] } if setting.call('ocr') == true
|
|
795
|
+
unless setting.call('linkDetection') == false
|
|
796
|
+
classification['linkDetection'] = { 'status' => 'evaluated', 'qrCodes' => [], 'links' => [] }
|
|
797
|
+
end
|
|
798
|
+
{ 'personDetection' => 'people', 'faceDetection' => 'faces' }.each do |feature, list|
|
|
799
|
+
classification[feature] = { 'status' => 'evaluated', list => [] } if setting.call(feature) == true
|
|
800
|
+
end
|
|
801
|
+
asset
|
|
802
|
+
end
|
|
663
803
|
|
|
664
804
|
# Asks `moderate` (unless `decision` is given) and settles a processing asset: a decision and
|
|
665
805
|
# its events, or nothing for `pending`.
|
|
@@ -681,6 +821,7 @@ module CalmLens
|
|
|
681
821
|
|
|
682
822
|
if decision.status == 'failed'
|
|
683
823
|
return add_event('type' => 'asset.failed', 'severity' => 'high', 'assetId' => item['id'], 'asset' => snapshot(item),
|
|
824
|
+
'actor' => deep_copy(AUTOMATED_ACTOR),
|
|
684
825
|
'facts' => [asset_name_fact(item), { 'key' => 'asset-type', 'value' => item['type'] },
|
|
685
826
|
{ 'key' => 'previous-status', 'value' => 'processing' },
|
|
686
827
|
{ 'key' => 'failure-reason',
|
|
@@ -689,40 +830,38 @@ module CalmLens
|
|
|
689
830
|
|
|
690
831
|
flagged = decision.report['flaggedCategories']
|
|
691
832
|
add_event('type' => 'asset.classified', 'severity' => 'low', 'assetId' => item['id'], 'asset' => snapshot(item),
|
|
833
|
+
'actor' => deep_copy(AUTOMATED_ACTOR),
|
|
692
834
|
'facts' => [{ 'key' => 'flagged-categories', 'value' => flagged },
|
|
693
835
|
{ 'key' => 'moderation-status', 'value' => decision.status },
|
|
694
836
|
{ 'key' => 'asset-type', 'value' => item['type'] }, asset_name_fact(item)])
|
|
695
837
|
# Like the event catalog: only allowing and limiting are low severity.
|
|
696
838
|
add_event('type' => "asset.#{decision.status}", 'severity' => %w[allowed limited].include?(decision.status) ? 'low' : 'high',
|
|
697
|
-
'assetId' => item['id'], 'asset' => snapshot(item),
|
|
698
|
-
'facts' => [{ 'key' => '
|
|
839
|
+
'assetId' => item['id'], 'asset' => snapshot(item), 'actor' => deep_copy(AUTOMATED_ACTOR),
|
|
840
|
+
'facts' => [{ 'key' => 'flagged-categories', 'value' => flagged },
|
|
699
841
|
{ 'key' => 'asset-type', 'value' => item['type'] }, asset_name_fact(item)])
|
|
700
842
|
end
|
|
701
843
|
|
|
702
844
|
# Moves an asset to an outcome like the pipeline would: a decision finishes processing, `purged`
|
|
703
845
|
# deletes the content (with `purge_reason`, `policy` for the pipeline), and finished processing
|
|
704
|
-
# removes content kept only temporarily. A
|
|
705
|
-
# placeholder, which
|
|
846
|
+
# removes content kept only temporarily. A decided asset gets its rendered file and preview
|
|
847
|
+
# (with a ThumbHash for its placeholder), which go with its content. `report` is a decision's
|
|
848
|
+
# categories, score and policy.
|
|
706
849
|
def apply_outcome(asset, outcome, purge_reason = 'policy', report = nil)
|
|
707
850
|
processing, moderation = AssetRecord.statuses(outcome)
|
|
708
851
|
now = @now.call
|
|
709
852
|
decided = processing == 'completed'
|
|
710
853
|
purge = moderation == 'purged' && asset['content']['status'] != 'removed'
|
|
711
854
|
remove = purge || (decided && asset['content']['status'] == 'temporary')
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
elsif moderation == 'purged'
|
|
719
|
-
image = without_placeholder(image)
|
|
720
|
-
end
|
|
721
|
-
asset['content']['image'] = image
|
|
855
|
+
# Like the API, an asset whose content is held has its rendered file and preview, completed by
|
|
856
|
+
# its decision, and removing content (when processing finishes or by purging) removes them
|
|
857
|
+
# with it, so no placeholder outlives the content. A safety hold withholds them.
|
|
858
|
+
content = asset['content']
|
|
859
|
+
held = content['status'] != 'removed' && !purge && !content['safetyHold']
|
|
860
|
+
content.merge!(held ? rendering(asset, decided) : NO_RENDERING)
|
|
722
861
|
if remove
|
|
723
862
|
forget_content(asset)
|
|
724
|
-
|
|
725
|
-
|
|
863
|
+
content.merge!('status' => 'removed', 'removalReason' => purge ? purge_reason : 'retention', 'updatedAt' => now,
|
|
864
|
+
'extractedText' => nil, 'metadata' => nil, **NO_RENDERING)
|
|
726
865
|
end
|
|
727
866
|
asset['upload'].merge!('status' => 'completed', 'updatedAt' => now)
|
|
728
867
|
update_classification(asset['classification'], processing, moderation, now)
|
|
@@ -744,8 +883,49 @@ module CalmLens
|
|
|
744
883
|
item['context'] = nil
|
|
745
884
|
end
|
|
746
885
|
|
|
747
|
-
#
|
|
748
|
-
def
|
|
886
|
+
# Whether browsers play an audio or video upload's format as it is, so it is served as stored.
|
|
887
|
+
def browser_safe_media?(format) = BROWSER_SAFE_MEDIA_FORMATS.include?(mime_type(format))
|
|
888
|
+
|
|
889
|
+
# A MIME type without parameters, in lower case.
|
|
890
|
+
def mime_type(format) = format.to_s.downcase.split(';', 2).first.to_s.strip
|
|
891
|
+
|
|
892
|
+
# The rendered file and preview the API reports for an asset with its content held (Asset.ts,
|
|
893
|
+
# getAssetRendered and getAssetPreview). Once uploaded, an image renders as itself with a preview
|
|
894
|
+
# that has no ThumbHash yet, and a PDF and browser-safe audio and video render as uploaded. Once
|
|
895
|
+
# processed, a website renders as its screenshot; a document or presentation as a PDF, with its
|
|
896
|
+
# first page as the preview; video and audio as a browser-safe file (MP4 or M4A unless browsers
|
|
897
|
+
# play the upload), audio with a waveform preview and video with no poster; and every preview has
|
|
898
|
+
# its ThumbHash. Text, spreadsheets and other files have neither. The mock decodes nothing, so
|
|
899
|
+
# sizes are unknown, audio has no peaks, and audio and video never get an adaptive stream.
|
|
900
|
+
def rendering(asset, processed) = { **rendered_file(asset, processed), 'hls' => nil }
|
|
901
|
+
|
|
902
|
+
# The rendered file and preview of {#rendering}.
|
|
903
|
+
def rendered_file(asset, processed)
|
|
904
|
+
format = asset['content']['format']
|
|
905
|
+
mime = mime_type(format)
|
|
906
|
+
safe = browser_safe_media?(format)
|
|
907
|
+
media = %w[video audio].include?(asset['type'])
|
|
908
|
+
return NO_RENDERING.dup unless processed || asset['type'] == 'image' || mime == 'application/pdf' || (media && safe)
|
|
909
|
+
|
|
910
|
+
unsized = { 'width' => nil, 'height' => nil }
|
|
911
|
+
preview = lambda do |size = unsized|
|
|
912
|
+
size.merge('thumbhash' => processed ? THUMBHASH : nil, 'averageColor' => processed ? '#7f7f7f' : nil)
|
|
913
|
+
end
|
|
914
|
+
case asset['type']
|
|
915
|
+
when 'image' then { 'rendered' => { 'kind' => 'image', 'format' => format || 'image/png', **unsized }, 'preview' => preview.call }
|
|
916
|
+
when 'website' then { 'rendered' => { 'kind' => 'image', 'format' => 'image/png', **unsized }, 'preview' => preview.call }
|
|
917
|
+
when 'multimedia-document'
|
|
918
|
+
return NO_RENDERING.dup if SPREADSHEET_MIME_TYPES.include?(mime)
|
|
919
|
+
|
|
920
|
+
{ 'rendered' => { 'kind' => 'pdf', 'format' => 'application/pdf', **unsized }, 'preview' => processed ? preview.call : nil }
|
|
921
|
+
when 'video' then { 'rendered' => { 'kind' => 'video', 'format' => safe ? format : 'video/mp4', **unsized }, 'preview' => nil }
|
|
922
|
+
when 'audio'
|
|
923
|
+
rendered = { 'kind' => 'audio', 'format' => safe ? format : 'audio/mp4', **unsized }
|
|
924
|
+
rendered['peaks'] = [] if processed
|
|
925
|
+
{ 'rendered' => rendered, 'preview' => processed ? preview.call(WAVEFORM_SIZE.dup) : nil }
|
|
926
|
+
else NO_RENDERING.dup
|
|
927
|
+
end
|
|
928
|
+
end
|
|
749
929
|
|
|
750
930
|
def asset(method, parts, query, json, body)
|
|
751
931
|
id = parts[1]
|
|
@@ -758,9 +938,10 @@ module CalmLens
|
|
|
758
938
|
in ['DELETE', nil]
|
|
759
939
|
@state.assets.delete(id)
|
|
760
940
|
@state.content.delete(id)
|
|
941
|
+
@processing_options.delete(id)
|
|
761
942
|
lifecycle_event('asset.deleted', item)
|
|
762
943
|
nil
|
|
763
|
-
in ['DELETE', 'content'] then
|
|
944
|
+
in ['DELETE', 'content'] then remove_content(item)
|
|
764
945
|
in ['GET', 'children'] then page(project_assets.select { |child| child['parentId'] == id }, query, id)
|
|
765
946
|
in ['POST', 'retry'] then retry_asset(item)
|
|
766
947
|
in ['POST', 'explain'] then explain(item, json['rerun'])
|
|
@@ -768,7 +949,6 @@ module CalmLens
|
|
|
768
949
|
in ['POST', 'move']
|
|
769
950
|
check_folder(json['folderId']) if json['folderId']
|
|
770
951
|
touch(item.merge!('folderId' => json['folderId']))
|
|
771
|
-
in ['POST', 'signed-url'] then signed_url(id, json)
|
|
772
952
|
in ['POST', 'part'] then part(item, query, body)
|
|
773
953
|
else route_not_found!(method, parts)
|
|
774
954
|
end
|
|
@@ -787,21 +967,21 @@ module CalmLens
|
|
|
787
967
|
touch(item.merge!(json.except('id', 'projectId', 'upload', 'classification', 'content')))
|
|
788
968
|
end
|
|
789
969
|
|
|
790
|
-
# Like the API: the record and report stay; stored content, extracted text
|
|
791
|
-
# go. Content already removed keeps its reason, and an `asset.content-removed` event is
|
|
792
|
-
# recorded either way.
|
|
793
|
-
def
|
|
970
|
+
# Like the API: the record and report stay; stored content, extracted text, metadata, the
|
|
971
|
+
# rendered file and the preview go. Content already removed keeps its reason, and an `asset.content-removed` event is
|
|
972
|
+
# recorded either way, with the `batch-id` fact of the batch that removed it, if any.
|
|
973
|
+
def remove_content(item, batch_id = nil)
|
|
794
974
|
forget_content(item)
|
|
795
975
|
content = item['content']
|
|
796
976
|
if content['status'] == 'removed'
|
|
797
|
-
content
|
|
977
|
+
content.merge!(NO_RENDERING)
|
|
798
978
|
else
|
|
799
979
|
now = @now.call
|
|
800
980
|
content.merge!('status' => 'removed', 'removalReason' => 'requested', 'updatedAt' => now, 'extractedText' => nil,
|
|
801
|
-
'
|
|
981
|
+
'metadata' => nil, **NO_RENDERING)
|
|
802
982
|
item['updatedAt'] = now
|
|
803
983
|
end
|
|
804
|
-
lifecycle_event('asset.content-removed', item)
|
|
984
|
+
lifecycle_event('asset.content-removed', item, batch_id: batch_id)
|
|
805
985
|
item
|
|
806
986
|
end
|
|
807
987
|
|
|
@@ -830,16 +1010,61 @@ module CalmLens
|
|
|
830
1010
|
fail!(ErrorCodes::ASSET_REVIEWED) unless moderation.fetch('manualReviews', []).empty?
|
|
831
1011
|
fail!(ErrorCodes::ASSET_CONTENT_UNAVAILABLE) if item['content']['status'] == 'removed'
|
|
832
1012
|
|
|
833
|
-
# Like the API:
|
|
834
|
-
|
|
835
|
-
|
|
1013
|
+
# Like the API: the request is recorded, and the asset returns as it was. It keeps its decision
|
|
1014
|
+
# while it's moderated again under the current policy.
|
|
1015
|
+
add_event('type' => 'asset.rerun-requested', 'severity' => 'low', 'assetId' => item['id'],
|
|
1016
|
+
'asset' => snapshot(item), 'actor' => deep_copy(API_KEY_ACTOR),
|
|
1017
|
+
'facts' => [asset_name_fact(item), { 'key' => 'asset-type', 'value' => item['type'] },
|
|
1018
|
+
{ 'key' => 'moderation-status', 'value' => moderation['status'] }])
|
|
1019
|
+
response = deep_copy(item)
|
|
1020
|
+
settle_rerun(item, 'rerun')
|
|
1021
|
+
response
|
|
836
1022
|
end
|
|
837
1023
|
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
1024
|
+
# Applies a re-run's decision to a decided asset like the pipeline: `pending` stays queued for
|
|
1025
|
+
# `complete_processing`, `failed` keeps the asset's decision, and any other replaces it with the
|
|
1026
|
+
# pipeline's events.
|
|
1027
|
+
def settle_rerun(item, trigger, decision = nil)
|
|
1028
|
+
decision ||= decision_for(item, trigger, true)
|
|
1029
|
+
@reruns.delete(item['id'])
|
|
1030
|
+
if decision.status == 'pending'
|
|
1031
|
+
@reruns[item['id']] = true
|
|
1032
|
+
elsif decision.status != 'failed'
|
|
1033
|
+
moderate_stored(item, trigger, decision)
|
|
1034
|
+
end
|
|
1035
|
+
end
|
|
1036
|
+
|
|
1037
|
+
# Signs an asset URL like the API (`POST /projects/:projectId/signed-asset-urls`): a grant to one
|
|
1038
|
+
# asset, by ID or external ID, until it expires, serving the original only when asked. Only the
|
|
1039
|
+
# mock's CDN accepts the grant.
|
|
1040
|
+
def sign_asset_url(json)
|
|
1041
|
+
# Like the API's schema, none of these fields takes null.
|
|
1042
|
+
%w[assetId externalId expiresIn original].each do |key|
|
|
1043
|
+
invalid!('validation/invalid-type', key) if json.key?(key) && json[key].nil?
|
|
1044
|
+
end
|
|
1045
|
+
if json.key?('assetId') == json.key?('externalId')
|
|
1046
|
+
message = 'Send either assetId or externalId'
|
|
1047
|
+
raise Failure.new(ErrorCodes::INVALID_REQUEST, message,
|
|
1048
|
+
details: { 'issues' => [{ 'path' => '', 'message' => message, 'code' => 'custom' }] }, status: 400)
|
|
1049
|
+
end
|
|
1050
|
+
expires = json.fetch('expiresIn', CDN::DEFAULT_SIGNED_URL_EXPIRES_IN)
|
|
1051
|
+
invalid!('validation/invalid-type', 'expiresIn') unless expires.is_a?(Integer) || (expires.is_a?(Float) && expires == expires.floor)
|
|
1052
|
+
invalid!('validation/too-small', 'expiresIn') if expires < 1
|
|
1053
|
+
invalid!('validation/too-big', 'expiresIn') if expires > CDN::MAX_SIGNED_URL_EXPIRES_IN
|
|
1054
|
+
original = json.fetch('original', false)
|
|
1055
|
+
invalid!('validation/invalid-type', 'original') unless [true, false].include?(original)
|
|
1056
|
+
|
|
1057
|
+
item = if json.key?('externalId')
|
|
1058
|
+
project_assets.find { |asset| asset['externalId'] == json['externalId'] }
|
|
1059
|
+
else
|
|
1060
|
+
own_asset(json['assetId'].to_s)
|
|
1061
|
+
end
|
|
1062
|
+
fail!(ErrorCodes::ASSET_NOT_FOUND) unless item
|
|
1063
|
+
fail!(ErrorCodes::ASSET_HELD) if item['content']['safetyHold']
|
|
1064
|
+
|
|
1065
|
+
expires_at = @now.call + (expires.to_i * 1000)
|
|
1066
|
+
{ 'url' => "#{@cdn_base_url}/assets/#{Internal.component(@project_id)}/#{MockGrant.encode(item['id'], original, expires_at)}",
|
|
1067
|
+
'expiresAt' => expires_at }
|
|
843
1068
|
end
|
|
844
1069
|
|
|
845
1070
|
# In the API's order: the upload must be open, parts arrive in order, image totals and the
|
|
@@ -857,7 +1082,7 @@ module CalmLens
|
|
|
857
1082
|
received = item['content']['sizeInBytes'].to_i + size
|
|
858
1083
|
upload = @state.uploads[item['id']] || {}
|
|
859
1084
|
code = image_size_error(item['type'], received, final) || total_size_error(upload['total'], received, final)
|
|
860
|
-
|
|
1085
|
+
size_failure!(code) if code
|
|
861
1086
|
check_part_size(size, final)
|
|
862
1087
|
|
|
863
1088
|
stored = (@state.content[item['id']] ||= { 'type' => item['content']['format'].to_s, 'parts' => {} })
|
|
@@ -870,7 +1095,8 @@ module CalmLens
|
|
|
870
1095
|
return item unless final
|
|
871
1096
|
|
|
872
1097
|
@state.uploads.delete(item['id'])
|
|
873
|
-
|
|
1098
|
+
# `wait=false` queues the final part; without `wait`, the upload's saved choice applies.
|
|
1099
|
+
queue = query.key?('wait') ? query['wait'] == false : upload.fetch('queue', false)
|
|
874
1100
|
apply_outcome(item, 'pending')
|
|
875
1101
|
upload_completed(item, queue)
|
|
876
1102
|
end
|
|
@@ -933,28 +1159,37 @@ module CalmLens
|
|
|
933
1159
|
end
|
|
934
1160
|
|
|
935
1161
|
# Policies: the project starts on the organisation's own General preset policy, following
|
|
936
|
-
# automatic updates. The mock
|
|
937
|
-
#
|
|
938
|
-
|
|
1162
|
+
# automatic updates (mode `live`). The mock's presets have one release, version 1, with
|
|
1163
|
+
# PRESET_SETTINGS. A stored policy is the API's shape without `projectIds`, with its preset's
|
|
1164
|
+
# `id`, `version` and `own`, whether it follows its preset (`followsPreset`) and its reset
|
|
1165
|
+
# `baseline`.
|
|
1166
|
+
def new_policy(name, settings, description: '', follows_preset: false, preset: nil)
|
|
939
1167
|
now = @now.call
|
|
940
1168
|
item = { 'id' => @create_id.call('pol'), 'name' => name, 'description' => description, 'settings' => settings,
|
|
941
|
-
'revision' => 1, '
|
|
1169
|
+
'revision' => 1, 'followsPreset' => follows_preset, 'preset' => preset,
|
|
942
1170
|
'baseline' => deep_copy(settings), 'createdAt' => now, 'updatedAt' => now }
|
|
943
1171
|
@policies[item['id']] = item
|
|
944
1172
|
end
|
|
945
1173
|
|
|
1174
|
+
# The project's policy, after creating the organisation's own General policy on first use.
|
|
946
1175
|
def ensure_policies
|
|
947
|
-
return if @project_policy_id
|
|
1176
|
+
return @project_policy_id if @project_policy_id
|
|
948
1177
|
|
|
949
1178
|
preset = { 'id' => 'general-community', 'version' => 1, 'own' => true }
|
|
950
|
-
@project_policy_id = new_policy(PRESET_NAMES['general-community'],
|
|
951
|
-
|
|
1179
|
+
@project_policy_id = new_policy(PRESET_NAMES['general-community'], deep_copy(PRESET_SETTINGS),
|
|
1180
|
+
follows_preset: true, preset: preset)['id']
|
|
952
1181
|
end
|
|
953
1182
|
|
|
954
|
-
# A stored policy as the API shows it
|
|
1183
|
+
# A stored policy as the API shows it: an own preset policy is `live` while it follows its
|
|
1184
|
+
# preset and `pinned` otherwise; any other preset policy is a `copy`.
|
|
955
1185
|
def show_policy(item)
|
|
956
|
-
shown = item.except('baseline')
|
|
957
|
-
|
|
1186
|
+
shown = item.except('baseline', 'followsPreset')
|
|
1187
|
+
if (preset = item['preset'])
|
|
1188
|
+
mode = if preset['own'] then item['followsPreset'] ? 'live' : 'pinned'
|
|
1189
|
+
else 'copy'
|
|
1190
|
+
end
|
|
1191
|
+
shown['preset'] = { 'id' => preset['id'], 'version' => preset['version'], 'latestVersion' => 1, 'mode' => mode }
|
|
1192
|
+
end
|
|
958
1193
|
shown.merge('projectIds' => item['id'] == @project_policy_id ? [@project_id] : [])
|
|
959
1194
|
end
|
|
960
1195
|
|
|
@@ -963,9 +1198,39 @@ module CalmLens
|
|
|
963
1198
|
@policies[id] || fail!(ErrorCodes::POLICY_NOT_FOUND)
|
|
964
1199
|
end
|
|
965
1200
|
|
|
966
|
-
|
|
1201
|
+
# `/policies` and `/policies/:id[/action]`, the API key's organisation's policies.
|
|
1202
|
+
def policies(method, parts, query, json)
|
|
1203
|
+
_, id, *rest = parts
|
|
1204
|
+
if id.nil?
|
|
1205
|
+
return policy_page(query) if method == 'GET'
|
|
1206
|
+
return create_policy(json) if method == 'POST'
|
|
1207
|
+
|
|
1208
|
+
route_not_found!(method, parts)
|
|
1209
|
+
end
|
|
1210
|
+
policy(method, id, rest, json)
|
|
1211
|
+
end
|
|
1212
|
+
|
|
1213
|
+
# A page of the organisation's policies, oldest first unless `sort` says otherwise.
|
|
1214
|
+
def policy_page(query)
|
|
967
1215
|
ensure_policies
|
|
968
|
-
|
|
1216
|
+
page(@policies.values.map { |item| show_policy(item) }, query, JSON.generate(['policies', @project_id]),
|
|
1217
|
+
default: { 'field' => 'createdAt', 'direction' => 'asc' })
|
|
1218
|
+
end
|
|
1219
|
+
|
|
1220
|
+
# Refuses settings the API refuses (MODERATION_POLICY_SETTINGS_SCHEMA): every category, each
|
|
1221
|
+
# with an `action` and its `allowances` and `restrictions` lists, and no other category.
|
|
1222
|
+
def check_policy_settings(settings)
|
|
1223
|
+
categories = settings.is_a?(Hash) ? settings['categorySettings'] : nil
|
|
1224
|
+
invalid!('validation/invalid-type', 'settings.categorySettings') unless categories.is_a?(Hash)
|
|
1225
|
+
MockClassification::CATEGORIES.each do |category|
|
|
1226
|
+
path = "settings.categorySettings.#{category}"
|
|
1227
|
+
value = categories[category]
|
|
1228
|
+
invalid!('validation/invalid-type', path) unless value.is_a?(Hash)
|
|
1229
|
+
%w[allowances restrictions].each { |key| invalid!('validation/invalid-type', "#{path}.#{key}") unless value[key].is_a?(Array) }
|
|
1230
|
+
invalid!('validation/invalid-value', "#{path}.action") unless value['action'].is_a?(String)
|
|
1231
|
+
end
|
|
1232
|
+
unknown = categories.keys.find { |key| !MockClassification::CATEGORIES.include?(key) }
|
|
1233
|
+
invalid!('validation/unknown-field', "settings.categorySettings.#{unknown}") if unknown
|
|
969
1234
|
end
|
|
970
1235
|
|
|
971
1236
|
def create_policy(json)
|
|
@@ -975,10 +1240,11 @@ module CalmLens
|
|
|
975
1240
|
if settings.nil? == preset_id.nil? || (preset_id && !PRESET_NAMES.key?(preset_id)) || (preset_id.nil? && !named)
|
|
976
1241
|
fail!(ErrorCodes::INVALID_VALUE, 'Send settings with a name, or presetId')
|
|
977
1242
|
end
|
|
1243
|
+
check_policy_settings(settings) unless preset_id
|
|
978
1244
|
description = json['description'].is_a?(String) ? json['description'] : ''
|
|
979
1245
|
item = if preset_id
|
|
980
1246
|
origin = { 'id' => preset_id, 'version' => 1, 'own' => false }
|
|
981
|
-
new_policy(named ? name.strip : PRESET_NAMES[preset_id],
|
|
1247
|
+
new_policy(named ? name.strip : PRESET_NAMES[preset_id], deep_copy(PRESET_SETTINGS),
|
|
982
1248
|
description: description, preset: origin)
|
|
983
1249
|
else
|
|
984
1250
|
new_policy(name.strip, settings, description: description)
|
|
@@ -1010,26 +1276,29 @@ module CalmLens
|
|
|
1010
1276
|
fail!(ErrorCodes::POLICY_IN_USE) if item['id'] == @project_policy_id
|
|
1011
1277
|
@policies.delete(id)
|
|
1012
1278
|
{}
|
|
1013
|
-
in ['POST', ['
|
|
1279
|
+
in ['POST', ['reset']] then restore_policy(item, 'reset', json)
|
|
1280
|
+
in ['POST', ['upgrade']] then restore_policy(item, 'upgrade', json)
|
|
1014
1281
|
in ['PUT', ['automatic-updates']] then set_policy_automatic_updates(item, json)
|
|
1015
1282
|
else route_not_found!(method, ['policies', id, *rest])
|
|
1016
1283
|
end
|
|
1017
1284
|
end
|
|
1018
1285
|
|
|
1019
1286
|
def update_policy(item, json)
|
|
1287
|
+
check_policy_settings(json['settings']) if json.key?('settings')
|
|
1020
1288
|
values = %w[name description].select { |key| json[key].is_a?(String) }
|
|
1021
1289
|
.to_h { |key| [key, key == 'name' ? json[key].strip : json[key]] }
|
|
1022
1290
|
values['settings'] = json['settings'] if json.key?('settings')
|
|
1023
|
-
# Saving a policy
|
|
1024
|
-
save_policy(item, json['revision'], values.merge('
|
|
1291
|
+
# Saving a `live` policy pins it.
|
|
1292
|
+
save_policy(item, json['revision'], values.merge('followsPreset' => false))
|
|
1025
1293
|
end
|
|
1026
1294
|
|
|
1027
|
-
|
|
1295
|
+
# Resets a `pinned` or `copy` policy to its preset release, or upgrades it to the latest.
|
|
1296
|
+
def restore_policy(item, operation, json)
|
|
1028
1297
|
preset = item['preset']
|
|
1029
|
-
fail!(ErrorCodes::POLICY_NO_PRESET) if preset.nil? || item['
|
|
1298
|
+
fail!(ErrorCodes::POLICY_NO_PRESET) if preset.nil? || item['followsPreset']
|
|
1030
1299
|
fail!(ErrorCodes::POLICY_PRESET_VERSION_CONFLICT) unless json['presetVersion'] == preset['version']
|
|
1031
|
-
# The mock's presets have one release, so an
|
|
1032
|
-
fail!(ErrorCodes::POLICY_PRESET_UPDATE_UNAVAILABLE) if
|
|
1300
|
+
# The mock's presets have one release, so an upgrade never has a newer one to apply.
|
|
1301
|
+
fail!(ErrorCodes::POLICY_PRESET_UPDATE_UNAVAILABLE) if operation == 'upgrade'
|
|
1033
1302
|
|
|
1034
1303
|
save_policy(item, json['revision'], 'settings' => deep_copy(item['baseline']))
|
|
1035
1304
|
end
|
|
@@ -1037,11 +1306,11 @@ module CalmLens
|
|
|
1037
1306
|
def set_policy_automatic_updates(item, json)
|
|
1038
1307
|
enabled = json['enabled'] == true
|
|
1039
1308
|
preset = item['preset']
|
|
1040
|
-
fail!(ErrorCodes::POLICY_AUTOMATIC_UPDATES_UNAVAILABLE) if preset.nil? || !preset['own'] || item['
|
|
1309
|
+
fail!(ErrorCodes::POLICY_AUTOMATIC_UPDATES_UNAVAILABLE) if preset.nil? || !preset['own'] || item['followsPreset'] == enabled
|
|
1041
1310
|
fail!(ErrorCodes::POLICY_REVISION_CONFLICT) unless json['revision'] == item['revision']
|
|
1042
1311
|
fail!(ErrorCodes::POLICY_PRESET_VERSION_CONFLICT) unless json['presetVersion'] == 1
|
|
1043
1312
|
|
|
1044
|
-
values = { '
|
|
1313
|
+
values = { 'followsPreset' => enabled }
|
|
1045
1314
|
values.merge!('name' => PRESET_NAMES[preset['id']], 'description' => '', 'settings' => deep_copy(item['baseline'])) if enabled
|
|
1046
1315
|
save_policy(item, json['revision'], values)
|
|
1047
1316
|
end
|
|
@@ -1103,10 +1372,10 @@ module CalmLens
|
|
|
1103
1372
|
if asset_id
|
|
1104
1373
|
asset = @state.assets[asset_id]
|
|
1105
1374
|
fail!(ErrorCodes::ASSET_NOT_FOUND) unless asset && asset['projectId'] == @project_id
|
|
1106
|
-
|
|
1107
|
-
fail!(ErrorCodes::HASH_LIST_ASSET_NOT_HASHED) unless
|
|
1108
|
-
hash =
|
|
1109
|
-
quality =
|
|
1375
|
+
content = asset['content']
|
|
1376
|
+
fail!(ErrorCodes::HASH_LIST_ASSET_NOT_HASHED) unless content['perceptualHash']
|
|
1377
|
+
hash = content['perceptualHash']
|
|
1378
|
+
quality = content['perceptualHashQuality']
|
|
1110
1379
|
end
|
|
1111
1380
|
fail!(ErrorCodes::INVALID_UPLOAD_PAYLOAD, 'Use a PDQ hash of 64 hexadecimal characters') unless hash.match?(/\A\h{64}\z/)
|
|
1112
1381
|
|
|
@@ -1115,7 +1384,7 @@ module CalmLens
|
|
|
1115
1384
|
|
|
1116
1385
|
entry = { 'id' => @create_id.call('hle'), 'projectId' => @project_id, 'hashListId' => list['id'], 'hash' => hash,
|
|
1117
1386
|
'quality' => quality, 'label' => json['label'].is_a?(String) ? json['label'] : nil,
|
|
1118
|
-
'
|
|
1387
|
+
'assetId' => asset_id, 'createdAt' => @now.call }
|
|
1119
1388
|
@state.hash_list_entries[entry['id']] = entry
|
|
1120
1389
|
respond(201, deep_copy(entry))
|
|
1121
1390
|
end
|
|
@@ -1129,17 +1398,43 @@ module CalmLens
|
|
|
1129
1398
|
{}
|
|
1130
1399
|
end
|
|
1131
1400
|
|
|
1132
|
-
#
|
|
1133
|
-
#
|
|
1134
|
-
#
|
|
1135
|
-
def
|
|
1401
|
+
# An asset's findings timeline, as the API lists it (publicGetAssetTimeline.ts): refused while a
|
|
1402
|
+
# safety hold is on. The mock analyses no audio or video, so it keeps no segments: every
|
|
1403
|
+
# timeline is empty, and no cursor continues one.
|
|
1404
|
+
def timeline(id, query)
|
|
1405
|
+
item = own_asset(id)
|
|
1406
|
+
fail!(ErrorCodes::ASSET_NOT_FOUND) unless item
|
|
1407
|
+
check_range(query.fetch('pageSize', DEFAULT_PAGE_SIZE), 'pageSize', MAX_PAGE_SIZE)
|
|
1408
|
+
fail!(ErrorCodes::ASSET_HELD, details: { 'hold' => 'safety' }) if item['content']['safetyHold']
|
|
1409
|
+
fail!(ErrorCodes::INVALID_CURSOR) if query.key?('cursor')
|
|
1410
|
+
|
|
1411
|
+
{ 'items' => [], 'nextCursor' => nil, 'hasMore' => false }
|
|
1412
|
+
end
|
|
1413
|
+
|
|
1414
|
+
# The page size and sort, refused like the API's query schema: `sort` is comma-separated
|
|
1415
|
+
# fields from `fields`, each descending when prefixed with `-`, none repeated and at most `max`
|
|
1416
|
+
# (`validation/invalid-value` at `sort` otherwise). Without one, `default` applies.
|
|
1417
|
+
def page_query(query, fields = %w[createdAt id], default = { 'field' => 'createdAt', 'direction' => 'desc' }, max: 2)
|
|
1136
1418
|
size = check_range(query.fetch('pageSize', DEFAULT_PAGE_SIZE), 'pageSize', MAX_PAGE_SIZE)
|
|
1137
|
-
|
|
1138
|
-
|
|
1139
|
-
|
|
1140
|
-
|
|
1141
|
-
|
|
1419
|
+
[size, query.key?('sort') ? parse_sort(query['sort'], fields, max) : [default.dup]]
|
|
1420
|
+
end
|
|
1421
|
+
|
|
1422
|
+
def parse_sort(value, fields, max)
|
|
1423
|
+
sort = value.to_s.split(',', -1).map(&:strip).map do |part|
|
|
1424
|
+
field = part.delete_prefix('-')
|
|
1425
|
+
invalid!('validation/invalid-value', 'sort') unless part.match?(/\A-?[A-Za-z][A-Za-z0-9.]*\z/) && fields.include?(field)
|
|
1426
|
+
{ 'field' => field, 'direction' => part.start_with?('-') ? 'desc' : 'asc' }
|
|
1142
1427
|
end
|
|
1428
|
+
fields_sent = sort.map { |rule| rule['field'] }
|
|
1429
|
+
invalid!('validation/invalid-value', 'sort') if sort.empty? || fields_sent.uniq.size != fields_sent.size || sort.size > max
|
|
1430
|
+
sort
|
|
1431
|
+
end
|
|
1432
|
+
|
|
1433
|
+
# One keyset page like the API's: sorted by `sort` (`createdAt` descending by default; an `id`
|
|
1434
|
+
# tie-breaker is added), starting after the cursor's values. The cursor is tied to `scope` (the
|
|
1435
|
+
# filter, folder or parent) and the sort, so it only continues the query it came from.
|
|
1436
|
+
def page(items, query, scope, fields: %w[createdAt id], max: 2, default: { 'field' => 'createdAt', 'direction' => 'desc' })
|
|
1437
|
+
size, ordering = page_query(query, fields, default, max: max)
|
|
1143
1438
|
ordering << { 'field' => 'id', 'direction' => ordering.first['direction'] } unless ordering.any? { |o| o['field'] == 'id' }
|
|
1144
1439
|
signature = JSON.generate([scope, ordering])
|
|
1145
1440
|
sorted = items.sort { |a, b| compare_items(a, b, ordering) }
|
|
@@ -1149,12 +1444,8 @@ module CalmLens
|
|
|
1149
1444
|
end
|
|
1150
1445
|
selected = sorted.first(size)
|
|
1151
1446
|
more = sorted.size > size
|
|
1152
|
-
cursor = if more
|
|
1153
|
-
|
|
1154
|
-
selected.last[o['field']]
|
|
1155
|
-
end })].pack('m0')
|
|
1156
|
-
end
|
|
1157
|
-
{ 'items' => selected, 'pageSize' => size, 'hasMore' => more, 'ordering' => ordering, 'nextCursor' => cursor }
|
|
1447
|
+
cursor = ([JSON.generate({ 'scope' => signature, 'values' => ordering.map { |o| selected.last[o['field']] } })].pack('m0') if more)
|
|
1448
|
+
{ 'items' => selected, 'hasMore' => more, 'nextCursor' => cursor }
|
|
1158
1449
|
end
|
|
1159
1450
|
|
|
1160
1451
|
def cursor_values(cursor, signature, ordering)
|
|
@@ -1177,15 +1468,6 @@ module CalmLens
|
|
|
1177
1468
|
false
|
|
1178
1469
|
end
|
|
1179
1470
|
|
|
1180
|
-
def ordering(query, default)
|
|
1181
|
-
entries = query.keys.filter_map { |key| key.match(/\Aordering\[(\d+)\]\[(field|direction)\]\z/) }
|
|
1182
|
-
return [default&.dup].compact if entries.empty?
|
|
1183
|
-
|
|
1184
|
-
entries.group_by { |match| Integer(match[1], 10) }.sort.map do |_, matches|
|
|
1185
|
-
matches.to_h { |match| [match[2], query[match[0]]] }
|
|
1186
|
-
end
|
|
1187
|
-
end
|
|
1188
|
-
|
|
1189
1471
|
def compare_items(left, right, ordering)
|
|
1190
1472
|
ordering.each do |rule|
|
|
1191
1473
|
result = compare_values(left[rule['field']], right[rule['field']])
|
|
@@ -1204,79 +1486,200 @@ module CalmLens
|
|
|
1204
1486
|
left.to_s <=> right.to_s
|
|
1205
1487
|
end
|
|
1206
1488
|
|
|
1207
|
-
# Answers like the CDN:
|
|
1208
|
-
#
|
|
1209
|
-
#
|
|
1210
|
-
#
|
|
1211
|
-
#
|
|
1212
|
-
|
|
1489
|
+
# Answers like the CDN (serveAssetResource.ts): `original` is the upload; `rendered` the image,
|
|
1490
|
+
# PDF, video or audio file `content.rendered` describes; `preview` the still `content.preview`
|
|
1491
|
+
# describes; `hls` the adaptive stream's files, which the mock never prepares; `embed` a small
|
|
1492
|
+
# page standing in for the embed page of any asset; and `placeholder` the preview's ThumbHash as a PNG, public
|
|
1493
|
+
# whatever the asset's visibility or moderation. A safety hold refuses every resource. Other resources pass delivery checks first,
|
|
1494
|
+
# which refuse everyone (content still uploading, blocked or purged, failed processing, removed
|
|
1495
|
+
# content), then an age-restricted asset needs `viewerClaims.ageConfirmed=true` whatever the key,
|
|
1496
|
+
# and a private asset needs an API key with `assets:download` for its folder. The mock converts
|
|
1497
|
+
# nothing: an upload stands in for its rendered file, an image for its own preview, and the
|
|
1498
|
+
# placeholder PNG for a screenshot, page or waveform.
|
|
1499
|
+
def serve(method, relative, headers)
|
|
1213
1500
|
path, query = relative.split('#', 2).first.split('?', 2)
|
|
1214
1501
|
raw_segments = path.split('/').reject(&:empty?)
|
|
1215
|
-
|
|
1216
|
-
|
|
1217
|
-
|
|
1218
|
-
|
|
1219
|
-
|
|
1220
|
-
|
|
1221
|
-
|
|
1502
|
+
root, project, _, resource = raw_segments.map { |part| URI.decode_www_form_component(part) }
|
|
1503
|
+
selected, options = selector(resource, raw_segments.drop(4))
|
|
1504
|
+
return invalid_transform if selected.nil?
|
|
1505
|
+
return asset_not_found unless root == 'assets' && project == @project_id && raw_segments[2]
|
|
1506
|
+
|
|
1507
|
+
grant, item = resolve_asset(raw_segments[2])
|
|
1508
|
+
return grant if grant.is_a?(Response)
|
|
1509
|
+
return asset_not_found unless item && RESOURCES.include?(resource)
|
|
1510
|
+
if resource == 'original' && grant && !grant[:original]
|
|
1511
|
+
return error(ErrorCodes::SIGNED_URL_ORIGINAL_NOT_GRANTED, "This signed asset URL doesn't grant the original file", status: 403)
|
|
1222
1512
|
end
|
|
1223
|
-
return invalid_transform if resource == 'original' && (lens || options)
|
|
1224
1513
|
|
|
1225
|
-
|
|
1226
|
-
|
|
1514
|
+
params = URI.decode_www_form(query.to_s).to_h
|
|
1515
|
+
# A grant authorises a private asset without the API key.
|
|
1516
|
+
authorised = !grant.nil? || can_download?(item['folderId'], headers)
|
|
1517
|
+
return info(method, item, authorised, params) if resource == 'info'
|
|
1518
|
+
return not_available if item['content']['safetyHold']
|
|
1227
1519
|
return placeholder(method, item) if resource == 'placeholder'
|
|
1228
1520
|
|
|
1229
|
-
deliver(method, item,
|
|
1521
|
+
deliver(method, item, resource, params, selected ? options || {} : nil, authorised)
|
|
1230
1522
|
end
|
|
1231
1523
|
|
|
1232
|
-
#
|
|
1233
|
-
#
|
|
1234
|
-
|
|
1524
|
+
# Whether the path after the resource selects a lens or transform, and its options; nil when the
|
|
1525
|
+
# CDN refuses it. Like the CDN, the adaptive stream's files sit beneath `/hls/` (its master
|
|
1526
|
+
# playlist is `/hls/index.m3u8`). Otherwise the selector after the resource is read
|
|
1527
|
+
# (AssetTransformParser.ts); the original, the embed page and the info take none.
|
|
1528
|
+
def selector(resource, path)
|
|
1529
|
+
lens, options = TransformRules.parse_selector(resource == 'hls' ? '' : path.join('/'))
|
|
1530
|
+
selected = !(lens || options).nil?
|
|
1531
|
+
%w[original embed info].include?(resource) && selected ? nil : [selected, options]
|
|
1532
|
+
rescue ArgumentError
|
|
1533
|
+
nil
|
|
1534
|
+
end
|
|
1535
|
+
|
|
1536
|
+
# Like the CDN (CdnUtils.ts), the asset segment is `ext:` and the URL-encoded external ID, a
|
|
1537
|
+
# grant (`sig_…`) or the asset ID. Returns `[grant, asset]`, or an error response for a grant
|
|
1538
|
+
# the mock didn't sign or that has expired.
|
|
1539
|
+
def resolve_asset(segment)
|
|
1540
|
+
if segment.start_with?('sig_')
|
|
1541
|
+
grant = MockGrant.decode(segment)
|
|
1542
|
+
return error(ErrorCodes::SIGNED_URL_INVALID, "The signed asset URL's grant is invalid", status: 403) unless grant
|
|
1543
|
+
return error(ErrorCodes::SIGNED_URL_EXPIRED, 'The signed URL has expired', status: 403) if grant[:expires_at] <= @now.call
|
|
1544
|
+
|
|
1545
|
+
[grant, project_assets.find { |asset| asset['id'] == grant[:asset_id] }]
|
|
1546
|
+
elsif segment.start_with?(CDN::EXTERNAL_ID_PREFIX)
|
|
1547
|
+
external = MockGrant.percent_decode(segment.delete_prefix(CDN::EXTERNAL_ID_PREFIX))
|
|
1548
|
+
[nil, external && !external.empty? ? project_assets.find { |asset| asset['externalId'] == external } : nil]
|
|
1549
|
+
else
|
|
1550
|
+
id = MockGrant.percent_decode(segment)
|
|
1551
|
+
[nil, project_assets.find { |asset| asset['id'] == id }]
|
|
1552
|
+
end
|
|
1553
|
+
end
|
|
1554
|
+
|
|
1555
|
+
def asset_not_found = error(ErrorCodes::ASSET_NOT_FOUND, 'Asset not found', status: 404)
|
|
1556
|
+
|
|
1557
|
+
# The display info at `/info`, like the CDN's (AssetInfo.ts): the allowlisted fields and whether
|
|
1558
|
+
# the viewer the claims describe can see the content, decided as the other resources decide it.
|
|
1559
|
+
# A private asset's info needs a grant or an API key with `assets:download`.
|
|
1560
|
+
def info(method, item, authorised, params)
|
|
1561
|
+
restricted = item['visibility'] != 'public'
|
|
1562
|
+
return error('asset/not-available', 'Asset is not available', status: 404) if restricted && !authorised
|
|
1563
|
+
|
|
1564
|
+
claims = JSON.parse(params.fetch('viewerClaims', '{}'))
|
|
1565
|
+
body = { 'id' => item['id'], 'projectId' => item['projectId'], 'type' => item['type'], 'externalUrl' => item['externalUrl'],
|
|
1566
|
+
'content' => item['content'].slice(*%w[format width height durationSeconds pageCount rendered preview hls]),
|
|
1567
|
+
'availability' => availability(item, claims.is_a?(Hash) && claims['ageConfirmed'] == true) }
|
|
1568
|
+
cdn_response(method, JSON.generate(body), { 'content-type' => 'application/json',
|
|
1569
|
+
'cache-control' => "#{restricted ? 'private' : 'public'}, max-age=60",
|
|
1570
|
+
'cache-tag' => "asset-#{item['id']}" })
|
|
1571
|
+
rescue JSON::ParserError
|
|
1572
|
+
error(ErrorCodes::INVALID_VIEWER_CLAIMS, 'Invalid viewer claims', status: 400)
|
|
1573
|
+
end
|
|
1574
|
+
|
|
1575
|
+
# Whether a viewer can see an asset's content now, decided as the other resources decide it.
|
|
1576
|
+
def availability(item, age_confirmed)
|
|
1235
1577
|
moderation = item['classification']['moderation']['status']
|
|
1236
|
-
|
|
1237
|
-
if stored.nil? || item['upload']['status'] == 'uploading' || %w[blocked purged].include?(moderation) ||
|
|
1578
|
+
if item['content']['safetyHold'] || item['upload']['status'] == 'uploading' || %w[blocked purged].include?(moderation) ||
|
|
1238
1579
|
item['classification']['status'] == 'failed' || item['content']['status'] == 'removed'
|
|
1239
|
-
|
|
1580
|
+
'unavailable'
|
|
1581
|
+
elsif moderation == 'age-restricted' && !age_confirmed
|
|
1582
|
+
'age-restricted'
|
|
1583
|
+
else
|
|
1584
|
+
'available'
|
|
1240
1585
|
end
|
|
1586
|
+
end
|
|
1241
1587
|
|
|
1242
|
-
|
|
1243
|
-
|
|
1244
|
-
|
|
1245
|
-
|
|
1588
|
+
# Serves a found asset's resource after the delivery, age and access checks, then whether it
|
|
1589
|
+
# has the resource and takes the selector, then the transform options' schema check.
|
|
1590
|
+
# `selector` is the selector's options (empty for a lens alone), or nil without one.
|
|
1591
|
+
def deliver(method, item, resource, params, selector, authorised)
|
|
1592
|
+
stored = @state.content[item['id']]
|
|
1593
|
+
refused = refuse_delivery(item, resource, stored, params, authorised)
|
|
1594
|
+
return refused if refused
|
|
1246
1595
|
|
|
1247
1596
|
restricted = item['visibility'] != 'public'
|
|
1248
|
-
|
|
1249
|
-
|
|
1250
|
-
|
|
1597
|
+
return embed(method, item, restricted, params) if resource == 'embed'
|
|
1598
|
+
|
|
1599
|
+
# Then whether the asset has the resource, and whether it is a still image that takes a lens
|
|
1600
|
+
# or transform: the preview always is, the rendered file only when it is an image.
|
|
1601
|
+
rendered, preview = item['content'].values_at('rendered', 'preview')
|
|
1602
|
+
return no_rendered_version if (resource == 'rendered' && !rendered) || (resource == 'preview' && !preview)
|
|
1603
|
+
# The mock prepares no stream, so it has no HLS files.
|
|
1604
|
+
return no_rendered_version if resource == 'hls'
|
|
1605
|
+
|
|
1606
|
+
still = resource == 'preview' || (resource == 'rendered' && rendered['kind'] == 'image')
|
|
1607
|
+
return invalid_transform if !still && selector
|
|
1608
|
+
|
|
1609
|
+
refused = selector ? TransformRules.check(selector).flatten : []
|
|
1610
|
+
return error(ErrorCodes::INVALID_TRANSFORM_OPTIONS, "Invalid transform options: #{refused.join(', ')}", status: 400) if refused.any?
|
|
1611
|
+
|
|
1612
|
+
bytes, type = resource_body(item, resource, stored)
|
|
1613
|
+
return not_available unless bytes
|
|
1614
|
+
|
|
1615
|
+
immutable = still || (resource == 'original' && item['type'] == 'image')
|
|
1616
|
+
cdn_response(method, bytes, viewer_delivery_headers(item, type, restricted, immutable, params))
|
|
1617
|
+
end
|
|
1618
|
+
|
|
1619
|
+
# The delivery checks, which refuse everyone, then age and access, as an error response, or nil.
|
|
1620
|
+
# Only the original needs the upload still stored.
|
|
1621
|
+
def refuse_delivery(item, resource, stored, params, authorised)
|
|
1622
|
+
moderation = item['classification']['moderation']['status']
|
|
1623
|
+
if (resource == 'original' && stored.nil?) || item['upload']['status'] == 'uploading' ||
|
|
1624
|
+
%w[blocked purged].include?(moderation) || item['classification']['status'] == 'failed' ||
|
|
1625
|
+
item['content']['status'] == 'removed'
|
|
1626
|
+
return not_available
|
|
1251
1627
|
end
|
|
1628
|
+
if moderation == 'age-restricted' && !viewer_age_confirmed?(params)
|
|
1629
|
+
return error('viewer/age-confirmation-required', 'Age confirmation required', status: 403)
|
|
1630
|
+
end
|
|
1631
|
+
return if item['visibility'] == 'public' || authorised
|
|
1632
|
+
|
|
1633
|
+
error(ErrorCodes::PERMISSION_DENIED, 'This API key needs assets:download permission for this asset’s folder.', status: 403)
|
|
1634
|
+
end
|
|
1252
1635
|
|
|
1253
|
-
|
|
1254
|
-
|
|
1636
|
+
# A resource's bytes and Content-Type: the upload, which stands in for its own rendered file
|
|
1637
|
+
# and an image's preview, or the placeholder PNG for a still the mock doesn't make. Nil bytes
|
|
1638
|
+
# when the upload is no longer stored.
|
|
1639
|
+
def resource_body(item, resource, stored)
|
|
1640
|
+
rendered = item['content']['rendered']
|
|
1641
|
+
serves_upload = resource == 'original' || (resource == 'rendered' && rendered['kind'] != 'image') || item['type'] == 'image'
|
|
1642
|
+
return [Placeholder.png(item.dig('content', 'preview', 'thumbhash') || THUMBHASH), 'image/png'] unless serves_upload
|
|
1643
|
+
return [nil, nil] unless stored
|
|
1255
1644
|
|
|
1256
|
-
type =
|
|
1257
|
-
|
|
1258
|
-
|
|
1645
|
+
type = if resource == 'rendered' then rendered['format']
|
|
1646
|
+
elsif stored['type'].to_s.empty? then 'application/octet-stream'
|
|
1647
|
+
else stored['type']
|
|
1648
|
+
end
|
|
1649
|
+
[stored['parts'].sort.map(&:last).join.b, type]
|
|
1259
1650
|
end
|
|
1260
1651
|
|
|
1261
1652
|
def viewer_age_confirmed?(params)
|
|
1262
1653
|
JSON.parse(params.fetch('viewerClaims', '{}'))['ageConfirmed'] == true
|
|
1263
1654
|
end
|
|
1264
1655
|
|
|
1265
|
-
def viewer_delivery_headers(item, type, restricted, params)
|
|
1266
|
-
headers = delivery_headers(item, type, restricted)
|
|
1656
|
+
def viewer_delivery_headers(item, type, restricted, immutable, params)
|
|
1657
|
+
headers = delivery_headers(item, type, restricted, immutable)
|
|
1267
1658
|
headers['Cache-Control'] = 'private, no-store' if params.key?('viewerClaims')
|
|
1268
1659
|
headers
|
|
1269
1660
|
end
|
|
1270
1661
|
|
|
1271
|
-
|
|
1662
|
+
# The embed page, which shows any asset that may be delivered, as its viewer for the asset's
|
|
1663
|
+
# type or a notice that the type has none; cached like other non-image content.
|
|
1664
|
+
def embed(method, item, restricted, params)
|
|
1665
|
+
cache = if params.key?('viewerClaims') then 'private, no-store'
|
|
1666
|
+
else "#{restricted ? 'private' : 'public'}, max-age=3600"
|
|
1667
|
+
end
|
|
1668
|
+
cdn_response(method, EMBED_HTML, { 'content-type' => 'text/html; charset=utf-8', 'cache-control' => cache,
|
|
1669
|
+
'cache-tag' => "asset-#{item['id']}", 'x-content-type-options' => 'nosniff' })
|
|
1670
|
+
end
|
|
1671
|
+
|
|
1672
|
+
def invalid_transform = error(ErrorCodes::INVALID_TRANSFORM, 'Invalid transform selector', status: 400)
|
|
1673
|
+
|
|
1674
|
+
def no_rendered_version = error('asset/no-rendered-version', 'This asset has no such resource', status: 404)
|
|
1272
1675
|
|
|
1273
1676
|
def not_available = error('asset/not-available', 'Asset is not available', status: 403)
|
|
1274
1677
|
|
|
1275
|
-
# Like the CDN's placeholder handler:
|
|
1678
|
+
# Like the CDN's placeholder handler: the preview's ThumbHash, public whatever the asset's
|
|
1276
1679
|
# visibility or moderation; refused for purged assets and assets without one (not yet
|
|
1277
|
-
#
|
|
1680
|
+
# processed, without a preview, or content removed).
|
|
1278
1681
|
def placeholder(method, item)
|
|
1279
|
-
thumbhash = item.dig('content', '
|
|
1682
|
+
thumbhash = item.dig('content', 'preview', 'thumbhash')
|
|
1280
1683
|
return not_available if thumbhash.nil? || item['classification']['moderation']['status'] == 'purged'
|
|
1281
1684
|
|
|
1282
1685
|
cdn_response(method, Placeholder.png(thumbhash),
|
|
@@ -1284,14 +1687,13 @@ module CalmLens
|
|
|
1284
1687
|
'cache-tag' => "asset-#{item['id']}", 'etag' => "\"#{thumbhash}\"")
|
|
1285
1688
|
end
|
|
1286
1689
|
|
|
1287
|
-
# Like the CDN:
|
|
1288
|
-
# by shared caches only when delivery is public. Active content is served as a download.
|
|
1289
|
-
def delivery_headers(item, type, restricted)
|
|
1290
|
-
|
|
1291
|
-
headers = { 'content-type' => type,
|
|
1292
|
-
'cache-control' => "#{restricted ? 'private' : 'public'}, #{image ? 'max-age=31536000, immutable' : 'max-age=3600'}",
|
|
1690
|
+
# Like the CDN: a still image's bytes are immutable for a year, other content is kept for an
|
|
1691
|
+
# hour, by shared caches only when delivery is public. Active content is served as a download.
|
|
1692
|
+
def delivery_headers(item, type, restricted, immutable)
|
|
1693
|
+
lifetime = immutable ? 'max-age=31536000, immutable' : 'max-age=3600'
|
|
1694
|
+
headers = { 'content-type' => type, 'cache-control' => "#{restricted ? 'private' : 'public'}, #{lifetime}",
|
|
1293
1695
|
'cache-tag' => "asset-#{item['id']}" }
|
|
1294
|
-
headers.merge!('etag' => "\"#{item['id']}\"", 'expires' => Time.at((@now.call / 1000) + 31_536_000).utc.httpdate) if
|
|
1696
|
+
headers.merge!('etag' => "\"#{item['id']}\"", 'expires' => Time.at((@now.call / 1000) + 31_536_000).utc.httpdate) if immutable
|
|
1295
1697
|
headers.merge!('x-content-type-options' => 'nosniff', 'content-security-policy' => "sandbox; default-src 'none'")
|
|
1296
1698
|
mime = type.downcase.split(';', 2).first.to_s.strip
|
|
1297
1699
|
headers['content-disposition'] = 'attachment' if ACTIVE_TYPES.include?(mime) || mime.end_with?('+xml')
|
|
@@ -1303,8 +1705,11 @@ module CalmLens
|
|
|
1303
1705
|
headers: headers.merge('content-length' => bytes.bytesize.to_s, 'x-request-id' => request_id))
|
|
1304
1706
|
end
|
|
1305
1707
|
|
|
1306
|
-
# Whether the mock API key
|
|
1307
|
-
|
|
1708
|
+
# Whether the request carries the mock API key and its grants give `assets:download` for
|
|
1709
|
+
# `folder_id`.
|
|
1710
|
+
def can_download?(folder_id, headers)
|
|
1711
|
+
return false unless header(headers, 'X-Api-Key') == @api_key
|
|
1712
|
+
|
|
1308
1713
|
@grants.any? do |grant|
|
|
1309
1714
|
next false unless Array(grant['actions']).include?('assets:download')
|
|
1310
1715
|
next true if grant['folderId'].nil?
|
|
@@ -1327,6 +1732,35 @@ module CalmLens
|
|
|
1327
1732
|
end
|
|
1328
1733
|
end
|
|
1329
1734
|
|
|
1735
|
+
# The mock's grant, the asset segment of a URL {CDN#sign_asset_url} signs on the mock: `sig_` and
|
|
1736
|
+
# the lowercase hex of the UTF-8 text `{assetId}.{1|0}.{expiresAt}`, its asset, original access and
|
|
1737
|
+
# expiry. Every SDK's mock writes it the same way. It is readable, not signed, as the mock has no
|
|
1738
|
+
# secret to keep.
|
|
1739
|
+
# @api private
|
|
1740
|
+
module MockGrant
|
|
1741
|
+
module_function
|
|
1742
|
+
|
|
1743
|
+
def encode(asset_id, original, expires_at)
|
|
1744
|
+
"sig_#{"#{asset_id}.#{original ? 1 : 0}.#{expires_at}".unpack1('H*')}"
|
|
1745
|
+
end
|
|
1746
|
+
|
|
1747
|
+
# `{asset_id:, original:, expires_at:}`, or nil for a segment the mock didn't sign.
|
|
1748
|
+
def decode(segment)
|
|
1749
|
+
hex = segment.delete_prefix('sig_')
|
|
1750
|
+
return unless hex.match?(/\A(?:[0-9a-f]{2})+\z/)
|
|
1751
|
+
|
|
1752
|
+
text = [hex].pack('H*').force_encoding(Encoding::UTF_8)
|
|
1753
|
+
match = text.valid_encoding? && /\A(.+)\.([01])\.(\d+)\z/m.match(text)
|
|
1754
|
+
match && { asset_id: match[1], original: match[2] == '1', expires_at: Integer(match[3], 10) }
|
|
1755
|
+
end
|
|
1756
|
+
|
|
1757
|
+
# Percent-decodes a path segment like decodeURIComponent; nil when it isn't valid UTF-8.
|
|
1758
|
+
def percent_decode(text)
|
|
1759
|
+
decoded = text.b.gsub(/%(\h\h)/) { Regexp.last_match(1).hex.chr }.force_encoding(Encoding::UTF_8)
|
|
1760
|
+
decoded if decoded.valid_encoding?
|
|
1761
|
+
end
|
|
1762
|
+
end
|
|
1763
|
+
|
|
1330
1764
|
# The processing and moderation status of each outcome.
|
|
1331
1765
|
# @api private
|
|
1332
1766
|
module AssetRecord
|
|
@@ -1345,125 +1779,5 @@ module CalmLens
|
|
|
1345
1779
|
end
|
|
1346
1780
|
end
|
|
1347
1781
|
end
|
|
1348
|
-
|
|
1349
|
-
# The RSQL subset the fake API understands: `==` and `!=` (with `*` wildcards), `=in=` and
|
|
1350
|
-
# `=out=` with a `(value,value)` list, and `=gt=`, `=ge=`, `=lt=`, `=le=`, with quoted values,
|
|
1351
|
-
# combined with `;` (and), which binds tighter than `,` (or), and grouped with `( … )`.
|
|
1352
|
-
# Anything else raises instead of being ignored, so a test never passes on an unfiltered list.
|
|
1353
|
-
# @api private
|
|
1354
|
-
module RSQL
|
|
1355
|
-
module_function
|
|
1356
|
-
|
|
1357
|
-
def filter(items, expression)
|
|
1358
|
-
return items if expression.nil? || expression.to_s.strip.empty?
|
|
1359
|
-
|
|
1360
|
-
test = parse(expression.to_s)
|
|
1361
|
-
items.select { |item| test.call(item) }
|
|
1362
|
-
end
|
|
1363
|
-
|
|
1364
|
-
# Compiles `,` (or) of `;` (and) terms, where a term is a comparison or a `( … )` group.
|
|
1365
|
-
def parse(expression)
|
|
1366
|
-
alternatives = split(expression, ',').map do |alternative|
|
|
1367
|
-
split(alternative, ';').map do |term|
|
|
1368
|
-
text = term.strip
|
|
1369
|
-
raise unsupported(expression) if text.empty?
|
|
1370
|
-
|
|
1371
|
-
group = unwrap(text)
|
|
1372
|
-
group ? parse(group) : comparison(text)
|
|
1373
|
-
end
|
|
1374
|
-
end
|
|
1375
|
-
->(item) { alternatives.any? { |terms| terms.all? { |test| test.call(item) } } }
|
|
1376
|
-
end
|
|
1377
|
-
|
|
1378
|
-
# The expression inside `( … )` when the parentheses enclose all of `text`, else nil.
|
|
1379
|
-
def unwrap(text)
|
|
1380
|
-
return unless text.start_with?('(') && text.end_with?(')')
|
|
1381
|
-
|
|
1382
|
-
depth = 0
|
|
1383
|
-
quote = nil
|
|
1384
|
-
text.each_char.with_index do |char, index|
|
|
1385
|
-
if quote
|
|
1386
|
-
quote = nil if char == quote
|
|
1387
|
-
elsif ['"', "'"].include?(char)
|
|
1388
|
-
quote = char
|
|
1389
|
-
elsif char == '('
|
|
1390
|
-
depth += 1
|
|
1391
|
-
elsif char == ')'
|
|
1392
|
-
depth -= 1
|
|
1393
|
-
return index == text.length - 1 ? text[1..-2] : nil if depth.zero?
|
|
1394
|
-
end
|
|
1395
|
-
end
|
|
1396
|
-
nil
|
|
1397
|
-
end
|
|
1398
|
-
|
|
1399
|
-
def unsupported(term)
|
|
1400
|
-
InvalidArgumentError.new(SdkErrorCodes::INVALID_ARGUMENT,
|
|
1401
|
-
'MockServerClient filters support ==, !=, =in=, =out=, =gt=, =ge=, =lt= and ' \
|
|
1402
|
-
"=le=, joined by ; and , and grouped with parentheses (got #{term.inspect})")
|
|
1403
|
-
end
|
|
1404
|
-
|
|
1405
|
-
# Splits on `separator` outside quotes and parentheses (groups and `=in=(...)` lists).
|
|
1406
|
-
def split(text, separator)
|
|
1407
|
-
parts = ['']
|
|
1408
|
-
quote = nil
|
|
1409
|
-
depth = 0
|
|
1410
|
-
text.each_char do |char|
|
|
1411
|
-
if quote
|
|
1412
|
-
quote = nil if char == quote
|
|
1413
|
-
elsif ['"', "'"].include?(char)
|
|
1414
|
-
quote = char
|
|
1415
|
-
elsif char == '('
|
|
1416
|
-
depth += 1
|
|
1417
|
-
elsif char == ')'
|
|
1418
|
-
depth -= 1
|
|
1419
|
-
elsif char == separator && depth.zero?
|
|
1420
|
-
parts << ''
|
|
1421
|
-
next
|
|
1422
|
-
end
|
|
1423
|
-
parts[-1] += char
|
|
1424
|
-
end
|
|
1425
|
-
parts
|
|
1426
|
-
end
|
|
1427
|
-
|
|
1428
|
-
def comparison(term)
|
|
1429
|
-
match = term.strip.match(/\A([A-Za-z][\w.]*)(==|!=|=in=|=out=|=gt=|=ge=|=lt=|=le=)(.+)\z/m)
|
|
1430
|
-
raise unsupported(term) unless match
|
|
1431
|
-
|
|
1432
|
-
field, operator, raw = match.captures
|
|
1433
|
-
values = list(operator, raw.strip)
|
|
1434
|
-
lambda do |item|
|
|
1435
|
-
actual = field.split('.').reduce(item) { |object, key| object.is_a?(Hash) ? object[key] : nil }
|
|
1436
|
-
entries = Array(actual.nil? ? 'null' : actual).map(&:to_s)
|
|
1437
|
-
case operator
|
|
1438
|
-
when '==', '=in=' then values.any? { |value| matches?(entries, value) }
|
|
1439
|
-
when '!=', '=out=' then values.none? { |value| matches?(entries, value) }
|
|
1440
|
-
else entries.any? { |entry| ordered?(entry, operator, values.first) }
|
|
1441
|
-
end
|
|
1442
|
-
end
|
|
1443
|
-
end
|
|
1444
|
-
|
|
1445
|
-
def list(operator, raw)
|
|
1446
|
-
return [unquote(raw)] unless %w[=in= =out=].include?(operator)
|
|
1447
|
-
|
|
1448
|
-
unless raw.start_with?('(') && raw.end_with?(')')
|
|
1449
|
-
raise InvalidArgumentError.new(SdkErrorCodes::INVALID_ARGUMENT, "#{operator} takes a (value,value) list")
|
|
1450
|
-
end
|
|
1451
|
-
|
|
1452
|
-
split(raw[1..-2], ',').map { |value| unquote(value.strip) }
|
|
1453
|
-
end
|
|
1454
|
-
|
|
1455
|
-
def unquote(value) = value.match?(/\A(["']).*\1\z/m) ? value[1..-2] : value
|
|
1456
|
-
|
|
1457
|
-
def matches?(entries, value)
|
|
1458
|
-
pattern = Regexp.new("\\A#{Regexp.escape(value).gsub('\*', '.*')}\\z", Regexp::MULTILINE)
|
|
1459
|
-
entries.any? { |entry| pattern.match?(entry) }
|
|
1460
|
-
end
|
|
1461
|
-
|
|
1462
|
-
# Numbers compare as numbers and everything else (such as ISO dates) as text.
|
|
1463
|
-
def ordered?(entry, operator, value)
|
|
1464
|
-
left, right = [entry, value].all? { |text| text.match?(/\A-?\d+(\.\d+)?\z/) } ? [entry.to_f, value.to_f] : [entry, value]
|
|
1465
|
-
{ '=gt=' => left > right, '=ge=' => left >= right, '=lt=' => left < right, '=le=' => left <= right }.fetch(operator)
|
|
1466
|
-
end
|
|
1467
|
-
end
|
|
1468
1782
|
end
|
|
1469
1783
|
end
|