calmlens 0.20.0 → 0.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +4 -1
- data/lib/calmlens/assets.rb +149 -110
- data/lib/calmlens/cdn.rb +197 -41
- data/lib/calmlens/error_codes.rb +100 -84
- data/lib/calmlens/errors.rb +11 -2
- data/lib/calmlens/events.rb +50 -1
- data/lib/calmlens/folders.rb +14 -9
- data/lib/calmlens/hash_lists.rb +29 -9
- data/lib/calmlens/internal.rb +9 -19
- data/lib/calmlens/lenses.rb +79 -6
- data/lib/calmlens/option_keys.rb +56 -26
- data/lib/calmlens/policies.rb +50 -18
- data/lib/calmlens/project_configuration.rb +113 -0
- data/lib/calmlens/resource.rb +24 -6
- data/lib/calmlens/server_client.rb +17 -6
- data/lib/calmlens/session.rb +29 -12
- data/lib/calmlens/testing/asset_filters.rb +39 -0
- data/lib/calmlens/testing/classification.rb +8 -2
- data/lib/calmlens/testing/fake_api.rb +950 -397
- data/lib/calmlens/testing/mock_decision.rb +6 -4
- data/lib/calmlens/testing/rsql.rb +245 -0
- data/lib/calmlens/testing/test_input_detections.rb +61 -0
- data/lib/calmlens/testing/test_input_scenario.rb +6 -2
- data/lib/calmlens/testing/test_inputs.rb +30 -10
- data/lib/calmlens/testing.rb +56 -15
- data/lib/calmlens/transform_rules.rb +1 -1
- data/lib/calmlens/upload_body.rb +85 -2
- data/lib/calmlens/upload_mime_types.rb +9 -1
- data/lib/calmlens/upload_tokens.rb +3 -3
- data/lib/calmlens/version.rb +1 -1
- data/lib/calmlens/webhooks.rb +58 -13
- data/lib/calmlens.rb +1 -0
- data/sig/calmlens.rbs +112 -13
- data/sig/error_codes.rbs +28 -20
- data/sig/qualifiers.rbs +2 -0
- data/sig/types.rbs +488 -190
- metadata +6 -2
|
@@ -6,6 +6,7 @@ require 'securerandom'
|
|
|
6
6
|
require 'uri'
|
|
7
7
|
require_relative 'placeholder'
|
|
8
8
|
require_relative 'mock_decision'
|
|
9
|
+
require_relative 'rsql'
|
|
9
10
|
|
|
10
11
|
module CalmLens
|
|
11
12
|
module Testing
|
|
@@ -36,35 +37,68 @@ module CalmLens
|
|
|
36
37
|
:hash_list_entries, keyword_init: true)
|
|
37
38
|
|
|
38
39
|
# A stateful stand-in for the CalmLens API and CDN. It stores assets and folders, applies
|
|
39
|
-
# filters and
|
|
40
|
+
# filters and sorting, serves stored bytes, and answers errors with the API's codes, statuses,
|
|
40
41
|
# details and body shape, checking requests in the API's order. It moderates through the
|
|
41
42
|
# `moderate` callback rather than classifying content, and does not run transforms or index for
|
|
42
43
|
# semantic search.
|
|
43
44
|
# @api private
|
|
44
45
|
class FakeAPI
|
|
45
|
-
JSON_QUERY = %w[
|
|
46
|
-
context].freeze
|
|
46
|
+
JSON_QUERY = %w[wait isFinal sizeInBytes totalSizeInBytes partNumber pageSize].freeze
|
|
47
47
|
# Keyword search fields, in the order matches are reported, and where each value sits in the asset.
|
|
48
48
|
SEARCH_FIELDS = { 'name' => %w[name], 'id' => %w[id], 'externalId' => %w[externalId],
|
|
49
49
|
'description' => %w[description], 'tags' => %w[tags], 'type' => %w[type],
|
|
50
50
|
'content.format' => %w[content format], 'content.extension' => %w[content extension],
|
|
51
51
|
'classification.moderation.status' => %w[classification moderation status] }.freeze
|
|
52
52
|
MAX_SEARCH_TOKENS = 8
|
|
53
|
-
|
|
53
|
+
MAX_SEARCH_PAGE_SIZE = 20
|
|
54
54
|
MAX_SEARCH_QUERY_LENGTH = 200
|
|
55
55
|
# Every API cursor list returns this many items unless `pageSize` says otherwise.
|
|
56
56
|
DEFAULT_PAGE_SIZE = 50
|
|
57
57
|
MAX_PAGE_SIZE = 100
|
|
58
58
|
REVIEW_DECISIONS = %w[allowed limited age-restricted blocked purged].freeze
|
|
59
|
+
# The moderation status each `asset.<status>` event announces.
|
|
60
|
+
ANNOUNCED_STATUSES = REVIEW_DECISIONS.to_h { |status| ["asset.#{status}", status] }.freeze
|
|
59
61
|
MAX_DECISION_NOTE_LENGTH = 1499
|
|
60
|
-
|
|
62
|
+
# The actor of events the mock's API key causes: mock keys have no ID.
|
|
63
|
+
API_KEY_ACTOR = { 'type' => 'api-key', 'id' => nil }.freeze
|
|
64
|
+
# The actor of a reviewer's decision through {MockServerClient#decide}.
|
|
65
|
+
REVIEWER_ACTOR = { 'type' => 'user', 'id' => nil }.freeze
|
|
66
|
+
# The actor of the pipeline's processing events.
|
|
67
|
+
AUTOMATED_ACTOR = { 'type' => 'automated', 'id' => nil }.freeze
|
|
68
|
+
# An event type's shape (`resource.action`), as the mock checks `types` without the catalog.
|
|
69
|
+
EVENT_TYPE_PATTERN = /\A[a-z][a-z-]*\.[a-z][a-z-]*\z/
|
|
70
|
+
# The resources an asset's CDN path names.
|
|
71
|
+
RESOURCES = %w[original rendered preview hls embed placeholder info].freeze
|
|
72
|
+
# The page the mock serves at `/embed`, standing in for the CDN's embed page.
|
|
73
|
+
EMBED_HTML = '<!doctype html><html lang="en"><head><meta charset="utf-8"><title>Embed</title></head>' \
|
|
74
|
+
'<body></body></html>'
|
|
75
|
+
|
|
61
76
|
MIN_PART = 5 * 1024 * 1024
|
|
62
77
|
MAX_MULTIPART_IMAGE = 20 * 1024 * 1024
|
|
63
78
|
# Like the API: an upload has at most this many parts.
|
|
64
79
|
MAX_PARTS = 10_000
|
|
65
|
-
# The ThumbHash the mock gives every
|
|
66
|
-
# pixels. The `placeholder` resource serves it as a PNG, like the CDN.
|
|
80
|
+
# The ThumbHash the mock gives every processed asset's preview, standing in for one computed
|
|
81
|
+
# from its pixels. The `placeholder` resource serves it as a PNG, like the CDN.
|
|
67
82
|
THUMBHASH = '1QcSHQRnh493V4dIh4eXh1h4kJUI'
|
|
83
|
+
# Removing or withholding content removes its rendered file, adaptive stream and preview, as the API does.
|
|
84
|
+
NO_RENDERING = { 'rendered' => nil, 'hls' => nil, 'hlsStatus' => nil, 'preview' => nil }.freeze
|
|
85
|
+
# The size of the waveform the API makes as an audio asset's preview.
|
|
86
|
+
WAVEFORM_SIZE = { 'width' => 1200, 'height' => 300 }.freeze
|
|
87
|
+
# Audio and video formats browsers play as uploaded, so the API serves them as stored (Asset.ts).
|
|
88
|
+
BROWSER_SAFE_MEDIA_FORMATS = %w[video/mp4 audio/mpeg audio/aac audio/mp4 audio/x-m4a audio/ogg].freeze
|
|
89
|
+
# Spreadsheets, which have no rendered file or preview (Asset.ts).
|
|
90
|
+
SPREADSHEET_MIME_TYPES = %w[application/vnd.openxmlformats-officedocument.spreadsheetml.sheet
|
|
91
|
+
application/vnd.ms-excel.sheet.macroenabled.12 application/vnd.ms-excel
|
|
92
|
+
application/vnd.oasis.opendocument.spreadsheet].freeze
|
|
93
|
+
# The settings of the mock's presets: every category with its default action (`allow` for
|
|
94
|
+
# personal information and profanity, `block` otherwise) and no rules. The mock holds no
|
|
95
|
+
# built-in preset settings, and uploads are never decided by a policy.
|
|
96
|
+
PRESET_SETTINGS = {
|
|
97
|
+
'categorySettings' => MockClassification::CATEGORIES.to_h do |category|
|
|
98
|
+
[category, { 'action' => %w[personal-information profanity].include?(category) ? 'allow' : 'block',
|
|
99
|
+
'allowances' => [], 'restrictions' => [] }]
|
|
100
|
+
end
|
|
101
|
+
}.freeze
|
|
68
102
|
# The built-in presets the mock knows, by display name.
|
|
69
103
|
PRESET_NAMES = { 'general-community' => 'General', 'children-platform' => 'Kids',
|
|
70
104
|
'education-support' => 'Learning and support', 'wholesome-community' => 'Teen-safe' }.freeze
|
|
@@ -77,8 +111,38 @@ module CalmLens
|
|
|
77
111
|
# The mock API key's `{ 'actions', 'folderId', 'includeSubfolders' }` grants.
|
|
78
112
|
attr_accessor :grants
|
|
79
113
|
|
|
80
|
-
|
|
114
|
+
# Splits an upload form as the API reads it: exactly an `options` part, then a `file` part, then
|
|
115
|
+
# the closing boundary.
|
|
116
|
+
# @return [Array(String, String), nil] the options part's text and the file's bytes, or nil
|
|
117
|
+
# when the body isn't such a form.
|
|
118
|
+
def self.upload_form(body, content_type)
|
|
119
|
+
boundary = content_type.to_s.match(%r{\Amultipart/form-data\s*;\s*boundary=(?:"([^"]+)"|([^;\s]+))}i)
|
|
120
|
+
return nil unless boundary
|
|
121
|
+
|
|
122
|
+
framed = "\r\n".b + body.to_s.b
|
|
123
|
+
delimiter = "\r\n--#{boundary[1] || boundary[2]}".b
|
|
124
|
+
return nil unless framed.start_with?(delimiter)
|
|
125
|
+
|
|
126
|
+
parts = []
|
|
127
|
+
at = 0
|
|
128
|
+
until framed.byteslice(at + delimiter.bytesize, 2) == '--'
|
|
129
|
+
start = at + delimiter.bytesize
|
|
130
|
+
header_end = framed.index("\r\n\r\n".b, start)
|
|
131
|
+
following = header_end && framed.index(delimiter, header_end + 4)
|
|
132
|
+
return nil unless following
|
|
133
|
+
|
|
134
|
+
name = framed.byteslice(start, header_end - start)[/content-disposition:\s*form-data\s*;[^\r\n]*?\bname="([^"]*)"/i, 1]
|
|
135
|
+
parts << [name, framed.byteslice(header_end + 4, following - header_end - 4)]
|
|
136
|
+
at = following
|
|
137
|
+
end
|
|
138
|
+
return nil unless parts.map(&:first) == %w[options file]
|
|
139
|
+
|
|
140
|
+
[parts[0][1].force_encoding(Encoding::UTF_8), parts[1][1]]
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
def initialize(project_id:, base_url:, cdn_base_url:, now:, create_id:, moderate:, api_key:, grants:)
|
|
81
144
|
@project_id = project_id
|
|
145
|
+
@api_key = api_key
|
|
82
146
|
@base_url = base_url
|
|
83
147
|
@cdn_base_url = cdn_base_url
|
|
84
148
|
@now = now
|
|
@@ -91,23 +155,48 @@ module CalmLens
|
|
|
91
155
|
# reset `baseline`) and the one the project moderates with.
|
|
92
156
|
@policies = {}
|
|
93
157
|
@project_policy_id = nil
|
|
158
|
+
# The project's configuration: upload presets by key, the settings it has set, its metadata
|
|
159
|
+
# fields in display order and the revision the settings and fields share.
|
|
160
|
+
@upload_presets = {}
|
|
161
|
+
@project_settings = {}
|
|
162
|
+
@metadata_fields = []
|
|
163
|
+
@settings_revision = 1
|
|
94
164
|
@request_count = 0
|
|
165
|
+
# Each asset's latest `assetSequence`, which the API keeps on the asset row.
|
|
166
|
+
@asset_event_sequences = Hash.new(0)
|
|
167
|
+
# The processing options each asset was uploaded with, which the API keeps but doesn't return.
|
|
168
|
+
@processing_options = {}
|
|
169
|
+
# Decided assets whose re-run (`assets.explain` with `rerun: true`) is still queued: they keep
|
|
170
|
+
# their decision until `complete_processing` decides them.
|
|
171
|
+
@reruns = {}
|
|
95
172
|
end
|
|
96
173
|
|
|
97
|
-
# Adds an event to the feed, filling the fields the API always sets.
|
|
174
|
+
# Adds an event to the feed, filling the fields the API always sets. Its `actor` is the mock's
|
|
175
|
+
# API key unless the event names another.
|
|
98
176
|
def add_event(event)
|
|
99
177
|
time = @now.call
|
|
100
|
-
record = { 'id' => @create_id.call('evt'), 'projectId' => @project_id, '
|
|
101
|
-
'
|
|
178
|
+
record = { 'id' => @create_id.call('evt'), 'projectId' => @project_id, 'actor' => deep_copy(API_KEY_ACTOR),
|
|
179
|
+
'severity' => 'low', 'description' => nil,
|
|
180
|
+
'assetId' => nil, 'assetSequence' => nil, 'asset' => nil, 'facts' => [],
|
|
181
|
+
'createdAt' => time }.merge(event)
|
|
102
182
|
record['title'] ||= record['type']
|
|
183
|
+
# Like the API: each event about an asset takes the asset's next number; others have none.
|
|
184
|
+
asset_id = record['assetId']
|
|
185
|
+
record['assetSequence'] = asset_id ? (@asset_event_sequences[asset_id] += 1) : nil
|
|
103
186
|
@state.events << record
|
|
104
187
|
record
|
|
105
188
|
end
|
|
106
189
|
|
|
107
|
-
# Adds an event like {#add_event}; one about a stored asset carries the asset as it is now
|
|
190
|
+
# Adds an event like {#add_event}; one about a stored asset carries the asset as it is now,
|
|
191
|
+
# with the moderation status an `asset.<status>` event announces.
|
|
108
192
|
def enqueue_event(event)
|
|
109
193
|
item = own_asset(event['assetId'])
|
|
110
|
-
|
|
194
|
+
if item && !event.key?('asset')
|
|
195
|
+
asset = snapshot(item)
|
|
196
|
+
announced = ANNOUNCED_STATUSES[event['type']]
|
|
197
|
+
asset['classification']['moderation']['status'] = announced if announced
|
|
198
|
+
event = event.merge('asset' => asset)
|
|
199
|
+
end
|
|
111
200
|
add_event(event)
|
|
112
201
|
end
|
|
113
202
|
|
|
@@ -121,25 +210,29 @@ module CalmLens
|
|
|
121
210
|
item = own_asset(asset_id)
|
|
122
211
|
return unless item
|
|
123
212
|
|
|
124
|
-
record_decision(item, decision, nil,
|
|
213
|
+
record_decision(item, decision, nil, false)
|
|
125
214
|
deep_copy(item)
|
|
126
215
|
end
|
|
127
216
|
|
|
128
217
|
# Finishes processing an asset as the pipeline would, with `decision` or else what `moderate`
|
|
129
|
-
# answers for the `complete` trigger, and returns it; nil when there is no such asset.
|
|
218
|
+
# answers for the `complete` trigger, and returns it; nil when there is no such asset. A decided
|
|
219
|
+
# asset whose re-run is still queued counts as processing.
|
|
130
220
|
def complete_processing(asset_id, decision)
|
|
131
221
|
item = own_asset(asset_id)
|
|
132
222
|
return unless item
|
|
133
|
-
raise Internal.invalid("Asset #{asset_id} isn't processing") unless item['classification']['status'] == 'processing'
|
|
134
223
|
|
|
135
|
-
|
|
224
|
+
rerun = @reruns.key?(asset_id)
|
|
225
|
+
raise Internal.invalid("Asset #{asset_id} isn't processing") unless rerun || item['classification']['status'] == 'processing'
|
|
226
|
+
|
|
227
|
+
settled = decision.nil? ? nil : MockDecision.from(decision)
|
|
228
|
+
rerun ? settle_rerun(item, 'complete', settled) : moderate_stored(item, 'complete', settled)
|
|
136
229
|
deep_copy(item)
|
|
137
230
|
end
|
|
138
231
|
|
|
139
232
|
# @return [Response]
|
|
140
233
|
def handle(method, url, headers, body)
|
|
141
234
|
if url.start_with?("#{@cdn_base_url}/")
|
|
142
|
-
serve(method, url.delete_prefix(@cdn_base_url))
|
|
235
|
+
serve(method, url.delete_prefix(@cdn_base_url), headers)
|
|
143
236
|
else
|
|
144
237
|
api(method, url.delete_prefix(@base_url), headers, body)
|
|
145
238
|
end
|
|
@@ -172,29 +265,29 @@ module CalmLens
|
|
|
172
265
|
end
|
|
173
266
|
|
|
174
267
|
# Applies a reviewer's decision like `decideAssetManually` in the API: the status and the review,
|
|
175
|
-
# then (for `purged`) the content removal and its event, then `asset.<decision>`.
|
|
176
|
-
|
|
268
|
+
# then (for `purged`) the content removal and its event, then `asset.<decision>`. Either way the
|
|
269
|
+
# review is the mock API key's, which has no ID. A decision through the API (`by_api_key`) has
|
|
270
|
+
# the API key as its events' actor; the test hook's is a reviewer's (a `user`).
|
|
271
|
+
def record_decision(item, decision, note, by_api_key)
|
|
272
|
+
# Like the API: a queued re-run no longer replaces a reviewed decision.
|
|
273
|
+
@reruns.delete(item['id'])
|
|
177
274
|
apply_outcome(item, decision, 'requested')
|
|
178
275
|
review = { 'id' => @create_id.call('rev'), 'decision' => decision, 'note' => note,
|
|
179
|
-
'reviewedAt' => @now.call, '
|
|
180
|
-
|
|
181
|
-
if reviewer_name
|
|
182
|
-
review.merge!('reviewerName' => reviewer_name, 'reviewerEmail' => nil)
|
|
183
|
-
reviewer_facts = [{ 'key' => 'reviewer-user-id', 'value' => nil }, { 'key' => 'reviewer-name', 'value' => reviewer_name }]
|
|
184
|
-
end
|
|
276
|
+
'decidedBy' => { 'type' => 'api-key', 'id' => nil }, 'reviewedAt' => @now.call, 'applied' => true }
|
|
277
|
+
actor = by_api_key ? API_KEY_ACTOR : REVIEWER_ACTOR
|
|
185
278
|
item['classification']['moderation']['manualReviews'] << review
|
|
186
|
-
lifecycle_event('asset.content-removed', item) if decision == 'purged'
|
|
279
|
+
lifecycle_event('asset.content-removed', item, actor: actor) if decision == 'purged'
|
|
187
280
|
# Like the event catalog: only allowing and limiting are low severity.
|
|
188
281
|
add_event('type' => "asset.#{decision}", 'severity' => %w[allowed limited].include?(decision) ? 'low' : 'high',
|
|
189
282
|
'title' => "asset.#{decision}", 'assetId' => item['id'], 'asset' => snapshot(item),
|
|
190
|
-
'
|
|
283
|
+
'actor' => deep_copy(actor),
|
|
284
|
+
'facts' => [{ 'key' => 'asset-type', 'value' => item['type'] },
|
|
191
285
|
{ 'key' => 'asset-name', 'value' => item['name'], 'url' => "https://calmlens.com/go/asset/#{item['id']}" },
|
|
192
|
-
|
|
286
|
+
{ 'key' => 'note', 'value' => note }])
|
|
193
287
|
end
|
|
194
288
|
|
|
195
|
-
# `POST /assets/:id/decision` as the API validates and applies it
|
|
196
|
-
#
|
|
197
|
-
# refused with `asset/processing`.
|
|
289
|
+
# `POST /assets/:id/decision` as the API validates and applies it, with the API key as its
|
|
290
|
+
# events' actor. An asset still uploading or processing is refused with `asset/processing`.
|
|
198
291
|
def decide_request(item, json)
|
|
199
292
|
decision = json['decision']
|
|
200
293
|
invalid!('validation/invalid-union', 'decision') unless REVIEW_DECISIONS.include?(decision)
|
|
@@ -211,7 +304,7 @@ module CalmLens
|
|
|
211
304
|
fail!(ErrorCodes::ASSET_DECISION_UNCHANGED) if status == decision
|
|
212
305
|
fail!(ErrorCodes::ASSET_CONTENT_PURGED) if status == 'purged' && decision != 'blocked'
|
|
213
306
|
|
|
214
|
-
record_decision(item, decision, note,
|
|
307
|
+
record_decision(item, decision, note, true)
|
|
215
308
|
item
|
|
216
309
|
end
|
|
217
310
|
|
|
@@ -219,26 +312,34 @@ module CalmLens
|
|
|
219
312
|
def snapshot(item)
|
|
220
313
|
{ 'id' => item['id'], 'externalId' => item['externalId'], 'uploadPresetId' => item['uploadPresetId'],
|
|
221
314
|
'upload' => { 'status' => item['upload']['status'] },
|
|
222
|
-
'classification' => { 'status' => item['classification']['status'],
|
|
315
|
+
'classification' => { 'status' => item['classification']['status'], 'testInput' => item['classification']['testInput'],
|
|
223
316
|
'moderation' => { 'status' => item['classification']['moderation']['status'] } },
|
|
224
317
|
'content' => { 'status' => item['content']['status'] } }
|
|
225
318
|
end
|
|
226
319
|
|
|
227
|
-
# The `asset.deleted`, `asset.content-removed` and `asset.retried` events the API records
|
|
228
|
-
|
|
320
|
+
# The `asset.deleted`, `asset.content-removed` and `asset.retried` events the API records, with
|
|
321
|
+
# the `batch-id` fact of the batch that caused them, if any.
|
|
322
|
+
def lifecycle_event(type, item, batch_id: nil, actor: API_KEY_ACTOR)
|
|
229
323
|
name = { 'key' => 'asset-name', 'value' => item['name'] }
|
|
230
324
|
# A deleted asset has no portal page to link to.
|
|
231
325
|
name['url'] = "https://calmlens.com/go/asset/#{item['id']}" unless type == 'asset.deleted'
|
|
326
|
+
facts = [name, { 'key' => 'asset-type', 'value' => item['type'] }]
|
|
327
|
+
facts << { 'key' => 'batch-id', 'value' => batch_id } if batch_id
|
|
232
328
|
add_event('type' => type, 'severity' => 'low', 'assetId' => item['id'], 'asset' => snapshot(item),
|
|
233
|
-
'
|
|
329
|
+
'actor' => deep_copy(actor), 'facts' => facts)
|
|
234
330
|
end
|
|
235
331
|
|
|
236
332
|
def fail!(code, message = nil, details: nil) = raise(Failure.new(code, message, details: details))
|
|
237
333
|
|
|
238
334
|
def route_not_found!(method, parts) = fail!(ErrorCodes::ROUTE_NOT_FOUND, "No route for #{method} /#{parts.join('/')}")
|
|
239
335
|
|
|
240
|
-
# A request the API's schema refuses: `validation
|
|
241
|
-
|
|
336
|
+
# A request the API's schema refuses: `validation/invalid-request` with HTTP 400, listing the
|
|
337
|
+
# field and its issue code (such as `validation/too-big`) in `details.issues`.
|
|
338
|
+
def invalid!(code, path)
|
|
339
|
+
message = "Invalid #{path}"
|
|
340
|
+
raise Failure.new(ErrorCodes::INVALID_REQUEST, message,
|
|
341
|
+
details: { 'issues' => [{ 'path' => path, 'message' => message, 'code' => code }] }, status: 400)
|
|
342
|
+
end
|
|
242
343
|
|
|
243
344
|
# Like the API's schema (METADATA_ITEM_SCHEMA): metadata is nil or a list of `{id, value}`
|
|
244
345
|
# Hashes with a String `id` and any JSON `value`.
|
|
@@ -253,32 +354,23 @@ module CalmLens
|
|
|
253
354
|
end
|
|
254
355
|
end
|
|
255
356
|
|
|
256
|
-
# Like the API's schema, checked before the asset is looked up: asset updates take no null
|
|
257
|
-
# name, and signed URLs no null field.
|
|
357
|
+
# Like the API's schema, checked before the asset is looked up: asset updates take no null name.
|
|
258
358
|
def check_asset_body(method, action, json)
|
|
259
|
-
|
|
260
|
-
in ['PATCH', nil]
|
|
261
|
-
check_metadata(json['metadata'])
|
|
262
|
-
invalid!('validation/invalid-type', 'name') if json.key?('name') && json['name'].nil?
|
|
263
|
-
in ['POST', 'signed-url']
|
|
264
|
-
%w[viewerClaims resource lens transform expiresInSeconds].each do |key|
|
|
265
|
-
next unless json.key?(key) && json[key].nil?
|
|
359
|
+
return unless method == 'PATCH' && action.nil?
|
|
266
360
|
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
else nil
|
|
270
|
-
end
|
|
361
|
+
check_metadata(json['metadata'])
|
|
362
|
+
invalid!('validation/invalid-type', 'name') if json.key?('name') && json['name'].nil?
|
|
271
363
|
end
|
|
272
364
|
|
|
273
365
|
# Like the API: an upload's metadata is checked, then its MIME type, which must be one the
|
|
274
|
-
# API can moderate (`upload/
|
|
366
|
+
# API can moderate (`upload/unsupported-type` with `details.mimeType` and `details.source`).
|
|
275
367
|
def check_upload(options)
|
|
276
368
|
check_metadata(options['metadata'])
|
|
277
369
|
type = options['contentType']
|
|
278
370
|
return if type.nil? || Internal.accepted_upload_mime_type?(type)
|
|
279
371
|
|
|
280
|
-
fail!(ErrorCodes::
|
|
281
|
-
details: { 'mimeType' => type })
|
|
372
|
+
fail!(ErrorCodes::UNSUPPORTED_UPLOAD_TYPE, "#{type} is not a valid upload file type",
|
|
373
|
+
details: { 'mimeType' => type, 'source' => 'calmlens' })
|
|
282
374
|
end
|
|
283
375
|
|
|
284
376
|
# Like the API's query schema: an integer from 1 to `max`.
|
|
@@ -299,19 +391,25 @@ module CalmLens
|
|
|
299
391
|
json = content_type == 'application/json' && body && upload.nil? ? JSON.parse(body) : {}
|
|
300
392
|
# HTTP stacks trim header values, so the API sees the key without surrounding whitespace.
|
|
301
393
|
key = header(headers, 'Idempotency-Key')&.to_s&.gsub(/\A[ \t\r\n]+|[ \t\r\n]+\z/, '')
|
|
302
|
-
|
|
394
|
+
request = { query: query, json: json, body: body, upload: upload, content_type: content_type }
|
|
395
|
+
return dispatch(method, parts, request) unless key && method == 'POST'
|
|
303
396
|
|
|
304
397
|
fail!(ErrorCodes::INVALID_IDEMPOTENCY_KEY) unless key.match?(IDEMPOTENCY_KEY)
|
|
305
398
|
# Like the API: a successful response is saved and replayed for the same key and request;
|
|
306
|
-
# a failed one frees the key.
|
|
307
|
-
|
|
399
|
+
# a failed one frees the key. An upload form counts by its options and file, not its boundary.
|
|
400
|
+
form = upload && FakeAPI.upload_form(body, content_type)
|
|
401
|
+
fingerprint = if form
|
|
402
|
+
[method, relative, 'multipart/form-data', form[0], Digest::SHA256.hexdigest(form[1])]
|
|
403
|
+
else
|
|
404
|
+
[method, relative, content_type, Digest::SHA256.hexdigest(body.to_s)]
|
|
405
|
+
end
|
|
308
406
|
saved = @state.idempotency[key]
|
|
309
407
|
if saved
|
|
310
408
|
fail!(ErrorCodes::IDEMPOTENCY_KEY_REUSED) unless saved[:fingerprint] == fingerprint
|
|
311
409
|
return Response.new(status: saved[:status], body: saved[:body],
|
|
312
410
|
headers: saved[:headers].merge('idempotent-replayed' => 'true', 'x-request-id' => request_id))
|
|
313
411
|
end
|
|
314
|
-
response = dispatch(method, parts,
|
|
412
|
+
response = dispatch(method, parts, request)
|
|
315
413
|
if response.success?
|
|
316
414
|
@state.idempotency[key] = { fingerprint: fingerprint, status: response.status, body: response.body,
|
|
317
415
|
headers: response.headers }
|
|
@@ -329,13 +427,17 @@ module CalmLens
|
|
|
329
427
|
fail!(ErrorCodes::INVALID_UPLOAD_PAYLOAD, "#{key} is not valid JSON")
|
|
330
428
|
end
|
|
331
429
|
|
|
332
|
-
def dispatch(method, parts,
|
|
430
|
+
def dispatch(method, parts, request)
|
|
431
|
+
query, json, body = request.values_at(:query, :json, :body)
|
|
333
432
|
data = case parts.first
|
|
334
|
-
when 'projects' then project(method, parts,
|
|
433
|
+
when 'projects' then project(method, parts, request)
|
|
335
434
|
when 'assets' then asset(method, parts, query, json, body)
|
|
336
435
|
when 'folders' then folder(method, parts, json)
|
|
337
436
|
when 'hash-lists' then hash_list(method, parts, query, json)
|
|
338
437
|
when 'hash-list-entries' then remove_hash_list_entry(method, parts)
|
|
438
|
+
when 'policies' then policies(method, parts, query, json)
|
|
439
|
+
when 'lenses' then lens(method, parts, json)
|
|
440
|
+
when 'upload-presets' then upload_preset(method, parts)
|
|
339
441
|
else route_not_found!(method, parts)
|
|
340
442
|
end
|
|
341
443
|
data.is_a?(Response) ? data : respond(200, deep_copy(data))
|
|
@@ -343,27 +445,40 @@ module CalmLens
|
|
|
343
445
|
|
|
344
446
|
def deep_copy(value) = JSON.parse(JSON.generate(value))
|
|
345
447
|
|
|
346
|
-
|
|
347
|
-
# request: absent is a JSON body, 'content' raw file content and 'part' a raw first part.
|
|
348
|
-
def project(method, parts, query, json, body, upload)
|
|
448
|
+
def project(method, parts, request)
|
|
349
449
|
fail!(ErrorCodes::API_KEY_PROJECT_MISMATCH) unless parts[1] == @project_id
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
in ['
|
|
353
|
-
in ['
|
|
354
|
-
in ['
|
|
355
|
-
in ['
|
|
356
|
-
in ['
|
|
357
|
-
in ['
|
|
358
|
-
in ['
|
|
359
|
-
in ['
|
|
360
|
-
in [
|
|
361
|
-
in ['
|
|
362
|
-
in ['GET', ['assets'
|
|
363
|
-
in ['
|
|
364
|
-
in ['
|
|
365
|
-
in ['POST', ['assets'
|
|
366
|
-
in ['POST', ['
|
|
450
|
+
query, json = request.values_at(:query, :json)
|
|
451
|
+
case [method, parts[2..]]
|
|
452
|
+
in ['GET', ['events']] then events(query)
|
|
453
|
+
in ['PUT', ['lenses']] then lenses(json.fetch('lenses', []))
|
|
454
|
+
in ['GET', ['lenses']] then page(@state.lenses.values, query, JSON.generate(['lenses', @project_id]))
|
|
455
|
+
in ['POST', ['lenses']] then create_lens(json)
|
|
456
|
+
in ['GET', ['folders']] then folder_page(query)
|
|
457
|
+
in ['POST', ['folders']] then create_folder(json)
|
|
458
|
+
in ['GET', ['hash-lists']] then page(project_hash_lists, query, JSON.generate([@project_id]))
|
|
459
|
+
in ['POST', ['hash-lists']] then create_hash_list(json)
|
|
460
|
+
in ['GET', ['policy']] then show_policy(find_policy(ensure_policies))
|
|
461
|
+
in ['PUT', ['policy']] then show_policy(assign_policy(json))
|
|
462
|
+
in ['GET', ['assets']] then list(query)
|
|
463
|
+
in ['GET', ['assets', 'search']] then search(query)
|
|
464
|
+
in ['GET', ['assets', String => asset_id, 'timeline']] then timeline(asset_id, query)
|
|
465
|
+
in ['POST', ['assets', 'batch']] then batch(json)
|
|
466
|
+
in ['POST', ['signed-asset-urls']] then sign_asset_url(json)
|
|
467
|
+
in ['POST', ['assets']] then create_upload(request)
|
|
468
|
+
else configuration(method, parts, query, json)
|
|
469
|
+
end
|
|
470
|
+
end
|
|
471
|
+
|
|
472
|
+
# A project's upload presets, settings and metadata fields.
|
|
473
|
+
def configuration(method, parts, query, json)
|
|
474
|
+
case [method, parts[2..]]
|
|
475
|
+
in ['GET', ['upload-presets']] then upload_preset_page(query)
|
|
476
|
+
in ['PUT', ['upload-presets']] then ensure_upload_presets(json)
|
|
477
|
+
in ['GET', ['settings']] then show_project_settings
|
|
478
|
+
in ['PUT', ['settings']] then update_project_settings(json)
|
|
479
|
+
in ['GET', ['metadata-fields']] then show_metadata_fields
|
|
480
|
+
in ['PUT', ['metadata-fields']] then ensure_metadata_fields(json)
|
|
481
|
+
in ['DELETE', ['metadata-fields', String => field_id]] then delete_metadata_field(field_id, query)
|
|
367
482
|
else route_not_found!(method, parts)
|
|
368
483
|
end
|
|
369
484
|
end
|
|
@@ -379,7 +494,8 @@ module CalmLens
|
|
|
379
494
|
# (`asset/not-found`) before anything happens. The operation then runs at once, in request
|
|
380
495
|
# order, rather than in the background: `delete` deletes each asset with its stored child
|
|
381
496
|
# assets and records `asset.deleted` with the asset as it was; `remove-content` removes the
|
|
382
|
-
# content like DELETE /assets/:id/content.
|
|
497
|
+
# content like DELETE /assets/:id/content. Each event carries the batch's `batch-id` fact. Mock
|
|
498
|
+
# assets carry no safety holds.
|
|
383
499
|
def batch(json)
|
|
384
500
|
ids = json['assetIds']
|
|
385
501
|
invalid!('validation/invalid-type', 'assetIds') unless ids.is_a?(Array)
|
|
@@ -394,15 +510,16 @@ module CalmLens
|
|
|
394
510
|
fail!(ErrorCodes::ASSET_NOT_FOUND) unless item && item['projectId'] == @project_id
|
|
395
511
|
item
|
|
396
512
|
end
|
|
513
|
+
batch_id = @create_id.call('batch')
|
|
397
514
|
items.each do |item|
|
|
398
515
|
if json['operation'] == 'delete'
|
|
399
516
|
delete_tree(item)
|
|
400
|
-
lifecycle_event('asset.deleted', item)
|
|
517
|
+
lifecycle_event('asset.deleted', item, batch_id: batch_id)
|
|
401
518
|
else
|
|
402
|
-
|
|
519
|
+
remove_content(item, batch_id)
|
|
403
520
|
end
|
|
404
521
|
end
|
|
405
|
-
respond(202, { 'queued' =>
|
|
522
|
+
respond(202, { 'batchId' => batch_id, 'status' => 'queued', 'assetCount' => items.size })
|
|
406
523
|
end
|
|
407
524
|
|
|
408
525
|
# Deletes an asset, its stored child assets and their content.
|
|
@@ -417,26 +534,26 @@ module CalmLens
|
|
|
417
534
|
deleting.each do |id|
|
|
418
535
|
@state.assets.delete(id)
|
|
419
536
|
@state.content.delete(id)
|
|
537
|
+
@processing_options.delete(id)
|
|
420
538
|
end
|
|
421
539
|
end
|
|
422
540
|
|
|
423
541
|
def list(query)
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
page(RSQL.filter(
|
|
542
|
+
# Like the API: the query schema is checked before the filter.
|
|
543
|
+
page_query(query)
|
|
544
|
+
page(RSQL.filter(project_assets, query['filter']), query, JSON.generate([@project_id, query['filter']]))
|
|
427
545
|
end
|
|
428
546
|
|
|
429
547
|
# Like the API: one keyset page of a parent's child folders, optionally matching `search`, by
|
|
430
|
-
# `name` ascending unless `
|
|
548
|
+
# `name` ascending unless `sort` (one field, `name` or `createdAt`) says otherwise, with an
|
|
431
549
|
# `id` tie-breaker, and the ancestors of the parent and the listed folders. Names compare by
|
|
432
550
|
# code point, so case matters.
|
|
433
551
|
def folder_page(query)
|
|
434
552
|
found = project_folders.select do |f|
|
|
435
553
|
f['parentId'] == query['parentId'] && (!query['search'] || f['name'].downcase.include?(query['search'].downcase))
|
|
436
554
|
end
|
|
437
|
-
invalid!('validation/too-big', 'ordering') if ordering(query, nil).size > 1
|
|
438
555
|
scope = JSON.generate([@project_id, query['parentId'], query['search']])
|
|
439
|
-
result = page(found, query, scope, fields: %w[name createdAt], default: { 'field' => 'name', 'direction' => 'asc' })
|
|
556
|
+
result = page(found, query, scope, fields: %w[name createdAt], max: 1, default: { 'field' => 'name', 'direction' => 'asc' })
|
|
440
557
|
page = result['items']
|
|
441
558
|
ids = ([query['parentId']] + page.map { |f| f['id'] }).compact.uniq
|
|
442
559
|
queue = ids.dup
|
|
@@ -453,46 +570,85 @@ module CalmLens
|
|
|
453
570
|
def project_folders = @state.folders.values.select { |item| item['projectId'] == @project_id }
|
|
454
571
|
|
|
455
572
|
# Like the API: without a cursor, no events and a `nextCursor` for now; with one, up to `pageSize`
|
|
456
|
-
# events added since, oldest first
|
|
457
|
-
#
|
|
573
|
+
# events added since, oldest first, of the `types` (comma-separated) and `assetId` asked for.
|
|
574
|
+
# `nextCursor` is never nil, and a cursor belongs to the filters it was issued with. The mock's
|
|
575
|
+
# key holds every permission, so no event type is excluded.
|
|
458
576
|
def events(query)
|
|
459
577
|
size = check_range(query.fetch('pageSize', DEFAULT_PAGE_SIZE), 'pageSize', MAX_PAGE_SIZE)
|
|
578
|
+
types = query.key?('types') ? query['types'].to_s.split(',', -1) : []
|
|
579
|
+
if query.key?('types') && (types.empty? || types.any? { |type| !type.match?(EVENT_TYPE_PATTERN) })
|
|
580
|
+
invalid!('validation/invalid-value', 'types')
|
|
581
|
+
end
|
|
582
|
+
asset_id = query['assetId']
|
|
583
|
+
invalid!('validation/invalid-format', 'assetId') if asset_id && !asset_id.match?(ASSET_ID_PATTERN)
|
|
584
|
+
filter = [JSON.generate([types.uniq.sort, asset_id])].pack('m0')
|
|
585
|
+
matches = ->(event) { (types.empty? || types.include?(event['type'])) && (asset_id.nil? || event['assetId'] == asset_id) }
|
|
460
586
|
feed = @state.events
|
|
461
|
-
|
|
587
|
+
unless query['cursor']
|
|
588
|
+
return { 'items' => [], 'nextCursor' => "mock_#{feed.size}~#{filter}", 'hasMore' => false, 'excludedTypes' => [] }
|
|
589
|
+
end
|
|
462
590
|
|
|
463
|
-
match = query['cursor'].to_s.match(/\Amock_(\d+)\z/)
|
|
464
|
-
position = match && Integer(match[1], 10)
|
|
591
|
+
match = query['cursor'].to_s.match(/\Amock_(\d+)~(.*)\z/)
|
|
592
|
+
position = match && match[2] == filter ? Integer(match[1], 10) : nil
|
|
465
593
|
fail!(ErrorCodes::INVALID_CURSOR) unless position && position <= feed.size
|
|
466
594
|
|
|
467
|
-
taken =
|
|
468
|
-
following = position
|
|
469
|
-
|
|
595
|
+
taken = []
|
|
596
|
+
following = position
|
|
597
|
+
while following < feed.size && taken.size < size
|
|
598
|
+
event = feed[following]
|
|
599
|
+
following += 1
|
|
600
|
+
taken << event if matches.call(event)
|
|
601
|
+
end
|
|
602
|
+
{ 'items' => taken, 'nextCursor' => "mock_#{following}~#{filter}", 'hasMore' => feed.drop(following).any?(&matches),
|
|
603
|
+
'excludedTypes' => [] }
|
|
470
604
|
end
|
|
471
605
|
|
|
472
|
-
# Saves lenses in order, like the API:
|
|
473
|
-
#
|
|
606
|
+
# Saves lenses in order, like the API: a new identifier creates a lens, an unchanged
|
|
607
|
+
# definition returns the stored lens and a changed one updates it in place.
|
|
474
608
|
def lenses(definitions)
|
|
475
609
|
{ 'lenses' => definitions.map { |definition| save_lens(definition) } }
|
|
476
610
|
end
|
|
477
611
|
|
|
478
612
|
def save_lens(definition)
|
|
479
|
-
now = @now.call
|
|
480
613
|
old = @state.lenses[definition['identifier']]
|
|
481
|
-
unless old
|
|
482
|
-
lens = { 'id' => @create_id.call('lens'), 'projectId' => @project_id }.merge(definition)
|
|
483
|
-
return @state.lenses[definition['identifier']] = lens.merge('createdAt' => now, 'updatedAt' => now)
|
|
484
|
-
end
|
|
614
|
+
return new_lens(definition) unless old
|
|
485
615
|
|
|
486
616
|
same = %w[name type active].all? { |key| old[key] == definition[key] } &&
|
|
487
617
|
old['description'] == definition['description'] && canonical(old['transform']) == canonical(definition['transform'])
|
|
488
|
-
return old if same
|
|
489
|
-
if definition['version'].to_i > old['version'].to_i
|
|
490
|
-
return @state.lenses[old['identifier']] =
|
|
491
|
-
old.merge(definition).merge('updatedAt' => now)
|
|
492
|
-
end
|
|
618
|
+
return old if same
|
|
493
619
|
|
|
494
|
-
|
|
495
|
-
|
|
620
|
+
@state.lenses[old['identifier']] = old.merge(definition).merge('updatedAt' => @now.call)
|
|
621
|
+
end
|
|
622
|
+
|
|
623
|
+
def new_lens(definition)
|
|
624
|
+
now = @now.call
|
|
625
|
+
lens = { 'id' => @create_id.call('lens'), 'projectId' => @project_id }.merge(definition)
|
|
626
|
+
@state.lenses[definition['identifier']] = lens.merge('createdAt' => now, 'updatedAt' => now)
|
|
627
|
+
end
|
|
628
|
+
|
|
629
|
+
# POST /projects/:id/lenses: like the API, the identifier must be new to the project.
|
|
630
|
+
def create_lens(json)
|
|
631
|
+
identifier = json['identifier']
|
|
632
|
+
invalid!('validation/invalid-type', 'identifier') unless identifier.is_a?(String) && !identifier.empty?
|
|
633
|
+
fail!(ErrorCodes::LENS_IDENTIFIER_TAKEN, details: { 'identifier' => identifier }) if @state.lenses.key?(identifier)
|
|
634
|
+
|
|
635
|
+
# Like the API, a new lens is 201 Created.
|
|
636
|
+
respond(201, deep_copy(new_lens(json)))
|
|
637
|
+
end
|
|
638
|
+
|
|
639
|
+
# GET, PATCH and DELETE /lenses/:id. An update keeps the identifier and changes the rest of
|
|
640
|
+
# the definition, as ensuring the changed definition would.
|
|
641
|
+
def lens(method, parts, json)
|
|
642
|
+
item = @state.lenses.values.find { |lens| lens['id'] == parts[1] && lens['projectId'] == @project_id }
|
|
643
|
+
fail!(ErrorCodes::LENS_NOT_FOUND) unless item && parts.size == 2
|
|
644
|
+
case method
|
|
645
|
+
when 'GET' then item
|
|
646
|
+
when 'PATCH' then save_lens(item.merge(json.except('identifier', 'id', 'projectId', 'createdAt', 'updatedAt')))
|
|
647
|
+
when 'DELETE'
|
|
648
|
+
@state.lenses.delete(item['identifier'])
|
|
649
|
+
{}
|
|
650
|
+
else route_not_found!(method, parts)
|
|
651
|
+
end
|
|
496
652
|
end
|
|
497
653
|
|
|
498
654
|
def canonical(value)
|
|
@@ -503,13 +659,136 @@ module CalmLens
|
|
|
503
659
|
end
|
|
504
660
|
end
|
|
505
661
|
|
|
662
|
+
# A page of the project's upload presets, oldest first unless `sort` says otherwise.
|
|
663
|
+
def upload_preset_page(query)
|
|
664
|
+
page(@upload_presets.values, query, JSON.generate(['upload-presets', @project_id]),
|
|
665
|
+
default: { 'field' => 'createdAt', 'direction' => 'asc' })
|
|
666
|
+
end
|
|
667
|
+
|
|
668
|
+
# Like the API: every preset's revision and settings are checked before any is saved, and the
|
|
669
|
+
# stored presets come back in the order sent.
|
|
670
|
+
def ensure_upload_presets(json)
|
|
671
|
+
definitions = json['uploadPresets']
|
|
672
|
+
invalid!('validation/too-small', 'uploadPresets') unless definitions.is_a?(Array) && !definitions.empty?
|
|
673
|
+
|
|
674
|
+
plans = definitions.each_with_index.map do |definition, index|
|
|
675
|
+
%w[key name].each do |field|
|
|
676
|
+
value = definition[field]
|
|
677
|
+
invalid!('validation/invalid-type', "uploadPresets.#{index}.#{field}") unless value.is_a?(String) && !value.empty?
|
|
678
|
+
end
|
|
679
|
+
existing = @upload_presets[definition['key']]
|
|
680
|
+
stored = existing ? existing['revision'] : 0
|
|
681
|
+
if definition.key?('revision') && definition['revision'] != stored
|
|
682
|
+
fail!(ErrorCodes::UPLOAD_PRESET_REVISION_CONFLICT, details: { 'key' => definition['key'], 'revision' => stored })
|
|
683
|
+
end
|
|
684
|
+
values = upload_preset_values(definition)
|
|
685
|
+
limited = ![nil, 0].include?(values['dailyUploadLimit'])
|
|
686
|
+
fail!(ErrorCodes::UPLOAD_PRESET_DAILY_LIMIT_REQUIRED) if values['access'] == 'anyone' && !limited
|
|
687
|
+
|
|
688
|
+
[existing, values]
|
|
689
|
+
end
|
|
690
|
+
{ 'uploadPresets' => plans.map { |existing, values| save_upload_preset(existing, values) } }
|
|
691
|
+
end
|
|
692
|
+
|
|
693
|
+
# The stored values a definition sets: fields left out take their defaults.
|
|
694
|
+
def upload_preset_values(definition)
|
|
695
|
+
given = ->(field, default = nil) { definition[field].nil? ? default : definition[field] }
|
|
696
|
+
list = ->(field) { definition[field].is_a?(Array) && !definition[field].empty? ? definition[field].uniq : nil }
|
|
697
|
+
{ 'key' => definition['key'], 'name' => definition['name'], 'access' => given.call('access', 'token'),
|
|
698
|
+
'enabled' => given.call('enabled', true), 'allowedContentTypes' => list.call('allowedContentTypes'),
|
|
699
|
+
'maxSizeInBytes' => given.call('maxSizeInBytes'), 'folderId' => given.call('folderId'),
|
|
700
|
+
'policyId' => given.call('policyId'), 'visibility' => given.call('visibility', 'private'),
|
|
701
|
+
'processingOptions' => given.call('processingOptions'), 'revealDecision' => given.call('revealDecision', false),
|
|
702
|
+
'dailyUploadLimit' => given.call('dailyUploadLimit'), 'allowedOrigins' => list.call('allowedOrigins') }
|
|
703
|
+
end
|
|
704
|
+
|
|
705
|
+
# Saves a preset's values under its key; an unchanged preset keeps its revision.
|
|
706
|
+
def save_upload_preset(existing, values)
|
|
707
|
+
return existing if existing && existing.slice(*values.keys) == values
|
|
708
|
+
|
|
709
|
+
now = @now.call
|
|
710
|
+
@upload_presets[values['key']] =
|
|
711
|
+
{ 'id' => existing ? existing['id'] : @create_id.call('preset'), 'projectId' => @project_id, 'createdByUserId' => nil,
|
|
712
|
+
**values, 'revision' => (existing ? existing['revision'] : 0) + 1,
|
|
713
|
+
'createdAt' => existing ? existing['createdAt'] : now, 'updatedAt' => now }
|
|
714
|
+
end
|
|
715
|
+
|
|
716
|
+
# DELETE /upload-presets/:id.
|
|
717
|
+
def upload_preset(method, parts)
|
|
718
|
+
route_not_found!(method, parts) unless method == 'DELETE' && parts.size == 2
|
|
719
|
+
preset = @upload_presets.values.find { |item| item['id'] == parts[1] }
|
|
720
|
+
fail!(ErrorCodes::UPLOAD_PRESET_NOT_FOUND) unless preset
|
|
721
|
+
@upload_presets.delete(preset['key'])
|
|
722
|
+
{}
|
|
723
|
+
end
|
|
724
|
+
|
|
725
|
+
def show_project_settings
|
|
726
|
+
settings = { 'optOutJurisdictions' => [], 'blockUnmoderatedAssets' => true, 'storeSensitiveMetadata' => false,
|
|
727
|
+
'streamingEnabled' => false, 'aiMediaSearchEnabled' => false, 'remoteUrlRule' => nil,
|
|
728
|
+
'websiteSubmissionsPerMinute' => nil }.merge(@project_settings)
|
|
729
|
+
settings['humanReview'] = { 'allowedSamplePercent' => 0, 'restrictedSamplePercent' => 0, 'uncertaintyMarginPercent' => 0,
|
|
730
|
+
'reviewMissingEvidence' => false }.merge(@project_settings['humanReview'] || {})
|
|
731
|
+
{ 'projectId' => @project_id, 'revision' => @settings_revision, 'settings' => settings }
|
|
732
|
+
end
|
|
733
|
+
|
|
734
|
+
# A revision to check, from a body or the query; none refuses nothing.
|
|
735
|
+
def check_settings_revision(revision)
|
|
736
|
+
return if revision.nil? || revision.to_s == @settings_revision.to_s
|
|
737
|
+
|
|
738
|
+
fail!(ErrorCodes::PROJECT_SETTINGS_REVISION_CONFLICT)
|
|
739
|
+
end
|
|
740
|
+
|
|
741
|
+
# Replaces the settings: those left out return to their defaults. A real change bumps the revision.
|
|
742
|
+
def update_project_settings(json)
|
|
743
|
+
check_settings_revision(json['revision'])
|
|
744
|
+
invalid!('validation/invalid-type', 'settings') unless json['settings'].is_a?(Hash)
|
|
745
|
+
|
|
746
|
+
before = show_project_settings['settings']
|
|
747
|
+
previous = @project_settings
|
|
748
|
+
@project_settings = json['settings']
|
|
749
|
+
if show_project_settings['settings'] == before
|
|
750
|
+
@project_settings = previous
|
|
751
|
+
else
|
|
752
|
+
@settings_revision += 1
|
|
753
|
+
end
|
|
754
|
+
show_project_settings
|
|
755
|
+
end
|
|
756
|
+
|
|
757
|
+
def show_metadata_fields = { 'projectId' => @project_id, 'revision' => @settings_revision, 'fields' => @metadata_fields }
|
|
758
|
+
|
|
759
|
+
# Creates or replaces fields by ID, keeping the others in order. A real change bumps the revision.
|
|
760
|
+
def ensure_metadata_fields(json)
|
|
761
|
+
check_settings_revision(json['revision'])
|
|
762
|
+
fields = json['fields']
|
|
763
|
+
invalid!('validation/too-small', 'fields') unless fields.is_a?(Array) && !fields.empty?
|
|
764
|
+
|
|
765
|
+
by_id = fields.to_h { |field| [field['id'], field] }
|
|
766
|
+
known = @metadata_fields.map { |field| field['id'] }
|
|
767
|
+
merged = @metadata_fields.map { |field| by_id.fetch(field['id'], field) } + fields.reject { |field| known.include?(field['id']) }
|
|
768
|
+
unless merged == @metadata_fields
|
|
769
|
+
@metadata_fields = merged
|
|
770
|
+
@settings_revision += 1
|
|
771
|
+
end
|
|
772
|
+
show_metadata_fields
|
|
773
|
+
end
|
|
774
|
+
|
|
775
|
+
def delete_metadata_field(field_id, query)
|
|
776
|
+
kept = @metadata_fields.reject { |field| field['id'] == field_id }
|
|
777
|
+
fail!(ErrorCodes::METADATA_FIELD_NOT_FOUND, details: { 'fieldId' => field_id }) if kept.size == @metadata_fields.size
|
|
778
|
+
|
|
779
|
+
check_settings_revision(query['revision'])
|
|
780
|
+
@metadata_fields = kept
|
|
781
|
+
@settings_revision += 1
|
|
782
|
+
show_metadata_fields
|
|
783
|
+
end
|
|
784
|
+
|
|
506
785
|
# Keyword search like the API: every whitespace-separated token (case-insensitive, at most 8
|
|
507
786
|
# distinct) must appear in some field. Results rank by the API's score, then newest first.
|
|
508
787
|
def search(query)
|
|
509
788
|
text = query['query'].to_s.strip
|
|
510
789
|
invalid!('validation/too-small', 'query') if text.empty?
|
|
511
790
|
invalid!('validation/too-big', 'query') if text.length > MAX_SEARCH_QUERY_LENGTH
|
|
512
|
-
limit = check_range(query.fetch('
|
|
791
|
+
limit = check_range(query.fetch('pageSize', 10), 'pageSize', MAX_SEARCH_PAGE_SIZE)
|
|
513
792
|
normalized = text.downcase
|
|
514
793
|
tokens = normalized.split(/\s+/).uniq.first(MAX_SEARCH_TOKENS)
|
|
515
794
|
items = project_assets.filter_map do |item|
|
|
@@ -560,17 +839,21 @@ module CalmLens
|
|
|
560
839
|
keep = processing.is_a?(Hash) && processing['keepContentAfterProcessing'] == true
|
|
561
840
|
context = options['context'].is_a?(Hash) ? ModerationContext.select_items(options['context'].fetch('items', [])) : []
|
|
562
841
|
# Fields the API returns as null when they are not set.
|
|
563
|
-
asset = { 'id' => @create_id.call('asset'), 'projectId' => @project_id, 'parentId' => nil,
|
|
842
|
+
asset = { 'id' => @create_id.call('asset'), 'projectId' => @project_id, 'parentId' => nil, 'parentPages' => [],
|
|
564
843
|
'folderId' => options['folderId'], 'uploadPresetId' => options['uploadPresetId'],
|
|
565
844
|
'createdAt' => now, 'updatedAt' => now, 'name' => options['name'], 'description' => options['description'],
|
|
566
845
|
'externalId' => options['externalId'], 'externalUrl' => url || options['externalUrl'],
|
|
567
|
-
'type' => kind(mime, url), 'metadata' => options['metadata'], 'authorId' => options['authorId'],
|
|
846
|
+
'type' => kind(mime, url), 'metadata' => options['metadata'] || [], 'authorId' => options['authorId'],
|
|
568
847
|
# Like the API, only the posts CalmLens uses are kept, and only while the content is held.
|
|
569
|
-
'context' => context.empty? ? nil : { 'items' => context }, 'tags' =>
|
|
570
|
-
'
|
|
848
|
+
'context' => context.empty? ? nil : { 'items' => context }, 'tags' => options['tags'] || [],
|
|
849
|
+
'visibility' => options.fetch('visibility', 'private'),
|
|
571
850
|
**sections(now: now, open: open, keep: keep, format: mime, size: data&.bytesize) }
|
|
572
851
|
@state.assets[asset['id']] = asset
|
|
573
|
-
@
|
|
852
|
+
@processing_options[asset['id']] = processing
|
|
853
|
+
if data
|
|
854
|
+
@state.content[asset['id']] = { 'type' => mime, 'parts' => { 1 => data } }
|
|
855
|
+
asset['content'].merge!(rendering(asset, false)) unless open
|
|
856
|
+
end
|
|
574
857
|
open ? asset : upload_completed(asset, queue, data)
|
|
575
858
|
end
|
|
576
859
|
|
|
@@ -578,20 +861,30 @@ module CalmLens
|
|
|
578
861
|
{
|
|
579
862
|
'content' => { 'status' => keep ? 'stored' : 'temporary', 'removalReason' => nil, 'updatedAt' => now,
|
|
580
863
|
'safetyHold' => nil, 'format' => format.empty? ? nil : format, 'extension' => nil,
|
|
581
|
-
|
|
582
|
-
|
|
864
|
+
# The mock decodes nothing, so the file's own details are unknown, and like
|
|
865
|
+
# the API, the rendered file and preview come with processing.
|
|
866
|
+
'sizeInBytes' => size, 'checksums' => nil, 'width' => nil, 'height' => nil,
|
|
867
|
+
'durationSeconds' => nil, 'pageCount' => nil, 'perceptualHash' => nil,
|
|
868
|
+
'perceptualHashQuality' => nil, 'metadata' => nil, 'extractedText' => nil, **NO_RENDERING },
|
|
583
869
|
# Like the API, only an open upload counts the parts received.
|
|
584
870
|
'upload' => { 'status' => open ? 'uploading' : 'completed', 'updatedAt' => now, 'partsReceived' => open ? 1 : nil },
|
|
585
|
-
'classification' => { 'status' => open ? '
|
|
586
|
-
|
|
587
|
-
'
|
|
588
|
-
|
|
871
|
+
'classification' => { 'status' => open ? 'queued' : 'processing', 'failureReason' => nil, 'updatedAt' => now,
|
|
872
|
+
# The mock never reads test inputs.
|
|
873
|
+
'completedAt' => nil, 'progress' => nil, 'testInput' => false,
|
|
874
|
+
# Like the API, moderation's results are null until it decides, and a
|
|
875
|
+
# feature's result is null until it has run.
|
|
876
|
+
'moderation' => { 'status' => 'pending', 'manualReviews' => [], 'automatedStatus' => nil,
|
|
877
|
+
'policy' => nil, 'pendingReview' => nil, 'source' => nil, 'rule' => nil,
|
|
878
|
+
'explanationComplete' => nil,
|
|
879
|
+
'categories' => nil, 'flaggedCategories' => [], 'regions' => [],
|
|
880
|
+
'raw' => nil, 'timeline' => nil },
|
|
881
|
+
'ocr' => nil, 'linkDetection' => nil }
|
|
589
882
|
}
|
|
590
883
|
end
|
|
591
884
|
|
|
592
885
|
# Like the API: image uploads in parts total at most 20 MiB, at any size below that.
|
|
593
886
|
def image_size_error(type, size, _final)
|
|
594
|
-
ErrorCodes::
|
|
887
|
+
ErrorCodes::UPLOAD_TOO_LARGE if type == 'image' && size > MAX_MULTIPART_IMAGE
|
|
595
888
|
end
|
|
596
889
|
|
|
597
890
|
# Like the API: the parts may not overshoot a declared total, and must reach it at the end.
|
|
@@ -601,28 +894,96 @@ module CalmLens
|
|
|
601
894
|
ErrorCodes::UPLOAD_SIZE_MISMATCH if received > total || (final && received != total)
|
|
602
895
|
end
|
|
603
896
|
|
|
897
|
+
# The failure for a size error code, with the limit `upload/too-large` carries.
|
|
898
|
+
def size_failure!(code)
|
|
899
|
+
details = code == ErrorCodes::UPLOAD_TOO_LARGE ? { 'limitBytes' => MAX_MULTIPART_IMAGE, 'source' => 'plan' } : nil
|
|
900
|
+
fail!(code, details: details)
|
|
901
|
+
end
|
|
902
|
+
|
|
604
903
|
def check_part_size(size, final)
|
|
605
904
|
return if final || size >= MIN_PART
|
|
606
905
|
|
|
607
906
|
fail!(ErrorCodes::INVALID_UPLOAD_PAYLOAD, 'Multipart upload parts must be at least 5MiB except the final part.')
|
|
608
907
|
end
|
|
609
908
|
|
|
909
|
+
# `POST .../assets` as the API reads it. Asset fields in the query are refused; then the
|
|
910
|
+
# `X-CalmLens-Upload` header picks the body: absent is JSON (text, a first text part or a URL),
|
|
911
|
+
# `content` an upload form with a whole file and `part` one with a first part.
|
|
912
|
+
def create_upload(request)
|
|
913
|
+
query, json, body, upload = request.values_at(:query, :json, :body, :upload)
|
|
914
|
+
in_query = Assets::UPLOAD_OPTION_KEYS.select { |key| query.key?(key) }
|
|
915
|
+
unless in_query.empty?
|
|
916
|
+
fail!(ErrorCodes::UPLOAD_OPTIONS_IN_QUERY, "Send the asset's fields in the request body, not in the query string",
|
|
917
|
+
details: { 'fields' => in_query })
|
|
918
|
+
end
|
|
919
|
+
queue = query['wait'] == false
|
|
920
|
+
return create_from_json(json, queue) if upload.nil?
|
|
921
|
+
|
|
922
|
+
route_not_found!('POST', ['assets']) unless %w[content part].include?(upload)
|
|
923
|
+
|
|
924
|
+
options, bytes = form_fields(body, request[:content_type])
|
|
925
|
+
fields = options.merge('contentType' => query['contentType'])
|
|
926
|
+
return create_asset(fields, queue, data: bytes) if upload == 'content'
|
|
927
|
+
|
|
928
|
+
start_multipart(fields.merge(query.slice('isFinal', 'totalSizeInBytes')), bytes, queue)
|
|
929
|
+
end
|
|
930
|
+
|
|
931
|
+
# A JSON create: text as `content` or a first `part` (`{ plainText, type }`), or a URL.
|
|
932
|
+
def create_from_json(json, queue)
|
|
933
|
+
key = %w[content part].find { |name| json.key?(name) }
|
|
934
|
+
return create_asset(json, queue, url: json['url']) unless key
|
|
935
|
+
|
|
936
|
+
text = json[key]
|
|
937
|
+
invalid!('validation/invalid-type', key) unless text.is_a?(Hash) && text['plainText'].is_a?(String) && text['type'].is_a?(String)
|
|
938
|
+
fields = json.except('content', 'part', 'sizeInBytes', 'totalSizeInBytes').merge('contentType' => text['type'])
|
|
939
|
+
data = text['plainText'].encode(Encoding::UTF_8).b
|
|
940
|
+
return create_asset(fields, queue, data: data) if key == 'content'
|
|
941
|
+
|
|
942
|
+
start_multipart(fields.merge('isFinal' => text['isFinal'] == true, 'totalSizeInBytes' => json['totalSizeInBytes']), data, queue)
|
|
943
|
+
end
|
|
944
|
+
|
|
945
|
+
# The asset fields in an upload form's options part, and the file's bytes. Like the API, a raw
|
|
946
|
+
# upload must be `multipart/form-data` (`request/unsupported-content-type`, 415), holding an
|
|
947
|
+
# `options` JSON object of known fields with a name, then a `file` part.
|
|
948
|
+
def form_fields(body, content_type)
|
|
949
|
+
mime = content_type.to_s.split(';').first.to_s.strip.downcase
|
|
950
|
+
unless mime == 'multipart/form-data'
|
|
951
|
+
fail!(ErrorCodes::UNSUPPORTED_CONTENT_TYPE, 'Send a raw upload as multipart/form-data: an options part, then a file part.',
|
|
952
|
+
details: { 'contentType' => mime.empty? ? nil : mime })
|
|
953
|
+
end
|
|
954
|
+
form = FakeAPI.upload_form(body, content_type)
|
|
955
|
+
fail!(ErrorCodes::INVALID_UPLOAD_PAYLOAD, 'The upload form must hold an options part, then a file part') unless form
|
|
956
|
+
|
|
957
|
+
options = begin
|
|
958
|
+
JSON.parse(form[0])
|
|
959
|
+
rescue JSON::ParserError
|
|
960
|
+
nil
|
|
961
|
+
end
|
|
962
|
+
unless options.is_a?(Hash)
|
|
963
|
+
fail!(ErrorCodes::INVALID_UPLOAD_PAYLOAD, "The options part must be a JSON object with the asset's fields")
|
|
964
|
+
end
|
|
965
|
+
unknown = options.keys.find { |name| !Assets::UPLOAD_OPTION_KEYS.include?(name) }
|
|
966
|
+
invalid!('validation/unknown-field', unknown) if unknown
|
|
967
|
+
invalid!('validation/invalid-type', 'name') unless options['name'].is_a?(String)
|
|
968
|
+
[options, form[1]]
|
|
969
|
+
end
|
|
970
|
+
|
|
610
971
|
# In the API's order: a declared total, then a single final part as the whole upload, then the
|
|
611
|
-
# part itself.
|
|
612
|
-
def start_multipart(
|
|
613
|
-
check_upload(
|
|
614
|
-
total =
|
|
615
|
-
type = kind(
|
|
616
|
-
final =
|
|
972
|
+
# part itself. Whether to wait and the declared total are saved with an open upload.
|
|
973
|
+
def start_multipart(fields, data, queue)
|
|
974
|
+
check_upload(fields)
|
|
975
|
+
total = fields['totalSizeInBytes']
|
|
976
|
+
type = kind(fields['contentType'].to_s, nil)
|
|
977
|
+
final = fields['isFinal'] == true
|
|
617
978
|
size = data.to_s.bytesize
|
|
618
979
|
code = (total && image_size_error(type, total, true)) ||
|
|
619
980
|
(final ? total_size_error(total, size, true) || image_size_error(type, size, true) : nil)
|
|
620
|
-
|
|
981
|
+
size_failure!(code) if code
|
|
621
982
|
check_part_size(size, final)
|
|
622
983
|
fail!(total_size_error(total, size, false)) if !final && total_size_error(total, size, false)
|
|
623
984
|
|
|
624
|
-
asset = create_asset(
|
|
625
|
-
@state.uploads[asset['id']] = { 'queue' =>
|
|
985
|
+
asset = create_asset(fields, queue, data: data, open: !final)
|
|
986
|
+
@state.uploads[asset['id']] = { 'queue' => queue, 'total' => total } unless final
|
|
626
987
|
asset
|
|
627
988
|
end
|
|
628
989
|
|
|
@@ -658,8 +1019,26 @@ module CalmLens
|
|
|
658
1019
|
# Without `moderate`, queued work stays pending until completed, which allows it.
|
|
659
1020
|
def default_moderation(trigger, queued) = queued && trigger != 'complete' ? 'pending' : 'allowed'
|
|
660
1021
|
|
|
661
|
-
# Applies `decision` to `asset` with its report.
|
|
662
|
-
|
|
1022
|
+
# Applies `decision` to `asset` with its report. The mock reads no text, QR codes, people or
|
|
1023
|
+
# faces in an image, so a decided image reports that the features its upload ran found nothing:
|
|
1024
|
+
# OCR and person and face detection when turned on, link detection unless turned off.
|
|
1025
|
+
def decided(asset, decision)
|
|
1026
|
+
apply_outcome(asset, decision.status, 'policy', decision.report)
|
|
1027
|
+
classification = asset['classification']
|
|
1028
|
+
return asset unless classification['status'] == 'completed' && asset['type'] == 'image'
|
|
1029
|
+
|
|
1030
|
+
processing = @processing_options[asset['id']]
|
|
1031
|
+
features = processing.is_a?(Hash) && processing['features'].is_a?(Hash) ? processing['features'] : {}
|
|
1032
|
+
setting = ->(name) { features[name].is_a?(Hash) ? features[name]['enabled'] : nil }
|
|
1033
|
+
classification['ocr'] = { 'status' => 'evaluated', 'textRegions' => [] } if setting.call('ocr') == true
|
|
1034
|
+
unless setting.call('linkDetection') == false
|
|
1035
|
+
classification['linkDetection'] = { 'status' => 'evaluated', 'qrCodes' => [], 'links' => [] }
|
|
1036
|
+
end
|
|
1037
|
+
{ 'personDetection' => 'people', 'faceDetection' => 'faces' }.each do |feature, list|
|
|
1038
|
+
classification[feature] = { 'status' => 'evaluated', list => [] } if setting.call(feature) == true
|
|
1039
|
+
end
|
|
1040
|
+
asset
|
|
1041
|
+
end
|
|
663
1042
|
|
|
664
1043
|
# Asks `moderate` (unless `decision` is given) and settles a processing asset: a decision and
|
|
665
1044
|
# its events, or nothing for `pending`.
|
|
@@ -681,6 +1060,7 @@ module CalmLens
|
|
|
681
1060
|
|
|
682
1061
|
if decision.status == 'failed'
|
|
683
1062
|
return add_event('type' => 'asset.failed', 'severity' => 'high', 'assetId' => item['id'], 'asset' => snapshot(item),
|
|
1063
|
+
'actor' => deep_copy(AUTOMATED_ACTOR),
|
|
684
1064
|
'facts' => [asset_name_fact(item), { 'key' => 'asset-type', 'value' => item['type'] },
|
|
685
1065
|
{ 'key' => 'previous-status', 'value' => 'processing' },
|
|
686
1066
|
{ 'key' => 'failure-reason',
|
|
@@ -689,40 +1069,38 @@ module CalmLens
|
|
|
689
1069
|
|
|
690
1070
|
flagged = decision.report['flaggedCategories']
|
|
691
1071
|
add_event('type' => 'asset.classified', 'severity' => 'low', 'assetId' => item['id'], 'asset' => snapshot(item),
|
|
1072
|
+
'actor' => deep_copy(AUTOMATED_ACTOR),
|
|
692
1073
|
'facts' => [{ 'key' => 'flagged-categories', 'value' => flagged },
|
|
693
1074
|
{ 'key' => 'moderation-status', 'value' => decision.status },
|
|
694
1075
|
{ 'key' => 'asset-type', 'value' => item['type'] }, asset_name_fact(item)])
|
|
695
1076
|
# Like the event catalog: only allowing and limiting are low severity.
|
|
696
1077
|
add_event('type' => "asset.#{decision.status}", 'severity' => %w[allowed limited].include?(decision.status) ? 'low' : 'high',
|
|
697
|
-
'assetId' => item['id'], 'asset' => snapshot(item),
|
|
698
|
-
'facts' => [{ 'key' => '
|
|
1078
|
+
'assetId' => item['id'], 'asset' => snapshot(item), 'actor' => deep_copy(AUTOMATED_ACTOR),
|
|
1079
|
+
'facts' => [{ 'key' => 'flagged-categories', 'value' => flagged },
|
|
699
1080
|
{ 'key' => 'asset-type', 'value' => item['type'] }, asset_name_fact(item)])
|
|
700
1081
|
end
|
|
701
1082
|
|
|
702
1083
|
# Moves an asset to an outcome like the pipeline would: a decision finishes processing, `purged`
|
|
703
1084
|
# deletes the content (with `purge_reason`, `policy` for the pipeline), and finished processing
|
|
704
|
-
# removes content kept only temporarily. A
|
|
705
|
-
# placeholder, which
|
|
1085
|
+
# removes content kept only temporarily. A decided asset gets its rendered file and preview
|
|
1086
|
+
# (with a ThumbHash for its placeholder), which go with its content. `report` is a decision's
|
|
1087
|
+
# categories, score and policy.
|
|
706
1088
|
def apply_outcome(asset, outcome, purge_reason = 'policy', report = nil)
|
|
707
1089
|
processing, moderation = AssetRecord.statuses(outcome)
|
|
708
1090
|
now = @now.call
|
|
709
1091
|
decided = processing == 'completed'
|
|
710
1092
|
purge = moderation == 'purged' && asset['content']['status'] != 'removed'
|
|
711
1093
|
remove = purge || (decided && asset['content']['status'] == 'temporary')
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
elsif moderation == 'purged'
|
|
719
|
-
image = without_placeholder(image)
|
|
720
|
-
end
|
|
721
|
-
asset['content']['image'] = image
|
|
1094
|
+
# Like the API, an asset whose content is held has its rendered file and preview, completed by
|
|
1095
|
+
# its decision, and removing content (when processing finishes or by purging) removes them
|
|
1096
|
+
# with it, so no placeholder outlives the content. A safety hold withholds them.
|
|
1097
|
+
content = asset['content']
|
|
1098
|
+
held = content['status'] != 'removed' && !purge && !content['safetyHold']
|
|
1099
|
+
content.merge!(held ? rendering(asset, decided) : NO_RENDERING)
|
|
722
1100
|
if remove
|
|
723
1101
|
forget_content(asset)
|
|
724
|
-
|
|
725
|
-
|
|
1102
|
+
content.merge!('status' => 'removed', 'removalReason' => purge ? purge_reason : 'retention', 'updatedAt' => now,
|
|
1103
|
+
'extractedText' => nil, 'metadata' => nil, **NO_RENDERING)
|
|
726
1104
|
end
|
|
727
1105
|
asset['upload'].merge!('status' => 'completed', 'updatedAt' => now)
|
|
728
1106
|
update_classification(asset['classification'], processing, moderation, now)
|
|
@@ -744,8 +1122,49 @@ module CalmLens
|
|
|
744
1122
|
item['context'] = nil
|
|
745
1123
|
end
|
|
746
1124
|
|
|
747
|
-
#
|
|
748
|
-
def
|
|
1125
|
+
# Whether browsers play an audio or video upload's format as it is, so it is served as stored.
|
|
1126
|
+
def browser_safe_media?(format) = BROWSER_SAFE_MEDIA_FORMATS.include?(mime_type(format))
|
|
1127
|
+
|
|
1128
|
+
# A MIME type without parameters, in lower case.
|
|
1129
|
+
def mime_type(format) = format.to_s.downcase.split(';', 2).first.to_s.strip
|
|
1130
|
+
|
|
1131
|
+
# The rendered file and preview the API reports for an asset with its content held (Asset.ts,
|
|
1132
|
+
# getAssetRendered and getAssetPreview). Once uploaded, an image renders as itself with a preview
|
|
1133
|
+
# that has no ThumbHash yet, and a PDF and browser-safe audio and video render as uploaded. Once
|
|
1134
|
+
# processed, a website renders as its screenshot; a document or presentation as a PDF, with its
|
|
1135
|
+
# first page as the preview; video and audio as a browser-safe file (MP4 or M4A unless browsers
|
|
1136
|
+
# play the upload), audio with a waveform preview and video with no poster; and every preview has
|
|
1137
|
+
# its ThumbHash. Text, spreadsheets and other files have neither. The mock decodes nothing, so
|
|
1138
|
+
# sizes are unknown, audio has no peaks, and audio and video never get an adaptive stream.
|
|
1139
|
+
def rendering(asset, processed) = { **rendered_file(asset, processed), 'hls' => nil, 'hlsStatus' => nil }
|
|
1140
|
+
|
|
1141
|
+
# The rendered file and preview of {#rendering}.
|
|
1142
|
+
def rendered_file(asset, processed)
|
|
1143
|
+
format = asset['content']['format']
|
|
1144
|
+
mime = mime_type(format)
|
|
1145
|
+
safe = browser_safe_media?(format)
|
|
1146
|
+
media = %w[video audio].include?(asset['type'])
|
|
1147
|
+
return NO_RENDERING.dup unless processed || asset['type'] == 'image' || mime == 'application/pdf' || (media && safe)
|
|
1148
|
+
|
|
1149
|
+
unsized = { 'width' => nil, 'height' => nil }
|
|
1150
|
+
preview = lambda do |size = unsized|
|
|
1151
|
+
size.merge('thumbhash' => processed ? THUMBHASH : nil, 'averageColor' => processed ? '#7f7f7f' : nil)
|
|
1152
|
+
end
|
|
1153
|
+
case asset['type']
|
|
1154
|
+
when 'image' then { 'rendered' => { 'kind' => 'image', 'format' => format || 'image/png', **unsized }, 'preview' => preview.call }
|
|
1155
|
+
when 'website' then { 'rendered' => { 'kind' => 'image', 'format' => 'image/png', **unsized }, 'preview' => preview.call }
|
|
1156
|
+
when 'multimedia-document'
|
|
1157
|
+
return NO_RENDERING.dup if SPREADSHEET_MIME_TYPES.include?(mime)
|
|
1158
|
+
|
|
1159
|
+
{ 'rendered' => { 'kind' => 'pdf', 'format' => 'application/pdf', **unsized }, 'preview' => processed ? preview.call : nil }
|
|
1160
|
+
when 'video' then { 'rendered' => { 'kind' => 'video', 'format' => safe ? format : 'video/mp4', **unsized }, 'preview' => nil }
|
|
1161
|
+
when 'audio'
|
|
1162
|
+
rendered = { 'kind' => 'audio', 'format' => safe ? format : 'audio/mp4', **unsized }
|
|
1163
|
+
rendered['peaks'] = [] if processed
|
|
1164
|
+
{ 'rendered' => rendered, 'preview' => processed ? preview.call(WAVEFORM_SIZE.dup) : nil }
|
|
1165
|
+
else NO_RENDERING.dup
|
|
1166
|
+
end
|
|
1167
|
+
end
|
|
749
1168
|
|
|
750
1169
|
def asset(method, parts, query, json, body)
|
|
751
1170
|
id = parts[1]
|
|
@@ -758,9 +1177,10 @@ module CalmLens
|
|
|
758
1177
|
in ['DELETE', nil]
|
|
759
1178
|
@state.assets.delete(id)
|
|
760
1179
|
@state.content.delete(id)
|
|
1180
|
+
@processing_options.delete(id)
|
|
761
1181
|
lifecycle_event('asset.deleted', item)
|
|
762
1182
|
nil
|
|
763
|
-
in ['DELETE', 'content'] then
|
|
1183
|
+
in ['DELETE', 'content'] then remove_content(item)
|
|
764
1184
|
in ['GET', 'children'] then page(project_assets.select { |child| child['parentId'] == id }, query, id)
|
|
765
1185
|
in ['POST', 'retry'] then retry_asset(item)
|
|
766
1186
|
in ['POST', 'explain'] then explain(item, json['rerun'])
|
|
@@ -768,7 +1188,6 @@ module CalmLens
|
|
|
768
1188
|
in ['POST', 'move']
|
|
769
1189
|
check_folder(json['folderId']) if json['folderId']
|
|
770
1190
|
touch(item.merge!('folderId' => json['folderId']))
|
|
771
|
-
in ['POST', 'signed-url'] then signed_url(id, json)
|
|
772
1191
|
in ['POST', 'part'] then part(item, query, body)
|
|
773
1192
|
else route_not_found!(method, parts)
|
|
774
1193
|
end
|
|
@@ -787,21 +1206,21 @@ module CalmLens
|
|
|
787
1206
|
touch(item.merge!(json.except('id', 'projectId', 'upload', 'classification', 'content')))
|
|
788
1207
|
end
|
|
789
1208
|
|
|
790
|
-
# Like the API: the record and report stay; stored content, extracted text
|
|
791
|
-
# go. Content already removed keeps its reason, and an `asset.content-removed` event is
|
|
792
|
-
# recorded either way.
|
|
793
|
-
def
|
|
1209
|
+
# Like the API: the record and report stay; stored content, extracted text, metadata, the
|
|
1210
|
+
# rendered file and the preview go. Content already removed keeps its reason, and an `asset.content-removed` event is
|
|
1211
|
+
# recorded either way, with the `batch-id` fact of the batch that removed it, if any.
|
|
1212
|
+
def remove_content(item, batch_id = nil)
|
|
794
1213
|
forget_content(item)
|
|
795
1214
|
content = item['content']
|
|
796
1215
|
if content['status'] == 'removed'
|
|
797
|
-
content
|
|
1216
|
+
content.merge!(NO_RENDERING)
|
|
798
1217
|
else
|
|
799
1218
|
now = @now.call
|
|
800
1219
|
content.merge!('status' => 'removed', 'removalReason' => 'requested', 'updatedAt' => now, 'extractedText' => nil,
|
|
801
|
-
'
|
|
1220
|
+
'metadata' => nil, **NO_RENDERING)
|
|
802
1221
|
item['updatedAt'] = now
|
|
803
1222
|
end
|
|
804
|
-
lifecycle_event('asset.content-removed', item)
|
|
1223
|
+
lifecycle_event('asset.content-removed', item, batch_id: batch_id)
|
|
805
1224
|
item
|
|
806
1225
|
end
|
|
807
1226
|
|
|
@@ -830,16 +1249,61 @@ module CalmLens
|
|
|
830
1249
|
fail!(ErrorCodes::ASSET_REVIEWED) unless moderation.fetch('manualReviews', []).empty?
|
|
831
1250
|
fail!(ErrorCodes::ASSET_CONTENT_UNAVAILABLE) if item['content']['status'] == 'removed'
|
|
832
1251
|
|
|
833
|
-
# Like the API:
|
|
834
|
-
|
|
835
|
-
|
|
1252
|
+
# Like the API: the request is recorded, and the asset returns as it was. It keeps its decision
|
|
1253
|
+
# while it's moderated again under the current policy.
|
|
1254
|
+
add_event('type' => 'asset.rerun-requested', 'severity' => 'low', 'assetId' => item['id'],
|
|
1255
|
+
'asset' => snapshot(item), 'actor' => deep_copy(API_KEY_ACTOR),
|
|
1256
|
+
'facts' => [asset_name_fact(item), { 'key' => 'asset-type', 'value' => item['type'] },
|
|
1257
|
+
{ 'key' => 'moderation-status', 'value' => moderation['status'] }])
|
|
1258
|
+
response = deep_copy(item)
|
|
1259
|
+
settle_rerun(item, 'rerun')
|
|
1260
|
+
response
|
|
836
1261
|
end
|
|
837
1262
|
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
1263
|
+
# Applies a re-run's decision to a decided asset like the pipeline: `pending` stays queued for
|
|
1264
|
+
# `complete_processing`, `failed` keeps the asset's decision, and any other replaces it with the
|
|
1265
|
+
# pipeline's events.
|
|
1266
|
+
def settle_rerun(item, trigger, decision = nil)
|
|
1267
|
+
decision ||= decision_for(item, trigger, true)
|
|
1268
|
+
@reruns.delete(item['id'])
|
|
1269
|
+
if decision.status == 'pending'
|
|
1270
|
+
@reruns[item['id']] = true
|
|
1271
|
+
elsif decision.status != 'failed'
|
|
1272
|
+
moderate_stored(item, trigger, decision)
|
|
1273
|
+
end
|
|
1274
|
+
end
|
|
1275
|
+
|
|
1276
|
+
# Signs an asset URL like the API (`POST /projects/:projectId/signed-asset-urls`): a grant to one
|
|
1277
|
+
# asset, by ID or external ID, until it expires, serving the original only when asked. Only the
|
|
1278
|
+
# mock's CDN accepts the grant.
|
|
1279
|
+
def sign_asset_url(json)
|
|
1280
|
+
# Like the API's schema, none of these fields takes null.
|
|
1281
|
+
%w[assetId externalId expiresIn original].each do |key|
|
|
1282
|
+
invalid!('validation/invalid-type', key) if json.key?(key) && json[key].nil?
|
|
1283
|
+
end
|
|
1284
|
+
if json.key?('assetId') == json.key?('externalId')
|
|
1285
|
+
message = 'Send either assetId or externalId'
|
|
1286
|
+
raise Failure.new(ErrorCodes::INVALID_REQUEST, message,
|
|
1287
|
+
details: { 'issues' => [{ 'path' => '', 'message' => message, 'code' => 'custom' }] }, status: 400)
|
|
1288
|
+
end
|
|
1289
|
+
expires = json.fetch('expiresIn', CDN::DEFAULT_SIGNED_URL_EXPIRES_IN)
|
|
1290
|
+
invalid!('validation/invalid-type', 'expiresIn') unless expires.is_a?(Integer) || (expires.is_a?(Float) && expires == expires.floor)
|
|
1291
|
+
invalid!('validation/too-small', 'expiresIn') if expires < 1
|
|
1292
|
+
invalid!('validation/too-big', 'expiresIn') if expires > CDN::MAX_SIGNED_URL_EXPIRES_IN
|
|
1293
|
+
original = json.fetch('original', false)
|
|
1294
|
+
invalid!('validation/invalid-type', 'original') unless [true, false].include?(original)
|
|
1295
|
+
|
|
1296
|
+
item = if json.key?('externalId')
|
|
1297
|
+
project_assets.find { |asset| asset['externalId'] == json['externalId'] }
|
|
1298
|
+
else
|
|
1299
|
+
own_asset(json['assetId'].to_s)
|
|
1300
|
+
end
|
|
1301
|
+
fail!(ErrorCodes::ASSET_NOT_FOUND) unless item
|
|
1302
|
+
fail!(ErrorCodes::ASSET_HELD) if item['content']['safetyHold']
|
|
1303
|
+
|
|
1304
|
+
expires_at = @now.call + (expires.to_i * 1000)
|
|
1305
|
+
{ 'url' => "#{@cdn_base_url}/assets/#{Internal.component(@project_id)}/#{MockGrant.encode(item['id'], original, expires_at)}",
|
|
1306
|
+
'expiresAt' => expires_at }
|
|
843
1307
|
end
|
|
844
1308
|
|
|
845
1309
|
# In the API's order: the upload must be open, parts arrive in order, image totals and the
|
|
@@ -857,7 +1321,7 @@ module CalmLens
|
|
|
857
1321
|
received = item['content']['sizeInBytes'].to_i + size
|
|
858
1322
|
upload = @state.uploads[item['id']] || {}
|
|
859
1323
|
code = image_size_error(item['type'], received, final) || total_size_error(upload['total'], received, final)
|
|
860
|
-
|
|
1324
|
+
size_failure!(code) if code
|
|
861
1325
|
check_part_size(size, final)
|
|
862
1326
|
|
|
863
1327
|
stored = (@state.content[item['id']] ||= { 'type' => item['content']['format'].to_s, 'parts' => {} })
|
|
@@ -870,7 +1334,8 @@ module CalmLens
|
|
|
870
1334
|
return item unless final
|
|
871
1335
|
|
|
872
1336
|
@state.uploads.delete(item['id'])
|
|
873
|
-
|
|
1337
|
+
# `wait=false` queues the final part; without `wait`, the upload's saved choice applies.
|
|
1338
|
+
queue = query.key?('wait') ? query['wait'] == false : upload.fetch('queue', false)
|
|
874
1339
|
apply_outcome(item, 'pending')
|
|
875
1340
|
upload_completed(item, queue)
|
|
876
1341
|
end
|
|
@@ -933,28 +1398,37 @@ module CalmLens
|
|
|
933
1398
|
end
|
|
934
1399
|
|
|
935
1400
|
# Policies: the project starts on the organisation's own General preset policy, following
|
|
936
|
-
# automatic updates. The mock
|
|
937
|
-
#
|
|
938
|
-
|
|
1401
|
+
# automatic updates (mode `live`). The mock's presets have one release, version 1, with
|
|
1402
|
+
# PRESET_SETTINGS. A stored policy is the API's shape without `projectIds`, with its preset's
|
|
1403
|
+
# `id`, `version` and `own`, whether it follows its preset (`followsPreset`) and its reset
|
|
1404
|
+
# `baseline`.
|
|
1405
|
+
def new_policy(name, settings, description: '', follows_preset: false, preset: nil)
|
|
939
1406
|
now = @now.call
|
|
940
1407
|
item = { 'id' => @create_id.call('pol'), 'name' => name, 'description' => description, 'settings' => settings,
|
|
941
|
-
'revision' => 1, '
|
|
1408
|
+
'revision' => 1, 'followsPreset' => follows_preset, 'preset' => preset,
|
|
942
1409
|
'baseline' => deep_copy(settings), 'createdAt' => now, 'updatedAt' => now }
|
|
943
1410
|
@policies[item['id']] = item
|
|
944
1411
|
end
|
|
945
1412
|
|
|
1413
|
+
# The project's policy, after creating the organisation's own General policy on first use.
|
|
946
1414
|
def ensure_policies
|
|
947
|
-
return if @project_policy_id
|
|
1415
|
+
return @project_policy_id if @project_policy_id
|
|
948
1416
|
|
|
949
1417
|
preset = { 'id' => 'general-community', 'version' => 1, 'own' => true }
|
|
950
|
-
@project_policy_id = new_policy(PRESET_NAMES['general-community'],
|
|
951
|
-
|
|
1418
|
+
@project_policy_id = new_policy(PRESET_NAMES['general-community'], deep_copy(PRESET_SETTINGS),
|
|
1419
|
+
follows_preset: true, preset: preset)['id']
|
|
952
1420
|
end
|
|
953
1421
|
|
|
954
|
-
# A stored policy as the API shows it
|
|
1422
|
+
# A stored policy as the API shows it: an own preset policy is `live` while it follows its
|
|
1423
|
+
# preset and `pinned` otherwise; any other preset policy is a `copy`.
|
|
955
1424
|
def show_policy(item)
|
|
956
|
-
shown = item.except('baseline')
|
|
957
|
-
|
|
1425
|
+
shown = item.except('baseline', 'followsPreset')
|
|
1426
|
+
if (preset = item['preset'])
|
|
1427
|
+
mode = if preset['own'] then item['followsPreset'] ? 'live' : 'pinned'
|
|
1428
|
+
else 'copy'
|
|
1429
|
+
end
|
|
1430
|
+
shown['preset'] = { 'id' => preset['id'], 'version' => preset['version'], 'latestVersion' => 1, 'mode' => mode }
|
|
1431
|
+
end
|
|
958
1432
|
shown.merge('projectIds' => item['id'] == @project_policy_id ? [@project_id] : [])
|
|
959
1433
|
end
|
|
960
1434
|
|
|
@@ -963,9 +1437,39 @@ module CalmLens
|
|
|
963
1437
|
@policies[id] || fail!(ErrorCodes::POLICY_NOT_FOUND)
|
|
964
1438
|
end
|
|
965
1439
|
|
|
966
|
-
|
|
1440
|
+
# `/policies` and `/policies/:id[/action]`, the API key's organisation's policies.
|
|
1441
|
+
def policies(method, parts, query, json)
|
|
1442
|
+
_, id, *rest = parts
|
|
1443
|
+
if id.nil?
|
|
1444
|
+
return policy_page(query) if method == 'GET'
|
|
1445
|
+
return create_policy(json) if method == 'POST'
|
|
1446
|
+
|
|
1447
|
+
route_not_found!(method, parts)
|
|
1448
|
+
end
|
|
1449
|
+
policy(method, id, rest, json)
|
|
1450
|
+
end
|
|
1451
|
+
|
|
1452
|
+
# A page of the organisation's policies, oldest first unless `sort` says otherwise.
|
|
1453
|
+
def policy_page(query)
|
|
967
1454
|
ensure_policies
|
|
968
|
-
|
|
1455
|
+
page(@policies.values.map { |item| show_policy(item) }, query, JSON.generate(['policies', @project_id]),
|
|
1456
|
+
default: { 'field' => 'createdAt', 'direction' => 'asc' })
|
|
1457
|
+
end
|
|
1458
|
+
|
|
1459
|
+
# Refuses settings the API refuses (MODERATION_POLICY_SETTINGS_SCHEMA): every category, each
|
|
1460
|
+
# with an `action` and its `allowances` and `restrictions` lists, and no other category.
|
|
1461
|
+
def check_policy_settings(settings)
|
|
1462
|
+
categories = settings.is_a?(Hash) ? settings['categorySettings'] : nil
|
|
1463
|
+
invalid!('validation/invalid-type', 'settings.categorySettings') unless categories.is_a?(Hash)
|
|
1464
|
+
MockClassification::CATEGORIES.each do |category|
|
|
1465
|
+
path = "settings.categorySettings.#{category}"
|
|
1466
|
+
value = categories[category]
|
|
1467
|
+
invalid!('validation/invalid-type', path) unless value.is_a?(Hash)
|
|
1468
|
+
%w[allowances restrictions].each { |key| invalid!('validation/invalid-type', "#{path}.#{key}") unless value[key].is_a?(Array) }
|
|
1469
|
+
invalid!('validation/invalid-value', "#{path}.action") unless value['action'].is_a?(String)
|
|
1470
|
+
end
|
|
1471
|
+
unknown = categories.keys.find { |key| !MockClassification::CATEGORIES.include?(key) }
|
|
1472
|
+
invalid!('validation/unknown-field', "settings.categorySettings.#{unknown}") if unknown
|
|
969
1473
|
end
|
|
970
1474
|
|
|
971
1475
|
def create_policy(json)
|
|
@@ -975,10 +1479,11 @@ module CalmLens
|
|
|
975
1479
|
if settings.nil? == preset_id.nil? || (preset_id && !PRESET_NAMES.key?(preset_id)) || (preset_id.nil? && !named)
|
|
976
1480
|
fail!(ErrorCodes::INVALID_VALUE, 'Send settings with a name, or presetId')
|
|
977
1481
|
end
|
|
1482
|
+
check_policy_settings(settings) unless preset_id
|
|
978
1483
|
description = json['description'].is_a?(String) ? json['description'] : ''
|
|
979
1484
|
item = if preset_id
|
|
980
1485
|
origin = { 'id' => preset_id, 'version' => 1, 'own' => false }
|
|
981
|
-
new_policy(named ? name.strip : PRESET_NAMES[preset_id],
|
|
1486
|
+
new_policy(named ? name.strip : PRESET_NAMES[preset_id], deep_copy(PRESET_SETTINGS),
|
|
982
1487
|
description: description, preset: origin)
|
|
983
1488
|
else
|
|
984
1489
|
new_policy(name.strip, settings, description: description)
|
|
@@ -1010,26 +1515,29 @@ module CalmLens
|
|
|
1010
1515
|
fail!(ErrorCodes::POLICY_IN_USE) if item['id'] == @project_policy_id
|
|
1011
1516
|
@policies.delete(id)
|
|
1012
1517
|
{}
|
|
1013
|
-
in ['POST', ['
|
|
1518
|
+
in ['POST', ['reset']] then restore_policy(item, 'reset', json)
|
|
1519
|
+
in ['POST', ['upgrade']] then restore_policy(item, 'upgrade', json)
|
|
1014
1520
|
in ['PUT', ['automatic-updates']] then set_policy_automatic_updates(item, json)
|
|
1015
1521
|
else route_not_found!(method, ['policies', id, *rest])
|
|
1016
1522
|
end
|
|
1017
1523
|
end
|
|
1018
1524
|
|
|
1019
1525
|
def update_policy(item, json)
|
|
1526
|
+
check_policy_settings(json['settings']) if json.key?('settings')
|
|
1020
1527
|
values = %w[name description].select { |key| json[key].is_a?(String) }
|
|
1021
1528
|
.to_h { |key| [key, key == 'name' ? json[key].strip : json[key]] }
|
|
1022
1529
|
values['settings'] = json['settings'] if json.key?('settings')
|
|
1023
|
-
# Saving a policy
|
|
1024
|
-
save_policy(item, json['revision'], values.merge('
|
|
1530
|
+
# Saving a `live` policy pins it.
|
|
1531
|
+
save_policy(item, json['revision'], values.merge('followsPreset' => false))
|
|
1025
1532
|
end
|
|
1026
1533
|
|
|
1027
|
-
|
|
1534
|
+
# Resets a `pinned` or `copy` policy to its preset release, or upgrades it to the latest.
|
|
1535
|
+
def restore_policy(item, operation, json)
|
|
1028
1536
|
preset = item['preset']
|
|
1029
|
-
fail!(ErrorCodes::POLICY_NO_PRESET) if preset.nil? || item['
|
|
1537
|
+
fail!(ErrorCodes::POLICY_NO_PRESET) if preset.nil? || item['followsPreset']
|
|
1030
1538
|
fail!(ErrorCodes::POLICY_PRESET_VERSION_CONFLICT) unless json['presetVersion'] == preset['version']
|
|
1031
|
-
# The mock's presets have one release, so an
|
|
1032
|
-
fail!(ErrorCodes::POLICY_PRESET_UPDATE_UNAVAILABLE) if
|
|
1539
|
+
# The mock's presets have one release, so an upgrade never has a newer one to apply.
|
|
1540
|
+
fail!(ErrorCodes::POLICY_PRESET_UPDATE_UNAVAILABLE) if operation == 'upgrade'
|
|
1033
1541
|
|
|
1034
1542
|
save_policy(item, json['revision'], 'settings' => deep_copy(item['baseline']))
|
|
1035
1543
|
end
|
|
@@ -1037,11 +1545,11 @@ module CalmLens
|
|
|
1037
1545
|
def set_policy_automatic_updates(item, json)
|
|
1038
1546
|
enabled = json['enabled'] == true
|
|
1039
1547
|
preset = item['preset']
|
|
1040
|
-
fail!(ErrorCodes::POLICY_AUTOMATIC_UPDATES_UNAVAILABLE) if preset.nil? || !preset['own'] || item['
|
|
1548
|
+
fail!(ErrorCodes::POLICY_AUTOMATIC_UPDATES_UNAVAILABLE) if preset.nil? || !preset['own'] || item['followsPreset'] == enabled
|
|
1041
1549
|
fail!(ErrorCodes::POLICY_REVISION_CONFLICT) unless json['revision'] == item['revision']
|
|
1042
1550
|
fail!(ErrorCodes::POLICY_PRESET_VERSION_CONFLICT) unless json['presetVersion'] == 1
|
|
1043
1551
|
|
|
1044
|
-
values = { '
|
|
1552
|
+
values = { 'followsPreset' => enabled }
|
|
1045
1553
|
values.merge!('name' => PRESET_NAMES[preset['id']], 'description' => '', 'settings' => deep_copy(item['baseline'])) if enabled
|
|
1046
1554
|
save_policy(item, json['revision'], values)
|
|
1047
1555
|
end
|
|
@@ -1103,10 +1611,10 @@ module CalmLens
|
|
|
1103
1611
|
if asset_id
|
|
1104
1612
|
asset = @state.assets[asset_id]
|
|
1105
1613
|
fail!(ErrorCodes::ASSET_NOT_FOUND) unless asset && asset['projectId'] == @project_id
|
|
1106
|
-
|
|
1107
|
-
fail!(ErrorCodes::HASH_LIST_ASSET_NOT_HASHED) unless
|
|
1108
|
-
hash =
|
|
1109
|
-
quality =
|
|
1614
|
+
content = asset['content']
|
|
1615
|
+
fail!(ErrorCodes::HASH_LIST_ASSET_NOT_HASHED) unless content['perceptualHash']
|
|
1616
|
+
hash = content['perceptualHash']
|
|
1617
|
+
quality = content['perceptualHashQuality']
|
|
1110
1618
|
end
|
|
1111
1619
|
fail!(ErrorCodes::INVALID_UPLOAD_PAYLOAD, 'Use a PDQ hash of 64 hexadecimal characters') unless hash.match?(/\A\h{64}\z/)
|
|
1112
1620
|
|
|
@@ -1115,7 +1623,7 @@ module CalmLens
|
|
|
1115
1623
|
|
|
1116
1624
|
entry = { 'id' => @create_id.call('hle'), 'projectId' => @project_id, 'hashListId' => list['id'], 'hash' => hash,
|
|
1117
1625
|
'quality' => quality, 'label' => json['label'].is_a?(String) ? json['label'] : nil,
|
|
1118
|
-
'
|
|
1626
|
+
'assetId' => asset_id, 'createdAt' => @now.call }
|
|
1119
1627
|
@state.hash_list_entries[entry['id']] = entry
|
|
1120
1628
|
respond(201, deep_copy(entry))
|
|
1121
1629
|
end
|
|
@@ -1129,17 +1637,43 @@ module CalmLens
|
|
|
1129
1637
|
{}
|
|
1130
1638
|
end
|
|
1131
1639
|
|
|
1132
|
-
#
|
|
1133
|
-
#
|
|
1134
|
-
#
|
|
1135
|
-
def
|
|
1640
|
+
# An asset's findings timeline, as the API lists it (publicGetAssetTimeline.ts): refused while a
|
|
1641
|
+
# safety hold is on. The mock analyses no audio or video, so it keeps no segments: every
|
|
1642
|
+
# timeline is empty, and no cursor continues one.
|
|
1643
|
+
def timeline(id, query)
|
|
1644
|
+
item = own_asset(id)
|
|
1645
|
+
fail!(ErrorCodes::ASSET_NOT_FOUND) unless item
|
|
1646
|
+
check_range(query.fetch('pageSize', DEFAULT_PAGE_SIZE), 'pageSize', MAX_PAGE_SIZE)
|
|
1647
|
+
fail!(ErrorCodes::ASSET_HELD, details: { 'hold' => 'safety' }) if item['content']['safetyHold']
|
|
1648
|
+
fail!(ErrorCodes::INVALID_CURSOR) if query.key?('cursor')
|
|
1649
|
+
|
|
1650
|
+
{ 'items' => [], 'nextCursor' => nil, 'hasMore' => false }
|
|
1651
|
+
end
|
|
1652
|
+
|
|
1653
|
+
# The page size and sort, refused like the API's query schema: `sort` is comma-separated
|
|
1654
|
+
# fields from `fields`, each descending when prefixed with `-`, none repeated and at most `max`
|
|
1655
|
+
# (`validation/invalid-value` at `sort` otherwise). Without one, `default` applies.
|
|
1656
|
+
def page_query(query, fields = %w[createdAt id], default = { 'field' => 'createdAt', 'direction' => 'desc' }, max: 2)
|
|
1136
1657
|
size = check_range(query.fetch('pageSize', DEFAULT_PAGE_SIZE), 'pageSize', MAX_PAGE_SIZE)
|
|
1137
|
-
|
|
1138
|
-
|
|
1139
|
-
|
|
1140
|
-
|
|
1141
|
-
|
|
1658
|
+
[size, query.key?('sort') ? parse_sort(query['sort'], fields, max) : [default.dup]]
|
|
1659
|
+
end
|
|
1660
|
+
|
|
1661
|
+
def parse_sort(value, fields, max)
|
|
1662
|
+
sort = value.to_s.split(',', -1).map(&:strip).map do |part|
|
|
1663
|
+
field = part.delete_prefix('-')
|
|
1664
|
+
invalid!('validation/invalid-value', 'sort') unless part.match?(/\A-?[A-Za-z][A-Za-z0-9.]*\z/) && fields.include?(field)
|
|
1665
|
+
{ 'field' => field, 'direction' => part.start_with?('-') ? 'desc' : 'asc' }
|
|
1142
1666
|
end
|
|
1667
|
+
fields_sent = sort.map { |rule| rule['field'] }
|
|
1668
|
+
invalid!('validation/invalid-value', 'sort') if sort.empty? || fields_sent.uniq.size != fields_sent.size || sort.size > max
|
|
1669
|
+
sort
|
|
1670
|
+
end
|
|
1671
|
+
|
|
1672
|
+
# One keyset page like the API's: sorted by `sort` (`createdAt` descending by default; an `id`
|
|
1673
|
+
# tie-breaker is added), starting after the cursor's values. The cursor is tied to `scope` (the
|
|
1674
|
+
# filter, folder or parent) and the sort, so it only continues the query it came from.
|
|
1675
|
+
def page(items, query, scope, fields: %w[createdAt id], max: 2, default: { 'field' => 'createdAt', 'direction' => 'desc' })
|
|
1676
|
+
size, ordering = page_query(query, fields, default, max: max)
|
|
1143
1677
|
ordering << { 'field' => 'id', 'direction' => ordering.first['direction'] } unless ordering.any? { |o| o['field'] == 'id' }
|
|
1144
1678
|
signature = JSON.generate([scope, ordering])
|
|
1145
1679
|
sorted = items.sort { |a, b| compare_items(a, b, ordering) }
|
|
@@ -1149,12 +1683,8 @@ module CalmLens
|
|
|
1149
1683
|
end
|
|
1150
1684
|
selected = sorted.first(size)
|
|
1151
1685
|
more = sorted.size > size
|
|
1152
|
-
cursor = if more
|
|
1153
|
-
|
|
1154
|
-
selected.last[o['field']]
|
|
1155
|
-
end })].pack('m0')
|
|
1156
|
-
end
|
|
1157
|
-
{ 'items' => selected, 'pageSize' => size, 'hasMore' => more, 'ordering' => ordering, 'nextCursor' => cursor }
|
|
1686
|
+
cursor = ([JSON.generate({ 'scope' => signature, 'values' => ordering.map { |o| selected.last[o['field']] } })].pack('m0') if more)
|
|
1687
|
+
{ 'items' => selected, 'hasMore' => more, 'nextCursor' => cursor }
|
|
1158
1688
|
end
|
|
1159
1689
|
|
|
1160
1690
|
def cursor_values(cursor, signature, ordering)
|
|
@@ -1177,15 +1707,6 @@ module CalmLens
|
|
|
1177
1707
|
false
|
|
1178
1708
|
end
|
|
1179
1709
|
|
|
1180
|
-
def ordering(query, default)
|
|
1181
|
-
entries = query.keys.filter_map { |key| key.match(/\Aordering\[(\d+)\]\[(field|direction)\]\z/) }
|
|
1182
|
-
return [default&.dup].compact if entries.empty?
|
|
1183
|
-
|
|
1184
|
-
entries.group_by { |match| Integer(match[1], 10) }.sort.map do |_, matches|
|
|
1185
|
-
matches.to_h { |match| [match[2], query[match[0]]] }
|
|
1186
|
-
end
|
|
1187
|
-
end
|
|
1188
|
-
|
|
1189
1710
|
def compare_items(left, right, ordering)
|
|
1190
1711
|
ordering.each do |rule|
|
|
1191
1712
|
result = compare_values(left[rule['field']], right[rule['field']])
|
|
@@ -1204,79 +1725,200 @@ module CalmLens
|
|
|
1204
1725
|
left.to_s <=> right.to_s
|
|
1205
1726
|
end
|
|
1206
1727
|
|
|
1207
|
-
# Answers like the CDN:
|
|
1208
|
-
#
|
|
1209
|
-
#
|
|
1210
|
-
#
|
|
1211
|
-
#
|
|
1212
|
-
|
|
1728
|
+
# Answers like the CDN (serveAssetResource.ts): `original` is the upload; `rendered` the image,
|
|
1729
|
+
# PDF, video or audio file `content.rendered` describes; `preview` the still `content.preview`
|
|
1730
|
+
# describes; `hls` the adaptive stream's files, which the mock never prepares; `embed` a small
|
|
1731
|
+
# page standing in for the embed page of any asset; and `placeholder` the preview's ThumbHash as a PNG, public
|
|
1732
|
+
# whatever the asset's visibility or moderation. A safety hold refuses every resource. Other resources pass delivery checks first,
|
|
1733
|
+
# which refuse everyone (content still uploading, blocked or purged, failed processing, removed
|
|
1734
|
+
# content), then an age-restricted asset needs `viewerClaims.ageConfirmed=true` whatever the key,
|
|
1735
|
+
# and a private asset needs an API key with `assets:download` for its folder. The mock converts
|
|
1736
|
+
# nothing: an upload stands in for its rendered file, an image for its own preview, and the
|
|
1737
|
+
# placeholder PNG for a screenshot, page or waveform.
|
|
1738
|
+
def serve(method, relative, headers)
|
|
1213
1739
|
path, query = relative.split('#', 2).first.split('?', 2)
|
|
1214
1740
|
raw_segments = path.split('/').reject(&:empty?)
|
|
1215
|
-
|
|
1216
|
-
|
|
1217
|
-
|
|
1218
|
-
|
|
1219
|
-
|
|
1220
|
-
|
|
1221
|
-
|
|
1222
|
-
|
|
1223
|
-
|
|
1224
|
-
|
|
1225
|
-
|
|
1226
|
-
|
|
1741
|
+
root, project, _, resource = raw_segments.map { |part| URI.decode_www_form_component(part) }
|
|
1742
|
+
selected, options = selector(resource, raw_segments.drop(4))
|
|
1743
|
+
return invalid_transform if selected.nil?
|
|
1744
|
+
return asset_not_found unless root == 'assets' && project == @project_id && raw_segments[2]
|
|
1745
|
+
|
|
1746
|
+
grant, item = resolve_asset(raw_segments[2])
|
|
1747
|
+
return grant if grant.is_a?(Response)
|
|
1748
|
+
return asset_not_found unless item && RESOURCES.include?(resource)
|
|
1749
|
+
if resource == 'original' && grant && !grant[:original]
|
|
1750
|
+
return error(ErrorCodes::SIGNED_URL_ORIGINAL_NOT_GRANTED, "This signed asset URL doesn't grant the original file", status: 403)
|
|
1751
|
+
end
|
|
1752
|
+
|
|
1753
|
+
params = URI.decode_www_form(query.to_s).to_h
|
|
1754
|
+
# A grant authorises a private asset without the API key.
|
|
1755
|
+
authorised = !grant.nil? || can_download?(item['folderId'], headers)
|
|
1756
|
+
return info(method, item, authorised, params) if resource == 'info'
|
|
1757
|
+
return not_available if item['content']['safetyHold']
|
|
1227
1758
|
return placeholder(method, item) if resource == 'placeholder'
|
|
1228
1759
|
|
|
1229
|
-
deliver(method, item,
|
|
1760
|
+
deliver(method, item, resource, params, selected ? options || {} : nil, authorised)
|
|
1761
|
+
end
|
|
1762
|
+
|
|
1763
|
+
# Whether the path after the resource selects a lens or transform, and its options; nil when the
|
|
1764
|
+
# CDN refuses it. Like the CDN, the adaptive stream's files sit beneath `/hls/` (its master
|
|
1765
|
+
# playlist is `/hls/index.m3u8`). Otherwise the selector after the resource is read
|
|
1766
|
+
# (AssetTransformParser.ts); the original, the embed page and the info take none.
|
|
1767
|
+
def selector(resource, path)
|
|
1768
|
+
lens, options = TransformRules.parse_selector(resource == 'hls' ? '' : path.join('/'))
|
|
1769
|
+
selected = !(lens || options).nil?
|
|
1770
|
+
%w[original embed info].include?(resource) && selected ? nil : [selected, options]
|
|
1771
|
+
rescue ArgumentError
|
|
1772
|
+
nil
|
|
1773
|
+
end
|
|
1774
|
+
|
|
1775
|
+
# Like the CDN (CdnUtils.ts), the asset segment is `ext:` and the URL-encoded external ID, a
|
|
1776
|
+
# grant (`sig_…`) or the asset ID. Returns `[grant, asset]`, or an error response for a grant
|
|
1777
|
+
# the mock didn't sign or that has expired.
|
|
1778
|
+
def resolve_asset(segment)
|
|
1779
|
+
if segment.start_with?('sig_')
|
|
1780
|
+
grant = MockGrant.decode(segment)
|
|
1781
|
+
return error(ErrorCodes::SIGNED_URL_INVALID, "The signed asset URL's grant is invalid", status: 403) unless grant
|
|
1782
|
+
return error(ErrorCodes::SIGNED_URL_EXPIRED, 'The signed URL has expired', status: 403) if grant[:expires_at] <= @now.call
|
|
1783
|
+
|
|
1784
|
+
[grant, project_assets.find { |asset| asset['id'] == grant[:asset_id] }]
|
|
1785
|
+
elsif segment.start_with?(CDN::EXTERNAL_ID_PREFIX)
|
|
1786
|
+
external = MockGrant.percent_decode(segment.delete_prefix(CDN::EXTERNAL_ID_PREFIX))
|
|
1787
|
+
[nil, external && !external.empty? ? project_assets.find { |asset| asset['externalId'] == external } : nil]
|
|
1788
|
+
else
|
|
1789
|
+
id = MockGrant.percent_decode(segment)
|
|
1790
|
+
[nil, project_assets.find { |asset| asset['id'] == id }]
|
|
1791
|
+
end
|
|
1792
|
+
end
|
|
1793
|
+
|
|
1794
|
+
def asset_not_found = error(ErrorCodes::ASSET_NOT_FOUND, 'Asset not found', status: 404)
|
|
1795
|
+
|
|
1796
|
+
# The display info at `/info`, like the CDN's (AssetInfo.ts): the allowlisted fields and whether
|
|
1797
|
+
# the viewer the claims describe can see the content, decided as the other resources decide it.
|
|
1798
|
+
# A private asset's info needs a grant or an API key with `assets:download`.
|
|
1799
|
+
def info(method, item, authorised, params)
|
|
1800
|
+
restricted = item['visibility'] != 'public'
|
|
1801
|
+
return asset_not_found if restricted && !authorised
|
|
1802
|
+
|
|
1803
|
+
claims = JSON.parse(params.fetch('viewerClaims', '{}'))
|
|
1804
|
+
body = { 'id' => item['id'], 'projectId' => item['projectId'], 'type' => item['type'], 'externalUrl' => item['externalUrl'],
|
|
1805
|
+
'content' => item['content'].slice(*%w[format width height durationSeconds pageCount rendered preview hls]),
|
|
1806
|
+
'availability' => availability(item, claims.is_a?(Hash) && claims['ageConfirmed'] == true) }
|
|
1807
|
+
cdn_response(method, JSON.generate(body), { 'content-type' => 'application/json',
|
|
1808
|
+
'cache-control' => "#{restricted ? 'private' : 'public'}, max-age=60",
|
|
1809
|
+
'cache-tag' => "asset-#{item['id']}" })
|
|
1810
|
+
rescue JSON::ParserError
|
|
1811
|
+
error(ErrorCodes::INVALID_VIEWER_CLAIMS, 'Invalid viewer claims', status: 400)
|
|
1230
1812
|
end
|
|
1231
1813
|
|
|
1232
|
-
#
|
|
1233
|
-
|
|
1234
|
-
def deliver(method, item, params, options)
|
|
1814
|
+
# Whether a viewer can see an asset's content now, decided as the other resources decide it.
|
|
1815
|
+
def availability(item, age_confirmed)
|
|
1235
1816
|
moderation = item['classification']['moderation']['status']
|
|
1236
|
-
|
|
1237
|
-
if stored.nil? || item['upload']['status'] == 'uploading' || %w[blocked purged].include?(moderation) ||
|
|
1817
|
+
if item['content']['safetyHold'] || item['upload']['status'] == 'uploading' || %w[blocked purged].include?(moderation) ||
|
|
1238
1818
|
item['classification']['status'] == 'failed' || item['content']['status'] == 'removed'
|
|
1239
|
-
|
|
1819
|
+
'unavailable'
|
|
1820
|
+
elsif moderation == 'age-restricted' && !age_confirmed
|
|
1821
|
+
'age-restricted'
|
|
1822
|
+
else
|
|
1823
|
+
'available'
|
|
1240
1824
|
end
|
|
1825
|
+
end
|
|
1241
1826
|
|
|
1242
|
-
|
|
1243
|
-
|
|
1244
|
-
|
|
1245
|
-
|
|
1827
|
+
# Serves a found asset's resource after the delivery, age and access checks, then whether it
|
|
1828
|
+
# has the resource and takes the selector, then the transform options' schema check.
|
|
1829
|
+
# `selector` is the selector's options (empty for a lens alone), or nil without one.
|
|
1830
|
+
def deliver(method, item, resource, params, selector, authorised)
|
|
1831
|
+
stored = @state.content[item['id']]
|
|
1832
|
+
refused = refuse_delivery(item, resource, stored, params, authorised)
|
|
1833
|
+
return refused if refused
|
|
1246
1834
|
|
|
1247
1835
|
restricted = item['visibility'] != 'public'
|
|
1248
|
-
|
|
1249
|
-
|
|
1250
|
-
|
|
1836
|
+
return embed(method, item, restricted, params) if resource == 'embed'
|
|
1837
|
+
|
|
1838
|
+
# Then whether the asset has the resource, and whether it is a still image that takes a lens
|
|
1839
|
+
# or transform: the preview always is, the rendered file only when it is an image.
|
|
1840
|
+
rendered, preview = item['content'].values_at('rendered', 'preview')
|
|
1841
|
+
return no_rendered_version if (resource == 'rendered' && !rendered) || (resource == 'preview' && !preview)
|
|
1842
|
+
# The mock prepares no stream, so it has no HLS files.
|
|
1843
|
+
return no_rendered_version if resource == 'hls'
|
|
1844
|
+
|
|
1845
|
+
still = resource == 'preview' || (resource == 'rendered' && rendered['kind'] == 'image')
|
|
1846
|
+
return invalid_transform if !still && selector
|
|
1847
|
+
|
|
1848
|
+
refused = selector ? TransformRules.check(selector).flatten : []
|
|
1849
|
+
return error(ErrorCodes::INVALID_TRANSFORM_OPTIONS, "Invalid transform options: #{refused.join(', ')}", status: 400) if refused.any?
|
|
1850
|
+
|
|
1851
|
+
bytes, type = resource_body(item, resource, stored)
|
|
1852
|
+
return not_available unless bytes
|
|
1853
|
+
|
|
1854
|
+
immutable = still || (resource == 'original' && item['type'] == 'image')
|
|
1855
|
+
cdn_response(method, bytes, viewer_delivery_headers(item, type, restricted, immutable, params))
|
|
1856
|
+
end
|
|
1857
|
+
|
|
1858
|
+
# The delivery checks, which refuse everyone, then age and access, as an error response, or nil.
|
|
1859
|
+
# Only the original needs the upload still stored.
|
|
1860
|
+
def refuse_delivery(item, resource, stored, params, authorised)
|
|
1861
|
+
moderation = item['classification']['moderation']['status']
|
|
1862
|
+
if (resource == 'original' && stored.nil?) || item['upload']['status'] == 'uploading' ||
|
|
1863
|
+
%w[blocked purged].include?(moderation) || item['classification']['status'] == 'failed' ||
|
|
1864
|
+
item['content']['status'] == 'removed'
|
|
1865
|
+
return not_available
|
|
1251
1866
|
end
|
|
1867
|
+
if moderation == 'age-restricted' && !viewer_age_confirmed?(params)
|
|
1868
|
+
return error('viewer/age-confirmation-required', 'Age confirmation required', status: 403)
|
|
1869
|
+
end
|
|
1870
|
+
return if item['visibility'] == 'public' || authorised
|
|
1871
|
+
|
|
1872
|
+
error(ErrorCodes::PERMISSION_DENIED, 'This API key needs assets:download permission for this asset’s folder.', status: 403)
|
|
1873
|
+
end
|
|
1252
1874
|
|
|
1253
|
-
|
|
1254
|
-
|
|
1875
|
+
# A resource's bytes and Content-Type: the upload, which stands in for its own rendered file
|
|
1876
|
+
# and an image's preview, or the placeholder PNG for a still the mock doesn't make. Nil bytes
|
|
1877
|
+
# when the upload is no longer stored.
|
|
1878
|
+
def resource_body(item, resource, stored)
|
|
1879
|
+
rendered = item['content']['rendered']
|
|
1880
|
+
serves_upload = resource == 'original' || (resource == 'rendered' && rendered['kind'] != 'image') || item['type'] == 'image'
|
|
1881
|
+
return [Placeholder.png(item.dig('content', 'preview', 'thumbhash') || THUMBHASH), 'image/png'] unless serves_upload
|
|
1882
|
+
return [nil, nil] unless stored
|
|
1255
1883
|
|
|
1256
|
-
type =
|
|
1257
|
-
|
|
1258
|
-
|
|
1884
|
+
type = if resource == 'rendered' then rendered['format']
|
|
1885
|
+
elsif stored['type'].to_s.empty? then 'application/octet-stream'
|
|
1886
|
+
else stored['type']
|
|
1887
|
+
end
|
|
1888
|
+
[stored['parts'].sort.map(&:last).join.b, type]
|
|
1259
1889
|
end
|
|
1260
1890
|
|
|
1261
1891
|
def viewer_age_confirmed?(params)
|
|
1262
1892
|
JSON.parse(params.fetch('viewerClaims', '{}'))['ageConfirmed'] == true
|
|
1263
1893
|
end
|
|
1264
1894
|
|
|
1265
|
-
def viewer_delivery_headers(item, type, restricted, params)
|
|
1266
|
-
headers = delivery_headers(item, type, restricted)
|
|
1895
|
+
def viewer_delivery_headers(item, type, restricted, immutable, params)
|
|
1896
|
+
headers = delivery_headers(item, type, restricted, immutable)
|
|
1267
1897
|
headers['Cache-Control'] = 'private, no-store' if params.key?('viewerClaims')
|
|
1268
1898
|
headers
|
|
1269
1899
|
end
|
|
1270
1900
|
|
|
1271
|
-
|
|
1901
|
+
# The embed page, which shows any asset that may be delivered, as its viewer for the asset's
|
|
1902
|
+
# type or a notice that the type has none; cached like other non-image content.
|
|
1903
|
+
def embed(method, item, restricted, params)
|
|
1904
|
+
cache = if params.key?('viewerClaims') then 'private, no-store'
|
|
1905
|
+
else "#{restricted ? 'private' : 'public'}, max-age=3600"
|
|
1906
|
+
end
|
|
1907
|
+
cdn_response(method, EMBED_HTML, { 'content-type' => 'text/html; charset=utf-8', 'cache-control' => cache,
|
|
1908
|
+
'cache-tag' => "asset-#{item['id']}", 'x-content-type-options' => 'nosniff' })
|
|
1909
|
+
end
|
|
1910
|
+
|
|
1911
|
+
def invalid_transform = error(ErrorCodes::INVALID_TRANSFORM, 'Invalid transform selector', status: 400)
|
|
1912
|
+
|
|
1913
|
+
def no_rendered_version = error('asset/no-rendered-version', 'This asset has no such resource', status: 404)
|
|
1272
1914
|
|
|
1273
1915
|
def not_available = error('asset/not-available', 'Asset is not available', status: 403)
|
|
1274
1916
|
|
|
1275
|
-
# Like the CDN's placeholder handler:
|
|
1917
|
+
# Like the CDN's placeholder handler: the preview's ThumbHash, public whatever the asset's
|
|
1276
1918
|
# visibility or moderation; refused for purged assets and assets without one (not yet
|
|
1277
|
-
#
|
|
1919
|
+
# processed, without a preview, or content removed).
|
|
1278
1920
|
def placeholder(method, item)
|
|
1279
|
-
thumbhash = item.dig('content', '
|
|
1921
|
+
thumbhash = item.dig('content', 'preview', 'thumbhash')
|
|
1280
1922
|
return not_available if thumbhash.nil? || item['classification']['moderation']['status'] == 'purged'
|
|
1281
1923
|
|
|
1282
1924
|
cdn_response(method, Placeholder.png(thumbhash),
|
|
@@ -1284,14 +1926,13 @@ module CalmLens
|
|
|
1284
1926
|
'cache-tag' => "asset-#{item['id']}", 'etag' => "\"#{thumbhash}\"")
|
|
1285
1927
|
end
|
|
1286
1928
|
|
|
1287
|
-
# Like the CDN:
|
|
1288
|
-
# by shared caches only when delivery is public. Active content is served as a download.
|
|
1289
|
-
def delivery_headers(item, type, restricted)
|
|
1290
|
-
|
|
1291
|
-
headers = { 'content-type' => type,
|
|
1292
|
-
'cache-control' => "#{restricted ? 'private' : 'public'}, #{image ? 'max-age=31536000, immutable' : 'max-age=3600'}",
|
|
1929
|
+
# Like the CDN: a still image's bytes are immutable for a year, other content is kept for an
|
|
1930
|
+
# hour, by shared caches only when delivery is public. Active content is served as a download.
|
|
1931
|
+
def delivery_headers(item, type, restricted, immutable)
|
|
1932
|
+
lifetime = immutable ? 'max-age=31536000, immutable' : 'max-age=3600'
|
|
1933
|
+
headers = { 'content-type' => type, 'cache-control' => "#{restricted ? 'private' : 'public'}, #{lifetime}",
|
|
1293
1934
|
'cache-tag' => "asset-#{item['id']}" }
|
|
1294
|
-
headers.merge!('etag' => "\"#{item['id']}\"", 'expires' => Time.at((@now.call / 1000) + 31_536_000).utc.httpdate) if
|
|
1935
|
+
headers.merge!('etag' => "\"#{item['id']}\"", 'expires' => Time.at((@now.call / 1000) + 31_536_000).utc.httpdate) if immutable
|
|
1295
1936
|
headers.merge!('x-content-type-options' => 'nosniff', 'content-security-policy' => "sandbox; default-src 'none'")
|
|
1296
1937
|
mime = type.downcase.split(';', 2).first.to_s.strip
|
|
1297
1938
|
headers['content-disposition'] = 'attachment' if ACTIVE_TYPES.include?(mime) || mime.end_with?('+xml')
|
|
@@ -1303,8 +1944,11 @@ module CalmLens
|
|
|
1303
1944
|
headers: headers.merge('content-length' => bytes.bytesize.to_s, 'x-request-id' => request_id))
|
|
1304
1945
|
end
|
|
1305
1946
|
|
|
1306
|
-
# Whether the mock API key
|
|
1307
|
-
|
|
1947
|
+
# Whether the request carries the mock API key and its grants give `assets:download` for
|
|
1948
|
+
# `folder_id`.
|
|
1949
|
+
def can_download?(folder_id, headers)
|
|
1950
|
+
return false unless header(headers, 'X-Api-Key') == @api_key
|
|
1951
|
+
|
|
1308
1952
|
@grants.any? do |grant|
|
|
1309
1953
|
next false unless Array(grant['actions']).include?('assets:download')
|
|
1310
1954
|
next true if grant['folderId'].nil?
|
|
@@ -1327,6 +1971,35 @@ module CalmLens
|
|
|
1327
1971
|
end
|
|
1328
1972
|
end
|
|
1329
1973
|
|
|
1974
|
+
# The mock's grant, the asset segment of a URL {CDN#sign_asset_url} signs on the mock: `sig_` and
|
|
1975
|
+
# the lowercase hex of the UTF-8 text `{assetId}.{1|0}.{expiresAt}`, its asset, original access and
|
|
1976
|
+
# expiry. Every SDK's mock writes it the same way. It is readable, not signed, as the mock has no
|
|
1977
|
+
# secret to keep.
|
|
1978
|
+
# @api private
|
|
1979
|
+
module MockGrant
|
|
1980
|
+
module_function
|
|
1981
|
+
|
|
1982
|
+
def encode(asset_id, original, expires_at)
|
|
1983
|
+
"sig_#{"#{asset_id}.#{original ? 1 : 0}.#{expires_at}".unpack1('H*')}"
|
|
1984
|
+
end
|
|
1985
|
+
|
|
1986
|
+
# `{asset_id:, original:, expires_at:}`, or nil for a segment the mock didn't sign.
|
|
1987
|
+
def decode(segment)
|
|
1988
|
+
hex = segment.delete_prefix('sig_')
|
|
1989
|
+
return unless hex.match?(/\A(?:[0-9a-f]{2})+\z/)
|
|
1990
|
+
|
|
1991
|
+
text = [hex].pack('H*').force_encoding(Encoding::UTF_8)
|
|
1992
|
+
match = text.valid_encoding? && /\A(.+)\.([01])\.(\d+)\z/m.match(text)
|
|
1993
|
+
match && { asset_id: match[1], original: match[2] == '1', expires_at: Integer(match[3], 10) }
|
|
1994
|
+
end
|
|
1995
|
+
|
|
1996
|
+
# Percent-decodes a path segment like decodeURIComponent; nil when it isn't valid UTF-8.
|
|
1997
|
+
def percent_decode(text)
|
|
1998
|
+
decoded = text.b.gsub(/%(\h\h)/) { Regexp.last_match(1).hex.chr }.force_encoding(Encoding::UTF_8)
|
|
1999
|
+
decoded if decoded.valid_encoding?
|
|
2000
|
+
end
|
|
2001
|
+
end
|
|
2002
|
+
|
|
1330
2003
|
# The processing and moderation status of each outcome.
|
|
1331
2004
|
# @api private
|
|
1332
2005
|
module AssetRecord
|
|
@@ -1345,125 +2018,5 @@ module CalmLens
|
|
|
1345
2018
|
end
|
|
1346
2019
|
end
|
|
1347
2020
|
end
|
|
1348
|
-
|
|
1349
|
-
# The RSQL subset the fake API understands: `==` and `!=` (with `*` wildcards), `=in=` and
|
|
1350
|
-
# `=out=` with a `(value,value)` list, and `=gt=`, `=ge=`, `=lt=`, `=le=`, with quoted values,
|
|
1351
|
-
# combined with `;` (and), which binds tighter than `,` (or), and grouped with `( … )`.
|
|
1352
|
-
# Anything else raises instead of being ignored, so a test never passes on an unfiltered list.
|
|
1353
|
-
# @api private
|
|
1354
|
-
module RSQL
|
|
1355
|
-
module_function
|
|
1356
|
-
|
|
1357
|
-
def filter(items, expression)
|
|
1358
|
-
return items if expression.nil? || expression.to_s.strip.empty?
|
|
1359
|
-
|
|
1360
|
-
test = parse(expression.to_s)
|
|
1361
|
-
items.select { |item| test.call(item) }
|
|
1362
|
-
end
|
|
1363
|
-
|
|
1364
|
-
# Compiles `,` (or) of `;` (and) terms, where a term is a comparison or a `( … )` group.
|
|
1365
|
-
def parse(expression)
|
|
1366
|
-
alternatives = split(expression, ',').map do |alternative|
|
|
1367
|
-
split(alternative, ';').map do |term|
|
|
1368
|
-
text = term.strip
|
|
1369
|
-
raise unsupported(expression) if text.empty?
|
|
1370
|
-
|
|
1371
|
-
group = unwrap(text)
|
|
1372
|
-
group ? parse(group) : comparison(text)
|
|
1373
|
-
end
|
|
1374
|
-
end
|
|
1375
|
-
->(item) { alternatives.any? { |terms| terms.all? { |test| test.call(item) } } }
|
|
1376
|
-
end
|
|
1377
|
-
|
|
1378
|
-
# The expression inside `( … )` when the parentheses enclose all of `text`, else nil.
|
|
1379
|
-
def unwrap(text)
|
|
1380
|
-
return unless text.start_with?('(') && text.end_with?(')')
|
|
1381
|
-
|
|
1382
|
-
depth = 0
|
|
1383
|
-
quote = nil
|
|
1384
|
-
text.each_char.with_index do |char, index|
|
|
1385
|
-
if quote
|
|
1386
|
-
quote = nil if char == quote
|
|
1387
|
-
elsif ['"', "'"].include?(char)
|
|
1388
|
-
quote = char
|
|
1389
|
-
elsif char == '('
|
|
1390
|
-
depth += 1
|
|
1391
|
-
elsif char == ')'
|
|
1392
|
-
depth -= 1
|
|
1393
|
-
return index == text.length - 1 ? text[1..-2] : nil if depth.zero?
|
|
1394
|
-
end
|
|
1395
|
-
end
|
|
1396
|
-
nil
|
|
1397
|
-
end
|
|
1398
|
-
|
|
1399
|
-
def unsupported(term)
|
|
1400
|
-
InvalidArgumentError.new(SdkErrorCodes::INVALID_ARGUMENT,
|
|
1401
|
-
'MockServerClient filters support ==, !=, =in=, =out=, =gt=, =ge=, =lt= and ' \
|
|
1402
|
-
"=le=, joined by ; and , and grouped with parentheses (got #{term.inspect})")
|
|
1403
|
-
end
|
|
1404
|
-
|
|
1405
|
-
# Splits on `separator` outside quotes and parentheses (groups and `=in=(...)` lists).
|
|
1406
|
-
def split(text, separator)
|
|
1407
|
-
parts = ['']
|
|
1408
|
-
quote = nil
|
|
1409
|
-
depth = 0
|
|
1410
|
-
text.each_char do |char|
|
|
1411
|
-
if quote
|
|
1412
|
-
quote = nil if char == quote
|
|
1413
|
-
elsif ['"', "'"].include?(char)
|
|
1414
|
-
quote = char
|
|
1415
|
-
elsif char == '('
|
|
1416
|
-
depth += 1
|
|
1417
|
-
elsif char == ')'
|
|
1418
|
-
depth -= 1
|
|
1419
|
-
elsif char == separator && depth.zero?
|
|
1420
|
-
parts << ''
|
|
1421
|
-
next
|
|
1422
|
-
end
|
|
1423
|
-
parts[-1] += char
|
|
1424
|
-
end
|
|
1425
|
-
parts
|
|
1426
|
-
end
|
|
1427
|
-
|
|
1428
|
-
def comparison(term)
|
|
1429
|
-
match = term.strip.match(/\A([A-Za-z][\w.]*)(==|!=|=in=|=out=|=gt=|=ge=|=lt=|=le=)(.+)\z/m)
|
|
1430
|
-
raise unsupported(term) unless match
|
|
1431
|
-
|
|
1432
|
-
field, operator, raw = match.captures
|
|
1433
|
-
values = list(operator, raw.strip)
|
|
1434
|
-
lambda do |item|
|
|
1435
|
-
actual = field.split('.').reduce(item) { |object, key| object.is_a?(Hash) ? object[key] : nil }
|
|
1436
|
-
entries = Array(actual.nil? ? 'null' : actual).map(&:to_s)
|
|
1437
|
-
case operator
|
|
1438
|
-
when '==', '=in=' then values.any? { |value| matches?(entries, value) }
|
|
1439
|
-
when '!=', '=out=' then values.none? { |value| matches?(entries, value) }
|
|
1440
|
-
else entries.any? { |entry| ordered?(entry, operator, values.first) }
|
|
1441
|
-
end
|
|
1442
|
-
end
|
|
1443
|
-
end
|
|
1444
|
-
|
|
1445
|
-
def list(operator, raw)
|
|
1446
|
-
return [unquote(raw)] unless %w[=in= =out=].include?(operator)
|
|
1447
|
-
|
|
1448
|
-
unless raw.start_with?('(') && raw.end_with?(')')
|
|
1449
|
-
raise InvalidArgumentError.new(SdkErrorCodes::INVALID_ARGUMENT, "#{operator} takes a (value,value) list")
|
|
1450
|
-
end
|
|
1451
|
-
|
|
1452
|
-
split(raw[1..-2], ',').map { |value| unquote(value.strip) }
|
|
1453
|
-
end
|
|
1454
|
-
|
|
1455
|
-
def unquote(value) = value.match?(/\A(["']).*\1\z/m) ? value[1..-2] : value
|
|
1456
|
-
|
|
1457
|
-
def matches?(entries, value)
|
|
1458
|
-
pattern = Regexp.new("\\A#{Regexp.escape(value).gsub('\*', '.*')}\\z", Regexp::MULTILINE)
|
|
1459
|
-
entries.any? { |entry| pattern.match?(entry) }
|
|
1460
|
-
end
|
|
1461
|
-
|
|
1462
|
-
# Numbers compare as numbers and everything else (such as ISO dates) as text.
|
|
1463
|
-
def ordered?(entry, operator, value)
|
|
1464
|
-
left, right = [entry, value].all? { |text| text.match?(/\A-?\d+(\.\d+)?\z/) } ? [entry.to_f, value.to_f] : [entry, value]
|
|
1465
|
-
{ '=gt=' => left > right, '=ge=' => left >= right, '=lt=' => left < right, '=le=' => left <= right }.fetch(operator)
|
|
1466
|
-
end
|
|
1467
|
-
end
|
|
1468
2021
|
end
|
|
1469
2022
|
end
|