calmlens 0.18.3 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +3 -1
- data/lib/calmlens/assets.rb +142 -42
- data/lib/calmlens/cdn.rb +14 -7
- data/lib/calmlens/error_codes.rb +28 -2
- data/lib/calmlens/events.rb +5 -2
- data/lib/calmlens/folders.rb +9 -6
- data/lib/calmlens/hash_lists.rb +11 -8
- data/lib/calmlens/internal.rb +10 -0
- data/lib/calmlens/lenses.rb +6 -2
- data/lib/calmlens/option_keys.rb +89 -0
- data/lib/calmlens/policies.rb +100 -0
- data/lib/calmlens/resource.rb +32 -1
- data/lib/calmlens/server_client.rb +23 -6
- data/lib/calmlens/session.rb +6 -3
- data/lib/calmlens/testing/classification.rb +3 -0
- data/lib/calmlens/testing/fake_api.rb +206 -20
- data/lib/calmlens/testing/test_image_grid.rb +161 -0
- data/lib/calmlens/testing/test_input_scenario.rb +127 -0
- data/lib/calmlens/testing/test_inputs.rb +157 -0
- data/lib/calmlens/testing.rb +10 -2
- data/lib/calmlens/text.rb +5 -4
- data/lib/calmlens/transform_rules.rb +120 -0
- data/lib/calmlens/transform_schema.rb +236 -0
- data/lib/calmlens/transforms.rb +37 -28
- data/lib/calmlens/upload_body.rb +14 -0
- data/lib/calmlens/upload_mime_types.rb +43 -0
- data/lib/calmlens/upload_tokens.rb +4 -1
- data/lib/calmlens/version.rb +1 -1
- data/lib/calmlens.rb +5 -0
- data/sig/calmlens.rbs +59 -3
- data/sig/error_codes.rbs +14 -1
- data/sig/types.rbs +104 -20
- metadata +10 -2
|
@@ -59,11 +59,15 @@ module CalmLens
|
|
|
59
59
|
MAX_DECISION_NOTE_LENGTH = 1499
|
|
60
60
|
RESOURCES = %w[original rendered placeholder].freeze
|
|
61
61
|
MIN_PART = 5 * 1024 * 1024
|
|
62
|
-
MIN_MULTIPART_IMAGE = 10 * 1024 * 1024
|
|
63
62
|
MAX_MULTIPART_IMAGE = 20 * 1024 * 1024
|
|
63
|
+
# Like the API: an upload has at most this many parts.
|
|
64
|
+
MAX_PARTS = 10_000
|
|
64
65
|
# The ThumbHash the mock gives every classified image, standing in for one computed from its
|
|
65
66
|
# pixels. The `placeholder` resource serves it as a PNG, like the CDN.
|
|
66
67
|
THUMBHASH = '1QcSHQRnh493V4dIh4eXh1h4kJUI'
|
|
68
|
+
# The built-in presets the mock knows, by display name.
|
|
69
|
+
PRESET_NAMES = { 'general-community' => 'General', 'children-platform' => 'Kids',
|
|
70
|
+
'education-support' => 'Learning and support', 'wholesome-community' => 'Teen-safe' }.freeze
|
|
67
71
|
# Like the API: 1 to 255 printable ASCII characters, without a space at either end.
|
|
68
72
|
IDEMPOTENCY_KEY = /\A[\x21-\x7e](?:[\x20-\x7e]{0,253}[\x21-\x7e])?\z/
|
|
69
73
|
# Content a browser could run as a document, which the CDN serves as a download.
|
|
@@ -83,6 +87,10 @@ module CalmLens
|
|
|
83
87
|
@grants = grants
|
|
84
88
|
@state = State.new(assets: {}, folders: {}, content: {}, events: [], lenses: {}, uploads: {}, idempotency: {},
|
|
85
89
|
hash_lists: {}, hash_list_entries: {})
|
|
90
|
+
# Stored policies (the API's shape without `projectIds` and `preset.latestVersion`, plus their
|
|
91
|
+
# reset `baseline`) and the one the project moderates with.
|
|
92
|
+
@policies = {}
|
|
93
|
+
@project_policy_id = nil
|
|
86
94
|
@request_count = 0
|
|
87
95
|
end
|
|
88
96
|
|
|
@@ -232,6 +240,47 @@ module CalmLens
|
|
|
232
240
|
# A request the API's schema refuses: `validation/<issue>` with HTTP 400 and the field.
|
|
233
241
|
def invalid!(code, path) = raise(Failure.new(code, "Invalid #{path}", details: { 'path' => path }, status: 400))
|
|
234
242
|
|
|
243
|
+
# Like the API's schema (METADATA_ITEM_SCHEMA): metadata is nil or a list of `{id, value}`
|
|
244
|
+
# Hashes with a String `id` and any JSON `value`.
|
|
245
|
+
def check_metadata(metadata)
|
|
246
|
+
return if metadata.nil?
|
|
247
|
+
|
|
248
|
+
invalid!('validation/invalid-type', 'metadata') unless metadata.is_a?(Array)
|
|
249
|
+
metadata.each_with_index do |item, index|
|
|
250
|
+
invalid!('validation/invalid-type', "metadata.#{index}") unless item.is_a?(Hash)
|
|
251
|
+
invalid!('validation/invalid-type', "metadata.#{index}.id") unless item['id'].is_a?(String)
|
|
252
|
+
invalid!('validation/invalid-value', "metadata.#{index}.value") unless item.key?('value')
|
|
253
|
+
end
|
|
254
|
+
end
|
|
255
|
+
|
|
256
|
+
# Like the API's schema, checked before the asset is looked up: asset updates take no null
|
|
257
|
+
# name, and signed URLs no null field.
|
|
258
|
+
def check_asset_body(method, action, json)
|
|
259
|
+
case [method, action]
|
|
260
|
+
in ['PATCH', nil]
|
|
261
|
+
check_metadata(json['metadata'])
|
|
262
|
+
invalid!('validation/invalid-type', 'name') if json.key?('name') && json['name'].nil?
|
|
263
|
+
in ['POST', 'signed-url']
|
|
264
|
+
%w[viewerClaims resource lens transform expiresInSeconds].each do |key|
|
|
265
|
+
next unless json.key?(key) && json[key].nil?
|
|
266
|
+
|
|
267
|
+
invalid!(key == 'resource' ? 'validation/invalid-value' : 'validation/invalid-type', key)
|
|
268
|
+
end
|
|
269
|
+
else nil
|
|
270
|
+
end
|
|
271
|
+
end
|
|
272
|
+
|
|
273
|
+
# Like the API: an upload's metadata is checked, then its MIME type, which must be one the
|
|
274
|
+
# API can moderate (`upload/invalid-mime-type` with `details.mimeType`).
|
|
275
|
+
def check_upload(options)
|
|
276
|
+
check_metadata(options['metadata'])
|
|
277
|
+
type = options['contentType']
|
|
278
|
+
return if type.nil? || Internal.accepted_upload_mime_type?(type)
|
|
279
|
+
|
|
280
|
+
fail!(ErrorCodes::INVALID_UPLOAD_MIME_TYPE, "#{type} is not a valid upload file type",
|
|
281
|
+
details: { 'mimeType' => type })
|
|
282
|
+
end
|
|
283
|
+
|
|
235
284
|
# Like the API's query schema: an integer from 1 to `max`.
|
|
236
285
|
def check_range(value, path, max)
|
|
237
286
|
invalid!('validation/too-small', path) if !value.is_a?(Numeric) || value < 1
|
|
@@ -245,10 +294,11 @@ module CalmLens
|
|
|
245
294
|
parts = uri.path.split('/').reject(&:empty?).map { |part| URI.decode_www_form_component(part) }
|
|
246
295
|
query = URI.decode_www_form(uri.query.to_s).to_h.to_h { |key, value| [key, query_value(key, value)] }
|
|
247
296
|
content_type = header(headers, 'Content-Type')
|
|
248
|
-
|
|
297
|
+
upload = header(headers, 'X-CalmLens-Upload')
|
|
298
|
+
# A raw upload's body is the file itself, whatever its type.
|
|
299
|
+
json = content_type == 'application/json' && body && upload.nil? ? JSON.parse(body) : {}
|
|
249
300
|
# HTTP stacks trim header values, so the API sees the key without surrounding whitespace.
|
|
250
301
|
key = header(headers, 'Idempotency-Key')&.to_s&.gsub(/\A[ \t\r\n]+|[ \t\r\n]+\z/, '')
|
|
251
|
-
upload = header(headers, 'X-CalmLens-Upload')
|
|
252
302
|
return dispatch(method, parts, query, json, body, upload) unless key && method == 'POST'
|
|
253
303
|
|
|
254
304
|
fail!(ErrorCodes::INVALID_IDEMPOTENCY_KEY) unless key.match?(IDEMPOTENCY_KEY)
|
|
@@ -304,6 +354,10 @@ module CalmLens
|
|
|
304
354
|
in ['POST', ['folders'], _] then create_folder(json)
|
|
305
355
|
in ['GET', ['hash-lists'], _] then page(project_hash_lists, query, JSON.generate([@project_id]))
|
|
306
356
|
in ['POST', ['hash-lists'], _] then create_hash_list(json)
|
|
357
|
+
in ['GET', ['policies'], _] then policy_list
|
|
358
|
+
in ['POST', ['policies'], _] then create_policy(json)
|
|
359
|
+
in ['PUT', ['policy'], _] then show_policy(assign_policy(json))
|
|
360
|
+
in [_, ['policies', String => policy_id, *rest], _] then policy(method, policy_id, rest, json)
|
|
307
361
|
in ['GET', ['assets'], _] then list(query)
|
|
308
362
|
in ['GET', ['assets', 'search'], _] then search(query)
|
|
309
363
|
in ['POST', ['assets', 'batch'], _] then batch(json)
|
|
@@ -399,11 +453,12 @@ module CalmLens
|
|
|
399
453
|
def project_folders = @state.folders.values.select { |item| item['projectId'] == @project_id }
|
|
400
454
|
|
|
401
455
|
# Like the API: without a cursor, no events and a `nextCursor` for now; with one, up to `pageSize`
|
|
402
|
-
# events added since, oldest first. `nextCursor` is never nil.
|
|
456
|
+
# events added since, oldest first. `nextCursor` is never nil. The mock's key holds every permission,
|
|
457
|
+
# so no event type is excluded.
|
|
403
458
|
def events(query)
|
|
404
459
|
size = check_range(query.fetch('pageSize', DEFAULT_PAGE_SIZE), 'pageSize', MAX_PAGE_SIZE)
|
|
405
460
|
feed = @state.events
|
|
406
|
-
return { 'items' => [], 'nextCursor' => "mock_#{feed.size}", 'hasMore' => false } unless query['cursor']
|
|
461
|
+
return { 'items' => [], 'nextCursor' => "mock_#{feed.size}", 'hasMore' => false, 'excludedTypes' => [] } unless query['cursor']
|
|
407
462
|
|
|
408
463
|
match = query['cursor'].to_s.match(/\Amock_(\d+)\z/)
|
|
409
464
|
position = match && Integer(match[1], 10)
|
|
@@ -411,7 +466,7 @@ module CalmLens
|
|
|
411
466
|
|
|
412
467
|
taken = feed[position, size]
|
|
413
468
|
following = position + taken.size
|
|
414
|
-
{ 'items' => taken, 'nextCursor' => "mock_#{following}", 'hasMore' => following < feed.size }
|
|
469
|
+
{ 'items' => taken, 'nextCursor' => "mock_#{following}", 'hasMore' => following < feed.size, 'excludedTypes' => [] }
|
|
415
470
|
end
|
|
416
471
|
|
|
417
472
|
# Saves lenses in order, like the API: an unchanged definition returns the stored lens, a
|
|
@@ -429,7 +484,7 @@ module CalmLens
|
|
|
429
484
|
end
|
|
430
485
|
|
|
431
486
|
same = %w[name type active].all? { |key| old[key] == definition[key] } &&
|
|
432
|
-
old['description'] == definition['description'] && canonical(old['
|
|
487
|
+
old['description'] == definition['description'] && canonical(old['transform']) == canonical(definition['transform'])
|
|
433
488
|
return old if same && old['version'] == definition['version']
|
|
434
489
|
if definition['version'].to_i > old['version'].to_i
|
|
435
490
|
return @state.lenses[old['identifier']] =
|
|
@@ -494,6 +549,7 @@ module CalmLens
|
|
|
494
549
|
end
|
|
495
550
|
|
|
496
551
|
def create_asset(options, queue, url: nil, data: nil, open: false)
|
|
552
|
+
check_upload(options)
|
|
497
553
|
check_folder(options['folderId']) if options['folderId']
|
|
498
554
|
if options['externalId'] && project_assets.any? { |item| item['externalId'] == options['externalId'] }
|
|
499
555
|
fail!(ErrorCodes::DUPLICATE_EXTERNAL_ID)
|
|
@@ -533,12 +589,9 @@ module CalmLens
|
|
|
533
589
|
}
|
|
534
590
|
end
|
|
535
591
|
|
|
536
|
-
# Like the API: image uploads in parts total
|
|
537
|
-
def image_size_error(type, size,
|
|
538
|
-
|
|
539
|
-
return ErrorCodes::MULTIPART_IMAGE_TOO_LARGE if size > MAX_MULTIPART_IMAGE
|
|
540
|
-
|
|
541
|
-
ErrorCodes::MULTIPART_IMAGE_TOO_SMALL if final && size < MIN_MULTIPART_IMAGE
|
|
592
|
+
# Like the API: image uploads in parts total at most 20 MiB, at any size below that.
|
|
593
|
+
def image_size_error(type, size, _final)
|
|
594
|
+
ErrorCodes::MULTIPART_IMAGE_TOO_LARGE if type == 'image' && size > MAX_MULTIPART_IMAGE
|
|
542
595
|
end
|
|
543
596
|
|
|
544
597
|
# Like the API: the parts may not overshoot a declared total, and must reach it at the end.
|
|
@@ -557,6 +610,7 @@ module CalmLens
|
|
|
557
610
|
# In the API's order: a declared total, then a single final part as the whole upload, then the
|
|
558
611
|
# part itself. The queue mode and declared total are saved with an open upload.
|
|
559
612
|
def start_multipart(query, data)
|
|
613
|
+
check_upload(query)
|
|
560
614
|
total = query['totalSizeInBytes']
|
|
561
615
|
type = kind(query['contentType'].to_s, nil)
|
|
562
616
|
final = query['isFinal'] == true
|
|
@@ -695,6 +749,7 @@ module CalmLens
|
|
|
695
749
|
|
|
696
750
|
def asset(method, parts, query, json, body)
|
|
697
751
|
id = parts[1]
|
|
752
|
+
check_asset_body(method, parts[2], json)
|
|
698
753
|
item = @state.assets[id]
|
|
699
754
|
fail!(ErrorCodes::ASSET_NOT_FOUND) unless item && item['projectId'] == @project_id
|
|
700
755
|
case [method, parts[2]]
|
|
@@ -793,7 +848,7 @@ module CalmLens
|
|
|
793
848
|
fail!(ErrorCodes::UPLOAD_NOT_OPEN) unless item['upload']['status'] == 'uploading'
|
|
794
849
|
number = query['partNumber']
|
|
795
850
|
expected = item['upload']['partsReceived'].to_i + 1
|
|
796
|
-
unless number == expected
|
|
851
|
+
unless number == expected && number <= MAX_PARTS
|
|
797
852
|
fail!(ErrorCodes::INVALID_UPLOAD_PART_NUMBER, "Expected part #{expected}",
|
|
798
853
|
details: { 'expectedPartNumber' => expected, 'receivedPartNumber' => number })
|
|
799
854
|
end
|
|
@@ -877,6 +932,120 @@ module CalmLens
|
|
|
877
932
|
touch(item.merge!('parentId' => parent_id))
|
|
878
933
|
end
|
|
879
934
|
|
|
935
|
+
# Policies: the project starts on the organisation's own General preset policy, following
|
|
936
|
+
# automatic updates. The mock holds no built-in preset settings: a preset's settings are
|
|
937
|
+
# `{ 'categorySettings' => {} }` at version 1, and uploads are never decided by a policy.
|
|
938
|
+
def new_policy(name, settings, description: '', automatic_updates: false, preset: nil)
|
|
939
|
+
now = @now.call
|
|
940
|
+
item = { 'id' => @create_id.call('pol'), 'name' => name, 'description' => description, 'settings' => settings,
|
|
941
|
+
'revision' => 1, 'automaticUpdates' => automatic_updates, 'preset' => preset,
|
|
942
|
+
'baseline' => deep_copy(settings), 'createdAt' => now, 'updatedAt' => now }
|
|
943
|
+
@policies[item['id']] = item
|
|
944
|
+
end
|
|
945
|
+
|
|
946
|
+
def ensure_policies
|
|
947
|
+
return if @project_policy_id
|
|
948
|
+
|
|
949
|
+
preset = { 'id' => 'general-community', 'version' => 1, 'own' => true }
|
|
950
|
+
@project_policy_id = new_policy(PRESET_NAMES['general-community'], { 'categorySettings' => {} },
|
|
951
|
+
automatic_updates: true, preset: preset)['id']
|
|
952
|
+
end
|
|
953
|
+
|
|
954
|
+
# A stored policy as the API shows it.
|
|
955
|
+
def show_policy(item)
|
|
956
|
+
shown = item.except('baseline')
|
|
957
|
+
shown['preset'] = item['preset'].merge('latestVersion' => 1) if item['preset']
|
|
958
|
+
shown.merge('projectIds' => item['id'] == @project_policy_id ? [@project_id] : [])
|
|
959
|
+
end
|
|
960
|
+
|
|
961
|
+
def find_policy(id)
|
|
962
|
+
ensure_policies
|
|
963
|
+
@policies[id] || fail!(ErrorCodes::POLICY_NOT_FOUND)
|
|
964
|
+
end
|
|
965
|
+
|
|
966
|
+
def policy_list
|
|
967
|
+
ensure_policies
|
|
968
|
+
{ 'policies' => @policies.values.map { |item| show_policy(item) }, 'projectPolicyId' => @project_policy_id }
|
|
969
|
+
end
|
|
970
|
+
|
|
971
|
+
def create_policy(json)
|
|
972
|
+
ensure_policies
|
|
973
|
+
name, preset_id, settings = json.values_at('name', 'presetId', 'settings')
|
|
974
|
+
named = name.is_a?(String) && !name.strip.empty?
|
|
975
|
+
if settings.nil? == preset_id.nil? || (preset_id && !PRESET_NAMES.key?(preset_id)) || (preset_id.nil? && !named)
|
|
976
|
+
fail!(ErrorCodes::INVALID_VALUE, 'Send settings with a name, or presetId')
|
|
977
|
+
end
|
|
978
|
+
description = json['description'].is_a?(String) ? json['description'] : ''
|
|
979
|
+
item = if preset_id
|
|
980
|
+
origin = { 'id' => preset_id, 'version' => 1, 'own' => false }
|
|
981
|
+
new_policy(named ? name.strip : PRESET_NAMES[preset_id], { 'categorySettings' => {} },
|
|
982
|
+
description: description, preset: origin)
|
|
983
|
+
else
|
|
984
|
+
new_policy(name.strip, settings, description: description)
|
|
985
|
+
end
|
|
986
|
+
# Like the API, a new policy is 201 Created.
|
|
987
|
+
respond(201, deep_copy(show_policy(item)))
|
|
988
|
+
end
|
|
989
|
+
|
|
990
|
+
def assign_policy(json)
|
|
991
|
+
item = find_policy(json['policyId'].to_s)
|
|
992
|
+
@project_policy_id = item['id']
|
|
993
|
+
item
|
|
994
|
+
end
|
|
995
|
+
|
|
996
|
+
# Saves new values under the next revision, once `revision` is the current one.
|
|
997
|
+
def save_policy(item, revision, values)
|
|
998
|
+
fail!(ErrorCodes::POLICY_REVISION_CONFLICT) unless revision.is_a?(Integer) && revision == item['revision']
|
|
999
|
+
|
|
1000
|
+
item.merge!(values, 'revision' => item['revision'] + 1, 'updatedAt' => @now.call)
|
|
1001
|
+
show_policy(item)
|
|
1002
|
+
end
|
|
1003
|
+
|
|
1004
|
+
def policy(method, id, rest, json)
|
|
1005
|
+
item = find_policy(id)
|
|
1006
|
+
case [method, rest]
|
|
1007
|
+
in ['GET', []] then show_policy(item)
|
|
1008
|
+
in ['PATCH', []] then update_policy(item, json)
|
|
1009
|
+
in ['DELETE', []]
|
|
1010
|
+
fail!(ErrorCodes::POLICY_IN_USE) if item['id'] == @project_policy_id
|
|
1011
|
+
@policies.delete(id)
|
|
1012
|
+
{}
|
|
1013
|
+
in ['POST', ['preset']] then apply_policy_preset(item, json)
|
|
1014
|
+
in ['PUT', ['automatic-updates']] then set_policy_automatic_updates(item, json)
|
|
1015
|
+
else route_not_found!(method, ['policies', id, *rest])
|
|
1016
|
+
end
|
|
1017
|
+
end
|
|
1018
|
+
|
|
1019
|
+
def update_policy(item, json)
|
|
1020
|
+
values = %w[name description].select { |key| json[key].is_a?(String) }
|
|
1021
|
+
.to_h { |key| [key, key == 'name' ? json[key].strip : json[key]] }
|
|
1022
|
+
values['settings'] = json['settings'] if json.key?('settings')
|
|
1023
|
+
# Saving a policy that follows its preset pins it as a copy.
|
|
1024
|
+
save_policy(item, json['revision'], values.merge('automaticUpdates' => false))
|
|
1025
|
+
end
|
|
1026
|
+
|
|
1027
|
+
def apply_policy_preset(item, json)
|
|
1028
|
+
preset = item['preset']
|
|
1029
|
+
fail!(ErrorCodes::POLICY_NO_PRESET) if preset.nil? || item['automaticUpdates']
|
|
1030
|
+
fail!(ErrorCodes::POLICY_PRESET_VERSION_CONFLICT) unless json['presetVersion'] == preset['version']
|
|
1031
|
+
# The mock's presets have one release, so an update never has a newer one to apply.
|
|
1032
|
+
fail!(ErrorCodes::POLICY_PRESET_UPDATE_UNAVAILABLE) if json['operation'] == 'update'
|
|
1033
|
+
|
|
1034
|
+
save_policy(item, json['revision'], 'settings' => deep_copy(item['baseline']))
|
|
1035
|
+
end
|
|
1036
|
+
|
|
1037
|
+
def set_policy_automatic_updates(item, json)
|
|
1038
|
+
enabled = json['enabled'] == true
|
|
1039
|
+
preset = item['preset']
|
|
1040
|
+
fail!(ErrorCodes::POLICY_AUTOMATIC_UPDATES_UNAVAILABLE) if preset.nil? || !preset['own'] || item['automaticUpdates'] == enabled
|
|
1041
|
+
fail!(ErrorCodes::POLICY_REVISION_CONFLICT) unless json['revision'] == item['revision']
|
|
1042
|
+
fail!(ErrorCodes::POLICY_PRESET_VERSION_CONFLICT) unless json['presetVersion'] == 1
|
|
1043
|
+
|
|
1044
|
+
values = { 'automaticUpdates' => enabled }
|
|
1045
|
+
values.merge!('name' => PRESET_NAMES[preset['id']], 'description' => '', 'settings' => deep_copy(item['baseline'])) if enabled
|
|
1046
|
+
save_policy(item, json['revision'], values)
|
|
1047
|
+
end
|
|
1048
|
+
|
|
880
1049
|
# Hash lists and their entries. Uploads are not compared with them.
|
|
881
1050
|
def project_hash_lists = @state.hash_lists.values.select { |item| item['projectId'] == @project_id }
|
|
882
1051
|
|
|
@@ -1042,13 +1211,27 @@ module CalmLens
|
|
|
1042
1211
|
# private asset needs an API key with `assets:download` for its folder.
|
|
1043
1212
|
def serve(method, relative)
|
|
1044
1213
|
path, query = relative.split('#', 2).first.split('?', 2)
|
|
1045
|
-
|
|
1046
|
-
_, project, identifier, resource =
|
|
1214
|
+
raw_segments = path.split('/').reject(&:empty?)
|
|
1215
|
+
_, project, identifier, resource = raw_segments.map { |part| URI.decode_www_form_component(part) }
|
|
1216
|
+
# Like the CDN, the selector after the resource is read first, and lenses and transforms
|
|
1217
|
+
# apply only to the rendered resource.
|
|
1218
|
+
begin
|
|
1219
|
+
lens, options = TransformRules.parse_selector(raw_segments.drop(4).join('/'))
|
|
1220
|
+
rescue ArgumentError
|
|
1221
|
+
return invalid_transform
|
|
1222
|
+
end
|
|
1223
|
+
return invalid_transform if resource == 'original' && (lens || options)
|
|
1224
|
+
|
|
1047
1225
|
item = project == @project_id && project_assets.find { |asset| [asset['id'], asset['externalId']].include?(identifier) }
|
|
1048
1226
|
return error('asset/not-found', 'Asset not found', status: 404) unless item && RESOURCES.include?(resource)
|
|
1049
|
-
|
|
1050
1227
|
return placeholder(method, item) if resource == 'placeholder'
|
|
1051
1228
|
|
|
1229
|
+
deliver(method, item, URI.decode_www_form(query.to_s).to_h, options)
|
|
1230
|
+
end
|
|
1231
|
+
|
|
1232
|
+
# Serves a found asset's content after the delivery, age and access checks, then the
|
|
1233
|
+
# transform options' schema check.
|
|
1234
|
+
def deliver(method, item, params, options)
|
|
1052
1235
|
moderation = item['classification']['moderation']['status']
|
|
1053
1236
|
stored = @state.content[item['id']]
|
|
1054
1237
|
if stored.nil? || item['upload']['status'] == 'uploading' || %w[blocked purged].include?(moderation) ||
|
|
@@ -1056,9 +1239,7 @@ module CalmLens
|
|
|
1056
1239
|
return not_available
|
|
1057
1240
|
end
|
|
1058
1241
|
|
|
1059
|
-
|
|
1060
|
-
opted_in = viewer_age_confirmed?(params)
|
|
1061
|
-
if moderation == 'age-restricted' && !opted_in
|
|
1242
|
+
if moderation == 'age-restricted' && !viewer_age_confirmed?(params)
|
|
1062
1243
|
return error('viewer/age-confirmation-required', 'Age confirmation required',
|
|
1063
1244
|
status: 403)
|
|
1064
1245
|
end
|
|
@@ -1069,6 +1250,9 @@ module CalmLens
|
|
|
1069
1250
|
'This API key needs assets:download permission for this asset’s folder.', status: 403)
|
|
1070
1251
|
end
|
|
1071
1252
|
|
|
1253
|
+
refused = options ? TransformRules.check(options).flatten : []
|
|
1254
|
+
return error('asset/invalid-transform-options', "Invalid transform options: #{refused.join(', ')}", status: 400) if refused.any?
|
|
1255
|
+
|
|
1072
1256
|
type = stored['type'].to_s.empty? ? 'application/octet-stream' : stored['type']
|
|
1073
1257
|
cdn_response(method, stored['parts'].sort.map(&:last).join.b,
|
|
1074
1258
|
viewer_delivery_headers(item, type, restricted, params))
|
|
@@ -1084,6 +1268,8 @@ module CalmLens
|
|
|
1084
1268
|
headers
|
|
1085
1269
|
end
|
|
1086
1270
|
|
|
1271
|
+
def invalid_transform = error('asset/invalid-transform', 'Invalid transform selector', status: 400)
|
|
1272
|
+
|
|
1087
1273
|
def not_available = error('asset/not-available', 'Asset is not available', status: 403)
|
|
1088
1274
|
|
|
1089
1275
|
# Like the CDN's placeholder handler: an image's ThumbHash, public whatever the asset's
|
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'zlib'
|
|
4
|
+
|
|
5
|
+
module CalmLens
|
|
6
|
+
# The image grid behind the signed test images (test_inputs.rb).
|
|
7
|
+
module TestInputs
|
|
8
|
+
# The test image grid of packages/shared/src/TestImageGrid.ts, painted byte for byte: an odd
|
|
9
|
+
# N×N grid with an alternating black-and-white ring, two Gray-coded bits per interior cell as
|
|
10
|
+
# one of four luma levels (colour in chroma only), and Reed–Solomon parity over GF(256).
|
|
11
|
+
# @api private
|
|
12
|
+
module Grid
|
|
13
|
+
# Target luma of the four data levels.
|
|
14
|
+
LEVELS = [72, 128, 184, 240].freeze
|
|
15
|
+
SYMBOL_LEVEL = [0, 1, 3, 2].freeze
|
|
16
|
+
PARITY = 32
|
|
17
|
+
# Category colours, carried in the chroma only.
|
|
18
|
+
COLORS = {
|
|
19
|
+
'sexual' => [232, 80, 160], 'violence' => [224, 56, 64], 'self-harm' => [24, 168, 152],
|
|
20
|
+
'harassment' => [240, 128, 48], 'hate' => [144, 80, 216], 'criminal-activity' => [224, 184, 40],
|
|
21
|
+
'personal-information' => [48, 128, 224], 'profanity' => [136, 192, 48]
|
|
22
|
+
}.freeze
|
|
23
|
+
NEUTRAL = [144, 152, 160].freeze
|
|
24
|
+
|
|
25
|
+
# GF(256) with the polynomial 0x11d and generator 2.
|
|
26
|
+
EXP = Array.new(512, 0)
|
|
27
|
+
LOG = Array.new(256, 0)
|
|
28
|
+
x = 1
|
|
29
|
+
255.times do |i|
|
|
30
|
+
EXP[i] = x
|
|
31
|
+
LOG[x] = i
|
|
32
|
+
x <<= 1
|
|
33
|
+
x ^= 0x11d if x.anybits?(256)
|
|
34
|
+
end
|
|
35
|
+
(255...512).each { |i| EXP[i] = EXP[i - 255] }
|
|
36
|
+
[EXP, LOG].each(&:freeze)
|
|
37
|
+
|
|
38
|
+
module_function
|
|
39
|
+
|
|
40
|
+
def mul(left, right) = left.zero? || right.zero? ? 0 : EXP[LOG[left] + LOG[right]]
|
|
41
|
+
|
|
42
|
+
# g(x) = (x - a^0)...(x - a^31), highest coefficient first.
|
|
43
|
+
GENERATOR = PARITY.times.reduce([1]) do |g, i|
|
|
44
|
+
following = Array.new(g.size + 1, 0)
|
|
45
|
+
g.each_with_index do |coefficient, j|
|
|
46
|
+
following[j] ^= coefficient
|
|
47
|
+
following[j + 1] ^= mul(coefficient, EXP[i])
|
|
48
|
+
end
|
|
49
|
+
following
|
|
50
|
+
end.freeze
|
|
51
|
+
|
|
52
|
+
def parity(block)
|
|
53
|
+
parity = Array.new(PARITY, 0)
|
|
54
|
+
block.each do |byte|
|
|
55
|
+
factor = byte ^ parity[0]
|
|
56
|
+
parity = parity.drop(1) << 0
|
|
57
|
+
PARITY.times { |j| parity[j] ^= mul(GENERATOR[j + 1], factor) } unless factor.zero?
|
|
58
|
+
end
|
|
59
|
+
parity
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
# The codeword bytes a grid of `cells` holds, its block lengths and its data bytes.
|
|
63
|
+
def layout(cells)
|
|
64
|
+
capacity = (cells - 2) * (cells - 2) / 4
|
|
65
|
+
blocks = (capacity + 254) / 255
|
|
66
|
+
lengths = Array.new(blocks) { |index| (capacity / blocks) + (index < capacity % blocks ? 1 : 0) }
|
|
67
|
+
[capacity, lengths, capacity - (blocks * PARITY)]
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
# The colour's chroma on the level's luma, scaled by the largest s/16 that keeps every
|
|
71
|
+
# channel in 0-255. Each scaled difference is truncated toward zero (not floored).
|
|
72
|
+
def pixel(colour, level)
|
|
73
|
+
target = LEVELS[level]
|
|
74
|
+
own = ((77 * colour[0]) + (150 * colour[1]) + (29 * colour[2])) >> 8
|
|
75
|
+
16.downto(0) do |scale|
|
|
76
|
+
pixel = colour.map do |channel|
|
|
77
|
+
difference = channel - own
|
|
78
|
+
magnitude = (difference.abs * scale) >> 4
|
|
79
|
+
difference.negative? ? target - magnitude : target + magnitude
|
|
80
|
+
end
|
|
81
|
+
return pixel if pixel.all? { |channel| channel.between?(0, 255) }
|
|
82
|
+
end
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
# The data cell's colour: the palette blended between opposite corners, in integers.
|
|
86
|
+
def blend(palette, column, row, cells)
|
|
87
|
+
span = cells - 1
|
|
88
|
+
denominator = span * span
|
|
89
|
+
position = ((column * (span - row)) + ((span - column) * row)) * (palette.size - 1)
|
|
90
|
+
start = position / denominator
|
|
91
|
+
remainder = position - (start * denominator)
|
|
92
|
+
finish = palette[[start + 1, palette.size - 1].min]
|
|
93
|
+
palette[start].each_with_index.map do |channel, c|
|
|
94
|
+
((channel * (denominator - remainder)) + (finish[c] * remainder)) / denominator
|
|
95
|
+
end
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
# Paints `data` (a binary String) as a grid of the categories' colours: [width, RGBA String].
|
|
99
|
+
def paint(data, categories)
|
|
100
|
+
cells = (15..127).step(2).find { |size| layout(size)[2] >= data.bytesize }
|
|
101
|
+
raise Internal.invalid('Test scenario is too large for a test image') unless cells
|
|
102
|
+
|
|
103
|
+
stream = codeword_stream(data, cells)
|
|
104
|
+
palette = categories.uniq.map { |category| COLORS.fetch(category) }
|
|
105
|
+
palette = [NEUTRAL] if palette.empty?
|
|
106
|
+
pixels = Array.new(cells * cells) { |cell| [*cell_pixel(stream, palette, cell, cells), 255].pack('C4') }
|
|
107
|
+
cell_size = 2 * (512 / 2 / cells)
|
|
108
|
+
rows = Array.new(cells) { |row| pixels[row * cells, cells].map { |pixel| pixel * cell_size }.join * cell_size }
|
|
109
|
+
[cells * cell_size, rows.join.b]
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
# The padded data with each block's parity, the blocks interleaved byte by byte and whitened, so
|
|
113
|
+
# a uniform area never reads as the all-zero codeword.
|
|
114
|
+
def codeword_stream(data, cells)
|
|
115
|
+
_, lengths, data_bytes = layout(cells)
|
|
116
|
+
padded = data.bytes + Array.new(data_bytes - data.bytesize) { |i| i.odd? ? 0x11 : 0xec }
|
|
117
|
+
offset = 0
|
|
118
|
+
codewords = lengths.map do |length|
|
|
119
|
+
block = padded[offset, length - PARITY]
|
|
120
|
+
offset += block.size
|
|
121
|
+
block + parity(block)
|
|
122
|
+
end
|
|
123
|
+
stream = Array.new(lengths.max) { |i| codewords.filter_map { |codeword| codeword[i] } }.flatten
|
|
124
|
+
stream.each_with_index.map { |byte, i| byte ^ (((i * 151) + 77) & 255) }
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
# A cell's RGB: the ring alternates black and white; data cells hold two bits each.
|
|
128
|
+
def cell_pixel(stream, palette, cell, cells)
|
|
129
|
+
row, column = cell.divmod(cells)
|
|
130
|
+
return [(row + column).odd? ? 255 : 0] * 3 if [row, column].any? { |at| at.zero? || at == cells - 1 }
|
|
131
|
+
|
|
132
|
+
symbol = ((row - 1) * (cells - 2)) + column - 1
|
|
133
|
+
index = symbol >> 2
|
|
134
|
+
level = index < stream.size ? SYMBOL_LEVEL[(stream[index] >> (6 - (2 * (symbol & 3)))) & 3] : 0
|
|
135
|
+
pixel(blend(palette, column, row, cells), level)
|
|
136
|
+
end
|
|
137
|
+
end
|
|
138
|
+
|
|
139
|
+
# The PNG encoding of test images.
|
|
140
|
+
# @api private
|
|
141
|
+
module Png
|
|
142
|
+
module_function
|
|
143
|
+
|
|
144
|
+
# 8-bit RGBA rows without filtering in stored deflate blocks, so every SDK writes the same bytes.
|
|
145
|
+
def png(width, height, rgba)
|
|
146
|
+
stride = width * 4
|
|
147
|
+
raw = Array.new(height) { |row| "\0".b + rgba.byteslice(row * stride, stride) }.join
|
|
148
|
+
zlib = "\x78\x01".b
|
|
149
|
+
blocks = (0...raw.bytesize).step(65_535).map { |offset| raw.byteslice(offset, 65_535) }
|
|
150
|
+
blocks.each_with_index do |block, index|
|
|
151
|
+
zlib << [index == blocks.size - 1 ? 1 : 0, block.bytesize, block.bytesize ^ 0xffff].pack('Cvv') << block
|
|
152
|
+
end
|
|
153
|
+
zlib << [Zlib.adler32(raw)].pack('N')
|
|
154
|
+
chunk = ->(type, body) { [body.bytesize].pack('N') + type.b + body + [Zlib.crc32(type.b + body)].pack('N') }
|
|
155
|
+
"\x89PNG\r\n\x1a\n".b + chunk.call('IHDR', [width, height, 8, 6, 0, 0, 0].pack('NNC5')) +
|
|
156
|
+
chunk.call('IDAT', zlib) + chunk.call('IEND', ''.b)
|
|
157
|
+
end
|
|
158
|
+
end
|
|
159
|
+
private_constant :Grid, :Png
|
|
160
|
+
end
|
|
161
|
+
end
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative 'classification'
|
|
4
|
+
|
|
5
|
+
module CalmLens
|
|
6
|
+
# The scenario checks behind the signed test inputs (test_inputs.rb).
|
|
7
|
+
module TestInputs
|
|
8
|
+
# Checks test input scenarios and writes them as the backend's canonical JSON.
|
|
9
|
+
# @api private
|
|
10
|
+
module Scenario
|
|
11
|
+
ACTIONS = %w[allow limit age-restrict block].freeze
|
|
12
|
+
REGIONAL_ACTIONS = %w[limit age-restrict block].freeze
|
|
13
|
+
# The qualifiers a scenario may list for each category: all but those the input type sets.
|
|
14
|
+
ALLOWED_QUALIFIERS = MockClassification::QUALIFIERS.transform_values do |list|
|
|
15
|
+
(list - MockClassification::INPUT_QUALIFIERS).freeze
|
|
16
|
+
end.freeze
|
|
17
|
+
|
|
18
|
+
module_function
|
|
19
|
+
|
|
20
|
+
# A checked scenario's JSON in the backend's key order. Every string is a validated ASCII token.
|
|
21
|
+
def canonical_json(value)
|
|
22
|
+
case value
|
|
23
|
+
when Hash then "{#{value.map { |key, item| "\"#{key}\":#{canonical_json(item)}" }.join(',')}}"
|
|
24
|
+
when Array then "[#{value.map { |item| canonical_json(item) }.join(',')}]"
|
|
25
|
+
when Float then js_number(value)
|
|
26
|
+
else "\"#{value}\""
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
# A score as JavaScript's JSON.stringify writes it: the shortest round-tripping digits, in
|
|
31
|
+
# exponent form below 1e-6, and 0 for -0.
|
|
32
|
+
def js_number(value)
|
|
33
|
+
return '0' if value.zero?
|
|
34
|
+
|
|
35
|
+
mantissa, exponent = value.to_s.split('e')
|
|
36
|
+
whole, fraction = mantissa.split('.')
|
|
37
|
+
digits = whole + fraction.to_s
|
|
38
|
+
point = whole.size + exponent.to_i
|
|
39
|
+
stripped = digits.sub(/\A0+/, '')
|
|
40
|
+
point -= digits.size - stripped.size
|
|
41
|
+
digits = stripped.sub(/0+\z/, '')
|
|
42
|
+
if value < 1e-6
|
|
43
|
+
"#{digits[0]}#{".#{digits[1..]}" if digits.size > 1}e#{point - 1}"
|
|
44
|
+
elsif point <= 0 then "0.#{'0' * -point}#{digits}"
|
|
45
|
+
elsif point >= digits.size then digits + ('0' * (point - digits.size))
|
|
46
|
+
else "#{digits[0, point]}.#{digits[point..]}"
|
|
47
|
+
end
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
# Checks a scenario and returns it with camelCase String keys in canonical order.
|
|
51
|
+
def validate(input)
|
|
52
|
+
input = fields(input, 'scenario', %w[regionalActions unassessedCategories instances], %w[instances])
|
|
53
|
+
instances = array(input['instances'], 'instances', 128).map { |instance| check_instance(instance) }
|
|
54
|
+
with_action = instances.count { |instance| instance.key?('action') }
|
|
55
|
+
check([0, instances.size].include?(with_action), 'Set an action for every instance or omit all actions')
|
|
56
|
+
checked = {}
|
|
57
|
+
if input.key?('regionalActions')
|
|
58
|
+
checked['regionalActions'] = array(input['regionalActions'], 'regionalActions', REGIONS.size).each_with_object([]) do |item, done|
|
|
59
|
+
item = fields(item, 'regional action', %w[region action], %w[region action])
|
|
60
|
+
check(REGIONS.include?(item['region']) && done.none? { |other| other['region'] == item['region'] },
|
|
61
|
+
"Regional actions need unique regions from #{REGIONS.join(', ')} (got #{item['region'].inspect})")
|
|
62
|
+
check(REGIONAL_ACTIONS.include?(item['action']), "Regional action must be one of #{REGIONAL_ACTIONS.join(', ')}")
|
|
63
|
+
done << { 'region' => item['region'], 'action' => item['action'] }
|
|
64
|
+
end
|
|
65
|
+
end
|
|
66
|
+
if input.key?('unassessedCategories')
|
|
67
|
+
unassessed = array(input['unassessedCategories'], 'unassessedCategories', MockClassification::CATEGORIES.size)
|
|
68
|
+
unassessed.each_with_index do |category, index|
|
|
69
|
+
check(MockClassification::CATEGORIES.include?(category) && !unassessed[0, index].include?(category) &&
|
|
70
|
+
instances.none? { |instance| instance['category'] == category },
|
|
71
|
+
"Unassessed categories must be unique categories without instances (got #{category.inspect})")
|
|
72
|
+
end
|
|
73
|
+
checked['unassessedCategories'] = unassessed
|
|
74
|
+
end
|
|
75
|
+
checked.merge('instances' => instances)
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
def check_instance(instance)
|
|
79
|
+
instance = fields(instance, 'instance', %w[category score action qualifiers], %w[category qualifiers])
|
|
80
|
+
category = instance['category']
|
|
81
|
+
allowed = ALLOWED_QUALIFIERS[category] if category.is_a?(String)
|
|
82
|
+
check(allowed, "Unknown category #{category.inspect}")
|
|
83
|
+
checked = { 'category' => category }
|
|
84
|
+
if instance.key?('score')
|
|
85
|
+
score = instance['score']
|
|
86
|
+
check(score.is_a?(Numeric) && score.real? && score.to_f.between?(0, 1), 'Score must be a number from 0 to 1')
|
|
87
|
+
checked['score'] = score.to_f
|
|
88
|
+
end
|
|
89
|
+
if instance.key?('action')
|
|
90
|
+
check(ACTIONS.include?(instance['action']), "Action must be one of #{ACTIONS.join(', ')}")
|
|
91
|
+
checked['action'] = instance['action']
|
|
92
|
+
end
|
|
93
|
+
qualifiers = array(instance['qualifiers'], 'qualifiers', 64)
|
|
94
|
+
qualifiers.each_with_index do |qualifier, index|
|
|
95
|
+
check(allowed.include?(qualifier) && !qualifiers[0, index].include?(qualifier),
|
|
96
|
+
"Choose unique #{category} qualifiers; input-type qualifiers are automatic (got #{qualifier.inspect})")
|
|
97
|
+
end
|
|
98
|
+
checked.merge('qualifiers' => qualifiers)
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
# A Hash's entries by camelCase key, refusing unknown, duplicate and missing keys.
|
|
102
|
+
def fields(value, name, allowed, required)
|
|
103
|
+
check(value.is_a?(Hash), "A test #{name} must be a Hash (got #{value.class})")
|
|
104
|
+
result = {}
|
|
105
|
+
value.each do |key, item|
|
|
106
|
+
field = Internal.camelize(key) if key.is_a?(String) || key.is_a?(Symbol)
|
|
107
|
+
check(allowed.include?(field), "Unknown test #{name} field #{key.inspect}")
|
|
108
|
+
check(!result.key?(field), "Test #{name} field #{field} is given twice")
|
|
109
|
+
result[field] = item
|
|
110
|
+
end
|
|
111
|
+
missing = required - result.keys
|
|
112
|
+
check(missing.empty?, "A test #{name} needs #{missing.join(', ')}")
|
|
113
|
+
result
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
def array(value, name, max)
|
|
117
|
+
check(value.is_a?(Array), "#{name} must be an Array")
|
|
118
|
+
check(value.size <= max, "#{name} has at most #{max} items")
|
|
119
|
+
value.dup
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
def check(condition, message) = condition || raise(Internal.invalid(message))
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
private_constant :Scenario
|
|
126
|
+
end
|
|
127
|
+
end
|