calmlens 0.18.3 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +3 -1
- data/lib/calmlens/assets.rb +142 -42
- data/lib/calmlens/cdn.rb +14 -7
- data/lib/calmlens/error_codes.rb +28 -2
- data/lib/calmlens/events.rb +5 -2
- data/lib/calmlens/folders.rb +9 -6
- data/lib/calmlens/hash_lists.rb +11 -8
- data/lib/calmlens/internal.rb +10 -0
- data/lib/calmlens/lenses.rb +6 -2
- data/lib/calmlens/option_keys.rb +89 -0
- data/lib/calmlens/policies.rb +100 -0
- data/lib/calmlens/resource.rb +32 -1
- data/lib/calmlens/server_client.rb +23 -6
- data/lib/calmlens/session.rb +6 -3
- data/lib/calmlens/testing/classification.rb +3 -0
- data/lib/calmlens/testing/fake_api.rb +206 -20
- data/lib/calmlens/testing/test_image_grid.rb +161 -0
- data/lib/calmlens/testing/test_input_scenario.rb +127 -0
- data/lib/calmlens/testing/test_inputs.rb +157 -0
- data/lib/calmlens/testing.rb +10 -2
- data/lib/calmlens/text.rb +5 -4
- data/lib/calmlens/transform_rules.rb +120 -0
- data/lib/calmlens/transform_schema.rb +236 -0
- data/lib/calmlens/transforms.rb +37 -28
- data/lib/calmlens/upload_body.rb +14 -0
- data/lib/calmlens/upload_mime_types.rb +43 -0
- data/lib/calmlens/upload_tokens.rb +4 -1
- data/lib/calmlens/version.rb +1 -1
- data/lib/calmlens.rb +5 -0
- data/sig/calmlens.rbs +59 -3
- data/sig/error_codes.rbs +14 -1
- data/sig/types.rbs +104 -20
- metadata +10 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: b3b2b65d6e8ec874bb4971fd4e4568e6cc424ecce6f060c6ce796bc2eb1a1c6d
|
|
4
|
+
data.tar.gz: 54a14554c90bec933b2ebaf0482918f3902497c6bdff6d8204e6b374fb881aab
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 899100986bd5d7534a535a92f1af8c91a06fe8eadd123dfb9ef03a63d22193536c6d3bdaf72936905bee03379e88a3bb53e8197ae259fddec8b32e1d0eb951b8
|
|
7
|
+
data.tar.gz: 33d8fa683ba8d59117f9be22611f0ba5a12b4b2d557a4bbf61cdfd3b782f1d4186a3197bdaf3e26df8e0ff07c8ed53ca79e0bdeb84feae3d1f9ce904976ae7c6
|
data/README.md
CHANGED
|
@@ -60,7 +60,7 @@ New statuses and fields may appear in later API versions. The SDK returns them u
|
|
|
60
60
|
|
|
61
61
|
- [Moderate text, images, documents, audio, video and websites](https://calmlens.com/docs/sdk/ruby#moderate-content), and [read classification results](https://calmlens.com/docs/guides/classification-results).
|
|
62
62
|
- [Wait for a queued decision](https://calmlens.com/docs/sdk/ruby#waiting-for-a-decision) or [receive it by webhook](https://calmlens.com/docs/guides/webhooks).
|
|
63
|
-
- [Stream uploads from any IO](https://calmlens.com/docs/sdk/ruby/reference#upload-content)
|
|
63
|
+
- [Stream uploads from any IO](https://calmlens.com/docs/sdk/ruby/reference#upload-content). Content larger than `part_size_bytes:` (8 MiB) that can be read again, such as a String or a File, goes in [parts](https://calmlens.com/docs/guides/multipart-uploads) automatically, each retried on its own and resumable with the same idempotency key.
|
|
64
64
|
- [Moderate replies with their conversation](https://calmlens.com/docs/sdk/ruby/reference#conversation-context).
|
|
65
65
|
- [Sign upload tokens for browser uploads](https://calmlens.com/docs/sdk/ruby/reference#upload-tokens) through [upload presets](https://calmlens.com/docs/guides/upload-presets).
|
|
66
66
|
- [Record reviewer decisions](https://calmlens.com/docs/sdk/ruby/reference#reviewer-decisions) and [explain a decision](https://calmlens.com/docs/sdk/ruby#methods).
|
|
@@ -77,6 +77,8 @@ Every method is listed in the [Ruby SDK documentation](https://calmlens.com/docs
|
|
|
77
77
|
|
|
78
78
|
`CalmLens::MockServerClient` from `calmlens/testing` runs the real client against an in-memory stand-in for the API and CDN, so you can test your integration offline. See [Ruby testing](https://calmlens.com/docs/sdk/ruby/testing).
|
|
79
79
|
|
|
80
|
+
`CalmLens::TestInputs` signs test inputs with your project's test input key from the portal: `generate_text_code` turns a scenario, preset or signal into a text code, `paint_image` paints it as a test image, and `generate_media_code` gives a phrase for audio and video, so the real API returns the scenario you declared.
|
|
81
|
+
|
|
80
82
|
## Links
|
|
81
83
|
|
|
82
84
|
- [Documentation](https://calmlens.com/docs/sdk/ruby)
|
data/lib/calmlens/assets.rb
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require 'securerandom'
|
|
4
|
+
|
|
3
5
|
module CalmLens
|
|
4
6
|
# Uploads, moderation and management of assets. Reach it through {ServerClient#assets}.
|
|
5
7
|
#
|
|
@@ -10,10 +12,19 @@ module CalmLens
|
|
|
10
12
|
# later appear as they are, so compare `classification.moderation.status` and similar values with known strings
|
|
11
13
|
# and treat anything else as not ready.
|
|
12
14
|
class Assets < Resource
|
|
15
|
+
# The prefix of this resource's operations in Internal::OPTION_KEYS.
|
|
16
|
+
OPERATIONS = 'assets'
|
|
17
|
+
|
|
13
18
|
SUBMIT_QUERY_KEYS = %w[name description processingOptions visibility metadata authorId context externalId externalUrl
|
|
14
19
|
folderId uploadPresetId].freeze
|
|
15
|
-
MIN_MULTIPART_IMAGE_BYTES = 10 * 1024 * 1024
|
|
16
20
|
MAX_MULTIPART_IMAGE_BYTES = 20 * 1024 * 1024
|
|
21
|
+
# The API's limits for one-request uploads: images up to 10 MiB (larger ones are uploaded in
|
|
22
|
+
# parts), audio and video up to 50 MiB and other files up to 5,000 MiB.
|
|
23
|
+
MAX_DIRECT_IMAGE_BYTES = 10 * 1024 * 1024
|
|
24
|
+
MAX_MEDIA_BYTES = 50 * 1024 * 1024
|
|
25
|
+
MAX_FILE_BYTES = 5000 * 1024 * 1024
|
|
26
|
+
# The longest `idempotency_key` of an upload sent in parts: each part's key adds `:part-<n>`.
|
|
27
|
+
MAX_SPLIT_IDEMPOTENCY_KEY_LENGTH = 240
|
|
17
28
|
# The moderation status before a decision; {#wait_for_decision} returns once an asset has any
|
|
18
29
|
# other, or once its classification status is in {FINISHED_PROCESSING_STATUSES}.
|
|
19
30
|
PENDING_MODERATION_STATUS = 'pending'
|
|
@@ -22,7 +33,7 @@ module CalmLens
|
|
|
22
33
|
DECISION_POLL_FIRST_DELAY = 0.5
|
|
23
34
|
# Each wait between decision checks is this many times the one before, up to DECISION_POLL_MAX_DELAY.
|
|
24
35
|
DECISION_POLL_FACTOR = 1.5
|
|
25
|
-
DECISION_POLL_MAX_DELAY =
|
|
36
|
+
DECISION_POLL_MAX_DELAY = 2.0
|
|
26
37
|
DEFAULT_DECISION_TIMEOUT = 60
|
|
27
38
|
# The lifetime of a signed URL when none is given: 15 minutes.
|
|
28
39
|
DEFAULT_SIGNED_URL_SECONDS = 15 * 60
|
|
@@ -40,7 +51,7 @@ module CalmLens
|
|
|
40
51
|
# @option options [Integer] :limit
|
|
41
52
|
# @return [Hash] `{'query', 'items' => [{'asset', 'score', 'matchedFields'}], 'semanticStatus'}`
|
|
42
53
|
def search(options = {})
|
|
43
|
-
call(options) { |api, request| request('GET', project('assets/search'), query: api, request_options: request) }
|
|
54
|
+
call(options, __method__) { |api, request| request('GET', project('assets/search'), query: api, request_options: request) }
|
|
44
55
|
end
|
|
45
56
|
|
|
46
57
|
# Lists one page of assets.
|
|
@@ -50,7 +61,7 @@ module CalmLens
|
|
|
50
61
|
# @option options [Array<Hash>] :ordering `[{field: 'createdAt', direction: 'desc'}]`
|
|
51
62
|
# @return [Hash] a cursor page: items, nextCursor, hasMore, pageSize and ordering.
|
|
52
63
|
def list(options = {})
|
|
53
|
-
call(options) do |api, request|
|
|
64
|
+
call(options, __method__) do |api, request|
|
|
54
65
|
request('GET', project('assets'), query: api, request_options: request)
|
|
55
66
|
end
|
|
56
67
|
end
|
|
@@ -62,7 +73,7 @@ module CalmLens
|
|
|
62
73
|
def each(options = {}, &block)
|
|
63
74
|
return enum_for(:each, options) unless block
|
|
64
75
|
|
|
65
|
-
api, request = call(options) { |*split| split }
|
|
76
|
+
api, request = call(options, __method__) { |*split| split }
|
|
66
77
|
listing = pages(api['cursor']) { |index| list(api.merge('cursor' => index, 'request_options' => request)) }
|
|
67
78
|
listing.each { |page| page['items'].each(&block) }
|
|
68
79
|
nil
|
|
@@ -72,7 +83,7 @@ module CalmLens
|
|
|
72
83
|
# @option options [String] :asset_id
|
|
73
84
|
# @return [Hash] a page like {#list}
|
|
74
85
|
def list_children(options)
|
|
75
|
-
call(options) do |api, request|
|
|
86
|
+
call(options, __method__) do |api, request|
|
|
76
87
|
query = without(api, 'assetId')
|
|
77
88
|
request('GET', "/assets/#{Internal.segment(api['assetId'])}/children", query: query, request_options: request)
|
|
78
89
|
end
|
|
@@ -85,7 +96,7 @@ module CalmLens
|
|
|
85
96
|
def each_child(options, &block)
|
|
86
97
|
return enum_for(:each_child, options) unless block
|
|
87
98
|
|
|
88
|
-
api, request = call(options) { |*split| split }
|
|
99
|
+
api, request = call(options, __method__) { |*split| split }
|
|
89
100
|
listing = pages(api['cursor']) { |index| list_children(api.merge('cursor' => index, 'request_options' => request)) }
|
|
90
101
|
listing.each { |page| page['items'].each(&block) }
|
|
91
102
|
nil
|
|
@@ -94,7 +105,7 @@ module CalmLens
|
|
|
94
105
|
# @option options [String] :asset_id
|
|
95
106
|
# @return [Hash] the asset
|
|
96
107
|
def retrieve(options)
|
|
97
|
-
call(options) do |api, request|
|
|
108
|
+
call(options, __method__) do |api, request|
|
|
98
109
|
request('GET', "/assets/#{Internal.segment(api['assetId'])}", request_options: request)
|
|
99
110
|
end
|
|
100
111
|
end
|
|
@@ -102,7 +113,7 @@ module CalmLens
|
|
|
102
113
|
# Waits until an asset's `classification.moderation.status` is no longer `pending` (`allowed`,
|
|
103
114
|
# `limited`, `age-restricted`, `blocked` or `purged`), or its `classification.status` is `failed`, and returns
|
|
104
115
|
# the asset. It retrieves the asset, waiting 0.5 seconds before the second check and 1.5 times
|
|
105
|
-
# longer each time after that, up to
|
|
116
|
+
# longer each time after that, up to 2 seconds.
|
|
106
117
|
# Webhooks report decisions without polling.
|
|
107
118
|
#
|
|
108
119
|
# `timeout:` is the whole wait in seconds; `request_options: {timeout:}` still limits each request.
|
|
@@ -113,7 +124,7 @@ module CalmLens
|
|
|
113
124
|
# @raise [TimeoutError] `sdk/timeout` when no decision arrives in time; `details` holds
|
|
114
125
|
# `'assetId'` and the last `'classificationStatus'` and `'moderationStatus'`.
|
|
115
126
|
def wait_for_decision(options)
|
|
116
|
-
call(options) do |api, request|
|
|
127
|
+
call(options, __method__) do |api, request|
|
|
117
128
|
timeout = api.fetch('timeout', DEFAULT_DECISION_TIMEOUT)
|
|
118
129
|
unless timeout.is_a?(Numeric) && timeout.real? && timeout.to_f.finite? && !timeout.negative?
|
|
119
130
|
raise Internal.invalid('timeout must be a non-negative number of seconds')
|
|
@@ -127,7 +138,7 @@ module CalmLens
|
|
|
127
138
|
# @option options [String] :asset_id
|
|
128
139
|
# @return [Hash] the updated asset
|
|
129
140
|
def update(options)
|
|
130
|
-
call(options) do |api, request|
|
|
141
|
+
call(options, __method__) do |api, request|
|
|
131
142
|
request('PATCH', "/assets/#{Internal.segment(api['assetId'])}", body: without(api, 'assetId'),
|
|
132
143
|
request_options: request)
|
|
133
144
|
end
|
|
@@ -137,7 +148,7 @@ module CalmLens
|
|
|
137
148
|
# @option options [String] :asset_id
|
|
138
149
|
# @return [Hash] the asset
|
|
139
150
|
def retry(options)
|
|
140
|
-
call(options) do |api, request|
|
|
151
|
+
call(options, __method__) do |api, request|
|
|
141
152
|
request('POST', "/assets/#{Internal.segment(api['assetId'])}/retry", request_options: request)
|
|
142
153
|
end
|
|
143
154
|
end
|
|
@@ -154,7 +165,7 @@ module CalmLens
|
|
|
154
165
|
# @option options [Boolean] :confirm_purge must be true to purge
|
|
155
166
|
# @return [Hash] the asset
|
|
156
167
|
def decide(options)
|
|
157
|
-
call(options) do |api, request|
|
|
168
|
+
call(options, __method__) do |api, request|
|
|
158
169
|
request('POST', "/assets/#{Internal.segment(api['assetId'])}/decision", body: without(api, 'assetId'),
|
|
159
170
|
request_options: request)
|
|
160
171
|
end
|
|
@@ -168,7 +179,7 @@ module CalmLens
|
|
|
168
179
|
# @option options [Boolean] :rerun
|
|
169
180
|
# @return [Hash] the asset
|
|
170
181
|
def explain(options)
|
|
171
|
-
call(options) do |api, request|
|
|
182
|
+
call(options, __method__) do |api, request|
|
|
172
183
|
request('POST', "/assets/#{Internal.segment(api['assetId'])}/explain",
|
|
173
184
|
body: api['rerun'] ? { 'rerun' => true } : {}, request_options: request)
|
|
174
185
|
end
|
|
@@ -179,7 +190,7 @@ module CalmLens
|
|
|
179
190
|
# @option options [String] :asset_id
|
|
180
191
|
# @return [Hash] the asset
|
|
181
192
|
def delete_content(options)
|
|
182
|
-
call(options) do |api, request|
|
|
193
|
+
call(options, __method__) do |api, request|
|
|
183
194
|
request('DELETE', "/assets/#{Internal.segment(api['assetId'])}/content", request_options: request)
|
|
184
195
|
end
|
|
185
196
|
end
|
|
@@ -188,7 +199,7 @@ module CalmLens
|
|
|
188
199
|
# @option options [String] :asset_id
|
|
189
200
|
# @return [nil]
|
|
190
201
|
def delete(options)
|
|
191
|
-
call(options) do |api, request|
|
|
202
|
+
call(options, __method__) do |api, request|
|
|
192
203
|
request('DELETE', "/assets/#{Internal.segment(api['assetId'])}", request_options: request)
|
|
193
204
|
nil
|
|
194
205
|
end
|
|
@@ -203,7 +214,7 @@ module CalmLens
|
|
|
203
214
|
# @option options [String] :operation one of {BATCH_OPERATIONS}
|
|
204
215
|
# @return [Hash] `{'queued' => true}`
|
|
205
216
|
def batch(options)
|
|
206
|
-
call(options) do |api, request|
|
|
217
|
+
call(options, __method__) do |api, request|
|
|
207
218
|
ids = api['assetIds']
|
|
208
219
|
unless ids.is_a?(Array) && ids.size.between?(1, MAX_BATCH_ASSETS)
|
|
209
220
|
raise Internal.invalid("asset_ids must list 1 to #{MAX_BATCH_ASSETS} asset IDs")
|
|
@@ -221,7 +232,7 @@ module CalmLens
|
|
|
221
232
|
# @option options [String, nil] :folder_id
|
|
222
233
|
# @return [Hash] the asset
|
|
223
234
|
def move(options)
|
|
224
|
-
call(options) do |api, request|
|
|
235
|
+
call(options, __method__) do |api, request|
|
|
225
236
|
request('POST', "/assets/#{Internal.segment(api['assetId'])}/move",
|
|
226
237
|
body: { 'folderId' => Internal.required(api, 'folderId') }, request_options: request)
|
|
227
238
|
end
|
|
@@ -232,11 +243,11 @@ module CalmLens
|
|
|
232
243
|
# @option options [String] :resource `rendered` (the API's default) or `original`, which takes no lens or transform
|
|
233
244
|
# @option options [Integer] :expires_in_seconds default 900
|
|
234
245
|
# @option options [String] :lens a lens identifier
|
|
235
|
-
# @option options [String, Hash
|
|
236
|
-
#
|
|
246
|
+
# @option options [String, Hash] :transform a selector String or structured options, as for
|
|
247
|
+
# {CDN#get_asset_url}; they replace the lens's options of the same name
|
|
237
248
|
# @return [Hash] `{'url', 'expiresAt'}`
|
|
238
249
|
def create_signed_url(options)
|
|
239
|
-
call(options) do |api, request|
|
|
250
|
+
call(options, __method__) do |api, request|
|
|
240
251
|
path = "/assets/#{Internal.segment(api['assetId'])}/signed-url"
|
|
241
252
|
body = { 'expiresInSeconds' => DEFAULT_SIGNED_URL_SECONDS }.merge(without(api, 'assetId'))
|
|
242
253
|
check_signed_url(body)
|
|
@@ -258,6 +269,13 @@ module CalmLens
|
|
|
258
269
|
#
|
|
259
270
|
# The request carries an `Idempotency-Key` (generated unless you pass
|
|
260
271
|
# `request_options: {idempotency_key:}`), so a retried upload never creates a second asset.
|
|
272
|
+
#
|
|
273
|
+
# `content:` larger than the client's `part_size_bytes` (8 MiB by default) whose size is known and
|
|
274
|
+
# that can be read again (a binary String, base64, or a File, StringIO or other seekable IO) is
|
|
275
|
+
# sent as a multipart upload in parts of that size. Each part carries the key
|
|
276
|
+
# `<idempotency_key>:part-<n>` and is retried on its own; calling again with the same
|
|
277
|
+
# `idempotency_key` (at most 240 characters) resumes an interrupted upload. Pipes, sockets and
|
|
278
|
+
# text are sent in one request.
|
|
261
279
|
# @option options [String] :name
|
|
262
280
|
# @option options [Hash] :content
|
|
263
281
|
# @option options [Hash] :part `{type:, data:, is_final:}` for the first multipart part
|
|
@@ -270,7 +288,7 @@ module CalmLens
|
|
|
270
288
|
# {ModerationContext::USED_ITEMS} that fit in {ModerationContext::USED_CHARACTERS} characters.
|
|
271
289
|
# @return [Hash] the asset
|
|
272
290
|
def create(options)
|
|
273
|
-
call(options) { |api, request| submit(api, request, true) }
|
|
291
|
+
call(options, __method__) { |api, request| submit(api, request, true) }
|
|
274
292
|
end
|
|
275
293
|
|
|
276
294
|
# Uploads content and waits for the moderation decision where the content type supports it.
|
|
@@ -279,7 +297,7 @@ module CalmLens
|
|
|
279
297
|
# `allowed` or `limited` (reduce the reach of `limited` assets). Serve `age-restricted` assets
|
|
280
298
|
# only behind your own age check, through signed URLs.
|
|
281
299
|
def classify(options)
|
|
282
|
-
call(options) do |api, request|
|
|
300
|
+
call(options, __method__) do |api, request|
|
|
283
301
|
content = api['content']
|
|
284
302
|
if content.is_a?(Hash) && content.key?('plainText') && (!content['plainText'].is_a?(String) || content['plainText'].strip.empty?)
|
|
285
303
|
raise Internal.invalid('Text to classify must be a non-empty string', ErrorCodes::INVALID_UPLOAD_PAYLOAD)
|
|
@@ -296,8 +314,8 @@ module CalmLens
|
|
|
296
314
|
# @option options [Boolean] :queue
|
|
297
315
|
# @return [Hash] the asset; `classification.status` stays `uploading` until the final part.
|
|
298
316
|
def upload_part(options)
|
|
299
|
-
call(options) do |api, request|
|
|
300
|
-
|
|
317
|
+
call(options, __method__) do |api, request|
|
|
318
|
+
Internal.segment(api['assetId'])
|
|
301
319
|
part = api['part']
|
|
302
320
|
raise Internal.invalid('part is required') unless part.is_a?(Hash)
|
|
303
321
|
|
|
@@ -308,10 +326,7 @@ module CalmLens
|
|
|
308
326
|
end
|
|
309
327
|
|
|
310
328
|
body = UploadBody.from(part, multipart: true, part_size: !@session.defer_upload_checks)
|
|
311
|
-
|
|
312
|
-
query['queue'] = api['queue'] if api.key?('queue')
|
|
313
|
-
expect_asset(request('POST', path, query: query, raw: body, idempotent: true, request_options: request, upload: true,
|
|
314
|
-
headers: { 'Content-Type' => 'application/octet-stream', 'X-CalmLens-Upload' => 'part' }))
|
|
329
|
+
send_part(api['assetId'], number, part.fetch('isFinal', false), body, api.fetch('queue', nil), request)
|
|
315
330
|
end
|
|
316
331
|
end
|
|
317
332
|
|
|
@@ -365,6 +380,9 @@ module CalmLens
|
|
|
365
380
|
end
|
|
366
381
|
|
|
367
382
|
body = UploadBody.from(content, multipart: multipart, part_size: !@session.defer_upload_checks)
|
|
383
|
+
return upload_in_parts(options, request_options, queue, mime, body) if !multipart && splits?(content, body)
|
|
384
|
+
|
|
385
|
+
check_direct_size(mime, body.size) unless multipart || @session.defer_upload_checks
|
|
368
386
|
query = options.slice(*SUBMIT_QUERY_KEYS).merge('queue' => queue, 'contentType' => content['type'],
|
|
369
387
|
'sizeInBytes' => body.size)
|
|
370
388
|
query.merge!(first_part_query(options, content, mime, body)) if multipart
|
|
@@ -374,6 +392,75 @@ module CalmLens
|
|
|
374
392
|
headers: { 'Content-Type' => content['type'], 'X-CalmLens-Upload' => multipart ? 'part' : 'content' }))
|
|
375
393
|
end
|
|
376
394
|
|
|
395
|
+
# Whether content goes in parts: not text, of known size, readable again and larger than a part.
|
|
396
|
+
def splits?(content, body)
|
|
397
|
+
!content.key?('plainText') && !body.size.nil? && body.replayable? && body.size > @session.part_size_bytes
|
|
398
|
+
end
|
|
399
|
+
|
|
400
|
+
# Uploads `body` as a multipart upload of `part_size_bytes` parts and returns the final part's
|
|
401
|
+
# asset. Part `n` after the first carries the key `<key>:part-<n>`, so repeating the call with
|
|
402
|
+
# the same key replays the parts the API already has and sends the rest.
|
|
403
|
+
def upload_in_parts(options, request_options, queue, mime, body)
|
|
404
|
+
key = request_options[:idempotency_key] || SecureRandom.uuid
|
|
405
|
+
if key.length > MAX_SPLIT_IDEMPOTENCY_KEY_LENGTH
|
|
406
|
+
raise Internal.invalid("idempotency_key can have at most #{MAX_SPLIT_IDEMPOTENCY_KEY_LENGTH} characters " \
|
|
407
|
+
'for content uploaded in parts')
|
|
408
|
+
end
|
|
409
|
+
unless @session.defer_upload_checks
|
|
410
|
+
mime.start_with?('image/') ? check_image_size(body.size) : check_direct_size(mime, body.size)
|
|
411
|
+
end
|
|
412
|
+
|
|
413
|
+
total = body.size
|
|
414
|
+
part_size = @session.part_size_bytes
|
|
415
|
+
count = (total + part_size - 1) / part_size
|
|
416
|
+
piece = ->(number) { body.slice((number - 1) * part_size, [part_size, total - ((number - 1) * part_size)].min) }
|
|
417
|
+
first = piece.call(1)
|
|
418
|
+
type = options.dig('content', 'type')
|
|
419
|
+
query = options.slice(*SUBMIT_QUERY_KEYS).merge('queue' => queue, 'contentType' => type, 'sizeInBytes' => first.size,
|
|
420
|
+
'partNumber' => 1, 'isFinal' => false, 'totalSizeInBytes' => total)
|
|
421
|
+
asset = expect_asset(request('POST', project('assets'),
|
|
422
|
+
query: query, raw: first, idempotent: true, upload: true,
|
|
423
|
+
request_options: request_options.merge(idempotency_key: key),
|
|
424
|
+
headers: { 'Content-Type' => type, 'X-CalmLens-Upload' => 'part' }))
|
|
425
|
+
send_remaining_parts(asset, count, piece, key, request_options)
|
|
426
|
+
end
|
|
427
|
+
|
|
428
|
+
# Sends parts 2 to `count`. A part the API already has (its response was lost) is skipped, and a
|
|
429
|
+
# final part that finds the upload completed reads the asset back.
|
|
430
|
+
def send_remaining_parts(asset, count, piece, key, request_options)
|
|
431
|
+
id = asset['id']
|
|
432
|
+
number = 2
|
|
433
|
+
while number <= count
|
|
434
|
+
final = number == count
|
|
435
|
+
begin
|
|
436
|
+
asset = send_part(id, number, final, piece.call(number), nil,
|
|
437
|
+
request_options.merge(idempotency_key: "#{key}:part-#{number}"))
|
|
438
|
+
rescue Error => e
|
|
439
|
+
expected = e.details.is_a?(Hash) ? e.details['expectedPartNumber'] : nil
|
|
440
|
+
if e.code == ErrorCodes::INVALID_UPLOAD_PART_NUMBER && expected.is_a?(Integer) && expected > number && expected <= count
|
|
441
|
+
number = expected
|
|
442
|
+
next
|
|
443
|
+
end
|
|
444
|
+
if final && e.code == ErrorCodes::UPLOAD_NOT_OPEN
|
|
445
|
+
return retrieve('assetId' => id, 'request_options' => request_options.except(:idempotency_key))
|
|
446
|
+
end
|
|
447
|
+
|
|
448
|
+
raise
|
|
449
|
+
end
|
|
450
|
+
number += 1
|
|
451
|
+
end
|
|
452
|
+
asset
|
|
453
|
+
end
|
|
454
|
+
|
|
455
|
+
def send_part(asset_id, number, final, body, queue, request_options)
|
|
456
|
+
query = { 'partNumber' => number, 'isFinal' => final, 'sizeInBytes' => body.size }
|
|
457
|
+
query['queue'] = queue unless queue.nil?
|
|
458
|
+
headers = { 'Content-Type' => 'application/octet-stream', 'X-CalmLens-Upload' => 'part' }
|
|
459
|
+
expect_asset(request('POST', "/assets/#{Internal.segment(asset_id)}/part", query: query, raw: body, idempotent: true,
|
|
460
|
+
request_options: request_options, upload: true,
|
|
461
|
+
headers: headers))
|
|
462
|
+
end
|
|
463
|
+
|
|
377
464
|
# Keeps only the earlier posts CalmLens uses, which keeps a raw upload's query short.
|
|
378
465
|
def with_used_context(options)
|
|
379
466
|
return options unless options.key?('context')
|
|
@@ -392,7 +479,7 @@ module CalmLens
|
|
|
392
479
|
end
|
|
393
480
|
|
|
394
481
|
final = content.fetch('isFinal', false)
|
|
395
|
-
check_image_size(total || body.size
|
|
482
|
+
check_image_size(total || body.size) if mime.start_with?('image/') && !@session.defer_upload_checks
|
|
396
483
|
{ 'partNumber' => 1, 'isFinal' => final, 'totalSizeInBytes' => total }
|
|
397
484
|
end
|
|
398
485
|
|
|
@@ -423,25 +510,38 @@ module CalmLens
|
|
|
423
510
|
|
|
424
511
|
def check_mime(type)
|
|
425
512
|
mime = type.to_s.split(';').first.to_s.strip.downcase
|
|
426
|
-
|
|
427
|
-
raise
|
|
428
|
-
|
|
429
|
-
|
|
513
|
+
unless Internal.accepted_upload_mime_type?(mime)
|
|
514
|
+
raise InvalidArgumentError.new(ErrorCodes::INVALID_UPLOAD_MIME_TYPE,
|
|
515
|
+
"#{mime} is not a valid upload file type. Uploads must use a concrete file MIME " \
|
|
516
|
+
'type that CalmLens can store and process, such as image/png, image/jpeg, ' \
|
|
517
|
+
'application/pdf, or text/plain.', details: { 'mimeType' => type })
|
|
430
518
|
end
|
|
431
519
|
raise Internal.invalid('Content type cannot contain line breaks') unless type.match?(Internal::HEADER_VALUE)
|
|
432
520
|
|
|
433
521
|
mime
|
|
434
522
|
end
|
|
435
523
|
|
|
436
|
-
#
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
524
|
+
# Refuses one-request content whose known size is over the API's limit for its type, before
|
|
525
|
+
# any of it is sent.
|
|
526
|
+
def check_direct_size(mime, size)
|
|
527
|
+
return if size.nil?
|
|
528
|
+
|
|
529
|
+
limit, message = if mime.start_with?('image/')
|
|
530
|
+
[MAX_DIRECT_IMAGE_BYTES, 'Images uploaded in one request can be at most 10 MiB; upload larger images in parts']
|
|
531
|
+
elsif mime.start_with?('audio/', 'video/')
|
|
532
|
+
[MAX_MEDIA_BYTES, 'Audio and video files can be at most 50 MiB']
|
|
533
|
+
else
|
|
534
|
+
[MAX_FILE_BYTES, 'Files can be at most 5,000 MiB']
|
|
535
|
+
end
|
|
536
|
+
raise Internal.invalid(message, ErrorCodes::FILE_TOO_LARGE) if size > limit
|
|
537
|
+
end
|
|
538
|
+
|
|
539
|
+
# Images in parts total at most 20 MiB, at any size below that; a declared total is checked
|
|
540
|
+
# from the start.
|
|
541
|
+
def check_image_size(size)
|
|
542
|
+
return unless size > MAX_MULTIPART_IMAGE_BYTES
|
|
443
543
|
|
|
444
|
-
raise Internal.invalid('Multipart image uploads
|
|
544
|
+
raise Internal.invalid('Multipart image uploads cannot exceed 20MB.', ErrorCodes::MULTIPART_IMAGE_TOO_LARGE)
|
|
445
545
|
end
|
|
446
546
|
end
|
|
447
547
|
end
|
data/lib/calmlens/cdn.rb
CHANGED
|
@@ -3,6 +3,9 @@
|
|
|
3
3
|
module CalmLens
|
|
4
4
|
# CDN URLs and authenticated content delivery. Reach it through {ServerClient#cdn}.
|
|
5
5
|
class CDN < Resource
|
|
6
|
+
# The prefix of this resource's operations in Internal::OPTION_KEYS.
|
|
7
|
+
OPERATIONS = 'cdn'
|
|
8
|
+
|
|
6
9
|
RESOURCES = %w[original rendered placeholder].freeze
|
|
7
10
|
private_constant :RESOURCES
|
|
8
11
|
|
|
@@ -13,15 +16,18 @@ module CalmLens
|
|
|
13
16
|
# takes a lens or transform. A placeholder is a small blurred PNG that the CDN serves publicly for every
|
|
14
17
|
# asset whatever its visibility or moderation status; it needs no age confirmation.
|
|
15
18
|
# @option options [String] :lens a lens identifier
|
|
16
|
-
# @option options [String, Hash
|
|
17
|
-
# or structured
|
|
19
|
+
# @option options [String, Hash] :transform options applied in one pass: a selector such as
|
|
20
|
+
# `width=200,fit=cover`, or structured options such as `{width: 200, fit: 'cover'}`. With a lens,
|
|
21
|
+
# they replace the lens's options of the same name
|
|
18
22
|
# @option options [Hash] :viewer_claims viewer assertions, such as `{age_confirmed: true}`, sent as a JSON
|
|
19
23
|
# `viewerClaims` query parameter. The CDN serves an asset whose moderation status is `age-restricted` only
|
|
20
24
|
# with `age_confirmed: true`, even with a signed URL or the API key. Set it only where your app has already
|
|
21
|
-
# decided the viewer may see age-restricted content.
|
|
25
|
+
# decided the viewer may see age-restricted content. Your platform confirms the viewer's age; CalmLens does
|
|
26
|
+
# not verify this claim, and anyone can add it to a public URL. To control who loads an age-restricted
|
|
27
|
+
# asset, make it private and include the claim in a signed URL.
|
|
22
28
|
# @return [String]
|
|
23
29
|
def get_asset_url(options)
|
|
24
|
-
call(options) { |api, _request| url(api) }
|
|
30
|
+
call(options, __method__) { |api, _request| url(api) }
|
|
25
31
|
end
|
|
26
32
|
|
|
27
33
|
# Fetches content from the CDN with the client's API key.
|
|
@@ -44,7 +50,7 @@ module CalmLens
|
|
|
44
50
|
# end
|
|
45
51
|
# end
|
|
46
52
|
def serve(options, &)
|
|
47
|
-
method, url, headers, request = call(options) do |api, request_options|
|
|
53
|
+
method, url, headers, request = call(options, __method__) do |api, request_options|
|
|
48
54
|
init = api.fetch('init', {})
|
|
49
55
|
raise Internal.invalid('init must be a Hash') unless init.is_a?(Hash)
|
|
50
56
|
|
|
@@ -68,9 +74,10 @@ module CalmLens
|
|
|
68
74
|
|
|
69
75
|
url = "#{@session.cdn_base_url}/assets/#{Internal.segment(@session.project_id)}/" \
|
|
70
76
|
"#{Internal.segment(api['assetIdentifier'])}/#{resource}"
|
|
71
|
-
url += "/@#{Internal.segment(api['lens'])}" if api['lens']
|
|
72
77
|
transform = api['transform']
|
|
73
|
-
|
|
78
|
+
selector = [("@#{Internal.segment(api['lens'])}" if api['lens']),
|
|
79
|
+
(Transforms.path(transform) unless transform.nil? || transform == '')].compact.join(',')
|
|
80
|
+
url += "/#{selector}" unless selector.empty?
|
|
74
81
|
|
|
75
82
|
claims = api['viewerClaims']
|
|
76
83
|
if !claims.nil? && (!claims.is_a?(Hash) || (claims.key?('ageConfirmed') && ![true, false].include?(claims['ageConfirmed'])))
|
data/lib/calmlens/error_codes.rb
CHANGED
|
@@ -10,6 +10,8 @@ module CalmLens
|
|
|
10
10
|
INVALID_VIEWER_REGION = 'viewer/invalid-region'
|
|
11
11
|
VIEWER_REGION_REQUIRED = 'viewer/region-required'
|
|
12
12
|
REGIONAL_EVIDENCE_REQUIRED = 'asset/regional-evidence-required'
|
|
13
|
+
SIGNED_URL_EXPIRED = 'signed-url/expired'
|
|
14
|
+
SIGNED_URL_INVALID = 'signed-url/invalid'
|
|
13
15
|
INVALID_TYPE = 'validation/invalid-type'
|
|
14
16
|
TOO_SMALL = 'validation/too-small'
|
|
15
17
|
TOO_BIG = 'validation/too-big'
|
|
@@ -37,9 +39,11 @@ module CalmLens
|
|
|
37
39
|
API_KEY_NOT_ACCEPTED = 'auth/api-key-not-accepted'
|
|
38
40
|
PERMISSION_DENIED = 'auth/permission-denied'
|
|
39
41
|
FILE_TOO_LARGE = 'upload/file-too-large'
|
|
40
|
-
MULTIPART_IMAGE_TOO_SMALL = 'upload/multipart-image-too-small'
|
|
41
42
|
MULTIPART_IMAGE_TOO_LARGE = 'upload/multipart-image-too-large'
|
|
42
43
|
IMAGE_EXCEEDED_MAX_DIMENSIONS = 'upload/image-exceeded-max-dimensions'
|
|
44
|
+
IMAGE_INVALID_INPUT = 'image/invalid-input'
|
|
45
|
+
IMAGE_UNSUPPORTED_FORMAT = 'image/unsupported-format'
|
|
46
|
+
IMAGE_INPUT_LIMIT = 'image/input-limit'
|
|
43
47
|
UPLOAD_LIMIT_EXCEEDED = 'upload/limit-exceeded'
|
|
44
48
|
SPEND_LIMIT_REACHED = 'upload/spend-limit-reached'
|
|
45
49
|
INVALID_UPLOAD_MIME_TYPE = 'upload/invalid-mime-type'
|
|
@@ -80,6 +84,15 @@ module CalmLens
|
|
|
80
84
|
HASH_LIST_LIMIT_REACHED = 'hash-list/limit-reached'
|
|
81
85
|
HASH_LIST_ENTRY_LIMIT_REACHED = 'hash-list/entry-limit-reached'
|
|
82
86
|
HASH_LIST_ASSET_NOT_HASHED = 'hash-list/asset-not-hashed'
|
|
87
|
+
POLICY_NOT_FOUND = 'policy/not-found'
|
|
88
|
+
POLICY_REVISION_CONFLICT = 'policy/revision-conflict'
|
|
89
|
+
POLICY_PROJECTS_CHANGED = 'policy/projects-changed'
|
|
90
|
+
POLICY_IN_USE = 'policy/in-use'
|
|
91
|
+
POLICY_USED_BY_ANOTHER_PROJECT = 'policy/used-by-another-project'
|
|
92
|
+
POLICY_NO_PRESET = 'policy/no-preset'
|
|
93
|
+
POLICY_PRESET_VERSION_CONFLICT = 'policy/preset-version-conflict'
|
|
94
|
+
POLICY_PRESET_UPDATE_UNAVAILABLE = 'policy/preset-update-unavailable'
|
|
95
|
+
POLICY_AUTOMATIC_UPDATES_UNAVAILABLE = 'policy/automatic-updates-unavailable'
|
|
83
96
|
UPLOAD_PRESET_NOT_FOUND = 'upload-preset/not-found'
|
|
84
97
|
UPLOAD_PRESET_DISABLED = 'upload-preset/disabled'
|
|
85
98
|
UPLOAD_PRESET_TOKEN_REQUIRED = 'upload-preset/token-required'
|
|
@@ -112,6 +125,8 @@ module CalmLens
|
|
|
112
125
|
'viewer/invalid-region' => 400,
|
|
113
126
|
'viewer/region-required' => 403,
|
|
114
127
|
'asset/regional-evidence-required' => 403,
|
|
128
|
+
'signed-url/expired' => 403,
|
|
129
|
+
'signed-url/invalid' => 403,
|
|
115
130
|
'validation/invalid-type' => 400,
|
|
116
131
|
'validation/too-small' => 400,
|
|
117
132
|
'validation/too-big' => 400,
|
|
@@ -139,9 +154,11 @@ module CalmLens
|
|
|
139
154
|
'auth/api-key-not-accepted' => 403,
|
|
140
155
|
'auth/permission-denied' => 403,
|
|
141
156
|
'upload/file-too-large' => 400,
|
|
142
|
-
'upload/multipart-image-too-small' => 400,
|
|
143
157
|
'upload/multipart-image-too-large' => 400,
|
|
144
158
|
'upload/image-exceeded-max-dimensions' => 400,
|
|
159
|
+
'image/invalid-input' => 422,
|
|
160
|
+
'image/unsupported-format' => 422,
|
|
161
|
+
'image/input-limit' => 422,
|
|
145
162
|
'upload/limit-exceeded' => 403,
|
|
146
163
|
'upload/spend-limit-reached' => 402,
|
|
147
164
|
'upload/invalid-mime-type' => 400,
|
|
@@ -182,6 +199,15 @@ module CalmLens
|
|
|
182
199
|
'hash-list/limit-reached' => 409,
|
|
183
200
|
'hash-list/entry-limit-reached' => 409,
|
|
184
201
|
'hash-list/asset-not-hashed' => 409,
|
|
202
|
+
'policy/not-found' => 404,
|
|
203
|
+
'policy/revision-conflict' => 409,
|
|
204
|
+
'policy/projects-changed' => 409,
|
|
205
|
+
'policy/in-use' => 409,
|
|
206
|
+
'policy/used-by-another-project' => 409,
|
|
207
|
+
'policy/no-preset' => 400,
|
|
208
|
+
'policy/preset-version-conflict' => 409,
|
|
209
|
+
'policy/preset-update-unavailable' => 409,
|
|
210
|
+
'policy/automatic-updates-unavailable' => 409,
|
|
185
211
|
'upload-preset/not-found' => 404,
|
|
186
212
|
'upload-preset/disabled' => 403,
|
|
187
213
|
'upload-preset/token-required' => 401,
|
data/lib/calmlens/events.rb
CHANGED
|
@@ -3,13 +3,16 @@
|
|
|
3
3
|
module CalmLens
|
|
4
4
|
# The project's events feed. Reach it through {ServerClient#events}.
|
|
5
5
|
class Events < Resource
|
|
6
|
+
# The prefix of this resource's operations in Internal::OPTION_KEYS.
|
|
7
|
+
OPERATIONS = 'events'
|
|
8
|
+
|
|
6
9
|
# Lists events after a cursor, oldest first. Without `cursor`, returns no events and a
|
|
7
10
|
# `nextCursor` for the current time: save it, then pass each response's `nextCursor` as `cursor`.
|
|
8
11
|
# @option options [String] :cursor the `nextCursor` of a previous response
|
|
9
12
|
# @option options [Integer] :page_size 1 to 100, default 50
|
|
10
13
|
# @return [Hash] `{'items', 'nextCursor', 'hasMore'}`; `nextCursor` is never nil
|
|
11
14
|
def list(options = {})
|
|
12
|
-
call(options) { |api, request| request('GET', project('events'), query: api, request_options: request) }
|
|
15
|
+
call(options, __method__) { |api, request| request('GET', project('events'), query: api, request_options: request) }
|
|
13
16
|
end
|
|
14
17
|
|
|
15
18
|
# Iterates pages from `cursor` until `hasMore` is false. Persist each page's `nextCursor` once its
|
|
@@ -20,7 +23,7 @@ module CalmLens
|
|
|
20
23
|
def each_page(options = {})
|
|
21
24
|
return enum_for(:each_page, options) unless block_given?
|
|
22
25
|
|
|
23
|
-
api, request = call(options) { |*split| split }
|
|
26
|
+
api, request = call(options, __method__) { |*split| split }
|
|
24
27
|
sent = api.key?('cursor') ? [api['cursor']] : []
|
|
25
28
|
loop do
|
|
26
29
|
page = list(api.merge('request_options' => request))
|
data/lib/calmlens/folders.rb
CHANGED
|
@@ -3,6 +3,9 @@
|
|
|
3
3
|
module CalmLens
|
|
4
4
|
# Folders organise a project's assets. Reach it through {ServerClient#folders}.
|
|
5
5
|
class Folders < Resource
|
|
6
|
+
# The prefix of this resource's operations in Internal::OPTION_KEYS.
|
|
7
|
+
OPERATIONS = 'folders'
|
|
8
|
+
|
|
6
9
|
# Lists one page of child folders, sorted by name unless `ordering` says otherwise.
|
|
7
10
|
# @option options [String] :parent_id list this folder's children; omit for the project root
|
|
8
11
|
# @option options [String] :cursor the previous page's `nextCursor`
|
|
@@ -11,7 +14,7 @@ module CalmLens
|
|
|
11
14
|
# @option options [String] :search only children whose name contains it
|
|
12
15
|
# @return [Hash] a cursor page (items, nextCursor, hasMore, pageSize, ordering) with ancestors.
|
|
13
16
|
def list(options = {})
|
|
14
|
-
call(options) do |api, request|
|
|
17
|
+
call(options, __method__) do |api, request|
|
|
15
18
|
response = request('GET', project('folders'), query: api, request_options: request)
|
|
16
19
|
valid = response.is_a?(Hash) && response['items'].is_a?(Array) && response['ancestors'].is_a?(Array) &&
|
|
17
20
|
[true, false].include?(response['hasMore'])
|
|
@@ -25,13 +28,13 @@ module CalmLens
|
|
|
25
28
|
# @option options [String, nil] :parent_id
|
|
26
29
|
# @return [Hash] the folder
|
|
27
30
|
def create(options)
|
|
28
|
-
call(options) { |api, request| request('POST', project('folders'), body: api, request_options: request) }
|
|
31
|
+
call(options, __method__) { |api, request| request('POST', project('folders'), body: api, request_options: request) }
|
|
29
32
|
end
|
|
30
33
|
|
|
31
34
|
# @option options [String] :folder_id
|
|
32
35
|
# @return [Hash] the folder
|
|
33
36
|
def retrieve(options)
|
|
34
|
-
call(options) do |api, request|
|
|
37
|
+
call(options, __method__) do |api, request|
|
|
35
38
|
request('GET', "/folders/#{Internal.segment(api['folderId'])}", request_options: request)
|
|
36
39
|
end
|
|
37
40
|
end
|
|
@@ -40,7 +43,7 @@ module CalmLens
|
|
|
40
43
|
# @option options [String] :name
|
|
41
44
|
# @return [Hash] the folder
|
|
42
45
|
def update(options)
|
|
43
|
-
call(options) do |api, request|
|
|
46
|
+
call(options, __method__) do |api, request|
|
|
44
47
|
request('PATCH', "/folders/#{Internal.segment(api['folderId'])}", body: without(api, 'folderId'),
|
|
45
48
|
request_options: request)
|
|
46
49
|
end
|
|
@@ -51,7 +54,7 @@ module CalmLens
|
|
|
51
54
|
# @option options [String, nil] :parent_id
|
|
52
55
|
# @return [Hash] the folder
|
|
53
56
|
def move(options)
|
|
54
|
-
call(options) do |api, request|
|
|
57
|
+
call(options, __method__) do |api, request|
|
|
55
58
|
request('POST', "/folders/#{Internal.segment(api['folderId'])}/move",
|
|
56
59
|
body: { 'parentId' => Internal.required(api, 'parentId') }, request_options: request)
|
|
57
60
|
end
|
|
@@ -61,7 +64,7 @@ module CalmLens
|
|
|
61
64
|
# @option options [String] :folder_id
|
|
62
65
|
# @return [nil]
|
|
63
66
|
def delete(options)
|
|
64
|
-
call(options) do |api, request|
|
|
67
|
+
call(options, __method__) do |api, request|
|
|
65
68
|
request('DELETE', "/folders/#{Internal.segment(api['folderId'])}", request_options: request)
|
|
66
69
|
nil
|
|
67
70
|
end
|