calmlens 0.20.0 → 0.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +3 -1
- data/lib/calmlens/assets.rb +107 -98
- data/lib/calmlens/cdn.rb +197 -41
- data/lib/calmlens/error_codes.rb +80 -84
- data/lib/calmlens/errors.rb +2 -2
- data/lib/calmlens/events.rb +50 -1
- data/lib/calmlens/folders.rb +14 -9
- data/lib/calmlens/hash_lists.rb +29 -9
- data/lib/calmlens/internal.rb +9 -19
- data/lib/calmlens/lenses.rb +79 -6
- data/lib/calmlens/option_keys.rb +46 -25
- data/lib/calmlens/policies.rb +50 -18
- data/lib/calmlens/resource.rb +24 -6
- data/lib/calmlens/server_client.rb +8 -6
- data/lib/calmlens/session.rb +29 -12
- data/lib/calmlens/testing/asset_filters.rb +39 -0
- data/lib/calmlens/testing/classification.rb +8 -2
- data/lib/calmlens/testing/fake_api.rb +682 -368
- data/lib/calmlens/testing/mock_decision.rb +5 -4
- data/lib/calmlens/testing/rsql.rb +245 -0
- data/lib/calmlens/testing/test_input_detections.rb +61 -0
- data/lib/calmlens/testing/test_input_scenario.rb +6 -2
- data/lib/calmlens/testing/test_inputs.rb +31 -10
- data/lib/calmlens/testing.rb +36 -12
- data/lib/calmlens/transform_rules.rb +1 -1
- data/lib/calmlens/upload_mime_types.rb +9 -1
- data/lib/calmlens/upload_tokens.rb +3 -3
- data/lib/calmlens/version.rb +1 -1
- data/lib/calmlens/webhooks.rb +58 -13
- data/sig/calmlens.rbs +87 -13
- data/sig/error_codes.rbs +18 -20
- data/sig/qualifiers.rbs +2 -0
- data/sig/types.rbs +326 -187
- metadata +5 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: b99a6fc15b38b4dd664a1df33e33eb8943cbd57483e16d7b7224f93cbc23d2a1
|
|
4
|
+
data.tar.gz: 668bf0e79e7fad1838be8d4e3c0bc536a379ea0de4cf3943a97057ed5025a72a
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 00023a2b77c5dd78ebd7f0563ab9d152bede63882abc80c7a3e47d667f8e2d3ec54bade2c18fa4bd901167da1e8e3b7be2181491d1b3d5d1700bca568403d6ba
|
|
7
|
+
data.tar.gz: 7e8a4bcab13c13c6278169e15a54bec41a9fc5849c366dbff1ef0208b6fceeeece777c157c52ea2286bfcca2135c64e41c8b878c6456e534fce51fd368dd61fe
|
data/README.md
CHANGED
|
@@ -16,6 +16,8 @@ gem "calmlens"
|
|
|
16
16
|
|
|
17
17
|
The gem has no runtime dependencies: it uses only the standard library (Net::HTTP, JSON, OpenSSL). It ships YARD documentation and RBS signatures in `sig/`, so Steep and other RBS tools can check your calls and the response keys you read.
|
|
18
18
|
|
|
19
|
+
Versions follow semantic versioning: see [Versioning and support](https://calmlens.com/docs/sdk#versioning-and-support).
|
|
20
|
+
|
|
19
21
|
## Quick start
|
|
20
22
|
|
|
21
23
|
Create an API key and copy your project ID in the CalmLens portal, and keep both on your server.
|
|
@@ -77,7 +79,7 @@ Every method is listed in the [Ruby SDK documentation](https://calmlens.com/docs
|
|
|
77
79
|
|
|
78
80
|
`CalmLens::MockServerClient` from `calmlens/testing` runs the real client against an in-memory stand-in for the API and CDN, so you can test your integration offline. See [Ruby testing](https://calmlens.com/docs/sdk/ruby/testing).
|
|
79
81
|
|
|
80
|
-
`CalmLens::TestInputs` signs test inputs with your project's test input key from the portal: `generate_text_code` turns a scenario, preset or signal into a text code, `paint_image` paints it as a test image, and `generate_media_code` gives a phrase for audio and video, so the real API returns the scenario you declared.
|
|
82
|
+
`CalmLens::TestInputs` signs test inputs with your project's test input key from the portal: `generate_text_code` turns a scenario, preset or signal into a text code, `paint_image` paints it as a test image, and `generate_media_code` gives a phrase for audio and video, so the real API returns the scenario you declared in projects with "Recognise test inputs" turned on. Submit a text code as the whole text and a test image as the whole upload.
|
|
81
83
|
|
|
82
84
|
## Links
|
|
83
85
|
|
data/lib/calmlens/assets.rb
CHANGED
|
@@ -15,30 +15,27 @@ module CalmLens
|
|
|
15
15
|
# The prefix of this resource's operations in Internal::OPTION_KEYS.
|
|
16
16
|
OPERATIONS = 'assets'
|
|
17
17
|
|
|
18
|
-
SUBMIT_QUERY_KEYS = %w[name description processingOptions visibility metadata authorId context externalId
|
|
19
|
-
folderId uploadPresetId].freeze
|
|
18
|
+
SUBMIT_QUERY_KEYS = %w[name description processingOptions visibility metadata tags authorId context externalId
|
|
19
|
+
externalUrl folderId uploadPresetId].freeze
|
|
20
20
|
MAX_MULTIPART_IMAGE_BYTES = 20 * 1024 * 1024
|
|
21
21
|
# The API's limits for one-request uploads: images up to 10 MiB (larger ones are uploaded in
|
|
22
22
|
# parts), audio and video up to 50 MiB and other files up to 5,000 MiB.
|
|
23
23
|
MAX_DIRECT_IMAGE_BYTES = 10 * 1024 * 1024
|
|
24
24
|
MAX_MEDIA_BYTES = 50 * 1024 * 1024
|
|
25
25
|
MAX_FILE_BYTES = 5000 * 1024 * 1024
|
|
26
|
+
# Audio and video sent in parts can be up to 2 GiB; the API holds free uploads, made before
|
|
27
|
+
# billing is set up, to MAX_MEDIA_BYTES itself.
|
|
28
|
+
MAX_LONG_MEDIA_BYTES = 2 * 1024 * 1024 * 1024
|
|
26
29
|
# The longest `idempotency_key` of an upload sent in parts: each part's key adds `:part-<n>`.
|
|
27
30
|
MAX_SPLIT_IDEMPOTENCY_KEY_LENGTH = 240
|
|
28
|
-
# The
|
|
29
|
-
#
|
|
30
|
-
|
|
31
|
-
# Processing statuses after which no decision will arrive without a retry.
|
|
32
|
-
FINISHED_PROCESSING_STATUSES = %w[failed].freeze
|
|
31
|
+
# The classification statuses of a settled asset: `completed` (with its decision in
|
|
32
|
+
# `classification.moderation.status`) or `failed`. {#wait_for_decision} returns at the first.
|
|
33
|
+
SETTLED_STATUSES = %w[completed failed].freeze
|
|
33
34
|
DECISION_POLL_FIRST_DELAY = 0.5
|
|
34
35
|
# Each wait between decision checks is this many times the one before, up to DECISION_POLL_MAX_DELAY.
|
|
35
36
|
DECISION_POLL_FACTOR = 1.5
|
|
36
37
|
DECISION_POLL_MAX_DELAY = 2.0
|
|
37
38
|
DEFAULT_DECISION_TIMEOUT = 60
|
|
38
|
-
# The lifetime of a signed URL when none is given: 15 minutes.
|
|
39
|
-
DEFAULT_SIGNED_URL_SECONDS = 15 * 60
|
|
40
|
-
# The longest signed URL lifetime the API accepts: 24 hours.
|
|
41
|
-
MAX_SIGNED_URL_SECONDS = 24 * 60 * 60
|
|
42
39
|
# The operations {#batch} runs: `delete` deletes each asset with its child assets and stored
|
|
43
40
|
# content; `remove-content` removes its stored content and keeps the record.
|
|
44
41
|
BATCH_OPERATIONS = %w[delete remove-content].freeze
|
|
@@ -48,18 +45,20 @@ module CalmLens
|
|
|
48
45
|
# Searches assets by meaning or keyword.
|
|
49
46
|
# @option options [String] :query
|
|
50
47
|
# @option options [String] :mode `hybrid`, `semantic` or `keyword`
|
|
51
|
-
# @option options [Integer] :
|
|
48
|
+
# @option options [Integer] :page_size how many results, 1 to 20 (default 10)
|
|
52
49
|
# @return [Hash] `{'query', 'items' => [{'asset', 'score', 'matchedFields'}], 'semanticStatus'}`
|
|
53
50
|
def search(options = {})
|
|
54
51
|
call(options, __method__) { |api, request| request('GET', project('assets/search'), query: api, request_options: request) }
|
|
55
52
|
end
|
|
56
53
|
|
|
57
|
-
# Lists one page of assets.
|
|
54
|
+
# Lists one page of assets, newest first unless `sort` says otherwise.
|
|
58
55
|
# @option options [String] :cursor from the previous page
|
|
59
56
|
# @option options [Integer] :page_size 1 to 100, default 50
|
|
60
|
-
# @option options [String] :filter an RSQL filter such as `classification.moderation.status==allowed
|
|
61
|
-
#
|
|
62
|
-
# @
|
|
57
|
+
# @option options [String, nil] :filter an RSQL filter such as `classification.moderation.status==allowed`;
|
|
58
|
+
# `folderId==fldr_…` lists one folder's assets and `folderId==null` the project root's
|
|
59
|
+
# @option options [String] :sort `createdAt` and `id`, comma-separated, descending when prefixed
|
|
60
|
+
# with `-`, such as `-createdAt,id`; `-createdAt` by default
|
|
61
|
+
# @return [Hash] a cursor page: items, nextCursor and hasMore.
|
|
63
62
|
def list(options = {})
|
|
64
63
|
call(options, __method__) do |api, request|
|
|
65
64
|
request('GET', project('assets'), query: api, request_options: request)
|
|
@@ -73,14 +72,14 @@ module CalmLens
|
|
|
73
72
|
def each(options = {}, &block)
|
|
74
73
|
return enum_for(:each, options) unless block
|
|
75
74
|
|
|
76
|
-
|
|
77
|
-
listing = pages(api['cursor']) { |index| list(api.merge('cursor' => index, 'request_options' => request)) }
|
|
78
|
-
listing.each { |page| page['items'].each(&block) }
|
|
79
|
-
nil
|
|
75
|
+
each_item(options, __method__, method(:list), &block)
|
|
80
76
|
end
|
|
81
77
|
|
|
82
78
|
# Lists one page of an asset's children, such as frames extracted from a video.
|
|
83
79
|
# @option options [String] :asset_id
|
|
80
|
+
# @option options [String] :cursor
|
|
81
|
+
# @option options [Integer] :page_size
|
|
82
|
+
# @option options [String] :sort as for {#list}
|
|
84
83
|
# @return [Hash] a page like {#list}
|
|
85
84
|
def list_children(options)
|
|
86
85
|
call(options, __method__) do |api, request|
|
|
@@ -89,6 +88,30 @@ module CalmLens
|
|
|
89
88
|
end
|
|
90
89
|
end
|
|
91
90
|
|
|
91
|
+
# Lists one page of an audio or video asset's findings timeline, in time order.
|
|
92
|
+
# `classification.moderation.timeline.segmentCount` says how many segments there are. Pass each
|
|
93
|
+
# page's `nextCursor` back unchanged; cursors are opaque.
|
|
94
|
+
# @option options [String] :asset_id
|
|
95
|
+
# @option options [String] :cursor
|
|
96
|
+
# @option options [Integer] :page_size
|
|
97
|
+
# @return [Hash] a cursor page of timeline segments: items, nextCursor and hasMore.
|
|
98
|
+
def list_timeline(options)
|
|
99
|
+
call(options, __method__) do |api, request|
|
|
100
|
+
query = without(api, 'assetId')
|
|
101
|
+
request('GET', project("assets/#{Internal.segment(api['assetId'])}/timeline"), query: query, request_options: request)
|
|
102
|
+
end
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
# Iterates every segment of an audio or video asset's findings timeline, in time order.
|
|
106
|
+
# @param options [Hash] the options of {#list_timeline}
|
|
107
|
+
# @yieldparam segment [Hash]
|
|
108
|
+
# @return [Enumerator, nil] an Enumerator when no block is given.
|
|
109
|
+
def each_timeline_segment(options, &block)
|
|
110
|
+
return enum_for(:each_timeline_segment, options) unless block
|
|
111
|
+
|
|
112
|
+
each_item(options, __method__, method(:list_timeline), &block)
|
|
113
|
+
end
|
|
114
|
+
|
|
92
115
|
# Iterates every child of an asset, fetching pages as needed.
|
|
93
116
|
# @param options [Hash] the options of {#list_children}
|
|
94
117
|
# @yieldparam asset [Hash]
|
|
@@ -96,10 +119,7 @@ module CalmLens
|
|
|
96
119
|
def each_child(options, &block)
|
|
97
120
|
return enum_for(:each_child, options) unless block
|
|
98
121
|
|
|
99
|
-
|
|
100
|
-
listing = pages(api['cursor']) { |index| list_children(api.merge('cursor' => index, 'request_options' => request)) }
|
|
101
|
-
listing.each { |page| page['items'].each(&block) }
|
|
102
|
-
nil
|
|
122
|
+
each_item(options, __method__, method(:list_children), &block)
|
|
103
123
|
end
|
|
104
124
|
|
|
105
125
|
# @option options [String] :asset_id
|
|
@@ -110,18 +130,19 @@ module CalmLens
|
|
|
110
130
|
end
|
|
111
131
|
end
|
|
112
132
|
|
|
113
|
-
# Waits until an asset
|
|
114
|
-
# `
|
|
115
|
-
#
|
|
116
|
-
# longer each time after that, up to 2
|
|
133
|
+
# Waits until an asset is settled, its `classification.status` `completed` (with its decision in
|
|
134
|
+
# `classification.moderation.status`) or `failed`, and returns the asset. A `queued` or `processing`
|
|
135
|
+
# asset is still waiting, even when a reviewer has already decided it. It retrieves the asset,
|
|
136
|
+
# waiting 0.5 seconds before the second check and 1.5 times longer each time after that, up to 2
|
|
137
|
+
# seconds.
|
|
117
138
|
# Webhooks report decisions without polling.
|
|
118
139
|
#
|
|
119
140
|
# `timeout:` is the whole wait in seconds; `request_options: {timeout:}` still limits each request.
|
|
120
141
|
# To stop waiting early, run the call in a thread you can stop: it only sleeps and sends requests.
|
|
121
142
|
# @option options [String] :asset_id
|
|
122
143
|
# @option options [Numeric] :timeout seconds to wait for a decision (default 60)
|
|
123
|
-
# @return [Hash] the
|
|
124
|
-
# @raise [TimeoutError] `sdk/timeout` when
|
|
144
|
+
# @return [Hash] the settled asset
|
|
145
|
+
# @raise [TimeoutError] `sdk/timeout` when the asset isn't settled in time; `details` holds
|
|
125
146
|
# `'assetId'` and the last `'classificationStatus'` and `'moderationStatus'`.
|
|
126
147
|
def wait_for_decision(options)
|
|
127
148
|
call(options, __method__) do |api, request|
|
|
@@ -155,7 +176,7 @@ module CalmLens
|
|
|
155
176
|
|
|
156
177
|
# Records a reviewer's decision, replacing the policy's: `classification.moderation.status`
|
|
157
178
|
# becomes the decision, the decision and note join `classification.moderation.manualReviews` and
|
|
158
|
-
# the matching `asset.<status>` event is sent with `
|
|
179
|
+
# the matching `asset.<status>` event is sent with the API key as its `actor`. An asset still pending
|
|
159
180
|
# moderation can only be blocked or purged. Purging needs `confirm_purge: true` and the
|
|
160
181
|
# `assets:delete` permission, and permanently deletes the stored content. Not retried: it carries
|
|
161
182
|
# no `Idempotency-Key`.
|
|
@@ -173,8 +194,12 @@ module CalmLens
|
|
|
173
194
|
|
|
174
195
|
# Completes the policy evidence behind a decided asset's moderation decision: the report's
|
|
175
196
|
# `classification.moderation.explanationComplete` becomes `true` and every action stays as decided.
|
|
176
|
-
# With `rerun: true` the asset is moderated again with the project's current policy and
|
|
177
|
-
#
|
|
197
|
+
# With `rerun: true` the asset is moderated again with the project's current policy and an
|
|
198
|
+
# `asset.rerun-requested` event is recorded; use it when the policy changed since the decision
|
|
199
|
+
# (`asset/policy-changed`). The asset returns unchanged: it keeps its decision until the re-run's
|
|
200
|
+
# is stored, reported by `asset.classified` and `asset.<status>` events, and a failed re-run keeps
|
|
201
|
+
# it. A reviewed asset raises `asset/reviewed`, and one whose content is gone
|
|
202
|
+
# `asset/content-unavailable`.
|
|
178
203
|
# @option options [String] :asset_id
|
|
179
204
|
# @option options [Boolean] :rerun
|
|
180
205
|
# @return [Hash] the asset
|
|
@@ -189,13 +214,14 @@ module CalmLens
|
|
|
189
214
|
# `requested`); the record and classification results are kept.
|
|
190
215
|
# @option options [String] :asset_id
|
|
191
216
|
# @return [Hash] the asset
|
|
192
|
-
def
|
|
217
|
+
def remove_content(options)
|
|
193
218
|
call(options, __method__) do |api, request|
|
|
194
219
|
request('DELETE', "/assets/#{Internal.segment(api['assetId'])}/content", request_options: request)
|
|
195
220
|
end
|
|
196
221
|
end
|
|
197
222
|
|
|
198
|
-
# Deletes the asset record and its stored content.
|
|
223
|
+
# Deletes the asset record and its stored content. An asset extracted from another (one with a
|
|
224
|
+
# `parentId`) raises `asset/child-not-deletable`: delete its parent instead.
|
|
199
225
|
# @option options [String] :asset_id
|
|
200
226
|
# @return [nil]
|
|
201
227
|
def delete(options)
|
|
@@ -207,12 +233,14 @@ module CalmLens
|
|
|
207
233
|
|
|
208
234
|
# Deletes up to 500 assets (`operation: 'delete'`) or removes their stored content
|
|
209
235
|
# (`'remove-content'`) in one request. Every asset must exist in the project and the key needs
|
|
210
|
-
# `assets:delete` for its folder,
|
|
236
|
+
# `assets:delete` for its folder, and a deletion can't name an asset extracted from another
|
|
237
|
+
# (`asset/child-not-deletable`), or nothing is queued. Returns once the operation is queued; the
|
|
211
238
|
# work runs in the background and records `asset.deleted` or `asset.content-removed` for each
|
|
212
|
-
# asset
|
|
239
|
+
# asset, with a `batch-id` fact naming `batchId`. Assets under a safety hold are skipped. Not
|
|
240
|
+
# retried: it carries no `Idempotency-Key`.
|
|
213
241
|
# @option options [Array<String>] :asset_ids 1 to 500 asset IDs
|
|
214
242
|
# @option options [String] :operation one of {BATCH_OPERATIONS}
|
|
215
|
-
# @return [Hash] `{'
|
|
243
|
+
# @return [Hash] `{'batchId', 'status' => 'queued', 'assetCount'}`
|
|
216
244
|
def batch(options)
|
|
217
245
|
call(options, __method__) do |api, request|
|
|
218
246
|
ids = api['assetIds']
|
|
@@ -238,27 +266,8 @@ module CalmLens
|
|
|
238
266
|
end
|
|
239
267
|
end
|
|
240
268
|
|
|
241
|
-
#
|
|
242
|
-
#
|
|
243
|
-
# @option options [String] :resource `rendered` (the API's default) or `original`, which takes no lens or transform
|
|
244
|
-
# @option options [Integer] :expires_in_seconds default 900
|
|
245
|
-
# @option options [String] :lens a lens identifier
|
|
246
|
-
# @option options [String, Hash] :transform a selector String or structured options, as for
|
|
247
|
-
# {CDN#get_asset_url}; they replace the lens's options of the same name
|
|
248
|
-
# @return [Hash] `{'url', 'expiresAt'}`
|
|
249
|
-
def create_signed_url(options)
|
|
250
|
-
call(options, __method__) do |api, request|
|
|
251
|
-
path = "/assets/#{Internal.segment(api['assetId'])}/signed-url"
|
|
252
|
-
body = { 'expiresInSeconds' => DEFAULT_SIGNED_URL_SECONDS }.merge(without(api, 'assetId'))
|
|
253
|
-
check_signed_url(body)
|
|
254
|
-
body['transform'] = Transforms.selector(body['transform']) if body.key?('transform')
|
|
255
|
-
request('POST', path,
|
|
256
|
-
body: body,
|
|
257
|
-
request_options: request)
|
|
258
|
-
end
|
|
259
|
-
end
|
|
260
|
-
|
|
261
|
-
# Uploads content and queues moderation; the call returns before classification finishes.
|
|
269
|
+
# Uploads content and returns without waiting for its decision (`wait=false`), with
|
|
270
|
+
# `classification.status` `queued`.
|
|
262
271
|
# Watch for `asset.allowed`, `asset.limited`, `asset.age-restricted`, `asset.blocked` and
|
|
263
272
|
# `asset.purged` events or retrieve the asset later.
|
|
264
273
|
#
|
|
@@ -281,6 +290,7 @@ module CalmLens
|
|
|
281
290
|
# @option options [Hash] :part `{type:, data:, is_final:}` for the first multipart part
|
|
282
291
|
# @option options [Integer] :total_size_in_bytes the whole file's size, for multipart uploads
|
|
283
292
|
# @option options [String] :url
|
|
293
|
+
# @option options [Array<String>] :tags tags for the asset, such as `['catalogue', 'backpacks']`
|
|
284
294
|
# @option options [String] :author_id your identifier for who wrote the content; CalmLens only
|
|
285
295
|
# compares it with the author IDs in `context`, to tell which posts share a speaker.
|
|
286
296
|
# @option options [Hash] :context `{items: [{text:, author_id:}]}`, earlier posts in the
|
|
@@ -288,7 +298,7 @@ module CalmLens
|
|
|
288
298
|
# {ModerationContext::USED_ITEMS} that fit in {ModerationContext::USED_CHARACTERS} characters.
|
|
289
299
|
# @return [Hash] the asset
|
|
290
300
|
def create(options)
|
|
291
|
-
call(options, __method__) { |api, request| submit(api, request,
|
|
301
|
+
call(options, __method__) { |api, request| submit(api, request, false) }
|
|
292
302
|
end
|
|
293
303
|
|
|
294
304
|
# Uploads content and waits for the moderation decision where the content type supports it.
|
|
@@ -303,15 +313,17 @@ module CalmLens
|
|
|
303
313
|
raise Internal.invalid('Text to classify must be a non-empty string', ErrorCodes::INVALID_UPLOAD_PAYLOAD)
|
|
304
314
|
end
|
|
305
315
|
|
|
306
|
-
submit(api, request,
|
|
316
|
+
submit(api, request, true)
|
|
307
317
|
end
|
|
308
318
|
end
|
|
309
319
|
|
|
310
320
|
# Uploads a later part of a multipart upload started with {#create} or {#classify} and `part:`.
|
|
311
|
-
#
|
|
321
|
+
# Whether the final part waits for the decision was chosen when the upload started; pass `wait:`
|
|
322
|
+
# only to override it.
|
|
312
323
|
# @option options [String] :asset_id
|
|
313
324
|
# @option options [Hash] :part `{data:, part_number:, is_final:}`; part numbers start at 2.
|
|
314
|
-
# @option options [Boolean] :
|
|
325
|
+
# @option options [Boolean] :wait `true` to wait for the decision, `false` to return with the
|
|
326
|
+
# asset `queued`
|
|
315
327
|
# @return [Hash] the asset; `classification.status` stays `uploading` until the final part.
|
|
316
328
|
def upload_part(options)
|
|
317
329
|
call(options, __method__) do |api, request|
|
|
@@ -326,7 +338,7 @@ module CalmLens
|
|
|
326
338
|
end
|
|
327
339
|
|
|
328
340
|
body = UploadBody.from(part, multipart: true, part_size: !@session.defer_upload_checks)
|
|
329
|
-
send_part(api['assetId'], number, part.fetch('isFinal', false), body, api.fetch('
|
|
341
|
+
send_part(api['assetId'], number, part.fetch('isFinal', false), body, api.fetch('wait', nil), request)
|
|
330
342
|
end
|
|
331
343
|
end
|
|
332
344
|
|
|
@@ -342,12 +354,12 @@ module CalmLens
|
|
|
342
354
|
asset = retrieve('assetId' => asset_id, 'request_options' => request_options)
|
|
343
355
|
processing = asset.dig('classification', 'status')
|
|
344
356
|
moderation = asset.dig('classification', 'moderation', 'status')
|
|
345
|
-
return asset if
|
|
357
|
+
return asset if SETTLED_STATUSES.include?(processing)
|
|
346
358
|
|
|
347
359
|
remaining = deadline - @session.monotonic
|
|
348
360
|
unless remaining.positive?
|
|
349
361
|
raise TimeoutError.new(SdkErrorCodes::TIMEOUT,
|
|
350
|
-
"Asset #{asset['id']}
|
|
362
|
+
"Asset #{asset['id']} was not settled after #{timeout}s " \
|
|
351
363
|
"(processing #{processing}, moderation #{moderation})",
|
|
352
364
|
details: { 'assetId' => asset['id'], 'classificationStatus' => processing,
|
|
353
365
|
'moderationStatus' => moderation })
|
|
@@ -358,7 +370,7 @@ module CalmLens
|
|
|
358
370
|
end
|
|
359
371
|
end
|
|
360
372
|
|
|
361
|
-
def submit(options, request_options,
|
|
373
|
+
def submit(options, request_options, wait)
|
|
362
374
|
raise Internal.invalid('Asset name is required', ErrorCodes::INVALID_UPLOAD_PAYLOAD) if options['name'].to_s.empty?
|
|
363
375
|
unless %w[content part url].one? { |key| options.key?(key) } && !options.key?('isFinal')
|
|
364
376
|
raise Internal.invalid('Supply exactly one of content, part, or url; isFinal belongs in part',
|
|
@@ -366,7 +378,7 @@ module CalmLens
|
|
|
366
378
|
end
|
|
367
379
|
options = with_used_context(options)
|
|
368
380
|
if options.key?('url')
|
|
369
|
-
return expect_asset(request('POST', project('assets'), query: { '
|
|
381
|
+
return expect_asset(request('POST', project('assets'), query: { 'wait' => wait }, body: options,
|
|
370
382
|
idempotent: true, request_options: request_options))
|
|
371
383
|
end
|
|
372
384
|
|
|
@@ -380,10 +392,10 @@ module CalmLens
|
|
|
380
392
|
end
|
|
381
393
|
|
|
382
394
|
body = UploadBody.from(content, multipart: multipart, part_size: !@session.defer_upload_checks)
|
|
383
|
-
return upload_in_parts(options, request_options,
|
|
395
|
+
return upload_in_parts(options, request_options, wait, mime, body) if !multipart && splits?(content, body)
|
|
384
396
|
|
|
385
|
-
|
|
386
|
-
query = options.slice(*SUBMIT_QUERY_KEYS).merge('
|
|
397
|
+
check_upload_size(mime, body.size) unless multipart || @session.defer_upload_checks
|
|
398
|
+
query = options.slice(*SUBMIT_QUERY_KEYS).merge('wait' => wait, 'contentType' => content['type'],
|
|
387
399
|
'sizeInBytes' => body.size)
|
|
388
400
|
query.merge!(first_part_query(options, content, mime, body)) if multipart
|
|
389
401
|
expect_asset(request('POST', project('assets'),
|
|
@@ -400,14 +412,14 @@ module CalmLens
|
|
|
400
412
|
# Uploads `body` as a multipart upload of `part_size_bytes` parts and returns the final part's
|
|
401
413
|
# asset. Part `n` after the first carries the key `<key>:part-<n>`, so repeating the call with
|
|
402
414
|
# the same key replays the parts the API already has and sends the rest.
|
|
403
|
-
def upload_in_parts(options, request_options,
|
|
415
|
+
def upload_in_parts(options, request_options, wait, mime, body)
|
|
404
416
|
key = request_options[:idempotency_key] || SecureRandom.uuid
|
|
405
417
|
if key.length > MAX_SPLIT_IDEMPOTENCY_KEY_LENGTH
|
|
406
418
|
raise Internal.invalid("idempotency_key can have at most #{MAX_SPLIT_IDEMPOTENCY_KEY_LENGTH} characters " \
|
|
407
419
|
'for content uploaded in parts')
|
|
408
420
|
end
|
|
409
421
|
unless @session.defer_upload_checks
|
|
410
|
-
mime.start_with?('image/') ? check_image_size(body.size) :
|
|
422
|
+
mime.start_with?('image/') ? check_image_size(body.size) : check_upload_size(mime, body.size, in_parts: true)
|
|
411
423
|
end
|
|
412
424
|
|
|
413
425
|
total = body.size
|
|
@@ -416,7 +428,7 @@ module CalmLens
|
|
|
416
428
|
piece = ->(number) { body.slice((number - 1) * part_size, [part_size, total - ((number - 1) * part_size)].min) }
|
|
417
429
|
first = piece.call(1)
|
|
418
430
|
type = options.dig('content', 'type')
|
|
419
|
-
query = options.slice(*SUBMIT_QUERY_KEYS).merge('
|
|
431
|
+
query = options.slice(*SUBMIT_QUERY_KEYS).merge('wait' => wait, 'contentType' => type, 'sizeInBytes' => first.size,
|
|
420
432
|
'partNumber' => 1, 'isFinal' => false, 'totalSizeInBytes' => total)
|
|
421
433
|
asset = expect_asset(request('POST', project('assets'),
|
|
422
434
|
query: query, raw: first, idempotent: true, upload: true,
|
|
@@ -452,9 +464,9 @@ module CalmLens
|
|
|
452
464
|
asset
|
|
453
465
|
end
|
|
454
466
|
|
|
455
|
-
def send_part(asset_id, number, final, body,
|
|
467
|
+
def send_part(asset_id, number, final, body, wait, request_options)
|
|
456
468
|
query = { 'partNumber' => number, 'isFinal' => final, 'sizeInBytes' => body.size }
|
|
457
|
-
query['
|
|
469
|
+
query['wait'] = wait unless wait.nil?
|
|
458
470
|
headers = { 'Content-Type' => 'application/octet-stream', 'X-CalmLens-Upload' => 'part' }
|
|
459
471
|
expect_asset(request('POST', "/assets/#{Internal.segment(asset_id)}/part", query: query, raw: body, idempotent: true,
|
|
460
472
|
request_options: request_options, upload: true,
|
|
@@ -479,7 +491,10 @@ module CalmLens
|
|
|
479
491
|
end
|
|
480
492
|
|
|
481
493
|
final = content.fetch('isFinal', false)
|
|
482
|
-
|
|
494
|
+
unless @session.defer_upload_checks
|
|
495
|
+
# A declared whole-file size over the limit for parts can never finish.
|
|
496
|
+
mime.start_with?('image/') ? check_image_size(total || body.size) : check_upload_size(mime, total, in_parts: true)
|
|
497
|
+
end
|
|
483
498
|
{ 'partNumber' => 1, 'isFinal' => final, 'totalSizeInBytes' => total }
|
|
484
499
|
end
|
|
485
500
|
|
|
@@ -495,45 +510,34 @@ module CalmLens
|
|
|
495
510
|
raise InvalidResponseError.new(SdkErrorCodes::INVALID_RESPONSE, 'Expected an asset response', status: 200)
|
|
496
511
|
end
|
|
497
512
|
|
|
498
|
-
def check_signed_url(body)
|
|
499
|
-
resource = body.fetch('resource', 'rendered')
|
|
500
|
-
raise Internal.invalid("A signed URL's resource must be original or rendered") unless %w[original rendered].include?(resource)
|
|
501
|
-
if resource == 'original' && (body.key?('lens') || body.key?('transform'))
|
|
502
|
-
raise Internal.invalid('Lenses and transforms apply only to the rendered resource')
|
|
503
|
-
end
|
|
504
|
-
|
|
505
|
-
expires = body['expiresInSeconds']
|
|
506
|
-
return if expires.is_a?(Integer) && expires.between?(1, MAX_SIGNED_URL_SECONDS)
|
|
507
|
-
|
|
508
|
-
raise Internal.invalid('expires_in_seconds must be a whole number from 1 to 86400')
|
|
509
|
-
end
|
|
510
|
-
|
|
511
513
|
def check_mime(type)
|
|
512
514
|
mime = type.to_s.split(';').first.to_s.strip.downcase
|
|
513
515
|
unless Internal.accepted_upload_mime_type?(mime)
|
|
514
|
-
raise InvalidArgumentError.new(ErrorCodes::
|
|
516
|
+
raise InvalidArgumentError.new(ErrorCodes::UNSUPPORTED_UPLOAD_TYPE,
|
|
515
517
|
"#{mime} is not a valid upload file type. Uploads must use a concrete file MIME " \
|
|
516
518
|
'type that CalmLens can store and process, such as image/png, image/jpeg, ' \
|
|
517
|
-
'application/pdf, or text/plain.', details: { 'mimeType' => type })
|
|
519
|
+
'application/pdf, or text/plain.', details: { 'mimeType' => type, 'source' => 'calmlens' })
|
|
518
520
|
end
|
|
519
521
|
raise Internal.invalid('Content type cannot contain line breaks') unless type.match?(Internal::HEADER_VALUE)
|
|
520
522
|
|
|
521
523
|
mime
|
|
522
524
|
end
|
|
523
525
|
|
|
524
|
-
# Refuses
|
|
525
|
-
#
|
|
526
|
-
|
|
526
|
+
# Refuses content whose known size is over the API's limit for its type, before any of it is
|
|
527
|
+
# sent: `in_parts` selects the limits for content sent in parts. Images in parts are checked by
|
|
528
|
+
# {#check_image_size}.
|
|
529
|
+
def check_upload_size(mime, size, in_parts: false)
|
|
527
530
|
return if size.nil?
|
|
528
531
|
|
|
529
532
|
limit, message = if mime.start_with?('image/')
|
|
530
533
|
[MAX_DIRECT_IMAGE_BYTES, 'Images uploaded in one request can be at most 10 MiB; upload larger images in parts']
|
|
531
534
|
elsif mime.start_with?('audio/', 'video/')
|
|
532
|
-
[
|
|
535
|
+
[in_parts ? MAX_LONG_MEDIA_BYTES : MAX_MEDIA_BYTES,
|
|
536
|
+
'Audio and video files can be at most 50 MiB in one request, or 2 GiB in parts']
|
|
533
537
|
else
|
|
534
538
|
[MAX_FILE_BYTES, 'Files can be at most 5,000 MiB']
|
|
535
539
|
end
|
|
536
|
-
raise
|
|
540
|
+
raise too_large(message, limit) if size > limit
|
|
537
541
|
end
|
|
538
542
|
|
|
539
543
|
# Images in parts total at most 20 MiB, at any size below that; a declared total is checked
|
|
@@ -541,7 +545,12 @@ module CalmLens
|
|
|
541
545
|
def check_image_size(size)
|
|
542
546
|
return unless size > MAX_MULTIPART_IMAGE_BYTES
|
|
543
547
|
|
|
544
|
-
raise
|
|
548
|
+
raise too_large('Multipart image uploads cannot exceed 20 MiB.', MAX_MULTIPART_IMAGE_BYTES)
|
|
549
|
+
end
|
|
550
|
+
|
|
551
|
+
# `upload/too-large` for content over the plan's `limit` bytes, as the API answers it.
|
|
552
|
+
def too_large(message, limit)
|
|
553
|
+
InvalidArgumentError.new(ErrorCodes::UPLOAD_TOO_LARGE, message, details: { 'limitBytes' => limit, 'source' => 'plan' })
|
|
545
554
|
end
|
|
546
555
|
end
|
|
547
556
|
end
|