@vertesia/client 1.6.0-dev.20261022.120426Z → 1.6.0-dev.20261120.123957Z
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/AccountApi.d.ts +2 -0
- package/lib/AccountApi.d.ts.map +1 -1
- package/lib/AccountApi.js +3 -0
- package/lib/AccountApi.js.map +1 -1
- package/lib/AccountRolesApi.d.ts +11 -0
- package/lib/AccountRolesApi.d.ts.map +1 -0
- package/lib/AccountRolesApi.js +22 -0
- package/lib/AccountRolesApi.js.map +1 -0
- package/lib/AppsApi.d.ts +18 -3
- package/lib/AppsApi.d.ts.map +1 -1
- package/lib/AppsApi.js +34 -4
- package/lib/AppsApi.js.map +1 -1
- package/lib/AuditTrailApi.d.ts +6 -1
- package/lib/AuditTrailApi.d.ts.map +1 -1
- package/lib/AuditTrailApi.js +8 -0
- package/lib/AuditTrailApi.js.map +1 -1
- package/lib/IamApi.d.ts +1 -1
- package/lib/IamApi.js +1 -1
- package/lib/InteractionCatalogApi.d.ts +19 -1
- package/lib/InteractionCatalogApi.d.ts.map +1 -1
- package/lib/InteractionCatalogApi.js +30 -0
- package/lib/InteractionCatalogApi.js.map +1 -1
- package/lib/InteractionsApi.d.ts +3 -12
- package/lib/InteractionsApi.d.ts.map +1 -1
- package/lib/InteractionsApi.js +2 -23
- package/lib/InteractionsApi.js.map +1 -1
- package/lib/OAuthClientsApi.d.ts +3 -1
- package/lib/OAuthClientsApi.d.ts.map +1 -1
- package/lib/OAuthClientsApi.js +4 -0
- package/lib/OAuthClientsApi.js.map +1 -1
- package/lib/OAuthProvidersApi.d.ts +6 -2
- package/lib/OAuthProvidersApi.d.ts.map +1 -1
- package/lib/OAuthProvidersApi.js +6 -2
- package/lib/OAuthProvidersApi.js.map +1 -1
- package/lib/StreamSource.d.ts +19 -1
- package/lib/StreamSource.d.ts.map +1 -1
- package/lib/StreamSource.js +31 -1
- package/lib/StreamSource.js.map +1 -1
- package/lib/UsersApi.d.ts +7 -1
- package/lib/UsersApi.d.ts.map +1 -1
- package/lib/UsersApi.js +8 -0
- package/lib/UsersApi.js.map +1 -1
- package/lib/client.d.ts +14 -2
- package/lib/client.d.ts.map +1 -1
- package/lib/client.js +55 -8
- package/lib/client.js.map +1 -1
- package/lib/execute.d.ts.map +1 -1
- package/lib/execute.js +4 -2
- package/lib/execute.js.map +1 -1
- package/lib/idempotency-key.d.ts +19 -0
- package/lib/idempotency-key.d.ts.map +1 -0
- package/lib/idempotency-key.js +34 -0
- package/lib/idempotency-key.js.map +1 -0
- package/lib/nodejs/NodeStreamSource.d.ts +10 -1
- package/lib/nodejs/NodeStreamSource.d.ts.map +1 -1
- package/lib/nodejs/NodeStreamSource.js +40 -2
- package/lib/nodejs/NodeStreamSource.js.map +1 -1
- package/lib/store/AgentsApi.d.ts.map +1 -1
- package/lib/store/AgentsApi.js +5 -2
- package/lib/store/AgentsApi.js.map +1 -1
- package/lib/store/ContentSourcesApi.d.ts +52 -0
- package/lib/store/ContentSourcesApi.d.ts.map +1 -0
- package/lib/store/ContentSourcesApi.js +85 -0
- package/lib/store/ContentSourcesApi.js.map +1 -0
- package/lib/store/CostApi.d.ts +6 -1
- package/lib/store/CostApi.d.ts.map +1 -1
- package/lib/store/CostApi.js +7 -0
- package/lib/store/CostApi.js.map +1 -1
- package/lib/store/DashboardApi.d.ts +61 -19
- package/lib/store/DashboardApi.d.ts.map +1 -1
- package/lib/store/DashboardApi.js +75 -19
- package/lib/store/DashboardApi.js.map +1 -1
- package/lib/store/DataApi.d.ts +9 -9
- package/lib/store/DataApi.d.ts.map +1 -1
- package/lib/store/DataApi.js +14 -10
- package/lib/store/DataApi.js.map +1 -1
- package/lib/store/FilesApi.d.ts +15 -1
- package/lib/store/FilesApi.d.ts.map +1 -1
- package/lib/store/FilesApi.js +54 -75
- package/lib/store/FilesApi.js.map +1 -1
- package/lib/store/NotificationsApi.d.ts +39 -0
- package/lib/store/NotificationsApi.d.ts.map +1 -0
- package/lib/store/NotificationsApi.js +68 -0
- package/lib/store/NotificationsApi.js.map +1 -0
- package/lib/store/ObjectsApi.d.ts +6 -1
- package/lib/store/ObjectsApi.d.ts.map +1 -1
- package/lib/store/ObjectsApi.js +23 -30
- package/lib/store/ObjectsApi.js.map +1 -1
- package/lib/store/RenderingApi.d.ts.map +1 -1
- package/lib/store/RenderingApi.js +1 -0
- package/lib/store/RenderingApi.js.map +1 -1
- package/lib/store/WorkflowsApi.d.ts +19 -1
- package/lib/store/WorkflowsApi.d.ts.map +1 -1
- package/lib/store/WorkflowsApi.js +21 -0
- package/lib/store/WorkflowsApi.js.map +1 -1
- package/lib/store/chunked-upload.d.ts +71 -0
- package/lib/store/chunked-upload.d.ts.map +1 -0
- package/lib/store/chunked-upload.js +206 -0
- package/lib/store/chunked-upload.js.map +1 -0
- package/lib/store/client.d.ts +9 -1
- package/lib/store/client.d.ts.map +1 -1
- package/lib/store/client.js +15 -1
- package/lib/store/client.js.map +1 -1
- package/lib/store/index.d.ts +3 -0
- package/lib/store/index.d.ts.map +1 -1
- package/lib/store/index.js +3 -0
- package/lib/store/index.js.map +1 -1
- package/lib/store/node-request.browser.d.ts +14 -0
- package/lib/store/node-request.browser.d.ts.map +1 -0
- package/lib/store/node-request.browser.js +20 -0
- package/lib/store/node-request.browser.js.map +1 -0
- package/lib/store/node-request.d.ts +20 -0
- package/lib/store/node-request.d.ts.map +1 -0
- package/lib/store/node-request.js +116 -0
- package/lib/store/node-request.js.map +1 -0
- package/lib/store/node-sessions.browser.d.ts +3 -0
- package/lib/store/node-sessions.browser.d.ts.map +1 -0
- package/lib/store/node-sessions.browser.js +15 -0
- package/lib/store/node-sessions.browser.js.map +1 -0
- package/lib/store/node-sessions.d.ts +14 -0
- package/lib/store/node-sessions.d.ts.map +1 -0
- package/lib/store/node-sessions.js +88 -0
- package/lib/store/node-sessions.js.map +1 -0
- package/lib/store/signed-url.d.ts +12 -16
- package/lib/store/signed-url.d.ts.map +1 -1
- package/lib/store/signed-url.js +111 -71
- package/lib/store/signed-url.js.map +1 -1
- package/lib/store/stream-bytes.d.ts +16 -0
- package/lib/store/stream-bytes.d.ts.map +1 -0
- package/lib/store/stream-bytes.js +70 -0
- package/lib/store/stream-bytes.js.map +1 -0
- package/lib/store/upload-body.d.ts +21 -0
- package/lib/store/upload-body.d.ts.map +1 -0
- package/lib/store/upload-body.js +32 -0
- package/lib/store/upload-body.js.map +1 -0
- package/lib/store/upload-content.d.ts +21 -0
- package/lib/store/upload-content.d.ts.map +1 -0
- package/lib/store/upload-content.js +113 -0
- package/lib/store/upload-content.js.map +1 -0
- package/lib/store/upload-runtime.d.ts +21 -0
- package/lib/store/upload-runtime.d.ts.map +1 -0
- package/lib/store/upload-runtime.js +40 -0
- package/lib/store/upload-runtime.js.map +1 -0
- package/lib/vertesia-client.js +10 -9
- package/lib/vertesia-client.js.map +1 -1
- package/package.json +35 -11
- package/src/AccountApi.ts +3 -0
- package/src/AccountRolesApi.ts +29 -0
- package/src/AppsApi.ts +58 -4
- package/src/AuditTrailApi.ts +11 -0
- package/src/IamApi.ts +1 -1
- package/src/InteractionCatalogApi.ts +44 -1
- package/src/InteractionsApi.ts +2 -31
- package/src/OAuthClientsApi.ts +6 -0
- package/src/OAuthProvidersApi.ts +7 -2
- package/src/StreamSource.ts +29 -0
- package/src/UsersApi.ts +10 -1
- package/src/client.ts +72 -8
- package/src/execute.ts +4 -2
- package/src/idempotency-key.ts +34 -0
- package/src/nodejs/NodeStreamSource.ts +37 -2
- package/src/store/AgentsApi.ts +5 -2
- package/src/store/ContentSourcesApi.ts +126 -0
- package/src/store/CostApi.ts +10 -0
- package/src/store/DashboardApi.ts +93 -18
- package/src/store/DataApi.ts +14 -10
- package/src/store/FilesApi.ts +84 -78
- package/src/store/NotificationsApi.ts +103 -0
- package/src/store/ObjectsApi.ts +30 -34
- package/src/store/RenderingApi.ts +1 -0
- package/src/store/WorkflowsApi.ts +25 -0
- package/src/store/chunked-upload.ts +272 -0
- package/src/store/client.ts +21 -1
- package/src/store/index.ts +3 -0
- package/src/store/node-request.browser.ts +22 -0
- package/src/store/node-request.ts +120 -0
- package/src/store/node-sessions.browser.ts +16 -0
- package/src/store/node-sessions.ts +109 -0
- package/src/store/signed-url.ts +126 -72
- package/src/store/stream-bytes.ts +79 -0
- package/src/store/upload-body.ts +37 -0
- package/src/store/upload-content.ts +129 -0
- package/src/store/upload-runtime.ts +42 -0
|
@@ -0,0 +1,272 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Uploads in parts, for stream sources longer than one part (Node.js only).
|
|
3
|
+
*
|
|
4
|
+
* A single signed PUT is capped by the store (5 GB on S3, about 5,000 MiB on Azure) and cannot resume. A source
|
|
5
|
+
* longer than {@link UPLOAD_PART_SIZE} therefore goes through an upload session, sent with the store's own protocol:
|
|
6
|
+
*
|
|
7
|
+
* - Google Cloud Storage: a resumable upload, written by `@google-cloud/storage`, which retries a failed chunk from
|
|
8
|
+
* the offset the service persisted.
|
|
9
|
+
* - Azure Blob Storage: Put Block for each part then Put Block List, written by `@azure/storage-blob`.
|
|
10
|
+
* - Amazon S3: a multipart upload. Each part is a `PUT` to a URL the server signs, retried like any signed-URL
|
|
11
|
+
* transfer, and the server completes the upload, or aborts it when a part cannot be sent.
|
|
12
|
+
*
|
|
13
|
+
* Memory stays about one part per upload, and nothing is written to disk. Parts are 16 MiB, or larger for content
|
|
14
|
+
* whose length would need more parts than the store allows: S3 takes 10,000 and Azure 50,000.
|
|
15
|
+
*
|
|
16
|
+
* The two SDKs are optional peer dependencies, loaded only on this path. Without the one a session needs, the
|
|
17
|
+
* upload falls back to a single signed PUT.
|
|
18
|
+
*/
|
|
19
|
+
import type { CompletedUploadPart, S3MultipartUploadSession, UploadSession } from '@vertesia/common';
|
|
20
|
+
import { StreamSource } from '../StreamSource.js';
|
|
21
|
+
// hasNodeHttp from node-request, whose browser build answers false: a bundle without the Node transport never
|
|
22
|
+
// opens a session.
|
|
23
|
+
import { hasNodeHttp } from './node-request.js';
|
|
24
|
+
import { uploadAzureBlocks, uploadGcsResumable } from './node-sessions.js';
|
|
25
|
+
import { fetchSignedUrl, MAX_BUFFERED_STREAM_BYTES } from './signed-url.js';
|
|
26
|
+
import { chunkToBlobPart, continueStream } from './stream-bytes.js';
|
|
27
|
+
import { getUploadBody, type UploadBody, unquoteEtag } from './upload-body.js';
|
|
28
|
+
import { ContentLengthMismatchError, UploadSessionUnavailableError } from './upload-runtime.js';
|
|
29
|
+
|
|
30
|
+
export { UploadSessionUnavailableError } from './upload-runtime.js';
|
|
31
|
+
|
|
32
|
+
/** Sources longer than this are sent in parts: the most of a stream that is held in memory at once. */
|
|
33
|
+
export const UPLOAD_PART_SIZE = MAX_BUFFERED_STREAM_BYTES;
|
|
34
|
+
|
|
35
|
+
const MiB = 1024 * 1024;
|
|
36
|
+
/** S3 numbers parts from 1 to 10,000 and takes objects up to 5 TiB. */
|
|
37
|
+
const S3_MAX_PARTS = 10_000;
|
|
38
|
+
const S3_MAX_OBJECT_SIZE = 5 * 1024 * 1024 * MiB;
|
|
39
|
+
/** For a stream of unknown length, S3 parts double in size every this many parts: 16 MiB parts reach 968 GiB. */
|
|
40
|
+
const S3_PARTS_PER_SIZE = 2_000;
|
|
41
|
+
/** Azure commits at most 50,000 blocks. */
|
|
42
|
+
const AZURE_MAX_BLOCKS = 50_000;
|
|
43
|
+
/**
|
|
44
|
+
* Part URLs are signed this many at a time, and signed again once this old, since they expire (S3 default, one hour,
|
|
45
|
+
* or sooner with the signer's credentials) while a slow source fills the parts before them.
|
|
46
|
+
*/
|
|
47
|
+
const PART_URL_BATCH = 100;
|
|
48
|
+
const PART_URL_MAX_AGE_MS = 15 * 60_000;
|
|
49
|
+
|
|
50
|
+
/** Content to send in parts, with its declared length when it is a stream whose length is known. */
|
|
51
|
+
export interface PartsBody {
|
|
52
|
+
body: Blob | ReadableStream<Uint8Array>;
|
|
53
|
+
size?: number;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
export type UploadPlan = { kind: 'single'; upload: UploadBody } | { kind: 'parts'; parts: PartsBody };
|
|
57
|
+
|
|
58
|
+
/** The operations an S3 multipart upload needs from the API. */
|
|
59
|
+
export interface MultipartUploadApi {
|
|
60
|
+
getUploadPartUrls(payload: {
|
|
61
|
+
id: string;
|
|
62
|
+
upload_id: string;
|
|
63
|
+
part_numbers: number[];
|
|
64
|
+
}): Promise<{ parts: { part_number: number; url: string }[] }>;
|
|
65
|
+
completeUploadSession(payload: {
|
|
66
|
+
id: string;
|
|
67
|
+
upload_id: string;
|
|
68
|
+
parts: CompletedUploadPart[];
|
|
69
|
+
}): Promise<{ etag?: string }>;
|
|
70
|
+
abortUploadSession(payload: { id: string; upload_id: string }): Promise<unknown>;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* How to upload a source. A `File`, and anything in a browser, is sent in one PUT as before. In Node.js, a stream
|
|
75
|
+
* source longer than one part is sent in parts. A plain stream is read up to one part to find out: one that ends
|
|
76
|
+
* first is sent whole, from memory, so it keeps its retries.
|
|
77
|
+
*/
|
|
78
|
+
export async function planUpload(source: StreamSource | File): Promise<UploadPlan> {
|
|
79
|
+
const upload = await getUploadBody(source);
|
|
80
|
+
if (!(source instanceof StreamSource) || !hasNodeHttp()) return { kind: 'single', upload };
|
|
81
|
+
const { body, contentLength } = upload;
|
|
82
|
+
if (body instanceof Blob) {
|
|
83
|
+
return body.size > UPLOAD_PART_SIZE ? { kind: 'parts', parts: { body } } : { kind: 'single', upload };
|
|
84
|
+
}
|
|
85
|
+
if (contentLength !== undefined && contentLength <= UPLOAD_PART_SIZE) return { kind: 'single', upload };
|
|
86
|
+
|
|
87
|
+
const reader = (body as ReadableStream<unknown>).getReader();
|
|
88
|
+
const head: BlobPart[] = [];
|
|
89
|
+
let read = 0;
|
|
90
|
+
try {
|
|
91
|
+
while (read <= UPLOAD_PART_SIZE) {
|
|
92
|
+
const { done, value } = await reader.read();
|
|
93
|
+
if (done) {
|
|
94
|
+
reader.releaseLock();
|
|
95
|
+
// a declared length is over one part here, so a stream that ended is short of it
|
|
96
|
+
if (contentLength !== undefined) throw new ContentLengthMismatchError(contentLength, read, 'short');
|
|
97
|
+
return { kind: 'single', upload: { body: new Blob(head) } };
|
|
98
|
+
}
|
|
99
|
+
const part = chunkToBlobPart(value);
|
|
100
|
+
head.push(part);
|
|
101
|
+
read += part instanceof Blob ? part.size : (part as Uint8Array).byteLength;
|
|
102
|
+
}
|
|
103
|
+
} catch (err) {
|
|
104
|
+
await reader.cancel(err).catch(() => undefined);
|
|
105
|
+
throw err;
|
|
106
|
+
}
|
|
107
|
+
return { kind: 'parts', parts: { body: continueStream(head, reader), size: contentLength } };
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
/** The same content as one stream, for a single PUT when no session can be used. */
|
|
111
|
+
export function partsAsUploadBody(parts: PartsBody): UploadBody {
|
|
112
|
+
return parts.body instanceof Blob ? { body: parts.body } : { body: parts.body, contentLength: parts.size };
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
/** Sends the content through the session. Returns the stored object's ETag, unquoted, when the store gives one. */
|
|
116
|
+
export async function uploadInParts(
|
|
117
|
+
api: MultipartUploadApi,
|
|
118
|
+
session: UploadSession,
|
|
119
|
+
parts: PartsBody,
|
|
120
|
+
): Promise<string | undefined> {
|
|
121
|
+
switch (session.protocol) {
|
|
122
|
+
case 's3_multipart':
|
|
123
|
+
return uploadS3Parts(api, session, parts);
|
|
124
|
+
case 'gcs_resumable':
|
|
125
|
+
return uploadGcsResumable(session, parts);
|
|
126
|
+
case 'azure_block_blob':
|
|
127
|
+
return unquoteEtag(
|
|
128
|
+
await uploadAzureBlocks(
|
|
129
|
+
session.url,
|
|
130
|
+
fitParts(session.part_size, knownSize(parts), AZURE_MAX_BLOCKS),
|
|
131
|
+
session.mime_type,
|
|
132
|
+
parts,
|
|
133
|
+
),
|
|
134
|
+
);
|
|
135
|
+
default:
|
|
136
|
+
// A protocol from a newer server: upload in one request instead, before reading anything.
|
|
137
|
+
throw new UploadSessionUnavailableError(
|
|
138
|
+
`support for the ${(session as { protocol: unknown }).protocol} protocol`,
|
|
139
|
+
);
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
function knownSize(parts: PartsBody): number | undefined {
|
|
144
|
+
return parts.body instanceof Blob ? parts.body.size : parts.size;
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
/** `base`, or when `size` bytes would need more than `maxParts` parts of it, the whole number of MiB that fits. */
|
|
148
|
+
function fitParts(base: number, size: number | undefined, maxParts: number): number {
|
|
149
|
+
const needed = size === undefined ? 0 : Math.ceil(size / maxParts);
|
|
150
|
+
return needed <= base ? base : Math.ceil(needed / MiB) * MiB;
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
/**
|
|
154
|
+
* The size of each S3 part. With the length known, one size that fits it in 10,000 parts. Without, parts grow as the
|
|
155
|
+
* upload does, so a stream of unknown length can reach 968 GiB while short uploads keep small parts.
|
|
156
|
+
*/
|
|
157
|
+
function s3PartSizes(base: number, size: number | undefined): (partNumber: number) => number {
|
|
158
|
+
if (size !== undefined) {
|
|
159
|
+
if (size > S3_MAX_OBJECT_SIZE) {
|
|
160
|
+
throw new Error(`S3 takes objects up to 5 TiB; this upload is declared at ${size} bytes`);
|
|
161
|
+
}
|
|
162
|
+
const fixed = fitParts(base, size, S3_MAX_PARTS);
|
|
163
|
+
return () => fixed;
|
|
164
|
+
}
|
|
165
|
+
return (partNumber) => base * 2 ** Math.floor((partNumber - 1) / S3_PARTS_PER_SIZE);
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
/** Splits the content into parts, part `n` of `partSize(n)` bytes; the last may be shorter. */
|
|
169
|
+
async function* blobParts(
|
|
170
|
+
body: Blob | ReadableStream<Uint8Array>,
|
|
171
|
+
partSize: (partNumber: number) => number,
|
|
172
|
+
): AsyncGenerator<Blob> {
|
|
173
|
+
let partNumber = 1;
|
|
174
|
+
if (body instanceof Blob) {
|
|
175
|
+
for (let offset = 0; offset < body.size; partNumber++) {
|
|
176
|
+
const end = Math.min(offset + partSize(partNumber), body.size);
|
|
177
|
+
yield body.slice(offset, end);
|
|
178
|
+
offset = end;
|
|
179
|
+
}
|
|
180
|
+
return;
|
|
181
|
+
}
|
|
182
|
+
const reader = body.getReader();
|
|
183
|
+
let pending: Uint8Array[] = [];
|
|
184
|
+
let pendingBytes = 0;
|
|
185
|
+
let finished = false;
|
|
186
|
+
try {
|
|
187
|
+
for (;;) {
|
|
188
|
+
const { done, value } = await reader.read();
|
|
189
|
+
if (done) break;
|
|
190
|
+
let chunk = value;
|
|
191
|
+
while (pendingBytes + chunk.byteLength >= partSize(partNumber)) {
|
|
192
|
+
const take = partSize(partNumber) - pendingBytes;
|
|
193
|
+
pending.push(chunk.subarray(0, take));
|
|
194
|
+
yield new Blob(pending as BlobPart[]);
|
|
195
|
+
partNumber++;
|
|
196
|
+
pending = [];
|
|
197
|
+
pendingBytes = 0;
|
|
198
|
+
chunk = chunk.subarray(take);
|
|
199
|
+
}
|
|
200
|
+
if (chunk.byteLength > 0) {
|
|
201
|
+
pending.push(chunk);
|
|
202
|
+
pendingBytes += chunk.byteLength;
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
finished = true;
|
|
206
|
+
if (pendingBytes > 0) yield new Blob(pending as BlobPart[]);
|
|
207
|
+
} finally {
|
|
208
|
+
if (!finished) await reader.cancel().catch(() => undefined);
|
|
209
|
+
reader.releaseLock();
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
async function uploadS3Parts(
|
|
214
|
+
api: MultipartUploadApi,
|
|
215
|
+
session: S3MultipartUploadSession,
|
|
216
|
+
parts: PartsBody,
|
|
217
|
+
): Promise<string | undefined> {
|
|
218
|
+
const { id, upload_id } = session;
|
|
219
|
+
const uploaded: CompletedUploadPart[] = [];
|
|
220
|
+
let urls = new Map<number, string>();
|
|
221
|
+
let signedAt = 0;
|
|
222
|
+
let sent = 0;
|
|
223
|
+
|
|
224
|
+
async function partUrl(partNumber: number, renew: boolean): Promise<string> {
|
|
225
|
+
if (renew || !urls.has(partNumber) || Date.now() - signedAt > PART_URL_MAX_AGE_MS) {
|
|
226
|
+
const last = Math.min(partNumber + PART_URL_BATCH - 1, S3_MAX_PARTS);
|
|
227
|
+
const part_numbers = Array.from({ length: last - partNumber + 1 }, (_, i) => partNumber + i);
|
|
228
|
+
const signed = await api.getUploadPartUrls({ id, upload_id, part_numbers });
|
|
229
|
+
urls = new Map(signed.parts.map((p) => [p.part_number, p.url]));
|
|
230
|
+
signedAt = Date.now();
|
|
231
|
+
}
|
|
232
|
+
const url = urls.get(partNumber);
|
|
233
|
+
if (!url) throw new Error(`No upload URL was signed for part ${partNumber}`);
|
|
234
|
+
return url;
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
try {
|
|
238
|
+
for await (const part of blobParts(parts.body, s3PartSizes(session.part_size, knownSize(parts)))) {
|
|
239
|
+
const partNumber = uploaded.length + 1;
|
|
240
|
+
if (partNumber > S3_MAX_PARTS) {
|
|
241
|
+
throw new Error(`Upload needs more than ${S3_MAX_PARTS} parts`);
|
|
242
|
+
}
|
|
243
|
+
sent += part.size;
|
|
244
|
+
if (parts.size !== undefined && sent > parts.size) {
|
|
245
|
+
throw new ContentLengthMismatchError(parts.size, sent, 'long');
|
|
246
|
+
}
|
|
247
|
+
let res = await fetchSignedUrl(await partUrl(partNumber, false), { method: 'PUT', body: part });
|
|
248
|
+
if (res.status === 403) {
|
|
249
|
+
// An expired URL, or one whose signing credentials expired: sign it again, once.
|
|
250
|
+
await res.body?.cancel().catch(() => undefined);
|
|
251
|
+
res = await fetchSignedUrl(await partUrl(partNumber, true), { method: 'PUT', body: part });
|
|
252
|
+
}
|
|
253
|
+
const detail = res.ok ? '' : await res.text().catch(() => '');
|
|
254
|
+
if (res.ok) await res.body?.cancel().catch(() => undefined);
|
|
255
|
+
const etag = res.headers.get('etag');
|
|
256
|
+
if (!res.ok || !etag) {
|
|
257
|
+
throw new Error(
|
|
258
|
+
`Failed to upload part ${partNumber}: ${res.status} ${res.statusText}${detail ? ` ${detail}` : ''}`,
|
|
259
|
+
);
|
|
260
|
+
}
|
|
261
|
+
uploaded.push({ part_number: partNumber, etag });
|
|
262
|
+
}
|
|
263
|
+
if (parts.size !== undefined && sent < parts.size) {
|
|
264
|
+
throw new ContentLengthMismatchError(parts.size, sent, 'short');
|
|
265
|
+
}
|
|
266
|
+
const completed = await api.completeUploadSession({ id, upload_id, parts: uploaded });
|
|
267
|
+
return unquoteEtag(completed.etag);
|
|
268
|
+
} catch (err) {
|
|
269
|
+
await api.abortUploadSession({ id, upload_id }).catch(() => undefined);
|
|
270
|
+
throw err;
|
|
271
|
+
}
|
|
272
|
+
}
|
package/src/store/client.ts
CHANGED
|
@@ -1,14 +1,17 @@
|
|
|
1
1
|
import {
|
|
2
2
|
AbstractFetchClient,
|
|
3
3
|
type FETCH_FN,
|
|
4
|
+
type IPacingRetryPolicy,
|
|
4
5
|
type IRequestRetryPolicy,
|
|
5
6
|
type RequestError,
|
|
7
|
+
type RequestIdentityProvider,
|
|
6
8
|
} from '@vertesia/api-fetch-client';
|
|
7
9
|
import { APP_VERSION_HEADER, type BulkOperationPayload, type BulkOperationResponse } from '@vertesia/common';
|
|
8
10
|
import { warnUnknownOptions } from '../unknown-options.js';
|
|
9
11
|
import { AgentsApi } from './AgentsApi.js';
|
|
10
12
|
import { CollectionsApi } from './CollectionsApi.js';
|
|
11
13
|
import { ContentApi } from './ContentApi.js';
|
|
14
|
+
import { ContentSourcesApi } from './ContentSourcesApi.js';
|
|
12
15
|
import { CostApi } from './CostApi.js';
|
|
13
16
|
import { DataApi } from './DataApi.js';
|
|
14
17
|
import { EmailApi } from './EmailApi.js';
|
|
@@ -17,6 +20,7 @@ import { EventsApi } from './EventsApi.js';
|
|
|
17
20
|
import { ZenoClientNotFoundError } from './errors.js';
|
|
18
21
|
import { FilesApi } from './FilesApi.js';
|
|
19
22
|
import { IndexingApi } from './IndexingApi.js';
|
|
23
|
+
import { NotificationsApi } from './NotificationsApi.js';
|
|
20
24
|
import { ObjectsApi } from './ObjectsApi.js';
|
|
21
25
|
import { PendingAsksApi } from './PendingAsksApi.js';
|
|
22
26
|
import { ProcessApi } from './ProcessApi.js';
|
|
@@ -34,9 +38,13 @@ export interface ZenoClientProps {
|
|
|
34
38
|
serverUrl?: string;
|
|
35
39
|
tokenServerUrl?: string;
|
|
36
40
|
apikey?: string;
|
|
41
|
+
/** Supplies the complete Authorization header; takes precedence over apikey. */
|
|
42
|
+
authCallback?: () => Promise<string>;
|
|
43
|
+
requestIdentity?: RequestIdentityProvider;
|
|
37
44
|
onRequest?: (request: Request) => void;
|
|
38
45
|
onResponse?: (response: Response) => void;
|
|
39
46
|
retryPolicy?: IRequestRetryPolicy;
|
|
47
|
+
pacingRetry?: IPacingRetryPolicy | false;
|
|
40
48
|
timeout?: number | false | null;
|
|
41
49
|
fetch?: FETCH_FN | Promise<FETCH_FN>;
|
|
42
50
|
}
|
|
@@ -46,9 +54,12 @@ const KNOWN_STORE_OPTIONS: Record<keyof Required<ZenoClientProps>, true> = {
|
|
|
46
54
|
serverUrl: true,
|
|
47
55
|
tokenServerUrl: true,
|
|
48
56
|
apikey: true,
|
|
57
|
+
authCallback: true,
|
|
58
|
+
requestIdentity: true,
|
|
49
59
|
onRequest: true,
|
|
50
60
|
onResponse: true,
|
|
51
61
|
retryPolicy: true,
|
|
62
|
+
pacingRetry: true,
|
|
52
63
|
timeout: true,
|
|
53
64
|
fetch: true,
|
|
54
65
|
};
|
|
@@ -64,15 +75,22 @@ export class ZenoClient extends AbstractFetchClient<ZenoClient> {
|
|
|
64
75
|
constructor(opts: ZenoClientProps = {}) {
|
|
65
76
|
warnUnknownOptions('ZenoClient', opts, KNOWN_STORE_OPTIONS);
|
|
66
77
|
super(ensureDefined(opts.serverUrl), opts.fetch);
|
|
67
|
-
if (opts.
|
|
78
|
+
if (opts.authCallback) {
|
|
79
|
+
this.withAuthCallback(opts.authCallback);
|
|
80
|
+
} else if (opts.apikey) {
|
|
68
81
|
this.withApiKey(opts.apikey);
|
|
69
82
|
}
|
|
70
83
|
if (opts.retryPolicy) {
|
|
71
84
|
this.withRetryPolicy(opts.retryPolicy);
|
|
72
85
|
}
|
|
86
|
+
if (opts.pacingRetry !== undefined) {
|
|
87
|
+
this.withPacingRetry(opts.pacingRetry);
|
|
88
|
+
}
|
|
89
|
+
|
|
73
90
|
if (opts.timeout !== undefined) {
|
|
74
91
|
this.withTimeout(opts.timeout);
|
|
75
92
|
}
|
|
93
|
+
this.requestIdentity = opts.requestIdentity;
|
|
76
94
|
this.onRequest = opts.onRequest;
|
|
77
95
|
this.onResponse = opts.onResponse;
|
|
78
96
|
this.errorFactory = (err: RequestError) => {
|
|
@@ -127,6 +145,7 @@ export class ZenoClient extends AbstractFetchClient<ZenoClient> {
|
|
|
127
145
|
types = new TypesApi(this);
|
|
128
146
|
workflows = new WorkflowsApi(this);
|
|
129
147
|
schedules = new SchedulesApi(this);
|
|
148
|
+
contentSources = new ContentSourcesApi(this);
|
|
130
149
|
processes = new ProcessApi(this);
|
|
131
150
|
processTestRuns = new ProcessTestRunApi(this);
|
|
132
151
|
tasks = new TaskApi(this);
|
|
@@ -138,6 +157,7 @@ export class ZenoClient extends AbstractFetchClient<ZenoClient> {
|
|
|
138
157
|
/** @deprecated Use events.subscriptions. */
|
|
139
158
|
eventSubscriptions = this.events.subscriptions;
|
|
140
159
|
pendingAsks = new PendingAsksApi(this);
|
|
160
|
+
notifications = new NotificationsApi(this);
|
|
141
161
|
data = new DataApi(this);
|
|
142
162
|
indexing = new IndexingApi(this);
|
|
143
163
|
query = new QueryApi(this);
|
package/src/store/index.ts
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
export * from './AgentsApi.js';
|
|
2
2
|
export * from './ContentApi.js';
|
|
3
|
+
export * from './ContentSourcesApi.js';
|
|
3
4
|
export * from './client.js';
|
|
5
|
+
export * from './DashboardApi.js';
|
|
4
6
|
export * from './DataApi.js';
|
|
5
7
|
export * from './EmailApi.js';
|
|
6
8
|
export * from './EmbeddingsApi.js';
|
|
@@ -10,6 +12,7 @@ export * from './EventsApi.js';
|
|
|
10
12
|
export * from './errors.js';
|
|
11
13
|
export * from './FilesApi.js';
|
|
12
14
|
export * from './IndexingApi.js';
|
|
15
|
+
export * from './NotificationsApi.js';
|
|
13
16
|
export * from './ObjectsApi.js';
|
|
14
17
|
export * from './PendingAsksApi.js';
|
|
15
18
|
export * from './ProcessApi.js';
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The browser build of `node-request.ts`, which the `browser` field of `package.json` substitutes for it, so that
|
|
3
|
+
* browser bundles never resolve Node's `http`, `https` and `stream` modules. Uploads only take the Node transport
|
|
4
|
+
* when {@link hasNodeHttp} is true, which it never is here.
|
|
5
|
+
*/
|
|
6
|
+
export { ContentLengthMismatchError } from './upload-runtime.js';
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* False: this build has no Node transport, even in a runtime that defines `process.versions.node` (an Electron
|
|
10
|
+
* renderer, or a worker with Node compatibility) and was bundled for the browser.
|
|
11
|
+
*/
|
|
12
|
+
export function hasNodeHttp(): boolean {
|
|
13
|
+
return false;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
export function nodeRequest(): never {
|
|
17
|
+
throw new Error('nodeRequest is only available in Node.js');
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
export function exactLength(): never {
|
|
21
|
+
throw new Error('exactLength is only available in Node.js');
|
|
22
|
+
}
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Large signed-URL uploads in Node.js, sent with Node's own `http`/`https` client instead of `fetch`.
|
|
3
|
+
*
|
|
4
|
+
* Node's `fetch` keeps every chunk of a streamed request body in memory until the request completes, so uploading a
|
|
5
|
+
* file through it costs the file's size in memory even when the body is a stream or a file-backed `Blob`. `http`
|
|
6
|
+
* writes the body with backpressure and releases each chunk once it is sent.
|
|
7
|
+
*
|
|
8
|
+
* Node.js only. Browser bundles get `node-request.browser.ts` instead, through the `browser` field of `package.json`,
|
|
9
|
+
* so they never resolve the Node modules this file imports.
|
|
10
|
+
*/
|
|
11
|
+
import { ContentLengthMismatchError, contentLengthHeader } from './upload-runtime.js';
|
|
12
|
+
|
|
13
|
+
export { ContentLengthMismatchError, hasNodeHttp } from './upload-runtime.js';
|
|
14
|
+
|
|
15
|
+
type WebReadableStream = Parameters<typeof import('node:stream').Readable.fromWeb>[0];
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* A pass-through that fails once more than `length` bytes go through it, or when it ends with fewer. The chunk that
|
|
19
|
+
* completes `length` is held until the source ends, so the server never receives a complete body from a stream
|
|
20
|
+
* that turns out to be longer.
|
|
21
|
+
*/
|
|
22
|
+
export function exactLength(length: number, TransformClass: typeof import('node:stream').Transform) {
|
|
23
|
+
let seen = 0;
|
|
24
|
+
let last: Buffer | undefined;
|
|
25
|
+
return new TransformClass({
|
|
26
|
+
transform(chunk: Buffer, _encoding, callback) {
|
|
27
|
+
seen += chunk.byteLength;
|
|
28
|
+
if (seen > length) {
|
|
29
|
+
callback(new ContentLengthMismatchError(length, seen, 'long'));
|
|
30
|
+
} else if (seen === length) {
|
|
31
|
+
last = chunk;
|
|
32
|
+
callback();
|
|
33
|
+
} else {
|
|
34
|
+
callback(null, chunk);
|
|
35
|
+
}
|
|
36
|
+
},
|
|
37
|
+
flush(callback) {
|
|
38
|
+
if (seen < length) {
|
|
39
|
+
callback(new ContentLengthMismatchError(length, seen, 'short'));
|
|
40
|
+
} else {
|
|
41
|
+
callback(null, last);
|
|
42
|
+
}
|
|
43
|
+
},
|
|
44
|
+
});
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
const NULL_BODY_STATUSES = new Set([101, 204, 205, 304]);
|
|
48
|
+
|
|
49
|
+
/** How long the connection may stay idle before the request fails: the headers and body timeouts of undici's fetch. */
|
|
50
|
+
const IDLE_TIMEOUT_MS = 300_000;
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* Send a request with a `Blob` or `ReadableStream` body. A `Blob` is sent with its size as `Content-Length`; a
|
|
54
|
+
* stream with the `Content-Length` header it is given, or with chunked transfer encoding when it has none. A body
|
|
55
|
+
* that ends short of its `Content-Length` or runs past it fails the request. Resolves with the response once its
|
|
56
|
+
* headers arrive, like `fetch`; rejects on a connection error, or when the connection is idle for `idleTimeoutMs`.
|
|
57
|
+
*/
|
|
58
|
+
export async function nodeRequest(
|
|
59
|
+
url: string,
|
|
60
|
+
init: { method: string; headers?: Record<string, string>; body: Blob | ReadableStream; idleTimeoutMs?: number },
|
|
61
|
+
): Promise<Response> {
|
|
62
|
+
const [http, https, { Readable, Transform }, { pipeline }] = await Promise.all([
|
|
63
|
+
import('node:http'),
|
|
64
|
+
import('node:https'),
|
|
65
|
+
import('node:stream'),
|
|
66
|
+
import('node:stream/promises'),
|
|
67
|
+
]);
|
|
68
|
+
const { body } = init;
|
|
69
|
+
const headers: Record<string, string> = { ...init.headers };
|
|
70
|
+
if (body instanceof Blob) {
|
|
71
|
+
for (const name of Object.keys(headers)) {
|
|
72
|
+
if (name.toLowerCase() === 'content-length') delete headers[name];
|
|
73
|
+
}
|
|
74
|
+
headers['content-length'] = String(body.size);
|
|
75
|
+
}
|
|
76
|
+
const target = new URL(url);
|
|
77
|
+
const send = target.protocol === 'http:' ? http.request : https.request;
|
|
78
|
+
|
|
79
|
+
return new Promise<Response>((resolve, reject) => {
|
|
80
|
+
const req = send(target, { method: init.method, headers });
|
|
81
|
+
const idleTimeoutMs = init.idleTimeoutMs ?? IDLE_TIMEOUT_MS;
|
|
82
|
+
req.setTimeout(idleTimeoutMs, () => {
|
|
83
|
+
req.destroy(new Error(`Request to ${target.host} was idle for ${idleTimeoutMs} ms`));
|
|
84
|
+
});
|
|
85
|
+
req.on('response', (res) => {
|
|
86
|
+
const responseHeaders = new Headers();
|
|
87
|
+
for (const [name, value] of Object.entries(res.headers)) {
|
|
88
|
+
if (value === undefined) continue;
|
|
89
|
+
for (const v of Array.isArray(value) ? value : [value]) responseHeaders.append(name, v);
|
|
90
|
+
}
|
|
91
|
+
const status = res.statusCode ?? 0;
|
|
92
|
+
const responseBody = NULL_BODY_STATUSES.has(status)
|
|
93
|
+
? null
|
|
94
|
+
: (Readable.toWeb(res) as unknown as ReadableStream<Uint8Array>);
|
|
95
|
+
if (!responseBody) res.resume();
|
|
96
|
+
try {
|
|
97
|
+
resolve(
|
|
98
|
+
new Response(responseBody, { status, statusText: res.statusMessage, headers: responseHeaders }),
|
|
99
|
+
);
|
|
100
|
+
} catch (err: unknown) {
|
|
101
|
+
// A status or reason phrase `Response` does not accept: fail the request, not the event loop.
|
|
102
|
+
res.destroy();
|
|
103
|
+
reject(err);
|
|
104
|
+
}
|
|
105
|
+
});
|
|
106
|
+
const source = Readable.fromWeb((body instanceof Blob ? body.stream() : body) as WebReadableStream);
|
|
107
|
+
// An error reading the body ends the request with a "socket hang up"; report the body's error instead.
|
|
108
|
+
let bodyError: unknown;
|
|
109
|
+
source.once('error', (err) => {
|
|
110
|
+
bodyError = err;
|
|
111
|
+
});
|
|
112
|
+
req.on('error', (err) => reject(bodyError ?? err));
|
|
113
|
+
// A stream that does not match its declared length aborts the request instead of sending a truncated or
|
|
114
|
+
// overlong body. (`strictContentLength` reports a short body from `end()` outside the pipeline, as an
|
|
115
|
+
// uncaught exception, so the length is checked here.)
|
|
116
|
+
const length = contentLengthHeader(headers);
|
|
117
|
+
const stages = length === undefined ? [source] : [source, exactLength(length, Transform)];
|
|
118
|
+
pipeline([...stages, req]).catch((err: unknown) => reject(bodyError ?? err));
|
|
119
|
+
});
|
|
120
|
+
}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The browser build of `node-sessions.ts`, which the `browser` field of `package.json` substitutes for it, so that
|
|
3
|
+
* browser bundles never resolve the storage SDKs. Uploads only open a session when `hasNodeHttp()` is true, which it
|
|
4
|
+
* never is in a browser.
|
|
5
|
+
*/
|
|
6
|
+
function nodeOnly(): never {
|
|
7
|
+
throw new Error('Uploading in parts is only available in Node.js');
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
export function uploadGcsResumable(): never {
|
|
11
|
+
return nodeOnly();
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
export function uploadAzureBlocks(): never {
|
|
15
|
+
return nodeOnly();
|
|
16
|
+
}
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The upload-session writers that run on the storage services' own SDKs: Google Cloud Storage resumable uploads and
|
|
3
|
+
* Azure block blobs (see `chunked-upload.ts`). The SDKs are optional peer dependencies, loaded on first use.
|
|
4
|
+
*
|
|
5
|
+
* Node.js only. Browser bundles get `node-sessions.browser.ts` instead, through the `browser` field of
|
|
6
|
+
* `package.json`, so they never resolve the SDKs or Node's `stream` module.
|
|
7
|
+
*/
|
|
8
|
+
import type { GcsResumableUploadSession } from '@vertesia/common';
|
|
9
|
+
import type { PartsBody } from './chunked-upload.js';
|
|
10
|
+
import { exactLength } from './node-request.js';
|
|
11
|
+
import { UploadSessionUnavailableError } from './upload-runtime.js';
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* The `catch` handler of an optional package's `import()`: a missing package means uploads in parts are unavailable.
|
|
15
|
+
*
|
|
16
|
+
* Each `import()` of an optional package carries webpack's `webpackIgnore` comment and chains this handler directly,
|
|
17
|
+
* the forms webpack and esbuild leave to run time, so a consumer's Node.js bundle builds without the packages.
|
|
18
|
+
*/
|
|
19
|
+
function missingPackage(name: string): (err: unknown) => never {
|
|
20
|
+
return (err) => {
|
|
21
|
+
const code = (err as { code?: unknown } | undefined)?.code;
|
|
22
|
+
if (code === 'ERR_MODULE_NOT_FOUND' || code === 'MODULE_NOT_FOUND') {
|
|
23
|
+
throw new UploadSessionUnavailableError(`the optional package ${name}`, err);
|
|
24
|
+
}
|
|
25
|
+
throw err;
|
|
26
|
+
};
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/** The content as a Node stream that fails, before it ends, when it does not match its declared length. */
|
|
30
|
+
async function nodeReadable(parts: PartsBody): Promise<import('node:stream').Readable> {
|
|
31
|
+
const { Readable, Transform } = await import('node:stream');
|
|
32
|
+
const { pipeline } = await import('node:stream/promises');
|
|
33
|
+
const web = parts.body instanceof Blob ? parts.body.stream() : parts.body;
|
|
34
|
+
const source = Readable.fromWeb(web as Parameters<typeof Readable.fromWeb>[0]);
|
|
35
|
+
if (parts.size === undefined) return source;
|
|
36
|
+
const counted = exactLength(parts.size, Transform);
|
|
37
|
+
// The pipeline forwards an error of the source to `counted`, which is what the SDK reads.
|
|
38
|
+
pipeline(source, counted).catch(() => undefined);
|
|
39
|
+
return counted;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* The SDK's ceiling on retrying, counted from the start of the upload. A session URI expires a week after it is
|
|
44
|
+
* created, so a later retry could not succeed. The retry count is what normally ends the retries, within minutes.
|
|
45
|
+
* See https://cloud.google.com/storage/docs/resumable-uploads
|
|
46
|
+
*/
|
|
47
|
+
const GCS_SESSION_LIFETIME_SECONDS = 7 * 24 * 60 * 60;
|
|
48
|
+
/** Retries of failed chunks across the whole upload: the SDK does not reset its count after a chunk succeeds. */
|
|
49
|
+
const GCS_SESSION_MAX_RETRIES = 10;
|
|
50
|
+
|
|
51
|
+
/** Writes the content to a GCS resumable session. Returns the object's MD5 in hex, the ETag a single PUT gives. */
|
|
52
|
+
export async function uploadGcsResumable(
|
|
53
|
+
session: GcsResumableUploadSession,
|
|
54
|
+
parts: PartsBody,
|
|
55
|
+
): Promise<string | undefined> {
|
|
56
|
+
const [{ IdempotencyStrategy, Storage }, { PassThroughClient }] = await Promise.all([
|
|
57
|
+
import(/* webpackIgnore: true */ '@google-cloud/storage').catch(missingPackage('@google-cloud/storage')),
|
|
58
|
+
import(/* webpackIgnore: true */ 'google-auth-library').catch(missingPackage('google-auth-library')),
|
|
59
|
+
]);
|
|
60
|
+
const { pipeline } = await import('node:stream/promises');
|
|
61
|
+
const slash = session.path.indexOf('/');
|
|
62
|
+
// The session URI is the only credential the upload needs, so requests carry no other.
|
|
63
|
+
const file = new Storage({
|
|
64
|
+
authClient: new PassThroughClient(),
|
|
65
|
+
retryOptions: {
|
|
66
|
+
// The default strategy retries only uploads with a precondition, which leaves a session with no retries.
|
|
67
|
+
// Resending a chunk is safe: the SDK first asks the session for the offset it persisted.
|
|
68
|
+
idempotencyStrategy: IdempotencyStrategy.RetryAlways,
|
|
69
|
+
// Both limits count from the start of the upload, not per chunk, so the defaults (3 retries within 600
|
|
70
|
+
// seconds) would leave a long upload with none.
|
|
71
|
+
maxRetries: GCS_SESSION_MAX_RETRIES,
|
|
72
|
+
totalTimeout: GCS_SESSION_LIFETIME_SECONDS,
|
|
73
|
+
},
|
|
74
|
+
})
|
|
75
|
+
.bucket(session.path.slice(0, slash))
|
|
76
|
+
.file(session.path.slice(slash + 1));
|
|
77
|
+
const writable = file.createWriteStream({
|
|
78
|
+
uri: session.url,
|
|
79
|
+
resumable: true,
|
|
80
|
+
chunkSize: session.part_size,
|
|
81
|
+
metadata: { contentType: session.mime_type },
|
|
82
|
+
});
|
|
83
|
+
await pipeline(await nodeReadable(parts), writable);
|
|
84
|
+
const md5 = file.metadata?.md5Hash;
|
|
85
|
+
return typeof md5 === 'string' ? Buffer.from(md5, 'base64').toString('hex') : undefined;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/** Uploads the content as blocks of `blockSize` bytes, one at a time, then commits them. Returns the blob's ETag. */
|
|
89
|
+
export async function uploadAzureBlocks(
|
|
90
|
+
url: string,
|
|
91
|
+
blockSize: number,
|
|
92
|
+
contentType: string | undefined,
|
|
93
|
+
parts: PartsBody,
|
|
94
|
+
): Promise<string | undefined> {
|
|
95
|
+
const { BlockBlobClient } = await import(/* webpackIgnore: true */ '@azure/storage-blob').catch(
|
|
96
|
+
missingPackage('@azure/storage-blob'),
|
|
97
|
+
);
|
|
98
|
+
const readable = await nodeReadable(parts);
|
|
99
|
+
try {
|
|
100
|
+
const response = await new BlockBlobClient(url).uploadStream(readable, blockSize, 1, {
|
|
101
|
+
blobHTTPHeaders: { blobContentType: contentType },
|
|
102
|
+
});
|
|
103
|
+
return response.etag;
|
|
104
|
+
} catch (err: unknown) {
|
|
105
|
+
// The SDK stops reading but leaves the stream open; destroying it cancels the source.
|
|
106
|
+
readable.destroy();
|
|
107
|
+
throw err;
|
|
108
|
+
}
|
|
109
|
+
}
|