@uploadfilm/sdk 0.0.0-stage → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +4 -0
- package/README.md +529 -2
- package/dist/ai/embed.d.ts +58 -0
- package/dist/ai/fake.d.ts +41 -0
- package/dist/ai/http.d.ts +17 -0
- package/dist/ai/index.d.ts +27 -0
- package/dist/ai/languages.d.ts +2 -0
- package/dist/ai/ollama.d.ts +23 -0
- package/dist/ai/openaiCompatible.d.ts +54 -0
- package/dist/ai/passages.d.ts +28 -0
- package/dist/ai/transcribe.d.ts +84 -0
- package/dist/ai/types.d.ts +141 -0
- package/dist/ai/usage.d.ts +66 -0
- package/dist/ai/whisperJson.d.ts +15 -0
- package/dist/ai/workersAi.d.ts +56 -0
- package/dist/ai.js +1 -0
- package/dist/ai.mjs +1 -0
- package/dist/cards/files.d.ts +91 -0
- package/dist/cards/index.d.ts +15 -0
- package/dist/cards/layouts.d.ts +121 -0
- package/dist/cards/probe.d.ts +69 -0
- package/dist/cards/proxies.d.ts +70 -0
- package/dist/cards/select.d.ts +68 -0
- package/dist/cards/volumes.d.ts +170 -0
- package/dist/cards/xml.d.ts +80 -0
- package/dist/cards.js +1 -0
- package/dist/cards.mjs +1 -0
- package/dist/copy/checksums.d.ts +66 -0
- package/dist/copy/engine.d.ts +24 -0
- package/dist/copy/errors.d.ts +109 -0
- package/dist/copy/index.d.ts +15 -0
- package/dist/copy/steps.d.ts +28 -0
- package/dist/copy/sweep.d.ts +24 -0
- package/dist/copy/types.d.ts +363 -0
- package/dist/embed/player.js +2 -0
- package/dist/host/adapters.d.ts +107 -0
- package/dist/host/detect.d.ts +68 -0
- package/dist/host/encoders.d.ts +34 -0
- package/dist/host/environment.d.ts +53 -0
- package/dist/host/index.d.ts +20 -0
- package/dist/host/linux.d.ts +94 -0
- package/dist/host/mac.d.ts +47 -0
- package/dist/host/network.d.ts +54 -0
- package/dist/host/node.d.ts +13 -0
- package/dist/host/policy.d.ts +101 -0
- package/dist/host/tuner.d.ts +48 -0
- package/dist/host/types.d.ts +118 -0
- package/dist/host-node.js +1 -0
- package/dist/host-node.mjs +1 -0
- package/dist/host.js +1 -0
- package/dist/host.mjs +1 -0
- package/dist/images/index.d.ts +11 -0
- package/dist/images/settings.d.ts +43 -0
- package/dist/images/types.d.ts +83 -0
- package/dist/images/url.d.ts +65 -0
- package/dist/index.d.ts +16 -0
- package/dist/index.js +1 -0
- package/dist/index.mjs +1 -0
- package/dist/player/adapters/hlsjs.d.ts +28 -0
- package/dist/player/bandwidthMemory.d.ts +28 -0
- package/dist/player/captions/index.d.ts +55 -0
- package/dist/player/embed/code.d.ts +195 -0
- package/dist/player/embed/constants.d.ts +4 -0
- package/dist/player/embed/controller.d.ts +62 -0
- package/dist/player/embed/element.d.ts +32 -0
- package/dist/player/embed/frame.d.ts +65 -0
- package/dist/player/embed/global.d.ts +14 -0
- package/dist/player/embed/index.d.ts +15 -0
- package/dist/player/embed/loader.d.ts +1 -0
- package/dist/player/embed/oembed.d.ts +74 -0
- package/dist/player/embed/protocol.d.ts +64 -0
- package/dist/player/engine.d.ts +215 -0
- package/dist/player/index.d.ts +18 -0
- package/dist/player/start.d.ts +62 -0
- package/dist/player/store.d.ts +10 -0
- package/dist/player/storyboard/index.d.ts +161 -0
- package/dist/player/stream.d.ts +206 -0
- package/dist/player/tracks.d.ts +41 -0
- package/dist/player.js +1 -0
- package/dist/player.mjs +1 -0
- package/dist/proxies/args.d.ts +19 -0
- package/dist/proxies/budget.d.ts +57 -0
- package/dist/proxies/camera.d.ts +104 -0
- package/dist/proxies/formats.d.ts +36 -0
- package/dist/proxies/index.d.ts +18 -0
- package/dist/proxies/jobs.d.ts +98 -0
- package/dist/proxies/node.d.ts +32 -0
- package/dist/proxies/plan.d.ts +89 -0
- package/dist/proxies/probe.d.ts +39 -0
- package/dist/proxies/render.d.ts +104 -0
- package/dist/proxies/safety.d.ts +45 -0
- package/dist/proxies/types.d.ts +83 -0
- package/dist/proxies/upload.d.ts +14 -0
- package/dist/react/BigPlay.d.ts +26 -0
- package/dist/react/Player.d.ts +58 -0
- package/dist/react/embed/index.d.ts +45 -0
- package/dist/react/examples/CustomControls.d.ts +8 -0
- package/dist/react/images/Image.d.ts +16 -0
- package/dist/react/images/index.d.ts +4 -0
- package/dist/react/images/useImage.d.ts +47 -0
- package/dist/react/index.d.ts +17 -0
- package/dist/react/storyboard/index.d.ts +58 -0
- package/dist/react/theme.d.ts +37 -0
- package/dist/react/useAdapterErrors.d.ts +8 -0
- package/dist/react/usePlaybackStart.d.ts +18 -0
- package/dist/react/usePlayer.d.ts +28 -0
- package/dist/react/useReducedMotion.d.ts +2 -0
- package/dist/react/useTimedTrack.d.ts +22 -0
- package/dist/react.js +1 -0
- package/dist/react.mjs +1 -0
- package/dist/server/auth/apiKeys.d.ts +30 -0
- package/dist/server/auth/cache.d.ts +21 -0
- package/dist/server/auth/deviceTokens.d.ts +38 -0
- package/dist/server/auth/envelope.d.ts +15 -0
- package/dist/server/auth/grants.d.ts +76 -0
- package/dist/server/auth/index.d.ts +19 -0
- package/dist/server/auth/jwt.d.ts +65 -0
- package/dist/server/auth/keyring.d.ts +31 -0
- package/dist/server/auth/kv.d.ts +12 -0
- package/dist/server/auth/repository.d.ts +45 -0
- package/dist/server/auth/revocations.d.ts +21 -0
- package/dist/server/auth/scope.d.ts +4 -0
- package/dist/server/auth/serviceAuth.d.ts +24 -0
- package/dist/server/auth/signedUrls.d.ts +28 -0
- package/dist/server/embed/index.d.ts +36 -0
- package/dist/server/embed/lifecycle.d.ts +53 -0
- package/dist/server/embed/playerScript.d.ts +4 -0
- package/dist/server/images/index.d.ts +15 -0
- package/dist/server/images/job.d.ts +80 -0
- package/dist/server/images/layout.d.ts +24 -0
- package/dist/server/images/params.d.ts +90 -0
- package/dist/server/images/route.d.ts +101 -0
- package/dist/server/images/sign.d.ts +55 -0
- package/dist/server/index.d.ts +42 -0
- package/dist/server/media/client.d.ts +99 -0
- package/dist/server/media/frameCheck.d.ts +62 -0
- package/dist/server/media/index.d.ts +10 -0
- package/dist/server/media/protocol.d.ts +556 -0
- package/dist/server/media/storyboard.d.ts +144 -0
- package/dist/server/media/transcript.d.ts +143 -0
- package/dist/server/pipeline/index.d.ts +12 -0
- package/dist/server/pipeline/match.d.ts +10 -0
- package/dist/server/pipeline/pipeline.d.ts +182 -0
- package/dist/server/pipeline/queues.d.ts +108 -0
- package/dist/server/pipeline/sources.d.ts +54 -0
- package/dist/server/r2/client.d.ts +164 -0
- package/dist/server/r2/crc32.d.ts +24 -0
- package/dist/server/r2/encoding.d.ts +35 -0
- package/dist/server/r2/errors.d.ts +9 -0
- package/dist/server/r2/integrity.d.ts +25 -0
- package/dist/server/r2/parts.d.ts +46 -0
- package/dist/server/r2/sign.d.ts +47 -0
- package/dist/server/redis/index.d.ts +39 -0
- package/dist/server/state/memory.d.ts +18 -0
- package/dist/server/state/r2.d.ts +73 -0
- package/dist/server/state/types.d.ts +57 -0
- package/dist/server/storage/chunked.d.ts +25 -0
- package/dist/server/storage/index.d.ts +12 -0
- package/dist/server/storage/r2.d.ts +96 -0
- package/dist/server/storage/s3.d.ts +144 -0
- package/dist/server/testing/fake-r2.d.ts +410 -0
- package/dist/server/testing/index.d.ts +10 -0
- package/dist/server/testing/kv-contract.d.ts +14 -0
- package/dist/server/transfer/handler.d.ts +217 -0
- package/dist/server/transfer/keys.d.ts +45 -0
- package/dist/server/transfer/protocol.d.ts +190 -0
- package/dist/server/zip/client.d.ts +32 -0
- package/dist/server/zip/generations.d.ts +83 -0
- package/dist/server/zip/index.d.ts +12 -0
- package/dist/server/zip/planner.d.ts +94 -0
- package/dist/server/zip/sign.d.ts +22 -0
- package/dist/server-media.js +1 -0
- package/dist/server-media.mjs +1 -0
- package/dist/server-redis.js +1 -0
- package/dist/server-redis.mjs +1 -0
- package/dist/server-testing.js +1 -0
- package/dist/server-testing.mjs +1 -0
- package/dist/server.js +1 -0
- package/dist/server.mjs +1 -0
- package/dist/storage/browserFile.d.ts +19 -0
- package/dist/storage/index.d.ts +12 -0
- package/dist/storage/local.d.ts +9 -0
- package/dist/storage/presigned.d.ts +22 -0
- package/dist/sync/ffmpeg.d.ts +44 -0
- package/dist/sync/index.d.ts +17 -0
- package/dist/sync/match.d.ts +28 -0
- package/dist/sync/probe.d.ts +30 -0
- package/dist/sync/timecode.d.ts +50 -0
- package/dist/sync/transcript.d.ts +35 -0
- package/dist/sync/types.d.ts +172 -0
- package/dist/sync/waveform.d.ts +89 -0
- package/dist/sync.js +1 -0
- package/dist/sync.mjs +1 -0
- package/dist/transfer/claims.d.ts +41 -0
- package/dist/transfer/client.d.ts +267 -0
- package/dist/transfer/download.d.ts +192 -0
- package/dist/transfer/engine.d.ts +402 -0
- package/dist/transfer/errors.d.ts +50 -0
- package/dist/transfer/growing/bytes.d.ts +17 -0
- package/dist/transfer/growing/index.d.ts +12 -0
- package/dist/transfer/growing/parts.d.ts +157 -0
- package/dist/transfer/growing/retry.d.ts +38 -0
- package/dist/transfer/growing/store.d.ts +66 -0
- package/dist/transfer/growing/upload.d.ts +126 -0
- package/dist/transfer/index.d.ts +40 -0
- package/dist/transfer/integrity.d.ts +93 -0
- package/dist/transfer/limits.d.ts +34 -0
- package/dist/transfer/node.d.ts +63 -0
- package/dist/transfer/pool.d.ts +83 -0
- package/dist/transfer/protocol.d.ts +172 -0
- package/dist/transfer/signer.d.ts +29 -0
- package/dist/transfer/storage.d.ts +135 -0
- package/dist/transfer/timing.d.ts +132 -0
- package/dist/transfer/transports/electronNet.d.ts +62 -0
- package/dist/transfer/transports/fetch.d.ts +15 -0
- package/dist/transfer/transports/retryAfter.d.ts +12 -0
- package/dist/transfer/transports/undici.d.ts +72 -0
- package/dist/transfer/tuner.d.ts +56 -0
- package/dist/workers/cli.mjs +115 -0
- package/dist/workers/media/config.mjs +53 -0
- package/dist/workers/media/container/Dockerfile +19 -0
- package/dist/workers/media/container/FFMPEG.md +129 -0
- package/dist/workers/media/container/runner.mjs +2690 -0
- package/dist/workers/media/index.mjs +1011 -0
- package/dist/workers/media/init/config.ts +52 -0
- package/dist/workers/media/init/worker.ts +8 -0
- package/dist/workers/media/init/wrangler.toml +44 -0
- package/dist/workers/media/runner.mjs +2656 -0
- package/dist/workers/media/types/build/entry/config.d.mts +6 -0
- package/dist/workers/media/types/build/entry/index.d.mts +43 -0
- package/dist/workers/media/types/build/entry/program.d.mts +13 -0
- package/dist/workers/media/types/build/entry/runner.d.mts +12 -0
- package/dist/workers/media/types/build/entry/settings.d.mts +40 -0
- package/dist/workers/media/types/src/config.d.mts +135 -0
- package/dist/workers/media/types/src/jobs.d.mts +73 -0
- package/dist/workers/media/types/src/runner/container.d.mts +32 -0
- package/dist/workers/media/types/src/runner/ffmpeg.d.mts +227 -0
- package/dist/workers/media/types/src/runner/job.d.mts +107 -0
- package/dist/workers/media/types/src/runner/node.d.mts +71 -0
- package/dist/workers/media/types/src/runner/plugins.d.mts +45 -0
- package/dist/workers/media/types/src/serve.d.mts +107 -0
- package/dist/workers/media/types/src/steps/audio.d.mts +33 -0
- package/dist/workers/media/types/src/steps/edit.d.mts +62 -0
- package/dist/workers/media/types/src/steps/frameCheck.d.mts +44 -0
- package/dist/workers/media/types/src/steps/gate.d.mts +18 -0
- package/dist/workers/media/types/src/steps/paths.d.mts +1 -0
- package/dist/workers/media/types/src/steps/prep.d.mts +60 -0
- package/dist/workers/media/types/src/steps/render.d.mts +40 -0
- package/dist/workers/media/types/src/steps/runner.d.mts +30 -0
- package/dist/workers/media/types/src/steps/transcribe.d.mts +89 -0
- package/dist/workers/media/types/src/storyboard/index.d.mts +61 -0
- package/dist/workers/media/types/src/transcoder.d.mts +30 -0
- package/dist/workers/media/types/src/worker.d.mts +30 -0
- package/dist/workers/zip/index.mjs +589 -0
- package/dist/workers/zip/init/config.ts +22 -0
- package/dist/workers/zip/init/worker.ts +5 -0
- package/dist/workers/zip/init/wrangler.toml +25 -0
- package/dist/workers/zip/types/build/entry/index.d.mts +5 -0
- package/dist/workers/zip/types/src/config.d.mts +35 -0
- package/dist/workers/zip/types/src/serve.d.mts +86 -0
- package/dist/workers/zip/types/src/worker.d.mts +12 -0
- package/dist/zip/index.d.ts +13 -0
- package/dist/zip/layout.d.ts +129 -0
- package/dist/zip/names.d.ts +34 -0
- package/dist/zip/plan.d.ts +115 -0
- package/dist/zip/sw.d.ts +7 -0
- package/package.json +313 -4
|
@@ -0,0 +1,402 @@
|
|
|
1
|
+
import { type ConnectionLimits, type PartSizeHook } from "./limits";
|
|
2
|
+
import { type Stamp } from "./node";
|
|
3
|
+
import type { ByteSource, ClaimStore, Hasher, PutReceipt, SourceStamp, Yieldable } from "./protocol";
|
|
4
|
+
import type { ListedPart, Storage } from "./storage";
|
|
5
|
+
import { type TransferTiming } from "./timing";
|
|
6
|
+
export interface Multipart {
|
|
7
|
+
uploadId: string;
|
|
8
|
+
partSize: number;
|
|
9
|
+
/** part number → its MD5 hex (= storage's part ETag). */
|
|
10
|
+
parts: Record<number, string>;
|
|
11
|
+
/** part number → zlib crc32 of its bytes. */
|
|
12
|
+
crcs?: Record<number, number>;
|
|
13
|
+
/** integrity "sha256": part number → the SHA-256 hex storage receipted for it. */
|
|
14
|
+
sha256s?: Record<number, string>;
|
|
15
|
+
/**
|
|
16
|
+
* Taken over from a claim holder that died (its upload id, no parts of ours yet): the resume proof adopts every part
|
|
17
|
+
* storage lists whose bytes re-read from the source hash to what storage holds, and sends the rest.
|
|
18
|
+
*/
|
|
19
|
+
adopt?: boolean;
|
|
20
|
+
}
|
|
21
|
+
/** The record of one object's upload (UploadFilm's Version). Its identity matters: a replaced record is a new version. */
|
|
22
|
+
export interface ItemVersion {
|
|
23
|
+
key: string;
|
|
24
|
+
size: number;
|
|
25
|
+
uploaded: number;
|
|
26
|
+
status: string;
|
|
27
|
+
error?: string;
|
|
28
|
+
/** `sha256`: the object's SHA-256 (hex); `sha256Composite`: storage's multipart composite ("hex-N"). */
|
|
29
|
+
verified?: {
|
|
30
|
+
etag: string;
|
|
31
|
+
at: string;
|
|
32
|
+
crc32?: string;
|
|
33
|
+
sha256?: string;
|
|
34
|
+
sha256Composite?: string;
|
|
35
|
+
};
|
|
36
|
+
/**
|
|
37
|
+
* RL-10: when a CreateMultipartUpload was first asked for this version and its upload id isn't recorded yet (saved
|
|
38
|
+
* before the create). A run that finds it set (killed in between, or the answer was lost) aborts the multipart
|
|
39
|
+
* storage opened for it before creating another.
|
|
40
|
+
*/
|
|
41
|
+
creating?: string;
|
|
42
|
+
}
|
|
43
|
+
/** A whole small object the product prepares itself (UploadFilm: posters): no Content-MD5, no item status. */
|
|
44
|
+
export interface BlobSend<S = unknown> {
|
|
45
|
+
/** What is about to be sent (UploadFilm: the poster's path at this moment). */
|
|
46
|
+
snapshot(): S;
|
|
47
|
+
read(s: S): Promise<Uint8Array>;
|
|
48
|
+
/** Stored. */
|
|
49
|
+
done(s: S): void;
|
|
50
|
+
/** A failure about `s` says nothing any more (another blob was chosen meanwhile). */
|
|
51
|
+
stale(s: S): boolean;
|
|
52
|
+
/** Failed for good. */
|
|
53
|
+
fail(msg: string): void;
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* One object to upload, as the product's view of it (getters over its own records). Created fresh by the host for
|
|
57
|
+
* every pick; the engine keys everything by `id`.
|
|
58
|
+
*/
|
|
59
|
+
export interface UploadItem {
|
|
60
|
+
/** Unique and stable; retry keys are `${id}`, `${id}#<part>`, `${id}#resume`, `${id}#complete`. */
|
|
61
|
+
readonly id: string;
|
|
62
|
+
/** Signer scope (UploadFilm: the project id); also the per-scope rate's key. */
|
|
63
|
+
readonly scope: string;
|
|
64
|
+
readonly lane: string;
|
|
65
|
+
/** For messages: "<name>: …". */
|
|
66
|
+
readonly name: string;
|
|
67
|
+
/** For the failure message: "<path>: …". */
|
|
68
|
+
readonly path: string;
|
|
69
|
+
/** The version record (identity compared to tell a replaced version). */
|
|
70
|
+
readonly version: ItemVersion;
|
|
71
|
+
mp: Multipart | undefined;
|
|
72
|
+
/** Forget the multipart (adopting a stored copy deletes the field). */
|
|
73
|
+
deleteMp(): void;
|
|
74
|
+
readonly contentType: string;
|
|
75
|
+
/** Counted against the scope's quota (UploadFilm: originals). */
|
|
76
|
+
readonly counts: boolean;
|
|
77
|
+
/** Written once, never overwritten: a different stored object is a conflict, not a reason to send again. */
|
|
78
|
+
readonly immutable: boolean;
|
|
79
|
+
/** When set, the source's mtime must still be this (an app rewriting metadata in place keeps the size). */
|
|
80
|
+
readonly mtimeMs?: number;
|
|
81
|
+
/** The bytes: a local path (read with node:fs exactly as before ByteSource existed), or any ByteSource. */
|
|
82
|
+
readonly source: string | ByteSource;
|
|
83
|
+
/** What "waiting for its source" is keyed by (UploadFilm: the original's path, even for a proxy). */
|
|
84
|
+
readonly waitKey: string;
|
|
85
|
+
/** A product-prepared small object instead of `source`. */
|
|
86
|
+
readonly blob?: BlobSend<any>;
|
|
87
|
+
/** sha256 hex of the whole source, when known: lets the server's dedupe hook answer "already stored". */
|
|
88
|
+
readonly sha256?: string;
|
|
89
|
+
/** The source as it was when `sha256` was computed: the hash is only used (and a dedupe answer only trusted) while
|
|
90
|
+
* the source is still exactly that file. */
|
|
91
|
+
readonly sha256Stamp?: Stamp | SourceStamp;
|
|
92
|
+
/**
|
|
93
|
+
* The caller declared what the bytes are (a content id): only the source as it was then may be sent. A source that
|
|
94
|
+
* isn't that one any more fails the item (SourceChanged), never uploads other bytes under the declaration (R2-3).
|
|
95
|
+
*/
|
|
96
|
+
readonly boundTo?: SourceStamp;
|
|
97
|
+
}
|
|
98
|
+
export type SourceCheck = "match" | "changed" | "otherCard" | "unknown";
|
|
99
|
+
export interface EngineHost<I extends UploadItem = UploadItem> {
|
|
100
|
+
/** Uploads can run (credentials/device ready). */
|
|
101
|
+
ready(): boolean;
|
|
102
|
+
/** Items of a lane that are ready to send, in priority order. */
|
|
103
|
+
candidates(lane: string): Iterable<I>;
|
|
104
|
+
/** The item still belongs to the product (a unit finishing for a removed item reports nothing). */
|
|
105
|
+
live(item: I): boolean;
|
|
106
|
+
/** A failure the product handles itself (UploadFilm: Deleted, ProjectDeleted). true = handled. */
|
|
107
|
+
onFatal?(item: I, e: unknown): boolean;
|
|
108
|
+
/** The item's version went to "error". */
|
|
109
|
+
onFailed?(item: I, msg: string): void;
|
|
110
|
+
/** The server no longer knows this client may write the scope (UploadFilm: NotClaimed). true = handled, retry later. */
|
|
111
|
+
claimLost?(item: I, e: unknown): boolean;
|
|
112
|
+
/** Extra check before reading/resuming/stitching (UploadFilm: camera proxies). false = not now (it backed off). */
|
|
113
|
+
guard?(item: I, retryKey: string): boolean;
|
|
114
|
+
/** Is the file at `item.source` still this item's source? Default: path + size (+ mtime). */
|
|
115
|
+
sourceCheck?(item: I): Promise<SourceCheck>;
|
|
116
|
+
/** The source stamp approved for a multipart created now (default: the stamp itself). null = not approved. */
|
|
117
|
+
approveOnCreate?(item: I, stamp: SourceStamp | null): SourceStamp | null;
|
|
118
|
+
/** May a resumed multipart proven from `stamp` be approved? (default: yes) */
|
|
119
|
+
stampApproved?(item: I, stamp: SourceStamp | null): boolean;
|
|
120
|
+
/**
|
|
121
|
+
* integrity "sha256": storage completed a multipart without reporting the composite SHA-256 the engine could check
|
|
122
|
+
* (or a single PUT's copy was only proven by MD5). "verified" = the product proved the stored bytes itself (e.g.
|
|
123
|
+
* SuperBuilder's server-side SHA-256 re-read); anything else leaves it stored but unverified (never a success).
|
|
124
|
+
*/
|
|
125
|
+
confirmStored?(item: I, stored: {
|
|
126
|
+
key: string;
|
|
127
|
+
size: number | null;
|
|
128
|
+
etag: string | null;
|
|
129
|
+
sha256Parts?: Record<number, string>;
|
|
130
|
+
}): Promise<"verified" | "unverified">;
|
|
131
|
+
/** Save soon / save now / the item's published status changed. */
|
|
132
|
+
touch(item: I): void;
|
|
133
|
+
save(item: I): void;
|
|
134
|
+
dirty(item: I): void;
|
|
135
|
+
/** An item was refused for quota (first: it wasn't waiting before). */
|
|
136
|
+
onQuota?(item: I, first: boolean): void;
|
|
137
|
+
/** The stored copy at `key` (the version that was stitched, not whatever the item holds now) doesn't match its source. */
|
|
138
|
+
onDemoted?(item: I, key: string): void;
|
|
139
|
+
/** Quota waiters still worth waiting for (removed, finished or failed ones are forgotten). */
|
|
140
|
+
stillWaiting?(id: string): boolean;
|
|
141
|
+
}
|
|
142
|
+
export interface LaneSpec {
|
|
143
|
+
name: string;
|
|
144
|
+
/** Picked before anything else. */
|
|
145
|
+
first?: boolean;
|
|
146
|
+
/** The primary lane: keeps at least ⌈minShare × sockets⌉ of them while it has work. At most one lane. */
|
|
147
|
+
minShare?: number;
|
|
148
|
+
/** A borrower whose queued PUTs may be cancelled to give the primary its share back. */
|
|
149
|
+
yieldable?: boolean;
|
|
150
|
+
/** At most this many units of the lane at once. */
|
|
151
|
+
maxActive?: number;
|
|
152
|
+
}
|
|
153
|
+
export interface EngineCoreOptions<I extends UploadItem> {
|
|
154
|
+
host: EngineHost<I>;
|
|
155
|
+
storage: Storage;
|
|
156
|
+
timing: TransferTiming;
|
|
157
|
+
hasher: Hasher;
|
|
158
|
+
lanes: LaneSpec[];
|
|
159
|
+
/** Tuner bounds (default 4..32, starting at 8). Changed live with `setConnectionLimits`. */
|
|
160
|
+
connections?: {
|
|
161
|
+
min?: number;
|
|
162
|
+
max?: number;
|
|
163
|
+
start?: number;
|
|
164
|
+
};
|
|
165
|
+
/**
|
|
166
|
+
* Part size for a multipart created from now on (bytes), given the object's size; e.g. `partSizeFor(size,
|
|
167
|
+
* tuner.decision)` from `@uploadfilm/sdk/host`. Raised to the engine's own floor for the object (`partSizeFor`:
|
|
168
|
+
* 16 MiB, or more so it fits in 9,500 parts), rounded up to a whole MiB, at most 5 GiB; a hook that throws or answers
|
|
169
|
+
* something that isn't a positive number gets the floor. A multipart already created keeps its part size (its parts
|
|
170
|
+
* are in storage). Default: none (the floor, today's rule). Changed live with `setPartSizeHook`.
|
|
171
|
+
*/
|
|
172
|
+
partSize?: PartSizeHook;
|
|
173
|
+
/**
|
|
174
|
+
* Most bytes of part bodies and small objects buffered in flight at once (e.g. a host decision's `inflightBytes`).
|
|
175
|
+
* One unit may always start when nothing is buffered, so an upload always moves however big its parts. Default: no
|
|
176
|
+
* cap. Changed live with `setInflightBytes`.
|
|
177
|
+
*/
|
|
178
|
+
inflightBytes?: number | null;
|
|
179
|
+
/** Fresh multipart starts after storage refused to finish one, per version, before the item fails (default 3). */
|
|
180
|
+
maxRestarts?: number;
|
|
181
|
+
/**
|
|
182
|
+
* How "verified" is proven (gap 2). "md5" (default): Content-MD5 per part, part ETag = MD5, composite ETag.
|
|
183
|
+
* "sha256": every part and single PUT carries `x-amz-checksum-sha256`, and storage's receipt of it (the response
|
|
184
|
+
* header) must be present, well-formed, once, and equal: anything else fails closed (never stored as verified).
|
|
185
|
+
* The stitched object is verified by storage's composite SHA-256 (or `confirmStored`).
|
|
186
|
+
*/
|
|
187
|
+
integrity?: "md5" | "sha256";
|
|
188
|
+
/**
|
|
189
|
+
* Opt-in (default off): a copy found at the key (another uploader's, an earlier install's, anyone's who can write
|
|
190
|
+
* there, or an earlier attempt of this engine's) is adopted only after reading it back whole through the server's
|
|
191
|
+
* `get` op (createSignHandler `readable`) and matching its SHA-256 against the file; a server that refuses the read
|
|
192
|
+
* then gives `Unverified`, never `verified`. Every copy found at the key is proven that way, an earlier attempt of this
|
|
193
|
+
* engine's whose answer was lost included (it is these bytes: adopted, at the cost of one read; review round 2: an
|
|
194
|
+
* "it may be ours" marker let foreign copies through on MD5). Off: such an object is adopted on its MD5 ETag (cheap; a collision crafted
|
|
195
|
+
* against the file's MD5 would pass: see SECURITY.md "MD5 adoption"). Ignored with integrity "sha256" (storage's SHA-256
|
|
196
|
+
* proves it already).
|
|
197
|
+
*/
|
|
198
|
+
readBack?: boolean;
|
|
199
|
+
/** The client's part size for a new multipart (default partSizeFor). Given → also proposed to the server on create. */
|
|
200
|
+
partSizeFor?(size: number): number;
|
|
201
|
+
/** Below this size an object goes as one PUT (default SMALL, 8 MiB). */
|
|
202
|
+
singlePutMax?: number;
|
|
203
|
+
/** Most parts a server-chosen part size may cut an object into (default 10,000, S3's limit). */
|
|
204
|
+
maxParts?: number;
|
|
205
|
+
/** Use the part size the server answers on create (gap 3). On by default with integrity "sha256", partSizeFor or claims. */
|
|
206
|
+
serverPartSize?: boolean;
|
|
207
|
+
/**
|
|
208
|
+
* Claims (gap 4): before sending anything for an item the engine takes its claim (key = scope + storage key); held
|
|
209
|
+
* by someone else → the item waits ("waiting") and polls (claimPollMs doubling to RETRY_CAP_MS, never past the
|
|
210
|
+
* holder's expiry), then adopts what the holder stored (immutable keys) or takes over an expired claim, keeping the
|
|
211
|
+
* dead holder's proven parts. The claim is renewed every claimTtlMs / 3; a renewal that fails means another engine
|
|
212
|
+
* took over: this one stops sending for the item. Released when the item finishes, fails or is forgotten.
|
|
213
|
+
*/
|
|
214
|
+
claims?: ClaimStore;
|
|
215
|
+
/** Lease length (default 60 s). */
|
|
216
|
+
claimTtlMs?: number;
|
|
217
|
+
/** First wait between looks at a held claim (default 2 s). */
|
|
218
|
+
claimPollMs?: number;
|
|
219
|
+
/** Give up waiting for a holder after this long (code ClaimTimeout, retryable by the product). Default: no limit. */
|
|
220
|
+
maxClaimWaitMs?: number;
|
|
221
|
+
}
|
|
222
|
+
/** Failures the engine's own checks raise (bad bytes): never kept, resent with a budget. */
|
|
223
|
+
export declare class IntegrityError extends Error {
|
|
224
|
+
}
|
|
225
|
+
/** integrity "sha256": storage answered a PUT without its SHA-256 receipt (the bytes may be stored, but unproven). */
|
|
226
|
+
export declare class ReceiptMissing extends IntegrityError {
|
|
227
|
+
}
|
|
228
|
+
/**
|
|
229
|
+
* Borrower lanes' room: what's left of `sockets` after the primary's ⌈minShare × sockets⌉ while the primary has work;
|
|
230
|
+
* everything when it has none.
|
|
231
|
+
*/
|
|
232
|
+
export declare function laneRoom(sockets: number, primaryWaiting: boolean, minShare: number): number;
|
|
233
|
+
/**
|
|
234
|
+
* Borrower lanes' room with the lane share (perf-lab finding 3, docs/platform/TRANSFER-PERF.md): while the primary has
|
|
235
|
+
* a unit waiting, laneRoom(sockets); while it has nothing to start but PUTs in flight (`primaryPuts`), at most
|
|
236
|
+
* ⌊primaryPuts × (1 − minShare) / minShare⌋ (2/3: half as many as the primary's), so the primary keeps ≥ minShare of
|
|
237
|
+
* the connections that carry bytes, not only of those being handed out; with neither, everything.
|
|
238
|
+
*/
|
|
239
|
+
export declare function laneShare(sockets: number, primaryWaiting: boolean, primaryPuts: number, minShare: number): number;
|
|
240
|
+
export declare const EXISTS_MSG = "a different file is already stored under this name in the cloud; nothing was overwritten";
|
|
241
|
+
export declare const MISSING_MSG = "the platform has this original recorded as stored, but storage has no copy of it; nothing was overwritten";
|
|
242
|
+
type Unit<I> = {
|
|
243
|
+
kind: "small";
|
|
244
|
+
item: I;
|
|
245
|
+
} | {
|
|
246
|
+
kind: "part";
|
|
247
|
+
item: I;
|
|
248
|
+
n: number;
|
|
249
|
+
} | {
|
|
250
|
+
kind: "create";
|
|
251
|
+
item: I;
|
|
252
|
+
} | {
|
|
253
|
+
kind: "complete";
|
|
254
|
+
item: I;
|
|
255
|
+
} | {
|
|
256
|
+
kind: "resume";
|
|
257
|
+
item: I;
|
|
258
|
+
};
|
|
259
|
+
export type EngineCore<I extends UploadItem = UploadItem> = ReturnType<typeof createEngineCore<I>>;
|
|
260
|
+
export declare function createEngineCore<I extends UploadItem>(o: EngineCoreOptions<I>): {
|
|
261
|
+
/** Start the loops: every ENGINE_TICK_MS measure the rate and start work (plus `onTick`), every TUNE_INTERVAL_MS tune. */
|
|
262
|
+
start(onTick?: () => void): void;
|
|
263
|
+
stop(): void;
|
|
264
|
+
pump: () => void;
|
|
265
|
+
tickRate: () => void;
|
|
266
|
+
tune: () => void;
|
|
267
|
+
nextUnit: () => Unit<I> | null;
|
|
268
|
+
localDigest: (src: string, size: number, partSize?: number) => Promise<{
|
|
269
|
+
etag: string;
|
|
270
|
+
crc32: string;
|
|
271
|
+
}>;
|
|
272
|
+
storage: {
|
|
273
|
+
createMultipart: (scope: string, key: string, contentType: string, size?: number, counts?: boolean) => Promise<string>;
|
|
274
|
+
createOrDedupe: (scope: string, key: string, contentType: string, size?: number, counts?: boolean, sha256?: string) => Promise<import("./storage").Deduped | {
|
|
275
|
+
uploadId: string;
|
|
276
|
+
}>;
|
|
277
|
+
completeMultipart: (scope: string, key: string, uploadId: string, parts: {
|
|
278
|
+
n: number;
|
|
279
|
+
etag: string;
|
|
280
|
+
}[]) => Promise<{
|
|
281
|
+
recovered?: true | undefined;
|
|
282
|
+
size: number | null;
|
|
283
|
+
etag: string | null;
|
|
284
|
+
weak?: boolean;
|
|
285
|
+
}>;
|
|
286
|
+
abortMultipart: (scope: string, key: string, uploadId: string) => Promise<void | undefined>;
|
|
287
|
+
head: (scope: string, key: string) => Promise<import("./storage").Stat>;
|
|
288
|
+
listParts: (scope: string, key: string, uploadId: string) => Promise<Map<number, string>>;
|
|
289
|
+
listUploads: (scope: string, key: string) => Promise<{
|
|
290
|
+
uploadId: string;
|
|
291
|
+
initiated?: string;
|
|
292
|
+
}[]>;
|
|
293
|
+
uploadPart: (scope: string, key: string, uploadId: string, partNumber: number, body: Uint8Array, contentMd5: string, onProgress: (sent: number) => void, timeoutMs: number, y?: Yieldable) => Promise<string>;
|
|
294
|
+
putObject: (scope: string, key: string, body: Uint8Array, contentType: string, onProgress?: (sent: number) => void, timeoutMs?: number, contentMd5?: string, counts?: boolean, y?: Yieldable, sha256?: string, w?: import("./storage").WriteMark) => Promise<string>;
|
|
295
|
+
transport: import("./protocol").Transport;
|
|
296
|
+
createSession: (scope: string, key: string, contentType: string, size: number, counts: boolean, x: {
|
|
297
|
+
sha256?: string;
|
|
298
|
+
checksumAlgorithm?: "SHA256";
|
|
299
|
+
partSize?: number;
|
|
300
|
+
resumeUploadId?: string;
|
|
301
|
+
}) => Promise<import("./storage").CreateAnswer>;
|
|
302
|
+
uploadPartChecked: (scope: string, key: string, uploadId: string, partNumber: number, body: Uint8Array, contentMd5: string, checksumSha256: string, onProgress: (sent: number) => void, timeoutMs: number, y?: Yieldable) => Promise<PutReceipt>;
|
|
303
|
+
putObjectChecked: (scope: string, key: string, body: Uint8Array, contentType: string, onProgress: (sent: number) => void, timeoutMs: number, contentMd5: string, checksumSha256: string, counts: boolean, y?: Yieldable, sha256?: string) => Promise<{
|
|
304
|
+
receipt: PutReceipt;
|
|
305
|
+
} | {
|
|
306
|
+
deduped: import("./storage").Deduped;
|
|
307
|
+
} | {
|
|
308
|
+
exists: import("./storage").CheckedStat;
|
|
309
|
+
}>;
|
|
310
|
+
listPartsChecked: (scope: string, key: string, uploadId: string) => Promise<Map<number, ListedPart>>;
|
|
311
|
+
completeChecked: (scope: string, key: string, uploadId: string, parts: {
|
|
312
|
+
n: number;
|
|
313
|
+
etag: string;
|
|
314
|
+
checksumSha256: string;
|
|
315
|
+
}[]) => Promise<import("./storage").CheckedStat>;
|
|
316
|
+
headWithChecksum: (scope: string, key: string) => Promise<import("./storage").CheckedStat>;
|
|
317
|
+
readBack: (scope: string, key: string, want: number, sha256Stream: () => {
|
|
318
|
+
update(b: Uint8Array): unknown;
|
|
319
|
+
digest(): Uint8Array;
|
|
320
|
+
}, crc32: (b: Uint8Array, prev?: number) => number) => Promise<import("./storage").ReadBack | null>;
|
|
321
|
+
};
|
|
322
|
+
readonly rate: number;
|
|
323
|
+
scopeRates: () => {
|
|
324
|
+
[x: string]: number;
|
|
325
|
+
};
|
|
326
|
+
readonly connections: number;
|
|
327
|
+
readonly target: number;
|
|
328
|
+
/** The tuner's bounds, the in-flight byte cap, what is buffered now and how many reads wait for room. */
|
|
329
|
+
readonly limits: {
|
|
330
|
+
min: number;
|
|
331
|
+
max: number;
|
|
332
|
+
inflightBytes: number | null;
|
|
333
|
+
buffered: number;
|
|
334
|
+
waiting: number;
|
|
335
|
+
};
|
|
336
|
+
setConnectionLimits(a: ConnectionLimits | number, max?: number): void;
|
|
337
|
+
/** Part size for multiparts created from now on (null: the default rule). Saved multiparts keep theirs. */
|
|
338
|
+
setPartSizeHook(fn: PartSizeHook | null): void;
|
|
339
|
+
/** Cap on buffered bytes in flight (null: none). Units already running finish; new ones wait for room. */
|
|
340
|
+
setInflightBytes(n: number | null): void;
|
|
341
|
+
readonly lastTick: number;
|
|
342
|
+
readonly lastBytesAt: number;
|
|
343
|
+
lastError: string | undefined;
|
|
344
|
+
readonly paused: boolean;
|
|
345
|
+
setPaused(v: boolean): void;
|
|
346
|
+
hold(lane: string, v: boolean): void;
|
|
347
|
+
isHeld: (lane: string) => boolean;
|
|
348
|
+
/** `false`: the account shows room again: items waiting for room try again now. */
|
|
349
|
+
roomAgain(): void;
|
|
350
|
+
isQuotaBlocked: () => boolean;
|
|
351
|
+
isWaitingForRoom: (id: string) => boolean;
|
|
352
|
+
/** Every request has failed with a network-class error for OFFLINE_NO_BYTES_MS, and nothing has worked since. */
|
|
353
|
+
looksOffline: () => boolean;
|
|
354
|
+
offlineReason: () => "network" | "server";
|
|
355
|
+
isWaitingForSource: (waitKey: string) => boolean;
|
|
356
|
+
readonly waitingForSourceCount: number;
|
|
357
|
+
/** Forget every retry counter (and restart count) of items whose id starts with `prefix` (Retry). */
|
|
358
|
+
forgetPrefix(prefix: string): void;
|
|
359
|
+
/**
|
|
360
|
+
* A source was found again (re-added, maybe at another mount point): forget the old wait and the backoff it built
|
|
361
|
+
* up, and prove whatever is there now before anything else is sent or stitched.
|
|
362
|
+
*/
|
|
363
|
+
sourceReconnected(prefix: string, oldWaitKey: string, uploadIds: string[]): void;
|
|
364
|
+
/** Drop a multipart's approval (its source changed): the full proof runs before it is used again. */
|
|
365
|
+
revoke: (uploadId: string) => void;
|
|
366
|
+
/** Give up the item's claim (claims mode): the product removed or cancelled it. */
|
|
367
|
+
releaseClaim: (id: string) => undefined;
|
|
368
|
+
/** Units of this item are running (a request may still land). */
|
|
369
|
+
busy: (id: string) => boolean;
|
|
370
|
+
/** Resolves once no unit of this item runs (R2-4: a replacement never races its cancelled predecessor). */
|
|
371
|
+
whenIdle: (id: string) => Promise<void>;
|
|
372
|
+
/** Renew claims now (the tick does it every ENGINE_TICK_MS). */
|
|
373
|
+
renewClaims: () => void;
|
|
374
|
+
/** The claim this engine holds for an item (tests, diagnostics). */
|
|
375
|
+
claimOf: (id: string) => {
|
|
376
|
+
key: string;
|
|
377
|
+
lost: boolean;
|
|
378
|
+
} | undefined;
|
|
379
|
+
tuner: {
|
|
380
|
+
readonly target: number;
|
|
381
|
+
readonly pinned: number | null;
|
|
382
|
+
tune(rate: number, active: number): void;
|
|
383
|
+
setBounds(b: {
|
|
384
|
+
min: number;
|
|
385
|
+
max: number;
|
|
386
|
+
start?: number;
|
|
387
|
+
}): void;
|
|
388
|
+
onError(): void;
|
|
389
|
+
pin(n: number | null): void;
|
|
390
|
+
};
|
|
391
|
+
backoff: (key: string, e?: unknown) => number;
|
|
392
|
+
waitForSource: (waitKey: string) => undefined;
|
|
393
|
+
attempts: () => Map<string, number>;
|
|
394
|
+
inFlight: (lane: string) => number;
|
|
395
|
+
/** Units running now (a stopped engine finishes these: stop() ends the loops, not the work in flight). */
|
|
396
|
+
readonly active: number;
|
|
397
|
+
readonly activePuts: number;
|
|
398
|
+
pinTarget(n: number | null): void;
|
|
399
|
+
/** Borrower lanes' share of the sockets while the primary has work (UploadFilm's proxyShare). */
|
|
400
|
+
borrowerShare(): number;
|
|
401
|
+
};
|
|
402
|
+
export {};
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The transfer engine's one error type and how storage answers are classified. Moved from UploadFilm's
|
|
3
|
+
* app/src/main/storage.ts.
|
|
4
|
+
*/
|
|
5
|
+
/**
|
|
6
|
+
* Storage (S3/R2) error codes that are transient although their HTTP status is a 4xx: send again (re-signing).
|
|
7
|
+
* - RequestTimeout (400): "Your socket connection to the server was not read from or written to within the
|
|
8
|
+
* timeout period": a slow or interrupted upload, not a bad request.
|
|
9
|
+
* - ExpiredToken (400): the presigned credentials expired; a new sign fixes it.
|
|
10
|
+
* - IncompleteBody (400): fewer bytes arrived than Content-Length said (the connection dropped mid-body).
|
|
11
|
+
* - ClientDisconnect (400/499): storage saw the client go away mid-request. 499 or a "Client disconnected"
|
|
12
|
+
* message counts too, whatever the code.
|
|
13
|
+
* 5xx codes (InternalError 500, ServiceUnavailable 503, SlowDown 503) are retryable by status, and SlowDown
|
|
14
|
+
* as 429 too. 403 AccessDenied "Request has expired" is matched by message.
|
|
15
|
+
*/
|
|
16
|
+
export declare const TRANSIENT_4XX: Set<string>;
|
|
17
|
+
/**
|
|
18
|
+
* `status` 0 = no answer (network, timeout, stall). `retryable`: retrying (re-signing) can fix it; the engine retries
|
|
19
|
+
* those for as long as it takes, and gives everything else a budget.
|
|
20
|
+
*/
|
|
21
|
+
export declare class StorageError extends Error {
|
|
22
|
+
status: number;
|
|
23
|
+
code?: string | undefined;
|
|
24
|
+
retryable: boolean;
|
|
25
|
+
constructor(message: string, status: number, code?: string | undefined, retryable?: boolean);
|
|
26
|
+
}
|
|
27
|
+
/** A PUT that gave its place to a higher-priority lane before sending a byte (not a failure). */
|
|
28
|
+
export declare const YIELDED = "Yielded";
|
|
29
|
+
/**
|
|
30
|
+
* How a non-2xx storage answer to a PUT becomes a StorageError: an expired URL, corrupted bytes (BadDigest) and the
|
|
31
|
+
* transient 4xx codes are retryable (503); anything else keeps its status.
|
|
32
|
+
*/
|
|
33
|
+
export declare function putError(status: number, text: string): StorageError;
|
|
34
|
+
/**
|
|
35
|
+
* Errors from a product's signer (its own error class for its HTTP client) become StorageErrors through this hook:
|
|
36
|
+
* return the shape, or undefined for "not one of mine". The default reads `status`/`code`/`retryable` off any error
|
|
37
|
+
* that carries a numeric status and a boolean retryable.
|
|
38
|
+
*/
|
|
39
|
+
export type ErrorShape = {
|
|
40
|
+
message: string;
|
|
41
|
+
status: number;
|
|
42
|
+
code?: string;
|
|
43
|
+
retryable: boolean;
|
|
44
|
+
};
|
|
45
|
+
export type ErrorAdapter = (e: unknown) => ErrorShape | undefined;
|
|
46
|
+
export declare const duckErrors: ErrorAdapter;
|
|
47
|
+
export declare function toStorageError(e: unknown, adapt?: ErrorAdapter): StorageError;
|
|
48
|
+
/** Retrying fixes it: network errors, timeouts, stalls, 5xx, 429, expired URLs, BadDigest. */
|
|
49
|
+
export declare const isTransient: (e: unknown) => boolean;
|
|
50
|
+
export declare const isYielded: (e: unknown) => boolean;
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Byte helpers for the growing uploader. Browser-safe on purpose (no Buffer, no node: imports): this module runs in a
|
|
3
|
+
* page, an extension's offscreen document, a Worker, Electron and Node alike.
|
|
4
|
+
*/
|
|
5
|
+
export declare const toHex: (b: Uint8Array) => string;
|
|
6
|
+
export declare const fromHex: (h: string) => Uint8Array;
|
|
7
|
+
export declare const toB64: (b: Uint8Array) => string;
|
|
8
|
+
export declare const concat: (parts: Uint8Array[]) => Uint8Array;
|
|
9
|
+
export declare const isMd5Hex: (s: unknown) => s is string;
|
|
10
|
+
/** An ETag as storage sends it → bare lowercase (no quotes, no W/). */
|
|
11
|
+
export declare const bareEtag: (e: unknown) => string;
|
|
12
|
+
/**
|
|
13
|
+
* A fragment as a Blob. Copies a Uint8Array's bytes, so the recorder is free to reuse its buffer (ClipMe's `toBlob`).
|
|
14
|
+
* Blobs let a slow network's backlog live outside the JS heap where the runtime can (browsers spill big Blobs to disk).
|
|
15
|
+
*/
|
|
16
|
+
export declare const toBlob: (fragment: Uint8Array | Blob) => Blob;
|
|
17
|
+
export declare const blobBytes: (b: Blob) => Promise<Uint8Array>;
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Growing uploads (upload while recording), ported from ClipMe's uploader. Browser-safe: no node: imports, no Buffer.
|
|
3
|
+
* See docs/platform/GROWING-UPLOADS.md.
|
|
4
|
+
*/
|
|
5
|
+
export { createGrowingUpload, growingRetryDelayMs, COMPLETE_FAILURES_BEFORE_RESTART } from "./upload";
|
|
6
|
+
export type { GrowingUpload, GrowingUploadOptions, GrowingResult, GrowingState, GrowingProgress } from "./upload";
|
|
7
|
+
export { createGrowingRetryPump } from "./retry";
|
|
8
|
+
export type { GrowingLocalCopy, GrowingRetryOptions } from "./retry";
|
|
9
|
+
export { GrowingPartUploader, GROWING_PART_BYTES, MAX_PARALLEL_PARTS, MAX_ATTEMPTS, MAX_PENDING_BYTES, AHEAD_MAX_AGE_MS, SignRefused, PartIntegrityError, isGone, md5Of } from "./parts";
|
|
10
|
+
export type { GrowingPart, PartSender, PartHasher, PartUploaderOptions, SignedPart } from "./parts";
|
|
11
|
+
export { memoryGrowingStore, listRecords as listGrowingUploads, loadRecord as loadGrowingRecord, RECORD_PREFIX as GROWING_RECORD_PREFIX } from "./store";
|
|
12
|
+
export type { GrowingStore, GrowingRecord, GrowingRecordState } from "./store";
|
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Turns a stream of fragments into multipart parts and gets them to storage while the recording runs.
|
|
3
|
+
*
|
|
4
|
+
* Ported from ClipMe's extension/src/upload/parts.ts (PartUploader, ClipMe origin/main 4684516), itself a port of its
|
|
5
|
+
* Electron uploader's part sizing. The behaviour is ClipMe's: every non-trailing part is EXACTLY `partSize` (R2
|
|
6
|
+
* refuses a multipart whose non-trailing parts differ in length, and only at complete, once a recording needs three
|
|
7
|
+
* parts), the trailing part is the only short one, at most 3 parts in flight, 6 attempts per part with backoff 0.5 s
|
|
8
|
+
* doubling to 20 s, the next part's URL signed while the current one uploads, a cap on unsent bytes that ends the
|
|
9
|
+
* upload rather than the recording, and parts an earlier attempt stored are kept by number.
|
|
10
|
+
*
|
|
11
|
+
* What the SDK adds (docs/platform/GROWING-UPLOADS.md lists every difference):
|
|
12
|
+
* - every part is proven: its MD5 is computed before it is sent, retries are signed with Content-MD5 (storage refuses
|
|
13
|
+
* a body that changed in transit), and storage's part ETag must equal that MD5 or the part is sent again;
|
|
14
|
+
* - a kept part (resume) is kept only when the bytes cut at its position now hash to what storage holds: ClipMe trusts
|
|
15
|
+
* the local copy to be the same bytes; here a different part is sent again;
|
|
16
|
+
* - a refusal the server marks as final (a quota, a key outside the layouts) ends the upload at once, without the
|
|
17
|
+
* remaining attempts.
|
|
18
|
+
*
|
|
19
|
+
* Buffers are Blobs so a slow network's backlog can live outside the JS heap (ClipMe's reason); a part's bytes are
|
|
20
|
+
* read into memory only once it has a connection, so memory stays near `maxParallel × partSize`.
|
|
21
|
+
*/
|
|
22
|
+
import { StorageError } from "../errors";
|
|
23
|
+
/** ClipMe's part size: R2's floor for a non-trailing part. */
|
|
24
|
+
export declare const GROWING_PART_BYTES: number;
|
|
25
|
+
export declare const MAX_PARALLEL_PARTS = 3;
|
|
26
|
+
export declare const MAX_ATTEMPTS = 6;
|
|
27
|
+
/** Unsent bytes past which the upload (not the recording) ends: memory is what fails first otherwise. */
|
|
28
|
+
export declare const MAX_PENDING_BYTES: number;
|
|
29
|
+
/**
|
|
30
|
+
* A URL signed ahead is used while younger than this (ClipMe: the server signs for 10 minutes; this leaves room for a
|
|
31
|
+
* slow PUT). The SDK also wants a minute left on its expiresAt, for servers that sign shorter.
|
|
32
|
+
*/
|
|
33
|
+
export declare const AHEAD_MAX_AGE_MS: number;
|
|
34
|
+
export interface GrowingPart {
|
|
35
|
+
n: number;
|
|
36
|
+
/** Storage's ETag for the part: its MD5, hex. */
|
|
37
|
+
etag: string;
|
|
38
|
+
size: number;
|
|
39
|
+
}
|
|
40
|
+
export interface SignedPart {
|
|
41
|
+
url: string;
|
|
42
|
+
headers: Record<string, string>;
|
|
43
|
+
expiresAt?: string;
|
|
44
|
+
}
|
|
45
|
+
/** How parts reach storage: the growing upload wires these to the SDK Signer and Transport. */
|
|
46
|
+
export interface PartSender {
|
|
47
|
+
/**
|
|
48
|
+
* A URL for part `n`. With `contentMd5` (base64) and `size` when the bytes are known; with neither when it is signed
|
|
49
|
+
* ahead, before the part exists.
|
|
50
|
+
*/
|
|
51
|
+
sign(n: number, o: {
|
|
52
|
+
contentMd5?: string;
|
|
53
|
+
size?: number;
|
|
54
|
+
}, signal: AbortSignal): Promise<SignedPart>;
|
|
55
|
+
/** PUT the part; resolves with storage's ETag (quotes stripped, may be ""). */
|
|
56
|
+
put(signed: SignedPart, body: Uint8Array, n: number, signal: AbortSignal, onProgress: (sent: number) => void): Promise<string>;
|
|
57
|
+
}
|
|
58
|
+
export interface PartHasher {
|
|
59
|
+
/**
|
|
60
|
+
* MD5 of `b`: 16 bytes or 32 hex characters, sync (nodeHasher) or async (hash-wasm's `md5`, which answers hex, fits
|
|
61
|
+
* as it is: `hasher: { md5 }`).
|
|
62
|
+
*/
|
|
63
|
+
md5(b: Uint8Array): Uint8Array | string | Promise<Uint8Array | string>;
|
|
64
|
+
}
|
|
65
|
+
/** The hasher's answer as 16 bytes (a malformed one is an error, never a digest). */
|
|
66
|
+
export declare function md5Of(h: PartHasher, b: Uint8Array): Promise<Uint8Array>;
|
|
67
|
+
export interface PartUploaderOptions {
|
|
68
|
+
partSize?: number;
|
|
69
|
+
maxParallel?: number;
|
|
70
|
+
maxAttempts?: number;
|
|
71
|
+
maxPendingBytes?: number;
|
|
72
|
+
/** Parts storage already holds for this multipart (a resume, proven by ListParts): kept when the same bytes come again. */
|
|
73
|
+
seed?: GrowingPart[];
|
|
74
|
+
/** Sign the next part while the current one uploads (ClipMe's tail-latency win). Default true. */
|
|
75
|
+
signAhead?: boolean;
|
|
76
|
+
sleep?: (ms: number) => Promise<void>;
|
|
77
|
+
now?: () => number;
|
|
78
|
+
/** A part stored and proven (`kept`: an earlier run's, proven again by its bytes). */
|
|
79
|
+
onPart?: (part: GrowingPart, acknowledged: number, kept: boolean) => void;
|
|
80
|
+
onFailure?: (error: Error) => void;
|
|
81
|
+
/** Bytes of in-flight PUTs as the transport reports them. */
|
|
82
|
+
onProgress?: () => void;
|
|
83
|
+
}
|
|
84
|
+
/** The multipart upload no longer exists in storage, so its parts are gone too (ClipMe's UploadGoneError). */
|
|
85
|
+
export declare const isGone: (e: unknown) => boolean;
|
|
86
|
+
/**
|
|
87
|
+
* The product's server refused to sign (a quota, a key outside the layouts, a deleted scope…). `retryable` false: the
|
|
88
|
+
* answer is final, so the part isn't tried again (ClipMe tried every failure 6 times).
|
|
89
|
+
*/
|
|
90
|
+
export declare class SignRefused extends StorageError {
|
|
91
|
+
}
|
|
92
|
+
/** A part whose stored ETag isn't the MD5 of what was sent (or storage hid the ETag). Sent again. */
|
|
93
|
+
export declare class PartIntegrityError extends Error {
|
|
94
|
+
}
|
|
95
|
+
export declare class GrowingPartUploader {
|
|
96
|
+
private readonly sender;
|
|
97
|
+
private readonly hasher;
|
|
98
|
+
private readonly options;
|
|
99
|
+
readonly partSize: number;
|
|
100
|
+
private buffered;
|
|
101
|
+
private bufferedBytes;
|
|
102
|
+
private nextPartNumber;
|
|
103
|
+
private readonly parts;
|
|
104
|
+
private readonly seed;
|
|
105
|
+
private readonly inFlight;
|
|
106
|
+
private readonly inflightProgress;
|
|
107
|
+
private pendingBytes;
|
|
108
|
+
private slots;
|
|
109
|
+
private readonly waiters;
|
|
110
|
+
private failure;
|
|
111
|
+
private acked;
|
|
112
|
+
private appended;
|
|
113
|
+
private readonly controller;
|
|
114
|
+
private readonly aborted;
|
|
115
|
+
private readonly sleep;
|
|
116
|
+
private readonly now;
|
|
117
|
+
private readonly maxParallel;
|
|
118
|
+
private readonly maxAttempts;
|
|
119
|
+
private readonly maxPendingBytes;
|
|
120
|
+
private ahead;
|
|
121
|
+
constructor(sender: PartSender, hasher: PartHasher, options?: PartUploaderOptions);
|
|
122
|
+
/** Sign part `n` now, for when it is cut. Never throws: a failed signature is asked for again. */
|
|
123
|
+
private signAhead;
|
|
124
|
+
private takeAhead;
|
|
125
|
+
get failed(): Error | null;
|
|
126
|
+
/** Bytes storage holds and the uploader proved (sent now, or kept from an earlier run). */
|
|
127
|
+
get acknowledged(): number;
|
|
128
|
+
/** Bytes handed over so far. */
|
|
129
|
+
get total(): number;
|
|
130
|
+
/** Handed over and not yet acknowledged by storage. */
|
|
131
|
+
get pending(): number;
|
|
132
|
+
/** Bytes of in-flight PUTs the transport has sent (live progress, below `pending`). */
|
|
133
|
+
get sending(): number;
|
|
134
|
+
/** Never throws and never blocks: the recording must not notice that its upload died. */
|
|
135
|
+
add(chunk: Blob): void;
|
|
136
|
+
private send;
|
|
137
|
+
private fail;
|
|
138
|
+
private acquire;
|
|
139
|
+
private release;
|
|
140
|
+
private isFinal;
|
|
141
|
+
private sendWithRetry;
|
|
142
|
+
private record;
|
|
143
|
+
/** For a reader of a finished local copy: wait until the network has caught up a little (ClipMe's `capacity`). */
|
|
144
|
+
capacity(): Promise<void>;
|
|
145
|
+
/** Every PUT started has settled (stored, failed or aborted): nothing in flight can report a part any more. */
|
|
146
|
+
settle(): Promise<void>;
|
|
147
|
+
private drain;
|
|
148
|
+
/**
|
|
149
|
+
* Send the tail and wait for everything. Returns every part, in order.
|
|
150
|
+
*
|
|
151
|
+
* The tail goes out at once, beside the parts still in flight, not after them: it is what Stop is waiting on, and its
|
|
152
|
+
* part number was fixed when it was cut, so the order the parts land in does not matter (ClipMe).
|
|
153
|
+
*/
|
|
154
|
+
finish(): Promise<GrowingPart[]>;
|
|
155
|
+
/** Stop now: in-flight requests are aborted and nothing further is sent. */
|
|
156
|
+
abort(): Promise<void>;
|
|
157
|
+
}
|