@uploadfilm/sdk 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +4 -0
- package/README.md +509 -2
- package/dist/ai/embed.d.ts +58 -0
- package/dist/ai/fake.d.ts +41 -0
- package/dist/ai/http.d.ts +17 -0
- package/dist/ai/index.d.ts +27 -0
- package/dist/ai/languages.d.ts +2 -0
- package/dist/ai/ollama.d.ts +23 -0
- package/dist/ai/openaiCompatible.d.ts +54 -0
- package/dist/ai/passages.d.ts +28 -0
- package/dist/ai/transcribe.d.ts +84 -0
- package/dist/ai/types.d.ts +141 -0
- package/dist/ai/usage.d.ts +66 -0
- package/dist/ai/whisperJson.d.ts +15 -0
- package/dist/ai/workersAi.d.ts +56 -0
- package/dist/ai.js +1 -0
- package/dist/ai.mjs +1 -0
- package/dist/cards/files.d.ts +91 -0
- package/dist/cards/index.d.ts +15 -0
- package/dist/cards/layouts.d.ts +121 -0
- package/dist/cards/probe.d.ts +69 -0
- package/dist/cards/proxies.d.ts +70 -0
- package/dist/cards/select.d.ts +68 -0
- package/dist/cards/volumes.d.ts +170 -0
- package/dist/cards/xml.d.ts +80 -0
- package/dist/cards.js +1 -0
- package/dist/cards.mjs +1 -0
- package/dist/copy/checksums.d.ts +66 -0
- package/dist/copy/engine.d.ts +24 -0
- package/dist/copy/errors.d.ts +109 -0
- package/dist/copy/index.d.ts +15 -0
- package/dist/copy/steps.d.ts +28 -0
- package/dist/copy/sweep.d.ts +24 -0
- package/dist/copy/types.d.ts +336 -0
- package/dist/embed/player.js +2 -0
- package/dist/host/adapters.d.ts +107 -0
- package/dist/host/detect.d.ts +68 -0
- package/dist/host/encoders.d.ts +34 -0
- package/dist/host/environment.d.ts +53 -0
- package/dist/host/index.d.ts +20 -0
- package/dist/host/linux.d.ts +94 -0
- package/dist/host/mac.d.ts +47 -0
- package/dist/host/network.d.ts +54 -0
- package/dist/host/node.d.ts +13 -0
- package/dist/host/policy.d.ts +101 -0
- package/dist/host/tuner.d.ts +48 -0
- package/dist/host/types.d.ts +118 -0
- package/dist/host-node.js +1 -0
- package/dist/host-node.mjs +1 -0
- package/dist/host.js +1 -0
- package/dist/host.mjs +1 -0
- package/dist/images/index.d.ts +11 -0
- package/dist/images/settings.d.ts +43 -0
- package/dist/images/types.d.ts +83 -0
- package/dist/images/url.d.ts +65 -0
- package/dist/index.d.ts +16 -0
- package/dist/index.js +1 -0
- package/dist/index.mjs +1 -0
- package/dist/player/adapters/hlsjs.d.ts +28 -0
- package/dist/player/bandwidthMemory.d.ts +28 -0
- package/dist/player/captions/index.d.ts +55 -0
- package/dist/player/embed/code.d.ts +195 -0
- package/dist/player/embed/constants.d.ts +4 -0
- package/dist/player/embed/controller.d.ts +62 -0
- package/dist/player/embed/element.d.ts +32 -0
- package/dist/player/embed/frame.d.ts +65 -0
- package/dist/player/embed/global.d.ts +14 -0
- package/dist/player/embed/index.d.ts +15 -0
- package/dist/player/embed/loader.d.ts +1 -0
- package/dist/player/embed/oembed.d.ts +74 -0
- package/dist/player/embed/protocol.d.ts +64 -0
- package/dist/player/engine.d.ts +215 -0
- package/dist/player/index.d.ts +18 -0
- package/dist/player/start.d.ts +62 -0
- package/dist/player/store.d.ts +10 -0
- package/dist/player/storyboard/index.d.ts +161 -0
- package/dist/player/stream.d.ts +206 -0
- package/dist/player/tracks.d.ts +41 -0
- package/dist/player.js +1 -0
- package/dist/player.mjs +1 -0
- package/dist/proxies/args.d.ts +19 -0
- package/dist/proxies/budget.d.ts +57 -0
- package/dist/proxies/camera.d.ts +104 -0
- package/dist/proxies/formats.d.ts +36 -0
- package/dist/proxies/index.d.ts +18 -0
- package/dist/proxies/jobs.d.ts +98 -0
- package/dist/proxies/node.d.ts +32 -0
- package/dist/proxies/plan.d.ts +89 -0
- package/dist/proxies/probe.d.ts +39 -0
- package/dist/proxies/render.d.ts +104 -0
- package/dist/proxies/safety.d.ts +45 -0
- package/dist/proxies/types.d.ts +83 -0
- package/dist/proxies/upload.d.ts +14 -0
- package/dist/react/BigPlay.d.ts +26 -0
- package/dist/react/Player.d.ts +58 -0
- package/dist/react/embed/index.d.ts +45 -0
- package/dist/react/examples/CustomControls.d.ts +8 -0
- package/dist/react/images/Image.d.ts +16 -0
- package/dist/react/images/index.d.ts +4 -0
- package/dist/react/images/useImage.d.ts +47 -0
- package/dist/react/index.d.ts +17 -0
- package/dist/react/storyboard/index.d.ts +58 -0
- package/dist/react/theme.d.ts +37 -0
- package/dist/react/useAdapterErrors.d.ts +8 -0
- package/dist/react/usePlaybackStart.d.ts +18 -0
- package/dist/react/usePlayer.d.ts +28 -0
- package/dist/react/useReducedMotion.d.ts +2 -0
- package/dist/react/useTimedTrack.d.ts +22 -0
- package/dist/react.js +1 -0
- package/dist/react.mjs +1 -0
- package/dist/server/auth/apiKeys.d.ts +30 -0
- package/dist/server/auth/cache.d.ts +21 -0
- package/dist/server/auth/deviceTokens.d.ts +38 -0
- package/dist/server/auth/envelope.d.ts +15 -0
- package/dist/server/auth/grants.d.ts +76 -0
- package/dist/server/auth/index.d.ts +19 -0
- package/dist/server/auth/jwt.d.ts +65 -0
- package/dist/server/auth/keyring.d.ts +31 -0
- package/dist/server/auth/kv.d.ts +12 -0
- package/dist/server/auth/repository.d.ts +45 -0
- package/dist/server/auth/revocations.d.ts +21 -0
- package/dist/server/auth/scope.d.ts +4 -0
- package/dist/server/auth/serviceAuth.d.ts +24 -0
- package/dist/server/auth/signedUrls.d.ts +28 -0
- package/dist/server/embed/index.d.ts +36 -0
- package/dist/server/embed/lifecycle.d.ts +53 -0
- package/dist/server/embed/playerScript.d.ts +4 -0
- package/dist/server/images/index.d.ts +15 -0
- package/dist/server/images/job.d.ts +80 -0
- package/dist/server/images/layout.d.ts +24 -0
- package/dist/server/images/params.d.ts +90 -0
- package/dist/server/images/route.d.ts +101 -0
- package/dist/server/images/sign.d.ts +55 -0
- package/dist/server/index.d.ts +42 -0
- package/dist/server/media/client.d.ts +66 -0
- package/dist/server/media/frameCheck.d.ts +62 -0
- package/dist/server/media/index.d.ts +10 -0
- package/dist/server/media/protocol.d.ts +318 -0
- package/dist/server/media/storyboard.d.ts +144 -0
- package/dist/server/media/transcript.d.ts +143 -0
- package/dist/server/pipeline/index.d.ts +12 -0
- package/dist/server/pipeline/match.d.ts +10 -0
- package/dist/server/pipeline/pipeline.d.ts +182 -0
- package/dist/server/pipeline/queues.d.ts +108 -0
- package/dist/server/pipeline/sources.d.ts +54 -0
- package/dist/server/r2/client.d.ts +150 -0
- package/dist/server/r2/crc32.d.ts +24 -0
- package/dist/server/r2/encoding.d.ts +35 -0
- package/dist/server/r2/errors.d.ts +9 -0
- package/dist/server/r2/integrity.d.ts +25 -0
- package/dist/server/r2/parts.d.ts +46 -0
- package/dist/server/r2/sign.d.ts +47 -0
- package/dist/server/redis/index.d.ts +39 -0
- package/dist/server/state/memory.d.ts +18 -0
- package/dist/server/state/r2.d.ts +73 -0
- package/dist/server/state/types.d.ts +57 -0
- package/dist/server/storage/chunked.d.ts +21 -0
- package/dist/server/storage/index.d.ts +12 -0
- package/dist/server/storage/r2.d.ts +96 -0
- package/dist/server/storage/s3.d.ts +93 -0
- package/dist/server/testing/fake-r2.d.ts +390 -0
- package/dist/server/testing/index.d.ts +10 -0
- package/dist/server/testing/kv-contract.d.ts +14 -0
- package/dist/server/transfer/handler.d.ts +167 -0
- package/dist/server/transfer/keys.d.ts +45 -0
- package/dist/server/transfer/protocol.d.ts +141 -0
- package/dist/server/zip/client.d.ts +32 -0
- package/dist/server/zip/generations.d.ts +83 -0
- package/dist/server/zip/index.d.ts +12 -0
- package/dist/server/zip/planner.d.ts +94 -0
- package/dist/server/zip/sign.d.ts +22 -0
- package/dist/server-media.js +1 -0
- package/dist/server-media.mjs +1 -0
- package/dist/server-redis.js +1 -0
- package/dist/server-redis.mjs +1 -0
- package/dist/server-testing.js +1 -0
- package/dist/server-testing.mjs +1 -0
- package/dist/server.js +1 -0
- package/dist/server.mjs +1 -0
- package/dist/storage/browserFile.d.ts +19 -0
- package/dist/storage/index.d.ts +12 -0
- package/dist/storage/local.d.ts +9 -0
- package/dist/storage/presigned.d.ts +22 -0
- package/dist/sync/ffmpeg.d.ts +44 -0
- package/dist/sync/index.d.ts +17 -0
- package/dist/sync/match.d.ts +28 -0
- package/dist/sync/probe.d.ts +30 -0
- package/dist/sync/timecode.d.ts +50 -0
- package/dist/sync/transcript.d.ts +35 -0
- package/dist/sync/types.d.ts +172 -0
- package/dist/sync/waveform.d.ts +89 -0
- package/dist/sync.js +1 -0
- package/dist/sync.mjs +1 -0
- package/dist/transfer/claims.d.ts +41 -0
- package/dist/transfer/client.d.ts +256 -0
- package/dist/transfer/download.d.ts +192 -0
- package/dist/transfer/engine.d.ts +382 -0
- package/dist/transfer/errors.d.ts +50 -0
- package/dist/transfer/index.d.ts +39 -0
- package/dist/transfer/integrity.d.ts +93 -0
- package/dist/transfer/limits.d.ts +34 -0
- package/dist/transfer/node.d.ts +63 -0
- package/dist/transfer/pool.d.ts +83 -0
- package/dist/transfer/protocol.d.ts +167 -0
- package/dist/transfer/signer.d.ts +29 -0
- package/dist/transfer/storage.d.ts +110 -0
- package/dist/transfer/timing.d.ts +132 -0
- package/dist/transfer/transports/electronNet.d.ts +62 -0
- package/dist/transfer/transports/fetch.d.ts +15 -0
- package/dist/transfer/transports/retryAfter.d.ts +12 -0
- package/dist/transfer/transports/undici.d.ts +72 -0
- package/dist/transfer/tuner.d.ts +56 -0
- package/dist/zip/index.d.ts +13 -0
- package/dist/zip/layout.d.ts +129 -0
- package/dist/zip/names.d.ts +34 -0
- package/dist/zip/plan.d.ts +115 -0
- package/dist/zip/sw.d.ts +7 -0
- package/package.json +265 -4
|
@@ -0,0 +1,382 @@
|
|
|
1
|
+
import { type ConnectionLimits, type PartSizeHook } from "./limits";
|
|
2
|
+
import { type Stamp } from "./node";
|
|
3
|
+
import type { ByteSource, ClaimStore, Hasher, PutReceipt, SourceStamp, Yieldable } from "./protocol";
|
|
4
|
+
import type { ListedPart, Storage } from "./storage";
|
|
5
|
+
import { type TransferTiming } from "./timing";
|
|
6
|
+
export interface Multipart {
|
|
7
|
+
uploadId: string;
|
|
8
|
+
partSize: number;
|
|
9
|
+
/** part number → its MD5 hex (= storage's part ETag). */
|
|
10
|
+
parts: Record<number, string>;
|
|
11
|
+
/** part number → zlib crc32 of its bytes. */
|
|
12
|
+
crcs?: Record<number, number>;
|
|
13
|
+
/** integrity "sha256": part number → the SHA-256 hex storage receipted for it. */
|
|
14
|
+
sha256s?: Record<number, string>;
|
|
15
|
+
/**
|
|
16
|
+
* Taken over from a claim holder that died (its upload id, no parts of ours yet): the resume proof adopts every part
|
|
17
|
+
* storage lists whose bytes re-read from the source hash to what storage holds, and sends the rest.
|
|
18
|
+
*/
|
|
19
|
+
adopt?: boolean;
|
|
20
|
+
}
|
|
21
|
+
/** The record of one object's upload (UploadFilm's Version). Its identity matters: a replaced record is a new version. */
|
|
22
|
+
export interface ItemVersion {
|
|
23
|
+
key: string;
|
|
24
|
+
size: number;
|
|
25
|
+
uploaded: number;
|
|
26
|
+
status: string;
|
|
27
|
+
error?: string;
|
|
28
|
+
/** `sha256`: the object's SHA-256 (hex); `sha256Composite`: storage's multipart composite ("hex-N"). */
|
|
29
|
+
verified?: {
|
|
30
|
+
etag: string;
|
|
31
|
+
at: string;
|
|
32
|
+
crc32?: string;
|
|
33
|
+
sha256?: string;
|
|
34
|
+
sha256Composite?: string;
|
|
35
|
+
};
|
|
36
|
+
/**
|
|
37
|
+
* RL-10: when a CreateMultipartUpload was first asked for this version and its upload id isn't recorded yet (saved
|
|
38
|
+
* before the create). A run that finds it set (killed in between, or the answer was lost) aborts the multipart
|
|
39
|
+
* storage opened for it before creating another.
|
|
40
|
+
*/
|
|
41
|
+
creating?: string;
|
|
42
|
+
}
|
|
43
|
+
/** A whole small object the product prepares itself (UploadFilm: posters): no Content-MD5, no item status. */
|
|
44
|
+
export interface BlobSend<S = unknown> {
|
|
45
|
+
/** What is about to be sent (UploadFilm: the poster's path at this moment). */
|
|
46
|
+
snapshot(): S;
|
|
47
|
+
read(s: S): Promise<Uint8Array>;
|
|
48
|
+
/** Stored. */
|
|
49
|
+
done(s: S): void;
|
|
50
|
+
/** A failure about `s` says nothing any more (another blob was chosen meanwhile). */
|
|
51
|
+
stale(s: S): boolean;
|
|
52
|
+
/** Failed for good. */
|
|
53
|
+
fail(msg: string): void;
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* One object to upload, as the product's view of it (getters over its own records). Created fresh by the host for
|
|
57
|
+
* every pick; the engine keys everything by `id`.
|
|
58
|
+
*/
|
|
59
|
+
export interface UploadItem {
|
|
60
|
+
/** Unique and stable; retry keys are `${id}`, `${id}#<part>`, `${id}#resume`, `${id}#complete`. */
|
|
61
|
+
readonly id: string;
|
|
62
|
+
/** Signer scope (UploadFilm: the project id); also the per-scope rate's key. */
|
|
63
|
+
readonly scope: string;
|
|
64
|
+
readonly lane: string;
|
|
65
|
+
/** For messages: "<name>: …". */
|
|
66
|
+
readonly name: string;
|
|
67
|
+
/** For the failure message: "<path>: …". */
|
|
68
|
+
readonly path: string;
|
|
69
|
+
/** The version record (identity compared to tell a replaced version). */
|
|
70
|
+
readonly version: ItemVersion;
|
|
71
|
+
mp: Multipart | undefined;
|
|
72
|
+
/** Forget the multipart (adopting a stored copy deletes the field). */
|
|
73
|
+
deleteMp(): void;
|
|
74
|
+
readonly contentType: string;
|
|
75
|
+
/** Counted against the scope's quota (UploadFilm: originals). */
|
|
76
|
+
readonly counts: boolean;
|
|
77
|
+
/** Written once, never overwritten: a different stored object is a conflict, not a reason to send again. */
|
|
78
|
+
readonly immutable: boolean;
|
|
79
|
+
/** When set, the source's mtime must still be this (an app rewriting metadata in place keeps the size). */
|
|
80
|
+
readonly mtimeMs?: number;
|
|
81
|
+
/** The bytes: a local path (read with node:fs exactly as before ByteSource existed), or any ByteSource. */
|
|
82
|
+
readonly source: string | ByteSource;
|
|
83
|
+
/** What "waiting for its source" is keyed by (UploadFilm: the original's path, even for a proxy). */
|
|
84
|
+
readonly waitKey: string;
|
|
85
|
+
/** A product-prepared small object instead of `source`. */
|
|
86
|
+
readonly blob?: BlobSend<any>;
|
|
87
|
+
/** sha256 hex of the whole source, when known: lets the server's dedupe hook answer "already stored". */
|
|
88
|
+
readonly sha256?: string;
|
|
89
|
+
/** The source as it was when `sha256` was computed: the hash is only used (and a dedupe answer only trusted) while
|
|
90
|
+
* the source is still exactly that file. */
|
|
91
|
+
readonly sha256Stamp?: Stamp | SourceStamp;
|
|
92
|
+
/**
|
|
93
|
+
* The caller declared what the bytes are (a content id): only the source as it was then may be sent. A source that
|
|
94
|
+
* isn't that one any more fails the item (SourceChanged), never uploads other bytes under the declaration (R2-3).
|
|
95
|
+
*/
|
|
96
|
+
readonly boundTo?: SourceStamp;
|
|
97
|
+
}
|
|
98
|
+
export type SourceCheck = "match" | "changed" | "otherCard" | "unknown";
|
|
99
|
+
export interface EngineHost<I extends UploadItem = UploadItem> {
|
|
100
|
+
/** Uploads can run (credentials/device ready). */
|
|
101
|
+
ready(): boolean;
|
|
102
|
+
/** Items of a lane that are ready to send, in priority order. */
|
|
103
|
+
candidates(lane: string): Iterable<I>;
|
|
104
|
+
/** The item still belongs to the product (a unit finishing for a removed item reports nothing). */
|
|
105
|
+
live(item: I): boolean;
|
|
106
|
+
/** A failure the product handles itself (UploadFilm: Deleted, ProjectDeleted). true = handled. */
|
|
107
|
+
onFatal?(item: I, e: unknown): boolean;
|
|
108
|
+
/** The item's version went to "error". */
|
|
109
|
+
onFailed?(item: I, msg: string): void;
|
|
110
|
+
/** The server no longer knows this client may write the scope (UploadFilm: NotClaimed). true = handled, retry later. */
|
|
111
|
+
claimLost?(item: I, e: unknown): boolean;
|
|
112
|
+
/** Extra check before reading/resuming/stitching (UploadFilm: camera proxies). false = not now (it backed off). */
|
|
113
|
+
guard?(item: I, retryKey: string): boolean;
|
|
114
|
+
/** Is the file at `item.source` still this item's source? Default: path + size (+ mtime). */
|
|
115
|
+
sourceCheck?(item: I): Promise<SourceCheck>;
|
|
116
|
+
/** The source stamp approved for a multipart created now (default: the stamp itself). null = not approved. */
|
|
117
|
+
approveOnCreate?(item: I, stamp: SourceStamp | null): SourceStamp | null;
|
|
118
|
+
/** May a resumed multipart proven from `stamp` be approved? (default: yes) */
|
|
119
|
+
stampApproved?(item: I, stamp: SourceStamp | null): boolean;
|
|
120
|
+
/**
|
|
121
|
+
* integrity "sha256": storage completed a multipart without reporting the composite SHA-256 the engine could check
|
|
122
|
+
* (or a single PUT's copy was only proven by MD5). "verified" = the product proved the stored bytes itself (e.g.
|
|
123
|
+
* SuperBuilder's server-side SHA-256 re-read); anything else leaves it stored but unverified (never a success).
|
|
124
|
+
*/
|
|
125
|
+
confirmStored?(item: I, stored: {
|
|
126
|
+
key: string;
|
|
127
|
+
size: number | null;
|
|
128
|
+
etag: string | null;
|
|
129
|
+
sha256Parts?: Record<number, string>;
|
|
130
|
+
}): Promise<"verified" | "unverified">;
|
|
131
|
+
/** Save soon / save now / the item's published status changed. */
|
|
132
|
+
touch(item: I): void;
|
|
133
|
+
save(item: I): void;
|
|
134
|
+
dirty(item: I): void;
|
|
135
|
+
/** An item was refused for quota (first: it wasn't waiting before). */
|
|
136
|
+
onQuota?(item: I, first: boolean): void;
|
|
137
|
+
/** The stored copy at `key` (the version that was stitched, not whatever the item holds now) doesn't match its source. */
|
|
138
|
+
onDemoted?(item: I, key: string): void;
|
|
139
|
+
/** Quota waiters still worth waiting for (removed, finished or failed ones are forgotten). */
|
|
140
|
+
stillWaiting?(id: string): boolean;
|
|
141
|
+
}
|
|
142
|
+
export interface LaneSpec {
|
|
143
|
+
name: string;
|
|
144
|
+
/** Picked before anything else. */
|
|
145
|
+
first?: boolean;
|
|
146
|
+
/** The primary lane: keeps at least ⌈minShare × sockets⌉ of them while it has work. At most one lane. */
|
|
147
|
+
minShare?: number;
|
|
148
|
+
/** A borrower whose queued PUTs may be cancelled to give the primary its share back. */
|
|
149
|
+
yieldable?: boolean;
|
|
150
|
+
/** At most this many units of the lane at once. */
|
|
151
|
+
maxActive?: number;
|
|
152
|
+
}
|
|
153
|
+
export interface EngineCoreOptions<I extends UploadItem> {
|
|
154
|
+
host: EngineHost<I>;
|
|
155
|
+
storage: Storage;
|
|
156
|
+
timing: TransferTiming;
|
|
157
|
+
hasher: Hasher;
|
|
158
|
+
lanes: LaneSpec[];
|
|
159
|
+
/** Tuner bounds (default 4..32, starting at 8). Changed live with `setConnectionLimits`. */
|
|
160
|
+
connections?: {
|
|
161
|
+
min?: number;
|
|
162
|
+
max?: number;
|
|
163
|
+
start?: number;
|
|
164
|
+
};
|
|
165
|
+
/**
|
|
166
|
+
* Part size for a multipart created from now on (bytes), given the object's size; e.g. `partSizeFor(size,
|
|
167
|
+
* tuner.decision)` from `@uploadfilm/sdk/host`. Raised to the engine's own floor for the object (`partSizeFor`:
|
|
168
|
+
* 16 MiB, or more so it fits in 9,500 parts), rounded up to a whole MiB, at most 5 GiB; a hook that throws or answers
|
|
169
|
+
* something that isn't a positive number gets the floor. A multipart already created keeps its part size (its parts
|
|
170
|
+
* are in storage). Default: none (the floor, today's rule). Changed live with `setPartSizeHook`.
|
|
171
|
+
*/
|
|
172
|
+
partSize?: PartSizeHook;
|
|
173
|
+
/**
|
|
174
|
+
* Most bytes of part bodies and small objects buffered in flight at once (e.g. a host decision's `inflightBytes`).
|
|
175
|
+
* One unit may always start when nothing is buffered, so an upload always moves however big its parts. Default: no
|
|
176
|
+
* cap. Changed live with `setInflightBytes`.
|
|
177
|
+
*/
|
|
178
|
+
inflightBytes?: number | null;
|
|
179
|
+
/** Fresh multipart starts after storage refused to finish one, per version, before the item fails (default 3). */
|
|
180
|
+
maxRestarts?: number;
|
|
181
|
+
/**
|
|
182
|
+
* How "verified" is proven (gap 2). "md5" (default): Content-MD5 per part, part ETag = MD5, composite ETag.
|
|
183
|
+
* "sha256": every part and single PUT carries `x-amz-checksum-sha256`, and storage's receipt of it (the response
|
|
184
|
+
* header) must be present, well-formed, once, and equal: anything else fails closed (never stored as verified).
|
|
185
|
+
* The stitched object is verified by storage's composite SHA-256 (or `confirmStored`).
|
|
186
|
+
*/
|
|
187
|
+
integrity?: "md5" | "sha256";
|
|
188
|
+
/** The client's part size for a new multipart (default partSizeFor). Given → also proposed to the server on create. */
|
|
189
|
+
partSizeFor?(size: number): number;
|
|
190
|
+
/** Below this size an object goes as one PUT (default SMALL, 8 MiB). */
|
|
191
|
+
singlePutMax?: number;
|
|
192
|
+
/** Most parts a server-chosen part size may cut an object into (default 10,000, S3's limit). */
|
|
193
|
+
maxParts?: number;
|
|
194
|
+
/** Use the part size the server answers on create (gap 3). On by default with integrity "sha256", partSizeFor or claims. */
|
|
195
|
+
serverPartSize?: boolean;
|
|
196
|
+
/**
|
|
197
|
+
* Claims (gap 4): before sending anything for an item the engine takes its claim (key = scope + storage key); held
|
|
198
|
+
* by someone else → the item waits ("waiting") and polls (claimPollMs doubling to RETRY_CAP_MS, never past the
|
|
199
|
+
* holder's expiry), then adopts what the holder stored (immutable keys) or takes over an expired claim, keeping the
|
|
200
|
+
* dead holder's proven parts. The claim is renewed every claimTtlMs / 3; a renewal that fails means another engine
|
|
201
|
+
* took over: this one stops sending for the item. Released when the item finishes, fails or is forgotten.
|
|
202
|
+
*/
|
|
203
|
+
claims?: ClaimStore;
|
|
204
|
+
/** Lease length (default 60 s). */
|
|
205
|
+
claimTtlMs?: number;
|
|
206
|
+
/** First wait between looks at a held claim (default 2 s). */
|
|
207
|
+
claimPollMs?: number;
|
|
208
|
+
/** Give up waiting for a holder after this long (code ClaimTimeout, retryable by the product). Default: no limit. */
|
|
209
|
+
maxClaimWaitMs?: number;
|
|
210
|
+
}
|
|
211
|
+
/** Failures the engine's own checks raise (bad bytes): never kept, resent with a budget. */
|
|
212
|
+
export declare class IntegrityError extends Error {
|
|
213
|
+
}
|
|
214
|
+
/** integrity "sha256": storage answered a PUT without its SHA-256 receipt (the bytes may be stored, but unproven). */
|
|
215
|
+
export declare class ReceiptMissing extends IntegrityError {
|
|
216
|
+
}
|
|
217
|
+
/**
|
|
218
|
+
* Borrower lanes' room: what's left of `sockets` after the primary's ⌈minShare × sockets⌉ while the primary has work;
|
|
219
|
+
* everything when it has none.
|
|
220
|
+
*/
|
|
221
|
+
export declare function laneRoom(sockets: number, primaryWaiting: boolean, minShare: number): number;
|
|
222
|
+
/**
|
|
223
|
+
* Borrower lanes' room with the lane share (perf-lab finding 3, docs/platform/TRANSFER-PERF.md): while the primary has
|
|
224
|
+
* a unit waiting, laneRoom(sockets); while it has nothing to start but PUTs in flight (`primaryPuts`), at most
|
|
225
|
+
* ⌊primaryPuts × (1 − minShare) / minShare⌋ (2/3: half as many as the primary's), so the primary keeps ≥ minShare of
|
|
226
|
+
* the connections that carry bytes, not only of those being handed out; with neither, everything.
|
|
227
|
+
*/
|
|
228
|
+
export declare function laneShare(sockets: number, primaryWaiting: boolean, primaryPuts: number, minShare: number): number;
|
|
229
|
+
export declare const EXISTS_MSG = "a different file is already stored under this name in the cloud; nothing was overwritten";
|
|
230
|
+
export declare const MISSING_MSG = "the platform has this original recorded as stored, but storage has no copy of it; nothing was overwritten";
|
|
231
|
+
type Unit<I> = {
|
|
232
|
+
kind: "small";
|
|
233
|
+
item: I;
|
|
234
|
+
} | {
|
|
235
|
+
kind: "part";
|
|
236
|
+
item: I;
|
|
237
|
+
n: number;
|
|
238
|
+
} | {
|
|
239
|
+
kind: "create";
|
|
240
|
+
item: I;
|
|
241
|
+
} | {
|
|
242
|
+
kind: "complete";
|
|
243
|
+
item: I;
|
|
244
|
+
} | {
|
|
245
|
+
kind: "resume";
|
|
246
|
+
item: I;
|
|
247
|
+
};
|
|
248
|
+
export type EngineCore<I extends UploadItem = UploadItem> = ReturnType<typeof createEngineCore<I>>;
|
|
249
|
+
export declare function createEngineCore<I extends UploadItem>(o: EngineCoreOptions<I>): {
|
|
250
|
+
/** Start the loops: every ENGINE_TICK_MS measure the rate and start work (plus `onTick`), every TUNE_INTERVAL_MS tune. */
|
|
251
|
+
start(onTick?: () => void): void;
|
|
252
|
+
stop(): void;
|
|
253
|
+
pump: () => void;
|
|
254
|
+
tickRate: () => void;
|
|
255
|
+
tune: () => void;
|
|
256
|
+
nextUnit: () => Unit<I> | null;
|
|
257
|
+
localDigest: (src: string, size: number, partSize?: number) => Promise<{
|
|
258
|
+
etag: string;
|
|
259
|
+
crc32: string;
|
|
260
|
+
}>;
|
|
261
|
+
storage: {
|
|
262
|
+
createMultipart: (scope: string, key: string, contentType: string, size?: number, counts?: boolean) => Promise<string>;
|
|
263
|
+
createOrDedupe: (scope: string, key: string, contentType: string, size?: number, counts?: boolean, sha256?: string) => Promise<import("./storage").Deduped | {
|
|
264
|
+
uploadId: string;
|
|
265
|
+
}>;
|
|
266
|
+
completeMultipart: (scope: string, key: string, uploadId: string, parts: {
|
|
267
|
+
n: number;
|
|
268
|
+
etag: string;
|
|
269
|
+
}[]) => Promise<import("./storage").Stat>;
|
|
270
|
+
abortMultipart: (scope: string, key: string, uploadId: string) => Promise<void | undefined>;
|
|
271
|
+
head: (scope: string, key: string) => Promise<import("./storage").Stat>;
|
|
272
|
+
listParts: (scope: string, key: string, uploadId: string) => Promise<Map<number, string>>;
|
|
273
|
+
listUploads: (scope: string, key: string) => Promise<{
|
|
274
|
+
uploadId: string;
|
|
275
|
+
initiated?: string;
|
|
276
|
+
}[]>;
|
|
277
|
+
uploadPart: (scope: string, key: string, uploadId: string, partNumber: number, body: Uint8Array, contentMd5: string, onProgress: (sent: number) => void, timeoutMs: number, y?: Yieldable) => Promise<string>;
|
|
278
|
+
putObject: (scope: string, key: string, body: Uint8Array, contentType: string, onProgress?: (sent: number) => void, timeoutMs?: number, contentMd5?: string, counts?: boolean, y?: Yieldable, sha256?: string) => Promise<string>;
|
|
279
|
+
transport: import("./protocol").Transport;
|
|
280
|
+
createSession: (scope: string, key: string, contentType: string, size: number, counts: boolean, x: {
|
|
281
|
+
sha256?: string;
|
|
282
|
+
checksumAlgorithm?: "SHA256";
|
|
283
|
+
partSize?: number;
|
|
284
|
+
resumeUploadId?: string;
|
|
285
|
+
}) => Promise<import("./storage").CreateAnswer>;
|
|
286
|
+
uploadPartChecked: (scope: string, key: string, uploadId: string, partNumber: number, body: Uint8Array, contentMd5: string, checksumSha256: string, onProgress: (sent: number) => void, timeoutMs: number, y?: Yieldable) => Promise<PutReceipt>;
|
|
287
|
+
putObjectChecked: (scope: string, key: string, body: Uint8Array, contentType: string, onProgress: (sent: number) => void, timeoutMs: number, contentMd5: string, checksumSha256: string, counts: boolean, y?: Yieldable, sha256?: string) => Promise<{
|
|
288
|
+
receipt: PutReceipt;
|
|
289
|
+
} | {
|
|
290
|
+
deduped: import("./storage").Deduped;
|
|
291
|
+
} | {
|
|
292
|
+
exists: import("./storage").CheckedStat;
|
|
293
|
+
}>;
|
|
294
|
+
listPartsChecked: (scope: string, key: string, uploadId: string) => Promise<Map<number, ListedPart>>;
|
|
295
|
+
completeChecked: (scope: string, key: string, uploadId: string, parts: {
|
|
296
|
+
n: number;
|
|
297
|
+
etag: string;
|
|
298
|
+
checksumSha256: string;
|
|
299
|
+
}[]) => Promise<import("./storage").CheckedStat>;
|
|
300
|
+
headWithChecksum: (scope: string, key: string) => Promise<import("./storage").CheckedStat>;
|
|
301
|
+
};
|
|
302
|
+
readonly rate: number;
|
|
303
|
+
scopeRates: () => {
|
|
304
|
+
[x: string]: number;
|
|
305
|
+
};
|
|
306
|
+
readonly connections: number;
|
|
307
|
+
readonly target: number;
|
|
308
|
+
/** The tuner's bounds, the in-flight byte cap, what is buffered now and how many reads wait for room. */
|
|
309
|
+
readonly limits: {
|
|
310
|
+
min: number;
|
|
311
|
+
max: number;
|
|
312
|
+
inflightBytes: number | null;
|
|
313
|
+
buffered: number;
|
|
314
|
+
waiting: number;
|
|
315
|
+
};
|
|
316
|
+
setConnectionLimits(a: ConnectionLimits | number, max?: number): void;
|
|
317
|
+
/** Part size for multiparts created from now on (null: the default rule). Saved multiparts keep theirs. */
|
|
318
|
+
setPartSizeHook(fn: PartSizeHook | null): void;
|
|
319
|
+
/** Cap on buffered bytes in flight (null: none). Units already running finish; new ones wait for room. */
|
|
320
|
+
setInflightBytes(n: number | null): void;
|
|
321
|
+
readonly lastTick: number;
|
|
322
|
+
readonly lastBytesAt: number;
|
|
323
|
+
lastError: string | undefined;
|
|
324
|
+
readonly paused: boolean;
|
|
325
|
+
setPaused(v: boolean): void;
|
|
326
|
+
hold(lane: string, v: boolean): void;
|
|
327
|
+
isHeld: (lane: string) => boolean;
|
|
328
|
+
/** `false`: the account shows room again: items waiting for room try again now. */
|
|
329
|
+
roomAgain(): void;
|
|
330
|
+
isQuotaBlocked: () => boolean;
|
|
331
|
+
isWaitingForRoom: (id: string) => boolean;
|
|
332
|
+
/** Every request has failed with a network-class error for OFFLINE_NO_BYTES_MS, and nothing has worked since. */
|
|
333
|
+
looksOffline: () => boolean;
|
|
334
|
+
offlineReason: () => "network" | "server";
|
|
335
|
+
isWaitingForSource: (waitKey: string) => boolean;
|
|
336
|
+
readonly waitingForSourceCount: number;
|
|
337
|
+
/** Forget every retry counter (and restart count) of items whose id starts with `prefix` (Retry). */
|
|
338
|
+
forgetPrefix(prefix: string): void;
|
|
339
|
+
/**
|
|
340
|
+
* A source was found again (re-added, maybe at another mount point): forget the old wait and the backoff it built
|
|
341
|
+
* up, and prove whatever is there now before anything else is sent or stitched.
|
|
342
|
+
*/
|
|
343
|
+
sourceReconnected(prefix: string, oldWaitKey: string, uploadIds: string[]): void;
|
|
344
|
+
/** Drop a multipart's approval (its source changed): the full proof runs before it is used again. */
|
|
345
|
+
revoke: (uploadId: string) => void;
|
|
346
|
+
/** Give up the item's claim (claims mode): the product removed or cancelled it. */
|
|
347
|
+
releaseClaim: (id: string) => undefined;
|
|
348
|
+
/** Units of this item are running (a request may still land). */
|
|
349
|
+
busy: (id: string) => boolean;
|
|
350
|
+
/** Resolves once no unit of this item runs (R2-4: a replacement never races its cancelled predecessor). */
|
|
351
|
+
whenIdle: (id: string) => Promise<void>;
|
|
352
|
+
/** Renew claims now (the tick does it every ENGINE_TICK_MS). */
|
|
353
|
+
renewClaims: () => void;
|
|
354
|
+
/** The claim this engine holds for an item (tests, diagnostics). */
|
|
355
|
+
claimOf: (id: string) => {
|
|
356
|
+
key: string;
|
|
357
|
+
lost: boolean;
|
|
358
|
+
} | undefined;
|
|
359
|
+
tuner: {
|
|
360
|
+
readonly target: number;
|
|
361
|
+
readonly pinned: number | null;
|
|
362
|
+
tune(rate: number, active: number): void;
|
|
363
|
+
setBounds(b: {
|
|
364
|
+
min: number;
|
|
365
|
+
max: number;
|
|
366
|
+
start?: number;
|
|
367
|
+
}): void;
|
|
368
|
+
onError(): void;
|
|
369
|
+
pin(n: number | null): void;
|
|
370
|
+
};
|
|
371
|
+
backoff: (key: string, e?: unknown) => number;
|
|
372
|
+
waitForSource: (waitKey: string) => undefined;
|
|
373
|
+
attempts: () => Map<string, number>;
|
|
374
|
+
inFlight: (lane: string) => number;
|
|
375
|
+
/** Units running now (a stopped engine finishes these: stop() ends the loops, not the work in flight). */
|
|
376
|
+
readonly active: number;
|
|
377
|
+
readonly activePuts: number;
|
|
378
|
+
pinTarget(n: number | null): void;
|
|
379
|
+
/** Borrower lanes' share of the sockets while the primary has work (UploadFilm's proxyShare). */
|
|
380
|
+
borrowerShare(): number;
|
|
381
|
+
};
|
|
382
|
+
export {};
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The transfer engine's one error type and how storage answers are classified. Moved from UploadFilm's
|
|
3
|
+
* app/src/main/storage.ts.
|
|
4
|
+
*/
|
|
5
|
+
/**
|
|
6
|
+
* Storage (S3/R2) error codes that are transient although their HTTP status is a 4xx: send again (re-signing).
|
|
7
|
+
* - RequestTimeout (400): "Your socket connection to the server was not read from or written to within the
|
|
8
|
+
* timeout period": a slow or interrupted upload, not a bad request.
|
|
9
|
+
* - ExpiredToken (400): the presigned credentials expired; a new sign fixes it.
|
|
10
|
+
* - IncompleteBody (400): fewer bytes arrived than Content-Length said (the connection dropped mid-body).
|
|
11
|
+
* - ClientDisconnect (400/499): storage saw the client go away mid-request. 499 or a "Client disconnected"
|
|
12
|
+
* message counts too, whatever the code.
|
|
13
|
+
* 5xx codes (InternalError 500, ServiceUnavailable 503, SlowDown 503) are retryable by status, and SlowDown
|
|
14
|
+
* as 429 too. 403 AccessDenied "Request has expired" is matched by message.
|
|
15
|
+
*/
|
|
16
|
+
export declare const TRANSIENT_4XX: Set<string>;
|
|
17
|
+
/**
|
|
18
|
+
* `status` 0 = no answer (network, timeout, stall). `retryable`: retrying (re-signing) can fix it; the engine retries
|
|
19
|
+
* those for as long as it takes, and gives everything else a budget.
|
|
20
|
+
*/
|
|
21
|
+
export declare class StorageError extends Error {
|
|
22
|
+
status: number;
|
|
23
|
+
code?: string | undefined;
|
|
24
|
+
retryable: boolean;
|
|
25
|
+
constructor(message: string, status: number, code?: string | undefined, retryable?: boolean);
|
|
26
|
+
}
|
|
27
|
+
/** A PUT that gave its place to a higher-priority lane before sending a byte (not a failure). */
|
|
28
|
+
export declare const YIELDED = "Yielded";
|
|
29
|
+
/**
|
|
30
|
+
* How a non-2xx storage answer to a PUT becomes a StorageError: an expired URL, corrupted bytes (BadDigest) and the
|
|
31
|
+
* transient 4xx codes are retryable (503); anything else keeps its status.
|
|
32
|
+
*/
|
|
33
|
+
export declare function putError(status: number, text: string): StorageError;
|
|
34
|
+
/**
|
|
35
|
+
* Errors from a product's signer (its own error class for its HTTP client) become StorageErrors through this hook:
|
|
36
|
+
* return the shape, or undefined for "not one of mine". The default reads `status`/`code`/`retryable` off any error
|
|
37
|
+
* that carries a numeric status and a boolean retryable.
|
|
38
|
+
*/
|
|
39
|
+
export type ErrorShape = {
|
|
40
|
+
message: string;
|
|
41
|
+
status: number;
|
|
42
|
+
code?: string;
|
|
43
|
+
retryable: boolean;
|
|
44
|
+
};
|
|
45
|
+
export type ErrorAdapter = (e: unknown) => ErrorShape | undefined;
|
|
46
|
+
export declare const duckErrors: ErrorAdapter;
|
|
47
|
+
export declare function toStorageError(e: unknown, adapt?: ErrorAdapter): StorageError;
|
|
48
|
+
/** Retrying fixes it: network errors, timeouts, stalls, 5xx, 429, expired URLs, BadDigest. */
|
|
49
|
+
export declare const isTransient: (e: unknown) => boolean;
|
|
50
|
+
export declare const isYielded: (e: unknown) => boolean;
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The client transfer engine: resumable, verified multipart uploads and verified ranged downloads over presigned URLs.
|
|
3
|
+
* Node/Electron today (Buffer, node:fs); the network, the signer, the hasher and the saved state are swapped in.
|
|
4
|
+
*/
|
|
5
|
+
export { TRANSFER_TIMING_DEFAULTS, createTiming, realClock, clockOf, monoOf } from "./timing";
|
|
6
|
+
export type { Clock, TimerHandle, TransferTiming, TransferTimingName, TransferTimingValues } from "./timing";
|
|
7
|
+
export { StorageError, YIELDED, TRANSIENT_4XX, putError, toStorageError, duckErrors, isTransient, isYielded } from "./errors";
|
|
8
|
+
export type { ErrorAdapter, ErrorShape } from "./errors";
|
|
9
|
+
export type { Signer, Transport, Hasher, Yieldable, TransferOp, TransferResult, PutReceipt, ByteSource, SourceStamp, Sink, ClaimStore, ClaimRecord } from "./protocol";
|
|
10
|
+
export { receiptOf, headerOf } from "./protocol";
|
|
11
|
+
export { createStorage, stat, unwrap, b64ToHex } from "./storage";
|
|
12
|
+
export type { Storage, StorageOptions, Stat, Deduped, CreateAnswer, CheckedStat, ListedPart } from "./storage";
|
|
13
|
+
export { createElectronNetTransport, POOL_PER_HOST, electronKeepAwake } from "./transports/electronNet";
|
|
14
|
+
export type { ElectronNetTransport, NetLike, NetRequestLike, NetResponseLike, PowerSaveBlockerLike } from "./transports/electronNet";
|
|
15
|
+
export { parseRetryAfter } from "./transports/retryAfter";
|
|
16
|
+
export { createUndiciTransport } from "./transports/undici";
|
|
17
|
+
export type { UndiciLike, UndiciTransport, UndiciTransportOptions, PutEvent } from "./transports/undici";
|
|
18
|
+
export { createFetchTransport } from "./transports/fetch";
|
|
19
|
+
export type { FetchTransportOptions } from "./transports/fetch";
|
|
20
|
+
export { MIN_PART, SMALL, MAX_PARTS, partSizeFor, isHex32, compositeEtag, partsCrc, strongStat, digestOf, etagsOf, checkEtag, crcHex, toHex, SHA256_B64, isHex64, bytesToB64, hexToB64, sha256B64ToHex, parseSha256Checksum, compositeSha256 } from "./integrity";
|
|
21
|
+
export type { RangeReader, Md5Stream } from "./integrity";
|
|
22
|
+
export { createWorkerHasher, nodeHasher, nodeMd5, nodeSha256Hex, stampOf, sameStamp, sameContentStamp, sameSource, openFile, fileSource, bufferSource } from "./node";
|
|
23
|
+
export type { Stamp, WorkerHasherOptions } from "./node";
|
|
24
|
+
export { createEngineCore, laneRoom, laneShare, IntegrityError, ReceiptMissing, EXISTS_MSG, MISSING_MSG } from "./engine";
|
|
25
|
+
export type { EngineCore, EngineCoreOptions, EngineHost, UploadItem, ItemVersion, Multipart, BlobSend, LaneSpec, SourceCheck } from "./engine";
|
|
26
|
+
export { HttpError, VersionChanged, LocalError, normEtag as normHeaderEtag, covered, addRange, nextGap, RANGE_CHUNK, isTransientDownloadError, downloadBackoff, fetchRangeInto, keyedLock, checkFileEtag, withSuffix, NO_LINK, publishNoOverwrite, publishCopy, downloadObject, sinkWriter, createDownloadLimiter, downloadLimiter, } from "./download";
|
|
27
|
+
export type { Range, RangeFetch, RangeWriter, PublishOptions, DownloadState, DownloadObjectOptions, DownloadResult, DownloadLimiter } from "./download";
|
|
28
|
+
export { memoryClaims, kvClaims } from "./claims";
|
|
29
|
+
export type { ClaimKv } from "./claims";
|
|
30
|
+
export { createTuner, createHalvingTuner } from "./tuner";
|
|
31
|
+
export type { Tuner, HalvingTuner, TunerBounds } from "./tuner";
|
|
32
|
+
export { createPartPool, PoolCancelled } from "./pool";
|
|
33
|
+
export type { PartPool, PartPoolOptions, PoolLane } from "./pool";
|
|
34
|
+
export { createHttpSigner, SignerError } from "./signer";
|
|
35
|
+
export type { HttpSignerOptions } from "./signer";
|
|
36
|
+
export { createTransferEngine, memoryStore, fileStore } from "./client";
|
|
37
|
+
export type { ResumeStore, UploadRequest, UploadHandle, UploadResult, UploadState, TransferEngineOptions } from "./client";
|
|
38
|
+
export { sanitizeLimits, sanitizeCap, resolvePartSize, MAX_PART_BYTES } from "./limits";
|
|
39
|
+
export type { ConnectionLimits, PartSizeHook } from "./limits";
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Part sizing and the checks that make "verified" mean something: per-part Content-MD5 (storage refuses changed
|
|
3
|
+
* bytes), the part ETag = MD5 of what was read, the multipart ETag = md5 of the part MD5s, the whole file's CRC-32
|
|
4
|
+
* combined from its parts', and the ETag a local file must have.
|
|
5
|
+
*
|
|
6
|
+
* Moved from UploadFilm's app/src/main/engine.ts (partSizeFor, compositeEtag, partsCrc, localDigest) and
|
|
7
|
+
* app/src/main/downloads.ts (checkEtag, etags).
|
|
8
|
+
*/
|
|
9
|
+
import { crcHex } from "../server/r2/crc32";
|
|
10
|
+
import type { Hasher } from "./protocol";
|
|
11
|
+
export { crcHex };
|
|
12
|
+
/** Smallest part (16 MiB). */
|
|
13
|
+
export declare const MIN_PART: number;
|
|
14
|
+
/** Objects under this go as one PUT, not a multipart upload. */
|
|
15
|
+
export declare const SMALL: number;
|
|
16
|
+
/** Most parts we cut an object into (headroom under S3's 10,000). */
|
|
17
|
+
export declare const MAX_PARTS = 9500;
|
|
18
|
+
/** Same as `@uploadfilm/sdk/server` partSizeFor with its defaults, without its range checks (never throws). */
|
|
19
|
+
export declare const partSizeFor: (size: number) => number;
|
|
20
|
+
export declare const isHex32: (s: string) => boolean;
|
|
21
|
+
export declare const toHex: (b: Uint8Array) => string;
|
|
22
|
+
/** S3/R2 multipart ETag: md5 of the concatenated binary part MD5s, then "-<parts>". null unless every part is an MD5. */
|
|
23
|
+
export declare function compositeEtag(hasher: Hasher, parts: Record<number, string>): string | null;
|
|
24
|
+
/**
|
|
25
|
+
* The whole file's CRC-32 from its parts' (crc32_combine, in part order), or undefined unless every part has
|
|
26
|
+
* one. Only called with a complete, all-MD5 parts list (the stitch), so the CRCs describe exactly the bytes
|
|
27
|
+
* whose MD5s storage confirmed.
|
|
28
|
+
*/
|
|
29
|
+
export declare function partsCrc(mp: {
|
|
30
|
+
partSize: number;
|
|
31
|
+
parts: Record<number, string>;
|
|
32
|
+
crcs?: Record<number, number>;
|
|
33
|
+
}, size: number): string | undefined;
|
|
34
|
+
/**
|
|
35
|
+
* R2 answers HEAD (and so the server's complete/head) of text-like objects over ~1 KiB with a WEAK ETag
|
|
36
|
+
* W/"…" because it may compress them in transit; the value is still the stored object's ETag. A weak prefix is
|
|
37
|
+
* not a difference: compare what's behind it.
|
|
38
|
+
*/
|
|
39
|
+
export declare const strongStat: <S extends {
|
|
40
|
+
etag: string | null;
|
|
41
|
+
}>(st: S) => S;
|
|
42
|
+
/** Reads `len` bytes at `pos` into `buf` (a FileHandle's read). */
|
|
43
|
+
export interface RangeReader {
|
|
44
|
+
read(buf: Uint8Array, off: number, len: number, pos: number): Promise<{
|
|
45
|
+
bytesRead: number;
|
|
46
|
+
}>;
|
|
47
|
+
close(): Promise<void>;
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* The ETag storage computes for `size` bytes read through `r`: MD5 for one PUT (under SMALL), md5(part MD5s)-N for a
|
|
51
|
+
* multipart upload with partSizeFor's part size (or `partSize`); plus the CRC-32 (8 hex), from the same single read.
|
|
52
|
+
*/
|
|
53
|
+
export declare function digestOf(hasher: Hasher, r: RangeReader, size: number, partSize?: number): Promise<{
|
|
54
|
+
etag: string;
|
|
55
|
+
crc32: string;
|
|
56
|
+
}>;
|
|
57
|
+
/** An incremental MD5 (node:crypto createHash("md5") fits). */
|
|
58
|
+
export interface Md5Stream {
|
|
59
|
+
update(b: Uint8Array): unknown;
|
|
60
|
+
digest(): Uint8Array;
|
|
61
|
+
}
|
|
62
|
+
/**
|
|
63
|
+
* MD5 of the bytes (no part sizes), or the multipart ETag hex for each part size, in one read of `size` bytes.
|
|
64
|
+
* `yieldEvery`: awaited between 8 MiB reads (let a UI breathe on huge files).
|
|
65
|
+
*/
|
|
66
|
+
export declare function etagsOf(newMd5: () => Md5Stream, hasher: Hasher, r: Pick<RangeReader, "read">, size: number, sizes: number[], parts?: number, yieldEvery?: () => Promise<unknown>): Promise<string[]>;
|
|
67
|
+
/**
|
|
68
|
+
* Does `size` bytes read through `r` have this S3 ETag? true / false, or null when it can't be reproduced (multipart
|
|
69
|
+
* with a part size we can't find). The uploader's part-size rule first (one read); only if that isn't it, the
|
|
70
|
+
* power-of-two MiB sizes and the first 12 whole MiB that give the same part count (a tuned or an older uploader), in
|
|
71
|
+
* one more read.
|
|
72
|
+
*/
|
|
73
|
+
export declare function checkEtag(newMd5: () => Md5Stream, hasher: Hasher, open: () => Promise<Pick<RangeReader, "read"> & {
|
|
74
|
+
close?: () => Promise<void>;
|
|
75
|
+
}>, size: number, etag: string, yieldEvery?: () => Promise<unknown>): Promise<boolean | null>;
|
|
76
|
+
/** A base64 SHA-256 exactly as S3/R2 send it in `x-amz-checksum-sha256` (44 chars, one "=" of padding). */
|
|
77
|
+
export declare const SHA256_B64: RegExp;
|
|
78
|
+
export declare const isHex64: (s: string) => boolean;
|
|
79
|
+
/** Bytes → base64 (no Buffer: any runtime). */
|
|
80
|
+
export declare function bytesToB64(b: Uint8Array): string;
|
|
81
|
+
export declare const hexToB64: (h: string) => string;
|
|
82
|
+
/** A strict base64 SHA-256 → hex, or null (missing, malformed, repeated "a, b", wrong length). */
|
|
83
|
+
export declare function sha256B64ToHex(v: string | undefined | null): string | null;
|
|
84
|
+
/**
|
|
85
|
+
* A checksum a server reported for a stored object: "<b64>" (one PUT: the object's SHA-256) or "<b64>-<parts>" (S3's
|
|
86
|
+
* COMPOSITE multipart checksum: sha256 of the concatenated part digests). → { hex, parts } or null when malformed.
|
|
87
|
+
*/
|
|
88
|
+
export declare function parseSha256Checksum(v: string | undefined | null): {
|
|
89
|
+
hex: string;
|
|
90
|
+
parts?: number;
|
|
91
|
+
} | null;
|
|
92
|
+
/** S3 COMPOSITE SHA-256 of a multipart upload: sha256 of the concatenated binary part SHA-256s, "-<parts>". null unless every part has one. */
|
|
93
|
+
export declare function compositeSha256(sha256: (b: Uint8Array) => Uint8Array, parts: Record<number, string>, count: number): string | null;
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
/** S3's (and R2's) largest part. */
|
|
2
|
+
export declare const MAX_PART_BYTES: number;
|
|
3
|
+
export interface ConnectionLimits {
|
|
4
|
+
min: number;
|
|
5
|
+
max: number;
|
|
6
|
+
/** Where the tuner starts; only before its first step (later it climbs from wherever it is). */
|
|
7
|
+
start?: number;
|
|
8
|
+
}
|
|
9
|
+
/** Part size for a multipart of `size` bytes created now. */
|
|
10
|
+
export type PartSizeHook = (size: number) => number;
|
|
11
|
+
/**
|
|
12
|
+
* Whole numbers, 1 ≤ min ≤ (start) ≤ max. A max that isn't a number ≥ 1 is refused (RangeError): the engine never
|
|
13
|
+
* runs on a limit it made up. min is clamped into 1..max; start into min..max.
|
|
14
|
+
*/
|
|
15
|
+
export declare function sanitizeLimits(l: ConnectionLimits): {
|
|
16
|
+
min: number;
|
|
17
|
+
max: number;
|
|
18
|
+
start?: number;
|
|
19
|
+
};
|
|
20
|
+
/** The in-flight byte cap: a positive number of bytes, or null for none (null, undefined, Infinity, 0 or less, NaN). */
|
|
21
|
+
export declare function sanitizeCap(n: number | null | undefined): number | null;
|
|
22
|
+
/**
|
|
23
|
+
* The part size a new multipart of `size` bytes gets: the default rule (16 MiB, or more so it fits in 9,500 parts)
|
|
24
|
+
* unless the hook asks for bigger; rounded up to a power-of-two MiB (16 MiB .. 4 GiB; never below the rule, so the part
|
|
25
|
+
* count stays within storage's limit). A hook that throws or answers anything but a positive number: the rule.
|
|
26
|
+
*/
|
|
27
|
+
export declare function resolvePartSize(size: number, hook: PartSizeHook | null | undefined): number;
|
|
28
|
+
/** Part sizes a hook resolves to: 16 MiB × 2^k up to 4 GiB (whole MiB, ≤ 5 GiB). */
|
|
29
|
+
export declare const POW2_PARTS: readonly number[];
|
|
30
|
+
/**
|
|
31
|
+
* The power-of-two MiB at or above `want` (16 MiB .. 4 GiB). Tuned part sizes are powers of two so anyone checking a
|
|
32
|
+
* stored ETag can find the layout without knowing the uploader's settings (checkEtag tries these first; adoption too).
|
|
33
|
+
*/
|
|
34
|
+
export declare function pow2Part(want: number): number;
|