@uploadfilm/sdk 0.0.0-stage → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +4 -0
- package/README.md +529 -2
- package/dist/ai/embed.d.ts +58 -0
- package/dist/ai/fake.d.ts +41 -0
- package/dist/ai/http.d.ts +17 -0
- package/dist/ai/index.d.ts +27 -0
- package/dist/ai/languages.d.ts +2 -0
- package/dist/ai/ollama.d.ts +23 -0
- package/dist/ai/openaiCompatible.d.ts +54 -0
- package/dist/ai/passages.d.ts +28 -0
- package/dist/ai/transcribe.d.ts +84 -0
- package/dist/ai/types.d.ts +141 -0
- package/dist/ai/usage.d.ts +66 -0
- package/dist/ai/whisperJson.d.ts +15 -0
- package/dist/ai/workersAi.d.ts +56 -0
- package/dist/ai.js +1 -0
- package/dist/ai.mjs +1 -0
- package/dist/cards/files.d.ts +91 -0
- package/dist/cards/index.d.ts +15 -0
- package/dist/cards/layouts.d.ts +121 -0
- package/dist/cards/probe.d.ts +69 -0
- package/dist/cards/proxies.d.ts +70 -0
- package/dist/cards/select.d.ts +68 -0
- package/dist/cards/volumes.d.ts +170 -0
- package/dist/cards/xml.d.ts +80 -0
- package/dist/cards.js +1 -0
- package/dist/cards.mjs +1 -0
- package/dist/copy/checksums.d.ts +66 -0
- package/dist/copy/engine.d.ts +24 -0
- package/dist/copy/errors.d.ts +109 -0
- package/dist/copy/index.d.ts +15 -0
- package/dist/copy/steps.d.ts +28 -0
- package/dist/copy/sweep.d.ts +24 -0
- package/dist/copy/types.d.ts +363 -0
- package/dist/embed/player.js +2 -0
- package/dist/host/adapters.d.ts +107 -0
- package/dist/host/detect.d.ts +68 -0
- package/dist/host/encoders.d.ts +34 -0
- package/dist/host/environment.d.ts +53 -0
- package/dist/host/index.d.ts +20 -0
- package/dist/host/linux.d.ts +94 -0
- package/dist/host/mac.d.ts +47 -0
- package/dist/host/network.d.ts +54 -0
- package/dist/host/node.d.ts +13 -0
- package/dist/host/policy.d.ts +101 -0
- package/dist/host/tuner.d.ts +48 -0
- package/dist/host/types.d.ts +118 -0
- package/dist/host-node.js +1 -0
- package/dist/host-node.mjs +1 -0
- package/dist/host.js +1 -0
- package/dist/host.mjs +1 -0
- package/dist/images/index.d.ts +11 -0
- package/dist/images/settings.d.ts +43 -0
- package/dist/images/types.d.ts +83 -0
- package/dist/images/url.d.ts +65 -0
- package/dist/index.d.ts +16 -0
- package/dist/index.js +1 -0
- package/dist/index.mjs +1 -0
- package/dist/player/adapters/hlsjs.d.ts +28 -0
- package/dist/player/bandwidthMemory.d.ts +28 -0
- package/dist/player/captions/index.d.ts +55 -0
- package/dist/player/embed/code.d.ts +195 -0
- package/dist/player/embed/constants.d.ts +4 -0
- package/dist/player/embed/controller.d.ts +62 -0
- package/dist/player/embed/element.d.ts +32 -0
- package/dist/player/embed/frame.d.ts +65 -0
- package/dist/player/embed/global.d.ts +14 -0
- package/dist/player/embed/index.d.ts +15 -0
- package/dist/player/embed/loader.d.ts +1 -0
- package/dist/player/embed/oembed.d.ts +74 -0
- package/dist/player/embed/protocol.d.ts +64 -0
- package/dist/player/engine.d.ts +215 -0
- package/dist/player/index.d.ts +18 -0
- package/dist/player/start.d.ts +62 -0
- package/dist/player/store.d.ts +10 -0
- package/dist/player/storyboard/index.d.ts +161 -0
- package/dist/player/stream.d.ts +206 -0
- package/dist/player/tracks.d.ts +41 -0
- package/dist/player.js +1 -0
- package/dist/player.mjs +1 -0
- package/dist/proxies/args.d.ts +19 -0
- package/dist/proxies/budget.d.ts +57 -0
- package/dist/proxies/camera.d.ts +104 -0
- package/dist/proxies/formats.d.ts +36 -0
- package/dist/proxies/index.d.ts +18 -0
- package/dist/proxies/jobs.d.ts +98 -0
- package/dist/proxies/node.d.ts +32 -0
- package/dist/proxies/plan.d.ts +89 -0
- package/dist/proxies/probe.d.ts +39 -0
- package/dist/proxies/render.d.ts +104 -0
- package/dist/proxies/safety.d.ts +45 -0
- package/dist/proxies/types.d.ts +83 -0
- package/dist/proxies/upload.d.ts +14 -0
- package/dist/react/BigPlay.d.ts +26 -0
- package/dist/react/Player.d.ts +58 -0
- package/dist/react/embed/index.d.ts +45 -0
- package/dist/react/examples/CustomControls.d.ts +8 -0
- package/dist/react/images/Image.d.ts +16 -0
- package/dist/react/images/index.d.ts +4 -0
- package/dist/react/images/useImage.d.ts +47 -0
- package/dist/react/index.d.ts +17 -0
- package/dist/react/storyboard/index.d.ts +58 -0
- package/dist/react/theme.d.ts +37 -0
- package/dist/react/useAdapterErrors.d.ts +8 -0
- package/dist/react/usePlaybackStart.d.ts +18 -0
- package/dist/react/usePlayer.d.ts +28 -0
- package/dist/react/useReducedMotion.d.ts +2 -0
- package/dist/react/useTimedTrack.d.ts +22 -0
- package/dist/react.js +1 -0
- package/dist/react.mjs +1 -0
- package/dist/server/auth/apiKeys.d.ts +30 -0
- package/dist/server/auth/cache.d.ts +21 -0
- package/dist/server/auth/deviceTokens.d.ts +38 -0
- package/dist/server/auth/envelope.d.ts +15 -0
- package/dist/server/auth/grants.d.ts +76 -0
- package/dist/server/auth/index.d.ts +19 -0
- package/dist/server/auth/jwt.d.ts +65 -0
- package/dist/server/auth/keyring.d.ts +31 -0
- package/dist/server/auth/kv.d.ts +12 -0
- package/dist/server/auth/repository.d.ts +45 -0
- package/dist/server/auth/revocations.d.ts +21 -0
- package/dist/server/auth/scope.d.ts +4 -0
- package/dist/server/auth/serviceAuth.d.ts +24 -0
- package/dist/server/auth/signedUrls.d.ts +28 -0
- package/dist/server/embed/index.d.ts +36 -0
- package/dist/server/embed/lifecycle.d.ts +53 -0
- package/dist/server/embed/playerScript.d.ts +4 -0
- package/dist/server/images/index.d.ts +15 -0
- package/dist/server/images/job.d.ts +80 -0
- package/dist/server/images/layout.d.ts +24 -0
- package/dist/server/images/params.d.ts +90 -0
- package/dist/server/images/route.d.ts +101 -0
- package/dist/server/images/sign.d.ts +55 -0
- package/dist/server/index.d.ts +42 -0
- package/dist/server/media/client.d.ts +99 -0
- package/dist/server/media/frameCheck.d.ts +62 -0
- package/dist/server/media/index.d.ts +10 -0
- package/dist/server/media/protocol.d.ts +556 -0
- package/dist/server/media/storyboard.d.ts +144 -0
- package/dist/server/media/transcript.d.ts +143 -0
- package/dist/server/pipeline/index.d.ts +12 -0
- package/dist/server/pipeline/match.d.ts +10 -0
- package/dist/server/pipeline/pipeline.d.ts +182 -0
- package/dist/server/pipeline/queues.d.ts +108 -0
- package/dist/server/pipeline/sources.d.ts +54 -0
- package/dist/server/r2/client.d.ts +164 -0
- package/dist/server/r2/crc32.d.ts +24 -0
- package/dist/server/r2/encoding.d.ts +35 -0
- package/dist/server/r2/errors.d.ts +9 -0
- package/dist/server/r2/integrity.d.ts +25 -0
- package/dist/server/r2/parts.d.ts +46 -0
- package/dist/server/r2/sign.d.ts +47 -0
- package/dist/server/redis/index.d.ts +39 -0
- package/dist/server/state/memory.d.ts +18 -0
- package/dist/server/state/r2.d.ts +73 -0
- package/dist/server/state/types.d.ts +57 -0
- package/dist/server/storage/chunked.d.ts +25 -0
- package/dist/server/storage/index.d.ts +12 -0
- package/dist/server/storage/r2.d.ts +96 -0
- package/dist/server/storage/s3.d.ts +144 -0
- package/dist/server/testing/fake-r2.d.ts +410 -0
- package/dist/server/testing/index.d.ts +10 -0
- package/dist/server/testing/kv-contract.d.ts +14 -0
- package/dist/server/transfer/handler.d.ts +217 -0
- package/dist/server/transfer/keys.d.ts +45 -0
- package/dist/server/transfer/protocol.d.ts +190 -0
- package/dist/server/zip/client.d.ts +32 -0
- package/dist/server/zip/generations.d.ts +83 -0
- package/dist/server/zip/index.d.ts +12 -0
- package/dist/server/zip/planner.d.ts +94 -0
- package/dist/server/zip/sign.d.ts +22 -0
- package/dist/server-media.js +1 -0
- package/dist/server-media.mjs +1 -0
- package/dist/server-redis.js +1 -0
- package/dist/server-redis.mjs +1 -0
- package/dist/server-testing.js +1 -0
- package/dist/server-testing.mjs +1 -0
- package/dist/server.js +1 -0
- package/dist/server.mjs +1 -0
- package/dist/storage/browserFile.d.ts +19 -0
- package/dist/storage/index.d.ts +12 -0
- package/dist/storage/local.d.ts +9 -0
- package/dist/storage/presigned.d.ts +22 -0
- package/dist/sync/ffmpeg.d.ts +44 -0
- package/dist/sync/index.d.ts +17 -0
- package/dist/sync/match.d.ts +28 -0
- package/dist/sync/probe.d.ts +30 -0
- package/dist/sync/timecode.d.ts +50 -0
- package/dist/sync/transcript.d.ts +35 -0
- package/dist/sync/types.d.ts +172 -0
- package/dist/sync/waveform.d.ts +89 -0
- package/dist/sync.js +1 -0
- package/dist/sync.mjs +1 -0
- package/dist/transfer/claims.d.ts +41 -0
- package/dist/transfer/client.d.ts +267 -0
- package/dist/transfer/download.d.ts +192 -0
- package/dist/transfer/engine.d.ts +402 -0
- package/dist/transfer/errors.d.ts +50 -0
- package/dist/transfer/growing/bytes.d.ts +17 -0
- package/dist/transfer/growing/index.d.ts +12 -0
- package/dist/transfer/growing/parts.d.ts +157 -0
- package/dist/transfer/growing/retry.d.ts +38 -0
- package/dist/transfer/growing/store.d.ts +66 -0
- package/dist/transfer/growing/upload.d.ts +126 -0
- package/dist/transfer/index.d.ts +40 -0
- package/dist/transfer/integrity.d.ts +93 -0
- package/dist/transfer/limits.d.ts +34 -0
- package/dist/transfer/node.d.ts +63 -0
- package/dist/transfer/pool.d.ts +83 -0
- package/dist/transfer/protocol.d.ts +172 -0
- package/dist/transfer/signer.d.ts +29 -0
- package/dist/transfer/storage.d.ts +135 -0
- package/dist/transfer/timing.d.ts +132 -0
- package/dist/transfer/transports/electronNet.d.ts +62 -0
- package/dist/transfer/transports/fetch.d.ts +15 -0
- package/dist/transfer/transports/retryAfter.d.ts +12 -0
- package/dist/transfer/transports/undici.d.ts +72 -0
- package/dist/transfer/tuner.d.ts +56 -0
- package/dist/workers/cli.mjs +115 -0
- package/dist/workers/media/config.mjs +53 -0
- package/dist/workers/media/container/Dockerfile +19 -0
- package/dist/workers/media/container/FFMPEG.md +129 -0
- package/dist/workers/media/container/runner.mjs +2690 -0
- package/dist/workers/media/index.mjs +1011 -0
- package/dist/workers/media/init/config.ts +52 -0
- package/dist/workers/media/init/worker.ts +8 -0
- package/dist/workers/media/init/wrangler.toml +44 -0
- package/dist/workers/media/runner.mjs +2656 -0
- package/dist/workers/media/types/build/entry/config.d.mts +6 -0
- package/dist/workers/media/types/build/entry/index.d.mts +43 -0
- package/dist/workers/media/types/build/entry/program.d.mts +13 -0
- package/dist/workers/media/types/build/entry/runner.d.mts +12 -0
- package/dist/workers/media/types/build/entry/settings.d.mts +40 -0
- package/dist/workers/media/types/src/config.d.mts +135 -0
- package/dist/workers/media/types/src/jobs.d.mts +73 -0
- package/dist/workers/media/types/src/runner/container.d.mts +32 -0
- package/dist/workers/media/types/src/runner/ffmpeg.d.mts +227 -0
- package/dist/workers/media/types/src/runner/job.d.mts +107 -0
- package/dist/workers/media/types/src/runner/node.d.mts +71 -0
- package/dist/workers/media/types/src/runner/plugins.d.mts +45 -0
- package/dist/workers/media/types/src/serve.d.mts +107 -0
- package/dist/workers/media/types/src/steps/audio.d.mts +33 -0
- package/dist/workers/media/types/src/steps/edit.d.mts +62 -0
- package/dist/workers/media/types/src/steps/frameCheck.d.mts +44 -0
- package/dist/workers/media/types/src/steps/gate.d.mts +18 -0
- package/dist/workers/media/types/src/steps/paths.d.mts +1 -0
- package/dist/workers/media/types/src/steps/prep.d.mts +60 -0
- package/dist/workers/media/types/src/steps/render.d.mts +40 -0
- package/dist/workers/media/types/src/steps/runner.d.mts +30 -0
- package/dist/workers/media/types/src/steps/transcribe.d.mts +89 -0
- package/dist/workers/media/types/src/storyboard/index.d.mts +61 -0
- package/dist/workers/media/types/src/transcoder.d.mts +30 -0
- package/dist/workers/media/types/src/worker.d.mts +30 -0
- package/dist/workers/zip/index.mjs +589 -0
- package/dist/workers/zip/init/config.ts +22 -0
- package/dist/workers/zip/init/worker.ts +5 -0
- package/dist/workers/zip/init/wrangler.toml +25 -0
- package/dist/workers/zip/types/build/entry/index.d.mts +5 -0
- package/dist/workers/zip/types/src/config.d.mts +35 -0
- package/dist/workers/zip/types/src/serve.d.mts +86 -0
- package/dist/workers/zip/types/src/worker.d.mts +12 -0
- package/dist/zip/index.d.ts +13 -0
- package/dist/zip/layout.d.ts +129 -0
- package/dist/zip/names.d.ts +34 -0
- package/dist/zip/plan.d.ts +115 -0
- package/dist/zip/sw.d.ts +7 -0
- package/package.json +313 -4
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Uploading what could not be uploaded live, from the product's local copy.
|
|
3
|
+
*
|
|
4
|
+
* Ported from ClipMe's extension/src/upload/retry.ts (createRetryPump, ClipMe origin/main 4684516). Everything it
|
|
5
|
+
* needs is in the record, because the context that started the upload may be long gone (a killed tab, a reload, a
|
|
6
|
+
* crash): the parts storage holds are proven and skipped, the rest is read again from the local copy. ClipMe's rules
|
|
7
|
+
* are kept: one pump at a time per context; nothing is tried while offline (an attempt that can't work doesn't count);
|
|
8
|
+
* uploads live in this context are left alone; a recording still being written waits; a deleted one is dropped (its
|
|
9
|
+
* multipart aborted); a failure backs off from one minute to fifteen and never gives up unless retrying can't help.
|
|
10
|
+
* ClipMe's product steps (opening its clip, finalize, cursor.json) are the product's: `onResult` is where they go.
|
|
11
|
+
*/
|
|
12
|
+
import { type GrowingRecord, type GrowingStore } from "./store";
|
|
13
|
+
import { type GrowingResult, type GrowingUploadOptions } from "./upload";
|
|
14
|
+
export interface GrowingLocalCopy {
|
|
15
|
+
/**
|
|
16
|
+
* Where the local copy stands: "complete" (closed, every byte there: resume it), "recording" (still being written,
|
|
17
|
+
* or waiting for crash recovery to close it: not now), "gone" (deleted from the device, or never had anything:
|
|
18
|
+
* drop the record and abort its multipart). Default "complete".
|
|
19
|
+
*/
|
|
20
|
+
state?(rec: GrowingRecord): Promise<"complete" | "recording" | "gone"> | "complete" | "recording" | "gone";
|
|
21
|
+
/**
|
|
22
|
+
* The whole recording from byte 0, in order. It MUST yield the same bytes as were appended live: parts are kept by
|
|
23
|
+
* position (a part whose bytes differ is sent again, so a differently muxed file is never stitched from both).
|
|
24
|
+
*/
|
|
25
|
+
read(rec: GrowingRecord): AsyncIterable<Uint8Array | Blob>;
|
|
26
|
+
}
|
|
27
|
+
export interface GrowingRetryOptions extends Omit<GrowingUploadOptions, "key" | "scope" | "id" | "contentType" | "meta" | "partSize" | "store"> {
|
|
28
|
+
store: GrowingStore;
|
|
29
|
+
local: GrowingLocalCopy;
|
|
30
|
+
/** navigator.onLine. An attempt that cannot work should not count as one. */
|
|
31
|
+
isOnline?: () => boolean;
|
|
32
|
+
/** Each attempt's outcome (verified, or failed with the record's new state). */
|
|
33
|
+
onResult?(rec: GrowingRecord, result: GrowingResult): void;
|
|
34
|
+
}
|
|
35
|
+
/** One pump at a time per context; a wake-up that lands mid-run has nothing to add. */
|
|
36
|
+
export declare function createGrowingRetryPump(o: GrowingRetryOptions): {
|
|
37
|
+
run(): Promise<void>;
|
|
38
|
+
};
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* What a growing upload keeps between runs (a crash, a reload, a killed tab): one record per upload, under
|
|
3
|
+
* `growing:<id>`. The record is enough to resume: the multipart's upload id, its part size, and every part storage
|
|
4
|
+
* acknowledged (number → MD5 and size). The bytes themselves are the product's local copy (ClipMe: OPFS), which the
|
|
5
|
+
* resumed upload reads again from byte 0; parts proven against ListParts and the re-read bytes are not sent again.
|
|
6
|
+
*
|
|
7
|
+
* Stores are async (IndexedDB, chrome.storage, a server); the engine's synchronous ResumeStore (memoryStore,
|
|
8
|
+
* fileStore) fits too. ClipMe's rule is kept: one key per record, so two contexts never lose each other's writes to a
|
|
9
|
+
* read-modify-write.
|
|
10
|
+
*/
|
|
11
|
+
export interface GrowingStore {
|
|
12
|
+
get(key: string): unknown | Promise<unknown>;
|
|
13
|
+
set(key: string, value: unknown): void | Promise<void>;
|
|
14
|
+
del(key: string): void | Promise<void>;
|
|
15
|
+
keys(): string[] | Promise<string[]>;
|
|
16
|
+
}
|
|
17
|
+
/** Where a record stands. */
|
|
18
|
+
export type GrowingRecordState =
|
|
19
|
+
/** Uploading while recording, or the context that did so died. */
|
|
20
|
+
"live"
|
|
21
|
+
/** To be uploaded again from the local copy (a retryable failure). */
|
|
22
|
+
| "queued"
|
|
23
|
+
/** Retrying cannot help (refused by the server, nothing recorded). The local copy is still the product's. */
|
|
24
|
+
| "failed";
|
|
25
|
+
export interface GrowingRecord {
|
|
26
|
+
v: 1;
|
|
27
|
+
id: string;
|
|
28
|
+
scope: string;
|
|
29
|
+
key: string;
|
|
30
|
+
contentType: string;
|
|
31
|
+
partSize: number;
|
|
32
|
+
state: GrowingRecordState;
|
|
33
|
+
createdAt: number;
|
|
34
|
+
uploadId?: string;
|
|
35
|
+
/**
|
|
36
|
+
* When a CreateMultipartUpload was asked for and its upload id isn't recorded yet (RL-10): a run that finds it set
|
|
37
|
+
* aborts what storage opened on the key before creating another.
|
|
38
|
+
*/
|
|
39
|
+
creating?: number;
|
|
40
|
+
/** Parts storage acknowledged: number → { etag: MD5 hex, size }. */
|
|
41
|
+
parts: Record<number, {
|
|
42
|
+
etag: string;
|
|
43
|
+
size: number;
|
|
44
|
+
}>;
|
|
45
|
+
/** end() was called: the recording's total length. */
|
|
46
|
+
ended?: number;
|
|
47
|
+
/** Completes storage refused as unstitchable, or that kept failing: at COMPLETE_FAILURES_BEFORE_RESTART a fresh multipart. */
|
|
48
|
+
completeFailures?: number;
|
|
49
|
+
attempts: number;
|
|
50
|
+
nextAttemptAt: number;
|
|
51
|
+
lastError?: string;
|
|
52
|
+
lastCode?: string;
|
|
53
|
+
/** The product's own data (ClipMe: its clip, title, finish metadata). Kept as given. */
|
|
54
|
+
meta?: unknown;
|
|
55
|
+
}
|
|
56
|
+
export declare const RECORD_PREFIX = "growing:";
|
|
57
|
+
export declare const recordKey: (id: string) => string;
|
|
58
|
+
export declare function loadRecord(store: GrowingStore, id: string): Promise<GrowingRecord | null>;
|
|
59
|
+
export declare const saveRecord: (store: GrowingStore, rec: GrowingRecord) => Promise<undefined>;
|
|
60
|
+
export declare const removeRecord: (store: GrowingStore, id: string) => Promise<undefined>;
|
|
61
|
+
/** Every unfinished growing upload in the store (what a retry pump walks). */
|
|
62
|
+
export declare function listRecords(store: GrowingStore): Promise<GrowingRecord[]>;
|
|
63
|
+
/** An in-memory store (tests, or a product that keeps nothing across reloads). Values are copied as JSON. */
|
|
64
|
+
export declare function memoryGrowingStore(): GrowingStore & {
|
|
65
|
+
readonly size: number;
|
|
66
|
+
};
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Upload while recording: one growing object's trip from appended chunks to a verified object in storage.
|
|
3
|
+
*
|
|
4
|
+
* Ported from ClipMe's extension/src/upload/session.ts (LiveUpload, ClipMe origin/main 4684516), keeping its
|
|
5
|
+
* behaviour: `createGrowingUpload` returns at once and opens the multipart in the background, so the recorder is never
|
|
6
|
+
* held up; chunks that arrive before the multipart exists are buffered (and count toward the backlog cap); whatever
|
|
7
|
+
* goes wrong, `append` never throws and the recording carries on: the product's local copy has every byte, so a
|
|
8
|
+
* failure just leaves a record for a resume (`createGrowingRetryPump`, or `createGrowingUpload` again with the same id).
|
|
9
|
+
* ClipMe's product steps (its clip, link, clipboard, poster frames, cursor track, finalize) stay ClipMe's.
|
|
10
|
+
*
|
|
11
|
+
* What the SDK adds is the proof (docs/platform/GROWING-UPLOADS.md):
|
|
12
|
+
* - a resumed multipart is checked against ListParts before anything is kept: a saved part is kept only if storage
|
|
13
|
+
* lists it with the same ETag and size, and the bytes read again at its position hash to that ETag;
|
|
14
|
+
* - `end()` resolves "verified" only after end() was called, every part 1..N is stored with ETag = its MD5, the
|
|
15
|
+
* non-trailing parts are exactly `partSize`, their sizes add up to every byte appended, and storage's answer to the
|
|
16
|
+
* complete (a HEAD) has that size and the composite ETag md5(part MD5s)-N. Anything else is "failed", never done;
|
|
17
|
+
* - the server signs every op (the SDK's Signer protocol), so its `signParts` hook can refuse a recording over budget.
|
|
18
|
+
*/
|
|
19
|
+
import { type ErrorAdapter } from "../errors";
|
|
20
|
+
import type { Signer, Transport } from "../protocol";
|
|
21
|
+
import { type PartHasher } from "./parts";
|
|
22
|
+
import { type GrowingStore } from "./store";
|
|
23
|
+
/** Complete refusals after which the multipart is given up and a fresh one is started from the local copy (ClipMe). */
|
|
24
|
+
export declare const COMPLETE_FAILURES_BEFORE_RESTART = 3;
|
|
25
|
+
/** Backoff between resume attempts, from one minute up to fifteen. Never gives up (ClipMe's `nextDelayMs`). */
|
|
26
|
+
export declare function growingRetryDelayMs(attempts: number): number;
|
|
27
|
+
export type GrowingState = "opening" | "sending" | "finishing" | "verified" | "failed" | "cancelled";
|
|
28
|
+
export type GrowingResult = {
|
|
29
|
+
state: "verified";
|
|
30
|
+
id: string;
|
|
31
|
+
key: string;
|
|
32
|
+
size: number;
|
|
33
|
+
etag: string;
|
|
34
|
+
parts: number;
|
|
35
|
+
uploadId: string;
|
|
36
|
+
}
|
|
37
|
+
/** `retryable`: a resume from the local copy can still get it there (the record is kept, state "queued"). */
|
|
38
|
+
| {
|
|
39
|
+
state: "failed";
|
|
40
|
+
id: string;
|
|
41
|
+
key: string;
|
|
42
|
+
error: string;
|
|
43
|
+
code?: string;
|
|
44
|
+
retryable: boolean;
|
|
45
|
+
} | {
|
|
46
|
+
state: "cancelled";
|
|
47
|
+
id: string;
|
|
48
|
+
key: string;
|
|
49
|
+
};
|
|
50
|
+
export interface GrowingProgress {
|
|
51
|
+
/** Bytes appended so far. */
|
|
52
|
+
total: number;
|
|
53
|
+
/** Bytes storage holds and the uploader proved. */
|
|
54
|
+
acknowledged: number;
|
|
55
|
+
/** Appended and not yet acknowledged. */
|
|
56
|
+
pending: number;
|
|
57
|
+
/** Of `pending`, bytes in-flight PUTs have sent. */
|
|
58
|
+
sending: number;
|
|
59
|
+
}
|
|
60
|
+
export interface GrowingUploadOptions {
|
|
61
|
+
signer: Signer;
|
|
62
|
+
transport: Transport;
|
|
63
|
+
/** MD5 of each part: nodeHasher in Node/Electron; in browsers a WebAssembly MD5 (hash-wasm: `{ md5 }`). */
|
|
64
|
+
hasher: PartHasher;
|
|
65
|
+
/** Storage key. Must be a kind the server lets grow: an immutable kind needs its size at create, which a recording doesn't know. */
|
|
66
|
+
key: string;
|
|
67
|
+
/** Signer scope (default "default"). */
|
|
68
|
+
scope?: string;
|
|
69
|
+
/** Stable id (default `${scope}:${key}`). A record saved under it is resumed. */
|
|
70
|
+
id?: string;
|
|
71
|
+
contentType?: string;
|
|
72
|
+
/** Where the record lives between runs (default: none, nothing survives a crash). */
|
|
73
|
+
store?: GrowingStore;
|
|
74
|
+
/** The product's own data, kept on the record (ClipMe: its clip and finish metadata). */
|
|
75
|
+
meta?: unknown;
|
|
76
|
+
/** Part size for a new multipart (default 5 MiB, ClipMe's; at least 5 MiB). A resumed multipart keeps its own. */
|
|
77
|
+
partSize?: number;
|
|
78
|
+
maxParallel?: number;
|
|
79
|
+
maxAttempts?: number;
|
|
80
|
+
maxPendingBytes?: number;
|
|
81
|
+
/** Sign the next part ahead (default true). */
|
|
82
|
+
signAhead?: boolean;
|
|
83
|
+
/** PUT deadline basis passed to the transport (default 60 s). */
|
|
84
|
+
partTimeoutMs?: number;
|
|
85
|
+
/** Ids uploading in this context, which a retry pump must leave alone (ClipMe's `active`). */
|
|
86
|
+
active?: Set<string>;
|
|
87
|
+
/** The product's signer errors → StorageErrors (default duckErrors). */
|
|
88
|
+
errors?: ErrorAdapter;
|
|
89
|
+
sleep?: (ms: number) => Promise<void>;
|
|
90
|
+
now?: () => number;
|
|
91
|
+
}
|
|
92
|
+
export interface GrowingUpload {
|
|
93
|
+
readonly id: string;
|
|
94
|
+
readonly key: string;
|
|
95
|
+
/** Resolves true once the multipart is open (or resumed and proven), false if it could not be. */
|
|
96
|
+
readonly ready: Promise<boolean>;
|
|
97
|
+
readonly state: GrowingState;
|
|
98
|
+
/** A saved record was found and resumed. Known once `ready` resolved. */
|
|
99
|
+
readonly resumed: boolean;
|
|
100
|
+
/** Parts an earlier run stored that this one proved (ListParts + the same bytes) and didn't send again. */
|
|
101
|
+
readonly kept: number;
|
|
102
|
+
/** Why the upload ended early (null while it hasn't). */
|
|
103
|
+
readonly failure: {
|
|
104
|
+
error: string;
|
|
105
|
+
code?: string;
|
|
106
|
+
retryable: boolean;
|
|
107
|
+
} | null;
|
|
108
|
+
/** One chunk, in order. Never throws and never blocks; ignored once the upload has failed or ended. */
|
|
109
|
+
append(chunk: Uint8Array | Blob): void;
|
|
110
|
+
/** For a reader of a finished local copy: resolves when the network has caught up a little. */
|
|
111
|
+
capacity(): Promise<void>;
|
|
112
|
+
/** The recording stopped: send the tail, prove and complete. The same promise on every call. */
|
|
113
|
+
end(): Promise<GrowingResult>;
|
|
114
|
+
/** Stop, abort the multipart and forget the record. */
|
|
115
|
+
abort(): Promise<void>;
|
|
116
|
+
/**
|
|
117
|
+
* Stop sending and keep everything for a resume: the multipart, the parts stored, the record (state "queued", this
|
|
118
|
+
* reason). Resolves once nothing in flight can touch the record any more. For when the local copy can't be read to
|
|
119
|
+
* its end: never call end() then, it would complete a truncated object.
|
|
120
|
+
*/
|
|
121
|
+
stop(reason?: string): Promise<void>;
|
|
122
|
+
progress(): GrowingProgress;
|
|
123
|
+
on(ev: "state", fn: (s: GrowingState) => void): () => void;
|
|
124
|
+
on(ev: "progress", fn: (p: GrowingProgress) => void): () => void;
|
|
125
|
+
}
|
|
126
|
+
export declare function createGrowingUpload(o: GrowingUploadOptions): GrowingUpload;
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The client transfer engine: resumable, verified multipart uploads and verified ranged downloads over presigned URLs.
|
|
3
|
+
* Node/Electron today (Buffer, node:fs); the network, the signer, the hasher and the saved state are swapped in.
|
|
4
|
+
*/
|
|
5
|
+
export { TRANSFER_TIMING_DEFAULTS, createTiming, realClock, clockOf, monoOf } from "./timing";
|
|
6
|
+
export type { Clock, TimerHandle, TransferTiming, TransferTimingName, TransferTimingValues } from "./timing";
|
|
7
|
+
export { StorageError, YIELDED, TRANSIENT_4XX, putError, toStorageError, duckErrors, isTransient, isYielded } from "./errors";
|
|
8
|
+
export type { ErrorAdapter, ErrorShape } from "./errors";
|
|
9
|
+
export type { Signer, Transport, Hasher, Yieldable, TransferOp, TransferResult, PutReceipt, ByteSource, SourceStamp, Sink, ClaimStore, ClaimRecord } from "./protocol";
|
|
10
|
+
export { receiptOf, headerOf } from "./protocol";
|
|
11
|
+
export { createStorage, stat, unwrap, b64ToHex } from "./storage";
|
|
12
|
+
export type { Storage, StorageOptions, Stat, Deduped, CreateAnswer, CheckedStat, ListedPart } from "./storage";
|
|
13
|
+
export { createElectronNetTransport, POOL_PER_HOST, electronKeepAwake } from "./transports/electronNet";
|
|
14
|
+
export type { ElectronNetTransport, NetLike, NetRequestLike, NetResponseLike, PowerSaveBlockerLike } from "./transports/electronNet";
|
|
15
|
+
export { parseRetryAfter } from "./transports/retryAfter";
|
|
16
|
+
export { createUndiciTransport } from "./transports/undici";
|
|
17
|
+
export type { UndiciLike, UndiciTransport, UndiciTransportOptions, PutEvent } from "./transports/undici";
|
|
18
|
+
export { createFetchTransport } from "./transports/fetch";
|
|
19
|
+
export type { FetchTransportOptions } from "./transports/fetch";
|
|
20
|
+
export { MIN_PART, SMALL, MAX_PARTS, partSizeFor, isHex32, compositeEtag, partsCrc, strongStat, digestOf, etagsOf, checkEtag, crcHex, toHex, SHA256_B64, isHex64, bytesToB64, hexToB64, sha256B64ToHex, parseSha256Checksum, compositeSha256 } from "./integrity";
|
|
21
|
+
export type { RangeReader, Md5Stream } from "./integrity";
|
|
22
|
+
export { createWorkerHasher, nodeHasher, nodeMd5, nodeSha256Hex, stampOf, sameStamp, sameContentStamp, sameSource, openFile, fileSource, bufferSource } from "./node";
|
|
23
|
+
export type { Stamp, WorkerHasherOptions } from "./node";
|
|
24
|
+
export { createEngineCore, laneRoom, laneShare, IntegrityError, ReceiptMissing, EXISTS_MSG, MISSING_MSG } from "./engine";
|
|
25
|
+
export type { EngineCore, EngineCoreOptions, EngineHost, UploadItem, ItemVersion, Multipart, BlobSend, LaneSpec, SourceCheck } from "./engine";
|
|
26
|
+
export { HttpError, VersionChanged, LocalError, normEtag as normHeaderEtag, covered, addRange, nextGap, RANGE_CHUNK, isTransientDownloadError, downloadBackoff, fetchRangeInto, keyedLock, checkFileEtag, withSuffix, NO_LINK, publishNoOverwrite, publishCopy, downloadObject, sinkWriter, createDownloadLimiter, downloadLimiter, } from "./download";
|
|
27
|
+
export type { Range, RangeFetch, RangeWriter, PublishOptions, DownloadState, DownloadObjectOptions, DownloadResult, DownloadLimiter } from "./download";
|
|
28
|
+
export { memoryClaims, kvClaims } from "./claims";
|
|
29
|
+
export type { ClaimKv } from "./claims";
|
|
30
|
+
export { createTuner, createHalvingTuner } from "./tuner";
|
|
31
|
+
export type { Tuner, HalvingTuner, TunerBounds } from "./tuner";
|
|
32
|
+
export { createPartPool, PoolCancelled } from "./pool";
|
|
33
|
+
export type { PartPool, PartPoolOptions, PoolLane } from "./pool";
|
|
34
|
+
export { createHttpSigner, SignerError } from "./signer";
|
|
35
|
+
export type { HttpSignerOptions } from "./signer";
|
|
36
|
+
export { createTransferEngine, memoryStore, fileStore } from "./client";
|
|
37
|
+
export type { ResumeStore, UploadRequest, UploadHandle, UploadResult, UploadState, TransferEngineOptions } from "./client";
|
|
38
|
+
export * from "./growing";
|
|
39
|
+
export { sanitizeLimits, sanitizeCap, resolvePartSize, MAX_PART_BYTES } from "./limits";
|
|
40
|
+
export type { ConnectionLimits, PartSizeHook } from "./limits";
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Part sizing and the checks that make "verified" mean something: per-part Content-MD5 (storage refuses changed
|
|
3
|
+
* bytes), the part ETag = MD5 of what was read, the multipart ETag = md5 of the part MD5s, the whole file's CRC-32
|
|
4
|
+
* combined from its parts', and the ETag a local file must have.
|
|
5
|
+
*
|
|
6
|
+
* Moved from UploadFilm's app/src/main/engine.ts (partSizeFor, compositeEtag, partsCrc, localDigest) and
|
|
7
|
+
* app/src/main/downloads.ts (checkEtag, etags).
|
|
8
|
+
*/
|
|
9
|
+
import { crcHex } from "../server/r2/crc32";
|
|
10
|
+
import type { Hasher } from "./protocol";
|
|
11
|
+
export { crcHex };
|
|
12
|
+
/** Smallest part (16 MiB). */
|
|
13
|
+
export declare const MIN_PART: number;
|
|
14
|
+
/** Objects under this go as one PUT, not a multipart upload. */
|
|
15
|
+
export declare const SMALL: number;
|
|
16
|
+
/** Most parts we cut an object into (headroom under S3's 10,000). */
|
|
17
|
+
export declare const MAX_PARTS = 9500;
|
|
18
|
+
/** Same as `@uploadfilm/sdk/server` partSizeFor with its defaults, without its range checks (never throws). */
|
|
19
|
+
export declare const partSizeFor: (size: number) => number;
|
|
20
|
+
export declare const isHex32: (s: string) => boolean;
|
|
21
|
+
export declare const toHex: (b: Uint8Array) => string;
|
|
22
|
+
/** S3/R2 multipart ETag: md5 of the concatenated binary part MD5s, then "-<parts>". null unless every part is an MD5. */
|
|
23
|
+
export declare function compositeEtag(hasher: Hasher, parts: Record<number, string>): string | null;
|
|
24
|
+
/**
|
|
25
|
+
* The whole file's CRC-32 from its parts' (crc32_combine, in part order), or undefined unless every part has
|
|
26
|
+
* one. Only called with a complete, all-MD5 parts list (the stitch), so the CRCs describe exactly the bytes
|
|
27
|
+
* whose MD5s storage confirmed.
|
|
28
|
+
*/
|
|
29
|
+
export declare function partsCrc(mp: {
|
|
30
|
+
partSize: number;
|
|
31
|
+
parts: Record<number, string>;
|
|
32
|
+
crcs?: Record<number, number>;
|
|
33
|
+
}, size: number): string | undefined;
|
|
34
|
+
/**
|
|
35
|
+
* R2 answers HEAD (and so the server's complete/head) of text-like objects over ~1 KiB with a WEAK ETag
|
|
36
|
+
* W/"…" because it may compress them in transit; the value is still the stored object's ETag. A weak prefix is
|
|
37
|
+
* not a difference: compare what's behind it.
|
|
38
|
+
*/
|
|
39
|
+
export declare const strongStat: <S extends {
|
|
40
|
+
etag: string | null;
|
|
41
|
+
}>(st: S) => S;
|
|
42
|
+
/** Reads `len` bytes at `pos` into `buf` (a FileHandle's read). */
|
|
43
|
+
export interface RangeReader {
|
|
44
|
+
read(buf: Uint8Array, off: number, len: number, pos: number): Promise<{
|
|
45
|
+
bytesRead: number;
|
|
46
|
+
}>;
|
|
47
|
+
close(): Promise<void>;
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* The ETag storage computes for `size` bytes read through `r`: MD5 for one PUT (under SMALL), md5(part MD5s)-N for a
|
|
51
|
+
* multipart upload with partSizeFor's part size (or `partSize`); plus the CRC-32 (8 hex), from the same single read.
|
|
52
|
+
*/
|
|
53
|
+
export declare function digestOf(hasher: Hasher, r: RangeReader, size: number, partSize?: number): Promise<{
|
|
54
|
+
etag: string;
|
|
55
|
+
crc32: string;
|
|
56
|
+
}>;
|
|
57
|
+
/** An incremental MD5 (node:crypto createHash("md5") fits). */
|
|
58
|
+
export interface Md5Stream {
|
|
59
|
+
update(b: Uint8Array): unknown;
|
|
60
|
+
digest(): Uint8Array;
|
|
61
|
+
}
|
|
62
|
+
/**
|
|
63
|
+
* MD5 of the bytes (no part sizes), or the multipart ETag hex for each part size, in one read of `size` bytes.
|
|
64
|
+
* `yieldEvery`: awaited between 8 MiB reads (let a UI breathe on huge files).
|
|
65
|
+
*/
|
|
66
|
+
export declare function etagsOf(newMd5: () => Md5Stream, hasher: Hasher, r: Pick<RangeReader, "read">, size: number, sizes: number[], parts?: number, yieldEvery?: () => Promise<unknown>): Promise<string[]>;
|
|
67
|
+
/**
|
|
68
|
+
* Does `size` bytes read through `r` have this S3 ETag? true / false, or null when it can't be reproduced (multipart
|
|
69
|
+
* with a part size we can't find). The uploader's part-size rule first (one read); only if that isn't it, the
|
|
70
|
+
* power-of-two MiB sizes and the first 12 whole MiB that give the same part count (a tuned or an older uploader), in
|
|
71
|
+
* one more read.
|
|
72
|
+
*/
|
|
73
|
+
export declare function checkEtag(newMd5: () => Md5Stream, hasher: Hasher, open: () => Promise<Pick<RangeReader, "read"> & {
|
|
74
|
+
close?: () => Promise<void>;
|
|
75
|
+
}>, size: number, etag: string, yieldEvery?: () => Promise<unknown>): Promise<boolean | null>;
|
|
76
|
+
/** A base64 SHA-256 exactly as S3/R2 send it in `x-amz-checksum-sha256` (44 chars, one "=" of padding). */
|
|
77
|
+
export declare const SHA256_B64: RegExp;
|
|
78
|
+
export declare const isHex64: (s: string) => boolean;
|
|
79
|
+
/** Bytes → base64 (no Buffer: any runtime). */
|
|
80
|
+
export declare function bytesToB64(b: Uint8Array): string;
|
|
81
|
+
export declare const hexToB64: (h: string) => string;
|
|
82
|
+
/** A strict base64 SHA-256 → hex, or null (missing, malformed, repeated "a, b", wrong length). */
|
|
83
|
+
export declare function sha256B64ToHex(v: string | undefined | null): string | null;
|
|
84
|
+
/**
|
|
85
|
+
* A checksum a server reported for a stored object: "<b64>" (one PUT: the object's SHA-256) or "<b64>-<parts>" (S3's
|
|
86
|
+
* COMPOSITE multipart checksum: sha256 of the concatenated part digests). → { hex, parts } or null when malformed.
|
|
87
|
+
*/
|
|
88
|
+
export declare function parseSha256Checksum(v: string | undefined | null): {
|
|
89
|
+
hex: string;
|
|
90
|
+
parts?: number;
|
|
91
|
+
} | null;
|
|
92
|
+
/** S3 COMPOSITE SHA-256 of a multipart upload: sha256 of the concatenated binary part SHA-256s, "-<parts>". null unless every part has one. */
|
|
93
|
+
export declare function compositeSha256(sha256: (b: Uint8Array) => Uint8Array, parts: Record<number, string>, count: number): string | null;
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
/** S3's (and R2's) largest part. */
|
|
2
|
+
export declare const MAX_PART_BYTES: number;
|
|
3
|
+
export interface ConnectionLimits {
|
|
4
|
+
min: number;
|
|
5
|
+
max: number;
|
|
6
|
+
/** Where the tuner starts; only before its first step (later it climbs from wherever it is). */
|
|
7
|
+
start?: number;
|
|
8
|
+
}
|
|
9
|
+
/** Part size for a multipart of `size` bytes created now. */
|
|
10
|
+
export type PartSizeHook = (size: number) => number;
|
|
11
|
+
/**
|
|
12
|
+
* Whole numbers, 1 ≤ min ≤ (start) ≤ max. A max that isn't a number ≥ 1 is refused (RangeError): the engine never
|
|
13
|
+
* runs on a limit it made up. min is clamped into 1..max; start into min..max.
|
|
14
|
+
*/
|
|
15
|
+
export declare function sanitizeLimits(l: ConnectionLimits): {
|
|
16
|
+
min: number;
|
|
17
|
+
max: number;
|
|
18
|
+
start?: number;
|
|
19
|
+
};
|
|
20
|
+
/** The in-flight byte cap: a positive number of bytes, or null for none (null, undefined, Infinity, 0 or less, NaN). */
|
|
21
|
+
export declare function sanitizeCap(n: number | null | undefined): number | null;
|
|
22
|
+
/**
|
|
23
|
+
* The part size a new multipart of `size` bytes gets: the default rule (16 MiB, or more so it fits in 9,500 parts)
|
|
24
|
+
* unless the hook asks for bigger; rounded up to a power-of-two MiB (16 MiB .. 4 GiB; never below the rule, so the part
|
|
25
|
+
* count stays within storage's limit). A hook that throws or answers anything but a positive number: the rule.
|
|
26
|
+
*/
|
|
27
|
+
export declare function resolvePartSize(size: number, hook: PartSizeHook | null | undefined): number;
|
|
28
|
+
/** Part sizes a hook resolves to: 16 MiB × 2^k up to 4 GiB (whole MiB, ≤ 5 GiB). */
|
|
29
|
+
export declare const POW2_PARTS: readonly number[];
|
|
30
|
+
/**
|
|
31
|
+
* The power-of-two MiB at or above `want` (16 MiB .. 4 GiB). Tuned part sizes are powers of two so anyone checking a
|
|
32
|
+
* stored ETag can find the layout without knowing the uploader's settings (checkEtag tries these first; adoption too).
|
|
33
|
+
*/
|
|
34
|
+
export declare function pow2Part(want: number): number;
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
import { Worker } from "node:worker_threads";
|
|
2
|
+
import type { Md5Stream } from "./integrity";
|
|
3
|
+
import type { ByteSource, Hasher, SourceStamp } from "./protocol";
|
|
4
|
+
export declare const nodeHasher: Hasher;
|
|
5
|
+
export declare const nodeMd5: () => Md5Stream;
|
|
6
|
+
export interface WorkerHasherOptions {
|
|
7
|
+
/** Worker threads (default 2: perf-lab's measured setting). */
|
|
8
|
+
workers?: number;
|
|
9
|
+
/** Called with each worker thread started (diagnostics and tests: a test kills one to prove the failover). */
|
|
10
|
+
onSpawn?(w: Worker): void;
|
|
11
|
+
}
|
|
12
|
+
/**
|
|
13
|
+
* nodeHasher with parts' MD5 + CRC-32 in worker threads (perf-lab finding 2, docs/platform/TRANSFER-PERF.md): the
|
|
14
|
+
* engine's main thread (Electron's main process in a Mac app) no longer spends ~1.1 s per GB in 25 ms blocks hashing.
|
|
15
|
+
* Same digests as nodeHasher (property-tested). Workers start on the first part and are unref'd (they never keep a
|
|
16
|
+
* process alive); `close()` terminates them (the engine's stop() calls it) and the next part starts them again. A
|
|
17
|
+
* worker that errors or exits fails its parts' digests (the engine then hashes those on its own thread) and is
|
|
18
|
+
* replaced on the next part; after 3 such losses the hasher stays on the main thread (`degraded`).
|
|
19
|
+
*/
|
|
20
|
+
export declare function createWorkerHasher(o?: WorkerHasherOptions): Hasher & {
|
|
21
|
+
readonly degraded: boolean;
|
|
22
|
+
close(): Promise<void>;
|
|
23
|
+
};
|
|
24
|
+
export declare const nodeSha256Hex: (b: Uint8Array) => string;
|
|
25
|
+
/**
|
|
26
|
+
* The file as it is now: device, inode, size, mtime. Two stamps of the same file are equal (sameStamp); a card pulled
|
|
27
|
+
* and another mounted at the same path (same name, size, even mtime) is a different device/inode.
|
|
28
|
+
*/
|
|
29
|
+
export type Stamp = {
|
|
30
|
+
dev: number;
|
|
31
|
+
ino: number;
|
|
32
|
+
size: number;
|
|
33
|
+
mtimeMs: number;
|
|
34
|
+
ctimeMs?: number;
|
|
35
|
+
birthtimeMs?: number;
|
|
36
|
+
};
|
|
37
|
+
export declare function stampOf(src: string): Stamp | null;
|
|
38
|
+
/**
|
|
39
|
+
* Same file (device, inode, size, mtime, and birth time when both stamps have one: another card with the same names,
|
|
40
|
+
* size and mtime in the same reader can get the same device number and inode, but not the same birth time). Takes any
|
|
41
|
+
* SourceStamp (hosts get them in callbacks); only files ever match. A chmod or an xattr doesn't make it another file.
|
|
42
|
+
*/
|
|
43
|
+
export declare const sameStamp: (a: SourceStamp | null | undefined, b: SourceStamp | null | undefined) => boolean;
|
|
44
|
+
/**
|
|
45
|
+
* sameStamp and the same ctime (when both have one): the engine's check that the bytes it read are still the file's
|
|
46
|
+
* (RL-2: a rewrite that put the mtime back keeps dev/inode/size/mtime, not ctime). A metadata-only change (chmod,
|
|
47
|
+
* xattr) fails it too; the engine then proves the content again rather than trusting it.
|
|
48
|
+
*/
|
|
49
|
+
export declare const sameContentStamp: (a: SourceStamp | null | undefined, b: SourceStamp | null | undefined) => boolean;
|
|
50
|
+
/** fs/promises `open`, looked up per call (so a test or product may wrap it). */
|
|
51
|
+
export declare const openFile: (path: string, flags?: string) => Promise<import("node:fs/promises").FileHandle>;
|
|
52
|
+
/** Two source stamps describe the same source: files by device/inode/size/mtime, other sources by id/size(/mtime). */
|
|
53
|
+
export declare function sameSource(a: SourceStamp | null | undefined, b: SourceStamp | null | undefined): boolean;
|
|
54
|
+
/**
|
|
55
|
+
* A local file as a ByteSource (identity = device/inode/size/mtime). The engine reads plain paths itself (the same
|
|
56
|
+
* calls as before ByteSource existed); this is for code that wants a ByteSource for a file, e.g. to mix files and
|
|
57
|
+
* buffers behind one interface. `size` is the file's size when it was opened.
|
|
58
|
+
*/
|
|
59
|
+
export declare function fileSource(path: string): ByteSource;
|
|
60
|
+
/** Bytes in memory as a ByteSource (tests, generated objects). `id` defaults to the bytes' SHA-256. */
|
|
61
|
+
export declare function bufferSource(bytes: Uint8Array, id?: string): ByteSource;
|
|
62
|
+
/** sameSource with sameContentStamp for files (the engine's content checks: a rewrite with its mtime put back fails). */
|
|
63
|
+
export declare function sameContentSource(a: SourceStamp | null | undefined, b: SourceStamp | null | undefined): boolean;
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The part pool as a standalone API (gap 5): one pool of connections shared by every transfer of a process, split
|
|
3
|
+
* between lanes, sized by a tuner. Jobs are functions; the pool only decides when each one starts.
|
|
4
|
+
*
|
|
5
|
+
* const pool = createPartPool({
|
|
6
|
+
* lanes: [{ name: "poster", first: true }, { name: "original", minShare: 2 / 3 }, { name: "proxy", yieldable: true }],
|
|
7
|
+
* connections: { min: 4, start: 8, max: 32 },
|
|
8
|
+
* socketsPerHost: transport.socketsPerHost,
|
|
9
|
+
* });
|
|
10
|
+
* await pool.run("original", signal, (y) => transport.put(url, headers, body, onProgress, timeoutMs, y));
|
|
11
|
+
* pool.sample(bytes); // feed the tuner
|
|
12
|
+
*
|
|
13
|
+
* Lane rules (the same as the engine core's, through the same laneShare()):
|
|
14
|
+
* - `first` lanes start before anything else;
|
|
15
|
+
* - the primary lane (`minShare`) keeps at least ⌈minShare × sockets⌉ of the connections while it has jobs queued
|
|
16
|
+
* (AGENTS.md: originals keep ⌈2/3⌉);
|
|
17
|
+
* - borrower lanes run in what the primary leaves; while it has nothing queued but jobs running, at most
|
|
18
|
+
* ⌊running × (1 − minShare) / minShare⌋ (laneShare: half as many as the primary's at 2/3), and every idle connection
|
|
19
|
+
* once it has neither;
|
|
20
|
+
* `yieldable` borrowers hand their place back: a job that passed the Yieldable it was given to its transport is
|
|
21
|
+
* cancelled while still queued in the stack (`cancelIfQueued`), rejects with YIELDED, and is queued again by the pool;
|
|
22
|
+
* - `maxActive`: at most this many jobs of the lane at once.
|
|
23
|
+
* Or strict priorities (cloud-transfer's CONTEXT > SMALL > ATTACHMENT): lanes with `priority` (lower first) start in
|
|
24
|
+
* that order, nothing reserved.
|
|
25
|
+
*
|
|
26
|
+
* A queued job consumes nothing: its function isn't called (no request timeout or retry budget runs) until it starts.
|
|
27
|
+
* Aborting a queued job removes it from the queue at once (rejects with an AbortError-like `Cancelled`).
|
|
28
|
+
*
|
|
29
|
+
* Tuners (policy): "uploadfilm" (createTuner: slow start then ±2 hill climbing every TUNE_INTERVAL_MS, ×0.7 on error
|
|
30
|
+
* bursts) or "halve-on-error" (createHalvingTuner: ±1 every tuneIntervalMs while work is queued, halve on every
|
|
31
|
+
* retryable error). `fail()` reports an error to the tuner; `sample(bytes)` reports throughput.
|
|
32
|
+
*/
|
|
33
|
+
import { StorageError } from "./errors";
|
|
34
|
+
import { type LaneSpec } from "./engine";
|
|
35
|
+
import type { Yieldable } from "./protocol";
|
|
36
|
+
import { type TransferTiming } from "./timing";
|
|
37
|
+
export interface PoolLane extends LaneSpec {
|
|
38
|
+
/** Strict order: lanes with a priority start lowest first (and reserve nothing). */
|
|
39
|
+
priority?: number;
|
|
40
|
+
}
|
|
41
|
+
export interface PartPoolOptions {
|
|
42
|
+
lanes: PoolLane[];
|
|
43
|
+
connections?: {
|
|
44
|
+
min?: number;
|
|
45
|
+
start?: number;
|
|
46
|
+
max?: number;
|
|
47
|
+
};
|
|
48
|
+
/** The transport's cap per host (Electron's net: 6): the pool never runs more than this at once. */
|
|
49
|
+
socketsPerHost?: number;
|
|
50
|
+
policy?: "uploadfilm" | "halve-on-error";
|
|
51
|
+
/** halve-on-error: how often the tuner moves (default 2 s) and how long an error holds it (default 5 s). */
|
|
52
|
+
tuneIntervalMs?: number;
|
|
53
|
+
errorCooldownMs?: number;
|
|
54
|
+
/** uploadfilm policy: its timing table and clock (default createTiming()). */
|
|
55
|
+
timing?: TransferTiming;
|
|
56
|
+
}
|
|
57
|
+
export declare class PoolCancelled extends Error {
|
|
58
|
+
readonly code = "Cancelled";
|
|
59
|
+
constructor();
|
|
60
|
+
}
|
|
61
|
+
export declare function createPartPool(o: PartPoolOptions): {
|
|
62
|
+
/** Run `job` in `lane` when the pool has room for it. `job` gets a Yieldable to pass to its transport. */
|
|
63
|
+
run<T>(lane: string, signal: AbortSignal | undefined, job: (y: Yieldable) => Promise<T>): Promise<T>;
|
|
64
|
+
/** `bytes` more went out (feeds the tuner). */
|
|
65
|
+
sample(bytes: number, at?: number): void;
|
|
66
|
+
/** A job failed with a retryable error. */
|
|
67
|
+
fail(): void;
|
|
68
|
+
/** uploadfilm policy: measure the rate (every ENGINE_TICK_MS) and tune (every TUNE_INTERVAL_MS). */
|
|
69
|
+
start(): void;
|
|
70
|
+
stop(): void;
|
|
71
|
+
/** The tuner's target (before the per-host cap). */
|
|
72
|
+
readonly connections: number;
|
|
73
|
+
readonly active: number;
|
|
74
|
+
readonly queued: number;
|
|
75
|
+
activeIn: (name: string) => number;
|
|
76
|
+
/** Jobs of a lane waiting to start. */
|
|
77
|
+
queuedIn: (name: string) => number;
|
|
78
|
+
/** Hold the target (tests; uploadfilm policy only). */
|
|
79
|
+
pin(n: number | null): void;
|
|
80
|
+
};
|
|
81
|
+
export type PartPool = ReturnType<typeof createPartPool>;
|
|
82
|
+
/** A job's transport answered "gave its place back" (re-exported for job authors). */
|
|
83
|
+
export declare const yielded: () => StorageError;
|