@uploadfilm/sdk 0.0.0-stage → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +4 -0
- package/README.md +529 -2
- package/dist/ai/embed.d.ts +58 -0
- package/dist/ai/fake.d.ts +41 -0
- package/dist/ai/http.d.ts +17 -0
- package/dist/ai/index.d.ts +27 -0
- package/dist/ai/languages.d.ts +2 -0
- package/dist/ai/ollama.d.ts +23 -0
- package/dist/ai/openaiCompatible.d.ts +54 -0
- package/dist/ai/passages.d.ts +28 -0
- package/dist/ai/transcribe.d.ts +84 -0
- package/dist/ai/types.d.ts +141 -0
- package/dist/ai/usage.d.ts +66 -0
- package/dist/ai/whisperJson.d.ts +15 -0
- package/dist/ai/workersAi.d.ts +56 -0
- package/dist/ai.js +1 -0
- package/dist/ai.mjs +1 -0
- package/dist/cards/files.d.ts +91 -0
- package/dist/cards/index.d.ts +15 -0
- package/dist/cards/layouts.d.ts +121 -0
- package/dist/cards/probe.d.ts +69 -0
- package/dist/cards/proxies.d.ts +70 -0
- package/dist/cards/select.d.ts +68 -0
- package/dist/cards/volumes.d.ts +170 -0
- package/dist/cards/xml.d.ts +80 -0
- package/dist/cards.js +1 -0
- package/dist/cards.mjs +1 -0
- package/dist/copy/checksums.d.ts +66 -0
- package/dist/copy/engine.d.ts +24 -0
- package/dist/copy/errors.d.ts +109 -0
- package/dist/copy/index.d.ts +15 -0
- package/dist/copy/steps.d.ts +28 -0
- package/dist/copy/sweep.d.ts +24 -0
- package/dist/copy/types.d.ts +363 -0
- package/dist/embed/player.js +2 -0
- package/dist/host/adapters.d.ts +107 -0
- package/dist/host/detect.d.ts +68 -0
- package/dist/host/encoders.d.ts +34 -0
- package/dist/host/environment.d.ts +53 -0
- package/dist/host/index.d.ts +20 -0
- package/dist/host/linux.d.ts +94 -0
- package/dist/host/mac.d.ts +47 -0
- package/dist/host/network.d.ts +54 -0
- package/dist/host/node.d.ts +13 -0
- package/dist/host/policy.d.ts +101 -0
- package/dist/host/tuner.d.ts +48 -0
- package/dist/host/types.d.ts +118 -0
- package/dist/host-node.js +1 -0
- package/dist/host-node.mjs +1 -0
- package/dist/host.js +1 -0
- package/dist/host.mjs +1 -0
- package/dist/images/index.d.ts +11 -0
- package/dist/images/settings.d.ts +43 -0
- package/dist/images/types.d.ts +83 -0
- package/dist/images/url.d.ts +65 -0
- package/dist/index.d.ts +16 -0
- package/dist/index.js +1 -0
- package/dist/index.mjs +1 -0
- package/dist/player/adapters/hlsjs.d.ts +28 -0
- package/dist/player/bandwidthMemory.d.ts +28 -0
- package/dist/player/captions/index.d.ts +55 -0
- package/dist/player/embed/code.d.ts +195 -0
- package/dist/player/embed/constants.d.ts +4 -0
- package/dist/player/embed/controller.d.ts +62 -0
- package/dist/player/embed/element.d.ts +32 -0
- package/dist/player/embed/frame.d.ts +65 -0
- package/dist/player/embed/global.d.ts +14 -0
- package/dist/player/embed/index.d.ts +15 -0
- package/dist/player/embed/loader.d.ts +1 -0
- package/dist/player/embed/oembed.d.ts +74 -0
- package/dist/player/embed/protocol.d.ts +64 -0
- package/dist/player/engine.d.ts +215 -0
- package/dist/player/index.d.ts +18 -0
- package/dist/player/start.d.ts +62 -0
- package/dist/player/store.d.ts +10 -0
- package/dist/player/storyboard/index.d.ts +161 -0
- package/dist/player/stream.d.ts +206 -0
- package/dist/player/tracks.d.ts +41 -0
- package/dist/player.js +1 -0
- package/dist/player.mjs +1 -0
- package/dist/proxies/args.d.ts +19 -0
- package/dist/proxies/budget.d.ts +57 -0
- package/dist/proxies/camera.d.ts +104 -0
- package/dist/proxies/formats.d.ts +36 -0
- package/dist/proxies/index.d.ts +18 -0
- package/dist/proxies/jobs.d.ts +98 -0
- package/dist/proxies/node.d.ts +32 -0
- package/dist/proxies/plan.d.ts +89 -0
- package/dist/proxies/probe.d.ts +39 -0
- package/dist/proxies/render.d.ts +104 -0
- package/dist/proxies/safety.d.ts +45 -0
- package/dist/proxies/types.d.ts +83 -0
- package/dist/proxies/upload.d.ts +14 -0
- package/dist/react/BigPlay.d.ts +26 -0
- package/dist/react/Player.d.ts +58 -0
- package/dist/react/embed/index.d.ts +45 -0
- package/dist/react/examples/CustomControls.d.ts +8 -0
- package/dist/react/images/Image.d.ts +16 -0
- package/dist/react/images/index.d.ts +4 -0
- package/dist/react/images/useImage.d.ts +47 -0
- package/dist/react/index.d.ts +17 -0
- package/dist/react/storyboard/index.d.ts +58 -0
- package/dist/react/theme.d.ts +37 -0
- package/dist/react/useAdapterErrors.d.ts +8 -0
- package/dist/react/usePlaybackStart.d.ts +18 -0
- package/dist/react/usePlayer.d.ts +28 -0
- package/dist/react/useReducedMotion.d.ts +2 -0
- package/dist/react/useTimedTrack.d.ts +22 -0
- package/dist/react.js +1 -0
- package/dist/react.mjs +1 -0
- package/dist/server/auth/apiKeys.d.ts +30 -0
- package/dist/server/auth/cache.d.ts +21 -0
- package/dist/server/auth/deviceTokens.d.ts +38 -0
- package/dist/server/auth/envelope.d.ts +15 -0
- package/dist/server/auth/grants.d.ts +76 -0
- package/dist/server/auth/index.d.ts +19 -0
- package/dist/server/auth/jwt.d.ts +65 -0
- package/dist/server/auth/keyring.d.ts +31 -0
- package/dist/server/auth/kv.d.ts +12 -0
- package/dist/server/auth/repository.d.ts +45 -0
- package/dist/server/auth/revocations.d.ts +21 -0
- package/dist/server/auth/scope.d.ts +4 -0
- package/dist/server/auth/serviceAuth.d.ts +24 -0
- package/dist/server/auth/signedUrls.d.ts +28 -0
- package/dist/server/embed/index.d.ts +36 -0
- package/dist/server/embed/lifecycle.d.ts +53 -0
- package/dist/server/embed/playerScript.d.ts +4 -0
- package/dist/server/images/index.d.ts +15 -0
- package/dist/server/images/job.d.ts +80 -0
- package/dist/server/images/layout.d.ts +24 -0
- package/dist/server/images/params.d.ts +90 -0
- package/dist/server/images/route.d.ts +101 -0
- package/dist/server/images/sign.d.ts +55 -0
- package/dist/server/index.d.ts +42 -0
- package/dist/server/media/client.d.ts +99 -0
- package/dist/server/media/frameCheck.d.ts +62 -0
- package/dist/server/media/index.d.ts +10 -0
- package/dist/server/media/protocol.d.ts +556 -0
- package/dist/server/media/storyboard.d.ts +144 -0
- package/dist/server/media/transcript.d.ts +143 -0
- package/dist/server/pipeline/index.d.ts +12 -0
- package/dist/server/pipeline/match.d.ts +10 -0
- package/dist/server/pipeline/pipeline.d.ts +182 -0
- package/dist/server/pipeline/queues.d.ts +108 -0
- package/dist/server/pipeline/sources.d.ts +54 -0
- package/dist/server/r2/client.d.ts +164 -0
- package/dist/server/r2/crc32.d.ts +24 -0
- package/dist/server/r2/encoding.d.ts +35 -0
- package/dist/server/r2/errors.d.ts +9 -0
- package/dist/server/r2/integrity.d.ts +25 -0
- package/dist/server/r2/parts.d.ts +46 -0
- package/dist/server/r2/sign.d.ts +47 -0
- package/dist/server/redis/index.d.ts +39 -0
- package/dist/server/state/memory.d.ts +18 -0
- package/dist/server/state/r2.d.ts +73 -0
- package/dist/server/state/types.d.ts +57 -0
- package/dist/server/storage/chunked.d.ts +25 -0
- package/dist/server/storage/index.d.ts +12 -0
- package/dist/server/storage/r2.d.ts +96 -0
- package/dist/server/storage/s3.d.ts +144 -0
- package/dist/server/testing/fake-r2.d.ts +410 -0
- package/dist/server/testing/index.d.ts +10 -0
- package/dist/server/testing/kv-contract.d.ts +14 -0
- package/dist/server/transfer/handler.d.ts +217 -0
- package/dist/server/transfer/keys.d.ts +45 -0
- package/dist/server/transfer/protocol.d.ts +190 -0
- package/dist/server/zip/client.d.ts +32 -0
- package/dist/server/zip/generations.d.ts +83 -0
- package/dist/server/zip/index.d.ts +12 -0
- package/dist/server/zip/planner.d.ts +94 -0
- package/dist/server/zip/sign.d.ts +22 -0
- package/dist/server-media.js +1 -0
- package/dist/server-media.mjs +1 -0
- package/dist/server-redis.js +1 -0
- package/dist/server-redis.mjs +1 -0
- package/dist/server-testing.js +1 -0
- package/dist/server-testing.mjs +1 -0
- package/dist/server.js +1 -0
- package/dist/server.mjs +1 -0
- package/dist/storage/browserFile.d.ts +19 -0
- package/dist/storage/index.d.ts +12 -0
- package/dist/storage/local.d.ts +9 -0
- package/dist/storage/presigned.d.ts +22 -0
- package/dist/sync/ffmpeg.d.ts +44 -0
- package/dist/sync/index.d.ts +17 -0
- package/dist/sync/match.d.ts +28 -0
- package/dist/sync/probe.d.ts +30 -0
- package/dist/sync/timecode.d.ts +50 -0
- package/dist/sync/transcript.d.ts +35 -0
- package/dist/sync/types.d.ts +172 -0
- package/dist/sync/waveform.d.ts +89 -0
- package/dist/sync.js +1 -0
- package/dist/sync.mjs +1 -0
- package/dist/transfer/claims.d.ts +41 -0
- package/dist/transfer/client.d.ts +267 -0
- package/dist/transfer/download.d.ts +192 -0
- package/dist/transfer/engine.d.ts +402 -0
- package/dist/transfer/errors.d.ts +50 -0
- package/dist/transfer/growing/bytes.d.ts +17 -0
- package/dist/transfer/growing/index.d.ts +12 -0
- package/dist/transfer/growing/parts.d.ts +157 -0
- package/dist/transfer/growing/retry.d.ts +38 -0
- package/dist/transfer/growing/store.d.ts +66 -0
- package/dist/transfer/growing/upload.d.ts +126 -0
- package/dist/transfer/index.d.ts +40 -0
- package/dist/transfer/integrity.d.ts +93 -0
- package/dist/transfer/limits.d.ts +34 -0
- package/dist/transfer/node.d.ts +63 -0
- package/dist/transfer/pool.d.ts +83 -0
- package/dist/transfer/protocol.d.ts +172 -0
- package/dist/transfer/signer.d.ts +29 -0
- package/dist/transfer/storage.d.ts +135 -0
- package/dist/transfer/timing.d.ts +132 -0
- package/dist/transfer/transports/electronNet.d.ts +62 -0
- package/dist/transfer/transports/fetch.d.ts +15 -0
- package/dist/transfer/transports/retryAfter.d.ts +12 -0
- package/dist/transfer/transports/undici.d.ts +72 -0
- package/dist/transfer/tuner.d.ts +56 -0
- package/dist/workers/cli.mjs +115 -0
- package/dist/workers/media/config.mjs +53 -0
- package/dist/workers/media/container/Dockerfile +19 -0
- package/dist/workers/media/container/FFMPEG.md +129 -0
- package/dist/workers/media/container/runner.mjs +2690 -0
- package/dist/workers/media/index.mjs +1011 -0
- package/dist/workers/media/init/config.ts +52 -0
- package/dist/workers/media/init/worker.ts +8 -0
- package/dist/workers/media/init/wrangler.toml +44 -0
- package/dist/workers/media/runner.mjs +2656 -0
- package/dist/workers/media/types/build/entry/config.d.mts +6 -0
- package/dist/workers/media/types/build/entry/index.d.mts +43 -0
- package/dist/workers/media/types/build/entry/program.d.mts +13 -0
- package/dist/workers/media/types/build/entry/runner.d.mts +12 -0
- package/dist/workers/media/types/build/entry/settings.d.mts +40 -0
- package/dist/workers/media/types/src/config.d.mts +135 -0
- package/dist/workers/media/types/src/jobs.d.mts +73 -0
- package/dist/workers/media/types/src/runner/container.d.mts +32 -0
- package/dist/workers/media/types/src/runner/ffmpeg.d.mts +227 -0
- package/dist/workers/media/types/src/runner/job.d.mts +107 -0
- package/dist/workers/media/types/src/runner/node.d.mts +71 -0
- package/dist/workers/media/types/src/runner/plugins.d.mts +45 -0
- package/dist/workers/media/types/src/serve.d.mts +107 -0
- package/dist/workers/media/types/src/steps/audio.d.mts +33 -0
- package/dist/workers/media/types/src/steps/edit.d.mts +62 -0
- package/dist/workers/media/types/src/steps/frameCheck.d.mts +44 -0
- package/dist/workers/media/types/src/steps/gate.d.mts +18 -0
- package/dist/workers/media/types/src/steps/paths.d.mts +1 -0
- package/dist/workers/media/types/src/steps/prep.d.mts +60 -0
- package/dist/workers/media/types/src/steps/render.d.mts +40 -0
- package/dist/workers/media/types/src/steps/runner.d.mts +30 -0
- package/dist/workers/media/types/src/steps/transcribe.d.mts +89 -0
- package/dist/workers/media/types/src/storyboard/index.d.mts +61 -0
- package/dist/workers/media/types/src/transcoder.d.mts +30 -0
- package/dist/workers/media/types/src/worker.d.mts +30 -0
- package/dist/workers/zip/index.mjs +589 -0
- package/dist/workers/zip/init/config.ts +22 -0
- package/dist/workers/zip/init/worker.ts +5 -0
- package/dist/workers/zip/init/wrangler.toml +25 -0
- package/dist/workers/zip/types/build/entry/index.d.mts +5 -0
- package/dist/workers/zip/types/src/config.d.mts +35 -0
- package/dist/workers/zip/types/src/serve.d.mts +86 -0
- package/dist/workers/zip/types/src/worker.d.mts +12 -0
- package/dist/zip/index.d.ts +13 -0
- package/dist/zip/layout.d.ts +129 -0
- package/dist/zip/names.d.ts +34 -0
- package/dist/zip/plan.d.ts +115 -0
- package/dist/zip/sw.d.ts +7 -0
- package/package.json +313 -4
|
@@ -0,0 +1,267 @@
|
|
|
1
|
+
import { type EngineCore, type EngineCoreOptions, type ItemVersion, type LaneSpec, type Multipart, type UploadItem } from "./engine";
|
|
2
|
+
import type { ErrorAdapter } from "./errors";
|
|
3
|
+
import { type Stamp } from "./node";
|
|
4
|
+
import type { ByteSource, ClaimStore, Hasher, Signer, SourceStamp, Transport } from "./protocol";
|
|
5
|
+
import { type TransferTiming } from "./timing";
|
|
6
|
+
/** Saved state, one JSON value per key. Writes are synchronous so a record is on disk when the engine moves on. */
|
|
7
|
+
export interface ResumeStore {
|
|
8
|
+
get(key: string): unknown | undefined;
|
|
9
|
+
set(key: string, value: unknown): void;
|
|
10
|
+
del(key: string): void;
|
|
11
|
+
keys(): string[];
|
|
12
|
+
}
|
|
13
|
+
export declare function memoryStore(): ResumeStore;
|
|
14
|
+
/**
|
|
15
|
+
* One JSON file per key under `dir`, written to a temp name, fsynced and renamed (a crash or power loss leaves the old
|
|
16
|
+
* or the new record, never an empty one: RL-9). A key whose base64url name would pass the filesystem's 255-byte limit
|
|
17
|
+
* (Reville gap 5: long project/key ids) gets a file named after its SHA-256, holding the key next to the value.
|
|
18
|
+
*/
|
|
19
|
+
export declare function fileStore(dir: string): ResumeStore;
|
|
20
|
+
/** `waiting`: another uploader holds this content's claim (claims mode). */
|
|
21
|
+
export type UploadState = "queued" | "waiting" | "sending" | "verified" | "failed";
|
|
22
|
+
/**
|
|
23
|
+
* `sha256`: the object's SHA-256 (hex), when proven (single PUT receipt; a multipart's only when the source's whole hash
|
|
24
|
+
* is known and still valid). `sha256Composite`: storage's multipart composite ("hex-N"). Failures: `error` never carries
|
|
25
|
+
* a URL; `retryable`: calling upload() again can help.
|
|
26
|
+
*/
|
|
27
|
+
export type UploadResult = {
|
|
28
|
+
state: "verified";
|
|
29
|
+
key: string;
|
|
30
|
+
size: number;
|
|
31
|
+
etag: string;
|
|
32
|
+
crc32?: string;
|
|
33
|
+
sha256?: string;
|
|
34
|
+
sha256Composite?: string;
|
|
35
|
+
deduped?: boolean;
|
|
36
|
+
} | {
|
|
37
|
+
state: "failed";
|
|
38
|
+
key: string;
|
|
39
|
+
error: string;
|
|
40
|
+
code?: string;
|
|
41
|
+
retryable?: boolean;
|
|
42
|
+
};
|
|
43
|
+
export interface UploadRequest {
|
|
44
|
+
/** Storage key (one of the server's layouts for the scope). */
|
|
45
|
+
key: string;
|
|
46
|
+
/** Local file (or give `source`). */
|
|
47
|
+
path?: string;
|
|
48
|
+
/** The bytes as a ByteSource (a buffer, a File, a private partial file…) instead of a path. */
|
|
49
|
+
source?: ByteSource;
|
|
50
|
+
/**
|
|
51
|
+
* The source's SHA-256 (hex), when the caller knows it (a content-addressed id): with `dedupe` nothing is read before
|
|
52
|
+
* the server is asked (no hashing pass). Trusted only while the source is unchanged (same stamp as at upload()).
|
|
53
|
+
*/
|
|
54
|
+
sha256?: string;
|
|
55
|
+
/** This caller's cancel (handle.cancel() does the same). With shareUploads, the run stops when its last caller left. */
|
|
56
|
+
signal?: AbortSignal;
|
|
57
|
+
/** Signer scope (default: the engine's). */
|
|
58
|
+
scope?: string;
|
|
59
|
+
/** Default: `${scope}:${key}`. Calling upload() again with the same id resumes the saved multipart. */
|
|
60
|
+
id?: string;
|
|
61
|
+
/** Default "default" (or the engine's first non-"first" lane). */
|
|
62
|
+
lane?: string;
|
|
63
|
+
contentType?: string;
|
|
64
|
+
/** Counted against the scope's quota by the server. */
|
|
65
|
+
counts?: boolean;
|
|
66
|
+
/** Written once: a different object already stored is a conflict, never overwritten. */
|
|
67
|
+
immutable?: boolean;
|
|
68
|
+
/** Ask the server's dedupe hook first (sha256 of the file, one extra read): already stored → nothing sent. */
|
|
69
|
+
dedupe?: boolean;
|
|
70
|
+
}
|
|
71
|
+
export interface UploadHandle {
|
|
72
|
+
id: string;
|
|
73
|
+
done: Promise<UploadResult>;
|
|
74
|
+
/** This caller leaves: its `done` resolves "failed" (Cancelled). The upload itself stops once no caller is left. */
|
|
75
|
+
cancel(): void;
|
|
76
|
+
on(ev: "progress", fn: (sent: number, size: number) => void): void;
|
|
77
|
+
on(ev: "state", fn: (s: UploadState) => void): void;
|
|
78
|
+
}
|
|
79
|
+
export interface TransferEngineOptions {
|
|
80
|
+
signer: Signer;
|
|
81
|
+
transport: Transport;
|
|
82
|
+
timing?: TransferTiming;
|
|
83
|
+
/** Default createWorkerHasher() (parts' MD5/CRC-32 off the main thread); nodeHasher hashes on the caller's thread. */
|
|
84
|
+
hasher?: Hasher;
|
|
85
|
+
store?: ResumeStore;
|
|
86
|
+
/** Default scope for uploads that don't name one. */
|
|
87
|
+
scope?: string;
|
|
88
|
+
/** Lanes in priority order (default: one lane, "default"). */
|
|
89
|
+
lanes?: LaneSpec[];
|
|
90
|
+
errors?: ErrorAdapter;
|
|
91
|
+
connections?: {
|
|
92
|
+
min?: number;
|
|
93
|
+
max?: number;
|
|
94
|
+
start?: number;
|
|
95
|
+
};
|
|
96
|
+
/** See EngineCoreOptions (gaps 2–4): "md5" (default) or "sha256" with storage receipts, failing closed. */
|
|
97
|
+
integrity?: "md5" | "sha256";
|
|
98
|
+
/** See EngineCoreOptions.readBack: pre-existing copies proven by a read-back SHA-256 (opt-in; the server's `readable`). */
|
|
99
|
+
readBack?: boolean;
|
|
100
|
+
partSizeFor?(size: number): number;
|
|
101
|
+
singlePutMax?: number;
|
|
102
|
+
maxParts?: number;
|
|
103
|
+
serverPartSize?: boolean;
|
|
104
|
+
claims?: ClaimStore;
|
|
105
|
+
claimTtlMs?: number;
|
|
106
|
+
claimPollMs?: number;
|
|
107
|
+
maxClaimWaitMs?: number;
|
|
108
|
+
/** integrity "sha256": the product's own proof of a stored copy storage gave no SHA-256 for (see EngineHost.confirmStored). */
|
|
109
|
+
confirmStored?(stored: {
|
|
110
|
+
id: string;
|
|
111
|
+
key: string;
|
|
112
|
+
size: number | null;
|
|
113
|
+
etag: string | null;
|
|
114
|
+
sha256Parts?: Record<number, string>;
|
|
115
|
+
}): Promise<"verified" | "unverified">;
|
|
116
|
+
/**
|
|
117
|
+
* upload() of an id already in flight, for the same source, joins that run instead of replacing it: every caller
|
|
118
|
+
* gets the same result; each can leave (cancel/signal), and the run stops when the last one left. The same id for a
|
|
119
|
+
* different source gets "failed" (InFlightConflict) and the running upload goes on. Default false (replace).
|
|
120
|
+
*/
|
|
121
|
+
shareUploads?: boolean;
|
|
122
|
+
/** Part size for multiparts created from now on (EngineCoreOptions.partSize). Default: the engine's rule. */
|
|
123
|
+
partSize?: EngineCoreOptions<UploadItem>["partSize"];
|
|
124
|
+
/** Cap on buffered bytes in flight (EngineCoreOptions.inflightBytes). Default: none. */
|
|
125
|
+
inflightBytes?: number | null;
|
|
126
|
+
/**
|
|
127
|
+
* Host power hook ("host-power", RC-T7): called with true when the engine has work to do (started, not paused, at
|
|
128
|
+
* least one upload or bundle not settled) and false when it has none, only on changes. A product follows it with a
|
|
129
|
+
* power assertion so the machine doesn't sleep mid-transfer, e.g. `electronKeepAwake(powerSaveBlocker)`.
|
|
130
|
+
*/
|
|
131
|
+
onBusyChange?(busy: boolean): void;
|
|
132
|
+
}
|
|
133
|
+
interface Rec {
|
|
134
|
+
id: string;
|
|
135
|
+
scope: string;
|
|
136
|
+
lane: string;
|
|
137
|
+
name: string;
|
|
138
|
+
contentType: string;
|
|
139
|
+
counts: boolean;
|
|
140
|
+
immutable: boolean;
|
|
141
|
+
path: string;
|
|
142
|
+
/** The ByteSource (when the request gave one; `path` is then its name). */
|
|
143
|
+
src?: ByteSource;
|
|
144
|
+
/** The ByteSource's identity id at upload() (part of the saved record: a resume needs the same id). */
|
|
145
|
+
sourceId?: string;
|
|
146
|
+
mtimeMs: number;
|
|
147
|
+
version: ItemVersion;
|
|
148
|
+
mp?: Multipart;
|
|
149
|
+
sha256?: string;
|
|
150
|
+
sha256Stamp?: Stamp | SourceStamp;
|
|
151
|
+
/** cancel() came while it was still being hashed: never started. */
|
|
152
|
+
cancelled?: boolean;
|
|
153
|
+
/** The failure's code, when the engine gave one (Unverified, ClaimTimeout, InvalidPartSize…), and whether retrying helps. */
|
|
154
|
+
failCode?: string;
|
|
155
|
+
failRetryable?: boolean;
|
|
156
|
+
/** The source's stamp at upload() (paths): a joining caller must see the very same file. */
|
|
157
|
+
stamp?: Stamp | null;
|
|
158
|
+
/** The content SHA-256 the caller gave (ByteSources join only on an equal one). */
|
|
159
|
+
reqSha256?: string;
|
|
160
|
+
/** With a declared sha256: the source as it was at upload(); only those bytes may be sent (R2-3). */
|
|
161
|
+
boundTo?: SourceStamp;
|
|
162
|
+
listeners: {
|
|
163
|
+
progress: ((s: number, n: number) => void)[];
|
|
164
|
+
state: ((s: UploadState) => void)[];
|
|
165
|
+
};
|
|
166
|
+
/** Settles every caller still attached. */
|
|
167
|
+
resolve: (r: UploadResult) => void;
|
|
168
|
+
/** The callers attached (shareUploads): each one's own settle. */
|
|
169
|
+
waiters: Map<symbol, (r: UploadResult) => void>;
|
|
170
|
+
lastState?: UploadState;
|
|
171
|
+
settled: boolean;
|
|
172
|
+
}
|
|
173
|
+
export declare function createTransferEngine(o: TransferEngineOptions): {
|
|
174
|
+
upload: (req: UploadRequest) => UploadHandle;
|
|
175
|
+
uploadBundle: (reqs: UploadRequest[], opts?: {
|
|
176
|
+
maxFiles?: number;
|
|
177
|
+
}) => Promise<{
|
|
178
|
+
id: string;
|
|
179
|
+
result: UploadResult;
|
|
180
|
+
}[]>;
|
|
181
|
+
/** Start the engine's loops (rate, pump every ENGINE_TICK_MS; tuner every TUNE_INTERVAL_MS). */
|
|
182
|
+
start: () => void;
|
|
183
|
+
stop: () => void;
|
|
184
|
+
pause: () => void;
|
|
185
|
+
resume: () => void;
|
|
186
|
+
/** Stop an upload: its multipart is aborted (parts released); its promise resolves "failed" (Cancelled). */
|
|
187
|
+
cancel(id: string): Promise<void>;
|
|
188
|
+
stats: () => {
|
|
189
|
+
rate: number;
|
|
190
|
+
connections: number;
|
|
191
|
+
target: number;
|
|
192
|
+
offline: boolean;
|
|
193
|
+
lastError: string | undefined;
|
|
194
|
+
};
|
|
195
|
+
core: EngineCore<{
|
|
196
|
+
readonly r: Rec;
|
|
197
|
+
get id(): string;
|
|
198
|
+
get scope(): string;
|
|
199
|
+
get lane(): string;
|
|
200
|
+
get name(): string;
|
|
201
|
+
get path(): string;
|
|
202
|
+
get version(): ItemVersion;
|
|
203
|
+
get mp(): Multipart | undefined;
|
|
204
|
+
set mp(m: Multipart | undefined);
|
|
205
|
+
deleteMp(): void;
|
|
206
|
+
get contentType(): string;
|
|
207
|
+
get counts(): boolean;
|
|
208
|
+
get immutable(): boolean;
|
|
209
|
+
get mtimeMs(): number;
|
|
210
|
+
get source(): string | ByteSource;
|
|
211
|
+
get waitKey(): string;
|
|
212
|
+
get sha256(): string | undefined;
|
|
213
|
+
get sha256Stamp(): SourceStamp | Stamp | undefined;
|
|
214
|
+
get boundTo(): SourceStamp | undefined;
|
|
215
|
+
}>;
|
|
216
|
+
storage: {
|
|
217
|
+
createMultipart: (scope: string, key: string, contentType: string, size?: number, counts?: boolean) => Promise<string>;
|
|
218
|
+
createOrDedupe: (scope: string, key: string, contentType: string, size?: number, counts?: boolean, sha256?: string) => Promise<import("./storage").Deduped | {
|
|
219
|
+
uploadId: string;
|
|
220
|
+
}>;
|
|
221
|
+
completeMultipart: (scope: string, key: string, uploadId: string, parts: {
|
|
222
|
+
n: number;
|
|
223
|
+
etag: string;
|
|
224
|
+
}[]) => Promise<{
|
|
225
|
+
recovered?: true | undefined;
|
|
226
|
+
size: number | null;
|
|
227
|
+
etag: string | null;
|
|
228
|
+
weak?: boolean;
|
|
229
|
+
}>;
|
|
230
|
+
abortMultipart: (scope: string, key: string, uploadId: string) => Promise<void | undefined>;
|
|
231
|
+
head: (scope: string, key: string) => Promise<import("./storage").Stat>;
|
|
232
|
+
listParts: (scope: string, key: string, uploadId: string) => Promise<Map<number, string>>;
|
|
233
|
+
listUploads: (scope: string, key: string) => Promise<{
|
|
234
|
+
uploadId: string;
|
|
235
|
+
initiated?: string;
|
|
236
|
+
}[]>;
|
|
237
|
+
uploadPart: (scope: string, key: string, uploadId: string, partNumber: number, body: Uint8Array, contentMd5: string, onProgress: (sent: number) => void, timeoutMs: number, y?: import("./protocol").Yieldable) => Promise<string>;
|
|
238
|
+
putObject: (scope: string, key: string, body: Uint8Array, contentType: string, onProgress?: (sent: number) => void, timeoutMs?: number, contentMd5?: string, counts?: boolean, y?: import("./protocol").Yieldable, sha256?: string, w?: import("./storage").WriteMark) => Promise<string>;
|
|
239
|
+
transport: Transport;
|
|
240
|
+
createSession: (scope: string, key: string, contentType: string, size: number, counts: boolean, x: {
|
|
241
|
+
sha256?: string;
|
|
242
|
+
checksumAlgorithm?: "SHA256";
|
|
243
|
+
partSize?: number;
|
|
244
|
+
resumeUploadId?: string;
|
|
245
|
+
}) => Promise<import("./storage").CreateAnswer>;
|
|
246
|
+
uploadPartChecked: (scope: string, key: string, uploadId: string, partNumber: number, body: Uint8Array, contentMd5: string, checksumSha256: string, onProgress: (sent: number) => void, timeoutMs: number, y?: import("./protocol").Yieldable) => Promise<import("./protocol").PutReceipt>;
|
|
247
|
+
putObjectChecked: (scope: string, key: string, body: Uint8Array, contentType: string, onProgress: (sent: number) => void, timeoutMs: number, contentMd5: string, checksumSha256: string, counts: boolean, y?: import("./protocol").Yieldable, sha256?: string) => Promise<{
|
|
248
|
+
receipt: import("./protocol").PutReceipt;
|
|
249
|
+
} | {
|
|
250
|
+
deduped: import("./storage").Deduped;
|
|
251
|
+
} | {
|
|
252
|
+
exists: import("./storage").CheckedStat;
|
|
253
|
+
}>;
|
|
254
|
+
listPartsChecked: (scope: string, key: string, uploadId: string) => Promise<Map<number, import("./storage").ListedPart>>;
|
|
255
|
+
completeChecked: (scope: string, key: string, uploadId: string, parts: {
|
|
256
|
+
n: number;
|
|
257
|
+
etag: string;
|
|
258
|
+
checksumSha256: string;
|
|
259
|
+
}[]) => Promise<import("./storage").CheckedStat>;
|
|
260
|
+
headWithChecksum: (scope: string, key: string) => Promise<import("./storage").CheckedStat>;
|
|
261
|
+
readBack: (scope: string, key: string, want: number, sha256Stream: () => {
|
|
262
|
+
update(b: Uint8Array): unknown;
|
|
263
|
+
digest(): Uint8Array;
|
|
264
|
+
}, crc32: (b: Uint8Array, prev?: number) => number) => Promise<import("./storage").ReadBack | null>;
|
|
265
|
+
};
|
|
266
|
+
};
|
|
267
|
+
export {};
|
|
@@ -0,0 +1,192 @@
|
|
|
1
|
+
import { type FileHandle } from "node:fs/promises";
|
|
2
|
+
import type { Sink } from "./protocol";
|
|
3
|
+
import { type TransferTiming } from "./timing";
|
|
4
|
+
export type Range = [number, number];
|
|
5
|
+
/** Storage answered with a status the download can act on (403/400: sign again; 404: gone). `status` 0 = network. */
|
|
6
|
+
export declare class HttpError extends Error {
|
|
7
|
+
status: number;
|
|
8
|
+
/** The server's error code, when it sent one. */
|
|
9
|
+
code?: string | undefined;
|
|
10
|
+
constructor(message: string, status: number,
|
|
11
|
+
/** The server's error code, when it sent one. */
|
|
12
|
+
code?: string | undefined);
|
|
13
|
+
}
|
|
14
|
+
/** Storage no longer has the version this file's bytes came from (412, or a different ETag came back). */
|
|
15
|
+
export declare class VersionChanged extends Error {
|
|
16
|
+
tried?: string | undefined;
|
|
17
|
+
constructor(tried?: string | undefined);
|
|
18
|
+
}
|
|
19
|
+
/** Our own disk, not the network (never retried forever). */
|
|
20
|
+
export declare class LocalError extends Error {
|
|
21
|
+
}
|
|
22
|
+
/** ETag without W/, quotes, whitespace; lower case. */
|
|
23
|
+
export declare const normEtag: (e?: string | null) => string | undefined;
|
|
24
|
+
export declare const covered: (ranges: Range[]) => number;
|
|
25
|
+
/** `ranges` with [s, e) added, merged and sorted (a new array). */
|
|
26
|
+
export declare function addRange(ranges: Range[], s: number, e: number): Range[];
|
|
27
|
+
/** 16 MiB: the most one range request fetches. */
|
|
28
|
+
export declare const RANGE_CHUNK: number;
|
|
29
|
+
/** The first byte range of [0, size) that's neither done nor in flight, at most `chunk` long. */
|
|
30
|
+
export declare function nextGap(done: Range[], inflight: Range[], size: number, chunk?: number): Range | null;
|
|
31
|
+
/** Worth retrying for as long as it takes: the network, a timeout/stall, or storage/server having a bad moment. */
|
|
32
|
+
export declare function isTransientDownloadError(err: Error): boolean;
|
|
33
|
+
export declare const downloadBackoff: (timing: TransferTiming, tries: number) => number;
|
|
34
|
+
/** Positional writes (a FileHandle fits). May write fewer bytes than asked: the rest is written again. */
|
|
35
|
+
export interface RangeWriter {
|
|
36
|
+
write(buf: Uint8Array, off: number, len: number, pos: number): Promise<{
|
|
37
|
+
bytesWritten: number;
|
|
38
|
+
}>;
|
|
39
|
+
}
|
|
40
|
+
/** A Sink as a RangeWriter. */
|
|
41
|
+
export declare const sinkWriter: (sink: Sink) => RangeWriter;
|
|
42
|
+
export interface RangeFetch {
|
|
43
|
+
url: string;
|
|
44
|
+
/** [s, e) as the request asks for it (the Range header, and whether a whole-object 200 is acceptable). */
|
|
45
|
+
bounds: Range;
|
|
46
|
+
/** [s, e) still to write: `range[1]` may shrink while it runs (another connection took the rest); range[0] advances. */
|
|
47
|
+
range: Range;
|
|
48
|
+
/** The ETag the bytes must have (If-Match), when known. */
|
|
49
|
+
want?: string;
|
|
50
|
+
/** The ETag the file is pinned to now (bytes on disk came from it); read when the answer arrives. */
|
|
51
|
+
pin(): string | undefined;
|
|
52
|
+
/** Storage refused the URL (403/400: expired or bad signature; 404) before the error is thrown: drop cached URLs. */
|
|
53
|
+
onRefused?(status: number): void;
|
|
54
|
+
signal: AbortSignal;
|
|
55
|
+
/** Abort after this long without bytes (re-armed on every chunk). */
|
|
56
|
+
stallMs: number;
|
|
57
|
+
/** Abort the request (stall). */
|
|
58
|
+
abort(reason: Error): void;
|
|
59
|
+
/** The file started over meanwhile: stop, nothing of this range counts. */
|
|
60
|
+
stale(): boolean;
|
|
61
|
+
/** Storage's ETag for the object, from the first answer of a file not pinned yet. */
|
|
62
|
+
onPin?(etag: string): void;
|
|
63
|
+
/** Storage sent no ETag and none was wanted: the file can't be pinned. */
|
|
64
|
+
onUnpinnable?(): void;
|
|
65
|
+
/** Where the bytes go (positional writes: a FileHandle, or a Sink's writer); null = the file was closed. */
|
|
66
|
+
handle(): RangeWriter | null;
|
|
67
|
+
/** [pos, pos + len) is on disk. */
|
|
68
|
+
wrote(pos: number, len: number): void;
|
|
69
|
+
/** Bytes arrived off the wire (rate meters). */
|
|
70
|
+
count?(n: number): void;
|
|
71
|
+
/** Total object size (a 200 is accepted only for a whole-object range). */
|
|
72
|
+
size: number;
|
|
73
|
+
timing: TransferTiming;
|
|
74
|
+
fetch?: typeof fetch;
|
|
75
|
+
}
|
|
76
|
+
/**
|
|
77
|
+
* Fetch one range into the file. Resolves "done" when the range (as far as it still reaches) is on disk, or "stale"
|
|
78
|
+
* when the file started over meanwhile (nothing of this range counts: the caller must not touch the file's state);
|
|
79
|
+
* throws VersionChanged (412, or another ETag than the pin/want), HttpError (403/400/404/other status), LocalError
|
|
80
|
+
* (the disk took no bytes), or the network's error. A response it doesn't read is cancelled.
|
|
81
|
+
*/
|
|
82
|
+
export declare function fetchRangeInto(o: RangeFetch): Promise<"done" | "stale">;
|
|
83
|
+
/**
|
|
84
|
+
* One lock per key. Every change of a file's version identity and every publication runs under it, so they never
|
|
85
|
+
* interleave. Not reentrant: nothing inside takes it again.
|
|
86
|
+
*/
|
|
87
|
+
export declare function keyedLock<K>(): <R>(k: K, fn: () => R | Promise<R>) => Promise<R>;
|
|
88
|
+
/** Does the file (a path, or an open handle) have this S3 ETag? true / false / null (part size not reproducible). */
|
|
89
|
+
export declare function checkFileEtag(path: string | FileHandle, size: number, etag: string): Promise<boolean | null>;
|
|
90
|
+
/** "a/b.mov" → "a/b (2).mov". */
|
|
91
|
+
export declare function withSuffix(rel: string, n: number): string;
|
|
92
|
+
/** Filesystems that can't hard-link (exFAT gives ENOTSUP; some network shares): publishing copies instead. */
|
|
93
|
+
export declare const NO_LINK: Set<string>;
|
|
94
|
+
export interface PublishOptions {
|
|
95
|
+
/** Another unfinished download already writes to this name (skipped for n > 1). */
|
|
96
|
+
claimed(candidate: string): boolean;
|
|
97
|
+
/** Before a copy: record the name it creates (flushed), so a copy cut short by a crash is recognised later. */
|
|
98
|
+
reserve(candidate: string | undefined): void;
|
|
99
|
+
}
|
|
100
|
+
/**
|
|
101
|
+
* Give the verified part (inode `ino`) its final name without ever replacing anything. link(2) fails with EEXIST if
|
|
102
|
+
* the name exists (whoever put it there), so the first free one of want, "want (2)", … gets it; another download's
|
|
103
|
+
* unfinished name is skipped. A name that already IS this inode is ours (a crash between link and unlink). Where hard
|
|
104
|
+
* links aren't supported, it copies with COPYFILE_EXCL, which can't overwrite either.
|
|
105
|
+
*/
|
|
106
|
+
export declare function publishNoOverwrite(part: string, want: string, ino: number, o: PublishOptions): Promise<{
|
|
107
|
+
out: string;
|
|
108
|
+
copied: boolean;
|
|
109
|
+
}>;
|
|
110
|
+
/**
|
|
111
|
+
* No hard links here (exFAT, some SMB): copy the part to the first free name with COPYFILE_EXCL: the copy creates the
|
|
112
|
+
* name exclusively, so an existing file is never replaced; the name is recorded first (reserve), so a copy cut short
|
|
113
|
+
* by a crash is recognised and removed next time. Costs one local copy, on these filesystems only.
|
|
114
|
+
*/
|
|
115
|
+
export declare function publishCopy(part: string, want: string, from: number, o: PublishOptions): Promise<string>;
|
|
116
|
+
/** What a resumable download keeps between runs (a ResumeStore entry). */
|
|
117
|
+
export interface DownloadState {
|
|
118
|
+
ranges: Range[];
|
|
119
|
+
pin?: string;
|
|
120
|
+
/** Sinks: 1 once a download was thrown away after a mismatch (one fresh attempt, then it fails). */
|
|
121
|
+
attempt?: 0 | 1;
|
|
122
|
+
/**
|
|
123
|
+
* The staging file this download created (its base name, next to `dest`): only that file is ever resumed. A state
|
|
124
|
+
* without it (saved by an older SDK) owns the legacy `${dest}.part` when it has ranges.
|
|
125
|
+
*/
|
|
126
|
+
part?: string;
|
|
127
|
+
}
|
|
128
|
+
/**
|
|
129
|
+
* How many downloads may hold their part file (and its 2-second datasync checkpoint) open at once (RC-T6, production
|
|
130
|
+
* D5: at most 2 × connections part files open). A naive share download that calls downloadObject for 20,000 objects at
|
|
131
|
+
* once would otherwise open 20,000 handles; the rest wait here, holding nothing (no file, no timer, no request).
|
|
132
|
+
*/
|
|
133
|
+
export interface DownloadLimiter {
|
|
134
|
+
/** Wait for a slot (rejects with the signal's reason if it aborts first); call the returned function to free it. */
|
|
135
|
+
acquire(signal?: AbortSignal): Promise<() => void>;
|
|
136
|
+
readonly active: number;
|
|
137
|
+
readonly queued: number;
|
|
138
|
+
/** The cap (at least 1); raising it starts waiting downloads at once. */
|
|
139
|
+
max: number;
|
|
140
|
+
}
|
|
141
|
+
export declare function createDownloadLimiter(max: number): DownloadLimiter;
|
|
142
|
+
/** The process-wide cap every downloadObject uses unless given its own: 16 (2 × the default 8 connections). */
|
|
143
|
+
export declare const downloadLimiter: DownloadLimiter;
|
|
144
|
+
export interface DownloadObjectOptions {
|
|
145
|
+
/** A presigned GET URL, or a function that signs one (called again after 403/400). */
|
|
146
|
+
url: string | (() => Promise<string>);
|
|
147
|
+
/**
|
|
148
|
+
* Final path; bytes go to a staging file next to it first (`${dest}.<random>.part`, created exclusively, its name
|
|
149
|
+
* kept in the saved state so a resume finds it): a `.part` this download didn't create is never written or removed.
|
|
150
|
+
* Or give `sink`.
|
|
151
|
+
*/
|
|
152
|
+
dest?: string;
|
|
153
|
+
/**
|
|
154
|
+
* Where the bytes go instead of a path (gap 1): ranges are written with sink.write, checked by reading the sink back,
|
|
155
|
+
* and published with sink.commit(size) only once verified; a mismatch discards it (one fresh attempt, kept in the
|
|
156
|
+
* saved state, then a failure). The saved ranges trust the sink: they are saved after sink.flush() (when it has one).
|
|
157
|
+
*/
|
|
158
|
+
sink?: Sink & {
|
|
159
|
+
flush?(): Promise<void>;
|
|
160
|
+
};
|
|
161
|
+
size: number;
|
|
162
|
+
/** The ETag the bytes must have (the uploader's proven one). Without it (and without sha256) the result is "size" only. */
|
|
163
|
+
etag?: string;
|
|
164
|
+
/** The SHA-256 (hex) the whole object must have: checked instead of the ETag when given (integrity "sha256"). */
|
|
165
|
+
sha256?: string;
|
|
166
|
+
timing: TransferTiming;
|
|
167
|
+
connections?: number;
|
|
168
|
+
/** Saved state (ranges + pin): survives a kill; resumed on the next call. */
|
|
169
|
+
state?: {
|
|
170
|
+
load(): Promise<DownloadState | undefined>;
|
|
171
|
+
save(s: DownloadState): Promise<void>;
|
|
172
|
+
remove(): Promise<void>;
|
|
173
|
+
};
|
|
174
|
+
onProgress?(done: number, size: number): void;
|
|
175
|
+
signal?: AbortSignal;
|
|
176
|
+
fetch?: typeof fetch;
|
|
177
|
+
/** Retries of non-network failures before giving up (default DL_MAX_TRIES). */
|
|
178
|
+
maxTries?: number;
|
|
179
|
+
/** The cap on downloads open at once this one counts against (default: the process-wide `downloadLimiter`); null = none. */
|
|
180
|
+
limiter?: DownloadLimiter | null;
|
|
181
|
+
}
|
|
182
|
+
export type DownloadResult = {
|
|
183
|
+
state: "verified";
|
|
184
|
+
out: string;
|
|
185
|
+
etag: string;
|
|
186
|
+
sha256?: string;
|
|
187
|
+
} | {
|
|
188
|
+
state: "unverified";
|
|
189
|
+
out: string;
|
|
190
|
+
reason: string;
|
|
191
|
+
};
|
|
192
|
+
export declare function downloadObject(o: DownloadObjectOptions): Promise<DownloadResult>;
|