@uploadfilm/sdk 0.0.0-stage → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +4 -0
- package/README.md +529 -2
- package/dist/ai/embed.d.ts +58 -0
- package/dist/ai/fake.d.ts +41 -0
- package/dist/ai/http.d.ts +17 -0
- package/dist/ai/index.d.ts +27 -0
- package/dist/ai/languages.d.ts +2 -0
- package/dist/ai/ollama.d.ts +23 -0
- package/dist/ai/openaiCompatible.d.ts +54 -0
- package/dist/ai/passages.d.ts +28 -0
- package/dist/ai/transcribe.d.ts +84 -0
- package/dist/ai/types.d.ts +141 -0
- package/dist/ai/usage.d.ts +66 -0
- package/dist/ai/whisperJson.d.ts +15 -0
- package/dist/ai/workersAi.d.ts +56 -0
- package/dist/ai.js +1 -0
- package/dist/ai.mjs +1 -0
- package/dist/cards/files.d.ts +91 -0
- package/dist/cards/index.d.ts +15 -0
- package/dist/cards/layouts.d.ts +121 -0
- package/dist/cards/probe.d.ts +69 -0
- package/dist/cards/proxies.d.ts +70 -0
- package/dist/cards/select.d.ts +68 -0
- package/dist/cards/volumes.d.ts +170 -0
- package/dist/cards/xml.d.ts +80 -0
- package/dist/cards.js +1 -0
- package/dist/cards.mjs +1 -0
- package/dist/copy/checksums.d.ts +66 -0
- package/dist/copy/engine.d.ts +24 -0
- package/dist/copy/errors.d.ts +109 -0
- package/dist/copy/index.d.ts +15 -0
- package/dist/copy/steps.d.ts +28 -0
- package/dist/copy/sweep.d.ts +24 -0
- package/dist/copy/types.d.ts +363 -0
- package/dist/embed/player.js +2 -0
- package/dist/host/adapters.d.ts +107 -0
- package/dist/host/detect.d.ts +68 -0
- package/dist/host/encoders.d.ts +34 -0
- package/dist/host/environment.d.ts +53 -0
- package/dist/host/index.d.ts +20 -0
- package/dist/host/linux.d.ts +94 -0
- package/dist/host/mac.d.ts +47 -0
- package/dist/host/network.d.ts +54 -0
- package/dist/host/node.d.ts +13 -0
- package/dist/host/policy.d.ts +101 -0
- package/dist/host/tuner.d.ts +48 -0
- package/dist/host/types.d.ts +118 -0
- package/dist/host-node.js +1 -0
- package/dist/host-node.mjs +1 -0
- package/dist/host.js +1 -0
- package/dist/host.mjs +1 -0
- package/dist/images/index.d.ts +11 -0
- package/dist/images/settings.d.ts +43 -0
- package/dist/images/types.d.ts +83 -0
- package/dist/images/url.d.ts +65 -0
- package/dist/index.d.ts +16 -0
- package/dist/index.js +1 -0
- package/dist/index.mjs +1 -0
- package/dist/player/adapters/hlsjs.d.ts +28 -0
- package/dist/player/bandwidthMemory.d.ts +28 -0
- package/dist/player/captions/index.d.ts +55 -0
- package/dist/player/embed/code.d.ts +195 -0
- package/dist/player/embed/constants.d.ts +4 -0
- package/dist/player/embed/controller.d.ts +62 -0
- package/dist/player/embed/element.d.ts +32 -0
- package/dist/player/embed/frame.d.ts +65 -0
- package/dist/player/embed/global.d.ts +14 -0
- package/dist/player/embed/index.d.ts +15 -0
- package/dist/player/embed/loader.d.ts +1 -0
- package/dist/player/embed/oembed.d.ts +74 -0
- package/dist/player/embed/protocol.d.ts +64 -0
- package/dist/player/engine.d.ts +215 -0
- package/dist/player/index.d.ts +18 -0
- package/dist/player/start.d.ts +62 -0
- package/dist/player/store.d.ts +10 -0
- package/dist/player/storyboard/index.d.ts +161 -0
- package/dist/player/stream.d.ts +206 -0
- package/dist/player/tracks.d.ts +41 -0
- package/dist/player.js +1 -0
- package/dist/player.mjs +1 -0
- package/dist/proxies/args.d.ts +19 -0
- package/dist/proxies/budget.d.ts +57 -0
- package/dist/proxies/camera.d.ts +104 -0
- package/dist/proxies/formats.d.ts +36 -0
- package/dist/proxies/index.d.ts +18 -0
- package/dist/proxies/jobs.d.ts +98 -0
- package/dist/proxies/node.d.ts +32 -0
- package/dist/proxies/plan.d.ts +89 -0
- package/dist/proxies/probe.d.ts +39 -0
- package/dist/proxies/render.d.ts +104 -0
- package/dist/proxies/safety.d.ts +45 -0
- package/dist/proxies/types.d.ts +83 -0
- package/dist/proxies/upload.d.ts +14 -0
- package/dist/react/BigPlay.d.ts +26 -0
- package/dist/react/Player.d.ts +58 -0
- package/dist/react/embed/index.d.ts +45 -0
- package/dist/react/examples/CustomControls.d.ts +8 -0
- package/dist/react/images/Image.d.ts +16 -0
- package/dist/react/images/index.d.ts +4 -0
- package/dist/react/images/useImage.d.ts +47 -0
- package/dist/react/index.d.ts +17 -0
- package/dist/react/storyboard/index.d.ts +58 -0
- package/dist/react/theme.d.ts +37 -0
- package/dist/react/useAdapterErrors.d.ts +8 -0
- package/dist/react/usePlaybackStart.d.ts +18 -0
- package/dist/react/usePlayer.d.ts +28 -0
- package/dist/react/useReducedMotion.d.ts +2 -0
- package/dist/react/useTimedTrack.d.ts +22 -0
- package/dist/react.js +1 -0
- package/dist/react.mjs +1 -0
- package/dist/server/auth/apiKeys.d.ts +30 -0
- package/dist/server/auth/cache.d.ts +21 -0
- package/dist/server/auth/deviceTokens.d.ts +38 -0
- package/dist/server/auth/envelope.d.ts +15 -0
- package/dist/server/auth/grants.d.ts +76 -0
- package/dist/server/auth/index.d.ts +19 -0
- package/dist/server/auth/jwt.d.ts +65 -0
- package/dist/server/auth/keyring.d.ts +31 -0
- package/dist/server/auth/kv.d.ts +12 -0
- package/dist/server/auth/repository.d.ts +45 -0
- package/dist/server/auth/revocations.d.ts +21 -0
- package/dist/server/auth/scope.d.ts +4 -0
- package/dist/server/auth/serviceAuth.d.ts +24 -0
- package/dist/server/auth/signedUrls.d.ts +28 -0
- package/dist/server/embed/index.d.ts +36 -0
- package/dist/server/embed/lifecycle.d.ts +53 -0
- package/dist/server/embed/playerScript.d.ts +4 -0
- package/dist/server/images/index.d.ts +15 -0
- package/dist/server/images/job.d.ts +80 -0
- package/dist/server/images/layout.d.ts +24 -0
- package/dist/server/images/params.d.ts +90 -0
- package/dist/server/images/route.d.ts +101 -0
- package/dist/server/images/sign.d.ts +55 -0
- package/dist/server/index.d.ts +42 -0
- package/dist/server/media/client.d.ts +99 -0
- package/dist/server/media/frameCheck.d.ts +62 -0
- package/dist/server/media/index.d.ts +10 -0
- package/dist/server/media/protocol.d.ts +556 -0
- package/dist/server/media/storyboard.d.ts +144 -0
- package/dist/server/media/transcript.d.ts +143 -0
- package/dist/server/pipeline/index.d.ts +12 -0
- package/dist/server/pipeline/match.d.ts +10 -0
- package/dist/server/pipeline/pipeline.d.ts +182 -0
- package/dist/server/pipeline/queues.d.ts +108 -0
- package/dist/server/pipeline/sources.d.ts +54 -0
- package/dist/server/r2/client.d.ts +164 -0
- package/dist/server/r2/crc32.d.ts +24 -0
- package/dist/server/r2/encoding.d.ts +35 -0
- package/dist/server/r2/errors.d.ts +9 -0
- package/dist/server/r2/integrity.d.ts +25 -0
- package/dist/server/r2/parts.d.ts +46 -0
- package/dist/server/r2/sign.d.ts +47 -0
- package/dist/server/redis/index.d.ts +39 -0
- package/dist/server/state/memory.d.ts +18 -0
- package/dist/server/state/r2.d.ts +73 -0
- package/dist/server/state/types.d.ts +57 -0
- package/dist/server/storage/chunked.d.ts +25 -0
- package/dist/server/storage/index.d.ts +12 -0
- package/dist/server/storage/r2.d.ts +96 -0
- package/dist/server/storage/s3.d.ts +144 -0
- package/dist/server/testing/fake-r2.d.ts +410 -0
- package/dist/server/testing/index.d.ts +10 -0
- package/dist/server/testing/kv-contract.d.ts +14 -0
- package/dist/server/transfer/handler.d.ts +217 -0
- package/dist/server/transfer/keys.d.ts +45 -0
- package/dist/server/transfer/protocol.d.ts +190 -0
- package/dist/server/zip/client.d.ts +32 -0
- package/dist/server/zip/generations.d.ts +83 -0
- package/dist/server/zip/index.d.ts +12 -0
- package/dist/server/zip/planner.d.ts +94 -0
- package/dist/server/zip/sign.d.ts +22 -0
- package/dist/server-media.js +1 -0
- package/dist/server-media.mjs +1 -0
- package/dist/server-redis.js +1 -0
- package/dist/server-redis.mjs +1 -0
- package/dist/server-testing.js +1 -0
- package/dist/server-testing.mjs +1 -0
- package/dist/server.js +1 -0
- package/dist/server.mjs +1 -0
- package/dist/storage/browserFile.d.ts +19 -0
- package/dist/storage/index.d.ts +12 -0
- package/dist/storage/local.d.ts +9 -0
- package/dist/storage/presigned.d.ts +22 -0
- package/dist/sync/ffmpeg.d.ts +44 -0
- package/dist/sync/index.d.ts +17 -0
- package/dist/sync/match.d.ts +28 -0
- package/dist/sync/probe.d.ts +30 -0
- package/dist/sync/timecode.d.ts +50 -0
- package/dist/sync/transcript.d.ts +35 -0
- package/dist/sync/types.d.ts +172 -0
- package/dist/sync/waveform.d.ts +89 -0
- package/dist/sync.js +1 -0
- package/dist/sync.mjs +1 -0
- package/dist/transfer/claims.d.ts +41 -0
- package/dist/transfer/client.d.ts +267 -0
- package/dist/transfer/download.d.ts +192 -0
- package/dist/transfer/engine.d.ts +402 -0
- package/dist/transfer/errors.d.ts +50 -0
- package/dist/transfer/growing/bytes.d.ts +17 -0
- package/dist/transfer/growing/index.d.ts +12 -0
- package/dist/transfer/growing/parts.d.ts +157 -0
- package/dist/transfer/growing/retry.d.ts +38 -0
- package/dist/transfer/growing/store.d.ts +66 -0
- package/dist/transfer/growing/upload.d.ts +126 -0
- package/dist/transfer/index.d.ts +40 -0
- package/dist/transfer/integrity.d.ts +93 -0
- package/dist/transfer/limits.d.ts +34 -0
- package/dist/transfer/node.d.ts +63 -0
- package/dist/transfer/pool.d.ts +83 -0
- package/dist/transfer/protocol.d.ts +172 -0
- package/dist/transfer/signer.d.ts +29 -0
- package/dist/transfer/storage.d.ts +135 -0
- package/dist/transfer/timing.d.ts +132 -0
- package/dist/transfer/transports/electronNet.d.ts +62 -0
- package/dist/transfer/transports/fetch.d.ts +15 -0
- package/dist/transfer/transports/retryAfter.d.ts +12 -0
- package/dist/transfer/transports/undici.d.ts +72 -0
- package/dist/transfer/tuner.d.ts +56 -0
- package/dist/workers/cli.mjs +115 -0
- package/dist/workers/media/config.mjs +53 -0
- package/dist/workers/media/container/Dockerfile +19 -0
- package/dist/workers/media/container/FFMPEG.md +129 -0
- package/dist/workers/media/container/runner.mjs +2690 -0
- package/dist/workers/media/index.mjs +1011 -0
- package/dist/workers/media/init/config.ts +52 -0
- package/dist/workers/media/init/worker.ts +8 -0
- package/dist/workers/media/init/wrangler.toml +44 -0
- package/dist/workers/media/runner.mjs +2656 -0
- package/dist/workers/media/types/build/entry/config.d.mts +6 -0
- package/dist/workers/media/types/build/entry/index.d.mts +43 -0
- package/dist/workers/media/types/build/entry/program.d.mts +13 -0
- package/dist/workers/media/types/build/entry/runner.d.mts +12 -0
- package/dist/workers/media/types/build/entry/settings.d.mts +40 -0
- package/dist/workers/media/types/src/config.d.mts +135 -0
- package/dist/workers/media/types/src/jobs.d.mts +73 -0
- package/dist/workers/media/types/src/runner/container.d.mts +32 -0
- package/dist/workers/media/types/src/runner/ffmpeg.d.mts +227 -0
- package/dist/workers/media/types/src/runner/job.d.mts +107 -0
- package/dist/workers/media/types/src/runner/node.d.mts +71 -0
- package/dist/workers/media/types/src/runner/plugins.d.mts +45 -0
- package/dist/workers/media/types/src/serve.d.mts +107 -0
- package/dist/workers/media/types/src/steps/audio.d.mts +33 -0
- package/dist/workers/media/types/src/steps/edit.d.mts +62 -0
- package/dist/workers/media/types/src/steps/frameCheck.d.mts +44 -0
- package/dist/workers/media/types/src/steps/gate.d.mts +18 -0
- package/dist/workers/media/types/src/steps/paths.d.mts +1 -0
- package/dist/workers/media/types/src/steps/prep.d.mts +60 -0
- package/dist/workers/media/types/src/steps/render.d.mts +40 -0
- package/dist/workers/media/types/src/steps/runner.d.mts +30 -0
- package/dist/workers/media/types/src/steps/transcribe.d.mts +89 -0
- package/dist/workers/media/types/src/storyboard/index.d.mts +61 -0
- package/dist/workers/media/types/src/transcoder.d.mts +30 -0
- package/dist/workers/media/types/src/worker.d.mts +30 -0
- package/dist/workers/zip/index.mjs +589 -0
- package/dist/workers/zip/init/config.ts +22 -0
- package/dist/workers/zip/init/worker.ts +5 -0
- package/dist/workers/zip/init/wrangler.toml +25 -0
- package/dist/workers/zip/types/build/entry/index.d.mts +5 -0
- package/dist/workers/zip/types/src/config.d.mts +35 -0
- package/dist/workers/zip/types/src/serve.d.mts +86 -0
- package/dist/workers/zip/types/src/worker.d.mts +12 -0
- package/dist/zip/index.d.ts +13 -0
- package/dist/zip/layout.d.ts +129 -0
- package/dist/zip/names.d.ts +34 -0
- package/dist/zip/plan.d.ts +115 -0
- package/dist/zip/sw.d.ts +7 -0
- package/package.json +313 -4
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import type { KeyValueStore } from "@uploadfilm/sdk/server";
|
|
2
|
+
export interface KvHarness {
|
|
3
|
+
make(): Promise<{
|
|
4
|
+
store: KeyValueStore;
|
|
5
|
+
advance(ms: number): Promise<void>;
|
|
6
|
+
}>;
|
|
7
|
+
/** Milliseconds to use as a short TTL (Redis: real time; default 200). */
|
|
8
|
+
shortTtlMs?: number;
|
|
9
|
+
}
|
|
10
|
+
export interface KvCase {
|
|
11
|
+
name: string;
|
|
12
|
+
run(h: KvHarness): Promise<void>;
|
|
13
|
+
}
|
|
14
|
+
export declare function keyValueStoreContract(): KvCase[];
|
|
@@ -0,0 +1,217 @@
|
|
|
1
|
+
import type { R2Client } from "../r2/client";
|
|
2
|
+
import type { TransferOp, TransferResult } from "./protocol";
|
|
3
|
+
/** Base64 of a 16-byte MD5: 22 chars + "==". */
|
|
4
|
+
export declare const MD5_B64: RegExp;
|
|
5
|
+
/** Base64 of a 32-byte SHA-256, as `x-amz-checksum-sha256` carries it: 43 chars + "=". */
|
|
6
|
+
export declare const SHA256_B64: RegExp;
|
|
7
|
+
/** "ok" lets the write through; anything else refuses it (unknown answers too: fail closed). */
|
|
8
|
+
export type ReserveOutcome = "ok" | "quota" | "exists" | "deleted" | {
|
|
9
|
+
refuse: TransferResult;
|
|
10
|
+
};
|
|
11
|
+
/** What the dedupe hook found: a key in the caller's own scope, or one elsewhere it explicitly authorizes. */
|
|
12
|
+
export type DedupeSource = {
|
|
13
|
+
key: string;
|
|
14
|
+
etag: string;
|
|
15
|
+
crossScopeAuthorized?: true;
|
|
16
|
+
} | null;
|
|
17
|
+
/** One write about to be issued, as the grant hook sees it. */
|
|
18
|
+
export interface GrantEntry {
|
|
19
|
+
index: number;
|
|
20
|
+
op: string;
|
|
21
|
+
key: string;
|
|
22
|
+
/** How long what is being issued stays usable (URL TTL, or the server-side window for create/complete). */
|
|
23
|
+
ttlSeconds: number;
|
|
24
|
+
}
|
|
25
|
+
export interface CustomOp<Ctx> {
|
|
26
|
+
/**
|
|
27
|
+
* The key this op will write, for the grant hook (null: no grant). Keyless ops derive their key from the context
|
|
28
|
+
* and the op, never from the client.
|
|
29
|
+
*/
|
|
30
|
+
grantKey?: (op: Record<string, unknown>, ctx: Ctx) => string | null;
|
|
31
|
+
/** TTL recorded with the grant. */
|
|
32
|
+
grantTtlSeconds?: number;
|
|
33
|
+
/** What a refused grant answers for this op. */
|
|
34
|
+
refused?: TransferResult;
|
|
35
|
+
run: (op: Record<string, unknown>, ctx: Ctx) => Promise<TransferResult> | TransferResult;
|
|
36
|
+
}
|
|
37
|
+
export interface SignHandlerOptions<Kind extends string, Ctx> {
|
|
38
|
+
/** Storage access (presign + control plane). Per call, so a product can pick a bucket per context. */
|
|
39
|
+
r2: R2Client | ((ctx: Ctx) => R2Client);
|
|
40
|
+
/** The key-layout guard for this context's scope: the key's kind, or null when it isn't one of the layouts. */
|
|
41
|
+
classify: (key: unknown, ctx: Ctx) => Kind | null;
|
|
42
|
+
/** Kinds written once and counted (UploadFilm: originals). Default: none. */
|
|
43
|
+
immutable?: (kind: Kind) => boolean;
|
|
44
|
+
/**
|
|
45
|
+
* Kinds this caller may read with the `get` op (a presigned GET, pinned by the client with If-Match). Default: none,
|
|
46
|
+
* every `get` is refused (`ReadRefused`, not retryable). A client engine uses it to prove by SHA-256 that an object
|
|
47
|
+
* which was at the key before its first write is the file (otherwise it can't adopt it: `Unverified`).
|
|
48
|
+
*/
|
|
49
|
+
readable?: (kind: Kind, ctx: Ctx) => boolean | Promise<boolean>;
|
|
50
|
+
limits?: {
|
|
51
|
+
/** Ops per call (default 64). */
|
|
52
|
+
maxOps?: number;
|
|
53
|
+
/** Parts per multipart upload (default 10,000). */
|
|
54
|
+
maxParts?: number;
|
|
55
|
+
/** Largest object (default 5 TiB, S3's). */
|
|
56
|
+
maxObjectSize?: number;
|
|
57
|
+
/** Presigned part/put URL lifetime, seconds (default 3600). */
|
|
58
|
+
presignTtlSeconds?: number;
|
|
59
|
+
/** Grant window recorded for server-side writes (create/complete), seconds (default 300). */
|
|
60
|
+
serverWriteTtlSeconds?: number;
|
|
61
|
+
/** Content-Type values are cut to this many characters (default 200). */
|
|
62
|
+
maxContentType?: number;
|
|
63
|
+
};
|
|
64
|
+
/** Product copy for the refusals clients show. */
|
|
65
|
+
messages?: {
|
|
66
|
+
quota?: string;
|
|
67
|
+
exists?: string;
|
|
68
|
+
deleted?: string;
|
|
69
|
+
busy?: string;
|
|
70
|
+
refused?: string;
|
|
71
|
+
internal?: string;
|
|
72
|
+
/** The 403 body's error when a key is outside the layouts. */
|
|
73
|
+
foreignKey?: (ctx: Ctx) => string;
|
|
74
|
+
};
|
|
75
|
+
hooks?: {
|
|
76
|
+
/** Whole-batch and per-op refusals before anything runs (per-op results are final for that op). */
|
|
77
|
+
guard?: (batch: {
|
|
78
|
+
ops: Record<string, unknown>[];
|
|
79
|
+
writes: boolean[];
|
|
80
|
+
custom: boolean[];
|
|
81
|
+
}, ctx: Ctx) => Promise<GuardOutcome> | GuardOutcome;
|
|
82
|
+
/** Record/re-check writes right before they are issued. Returns the indices it refuses (→ `messages.deleted`, or the custom op's `refused`). */
|
|
83
|
+
grant?: (entries: GrantEntry[], ctx: Ctx) => Promise<Iterable<number>> | Iterable<number>;
|
|
84
|
+
/**
|
|
85
|
+
* Per-request lifetime of the presigned part/put URLs, seconds (e.g. what's left of the caller's token, so a URL
|
|
86
|
+
* never outlives the credential it was issued to). Called once per batch; it applies to the URLs, their
|
|
87
|
+
* `expiresAt` and the grants' `ttlSeconds`. It can only lower `limits.presignTtlSeconds`, never raise it.
|
|
88
|
+
* undefined/null: the global limit. Anything that isn't a positive number fails closed to 1 s.
|
|
89
|
+
*/
|
|
90
|
+
presignTtl?: (ctx: Ctx) => Promise<number | null | undefined> | number | null | undefined;
|
|
91
|
+
/**
|
|
92
|
+
* Charge an immutable write before it is issued. `uploadId` null for create (bound later), "put" for a single PUT.
|
|
93
|
+
* Only immutable kinds are reserved (see the module comment: "charge per key").
|
|
94
|
+
*/
|
|
95
|
+
reserve?: (ctx: Ctx, w: {
|
|
96
|
+
key: string;
|
|
97
|
+
size: number;
|
|
98
|
+
kind: Kind;
|
|
99
|
+
uploadId: string | null;
|
|
100
|
+
}) => Promise<ReserveOutcome> | ReserveOutcome;
|
|
101
|
+
/** The multipart upload created for a reserved key. */
|
|
102
|
+
bindUpload?: (ctx: Ctx, key: string, uploadId: string) => Promise<void> | void;
|
|
103
|
+
/** An immutable object is stored at `size` bytes (complete / head / dedupe copy). */
|
|
104
|
+
confirm?: (ctx: Ctx, key: string, size: number) => Promise<void> | void;
|
|
105
|
+
/**
|
|
106
|
+
* A reserved multipart upload was aborted BY THIS CALL (storage removed it: an open upload, so one that never
|
|
107
|
+
* completed) and nothing else can use the key's reservation now: no other multipart
|
|
108
|
+
* upload of the key is open in storage (one may be completing against it, or complete later) and nothing is stored
|
|
109
|
+
* at the key (a complete that just landed and whose `confirm` hasn't run yet). Otherwise the handler doesn't call
|
|
110
|
+
* it, and the reservation stays for that upload (BL-S22: releasing it let another key take the quota while an older
|
|
111
|
+
* multipart of the same key completed, over quota); a product's reconcile frees reservations nothing ever used.
|
|
112
|
+
* Not called when storage answers the abort with NoSuchUpload: the upload was completed (its object may have been
|
|
113
|
+
* deleted since: storage can't tell), aborted before (a repeated abort), expired, or never was this key's (a made-up
|
|
114
|
+
* id); the reservation stays for the product's reconcile (UploadFilm: lib/crons.ts) or a new create of the key.
|
|
115
|
+
* Aborts of one upload repeated in a batch run once (the others answer the same). Still, keep it idempotent: delete
|
|
116
|
+
* only a row that is unconfirmed and bound to this uploadId.
|
|
117
|
+
* Never called for the uploadId "put" (a single PUT's reservation). Storage can't show an outstanding single-PUT URL:
|
|
118
|
+
* a product whose ledger keeps one reservation per key must not let a create re-bind a reservation made for a PUT
|
|
119
|
+
* (UploadFilm keeps such a row bound to "put"), or aborting that multipart would free quota the URL can still use.
|
|
120
|
+
*/
|
|
121
|
+
release?: (ctx: Ctx, key: string, uploadId: string) => Promise<void> | void;
|
|
122
|
+
/**
|
|
123
|
+
* The bytes reserved for this multipart upload of an immutable kind (what `reserve` charged for it, bound by
|
|
124
|
+
* `bindUpload`), or null when there is no reservation for it. At `complete` the handler sums the sizes storage
|
|
125
|
+
* holds for the parts being completed (ListParts) and refuses, and aborts, an upload over its reservation (or over
|
|
126
|
+
* `maxObjectSize`): part URLs aren't length-bound, so without this a client could reserve 1 byte and store
|
|
127
|
+
* terabytes. Null refuses the complete (non-retryable `ReservationRefused`) without aborting. Without this hook
|
|
128
|
+
* only `maxObjectSize` is enforced at complete. The product's `reserve` must never shrink a key's reservation while
|
|
129
|
+
* one of its uploads may be completing (UploadFilm keeps the larger of the two), or a concurrent re-create could
|
|
130
|
+
* free quota between this check and the complete.
|
|
131
|
+
*/
|
|
132
|
+
reservation?: (ctx: Ctx, key: string, uploadId: string) => Promise<number | null> | number | null;
|
|
133
|
+
/**
|
|
134
|
+
* Dedupe: the same content (by `sha256`, which the client sends with create/put) already stored at another key.
|
|
135
|
+
* Return that key and the ETag of the version whose sha256 was verified, to have the server copy exactly that
|
|
136
|
+
* version into place instead of an upload (`{ ok, deduped: true }`), or null.
|
|
137
|
+
* Called for immutable kinds only, after the size and Exists checks and the reservation.
|
|
138
|
+
*
|
|
139
|
+
* THIS HOOK IS AN AUTHORIZATION BOUNDARY. The client only claims a digest; the handler can't hash stored bytes.
|
|
140
|
+
* Return only sources whose sha256 your server computed or verified itself, that this caller may read. A key
|
|
141
|
+
* outside the caller's scope (classify(key, ctx) === null) is ignored unless returned with
|
|
142
|
+
* `crossScopeAuthorized: true`. The copy is pinned to the returned ETag (a source replaced since it was verified
|
|
143
|
+
* is not copied: the client uploads instead) and never overwrites the destination (R2 conditional copy): a race
|
|
144
|
+
* leaves the stored object alone and answers Exists.
|
|
145
|
+
*/
|
|
146
|
+
dedupe?: (ctx: Ctx, w: {
|
|
147
|
+
key: string;
|
|
148
|
+
size: number;
|
|
149
|
+
kind: Kind;
|
|
150
|
+
contentType: string;
|
|
151
|
+
sha256: string;
|
|
152
|
+
}) => Promise<DedupeSource> | DedupeSource;
|
|
153
|
+
/**
|
|
154
|
+
* The budget of a multipart whose size nobody knows until it ends (growing uploads), ported from ClipMe's
|
|
155
|
+
* presign-part and complete routes (its AUDIT F-02): a client holding a scope's credentials must not park unlimited
|
|
156
|
+
* bytes under one key. Called for every "part" op before it is signed (`phase: "part"`) and for every "complete"
|
|
157
|
+
* (`phase: "complete"`), of every kind, with storage's own count (ListParts), never the client's word:
|
|
158
|
+
* - part: `stored` = the bytes storage holds for this upload's OTHER parts (a part sent again replaces itself, so
|
|
159
|
+
* its earlier bytes don't count twice), `size` = the part's declared length (null when the client didn't say:
|
|
160
|
+
* a part signed ahead; refuse that if you require sizes), `total` = stored + (size ?? 0). A declared size is
|
|
161
|
+
* always signed into the URL as Content-Length, so storage refuses a bigger body;
|
|
162
|
+
* - complete: `stored` = `total` = the bytes of the parts being completed; a refusal aborts the upload, so a
|
|
163
|
+
* too-large object never comes into existence;
|
|
164
|
+
* - put (a single PUT, which would otherwise go round the budget): `size` = `total` = its declared size, signed as
|
|
165
|
+
* Content-Length. A put with no size is refused (SizeRequired) while this hook is set.
|
|
166
|
+
* "ok" lets it through; anything else refuses (same outcomes as `reserve`; unknown answers fail closed). Parts in
|
|
167
|
+
* flight together are each judged against what is stored, so a burst can pass by up to the parts in flight: the
|
|
168
|
+
* complete check is the one that holds. Costs one ListParts per part signed: set it only where it is wanted (the
|
|
169
|
+
* hook gets the kind). An upload storage no longer has answers NoSuchUpload.
|
|
170
|
+
*/
|
|
171
|
+
signParts?: (ctx: Ctx, w: {
|
|
172
|
+
key: string;
|
|
173
|
+
kind: Kind;
|
|
174
|
+
uploadId: string;
|
|
175
|
+
phase: "part" | "complete" | "put";
|
|
176
|
+
partNumber: number | null;
|
|
177
|
+
size: number | null;
|
|
178
|
+
stored: number;
|
|
179
|
+
total: number;
|
|
180
|
+
}) => Promise<ReserveOutcome> | ReserveOutcome;
|
|
181
|
+
/** Storage/internal failures turned into generic results (default: console.error, as UploadFilm logs them). */
|
|
182
|
+
onError?: (e: unknown, retryable: boolean) => void;
|
|
183
|
+
};
|
|
184
|
+
/** The product's own keyless ops, by name. */
|
|
185
|
+
ops?: Record<string, CustomOp<Ctx>>;
|
|
186
|
+
/**
|
|
187
|
+
* SHA-256 checksums (a client with integrity "sha256"). Single PUTs are always signed with the `x-amz-checksum-sha256`
|
|
188
|
+
* a client sends (storage checks the body against it and reports it on HEAD).
|
|
189
|
+
*
|
|
190
|
+
* `multipart` (default true): multipart uploads with SHA-256 too: create with ChecksumAlgorithm SHA256, every part
|
|
191
|
+
* signed with its checksum, complete with each part's ChecksumSHA256, and the composite checksum on the answers. Set
|
|
192
|
+
* it to false where storage can't do that: on **R2** a part PUT with `x-amz-checksum-sha256` is refused (501
|
|
193
|
+
* NotImplemented) and a complete's part checksums are ignored (measured 2026-11-16). Then a create, part or complete
|
|
194
|
+
* that asks for SHA-256 is refused with `ChecksumUnsupported` (not retryable): the client learns at once that it can't
|
|
195
|
+
* upload that file with SHA-256 here, instead of sending it all and ending `Unverified`.
|
|
196
|
+
*/
|
|
197
|
+
checksums?: {
|
|
198
|
+
multipart?: boolean;
|
|
199
|
+
};
|
|
200
|
+
}
|
|
201
|
+
export type GuardOutcome = {
|
|
202
|
+
refuse: {
|
|
203
|
+
status: number;
|
|
204
|
+
body: unknown;
|
|
205
|
+
};
|
|
206
|
+
} | {
|
|
207
|
+
refuse?: undefined;
|
|
208
|
+
results?: Map<number, TransferResult> | void;
|
|
209
|
+
} | void;
|
|
210
|
+
export interface HandlerAnswer {
|
|
211
|
+
status: number;
|
|
212
|
+
body: unknown;
|
|
213
|
+
}
|
|
214
|
+
export declare function createSignHandler<Kind extends string, Ctx>(opts: SignHandlerOptions<Kind, Ctx>): {
|
|
215
|
+
handle: (body: unknown, ctx: Ctx) => Promise<HandlerAnswer>;
|
|
216
|
+
run: (op: TransferOp, kind: Kind, expiresAt: string, ctx: Ctx, urlTtl?: number, capUntil?: number) => Promise<TransferResult>;
|
|
217
|
+
};
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The key-layout guard: every key a server signs or reads for a scope (a project, an account, a user…) must be one of
|
|
3
|
+
* the product's layouts under that scope's prefix. The product declares the layouts; this module refuses everything
|
|
4
|
+
* that isn't a plain, well-formed key first, so a layout never sees a traversal or an encoding trick:
|
|
5
|
+
*
|
|
6
|
+
* - not a string, longer than `maxLength` (S3: 1024 bytes of UTF-8; checked in bytes), empty;
|
|
7
|
+
* - outside the prefix;
|
|
8
|
+
* - a segment that is empty, "." or ".." (URL parsers collapse them: `p/a/../b` would be signed and served as `p/b`);
|
|
9
|
+
* - backslashes (URL parsers treat `\` as `/` in http(s) paths) and ASCII control characters;
|
|
10
|
+
* - not well-formed Unicode (lone surrogates: no UTF-8 encoding, the signer would throw);
|
|
11
|
+
* - with `strictUnicode` (default on): C1 controls, bidi overrides/isolates, zero-width and other invisible format
|
|
12
|
+
* characters, line/paragraph separators, and the non-characters U+FFFE/U+FFFF — characters that make a key display
|
|
13
|
+
* as another key (`clip<U+202E>vom.exe`), break logs, or vanish in copy-paste; and segments that percent-decode to
|
|
14
|
+
* "." or ".." (`%2e%2e`): harmless through the SDK's signer (it encodes `%`), a traversal for any code that puts a
|
|
15
|
+
* raw key into a URL.
|
|
16
|
+
*
|
|
17
|
+
* A layout gets the segments after the prefix and answers its kind, or null. UploadFilm's layouts live in its
|
|
18
|
+
* web/lib/layout.ts (`classifyKey`).
|
|
19
|
+
*
|
|
20
|
+
* A leaf module (no imports): UploadFilm's web/lib/layout.ts imports it by path and plain Node loaders load it, so it
|
|
21
|
+
* stays free of imports and of TypeScript-only runtime syntax.
|
|
22
|
+
*/
|
|
23
|
+
export interface KeyGuardOptions {
|
|
24
|
+
/** Longest key, in UTF-8 bytes. Default 1024 (S3's limit). */
|
|
25
|
+
maxBytes?: number;
|
|
26
|
+
/** Longest key, in UTF-16 code units, checked first (cheap). Default `maxBytes`. */
|
|
27
|
+
maxLength?: number;
|
|
28
|
+
/** Refuse invisible / direction-changing / C1 characters too. Default true. */
|
|
29
|
+
strictUnicode?: boolean;
|
|
30
|
+
}
|
|
31
|
+
export type KeyLayout<K> = (segments: string[], key: string) => K | null;
|
|
32
|
+
/** The key's segments after `prefix`, or null when the key is not a plain, safe key under it (see above). */
|
|
33
|
+
export declare function keySegments(prefix: string, key: unknown, o?: KeyGuardOptions): string[] | null;
|
|
34
|
+
/**
|
|
35
|
+
* A classifier: `(prefix, key) → kind | null`. Layouts are tried in order; the first non-null kind wins.
|
|
36
|
+
*
|
|
37
|
+
* ```ts
|
|
38
|
+
* const classify = createKeyGuard([
|
|
39
|
+
* (s) => (s[0] === "orig" && s.length >= 2 ? "original" : null),
|
|
40
|
+
* (s) => (s[0] === "thumb" && s.length >= 2 && /\.jpe?g$/i.test(s[s.length - 1]) ? "thumb" : null),
|
|
41
|
+
* ]);
|
|
42
|
+
* classify(`p/${projectId}/`, key); // "original" | "thumb" | null
|
|
43
|
+
* ```
|
|
44
|
+
*/
|
|
45
|
+
export declare function createKeyGuard<K>(layouts: KeyLayout<K> | KeyLayout<K>[], o?: KeyGuardOptions): (prefix: string, key: unknown) => K | null;
|
|
@@ -0,0 +1,190 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The sign protocol between a client that sends bytes and the product's server that holds the storage credentials.
|
|
3
|
+
* The client never holds credentials: it asks the server to sign each operation, then sends bytes straight to storage
|
|
4
|
+
* with the presigned URL. The server never touches file bodies.
|
|
5
|
+
*
|
|
6
|
+
* POST <product route> `{ ops: TransferOp[] }` → `{ results: TransferResult[] }`, results positional. Every key must
|
|
7
|
+
* be one of the product's layouts for the scope (transfer/keys.ts), or the whole batch is refused (403).
|
|
8
|
+
* Products add their own keyless ops (UploadFilm: "manifest") through the handler's `ops` option.
|
|
9
|
+
*
|
|
10
|
+
* Types only: products re-export them (UploadFilm's shared/api.ts), and type-only re-exports cost nothing at runtime.
|
|
11
|
+
*/
|
|
12
|
+
export type TransferOp =
|
|
13
|
+
/**
|
|
14
|
+
* Server performs CreateMultipartUpload and returns the uploadId. `size` lets the server enforce plans (immutable
|
|
15
|
+
* kinds must give it). `sha256` (hex, optional) lets a dedupe hook recognise content that is already stored.
|
|
16
|
+
* `checksumAlgorithm: "SHA256"` (integrity "sha256"): the upload is created with SHA-256 checksums; a handler whose
|
|
17
|
+
* storage can't check them on parts (R2: `checksums: { multipart: false }`) refuses it (`ChecksumUnsupported`).
|
|
18
|
+
*/
|
|
19
|
+
{
|
|
20
|
+
op: "create";
|
|
21
|
+
key: string;
|
|
22
|
+
contentType: string;
|
|
23
|
+
size?: number;
|
|
24
|
+
counts?: boolean;
|
|
25
|
+
sha256?: string;
|
|
26
|
+
checksumAlgorithm?: "SHA256";
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* Presigned PUT for one part (body sent by the client). With `contentMd5` (base64 MD5 of the body) the URL signs
|
|
30
|
+
* Content-MD5, so storage rejects a body that changed in transit (BadDigest) instead of storing it.
|
|
31
|
+
*/
|
|
32
|
+
/**
|
|
33
|
+
* `size` (optional): the part's exact length. Signed into the URL as Content-Length (storage refuses another length)
|
|
34
|
+
* and given to the handler's `signParts` budget hook. A part signed ahead of its bytes (growing uploads) has none.
|
|
35
|
+
*/
|
|
36
|
+
/**
|
|
37
|
+
* `checksumSha256` (base64 SHA-256 of the body, integrity "sha256"): signed as `x-amz-checksum-sha256`, so storage
|
|
38
|
+
* checks the body against it and answers it back (the PUT's receipt).
|
|
39
|
+
*/
|
|
40
|
+
| {
|
|
41
|
+
op: "part";
|
|
42
|
+
key: string;
|
|
43
|
+
uploadId: string;
|
|
44
|
+
partNumber: number;
|
|
45
|
+
contentMd5?: string;
|
|
46
|
+
size?: number;
|
|
47
|
+
checksumSha256?: string;
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* Server performs CompleteMultipartUpload and returns the final size (HEAD). A part's `checksumSha256` (base64) is
|
|
51
|
+
* sent as its ChecksumSHA256 (every part has one, or none does).
|
|
52
|
+
*/
|
|
53
|
+
| {
|
|
54
|
+
op: "complete";
|
|
55
|
+
key: string;
|
|
56
|
+
uploadId: string;
|
|
57
|
+
parts: {
|
|
58
|
+
n: number;
|
|
59
|
+
etag: string;
|
|
60
|
+
checksumSha256?: string;
|
|
61
|
+
}[];
|
|
62
|
+
}
|
|
63
|
+
/** Server aborts a multipart upload. */
|
|
64
|
+
| {
|
|
65
|
+
op: "abort";
|
|
66
|
+
key: string;
|
|
67
|
+
uploadId: string;
|
|
68
|
+
}
|
|
69
|
+
/** Presigned single PUT (small files, posters). `contentMd5`, `checksumSha256` as for parts; `sha256` as for create. */
|
|
70
|
+
| {
|
|
71
|
+
op: "put";
|
|
72
|
+
key: string;
|
|
73
|
+
contentType: string;
|
|
74
|
+
contentMd5?: string;
|
|
75
|
+
size?: number;
|
|
76
|
+
counts?: boolean;
|
|
77
|
+
sha256?: string;
|
|
78
|
+
checksumSha256?: string;
|
|
79
|
+
}
|
|
80
|
+
/** Server HEADs the key: size or null. */
|
|
81
|
+
| {
|
|
82
|
+
op: "head";
|
|
83
|
+
key: string;
|
|
84
|
+
}
|
|
85
|
+
/**
|
|
86
|
+
* Server lists the parts storage holds for this multipart upload (all pages): what a resumed upload checks its
|
|
87
|
+
* saved parts against. NoSuchUpload (not retryable) when the upload is gone.
|
|
88
|
+
*/
|
|
89
|
+
| {
|
|
90
|
+
op: "listParts";
|
|
91
|
+
key: string;
|
|
92
|
+
uploadId: string;
|
|
93
|
+
}
|
|
94
|
+
/**
|
|
95
|
+
* Server lists the multipart uploads open on exactly this key (upload id, when it was initiated): a client that was
|
|
96
|
+
* killed between asking for a create and recording its upload id finds the orphan and aborts it (RL-10).
|
|
97
|
+
*/
|
|
98
|
+
| {
|
|
99
|
+
op: "listUploads";
|
|
100
|
+
key: string;
|
|
101
|
+
}
|
|
102
|
+
/**
|
|
103
|
+
* A presigned GET of the key, with the stored object's size and ETag (what to pin the read to with If-Match). Only
|
|
104
|
+
* when the product's `readable` hook allows reading that kind for the caller (fail closed: refused by default, code
|
|
105
|
+
* `ReadRefused`). The engine reads back an object that was at the key before its first write, to prove by SHA-256
|
|
106
|
+
* that it is the file (the MD5 ETag isn't collision resistant); downloads can use it too (Reville gap 9).
|
|
107
|
+
*/
|
|
108
|
+
| {
|
|
109
|
+
op: "get";
|
|
110
|
+
key: string;
|
|
111
|
+
};
|
|
112
|
+
export type TransferOpName = TransferOp["op"];
|
|
113
|
+
/** Ops that write (or lead to a write of) the key. */
|
|
114
|
+
export declare const WRITE_OP_NAMES: readonly TransferOpName[];
|
|
115
|
+
export type TransferResult = {
|
|
116
|
+
ok: true;
|
|
117
|
+
uploadId: string;
|
|
118
|
+
} | {
|
|
119
|
+
ok: true;
|
|
120
|
+
url: string;
|
|
121
|
+
headers: Record<string, string>;
|
|
122
|
+
expiresAt: string;
|
|
123
|
+
}
|
|
124
|
+
/**
|
|
125
|
+
* complete, head. `etag` is storage's ETag without quotes: MD5 hex for a single PUT, md5(part md5s)-N for multipart.
|
|
126
|
+
* complete `recovered: true`: the upload was already gone (a completion whose answer was lost, or not ours) and this
|
|
127
|
+
* is a HEAD of what is stored, not a stitch of the parts named.
|
|
128
|
+
* `checksumSha256`: the SHA-256 storage holds for the object (base64; "<b64>-<parts>" for an S3 composite multipart
|
|
129
|
+
* checksum), when it reports one.
|
|
130
|
+
*/
|
|
131
|
+
| {
|
|
132
|
+
ok: true;
|
|
133
|
+
size: number | null;
|
|
134
|
+
etag: string | null;
|
|
135
|
+
recovered?: true;
|
|
136
|
+
checksumSha256?: string;
|
|
137
|
+
}
|
|
138
|
+
/**
|
|
139
|
+
* create, put, when the server's dedupe hook found the same content already stored and copied it to the key
|
|
140
|
+
* server-side: nothing to upload. `etag`/`size` as for head. Only products that enable dedupe ever answer this.
|
|
141
|
+
*/
|
|
142
|
+
| {
|
|
143
|
+
ok: true;
|
|
144
|
+
deduped: true;
|
|
145
|
+
size: number | null;
|
|
146
|
+
etag: string | null;
|
|
147
|
+
} | {
|
|
148
|
+
ok: true;
|
|
149
|
+
}
|
|
150
|
+
/**
|
|
151
|
+
* listParts. `etag`: storage's part ETag without quotes (the part's MD5 hex on R2). `checksumSha256`: the part's
|
|
152
|
+
* (base64), when storage reports one (S3 with SHA-256 checksums; never R2).
|
|
153
|
+
*/
|
|
154
|
+
| {
|
|
155
|
+
ok: true;
|
|
156
|
+
parts: {
|
|
157
|
+
n: number;
|
|
158
|
+
etag: string;
|
|
159
|
+
size: number;
|
|
160
|
+
checksumSha256?: string;
|
|
161
|
+
}[];
|
|
162
|
+
}
|
|
163
|
+
/** listUploads: the multipart uploads open on the key (`initiated`: ISO time, when storage reports it). */
|
|
164
|
+
| {
|
|
165
|
+
ok: true;
|
|
166
|
+
uploads: {
|
|
167
|
+
uploadId: string;
|
|
168
|
+
initiated?: string;
|
|
169
|
+
}[];
|
|
170
|
+
}
|
|
171
|
+
/** get: the URL, its lifetime and the stored object's size/ETag. Nothing stored: the head shape with size null. */
|
|
172
|
+
| {
|
|
173
|
+
ok: true;
|
|
174
|
+
url: string;
|
|
175
|
+
headers: Record<string, string>;
|
|
176
|
+
expiresAt: string;
|
|
177
|
+
size: number;
|
|
178
|
+
etag: string | null;
|
|
179
|
+
} | {
|
|
180
|
+
ok: false;
|
|
181
|
+
error: string;
|
|
182
|
+
code?: string;
|
|
183
|
+
retryable: boolean;
|
|
184
|
+
};
|
|
185
|
+
export interface TransferRequest<Op = TransferOp> {
|
|
186
|
+
ops: Op[];
|
|
187
|
+
}
|
|
188
|
+
export interface TransferResponse<Result = TransferResult> {
|
|
189
|
+
results: Result[];
|
|
190
|
+
}
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import { type ZipDomains } from "./sign";
|
|
2
|
+
export interface ZipWorkerClientOptions {
|
|
3
|
+
/** The Worker's origin, e.g. https://zip.example.com (a trailing "/" is fine). */
|
|
4
|
+
url: () => string;
|
|
5
|
+
/** Shared with the Worker (≥ 32 characters). */
|
|
6
|
+
secret: () => string;
|
|
7
|
+
domains: ZipDomains;
|
|
8
|
+
/** Default: the global fetch. */
|
|
9
|
+
fetch?: typeof fetch;
|
|
10
|
+
/** Default: Date.now. */
|
|
11
|
+
now?: () => number;
|
|
12
|
+
}
|
|
13
|
+
export interface CrcAnswer {
|
|
14
|
+
status: number;
|
|
15
|
+
crc32?: string;
|
|
16
|
+
error?: string;
|
|
17
|
+
}
|
|
18
|
+
export interface ZipWorkerClient {
|
|
19
|
+
/** A signed link to archive `planId` of `shareId`, downloaded as `fileName`, valid until `exp` (Unix seconds). */
|
|
20
|
+
link(shareId: string, planId: string, fileName: string, exp: number): Promise<string>;
|
|
21
|
+
/**
|
|
22
|
+
* CRC-32 of [offset, offset+length) of an object, computed by the Worker next to storage, conditional on `etag`.
|
|
23
|
+
* The request is valid for 15 minutes. An answer for another length or without a CRC is a 502.
|
|
24
|
+
*/
|
|
25
|
+
crc(key: string, etag: string, offset: number, length: number): Promise<CrcAnswer>;
|
|
26
|
+
}
|
|
27
|
+
export declare function createZipWorkerClient(o: ZipWorkerClientOptions): ZipWorkerClient;
|
|
28
|
+
/** The CRC-32 of a whole object from its ranges' CRCs, in order (8 lowercase hex). */
|
|
29
|
+
export declare const combineCrcChunks: (chunks: {
|
|
30
|
+
crc32: string;
|
|
31
|
+
length: number;
|
|
32
|
+
}[]) => string;
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A collection's zip generation: how a product withdraws archive links. Every plan records the generation it was built
|
|
3
|
+
* under (../../zip/plan.ts); the Worker serves it only while that is still the collection's current generation and
|
|
4
|
+
* its id isn't revoked. Withdrawing every outstanding link of a collection = bumping its generation (files left it,
|
|
5
|
+
* or it was revoked/deleted). Withdrawing one plan without touching the others = revoking its id.
|
|
6
|
+
*
|
|
7
|
+
* Why a plan built from older records can't survive a removal: the product reads the generation BEFORE it reads its
|
|
8
|
+
* records, and a removal bumps it AFTER the records are written. A plan that still lists a removed file was read from
|
|
9
|
+
* the records before that write, so its generation is older than the bump: refused, even if written again later.
|
|
10
|
+
* publish() also re-reads the generation after writing a plan, so a link isn't handed out for a dead plan.
|
|
11
|
+
*
|
|
12
|
+
* Storage is behind GenStore (compare-and-swap on the marker); tested with an in-memory store. Moved from UploadFilm's
|
|
13
|
+
* web/lib/zipGen.ts; the marker bytes and the order of reads and writes are unchanged.
|
|
14
|
+
*/
|
|
15
|
+
import { type GenMarker, type ZipKeyLayout } from "../../zip/plan";
|
|
16
|
+
/** The marker's storage: a read with a version, and a conditional write. */
|
|
17
|
+
export interface GenStore {
|
|
18
|
+
get(key: string): Promise<{
|
|
19
|
+
text: string;
|
|
20
|
+
etag: string;
|
|
21
|
+
} | null>;
|
|
22
|
+
/** Conditional write: false when the condition failed (someone else wrote first). */
|
|
23
|
+
put(key: string, text: string, cond: {
|
|
24
|
+
ifMatch?: string;
|
|
25
|
+
ifNoneMatch?: boolean;
|
|
26
|
+
}): Promise<boolean>;
|
|
27
|
+
}
|
|
28
|
+
/** What the sweep reads. */
|
|
29
|
+
export interface SweepIo {
|
|
30
|
+
/** Keys of the collection's stored plans. */
|
|
31
|
+
listPlans(shareId: string): Promise<string[]>;
|
|
32
|
+
getText(key: string): Promise<string | null>;
|
|
33
|
+
del(key: string): Promise<unknown>;
|
|
34
|
+
/** Keys the collection still holds (any plan naming another key is stale), or null when the collection is gone. */
|
|
35
|
+
keys(shareId: string): Promise<Set<string> | null>;
|
|
36
|
+
}
|
|
37
|
+
export interface ZipGenerations {
|
|
38
|
+
read(shareId: string): Promise<GenMarker & {
|
|
39
|
+
etag: string | null;
|
|
40
|
+
}>;
|
|
41
|
+
/** Bump the generation (compare-and-swap, retried), adding `seen` removal ids. → the new generation. */
|
|
42
|
+
bump(shareId: string, seen?: string[], tries?: number): Promise<number>;
|
|
43
|
+
/**
|
|
44
|
+
* Withdraw single plans (their links stop; the collection's other links don't). Entries older than
|
|
45
|
+
* `revokedForMs` are dropped: by then no plan they name may still exist (see zipGenerations).
|
|
46
|
+
*/
|
|
47
|
+
revoke(shareId: string, ids: string[], now?: number, tries?: number): Promise<void>;
|
|
48
|
+
/**
|
|
49
|
+
* Write a plan built under generation `gen`, then make sure no removal happened meanwhile: if the generation moved,
|
|
50
|
+
* the plan is deleted and the answer is false (ask again). The Worker would refuse it anyway; this just avoids
|
|
51
|
+
* handing out a dead link.
|
|
52
|
+
*/
|
|
53
|
+
publish(io: {
|
|
54
|
+
put(): Promise<unknown>;
|
|
55
|
+
del(): Promise<unknown>;
|
|
56
|
+
}, shareId: string, gen: number): Promise<boolean>;
|
|
57
|
+
/**
|
|
58
|
+
* The sweep, for one collection. A plan whose generation is current but that lists a key the collection no longer
|
|
59
|
+
* holds is revoked BY ID: its links stop, the collection's other links don't. Older-generation plans are already
|
|
60
|
+
* refused and are only deleted. A collection that is gone gets a generation bump (every link stops).
|
|
61
|
+
*/
|
|
62
|
+
sweep(io: SweepIo, shareId: string): Promise<{
|
|
63
|
+
plans: number;
|
|
64
|
+
withdrawn: number;
|
|
65
|
+
bumped: boolean;
|
|
66
|
+
}>;
|
|
67
|
+
}
|
|
68
|
+
export interface ZipGenerationOptions extends Pick<ZipKeyLayout, "genKey" | "entryPrefix"> {
|
|
69
|
+
/** A stored plan's key → its id. */
|
|
70
|
+
planIdOfKey(key: string): string;
|
|
71
|
+
/**
|
|
72
|
+
* How long a revocation is kept. It MUST outlive every plan it can name: longer than a plan can stay in storage after
|
|
73
|
+
* its last write (a lifecycle rule) AND than any link signed for it. A revocation dropped earlier revives the link.
|
|
74
|
+
* No default: it depends on the product's plan lifetime (e.g. plans expiring 30 days after their last write need
|
|
75
|
+
* revocations kept longer than 30 days, and longer than the longest link).
|
|
76
|
+
*/
|
|
77
|
+
revokedForMs: number;
|
|
78
|
+
}
|
|
79
|
+
/**
|
|
80
|
+
* The generation protocol over `store`, with the product's key layout (`genKey` for the marker, `entryPrefix` for the
|
|
81
|
+
* keys a plan names), `planIdOfKey` and how long revocations are kept.
|
|
82
|
+
*/
|
|
83
|
+
export declare function zipGenerations(store: GenStore, keys: ZipGenerationOptions): ZipGenerations;
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Streaming ZIP, the site's side (Node or Workers): link and request signatures, planning (pinning, CRCs, limits),
|
|
3
|
+
* the generation protocol (withdrawing links) and the client for the zip Worker (sdks/zip-worker).
|
|
4
|
+
*/
|
|
5
|
+
export { sign, verify, zipLinkPath } from "./sign";
|
|
6
|
+
export type { ZipDomains } from "./sign";
|
|
7
|
+
export { crcChunkFor, crcKey, crcRanges, encodeForWorker, MAX_CRC_RANGE, MAX_CRC_STEPS, planFor, planStillValid, } from "./planner";
|
|
8
|
+
export type { MissingCrc, PlanResult, Stored, ZipSource } from "./planner";
|
|
9
|
+
export { zipGenerations } from "./generations";
|
|
10
|
+
export type { GenStore, SweepIo, ZipGenerationOptions, ZipGenerations } from "./generations";
|
|
11
|
+
export { combineCrcChunks, createZipWorkerClient } from "./client";
|
|
12
|
+
export type { CrcAnswer, ZipWorkerClient, ZipWorkerClientOptions } from "./client";
|