@uploadfilm/sdk 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +4 -0
- package/README.md +509 -2
- package/dist/ai/embed.d.ts +58 -0
- package/dist/ai/fake.d.ts +41 -0
- package/dist/ai/http.d.ts +17 -0
- package/dist/ai/index.d.ts +27 -0
- package/dist/ai/languages.d.ts +2 -0
- package/dist/ai/ollama.d.ts +23 -0
- package/dist/ai/openaiCompatible.d.ts +54 -0
- package/dist/ai/passages.d.ts +28 -0
- package/dist/ai/transcribe.d.ts +84 -0
- package/dist/ai/types.d.ts +141 -0
- package/dist/ai/usage.d.ts +66 -0
- package/dist/ai/whisperJson.d.ts +15 -0
- package/dist/ai/workersAi.d.ts +56 -0
- package/dist/ai.js +1 -0
- package/dist/ai.mjs +1 -0
- package/dist/cards/files.d.ts +91 -0
- package/dist/cards/index.d.ts +15 -0
- package/dist/cards/layouts.d.ts +121 -0
- package/dist/cards/probe.d.ts +69 -0
- package/dist/cards/proxies.d.ts +70 -0
- package/dist/cards/select.d.ts +68 -0
- package/dist/cards/volumes.d.ts +170 -0
- package/dist/cards/xml.d.ts +80 -0
- package/dist/cards.js +1 -0
- package/dist/cards.mjs +1 -0
- package/dist/copy/checksums.d.ts +66 -0
- package/dist/copy/engine.d.ts +24 -0
- package/dist/copy/errors.d.ts +109 -0
- package/dist/copy/index.d.ts +15 -0
- package/dist/copy/steps.d.ts +28 -0
- package/dist/copy/sweep.d.ts +24 -0
- package/dist/copy/types.d.ts +336 -0
- package/dist/embed/player.js +2 -0
- package/dist/host/adapters.d.ts +107 -0
- package/dist/host/detect.d.ts +68 -0
- package/dist/host/encoders.d.ts +34 -0
- package/dist/host/environment.d.ts +53 -0
- package/dist/host/index.d.ts +20 -0
- package/dist/host/linux.d.ts +94 -0
- package/dist/host/mac.d.ts +47 -0
- package/dist/host/network.d.ts +54 -0
- package/dist/host/node.d.ts +13 -0
- package/dist/host/policy.d.ts +101 -0
- package/dist/host/tuner.d.ts +48 -0
- package/dist/host/types.d.ts +118 -0
- package/dist/host-node.js +1 -0
- package/dist/host-node.mjs +1 -0
- package/dist/host.js +1 -0
- package/dist/host.mjs +1 -0
- package/dist/images/index.d.ts +11 -0
- package/dist/images/settings.d.ts +43 -0
- package/dist/images/types.d.ts +83 -0
- package/dist/images/url.d.ts +65 -0
- package/dist/index.d.ts +16 -0
- package/dist/index.js +1 -0
- package/dist/index.mjs +1 -0
- package/dist/player/adapters/hlsjs.d.ts +28 -0
- package/dist/player/bandwidthMemory.d.ts +28 -0
- package/dist/player/captions/index.d.ts +55 -0
- package/dist/player/embed/code.d.ts +195 -0
- package/dist/player/embed/constants.d.ts +4 -0
- package/dist/player/embed/controller.d.ts +62 -0
- package/dist/player/embed/element.d.ts +32 -0
- package/dist/player/embed/frame.d.ts +65 -0
- package/dist/player/embed/global.d.ts +14 -0
- package/dist/player/embed/index.d.ts +15 -0
- package/dist/player/embed/loader.d.ts +1 -0
- package/dist/player/embed/oembed.d.ts +74 -0
- package/dist/player/embed/protocol.d.ts +64 -0
- package/dist/player/engine.d.ts +215 -0
- package/dist/player/index.d.ts +18 -0
- package/dist/player/start.d.ts +62 -0
- package/dist/player/store.d.ts +10 -0
- package/dist/player/storyboard/index.d.ts +161 -0
- package/dist/player/stream.d.ts +206 -0
- package/dist/player/tracks.d.ts +41 -0
- package/dist/player.js +1 -0
- package/dist/player.mjs +1 -0
- package/dist/proxies/args.d.ts +19 -0
- package/dist/proxies/budget.d.ts +57 -0
- package/dist/proxies/camera.d.ts +104 -0
- package/dist/proxies/formats.d.ts +36 -0
- package/dist/proxies/index.d.ts +18 -0
- package/dist/proxies/jobs.d.ts +98 -0
- package/dist/proxies/node.d.ts +32 -0
- package/dist/proxies/plan.d.ts +89 -0
- package/dist/proxies/probe.d.ts +39 -0
- package/dist/proxies/render.d.ts +104 -0
- package/dist/proxies/safety.d.ts +45 -0
- package/dist/proxies/types.d.ts +83 -0
- package/dist/proxies/upload.d.ts +14 -0
- package/dist/react/BigPlay.d.ts +26 -0
- package/dist/react/Player.d.ts +58 -0
- package/dist/react/embed/index.d.ts +45 -0
- package/dist/react/examples/CustomControls.d.ts +8 -0
- package/dist/react/images/Image.d.ts +16 -0
- package/dist/react/images/index.d.ts +4 -0
- package/dist/react/images/useImage.d.ts +47 -0
- package/dist/react/index.d.ts +17 -0
- package/dist/react/storyboard/index.d.ts +58 -0
- package/dist/react/theme.d.ts +37 -0
- package/dist/react/useAdapterErrors.d.ts +8 -0
- package/dist/react/usePlaybackStart.d.ts +18 -0
- package/dist/react/usePlayer.d.ts +28 -0
- package/dist/react/useReducedMotion.d.ts +2 -0
- package/dist/react/useTimedTrack.d.ts +22 -0
- package/dist/react.js +1 -0
- package/dist/react.mjs +1 -0
- package/dist/server/auth/apiKeys.d.ts +30 -0
- package/dist/server/auth/cache.d.ts +21 -0
- package/dist/server/auth/deviceTokens.d.ts +38 -0
- package/dist/server/auth/envelope.d.ts +15 -0
- package/dist/server/auth/grants.d.ts +76 -0
- package/dist/server/auth/index.d.ts +19 -0
- package/dist/server/auth/jwt.d.ts +65 -0
- package/dist/server/auth/keyring.d.ts +31 -0
- package/dist/server/auth/kv.d.ts +12 -0
- package/dist/server/auth/repository.d.ts +45 -0
- package/dist/server/auth/revocations.d.ts +21 -0
- package/dist/server/auth/scope.d.ts +4 -0
- package/dist/server/auth/serviceAuth.d.ts +24 -0
- package/dist/server/auth/signedUrls.d.ts +28 -0
- package/dist/server/embed/index.d.ts +36 -0
- package/dist/server/embed/lifecycle.d.ts +53 -0
- package/dist/server/embed/playerScript.d.ts +4 -0
- package/dist/server/images/index.d.ts +15 -0
- package/dist/server/images/job.d.ts +80 -0
- package/dist/server/images/layout.d.ts +24 -0
- package/dist/server/images/params.d.ts +90 -0
- package/dist/server/images/route.d.ts +101 -0
- package/dist/server/images/sign.d.ts +55 -0
- package/dist/server/index.d.ts +42 -0
- package/dist/server/media/client.d.ts +66 -0
- package/dist/server/media/frameCheck.d.ts +62 -0
- package/dist/server/media/index.d.ts +10 -0
- package/dist/server/media/protocol.d.ts +318 -0
- package/dist/server/media/storyboard.d.ts +144 -0
- package/dist/server/media/transcript.d.ts +143 -0
- package/dist/server/pipeline/index.d.ts +12 -0
- package/dist/server/pipeline/match.d.ts +10 -0
- package/dist/server/pipeline/pipeline.d.ts +182 -0
- package/dist/server/pipeline/queues.d.ts +108 -0
- package/dist/server/pipeline/sources.d.ts +54 -0
- package/dist/server/r2/client.d.ts +150 -0
- package/dist/server/r2/crc32.d.ts +24 -0
- package/dist/server/r2/encoding.d.ts +35 -0
- package/dist/server/r2/errors.d.ts +9 -0
- package/dist/server/r2/integrity.d.ts +25 -0
- package/dist/server/r2/parts.d.ts +46 -0
- package/dist/server/r2/sign.d.ts +47 -0
- package/dist/server/redis/index.d.ts +39 -0
- package/dist/server/state/memory.d.ts +18 -0
- package/dist/server/state/r2.d.ts +73 -0
- package/dist/server/state/types.d.ts +57 -0
- package/dist/server/storage/chunked.d.ts +21 -0
- package/dist/server/storage/index.d.ts +12 -0
- package/dist/server/storage/r2.d.ts +96 -0
- package/dist/server/storage/s3.d.ts +93 -0
- package/dist/server/testing/fake-r2.d.ts +390 -0
- package/dist/server/testing/index.d.ts +10 -0
- package/dist/server/testing/kv-contract.d.ts +14 -0
- package/dist/server/transfer/handler.d.ts +167 -0
- package/dist/server/transfer/keys.d.ts +45 -0
- package/dist/server/transfer/protocol.d.ts +141 -0
- package/dist/server/zip/client.d.ts +32 -0
- package/dist/server/zip/generations.d.ts +83 -0
- package/dist/server/zip/index.d.ts +12 -0
- package/dist/server/zip/planner.d.ts +94 -0
- package/dist/server/zip/sign.d.ts +22 -0
- package/dist/server-media.js +1 -0
- package/dist/server-media.mjs +1 -0
- package/dist/server-redis.js +1 -0
- package/dist/server-redis.mjs +1 -0
- package/dist/server-testing.js +1 -0
- package/dist/server-testing.mjs +1 -0
- package/dist/server.js +1 -0
- package/dist/server.mjs +1 -0
- package/dist/storage/browserFile.d.ts +19 -0
- package/dist/storage/index.d.ts +12 -0
- package/dist/storage/local.d.ts +9 -0
- package/dist/storage/presigned.d.ts +22 -0
- package/dist/sync/ffmpeg.d.ts +44 -0
- package/dist/sync/index.d.ts +17 -0
- package/dist/sync/match.d.ts +28 -0
- package/dist/sync/probe.d.ts +30 -0
- package/dist/sync/timecode.d.ts +50 -0
- package/dist/sync/transcript.d.ts +35 -0
- package/dist/sync/types.d.ts +172 -0
- package/dist/sync/waveform.d.ts +89 -0
- package/dist/sync.js +1 -0
- package/dist/sync.mjs +1 -0
- package/dist/transfer/claims.d.ts +41 -0
- package/dist/transfer/client.d.ts +256 -0
- package/dist/transfer/download.d.ts +192 -0
- package/dist/transfer/engine.d.ts +382 -0
- package/dist/transfer/errors.d.ts +50 -0
- package/dist/transfer/index.d.ts +39 -0
- package/dist/transfer/integrity.d.ts +93 -0
- package/dist/transfer/limits.d.ts +34 -0
- package/dist/transfer/node.d.ts +63 -0
- package/dist/transfer/pool.d.ts +83 -0
- package/dist/transfer/protocol.d.ts +167 -0
- package/dist/transfer/signer.d.ts +29 -0
- package/dist/transfer/storage.d.ts +110 -0
- package/dist/transfer/timing.d.ts +132 -0
- package/dist/transfer/transports/electronNet.d.ts +62 -0
- package/dist/transfer/transports/fetch.d.ts +15 -0
- package/dist/transfer/transports/retryAfter.d.ts +12 -0
- package/dist/transfer/transports/undici.d.ts +72 -0
- package/dist/transfer/tuner.d.ts +56 -0
- package/dist/zip/index.d.ts +13 -0
- package/dist/zip/layout.d.ts +129 -0
- package/dist/zip/names.d.ts +34 -0
- package/dist/zip/plan.d.ts +115 -0
- package/dist/zip/sw.d.ts +7 -0
- package/package.json +265 -4
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import { type Transport } from "../protocol";
|
|
2
|
+
import { type TransferTiming } from "../timing";
|
|
3
|
+
export interface FetchTransportOptions {
|
|
4
|
+
fetch?: typeof fetch;
|
|
5
|
+
allowedOrigins?: string[];
|
|
6
|
+
/**
|
|
7
|
+
* Stream the body (supervised, live progress) instead of sending one buffer. Default: on where the runtime is Node
|
|
8
|
+
* (its fetch streams a fixed-length body), off elsewhere. Only force it on for a fetch that sends a ReadableStream
|
|
9
|
+
* body with the given Content-Length (never chunked).
|
|
10
|
+
*/
|
|
11
|
+
streamBody?: boolean;
|
|
12
|
+
}
|
|
13
|
+
export declare function createFetchTransport(timing: TransferTiming, opts?: FetchTransportOptions): Transport & {
|
|
14
|
+
send: NonNullable<Transport["send"]>;
|
|
15
|
+
};
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Retry-After on a storage answer (RC-T4): 429 SlowDown and 503 come with "Retry-After: <seconds>" or an HTTP date.
|
|
3
|
+
* The transports attach it to the PUT's StorageError as `retryAfterMs`, so the engine can wait at least that long
|
|
4
|
+
* before the next attempt instead of its own (shorter) backoff.
|
|
5
|
+
*/
|
|
6
|
+
import type { StorageError } from "../errors";
|
|
7
|
+
/** Milliseconds to wait, or undefined when there is no usable header. Capped at `capMs` (a bogus far date). */
|
|
8
|
+
export declare function parseRetryAfter(v: string | string[] | null | undefined, capMs?: number, nowMs?: number): number | undefined;
|
|
9
|
+
/** The StorageError with `retryAfterMs` set when the answer had a usable Retry-After (only for 429/503 answers). */
|
|
10
|
+
export declare function withRetryAfter<E extends StorageError>(e: E, status: number, header: string | string[] | null | undefined): E & {
|
|
11
|
+
retryAfterMs?: number;
|
|
12
|
+
};
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
import { type Transport } from "../protocol";
|
|
2
|
+
import { type TransferTiming } from "../timing";
|
|
3
|
+
/**
|
|
4
|
+
* The parts of undici this transport uses (structural, so any undici ≥ 6 fits). The real module goes in as it is
|
|
5
|
+
* (`createUndiciTransport(undici, timing)`, no cast). `O` is the module's own request-options type, inferred from
|
|
6
|
+
* the module passed in: undici's typings are narrower than what it accepts at run time (its `body` type has no async
|
|
7
|
+
* iterable, which undici documents and this transport sends), so the transport builds UndiciRequestOptions and hands
|
|
8
|
+
* them over as `O` rather than making callers cast the module.
|
|
9
|
+
*/
|
|
10
|
+
export interface UndiciLike<O = UndiciRequestOptions> {
|
|
11
|
+
Agent: new (o: Record<string, unknown>) => unknown;
|
|
12
|
+
/**
|
|
13
|
+
* undici's connector factory (every undici ≥ 6 exports it): wrapped so a socket option that throws can't kill us.
|
|
14
|
+
* A method (bivariant), so the real module's narrower connector types fit without a cast.
|
|
15
|
+
*/
|
|
16
|
+
buildConnector?(o: {
|
|
17
|
+
timeout?: number;
|
|
18
|
+
}): (o: never, cb: (e: Error | null, s: unknown) => void) => void;
|
|
19
|
+
request(url: string, o: O): Promise<UndiciResponse>;
|
|
20
|
+
}
|
|
21
|
+
/** What the transport passes to `undici.request`. */
|
|
22
|
+
export interface UndiciRequestOptions {
|
|
23
|
+
method: "PUT";
|
|
24
|
+
headers: Record<string, string>;
|
|
25
|
+
/** The whole body, or a one-shot async iterable of CHUNK slices (progress follows the socket). */
|
|
26
|
+
body: Uint8Array | AsyncIterable<Uint8Array>;
|
|
27
|
+
signal: AbortSignal;
|
|
28
|
+
/** The keep-alive Agent, or the override from UndiciTransportOptions.dispatcher / setDispatcher. */
|
|
29
|
+
dispatcher: unknown;
|
|
30
|
+
}
|
|
31
|
+
/** The parts of undici's response the transport reads. */
|
|
32
|
+
export interface UndiciResponse {
|
|
33
|
+
statusCode: number;
|
|
34
|
+
headers: Record<string, string | string[] | undefined>;
|
|
35
|
+
body: {
|
|
36
|
+
text(): Promise<string>;
|
|
37
|
+
destroy(): unknown;
|
|
38
|
+
};
|
|
39
|
+
}
|
|
40
|
+
/**
|
|
41
|
+
* `unknown` when the module's own options type `O` takes the options the transport sends (dispatcher aside, typed by
|
|
42
|
+
* each module its own way), `never` otherwise: a `request(url, o: string)` is refused at compile time.
|
|
43
|
+
*/
|
|
44
|
+
export type UndiciOptionsAccepted<O> = {
|
|
45
|
+
method: "PUT";
|
|
46
|
+
headers: Record<string, string>;
|
|
47
|
+
body: Uint8Array;
|
|
48
|
+
signal: AbortSignal;
|
|
49
|
+
} extends OmitEach<NonNullable<O>, "dispatcher"> ? unknown : never;
|
|
50
|
+
/** Omit applied to each member of a union (plain Omit merges a union's members into one type, review round 2). */
|
|
51
|
+
type OmitEach<T, K extends PropertyKey> = T extends unknown ? Omit<T, K> : never;
|
|
52
|
+
/** Each PUT's life, for observers (tests' net logs, race reports). "sending" = its first body byte goes out. */
|
|
53
|
+
export type PutEvent = {
|
|
54
|
+
id: symbol;
|
|
55
|
+
url: string;
|
|
56
|
+
kind: "start" | "sending" | "aborted" | "response" | "error";
|
|
57
|
+
status?: number;
|
|
58
|
+
reason?: string;
|
|
59
|
+
};
|
|
60
|
+
export interface UndiciTransportOptions {
|
|
61
|
+
/** Send through another dispatcher (a test's virtual link, a recording wrapper). Default: one keep-alive Agent. */
|
|
62
|
+
dispatcher?: unknown;
|
|
63
|
+
/** Only these origins may be PUT to (scheme://host[:port]); anything else is refused before a byte is sent. */
|
|
64
|
+
allowedOrigins?: string[];
|
|
65
|
+
}
|
|
66
|
+
export interface UndiciTransport extends Transport {
|
|
67
|
+
send: NonNullable<Transport["send"]>;
|
|
68
|
+
watchers: Set<(e: PutEvent) => void>;
|
|
69
|
+
setDispatcher(d: unknown | null): void;
|
|
70
|
+
}
|
|
71
|
+
export declare function createUndiciTransport<O = UndiciRequestOptions>(undici: UndiciLike<O> & UndiciOptionsAccepted<O>, timing: TransferTiming, opts?: UndiciTransportOptions): UndiciTransport;
|
|
72
|
+
export {};
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Connection tuners (gap 5): how many requests to run at once, decided from measured throughput.
|
|
3
|
+
*
|
|
4
|
+
* - createTuner (policy "uploadfilm", the engine core's own): slow start (double while throughput still gains ≥ 10%),
|
|
5
|
+
* then hill-climb in steps of 2 (turn around when a move cost > 5%; stop adding at ≥ 12 when flat within 3%), never
|
|
6
|
+
* outside [min, max]. Errors only cut it when they cluster: 3 within ERROR_BURST_WINDOW_MS → ×0.7 (and climbing
|
|
7
|
+
* waits TUNE_SETTLE_AFTER_ERROR_MS). It doesn't judge while fewer than target − 1 requests run (not enough work).
|
|
8
|
+
* - createHalvingTuner (policy "halve-on-error", SuperBuilder cloud-transfer's PartPool): every tuneIntervalMs, ±1
|
|
9
|
+
* from the measured rate (turn around below 95%, step down when ≤ 103%), only while work is queued; every
|
|
10
|
+
* retryable error halves it and holds it for errorCooldownMs.
|
|
11
|
+
*
|
|
12
|
+
* Both are plain state machines: the caller feeds them rates/bytes and errors, and reads `target`.
|
|
13
|
+
*/
|
|
14
|
+
import type { TransferTiming } from "./timing";
|
|
15
|
+
export interface TunerBounds {
|
|
16
|
+
min?: number;
|
|
17
|
+
max?: number;
|
|
18
|
+
start?: number;
|
|
19
|
+
}
|
|
20
|
+
/** The engine core's tuner. `now` and the timing table are read live (tests warp them). */
|
|
21
|
+
export declare function createTuner(o: TunerBounds & {
|
|
22
|
+
now: () => number;
|
|
23
|
+
timing: TransferTiming;
|
|
24
|
+
}): {
|
|
25
|
+
readonly target: number;
|
|
26
|
+
readonly pinned: number | null;
|
|
27
|
+
/** One tuning step from the current smoothed `rate` (bytes/s) with `active` requests running. */
|
|
28
|
+
tune(rate: number, active: number): void;
|
|
29
|
+
/**
|
|
30
|
+
* New bounds (already checked: limits.ts sanitizeLimits), applied at once: the target is clamped into them unless
|
|
31
|
+
* pinned; `start` only counts before the first step.
|
|
32
|
+
*/
|
|
33
|
+
setBounds(b: {
|
|
34
|
+
min: number;
|
|
35
|
+
max: number;
|
|
36
|
+
start?: number;
|
|
37
|
+
}): void;
|
|
38
|
+
/** A request failed (any error). */
|
|
39
|
+
onError(): void;
|
|
40
|
+
/** Hold the target at `n` (null: tune again). */
|
|
41
|
+
pin(n: number | null): void;
|
|
42
|
+
};
|
|
43
|
+
export type Tuner = ReturnType<typeof createTuner>;
|
|
44
|
+
/** cloud-transfer's PartPool tuner. */
|
|
45
|
+
export declare function createHalvingTuner(o: TunerBounds & {
|
|
46
|
+
tuneIntervalMs: number;
|
|
47
|
+
errorCooldownMs: number;
|
|
48
|
+
now?: () => number;
|
|
49
|
+
}): {
|
|
50
|
+
readonly target: number;
|
|
51
|
+
/** `bytes` more were sent; `queued`: work is waiting (only then is the target moved). */
|
|
52
|
+
sample(n: number, queued: boolean, at?: number): void;
|
|
53
|
+
/** A retryable failure: halve, and hold for errorCooldownMs. */
|
|
54
|
+
onError(): void;
|
|
55
|
+
};
|
|
56
|
+
export type HalvingTuner = ReturnType<typeof createHalvingTuner>;
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Streaming ZIP, the parts any runtime uses: the archive layout (any byte range of an archive of any size, ZIP64
|
|
3
|
+
* where needed, without building it), the stored plan format, and safe entry names. Signing, planning, the
|
|
4
|
+
* generation protocol and the Worker client are in `@uploadfilm/sdk/server`; the Worker itself is the deploy template
|
|
5
|
+
* in sdks/zip-worker.
|
|
6
|
+
*/
|
|
7
|
+
export { archiveEtag, LAYOUT_VERSION, LayoutError, MAX_COMPONENT_BYTES, MAX_NAME_BYTES, MAX_PATH_BYTES, safeZipPath, utf8Length, ZipLayout, } from "./layout";
|
|
8
|
+
export type { Piece, ZipItem } from "./layout";
|
|
9
|
+
export { encodePlan, MAX_ENTRIES, MAX_PLAN_BYTES, MAX_PLAN_NAME_BYTES, normEtag, parseGen, parsePlan, PLAN_ID, PLAN_VERSION, planIdOf, planServable, } from "./plan";
|
|
10
|
+
export type { GenMarker, ParsedPlan, Plan, PlanEntry, ZipKeyLayout } from "./plan";
|
|
11
|
+
export { allocateNames, claimPath, cleanComponent, foldName, simpleComponent, simplePath } from "./names";
|
|
12
|
+
export type { NameRequest } from "./names";
|
|
13
|
+
export { retiredServiceWorkerJs } from "./sw";
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A streaming ZIP laid out before its first byte: what a download Worker serves straight from object storage.
|
|
3
|
+
*
|
|
4
|
+
* Every entry is STORED (no compression), its CRC-32 and size known in advance, so:
|
|
5
|
+
* - local headers carry the real CRC and sizes (no data descriptors, general-purpose bit 3 off);
|
|
6
|
+
* - the total length and every offset are known up front, and any byte range can be produced on its own:
|
|
7
|
+
* headers are encoded on demand, file bytes are a range of a stored object.
|
|
8
|
+
*
|
|
9
|
+
* ZIP64 (APPNOTE 6.3.10 §4.3.9, §4.4.1.4, §4.5.3) exactly where needed:
|
|
10
|
+
* - an entry whose size is ≥ 0xFFFFFFFF: ZIP64 extra in its local header (both sizes) and central header;
|
|
11
|
+
* - an entry whose local header starts at ≥ 0xFFFFFFFF: ZIP64 offset in its central header;
|
|
12
|
+
* - ≥ 0xFFFF entries, or a central directory larger than / starting beyond 0xFFFFFFFF: ZIP64 end of central
|
|
13
|
+
* directory record + locator before the classic end record, whose overflowing fields hold the sentinels.
|
|
14
|
+
*
|
|
15
|
+
* Hand-rolled on purpose: no maintained library produces a ZIP at an arbitrary offset (a resumed download asks for
|
|
16
|
+
* bytes 40 GB in; the archive is never built whole). Correctness is checked by independent readers (Python zipfile,
|
|
17
|
+
* Info-ZIP unzip, 7-Zip, macOS ditto) in UploadFilm's worker/zip/test and sdks/sdk/test/zip.
|
|
18
|
+
*
|
|
19
|
+
* SELF-CONTAINED ON PURPOSE: no imports and only erasable TypeScript, so plain Node (type stripping), workerd and
|
|
20
|
+
* bundlers all load this file by path (moved from UploadFilm's shared/zip/layout.ts; the bytes are unchanged).
|
|
21
|
+
*/
|
|
22
|
+
/**
|
|
23
|
+
* Version of the bytes this file produces for a given plan. It is part of the archive's ETag (archiveEtag), so a
|
|
24
|
+
* deploy that changes any byte (header layout, extras) also changes the ETag: a browser resuming across that
|
|
25
|
+
* deploy sends If-Range with the old ETag and gets the whole new archive (200), never a splice of old and new bytes.
|
|
26
|
+
* Bump it with ANY change to the bytes. z2: far entries carry both sizes in their ZIP64 extra (Info-ZIP fix).
|
|
27
|
+
*/
|
|
28
|
+
export declare const LAYOUT_VERSION = "z2";
|
|
29
|
+
export declare const archiveEtag: (planId: string) => string;
|
|
30
|
+
/** One file in the archive. */
|
|
31
|
+
export interface ZipItem {
|
|
32
|
+
/** Path inside the archive, "/"-separated, relative, no ".." segments. */
|
|
33
|
+
path: string;
|
|
34
|
+
size: number;
|
|
35
|
+
/** CRC-32 of the file's bytes (unsigned). */
|
|
36
|
+
crc32: number;
|
|
37
|
+
/** Modification time, Unix seconds (UTC). */
|
|
38
|
+
mtime: number;
|
|
39
|
+
}
|
|
40
|
+
export declare class LayoutError extends Error {
|
|
41
|
+
}
|
|
42
|
+
/** A path is safe to extract anywhere: relative, no empty/"."/".." segments, no backslashes or NULs. */
|
|
43
|
+
export declare function safeZipPath(p: string): boolean;
|
|
44
|
+
/** What a range of the archive is made of, in order. */
|
|
45
|
+
export type Piece = {
|
|
46
|
+
kind: "bytes";
|
|
47
|
+
data: Uint8Array;
|
|
48
|
+
}
|
|
49
|
+
/** Bytes [from, to) of entry `index`'s file. */
|
|
50
|
+
| {
|
|
51
|
+
kind: "file";
|
|
52
|
+
index: number;
|
|
53
|
+
from: number;
|
|
54
|
+
to: number;
|
|
55
|
+
}
|
|
56
|
+
/**
|
|
57
|
+
* An empty file's header is in the range: it has no bytes to read, but its pinned version must still be checked
|
|
58
|
+
* (exists, same ETag, still empty). Contributes no bytes.
|
|
59
|
+
*/
|
|
60
|
+
| {
|
|
61
|
+
kind: "check";
|
|
62
|
+
index: number;
|
|
63
|
+
};
|
|
64
|
+
/** Longest path (UTF-8 bytes) an archive entry may have. Real paths are far shorter; it bounds Worker memory. */
|
|
65
|
+
export declare const MAX_PATH_BYTES = 1024;
|
|
66
|
+
/** Longest path component (UTF-8 bytes): APFS, HFS+, ext4 and NTFS all stop at 255. */
|
|
67
|
+
export declare const MAX_COMPONENT_BYTES = 255;
|
|
68
|
+
/** All entry paths of one archive together, UTF-8 bytes: bounds the Worker's memory for any accepted plan. */
|
|
69
|
+
export declare const MAX_NAME_BYTES: number;
|
|
70
|
+
/** UTF-8 byte length of a JS string (lone surrogates count 3, as TextEncoder writes U+FFFD). */
|
|
71
|
+
export declare function utf8Length(s: string): number;
|
|
72
|
+
export declare class ZipLayout {
|
|
73
|
+
readonly count: number;
|
|
74
|
+
/** Total archive length in bytes. */
|
|
75
|
+
readonly length: number;
|
|
76
|
+
readonly cdOffset: number;
|
|
77
|
+
readonly cdSize: number;
|
|
78
|
+
/** Whether the ZIP64 end records are present. */
|
|
79
|
+
readonly zip64End: boolean;
|
|
80
|
+
/**
|
|
81
|
+
* Compact on purpose (the Worker has 128 MB): one buffer of all names + offsets, typed arrays for the rest; no
|
|
82
|
+
* per-entry objects or strings are kept.
|
|
83
|
+
*/
|
|
84
|
+
private readonly nameBuf;
|
|
85
|
+
private readonly nameOff;
|
|
86
|
+
private readonly sizes;
|
|
87
|
+
private readonly crcs;
|
|
88
|
+
private readonly mtimes;
|
|
89
|
+
/** localOffset[i] = where entry i's local header starts; localOffset[count] = cdOffset. */
|
|
90
|
+
private readonly localOffset;
|
|
91
|
+
/** cdRel[i] = where entry i's central header starts, relative to cdOffset; cdRel[count] = cdSize. */
|
|
92
|
+
private readonly cdRel;
|
|
93
|
+
private tailBytes;
|
|
94
|
+
/** From items, or from columns (the Worker's parsed plan: no per-entry objects). */
|
|
95
|
+
constructor(src: ZipItem[] | {
|
|
96
|
+
count: number;
|
|
97
|
+
path(i: number): string;
|
|
98
|
+
size(i: number): number;
|
|
99
|
+
crc32(i: number): number;
|
|
100
|
+
mtime(i: number): number;
|
|
101
|
+
});
|
|
102
|
+
/** Bytes this layout keeps (for the Worker's cache budget). */
|
|
103
|
+
get retainedBytes(): number;
|
|
104
|
+
size(i: number): number;
|
|
105
|
+
private name;
|
|
106
|
+
private big;
|
|
107
|
+
private localLen;
|
|
108
|
+
/**
|
|
109
|
+
* ZIP64 extra in the central header: whenever the size or the offset doesn't fit. It then always holds BOTH sizes
|
|
110
|
+
* (with 0xFFFFFFFF in the fixed fields), plus the offset when that's what overflowed. An offset-only extra is
|
|
111
|
+
* spec-legal, but Info-ZIP unzip 6.00 (macOS /usr/bin/unzip, Linux distros) reads its first value as the size.
|
|
112
|
+
*/
|
|
113
|
+
private central64;
|
|
114
|
+
private centralLen;
|
|
115
|
+
/** Where entry i's file bytes start in the archive. */
|
|
116
|
+
dataOffset(i: number): number;
|
|
117
|
+
localHeader(i: number): Uint8Array;
|
|
118
|
+
centralHeader(i: number): Uint8Array;
|
|
119
|
+
/** [ZIP64 EOCD record + locator] + classic EOCD. */
|
|
120
|
+
tail(): Uint8Array;
|
|
121
|
+
/** Last i with arr[i] <= x, for i in [0, n). */
|
|
122
|
+
private static floor;
|
|
123
|
+
/**
|
|
124
|
+
* The pieces making up archive bytes [start, end) (end exclusive), in order. Header pieces are cut to the
|
|
125
|
+
* range; file pieces are object byte ranges. Empty files make no file piece, but a check piece. Lazy: a range late in a
|
|
126
|
+
* 250,000-entry archive costs a binary search, not a walk.
|
|
127
|
+
*/
|
|
128
|
+
pieces(start: number, end: number): Generator<Piece>;
|
|
129
|
+
}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
/** One safe component (never empty, never "." / ".."), within `max` bytes, extension kept. */
|
|
2
|
+
export declare function cleanComponent(raw: string, max?: number, fallback?: string): string;
|
|
3
|
+
/**
|
|
4
|
+
* Case- and normalization-insensitive key (review R2-B4). toLowerCase() alone misses full case mappings that
|
|
5
|
+
* APFS/HFS+ apply (Σ/ς, ß/ss/ẞ, ſ/s, fi/fi, K/k all name one file on this Mac's APFS). lower → upper → lower:
|
|
6
|
+
* the upper step applies the full mappings (ß→SS, ς→Σ, ſ→S, fi→FI), the first lower step brings ẞ to ß first.
|
|
7
|
+
* Idempotent over the BMP, and checked against the real APFS oracle (UploadFilm's web/test/fixtures/apfs-fold-classes.json,
|
|
8
|
+
* made by creating every candidate name on APFS). It merges a few names some file systems
|
|
9
|
+
* keep apart (ı/i, ß/ss on NTFS): those only get a " (2)" they didn't strictly need, never a collision.
|
|
10
|
+
*/
|
|
11
|
+
export declare const foldName: (p: string) => string;
|
|
12
|
+
export interface NameRequest {
|
|
13
|
+
/** Folder components (raw, from the manifest / the zip's top folder). */
|
|
14
|
+
dirs: string[];
|
|
15
|
+
/** File name (raw). */
|
|
16
|
+
name: string;
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* Paths for `reqs`, in order (callers pass them in their final order: earlier entries keep the plain names).
|
|
20
|
+
* null = left out: the path can't be made short enough.
|
|
21
|
+
*/
|
|
22
|
+
export declare function allocateNames(reqs: NameRequest[]): (string | null)[];
|
|
23
|
+
/**
|
|
24
|
+
* One path segment, safe on macOS, Windows and Linux, and never "." / "..": the simple rule (no NFC, no reserved
|
|
25
|
+
* names, no byte limit). "" when nothing is left. For download lists whose names clients already saved files under.
|
|
26
|
+
*/
|
|
27
|
+
export declare const simpleComponent: (s: string) => string;
|
|
28
|
+
/** Parts split on "/", each cleaned with simpleComponent, empty ones dropped, joined with "/". */
|
|
29
|
+
export declare const simplePath: (...parts: string[]) => string;
|
|
30
|
+
/**
|
|
31
|
+
* `path`, or "stem (n).ext" with the smallest n ≥ 2 that is free, compared lower-cased against `taken` (case-insensitive
|
|
32
|
+
* file systems: two entries may not collide). Adds the result (lower-cased) to `taken`.
|
|
33
|
+
*/
|
|
34
|
+
export declare function claimPath(path: string, taken: Set<string>): string;
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A "zip plan": everything a download Worker needs to serve one archive, pinned to exact object versions. A product
|
|
3
|
+
* writes it once to storage (where: its `ZipKeyLayout`); the plan id is the hash of the stored bytes, so a plan is
|
|
4
|
+
* immutable and its id is the archive's ETag.
|
|
5
|
+
*
|
|
6
|
+
* In code a plan is a list of entries [path, key, etag, size, crc32 hex, mtime unix s]. Stored, it is columnar (one
|
|
7
|
+
* flat array per field, keys without the collection's shared key prefix, CRCs as one hex string), which parses into
|
|
8
|
+
* far fewer objects: what keeps a 75,000-entry cold load inside a Worker isolate's 128 MB:
|
|
9
|
+
* { "v": "z2", "share", "name", "created", "gen", "n", "paths": [...], "keys": [...], "etags": [...], "sizes": [...],
|
|
10
|
+
* "crcs": "<8 hex per entry>", "mtimes": [...] }
|
|
11
|
+
* `share` is the id of the collection the archive is made from (whatever the product groups files by); the field
|
|
12
|
+
* name is part of the stored format, and so of every plan id.
|
|
13
|
+
*
|
|
14
|
+
* Generations: a collection has one marker object (`{ gen, seen, revoked }`); a plan records the generation it was
|
|
15
|
+
* built under and is served only while that is still current and its id isn't revoked. Moving the generation stops
|
|
16
|
+
* every outstanding link of the collection at once (server/zip/generations.ts).
|
|
17
|
+
*
|
|
18
|
+
* SELF-CONTAINED ON PURPOSE: no imports and only erasable TypeScript (plain Node, workerd and bundlers load it by
|
|
19
|
+
* path). Hashing uses WebCrypto (Node ≥ 19, browsers, Workers). Moved from UploadFilm's shared/zip/plan.ts; the
|
|
20
|
+
* stored bytes, ids and limits are unchanged.
|
|
21
|
+
*/
|
|
22
|
+
export declare const PLAN_VERSION = "z2";
|
|
23
|
+
/**
|
|
24
|
+
* Limits that keep a Worker isolate (128 MB) safe: one cold load of the biggest plan + a full plan cache fits with
|
|
25
|
+
* margin (measured in UploadFilm's worker/zip/test/tools/measure-memory.ts). A product refuses to make a plan over
|
|
26
|
+
* either (encodeForWorker answers null); the Worker refuses one over MAX_PLAN_BYTES unread.
|
|
27
|
+
*/
|
|
28
|
+
export declare const MAX_ENTRIES = 75000;
|
|
29
|
+
export declare const MAX_PLAN_BYTES: number;
|
|
30
|
+
/** Same as layout.ts MAX_NAME_BYTES (no import there: this file stays import-free for the plan format). */
|
|
31
|
+
export declare const MAX_PLAN_NAME_BYTES: number;
|
|
32
|
+
export type PlanEntry = [path: string, key: string, etag: string, size: number, crc32: string, mtime: number];
|
|
33
|
+
export interface Plan {
|
|
34
|
+
v: typeof PLAN_VERSION;
|
|
35
|
+
/** The collection the archive is made from (stored as "share"). */
|
|
36
|
+
share: string;
|
|
37
|
+
/** The archive's file name (Content-Disposition). */
|
|
38
|
+
name: string;
|
|
39
|
+
/** Last-Modified of the archive, Unix seconds. */
|
|
40
|
+
created: number;
|
|
41
|
+
/** The collection's generation when this plan was built: served only while it is still the current one. */
|
|
42
|
+
gen: number;
|
|
43
|
+
entries: PlanEntry[];
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* Where a product keeps zip objects in its bucket. The SDK never assumes a layout: every reader and writer (the
|
|
47
|
+
* Worker, the planner, the generation protocol) takes one of these.
|
|
48
|
+
*/
|
|
49
|
+
export interface ZipKeyLayout {
|
|
50
|
+
/** The stored plan `planId` of collection `share`. */
|
|
51
|
+
planKey(share: string, planId: string): string;
|
|
52
|
+
/** The collection's generation marker. Keep it outside any expiring prefix: a lost marker revives older plans. */
|
|
53
|
+
genKey(share: string): string;
|
|
54
|
+
/**
|
|
55
|
+
* Prefix every entry key of the collection starts with. Stored plans keep keys without it (smaller plans); readers
|
|
56
|
+
* add it back. A plan listing a key outside it can't be encoded.
|
|
57
|
+
*/
|
|
58
|
+
entryPrefix(share: string): string;
|
|
59
|
+
}
|
|
60
|
+
/**
|
|
61
|
+
* Per-collection generation marker, read by the Worker on every request and every 30 s while streaming:
|
|
62
|
+
* { "gen": <n>, "seen": [<removal ids already accounted for>], "revoked": [{ "id": <planId>, "at": <ms> }] }
|
|
63
|
+
* Missing = generation 0.
|
|
64
|
+
*/
|
|
65
|
+
export interface GenMarker {
|
|
66
|
+
gen: number;
|
|
67
|
+
/** Ids of removals already accounted for by a bump (the product decides what an id is). */
|
|
68
|
+
seen: string[];
|
|
69
|
+
/**
|
|
70
|
+
* Single plans withdrawn without moving the generation: their links stop, other links of the collection don't.
|
|
71
|
+
* Cleared by a generation bump; entries expire after 31 days (plans live at most 30 days since their last write).
|
|
72
|
+
*/
|
|
73
|
+
revoked: {
|
|
74
|
+
id: string;
|
|
75
|
+
at: number;
|
|
76
|
+
}[];
|
|
77
|
+
}
|
|
78
|
+
/** A marker's text → its contents (missing → gen 0). Junk throws: never treated as valid. */
|
|
79
|
+
export declare function parseGen(text: string | null): GenMarker;
|
|
80
|
+
/** Is a plan of generation `gen` with id `planId` still served under this marker? */
|
|
81
|
+
export declare const planServable: (m: GenMarker, gen: number, planId: string) => boolean;
|
|
82
|
+
/** A plan id: base64url SHA-256 (43 characters). */
|
|
83
|
+
export declare const PLAN_ID: RegExp;
|
|
84
|
+
/** The plan's bytes as stored, and its id (base64url sha-256 of those bytes). Keys are stored without `keyPrefix`. */
|
|
85
|
+
export declare function encodePlan(p: Plan, keyPrefix: string): Promise<{
|
|
86
|
+
text: string;
|
|
87
|
+
id: string;
|
|
88
|
+
}>;
|
|
89
|
+
export declare function planIdOf(text: string | Uint8Array): Promise<string>;
|
|
90
|
+
/** An S3/R2 ETag without quotes or a weak prefix, lowercase. */
|
|
91
|
+
export declare const normEtag: (e: string) => string;
|
|
92
|
+
/** A stored plan, parsed: columns (no per-entry objects). Full key of entry i = keyPrefix + keys[i]. */
|
|
93
|
+
export interface ParsedPlan {
|
|
94
|
+
share: string;
|
|
95
|
+
name: string;
|
|
96
|
+
created: number;
|
|
97
|
+
gen: number;
|
|
98
|
+
count: number;
|
|
99
|
+
keyPrefix: string;
|
|
100
|
+
keys: string[];
|
|
101
|
+
etags: string[];
|
|
102
|
+
/** For ZipLayout. */
|
|
103
|
+
source: {
|
|
104
|
+
count: number;
|
|
105
|
+
path(i: number): string;
|
|
106
|
+
size(i: number): number;
|
|
107
|
+
crc32(i: number): number;
|
|
108
|
+
mtime(i: number): number;
|
|
109
|
+
};
|
|
110
|
+
}
|
|
111
|
+
/**
|
|
112
|
+
* Parse and check a stored plan of collection `shareId` whose keys live under `keyPrefix`. `keyOk` decides whether a
|
|
113
|
+
* full key may be read for this collection (the product's key policy). Throws on anything off.
|
|
114
|
+
*/
|
|
115
|
+
export declare function parsePlan(text: string, shareId: string, keyPrefix: string, keyOk: (key: string) => boolean): ParsedPlan;
|
package/dist/zip/sw.d.ts
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A retired service worker: what a product serves at the URL a service worker it no longer wants was registered from.
|
|
3
|
+
* Browsers that still have the old one check that URL for updates: this version takes over, removes itself, and
|
|
4
|
+
* reloads nothing. Serve it as JavaScript for a while after retiring the old worker. `note` is the comment at the top
|
|
5
|
+
* (lines, without comment markers), so a product keeps its own wording and its file's exact bytes.
|
|
6
|
+
*/
|
|
7
|
+
export declare function retiredServiceWorkerJs(note: string[]): string;
|