@uploadfilm/sdk 0.0.0-stage → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +4 -0
- package/README.md +529 -2
- package/dist/ai/embed.d.ts +58 -0
- package/dist/ai/fake.d.ts +41 -0
- package/dist/ai/http.d.ts +17 -0
- package/dist/ai/index.d.ts +27 -0
- package/dist/ai/languages.d.ts +2 -0
- package/dist/ai/ollama.d.ts +23 -0
- package/dist/ai/openaiCompatible.d.ts +54 -0
- package/dist/ai/passages.d.ts +28 -0
- package/dist/ai/transcribe.d.ts +84 -0
- package/dist/ai/types.d.ts +141 -0
- package/dist/ai/usage.d.ts +66 -0
- package/dist/ai/whisperJson.d.ts +15 -0
- package/dist/ai/workersAi.d.ts +56 -0
- package/dist/ai.js +1 -0
- package/dist/ai.mjs +1 -0
- package/dist/cards/files.d.ts +91 -0
- package/dist/cards/index.d.ts +15 -0
- package/dist/cards/layouts.d.ts +121 -0
- package/dist/cards/probe.d.ts +69 -0
- package/dist/cards/proxies.d.ts +70 -0
- package/dist/cards/select.d.ts +68 -0
- package/dist/cards/volumes.d.ts +170 -0
- package/dist/cards/xml.d.ts +80 -0
- package/dist/cards.js +1 -0
- package/dist/cards.mjs +1 -0
- package/dist/copy/checksums.d.ts +66 -0
- package/dist/copy/engine.d.ts +24 -0
- package/dist/copy/errors.d.ts +109 -0
- package/dist/copy/index.d.ts +15 -0
- package/dist/copy/steps.d.ts +28 -0
- package/dist/copy/sweep.d.ts +24 -0
- package/dist/copy/types.d.ts +363 -0
- package/dist/embed/player.js +2 -0
- package/dist/host/adapters.d.ts +107 -0
- package/dist/host/detect.d.ts +68 -0
- package/dist/host/encoders.d.ts +34 -0
- package/dist/host/environment.d.ts +53 -0
- package/dist/host/index.d.ts +20 -0
- package/dist/host/linux.d.ts +94 -0
- package/dist/host/mac.d.ts +47 -0
- package/dist/host/network.d.ts +54 -0
- package/dist/host/node.d.ts +13 -0
- package/dist/host/policy.d.ts +101 -0
- package/dist/host/tuner.d.ts +48 -0
- package/dist/host/types.d.ts +118 -0
- package/dist/host-node.js +1 -0
- package/dist/host-node.mjs +1 -0
- package/dist/host.js +1 -0
- package/dist/host.mjs +1 -0
- package/dist/images/index.d.ts +11 -0
- package/dist/images/settings.d.ts +43 -0
- package/dist/images/types.d.ts +83 -0
- package/dist/images/url.d.ts +65 -0
- package/dist/index.d.ts +16 -0
- package/dist/index.js +1 -0
- package/dist/index.mjs +1 -0
- package/dist/player/adapters/hlsjs.d.ts +28 -0
- package/dist/player/bandwidthMemory.d.ts +28 -0
- package/dist/player/captions/index.d.ts +55 -0
- package/dist/player/embed/code.d.ts +195 -0
- package/dist/player/embed/constants.d.ts +4 -0
- package/dist/player/embed/controller.d.ts +62 -0
- package/dist/player/embed/element.d.ts +32 -0
- package/dist/player/embed/frame.d.ts +65 -0
- package/dist/player/embed/global.d.ts +14 -0
- package/dist/player/embed/index.d.ts +15 -0
- package/dist/player/embed/loader.d.ts +1 -0
- package/dist/player/embed/oembed.d.ts +74 -0
- package/dist/player/embed/protocol.d.ts +64 -0
- package/dist/player/engine.d.ts +215 -0
- package/dist/player/index.d.ts +18 -0
- package/dist/player/start.d.ts +62 -0
- package/dist/player/store.d.ts +10 -0
- package/dist/player/storyboard/index.d.ts +161 -0
- package/dist/player/stream.d.ts +206 -0
- package/dist/player/tracks.d.ts +41 -0
- package/dist/player.js +1 -0
- package/dist/player.mjs +1 -0
- package/dist/proxies/args.d.ts +19 -0
- package/dist/proxies/budget.d.ts +57 -0
- package/dist/proxies/camera.d.ts +104 -0
- package/dist/proxies/formats.d.ts +36 -0
- package/dist/proxies/index.d.ts +18 -0
- package/dist/proxies/jobs.d.ts +98 -0
- package/dist/proxies/node.d.ts +32 -0
- package/dist/proxies/plan.d.ts +89 -0
- package/dist/proxies/probe.d.ts +39 -0
- package/dist/proxies/render.d.ts +104 -0
- package/dist/proxies/safety.d.ts +45 -0
- package/dist/proxies/types.d.ts +83 -0
- package/dist/proxies/upload.d.ts +14 -0
- package/dist/react/BigPlay.d.ts +26 -0
- package/dist/react/Player.d.ts +58 -0
- package/dist/react/embed/index.d.ts +45 -0
- package/dist/react/examples/CustomControls.d.ts +8 -0
- package/dist/react/images/Image.d.ts +16 -0
- package/dist/react/images/index.d.ts +4 -0
- package/dist/react/images/useImage.d.ts +47 -0
- package/dist/react/index.d.ts +17 -0
- package/dist/react/storyboard/index.d.ts +58 -0
- package/dist/react/theme.d.ts +37 -0
- package/dist/react/useAdapterErrors.d.ts +8 -0
- package/dist/react/usePlaybackStart.d.ts +18 -0
- package/dist/react/usePlayer.d.ts +28 -0
- package/dist/react/useReducedMotion.d.ts +2 -0
- package/dist/react/useTimedTrack.d.ts +22 -0
- package/dist/react.js +1 -0
- package/dist/react.mjs +1 -0
- package/dist/server/auth/apiKeys.d.ts +30 -0
- package/dist/server/auth/cache.d.ts +21 -0
- package/dist/server/auth/deviceTokens.d.ts +38 -0
- package/dist/server/auth/envelope.d.ts +15 -0
- package/dist/server/auth/grants.d.ts +76 -0
- package/dist/server/auth/index.d.ts +19 -0
- package/dist/server/auth/jwt.d.ts +65 -0
- package/dist/server/auth/keyring.d.ts +31 -0
- package/dist/server/auth/kv.d.ts +12 -0
- package/dist/server/auth/repository.d.ts +45 -0
- package/dist/server/auth/revocations.d.ts +21 -0
- package/dist/server/auth/scope.d.ts +4 -0
- package/dist/server/auth/serviceAuth.d.ts +24 -0
- package/dist/server/auth/signedUrls.d.ts +28 -0
- package/dist/server/embed/index.d.ts +36 -0
- package/dist/server/embed/lifecycle.d.ts +53 -0
- package/dist/server/embed/playerScript.d.ts +4 -0
- package/dist/server/images/index.d.ts +15 -0
- package/dist/server/images/job.d.ts +80 -0
- package/dist/server/images/layout.d.ts +24 -0
- package/dist/server/images/params.d.ts +90 -0
- package/dist/server/images/route.d.ts +101 -0
- package/dist/server/images/sign.d.ts +55 -0
- package/dist/server/index.d.ts +42 -0
- package/dist/server/media/client.d.ts +99 -0
- package/dist/server/media/frameCheck.d.ts +62 -0
- package/dist/server/media/index.d.ts +10 -0
- package/dist/server/media/protocol.d.ts +556 -0
- package/dist/server/media/storyboard.d.ts +144 -0
- package/dist/server/media/transcript.d.ts +143 -0
- package/dist/server/pipeline/index.d.ts +12 -0
- package/dist/server/pipeline/match.d.ts +10 -0
- package/dist/server/pipeline/pipeline.d.ts +182 -0
- package/dist/server/pipeline/queues.d.ts +108 -0
- package/dist/server/pipeline/sources.d.ts +54 -0
- package/dist/server/r2/client.d.ts +164 -0
- package/dist/server/r2/crc32.d.ts +24 -0
- package/dist/server/r2/encoding.d.ts +35 -0
- package/dist/server/r2/errors.d.ts +9 -0
- package/dist/server/r2/integrity.d.ts +25 -0
- package/dist/server/r2/parts.d.ts +46 -0
- package/dist/server/r2/sign.d.ts +47 -0
- package/dist/server/redis/index.d.ts +39 -0
- package/dist/server/state/memory.d.ts +18 -0
- package/dist/server/state/r2.d.ts +73 -0
- package/dist/server/state/types.d.ts +57 -0
- package/dist/server/storage/chunked.d.ts +25 -0
- package/dist/server/storage/index.d.ts +12 -0
- package/dist/server/storage/r2.d.ts +96 -0
- package/dist/server/storage/s3.d.ts +144 -0
- package/dist/server/testing/fake-r2.d.ts +410 -0
- package/dist/server/testing/index.d.ts +10 -0
- package/dist/server/testing/kv-contract.d.ts +14 -0
- package/dist/server/transfer/handler.d.ts +217 -0
- package/dist/server/transfer/keys.d.ts +45 -0
- package/dist/server/transfer/protocol.d.ts +190 -0
- package/dist/server/zip/client.d.ts +32 -0
- package/dist/server/zip/generations.d.ts +83 -0
- package/dist/server/zip/index.d.ts +12 -0
- package/dist/server/zip/planner.d.ts +94 -0
- package/dist/server/zip/sign.d.ts +22 -0
- package/dist/server-media.js +1 -0
- package/dist/server-media.mjs +1 -0
- package/dist/server-redis.js +1 -0
- package/dist/server-redis.mjs +1 -0
- package/dist/server-testing.js +1 -0
- package/dist/server-testing.mjs +1 -0
- package/dist/server.js +1 -0
- package/dist/server.mjs +1 -0
- package/dist/storage/browserFile.d.ts +19 -0
- package/dist/storage/index.d.ts +12 -0
- package/dist/storage/local.d.ts +9 -0
- package/dist/storage/presigned.d.ts +22 -0
- package/dist/sync/ffmpeg.d.ts +44 -0
- package/dist/sync/index.d.ts +17 -0
- package/dist/sync/match.d.ts +28 -0
- package/dist/sync/probe.d.ts +30 -0
- package/dist/sync/timecode.d.ts +50 -0
- package/dist/sync/transcript.d.ts +35 -0
- package/dist/sync/types.d.ts +172 -0
- package/dist/sync/waveform.d.ts +89 -0
- package/dist/sync.js +1 -0
- package/dist/sync.mjs +1 -0
- package/dist/transfer/claims.d.ts +41 -0
- package/dist/transfer/client.d.ts +267 -0
- package/dist/transfer/download.d.ts +192 -0
- package/dist/transfer/engine.d.ts +402 -0
- package/dist/transfer/errors.d.ts +50 -0
- package/dist/transfer/growing/bytes.d.ts +17 -0
- package/dist/transfer/growing/index.d.ts +12 -0
- package/dist/transfer/growing/parts.d.ts +157 -0
- package/dist/transfer/growing/retry.d.ts +38 -0
- package/dist/transfer/growing/store.d.ts +66 -0
- package/dist/transfer/growing/upload.d.ts +126 -0
- package/dist/transfer/index.d.ts +40 -0
- package/dist/transfer/integrity.d.ts +93 -0
- package/dist/transfer/limits.d.ts +34 -0
- package/dist/transfer/node.d.ts +63 -0
- package/dist/transfer/pool.d.ts +83 -0
- package/dist/transfer/protocol.d.ts +172 -0
- package/dist/transfer/signer.d.ts +29 -0
- package/dist/transfer/storage.d.ts +135 -0
- package/dist/transfer/timing.d.ts +132 -0
- package/dist/transfer/transports/electronNet.d.ts +62 -0
- package/dist/transfer/transports/fetch.d.ts +15 -0
- package/dist/transfer/transports/retryAfter.d.ts +12 -0
- package/dist/transfer/transports/undici.d.ts +72 -0
- package/dist/transfer/tuner.d.ts +56 -0
- package/dist/workers/cli.mjs +115 -0
- package/dist/workers/media/config.mjs +53 -0
- package/dist/workers/media/container/Dockerfile +19 -0
- package/dist/workers/media/container/FFMPEG.md +129 -0
- package/dist/workers/media/container/runner.mjs +2690 -0
- package/dist/workers/media/index.mjs +1011 -0
- package/dist/workers/media/init/config.ts +52 -0
- package/dist/workers/media/init/worker.ts +8 -0
- package/dist/workers/media/init/wrangler.toml +44 -0
- package/dist/workers/media/runner.mjs +2656 -0
- package/dist/workers/media/types/build/entry/config.d.mts +6 -0
- package/dist/workers/media/types/build/entry/index.d.mts +43 -0
- package/dist/workers/media/types/build/entry/program.d.mts +13 -0
- package/dist/workers/media/types/build/entry/runner.d.mts +12 -0
- package/dist/workers/media/types/build/entry/settings.d.mts +40 -0
- package/dist/workers/media/types/src/config.d.mts +135 -0
- package/dist/workers/media/types/src/jobs.d.mts +73 -0
- package/dist/workers/media/types/src/runner/container.d.mts +32 -0
- package/dist/workers/media/types/src/runner/ffmpeg.d.mts +227 -0
- package/dist/workers/media/types/src/runner/job.d.mts +107 -0
- package/dist/workers/media/types/src/runner/node.d.mts +71 -0
- package/dist/workers/media/types/src/runner/plugins.d.mts +45 -0
- package/dist/workers/media/types/src/serve.d.mts +107 -0
- package/dist/workers/media/types/src/steps/audio.d.mts +33 -0
- package/dist/workers/media/types/src/steps/edit.d.mts +62 -0
- package/dist/workers/media/types/src/steps/frameCheck.d.mts +44 -0
- package/dist/workers/media/types/src/steps/gate.d.mts +18 -0
- package/dist/workers/media/types/src/steps/paths.d.mts +1 -0
- package/dist/workers/media/types/src/steps/prep.d.mts +60 -0
- package/dist/workers/media/types/src/steps/render.d.mts +40 -0
- package/dist/workers/media/types/src/steps/runner.d.mts +30 -0
- package/dist/workers/media/types/src/steps/transcribe.d.mts +89 -0
- package/dist/workers/media/types/src/storyboard/index.d.mts +61 -0
- package/dist/workers/media/types/src/transcoder.d.mts +30 -0
- package/dist/workers/media/types/src/worker.d.mts +30 -0
- package/dist/workers/zip/index.mjs +589 -0
- package/dist/workers/zip/init/config.ts +22 -0
- package/dist/workers/zip/init/worker.ts +5 -0
- package/dist/workers/zip/init/wrangler.toml +25 -0
- package/dist/workers/zip/types/build/entry/index.d.mts +5 -0
- package/dist/workers/zip/types/src/config.d.mts +35 -0
- package/dist/workers/zip/types/src/serve.d.mts +86 -0
- package/dist/workers/zip/types/src/worker.d.mts +12 -0
- package/dist/zip/index.d.ts +13 -0
- package/dist/zip/layout.d.ts +129 -0
- package/dist/zip/names.d.ts +34 -0
- package/dist/zip/plan.d.ts +115 -0
- package/dist/zip/sw.d.ts +7 -0
- package/package.json +313 -4
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Storyboards: the timeline hover previews a job's `storyboard` step makes, from the job's ONE decode of the source.
|
|
3
|
+
*
|
|
4
|
+
* storyboard/sb-NNN.webp sheets of `cols`×`rows` thumbnails (~160 px on the long side), WebP
|
|
5
|
+
* storyboard/storyboard.vtt a WebVTT "thumbnails" track: one cue per thumbnail, `sb-NNN.webp#xywh=x,y,w,h`
|
|
6
|
+
* storyboard/storyboard.json the same as data (StoryboardIndex), written last: the step's completion file
|
|
7
|
+
*
|
|
8
|
+
* One thumbnail every `intervalS` seconds (default 2), stretched to `seconds / maxFrames` for a long video so a job
|
|
9
|
+
* never makes more than `maxFrames` (default 200) thumbnails, and the sheets stay within a byte budget (a 10-minute
|
|
10
|
+
* video: well under 1 MB in all).
|
|
11
|
+
*
|
|
12
|
+
* Shared by the media worker (which makes these files: sdks/media-worker/src/storyboard), the site (descriptors for
|
|
13
|
+
* its pages: storyboardDescriptor) and tests. SELF-CONTAINED ON PURPOSE, like ./protocol.ts: no imports and only
|
|
14
|
+
* erasable TypeScript, so plain Node, workerd, a bundler and the ffmpeg container all load this one file by path.
|
|
15
|
+
*
|
|
16
|
+
* The files under a job's prefix never change once written (the prefix is named by the source and the settings
|
|
17
|
+
* version): serve them as immutable.
|
|
18
|
+
*/
|
|
19
|
+
/**
|
|
20
|
+
* Everything that decides what a storyboard looks like. `version` names these settings: change it whenever anything
|
|
21
|
+
* here changes (outputs made with other settings must not be served as these).
|
|
22
|
+
*/
|
|
23
|
+
export interface StoryboardSettings {
|
|
24
|
+
version: string;
|
|
25
|
+
/** Seconds between thumbnails for a short video. */
|
|
26
|
+
intervalS: number;
|
|
27
|
+
/** At most this many thumbnails: a longer video gets a longer interval (seconds / maxFrames). */
|
|
28
|
+
maxFrames: number;
|
|
29
|
+
/** A thumbnail's long side, in pixels (the short side follows the picture, even). */
|
|
30
|
+
size: number;
|
|
31
|
+
/** Thumbnails per sheet: `cols` across, `rows` down. */
|
|
32
|
+
cols: number;
|
|
33
|
+
rows: number;
|
|
34
|
+
/** WebP quality (0–100) the sheets are made with first… */
|
|
35
|
+
quality: number;
|
|
36
|
+
/** …and the lowest it may go to fit the budget. */
|
|
37
|
+
minQuality: number;
|
|
38
|
+
/** Byte budget: a sheet at most this… */
|
|
39
|
+
maxSheetBytes: number;
|
|
40
|
+
/** …and every sheet together at most this. */
|
|
41
|
+
maxBytes: number;
|
|
42
|
+
}
|
|
43
|
+
/** The default storyboard settings with the given version (and any overrides). */
|
|
44
|
+
export declare function storyboardSettings(version: string, over?: Partial<Omit<StoryboardSettings, "version">>): StoryboardSettings;
|
|
45
|
+
export declare const DEFAULT_STORYBOARD_SETTINGS: StoryboardSettings;
|
|
46
|
+
/** Throws on settings no storyboard can be made with (a product's config mistake, caught at startup). */
|
|
47
|
+
export declare function checkSettings(s: StoryboardSettings): void;
|
|
48
|
+
/** What a storyboard of one picture is: thumbnails every `interval` s, each `width`×`height`, `cols`×`rows` a sheet. */
|
|
49
|
+
export interface StoryboardPlan {
|
|
50
|
+
/** Seconds between thumbnails (3 decimals: the cue times are exact multiples of it). */
|
|
51
|
+
interval: number;
|
|
52
|
+
/** At most this many thumbnails. */
|
|
53
|
+
maxFrames: number;
|
|
54
|
+
/** Thumbnails the picture's length asks for (the made count may differ by one at the very end: see the runner). */
|
|
55
|
+
expected: number;
|
|
56
|
+
width: number;
|
|
57
|
+
height: number;
|
|
58
|
+
cols: number;
|
|
59
|
+
rows: number;
|
|
60
|
+
/** The picture's length (0: unknown). */
|
|
61
|
+
seconds: number;
|
|
62
|
+
}
|
|
63
|
+
/**
|
|
64
|
+
* The plan for a picture of this displayed size and length. `interval = max(intervalS, seconds / maxFrames)`, rounded
|
|
65
|
+
* up to the millisecond, so `seconds / interval` never exceeds maxFrames.
|
|
66
|
+
*/
|
|
67
|
+
export declare function planStoryboard(picture: {
|
|
68
|
+
width: number;
|
|
69
|
+
height: number;
|
|
70
|
+
seconds: number;
|
|
71
|
+
}, s?: StoryboardSettings): StoryboardPlan;
|
|
72
|
+
export declare const STORYBOARD_DIR = "storyboard";
|
|
73
|
+
export declare const STORYBOARD_VTT = "storyboard/storyboard.vtt";
|
|
74
|
+
export declare const STORYBOARD_INDEX = "storyboard/storyboard.json";
|
|
75
|
+
/** Sheet `i` (0-based) under the job's prefix. */
|
|
76
|
+
export declare const storyboardSheetName: (i: number) => string;
|
|
77
|
+
/** Is `rel` (a path under a job's prefix) one of a storyboard's files? */
|
|
78
|
+
export declare function storyboardFile(rel: string): "storyboard" | "storyboard-vtt" | "storyboard-index" | null;
|
|
79
|
+
/** Sheets `count` thumbnails fill. */
|
|
80
|
+
export declare const sheetCount: (plan: Pick<StoryboardPlan, "cols" | "rows">, count: number) => number;
|
|
81
|
+
/** One thumbnail: shown from `start` to `end` (seconds), the rectangle x,y,w,h of sheet `sheet` (0-based). */
|
|
82
|
+
export interface StoryboardCue {
|
|
83
|
+
start: number;
|
|
84
|
+
end: number;
|
|
85
|
+
sheet: number;
|
|
86
|
+
x: number;
|
|
87
|
+
y: number;
|
|
88
|
+
w: number;
|
|
89
|
+
h: number;
|
|
90
|
+
}
|
|
91
|
+
/**
|
|
92
|
+
* The cues of `count` thumbnails made with `plan`: thumbnail i shows [i·interval, (i+1)·interval), the last one until
|
|
93
|
+
* the picture's end (when it is known and later).
|
|
94
|
+
*/
|
|
95
|
+
export declare function storyboardCues(plan: StoryboardPlan, count: number): StoryboardCue[];
|
|
96
|
+
/**
|
|
97
|
+
* The WebVTT thumbnails track. Sheet URLs are relative to the VTT file itself (same directory), so the track works
|
|
98
|
+
* from wherever the job's files are served (a CDN, a signed path), and players that read `#xywh` (Video.js, Media
|
|
99
|
+
* Chrome, Vidstack, JW Player) show it as is.
|
|
100
|
+
*/
|
|
101
|
+
export declare function storyboardVtt(plan: StoryboardPlan, count: number): string;
|
|
102
|
+
/** storyboard/storyboard.json: the storyboard as data (a player can use it without reading the VTT). */
|
|
103
|
+
export interface StoryboardIndex {
|
|
104
|
+
/** The settings version it was made with. */
|
|
105
|
+
version: string;
|
|
106
|
+
interval: number;
|
|
107
|
+
cols: number;
|
|
108
|
+
rows: number;
|
|
109
|
+
width: number;
|
|
110
|
+
height: number;
|
|
111
|
+
/** Thumbnails made. */
|
|
112
|
+
count: number;
|
|
113
|
+
/** The picture's length, in seconds. */
|
|
114
|
+
seconds: number;
|
|
115
|
+
/** Paths under the job's prefix, in order. */
|
|
116
|
+
sheets: string[];
|
|
117
|
+
vtt: string;
|
|
118
|
+
/** Bytes of every sheet together. */
|
|
119
|
+
bytes: number;
|
|
120
|
+
}
|
|
121
|
+
export declare function storyboardIndex(plan: StoryboardPlan, count: number, version: string, bytes: number): StoryboardIndex;
|
|
122
|
+
/** Is this a storyboard index a player can use? (A fetched JSON is checked before anything trusts its numbers.) */
|
|
123
|
+
export declare function isStoryboardIndex(x: unknown): x is StoryboardIndex;
|
|
124
|
+
/**
|
|
125
|
+
* What a page hands the player: absolute URLs of the track and its sheets, and the layout. `base` is the URL of the
|
|
126
|
+
* job's prefix as the viewer reaches it (with any token path), ending in "/".
|
|
127
|
+
*
|
|
128
|
+
* storyboardDescriptor(base) the VTT's URL only (the player fetches and reads it)
|
|
129
|
+
* storyboardDescriptor(base, index) everything (no VTT fetch: getPreviewAt answers at once)
|
|
130
|
+
*/
|
|
131
|
+
export interface StoryboardDescriptor {
|
|
132
|
+
vtt: string;
|
|
133
|
+
index: string;
|
|
134
|
+
/** With an index: the layout and every sheet's URL. */
|
|
135
|
+
interval?: number;
|
|
136
|
+
cols?: number;
|
|
137
|
+
rows?: number;
|
|
138
|
+
width?: number;
|
|
139
|
+
height?: number;
|
|
140
|
+
count?: number;
|
|
141
|
+
seconds?: number;
|
|
142
|
+
sheets?: string[];
|
|
143
|
+
}
|
|
144
|
+
export declare function storyboardDescriptor(base: string, index?: StoryboardIndex): StoryboardDescriptor;
|
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The `transcript` step: speech to text for a job's audio chunks (audio/a-NNN.ogg, Opus mono 16 kHz, from the job's one
|
|
3
|
+
* decode), made by the media Worker after the container is done, stored as transcript.vtt (captions) and
|
|
4
|
+
* transcript.json (segments with words) under the job's prefix.
|
|
5
|
+
*
|
|
6
|
+
* Silence never becomes text. Speech-to-text models invent words over silence ("Thank you.", "Subtitles by…"); so the
|
|
7
|
+
* container measures silence on the same decode (FFmpeg's silencedetect, StepSettings.vad) and lists it in
|
|
8
|
+
* audio/audio.json; a chunk that is silence end to end is never sent, and a segment the model returns inside a silence
|
|
9
|
+
* is dropped (buildTranscript).
|
|
10
|
+
*
|
|
11
|
+
* SELF-CONTAINED ON PURPOSE (like ./protocol.ts): no imports, erasable TypeScript only; the Worker, the site and the
|
|
12
|
+
* player's tests load this one file.
|
|
13
|
+
*/
|
|
14
|
+
/** A stretch of silence, seconds from the start of the source's audio. `end` is the audio's end when it runs to it. */
|
|
15
|
+
export interface Silence {
|
|
16
|
+
start: number;
|
|
17
|
+
end: number;
|
|
18
|
+
}
|
|
19
|
+
/** One audio chunk, as audio/audio.json lists it. */
|
|
20
|
+
export interface AudioChunk {
|
|
21
|
+
/** Name under the job's prefix (audio/a-NNN.ogg). */
|
|
22
|
+
file: string;
|
|
23
|
+
/** Seconds from the start of the source's audio. */
|
|
24
|
+
start: number;
|
|
25
|
+
end: number;
|
|
26
|
+
}
|
|
27
|
+
/** audio/audio.json */
|
|
28
|
+
export interface AudioIndex {
|
|
29
|
+
format: 1;
|
|
30
|
+
/** Opus, mono. */
|
|
31
|
+
codec: "opus";
|
|
32
|
+
sampleRate: number;
|
|
33
|
+
/** False when the source has no audio stream: no chunks, an empty transcript. */
|
|
34
|
+
hasAudio: boolean;
|
|
35
|
+
/** Seconds of audio. */
|
|
36
|
+
seconds: number;
|
|
37
|
+
chunks: AudioChunk[];
|
|
38
|
+
silences: Silence[];
|
|
39
|
+
/** Silence settings these were measured with. */
|
|
40
|
+
vad: {
|
|
41
|
+
noiseDb: number;
|
|
42
|
+
minSilenceS: number;
|
|
43
|
+
};
|
|
44
|
+
}
|
|
45
|
+
/** What Workers AI's Whisper models answer (@cf/openai/whisper-large-v3-turbo); only what we read. */
|
|
46
|
+
export interface WhisperResult {
|
|
47
|
+
text?: string;
|
|
48
|
+
transcription_info?: {
|
|
49
|
+
language?: string;
|
|
50
|
+
language_probability?: number;
|
|
51
|
+
duration?: number;
|
|
52
|
+
duration_after_vad?: number;
|
|
53
|
+
};
|
|
54
|
+
segments?: {
|
|
55
|
+
start: number;
|
|
56
|
+
end: number;
|
|
57
|
+
text: string;
|
|
58
|
+
no_speech_prob?: number;
|
|
59
|
+
avg_logprob?: number;
|
|
60
|
+
words?: {
|
|
61
|
+
word: string;
|
|
62
|
+
start: number;
|
|
63
|
+
end: number;
|
|
64
|
+
}[];
|
|
65
|
+
}[];
|
|
66
|
+
vtt?: string;
|
|
67
|
+
}
|
|
68
|
+
export interface TranscriptWord {
|
|
69
|
+
word: string;
|
|
70
|
+
start: number;
|
|
71
|
+
end: number;
|
|
72
|
+
}
|
|
73
|
+
export interface TranscriptSegment {
|
|
74
|
+
start: number;
|
|
75
|
+
end: number;
|
|
76
|
+
text: string;
|
|
77
|
+
words?: TranscriptWord[];
|
|
78
|
+
}
|
|
79
|
+
/** transcript.json */
|
|
80
|
+
export interface Transcript {
|
|
81
|
+
format: 1;
|
|
82
|
+
/** The model that made it. */
|
|
83
|
+
model: string;
|
|
84
|
+
/** Detected (or forced) language, when the model said. */
|
|
85
|
+
language?: string;
|
|
86
|
+
/** Seconds of audio. */
|
|
87
|
+
seconds: number;
|
|
88
|
+
segments: TranscriptSegment[];
|
|
89
|
+
/** The whole text, segments joined by spaces. */
|
|
90
|
+
text: string;
|
|
91
|
+
/** Segments the model returned inside silence, dropped. */
|
|
92
|
+
dropped: number;
|
|
93
|
+
/** Chunks not sent at all (silence end to end). */
|
|
94
|
+
silentChunks: number;
|
|
95
|
+
}
|
|
96
|
+
export interface VadSettings {
|
|
97
|
+
/** A segment overlapping silence with at most this many seconds of it outside silence is dropped. */
|
|
98
|
+
edgeS: number;
|
|
99
|
+
}
|
|
100
|
+
/** …and only when silence covers at least this share of it (speech next to a silence is never dropped). */
|
|
101
|
+
export declare const SILENT_SHARE = 0.8;
|
|
102
|
+
/** Seconds of [start, end] that are silence. */
|
|
103
|
+
export declare function silentSeconds(start: number, end: number, silences: readonly Silence[]): number;
|
|
104
|
+
/**
|
|
105
|
+
* Is [start, end] the model talking over silence? Silence covers at least SILENT_SHARE of it and at most `edge` seconds
|
|
106
|
+
* of it are outside silence. A point (start = end) is silent when it lies in a silence. Speech merely next to a silence
|
|
107
|
+
* is never dropped.
|
|
108
|
+
*/
|
|
109
|
+
export declare function inSilence(start: number, end: number, silences: readonly Silence[], edge: number): boolean;
|
|
110
|
+
/** Is this chunk silence from end to end? (Then it isn't sent.) */
|
|
111
|
+
export declare const chunkSilent: (c: AudioChunk, silences: readonly Silence[]) => boolean;
|
|
112
|
+
/**
|
|
113
|
+
* The transcript from each sent chunk's model result (with the chunk's offset), dropping segments inside silence.
|
|
114
|
+
* `results` in chunk order; chunks not sent are counted in `silentChunks`.
|
|
115
|
+
*/
|
|
116
|
+
export declare function buildTranscript(o: {
|
|
117
|
+
model: string;
|
|
118
|
+
seconds: number;
|
|
119
|
+
silences: readonly Silence[];
|
|
120
|
+
results: readonly {
|
|
121
|
+
chunk: AudioChunk;
|
|
122
|
+
result: WhisperResult;
|
|
123
|
+
}[];
|
|
124
|
+
silentChunks?: number;
|
|
125
|
+
language?: string;
|
|
126
|
+
}, vad: VadSettings): Transcript;
|
|
127
|
+
/** 3725.5 → "01:02:05.500" (WebVTT timestamps always with hours: any length reads the same). */
|
|
128
|
+
export declare function vttTime(t: number): string;
|
|
129
|
+
/** Cue text that can't break the file: no blank lines, no "-->", and `<`/`&` escaped (WebVTT reads tags). */
|
|
130
|
+
export declare const vttText: (t: string) => string;
|
|
131
|
+
/** transcript.vtt: one cue per segment (a zero-length segment shows for half a second). */
|
|
132
|
+
export declare function toWebVtt(t: Pick<Transcript, "segments" | "language">): string;
|
|
133
|
+
/** A stored transcript.json, checked: null when it isn't one this version knows. */
|
|
134
|
+
export declare function readTranscript(json: string): Transcript | null;
|
|
135
|
+
/** A stored audio/audio.json, checked. */
|
|
136
|
+
export declare function readAudioIndex(json: string): AudioIndex | null;
|
|
137
|
+
/** Where a job's optional outputs live (keys under the job's prefix), for a site that stores or serves them. */
|
|
138
|
+
export declare function mediaStepFiles(prefix: string): {
|
|
139
|
+
audioIndex: string;
|
|
140
|
+
transcriptVtt: string;
|
|
141
|
+
transcriptJson: string;
|
|
142
|
+
frameCheck: string;
|
|
143
|
+
};
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The upload-completed pipeline (RFC §2.4): sources → dedupe → jobs with key filters → a queue module (Inngest,
|
|
3
|
+
* Cloudflare Queues, in-memory). Jobs end `jobDone` (proven), `alert` (unproven or failed for good) or `jobRetry`.
|
|
4
|
+
*/
|
|
5
|
+
export { createPipeline, messageId, provenCopy, errorInfo } from "./pipeline";
|
|
6
|
+
export type { DedupeStore, ErrorInfo, Job, JobContext, JobMessage, Pipeline, PipelineEvent, PipelineOptions, QueueModule, RunOutcome, UploadCompleted, } from "./pipeline";
|
|
7
|
+
export { globToRegExp, matcher } from "./match";
|
|
8
|
+
export type { KeyFilter } from "./match";
|
|
9
|
+
export { fromR2Notification, fromTransferComplete, r2Notifications } from "./sources";
|
|
10
|
+
export type { MessageBatchLike, QueueMessageLike, R2Notification } from "./sources";
|
|
11
|
+
export { memoryQueue, inngestQueue, cloudflareQueue } from "./queues";
|
|
12
|
+
export type { CloudflareQueue, CloudflareQueueLike, CloudflareQueueOptions, InngestLike, InngestQueue, InngestQueueOptions, MemoryOutcome, MemoryQueue, MemoryQueueOptions, } from "./queues";
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Key filters for pipeline jobs: globs over the product's key shapes, RegExps or predicates.
|
|
3
|
+
* `*` any characters within one path segment (not `/`)
|
|
4
|
+
* `**` any characters, `/` included
|
|
5
|
+
* `?` one character within a segment
|
|
6
|
+
* A glob matches the whole key: `originals/*` matches `originals/a.mov`, not `originals/x/a.mov` (`originals/**`).
|
|
7
|
+
*/
|
|
8
|
+
export type KeyFilter = string | RegExp | ((key: string) => boolean) | (string | RegExp)[];
|
|
9
|
+
export declare function globToRegExp(glob: string): RegExp;
|
|
10
|
+
export declare function matcher(f: KeyFilter): (key: string) => boolean;
|
|
@@ -0,0 +1,182 @@
|
|
|
1
|
+
import type { StepRunner } from "../../copy/types";
|
|
2
|
+
import { type KeyFilter } from "./match";
|
|
3
|
+
export interface UploadCompleted {
|
|
4
|
+
key: string;
|
|
5
|
+
size: number;
|
|
6
|
+
/** Bare ETag of the version that landed. */
|
|
7
|
+
etag: string;
|
|
8
|
+
versionId?: string;
|
|
9
|
+
bucket?: string;
|
|
10
|
+
/** Where the event came from ("r2-notification", "upload-hook", …). */
|
|
11
|
+
source?: string;
|
|
12
|
+
/** ms since epoch. */
|
|
13
|
+
at?: number;
|
|
14
|
+
}
|
|
15
|
+
export interface JobContext {
|
|
16
|
+
/** The job's steps (Inngest's step, or the queue module's memoized in-process steps). */
|
|
17
|
+
step: StepRunner;
|
|
18
|
+
/** 0 on the first run. */
|
|
19
|
+
attempt: number;
|
|
20
|
+
/** This is the last attempt the queue will make. */
|
|
21
|
+
final: boolean;
|
|
22
|
+
/** The message id (job + key + ETag). */
|
|
23
|
+
id: string;
|
|
24
|
+
/**
|
|
25
|
+
* Whether `run` reports a final failure itself (default true). Inngest's module passes false: there a step that
|
|
26
|
+
* fails for good fails the run before the job's code sees it, so its `onFailure` handler reports it, once.
|
|
27
|
+
*/
|
|
28
|
+
reportFailure?: boolean;
|
|
29
|
+
}
|
|
30
|
+
export interface Job<R = unknown> {
|
|
31
|
+
name: string;
|
|
32
|
+
/** Glob(s) over keys (`*` within a segment, `**` across), a RegExp, or a predicate. */
|
|
33
|
+
match: KeyFilter;
|
|
34
|
+
run(e: UploadCompleted, ctx: JobContext): Promise<R>;
|
|
35
|
+
/**
|
|
36
|
+
* Whether a result is proven. Default: a CopyResult-shaped result (`state`) is proven only when `verified`, or
|
|
37
|
+
* `alreadyCopied` with `verified: true`, or `sourceGone`; anything without a `state` counts as done.
|
|
38
|
+
*/
|
|
39
|
+
proven?: (r: R) => {
|
|
40
|
+
ok: true;
|
|
41
|
+
} | {
|
|
42
|
+
ok: false;
|
|
43
|
+
reason: string;
|
|
44
|
+
alert?: string;
|
|
45
|
+
};
|
|
46
|
+
}
|
|
47
|
+
export interface JobMessage {
|
|
48
|
+
id: string;
|
|
49
|
+
job: string;
|
|
50
|
+
event: UploadCompleted;
|
|
51
|
+
/** Set on a forced resend (ms): queues that dedupe by id (Inngest) give it its own delivery id. */
|
|
52
|
+
resend?: number;
|
|
53
|
+
}
|
|
54
|
+
export type PipelineEvent = {
|
|
55
|
+
type: "received";
|
|
56
|
+
event: UploadCompleted;
|
|
57
|
+
} | {
|
|
58
|
+
type: "ignored";
|
|
59
|
+
event?: UploadCompleted;
|
|
60
|
+
reason: string;
|
|
61
|
+
} | {
|
|
62
|
+
type: "duplicate";
|
|
63
|
+
id: string;
|
|
64
|
+
job: string;
|
|
65
|
+
event: UploadCompleted;
|
|
66
|
+
} | {
|
|
67
|
+
type: "enqueued";
|
|
68
|
+
id: string;
|
|
69
|
+
job: string;
|
|
70
|
+
event: UploadCompleted;
|
|
71
|
+
} | {
|
|
72
|
+
type: "jobDone";
|
|
73
|
+
id: string;
|
|
74
|
+
job: string;
|
|
75
|
+
event: UploadCompleted;
|
|
76
|
+
result: unknown;
|
|
77
|
+
} | {
|
|
78
|
+
type: "jobRetry";
|
|
79
|
+
id: string;
|
|
80
|
+
job: string;
|
|
81
|
+
event: UploadCompleted;
|
|
82
|
+
attempt: number;
|
|
83
|
+
error: ErrorInfo;
|
|
84
|
+
} | {
|
|
85
|
+
type: "alert";
|
|
86
|
+
id: string;
|
|
87
|
+
job: string;
|
|
88
|
+
event: UploadCompleted;
|
|
89
|
+
reason: string;
|
|
90
|
+
alert: string;
|
|
91
|
+
error?: ErrorInfo;
|
|
92
|
+
result?: unknown;
|
|
93
|
+
};
|
|
94
|
+
export interface ErrorInfo {
|
|
95
|
+
message: string;
|
|
96
|
+
code?: string;
|
|
97
|
+
alert?: string;
|
|
98
|
+
retryable: boolean;
|
|
99
|
+
}
|
|
100
|
+
export interface QueueModule {
|
|
101
|
+
readonly name: string;
|
|
102
|
+
enqueue(msgs: JobMessage[]): Promise<void>;
|
|
103
|
+
/** Called once by createPipeline so the module can run messages (memory queue) or build handlers. */
|
|
104
|
+
bind?(p: Pipeline): void;
|
|
105
|
+
}
|
|
106
|
+
/** KeyValueStore-shaped dedupe index (set with ifAbsent). */
|
|
107
|
+
export interface DedupeStore {
|
|
108
|
+
set(key: string, value: string, o?: {
|
|
109
|
+
ttlMs?: number;
|
|
110
|
+
ifAbsent?: boolean;
|
|
111
|
+
}): Promise<boolean>;
|
|
112
|
+
del(key: string): Promise<void>;
|
|
113
|
+
/** Tells a reservation still pending (another ingest under way, or a crashed one) from a confirmed one. */
|
|
114
|
+
get?(key: string): Promise<string | null>;
|
|
115
|
+
}
|
|
116
|
+
export interface PipelineOptions {
|
|
117
|
+
jobs: Job[];
|
|
118
|
+
queue: QueueModule;
|
|
119
|
+
/** Dedupe index; default in-memory (one process). Pass the product's Redis store across instances. */
|
|
120
|
+
dedupe?: DedupeStore;
|
|
121
|
+
/** How long a job + key + ETag stays deduplicated (default 7 days). */
|
|
122
|
+
dedupeTtlMs?: number;
|
|
123
|
+
/**
|
|
124
|
+
* How long a reservation made before the queue accepted the messages holds (default 10 min): a process that dies
|
|
125
|
+
* between the two leaves only this, so a redelivery after it runs the job (never a job lost for 7 days).
|
|
126
|
+
*/
|
|
127
|
+
pendingTtlMs?: number;
|
|
128
|
+
/** Prefix of the dedupe keys (default "pipeline"): two pipelines sharing a store never dedupe each other. */
|
|
129
|
+
name?: string;
|
|
130
|
+
/**
|
|
131
|
+
* Part of the message id before the key, for products with several buckets: e.g. `(e) => e.bucket ?? ""`. Default
|
|
132
|
+
* none (one bucket: a notification and the hook for the same key + ETag are the same message).
|
|
133
|
+
*/
|
|
134
|
+
scope?: (e: UploadCompleted) => string;
|
|
135
|
+
/** Telemetry: every event, synchronously; a throwing listener is ignored. */
|
|
136
|
+
onEvent?: (e: PipelineEvent) => void;
|
|
137
|
+
/**
|
|
138
|
+
* Where alerts must arrive (a pager, the product's DB). Awaited: when it throws, the message is retried instead
|
|
139
|
+
* of acknowledged, so an alert is never lost with the message.
|
|
140
|
+
*/
|
|
141
|
+
onAlert?: (e: Extract<PipelineEvent, {
|
|
142
|
+
type: "alert";
|
|
143
|
+
}>) => Promise<void> | void;
|
|
144
|
+
}
|
|
145
|
+
export interface RunOutcome {
|
|
146
|
+
state: "done" | "alert" | "retry";
|
|
147
|
+
result?: unknown;
|
|
148
|
+
error?: unknown;
|
|
149
|
+
}
|
|
150
|
+
export interface Pipeline {
|
|
151
|
+
readonly jobs: readonly Job[];
|
|
152
|
+
/** Fan one or more events out to the matching jobs. `force`: skip the dedupe (a manual resend). */
|
|
153
|
+
ingest(e: UploadCompleted | UploadCompleted[], o?: {
|
|
154
|
+
force?: boolean;
|
|
155
|
+
}): Promise<{
|
|
156
|
+
enqueued: string[];
|
|
157
|
+
duplicates: string[];
|
|
158
|
+
}>;
|
|
159
|
+
/** Run one message (queue modules call this). Throws when the queue should retry it. */
|
|
160
|
+
run(msg: JobMessage, ctx: Omit<JobContext, "id"> & {
|
|
161
|
+
id?: string;
|
|
162
|
+
}): Promise<RunOutcome>;
|
|
163
|
+
/** Report a message the queue gave up on outside `run` (Inngest onFailure, a dead-letter queue). */
|
|
164
|
+
alert(msg: JobMessage, error: unknown, reason?: string): Promise<void>;
|
|
165
|
+
emit(e: PipelineEvent): void;
|
|
166
|
+
}
|
|
167
|
+
/** Job + scope + key + ETag (+ version): the message id, and the dedupe key. Safe for Inngest event ids. */
|
|
168
|
+
export declare const messageId: (job: string, e: UploadCompleted, scope?: string) => string;
|
|
169
|
+
export declare function errorInfo(e: unknown): ErrorInfo;
|
|
170
|
+
/**
|
|
171
|
+
* The default proof rule for CopyResult-shaped results: only `verified`, or `alreadyCopied` + `verified`, is done.
|
|
172
|
+
* A source that is gone before it was copied is an alert too ("gone"): nothing was backed up. So is one skipped
|
|
173
|
+
* because it was replaced (`replaced: "skip"`: alert "replaced").
|
|
174
|
+
*/
|
|
175
|
+
export declare function provenCopy(r: unknown): {
|
|
176
|
+
ok: true;
|
|
177
|
+
} | {
|
|
178
|
+
ok: false;
|
|
179
|
+
reason: string;
|
|
180
|
+
alert?: string;
|
|
181
|
+
};
|
|
182
|
+
export declare function createPipeline(o: PipelineOptions): Pipeline;
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
import { type MemoStore } from "../../copy/steps";
|
|
2
|
+
import type { JobMessage, QueueModule, RunOutcome } from "./pipeline";
|
|
3
|
+
import type { MessageBatchLike } from "./sources";
|
|
4
|
+
export interface MemoryQueueOptions {
|
|
5
|
+
/** Retries after the first run (default 3). */
|
|
6
|
+
retries?: number;
|
|
7
|
+
/** Wait before retry `attempt` (1-based; default 0 ms: tests). */
|
|
8
|
+
backoffMs?: (attempt: number) => number;
|
|
9
|
+
sleep?: (ms: number) => Promise<void>;
|
|
10
|
+
/** Where steps are memoized (default: memory). */
|
|
11
|
+
state?: MemoStore;
|
|
12
|
+
/** Run messages as they arrive (default true); false: only on drain(). */
|
|
13
|
+
auto?: boolean;
|
|
14
|
+
/** Messages run at once (default 4). */
|
|
15
|
+
concurrency?: number;
|
|
16
|
+
}
|
|
17
|
+
export type MemoryOutcome = RunOutcome | {
|
|
18
|
+
state: "failed";
|
|
19
|
+
error: unknown;
|
|
20
|
+
};
|
|
21
|
+
export interface MemoryQueue extends QueueModule {
|
|
22
|
+
/** Resolves once every message (and its retries) has finished. */
|
|
23
|
+
drain(): Promise<void>;
|
|
24
|
+
/** The last outcome per message id. */
|
|
25
|
+
readonly outcomes: Map<string, MemoryOutcome>;
|
|
26
|
+
/** Runs per message id. */
|
|
27
|
+
readonly runs: Map<string, number>;
|
|
28
|
+
readonly messages: JobMessage[];
|
|
29
|
+
}
|
|
30
|
+
export declare function memoryQueue(o?: MemoryQueueOptions): MemoryQueue;
|
|
31
|
+
/**
|
|
32
|
+
* The slice of an Inngest client used. Inngest 4 (`createFunction({ id, triggers, … }, handler)`, what UploadFilm
|
|
33
|
+
* runs) by default; Inngest 3 (`createFunction(options, trigger, handler)`) with `api: "v3"`.
|
|
34
|
+
*/
|
|
35
|
+
export interface InngestLike {
|
|
36
|
+
send(payload: {
|
|
37
|
+
name: string;
|
|
38
|
+
data: unknown;
|
|
39
|
+
id?: string;
|
|
40
|
+
} | {
|
|
41
|
+
name: string;
|
|
42
|
+
data: unknown;
|
|
43
|
+
id?: string;
|
|
44
|
+
}[]): Promise<unknown>;
|
|
45
|
+
createFunction(...args: never[]): unknown;
|
|
46
|
+
}
|
|
47
|
+
export interface InngestQueueOptions {
|
|
48
|
+
/** The event every job message is sent as (default "sdk/pipeline.job"). */
|
|
49
|
+
eventName?: string;
|
|
50
|
+
/** Inngest retries per function (default 4). */
|
|
51
|
+
retries?: number;
|
|
52
|
+
/** Prefix of the function ids (default "pipeline-"). */
|
|
53
|
+
functionPrefix?: string;
|
|
54
|
+
/** Inngest's NonRetriableError class (default: an Error named "NonRetriableError", which Inngest honours by name). */
|
|
55
|
+
NonRetriableError?: new (message: string, opts?: {
|
|
56
|
+
cause?: unknown;
|
|
57
|
+
}) => Error;
|
|
58
|
+
/**
|
|
59
|
+
* Extra createFunction options per job; each one replaces the default of the same name. Defaults (Inngest 4):
|
|
60
|
+
* - `checkpointing: false`: one step per request. Inngest 4 otherwise runs every step of a run in one request,
|
|
61
|
+
* i.e. every part of a copy in one Worker invocation (its CPU, memory and subrequests at once); measured on a
|
|
62
|
+
* real Inngest dev server: 1 invocation for 3 steps (REALITY-CHECK.md RC-I1);
|
|
63
|
+
* - `concurrency: [{ limit: 1, key: "event.data.event.key" }]`: one step at a time per source key. Two runs for
|
|
64
|
+
* one key (a notification and a repair, a resend) would otherwise run at once and could adopt one upload
|
|
65
|
+
* (RC-I2). Other keys are not limited (add `{ limit: N }` for a total);
|
|
66
|
+
* - `singleton: { key: "event.data.id", mode: "skip" }` (the message id: job, scope, key, version): Inngest's
|
|
67
|
+
* concurrency limits running steps, not runs, so two runs for one version could still interleave between steps;
|
|
68
|
+
* a second run of the same message (a forced resend) while one runs is skipped (the running one copies it; a
|
|
69
|
+
* resend after it ended runs normally). As the backup Worker does, per key + ETag.
|
|
70
|
+
*/
|
|
71
|
+
functionOptions?: (job: string) => Record<string, unknown>;
|
|
72
|
+
/** The Inngest SDK's major API (default "v4"). */
|
|
73
|
+
api?: "v3" | "v4";
|
|
74
|
+
}
|
|
75
|
+
export interface InngestQueue extends QueueModule {
|
|
76
|
+
/** The Inngest functions to serve (`serve({ client, functions: queue.functions() })`). */
|
|
77
|
+
functions(): unknown[];
|
|
78
|
+
}
|
|
79
|
+
export declare function inngestQueue(inngest: InngestLike, o?: InngestQueueOptions): InngestQueue;
|
|
80
|
+
/** The producer side of a Cloudflare Queue binding (Queue fits). */
|
|
81
|
+
export interface CloudflareQueueLike {
|
|
82
|
+
send(body: unknown, o?: {
|
|
83
|
+
delaySeconds?: number;
|
|
84
|
+
}): Promise<void>;
|
|
85
|
+
sendBatch?(messages: {
|
|
86
|
+
body: unknown;
|
|
87
|
+
}[]): Promise<void>;
|
|
88
|
+
}
|
|
89
|
+
export interface CloudflareQueueOptions {
|
|
90
|
+
/**
|
|
91
|
+
* Deliveries before the consumer gives up and alerts (default 4: Cloudflare's default max_retries 3 + the first).
|
|
92
|
+
* Set it to the queue's max_retries + 1.
|
|
93
|
+
*/
|
|
94
|
+
maxAttempts?: number;
|
|
95
|
+
delaySeconds?: (attempts: number) => number;
|
|
96
|
+
/** Steps memoized here between deliveries (Workers KV / Redis store); default none (a retry starts over). */
|
|
97
|
+
state?: MemoStore;
|
|
98
|
+
}
|
|
99
|
+
export interface CloudflareQueue extends QueueModule {
|
|
100
|
+
/** The Worker's `queue(batch)` handler for this queue. */
|
|
101
|
+
consumer(): (batch: MessageBatchLike) => Promise<void>;
|
|
102
|
+
/**
|
|
103
|
+
* The handler for this queue's dead-letter queue (configure one: past max_retries Cloudflare otherwise deletes the
|
|
104
|
+
* message). Each message is reported as an alert; one whose alert can't be delivered is retried there.
|
|
105
|
+
*/
|
|
106
|
+
deadLetters(): (batch: MessageBatchLike) => Promise<void>;
|
|
107
|
+
}
|
|
108
|
+
export declare function cloudflareQueue(queue: CloudflareQueueLike, o?: CloudflareQueueOptions): CloudflareQueue;
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Event sources, normalised to `UploadCompleted`.
|
|
3
|
+
*
|
|
4
|
+
* - R2 event notifications (delivered through a Cloudflare Queue): `fromR2Notification(body)` for one message
|
|
5
|
+
* body, `r2Notifications(pipeline)` for a ready queue consumer. Only object-created actions count.
|
|
6
|
+
* - The product's own hook after a transfer completes: `fromTransferComplete(op, result)` turns the transfer
|
|
7
|
+
* protocol's `complete` op and its result (`{ size, etag }`, which the server HEADs) into the event; call
|
|
8
|
+
* `pipeline.ingest()` with it from the route's handler. A single presigned PUT is not seen by the server (the
|
|
9
|
+
* client sends the bytes): use R2 notifications for those.
|
|
10
|
+
*/
|
|
11
|
+
import type { TransferOp, TransferResult } from "../transfer/protocol";
|
|
12
|
+
import type { Pipeline, UploadCompleted } from "./pipeline";
|
|
13
|
+
/** The R2 event notification message body (Cloudflare's documented shape). */
|
|
14
|
+
export interface R2Notification {
|
|
15
|
+
account?: string;
|
|
16
|
+
bucket?: string;
|
|
17
|
+
action?: string;
|
|
18
|
+
eventTime?: string;
|
|
19
|
+
object?: {
|
|
20
|
+
key?: string;
|
|
21
|
+
size?: number;
|
|
22
|
+
eTag?: string;
|
|
23
|
+
version?: string;
|
|
24
|
+
};
|
|
25
|
+
copySource?: {
|
|
26
|
+
bucket?: string;
|
|
27
|
+
object?: string;
|
|
28
|
+
};
|
|
29
|
+
}
|
|
30
|
+
/** One R2 notification → the event, or null for anything that is not an object landing (deletes, lifecycle). */
|
|
31
|
+
export declare function fromR2Notification(body: unknown): UploadCompleted | null;
|
|
32
|
+
/** The transfer protocol's completed multipart upload → the event (null for other ops or a failed result). */
|
|
33
|
+
export declare function fromTransferComplete(op: TransferOp, result: TransferResult, bucket?: string): UploadCompleted | null;
|
|
34
|
+
/** The slice of a Cloudflare Queues batch a consumer uses (MessageBatch fits). */
|
|
35
|
+
export interface QueueMessageLike<B = unknown> {
|
|
36
|
+
readonly id: string;
|
|
37
|
+
readonly body: B;
|
|
38
|
+
readonly attempts: number;
|
|
39
|
+
ack(): void;
|
|
40
|
+
retry(o?: {
|
|
41
|
+
delaySeconds?: number;
|
|
42
|
+
}): void;
|
|
43
|
+
}
|
|
44
|
+
export interface MessageBatchLike<B = unknown> {
|
|
45
|
+
readonly messages: readonly QueueMessageLike<B>[];
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* A Cloudflare Queue consumer for R2 notifications: each message is ingested (fan-out + dedupe), then acked; a
|
|
49
|
+
* failed ingest (the job queue unreachable) is retried with backoff, never acked, and alerted at its last delivery.
|
|
50
|
+
*/
|
|
51
|
+
export declare function r2Notifications(pipeline: Pipeline, o?: {
|
|
52
|
+
delaySeconds?: (attempts: number) => number;
|
|
53
|
+
maxAttempts?: number;
|
|
54
|
+
}): (batch: MessageBatchLike) => Promise<void>;
|