@nodaro/shared 3.26.0 → 3.30.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +726 -174
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +569 -34
- package/dist/index.d.ts +569 -34
- package/dist/index.js +667 -175
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
- package/src/__tests__/auto-aspect-source.test.ts +372 -0
- package/src/__tests__/camera-switch-contract.test.ts +10 -0
- package/src/__tests__/collections.test.ts +10 -0
- package/src/__tests__/edl.test.ts +319 -4
- package/src/__tests__/json-kinds.test.ts +11 -1
- package/src/__tests__/llm-models.test.ts +120 -8
- package/src/__tests__/pitch-tempo-pricing.test.ts +62 -0
- package/src/__tests__/render-nodes.test.ts +84 -5
- package/src/__tests__/transcribe-preflight.test.ts +5 -0
- package/src/__tests__/video-depth.test.ts +146 -0
- package/src/__tests__/video-mask.test.ts +77 -0
- package/src/camera-switch-contract.ts +9 -6
- package/src/collections.ts +72 -2
- package/src/credit-estimators/index.ts +10 -0
- package/src/credit-estimators/pitch-tempo.ts +68 -0
- package/src/credit-identifiers.ts +87 -19
- package/src/edl.ts +143 -23
- package/src/fan-out-rows.ts +3 -1
- package/src/i18n/camera-format.ko.ts +3 -3
- package/src/index.ts +35 -0
- package/src/json-kinds.ts +11 -6
- package/src/llm-models.ts +87 -2
- package/src/mask-composite.ts +108 -0
- package/src/model-catalog.ts +146 -4
- package/src/model-constants.ts +2 -1
- package/src/model-tree.ts +2 -0
- package/src/node-mappable-fields.ts +2 -0
- package/src/node-runtime-keys.ts +3 -0
- package/src/normalize-node-params.ts +47 -4
- package/src/pipeline-types.ts +16 -3
- package/src/presentation-utils.ts +3 -2
- package/src/producer-types.ts +13 -0
- package/src/render-nodes.ts +71 -6
- package/src/transcribe-preflight.ts +11 -10
- package/src/video-depth.ts +151 -0
- package/src/video-mask.ts +96 -0
package/src/producer-types.ts
CHANGED
|
@@ -47,6 +47,7 @@ export const VIDEO_PRODUCER_TYPES: ReadonlySet<string> = new Set([
|
|
|
47
47
|
"speech-to-video",
|
|
48
48
|
"motion-transfer",
|
|
49
49
|
"video-upscale",
|
|
50
|
+
"video-depth",
|
|
50
51
|
"extend-video",
|
|
51
52
|
// face-swap output is video (writes generatedVideoUrl, per-result `url`).
|
|
52
53
|
// Frontend execution-graph already includes it in VIDEO_SOURCE_TYPES; this
|
|
@@ -72,6 +73,9 @@ export const VIDEO_PRODUCER_TYPES: ReadonlySet<string> = new Set([
|
|
|
72
73
|
"speed-ramp",
|
|
73
74
|
"loop-video",
|
|
74
75
|
"fade-video",
|
|
76
|
+
// Video Mask emits its mask as a video; Mask Composite its composite.
|
|
77
|
+
"video-mask",
|
|
78
|
+
"mask-composite",
|
|
75
79
|
"transcode-video",
|
|
76
80
|
"manual-edit",
|
|
77
81
|
// Remove Audio: strips the audio track, emits a silent video.
|
|
@@ -132,6 +136,10 @@ export const DYNAMIC_PRODUCER_TYPES: ReadonlySet<string> = new Set([
|
|
|
132
136
|
"list",
|
|
133
137
|
"sub-workflow",
|
|
134
138
|
"adjust-volume",
|
|
139
|
+
// Pitch & Tempo: audio in → audio out, video in → video out (picture
|
|
140
|
+
// copied). Dual-mode like adjust-volume; stays in AUDIO_PRODUCER_TYPES as
|
|
141
|
+
// its default.
|
|
142
|
+
"pitch-tempo",
|
|
135
143
|
// Dual-mode: audio in → audio out; video in → video out (+ revoiced audio).
|
|
136
144
|
// Listed here so canvas validators accept its output on BOTH audio and video
|
|
137
145
|
// input handles (it also stays in AUDIO_PRODUCER_TYPES as its default).
|
|
@@ -196,6 +204,7 @@ export const AUDIO_PRODUCER_TYPES: ReadonlySet<string> = new Set([
|
|
|
196
204
|
"combine-audio",
|
|
197
205
|
"adjust-volume",
|
|
198
206
|
"audio-fx",
|
|
207
|
+
"pitch-tempo",
|
|
199
208
|
"reference-audio",
|
|
200
209
|
"audio-isolation",
|
|
201
210
|
"text-to-dialogue",
|
|
@@ -386,6 +395,10 @@ export const FAN_IN_TARGETS: Readonly<Record<string, "*" | readonly string[]>> =
|
|
|
386
395
|
"merge-lists": "*",
|
|
387
396
|
"sort-list": "*",
|
|
388
397
|
selector: "*",
|
|
398
|
+
// Speaker Frames on a clip pack (P3-24 (a), decided 2026-10-06): ONE run over
|
|
399
|
+
// the union of the clips' spans — the pack's EDLs fold onto its `edl` wire
|
|
400
|
+
// instead of running one detection job (and one minimum charge) per clip.
|
|
401
|
+
"speaker-frames": ["edl"],
|
|
389
402
|
}
|
|
390
403
|
|
|
391
404
|
export function isFanInNodeType(nodeType: string | undefined | null): boolean {
|
package/src/render-nodes.ts
CHANGED
|
@@ -15,7 +15,7 @@
|
|
|
15
15
|
* (by quality or otherwise — only its credit id changes), so the same key
|
|
16
16
|
* answers for `node.type` and for `jobs.job_type`.
|
|
17
17
|
*/
|
|
18
|
-
import { applyEdlCreditId, speakerViewCreditId } from "./credit-identifiers.js"
|
|
18
|
+
import { applyEdlCreditId, speakerViewCreditId, SPEAKER_VIEW_CREDIT_IDS } from "./credit-identifiers.js"
|
|
19
19
|
import type { RenderMedium } from "./render-output.js"
|
|
20
20
|
|
|
21
21
|
/** Where a render's player maps the output clock back to the master clock
|
|
@@ -27,12 +27,34 @@ export type RenderClockSource = "input" | "output-json"
|
|
|
27
27
|
* on as one (Add Captions, the word-timing preflight). */
|
|
28
28
|
export type RenderJsonKind = "transcript" | "edl"
|
|
29
29
|
|
|
30
|
+
/** Where a render's wired transcript, remapped through its cut, comes out
|
|
31
|
+
* (Apply EDL's on its `json`; Speaker View's on its own `transcript` pip,
|
|
32
|
+
* decided 2026-10-08). */
|
|
33
|
+
export interface RenderTranscriptOutput {
|
|
34
|
+
/** The output pip — also the key it rides on the job's `output_data` and on
|
|
35
|
+
* a run's node output. */
|
|
36
|
+
readonly handle: string
|
|
37
|
+
/** The saved node-data field that holds it (the landed take's). */
|
|
38
|
+
readonly dataField: string
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/** The pip a render's `json` output (its `jsonKind`) comes out on, and its key
|
|
42
|
+
* on the job's `output_data`. */
|
|
43
|
+
export const RENDER_JSON_HANDLE = "json"
|
|
44
|
+
|
|
30
45
|
export interface RenderNodeDescriptor {
|
|
31
46
|
/** The medium the render's order asks for (its node data, or its job's
|
|
32
47
|
* `input_data`) — before any output exists to say. */
|
|
33
48
|
readonly mediumOf: (data: Readonly<Record<string, unknown>>) => RenderMedium
|
|
34
|
-
/** The credit id a render
|
|
35
|
-
|
|
49
|
+
/** The credit id a render reserves on — a per-OUTPUT-MINUTE row: every
|
|
50
|
+
* render is charged its row × the minutes it renders. Chosen from the
|
|
51
|
+
* render's `quality` and `output` (as its node data or its job's request
|
|
52
|
+
* carries them) and, for a render priced in tiers, the `priceTier` the
|
|
53
|
+
* render rule judged it to be in (Speaker View's, decided 2026-10-10). A
|
|
54
|
+
* tiered render given no tier names its dearer row. */
|
|
55
|
+
readonly creditId: (quality: unknown, output?: unknown, priceTier?: unknown) => string
|
|
56
|
+
/** Every id `creditId` can name (each must have a price row). */
|
|
57
|
+
readonly creditIds: readonly string[]
|
|
36
58
|
readonly clockMapFrom: RenderClockSource
|
|
37
59
|
/** Listed in its owner's own views only, never in a public one (decided
|
|
38
60
|
* 2026-10-06): a Preview is private, and a public final would be exposure
|
|
@@ -42,6 +64,9 @@ export interface RenderNodeDescriptor {
|
|
|
42
64
|
* (TA6, decided 2026-10-04). */
|
|
43
65
|
readonly latestBatch: boolean
|
|
44
66
|
readonly jsonKind: RenderJsonKind
|
|
67
|
+
/** Where its remapped transcript comes out. On the `json` pip exactly when
|
|
68
|
+
* `jsonKind` is `transcript` (a guard test holds the two together). */
|
|
69
|
+
readonly transcriptOutput: RenderTranscriptOutput
|
|
45
70
|
}
|
|
46
71
|
|
|
47
72
|
type Data = Readonly<Record<string, unknown>>
|
|
@@ -49,27 +74,34 @@ type Data = Readonly<Record<string, unknown>>
|
|
|
49
74
|
/** Its `output` setting, video when absent. */
|
|
50
75
|
const outputSettingMedium = (data: Data): RenderMedium => (data.output === "audio" ? "audio" : "video")
|
|
51
76
|
|
|
52
|
-
const descriptor = (d: RenderNodeDescriptor): RenderNodeDescriptor =>
|
|
77
|
+
const descriptor = (d: RenderNodeDescriptor): RenderNodeDescriptor =>
|
|
78
|
+
Object.freeze({ ...d, creditIds: Object.freeze([...d.creditIds]), transcriptOutput: Object.freeze({ ...d.transcriptOutput }) })
|
|
53
79
|
|
|
54
80
|
export const RENDER_NODE_TYPES: Readonly<Record<string, RenderNodeDescriptor>> = Object.freeze({
|
|
55
81
|
"apply-edl": descriptor({
|
|
56
82
|
mediumOf: outputSettingMedium,
|
|
57
83
|
creditId: (quality) => applyEdlCreditId(quality),
|
|
84
|
+
creditIds: ["apply-edl", "apply-edl:proxy"],
|
|
58
85
|
clockMapFrom: "input",
|
|
59
86
|
ownerOnlyListing: true,
|
|
60
87
|
latestBatch: true,
|
|
61
88
|
jsonKind: "transcript",
|
|
89
|
+
transcriptOutput: { handle: RENDER_JSON_HANDLE, dataField: "generatedJson" },
|
|
62
90
|
}),
|
|
63
91
|
// Speaker View (C3.2): video only; its `json` is the EDL as it drew it (the
|
|
64
92
|
// turns split, the layouts written), so a player maps the output clock
|
|
65
|
-
// through THAT, and Add Captions never takes it for a Transcript.
|
|
93
|
+
// through THAT, and Add Captions never takes it for a Transcript. The wired
|
|
94
|
+
// transcript, remapped through that same drawn edit, comes out on its own
|
|
95
|
+
// `transcript` pip (decided 2026-10-08) — what a caption step reads.
|
|
66
96
|
"speaker-view": descriptor({
|
|
67
97
|
mediumOf: () => "video",
|
|
68
|
-
creditId: (quality) => speakerViewCreditId(quality),
|
|
98
|
+
creditId: (quality, _output, priceTier) => speakerViewCreditId(quality, priceTier),
|
|
99
|
+
creditIds: SPEAKER_VIEW_CREDIT_IDS,
|
|
69
100
|
clockMapFrom: "output-json",
|
|
70
101
|
ownerOnlyListing: true,
|
|
71
102
|
latestBatch: true,
|
|
72
103
|
jsonKind: "edl",
|
|
104
|
+
transcriptOutput: { handle: "transcript", dataField: "generatedTranscript" },
|
|
73
105
|
}),
|
|
74
106
|
})
|
|
75
107
|
|
|
@@ -97,6 +129,39 @@ export function rendersTranscriptJson(type: unknown): boolean {
|
|
|
97
129
|
return renderNodeOf(type)?.jsonKind === "transcript"
|
|
98
130
|
}
|
|
99
131
|
|
|
132
|
+
/** Where a render's remapped transcript comes out; `undefined` for any other
|
|
133
|
+
* node type. */
|
|
134
|
+
export function renderTranscriptOutputOf(type: unknown): RenderTranscriptOutput | undefined {
|
|
135
|
+
return renderNodeOf(type)?.transcriptOutput
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
/** Is `handle` one of a render's data pips — its `json` or its transcript —
|
|
139
|
+
* rather than its media? Every other pip of a render (the default included)
|
|
140
|
+
* is the rendered media. `false` for any other node type. */
|
|
141
|
+
export function isRenderDataHandle(type: unknown, handle: unknown): boolean {
|
|
142
|
+
const d = renderNodeOf(type)
|
|
143
|
+
if (!d || typeof handle !== "string") return false
|
|
144
|
+
return handle === RENDER_JSON_HANDLE || handle === d.transcriptOutput.handle
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
/** One json output a render lands on its node: its key on the job's
|
|
148
|
+
* `output_data` (and on a run's node output) and its saved node-data field. */
|
|
149
|
+
export interface RenderSavedJsonOutput {
|
|
150
|
+
readonly outputKey: string
|
|
151
|
+
readonly dataField: string
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
/** The json outputs a render lands on its node, `json` first, each once — what
|
|
155
|
+
* every lane that lands a take (or picks one) writes or restores. Empty for
|
|
156
|
+
* any other node type. */
|
|
157
|
+
export function renderSavedJsonOutputs(type: unknown): readonly RenderSavedJsonOutput[] {
|
|
158
|
+
const d = renderNodeOf(type)
|
|
159
|
+
if (!d) return []
|
|
160
|
+
const json: RenderSavedJsonOutput = { outputKey: RENDER_JSON_HANDLE, dataField: "generatedJson" }
|
|
161
|
+
const t = d.transcriptOutput
|
|
162
|
+
return t.handle === RENDER_JSON_HANDLE ? [json] : [json, { outputKey: t.handle, dataField: t.dataField }]
|
|
163
|
+
}
|
|
164
|
+
|
|
100
165
|
/** The render job types that list in their owner's own views only. */
|
|
101
166
|
export const OWNER_ONLY_LISTING_RENDER_TYPES: readonly string[] = Object.freeze(
|
|
102
167
|
RENDER_NODE_TYPE_IDS.filter((t) => RENDER_NODE_TYPES[t]!.ownerOnlyListing),
|
|
@@ -3,7 +3,7 @@ import {
|
|
|
3
3
|
transcribeLaneSupportsWordTimestamps,
|
|
4
4
|
transcribeProvidersWithWordTimestamps,
|
|
5
5
|
} from "./model-constants.js"
|
|
6
|
-
import {
|
|
6
|
+
import { renderTranscriptOutputOf } from "./render-nodes.js"
|
|
7
7
|
|
|
8
8
|
/**
|
|
9
9
|
* Pre-run checks for the transcribe → captions chain.
|
|
@@ -47,14 +47,12 @@ export interface WordlessTranscriptFeed {
|
|
|
47
47
|
// Handle ids (generated map: backend/src/lib/mcp/generated/node-handles.ts).
|
|
48
48
|
const TRANSCRIBE_JSON_OUT = "json"
|
|
49
49
|
const TRANSCRIPT_IN = "transcript"
|
|
50
|
-
/** A render's `json` output — handed on as a Transcript only by a render whose
|
|
51
|
-
* `jsonKind` is `transcript` (Apply EDL's is; an EDL-emitting render's is not). */
|
|
52
|
-
const RENDER_JSON_OUT = "json"
|
|
53
50
|
|
|
54
51
|
/**
|
|
55
52
|
* Every transcribe node on a word-INCAPABLE lane whose `json` output reaches an
|
|
56
53
|
* add-captions `transcript` input — directly, or through apply-edl, which remaps
|
|
57
|
-
* the transcript and re-emits it on its own `json` handle
|
|
54
|
+
* the transcript and re-emits it on its own `json` handle (or any render, by
|
|
55
|
+
* its registry `transcriptOutput`: Speaker View's `transcript`). add-captions rejects a
|
|
58
56
|
* transcript with no words, so such a run can only fail, after paying for the
|
|
59
57
|
* transcription. Skipped nodes are ignored on both ends.
|
|
60
58
|
*
|
|
@@ -77,8 +75,8 @@ export function findWordlessTranscriptFeeds(
|
|
|
77
75
|
const refusal = transcribeWordTimestampsRefusal(provider)
|
|
78
76
|
if (!refusal) continue
|
|
79
77
|
|
|
80
|
-
// Walk the transcript's path: transcribe.json → [render.transcript →
|
|
81
|
-
//
|
|
78
|
+
// Walk the transcript's path: transcribe.json → [render.transcript → its transcript output]* → add-captions.transcript,
|
|
79
|
+
// out of every render on the pip its registry names (RENDER_NODE_TYPES transcriptOutput)
|
|
82
80
|
const seen = new Set<string>()
|
|
83
81
|
const frontier: Array<{ id: string; outHandle: string }> = [{ id: node.id, outHandle: TRANSCRIBE_JSON_OUT }]
|
|
84
82
|
while (frontier.length > 0) {
|
|
@@ -94,9 +92,12 @@ export function findWordlessTranscriptFeeds(
|
|
|
94
92
|
provider,
|
|
95
93
|
message: `Captions need word timings, but ${refusal}.`,
|
|
96
94
|
})
|
|
97
|
-
} else if (
|
|
98
|
-
|
|
99
|
-
|
|
95
|
+
} else if (!seen.has(target!.id)) {
|
|
96
|
+
const remapped = renderTranscriptOutputOf(target!.type)
|
|
97
|
+
if (remapped) {
|
|
98
|
+
seen.add(target!.id)
|
|
99
|
+
frontier.push({ id: target!.id, outHandle: remapped.handle })
|
|
100
|
+
}
|
|
100
101
|
}
|
|
101
102
|
}
|
|
102
103
|
}
|
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Video Depth — the one contract every layer reads: which providers exist and
|
|
3
|
+
* which lane serves each, which optional levers a provider honours, how long a
|
|
4
|
+
* clip may be, and which price row a run is charged from.
|
|
5
|
+
*
|
|
6
|
+
* Colormaps: fal renders every `VIDEO_DEPTH_COLORMAPS` entry natively. The
|
|
7
|
+
* Replicate model's native output is inferno-colorized, so it offers only
|
|
8
|
+
* `inferno` (passed through as-is) and `grayscale`, which a worker ffmpeg pass
|
|
9
|
+
* produces from the inferno output (`videoDepthNeedsGrayscalePass`).
|
|
10
|
+
*
|
|
11
|
+
* Price rows are `model_pricing` rows (`<provider>:<n>s`); the credit amounts
|
|
12
|
+
* live there, never here. Worked examples (repeated in
|
|
13
|
+
* docs/nodes/processing-video/video-depth.md):
|
|
14
|
+
* 5 s → `:15s`, 15 s → `:15s`, 15.01 s → `:30s`, 61 s → `:120s`.
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
export const VIDEO_DEPTH_PROVIDERS = ["video-depth-anything", "depth-anything-frames"] as const
|
|
18
|
+
export type VideoDepthProvider = (typeof VIDEO_DEPTH_PROVIDERS)[number]
|
|
19
|
+
|
|
20
|
+
export const FAL_VIDEO_DEPTH_PROVIDERS: ReadonlySet<string> = new Set(["video-depth-anything"])
|
|
21
|
+
export const REPLICATE_VIDEO_DEPTH_PROVIDERS: ReadonlySet<string> = new Set(["depth-anything-frames"])
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* The provider a run uses: `raw` when it is a Video Depth provider, else the
|
|
25
|
+
* default (the first entry). Raw workflow JSON or a FieldMapping can carry any
|
|
26
|
+
* value; the dispatch, the model deny and the price row all read this one
|
|
27
|
+
* resolution, so an unknown id never reaches a price row or slips the deny.
|
|
28
|
+
*/
|
|
29
|
+
export function resolveVideoDepthProvider(raw: unknown): VideoDepthProvider {
|
|
30
|
+
return typeof raw === "string" && (VIDEO_DEPTH_PROVIDERS as readonly string[]).includes(raw)
|
|
31
|
+
? (raw as VideoDepthProvider)
|
|
32
|
+
: VIDEO_DEPTH_PROVIDERS[0]
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
export const VIDEO_DEPTH_MODEL_SIZES = ["small", "base", "large"] as const
|
|
36
|
+
export type VideoDepthModelSize = (typeof VIDEO_DEPTH_MODEL_SIZES)[number]
|
|
37
|
+
|
|
38
|
+
/** The model size a run uses: `raw` when it is one, else `large` (the providers index lookup tables with it). */
|
|
39
|
+
export function resolveVideoDepthModelSize(raw: unknown): VideoDepthModelSize {
|
|
40
|
+
return typeof raw === "string" && (VIDEO_DEPTH_MODEL_SIZES as readonly string[]).includes(raw)
|
|
41
|
+
? (raw as VideoDepthModelSize)
|
|
42
|
+
: "large"
|
|
43
|
+
}
|
|
44
|
+
export const VIDEO_DEPTH_COLORMAPS = ["grayscale", "turbo", "inferno", "magma", "viridis"] as const
|
|
45
|
+
export type VideoDepthColormap = (typeof VIDEO_DEPTH_COLORMAPS)[number]
|
|
46
|
+
export const VIDEO_DEPTH_RESOLUTIONS = ["auto", "360p", "480p", "720p", "1080p"] as const
|
|
47
|
+
export type VideoDepthResolution = (typeof VIDEO_DEPTH_RESOLUTIONS)[number]
|
|
48
|
+
|
|
49
|
+
export type VideoDepthLever = "colormap" | "resolution" | "sideBySide"
|
|
50
|
+
const ALL_LEVERS: ReadonlySet<VideoDepthLever> = new Set(["colormap", "resolution", "sideBySide"])
|
|
51
|
+
const COLORMAP_ONLY: ReadonlySet<VideoDepthLever> = new Set(["colormap"])
|
|
52
|
+
const NO_LEVERS: ReadonlySet<VideoDepthLever> = new Set()
|
|
53
|
+
|
|
54
|
+
/** The optional levers `provider` honours. A lever absent here is dropped, never sent. */
|
|
55
|
+
export function videoDepthLevers(provider: string): ReadonlySet<VideoDepthLever> {
|
|
56
|
+
if (FAL_VIDEO_DEPTH_PROVIDERS.has(provider)) return ALL_LEVERS
|
|
57
|
+
if (REPLICATE_VIDEO_DEPTH_PROVIDERS.has(provider)) return COLORMAP_ONLY
|
|
58
|
+
return NO_LEVERS
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
const REPLICATE_COLORMAPS: readonly VideoDepthColormap[] = ["grayscale", "inferno"]
|
|
62
|
+
|
|
63
|
+
/** The colormaps `provider` can deliver, in display order; empty when it has no colormap lever. */
|
|
64
|
+
export function videoDepthColormaps(provider: string): readonly VideoDepthColormap[] {
|
|
65
|
+
if (FAL_VIDEO_DEPTH_PROVIDERS.has(provider)) return VIDEO_DEPTH_COLORMAPS
|
|
66
|
+
if (REPLICATE_VIDEO_DEPTH_PROVIDERS.has(provider)) return REPLICATE_COLORMAPS
|
|
67
|
+
return []
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/** The colormap a run uses: the requested one when `provider` offers it, else `grayscale`; `undefined` when the provider has no colormap lever. Coerces, never throws. */
|
|
71
|
+
export function resolveVideoDepthColormap(provider: string, requested: unknown): VideoDepthColormap | undefined {
|
|
72
|
+
if (!videoDepthLevers(provider).has("colormap")) return undefined
|
|
73
|
+
const offered = videoDepthColormaps(provider)
|
|
74
|
+
return typeof requested === "string" && (offered as readonly string[]).includes(requested)
|
|
75
|
+
? (requested as VideoDepthColormap)
|
|
76
|
+
: "grayscale"
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/** True when the worker must turn the Replicate model's native inferno output into grayscale with ffmpeg. */
|
|
80
|
+
export function videoDepthNeedsGrayscalePass(provider: string, colormap: VideoDepthColormap | undefined): boolean {
|
|
81
|
+
return REPLICATE_VIDEO_DEPTH_PROVIDERS.has(provider) && resolveVideoDepthColormap(provider, colormap) === "grayscale"
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
export const VIDEO_DEPTH_LIMITS = {
|
|
85
|
+
/** fal `max_frames` ceiling. */
|
|
86
|
+
FAL_MAX_FRAMES: 2400,
|
|
87
|
+
/** Longest clip the fal lane accepts, whatever its frame rate: the top price bucket (a low-fps clip under the frame limit can otherwise run past every price row). */
|
|
88
|
+
FAL_MAX_SECONDS: 300,
|
|
89
|
+
/** Frame rate assumed when the clip reports none (variable-rate or missing). */
|
|
90
|
+
ASSUMED_FPS: 30,
|
|
91
|
+
/** Longest clip the Replicate lane accepts. */
|
|
92
|
+
REPLICATE_MAX_SECONDS: 80,
|
|
93
|
+
} as const
|
|
94
|
+
|
|
95
|
+
const fmt = (n: number): string => n.toLocaleString("en-US")
|
|
96
|
+
|
|
97
|
+
/** Refuse a clip the provider cannot take — run BEFORE any credit reservation. */
|
|
98
|
+
export function checkVideoDepthLength(
|
|
99
|
+
provider: string,
|
|
100
|
+
probe: { durationSec: number; fps?: number },
|
|
101
|
+
): { ok: true } | { ok: false; message: string } {
|
|
102
|
+
if (REPLICATE_VIDEO_DEPTH_PROVIDERS.has(provider)) {
|
|
103
|
+
if (probe.durationSec <= VIDEO_DEPTH_LIMITS.REPLICATE_MAX_SECONDS) return { ok: true }
|
|
104
|
+
return {
|
|
105
|
+
ok: false,
|
|
106
|
+
message: `Video is ${Math.ceil(probe.durationSec)} s. This provider accepts at most ${VIDEO_DEPTH_LIMITS.REPLICATE_MAX_SECONDS} s.`,
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
if (probe.durationSec > VIDEO_DEPTH_LIMITS.FAL_MAX_SECONDS) {
|
|
110
|
+
return {
|
|
111
|
+
ok: false,
|
|
112
|
+
message: `Video is ${Math.ceil(probe.durationSec)} s. This provider accepts at most ${VIDEO_DEPTH_LIMITS.FAL_MAX_SECONDS} s. Trim the clip and run again.`,
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
const assumed = !(typeof probe.fps === "number" && probe.fps > 0)
|
|
116
|
+
const fps = assumed ? VIDEO_DEPTH_LIMITS.ASSUMED_FPS : (probe.fps as number)
|
|
117
|
+
const frames = Math.ceil(probe.durationSec * fps - 1e-9)
|
|
118
|
+
if (frames <= VIDEO_DEPTH_LIMITS.FAL_MAX_FRAMES) return { ok: true }
|
|
119
|
+
const maxSec = Math.floor(VIDEO_DEPTH_LIMITS.FAL_MAX_FRAMES / fps)
|
|
120
|
+
return {
|
|
121
|
+
ok: false,
|
|
122
|
+
message:
|
|
123
|
+
`Video is ${fmt(frames)} frames; the limit is ${fmt(VIDEO_DEPTH_LIMITS.FAL_MAX_FRAMES)} ` +
|
|
124
|
+
`(about ${maxSec} s at ${Math.round(fps)} fps${assumed ? ", assumed 30 fps because the clip reports no frame rate" : ""}). Trim the clip and run again.`,
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
export const VIDEO_DEPTH_BUCKETS = [15, 30, 60, 120, 300] as const
|
|
129
|
+
|
|
130
|
+
/** The longest clip `provider` accepts (`checkVideoDepthLength`); the fal limit for anything else. */
|
|
131
|
+
function videoDepthMaxSeconds(provider: string): number {
|
|
132
|
+
return REPLICATE_VIDEO_DEPTH_PROVIDERS.has(provider)
|
|
133
|
+
? VIDEO_DEPTH_LIMITS.REPLICATE_MAX_SECONDS
|
|
134
|
+
: VIDEO_DEPTH_LIMITS.FAL_MAX_SECONDS
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
/**
|
|
138
|
+
* The price row for a clip of `durationSec`. Unknown, non-positive or longer
|
|
139
|
+
* than the provider accepts → the provider's highest REACHABLE row: the
|
|
140
|
+
* smallest bucket at or above its longest clip (Replicate 80 s → `:120s`,
|
|
141
|
+
* fal 300 s → `:300s`) — the most a run on it can be charged, never a row a
|
|
142
|
+
* run could not reach.
|
|
143
|
+
*/
|
|
144
|
+
export function videoDepthCreditId(provider: string, durationSec: number | null | undefined): string {
|
|
145
|
+
const top = VIDEO_DEPTH_BUCKETS[VIDEO_DEPTH_BUCKETS.length - 1]
|
|
146
|
+
const maxSec = videoDepthMaxSeconds(provider)
|
|
147
|
+
const reachableCeiling = VIDEO_DEPTH_BUCKETS.find((b) => b >= maxSec) ?? top
|
|
148
|
+
const known = typeof durationSec === "number" && Number.isFinite(durationSec) && durationSec > 0 && durationSec <= maxSec
|
|
149
|
+
const bucket = known ? (VIDEO_DEPTH_BUCKETS.find((b) => durationSec <= b) ?? top) : reachableCeiling
|
|
150
|
+
return `${provider}:${bucket}s`
|
|
151
|
+
}
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Video Mask — text-prompted video segmentation (SAM 3.1 / SAM 3). The wire contract of
|
|
3
|
+
* the `video-mask` node, `POST /v1/video-mask`, the `video_mask` MCP verb,
|
|
4
|
+
* `client.media.videoMask` and `nodaro media video-mask`, defined ONCE:
|
|
5
|
+
* - every bound the route schema, the node panel and the MCP verb read;
|
|
6
|
+
* - `videoMaskBilledSeconds`: the length a run is priced at — the clip's
|
|
7
|
+
* length at the 30 fps basis, so a 60 fps clip is priced at twice its
|
|
8
|
+
* length (the segmentation works per FRAME);
|
|
9
|
+
* - `videoMaskCreditId`: which price row (`model_pricing`) that length is
|
|
10
|
+
* charged from. The credit amounts live in the rows, never here.
|
|
11
|
+
*
|
|
12
|
+
* Worked examples (repeated in docs/nodes/ai-video/video-mask.md):
|
|
13
|
+
* 4 s at 30 fps → `sam-3.1-video:5s`, 12 s at 24 fps → `sam-3.1-video:15s`,
|
|
14
|
+
* 10 s at 60 fps (priced as 20 s) → `sam-3.1-video:30s`, 45 s at 30 fps → `sam-3.1-video:60s`,
|
|
15
|
+
* 100 s at 30 fps → `sam-3.1-video:120s`, 4 min at 30 fps on SAM 3 → `sam-3-video:240s`.
|
|
16
|
+
*
|
|
17
|
+
* Wire contract only — no I/O.
|
|
18
|
+
*/
|
|
19
|
+
|
|
20
|
+
/** Segmentation models a Video Mask run can use. SAM 3.1 (the default)
|
|
21
|
+
* tracks several objects in one pass and runs faster; SAM 3 costs half. */
|
|
22
|
+
export const VIDEO_MASK_PROVIDERS = ["sam-3.1", "sam-3"] as const
|
|
23
|
+
export type VideoMaskProvider = (typeof VIDEO_MASK_PROVIDERS)[number]
|
|
24
|
+
export const DEFAULT_VIDEO_MASK_PROVIDER: VideoMaskProvider = "sam-3.1"
|
|
25
|
+
|
|
26
|
+
/** A new Video Mask node's settings — spread into its node definition. */
|
|
27
|
+
export const VIDEO_MASK_DEFAULTS = {
|
|
28
|
+
threshold: 0.5,
|
|
29
|
+
provider: "sam-3.1" as VideoMaskProvider,
|
|
30
|
+
} as const
|
|
31
|
+
|
|
32
|
+
export const VIDEO_MASK_LIMITS = {
|
|
33
|
+
/** The longest input clip a run accepts, in seconds of PRICED length (see `videoMaskBilledSeconds`). */
|
|
34
|
+
MAX_BILLED_SEC: 300,
|
|
35
|
+
/** Frame rate the price rows assume. A faster clip is priced proportionally longer. */
|
|
36
|
+
FPS_BASIS: 30,
|
|
37
|
+
/** Subject description length cap. */
|
|
38
|
+
PROMPT_MAX_LENGTH: 500,
|
|
39
|
+
/** Detection confidence bounds (the model's own range, narrowed to useful values). */
|
|
40
|
+
THRESHOLD_MIN: 0.05,
|
|
41
|
+
THRESHOLD_MAX: 0.95,
|
|
42
|
+
THRESHOLD_STEP: 0.05,
|
|
43
|
+
/** Default detection confidence. Lower finds more (and less precise) regions. */
|
|
44
|
+
THRESHOLD_DEFAULT: VIDEO_MASK_DEFAULTS.threshold,
|
|
45
|
+
} as const
|
|
46
|
+
|
|
47
|
+
const VIDEO_MASK_BUCKETS_SEC = [5, 10, 15, 30, 60, 90, 120, 180, 240, 300] as const
|
|
48
|
+
|
|
49
|
+
/** Every price row a provider can be charged from, smallest first. */
|
|
50
|
+
export function videoMaskCreditIds(provider: VideoMaskProvider = DEFAULT_VIDEO_MASK_PROVIDER): string[] {
|
|
51
|
+
return VIDEO_MASK_BUCKETS_SEC.map((sec) => `${provider}-video:${sec}s`)
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/**
|
|
55
|
+
* The length a clip is priced at: its duration at the 30 fps basis. A clip
|
|
56
|
+
* at or under 30 fps is priced at its own length; a 60 fps clip at twice it.
|
|
57
|
+
* An unknown frame rate counts as the basis.
|
|
58
|
+
*/
|
|
59
|
+
export function videoMaskBilledSeconds(durationSec: number, fps?: number | null): number {
|
|
60
|
+
const rate = typeof fps === "number" && Number.isFinite(fps) && fps > 0 ? fps : VIDEO_MASK_LIMITS.FPS_BASIS
|
|
61
|
+
return durationSec * Math.max(1, rate / VIDEO_MASK_LIMITS.FPS_BASIS)
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/**
|
|
65
|
+
* The price row for a clip priced at `billedSec` seconds. An unknown or
|
|
66
|
+
* non-positive length prices the LONGEST row (a clip the route could not
|
|
67
|
+
* measure is refused, so this is only the pre-run quote for a clip whose
|
|
68
|
+
* length is not known yet); anything past the cap prices the last row.
|
|
69
|
+
*/
|
|
70
|
+
export function videoMaskCreditId(
|
|
71
|
+
billedSec: number | null | undefined,
|
|
72
|
+
provider: VideoMaskProvider = DEFAULT_VIDEO_MASK_PROVIDER,
|
|
73
|
+
): string {
|
|
74
|
+
const ids = videoMaskCreditIds(provider)
|
|
75
|
+
if (typeof billedSec !== "number" || !Number.isFinite(billedSec) || billedSec <= 0) return ids[ids.length - 1]!
|
|
76
|
+
const i = VIDEO_MASK_BUCKETS_SEC.findIndex((b) => billedSec <= b)
|
|
77
|
+
return ids[i === -1 ? ids.length - 1 : i]!
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/** Narrow an arbitrary value to a known provider, falling back to the default. */
|
|
81
|
+
export function resolveVideoMaskProvider(value: unknown): VideoMaskProvider {
|
|
82
|
+
return (VIDEO_MASK_PROVIDERS as readonly string[]).includes(value as string)
|
|
83
|
+
? (value as VideoMaskProvider)
|
|
84
|
+
: DEFAULT_VIDEO_MASK_PROVIDER
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/** `POST /v1/video-mask` request body. */
|
|
88
|
+
export interface VideoMaskRequest {
|
|
89
|
+
/** The clip to segment. */
|
|
90
|
+
readonly videoUrl: string
|
|
91
|
+
/** What to track: a short noun phrase; separate several with commas. */
|
|
92
|
+
readonly prompt: string
|
|
93
|
+
/** Detection confidence, `VIDEO_MASK_LIMITS.THRESHOLD_MIN`–`THRESHOLD_MAX` (default 0.5). */
|
|
94
|
+
readonly threshold?: number
|
|
95
|
+
readonly provider?: VideoMaskProvider
|
|
96
|
+
}
|