@nodaro/shared 3.21.0 → 3.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +2284 -195
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +1915 -75
- package/dist/index.d.ts +1915 -75
- package/dist/index.js +2117 -196
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
- package/src/__tests__/add-captions-plan.test.ts +33 -0
- package/src/__tests__/caption-plan-parse.test.ts +23 -0
- package/src/__tests__/caption-segment-schema.test.ts +30 -0
- package/src/__tests__/collections.test.ts +253 -0
- package/src/__tests__/dialogue-capabilities.test.ts +157 -0
- package/src/__tests__/edit-plan-review.test.ts +70 -0
- package/src/__tests__/edl-clocks.test.ts +97 -0
- package/src/__tests__/edl.test.ts +34 -0
- package/src/__tests__/execution-outcome.test.ts +44 -0
- package/src/__tests__/fan-out-meta.test.ts +23 -0
- package/src/__tests__/fan-out-rows.test.ts +36 -0
- package/src/__tests__/fan-out-url-item.test.ts +44 -0
- package/src/__tests__/fixtures/speaker-tracks.fixtures.json +622 -0
- package/src/__tests__/input-field-extra-keys.test.ts +34 -0
- package/src/__tests__/list-consumers-fan-in.test.ts +40 -0
- package/src/__tests__/model-tree.test.ts +7 -0
- package/src/__tests__/node-pass-through.test.ts +23 -0
- package/src/__tests__/node-runtime-keys.test.ts +1 -0
- package/src/__tests__/picker-restrictions.test.ts +36 -0
- package/src/__tests__/preview-gate.test.ts +305 -0
- package/src/__tests__/render-basis.test.ts +176 -0
- package/src/__tests__/render-nodes.test.ts +73 -0
- package/src/__tests__/render-output.test.ts +392 -0
- package/src/__tests__/run-continuation.test.ts +32 -0
- package/src/__tests__/speaker-tracks.test.ts +474 -0
- package/src/__tests__/speech-pricing.test.ts +69 -0
- package/src/__tests__/telegram-channel.test.ts +91 -0
- package/src/__tests__/tts-capabilities.test.ts +81 -4
- package/src/__tests__/tts-neighbour-text.test.ts +19 -0
- package/src/__tests__/ugc-call-body.test.ts +38 -0
- package/src/__tests__/ugc-run-state.test.ts +68 -0
- package/src/__tests__/video-character-refs.test.ts +213 -0
- package/src/__tests__/video-overlay-layer-plan.test.ts +94 -0
- package/src/add-captions-plan.ts +50 -0
- package/src/caption-plan.ts +45 -0
- package/src/caption-segment-schema.ts +82 -0
- package/src/collections.ts +462 -0
- package/src/competitors.ts +77 -0
- package/src/credit-estimators/index.ts +9 -0
- package/src/credit-estimators/speech.ts +51 -0
- package/src/dialogue-capabilities.ts +48 -0
- package/src/edit-plan-contract.ts +20 -9
- package/src/edit-plan-review.ts +40 -1
- package/src/edl.ts +27 -1
- package/src/execution-outcome.ts +41 -0
- package/src/fan-out-meta.ts +23 -0
- package/src/fan-out-rows.ts +74 -5
- package/src/gemini-omni-voices.ts +59 -0
- package/src/handle-aliases.ts +295 -0
- package/src/i18n/camera-motions.pt-BR.ts +14 -14
- package/src/i18n/transitions.pt-BR.ts +13 -13
- package/src/index.ts +179 -1
- package/src/model-catalog.ts +122 -13
- package/src/model-constants.ts +24 -4
- package/src/model-tree.ts +5 -1
- package/src/node-execution-state.ts +13 -0
- package/src/node-mappable-fields.ts +7 -1
- package/src/node-pass-through.ts +37 -0
- package/src/node-runtime-keys.ts +14 -0
- package/src/picker-restrictions.ts +49 -0
- package/src/presentation-types.ts +26 -2
- package/src/presentation-utils.ts +25 -7
- package/src/preview-gate.ts +279 -0
- package/src/producer-types.ts +58 -0
- package/src/render-final.ts +20 -0
- package/src/render-nodes.ts +92 -0
- package/src/render-output.ts +565 -0
- package/src/run-continuation.ts +39 -0
- package/src/speaker-tracks.ts +638 -0
- package/src/telegram-channel.ts +140 -0
- package/src/transcribe-preflight.ts +8 -4
- package/src/tts-capabilities.ts +19 -5
- package/src/tts-neighbour-text.ts +46 -0
- package/src/ugc-call-body.ts +86 -0
- package/src/ugc-run-state.ts +64 -0
- package/src/video-character-refs.ts +233 -0
- package/src/video-overlay.ts +78 -21
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Telegram Channel Feed — the post shape the node emits on its `json` handle,
|
|
3
|
+
* the digest its `text` handle carries, and the ONE cursor rule (where the
|
|
4
|
+
* feed's position goes after a run). Structural vocabulary and a pure rule;
|
|
5
|
+
* the fetching lives in the backend (`services/social/telegram-channel.ts`).
|
|
6
|
+
*
|
|
7
|
+
* Media links (`cdn*.telesco.pe/...`) carry a `?token=` and expire within days:
|
|
8
|
+
* a consumer that needs the picture later copies it (an upload node, a
|
|
9
|
+
* collection), never stores the link alone.
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
export interface TelegramChannelPostMedia {
|
|
13
|
+
readonly type: "photo" | "video"
|
|
14
|
+
/** The file — a photo's, or a video's when the preview page embeds the file ("Watch in Telegram" players have none). */
|
|
15
|
+
readonly url?: string
|
|
16
|
+
/** A video's poster frame. */
|
|
17
|
+
readonly posterUrl?: string
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
export interface TelegramChannelPost {
|
|
21
|
+
/** Sequential per-channel message id. */
|
|
22
|
+
readonly id: number
|
|
23
|
+
/** The channel's bare id (no `@`). */
|
|
24
|
+
readonly channel: string
|
|
25
|
+
/** Canonical link to the post. */
|
|
26
|
+
readonly postUrl: string
|
|
27
|
+
/** Plain text (HTML stripped, entities decoded); a reply's quoted text is not part of it. */
|
|
28
|
+
readonly text: string
|
|
29
|
+
/** ISO timestamp of the post. */
|
|
30
|
+
readonly date?: string
|
|
31
|
+
/** The channel or person the post was forwarded from. */
|
|
32
|
+
readonly forwardedFrom?: { readonly name: string; readonly url?: string }
|
|
33
|
+
/** Every photo and video of the post, in page order (an album is several). */
|
|
34
|
+
readonly media: readonly TelegramChannelPostMedia[]
|
|
35
|
+
/** The first picture — the first photo, else the first video's poster — for a consumer that takes one image. */
|
|
36
|
+
readonly imageUrl?: string
|
|
37
|
+
/** The view counter as the page shows it ("1.52M"). */
|
|
38
|
+
readonly views?: string
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/** The most posts a poll emits per run — two preview pages at most. */
|
|
42
|
+
export const TELEGRAM_FEED_LIMIT_MAX = 30
|
|
43
|
+
/** The most posts a peek ("re-fetch the last N") can show — one preview page. */
|
|
44
|
+
export const TELEGRAM_FEED_PEEK_MAX = 20
|
|
45
|
+
/** What one preview page (`t.me/s/<channel>[?after=<id>]`) renders. */
|
|
46
|
+
export const TELEGRAM_FEED_PAGE_SIZE = 20
|
|
47
|
+
/** The default posts per run. */
|
|
48
|
+
export const TELEGRAM_FEED_DEFAULT_LIMIT = 5
|
|
49
|
+
export const TELEGRAM_FEED_DIGEST_SEPARATOR = "\n\n---\n\n"
|
|
50
|
+
|
|
51
|
+
/** The `text` handle: the posts' text, oldest first, joined by a rule. Posts with no text (a bare picture) add nothing. */
|
|
52
|
+
export function telegramFeedDigest(posts: ReadonlyArray<Pick<TelegramChannelPost, "text">>): string {
|
|
53
|
+
return posts
|
|
54
|
+
.map((p) => p.text)
|
|
55
|
+
.filter((t) => typeof t === "string" && t.trim().length > 0)
|
|
56
|
+
.join(TELEGRAM_FEED_DIGEST_SEPARATOR)
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
const isRecord = (v: unknown): v is Record<string, unknown> => typeof v === "object" && v !== null && !Array.isArray(v)
|
|
60
|
+
const str = (v: unknown): string | undefined => (typeof v === "string" ? v : undefined)
|
|
61
|
+
|
|
62
|
+
/**
|
|
63
|
+
* The posts held in a saved `generatedJson` / a run's `output.json`, normalized:
|
|
64
|
+
* anything that is not a post with a numeric id and a text is left out, and a
|
|
65
|
+
* post saved before a field existed reads with that field at its default.
|
|
66
|
+
*/
|
|
67
|
+
export function telegramPostsFrom(value: unknown): TelegramChannelPost[] {
|
|
68
|
+
if (!Array.isArray(value)) return []
|
|
69
|
+
const out: TelegramChannelPost[] = []
|
|
70
|
+
for (const item of value) {
|
|
71
|
+
if (!isRecord(item) || typeof item.id !== "number" || !Number.isFinite(item.id) || typeof item.text !== "string") continue
|
|
72
|
+
const media = Array.isArray(item.media)
|
|
73
|
+
? item.media.filter((m): m is TelegramChannelPostMedia => isRecord(m) && (m.type === "photo" || m.type === "video"))
|
|
74
|
+
: []
|
|
75
|
+
const forwarded = isRecord(item.forwardedFrom) && typeof item.forwardedFrom.name === "string"
|
|
76
|
+
? { name: item.forwardedFrom.name, ...(str(item.forwardedFrom.url) ? { url: item.forwardedFrom.url as string } : {}) }
|
|
77
|
+
: undefined
|
|
78
|
+
const channel = str(item.channel) ?? ""
|
|
79
|
+
out.push({
|
|
80
|
+
id: item.id,
|
|
81
|
+
channel,
|
|
82
|
+
postUrl: str(item.postUrl) ?? str(item.url) ?? (channel ? `https://t.me/${channel}/${item.id}` : ""),
|
|
83
|
+
text: item.text,
|
|
84
|
+
...(str(item.date) ? { date: item.date as string } : {}),
|
|
85
|
+
...(forwarded ? { forwardedFrom: forwarded } : {}),
|
|
86
|
+
media,
|
|
87
|
+
...(str(item.imageUrl) ? { imageUrl: item.imageUrl as string } : {}),
|
|
88
|
+
...(str(item.views) ? { views: item.views as string } : {}),
|
|
89
|
+
})
|
|
90
|
+
}
|
|
91
|
+
return out
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
export interface FeedEmissionPlan<P extends { readonly id: number }> {
|
|
95
|
+
/** The posts this run emits, oldest first. */
|
|
96
|
+
readonly emitted: P[]
|
|
97
|
+
/** The feed's position after this run (the cursor to store), or undefined when there is nothing to stand on. */
|
|
98
|
+
readonly latestId: number | undefined
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
/**
|
|
102
|
+
* Where the feed stands after a run — the ONE rule both the route and the
|
|
103
|
+
* tests read (decided 2026-10-05):
|
|
104
|
+
*
|
|
105
|
+
* - no position yet (first run, after Reset): the NEWEST `limit` posts, and
|
|
106
|
+
* the position jumps to the newest post — the person asked for what is new
|
|
107
|
+
* from now on; older posts are never revisited;
|
|
108
|
+
* - a stored position: the OLDEST `limit` posts above it, and the position
|
|
109
|
+
* becomes the highest post EMITTED — a backlog drains `limit` per run and
|
|
110
|
+
* nothing is lost (the old rule jumped to the newest post even when `limit`
|
|
111
|
+
* cut the fetch short, and the posts in between were gone for good).
|
|
112
|
+
*
|
|
113
|
+
* `posts` may come in any order and repeat (a pinned post renders twice).
|
|
114
|
+
*/
|
|
115
|
+
export function planFeedEmission<P extends { readonly id: number }>(
|
|
116
|
+
posts: ReadonlyArray<P>,
|
|
117
|
+
since: number | undefined,
|
|
118
|
+
limit: number,
|
|
119
|
+
/**
|
|
120
|
+
* The highest message id the fetched page(s) rendered above `since`, posts
|
|
121
|
+
* with nothing to read included. When nothing readable lies above the
|
|
122
|
+
* position but the page went further, the position moves past the unreadable
|
|
123
|
+
* posts — else the feed reads the same page every tick for ever.
|
|
124
|
+
*/
|
|
125
|
+
pageMaxId?: number,
|
|
126
|
+
): FeedEmissionPlan<P> {
|
|
127
|
+
const cap = Math.max(1, Math.min(TELEGRAM_FEED_LIMIT_MAX, Math.floor(Number.isFinite(limit) ? limit : TELEGRAM_FEED_DEFAULT_LIMIT) || 1))
|
|
128
|
+
const byId = new Map<number, P>()
|
|
129
|
+
for (const post of posts) byId.set(post.id, post)
|
|
130
|
+
const sorted = [...byId.values()].sort((a, b) => a.id - b.id)
|
|
131
|
+
if (since === undefined) {
|
|
132
|
+
const emitted = sorted.slice(-cap)
|
|
133
|
+
return { emitted, latestId: sorted.length > 0 ? sorted[sorted.length - 1]!.id : undefined }
|
|
134
|
+
}
|
|
135
|
+
const fresh = sorted.filter((p) => p.id > since)
|
|
136
|
+
const emitted = fresh.slice(0, cap)
|
|
137
|
+
if (emitted.length > 0) return { emitted, latestId: emitted[emitted.length - 1]!.id }
|
|
138
|
+
const pastUnreadable = pageMaxId !== undefined && Number.isFinite(pageMaxId) && pageMaxId > since
|
|
139
|
+
return { emitted, latestId: pastUnreadable ? pageMaxId : since }
|
|
140
|
+
}
|
|
@@ -3,6 +3,7 @@ import {
|
|
|
3
3
|
transcribeLaneSupportsWordTimestamps,
|
|
4
4
|
transcribeProvidersWithWordTimestamps,
|
|
5
5
|
} from "./model-constants.js"
|
|
6
|
+
import { rendersTranscriptJson } from "./render-nodes.js"
|
|
6
7
|
|
|
7
8
|
/**
|
|
8
9
|
* Pre-run checks for the transcribe → captions chain.
|
|
@@ -46,7 +47,9 @@ export interface WordlessTranscriptFeed {
|
|
|
46
47
|
// Handle ids (generated map: backend/src/lib/mcp/generated/node-handles.ts).
|
|
47
48
|
const TRANSCRIBE_JSON_OUT = "json"
|
|
48
49
|
const TRANSCRIPT_IN = "transcript"
|
|
49
|
-
|
|
50
|
+
/** A render's `json` output — handed on as a Transcript only by a render whose
|
|
51
|
+
* `jsonKind` is `transcript` (Apply EDL's is; an EDL-emitting render's is not). */
|
|
52
|
+
const RENDER_JSON_OUT = "json"
|
|
50
53
|
|
|
51
54
|
/**
|
|
52
55
|
* Every transcribe node on a word-INCAPABLE lane whose `json` output reaches an
|
|
@@ -74,7 +77,8 @@ export function findWordlessTranscriptFeeds(
|
|
|
74
77
|
const refusal = transcribeWordTimestampsRefusal(provider)
|
|
75
78
|
if (!refusal) continue
|
|
76
79
|
|
|
77
|
-
// Walk the transcript's path: transcribe.json → [
|
|
80
|
+
// Walk the transcript's path: transcribe.json → [render.transcript → render.json]* → add-captions.transcript,
|
|
81
|
+
// through every render whose json is a Transcript (RENDER_NODE_TYPES jsonKind)
|
|
78
82
|
const seen = new Set<string>()
|
|
79
83
|
const frontier: Array<{ id: string; outHandle: string }> = [{ id: node.id, outHandle: TRANSCRIBE_JSON_OUT }]
|
|
80
84
|
while (frontier.length > 0) {
|
|
@@ -90,9 +94,9 @@ export function findWordlessTranscriptFeeds(
|
|
|
90
94
|
provider,
|
|
91
95
|
message: `Captions need word timings, but ${refusal}.`,
|
|
92
96
|
})
|
|
93
|
-
} else if (target!.type
|
|
97
|
+
} else if (rendersTranscriptJson(target!.type) && !seen.has(target!.id)) {
|
|
94
98
|
seen.add(target!.id)
|
|
95
|
-
frontier.push({ id: target!.id, outHandle:
|
|
99
|
+
frontier.push({ id: target!.id, outHandle: RENDER_JSON_OUT })
|
|
96
100
|
}
|
|
97
101
|
}
|
|
98
102
|
}
|
package/src/tts-capabilities.ts
CHANGED
|
@@ -3,15 +3,15 @@
|
|
|
3
3
|
*
|
|
4
4
|
* Every "does this model do X" decision on that lane reads the model's `tts`
|
|
5
5
|
* sheet in `MODEL_CATALOG` through these helpers — tag stripping, which voice
|
|
6
|
-
* settings are sent and shown, the language picker
|
|
6
|
+
* settings are sent and shown, the language picker, whether neighbouring text
|
|
7
|
+
* is sent. Declare the sheet honestly
|
|
7
8
|
* on the catalog entry and a new model is covered everywhere; do not compare a
|
|
8
9
|
* provider id by hand.
|
|
9
10
|
*
|
|
10
11
|
* "The text-to-speech lane" means the models that list the `tts` mode. The
|
|
11
|
-
* dialogue
|
|
12
|
-
* runs as the fallback model, so these
|
|
13
|
-
*
|
|
14
|
-
* `MAX_TTS_CHARS_BY_PROVIDER` and the language picker.)
|
|
12
|
+
* dialogue models carry a sheet too, read by `dialogue-capabilities.ts`; a
|
|
13
|
+
* text-to-speech request that names one runs as the fallback model, so these
|
|
14
|
+
* helpers answer for the fallback there.
|
|
15
15
|
*/
|
|
16
16
|
import { MODEL_CATALOG, type TtsCapabilities, type TtsSettingLever } from "./model-catalog.js"
|
|
17
17
|
|
|
@@ -64,11 +64,25 @@ export function ttsSupportsSsmlBreaks(provider: string | undefined): boolean {
|
|
|
64
64
|
return getTtsCapabilities(provider).ssmlBreaks
|
|
65
65
|
}
|
|
66
66
|
|
|
67
|
+
/**
|
|
68
|
+
* The model conditions on neighbouring text (`previous_text` / `next_text`).
|
|
69
|
+
* false ⇒ the provider funnel never puts the fields on the wire, whatever the
|
|
70
|
+
* request carries — the model would reject them.
|
|
71
|
+
*/
|
|
72
|
+
export function ttsSupportsStitching(provider: string | undefined): boolean {
|
|
73
|
+
return getTtsCapabilities(provider).stitching
|
|
74
|
+
}
|
|
75
|
+
|
|
67
76
|
/** The model honours this voice-setting lever. */
|
|
68
77
|
export function ttsHasLever(provider: string | undefined, lever: TtsSettingLever): boolean {
|
|
69
78
|
return getTtsCapabilities(provider).levers.includes(lever)
|
|
70
79
|
}
|
|
71
80
|
|
|
81
|
+
/** The model answers `/v1/text-to-speech/{voice}/with-timestamps` (character timings). */
|
|
82
|
+
export function ttsSupportsTimestamps(provider: string | undefined): boolean {
|
|
83
|
+
return getTtsCapabilities(provider).timestamps
|
|
84
|
+
}
|
|
85
|
+
|
|
72
86
|
/** Language codes offered for the model, in catalog order. */
|
|
73
87
|
export function ttsLanguageCodes(provider: string | undefined): readonly string[] {
|
|
74
88
|
return getTtsCapabilities(provider).languages
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The neighbour-text rule of Text to Speech — what is spoken just BEFORE and just AFTER a clip in the finished
|
|
3
|
+
* piece — in ONE place, so the editor's run, the provider exits and the REST route cannot drift: a request that
|
|
4
|
+
* the editor sends must be one the route accepts.
|
|
5
|
+
*
|
|
6
|
+
* A string is trimmed; an empty one, or anything that is not a string, is ABSENT. Over the cap the value is
|
|
7
|
+
* TRIMMED, never rejected, by every sender and exit: previous text keeps its LAST characters (what leads into
|
|
8
|
+
* the clip), next text its FIRST (what the clip leads into). The REST route is the one place that rejects an
|
|
9
|
+
* over-cap value — measured on the trimmed text, for a model that stitches only. Whether a model takes the
|
|
10
|
+
* fields at all is its sheet (`ttsSupportsStitching`), never decided here.
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* The most characters of context sent on each side. ElevenLabs documents no limit for the text-to-speech
|
|
15
|
+
* endpoint's fields; this bounds what we send (and what the vendor may count) to about two sentences' worth of
|
|
16
|
+
* run-in and run-out — enough to carry intonation across a cut.
|
|
17
|
+
*/
|
|
18
|
+
export const TTS_NEIGHBOUR_TEXT_MAX_CHARS = 1000
|
|
19
|
+
|
|
20
|
+
/** The normalised pair: only non-empty strings within the cap; a side that was unusable is not a key. */
|
|
21
|
+
export interface TtsNeighbourText {
|
|
22
|
+
previousText?: string
|
|
23
|
+
nextText?: string
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
function usableText(value: unknown): string | undefined {
|
|
27
|
+
if (typeof value !== "string") return undefined
|
|
28
|
+
const trimmed = value.trim()
|
|
29
|
+
return trimmed ? trimmed : undefined
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/** `previousText` / `nextText` of `options`, normalised. Other keys are not carried over; the input is never mutated. */
|
|
33
|
+
export function normalizeTtsNeighbourText(
|
|
34
|
+
options: Readonly<{ previousText?: unknown; nextText?: unknown }> | undefined,
|
|
35
|
+
): TtsNeighbourText {
|
|
36
|
+
const out: TtsNeighbourText = {}
|
|
37
|
+
const previous = usableText(options?.previousText)
|
|
38
|
+
if (previous !== undefined) {
|
|
39
|
+
out.previousText = previous.length > TTS_NEIGHBOUR_TEXT_MAX_CHARS ? previous.slice(-TTS_NEIGHBOUR_TEXT_MAX_CHARS) : previous
|
|
40
|
+
}
|
|
41
|
+
const next = usableText(options?.nextText)
|
|
42
|
+
if (next !== undefined) {
|
|
43
|
+
out.nextText = next.length > TTS_NEIGHBOUR_TEXT_MAX_CHARS ? next.slice(0, TTS_NEIGHBOUR_TEXT_MAX_CHARS) : next
|
|
44
|
+
}
|
|
45
|
+
return out
|
|
46
|
+
}
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A UGC builder call `{ tool, args }` (MCP argument names) → the REST body of
|
|
3
|
+
* the public route that tool dispatches to. The builder sets every price lever;
|
|
4
|
+
* this adds none — its one addition is the image of a call that needs the
|
|
5
|
+
* previous step's image (`imageArg`, or the photo of `image_to_text`). A tool
|
|
6
|
+
* it does not know → undefined, so an app older than the builder fails plainly
|
|
7
|
+
* instead of guessing. Kept equal to the MCP verbs by ugc-call-body-parity.test.ts.
|
|
8
|
+
*/
|
|
9
|
+
export interface UgcToolCall {
|
|
10
|
+
readonly tool: string
|
|
11
|
+
readonly args: Readonly<Record<string, unknown>>
|
|
12
|
+
readonly imageArg?: string
|
|
13
|
+
}
|
|
14
|
+
export type UgcRouteBody = {
|
|
15
|
+
readonly path: "/v1/generate-image" | "/v1/image-to-image" | "/v1/image-to-text/describe"
|
|
16
|
+
readonly body: Record<string, unknown>
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
const pick = (a: Readonly<Record<string, unknown>>, from: string, to: string): Record<string, unknown> =>
|
|
20
|
+
a[from] !== undefined && a[from] !== null ? { [to]: a[from] } : {}
|
|
21
|
+
|
|
22
|
+
/** The LLM levers image_to_text forwards (the verb's llmPayloadFields). */
|
|
23
|
+
function llmFields(a: Readonly<Record<string, unknown>>): Record<string, unknown> {
|
|
24
|
+
return {
|
|
25
|
+
...pick(a, "llmModel", "llmModel"),
|
|
26
|
+
...pick(a, "reasoning_effort", "reasoningEffort"),
|
|
27
|
+
...(a.advanced_mode === true ? { advancedMode: true } : {}),
|
|
28
|
+
...pick(a, "temperature", "temperature"),
|
|
29
|
+
...pick(a, "max_tokens", "maxTokens"),
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
export function ugcCallToRouteBody(
|
|
34
|
+
call: UgcToolCall,
|
|
35
|
+
opts: { readonly image?: string } = {},
|
|
36
|
+
): UgcRouteBody | { readonly error: string } | undefined {
|
|
37
|
+
const args: Record<string, unknown> = { ...call.args }
|
|
38
|
+
if (call.imageArg) {
|
|
39
|
+
if (!opts.image) return { error: `${call.tool} needs the previous step's image` }
|
|
40
|
+
args[call.imageArg] = opts.image
|
|
41
|
+
}
|
|
42
|
+
switch (call.tool) {
|
|
43
|
+
case "generate_image":
|
|
44
|
+
return {
|
|
45
|
+
path: "/v1/generate-image",
|
|
46
|
+
body: {
|
|
47
|
+
...pick(args, "prompt", "prompt"),
|
|
48
|
+
...pick(args, "model", "provider"),
|
|
49
|
+
...pick(args, "aspect_ratio", "aspectRatio"),
|
|
50
|
+
...pick(args, "resolution", "resolution"),
|
|
51
|
+
...pick(args, "quality", "quality"),
|
|
52
|
+
...pick(args, "negative_prompt", "negativePrompt"),
|
|
53
|
+
},
|
|
54
|
+
}
|
|
55
|
+
case "image_to_image":
|
|
56
|
+
return {
|
|
57
|
+
path: "/v1/image-to-image",
|
|
58
|
+
body: {
|
|
59
|
+
...pick(args, "image_url", "imageUrl"),
|
|
60
|
+
...pick(args, "prompt", "prompt"),
|
|
61
|
+
...pick(args, "model", "provider"),
|
|
62
|
+
...pick(args, "resolution", "resolution"),
|
|
63
|
+
...pick(args, "quality", "quality"),
|
|
64
|
+
...pick(args, "strength", "strength"),
|
|
65
|
+
...pick(args, "aspect_ratio", "aspectRatio"),
|
|
66
|
+
...pick(args, "negative_prompt", "negativePrompt"),
|
|
67
|
+
...pick(args, "seed", "seed"),
|
|
68
|
+
},
|
|
69
|
+
}
|
|
70
|
+
case "image_to_text": {
|
|
71
|
+
const image = opts.image ?? (typeof args.image_url === "string" ? args.image_url : undefined)
|
|
72
|
+
if (!image) return { error: "image_to_text needs an image" }
|
|
73
|
+
return {
|
|
74
|
+
path: "/v1/image-to-text/describe",
|
|
75
|
+
body: {
|
|
76
|
+
imageUrl: image,
|
|
77
|
+
...pick(args, "detail_level", "detailLevel"),
|
|
78
|
+
...pick(args, "custom_prompt", "customPrompt"),
|
|
79
|
+
...llmFields(args),
|
|
80
|
+
},
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
default:
|
|
84
|
+
return undefined
|
|
85
|
+
}
|
|
86
|
+
}
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
/** The five UGC node types (Cloud only; served by a private plugin). */
|
|
2
|
+
export const UGC_NODE_TYPES: ReadonlySet<string> = new Set(["ugc-creator", "ugc-script", "ugc-clips", "ugc-clip", "ugc-cards"])
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* Per UGC type, the node-data keys that are a RUN's state, never a choice. A
|
|
6
|
+
* published app's snapshot is readable by every runner, so publishing strips
|
|
7
|
+
* them (the last plan, the clip tickets, the publisher's kept creator).
|
|
8
|
+
*/
|
|
9
|
+
export const UGC_NODE_RUN_STATE_KEYS: Readonly<Record<string, readonly string[]>> = {
|
|
10
|
+
"ugc-creator": ["result", "keepResult", "ugcOutputs"],
|
|
11
|
+
"ugc-script": ["result", "keepResult", "ugcOutputs"],
|
|
12
|
+
"ugc-clips": ["result", "quote", "__listResults", "ugcOutputs"],
|
|
13
|
+
"ugc-clip": ["warnings", "rerender", "__listResults", "__listResultMeta", "clipWarnings", "durationSec", "ugcOutputs"],
|
|
14
|
+
"ugc-cards": ["result", "ugcOutputs"],
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* The ONLY UGC keys a run request's `inputOverrides` may set — an allow-list,
|
|
19
|
+
* so a run-state field added later is locked without anyone listing it.
|
|
20
|
+
*
|
|
21
|
+
* No UGC type may gain a `LEGACY_EXPOSED_FIELD_KEYS` row: the lock reads the
|
|
22
|
+
* stored key, before that table renames it.
|
|
23
|
+
*/
|
|
24
|
+
export const UGC_OVERRIDABLE_FIELDS: Readonly<Record<string, readonly string[]>> = {
|
|
25
|
+
"ugc-creator": ["source", "gender", "photoUrl"],
|
|
26
|
+
"ugc-script": ["targetDurationSec"],
|
|
27
|
+
"ugc-clips": [],
|
|
28
|
+
"ugc-clip": [],
|
|
29
|
+
"ugc-cards": [],
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
const KEEP_RESULT_TYPES: ReadonlySet<string> = new Set(["ugc-creator", "ugc-script"])
|
|
33
|
+
|
|
34
|
+
/** A copy of `nodes` with every UGC node's run state removed (and `keepResult: false`). Other nodes are returned as they are. */
|
|
35
|
+
export function stripUgcRunState<T extends { type?: unknown; data?: unknown }>(nodes: readonly T[]): T[] {
|
|
36
|
+
return nodes.map((node) => {
|
|
37
|
+
const type = typeof node.type === "string" ? node.type : ""
|
|
38
|
+
if (!UGC_NODE_TYPES.has(type) || !node.data || typeof node.data !== "object") return node
|
|
39
|
+
const data: Record<string, unknown> = { ...(node.data as Record<string, unknown>) }
|
|
40
|
+
for (const key of UGC_NODE_RUN_STATE_KEYS[type] ?? []) delete data[key]
|
|
41
|
+
if (KEEP_RESULT_TYPES.has(type)) data.keepResult = false
|
|
42
|
+
return { ...node, data }
|
|
43
|
+
})
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/** Every override entry that would set a non-overridable key on a UGC node, in node then key order. Pure. */
|
|
47
|
+
export function findUgcLockedFields(
|
|
48
|
+
nodes: ReadonlyArray<{ id: string; type?: string }> | null | undefined,
|
|
49
|
+
inputOverrides: Record<string, Record<string, unknown>> | null | undefined,
|
|
50
|
+
): Array<{ nodeId: string; nodeType: string; field: string }> {
|
|
51
|
+
if (!nodes || !inputOverrides) return []
|
|
52
|
+
const found: Array<{ nodeId: string; nodeType: string; field: string }> = []
|
|
53
|
+
for (const node of nodes) {
|
|
54
|
+
const type = node?.type
|
|
55
|
+
if (typeof type !== "string" || !UGC_NODE_TYPES.has(type)) continue
|
|
56
|
+
const allowed = UGC_OVERRIDABLE_FIELDS[type] ?? []
|
|
57
|
+
const fields = inputOverrides[node.id]
|
|
58
|
+
if (!fields || typeof fields !== "object") continue
|
|
59
|
+
for (const field of Object.keys(fields)) {
|
|
60
|
+
if (!allowed.includes(field)) found.push({ nodeId: String(node.id), nodeType: type, field })
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
return found
|
|
64
|
+
}
|
|
@@ -0,0 +1,233 @@
|
|
|
1
|
+
import { VIDEO_REF_LIMITS_BY_PROVIDER } from "./model-constants.js"
|
|
2
|
+
import type { GeminiOmniVoicePresetId } from "./gemini-omni-voices.js"
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* Character references — the dedicated IDENTITY input of a video model, as
|
|
6
|
+
* opposed to a loose reference image.
|
|
7
|
+
*
|
|
8
|
+
* Gemini Omni treats `image_urls` as "reference images for characters, scenes,
|
|
9
|
+
* styles, or storyboard guidance": a portrait sent there steers the clip but
|
|
10
|
+
* the face drifts. Its separate `character_ids` input is the one that keeps a
|
|
11
|
+
* real person's face, and it takes ids minted from a portrait by a create
|
|
12
|
+
* call. The PUBLIC shape below is provider-neutral — a portrait and a
|
|
13
|
+
* description — and the provider layer mints (and caches) whatever ids its
|
|
14
|
+
* upstream needs. Nothing upstream-shaped ever appears in the API.
|
|
15
|
+
*
|
|
16
|
+
* WHICH models take them is data, never a name list: a positive `characters`
|
|
17
|
+
* cap on `VIDEO_REF_LIMITS_BY_PROVIDER`.
|
|
18
|
+
*/
|
|
19
|
+
export interface VideoCharacterVoice {
|
|
20
|
+
/** The base voice — one of `GEMINI_OMNI_VOICE_PRESETS`. */
|
|
21
|
+
preset: GeminiOmniVoicePresetId
|
|
22
|
+
/** Free-text timbre / pace / emotion for the persona. Optional. */
|
|
23
|
+
description?: string
|
|
24
|
+
/** One example line the persona says (≤ 120 chars). Optional. */
|
|
25
|
+
exampleLine?: string
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
export interface VideoCharacterReference {
|
|
29
|
+
/** Portrait of the person (public URL). Index 0 of the upstream image set. */
|
|
30
|
+
imageUrl: string
|
|
31
|
+
/** Optional full-body image; costs a second input unit. */
|
|
32
|
+
bodyImageUrl?: string
|
|
33
|
+
/** Who this is: appearance, identity, style, clothing. Required by the upstream. */
|
|
34
|
+
description: string
|
|
35
|
+
/** Optional display name. */
|
|
36
|
+
name?: string
|
|
37
|
+
/**
|
|
38
|
+
* Pin a voice persona to this character so it keeps ONE voice across clips
|
|
39
|
+
* (the model otherwise improvises a voice per generation). Only models with a
|
|
40
|
+
* positive `voices` cap take it.
|
|
41
|
+
*/
|
|
42
|
+
voice?: VideoCharacterVoice
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/** Zod `.max()` ceiling for the wire field — the widest cap any provider declares. */
|
|
46
|
+
export const VIDEO_CHARACTER_REFS_WIRE_MAX = Math.max(
|
|
47
|
+
0,
|
|
48
|
+
...Object.values(VIDEO_REF_LIMITS_BY_PROVIDER).map((l) => l?.characters ?? 0),
|
|
49
|
+
)
|
|
50
|
+
|
|
51
|
+
/** Longest `description` / `name` a client may send (route Zod reads these). */
|
|
52
|
+
export const VIDEO_CHARACTER_DESCRIPTION_MAX = 2000
|
|
53
|
+
export const VIDEO_CHARACTER_NAME_MAX = 100
|
|
54
|
+
/** Longest voice `description` / `exampleLine` a client may send. The upstream
|
|
55
|
+
* allows far more description (20000); we cap the PUBLIC field at the same
|
|
56
|
+
* ceiling as a character description so policy text and prompts stay bounded. */
|
|
57
|
+
export const VIDEO_CHARACTER_VOICE_DESCRIPTION_MAX = 2000
|
|
58
|
+
export const VIDEO_CHARACTER_VOICE_EXAMPLE_MAX = 120
|
|
59
|
+
|
|
60
|
+
/** How many character references the provider takes; 0 = none (or unknown). */
|
|
61
|
+
export function videoCharacterRefCap(provider: string | undefined): number {
|
|
62
|
+
return provider ? (VIDEO_REF_LIMITS_BY_PROVIDER[provider]?.characters ?? 0) : 0
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/** How many DISTINCT voice personas the provider takes per request; 0 = none. */
|
|
66
|
+
export function videoCharacterVoiceCap(provider: string | undefined): number {
|
|
67
|
+
return provider ? (VIDEO_REF_LIMITS_BY_PROVIDER[provider]?.voices ?? 0) : 0
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/**
|
|
71
|
+
* Identity of a voice persona — what makes two voices "the same voice". The
|
|
72
|
+
* persona is minted per (preset, description, example line) AND the character's
|
|
73
|
+
* name (the label it is created under), so the name is part of the identity:
|
|
74
|
+
* this count must equal the number of personas the provider will actually mint,
|
|
75
|
+
* or the cap would be checked against a different number than what ships.
|
|
76
|
+
*/
|
|
77
|
+
export function videoCharacterVoiceKey(ref: Pick<VideoCharacterReference, "voice" | "name">): string | null {
|
|
78
|
+
const voice = ref.voice
|
|
79
|
+
if (!voice) return null
|
|
80
|
+
return JSON.stringify([voice.preset, voice.description?.trim() ?? "", voice.exampleLine?.trim() ?? "", ref.name?.trim() ?? ""])
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/** How many distinct voice personas a set of references asks for. */
|
|
84
|
+
export function videoCharacterDistinctVoices(
|
|
85
|
+
characterReferences: readonly Pick<VideoCharacterReference, "voice" | "name">[] | undefined,
|
|
86
|
+
): number {
|
|
87
|
+
const keys = new Set<string>()
|
|
88
|
+
for (const c of characterReferences ?? []) {
|
|
89
|
+
const key = videoCharacterVoiceKey(c)
|
|
90
|
+
if (key) keys.add(key)
|
|
91
|
+
}
|
|
92
|
+
return keys.size
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
/** Every provider that declares a character-reference cap, in declaration order. */
|
|
96
|
+
export function videoCharacterRefProviders(): string[] {
|
|
97
|
+
return Object.entries(VIDEO_REF_LIMITS_BY_PROVIDER)
|
|
98
|
+
.filter(([, l]) => (l?.characters ?? 0) > 0)
|
|
99
|
+
.map(([id]) => id)
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/**
|
|
103
|
+
* Input units a set of character references draws from the provider's shared
|
|
104
|
+
* input budget: 1 per character, 2 when a body image is supplied (the upstream
|
|
105
|
+
* counts a dual-image character as two slots).
|
|
106
|
+
*/
|
|
107
|
+
export function videoCharacterRefUnits(
|
|
108
|
+
characterReferences: readonly Pick<VideoCharacterReference, "bodyImageUrl">[] | undefined,
|
|
109
|
+
): number {
|
|
110
|
+
let units = 0
|
|
111
|
+
for (const c of characterReferences ?? []) units += c.bodyImageUrl ? 2 : 1
|
|
112
|
+
return units
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
export interface VideoCharacterRefProblem {
|
|
116
|
+
code:
|
|
117
|
+
| "character_references_unsupported"
|
|
118
|
+
| "character_references_over_limit"
|
|
119
|
+
| "character_references_with_start_frame"
|
|
120
|
+
| "character_references_with_end_frame"
|
|
121
|
+
| "character_references_quota"
|
|
122
|
+
| "character_voice_unsupported"
|
|
123
|
+
| "character_voices_over_limit"
|
|
124
|
+
message: string
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
/**
|
|
128
|
+
* Voice-persona rule: the provider must take voices at all, and the DISTINCT
|
|
129
|
+
* voices must fit its cap (Gemini Omni: 3 `audio_ids` per task). Two characters
|
|
130
|
+
* with an identical voice AND name count once. `cap` overrides the provider's declared cap
|
|
131
|
+
* (tests; a future model with a different limit declares it as data instead).
|
|
132
|
+
*/
|
|
133
|
+
export function videoCharacterVoiceProblem(args: {
|
|
134
|
+
provider: string | undefined
|
|
135
|
+
characterReferences: readonly VideoCharacterReference[] | undefined
|
|
136
|
+
cap?: number
|
|
137
|
+
}): VideoCharacterRefProblem | null {
|
|
138
|
+
const distinct = videoCharacterDistinctVoices(args.characterReferences)
|
|
139
|
+
if (distinct === 0) return null
|
|
140
|
+
const cap = args.cap ?? videoCharacterVoiceCap(args.provider)
|
|
141
|
+
if (cap === 0) {
|
|
142
|
+
return {
|
|
143
|
+
code: "character_voice_unsupported",
|
|
144
|
+
message:
|
|
145
|
+
`A character voice is not supported by ${args.provider ?? "the selected model"}. ` +
|
|
146
|
+
"Drop the voice, or pick a model that takes characterReferences.",
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
if (distinct > cap) {
|
|
150
|
+
return {
|
|
151
|
+
code: "character_voices_over_limit",
|
|
152
|
+
message: `${args.provider} accepts at most ${cap} distinct character voices per video; got ${distinct}. Give some characters the same voice, or drop some.`,
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
return null
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
/**
|
|
159
|
+
* The one validation both video routes (and the provider backstop) run. Returns
|
|
160
|
+
* the first problem as a friendly 400 body, or null when the request is fine.
|
|
161
|
+
*
|
|
162
|
+
* The shared budget: for a model that declares `characters`, its `images` cap
|
|
163
|
+
* IS the total input budget (images + 2×videos + character units ≤ images cap —
|
|
164
|
+
* 7 for Gemini Omni). Callers pass the ASSEMBLED image count, after any
|
|
165
|
+
* `connectedReferences` expansion, so the budget is measured on what ships.
|
|
166
|
+
*/
|
|
167
|
+
export function videoCharacterRefProblem(args: {
|
|
168
|
+
provider: string | undefined
|
|
169
|
+
characterReferences: readonly VideoCharacterReference[] | undefined
|
|
170
|
+
/** Reference images that will ship (assembled, flat). */
|
|
171
|
+
imageCount: number
|
|
172
|
+
/** Source / reference videos that will ship. */
|
|
173
|
+
videoCount: number
|
|
174
|
+
/** True when the request carries a start frame (`imageUrl`). */
|
|
175
|
+
hasStartFrame: boolean
|
|
176
|
+
/** True when the request carries an end frame (`endFrameUrl`). */
|
|
177
|
+
hasEndFrame?: boolean
|
|
178
|
+
}): VideoCharacterRefProblem | null {
|
|
179
|
+
const { provider, characterReferences } = args
|
|
180
|
+
if (!characterReferences || characterReferences.length === 0) return null
|
|
181
|
+
|
|
182
|
+
const cap = videoCharacterRefCap(provider)
|
|
183
|
+
if (cap === 0) {
|
|
184
|
+
return {
|
|
185
|
+
code: "character_references_unsupported",
|
|
186
|
+
message:
|
|
187
|
+
`characterReferences is not supported by ${provider ?? "the selected model"}. ` +
|
|
188
|
+
`Models that take them: ${videoCharacterRefProviders().join(", ")}. ` +
|
|
189
|
+
"Otherwise send the portrait as a reference image instead.",
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
if (characterReferences.length > cap) {
|
|
193
|
+
return {
|
|
194
|
+
code: "character_references_over_limit",
|
|
195
|
+
message: `${provider} accepts at most ${cap} character references; got ${characterReferences.length}.`,
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
// A NODARO limit, not an upstream exclusivity: Gemini Omni's start frame is
|
|
199
|
+
// sent as the first entry of `image_urls` (never as `first_frame_url`), so
|
|
200
|
+
// the upstream would take it beside `character_ids`. We reject the pair
|
|
201
|
+
// because the frame / identity roles are not yet bound in the prompt
|
|
202
|
+
// (and the frame would need to be budgeted against the shared input quota).
|
|
203
|
+
if (args.hasStartFrame) {
|
|
204
|
+
return {
|
|
205
|
+
code: "character_references_with_start_frame",
|
|
206
|
+
message:
|
|
207
|
+
`${provider} does not yet support combining characterReferences with a start frame (imageUrl) — ` +
|
|
208
|
+
"the frame and the characters would compete for the same input slots without a defined role. " +
|
|
209
|
+
"Drop the start frame (the character's portrait carries the face), or drop characterReferences.",
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
// Gemini Omni has no end-frame input; without this the frame would be dropped
|
|
213
|
+
// silently once characters make a frame-less request look "referenced".
|
|
214
|
+
if (args.hasEndFrame) {
|
|
215
|
+
return {
|
|
216
|
+
code: "character_references_with_end_frame",
|
|
217
|
+
message:
|
|
218
|
+
`${provider} does not use an end frame (endFrameUrl) together with characterReferences. ` +
|
|
219
|
+
"Drop the end frame, or drop characterReferences.",
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
const budget = VIDEO_REF_LIMITS_BY_PROVIDER[provider as string]?.images ?? 7
|
|
223
|
+
const used = args.imageCount + args.videoCount * 2 + videoCharacterRefUnits(characterReferences)
|
|
224
|
+
if (used > budget) {
|
|
225
|
+
return {
|
|
226
|
+
code: "character_references_quota",
|
|
227
|
+
message:
|
|
228
|
+
`${provider} takes at most ${budget} input units: reference images count 1, each reference video 2, ` +
|
|
229
|
+
`each character 1 (2 with a body image). This request uses ${used}. Remove some reference images or characters.`,
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
return videoCharacterVoiceProblem({ provider, characterReferences })
|
|
233
|
+
}
|