@nodaro/shared 1.21.0 → 1.23.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/llm-models.ts CHANGED
@@ -51,8 +51,44 @@ export interface LlmModelDef {
51
51
  structuredOutputMode?: "anthropic-tool" | "kie-response-format" | "responses-json-schema"
52
52
  /** If set, fallback to direct Anthropic SDK with this model ID when KIE.ai fails */
53
53
  directFallbackModel?: string
54
+ /**
55
+ * Google Gemini API model id for the DIRECT lane (generativelanguage, keyed
56
+ * by `GEMINI_API_KEY`) — the Google-side twin of `directFallbackModel`.
57
+ * Presence declares "this model CAN be served straight from Google"; absence
58
+ * pins it to KIE forever. Stated, never derived: Google carries `-preview`
59
+ * suffixes on unreleased models, so the id routinely differs from both `id`
60
+ * and `kieSlugOrModel` (`gemini-3.1-pro` → `gemini-3.1-pro-preview`).
61
+ */
62
+ directGeminiModel?: string
63
+ /**
64
+ * Try the direct-vendor lane FIRST for this model, with KIE as the failure
65
+ * fallback. Absent (while `directGeminiModel` is set) = KIE first, direct
66
+ * only when KIE fails.
67
+ *
68
+ * This is a per-model COST decision, not just a routing one: the two lanes
69
+ * bill the same model at materially different unit rates, so a model that
70
+ * backs a high-volume default (see `LLM_FEATURE_DEFAULTS`) is usually better
71
+ * left on whichever lane is cheaper. The rate tables for both lanes live in
72
+ * `backend/src/lib/pricing/llm-cost.ts` — deliberately not in this package,
73
+ * which is published to npm.
74
+ *
75
+ * Mutually exclusive with `preferKie` — the Claude-side half of the same
76
+ * idea. Guarded by a registry test so the two can't both be set.
77
+ */
78
+ preferDirect?: true
54
79
  /** Effort levels this model accepts (ascending). Absent/empty = no effort lever, picker hidden. */
55
80
  reasoningEfforts?: readonly LlmReasoningEffort[]
81
+ /**
82
+ * Effort levels available on the DIRECT lane, when the vendor's own API
83
+ * accepts more than the aggregator does. Absent = the direct lane offers the
84
+ * same set as `reasoningEfforts`.
85
+ *
86
+ * This exists because `reasoningEfforts` has to stay at the KIE-safe
87
+ * intersection — sending a level KIE rejects is a hard failure — while the
88
+ * vendor API accepts the full ladder. Unlocking those extra levels is one of
89
+ * the concrete things Advanced mode buys.
90
+ */
91
+ directReasoningEfforts?: readonly LlmReasoningEffort[]
56
92
  /** false = model rejects `temperature` (Claude 5-era, GPT-5.6). Absent = accepts. */
57
93
  supportsTemperature?: false
58
94
  /** Claude-only: KIE is the preferred routing, direct Anthropic the fallback. */
@@ -85,6 +121,13 @@ export const LLM_MODELS: readonly LlmModelDef[] = [
85
121
  structuredOutputMode: "kie-response-format",
86
122
  supportsImages: true,
87
123
  maxOutputTokens: 8192,
124
+ // KIE-first: no `preferDirect` — direct is the reliability fallback only.
125
+ // (Per-lane rates are deliberately NOT in this published package; see
126
+ // backend/src/lib/pricing/llm-cost.ts.)
127
+ directGeminiModel: "gemini-3-flash-preview",
128
+ // No `reasoningEfforts` at all on the KIE lane, but the vendor API accepts
129
+ // the full minimal→high ladder (`none` maps to Google's `minimal`).
130
+ directReasoningEfforts: ["none", "low", "medium", "high"],
88
131
  },
89
132
  {
90
133
  id: "gemini-3.6-flash",
@@ -102,7 +145,19 @@ export const LLM_MODELS: readonly LlmModelDef[] = [
102
145
  maxOutputTokens: 8192,
103
146
  // KIE's 3.6 endpoint accepts `reasoning_effort: low | high` (thinking
104
147
  // level) — exactly the chat-completions wire mapping deriveParams sends.
148
+ // Google's own API additionally accepts `minimal` and `medium` on this
149
+ // model; the set stays at the KIE-safe intersection because ONE field
150
+ // feeds both lanes and this model is KIE-first. Widen it only if/when
151
+ // `preferDirect` is set here.
105
152
  reasoningEfforts: ["low", "high"],
153
+ // Google's own API additionally accepts `minimal` and `medium` here —
154
+ // live-verified 2026-07-28. Advanced mode unlocks them.
155
+ directReasoningEfforts: ["none", "low", "medium", "high"],
156
+ // KIE-first: this model backs 5 of the LLM_FEATURE_DEFAULTS plus the
157
+ // video-analysis fast tier, so it carries the highest call volume of any
158
+ // Gemini entry — the lane with the lower unit cost wins by default and
159
+ // direct is the reliability fallback only.
160
+ directGeminiModel: "gemini-3.6-flash",
106
161
  },
107
162
  {
108
163
  id: "claude-haiku-4.5",
@@ -153,6 +208,18 @@ export const LLM_MODELS: readonly LlmModelDef[] = [
153
208
  structuredOutputMode: "kie-response-format",
154
209
  supportsImages: true,
155
210
  maxOutputTokens: 16384,
211
+ directGeminiModel: "gemini-3.1-pro-preview",
212
+ // Google documents low/medium/high for 3.1 Pro — no `minimal` tier, so
213
+ // this ladder is deliberately shorter than the flash models'.
214
+ directReasoningEfforts: ["low", "medium", "high"],
215
+ // The ONE Gemini model routed direct-first. It is the premium/low-volume
216
+ // tier (video-analysis `pro`, no LLM_FEATURE_DEFAULTS entry), so the ~4×
217
+ // list-price premium lands on the smallest call volume — and it is where
218
+ // the direct lane's capability wins actually matter: real `thinkingLevel`
219
+ // control, native media ingestion, and a `responseJsonSchema` that honours
220
+ // `additionalProperties` (KIE's `response_format` silently DROPS
221
+ // record/map-shaped fields — see the z.record rule in backend/CLAUDE.md).
222
+ preferDirect: true,
156
223
  },
157
224
  {
158
225
  id: "claude-opus-4.7",
@@ -425,21 +492,45 @@ export function getLlmModel(id: string): LlmModelDef | undefined {
425
492
  if (aliased) return aliased
426
493
  }
427
494
  // Last resort: provider slugs double as historical aliases (e.g. the
428
- // dated Anthropic slugs) — accept any model whose slug matches exactly.
429
- return LLM_MODELS.find((m) => m.kieSlugOrModel === id || m.directFallbackModel === id)
495
+ // dated Anthropic slugs, the `-preview`-suffixed Google ids) — accept any
496
+ // model whose slug matches exactly, on either lane.
497
+ return LLM_MODELS.find(
498
+ (m) => m.kieSlugOrModel === id || m.directFallbackModel === id || m.directGeminiModel === id,
499
+ )
430
500
  }
431
501
 
432
502
  export function getLlmTier(id: string): LlmTier {
433
503
  return getLlmModel(id)?.tier ?? "standard"
434
504
  }
435
505
 
506
+ /**
507
+ * Effort levels this model actually accepts on the lane it will be served on.
508
+ *
509
+ * The two lanes do NOT offer the same ladder: the aggregator accepts a narrower
510
+ * set than the vendor's own API does, which is one of the concrete things
511
+ * Advanced mode buys. Kept as one lookup so the UI picker and the wire-side
512
+ * clamp can never disagree about what's selectable.
513
+ */
514
+ export function availableReasoningEfforts(
515
+ modelId: string | undefined,
516
+ advanced = false,
517
+ ): readonly LlmReasoningEffort[] {
518
+ const model = getLlmModel(modelId ?? "")
519
+ if (!model) return []
520
+ if (advanced && supportsAdvancedMode(modelId)) {
521
+ return model.directReasoningEfforts ?? model.reasoningEfforts ?? []
522
+ }
523
+ return model.reasoningEfforts ?? []
524
+ }
525
+
436
526
  /** Highest level the model supports that is ≤ the requested level; undefined = treat as Auto. */
437
527
  export function effectiveReasoningEffort(
438
528
  modelId: string | undefined,
439
529
  requested?: string,
530
+ advanced = false,
440
531
  ): LlmReasoningEffort | undefined {
441
532
  if (!requested || !(requested in EFFORT_RANK)) return undefined
442
- const levels = getLlmModel(modelId ?? "")?.reasoningEfforts
533
+ const levels = availableReasoningEfforts(modelId, advanced)
443
534
  if (!levels || levels.length === 0) return undefined
444
535
  const req = requested as LlmReasoningEffort
445
536
  let best: LlmReasoningEffort | undefined
@@ -459,25 +550,107 @@ export function effectiveReasoningEffort(
459
550
  * premium stays premium). `high` is the Claude-family server default and
460
551
  * never bumps.
461
552
  */
462
- export function buildLlmCreditIdentifier(feature: string, modelId?: string, reasoningEffort?: string): string {
553
+
554
+ /**
555
+ * The sampling defaults each LLM feature's route sends when Advanced mode is
556
+ * OFF — and therefore the values its Advanced panel must seed the sliders with.
557
+ *
558
+ * SINGLE SOURCE because the two used to disagree: the routes hardcoded their
559
+ * own literals while the toggle fell back to 0.7/2048, so the panel displayed
560
+ * a temperature the run never used, and one arrow-key press on 3D Title's Max
561
+ * Tokens silently cut its budget from 3072 to 2048 on a node that emits
562
+ * structured JSON.
563
+ *
564
+ * `structuredOutput` marks the features whose prompt asks the model for JSON —
565
+ * a high temperature measurably degrades schema adherence there, so the panel
566
+ * warns. Absent `temperature` means the route sends none (vendor default).
567
+ */
568
+ export interface LlmRouteDefaults {
569
+ temperature?: number
570
+ maxTokens?: number
571
+ structuredOutput?: true
572
+ }
573
+
574
+ export const LLM_ROUTE_DEFAULTS: Record<string, LlmRouteDefaults> = {
575
+ "llm-chat": { temperature: 0.7, maxTokens: 8192 },
576
+ "ai-writer": { temperature: 0.7, maxTokens: 8192 },
577
+ "prompt-helper": { temperature: 0.7, maxTokens: 8192, structuredOutput: true },
578
+ "generate-script": { maxTokens: 16384, structuredOutput: true },
579
+ "qa-check": { maxTokens: 1024, structuredOutput: true },
580
+ "image-critic": { maxTokens: 1024, structuredOutput: true },
581
+ "image-to-text": { maxTokens: 1024 },
582
+ "describe-to-picker": { structuredOutput: true },
583
+ "scene-graph-ai": { temperature: 0.3, maxTokens: 4096, structuredOutput: true },
584
+ "after-effects": { temperature: 0.3, maxTokens: 2048, structuredOutput: true },
585
+ "lottie-overlay": { temperature: 0.3, maxTokens: 2048, structuredOutput: true },
586
+ "motion-graphics": { temperature: 0.3, maxTokens: 2048, structuredOutput: true },
587
+ "motion-graphics-lottie": { temperature: 0.3, maxTokens: 8192, structuredOutput: true },
588
+ "3d-title": { temperature: 0.4, maxTokens: 3072, structuredOutput: true },
589
+ }
590
+
591
+ /** Route defaults for a feature; `{}` for an unknown one. */
592
+ export function llmRouteDefaults(feature: string | undefined): LlmRouteDefaults {
593
+ return (feature && LLM_ROUTE_DEFAULTS[feature]) || {}
594
+ }
595
+
596
+ /** One step up the economy → standard → premium ladder. Premium is the ceiling. */
597
+ function bumpTier(tier: LlmTier): LlmTier {
598
+ if (tier === "economy") return "standard"
599
+ if (tier === "standard") return "premium"
600
+ return tier
601
+ }
602
+
603
+ /**
604
+ * Can this model be run in Advanced mode?
605
+ *
606
+ * Advanced mode pins the call to the vendor's own API, which is the only lane
607
+ * where sampling levers (`temperature`, `maxTokens`) and the full effort range
608
+ * actually take effect. Capability-derived from the registry — a model without
609
+ * a direct lane simply cannot offer it, so UI and routes both gate on this
610
+ * rather than on a hand-maintained model list.
611
+ */
612
+ export function supportsAdvancedMode(modelId: string | undefined): boolean {
613
+ return Boolean(modelId && getLlmModel(modelId)?.directGeminiModel)
614
+ }
615
+
616
+ /** User-facing reason a model can't offer Advanced mode. Single-sourced so the
617
+ * config panel's disabled hint and the route's 400 say the same thing. */
618
+ export const ADVANCED_MODE_UNAVAILABLE_REASON =
619
+ "Advanced mode is available on Gemini models — switch the model to enable it."
620
+
621
+ export function buildLlmCreditIdentifier(
622
+ feature: string,
623
+ modelId?: string,
624
+ reasoningEffort?: string,
625
+ advancedMode?: boolean,
626
+ ): string {
463
627
  if (!modelId) return feature
464
628
  let tier = getLlmTier(modelId)
465
629
  const eff = effectiveReasoningEffort(modelId, reasoningEffort)
466
- if (eff !== undefined && EFFORT_TIER_BUMP.has(eff)) {
467
- if (tier === "economy") tier = "standard"
468
- else if (tier === "standard") tier = "premium"
469
- }
630
+ if (eff !== undefined && EFFORT_TIER_BUMP.has(eff)) tier = bumpTier(tier)
631
+ // Advanced mode routes to the vendor's own API, which bills materially more
632
+ // per token than the aggregator. It bumps INDEPENDENTLY of the effort bump —
633
+ // the two are separate cost levers and genuinely stack, so a max-effort
634
+ // advanced economy call lands at premium. The bump is ignored on a model that
635
+ // can't run advanced at all, so a stale flag never inflates a bill.
636
+ if (advancedMode && supportsAdvancedMode(modelId)) tier = bumpTier(tier)
470
637
  if (tier === "standard") return feature
471
638
  return `${feature}:${tier}`
472
639
  }
473
640
 
474
641
  /**
475
- * Resolve llmModel (+ reasoningEffort) from raw body for creditGuard preHandler
476
- * (before Zod parsing). Returns the credit identifier for the given feature.
642
+ * Resolve llmModel (+ reasoningEffort, advancedMode) from raw body for the
643
+ * creditGuard preHandler (before Zod parsing). Returns the credit identifier
644
+ * for the given feature.
477
645
  */
478
646
  export function resolveLlmCreditId(feature: string, body: unknown): string {
479
647
  const b = body as Record<string, unknown> | undefined
480
- return buildLlmCreditIdentifier(feature, b?.llmModel as string | undefined, b?.reasoningEffort as string | undefined)
648
+ return buildLlmCreditIdentifier(
649
+ feature,
650
+ b?.llmModel as string | undefined,
651
+ b?.reasoningEffort as string | undefined,
652
+ b?.advancedMode === true,
653
+ )
481
654
  }
482
655
 
483
656
  /** Models capable of video-analysis: capability-derived, never hand-listed (route-enum-sync convention). */
@@ -1815,11 +1815,11 @@ const VIDEO_MODELS: Record<string, ModelCatalogEntry> = {
1815
1815
  description: "Legacy fast-tier analysis model (pre-2026-07). Kept so stored raw-model configs keep running and keep pricing under their own identifier; new fast-tier runs use the current fast model.",
1816
1816
  useCases: ["video-analysis", "shot-list", "fast"],
1817
1817
  pricing: [
1818
- { identifier: "video-analysis:gemini-3-flash", credits: 9, note: "10-min ceiling (no duration given)" },
1819
- { identifier: "video-analysis:gemini-3-flash:60s", credits: 2 },
1820
- { identifier: "video-analysis:gemini-3-flash:180s", credits: 3 },
1821
- { identifier: "video-analysis:gemini-3-flash:360s", credits: 6 },
1822
- { identifier: "video-analysis:gemini-3-flash:600s", credits: 9, note: "10-min ceiling" },
1818
+ { identifier: "video-analysis:gemini-3-flash", credits: 112, note: "10-min ceiling (no duration given)" },
1819
+ { identifier: "video-analysis:gemini-3-flash:60s", credits: 21 },
1820
+ { identifier: "video-analysis:gemini-3-flash:180s", credits: 24 },
1821
+ { identifier: "video-analysis:gemini-3-flash:360s", credits: 68 },
1822
+ { identifier: "video-analysis:gemini-3-flash:600s", credits: 112, note: "10-min ceiling" },
1823
1823
  ],
1824
1824
  },
1825
1825
  "gemini-3.6-flash-video-analysis": {
@@ -1832,11 +1832,11 @@ const VIDEO_MODELS: Record<string, ModelCatalogEntry> = {
1832
1832
  description: "Analyze a video into a structured shot list (scenes, camera, audio) — fast, economy tier. Billed per duration bucket.",
1833
1833
  useCases: ["video-analysis", "shot-list", "fast"],
1834
1834
  pricing: [
1835
- { identifier: "video-analysis:gemini-3.6-flash", credits: 25, note: "10-min ceiling (no duration given)" },
1836
- { identifier: "video-analysis:gemini-3.6-flash:60s", credits: 5 },
1837
- { identifier: "video-analysis:gemini-3.6-flash:180s", credits: 6 },
1838
- { identifier: "video-analysis:gemini-3.6-flash:360s", credits: 15 },
1839
- { identifier: "video-analysis:gemini-3.6-flash:600s", credits: 25, note: "10-min ceiling" },
1835
+ { identifier: "video-analysis:gemini-3.6-flash", credits: 291, note: "10-min ceiling (no duration given)" },
1836
+ { identifier: "video-analysis:gemini-3.6-flash:60s", credits: 54 },
1837
+ { identifier: "video-analysis:gemini-3.6-flash:180s", credits: 63 },
1838
+ { identifier: "video-analysis:gemini-3.6-flash:360s", credits: 175 },
1839
+ { identifier: "video-analysis:gemini-3.6-flash:600s", credits: 291, note: "10-min ceiling" },
1840
1840
  ],
1841
1841
  },
1842
1842
  "gemini-3.1-pro-video-analysis": {
@@ -1849,11 +1849,11 @@ const VIDEO_MODELS: Record<string, ModelCatalogEntry> = {
1849
1849
  description: "Analyze a video into a structured shot list (scenes, camera, audio) — higher-fidelity, default tier. Billed per duration bucket.",
1850
1850
  useCases: ["video-analysis", "shot-list", "cinematic"],
1851
1851
  pricing: [
1852
- { identifier: "video-analysis:gemini-3.1-pro", credits: 33, note: "10-min ceiling (no duration given)" },
1853
- { identifier: "video-analysis:gemini-3.1-pro:60s", credits: 6 },
1854
- { identifier: "video-analysis:gemini-3.1-pro:180s", credits: 8 },
1855
- { identifier: "video-analysis:gemini-3.1-pro:360s", credits: 20 },
1856
- { identifier: "video-analysis:gemini-3.1-pro:600s", credits: 33, note: "10-min ceiling" },
1852
+ { identifier: "video-analysis:gemini-3.1-pro", credits: 448, note: "10-min ceiling (no duration given)" },
1853
+ { identifier: "video-analysis:gemini-3.1-pro:60s", credits: 84 },
1854
+ { identifier: "video-analysis:gemini-3.1-pro:180s", credits: 96 },
1855
+ { identifier: "video-analysis:gemini-3.1-pro:360s", credits: 269 },
1856
+ { identifier: "video-analysis:gemini-3.1-pro:600s", credits: 448, note: "10-min ceiling" },
1857
1857
  ],
1858
1858
  },
1859
1859
  // Both mixed tiers are variants of the same advanced multi-engine analysis
@@ -1869,11 +1869,11 @@ const VIDEO_MODELS: Record<string, ModelCatalogEntry> = {
1869
1869
  description: "Our most advanced analysis tier — multiple analysis engines combined into one result for maximum completeness and accuracy. Billed per duration bucket.",
1870
1870
  useCases: ["video-analysis", "shot-list", "premium", "most-complete"],
1871
1871
  pricing: [
1872
- { identifier: "video-analysis:mixed", credits: 57, note: "10-min ceiling (no duration given)" },
1873
- { identifier: "video-analysis:mixed:60s", credits: 10 },
1874
- { identifier: "video-analysis:mixed:180s", credits: 13 },
1875
- { identifier: "video-analysis:mixed:360s", credits: 35 },
1876
- { identifier: "video-analysis:mixed:600s", credits: 57, note: "10-min ceiling" },
1872
+ { identifier: "video-analysis:mixed", credits: 739, note: "10-min ceiling (no duration given)" },
1873
+ { identifier: "video-analysis:mixed:60s", credits: 137 },
1874
+ { identifier: "video-analysis:mixed:180s", credits: 158 },
1875
+ { identifier: "video-analysis:mixed:360s", credits: 443 },
1876
+ { identifier: "video-analysis:mixed:600s", credits: 739, note: "10-min ceiling" },
1877
1877
  ],
1878
1878
  },
1879
1879
  }
@@ -24,8 +24,11 @@
24
24
  * `VIDEO_CLIP_CREDITS` uses in `film-pricing.ts`. It is what the frontend's
25
25
  * client-side cost preview (`estimateNodeCredits` in
26
26
  * workflow-editor/types.ts) reads instead of calling the formula directly.
27
- * A backend test (`video-analysis-cost.test.ts`) cross-checks this table
28
- * against the live formula so it can't silently drift.
27
+ * The formula's own test in `@nodaroai/cloud-plugins`
28
+ * (`src/plugins/video-analysis/__tests__/cost.test.ts`) cross-checks this table
29
+ * against it and fails on drift. There is deliberately NO app-side formula to
30
+ * check against — it was moved private in 2026-07 and the old backend test
31
+ * went with it.
29
32
  */
30
33
 
31
34
  export const VIDEO_ANALYSIS_DURATION_BUCKETS = [60, 180, 360, 600] as const
@@ -47,27 +50,27 @@ export const VIDEO_ANALYSIS_WINDOW = { LEN: WINDOW_LEN, STRIDE: WINDOW_STRIDE, O
47
50
  */
48
51
  export const VIDEO_ANALYSIS_BUCKET_CREDITS: Record<string, number> = {
49
52
  // Legacy fast-tier model (pre-2026-07) — kept for stored raw-id configs.
50
- "video-analysis:gemini-3-flash:60s": 2,
51
- "video-analysis:gemini-3-flash:180s": 3,
52
- "video-analysis:gemini-3-flash:360s": 6,
53
- "video-analysis:gemini-3-flash:600s": 9,
53
+ "video-analysis:gemini-3-flash:60s": 21,
54
+ "video-analysis:gemini-3-flash:180s": 24,
55
+ "video-analysis:gemini-3-flash:360s": 68,
56
+ "video-analysis:gemini-3-flash:600s": 112,
54
57
  // Current fast tier — regenerated from the private formula for its backing
55
58
  // model; higher than the legacy fast schedule but still ≤ pro per bucket.
56
- "video-analysis:gemini-3.6-flash:60s": 5,
57
- "video-analysis:gemini-3.6-flash:180s": 6,
58
- "video-analysis:gemini-3.6-flash:360s": 15,
59
- "video-analysis:gemini-3.6-flash:600s": 25,
60
- "video-analysis:gemini-3.1-pro:60s": 6,
61
- "video-analysis:gemini-3.1-pro:180s": 8,
62
- "video-analysis:gemini-3.1-pro:360s": 20,
63
- "video-analysis:gemini-3.1-pro:600s": 33,
59
+ "video-analysis:gemini-3.6-flash:60s": 54,
60
+ "video-analysis:gemini-3.6-flash:180s": 63,
61
+ "video-analysis:gemini-3.6-flash:360s": 175,
62
+ "video-analysis:gemini-3.6-flash:600s": 291,
63
+ "video-analysis:gemini-3.1-pro:60s": 84,
64
+ "video-analysis:gemini-3.1-pro:180s": 96,
65
+ "video-analysis:gemini-3.1-pro:360s": 269,
66
+ "video-analysis:gemini-3.1-pro:600s": 448,
64
67
  // Mixed tiers (`mixed` + `mixed-fast`) share ONE credit family — they are
65
68
  // variants of the same engine plan (plan internals live in the private
66
69
  // analysis plugin). Admin-tunable via model_pricing like every other row.
67
- "video-analysis:mixed:60s": 10,
68
- "video-analysis:mixed:180s": 13,
69
- "video-analysis:mixed:360s": 35,
70
- "video-analysis:mixed:600s": 57,
70
+ "video-analysis:mixed:60s": 137,
71
+ "video-analysis:mixed:180s": 158,
72
+ "video-analysis:mixed:360s": 443,
73
+ "video-analysis:mixed:600s": 739,
71
74
  }
72
75
 
73
76
  /**
@@ -20,6 +20,148 @@
20
20
  import { z } from "zod"
21
21
 
22
22
  export const VIDEO_ANALYSIS_MAX_SCENE_SEC = 8
23
+
24
+ /**
25
+ * Camera VIEWPOINT — where the camera is relative to the subject. The axis
26
+ * `shotType` does not carry, and the one the analyzer had been improvising.
27
+ *
28
+ * Two failures this fixes, both from real jobs:
29
+ *
30
+ * 1. `shotType` is framing SIZE (Wide … Extreme Close-Up), so a true angle had
31
+ * nowhere to go and landed in the MOVEMENT field instead:
32
+ * `"camera": "low angle static"`. That loses the angle to anything reading
33
+ * `camera` and pollutes the movement vocabulary.
34
+ * 2. The relational viewpoints — `Over-the-Shoulder`, `POV` — were conventions
35
+ * inside the `shotType` list, competing with the sizes for one slot. So an
36
+ * over-the-shoulder MEDIUM had to pick one and threw the other away. They
37
+ * belong here, leaving `shotType` free to state the size: an OTS medium is
38
+ * `shotType: "Medium"` + `angle: "over-the-shoulder"`, which is strictly more
39
+ * than either field could carry alone.
40
+ *
41
+ * A closed enum rather than free text precisely because improvisation is the
42
+ * failure being fixed. Absent means EYE-LEVEL — the overwhelming default, so
43
+ * omitting it costs nothing on most shots (the same "absence is the default"
44
+ * shape as `transitionOut` and appearance variations).
45
+ *
46
+ * `from-behind` and `over-the-shoulder` also carry real meaning downstream: a
47
+ * face is not visible in either, which is what auto-cast needs to know before
48
+ * choosing one as an identity reference.
49
+ */
50
+ export const VIDEO_ANALYSIS_SHOT_ANGLES = [
51
+ // Vertical placement and roll — the classical "angles".
52
+ "eye-level", "low", "high", "overhead", "worms-eye", "dutch",
53
+ // Relational viewpoints — where the camera sits with respect to the subject.
54
+ "over-the-shoulder", "pov", "profile", "from-behind",
55
+ ] as const
56
+ export type VideoAnalysisShotAngle = (typeof VIDEO_ANALYSIS_SHOT_ANGLES)[number]
57
+
58
+ /** Viewpoints in which the subject's FACE is not visible — so a frame shot this
59
+ * way is a poor identity reference however good its framing otherwise is. */
60
+ export const VIDEO_ANALYSIS_FACELESS_ANGLES: ReadonlySet<string> = new Set(["over-the-shoulder", "from-behind"])
61
+
62
+ /**
63
+ * Effects applied to the PICTURE of a shot. An array — a shot can be grainy and
64
+ * vignetted at once — and absent when the image is clean, which is most shots.
65
+ *
66
+ * Scoped deliberately to things done to the IMAGE, and NOT to compositing that
67
+ * asserts what is in the shot (picture-in-picture, split screen). That line
68
+ * matters: a real job invented `{slot:creator} overlay talking to camera` across
69
+ * nine scenes for a man who is never seen, so a field for "there is an inset of a
70
+ * person here" would hand that fabrication a legitimate home. An effect is
71
+ * verifiable in the pixels; a claim about who is inset is not.
72
+ *
73
+ * `dissolve` and `fade` are NOT here either — they are edits BETWEEN shots and
74
+ * belong to `transitionOut`.
75
+ */
76
+ export const VIDEO_ANALYSIS_VISUAL_EFFECTS = [
77
+ "blur", "pixelate", "glitch", "grain", "vignette", "flash", "distortion", "double-exposure",
78
+ ] as const
79
+ export type VideoAnalysisVisualEffect = (typeof VIDEO_ANALYSIS_VISUAL_EFFECTS)[number]
80
+
81
+ /**
82
+ * Visible edit INTO the next shot.
83
+ *
84
+ * `dissolve` (a cross-fade from one image to the other) is distinct from `fade`
85
+ * (through black or white). Collapsing both onto `fade` — as this enum did — makes
86
+ * a recreation render the wrong edit, and the two look nothing alike.
87
+ */
88
+ export const VIDEO_ANALYSIS_TRANSITIONS = ["cut", "fade", "dissolve", "wipe", "whip"] as const
89
+ export type VideoAnalysisTransition = (typeof VIDEO_ANALYSIS_TRANSITIONS)[number]
90
+
91
+ /**
92
+ * Time manipulation — slow motion, ramps, timelapse, freeze, reverse.
93
+ *
94
+ * Previously unrepresentable anywhere in the schema, so a recreation rendered
95
+ * every shot at normal speed no matter what the footage did. It is a first-class
96
+ * lever in every video model and a real editing decision in most action footage.
97
+ *
98
+ * `"normal"` is deliberately NOT a member: absence is normal speed, so there is
99
+ * exactly one way to say "nothing unusual here" and the field costs nothing on
100
+ * the majority of shots.
101
+ */
102
+ export const VIDEO_ANALYSIS_SPEED_EFFECTS = ["slow-motion", "ramp-in", "ramp-out", "timelapse", "freeze", "reverse"] as const
103
+ export type VideoAnalysisSpeedEffect = (typeof VIDEO_ANALYSIS_SPEED_EFFECTS)[number]
104
+
105
+ /**
106
+ * The CLIP-LEVEL look — one source of truth for the properties that belong to
107
+ * the whole piece rather than any one shot.
108
+ *
109
+ * Colour grade, camera format and lens character were previously only ever prose
110
+ * inside each scene's `visual`, which meant a 43-scene analysis re-decided the
111
+ * grade forty-three independent times with nothing holding them consistent. That
112
+ * is the same drift problem entity slots solve for people: state it once, apply it
113
+ * everywhere. A recreation reads this alongside every scene.
114
+ *
115
+ * Every field optional — an analyzer that cannot read the format should say
116
+ * nothing rather than guess, and a per-scene deviation still belongs in that
117
+ * scene's `visual` prose.
118
+ */
119
+ export const clipLookSchema = z.object({
120
+ /**
121
+ * The rendering MEDIUM — "live-action photoreal", "2D anime", "stop-motion
122
+ * claymation", "3D render", "oil painting", "pixel art".
123
+ *
124
+ * The most consequential field in this object, and orthogonal to every other
125
+ * one: two shots with identical grade, lens, lighting and framing still look
126
+ * nothing alike when one is live action and the other is an oil painting. Get
127
+ * this wrong and a recreation renders the whole piece in the wrong medium,
128
+ * which no amount of correct grade or lighting can rescue.
129
+ *
130
+ * Mirrors the product's Style picker, whose catalog defines exactly this axis
131
+ * and states its independence from lighting, colour-look, atmosphere and lens.
132
+ * A clip that genuinely changes medium partway (live action with an animated
133
+ * insert) states the deviation in that scene's `visual`, as with `lighting`.
134
+ */
135
+ style: z.string().optional(),
136
+ /** Colour grade / palette — "muted teal-and-orange, crushed blacks". */
137
+ grade: z.string().optional(),
138
+ /** Camera or film FORMAT and stock — "anamorphic digital", "16mm film grain". */
139
+ format: z.string().optional(),
140
+ /** Lens character — "wide-angle, shallow depth of field throughout". */
141
+ lens: z.string().optional(),
142
+ /** Overall lighting style — "hard single-source daylight, deep shadow". */
143
+ lighting: z.string().optional(),
144
+ /** What KIND of piece this is — "cinematic trailer", "talking-head vlog". */
145
+ genre: z.string().optional(),
146
+ /**
147
+ * The visual INFLUENCE the piece clearly evokes — a cinematographer, director,
148
+ * photographer or named aesthetic ("shot like Deakins", "Wes Anderson
149
+ * symmetry", "80s Kodachrome editorial").
150
+ *
151
+ * The highest-leverage field here by a distance: a couple of words transfer a
152
+ * whole aesthetic that would otherwise take a paragraph of grade, lens and
153
+ * lighting prose to approximate — which is exactly why the product already
154
+ * exposes it as a curated picker ("Photographer / Artist") whose catalog ships
155
+ * `in the style of …` prompt hints. The analyzer had no way to read it back.
156
+ *
157
+ * Deliberately conservative: OMIT unless the footage genuinely evokes a
158
+ * well-known, nameable style. A confident misattribution is worse than silence,
159
+ * because it drags an entire wrong aesthetic into every regenerated shot — so
160
+ * describing the look in `grade`/`lighting` always beats guessing a name.
161
+ */
162
+ influence: z.string().optional(),
163
+ })
164
+ export type ClipLook = z.infer<typeof clipLookSchema>
23
165
  export const VIDEO_ANALYSIS_ENTITY_SOURCES = ["wired-character", "wired-object", "wired-location", "wired-creature"] as const
24
166
  export type VideoAnalysisEntitySource = (typeof VIDEO_ANALYSIS_ENTITY_SOURCES)[number]
25
167
  /** Matches {slot:<id>} tokens. Distinct from NODE_REF_PATTERN / {image:N} grammars. */
@@ -108,8 +250,27 @@ const windowSceneBase = z.object({
108
250
  label: z.string().min(1),
109
251
  shotType: z.string().min(1),
110
252
  camera: z.string(),
253
+ /** Camera VIEWPOINT. Absent ⇒ eye-level. Keeps `camera` to pure MOVEMENT and
254
+ * frees `shotType` to state the SIZE even on an over-the-shoulder or POV. */
255
+ angle: z.enum(VIDEO_ANALYSIS_SHOT_ANGLES).optional(),
256
+ /** Time manipulation. Absent ⇒ normal speed. */
257
+ speed: z.enum(VIDEO_ANALYSIS_SPEED_EFFECTS).optional(),
111
258
  visual: z.string().min(1),
112
- transitionOut: z.enum(["cut", "fade", "wipe", "whip"]).optional(),
259
+ /**
260
+ * Text burned into the PICTURE of this shot — titles, captions, lower-thirds,
261
+ * subtitles — verbatim, in its original script. Absent when the frame carries
262
+ * none.
263
+ *
264
+ * Doctrine already asks for on-screen text inside `visual` prose, but a
265
+ * recreation needs to know discretely whether to RENDER text at all, and
266
+ * `translateOnScreenTextToEnglish` had no structured field to land in. It is
267
+ * also the signal the auto-cast frame judge reads: picture text belonging to an
268
+ * earlier scene's speech means the shot is replayed footage under a voice-over.
269
+ */
270
+ onScreenText: z.string().optional(),
271
+ /** Effects on this shot's PICTURE. Absent ⇒ a clean image. */
272
+ effects: z.array(z.enum(VIDEO_ANALYSIS_VISUAL_EFFECTS)).optional(),
273
+ transitionOut: z.enum(VIDEO_ANALYSIS_TRANSITIONS).optional(),
113
274
  // Array of concurrent layers (music + speech + sfx together); [] = silence.
114
275
  audio: z.array(audioLayerSchema),
115
276
  /** slotId → variationId for slots wearing a NON-default look in this scene
@@ -125,6 +286,9 @@ export type WindowScene = z.infer<typeof windowSceneSchema>
125
286
  /** What the MODEL emits per window (strict-JSON footer schema). scenes has NO min. */
126
287
  export const windowAnalysisSchema = z.object({
127
288
  language: z.string().optional(),
289
+ /** The clip-level look as read from THIS window. Merge folds the windows
290
+ * field-by-field (first non-empty wins), like `language`. */
291
+ look: clipLookSchema.optional(),
128
292
  slots: z.array(entitySlotSchema),
129
293
  scenes: z.array(windowSceneSchema),
130
294
  })
@@ -147,6 +311,10 @@ export const videoAnalysisResultSchema = z.object({
147
311
  title: z.string().optional(),
148
312
  language: z.string().optional(),
149
313
  }),
314
+ /** Clip-level look, merged across windows. Deliberately a sibling of `meta`
315
+ * rather than a member: `meta` is probed fact (ffprobe dimensions, probed
316
+ * duration), while this is the model's reading of the photography. */
317
+ look: clipLookSchema.optional(),
150
318
  slots: z.array(entitySlotSchema),
151
319
  scenes: z.array(analyzedSceneSchema).min(1),
152
320
  /** CAST VARIATIONS (§4 cap handling): looks the analyzer's merge FOLDED into
@@ -267,6 +435,28 @@ export function dropUnknownSpeakers(
267
435
  return { audio: dropped.length > 0 ? out : audio, dropped }
268
436
  }
269
437
 
438
+ /**
439
+ * Fold each window's reading of the clip look into one, FIELD BY FIELD: the first
440
+ * window that had something to say about a field wins it.
441
+ *
442
+ * Per-field rather than first-window-wins-everything because the windows see
443
+ * different footage — an opening window may read the grade confidently while only
444
+ * a later one contains the shot that reveals the format. Mirrors how `language` is
445
+ * resolved across windows rather than taken from window 0.
446
+ *
447
+ * Returns undefined when no window said anything, so an analysis with nothing to
448
+ * report omits the field rather than shipping an empty object.
449
+ */
450
+ export function mergeClipLook(looks: ReadonlyArray<ClipLook | undefined>): ClipLook | undefined {
451
+ const out: Record<string, string> = {}
452
+ for (const look of looks) {
453
+ for (const [k, v] of Object.entries(look ?? {})) {
454
+ if (typeof v === "string" && v.trim() && !out[k]) out[k] = v.trim()
455
+ }
456
+ }
457
+ return Object.keys(out).length > 0 ? (out as ClipLook) : undefined
458
+ }
459
+
270
460
  /** Substitute {slot:x}: castMap binding wins, else the slot's description, else literal id. */
271
461
  export function renderAnalyzedScene(scene: { visual: string }, slots: EntitySlot[], castMap?: Record<string, string>): string {
272
462
  const byId = new Map(slots.map((s) => [s.slotId, s]))