davinci-resolve-mcp 2.40.0 → 2.42.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +68 -0
- package/README.md +1 -1
- package/docs/SKILL.md +39 -2
- package/docs/guides/control-panel.md +7 -2
- package/install.py +1 -1
- package/package.json +1 -1
- package/src/analysis_dashboard.py +25 -5
- package/src/granular/common.py +1 -1
- package/src/server.py +133 -1
- package/src/utils/analysis_store.py +927 -0
- package/src/utils/deep_vision.py +643 -0
- package/src/utils/media_analysis.py +77 -1
- package/src/utils/timeline_brain_db.py +179 -1
|
@@ -0,0 +1,643 @@
|
|
|
1
|
+
"""Deep shot-level vision tier (Phase B of the analysis program).
|
|
2
|
+
|
|
3
|
+
Fills the per-shot Visual / Content / Production / Editorial / Cuttability
|
|
4
|
+
field groups (v2 shot schema spec §3.2–3.8) via the same deferred-payload
|
|
5
|
+
host-vision pattern as standard analysis: the server prepares frames + a
|
|
6
|
+
schema, the host chat reads the frames and commits JSON back.
|
|
7
|
+
|
|
8
|
+
Two entry points share the schema:
|
|
9
|
+
|
|
10
|
+
- ``depth="deep"`` on analyze extends the single-pass payload (handled in
|
|
11
|
+
media_analysis.build_host_chat_paths_payload via deep_shot_schema()).
|
|
12
|
+
- ``deepen`` is the post-hoc per-clip/per-shot pass for already-analyzed
|
|
13
|
+
clips. It is estimate-first: the first call returns the frame/token cost
|
|
14
|
+
and a confirm_token; the second call (with the token) returns the deferred
|
|
15
|
+
payload. Both paths pass the caps pre-call refusal — confirmation never
|
|
16
|
+
bypasses budgets.
|
|
17
|
+
|
|
18
|
+
Deep results commit via ``commit_shot_vision``: subjective rows are written
|
|
19
|
+
with source ``vision_deep_v1`` (human rows still win), the canonical report
|
|
20
|
+
blob is updated, and analysis.json re-exports in lockstep.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
from __future__ import annotations
|
|
24
|
+
|
|
25
|
+
import json
|
|
26
|
+
import os
|
|
27
|
+
import time
|
|
28
|
+
from typing import Any, Dict, List, Optional, Tuple
|
|
29
|
+
|
|
30
|
+
from src.utils import analysis_store, timeline_brain_db
|
|
31
|
+
|
|
32
|
+
DEEP_SHOT_SCHEMA_REFERENCE = "davinci_resolve_mcp.shot_deep_analysis.v1"
|
|
33
|
+
DEEP_SOURCE = "vision_deep_v1"
|
|
34
|
+
|
|
35
|
+
# Threshold above which a shot gets one extra extracted frame for the deep pass.
|
|
36
|
+
LONG_SHOT_SECONDS = 15.0
|
|
37
|
+
|
|
38
|
+
# Per-shot field groups (v2-shot-schema-spec §3.2–3.8). Values are the schema
|
|
39
|
+
# template shown to the host chat; enums are listed inline.
|
|
40
|
+
DEEP_SHOT_FIELD_GROUPS: Dict[str, Any] = {
|
|
41
|
+
"visual": {
|
|
42
|
+
"shot_size": "wide|medium_wide|medium|medium_close|close|extreme_close|insert|establishing|other",
|
|
43
|
+
"framing": "single|two_shot|group|crowd|empty|insert|establishing|abstract",
|
|
44
|
+
"camera_height": "eye_level|high_angle|low_angle|birds_eye|dutch|unknown",
|
|
45
|
+
"camera_motion": "locked|pan|tilt|dolly|handheld|crane|drone|zoom|composite|other",
|
|
46
|
+
"motion_direction": "left|right|up|down|in|out|clockwise|counter_clockwise|none",
|
|
47
|
+
"depth_of_field": "deep|shallow|rack_focus|unknown",
|
|
48
|
+
"lens_character": "wide|normal|tele|fisheye|unknown",
|
|
49
|
+
"lighting": "natural|high_key|low_key|practical|backlit|silhouette|mixed|unknown",
|
|
50
|
+
"color_mood": "warm|cool|neutral|desaturated|saturated|monochrome|unnatural|unknown",
|
|
51
|
+
"composition_notes": "<short free text>",
|
|
52
|
+
},
|
|
53
|
+
"content": {
|
|
54
|
+
"primary_subject": {
|
|
55
|
+
"type": "person|object|landscape|interior|vehicle|animal|text_graphic|abstract",
|
|
56
|
+
"description": "<one phrase>",
|
|
57
|
+
"performance": "null unless type==person: {eye_line: to_camera|off_left|off_right|down|up|closed|unknown, energy: low|medium|high, emotional_register: <short free text>}",
|
|
58
|
+
},
|
|
59
|
+
"secondary_subjects": ["<[{type, description}]>"],
|
|
60
|
+
"action": "<one sentence — what is happening>",
|
|
61
|
+
"location": "<one sentence — where this is>",
|
|
62
|
+
"visible_text": ["<readable on-screen text>"],
|
|
63
|
+
"objects_of_note": ["<notable props/elements>"],
|
|
64
|
+
"audio_character": "silence|sync_dialogue|vo_dialogue|music|ambient|sfx|mixed|unknown",
|
|
65
|
+
},
|
|
66
|
+
"production": {
|
|
67
|
+
"composite_shot": "<bool — split-screen / PiP / multi-angle composite>",
|
|
68
|
+
"composite_panels": "null unless composite_shot: [{region, primary_subject, action}]",
|
|
69
|
+
"vfx_present": "none|minor|major|unknown",
|
|
70
|
+
},
|
|
71
|
+
"editorial": {
|
|
72
|
+
"editorial_role": "establishing|coverage|reaction|insert|transition|b_roll|montage_element|titles_or_graphics|bumper|other",
|
|
73
|
+
"select_potential": "low|medium|high",
|
|
74
|
+
"best_moment": "null if the shot is a sustained flat beat, else {time_seconds, why}",
|
|
75
|
+
"best_moment_present": "<bool>",
|
|
76
|
+
"pacing": "still|moderate|kinetic|variable",
|
|
77
|
+
"stillness_type": "held_tension|quiet|contemplative|transitional|dead_air|unknown — null unless pacing is still/variable",
|
|
78
|
+
"pacing_note": "<free text or null — only when pacing is still/variable>",
|
|
79
|
+
},
|
|
80
|
+
"cuttability": {
|
|
81
|
+
"cut_in": {"quality": "poor|ok|clean", "notes": "<short>"},
|
|
82
|
+
"cut_out": {"quality": "poor|ok|clean", "notes": "<short>"},
|
|
83
|
+
"match_action_in": "<bool — could receive a match cut>",
|
|
84
|
+
"match_action_out": "<bool — could send a match cut>",
|
|
85
|
+
"cut_compatibility_hints": "<free text>",
|
|
86
|
+
},
|
|
87
|
+
"description": "<1-3 sentences, editorially useful, colleague-style note>",
|
|
88
|
+
"confidence": {
|
|
89
|
+
"visual": "low|medium|high",
|
|
90
|
+
"content": "low|medium|high",
|
|
91
|
+
"audio": "low|medium|high",
|
|
92
|
+
"editorial": "low|medium|high",
|
|
93
|
+
"cuttability": "low|medium|high",
|
|
94
|
+
},
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
DEEP_SHOT_PROMPT = (
|
|
98
|
+
"Deep per-shot editorial analysis. For EACH shot listed in shot_table, look "
|
|
99
|
+
"only at the frames whose indices appear in that shot's frame_indices and "
|
|
100
|
+
"fill every field group in deep_shot_schema (visual, content, production, "
|
|
101
|
+
"editorial, cuttability, description, confidence). Use the enum values "
|
|
102
|
+
"verbatim; use 'unknown' or null when the frames do not support a claim — "
|
|
103
|
+
"hedge identity/intent/value when frame evidence is thin. Return strict "
|
|
104
|
+
"JSON only: {\"shots\": [{\"shot_index\": <int>, ...field groups...}]}."
|
|
105
|
+
)
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def _ma():
|
|
109
|
+
from src.utils import media_analysis
|
|
110
|
+
|
|
111
|
+
return media_analysis
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def deep_shot_schema() -> Dict[str, Any]:
|
|
115
|
+
"""The per-shot deep field groups (shared by deep-depth analyze + deepen)."""
|
|
116
|
+
return json.loads(json.dumps(DEEP_SHOT_FIELD_GROUPS))
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def _now() -> str:
|
|
120
|
+
return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def _clip_context(project_root: str, clip_ref: Any) -> Tuple[Optional[Dict[str, Any]], Optional[str]]:
|
|
124
|
+
"""Resolve a clip to (context, error). Auto-ingests the JSON report when
|
|
125
|
+
the DB has no rows yet (pre-v9 analysis roots)."""
|
|
126
|
+
ma = _ma()
|
|
127
|
+
conn = timeline_brain_db.connect(project_root)
|
|
128
|
+
clip_uuid = analysis_store.resolve_clip_uuid(conn, clip_ref)
|
|
129
|
+
if not clip_uuid:
|
|
130
|
+
# Pre-v9 report? Find the analysis.json by walking clips/ and ingest it.
|
|
131
|
+
clips_root = os.path.join(project_root, "clips")
|
|
132
|
+
candidate = str(clip_ref or "")
|
|
133
|
+
if os.path.isdir(clips_root):
|
|
134
|
+
for entry in sorted(os.listdir(clips_root)):
|
|
135
|
+
report_path = os.path.join(clips_root, entry, "analysis.json")
|
|
136
|
+
if not os.path.isfile(report_path):
|
|
137
|
+
continue
|
|
138
|
+
try:
|
|
139
|
+
with open(report_path, "r", encoding="utf-8") as handle:
|
|
140
|
+
report = json.load(handle)
|
|
141
|
+
except (OSError, json.JSONDecodeError):
|
|
142
|
+
continue
|
|
143
|
+
clip_block = report.get("clip") or {}
|
|
144
|
+
if candidate not in (
|
|
145
|
+
entry,
|
|
146
|
+
str(clip_block.get("clip_id") or ""),
|
|
147
|
+
str(clip_block.get("media_id") or ""),
|
|
148
|
+
ma.normalize_path(clip_block.get("file_path") or ""),
|
|
149
|
+
):
|
|
150
|
+
continue
|
|
151
|
+
ingest = analysis_store.ingest_report(
|
|
152
|
+
project_root, report, clip_dir=os.path.join(clips_root, entry)
|
|
153
|
+
)
|
|
154
|
+
if ingest.get("success"):
|
|
155
|
+
clip_uuid = ingest["clip_uuid"]
|
|
156
|
+
break
|
|
157
|
+
if not clip_uuid:
|
|
158
|
+
return None, f"No analyzed clip found for {clip_ref!r} (run db_ingest if this is an older analysis root)"
|
|
159
|
+
clip_row = conn.execute("SELECT * FROM clips WHERE clip_uuid = ?", (clip_uuid,)).fetchone()
|
|
160
|
+
shots = conn.execute(
|
|
161
|
+
"SELECT * FROM shots WHERE clip_uuid = ? ORDER BY shot_index", (clip_uuid,)
|
|
162
|
+
).fetchall()
|
|
163
|
+
frames = conn.execute(
|
|
164
|
+
"SELECT * FROM frames WHERE clip_uuid = ? ORDER BY frame_index", (clip_uuid,)
|
|
165
|
+
).fetchall()
|
|
166
|
+
if not shots:
|
|
167
|
+
return None, "Clip has no detected shots — deep shot analysis needs a standard analysis with cut detection first"
|
|
168
|
+
return {
|
|
169
|
+
"clip_uuid": clip_uuid,
|
|
170
|
+
"clip": dict(clip_row) if clip_row else {},
|
|
171
|
+
"shots": [dict(s) for s in shots],
|
|
172
|
+
"frames": [dict(f) for f in frames],
|
|
173
|
+
}, None
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def _select_shots(shots: List[Dict[str, Any]], shot_indices: Optional[List[int]]) -> Tuple[List[Dict[str, Any]], List[int]]:
|
|
177
|
+
if not shot_indices:
|
|
178
|
+
return shots, []
|
|
179
|
+
wanted = {int(i) for i in shot_indices}
|
|
180
|
+
selected = [s for s in shots if int(s["shot_index"]) in wanted]
|
|
181
|
+
missing = sorted(wanted - {int(s["shot_index"]) for s in selected})
|
|
182
|
+
return selected, missing
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
def _frames_for_shot(shot: Dict[str, Any], frames: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
|
|
186
|
+
rows = [f for f in frames if f.get("shot_uuid") == shot["shot_uuid"]]
|
|
187
|
+
if rows:
|
|
188
|
+
return rows
|
|
189
|
+
start, end = shot.get("time_seconds_start"), shot.get("time_seconds_end")
|
|
190
|
+
if start is None or end is None:
|
|
191
|
+
return []
|
|
192
|
+
return [
|
|
193
|
+
f for f in frames
|
|
194
|
+
if f.get("time_seconds") is not None and start <= float(f["time_seconds"]) < end
|
|
195
|
+
]
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def _extract_frame(source: str, time_seconds: float, out_path: str) -> Optional[str]:
|
|
199
|
+
"""Extract one frame with ffmpeg (read-only on source). None on failure."""
|
|
200
|
+
ma = _ma()
|
|
201
|
+
os.makedirs(os.path.dirname(out_path), exist_ok=True)
|
|
202
|
+
code, _out, _err = ma._run_command([
|
|
203
|
+
"ffmpeg", "-y", "-ss", f"{max(0.0, time_seconds):.3f}", "-i", source,
|
|
204
|
+
"-frames:v", "1", "-q:v", "3", out_path,
|
|
205
|
+
])
|
|
206
|
+
if code != 0 or not os.path.isfile(out_path):
|
|
207
|
+
return None
|
|
208
|
+
try:
|
|
209
|
+
from src.utils import analysis_caps
|
|
210
|
+
|
|
211
|
+
caps = ma._resolve_active_caps()
|
|
212
|
+
analysis_caps.downscale_frame_if_needed(out_path, caps.max_frame_dim_pixels)
|
|
213
|
+
except Exception:
|
|
214
|
+
pass
|
|
215
|
+
return out_path
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
def _confirm_token_for(clip_uuid: str, shot_uuids: List[str]) -> str:
|
|
219
|
+
return _ma().short_hash(f"deepen:{clip_uuid}:{','.join(sorted(shot_uuids))}", 16)
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
def _vision_token_for(clip_uuid: str, shot_uuids: List[str]) -> str:
|
|
223
|
+
return _ma().short_hash(f"deep_vision:{clip_uuid}:{','.join(sorted(shot_uuids))}", 16)
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
def deepen_clip(
|
|
227
|
+
project_root: str,
|
|
228
|
+
*,
|
|
229
|
+
clip_ref: Any,
|
|
230
|
+
shot_indices: Optional[List[int]] = None,
|
|
231
|
+
confirm_token: Optional[str] = None,
|
|
232
|
+
job_id: Optional[str] = None,
|
|
233
|
+
) -> Dict[str, Any]:
|
|
234
|
+
"""Estimate-first deep pass over a clip (or selected shots).
|
|
235
|
+
|
|
236
|
+
First call (no confirm_token) returns the cost estimate + confirm_token.
|
|
237
|
+
Second call with the token returns the deferred host-vision payload.
|
|
238
|
+
"""
|
|
239
|
+
ma = _ma()
|
|
240
|
+
context, err = _clip_context(project_root, clip_ref)
|
|
241
|
+
if err:
|
|
242
|
+
return {"success": False, "error": err}
|
|
243
|
+
clip_uuid = context["clip_uuid"]
|
|
244
|
+
clip = context["clip"]
|
|
245
|
+
shots, missing = _select_shots(context["shots"], shot_indices)
|
|
246
|
+
if missing:
|
|
247
|
+
return {"success": False, "error": f"shot_index not found: {missing}", "available": [s['shot_index'] for s in context['shots']]}
|
|
248
|
+
if not shots:
|
|
249
|
+
return {"success": False, "error": "No shots selected"}
|
|
250
|
+
|
|
251
|
+
shot_uuids = [s["shot_uuid"] for s in shots]
|
|
252
|
+
expected_confirm = _confirm_token_for(clip_uuid, shot_uuids)
|
|
253
|
+
|
|
254
|
+
# Frame plan: existing on-disk frames per shot; shots with none get
|
|
255
|
+
# extraction placeholders; long shots get one extra mid-late frame.
|
|
256
|
+
source = clip.get("file_path")
|
|
257
|
+
frame_plan: List[Dict[str, Any]] = []
|
|
258
|
+
total_frames = 0
|
|
259
|
+
for shot in shots:
|
|
260
|
+
rows = _frames_for_shot(shot, context["frames"])
|
|
261
|
+
on_disk = [r for r in rows if r.get("frame_path") and os.path.isfile(str(r["frame_path"]))]
|
|
262
|
+
start = float(shot.get("time_seconds_start") or 0.0)
|
|
263
|
+
end = float(shot.get("time_seconds_end") or start)
|
|
264
|
+
duration = max(0.0, end - start)
|
|
265
|
+
to_extract: List[float] = []
|
|
266
|
+
if not on_disk:
|
|
267
|
+
to_extract.append(start + duration / 2.0)
|
|
268
|
+
if duration > LONG_SHOT_SECONDS:
|
|
269
|
+
to_extract.append(start + duration * 2.0 / 3.0)
|
|
270
|
+
count = len(on_disk) + len(to_extract)
|
|
271
|
+
total_frames += count
|
|
272
|
+
frame_plan.append({
|
|
273
|
+
"shot_index": shot["shot_index"],
|
|
274
|
+
"shot_uuid": shot["shot_uuid"],
|
|
275
|
+
"existing_frames": len(on_disk),
|
|
276
|
+
"frames_to_extract": len(to_extract),
|
|
277
|
+
"extract_times": to_extract,
|
|
278
|
+
"rows": on_disk,
|
|
279
|
+
})
|
|
280
|
+
|
|
281
|
+
estimated_tokens = total_frames * ma.AVG_VISION_TOKENS_PER_FRAME
|
|
282
|
+
refusal = ma._check_caps_pre_call(
|
|
283
|
+
project_root=project_root,
|
|
284
|
+
estimated_vision_tokens=estimated_tokens,
|
|
285
|
+
clip_id=clip.get("resolve_clip_id") or clip_uuid,
|
|
286
|
+
job_id=job_id,
|
|
287
|
+
)
|
|
288
|
+
if refusal is not None:
|
|
289
|
+
return refusal
|
|
290
|
+
|
|
291
|
+
estimate = {
|
|
292
|
+
"clip_uuid": clip_uuid,
|
|
293
|
+
"clip_name": clip.get("clip_name"),
|
|
294
|
+
"shot_count": len(shots),
|
|
295
|
+
"frame_count": total_frames,
|
|
296
|
+
"estimated_vision_tokens": estimated_tokens,
|
|
297
|
+
"tokens_per_frame_assumption": ma.AVG_VISION_TOKENS_PER_FRAME,
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
if confirm_token != expected_confirm:
|
|
301
|
+
return {
|
|
302
|
+
"success": True,
|
|
303
|
+
"status": "confirmation_required",
|
|
304
|
+
"estimate": estimate,
|
|
305
|
+
"confirm_token": expected_confirm,
|
|
306
|
+
"note": (
|
|
307
|
+
"Deep shot analysis is opt-in and costs vision tokens. Re-call "
|
|
308
|
+
"media_analysis(action='deepen') with this confirm_token to "
|
|
309
|
+
"proceed. Confirmation does not bypass caps."
|
|
310
|
+
),
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
# Confirmed: materialize missing frames, then build the deferred payload.
|
|
314
|
+
clip_dir_name = clip.get("clip_dir")
|
|
315
|
+
clip_dir = os.path.join(project_root, "clips", clip_dir_name) if clip_dir_name else None
|
|
316
|
+
shot_table: List[Dict[str, Any]] = []
|
|
317
|
+
frame_metadata: List[Dict[str, Any]] = []
|
|
318
|
+
frame_paths: List[str] = []
|
|
319
|
+
synthetic_index = 100000 # extraction-only frames get indices above sampled ones
|
|
320
|
+
for plan in frame_plan:
|
|
321
|
+
indices: List[int] = []
|
|
322
|
+
for row in plan["rows"]:
|
|
323
|
+
path = ma.normalize_path(str(row["frame_path"]))
|
|
324
|
+
if path not in frame_paths:
|
|
325
|
+
frame_paths.append(path)
|
|
326
|
+
frame_metadata.append({
|
|
327
|
+
"frame_index": int(row["frame_index"]),
|
|
328
|
+
"frame_path": path,
|
|
329
|
+
"time_seconds": row.get("time_seconds"),
|
|
330
|
+
"selection_reason": row.get("selection_reason"),
|
|
331
|
+
"shot_index": plan["shot_index"],
|
|
332
|
+
})
|
|
333
|
+
indices.append(int(row["frame_index"]))
|
|
334
|
+
for t in plan["extract_times"]:
|
|
335
|
+
if not source or not os.path.isfile(source):
|
|
336
|
+
continue
|
|
337
|
+
if not clip_dir:
|
|
338
|
+
continue
|
|
339
|
+
out_path = os.path.join(clip_dir, "frames", f"deep_shot{int(plan['shot_index']):03d}_{int(t * 1000):08d}.jpg")
|
|
340
|
+
extracted = out_path if os.path.isfile(out_path) else _extract_frame(source, t, out_path)
|
|
341
|
+
if not extracted:
|
|
342
|
+
continue
|
|
343
|
+
synthetic_index += 1
|
|
344
|
+
path = ma.normalize_path(extracted)
|
|
345
|
+
frame_paths.append(path)
|
|
346
|
+
frame_metadata.append({
|
|
347
|
+
"frame_index": synthetic_index,
|
|
348
|
+
"frame_path": path,
|
|
349
|
+
"time_seconds": t,
|
|
350
|
+
"selection_reason": "deep_pass_extraction",
|
|
351
|
+
"shot_index": plan["shot_index"],
|
|
352
|
+
})
|
|
353
|
+
indices.append(synthetic_index)
|
|
354
|
+
shot = next(s for s in shots if s["shot_index"] == plan["shot_index"])
|
|
355
|
+
shot_table.append({
|
|
356
|
+
"shot_index": int(shot["shot_index"]),
|
|
357
|
+
"shot_uuid": shot["shot_uuid"],
|
|
358
|
+
"time_seconds_start": shot.get("time_seconds_start"),
|
|
359
|
+
"time_seconds_end": shot.get("time_seconds_end"),
|
|
360
|
+
"current_description": shot.get("description"),
|
|
361
|
+
"frame_indices": indices,
|
|
362
|
+
})
|
|
363
|
+
|
|
364
|
+
vision_token = _vision_token_for(clip_uuid, shot_uuids)
|
|
365
|
+
commit_params: Dict[str, Any] = {
|
|
366
|
+
"vision_token": vision_token,
|
|
367
|
+
"shots": "<host chat: [{shot_index, ...deep_shot_schema groups...}]>",
|
|
368
|
+
"analysis_root": project_root,
|
|
369
|
+
}
|
|
370
|
+
if clip.get("resolve_clip_id"):
|
|
371
|
+
commit_params["clip_id"] = clip["resolve_clip_id"]
|
|
372
|
+
elif clip_dir_name:
|
|
373
|
+
commit_params["clip_dir"] = clip_dir_name
|
|
374
|
+
|
|
375
|
+
return {
|
|
376
|
+
"success": True,
|
|
377
|
+
"status": "pending_host_analysis",
|
|
378
|
+
"provider": "host_chat_paths",
|
|
379
|
+
"mode": "deep_shots",
|
|
380
|
+
"vision_token": vision_token,
|
|
381
|
+
"estimate": estimate,
|
|
382
|
+
"clip": {
|
|
383
|
+
"clip_uuid": clip_uuid,
|
|
384
|
+
"clip_id": clip.get("resolve_clip_id"),
|
|
385
|
+
"clip_name": clip.get("clip_name"),
|
|
386
|
+
"file_path": source,
|
|
387
|
+
},
|
|
388
|
+
"frame_count": len(frame_paths),
|
|
389
|
+
"frame_paths": frame_paths,
|
|
390
|
+
"frame_metadata": frame_metadata,
|
|
391
|
+
"shot_table": shot_table,
|
|
392
|
+
"deep_shot_schema": deep_shot_schema(),
|
|
393
|
+
"schema_reference": DEEP_SHOT_SCHEMA_REFERENCE,
|
|
394
|
+
"prompt": DEEP_SHOT_PROMPT,
|
|
395
|
+
"commit_action": {
|
|
396
|
+
"tool": "media_analysis",
|
|
397
|
+
"action": "commit_shot_vision",
|
|
398
|
+
"params": commit_params,
|
|
399
|
+
},
|
|
400
|
+
"instructions": (
|
|
401
|
+
"Read every file under frame_paths as a local image. For each entry in "
|
|
402
|
+
"shot_table, produce one object in `shots` with its shot_index and the "
|
|
403
|
+
"deep_shot_schema field groups, grounded ONLY in the frames listed in "
|
|
404
|
+
"that shot's frame_indices. Then call the tool in commit_action with "
|
|
405
|
+
"`shots` set to that array. Skipping the commit leaves the deep pass "
|
|
406
|
+
"incomplete — surface that rather than silently stopping."
|
|
407
|
+
),
|
|
408
|
+
}
|
|
409
|
+
|
|
410
|
+
|
|
411
|
+
_DEEP_GROUP_KEYS = ("visual", "content", "production", "editorial", "cuttability")
|
|
412
|
+
|
|
413
|
+
|
|
414
|
+
def commit_shot_vision(
|
|
415
|
+
project_root: str,
|
|
416
|
+
*,
|
|
417
|
+
shots: Any,
|
|
418
|
+
vision_token: Optional[str] = None,
|
|
419
|
+
clip_ref: Any = None,
|
|
420
|
+
author: str = "host_chat",
|
|
421
|
+
) -> Dict[str, Any]:
|
|
422
|
+
"""Commit a deep per-shot payload: subjective rows (source vision_deep_v1),
|
|
423
|
+
canonical blob update, and lockstep analysis.json re-export."""
|
|
424
|
+
if isinstance(shots, str):
|
|
425
|
+
try:
|
|
426
|
+
shots = json.loads(shots)
|
|
427
|
+
except json.JSONDecodeError as exc:
|
|
428
|
+
return {"success": False, "error": f"shots was a string but not valid JSON: {exc}"}
|
|
429
|
+
if isinstance(shots, dict) and isinstance(shots.get("shots"), list):
|
|
430
|
+
shots = shots["shots"]
|
|
431
|
+
if not isinstance(shots, list) or not shots:
|
|
432
|
+
return {"success": False, "error": "commit_shot_vision requires `shots`: a non-empty array of per-shot objects"}
|
|
433
|
+
|
|
434
|
+
context, err = _clip_context(project_root, clip_ref)
|
|
435
|
+
if err:
|
|
436
|
+
return {"success": False, "error": err}
|
|
437
|
+
clip_uuid = context["clip_uuid"]
|
|
438
|
+
db_shots = {int(s["shot_index"]): s for s in context["shots"]}
|
|
439
|
+
by_uuid = {s["shot_uuid"]: s for s in context["shots"]}
|
|
440
|
+
|
|
441
|
+
# Token check: recompute over the shots being committed.
|
|
442
|
+
target_uuids = []
|
|
443
|
+
normalized_entries: List[Tuple[Dict[str, Any], Dict[str, Any]]] = [] # (db_shot, payload_entry)
|
|
444
|
+
for entry in shots:
|
|
445
|
+
if not isinstance(entry, dict):
|
|
446
|
+
continue
|
|
447
|
+
db_shot = None
|
|
448
|
+
if entry.get("shot_uuid") and entry["shot_uuid"] in by_uuid:
|
|
449
|
+
db_shot = by_uuid[entry["shot_uuid"]]
|
|
450
|
+
else:
|
|
451
|
+
try:
|
|
452
|
+
db_shot = db_shots.get(int(entry.get("shot_index")))
|
|
453
|
+
except (TypeError, ValueError):
|
|
454
|
+
db_shot = None
|
|
455
|
+
if db_shot is None:
|
|
456
|
+
return {"success": False, "error": f"shot not found for entry: {entry.get('shot_uuid') or entry.get('shot_index')!r}"}
|
|
457
|
+
target_uuids.append(db_shot["shot_uuid"])
|
|
458
|
+
normalized_entries.append((db_shot, entry))
|
|
459
|
+
if not normalized_entries:
|
|
460
|
+
return {"success": False, "error": "No valid shot entries in payload"}
|
|
461
|
+
expected = _vision_token_for(clip_uuid, target_uuids)
|
|
462
|
+
if vision_token and str(vision_token) != expected:
|
|
463
|
+
return {
|
|
464
|
+
"success": False,
|
|
465
|
+
"error": "vision_token mismatch; the clip was re-analyzed or the shot selection changed since the deepen payload was issued.",
|
|
466
|
+
"expected_vision_token": expected,
|
|
467
|
+
"received_vision_token": vision_token,
|
|
468
|
+
}
|
|
469
|
+
|
|
470
|
+
ma = _ma()
|
|
471
|
+
conn = timeline_brain_db.connect(project_root)
|
|
472
|
+
report_row = conn.execute(
|
|
473
|
+
"SELECT report_json FROM analysis_reports WHERE clip_uuid = ?", (clip_uuid,)
|
|
474
|
+
).fetchone()
|
|
475
|
+
if not report_row:
|
|
476
|
+
return {"success": False, "error": "No canonical report for clip — run db_ingest first"}
|
|
477
|
+
report = json.loads(report_row["report_json"])
|
|
478
|
+
visual = report.get("visual") if isinstance(report.get("visual"), dict) else {}
|
|
479
|
+
shot_entries = visual.get("shot_descriptions") if isinstance(visual.get("shot_descriptions"), list) else []
|
|
480
|
+
entries_by_index: Dict[int, Dict[str, Any]] = {}
|
|
481
|
+
for se in shot_entries:
|
|
482
|
+
if isinstance(se, dict) and se.get("shot_index") is not None:
|
|
483
|
+
try:
|
|
484
|
+
entries_by_index[int(se["shot_index"])] = se
|
|
485
|
+
except (TypeError, ValueError):
|
|
486
|
+
continue
|
|
487
|
+
|
|
488
|
+
updated = 0
|
|
489
|
+
for db_shot, entry in normalized_entries:
|
|
490
|
+
target = entries_by_index.get(int(db_shot["shot_index"]))
|
|
491
|
+
if target is None:
|
|
492
|
+
target = {
|
|
493
|
+
"shot_index": int(db_shot["shot_index"]),
|
|
494
|
+
"time_seconds_start": db_shot.get("time_seconds_start"),
|
|
495
|
+
"time_seconds_end": db_shot.get("time_seconds_end"),
|
|
496
|
+
"description": db_shot.get("description") or "",
|
|
497
|
+
"qc_flags": [],
|
|
498
|
+
}
|
|
499
|
+
shot_entries.append(target)
|
|
500
|
+
entries_by_index[int(db_shot["shot_index"])] = target
|
|
501
|
+
for group in _DEEP_GROUP_KEYS:
|
|
502
|
+
if isinstance(entry.get(group), dict):
|
|
503
|
+
target[group] = entry[group]
|
|
504
|
+
if isinstance(entry.get("confidence"), dict):
|
|
505
|
+
target["confidence"] = entry["confidence"]
|
|
506
|
+
description = entry.get("description")
|
|
507
|
+
if isinstance(description, str) and description.strip():
|
|
508
|
+
target["description"] = description.strip()
|
|
509
|
+
updated += 1
|
|
510
|
+
|
|
511
|
+
visual["shot_descriptions"] = shot_entries
|
|
512
|
+
report["visual"] = visual
|
|
513
|
+
report["deep_shots_committed_at"] = _now()
|
|
514
|
+
|
|
515
|
+
# Rows first (canonical), deep source label; human rows preserved inside.
|
|
516
|
+
ingest = analysis_store.ingest_report(
|
|
517
|
+
project_root,
|
|
518
|
+
report,
|
|
519
|
+
clip_dir=os.path.join(project_root, "clips", context["clip"].get("clip_dir") or ""),
|
|
520
|
+
author=author,
|
|
521
|
+
source=DEEP_SOURCE,
|
|
522
|
+
)
|
|
523
|
+
if not ingest.get("success"):
|
|
524
|
+
return {"success": False, "error": f"DB ingest failed: {ingest.get('error')}"}
|
|
525
|
+
|
|
526
|
+
# Lockstep JSON export.
|
|
527
|
+
export_path = None
|
|
528
|
+
clip_dir_name = context["clip"].get("clip_dir")
|
|
529
|
+
if clip_dir_name:
|
|
530
|
+
candidate = os.path.join(project_root, "clips", clip_dir_name, "analysis.json")
|
|
531
|
+
if os.path.isfile(candidate):
|
|
532
|
+
export_path = analysis_store.export_report_file(project_root, clip_uuid, candidate)
|
|
533
|
+
|
|
534
|
+
# Caps usage: frames that fed this pass (frames table rows for the target
|
|
535
|
+
# shots), falling back to one frame per shot when none are recorded.
|
|
536
|
+
placeholders = ",".join("?" for _ in target_uuids)
|
|
537
|
+
frames_seen = int(conn.execute(
|
|
538
|
+
f"SELECT COUNT(*) FROM frames WHERE clip_uuid = ? AND shot_uuid IN ({placeholders})",
|
|
539
|
+
(clip_uuid, *target_uuids),
|
|
540
|
+
).fetchone()[0]) or len(normalized_entries)
|
|
541
|
+
ma._record_caps_usage(
|
|
542
|
+
project_root=project_root,
|
|
543
|
+
clip_id=context["clip"].get("resolve_clip_id") or clip_uuid,
|
|
544
|
+
vision_tokens=frames_seen * ma.AVG_VISION_TOKENS_PER_FRAME,
|
|
545
|
+
frames_uploaded=frames_seen,
|
|
546
|
+
)
|
|
547
|
+
|
|
548
|
+
return {
|
|
549
|
+
"success": True,
|
|
550
|
+
"clip_uuid": clip_uuid,
|
|
551
|
+
"shots_updated": updated,
|
|
552
|
+
"subjective_fields_written": ingest.get("subjective_fields_written"),
|
|
553
|
+
"subjective_fields_preserved_human": ingest.get("subjective_fields_preserved_human"),
|
|
554
|
+
"source": DEEP_SOURCE,
|
|
555
|
+
"analysis_json": export_path,
|
|
556
|
+
}
|
|
557
|
+
|
|
558
|
+
|
|
559
|
+
def vision_pending_sweep(
|
|
560
|
+
project_root: str,
|
|
561
|
+
*,
|
|
562
|
+
expire: bool = False,
|
|
563
|
+
max_age_days: Optional[float] = None,
|
|
564
|
+
reoffer: bool = False,
|
|
565
|
+
) -> Dict[str, Any]:
|
|
566
|
+
"""List clips stuck in pending_host_analysis; optionally expire or re-offer.
|
|
567
|
+
|
|
568
|
+
Re-offer returns each clip's stored deferred payload (report["visual"])
|
|
569
|
+
after verifying its frame files still exist. Expire stamps the report
|
|
570
|
+
``expired_host_analysis`` so pendings never linger silently.
|
|
571
|
+
"""
|
|
572
|
+
ma = _ma()
|
|
573
|
+
root = ma.normalize_path(project_root)
|
|
574
|
+
clips_root = os.path.join(root, "clips")
|
|
575
|
+
now = time.time()
|
|
576
|
+
pending: List[Dict[str, Any]] = []
|
|
577
|
+
expired: List[str] = []
|
|
578
|
+
reoffers: List[Dict[str, Any]] = []
|
|
579
|
+
if os.path.isdir(clips_root):
|
|
580
|
+
for entry in sorted(os.listdir(clips_root)):
|
|
581
|
+
clip_dir = os.path.join(clips_root, entry)
|
|
582
|
+
report_path = os.path.join(clip_dir, "analysis.json")
|
|
583
|
+
if not os.path.isfile(report_path):
|
|
584
|
+
continue
|
|
585
|
+
report = analysis_store.load_db_report(root, clip_dir=entry)
|
|
586
|
+
if report is None:
|
|
587
|
+
try:
|
|
588
|
+
with open(report_path, "r", encoding="utf-8") as handle:
|
|
589
|
+
report = json.load(handle)
|
|
590
|
+
except (OSError, json.JSONDecodeError):
|
|
591
|
+
continue
|
|
592
|
+
if report.get("vision_status") != "pending_host_analysis":
|
|
593
|
+
continue
|
|
594
|
+
analyzed_at = report.get("analyzed_at")
|
|
595
|
+
age_days = None
|
|
596
|
+
ts = ma._timestamp_from_analyzed_at(analyzed_at)
|
|
597
|
+
if ts:
|
|
598
|
+
age_days = round((now - ts) / 86400.0, 2)
|
|
599
|
+
visual = report.get("visual") if isinstance(report.get("visual"), dict) else {}
|
|
600
|
+
frame_paths = visual.get("frame_paths") if isinstance(visual.get("frame_paths"), list) else []
|
|
601
|
+
frames_present = sum(1 for p in frame_paths if p and os.path.isfile(str(p)))
|
|
602
|
+
row = {
|
|
603
|
+
"clip_dir": entry,
|
|
604
|
+
"clip_name": (report.get("clip") or {}).get("clip_name"),
|
|
605
|
+
"clip_id": (report.get("clip") or {}).get("clip_id"),
|
|
606
|
+
"analyzed_at": analyzed_at,
|
|
607
|
+
"age_days": age_days,
|
|
608
|
+
"vision_token": report.get("vision_token") or visual.get("vision_token"),
|
|
609
|
+
"frame_count": len(frame_paths),
|
|
610
|
+
"frames_still_on_disk": frames_present,
|
|
611
|
+
"reofferable": frames_present > 0,
|
|
612
|
+
}
|
|
613
|
+
pending.append(row)
|
|
614
|
+
over_age = max_age_days is not None and (age_days or 0) >= float(max_age_days)
|
|
615
|
+
if expire and (max_age_days is None or over_age):
|
|
616
|
+
report["vision_status"] = "expired_host_analysis"
|
|
617
|
+
report["vision_expired_at"] = _now()
|
|
618
|
+
ingest = analysis_store.ingest_report(root, report, clip_dir=clip_dir)
|
|
619
|
+
if ingest.get("success"):
|
|
620
|
+
analysis_store.export_report_file(root, ingest["clip_uuid"], report_path)
|
|
621
|
+
else:
|
|
622
|
+
ma._write_json(report_path, report)
|
|
623
|
+
expired.append(entry)
|
|
624
|
+
row["expired"] = True
|
|
625
|
+
elif reoffer and frames_present:
|
|
626
|
+
reoffers.append({
|
|
627
|
+
"clip_dir": entry,
|
|
628
|
+
"payload": visual,
|
|
629
|
+
})
|
|
630
|
+
result: Dict[str, Any] = {
|
|
631
|
+
"success": True,
|
|
632
|
+
"project_root": root,
|
|
633
|
+
"pending_count": len(pending),
|
|
634
|
+
"pending": pending,
|
|
635
|
+
"expired": expired,
|
|
636
|
+
}
|
|
637
|
+
if reoffer:
|
|
638
|
+
result["reoffers"] = reoffers
|
|
639
|
+
result["note"] = (
|
|
640
|
+
"Each reoffer.payload is the original deferred host-vision payload; "
|
|
641
|
+
"read its frame_paths and call its commit_action to finish the run."
|
|
642
|
+
)
|
|
643
|
+
return result
|