ffmpeg-skill 3.1.6 → 3.1.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/docs/contract.md +22 -6
- package/package.json +1 -1
- package/references/scripts.md +19 -6
- package/scripts/_common/asr.py +49 -10
- package/scripts/_common/decision.py +13 -1
- package/scripts/_common/fonts.py +26 -0
- package/scripts/_common/probe.py +20 -3
- package/scripts/_common/runner.py +136 -22
- package/scripts/_common/timeline.py +118 -23
- package/scripts/_contract.py +33 -2
- package/scripts/batch.py +175 -28
- package/scripts/caption.py +95 -11
- package/scripts/check.py +7 -0
- package/scripts/color.py +3 -0
- package/scripts/cut.py +3 -0
- package/scripts/export.py +22 -15
- package/scripts/metadata.py +9 -2
- package/scripts/render.py +39 -4
- package/scripts/silence.py +30 -5
- package/scripts/sync.py +3 -0
- package/scripts/waveform.py +8 -0
package/scripts/batch.py
CHANGED
|
@@ -33,7 +33,7 @@ from pathlib import Path
|
|
|
33
33
|
from typing import Any, Dict, List, Optional
|
|
34
34
|
|
|
35
35
|
from _common import STATE, add_common, apply_common, child_args, die, emit, info, run_tool, MEDIA_EXT as _MEDIA_EXT
|
|
36
|
-
from _common.runner import HW_FORCED_ENV, add_hw_orchestrator_args, env_hw, stop_children
|
|
36
|
+
from _common.runner import HW_FORCED_ENV, _OutputLock, add_hw_orchestrator_args, env_hw, stop_children
|
|
37
37
|
from _common.emit import absorb_stage_hw, audit, note, shown, steps_encoder_report, warn
|
|
38
38
|
|
|
39
39
|
HERE = Path(__file__).resolve().parent
|
|
@@ -59,6 +59,30 @@ def file_key(path: Path) -> str:
|
|
|
59
59
|
return h.hexdigest()
|
|
60
60
|
|
|
61
61
|
|
|
62
|
+
def side_inputs_key(recipe: Dict[str, Any]) -> str:
|
|
63
|
+
"""The files a recipe's steps name besides {in}/{out} (check.py --compare REF, a LUT, a logo),
|
|
64
|
+
keyed by their own content: editing or removing one is a change the cache must see, or a QC
|
|
65
|
+
pass is served its old verdict against a reference that changed or is gone (#428 review).
|
|
66
|
+
Read on every pass, so a --watch loop sees the edit too."""
|
|
67
|
+
h = hashlib.sha1(usedforsecurity=False)
|
|
68
|
+
for step in recipe.get("steps") or []:
|
|
69
|
+
for arg in step[1:] if isinstance(step, list) else []:
|
|
70
|
+
if not isinstance(arg, str) or "{" in arg:
|
|
71
|
+
continue
|
|
72
|
+
value = arg.split("=", 1)[1] if arg.startswith("-") and "=" in arg else arg
|
|
73
|
+
if value.startswith("-") or not value:
|
|
74
|
+
continue
|
|
75
|
+
path = Path(value)
|
|
76
|
+
try:
|
|
77
|
+
if path.is_file():
|
|
78
|
+
h.update(f"{value}\0{file_key(path)}\0".encode())
|
|
79
|
+
continue
|
|
80
|
+
except OSError:
|
|
81
|
+
pass
|
|
82
|
+
h.update(f"{value}\0-\0".encode()) # absent now: different from when it existed
|
|
83
|
+
return h.hexdigest()[:8]
|
|
84
|
+
|
|
85
|
+
|
|
62
86
|
def recipe_key(recipe: Dict[str, Any]) -> str:
|
|
63
87
|
# A "project" recipe is just {"project": "<path>", "clip_key": N} -- the actual settings
|
|
64
88
|
# (export preset, captions, everything) live in the file at that path, not in this dict.
|
|
@@ -182,9 +206,80 @@ def final_path(src: Path, recipe: Dict[str, Any], outdir: Path) -> Path:
|
|
|
182
206
|
# detection of its own; see one_pass()'s pre-flight check, which uses this same computation
|
|
183
207
|
# to catch that before any file is actually processed (and the earlier one silently clobbered).
|
|
184
208
|
final_ext = recipe.get("ext") or src.suffix.lstrip(".") or "mp4"
|
|
209
|
+
steps = recipe.get("steps") or []
|
|
210
|
+
if not recipe.get("ext") and not recipe.get("project") and steps and not step_writes(steps[-1]) and not writes_nothing(recipe):
|
|
211
|
+
# the last step writes nothing, so what is delivered is an intermediate: its name says its
|
|
212
|
+
# container (a .webm source's fit.py step writes MP4, which must not be named .webm)
|
|
213
|
+
final_ext = intermediate_ext(src)
|
|
185
214
|
return outdir / f"{src.stem}{suffix}.{final_ext}"
|
|
186
215
|
|
|
187
216
|
|
|
217
|
+
def intermediate_ext(src: Path) -> str:
|
|
218
|
+
"""The extension of an item's intermediate step files: the source's own for an audio file,
|
|
219
|
+
else mp4 (the tools' encoders write x264/AAC, which .webm and friends do not take)."""
|
|
220
|
+
return src.suffix.lstrip(".") if src.suffix.lower() in (".wav", ".mp3", ".m4a", ".flac") else "mp4"
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
def step_writes(step: Any) -> bool:
|
|
224
|
+
"""Whether a recipe step writes a file: it names {out} (check.py and probe.py do not)."""
|
|
225
|
+
return isinstance(step, list) and any("{out}" in str(a) for a in step[1:])
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
def writes_nothing(recipe: Dict[str, Any]) -> bool:
|
|
229
|
+
"""A steps recipe in which no step names {out}: it measures each file and writes none."""
|
|
230
|
+
return not recipe.get("project") and not any(step_writes(st) for st in recipe.get("steps") or [])
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
def row_output(src: Path, recipe: Dict[str, Any], outdir: Path) -> Dict[str, Any]:
|
|
234
|
+
"""An item's output fields for its row: `output` is always the final path, a string as it has
|
|
235
|
+
been since 1.x (a --json key never changes type within a major), and a recipe that writes
|
|
236
|
+
nothing adds `written: false` (#415, #428 review)."""
|
|
237
|
+
fields: Dict[str, Any] = {"output": str(final_path(src, recipe, outdir))}
|
|
238
|
+
if writes_nothing(recipe):
|
|
239
|
+
fields["written"] = False
|
|
240
|
+
return fields
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
def deliver(src: str, final: Path) -> None:
|
|
244
|
+
"""Move an item's last written intermediate to its final path, under the output guard every
|
|
245
|
+
tool applies: an existing file is replaced only with --overwrite. A dry run writes nothing."""
|
|
246
|
+
if STATE.dry_run:
|
|
247
|
+
# the refusal the real run would meet, as a step writing {out} plans it (#428 review)
|
|
248
|
+
if final.exists() and not STATE.overwrite:
|
|
249
|
+
raise ItemError({"step": None, "tool": None, "kind": "input", "reason": "output_exists",
|
|
250
|
+
"message": f"refusing to overwrite existing output {str(final)!r}: pass --overwrite to replace it"})
|
|
251
|
+
log(f" [dry-run] would deliver the last written file to {final}")
|
|
252
|
+
return
|
|
253
|
+
|
|
254
|
+
def locked(msg: str, hint: str, details: Dict[str, Any]) -> None:
|
|
255
|
+
raise ItemError({"step": None, "tool": None, "kind": "input", "reason": "output_locked",
|
|
256
|
+
"message": msg, "hint": hint, "details": details})
|
|
257
|
+
# under the output lock every tool's write holds, so the existence check and the move are one
|
|
258
|
+
# step for another run delivering or writing the same path
|
|
259
|
+
with _OutputLock(str(final), on_locked=locked):
|
|
260
|
+
if final.exists() and not STATE.overwrite:
|
|
261
|
+
raise ItemError({"step": None, "tool": None, "kind": "input", "reason": "output_exists",
|
|
262
|
+
"message": f"refusing to overwrite existing output {str(final)!r}: pass --overwrite to replace it"})
|
|
263
|
+
try:
|
|
264
|
+
os.replace(src, final)
|
|
265
|
+
except OSError:
|
|
266
|
+
# another filesystem: copy beside the final path and rename, so a copy that fails part
|
|
267
|
+
# way never leaves a truncated file where the caller's was
|
|
268
|
+
import shutil
|
|
269
|
+
tmp = final.with_name(f".{final.stem}.ffskill-{os.getpid()}-{threading.get_ident()}{final.suffix}")
|
|
270
|
+
try:
|
|
271
|
+
shutil.copyfile(src, tmp)
|
|
272
|
+
os.replace(tmp, final)
|
|
273
|
+
except OSError as exc:
|
|
274
|
+
try:
|
|
275
|
+
os.remove(tmp)
|
|
276
|
+
except OSError:
|
|
277
|
+
pass
|
|
278
|
+
raise ItemError({"step": None, "tool": None, "kind": "output",
|
|
279
|
+
"message": f"could not deliver {src} to {final}: {exc}"})
|
|
280
|
+
log(f" delivered the last written file to {final}")
|
|
281
|
+
|
|
282
|
+
|
|
188
283
|
def process(src: Path, recipe: Dict[str, Any], outdir: Path, work: Path,
|
|
189
284
|
deadline: "Optional[float]" = None) -> Dict[str, Any]:
|
|
190
285
|
final = final_path(src, recipe, outdir)
|
|
@@ -211,7 +306,7 @@ def process(src: Path, recipe: Dict[str, Any], outdir: Path, work: Path,
|
|
|
211
306
|
proj["clips"].append({})
|
|
212
307
|
proj["clips"][idx]["src"] = str(src.resolve())
|
|
213
308
|
proj["output"] = str(final.resolve())
|
|
214
|
-
pj = work /
|
|
309
|
+
pj = work / "project.json"
|
|
215
310
|
pj.write_text(json.dumps(proj, indent=2), encoding="utf-8")
|
|
216
311
|
ok, _doc = run_step(["render.py", str(pj)], budget())
|
|
217
312
|
step_docs = [("render", _doc)]
|
|
@@ -231,7 +326,11 @@ def process(src: Path, recipe: Dict[str, Any], outdir: Path, work: Path,
|
|
|
231
326
|
cut_reasons = []
|
|
232
327
|
for i, step in enumerate(steps):
|
|
233
328
|
last = i == len(steps) - 1
|
|
234
|
-
|
|
329
|
+
# the item's directory is its own, so the name needs no stem (a long one overflowed it)
|
|
330
|
+
out = str(final) if last else str(work / f"step{i}.{intermediate_ext(src)}")
|
|
331
|
+
# a step that names no {out} (check.py, probe.py) writes nothing: the next step reads
|
|
332
|
+
# what this one read, and when it is the last, the last file written is delivered (#415)
|
|
333
|
+
writes = step_writes(step)
|
|
235
334
|
argv = [str(a).replace("{in}", cur).replace("{out}", out) for a in step]
|
|
236
335
|
step_ok, doc = run_step(argv, budget())
|
|
237
336
|
step_docs.append((Path(str(argv[0])).stem if argv else "step", doc))
|
|
@@ -246,8 +345,16 @@ def process(src: Path, recipe: Dict[str, Any], outdir: Path, work: Path,
|
|
|
246
345
|
ok = False
|
|
247
346
|
failure = step_error(i, str(argv[0]) if argv else None, doc)
|
|
248
347
|
break
|
|
249
|
-
|
|
250
|
-
|
|
348
|
+
if writes:
|
|
349
|
+
cur = out
|
|
350
|
+
if ok and cur != str(final) and not writes_nothing(recipe):
|
|
351
|
+
# the last step wrote nothing: the item's deliverable is the last file a step wrote,
|
|
352
|
+
# which is in this item's work directory and would be removed with it (#415)
|
|
353
|
+
try:
|
|
354
|
+
deliver(cur, final)
|
|
355
|
+
except ItemError as exc:
|
|
356
|
+
ok, failure = False, exc.error
|
|
357
|
+
result: Dict[str, Any] = {"file": str(src), **row_output(src, recipe, outdir), "ok": ok, "seconds": round(time.time() - t0, 1)}
|
|
251
358
|
if failure:
|
|
252
359
|
result["error"] = failure
|
|
253
360
|
# `ok` is "every step completed" (design-decisions: a spec miss a tool cannot fix is completed +
|
|
@@ -291,6 +398,8 @@ def cached_output_changed(hit: Dict[str, Any]) -> "Optional[Dict[str, Any]]":
|
|
|
291
398
|
output_missing, output_changed (size or mtime differ: something replaced or edited it) or
|
|
292
399
|
entry_unverifiable (an entry written by 2.10.11 or earlier recorded no identity, so nothing can say
|
|
293
400
|
the file is the one it describes)."""
|
|
401
|
+
if hit.get("written") is False or ("output" in hit and hit["output"] is None):
|
|
402
|
+
return None # a recipe that writes nothing (check.py over a folder): no file to have changed
|
|
294
403
|
out = str(hit.get("output") or "")
|
|
295
404
|
now = output_identity(out) if out else None
|
|
296
405
|
if now is None:
|
|
@@ -344,9 +453,22 @@ def main() -> int:
|
|
|
344
453
|
if not isinstance(name, str) or name not in ALLOWED_STEP_SCRIPTS:
|
|
345
454
|
die(f"recipe step names a script that isn't one of this skill's own tools: {name!r} "
|
|
346
455
|
f"(must be a bare filename like 'silence.py', found in scripts/)")
|
|
456
|
+
if writes_nothing(recipe):
|
|
457
|
+
# a QC batch (check.py / probe.py on every file): its rows carry written: false, and its
|
|
458
|
+
# cache entries claim no file, so they are served rather than missed as output_missing (#415)
|
|
459
|
+
note("no recipe step names {out}: the steps only read each file, so no output is written "
|
|
460
|
+
"(each row says written: false; its output path is not created)")
|
|
347
461
|
outdir = Path(recipe.get("output_dir") or (folder / "out"))
|
|
348
462
|
if not outdir.is_absolute():
|
|
349
463
|
outdir = folder / outdir
|
|
464
|
+
# resolved, so an alias of the input folder ("sub/..", a symlink) is recognised as one: the
|
|
465
|
+
# input scan below excludes outdir's files by parentage, and the input check by resolved path
|
|
466
|
+
outdir = outdir.resolve()
|
|
467
|
+
if outdir == folder:
|
|
468
|
+
# the scan skips files under outdir, so it found nothing here, and with suffix "" a
|
|
469
|
+
# final path was the source itself, which --overwrite replaced (#428 review)
|
|
470
|
+
die(f"output_dir {recipe.get('output_dir')!r} is the input folder {folder}: its outputs would mix with "
|
|
471
|
+
"or replace the batch inputs -- set an \"output_dir\" outside it (the default is out/)", kind="input")
|
|
350
472
|
work = Path(args.work) if args.work else outdir / ".work"
|
|
351
473
|
# the children refuse an output whose directory does not exist, under --dry-run too, so the
|
|
352
474
|
# directories are created for the plan as well -- and removed again afterwards when a dry
|
|
@@ -435,23 +557,47 @@ def main() -> int:
|
|
|
435
557
|
# collapsing e.g. clip.mp4 and clip.mov to the same clip_out.mp4) -- catch that before
|
|
436
558
|
# processing anything, rather than letting the later one silently overwrite the earlier
|
|
437
559
|
# one's finished output with the cache still recording both as "ok".
|
|
438
|
-
|
|
560
|
+
# Compared case-folded: Clip_out.mp4 and clip_out.mp4 are one file on macOS's and
|
|
561
|
+
# Windows' default filesystems, so a batch that only works on Linux is refused everywhere (#415).
|
|
562
|
+
by_final: Dict[str, List[Path]] = {}
|
|
439
563
|
for src in files:
|
|
440
|
-
by_final.setdefault(final_path(src, recipe, outdir), []).append(src)
|
|
441
|
-
collisions = {
|
|
564
|
+
by_final.setdefault(str(final_path(src, recipe, outdir)).casefold(), []).append(src)
|
|
565
|
+
collisions = {key: srcs for key, srcs in by_final.items() if len(srcs) > 1 and not writes_nothing(recipe)}
|
|
442
566
|
if collisions:
|
|
443
|
-
detail = "; ".join(
|
|
444
|
-
|
|
567
|
+
detail = "; ".join(" / ".join(sorted({final_path(s, recipe, outdir).name for s in srcs}))
|
|
568
|
+
+ " <- " + ", ".join(s.name for s in srcs) for srcs in collisions.values())
|
|
569
|
+
die(f"{len(collisions)} output filename collision(s) in this batch (names that differ only in case are one "
|
|
570
|
+
f"file on macOS and Windows) -- rename the sources, "
|
|
445
571
|
f"or add a distinguishing \"suffix\"/\"ext\" per run, or split into separate globs: {detail}")
|
|
572
|
+
# A final path that is one of the inputs would replace a source -- an output_dir that is
|
|
573
|
+
# the input folder under another name ("sub/..") with suffix "" -- and --overwrite does
|
|
574
|
+
# not license that (#428 review). A recipe that writes nothing delivers no file.
|
|
575
|
+
if not writes_nothing(recipe):
|
|
576
|
+
inputs = {str(p.resolve()).casefold() for p in files}
|
|
577
|
+
hits = [s for s in files if str(final_path(s, recipe, outdir).resolve()).casefold() in inputs]
|
|
578
|
+
if hits:
|
|
579
|
+
die(f"{len(hits)} final output path(s) would replace a batch input ({', '.join(s.name for s in hits)}); "
|
|
580
|
+
"the output_dir is the input folder -- set a different \"output_dir\" or a non-empty \"suffix\"",
|
|
581
|
+
kind="input")
|
|
446
582
|
def item_work(i: int, src: Path) -> Path:
|
|
447
|
-
"""Where this item's intermediates go
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
583
|
+
"""Where this item's intermediates go: a new directory of its own under the work
|
|
584
|
+
directory, removed with everything in it when the item ends (run_item()). The step
|
|
585
|
+
file names are stem-derived, so two items with one stem (clip.mp4, clip.mov) sharing a
|
|
586
|
+
directory refused each other's files, and a later run refused the files an earlier one
|
|
587
|
+
left there as someone else's (#415); parallel items had their own directories since
|
|
588
|
+
1.17, serial ones now do too."""
|
|
589
|
+
import tempfile
|
|
590
|
+
# no stem in the name: a stem near the 255-byte limit made the directory name too long
|
|
591
|
+
return Path(tempfile.mkdtemp(prefix=f"{i}-", dir=str(work)))
|
|
592
|
+
|
|
593
|
+
def run_item(i: int, src: Path) -> Dict[str, Any]:
|
|
594
|
+
"""process() in the item's own work directory, which goes when it ends, however it ends."""
|
|
595
|
+
import shutil
|
|
596
|
+
iw = item_work(i, src)
|
|
597
|
+
try:
|
|
598
|
+
return process(src, recipe, outdir, iw, deadline)
|
|
599
|
+
finally:
|
|
600
|
+
shutil.rmtree(iw, ignore_errors=True)
|
|
455
601
|
|
|
456
602
|
def store(key: str, r: Dict[str, Any]) -> None:
|
|
457
603
|
if not (r["ok"] and not STATE.dry_run):
|
|
@@ -462,10 +608,10 @@ def main() -> int:
|
|
|
462
608
|
with cache_lock:
|
|
463
609
|
# this run just rewrote r["output"]: an entry under another key (another recipe,
|
|
464
610
|
# the other GPU setting) that names the same file now describes bytes that are gone
|
|
465
|
-
for stale in [k for k, v in cache.items() if k != key and isinstance(v, dict) and v.get("output") == r["output"]]:
|
|
611
|
+
for stale in [k for k, v in cache.items() if r["output"] and r.get("written", True) and k != key and isinstance(v, dict) and v.get("output") == r["output"]]:
|
|
466
612
|
del cache[stale]
|
|
467
613
|
# the output as this run left it: a later hit is served only while it is unchanged
|
|
468
|
-
cache[key] = dict(r, output_identity=output_identity(r["output"]))
|
|
614
|
+
cache[key] = dict(r, output_identity=output_identity(r["output"]) if r["output"] and r.get("written", True) else None)
|
|
469
615
|
# write_text isn't atomic -- a process killed mid-write (or a --watch loop racing
|
|
470
616
|
# a concurrent manual run) could leave a truncated file that json.loads() above
|
|
471
617
|
# then silently treats as "no cache" (a ValueError -> {}), discarding every prior
|
|
@@ -483,8 +629,9 @@ def main() -> int:
|
|
|
483
629
|
slots: "List[Optional[Dict[str, Any]]]" = [None] * len(files)
|
|
484
630
|
pending: List[tuple] = []
|
|
485
631
|
cache_miss: Dict[int, Dict[str, Any]] = {}
|
|
632
|
+
pass_key = rkey + side_inputs_key(recipe)
|
|
486
633
|
for i, src in enumerate(files):
|
|
487
|
-
key = f"{file_key(src)}:{
|
|
634
|
+
key = f"{file_key(src)}:{pass_key}"
|
|
488
635
|
hit = cache.get(key)
|
|
489
636
|
if isinstance(hit, dict) and not args.force:
|
|
490
637
|
why = cached_output_changed(hit)
|
|
@@ -503,7 +650,7 @@ def main() -> int:
|
|
|
503
650
|
|
|
504
651
|
def timed_out_row(src: Path) -> "Dict[str, Any]":
|
|
505
652
|
timed_out["hit"] = True
|
|
506
|
-
return {"file": str(src),
|
|
653
|
+
return {"file": str(src), **row_output(src, recipe, outdir),
|
|
507
654
|
"ok": False, "seconds": 0.0, "skipped": "timeout"}
|
|
508
655
|
|
|
509
656
|
def failed_row(src: Path, exc: BaseException) -> "Dict[str, Any]":
|
|
@@ -522,13 +669,13 @@ def main() -> int:
|
|
|
522
669
|
error = {"step": None, "tool": None, "exception": exc.__class__.__name__,
|
|
523
670
|
"message": (str(exc) or exc.__class__.__name__)[:300]}
|
|
524
671
|
log(f" failed: {error['message']}")
|
|
525
|
-
return {"file": str(src),
|
|
672
|
+
return {"file": str(src), **row_output(src, recipe, outdir),
|
|
526
673
|
"ok": False, "verified": False, "seconds": 0.0, "error": error}
|
|
527
674
|
|
|
528
675
|
def interrupted_row(src: Path, started: bool, r: "Optional[Dict[str, Any]]" = None) -> "Dict[str, Any]":
|
|
529
676
|
"""An item the interrupt reached: `interrupted: true` when it was running (its steps
|
|
530
677
|
were stopped), `skipped: "interrupted"` when it had not started."""
|
|
531
|
-
row = dict(r) if r else {"file": str(src),
|
|
678
|
+
row = dict(r) if r else {"file": str(src), **row_output(src, recipe, outdir), "seconds": 0.0}
|
|
532
679
|
row.update(ok=False, verified=False)
|
|
533
680
|
if started:
|
|
534
681
|
row["interrupted"] = True
|
|
@@ -550,7 +697,7 @@ def main() -> int:
|
|
|
550
697
|
continue
|
|
551
698
|
info(f"=== {src.name}")
|
|
552
699
|
try:
|
|
553
|
-
r =
|
|
700
|
+
r = run_item(i, src)
|
|
554
701
|
except KeyboardInterrupt:
|
|
555
702
|
interrupted["hit"] = True
|
|
556
703
|
# the running item, then every one not started: each keeps its row
|
|
@@ -572,7 +719,7 @@ def main() -> int:
|
|
|
572
719
|
def work_one(i: int, src: Path, key: str) -> Dict[str, Any]:
|
|
573
720
|
_LOG.buffer = [f"=== {src.name}"]
|
|
574
721
|
try:
|
|
575
|
-
r =
|
|
722
|
+
r = run_item(i, src)
|
|
576
723
|
if not interrupted["hit"]: # a step the interrupt stopped is no result to keep
|
|
577
724
|
store(key, r)
|
|
578
725
|
return with_miss(i, r)
|
|
@@ -670,7 +817,7 @@ def main() -> int:
|
|
|
670
817
|
info(f"{done}/{len(results)} processed, {sum(1 for r in results if r.get('cached'))} from cache")
|
|
671
818
|
if not args.json:
|
|
672
819
|
for r in results:
|
|
673
|
-
print(f"{'OK ' if r['ok'] else 'FAIL'} {r['file']} -> {r['output']}" + (" (cached)" if r.get("cached") else ""))
|
|
820
|
+
print(f"{'OK ' if r['ok'] else 'FAIL'} {r['file']} -> {r['output'] or '(no output: the steps only read it)'}" + (" (cached)" if r.get("cached") else ""))
|
|
674
821
|
if interrupted["hit"]:
|
|
675
822
|
# Ctrl-C on a batch that already produced files: the user needs the partial table, not a
|
|
676
823
|
# traceback and not "nothing was written". Exit 130 (143 for SIGTERM) with every item:
|
|
@@ -684,7 +831,7 @@ def main() -> int:
|
|
|
684
831
|
timed_out=timed_out["hit"])
|
|
685
832
|
if timed_out["hit"]:
|
|
686
833
|
skipped = [r["file"] for r in results if r.get("skipped") == "timeout"]
|
|
687
|
-
die(f"the batch's {STATE.timeout
|
|
834
|
+
die(f"the batch's {STATE.timeout:g} s budget ran out with {len(skipped)} item(s) not "
|
|
688
835
|
f"started: {', '.join(os.path.basename(f) for f in skipped[:5])}"
|
|
689
836
|
+ (" ..." if len(skipped) > 5 else "")
|
|
690
837
|
+ ". --timeout is the whole run's limit, not each item's; raise it or split the folder.",
|
package/scripts/caption.py
CHANGED
|
@@ -54,6 +54,7 @@ from _common import asr as _asr
|
|
|
54
54
|
from _common.asr import ENGINE_CHOICES
|
|
55
55
|
from _common.emit import warn
|
|
56
56
|
from _common.probe import display_size, square_pixels, square_pixels_note
|
|
57
|
+
from _common.runner import subtitle_codec_for, subtitle_container_holds
|
|
57
58
|
from _common.fonts import font_choice, record_font_choice
|
|
58
59
|
from _common import (ASR_INSTALL_HINT, die_no_engine, parse_srt, transcribe, whisper_word_timings,
|
|
59
60
|
write_srt)
|
|
@@ -145,7 +146,9 @@ def word_durations_from_audio(video: str, start: float, end: float, n_words: int
|
|
|
145
146
|
return [total_cs]
|
|
146
147
|
ffmpeg = require_tool("ffmpeg")
|
|
147
148
|
cmd = [ffmpeg, "-hide_banner", "-loglevel", "error", "-nostdin", "-ss", f"{start:.3f}", "-i", video,
|
|
148
|
-
"-map", f"0:a:{audio_stream}", "-t", f"{end - start:.3f}", "-vn",
|
|
149
|
+
"-map", f"0:a:{audio_stream}", "-t", f"{end - start:.3f}", "-vn",
|
|
150
|
+
# a sound that starts late is padded to its place on the file's clock (#414)
|
|
151
|
+
"-af", "aresample=async=1:first_pts=0", "-ac", "1", "-ar", "8000", "-f", "s16le", "-"]
|
|
149
152
|
proc = run_analysis(cmd, check=False, text=False) # under --timeout like every other measurement
|
|
150
153
|
n = len(proc.stdout) // 2
|
|
151
154
|
if proc.returncode != 0 or n < 800:
|
|
@@ -887,6 +890,23 @@ def track_title_for(code: Optional[str], given: Optional[str]) -> Optional[str]:
|
|
|
887
890
|
return LANG_TITLES.get(code.split("-")[0].lower(), code)
|
|
888
891
|
|
|
889
892
|
|
|
893
|
+
# a timing line in the grammar ffmpeg's SRT demuxer and parse_srt() read alike: hours, minutes,
|
|
894
|
+
# seconds and three-digit milliseconds on both sides (ffmpeg allows position fields after it)
|
|
895
|
+
_SHARED_SRT_TIMING = re.compile(r"^\s*\d{2,}:\d{2}:\d{2}[,.]\d{3}\s*-->\s*\d{2,}:\d{2}:\d{2}[,.]\d{3}(\s.*)?$")
|
|
896
|
+
|
|
897
|
+
|
|
898
|
+
def srt_timings_shared_grammar(path: str) -> bool:
|
|
899
|
+
"""True when every timing line of the SRT at `path` reads the same to ffmpeg's SRT demuxer as
|
|
900
|
+
to parse_srt(). ffmpeg refuses `00:01,000` (no hours) and reads `00:00:01,5` as 1.005 s, where
|
|
901
|
+
parse_srt() reads 1.5 s (#414); such a file is handed to ffmpeg as a rewritten copy."""
|
|
902
|
+
text = read_text_or_die(path, "--srt").lstrip("\ufeff").replace("\r\n", "\n")
|
|
903
|
+
# a cue's timing line is its first line with "-->", as parse_srt() reads it (and ffmpeg too);
|
|
904
|
+
# a later line holding "-->" is dialogue, not a timing this check should judge
|
|
905
|
+
blocks = [[ln for ln in b.split("\n") if ln.strip()] for b in re.split(r"\n\s*\n", text)]
|
|
906
|
+
timings = [next((ln for ln in b if "-->" in ln), None) for b in blocks]
|
|
907
|
+
return all(_SHARED_SRT_TIMING.match(line) for line in timings if line is not None)
|
|
908
|
+
|
|
909
|
+
|
|
890
910
|
def mux_subtitle_codec(output: str) -> str:
|
|
891
911
|
ext = Path(output).suffix.lower()
|
|
892
912
|
if ext in (".mp4", ".m4v", ".mov"):
|
|
@@ -1130,6 +1150,11 @@ def main() -> int:
|
|
|
1130
1150
|
die(f"--audio-stream {args.audio_stream}: input has {len(audio_streams)} audio stream(s), 0..{len(audio_streams) - 1}")
|
|
1131
1151
|
if args.audio_stream and not audio_streams:
|
|
1132
1152
|
die("--audio-stream needs an input with audio streams")
|
|
1153
|
+
if args.transcribe and not audio_streams:
|
|
1154
|
+
# refused before any engine runs, as silence.py refuses it, not an unclassified
|
|
1155
|
+
# "Stream map '0:a:0' matches no streams" from the WAV extraction (#414)
|
|
1156
|
+
die(f"--transcribe needs speech to transcribe, and {args.input} has no audio stream",
|
|
1157
|
+
kind="input", hint="pick the file that carries the sound, or write the cues with --text")
|
|
1133
1158
|
if args.mode != "mux":
|
|
1134
1159
|
# before any transcription or sidecar: a build without libass fails these at parse time
|
|
1135
1160
|
# with a message that names no filter (#393), so ask the build first, dry run included
|
|
@@ -1380,9 +1405,15 @@ def main() -> int:
|
|
|
1380
1405
|
side_notes: List[str] = list(getattr(args, "_asr_notes", None) or [])
|
|
1381
1406
|
ass_counts: dict = {} # --ass: ass_shifted / ass_unshifted (with --offset) and ass_malformed, into `caption`
|
|
1382
1407
|
ass_sample_path = args.ass
|
|
1408
|
+
normalised_only = False # srt_path is a copy of the caller's SRT that only rewrites its timings
|
|
1383
1409
|
if args.srt and not (args.text or args.transcribe) and os.path.exists(srt_path or ""):
|
|
1384
1410
|
adjusted, changed = lay_out(parse_srt(srt_path))
|
|
1385
|
-
|
|
1411
|
+
# ffmpeg's own SRT reader takes a narrower grammar than parse_srt(): a time without hours
|
|
1412
|
+
# is refused and ",5" is read as 5 ms where this tool reads 500. A file outside the grammar
|
|
1413
|
+
# both share goes to ffmpeg as the copy this tool writes, so it gets the cues counted
|
|
1414
|
+
# here (#414)
|
|
1415
|
+
normalised_only = not changed and not srt_timings_shared_grammar(srt_path)
|
|
1416
|
+
if changed or normalised_only:
|
|
1386
1417
|
new_srt = os.path.splitext(output)[0] + "_adjusted.srt"
|
|
1387
1418
|
claim([(new_srt, "adjusted SRT")])
|
|
1388
1419
|
if STATE.dry_run:
|
|
@@ -1392,8 +1423,10 @@ def main() -> int:
|
|
|
1392
1423
|
info(f"wrote {new_srt} ({len(adjusted)} cues, adjusted from {os.path.basename(srt_path)})")
|
|
1393
1424
|
srt_path = new_srt
|
|
1394
1425
|
planned_cues = adjusted
|
|
1395
|
-
|
|
1396
|
-
|
|
1426
|
+
why = ("--offset/--max-lines/--min-duration" if changed else
|
|
1427
|
+
"its timings rewritten as hh:mm:ss,mmm, which ffmpeg reads as this tool does")
|
|
1428
|
+
side_notes.append(f"the {'muxed' if args.mode == 'mux' else 'burned'} subtitles are {new_srt}, the adjusted copy of "
|
|
1429
|
+
f"{os.path.basename(args.srt)} this run writes ({why})"
|
|
1397
1430
|
+ ("; re-run this command without --dry-run to produce it" if STATE.dry_run else ""))
|
|
1398
1431
|
if args.ass and args.offset and os.path.exists(args.ass):
|
|
1399
1432
|
shifted = os.path.splitext(output)[0] + "_offset.ass"
|
|
@@ -1424,10 +1457,34 @@ def main() -> int:
|
|
|
1424
1457
|
and not (STATE.dry_run and (args.text or args.transcribe))):
|
|
1425
1458
|
die(f"SRT file not found: {srt_path}")
|
|
1426
1459
|
codec = mux_subtitle_codec(output)
|
|
1460
|
+
out_ext = Path(output).suffix.lower().lstrip(".")
|
|
1461
|
+
# The picture and the sound are copied: a WebM holds only VP8/VP9/AV1 and Opus/Vorbis, so
|
|
1462
|
+
# anything else is refused here, not left to fail unclassified in the muxer (#414)
|
|
1463
|
+
if out_ext == "webm":
|
|
1464
|
+
vcodec = (meta.get("video") or {}).get("codec")
|
|
1465
|
+
# the stream -map 0:a:N copies (--audio-stream), not always the first one
|
|
1466
|
+
audio_streams = meta.get("audio_streams") or []
|
|
1467
|
+
acodec = audio_streams[args.audio_stream].get("codec") if args.audio_stream < len(audio_streams) else None
|
|
1468
|
+
bad = [c for c in (vcodec,) if c and c not in ("vp8", "vp9", "av1")] + \
|
|
1469
|
+
[c for c in (acodec,) if c and c not in ("opus", "vorbis")]
|
|
1470
|
+
if bad:
|
|
1471
|
+
die(f"--mode mux copies the picture and the sound, and a .webm holds only VP8/VP9/AV1 video and "
|
|
1472
|
+
f"Opus/Vorbis audio; {args.input} carries {' and '.join(bad)}", kind="input",
|
|
1473
|
+
hint="write .mkv (any codec) or .mp4 instead, or re-encode first with export.py")
|
|
1427
1474
|
# Keep any subtitle track(s) the input already has (e.g. chaining --mode mux once per
|
|
1428
|
-
# language to build a multi-language set)
|
|
1429
|
-
#
|
|
1475
|
+
# language to build a multi-language set): copied where the output container holds them as
|
|
1476
|
+
# they are, else converted to its own text format as export.py converts them (#380, #414);
|
|
1477
|
+
# a track the container cannot hold at all (a bitmap one in MPEG-4 or WebM) is refused.
|
|
1430
1478
|
existing_subs = meta.get("subtitle_streams") or 0
|
|
1479
|
+
existing_details = list(meta.get("subtitle_stream_details") or [])
|
|
1480
|
+
existing_codecs = [(existing_details[i] if i < len(existing_details) else {}).get("codec") or ""
|
|
1481
|
+
for i in range(existing_subs)]
|
|
1482
|
+
unheld = sorted({c or "unknown" for c in existing_codecs if not subtitle_container_holds(c, out_ext)})
|
|
1483
|
+
if unheld:
|
|
1484
|
+
die(f"{args.input} already carries a subtitle track ({', '.join(unheld)}) that a .{out_ext} file cannot "
|
|
1485
|
+
"hold, and --mode mux keeps every existing track", kind="input",
|
|
1486
|
+
hint="write .mkv, which holds any subtitle codec")
|
|
1487
|
+
existing_out = [subtitle_codec_for(c, out_ext) for c in existing_codecs]
|
|
1431
1488
|
# the first entry's path is srt_path, which --offset/--max-lines may have repointed at an
|
|
1432
1489
|
# adjusted copy; the rest are taken as written
|
|
1433
1490
|
added = [(srt_path, srt_tracks[0][1] if srt_tracks else args.language)] + \
|
|
@@ -1435,6 +1492,18 @@ def main() -> int:
|
|
|
1435
1492
|
for path, _lang in added[1:]:
|
|
1436
1493
|
if not os.path.exists(path) and not STATE.dry_run:
|
|
1437
1494
|
die(f"SRT file not found: {path}")
|
|
1495
|
+
for n in range(1, len(added)):
|
|
1496
|
+
path, lang = added[n]
|
|
1497
|
+
if os.path.exists(path) and not srt_timings_shared_grammar(path):
|
|
1498
|
+
# the same rule as the first track (#414): ffmpeg is given what parse_srt() reads
|
|
1499
|
+
copy = f"{os.path.splitext(output)[0]}_track{n}.srt"
|
|
1500
|
+
claim([(copy, f"normalised SRT {n}")])
|
|
1501
|
+
if not STATE.dry_run:
|
|
1502
|
+
write_srt(parse_srt(path), copy)
|
|
1503
|
+
info(f"wrote {copy} (timings of {os.path.basename(path)} rewritten as hh:mm:ss,mmm)")
|
|
1504
|
+
side_notes.append(f"subtitle track {n} is {copy}, a copy of {os.path.basename(path)} with its timings "
|
|
1505
|
+
"rewritten as hh:mm:ss,mmm, which ffmpeg reads as this tool does")
|
|
1506
|
+
added[n] = (copy, lang)
|
|
1438
1507
|
titles = list(args.track_title or [])
|
|
1439
1508
|
mp4_family = Path(output).suffix.lower() in (".mp4", ".m4v", ".mov")
|
|
1440
1509
|
dropped_titles: List[str] = []
|
|
@@ -1453,13 +1522,16 @@ def main() -> int:
|
|
|
1453
1522
|
if mp4_family:
|
|
1454
1523
|
cmd += ["-movflags", "+faststart"]
|
|
1455
1524
|
for i in range(existing_subs):
|
|
1456
|
-
cmd += [f"-c:s:{i}",
|
|
1525
|
+
cmd += [f"-c:s:{i}", existing_out[i]]
|
|
1457
1526
|
tracks: List[Dict[str, Any]] = []
|
|
1527
|
+
converted: List[str] = []
|
|
1458
1528
|
for i in range(existing_subs):
|
|
1459
|
-
|
|
1460
|
-
|
|
1529
|
+
detail = existing_details[i] if i < len(existing_details) else {}
|
|
1530
|
+
written = detail.get("codec") if existing_out[i] == "copy" else existing_out[i]
|
|
1531
|
+
if existing_out[i] != "copy":
|
|
1532
|
+
converted.append(f"{detail.get('codec')} -> {existing_out[i]}")
|
|
1461
1533
|
tracks.append({"index": i, "file": None, "language": detail.get("language"),
|
|
1462
|
-
"title": detail.get("title"), "codec":
|
|
1534
|
+
"title": detail.get("title"), "codec": written,
|
|
1463
1535
|
"default": False, "cues": None, "kept_from_input": True})
|
|
1464
1536
|
for n, (path, lang) in enumerate(added):
|
|
1465
1537
|
idx = existing_subs + n
|
|
@@ -1510,6 +1582,9 @@ def main() -> int:
|
|
|
1510
1582
|
notes.append(f"{total} subtitle tracks in an MPEG-4 container: the tracks are all there, "
|
|
1511
1583
|
"but many players only ever show the first -- write to .mkv for a "
|
|
1512
1584
|
"deliverable a viewer can actually switch")
|
|
1585
|
+
if converted:
|
|
1586
|
+
notes.append(f"the input's own subtitle track(s) were converted to the .{out_ext} container's text format "
|
|
1587
|
+
f"({', '.join(converted)}): their text and timing are kept, styling a format cannot carry is not")
|
|
1513
1588
|
if dropped_titles:
|
|
1514
1589
|
notes.append("an MPEG-4 container has no per-track title this tool can write back "
|
|
1515
1590
|
f"({', '.join(dropped_titles)} would be dropped), so the tracks are "
|
|
@@ -1521,6 +1596,14 @@ def main() -> int:
|
|
|
1521
1596
|
"as given and may not survive")
|
|
1522
1597
|
run(cmd)
|
|
1523
1598
|
result = probe(output, role="output")
|
|
1599
|
+
# the language and title each track carries in the written file: a converted track keeps
|
|
1600
|
+
# only what the container can store (MPEG-4 drops a two-letter code and every title)
|
|
1601
|
+
written = result.get("subtitle_stream_details") or []
|
|
1602
|
+
if len(written) == len(tracks):
|
|
1603
|
+
for track, detail in zip(tracks, written):
|
|
1604
|
+
track["language"] = detail.get("language")
|
|
1605
|
+
if track["kept_from_input"]:
|
|
1606
|
+
track["title"] = detail.get("title")
|
|
1524
1607
|
info(f"wrote {output} ({fmt_secs(result.get('duration'))}, mux, {len(added)} "
|
|
1525
1608
|
f"subtitle track(s) added, codec {codec})")
|
|
1526
1609
|
emit(output, tracks=tracks, subtitle_tracks=total, **({"notes": notes} if notes else {}),
|
|
@@ -1587,7 +1670,8 @@ def main() -> int:
|
|
|
1587
1670
|
cues if (args.text or args.transcribe) else (
|
|
1588
1671
|
planned_cues if planned_cues is not None else parse_srt(srt_path)))
|
|
1589
1672
|
if args.karaoke and not getattr(args, "_word_timings", None):
|
|
1590
|
-
|
|
1673
|
+
# a copy that only rewrote the timings keeps the caller's word-timing JSON beside it
|
|
1674
|
+
args._word_timings = whisper_word_timings(args.srt if normalised_only else srt_path)
|
|
1591
1675
|
ass_path = args.write_ass or os.path.splitext(output)[0] + ".ass"
|
|
1592
1676
|
claim([(ass_path, "ASS")])
|
|
1593
1677
|
w, h = display_size(meta["video"])
|
package/scripts/check.py
CHANGED
|
@@ -32,6 +32,7 @@ the row set is unchanged.
|
|
|
32
32
|
"""
|
|
33
33
|
import argparse
|
|
34
34
|
import json
|
|
35
|
+
import os
|
|
35
36
|
import re
|
|
36
37
|
import sys
|
|
37
38
|
from fractions import Fraction
|
|
@@ -164,6 +165,12 @@ def main() -> int:
|
|
|
164
165
|
|
|
165
166
|
# read before anything is measured and again after: a file replaced mid-check is not the one graded
|
|
166
167
|
identity_before = None if dry_run_input_pending(args.input) else file_identity(args.input)
|
|
168
|
+
if identity_before is None and STATE.dry_run and not os.path.exists(args.input):
|
|
169
|
+
# a render/batch plan's earlier stage would write this file: there is nothing to grade yet,
|
|
170
|
+
# and grading the probe's empty answer divided by a 0 height
|
|
171
|
+
emit(None, platform=args.platform, checks=[], failed=0, warnings=0, ok=False,
|
|
172
|
+
notes=[f"[dry-run] {args.input} does not exist yet, so nothing was checked"])
|
|
173
|
+
return 0
|
|
167
174
|
meta = probe(args.input)
|
|
168
175
|
v, a = meta.get("video") or {}, meta.get("audio") or {}
|
|
169
176
|
rows: List[Dict[str, Any]] = []
|
package/scripts/color.py
CHANGED
|
@@ -23,6 +23,7 @@ import os
|
|
|
23
23
|
import sys
|
|
24
24
|
from typing import List, Optional
|
|
25
25
|
|
|
26
|
+
from _common.runner import fail_if_classified
|
|
26
27
|
from _common import STATE, add_common, analyze_levels, apply_common, emit, aac_args, cfr_args, default_output, die, escape_filter_path, ffmpeg_base, ffmpeg_bsfs, info, probe, require_tool, run, run_analysis, run_keeping_subtitles, x264_args, X264_PRESETS, fmt_secs
|
|
27
28
|
from _common.emit import warn
|
|
28
29
|
|
|
@@ -320,6 +321,7 @@ def retag(args: argparse.Namespace, meta: dict, has_audio: bool) -> int:
|
|
|
320
321
|
cmd += ["-movflags", "+faststart"]
|
|
321
322
|
cmd.append(output)
|
|
322
323
|
proc = run(cmd, check=False)
|
|
324
|
+
fail_if_classified(proc) # a full disk fails the re-encode too: its error, not two more encodes (#415)
|
|
323
325
|
notes: List[str] = []
|
|
324
326
|
reencoded = proc.returncode != 0
|
|
325
327
|
if not reencoded and not STATE.dry_run:
|
|
@@ -345,6 +347,7 @@ def retag(args: argparse.Namespace, meta: dict, has_audio: bool) -> int:
|
|
|
345
347
|
cmd = ffmpeg_base() + ["-i", args.input, "-map", "0:v:0", "-map", f"0:a:{args.audio_stream}?",
|
|
346
348
|
"-map", "0:s?", "-map", "0:d?"] + label + enc + ["-c:s", "copy", "-c:d", "copy", output]
|
|
347
349
|
proc2 = run(cmd, check=False)
|
|
350
|
+
fail_if_classified(proc2)
|
|
348
351
|
if proc2.returncode != 0:
|
|
349
352
|
info("re-encode with subtitles/data streams kept also failed; dropping them")
|
|
350
353
|
dropped_streams = True
|
package/scripts/cut.py
CHANGED
|
@@ -58,6 +58,7 @@ import tempfile
|
|
|
58
58
|
import zlib
|
|
59
59
|
from typing import List, Tuple
|
|
60
60
|
|
|
61
|
+
from _common.runner import fail_if_classified
|
|
61
62
|
from _common import (beat_grid, snap_points, decode_pcm_mono, rms_envelope, BEAT_MIN_CONFIDENCE)
|
|
62
63
|
from _common import require_tool
|
|
63
64
|
from _common.probe import TS_EXTS, signatures_match, stream_signature
|
|
@@ -893,6 +894,7 @@ def cut_one(src: str, start: float, end: float, dst: str, reencode: bool, crf: i
|
|
|
893
894
|
proc = run(cmd, check=False)
|
|
894
895
|
if proc.returncode != 0:
|
|
895
896
|
if not reencode:
|
|
897
|
+
fail_if_classified(proc) # a full disk fails the re-encode too: its error, not a retry (#415)
|
|
896
898
|
info("stream copy failed, falling back to re-encode")
|
|
897
899
|
return cut_one(src, start, end, dst, True, crf, preset, tolerance, meta, reasons + ["copy_failed"], edit_list_ok)
|
|
898
900
|
die(f"ffmpeg failed:\n{proc.stderr.strip()}", kind="ffmpeg")
|
|
@@ -1604,6 +1606,7 @@ def main() -> int:
|
|
|
1604
1606
|
cmd += ["-movflags", "+faststart"]
|
|
1605
1607
|
cmd += [joined]
|
|
1606
1608
|
proc = run(cmd, check=False)
|
|
1609
|
+
fail_if_classified(proc) # #415: a full disk is not a reason to re-cut every segment
|
|
1607
1610
|
if proc.returncode != 0:
|
|
1608
1611
|
info("concat with stream copy failed; re-cutting every segment from the source into one re-encode")
|
|
1609
1612
|
compatible = False
|