ffmpeg-skill 2.10.10 → 2.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -3
- package/SKILL.md +1 -1
- package/docs/contract.md +53 -10
- package/package.json +1 -1
- package/references/ci-platform-pitfalls.md +9 -1
- package/references/scripts.md +34 -8
- package/scripts/_common/__init__.py +9 -7
- package/scripts/_common/drawtext.py +65 -6
- package/scripts/_common/emit.py +154 -2
- package/scripts/_common/runner.py +155 -33
- package/scripts/_common/text.py +1 -1
- package/scripts/_contract.py +59 -8
- package/scripts/color.py +142 -41
- package/scripts/overlay.py +66 -22
- package/scripts/render.py +91 -31
package/scripts/_common/emit.py
CHANGED
|
@@ -13,7 +13,8 @@ from typing import Any, Dict, List, Optional, Sequence, Tuple
|
|
|
13
13
|
from _common.runner import Context, ERROR_CODE, ERROR_RETRYABLE, STATE
|
|
14
14
|
|
|
15
15
|
|
|
16
|
-
def die(msg: str, code: int = 1, kind: str = "input", *, ctx: "Optional[Context]" = None,
|
|
16
|
+
def die(msg: str, code: int = 1, kind: str = "input", *, ctx: "Optional[Context]" = None,
|
|
17
|
+
error_extra: "Optional[Dict[str, Any]]" = None, **extra: Any) -> "None":
|
|
17
18
|
"""Exit with a message. Under --json also print a machine-readable failure document
|
|
18
19
|
(status: failed) on stdout so callers get the same shape as a success; exit codes are unchanged.
|
|
19
20
|
|
|
@@ -21,7 +22,10 @@ def die(msg: str, code: int = 1, kind: str = "input", *, ctx: "Optional[Context]
|
|
|
21
22
|
platform rows, render.py's check stage, batch.py's per-item results, verify.py's steps) keeps
|
|
22
23
|
reporting that detail while the top-level status says failed. Before 1.4.3 those four printed
|
|
23
24
|
`status: "completed"` next to a non-zero exit code, so a caller keying on the status alone
|
|
24
|
-
read a failed delivery as a success.
|
|
25
|
+
read a failed delivery as a success.
|
|
26
|
+
|
|
27
|
+
`error_extra` goes inside `error` (refuse() passes `reason` and `details` there); the
|
|
28
|
+
collector's records (warn() and friends) are merged into the document as emit() merges them."""
|
|
25
29
|
hint = extra.pop("hint", None)
|
|
26
30
|
ctx = ctx or STATE # 1.10: the optional per-request Context (2.0 makes it required); STATE is the default instance
|
|
27
31
|
_set_current_ctx(ctx) # the atexit hook has no argument: it reads the ctx emit()/die() last used
|
|
@@ -48,11 +52,103 @@ def die(msg: str, code: int = 1, kind: str = "input", *, ctx: "Optional[Context]
|
|
|
48
52
|
}
|
|
49
53
|
if hint:
|
|
50
54
|
doc["error"]["hint"] = hint
|
|
55
|
+
for key, value in (error_extra or {}).items():
|
|
56
|
+
if value is not None:
|
|
57
|
+
doc["error"][key] = value
|
|
51
58
|
doc.update(extra)
|
|
59
|
+
_merge_collector(doc, ctx)
|
|
52
60
|
print_json(doc)
|
|
53
61
|
sys.exit(code)
|
|
54
62
|
|
|
55
63
|
|
|
64
|
+
def refuse(kind: str, reason: str, msg: str, *, hint: Optional[str] = None, code: int = 1,
|
|
65
|
+
ctx: "Optional[Context]" = None, extra: "Optional[Dict[str, Any]]" = None, **details: Any) -> "None":
|
|
66
|
+
"""die() for a refusal a caller has to act on: `error.reason` is a stable snake_case code for
|
|
67
|
+
*why* (output_exists, output_is_input, output_locked, plan_inputs_changed, ...), and
|
|
68
|
+
`error.details` holds the facts the message is written from (paths, expected and actual
|
|
69
|
+
values), so a caller never parses the prose. `extra` adds top-level keys as die()'s do."""
|
|
70
|
+
die(msg, code=code, kind=kind, ctx=ctx, hint=hint, error_extra={"reason": reason, "details": json_safe(details)}, **(extra or {}))
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _record(bucket: str, msg: str, code: str, ctx: "Optional[Context]", data: Dict[str, Any], prefix: str) -> Dict[str, Any]:
|
|
74
|
+
ctx = ctx or STATE
|
|
75
|
+
entry: Dict[str, Any] = {"code": code, "message": msg}
|
|
76
|
+
entry.update(data)
|
|
77
|
+
getattr(ctx, bucket).append(entry)
|
|
78
|
+
sys.stderr.write(f"{prefix}{msg}\n")
|
|
79
|
+
return entry
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def warn(msg: str, code: str, *, ctx: "Optional[Context]" = None, **data: Any) -> Dict[str, Any]:
|
|
83
|
+
"""Say something went other than asked, on stderr AND in the document
|
|
84
|
+
(`diagnostics.warnings`: {code, message, ...data}). Use it where a bare info("warning: ...")
|
|
85
|
+
would leave the caller of --json nothing to read."""
|
|
86
|
+
return _record("warnings", msg, code, ctx, data, "warning: ")
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def record_fallback(msg: str, code: str, *, ctx: "Optional[Context]" = None, **data: Any) -> Dict[str, Any]:
|
|
90
|
+
"""The run took another route than the asked one and went on (`diagnostics.fallbacks`)."""
|
|
91
|
+
return _record("fallbacks", msg, code, ctx, data, "")
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def record_skip(msg: str, code: str, *, ctx: "Optional[Context]" = None, **data: Any) -> Dict[str, Any]:
|
|
95
|
+
"""A step that was asked for, or would normally run, did not (`diagnostics.skipped`)."""
|
|
96
|
+
return _record("skipped", msg, code, ctx, data, "")
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def note(msg: str, ctx: "Optional[Context]" = None) -> None:
|
|
100
|
+
"""A plain `notes` line that also goes to stderr."""
|
|
101
|
+
ctx = ctx or STATE
|
|
102
|
+
if msg not in ctx.notes:
|
|
103
|
+
ctx.notes.append(msg)
|
|
104
|
+
sys.stderr.write(f"{msg}\n")
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def _merge_collector(doc: Dict[str, Any], ctx: "Context") -> None:
|
|
108
|
+
"""The collector into a result or failure document: ctx.notes join `notes` (once each), and
|
|
109
|
+
the three record lists become `diagnostics`, present only when one of them is not empty
|
|
110
|
+
(`warnings` and `skipped` are not top-level keys: check.py's `warnings` is a count and join.py's
|
|
111
|
+
`skipped` its left-out inputs)."""
|
|
112
|
+
extra_notes = [n for n in ctx.notes if n not in (doc.get("notes") or [])]
|
|
113
|
+
if extra_notes:
|
|
114
|
+
doc["notes"] = list(doc.get("notes") or []) + extra_notes
|
|
115
|
+
diag = {k: list(getattr(ctx, k)) for k in ("warnings", "fallbacks", "skipped") if getattr(ctx, k)}
|
|
116
|
+
if diag:
|
|
117
|
+
merged = dict(doc.get("diagnostics") or {})
|
|
118
|
+
for k, v in diag.items():
|
|
119
|
+
merged[k] = list(merged.get(k) or []) + [e for e in v if e not in (merged.get(k) or [])]
|
|
120
|
+
doc["diagnostics"] = merged
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def audit(event: str, details: Dict[str, Any], *, default_path: Optional[str] = None, plan_sha256: Optional[str] = None,
|
|
124
|
+
ctx: "Optional[Context]" = None) -> Dict[str, Any]:
|
|
125
|
+
"""Append one JSON line to the audit log: {ts, tool, event, pid, plan_sha256?, details}.
|
|
126
|
+
The log is render.py's --audit-log FILE when given, else the event's own `default_path` (a
|
|
127
|
+
plan's `<plan>.audit.jsonl`, a lock's hidden sibling); neither = nothing to write. Never under
|
|
128
|
+
--dry-run (a dry run writes nothing). A write that fails is a recorded warning, never a
|
|
129
|
+
failure of the run. Returns what happened, for the caller's `details.audit_log`."""
|
|
130
|
+
import datetime
|
|
131
|
+
ctx = ctx or STATE
|
|
132
|
+
path = ctx.audit_log or default_path
|
|
133
|
+
if not path:
|
|
134
|
+
return {"path": None, "written": False, "reason": "no_audit_log"}
|
|
135
|
+
if ctx.dry_run:
|
|
136
|
+
return {"path": path, "written": False, "reason": "dry_run"}
|
|
137
|
+
rec: Dict[str, Any] = {"ts": datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%fZ"),
|
|
138
|
+
"tool": os.path.splitext(os.path.basename(sys.argv[0] or "?"))[0], "event": event, "pid": os.getpid()}
|
|
139
|
+
if plan_sha256:
|
|
140
|
+
rec["plan_sha256"] = plan_sha256
|
|
141
|
+
rec["details"] = details
|
|
142
|
+
try:
|
|
143
|
+
with open(path, "a", encoding="utf-8") as f:
|
|
144
|
+
f.write(json.dumps(json_safe(rec), ensure_ascii=False, sort_keys=True) + "\n")
|
|
145
|
+
except OSError as exc:
|
|
146
|
+
warn(f"could not append the {event} record to the audit log {path} ({exc}); the run is unaffected",
|
|
147
|
+
"audit_log_unwritable", ctx=ctx, path=path, event=event, error=str(exc))
|
|
148
|
+
return {"path": path, "written": False, "reason": "unwritable", "error": str(exc)}
|
|
149
|
+
return {"path": path, "written": True}
|
|
150
|
+
|
|
151
|
+
|
|
56
152
|
def info(msg: str, ctx: "Optional[Context]" = None) -> None:
|
|
57
153
|
# under --dry-run nothing is written; do not let scripts claim otherwise
|
|
58
154
|
ctx = ctx or STATE
|
|
@@ -195,6 +291,24 @@ def absorb_stage_hw(stage: str, doc: Any, ctx: "Optional[Context]" = None) -> No
|
|
|
195
291
|
"used": hw.get("used"), "fallback": bool(hw.get("fallback")), "quality_note": quality})
|
|
196
292
|
|
|
197
293
|
|
|
294
|
+
def absorb_stage(stage: str, doc: Any, ctx: "Optional[Context]" = None) -> Dict[str, Any]:
|
|
295
|
+
"""A child tool's --json document into this result: its `hw` facts (absorb_stage_hw) and its
|
|
296
|
+
`diagnostics` records, each tagged with the stage it came from. Returns the fields a re-raise
|
|
297
|
+
of the stage's failure carries in its own `error`: the child's `reason` and `details`."""
|
|
298
|
+
ctx = ctx or STATE
|
|
299
|
+
absorb_stage_hw(stage, doc, ctx)
|
|
300
|
+
if not isinstance(doc, dict):
|
|
301
|
+
return {}
|
|
302
|
+
for bucket in ("warnings", "fallbacks", "skipped"):
|
|
303
|
+
for entry in (doc.get("diagnostics") or {}).get(bucket) or []:
|
|
304
|
+
if isinstance(entry, dict):
|
|
305
|
+
tagged = dict(entry, stage=entry.get("stage") or stage)
|
|
306
|
+
if tagged not in getattr(ctx, bucket):
|
|
307
|
+
getattr(ctx, bucket).append(tagged)
|
|
308
|
+
err = doc.get("error") if isinstance(doc.get("error"), dict) else {}
|
|
309
|
+
return {k: err[k] for k in ("reason", "details") if err.get(k) is not None}
|
|
310
|
+
|
|
311
|
+
|
|
198
312
|
def steps_encoder_report(steps: Sequence["Tuple[str, Any]"]) -> Dict[str, Any]:
|
|
199
313
|
"""batch.py's per-item `encoder`, `hw` and `notes`, from the --json documents of the steps
|
|
200
314
|
(child processes) that made the item: the last encoder a step ran (a later stream copy keeps
|
|
@@ -246,6 +360,7 @@ def emit(output: Optional[str], *, ctx: "Optional[Context]" = None, **extra: Any
|
|
|
246
360
|
hw_notes = [n for n in _hw_notes_for(ctx) if n not in (doc.get("notes") or [])]
|
|
247
361
|
if hw_notes:
|
|
248
362
|
doc["notes"] = list(doc.get("notes") or []) + hw_notes
|
|
363
|
+
_merge_collector(doc, ctx)
|
|
249
364
|
if ctx.plan:
|
|
250
365
|
doc["plan"] = write_plan(ctx.plan, output, extra, ctx=ctx)
|
|
251
366
|
print_json(_brief(doc, meta) if ctx.json_brief else doc)
|
|
@@ -345,6 +460,41 @@ def _plan_inputs(commands: Sequence[str], argv: Sequence[str] = (), ctx: "Option
|
|
|
345
460
|
return seen
|
|
346
461
|
|
|
347
462
|
|
|
463
|
+
def _plan_unbound_inputs(commands: Sequence[str], ctx: "Optional[Context]" = None) -> List[str]:
|
|
464
|
+
"""The inputs _plan_inputs() could not fingerprint because they did not exist when the plan
|
|
465
|
+
was made: a planned command's `-i` file (or a side input registered through
|
|
466
|
+
escape_filter_path) that is not on disk and is not written by an earlier planned command (the
|
|
467
|
+
tool's own intermediates are bound by the command that makes them). render.py reports them on
|
|
468
|
+
execution: nothing checks that their content is what the plan was made for."""
|
|
469
|
+
import shlex
|
|
470
|
+
produced: List[str] = []
|
|
471
|
+
out: List[str] = []
|
|
472
|
+
|
|
473
|
+
def consider(p: str) -> None:
|
|
474
|
+
if (not p or p == "-" or p.startswith("pipe:") or os.path.exists(p)
|
|
475
|
+
or os.path.abspath(p) in produced or os.path.abspath(p) in out):
|
|
476
|
+
return
|
|
477
|
+
out.append(os.path.abspath(p))
|
|
478
|
+
for p in (ctx or STATE).plan_inputs:
|
|
479
|
+
consider(p)
|
|
480
|
+
for line in commands:
|
|
481
|
+
try:
|
|
482
|
+
toks = shlex.split(line.split("] ", 1)[1] if line.startswith("[dry-run] ") else line)
|
|
483
|
+
except ValueError:
|
|
484
|
+
continue
|
|
485
|
+
fmt = None # an input's -f (lavfi's "inputs" are filter graphs, not files)
|
|
486
|
+
for i, tok in enumerate(toks[:-1]):
|
|
487
|
+
if tok == "-f":
|
|
488
|
+
fmt = toks[i + 1]
|
|
489
|
+
elif tok == "-i":
|
|
490
|
+
if fmt != "lavfi":
|
|
491
|
+
consider(toks[i + 1])
|
|
492
|
+
fmt = None
|
|
493
|
+
if toks:
|
|
494
|
+
produced.append(os.path.abspath(toks[-1]))
|
|
495
|
+
return out
|
|
496
|
+
|
|
497
|
+
|
|
348
498
|
def write_plan(path: str, output: Optional[str], extra: Dict[str, Any], ctx: "Optional[Context]" = None) -> str:
|
|
349
499
|
"""The dry run as an artifact: what will run, on which exact inputs, producing what, checked
|
|
350
500
|
how. `render.py PLAN` executes it after re-fingerprinting the inputs (issue #189 C).
|
|
@@ -382,6 +532,8 @@ def write_plan(path: str, output: Optional[str], extra: Dict[str, Any], ctx: "Op
|
|
|
382
532
|
"argv": cleaned,
|
|
383
533
|
"cwd": os.getcwd(),
|
|
384
534
|
"inputs": [fingerprint(p) for p in _plan_inputs(ctx.commands, cleaned, ctx)],
|
|
535
|
+
# inputs that did not exist yet, so carry no fingerprint (render.py reports them on execution)
|
|
536
|
+
"unbound_inputs": _plan_unbound_inputs(ctx.commands, ctx),
|
|
385
537
|
"commands": list(ctx.commands),
|
|
386
538
|
"output": os.path.abspath(output) if output else None,
|
|
387
539
|
"verify": verify,
|
|
@@ -205,6 +205,7 @@ CODECS = ("h264", "hevc", "av1", "prores")
|
|
|
205
205
|
|
|
206
206
|
|
|
207
207
|
_ENCODERS: Optional[set] = None
|
|
208
|
+
_BSFS: Optional[set] = None
|
|
208
209
|
|
|
209
210
|
|
|
210
211
|
class Context:
|
|
@@ -216,7 +217,8 @@ class Context:
|
|
|
216
217
|
"""
|
|
217
218
|
|
|
218
219
|
__slots__ = ("dry_run", "json", "json_brief", "progress", "fast", "duration_hint", "commands", "timeout", "overwrite", "written", "preexisting", "plan", "plan_written", "plan_inputs", "codec",
|
|
219
|
-
"hw", "hw_source", "hw_notes", "hw_swaps", "hw_fallback", "hw_env_ignored", "hw_stages", "hw_auto"
|
|
220
|
+
"hw", "hw_source", "hw_notes", "hw_swaps", "hw_fallback", "hw_env_ignored", "hw_stages", "hw_auto",
|
|
221
|
+
"notes", "warnings", "fallbacks", "skipped", "audit_log")
|
|
220
222
|
|
|
221
223
|
def __init__(self) -> None:
|
|
222
224
|
self.reset()
|
|
@@ -245,6 +247,15 @@ class Context:
|
|
|
245
247
|
self.hw_env_ignored = False # $FFMPEG_SKILL_HW=1 was set but this tool's delivery preset does not take it (export.py)
|
|
246
248
|
self.hw_stages: List[Dict[str, Any]] = [] # render.py: the `hw` facts each stage (a child process) reported
|
|
247
249
|
self.hw_auto: List[str] = [] # CPU encoders (libx264/libx265) this ffmpeg build lacks, whose encodes ran on VideoToolbox without --hw
|
|
250
|
+
# The collector (docs/design-decisions.md, "Nothing happens silently"): what this run
|
|
251
|
+
# decided on its own -- a warning, a fallback, a step it skipped -- recorded as it happens
|
|
252
|
+
# by warn()/record_fallback()/record_skip() (stderr too) and merged by emit()/die() into
|
|
253
|
+
# the document: `notes` (strings) and `diagnostics: {warnings, fallbacks, skipped}`.
|
|
254
|
+
self.notes: List[str] = []
|
|
255
|
+
self.warnings: List[Dict[str, Any]] = []
|
|
256
|
+
self.fallbacks: List[Dict[str, Any]] = []
|
|
257
|
+
self.skipped: List[Dict[str, Any]] = []
|
|
258
|
+
self.audit_log: Optional[str] = None # render.py --audit-log FILE: where audit() appends; None = the event's own default
|
|
248
259
|
|
|
249
260
|
|
|
250
261
|
STATE = Context()
|
|
@@ -530,12 +541,21 @@ def _check_no_overwrite_input(cmd: Sequence[str]) -> None:
|
|
|
530
541
|
if a == "-i" and i + 1 < len(cmd):
|
|
531
542
|
inp = cmd[i + 1]
|
|
532
543
|
try:
|
|
533
|
-
|
|
534
|
-
die(f"refusing to run: output {output!r} is the same file as input {inp!r} "
|
|
535
|
-
f"(would overwrite it while ffmpeg is still reading it) -- choose a different --output/-o path",
|
|
536
|
-
kind="input")
|
|
544
|
+
same = os.path.realpath(inp) == out_real
|
|
537
545
|
except OSError:
|
|
538
546
|
continue
|
|
547
|
+
if same:
|
|
548
|
+
_refuse_output_is_input(output, inp, out_real, "would overwrite it while ffmpeg is still reading it")
|
|
549
|
+
|
|
550
|
+
|
|
551
|
+
def _refuse_output_is_input(output: str, inp: str, real: str, why: str) -> None:
|
|
552
|
+
"""The one refusal both output-is-input guards end in: structured, and in the audit log when
|
|
553
|
+
one is configured (render.py --audit-log)."""
|
|
554
|
+
details = {"output": output, "input": inp, "resolved": real}
|
|
555
|
+
audit("output_is_input_refused", details)
|
|
556
|
+
refuse("input", "output_is_input",
|
|
557
|
+
f"refusing to run: output {output!r} is the same file as input {inp!r} ({why}) -- choose a different --output/-o path",
|
|
558
|
+
**details)
|
|
539
559
|
|
|
540
560
|
|
|
541
561
|
def refuse_output_is_input(output: str, *inputs: str) -> None:
|
|
@@ -554,8 +574,7 @@ def refuse_output_is_input(output: str, *inputs: str) -> None:
|
|
|
554
574
|
except OSError:
|
|
555
575
|
continue
|
|
556
576
|
if same:
|
|
557
|
-
|
|
558
|
-
f"(the result would replace the source) -- choose a different --output/-o path", kind="input")
|
|
577
|
+
_refuse_output_is_input(output, inp, out_real, "the result would replace the source")
|
|
559
578
|
|
|
560
579
|
|
|
561
580
|
def _check_output_path(cmd: Sequence[str]) -> None:
|
|
@@ -566,12 +585,15 @@ def _check_output_path(cmd: Sequence[str]) -> None:
|
|
|
566
585
|
if output == "-" or output.startswith("pipe:") or output.startswith("-"):
|
|
567
586
|
return
|
|
568
587
|
if os.path.isdir(output):
|
|
569
|
-
|
|
588
|
+
refuse("input", "output_is_directory",
|
|
589
|
+
f"output {output!r} is a directory; pass a file path (e.g. {os.path.join(output, 'result.mp4')!r})", output=output)
|
|
570
590
|
parent = os.path.dirname(os.path.abspath(output))
|
|
571
591
|
if not os.path.isdir(parent):
|
|
572
|
-
|
|
592
|
+
refuse("input", "output_dir_missing",
|
|
593
|
+
f"output directory {parent!r} does not exist; create it first (this tool never creates directories)",
|
|
594
|
+
output=output, directory=parent)
|
|
573
595
|
if not os.access(parent, os.W_OK):
|
|
574
|
-
|
|
596
|
+
refuse("input", "output_dir_not_writable", f"output directory {parent!r} is not writable", output=output, directory=parent)
|
|
575
597
|
|
|
576
598
|
|
|
577
599
|
EVEN_SCALE = "scale=trunc(iw/2)*2:trunc(ih/2)*2"
|
|
@@ -601,14 +623,29 @@ def _pid_dead(pid: int) -> bool:
|
|
|
601
623
|
return False
|
|
602
624
|
|
|
603
625
|
|
|
626
|
+
LOCK_HOUR = 3600.0
|
|
627
|
+
LOCK_UNLIMITED = 10 * 365 * 86400.0 # --timeout 0: the deadline is pushed past any run (and kept short of 2038)
|
|
628
|
+
|
|
629
|
+
|
|
604
630
|
class _OutputLock:
|
|
605
631
|
"""Two runs writing the same output at once used to both report `completed` while one of
|
|
606
632
|
them described the other's file (sweep F1). A lock file next to the output, created with
|
|
607
|
-
O_EXCL and holding the writer's pid, makes the second run refuse
|
|
608
|
-
|
|
633
|
+
O_EXCL and holding the writer's pid, makes the second run refuse (`kind: input`,
|
|
634
|
+
`reason: output_locked`, the holder in `details`).
|
|
635
|
+
|
|
636
|
+
Stale locks: a lock whose pid is dead is taken over. A lock whose pid is alive is taken over
|
|
637
|
+
only once its holder's own time limit has passed: the writer sets the lock's mtime to that
|
|
638
|
+
deadline minus an hour, so "older than an hour" -- the rule every version since 1.x applies,
|
|
639
|
+
which this keeps compatible in both directions -- means "past the deadline". The deadline is
|
|
640
|
+
child_limit() from now (4x --timeout + 60 s, the ceiling on everything one run() does under
|
|
641
|
+
the lock); with --timeout 0 there is none and a live holder is never taken over. Through 2.10
|
|
642
|
+
the hour ran from the lock's creation, so a legitimate run longer than an hour lost its lock
|
|
643
|
+
to a second writer. A takeover is recorded (`diagnostics.fallbacks`, stderr, and the audit
|
|
644
|
+
log); so is running without a lock where none can be created (`diagnostics.warnings`)."""
|
|
609
645
|
def __init__(self, output: str) -> None:
|
|
610
646
|
self.path: Optional[str] = None
|
|
611
647
|
self.fd: Optional[int] = None
|
|
648
|
+
self.output = output
|
|
612
649
|
if output == "-" or output.startswith("pipe:") or output.startswith("-"):
|
|
613
650
|
return
|
|
614
651
|
d, base = os.path.split(os.path.abspath(output))
|
|
@@ -617,33 +654,95 @@ class _OutputLock:
|
|
|
617
654
|
def __enter__(self) -> "_OutputLock":
|
|
618
655
|
if not self.path:
|
|
619
656
|
return self
|
|
657
|
+
taken: Optional[Dict[str, Any]] = None
|
|
620
658
|
for attempt in (0, 1):
|
|
621
659
|
try:
|
|
622
660
|
self.fd = os.open(self.path, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o644)
|
|
623
661
|
os.write(self.fd, str(os.getpid()).encode())
|
|
662
|
+
self._set_deadline()
|
|
663
|
+
if taken is not None:
|
|
664
|
+
self._record_takeover(taken)
|
|
624
665
|
return self
|
|
625
666
|
except FileExistsError:
|
|
626
|
-
|
|
667
|
+
holder = self._holder()
|
|
668
|
+
if attempt == 0 and holder["stale"]:
|
|
627
669
|
try:
|
|
628
670
|
os.remove(self.path)
|
|
629
|
-
except OSError:
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
671
|
+
except OSError as exc:
|
|
672
|
+
# Windows will not delete a file another process holds open, and a live
|
|
673
|
+
# holder keeps its lock open until it finishes: the takeover fails and
|
|
674
|
+
# the run is refused below, with the reason, instead of passing silently
|
|
675
|
+
holder["takeover_error"] = str(exc)
|
|
676
|
+
holder["stale"] = False
|
|
677
|
+
else:
|
|
678
|
+
taken = holder
|
|
679
|
+
continue
|
|
680
|
+
name = os.path.basename(self.path)[1:-len(".ffskill-lock")]
|
|
681
|
+
refuse("input", "output_locked",
|
|
682
|
+
f"another run is writing {name!r} right now (lock {self.path}"
|
|
683
|
+
+ (f", pid {holder['pid']}" if holder.get("pid") else "") + "); wait for it or choose a different --output/-o path",
|
|
684
|
+
hint=("wait for that run, or choose another -o path; if pid "
|
|
685
|
+
f"{holder.get('pid')} is not an ffmpeg-skill run, delete {self.path}"),
|
|
686
|
+
lock=self.path, output=os.path.abspath(self.output), **{k: v for k, v in holder.items() if k != "stale"})
|
|
687
|
+
except OSError as exc:
|
|
688
|
+
# an unlockable location (a read-only directory surfaces elsewhere): the run goes
|
|
689
|
+
# on without a lock, and says so -- two runs could now both write this output
|
|
690
|
+
details = {"lock": self.path, "output": os.path.abspath(self.output), "error": str(exc)}
|
|
691
|
+
warn(f"could not create the output lock {self.path} ({exc}); running without one, so a second run "
|
|
692
|
+
"writing the same output at the same time would not be refused", "output_unlocked", **details)
|
|
693
|
+
audit("output_unlocked", details, default_path=self._audit_default())
|
|
694
|
+
return self
|
|
636
695
|
return self
|
|
637
696
|
|
|
638
|
-
def
|
|
697
|
+
def _audit_default(self) -> Optional[str]:
|
|
698
|
+
if not self.path:
|
|
699
|
+
return None
|
|
700
|
+
d, base = os.path.split(os.path.abspath(self.output))
|
|
701
|
+
return os.path.join(d, f".{base}.ffskill-audit.jsonl")
|
|
702
|
+
|
|
703
|
+
def _set_deadline(self) -> None:
|
|
704
|
+
import time
|
|
705
|
+
limit = child_limit()
|
|
706
|
+
deadline = time.time() + (limit if limit else LOCK_UNLIMITED)
|
|
639
707
|
try:
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
708
|
+
os.utime(self.path, (deadline - LOCK_HOUR, deadline - LOCK_HOUR))
|
|
709
|
+
except (OSError, OverflowError, ValueError) as exc:
|
|
710
|
+
warn(f"could not date the output lock {self.path} ({exc}); another run may take it over an hour from now",
|
|
711
|
+
"lock_deadline_unset", lock=self.path, error=str(exc))
|
|
712
|
+
|
|
713
|
+
def _holder(self) -> Dict[str, Any]:
|
|
714
|
+
"""Who holds the lock and whether it is stale: {pid, pid_alive, deadline (UTC, the
|
|
715
|
+
mtime plus an hour), why: pid_dead | past_deadline | unreadable | held, stale}."""
|
|
716
|
+
import datetime
|
|
717
|
+
import time
|
|
718
|
+
out: Dict[str, Any] = {"pid": None, "pid_alive": None, "deadline": None, "why": "unreadable", "stale": True}
|
|
719
|
+
try:
|
|
720
|
+
with open(self.path, encoding="utf-8", errors="replace") as f:
|
|
721
|
+
text = f.read().strip()
|
|
722
|
+
deadline = os.path.getmtime(self.path) + LOCK_HOUR
|
|
723
|
+
pid = int(text or "0")
|
|
645
724
|
except (OSError, ValueError):
|
|
646
|
-
return
|
|
725
|
+
return out
|
|
726
|
+
out["deadline"] = datetime.datetime.fromtimestamp(deadline, datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
|
727
|
+
out["pid"] = pid if pid > 0 else None
|
|
728
|
+
if pid > 0 and _pid_dead(pid):
|
|
729
|
+
out.update(pid_alive=False, why="pid_dead")
|
|
730
|
+
return out
|
|
731
|
+
out["pid_alive"] = pid > 0 or None
|
|
732
|
+
if time.time() > deadline:
|
|
733
|
+
out["why"] = "past_deadline"
|
|
734
|
+
return out
|
|
735
|
+
out.update(why="held", stale=False)
|
|
736
|
+
return out
|
|
737
|
+
|
|
738
|
+
def _record_takeover(self, holder: Dict[str, Any]) -> None:
|
|
739
|
+
details = {"lock": self.path, "output": os.path.abspath(self.output),
|
|
740
|
+
**{k: v for k, v in holder.items() if k != "stale"}}
|
|
741
|
+
why = {"pid_dead": f"its pid {holder.get('pid')} no longer runs",
|
|
742
|
+
"past_deadline": f"its holder (pid {holder.get('pid')}, still running) is past its own time limit",
|
|
743
|
+
}.get(holder.get("why") or "", "it could not be read")
|
|
744
|
+
details["audit_log"] = audit("lock_takeover", details, default_path=self._audit_default())
|
|
745
|
+
record_fallback(f"took over the stale output lock {self.path}: {why}", "stale_lock_taken_over", **details)
|
|
647
746
|
|
|
648
747
|
def __exit__(self, *exc: Any) -> None:
|
|
649
748
|
if self.fd is not None:
|
|
@@ -697,6 +796,7 @@ def refuse_existing_outputs(paths: Sequence[str]) -> None:
|
|
|
697
796
|
.srt/.ass sidecars) calls this up front so a hand-edited sidecar is refused like the video
|
|
698
797
|
is, and a dry run predicts the same refusal."""
|
|
699
798
|
existing: List[str] = []
|
|
799
|
+
facts: List[Dict[str, Any]] = []
|
|
700
800
|
for output in paths:
|
|
701
801
|
if not output:
|
|
702
802
|
continue
|
|
@@ -707,21 +807,25 @@ def refuse_existing_outputs(paths: Sequence[str]) -> None:
|
|
|
707
807
|
continue
|
|
708
808
|
if not exists or real in STATE.written:
|
|
709
809
|
continue
|
|
810
|
+
size = mtime_ns = None
|
|
710
811
|
try:
|
|
711
812
|
st = os.stat(output)
|
|
712
813
|
STATE.preexisting[real] = (st.st_size, st.st_mtime_ns)
|
|
814
|
+
size, mtime_ns = st.st_size, st.st_mtime_ns
|
|
713
815
|
except OSError:
|
|
714
816
|
pass
|
|
715
817
|
if output not in existing:
|
|
716
818
|
existing.append(output)
|
|
819
|
+
facts.append({"path": output, "resolved": real, "size": size, "mtime_ns": mtime_ns})
|
|
717
820
|
if not existing or STATE.overwrite:
|
|
718
821
|
return
|
|
822
|
+
hint = "pass --overwrite, or choose another -o path"
|
|
719
823
|
if len(existing) == 1:
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
824
|
+
refuse("input", "output_exists",
|
|
825
|
+
f"refusing to overwrite existing output {existing[0]!r}: pass --overwrite to replace it, or choose another -o path",
|
|
826
|
+
hint=hint, outputs=facts)
|
|
827
|
+
refuse("input", "output_exists", "refusing to overwrite existing outputs " + ", ".join(repr(p) for p in existing)
|
|
828
|
+
+ ": pass --overwrite to replace them, or choose another -o path", hint=hint, outputs=facts)
|
|
725
829
|
|
|
726
830
|
|
|
727
831
|
def _remember_output(cmd: Sequence[str]) -> None:
|
|
@@ -1372,6 +1476,24 @@ def ffmpeg_encoders() -> set:
|
|
|
1372
1476
|
return _ENCODERS
|
|
1373
1477
|
|
|
1374
1478
|
|
|
1479
|
+
def ffmpeg_bsfs() -> set:
|
|
1480
|
+
"""Names from `ffmpeg -bsfs`, read once, empty when it cannot be read; from ffprobe under
|
|
1481
|
+
--dry-run, for the reason ffmpeg_encoders() gives. color.py --retag rewrites an H.264/HEVC
|
|
1482
|
+
stream's colour description with h264_metadata/hevc_metadata only when the build has it."""
|
|
1483
|
+
global _BSFS
|
|
1484
|
+
if _BSFS is None:
|
|
1485
|
+
_BSFS = set()
|
|
1486
|
+
tool = "ffprobe" if STATE.dry_run else "ffmpeg"
|
|
1487
|
+
limit = min(PROBE_TIMEOUT, STATE.timeout) if STATE.timeout else PROBE_TIMEOUT
|
|
1488
|
+
try:
|
|
1489
|
+
out = subprocess.run([ffmpeg_exe(tool)[0] or tool, "-hide_banner", "-bsfs"], stdout=subprocess.PIPE,
|
|
1490
|
+
stderr=subprocess.DEVNULL, text=True, encoding="utf-8", errors="replace", timeout=limit).stdout
|
|
1491
|
+
_BSFS = set(re.findall(r"^\s*([a-z0-9_]+)\s*$", out, re.M))
|
|
1492
|
+
except (OSError, subprocess.SubprocessError):
|
|
1493
|
+
pass
|
|
1494
|
+
return _BSFS
|
|
1495
|
+
|
|
1496
|
+
|
|
1375
1497
|
def read_text_or_die(path: str, flag: str) -> str:
|
|
1376
1498
|
"""Read a caller-supplied UTF-8 text file (a cue list, chapters, notes) or fail as kind input
|
|
1377
1499
|
with the flag named, instead of a FileNotFoundError / UnicodeDecodeError traceback."""
|
|
@@ -1396,4 +1518,4 @@ def read_text_or_die(path: str, flag: str) -> str:
|
|
|
1396
1518
|
# emit needs runner's Context/STATE/ERROR_CODE, so the two form a cycle that has to be cut
|
|
1397
1519
|
# somewhere: by the time this line runs every name emit reads from runner above is defined, and
|
|
1398
1520
|
# every name runner reads from emit is only ever read inside a function body, never at import.
|
|
1399
|
-
from _common.emit import _plan_at_exit, die, hw_report, info # noqa: E402
|
|
1521
|
+
from _common.emit import _plan_at_exit, audit, die, hw_report, info, record_fallback, refuse, warn # noqa: E402
|
package/scripts/_common/text.py
CHANGED
|
@@ -32,7 +32,7 @@ from _common.emoji import (
|
|
|
32
32
|
from _common.drawtext import (
|
|
33
33
|
drawtext_boxborderw, SHAPING_SCRIPTS, BIDI_SCRIPTS, _SHAPING_BUILD_CACHE, drawtext_shaping, needs_shaping,
|
|
34
34
|
escape_drawtext, drawtext_text_opts, drawtext_center_align, NEWLINE_GLYPH_RELEASES,
|
|
35
|
-
drawtext_draws_newline_glyph, is_multiline,
|
|
35
|
+
drawtext_draws_newline_glyph, is_multiline, drawtext_clean, drawtext_line_metrics,
|
|
36
36
|
)
|
|
37
37
|
from _common.wrap import (
|
|
38
38
|
ADVANCE_EM, NO_SPACE_SCRIPTS, NO_BOUNDARY_SCRIPTS, LATIN_EM, LEADING_VOWELS, _is_mark, _char_em,
|