ffmpeg-skill 3.8.15 → 3.8.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/docs/contract.md +3 -3
- package/mcp/server.py +143 -21
- package/package.json +1 -1
- package/references/gotchas.md +12 -2
- package/scripts/_common/__init__.py +2 -2
- package/scripts/_common/alpha.py +1 -0
- package/scripts/_common/color.py +30 -1
- package/scripts/_common/decision.py +57 -0
- package/scripts/_common/emit.py +55 -6
- package/scripts/_common/probe.py +14 -1
- package/scripts/_common/runner.py +452 -80
- package/scripts/_common/wrap.py +24 -8
- package/scripts/_contract.py +18 -2
- package/scripts/batch.py +7 -1
- package/scripts/broll.py +5 -2
- package/scripts/caption.py +11 -7
- package/scripts/color.py +3 -2
- package/scripts/crop.py +2 -2
- package/scripts/cut.py +9 -3
- package/scripts/deinterlace.py +12 -7
- package/scripts/denoise.py +2 -2
- package/scripts/export.py +23 -4
- package/scripts/fit.py +45 -7
- package/scripts/freeze.py +9 -7
- package/scripts/graphics.py +7 -4
- package/scripts/loop.py +7 -2
- package/scripts/metadata.py +5 -2
- package/scripts/overlay.py +92 -4
- package/scripts/pad.py +7 -2
- package/scripts/proxy.py +2 -2
- package/scripts/redact.py +5 -2
- package/scripts/render.py +24 -12
- package/scripts/silence.py +2 -2
- package/scripts/speedramp.py +49 -14
- package/scripts/sphere.py +2 -2
- package/scripts/stabilize.py +2 -2
- package/scripts/straighten.py +2 -2
- package/scripts/verify.py +32 -2
|
@@ -608,8 +608,13 @@ def stop_children() -> None:
|
|
|
608
608
|
signal handler does before it exits, for a tool that reports more than "interrupted" after a
|
|
609
609
|
signal (batch.py's partial table)."""
|
|
610
610
|
for proc, cmd in list(_CHILDREN):
|
|
611
|
+
if not cmd:
|
|
612
|
+
# a sibling script (run_tool(), verify.py's steps): its own handler stops its ffmpeg
|
|
613
|
+
# and removes its partial output, given STOP_GRACE; on Windows its whole tree is killed
|
|
614
|
+
_stop_child(proc, communicate=False)
|
|
615
|
+
continue
|
|
611
616
|
try:
|
|
612
|
-
proc.terminate() # ffmpeg exits promptly on SIGTERM
|
|
617
|
+
proc.terminate() # ffmpeg exits promptly on SIGTERM
|
|
613
618
|
try:
|
|
614
619
|
proc.wait(timeout=5)
|
|
615
620
|
except subprocess.TimeoutExpired:
|
|
@@ -978,14 +983,24 @@ def _check_no_overwrite_input(cmd: Sequence[str]) -> None:
|
|
|
978
983
|
for i, a in enumerate(cmd):
|
|
979
984
|
if a == "-i" and i + 1 < len(cmd):
|
|
980
985
|
inp = cmd[i + 1]
|
|
981
|
-
|
|
982
|
-
same = os.path.realpath(inp) == out_real
|
|
983
|
-
except OSError:
|
|
984
|
-
continue
|
|
985
|
-
if same:
|
|
986
|
+
if same_file(inp, output):
|
|
986
987
|
_refuse_output_is_input(output, inp, out_real, "would overwrite it while ffmpeg is still reading it")
|
|
987
988
|
|
|
988
989
|
|
|
990
|
+
def same_file(a: str, b: str) -> bool:
|
|
991
|
+
"""Whether two paths name one file: the same resolved path, or, when both exist, the same
|
|
992
|
+
file by identity (os.path.samefile: device and inode). The second catches what the strings
|
|
993
|
+
do not (A5, #465): a hard link, and on a case-insensitive volume (macOS's and Windows'
|
|
994
|
+
defaults) `Clip.mp4` for `clip.mp4`, which --overwrite then replaced. False when either path
|
|
995
|
+
cannot be resolved."""
|
|
996
|
+
try:
|
|
997
|
+
if os.path.realpath(a) == os.path.realpath(b):
|
|
998
|
+
return True
|
|
999
|
+
return os.path.exists(a) and os.path.exists(b) and os.path.samefile(a, b)
|
|
1000
|
+
except (OSError, ValueError):
|
|
1001
|
+
return False
|
|
1002
|
+
|
|
1003
|
+
|
|
989
1004
|
def _refuse_output_is_input(output: str, inp: str, real: str, why: str) -> None:
|
|
990
1005
|
"""The one refusal both output-is-input guards end in: structured, and in the audit log when
|
|
991
1006
|
one is configured (render.py --audit-log)."""
|
|
@@ -1007,13 +1022,8 @@ def refuse_output_is_input(output: str, *inputs: str, why: str = "the result wou
|
|
|
1007
1022
|
except OSError:
|
|
1008
1023
|
return
|
|
1009
1024
|
for inp in inputs:
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
same = os.path.realpath(inp) == out_real or (
|
|
1013
|
-
os.path.exists(output) and os.path.exists(inp) and os.path.samefile(output, inp))
|
|
1014
|
-
except OSError:
|
|
1015
|
-
continue
|
|
1016
|
-
if same:
|
|
1025
|
+
# a hard link is another name for the same file: compare the files, not just the names
|
|
1026
|
+
if same_file(inp, output):
|
|
1017
1027
|
_refuse_output_is_input(output, inp, out_real, why)
|
|
1018
1028
|
|
|
1019
1029
|
|
|
@@ -1087,7 +1097,7 @@ def _an_input_has(cmd: Sequence[str], kind: str) -> bool:
|
|
|
1087
1097
|
if not os.path.isfile(path):
|
|
1088
1098
|
return True
|
|
1089
1099
|
probe_cmd = [shutil.which("ffprobe") or "ffprobe", "-v", "error", "-select_streams", kind,
|
|
1090
|
-
"-show_entries", "stream=index", "-of", "csv=p=0", path]
|
|
1100
|
+
"-show_entries", "stream=index", "-of", "csv=p=0", *still_input_args(path), path]
|
|
1091
1101
|
limit = _probe_limit() # PROBE_TIMEOUT, or --timeout when that is lower; never unlimited (--timeout 0)
|
|
1092
1102
|
try:
|
|
1093
1103
|
proc = subprocess.run(probe_cmd, capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=limit)
|
|
@@ -1251,9 +1261,12 @@ EVEN_SCALE = "scale=trunc(iw/2)*2:trunc(ih/2)*2"
|
|
|
1251
1261
|
|
|
1252
1262
|
|
|
1253
1263
|
def _pid_dead(pid: int) -> bool:
|
|
1254
|
-
"""True only when the process is known not to
|
|
1255
|
-
fails with ERROR_INVALID_PARAMETER (87) for a pid that is not in use;
|
|
1256
|
-
|
|
1264
|
+
"""True only when the process is known not to have exited. POSIX: signal 0. Windows:
|
|
1265
|
+
OpenProcess fails with ERROR_INVALID_PARAMETER (87) for a pid that is not in use; a process
|
|
1266
|
+
that has exited still opens while any handle to it is open (its parent's Popen keeps one), so
|
|
1267
|
+
its exit code is read too, and one other than STILL_ACTIVE (259) is dead (review of #530: the
|
|
1268
|
+
Windows CI judged a finished child alive and refused its stale lock as held). Any other
|
|
1269
|
+
outcome (access denied, an unreadable code) means it is live. Unknown is treated as live."""
|
|
1257
1270
|
if os.name != "nt":
|
|
1258
1271
|
try:
|
|
1259
1272
|
os.kill(pid, 0)
|
|
@@ -1267,8 +1280,13 @@ def _pid_dead(pid: int) -> bool:
|
|
|
1267
1280
|
k32 = ctypes.windll.kernel32 # type: ignore[attr-defined]
|
|
1268
1281
|
handle = k32.OpenProcess(0x1000, False, pid) # PROCESS_QUERY_LIMITED_INFORMATION
|
|
1269
1282
|
if handle:
|
|
1270
|
-
|
|
1271
|
-
|
|
1283
|
+
try:
|
|
1284
|
+
code = ctypes.c_ulong(0)
|
|
1285
|
+
if k32.GetExitCodeProcess(handle, ctypes.byref(code)):
|
|
1286
|
+
return code.value != 259 # STILL_ACTIVE
|
|
1287
|
+
return False
|
|
1288
|
+
finally:
|
|
1289
|
+
k32.CloseHandle(handle)
|
|
1272
1290
|
return k32.GetLastError() == 87
|
|
1273
1291
|
except Exception:
|
|
1274
1292
|
return False
|
|
@@ -1278,6 +1296,114 @@ LOCK_HOUR = 3600.0
|
|
|
1278
1296
|
LOCK_UNLIMITED = 10 * 365 * 86400.0 # --timeout 0: the deadline is pushed past any run (and kept short of 2038)
|
|
1279
1297
|
|
|
1280
1298
|
|
|
1299
|
+
SECTION_WAIT = 2.0 # seconds a run waits for another's takeover section before it is refused
|
|
1300
|
+
|
|
1301
|
+
|
|
1302
|
+
class _SectionUnavailable(OSError):
|
|
1303
|
+
"""This filesystem (or platform) gives no OS file lock to the takeover section."""
|
|
1304
|
+
|
|
1305
|
+
|
|
1306
|
+
def _os_lock(fd: int) -> bool:
|
|
1307
|
+
"""Lock `fd` exclusively through the OS without waiting: True when locked, False when another
|
|
1308
|
+
open file holds it. The kernel releases it when the file is closed or its process ends, killed
|
|
1309
|
+
or not. _SectionUnavailable when the filesystem or platform has no such lock."""
|
|
1310
|
+
if os.name == "nt":
|
|
1311
|
+
import errno
|
|
1312
|
+
import msvcrt
|
|
1313
|
+
try:
|
|
1314
|
+
os.lseek(fd, 0, os.SEEK_SET)
|
|
1315
|
+
msvcrt.locking(fd, msvcrt.LK_NBLCK, 1) # one byte; a lock past the end of a file is allowed
|
|
1316
|
+
return True
|
|
1317
|
+
except OSError as exc:
|
|
1318
|
+
if exc.errno in (errno.EACCES, getattr(errno, "EDEADLOCK", errno.EDEADLK), errno.EDEADLK):
|
|
1319
|
+
return False
|
|
1320
|
+
raise _SectionUnavailable(str(exc)) from exc
|
|
1321
|
+
try:
|
|
1322
|
+
import fcntl
|
|
1323
|
+
except ImportError as exc:
|
|
1324
|
+
raise _SectionUnavailable(f"no fcntl here: {exc}") from exc
|
|
1325
|
+
try:
|
|
1326
|
+
fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
1327
|
+
return True
|
|
1328
|
+
except BlockingIOError:
|
|
1329
|
+
return False
|
|
1330
|
+
except OSError as exc:
|
|
1331
|
+
raise _SectionUnavailable(str(exc)) from exc
|
|
1332
|
+
|
|
1333
|
+
|
|
1334
|
+
class _TakeoverSection:
|
|
1335
|
+
"""The section in which a stale output lock is judged, removed and replaced, and in which a
|
|
1336
|
+
holder removes its own lock: `<lock>.takeover`, held through an OS file lock (flock on POSIX,
|
|
1337
|
+
msvcrt.locking on Windows), never through the file's existence or age. The kernel releases it
|
|
1338
|
+
when the run closes it or ends, killed included, so nothing in it expires while its holder
|
|
1339
|
+
lives -- a paused holder still holds it -- and nothing is left for anyone to recover (review of
|
|
1340
|
+
#530: every timed or claimed scheme before this had a window where a paused run lost it).
|
|
1341
|
+
|
|
1342
|
+
The file itself means nothing: its name is removed when the section is left, so no file stays
|
|
1343
|
+
beside the output. On POSIX the name is removed while the lock is still held, and a run that
|
|
1344
|
+
opened the old file before that finds, once it has the lock, that the name is now another file
|
|
1345
|
+
or none, and starts again (the usual lock-file race). On Windows a file another handle has open
|
|
1346
|
+
cannot be removed: the holder closes it first and a failed removal is left alone; the file then
|
|
1347
|
+
serves the next run as it is."""
|
|
1348
|
+
|
|
1349
|
+
def __init__(self, lock_path: str) -> None:
|
|
1350
|
+
self.path = lock_path + ".takeover"
|
|
1351
|
+
self.fd: Optional[int] = None
|
|
1352
|
+
|
|
1353
|
+
def acquire(self, wait: float = SECTION_WAIT) -> Optional[str]:
|
|
1354
|
+
"""None when held; "busy" when another run held it all through `wait` seconds;
|
|
1355
|
+
_SectionUnavailable when the OS lock cannot be had here at all."""
|
|
1356
|
+
import time
|
|
1357
|
+
deadline = time.monotonic() + max(0.0, wait)
|
|
1358
|
+
while True:
|
|
1359
|
+
fd = os.open(self.path, os.O_CREAT | os.O_RDWR, 0o644)
|
|
1360
|
+
try:
|
|
1361
|
+
got = _os_lock(fd)
|
|
1362
|
+
except BaseException:
|
|
1363
|
+
os.close(fd)
|
|
1364
|
+
raise
|
|
1365
|
+
if got:
|
|
1366
|
+
if os.name == "nt" or self._names(fd):
|
|
1367
|
+
self.fd = fd
|
|
1368
|
+
return None
|
|
1369
|
+
os.close(fd) # locked a file whose name was removed meanwhile: start again
|
|
1370
|
+
continue
|
|
1371
|
+
os.close(fd)
|
|
1372
|
+
if time.monotonic() >= deadline:
|
|
1373
|
+
return "busy"
|
|
1374
|
+
time.sleep(0.05)
|
|
1375
|
+
|
|
1376
|
+
def _names(self, fd: int) -> bool:
|
|
1377
|
+
try:
|
|
1378
|
+
mine, there = os.fstat(fd), os.stat(self.path)
|
|
1379
|
+
except FileNotFoundError:
|
|
1380
|
+
return False
|
|
1381
|
+
return (mine.st_dev, mine.st_ino) == (there.st_dev, there.st_ino)
|
|
1382
|
+
|
|
1383
|
+
def release(self) -> None:
|
|
1384
|
+
fd, self.fd = self.fd, None
|
|
1385
|
+
if fd is None:
|
|
1386
|
+
return
|
|
1387
|
+
if os.name != "nt":
|
|
1388
|
+
try:
|
|
1389
|
+
os.remove(self.path) # still locked: whoever opened this file re-checks its name
|
|
1390
|
+
except OSError:
|
|
1391
|
+
pass
|
|
1392
|
+
os.close(fd) # closing releases the lock
|
|
1393
|
+
return
|
|
1394
|
+
try:
|
|
1395
|
+
import msvcrt
|
|
1396
|
+
os.lseek(fd, 0, os.SEEK_SET)
|
|
1397
|
+
msvcrt.locking(fd, msvcrt.LK_UNLCK, 1)
|
|
1398
|
+
except OSError:
|
|
1399
|
+
pass
|
|
1400
|
+
os.close(fd)
|
|
1401
|
+
try:
|
|
1402
|
+
os.remove(self.path) # fails while another run has it open: it is left to that run
|
|
1403
|
+
except OSError:
|
|
1404
|
+
pass
|
|
1405
|
+
|
|
1406
|
+
|
|
1281
1407
|
class _OutputLock:
|
|
1282
1408
|
"""Two runs writing the same output at once used to both report `completed` while one of
|
|
1283
1409
|
them described the other's file (sweep F1). A lock file next to the output, created with
|
|
@@ -1308,67 +1434,153 @@ class _OutputLock:
|
|
|
1308
1434
|
def __enter__(self) -> "_OutputLock":
|
|
1309
1435
|
if not self.path:
|
|
1310
1436
|
return self
|
|
1311
|
-
|
|
1312
|
-
|
|
1313
|
-
|
|
1314
|
-
|
|
1315
|
-
|
|
1316
|
-
|
|
1317
|
-
|
|
1318
|
-
|
|
1437
|
+
try:
|
|
1438
|
+
self.fd = self._create()
|
|
1439
|
+
return self
|
|
1440
|
+
except FileExistsError:
|
|
1441
|
+
pass
|
|
1442
|
+
except OSError as exc:
|
|
1443
|
+
return self._unlocked(exc)
|
|
1444
|
+
holder = self._holder()
|
|
1445
|
+
if holder["stale"]:
|
|
1446
|
+
holder = self._take_over(holder)
|
|
1447
|
+
if holder.get("unlocked"):
|
|
1319
1448
|
return self
|
|
1320
|
-
|
|
1321
|
-
|
|
1322
|
-
if attempt == 0 and holder["stale"]:
|
|
1323
|
-
try:
|
|
1324
|
-
os.remove(self.path)
|
|
1325
|
-
except OSError as exc:
|
|
1326
|
-
# Windows will not delete a file another process holds open, and a live
|
|
1327
|
-
# holder keeps its lock open until it finishes: the takeover fails and
|
|
1328
|
-
# the run is refused below, with the reason, instead of passing silently
|
|
1329
|
-
holder["takeover_error"] = str(exc)
|
|
1330
|
-
holder["stale"] = False
|
|
1331
|
-
else:
|
|
1332
|
-
taken = holder
|
|
1333
|
-
continue
|
|
1334
|
-
name = os.path.basename(self.path)[1:-len(".ffskill-lock")]
|
|
1335
|
-
msg = (f"another run is writing {name!r} right now (lock {self.path}"
|
|
1336
|
-
+ (f", pid {holder['pid']}" if holder.get("pid") else "") + "); wait for it or choose a different --output/-o path")
|
|
1337
|
-
hint = ("wait for that run, or choose another -o path; if pid "
|
|
1338
|
-
f"{holder.get('pid')} is not an ffmpeg-skill run, delete {self.path}")
|
|
1339
|
-
details = dict(lock=self.path, output=os.path.abspath(self.output), **{k: v for k, v in holder.items() if k != "stale"})
|
|
1340
|
-
if self.on_locked:
|
|
1341
|
-
self.on_locked(msg, hint, details)
|
|
1342
|
-
refuse("input", "output_locked", msg, hint=hint, **details)
|
|
1343
|
-
except OSError as exc:
|
|
1344
|
-
# an unlockable location (a read-only directory surfaces elsewhere): the run goes
|
|
1345
|
-
# on without a lock, and says so -- two runs could now both write this output
|
|
1346
|
-
details = {"lock": self.path, "output": os.path.abspath(self.output), "error": str(exc)}
|
|
1347
|
-
warn(f"could not create the output lock {self.path} ({exc}); running without one, so a second run "
|
|
1348
|
-
"writing the same output at the same time would not be refused", "output_unlocked", **details)
|
|
1349
|
-
audit("output_unlocked", details, default_path=self._audit_default())
|
|
1449
|
+
if self.fd is not None:
|
|
1450
|
+
self._record_takeover(holder)
|
|
1350
1451
|
return self
|
|
1452
|
+
name = os.path.basename(self.path)[1:-len(".ffskill-lock")]
|
|
1453
|
+
if holder.get("why") == "takeover_unavailable":
|
|
1454
|
+
msg = (f"the lock on {name!r} ({self.path}) is stale ({holder.get('stale_why')}), but it cannot be taken over "
|
|
1455
|
+
f"safely here: this filesystem gives no OS file lock ({holder.get('takeover_error')})")
|
|
1456
|
+
hint = f"if no other run is writing {name!r}, delete {self.path} and run again"
|
|
1457
|
+
else:
|
|
1458
|
+
msg = (f"another run is writing {name!r} right now (lock {self.path}"
|
|
1459
|
+
+ (f", pid {holder['pid']}" if holder.get("pid") else "") + "); wait for it or choose a different --output/-o path")
|
|
1460
|
+
hint = ("wait for that run, or choose another -o path; if pid "
|
|
1461
|
+
f"{holder.get('pid')} is not an ffmpeg-skill run, delete {self.path}")
|
|
1462
|
+
details = dict(lock=self.path, output=os.path.abspath(self.output), **{k: v for k, v in holder.items() if k != "stale"})
|
|
1463
|
+
if self.on_locked:
|
|
1464
|
+
self.on_locked(msg, hint, details)
|
|
1465
|
+
refuse("input", "output_locked", msg, hint=hint, **details)
|
|
1351
1466
|
return self
|
|
1352
1467
|
|
|
1468
|
+
def _unlocked(self, exc: OSError) -> "_OutputLock":
|
|
1469
|
+
# an unlockable location (a read-only directory surfaces elsewhere): the run goes
|
|
1470
|
+
# on without a lock, and says so -- two runs could now both write this output
|
|
1471
|
+
details = {"lock": self.path, "output": os.path.abspath(self.output), "error": str(exc)}
|
|
1472
|
+
warn(f"could not create the output lock {self.path} ({exc}); running without one, so a second run "
|
|
1473
|
+
"writing the same output at the same time would not be refused", "output_unlocked", **details)
|
|
1474
|
+
audit("output_unlocked", details, default_path=self._audit_default())
|
|
1475
|
+
return self
|
|
1476
|
+
|
|
1477
|
+
def _take_over(self, judged: Dict[str, Any]) -> Dict[str, Any]:
|
|
1478
|
+
"""Take a lock judged stale over inside the takeover section (A186, #465; review of #530):
|
|
1479
|
+
judge it again there, remove it only if it is still stale, and create this run's own lock
|
|
1480
|
+
before leaving, so no other run can judge, remove or create between those steps. Sets
|
|
1481
|
+
self.fd and returns the holder as judged last when this run now holds the lock; otherwise
|
|
1482
|
+
returns why not (held, `takeover_in_progress`, `takeover_unavailable`, `takeover_error`).
|
|
1483
|
+
Returns {"unlocked": True} when the lock cannot be created at all (the run goes on
|
|
1484
|
+
without one, as __enter__ says)."""
|
|
1485
|
+
section = _TakeoverSection(self.path)
|
|
1486
|
+
try:
|
|
1487
|
+
busy = section.acquire()
|
|
1488
|
+
except OSError as exc:
|
|
1489
|
+
# no OS lock on this filesystem: nothing safe to fall back on, so the stale lock is
|
|
1490
|
+
# left for a person to remove (the refusal says so)
|
|
1491
|
+
return dict(judged, stale=False, why="takeover_unavailable", stale_why=judged.get("why"),
|
|
1492
|
+
takeover_error=str(exc))
|
|
1493
|
+
if busy:
|
|
1494
|
+
return dict(judged, stale=False, why="takeover_in_progress")
|
|
1495
|
+
try:
|
|
1496
|
+
now = judged
|
|
1497
|
+
if os.path.lexists(self.path):
|
|
1498
|
+
now = self._holder()
|
|
1499
|
+
if not now["stale"]:
|
|
1500
|
+
return now # taken over, or written again, since our look
|
|
1501
|
+
try:
|
|
1502
|
+
os.remove(self.path)
|
|
1503
|
+
except FileNotFoundError:
|
|
1504
|
+
pass
|
|
1505
|
+
except OSError as exc:
|
|
1506
|
+
# Windows will not delete a file another process holds open, and a live
|
|
1507
|
+
# holder keeps its lock open until it finishes: the takeover fails and
|
|
1508
|
+
# the run is refused, with the reason, instead of passing silently
|
|
1509
|
+
return dict(now, stale=False, takeover_error=str(exc))
|
|
1510
|
+
try:
|
|
1511
|
+
self.fd = self._create()
|
|
1512
|
+
except FileExistsError:
|
|
1513
|
+
return self._holder() # a run that found no lock created one: it holds it
|
|
1514
|
+
except OSError as exc:
|
|
1515
|
+
self._unlocked(exc)
|
|
1516
|
+
return {"unlocked": True}
|
|
1517
|
+
return now
|
|
1518
|
+
finally:
|
|
1519
|
+
section.release()
|
|
1520
|
+
|
|
1353
1521
|
def _audit_default(self) -> Optional[str]:
|
|
1354
1522
|
if not self.path:
|
|
1355
1523
|
return None
|
|
1356
1524
|
d, base = os.path.split(os.path.abspath(self.output))
|
|
1357
1525
|
return os.path.join(d, f".{base}.ffskill-audit.jsonl")
|
|
1358
1526
|
|
|
1359
|
-
def
|
|
1527
|
+
def _create(self) -> int:
|
|
1528
|
+
"""Create the lock, already holding this run's pid and dated with its deadline, and
|
|
1529
|
+
return an open descriptor on it; FileExistsError when another run's lock is there.
|
|
1530
|
+
|
|
1531
|
+
The pid and the date are written to a file of this run's own first, which is then
|
|
1532
|
+
hard-linked to the lock's name: the link is atomic and fails when the name exists, as
|
|
1533
|
+
O_EXCL does, so no other run ever reads this lock empty (an empty lock reads as
|
|
1534
|
+
`unreadable`, which is stale, and was open to a takeover between the create and the
|
|
1535
|
+
write) or undated. Where a hard link cannot be made (a FAT volume, some network shares)
|
|
1536
|
+
the lock is created with O_EXCL and written at once, as before."""
|
|
1537
|
+
import uuid
|
|
1538
|
+
tmp = f"{self.path}.{os.getpid()}-{uuid.uuid4().hex[:12]}.new"
|
|
1539
|
+
try:
|
|
1540
|
+
fd = os.open(tmp, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o644)
|
|
1541
|
+
except OSError:
|
|
1542
|
+
return self._create_in_place()
|
|
1543
|
+
try:
|
|
1544
|
+
try:
|
|
1545
|
+
os.write(fd, str(os.getpid()).encode())
|
|
1546
|
+
finally:
|
|
1547
|
+
os.close(fd) # closed before the link: Windows links and removes only a closed file
|
|
1548
|
+
self._set_deadline(tmp)
|
|
1549
|
+
try:
|
|
1550
|
+
os.link(tmp, self.path)
|
|
1551
|
+
except FileExistsError:
|
|
1552
|
+
raise
|
|
1553
|
+
except (OSError, NotImplementedError, AttributeError):
|
|
1554
|
+
return self._create_in_place()
|
|
1555
|
+
return os.open(self.path, os.O_WRONLY)
|
|
1556
|
+
finally:
|
|
1557
|
+
try:
|
|
1558
|
+
os.remove(tmp)
|
|
1559
|
+
except OSError:
|
|
1560
|
+
pass
|
|
1561
|
+
|
|
1562
|
+
def _create_in_place(self) -> int:
|
|
1563
|
+
fd = os.open(self.path, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o644)
|
|
1564
|
+
os.write(fd, str(os.getpid()).encode())
|
|
1565
|
+
self._set_deadline(self.path)
|
|
1566
|
+
return fd
|
|
1567
|
+
|
|
1568
|
+
def _set_deadline(self, path: Optional[str] = None) -> None:
|
|
1360
1569
|
import time
|
|
1570
|
+
path = path or self.path
|
|
1361
1571
|
limit = child_limit()
|
|
1362
1572
|
deadline = time.time() + (limit if limit else LOCK_UNLIMITED)
|
|
1363
1573
|
try:
|
|
1364
|
-
os.utime(
|
|
1574
|
+
os.utime(path, (deadline - LOCK_HOUR, deadline - LOCK_HOUR))
|
|
1365
1575
|
except (OSError, OverflowError, ValueError) as exc:
|
|
1366
1576
|
warn(f"could not date the output lock {self.path} ({exc}); another run may take it over an hour from now",
|
|
1367
1577
|
"lock_deadline_unset", lock=self.path, error=str(exc))
|
|
1368
1578
|
|
|
1369
1579
|
def _holder(self) -> Dict[str, Any]:
|
|
1370
1580
|
"""Who holds the lock and whether it is stale: {pid, pid_alive, deadline (UTC, the
|
|
1371
|
-
mtime plus an hour), why: pid_dead | past_deadline | unreadable | held, stale}.
|
|
1581
|
+
mtime plus an hour), why: pid_dead | past_deadline | unreadable | held, stale}.
|
|
1582
|
+
_take_over() adds why `takeover_in_progress` (another run is in the takeover section) and
|
|
1583
|
+
`takeover_unavailable` (no OS file lock here)."""
|
|
1372
1584
|
import datetime
|
|
1373
1585
|
import time
|
|
1374
1586
|
out: Dict[str, Any] = {"pid": None, "pid_alive": None, "deadline": None, "why": "unreadable", "stale": True}
|
|
@@ -1401,16 +1613,46 @@ class _OutputLock:
|
|
|
1401
1613
|
record_fallback(f"took over the stale output lock {self.path}: {why}", "stale_lock_taken_over", **details)
|
|
1402
1614
|
|
|
1403
1615
|
def __exit__(self, *exc: Any) -> None:
|
|
1404
|
-
|
|
1405
|
-
|
|
1406
|
-
|
|
1407
|
-
|
|
1408
|
-
|
|
1409
|
-
|
|
1616
|
+
"""Remove this run's own lock, inside the takeover section: a holder past its deadline
|
|
1617
|
+
whose lock another run is taking over, or has taken over, never removes that run's lock
|
|
1618
|
+
by its name. On POSIX the lock at the name is removed only when it is still this run's
|
|
1619
|
+
file; on Windows no other run can remove a lock this run has open, so the name is still
|
|
1620
|
+
ours. When the section is busy (a takeover of this lock is under way) the lock is left to
|
|
1621
|
+
that run; when there is no OS lock here, the same file check alone decides."""
|
|
1622
|
+
fd, self.fd = self.fd, None
|
|
1623
|
+
if fd is None or not self.path:
|
|
1624
|
+
if fd is not None:
|
|
1625
|
+
os.close(fd)
|
|
1626
|
+
return
|
|
1627
|
+
section = _TakeoverSection(self.path)
|
|
1628
|
+
try:
|
|
1629
|
+
busy = section.acquire()
|
|
1630
|
+
except OSError:
|
|
1631
|
+
busy = None # no OS lock: the file check below alone
|
|
1632
|
+
try:
|
|
1633
|
+
if busy == "busy":
|
|
1634
|
+
return
|
|
1635
|
+
if os.name == "nt":
|
|
1636
|
+
os.close(fd)
|
|
1637
|
+
fd = None
|
|
1638
|
+
try:
|
|
1639
|
+
os.remove(self.path)
|
|
1640
|
+
except OSError:
|
|
1641
|
+
pass
|
|
1642
|
+
return
|
|
1410
1643
|
try:
|
|
1411
|
-
os.
|
|
1644
|
+
mine, there = os.fstat(fd), os.stat(self.path)
|
|
1645
|
+
if (mine.st_dev, mine.st_ino) == (there.st_dev, there.st_ino):
|
|
1646
|
+
os.remove(self.path)
|
|
1412
1647
|
except OSError:
|
|
1413
1648
|
pass
|
|
1649
|
+
finally:
|
|
1650
|
+
if fd is not None:
|
|
1651
|
+
try:
|
|
1652
|
+
os.close(fd)
|
|
1653
|
+
except OSError:
|
|
1654
|
+
pass
|
|
1655
|
+
section.release()
|
|
1414
1656
|
|
|
1415
1657
|
|
|
1416
1658
|
def _odd_dimension_retry(cmd: List[str], stderr: str) -> Optional[List[str]]:
|
|
@@ -1460,8 +1702,7 @@ def guard_sidecar(path: str, *inputs: str, siblings: Sequence[str] = ()) -> None
|
|
|
1460
1702
|
if not path:
|
|
1461
1703
|
return
|
|
1462
1704
|
for other in siblings:
|
|
1463
|
-
if other and (
|
|
1464
|
-
or (os.path.exists(other) and os.path.exists(path) and os.path.samefile(other, path))):
|
|
1705
|
+
if other and same_file(other, path):
|
|
1465
1706
|
refuse("input", "output_path_taken",
|
|
1466
1707
|
f"{path!r} is also another file this run writes ({other!r}); give each its own path",
|
|
1467
1708
|
output=path, other=other)
|
|
@@ -2029,6 +2270,7 @@ def run(cmd: Sequence[str], *, quiet: bool = False, check: bool = True, ctx: "Op
|
|
|
2029
2270
|
"""
|
|
2030
2271
|
ctx = ctx or STATE
|
|
2031
2272
|
is_ffmpeg = _is_ffmpeg(cmd)
|
|
2273
|
+
cmd = guard_still_patterns(cmd) # before it is checked, recorded or run
|
|
2032
2274
|
if not is_ffmpeg:
|
|
2033
2275
|
_refuse_url_media(cmd) # ffprobe reads a URL or a playlist as readily as ffmpeg does
|
|
2034
2276
|
if is_ffmpeg:
|
|
@@ -2194,6 +2436,7 @@ def run_analysis(cmd: Sequence[str], *, check: bool = True, text: bool = True, r
|
|
|
2194
2436
|
wall-clock limit as any other ffmpeg invocation and, with check=True, the same `kind: ffmpeg`
|
|
2195
2437
|
failure instead of an exit-0 "0 scenes found" over a file ffmpeg could not read. record=True
|
|
2196
2438
|
lists the command in the --json `commands` like run() does."""
|
|
2439
|
+
cmd = guard_still_patterns(cmd)
|
|
2197
2440
|
_refuse_url_media(cmd)
|
|
2198
2441
|
if record:
|
|
2199
2442
|
STATE.commands.append(_cmdline(cmd))
|
|
@@ -2241,30 +2484,125 @@ def child_limit(per_call: Optional[float] = None) -> Optional[float]:
|
|
|
2241
2484
|
return limit * 4 + 60
|
|
2242
2485
|
|
|
2243
2486
|
|
|
2487
|
+
# How long a sibling script past its outer ceiling gets to stop on SIGTERM before it is killed:
|
|
2488
|
+
# its handler waits up to 5 s for each ffmpeg it runs (stop_children()), then removes the partial
|
|
2489
|
+
# output and exits with its failure document.
|
|
2490
|
+
STOP_GRACE = 15.0
|
|
2491
|
+
|
|
2492
|
+
|
|
2493
|
+
def _taskkill() -> Optional[str]:
|
|
2494
|
+
"""Windows' own taskkill.exe, by its full path under %SystemRoot% (never a `taskkill` found on
|
|
2495
|
+
PATH), or None when it is not there."""
|
|
2496
|
+
root = os.environ.get("SystemRoot") or os.environ.get("windir") or r"C:\Windows"
|
|
2497
|
+
path = os.path.join(root, "System32", "taskkill.exe")
|
|
2498
|
+
return path if os.path.isfile(path) else None
|
|
2499
|
+
|
|
2500
|
+
|
|
2501
|
+
TREE_KILL_TIMEOUT = 30.0
|
|
2502
|
+
|
|
2503
|
+
|
|
2504
|
+
def terminate_tree(child: subprocess.Popen, windows: Optional[bool] = None) -> bool:
|
|
2505
|
+
"""Ask a sibling script to stop, so that its ffmpeg stops too (review of #530).
|
|
2506
|
+
|
|
2507
|
+
POSIX: SIGTERM to the script; its handler stops its ffmpeg (up to 5 s each), removes the
|
|
2508
|
+
partial output and releases the output lock, then exits. Returns True.
|
|
2509
|
+
|
|
2510
|
+
Windows: terminate() is TerminateProcess on the script alone, no handler runs, and its ffmpeg
|
|
2511
|
+
would keep encoding with the lock and the partial output in place. The whole tree is killed
|
|
2512
|
+
with taskkill /T /F and this returns once taskkill has (True when it says it did). Without
|
|
2513
|
+
taskkill, or when it fails or does not finish in TREE_KILL_TIMEOUT, the script alone is
|
|
2514
|
+
killed and this returns False: its ffmpeg may outlive it."""
|
|
2515
|
+
if windows is None:
|
|
2516
|
+
windows = os.name == "nt"
|
|
2517
|
+
if child.poll() is not None:
|
|
2518
|
+
return True
|
|
2519
|
+
if not windows:
|
|
2520
|
+
try:
|
|
2521
|
+
child.terminate()
|
|
2522
|
+
except OSError:
|
|
2523
|
+
pass
|
|
2524
|
+
return True
|
|
2525
|
+
exe = _taskkill()
|
|
2526
|
+
if exe:
|
|
2527
|
+
try:
|
|
2528
|
+
done = subprocess.run([exe, "/T", "/F", "/PID", str(child.pid)], stdout=subprocess.DEVNULL,
|
|
2529
|
+
stderr=subprocess.DEVNULL, timeout=TREE_KILL_TIMEOUT)
|
|
2530
|
+
if done.returncode == 0:
|
|
2531
|
+
return True
|
|
2532
|
+
except (OSError, subprocess.TimeoutExpired):
|
|
2533
|
+
pass
|
|
2534
|
+
if child.poll() is not None:
|
|
2535
|
+
return True # taskkill failed because the tree was already gone
|
|
2536
|
+
try:
|
|
2537
|
+
child.kill()
|
|
2538
|
+
except OSError:
|
|
2539
|
+
pass
|
|
2540
|
+
return False
|
|
2541
|
+
|
|
2542
|
+
|
|
2543
|
+
def _stop_child(child: subprocess.Popen, communicate: bool = True) -> "Tuple[bool, str]":
|
|
2544
|
+
"""Stop a sibling script with terminate_tree(): on POSIX SIGTERM, then SIGKILL if it has not
|
|
2545
|
+
exited within STOP_GRACE; on Windows its whole tree at once (taskkill /T /F). Returns
|
|
2546
|
+
(stopped, stderr): stopped is True when it exited on the SIGTERM, which a sibling script
|
|
2547
|
+
handles by removing its partial output and its output lock, and False when it had to be
|
|
2548
|
+
killed (always on Windows, where no handler runs). communicate=False waits without reading the
|
|
2549
|
+
pipes (stop_children(), whose caller is already reading them or exiting)."""
|
|
2550
|
+
def wait(timeout: Optional[float] = None) -> str:
|
|
2551
|
+
if communicate:
|
|
2552
|
+
return child.communicate(timeout=timeout)[1] or ""
|
|
2553
|
+
child.wait(timeout=timeout)
|
|
2554
|
+
return ""
|
|
2555
|
+
if os.name != "nt":
|
|
2556
|
+
try:
|
|
2557
|
+
terminate_tree(child, windows=False)
|
|
2558
|
+
return True, wait(STOP_GRACE)
|
|
2559
|
+
except subprocess.TimeoutExpired:
|
|
2560
|
+
pass
|
|
2561
|
+
except OSError:
|
|
2562
|
+
return False, wait()
|
|
2563
|
+
try:
|
|
2564
|
+
child.kill()
|
|
2565
|
+
except OSError:
|
|
2566
|
+
pass
|
|
2567
|
+
return False, wait()
|
|
2568
|
+
terminate_tree(child, windows=True)
|
|
2569
|
+
return False, wait()
|
|
2570
|
+
|
|
2571
|
+
|
|
2244
2572
|
def run_tool(argv: Sequence[str], *, per_call: Optional[float] = None,
|
|
2245
|
-
env: Optional[Dict[str, str]] = None
|
|
2573
|
+
env: Optional[Dict[str, str]] = None,
|
|
2574
|
+
on_start: "Optional[Callable[[subprocess.Popen], None]]" = None) -> subprocess.CompletedProcess:
|
|
2246
2575
|
"""Run a sibling script (`argv[0]` is the script path) under child_limit(). On overrun the
|
|
2247
|
-
child is
|
|
2576
|
+
child is stopped (_stop_child()) and a CompletedProcess is returned whose stdout is this skill's own failure
|
|
2248
2577
|
document (kind timeout, exit 124), so callers that parse the child's --json see a timeout
|
|
2249
|
-
exactly as they would from the child itself. `env` replaces the inherited environment.
|
|
2578
|
+
exactly as they would from the child itself. `env` replaces the inherited environment.
|
|
2579
|
+
`on_start(child)` is called once the child is running (the MCP server keeps it to stop a call
|
|
2580
|
+
the client cancels)."""
|
|
2250
2581
|
limit = child_limit(per_call)
|
|
2251
2582
|
child = subprocess.Popen([sys.executable] + list(argv), stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, encoding="utf-8", errors="replace", env=env)
|
|
2252
2583
|
_watch(child, []) # a sibling script removes its own partial output; there is none of ours to clean
|
|
2584
|
+
if on_start is not None:
|
|
2585
|
+
on_start(child)
|
|
2253
2586
|
try:
|
|
2254
2587
|
out, err = child.communicate(timeout=limit)
|
|
2255
2588
|
_unwatch(child)
|
|
2256
2589
|
return subprocess.CompletedProcess(child.args, child.returncode, out, err)
|
|
2257
|
-
except subprocess.TimeoutExpired
|
|
2258
|
-
child.
|
|
2259
|
-
|
|
2590
|
+
except subprocess.TimeoutExpired:
|
|
2591
|
+
# A183: the child is asked to stop first. A sibling script's SIGTERM handler stops its
|
|
2592
|
+
# ffmpeg, removes the partial output and releases its output lock before it exits; a
|
|
2593
|
+
# SIGKILL left the partial file under the final output name and a stale lock beside it,
|
|
2594
|
+
# under a document that said kind timeout. Killed only when it has not exited in
|
|
2595
|
+
# STOP_GRACE seconds (and at once on Windows, where terminate() is a kill).
|
|
2596
|
+
stopped_cleanly, rest = _stop_child(child)
|
|
2260
2597
|
_unwatch(child)
|
|
2261
2598
|
name = os.path.basename(str(argv[0]))
|
|
2262
|
-
msg = f"{name} did not finish within {limit:g} s (4x the per-ffmpeg --timeout plus 60 s) and was
|
|
2599
|
+
msg = (f"{name} did not finish within {limit:g} s (4x the per-ffmpeg --timeout plus 60 s) and was "
|
|
2600
|
+
+ ("stopped" if stopped_cleanly else "killed"))
|
|
2263
2601
|
doc = {"status": "failed", "exit_code": 124,
|
|
2264
2602
|
"error": {"kind": "timeout", "message": msg, "code": ERROR_CODE["timeout"], "retryable": ERROR_RETRYABLE},
|
|
2265
2603
|
"commands": []}
|
|
2266
|
-
|
|
2267
|
-
return subprocess.CompletedProcess(list(argv), 124, json.dumps(doc),
|
|
2604
|
+
# communicate() after a timeout returns everything the child wrote, the part read before too
|
|
2605
|
+
return subprocess.CompletedProcess(list(argv), 124, json.dumps(doc), rest + f"\nerror: {msg}\n")
|
|
2268
2606
|
|
|
2269
2607
|
|
|
2270
2608
|
def option_args(flag: str, value: Any) -> List[str]:
|
|
@@ -2304,7 +2642,7 @@ def _has_non_av_streams(cmd: Sequence[str]) -> bool:
|
|
|
2304
2642
|
if not src or not os.path.isfile(src):
|
|
2305
2643
|
return True
|
|
2306
2644
|
try:
|
|
2307
|
-
out = subprocess.run([found_exe("ffprobe"), "-v", "error", "-show_entries", "stream=codec_type", "-of", "csv=p=0", src],
|
|
2645
|
+
out = subprocess.run([found_exe("ffprobe"), "-v", "error", "-show_entries", "stream=codec_type", "-of", "csv=p=0", *still_input_args(src), src],
|
|
2308
2646
|
stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True, encoding="utf-8", errors="replace",
|
|
2309
2647
|
timeout=_probe_limit()).stdout
|
|
2310
2648
|
except (OSError, subprocess.TimeoutExpired):
|
|
@@ -2345,7 +2683,7 @@ def carried_stream_maps(src: Optional[str], output: str, *, chapters: bool = Tru
|
|
|
2345
2683
|
return []
|
|
2346
2684
|
try:
|
|
2347
2685
|
out = subprocess.run([found_exe("ffprobe"), "-v", "error", "-show_entries",
|
|
2348
|
-
"stream=index,codec_type,codec_tag_string:chapter=id", "-of", "json", src],
|
|
2686
|
+
"stream=index,codec_type,codec_tag_string:chapter=id", "-of", "json", *still_input_args(src), src],
|
|
2349
2687
|
stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True, encoding="utf-8",
|
|
2350
2688
|
errors="replace", timeout=_probe_limit()).stdout
|
|
2351
2689
|
doc = json.loads(out or "{}")
|
|
@@ -2686,6 +3024,40 @@ def still_input_args(path: str) -> List[str]:
|
|
|
2686
3024
|
return ["-pattern_type", "none"] if "%" in p and os.path.splitext(p)[1].lower() in _IMAGE2_EXT else []
|
|
2687
3025
|
|
|
2688
3026
|
|
|
3027
|
+
def guard_still_patterns(cmd: Sequence[str]) -> List[str]:
|
|
3028
|
+
"""`cmd` with still_input_args() in front of every input that needs it and has neither its
|
|
3029
|
+
own `-pattern_type` nor a forced `-f`: each `-i` of an ffmpeg command, the last argument of an
|
|
3030
|
+
ffprobe one. run() and run_analysis() apply it to every command (A7, #465): the emoji overlay
|
|
3031
|
+
PNGs, grid.py's and broll.py's stills and join.py's had no guard, so `%d` in such a path made
|
|
3032
|
+
image2 open a different file. Every other command is returned as it is."""
|
|
3033
|
+
cmd = [str(a) for a in cmd]
|
|
3034
|
+
base = os.path.basename(cmd[0]) if cmd else ""
|
|
3035
|
+
if not any("%" in a for a in cmd[1:]):
|
|
3036
|
+
return cmd
|
|
3037
|
+
if base.startswith("ffprobe"):
|
|
3038
|
+
if still_input_args(cmd[-1]) and "-pattern_type" not in cmd and "-f" not in cmd[:-1]:
|
|
3039
|
+
return cmd[:-1] + still_input_args(cmd[-1]) + cmd[-1:]
|
|
3040
|
+
return cmd
|
|
3041
|
+
if not base.startswith("ffmpeg"):
|
|
3042
|
+
return cmd
|
|
3043
|
+
out: List[str] = []
|
|
3044
|
+
since = 0 # where the current input's options start in `out`
|
|
3045
|
+
i = 0
|
|
3046
|
+
while i < len(cmd):
|
|
3047
|
+
a = cmd[i]
|
|
3048
|
+
if a == "-i" and i + 1 < len(cmd):
|
|
3049
|
+
opts = out[since:]
|
|
3050
|
+
if still_input_args(cmd[i + 1]) and "-pattern_type" not in opts and "-f" not in opts:
|
|
3051
|
+
out += still_input_args(cmd[i + 1])
|
|
3052
|
+
out += [a, cmd[i + 1]]
|
|
3053
|
+
since = len(out)
|
|
3054
|
+
i += 2
|
|
3055
|
+
continue
|
|
3056
|
+
out.append(a)
|
|
3057
|
+
i += 1
|
|
3058
|
+
return out
|
|
3059
|
+
|
|
3060
|
+
|
|
2689
3061
|
def sibling_temp(dst: str, tag: str) -> str:
|
|
2690
3062
|
"""A hidden path next to `dst`, with its extension (the muxer is chosen by it), for a file
|
|
2691
3063
|
place_output(..., move=True) renames into place: same directory, so the rename never
|