ffmpeg-skill 3.8.19 → 3.8.21

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/scripts/batch.py CHANGED
@@ -34,10 +34,10 @@ from pathlib import Path
34
34
  from typing import Any, Dict, List, Optional
35
35
 
36
36
  from _common import STATE, add_common, apply_common, child_args, die, emit, info, run_tool, MEDIA_EXT as _MEDIA_EXT
37
- from _common.runner import (HW_FORCED_ENV, OWNED_ENV, _OutputLock, add_hw_orchestrator_args, copy_to_sibling, env_hw, read_json,
37
+ from _common.runner import (with_flags, HW_FORCED_ENV, OWNED_ENV, _OutputLock, add_hw_orchestrator_args, copy_to_sibling, env_hw, read_json,
38
38
  owned_identity, stop_children)
39
- from _common.emit import absorb_stage_hw, audit, note, shown, steps_encoder_report, warn
40
- from _common.decision import NO_DEFAULT_ENCODE_EXT, rebase_project_paths
39
+ from _common.emit import absorb_stage_hw, audit, note, record_skip, shown, steps_encoder_report, warn
40
+ from _common.decision import NO_DEFAULT_ENCODE_EXT, brand_asset_files, rebase_project_paths
41
41
  from _common.runner import parse_json
42
42
 
43
43
  HERE = Path(__file__).resolve().parent
@@ -70,6 +70,40 @@ def file_key(path: Path) -> str:
70
70
  return h.hexdigest()
71
71
 
72
72
 
73
+ def side_input_file(value: str) -> Path:
74
+ """The file a side-input argument reads: the argument itself, or, when that is no file and it
75
+ ends in `:SUFFIX`, the part before the last colon when that is one (caption.py's
76
+ `--srt FILE:LANG`, which split_srt_lang() reads the same way). Keying the whole token hashed a
77
+ path that does not exist, so an edited subtitle file kept its key (#510 review)."""
78
+ path = Path(value)
79
+ try:
80
+ if path.is_file() or ":" not in value:
81
+ return path
82
+ head = Path(value.rpartition(":")[0])
83
+ return head if value.rpartition(":")[0] and head.is_file() else path
84
+ except OSError:
85
+ return path
86
+
87
+
88
+ def is_brand_value(step: List[Any], j: int) -> bool:
89
+ """Whether step[j] is the value of --brand (`--brand X` or `--brand=X`)."""
90
+ arg = str(step[j])
91
+ return arg.startswith("--brand=") or (j > 0 and str(step[j - 1]) == "--brand")
92
+
93
+
94
+ def key_brand_assets(h: Any, path: Path) -> None:
95
+ """Hash the logo and font file a brand file names into `h`: the brand file's own bytes do not
96
+ change when one of them is replaced under its name (#510 review; render.py's stage key and a
97
+ project recipe's key already cover them)."""
98
+ for key, asset in brand_asset_files(str(path)):
99
+ a = Path(asset)
100
+ try:
101
+ ident = file_key(a) if a.is_file() else "-"
102
+ except OSError:
103
+ ident = "-"
104
+ h.update(f"brand:{key}\0{asset}\0{ident}\0".encode())
105
+
106
+
73
107
  def side_inputs_key(recipe: Dict[str, Any]) -> str:
74
108
  """The files a recipe's steps name besides {in}/{out} (check.py --compare REF, a LUT, a logo),
75
109
  keyed by their own content: editing or removing one is a change the cache must see, or a QC
@@ -77,16 +111,18 @@ def side_inputs_key(recipe: Dict[str, Any]) -> str:
77
111
  Read on every pass, so a --watch loop sees the edit too."""
78
112
  h = hashlib.sha1(usedforsecurity=False)
79
113
  for step in recipe.get("steps") or []:
80
- for arg in step[1:] if isinstance(step, list) else []:
81
- if not isinstance(arg, str) or "{" in arg:
114
+ for j, arg in enumerate(step) if isinstance(step, list) else []:
115
+ if j == 0 or not isinstance(arg, str) or "{" in arg:
82
116
  continue
83
117
  value = arg.split("=", 1)[1] if arg.startswith("-") and "=" in arg else arg
84
118
  if value.startswith("-") or not value:
85
119
  continue
86
- path = Path(value)
120
+ path = side_input_file(value)
87
121
  try:
88
122
  if path.is_file():
89
123
  h.update(f"{value}\0{file_key(path)}\0".encode())
124
+ if is_brand_value(step, j):
125
+ key_brand_assets(h, path)
90
126
  continue
91
127
  except OSError:
92
128
  pass
@@ -94,6 +130,40 @@ def side_inputs_key(recipe: Dict[str, Any]) -> str:
94
130
  return h.hexdigest()[:8]
95
131
 
96
132
 
133
+ def item_side_inputs_key(recipe: Dict[str, Any], src: Path) -> "tuple[str, Optional[Dict[str, Any]]]":
134
+ """The side inputs a step names through a placeholder, for one item (#510): `--image {in}.png`
135
+ in the first step is a file beside this item's source, and editing it is a change the cache
136
+ must see as side_inputs_key() sees a literal path's. Returns (key part, None), "" when the
137
+ recipe names none, or ("", why) when one cannot be resolved before the item runs: a later
138
+ step's {in} is an intermediate that does not exist yet, so its file cannot be keyed, and the
139
+ item is not served from the cache at all."""
140
+ h = hashlib.sha1(usedforsecurity=False)
141
+ named = False
142
+ for i, step in enumerate(recipe.get("steps") or []):
143
+ for j, arg in enumerate(step) if isinstance(step, list) else []:
144
+ if j == 0 or not isinstance(arg, str) or "{in}" not in arg or "{out}" in arg:
145
+ continue # literal paths are side_inputs_key()'s; {out} names an output, not an input
146
+ value = arg.split("=", 1)[1] if arg.startswith("-") and "=" in arg else arg
147
+ if value == "{in}" or "{in}" not in value:
148
+ continue # the step's own input
149
+ if i > 0:
150
+ return "", {"reason": "side_input_unresolved", "step": i, "argument": arg,
151
+ "message": f"step {i} names {arg!r}, a file derived from an intermediate that does not "
152
+ "exist before the item runs, so the cache cannot key it; the item always runs"}
153
+ path = side_input_file(value.replace("{in}", str(src)))
154
+ named = True
155
+ try:
156
+ if path.is_file():
157
+ h.update(f"{path}\0{file_key(path)}\0".encode())
158
+ if is_brand_value(step, j):
159
+ key_brand_assets(h, path)
160
+ continue
161
+ except OSError:
162
+ pass
163
+ h.update(f"{path}\0-\0".encode())
164
+ return (h.hexdigest()[:8] if named else ""), None
165
+
166
+
97
167
  def caseless_glob(pattern: str) -> str:
98
168
  """`pattern` with every letter outside a [...] class matched in either case: "*.mp4" finds
99
169
  GOPR0001.MP4, as it already did on macOS's and Windows' default filesystems, so a recipe means
@@ -201,6 +271,8 @@ _LOG = threading.local()
201
271
  # left, or one another item delivered (two items naming one side file) is still refused.
202
272
  _OWN_OUTPUTS: Dict[str, Dict[str, Dict[str, int]]] = {}
203
273
  _OWN_LOCK = threading.Lock()
274
+ # source -> (final path, its owned_identity() before the last step started), while that step runs
275
+ _WATCHED: Dict[str, Any] = {}
204
276
  # set by batch.py's signal handler: a --jobs worker between two steps must not start the next one
205
277
  _STOP = threading.Event()
206
278
 
@@ -257,7 +329,8 @@ def run_step(argv: List[str], per_call: "Optional[float]" = None,
257
329
  if _STOP.is_set():
258
330
  return False, {"status": "failed", "exit_code": 130,
259
331
  "error": {"kind": "interrupted", "message": f"interrupted before {script} started"}}
260
- cmd = [str(HERE / script)] + argv[1:] + child_args() + (["--json"] if "--json" not in argv[1:] else [])
332
+ # before a bare `--` in the step, where argparse still reads them as options (#510 review)
333
+ cmd = [str(HERE / script)] + with_flags(argv[1:], child_args() + (["--json"] if "--json" not in argv[1:] else []))
261
334
  log(" → " + " ".join(os.path.basename(c) if i < 1 else c for i, c in enumerate(cmd)))
262
335
  env = dict(os.environ, **{OWNED_ENV: str(owned)}) if owned else None
263
336
  proc = run_tool(cmd, per_call=per_call, env=env)
@@ -354,6 +427,41 @@ def check_recipe_types(recipe: Dict[str, Any]) -> None:
354
427
  kind="input")
355
428
 
356
429
 
430
+ STEP_WRITES_WITHOUT_OUT = "step_writes_without_out"
431
+
432
+
433
+ def names_output(step: Any) -> bool:
434
+ """Whether a step names where it writes itself (-o/--output with a path that is not {out}): a
435
+ side product the caller placed on purpose (look.py {in} -o {in}.png), not an implicit output."""
436
+ for tok in (step[1:] if isinstance(step, list) else []):
437
+ tok = str(tok)
438
+ if tok in ("-o", "--output") or tok.startswith("--output=") or (tok.startswith("-o") and not tok.startswith("--") and len(tok) > 2):
439
+ return True
440
+ return False
441
+
442
+
443
+ def work_files(work: Path) -> "set":
444
+ """The names of the files directly in an item's work directory, batch's own excepted (the
445
+ owned*.json snapshots, the owned-report directory, hidden temporaries)."""
446
+ try:
447
+ return {p.name for p in work.iterdir()
448
+ if p.is_file() and not p.name.startswith(".") and not (p.name.startswith("owned") and p.suffix == ".json")}
449
+ except OSError:
450
+ return set()
451
+
452
+
453
+ def wrote_without_out(i: int, tool: str, path: str, src: Path) -> Dict[str, Any]:
454
+ """The error of a step that reports an output file while naming no {out}: the batch stops."""
455
+ written = not STATE.dry_run and os.path.isfile(path)
456
+ where = (f"this run wrote {path} for {src.name} before stopping and left it there; no later item ran"
457
+ if written else "nothing was written" + (" (dry run)" if STATE.dry_run else ""))
458
+ return {"step": i, "tool": tool, "kind": "input", "reason": STEP_WRITES_WITHOUT_OUT,
459
+ "message": (f"step {i} ({tool}) names no {{out}} but writes {path}, its own default output: the next "
460
+ f"step does not read it and the batch does not deliver it; {where}"),
461
+ "hint": f"give step {i} \"-o\", \"{{out}}\" so its file is the one the next step reads and the batch delivers",
462
+ "details": {"step": i, "tool": tool, "file": path, "written": written}}
463
+
464
+
357
465
  def step_writes(step: Any) -> bool:
358
466
  """Whether a recipe step writes a file: it names {out} (check.py and probe.py do not)."""
359
467
  return isinstance(step, list) and any("{out}" in str(a) for a in step[1:])
@@ -520,11 +628,21 @@ def process(src: Path, recipe: Dict[str, Any], outdir: Path, work: Path,
520
628
  proj["output"] = str(final.resolve())
521
629
  pj = work / "project.json"
522
630
  pj.write_text(json.dumps(proj, indent=2), encoding="utf-8")
523
- ok, _doc = run_step(["render.py", str(pj)], budget(), owned=owned(0))
631
+ owned_cfg = owned(0)
632
+ if not STATE.dry_run:
633
+ # the render writes the final path itself: a stop while it runs reports a file it left
634
+ # there as kept, as for a steps recipe's last step (#510 review)
635
+ with _OWN_LOCK:
636
+ _WATCHED[str(src)] = (str(final), owned_identity(str(final)))
637
+ ok, _doc = run_step(["render.py", str(pj)], budget(), owned=owned_cfg)
638
+ if ok:
639
+ with _OWN_LOCK:
640
+ _WATCHED.pop(str(src), None)
524
641
  step_docs = [("render", _doc)]
525
642
  failure = None if ok else step_error(0, "render.py", _doc)
526
643
  cut_reencoded: List[bool] = []
527
644
  cut_reasons: List[str] = []
645
+ cut_possible: List[List[str]] = []
528
646
  else:
529
647
  steps = recipe.get("steps") or []
530
648
  if not steps:
@@ -536,6 +654,7 @@ def process(src: Path, recipe: Dict[str, Any], outdir: Path, work: Path,
536
654
  cut_reencoded = []
537
655
  step_docs = []
538
656
  cut_reasons = []
657
+ cut_possible = []
539
658
  for i, step in enumerate(steps):
540
659
  last = i == len(steps) - 1
541
660
  # the item's directory is its own, so the name needs no stem (a long one overflowed it)
@@ -546,7 +665,17 @@ def process(src: Path, recipe: Dict[str, Any], outdir: Path, work: Path,
546
665
  argv = [str(a).replace("{in}", cur).replace("{out}", out) for a in step]
547
666
  if last and writes and track and is_own_output(src, final):
548
667
  log(f" {final.name} is this run's own earlier output, unchanged since: it may be replaced")
549
- step_ok, doc = run_step(argv, budget(), owned=owned(i))
668
+ owned_cfg = owned(i)
669
+ before = set() if last else work_files(work)
670
+ if last and writes and not STATE.dry_run:
671
+ # what is at the final path before the step that writes it: a stop while it runs
672
+ # reports a file the step left there as kept, not as nothing (#510)
673
+ with _OWN_LOCK:
674
+ _WATCHED[str(src)] = (str(final), owned_identity(str(final)))
675
+ step_ok, doc = run_step(argv, budget(), owned=owned_cfg)
676
+ if step_ok:
677
+ with _OWN_LOCK:
678
+ _WATCHED.pop(str(src), None) # a stopped step's stays for interrupted_row()
550
679
  step_docs.append((Path(str(argv[0])).stem if argv else "step", doc))
551
680
  # only cut.py's own doc carries `reencoded` -- true if ANY range this call cut (or its
552
681
  # --segments join) re-encoded, for the reasons in its `reencode_reason` (distinct per
@@ -555,10 +684,26 @@ def process(src: Path, recipe: Dict[str, Any], outdir: Path, work: Path,
555
684
  if step_ok and argv and argv[0] == "cut.py" and isinstance(doc, dict) and "reencoded" in doc:
556
685
  cut_reencoded.append(bool(doc["reencoded"]))
557
686
  cut_reasons.extend(r for r in (doc.get("reencode_reason") or []) if isinstance(r, str))
687
+ # a dry run's planned copy may still re-encode: what each call's plan says (#510 review)
688
+ cut_possible.append([r for r in (doc.get("possible_reencode_reason") or []) if isinstance(r, str)])
558
689
  if not step_ok:
559
690
  ok = False
560
691
  failure = step_error(i, str(argv[0]) if argv else None, doc)
561
692
  break
693
+ reported = doc.get("output") if isinstance(doc, dict) else None
694
+ if not writes and isinstance(reported, str) and reported and not names_output(step):
695
+ # #510: the step wrote (or under --dry-run would write) its own default output, which
696
+ # the next step does not read and the batch does not deliver: refused, and the batch stops
697
+ raise ItemError(wrote_without_out(i, str(argv[0]), os.path.abspath(reported), src))
698
+ if writes and not last:
699
+ lost = sorted(set(work_files(work)) - before - {os.path.basename(out)})
700
+ if lost:
701
+ # a side file the step wrote beside its intermediate (caption.py's .srt, an .ass):
702
+ # the item's work directory goes when the item ends, and it with it (#510)
703
+ record_skip(f"{src.name}: step {i} ({argv[0]}) also wrote {', '.join(lost)} beside its intermediate; "
704
+ "it is removed with the item's work directory, not delivered (only the last step's "
705
+ "side files land beside the output)",
706
+ "batch_side_file_not_delivered", file=str(src), step=i, tool=str(argv[0]), files=lost)
562
707
  if writes:
563
708
  cur = out
564
709
  if ok and cur != str(final) and not writes_nothing(recipe):
@@ -598,6 +743,8 @@ def process(src: Path, recipe: Dict[str, Any], outdir: Path, work: Path,
598
743
  if cut_reencoded:
599
744
  result["cut_reencoded"] = cut_reencoded
600
745
  result["cut_reencode_reasons"] = cut_reasons
746
+ if STATE.dry_run:
747
+ result["cut_possible_reencode_reason"] = cut_possible
601
748
  # the encoder each item really ran and its GPU facts (--hw / FFMPEG_SKILL_HW: on VideoToolbox,
602
749
  # fell back and why): this process encodes nothing itself, so its top-level hw.used is null
603
750
  result.update(steps_encoder_report(step_docs))
@@ -613,11 +760,17 @@ def output_identity(path: str) -> "Optional[Dict[str, int]]":
613
760
  return {"size": st.st_size, "mtime_ns": st.st_mtime_ns}
614
761
 
615
762
 
616
- def cached_output_changed(hit: Dict[str, Any]) -> "Optional[Dict[str, Any]]":
763
+ def cached_output_changed(hit: Dict[str, Any], recipe: "Optional[Dict[str, Any]]" = None) -> "Optional[Dict[str, Any]]":
617
764
  """None when the entry's output is still the file this batch wrote; else why it is not served:
618
- output_missing, output_changed (size or mtime differ: something replaced or edited it) or
765
+ output_missing, output_changed (size or mtime differ: something replaced or edited it),
619
766
  entry_unverifiable (an entry written by 2.10.11 or earlier recorded no identity, so nothing can say
620
- the file is the one it describes)."""
767
+ the file is the one it describes), or entry_predates_output_check: the recipe has a step that
768
+ names no {out}, and the entry was stored before such a step's own output was checked (#510), so
769
+ a step that writes its default output (metadata.py {in}) was cached as one that writes nothing."""
770
+ steps = (recipe or {}).get("steps") or []
771
+ if not (recipe or {}).get("project") and any(not step_writes(st) for st in steps) and not hit.get("output_checked"):
772
+ return {"reason": "entry_predates_output_check", "output": hit.get("output"),
773
+ "message": "the entry was stored before batch checked what a step naming no {out} writes"}
621
774
  if hit.get("written") is False or ("output" in hit and hit["output"] is None):
622
775
  return None # a recipe that writes nothing (check.py over a folder): no file to have changed
623
776
  out = str(hit.get("output") or "")
@@ -654,6 +807,7 @@ def main() -> int:
654
807
  started = time.time()
655
808
  with _OWN_LOCK:
656
809
  _OWN_OUTPUTS.clear()
810
+ _WATCHED.clear()
657
811
  folder = Path(args.folder).resolve() # relative 'bdir' used to become bdir/bdir/out once joined with the default outdir
658
812
  if not folder.is_dir():
659
813
  die(f"not a folder: {folder}")
@@ -759,6 +913,7 @@ def main() -> int:
759
913
  cache_lock = threading.Lock()
760
914
  timed_out = {"hit": False}
761
915
  interrupted: Dict[str, Any] = {"hit": False, "signal": None}
916
+ refused: Dict[str, Any] = {"hit": False} # a step wrote without naming {out}: no later item runs (#510)
762
917
  _STOP.clear()
763
918
  # Ctrl-C (and SIGTERM where it is delivered) stops the running steps and reports the table:
764
919
  # the shared handler would exit with "nothing was written" and no rows, and inside an item it
@@ -766,6 +921,11 @@ def main() -> int:
766
921
  import signal as _signal
767
922
 
768
923
  def _interrupt(signum: int, frame: Any) -> None:
924
+ if interrupted["signal"] is not None:
925
+ # a second signal while the first one's stop runs (Ctrl-C reaches the whole group, and a
926
+ # parent then sends SIGTERM): raising again broke into the stop or the table, and the
927
+ # exit code became the second signal's (#510 review; runner._on_signal() does the same)
928
+ return
769
929
  interrupted["signal"] = signum
770
930
  _STOP.set()
771
931
  stop_children()
@@ -862,8 +1022,19 @@ def main() -> int:
862
1022
  finally:
863
1023
  shutil.rmtree(iw, ignore_errors=True)
864
1024
 
865
- def store(key: str, r: Dict[str, Any]) -> None:
866
- if not (r["ok"] and not STATE.dry_run):
1025
+ def store(key: "Optional[str]", r: Dict[str, Any]) -> None:
1026
+ if not key or not (r["ok"] and not STATE.dry_run):
1027
+ return
1028
+ # the key was made before the item ran: a source or side input edited while it waited
1029
+ # or ran made an output the key does not describe, and a later restore of the old file
1030
+ # would have served it (#510 review). Made again here; not stored when it changed
1031
+ try:
1032
+ now = item_key(Path(r["file"]))
1033
+ except OSError:
1034
+ now = None
1035
+ if now != key:
1036
+ record_skip(f"{Path(r['file']).name}: its source or a side input changed while it was processed, so its "
1037
+ "output is not cached (the next run makes it again)", "batch_cache_inputs_changed", file=r["file"])
867
1038
  return
868
1039
  # The read-modify-write of the in-memory dict needs the lock even though the file
869
1040
  # write is already atomic: two finishers could otherwise serialise from two different
@@ -874,7 +1045,8 @@ def main() -> int:
874
1045
  for stale in [k for k, v in cache.items() if r["output"] and r.get("written", True) and k != key and isinstance(v, dict) and v.get("output") == r["output"]]:
875
1046
  del cache[stale]
876
1047
  # the output as this run left it: a later hit is served only while it is unchanged
877
- cache[key] = dict(r, output_identity=output_identity(r["output"]) if r["output"] and r.get("written", True) else None)
1048
+ cache[key] = dict(r, output_identity=output_identity(r["output"]) if r["output"] and r.get("written", True) else None,
1049
+ output_checked=True) # every step's own output was checked (#510): see cached_output_changed()
878
1050
  # write_text isn't atomic -- a process killed mid-write (or a --watch loop racing
879
1051
  # a concurrent manual run) could leave a truncated file that json.loads() above
880
1052
  # then silently treats as "no cache" (a ValueError -> {}), discarding every prior
@@ -895,6 +1067,16 @@ def main() -> int:
895
1067
  # read on every pass: a --watch loop serves nothing from an edited project.json, cue file or
896
1068
  # music bed (computed once before the loop, it did: A81)
897
1069
  pass_key = recipe_key(recipe) + gpu_key() + side_inputs_key(recipe)
1070
+
1071
+ def item_key(src: Path) -> "Optional[str]":
1072
+ """The cache key of one item as its files are now: file_key(src), the recipe, the GPU
1073
+ setting and the side inputs (literal and per item); None when one cannot be keyed.
1074
+ Recomputed by store(), so the side inputs are read again then (pass_key's included)."""
1075
+ per, why = item_side_inputs_key(recipe, src)
1076
+ if why:
1077
+ return None
1078
+ base = recipe_key(recipe) + gpu_key() + side_inputs_key(recipe)
1079
+ return f"{file_key(src)}:{base}" + (":" + per if per else "")
898
1080
  polled = settle_poll(files)
899
1081
  for i, src in enumerate(files):
900
1082
  if str(src) in polled:
@@ -903,12 +1085,21 @@ def main() -> int:
903
1085
  "seconds": 0.0, "skipped": "settling"}
904
1086
  continue
905
1087
  key = f"{file_key(src)}:{pass_key}"
1088
+ per_item, unkeyed = item_side_inputs_key(recipe, src)
1089
+ if unkeyed:
1090
+ # nothing can say the cached output still matches its side input: run it, unstored
1091
+ cache_miss[i] = unkeyed
1092
+ info(f"{src.name}: {unkeyed['message']}")
1093
+ pending.append((i, src, None))
1094
+ continue
1095
+ if per_item:
1096
+ key += ":" + per_item
906
1097
  hit = cache.get(key)
907
1098
  if isinstance(hit, dict) and not args.force:
908
- why = cached_output_changed(hit)
1099
+ why = cached_output_changed(hit, recipe)
909
1100
  if why is None:
910
1101
  info(f"skip (cached) {src.name}")
911
- slots[i] = {**{k: v for k, v in hit.items() if k != "output_identity"}, "cached": True}
1102
+ slots[i] = {**{k: v for k, v in hit.items() if k not in ("output_identity", "output_checked")}, "cached": True}
912
1103
  continue
913
1104
  cache_miss[i] = why
914
1105
  facts = {k: v for k, v in why.items() if k not in ("reason", "message")}
@@ -950,12 +1141,26 @@ def main() -> int:
950
1141
  row.update(ok=False, verified=False)
951
1142
  if started:
952
1143
  row["interrupted"] = True
1144
+ with _OWN_LOCK:
1145
+ watched = _WATCHED.pop(str(src), None)
1146
+ if watched and owned_identity(watched[0]) not in (None, watched[1]):
1147
+ # the stopped step had written the final file (export.py's --normalize pass keeps
1148
+ # its encode): it is there, unverified, without the step that was stopped (#510)
1149
+ row["kept"] = watched[0]
953
1150
  row.setdefault("error", {"step": None, "tool": None, "kind": "interrupted",
954
1151
  "message": "interrupted while this item was running; its running step was stopped"})
955
1152
  else:
956
1153
  row["skipped"] = "interrupted"
957
1154
  return row
958
1155
 
1156
+ def refused_row(src: Path) -> "Dict[str, Any]":
1157
+ """An item not started because an earlier one's step wrote a file without naming {out}."""
1158
+ return {"file": str(src), **row_output(src, recipe, outdir), "ok": False, "verified": False,
1159
+ "seconds": 0.0, "skipped": "recipe_refused"}
1160
+
1161
+ def is_refusal(r: "Optional[Dict[str, Any]]") -> bool:
1162
+ return bool(r) and ((r or {}).get("error") or {}).get("reason") == STEP_WRITES_WITHOUT_OUT
1163
+
959
1164
  def with_miss(i: int, r: Dict[str, Any]) -> Dict[str, Any]:
960
1165
  if i in cache_miss:
961
1166
  r["cache_miss"] = {k: v for k, v in cache_miss[i].items() if k != "message"}
@@ -980,6 +1185,11 @@ def main() -> int:
980
1185
  if isinstance(exc, SystemExit) and not exc.code:
981
1186
  raise
982
1187
  slots[i] = with_miss(i, failed_row(src, exc))
1188
+ if is_refusal(slots[i]):
1189
+ refused["hit"] = True
1190
+ for j, later, _k in pending[n + 1:]:
1191
+ slots[j] = refused_row(later)
1192
+ break
983
1193
  continue
984
1194
  slots[i] = with_miss(i, r)
985
1195
  store(key, r)
@@ -1011,7 +1221,7 @@ def main() -> int:
1011
1221
  # list up front would put every item past the deadline check before the first one
1012
1222
  # had finished, and the shared budget could then never stop anything.
1013
1223
  while queue or in_flight:
1014
- while queue and len(in_flight) < jobs:
1224
+ while queue and len(in_flight) < jobs and not refused["hit"]:
1015
1225
  if deadline is not None and time.monotonic() >= deadline:
1016
1226
  break
1017
1227
  i, src, key = queue.pop(0)
@@ -1030,8 +1240,10 @@ def main() -> int:
1030
1240
  except BaseException as exc: # noqa: BLE001 - reported, never swallowed
1031
1241
  slots[i] = failed_row(src, exc)
1032
1242
  lines[i] = [f"=== {src.name}", f" failed: {exc}"]
1243
+ if is_refusal(slots[i]):
1244
+ refused["hit"] = True # the items in flight finish; none is started after
1033
1245
  for i, src, key in queue:
1034
- slots[i] = timed_out_row(src)
1246
+ slots[i] = refused_row(src) if refused["hit"] else timed_out_row(src)
1035
1247
  except KeyboardInterrupt:
1036
1248
  # Spec 4.1: cancel what has not started, stop the running steps (the handler
1037
1249
  # already did), and REPORT every item -- exit 130 with the table, never a
@@ -1062,7 +1274,7 @@ def main() -> int:
1062
1274
  return [r for r in slots if r is not None]
1063
1275
 
1064
1276
  results = one_pass()
1065
- if args.watch:
1277
+ if args.watch and not refused["hit"]:
1066
1278
  info(f"watching {folder} every {args.watch:g}s (Ctrl-C to stop)")
1067
1279
  # the shared SIGINT handler (install_signal_handlers) exits 130 with "nothing was written",
1068
1280
  # which is wrong for a watch that already processed files: while idle between passes,
@@ -1076,7 +1288,7 @@ def main() -> int:
1076
1288
  finally:
1077
1289
  signal.signal(signal.SIGINT, previous)
1078
1290
  results = one_pass()
1079
- if interrupted["hit"]:
1291
+ if interrupted["hit"] or refused["hit"]:
1080
1292
  break # the pass reported its rows; a watch interrupted mid-pass stops here
1081
1293
  except KeyboardInterrupt:
1082
1294
  info("watch stopped")
@@ -1095,11 +1307,27 @@ def main() -> int:
1095
1307
  # finished, interrupted while running, or not started.
1096
1308
  stopped = sum(1 for r in results if r.get("interrupted"))
1097
1309
  not_started = sum(1 for r in results if r.get("skipped") == "interrupted")
1310
+ kept = [r["kept"] for r in results if r.get("kept")]
1098
1311
  die(f"interrupted after {done} of {len(results)} item(s): {stopped} stopped while running and "
1099
- f"{not_started} not started; the finished outputs are kept",
1312
+ f"{not_started} not started; the finished outputs are kept"
1313
+ + ((f", and {', '.join(kept)} " + ("was written by a step that was stopped and is" if len(kept) == 1 else
1314
+ "were written by steps that were stopped and are") + " kept, unverified")
1315
+ if kept else ""),
1100
1316
  code=128 + int(interrupted["signal"] or 2), kind="interrupted", output=None, dry_run=STATE.dry_run, results=results,
1101
1317
  processed=done, total=len(results), jobs=jobs, jobs_requested=requested,
1102
1318
  timed_out=timed_out["hit"])
1319
+ if refused["hit"]:
1320
+ rows = [r for r in results if ((r.get("error") or {}).get("reason")) == STEP_WRITES_WITHOUT_OUT]
1321
+ first = rows[0]["error"]
1322
+ files = [r["error"]["details"]["file"] for r in rows]
1323
+ written = [f for r, f in zip(rows, files) if r["error"]["details"].get("written")]
1324
+ not_run = sum(1 for r in results if r.get("skipped") == "recipe_refused")
1325
+ die(first["message"] + (f" ({len(rows)} items in flight wrote one each: {', '.join(written)})" if len(written) > 1 else "")
1326
+ + (f"; {not_run} item(s) not started" if not_run else ""),
1327
+ kind="input", hint=first.get("hint"), output=None, dry_run=STATE.dry_run, results=results,
1328
+ processed=done, total=len(results),
1329
+ error_extra={"reason": STEP_WRITES_WITHOUT_OUT,
1330
+ "details": dict(first["details"], files=files, written_files=written, not_started=not_run)})
1103
1331
  if timed_out["hit"]:
1104
1332
  skipped = [r["file"] for r in results if r.get("skipped") == "timeout"]
1105
1333
  die(f"the batch's {STATE.timeout:g} s budget ran out with {len(skipped)} item(s) not "
@@ -1140,8 +1368,15 @@ def main() -> int:
1140
1368
  for r in res.get("cut_reencode_reasons") or []:
1141
1369
  reasons[r] = reasons.get(r, 0) + 1
1142
1370
  why = (" (reencode_reason: " + ", ".join(f"{k} x{v}" for k, v in reasons.items()) + ")") if reasons else ""
1143
- info(f"cut.py: {copied}/{len(all_cuts)} call(s) stayed fully lossless stream-copy "
1144
- f"({cut_summary['stream_copy_rate']:.0%}), {len(all_cuts) - copied} re-encoded a range or its --segments join{why}")
1371
+ if STATE.dry_run:
1372
+ # a dry run's counts are cut.py's plans: a planned copy may still re-encode in a real run,
1373
+ # which measures where it lands (cut.py's possible_reencode_reason, #510)
1374
+ info(f"cut.py: {copied}/{len(all_cuts)} call(s) planned as a lossless stream copy "
1375
+ f"({cut_summary['stream_copy_rate']:.0%}), which a real run may still re-encode (each row's "
1376
+ f"cut_possible_reencode_reason); {len(all_cuts) - copied} planned to re-encode a range or its --segments join{why}")
1377
+ else:
1378
+ info(f"cut.py: {copied}/{len(all_cuts)} call(s) stayed fully lossless stream-copy "
1379
+ f"({cut_summary['stream_copy_rate']:.0%}), {len(all_cuts) - copied} re-encoded a range or its --segments join{why}")
1145
1380
  unverified = [shown(os.path.basename(r["file"])) for r in results if r.get("ok") and not r.get("verified")]
1146
1381
  if unverified and not STATE.dry_run:
1147
1382
  note(f"{len(unverified)} item(s) completed but did not verify everything they wrote (see each row's unverified): "
package/scripts/cut.py CHANGED
@@ -260,6 +260,38 @@ def _outcome(meta: dict, dst: str, reencoded: bool, reasons: List[str], **copy)
260
260
  EDIT_LIST_EXTS = (".mp4", ".mov", ".m4v")
261
261
 
262
262
 
263
+ def possible_reencode_reason(mode: str, outcomes: List[dict], segments: int) -> List[str]:
264
+ """Under --dry-run, the reencode_reason values a real run may still add to a plan of mode copy
265
+ or smart, which would then report hybrid or accurate (#510): the dry run reads no packets for a
266
+ single copy, so where its copy lands, and every check a real run makes on the written file,
267
+ are not known yet. [] for a real run and for a plan that already re-encodes."""
268
+ if not STATE.dry_run:
269
+ return []
270
+ if mode == "smart":
271
+ return ["smart_fallback"] # the checks run on the written file (smart.checks is null)
272
+ if mode != "copy":
273
+ return []
274
+ possible = ["copy_failed", "stream_lost"]
275
+ if any(o.get("length_unjudged") for o in outcomes):
276
+ possible.append("tolerance")
277
+ if segments > 1:
278
+ possible.append("concat_fallback")
279
+ return possible
280
+
281
+
282
+ def possible_reencode_note(mode: str, possible: List[str], tolerance: float) -> str:
283
+ """The plan's notes line for possible_reencode_reason."""
284
+ if mode == "smart":
285
+ return ("planned as a smart cut: a real run checks the written cut against the source and re-encodes "
286
+ "the whole cut instead (mode accurate, reencode_reason smart_fallback) when a check fails")
287
+ why = ("planned as a stream copy: a real run may re-encode it instead (mode hybrid, reencode_reason "
288
+ + ", ".join(possible) + ")")
289
+ if "tolerance" in possible:
290
+ why += (f"; where the copy lands is measured only from the written file, and a copy that lands "
291
+ f"{tolerance:.2f}s (--tolerance) or more from the requested cut is re-encoded")
292
+ return why
293
+
294
+
263
295
  def _with_notes(outcome: dict, notes: List[str]) -> dict:
264
296
  """An outcome with the notes of the attempt that led to it put first."""
265
297
  outcome["notes"] = list(notes) + list(outcome.get("notes") or [])
@@ -1013,10 +1045,15 @@ def cut_one(src: str, start: float, end: float, dst: str, reencode: bool, crf: i
1013
1045
  f"(> {tolerance:.2f}s tolerance); re-encoding this segment for accuracy{alt}")
1014
1046
  return _with_notes(cut_one(src, start, end, dst, True, crf, preset, tolerance, meta, reasons + ["tolerance"], edit_list_ok), notes)
1015
1047
  if reencode or audio_only:
1016
- return _outcome(meta, dst, reencode, reasons)
1048
+ # an audio-only copy is judged by its written length too (above): a dry run that wrote
1049
+ # nothing says a real run may still re-encode it (#510 review)
1050
+ pending = {"length_unjudged": True} if (STATE.dry_run and not reencode and copy_t is None and tolerance >= 0) else {}
1051
+ return _outcome(meta, dst, reencode, reasons, **pending)
1017
1052
  if STATE.dry_run:
1018
- # the planned copy: whether it keeps an edit list is known; where it lands is not
1019
- return _outcome(meta, dst, False, reasons, edit_list=edit_list and edit_list_ok, start_snapped=None)
1053
+ # the planned copy: whether it keeps an edit list is known; where it lands is not, so a
1054
+ # real run that judges the written copy by its length may still re-encode it (#510)
1055
+ return _outcome(meta, dst, False, reasons, edit_list=edit_list and edit_list_ok, start_snapped=None,
1056
+ length_unjudged=copy_t is None and tolerance >= 0)
1020
1057
  fps = (meta.get("video") or {}).get("fps")
1021
1058
  # a concat part's edit list does not hide its pre-roll in the join, so it presents from the keyframe
1022
1059
  shown = edit_list and edit_list_ok
@@ -1877,6 +1914,9 @@ def main() -> int:
1877
1914
  mode = ("smart" if smart_doc is not None and smart_doc["applied"] else
1878
1915
  "copy" if not reencoded else ("accurate" if args.accurate else "hybrid"))
1879
1916
  notes: List[str] = vfr_notes + join_notes + [n for o in outcomes for n in o.get("notes") or []]
1917
+ possible = possible_reencode_reason(mode, outcomes, len(segments))
1918
+ if possible:
1919
+ notes.append(possible_reencode_note(mode, possible, args.tolerance))
1880
1920
  single = outcomes[0] if len(outcomes) == 1 else {}
1881
1921
  stored = single.get("stored_preroll_seconds")
1882
1922
  if single.get("edit_list") and stored:
@@ -1930,7 +1970,7 @@ def main() -> int:
1930
1970
  requested_segments=[[round(s, 6), round(e, 6)] for s, e in requested] if len(requested) > 1 else None,
1931
1971
  requested_duration=round(expected, 6), output_duration=round(got, 6) if got is not None and not STATE.dry_run else None,
1932
1972
  duration_delta_seconds=round(error_ms / 1000, 6) if error_ms is not None else None,
1933
- mode=mode, keyframe_snapped=report["keyframe_snapped"], start_snapped=report["start_snapped"],
1973
+ mode=mode, possible_reencode_reason=possible, keyframe_snapped=report["keyframe_snapped"], start_snapped=report["start_snapped"],
1934
1974
  reencode_reason=reencode_reason, least_exact_precision=report["least_exact_precision"],
1935
1975
  segment_precision=report["segment_precision"],
1936
1976
  edit_list=bool(single.get("edit_list")), stored_preroll_seconds=stored, av_start_skew_seconds=skew,