ffmpeg-skill 2.23.0 → 2.24.0

package/scripts/_common/probe.py

@@ -827,3 +827,89 @@ def detect_scenes(path: str, threshold: float, min_len: float, duration: float,
     if duration - cuts[-1] < min_len and len(cuts) > 1:
         cuts.pop()
     return cuts
+
+
+# What a stream-copy join compares (cut.py --segments, join.py --copy).
+IN_BAND_CONFIG = ("mp3", "mp2")
+TS_EXTS = (".ts", ".m2ts", ".mts")
+_SIG_FIELDS = {"video": ("codec_name", "profile", "pix_fmt", "width", "height", "r_frame_rate", "time_base",
+                         "sample_aspect_ratio", "color_space", "color_range", "color_transfer", "color_primaries"),
+               "audio": ("codec_name", "profile", "sample_rate", "channels", "time_base")}
+
+
+def stream_signature(doc: Dict[str, Any]) -> "List[Dict[str, Any]]":
+    """What a stream-copy join needs to be identical across parts, from one ffprobe document
+    (-show_streams with -show_data_hash sha256): each stream's type, codec parameters, rotation
+    and extradata hash. Shared by cut.py's --segments join and join.py --copy."""
+    sig = []
+    for st in doc.get("streams") or []:
+        kind = st.get("codec_type")
+        entry = {"type": kind, "extradata_hash": st.get("extradata_hash")}
+        entry.update({f: st.get(f) for f in _SIG_FIELDS.get(kind, ("codec_name",))})
+        if kind == "video":
+            entry["rotation"] = next((sd.get("rotation") for sd in st.get("side_data_list") or []
+                                      if "rotation" in sd), None)
+        sig.append(entry)
+    return sig
+
+
+def signatures_match(sigs: list, ext: str) -> bool:
+    """True when every part can be joined by stream copy: same streams in the same order with the
+    same parameters. A missing extradata hash only matches another missing one where the codec (or
+    an MPEG-TS output) carries its configuration in-band -- PCM has none at all."""
+    if not sigs or any(sig is None for sig in sigs):
+        return False
+    first = sigs[0]
+    for sig in sigs[1:]:
+        if len(sig) != len(first):
+            return False
+        for a, b in zip(first, sig):
+            if {k: v for k, v in a.items() if k != "extradata_hash"} != {k: v for k, v in b.items() if k != "extradata_hash"}:
+                return False
+            ha, hb = a.get("extradata_hash"), b.get("extradata_hash")
+            if ha or hb:
+                if ha != hb:
+                    return False
+            else:
+                codec = str(a.get("codec_name") or "")
+                if not (codec.startswith("pcm_") or codec in IN_BAND_CONFIG or ext in TS_EXTS):
+                    return False
+    return True
+
+
+# check.py --compare (2.24): the output's picture against a reference, frame by frame.
+COMPARE_SSIM_WARN = 0.98   # mean SSIM below this: a visible difference somewhere is likely
+_SSIM_SUMMARY = re.compile(r"SSIM Y:[0-9.]+ .*All:([0-9.]+)")
+_PSNR_SUMMARY = re.compile(r"PSNR y:\S+ .*average:(\S+)")
+_SSIM_FRAME = re.compile(r"n:(\d+) .*All:([0-9.]+)")
+
+
+def compare_pictures(path: str, reference: str) -> "Optional[Dict[str, Any]]":
+    """SSIM (mean and the worst frame) and PSNR (mean) of `path`'s first video stream against
+    `reference`'s, frame by frame in decode order, in one pass. ffmpeg's ssim and psnr filters
+    need equal sizes; the caller checks that first. None when the pass fails."""
+    import tempfile
+    from _common.decision import escape_filter_path
+    with tempfile.TemporaryDirectory(prefix="ffskill_compare_") as tmp:
+        stats = os.path.join(tmp, "ssim.log")
+        # frames paired by index (settb/setpts=N): ssim and psnr otherwise pair by timestamp, so a
+        # shifted or irregular timestamp would meet the wrong reference frame
+        graph = ("[0:V:0]settb=1/1000,setpts=N,split[a0][a1];[1:V:0]settb=1/1000,setpts=N,split[b0][b1];"
+                 f"[a0][b0]ssim=stats_file={escape_filter_path(stats, bind=False)};[a1][b1]psnr")
+        proc = run_analysis([require_tool("ffmpeg"), "-hide_banner", "-nostdin", "-i", path, "-i", reference,
+                             "-filter_complex", graph, "-f", "null", "-"], check=False, record=True)
+        if proc.returncode != 0:
+            return None
+        ssim, psnr = _SSIM_SUMMARY.search(proc.stderr), _PSNR_SUMMARY.search(proc.stderr)
+        frames = []
+        try:
+            with open(stats, encoding="utf-8") as fh:
+                frames = [(int(m.group(1)), float(m.group(2))) for m in map(_SSIM_FRAME.search, fh) if m]
+        except OSError:
+            pass
+    if not ssim or not frames:
+        return None
+    worst = min(frames, key=lambda f: f[1])
+    mean_psnr = psnr.group(1) if psnr else None
+    return {"ssim": float(ssim.group(1)), "worst_ssim": worst[1], "worst_frame": worst[0] - 1, "frames": len(frames),
+            "psnr": None if mean_psnr is None else (float("inf") if mean_psnr == "inf" else float(mean_psnr))}

package/scripts/_common/timeline.py

@@ -492,13 +492,42 @@ def parses(path: str, fmt: str) -> bool:
     return why is None
 
 
+def _xml_root_tag(data: bytes) -> str:
+    """The root element of a well-formed XML document, read with expat directly.
+
+    ElementTree expands entities declared in an internal DTD, and on Python 3.9/3.10 (both
+    supported) that is an entity-expansion ("billion laughs") risk on a file the tool did not
+    write. A timeline never declares an entity (to_fcpxml writes a bare `<!DOCTYPE fcpxml>`), so
+    a declaration is refused rather than expanded. Errors are ElementTree's ParseError, so a
+    failure reads the same as before."""
+    from xml.etree.ElementTree import ParseError
+    from xml.parsers import expat
+    parser = expat.ParserCreate()
+    parser.SetParamEntityParsing(expat.XML_PARAM_ENTITY_PARSING_NEVER)
+    root: List[str] = []
+
+    def start(name: str, _attrs: Any) -> None:
+        if not root:
+            root.append(name)
+
+    def entity(name: str, *_rest: Any) -> None:
+        raise ParseError(f"the document declares the entity {name!r}; a timeline never declares one")
+
+    parser.StartElementHandler = start
+    parser.EntityDeclHandler = entity
+    try:
+        parser.Parse(data, True)
+    except expat.ExpatError as exc:
+        raise ParseError(str(exc)) from None
+    return root[0]
+
+
 def parse_error(path: str, fmt: str) -> Optional[str]:
     """Why `path` does not read back as `fmt`, or None when it does (parses() without the record)."""
     try:
         text = Path(path).read_text(encoding="utf-8")
         if fmt == "fcpxml":
-            import xml.etree.ElementTree as ET
-            tag = ET.fromstring(text.encode("utf-8")).tag
+            tag = _xml_root_tag(text.encode("utf-8"))
             return None if tag == "fcpxml" else f"the root element is <{tag}>, not <fcpxml>"
         if fmt == "otio":
             doc = json.loads(text)

package/scripts/_contract.py

@@ -290,7 +290,7 @@ REENCODE_META: Dict[str, Dict[str, str]] = {
     "audio":     dict(video="never", audio="always", note="video stream is always -c:v copy when present; this tool's job is the audio"),
     "loudness":  dict(video="never", audio="always"),
     "silence":   dict(video="always", audio="always", note="removing gaps requires cutting on non-keyframe boundaries"),
-    "join":      dict(video="always", audio="always"),
+    "join":      dict(video="conditional", audio="conditional", note="--copy --transition none is -c copy of the first video and audio stream (clips with matching parameters only); every other join re-encodes both"),
     "color":     dict(video="conditional", audio="conditional", note="--strip-dovi and --retag (when the stream copy succeeds) are -c copy of both streams; --retag falls back to re-encoding only if the copy attempt fails or ffprobe still reads the old tags on the copy (a codec with no metadata bitstream filter here); --to-sdr / --lut / --correct always re-encode both"),
     "proxy":     dict(video="always", audio="conditional", note="video is always re-encoded at proxy-grade quality; audio is re-encoded when present, dropped entirely with --no-audio or when the source has none"),
     "export":    dict(video="conditional", audio="conditional", note="--preset copy is -c:v copy -c:a copy (no re-encode); every other preset re-encodes both"),
@@ -417,6 +417,7 @@ ERROR_REASONS = {
     "output_is_directory": "the output path is a directory; details: {output}",
     "media_path_is_playlist": "an input is a playlist (HLS .m3u8/.m3u or an ffconcat list, by name or by its first bytes) that ffmpeg would follow to the files it names; the skill takes media files only; details: {path}",
     "media_path_not_a_file": "an input exists but is not a regular file (a FIFO or a device), which ffmpeg would block on or read as a stream; details: {path}",
+    "join_copy_mismatch": "join.py --copy: a clip's streams do not match the first clip's, it hides pre-roll frames with an edit list, or its picture and sound end more than 50 ms apart; details: {clips: [{index?, path, differs}]}",
     "output_path_taken": "two files one run writes (a side file and the output, or two side files) resolve to the same path, so the second would replace the first; details: {output, other}",
     "output_looks_like_option": "the output starts with '-' (other than '-' for stdout), which ffmpeg would read as an option; ./name writes a file of that name; details: {output}",
     "media_path_is_url": "an input or output has a protocol prefix (tcp:, http://, file:, ...) that ffmpeg would open as a URL; the skill reads and writes local files only; details: {path, scheme}",
@@ -496,6 +497,8 @@ DIAGNOSTIC_CODES = {
     "colour_not_converted": ("warnings", "export.py: an HDR or wide-gamut source written by a preset that tags SDR BT.709 without converting it; {preset, hdr_format}"),
     "output_audio_silent": ("warnings", "export.py: the written audio is silent (-inf LUFS), so no gain meets the platform's loudness; {platform}"),
     "flash_pass_failed": ("warnings", "check.py --content: the flash pass failed, so the flashes row is not measured (WARN, could not measure); the other rows stand"),
+    "repair_incomplete": ("warnings", "export.py --repair: decode errors remain after the remux (a damaged frame needs a re-encode); {timestamp_warnings, decode_errors}"),
+    "size_over_limit": ("warnings", "export.py --max-mb: the written file is over the size asked for (the encoder overshot its bitrate); {size, limit}"),
     "loudness_outside_spec": ("warnings", "export.py: the written file's loudness is outside the platform's spec (normalized: after --normalize too); {platform, lufs, tp, normalized}"),
     "camera_not_started": ("fallbacks", "multicam.py: a range asked for a camera that had not started; camera 0 played it ({start, end, requested_camera, camera, starts_at}); `cuts` names the camera rendered"),
     "loudness_not_measured": ("warnings", "export.py: the written file's loudness could not be read, so the platform's loudness was not verified (loudness.ok null, a failing loudness verification step)"),
@@ -542,6 +545,7 @@ def output_schema(name: str, meta: Dict[str, Any], hw: bool = False) -> Dict[str
     base = {"status": {"enum": ["completed"]}, "output": {"type": ["string", "null"], "description": "path written, or null"},
             "dry_run": {"type": "boolean"}, "commands": {"type": "array", "items": {"type": "string"}, "description": "every ffmpeg command line planned or run"},
             "probe": {"type": "object", "description": "probe of the output when a file was written"},
+            "output_identity": {"type": "object", "description": "2.24: {size, mtime_ns} of the file this run wrote, absent under --dry-run; pass it to check.py --expect-identity SIZE:MTIME_NS so the check grades this file and not whatever later sits at the path"},
             "encoder": {"type": "string", "description": "the video encoder the last ffmpeg command used (after any GPU->CPU fallback; under --dry-run the planned one; render.py and waveform.py: their stages' commands included), `copy` for a stream copy; absent when no command encoded video"},
             "hw": {"type": "object", "description": "present when VideoToolbox was asked for (--hw, $FFMPEG_SKILL_HW, a render.py stage, a batch.py item), on export.py when $FFMPEG_SKILL_HW=1 is set but its delivery preset does not take it, and when this ffmpeg build has no libx264/libx265 and an encode ran on VideoToolbox instead: {requested: whether this run's encodes were asked onto VideoToolbox (false for that export, and for the build without libx264/libx265), source: flag|env|auto (auto: nobody asked, the build has no libx264/libx265 and this Apple Silicon machine ran VideoToolbox in its place), used: true|false whether the encoder that ran last is VideoToolbox, null under --dry-run (nothing ran) and when this process ran no encode itself (batch.py: its stages are child processes; its top-level hw gathers the items' requested/source/fallback/notes and each results row carries its item's encoder and hw), fallback: a job VideoToolbox refused (encoder open or compression session failure; nothing else is retried) was re-encoded on the CPU, notes: why an encode stayed on or fell back to the CPU, and, when $FFMPEG_SKILL_HW chose the GPU and it ran, what that costs and that --no-hw opts out}. A VideoToolbox encode also adds a top-level notes line: its quality is not CRF-equivalent. A kind: ffmpeg failure under --hw / $FFMPEG_SKILL_HW carries encoder and hw too"},
             "plan": {"type": "string", "description": "with --plan FILE: the plan document written (the run itself is a dry run)"},
@@ -552,8 +556,9 @@ def output_schema(name: str, meta: Dict[str, Any], hw: bool = False) -> Dict[str
     extra: Dict[str, Any] = {}
     if name == "check":
         extra = {"platform": {"type": "string"}, "ok": {"type": "boolean"}, "failed": {"type": "integer"}, "warnings": {"type": "integer"},
-                 "notes": {"type": "array", "items": {"type": "string"}, "description": "present when no --platform was named: youtube was assumed and judgement rows are WARN"},
-                 "checks": {"type": "array", "items": {"type": "object", "properties": {"check": {"type": "string", "description": "row name; --content adds black, frozen and silence; audio FAILs when the track is present but silent (peak <= -50 dBFS)"}, "status": {"enum": ["PASS", "WARN", "FAIL"]}, "value": {}, "expected": {}, "fix": {"type": "string"}, "kind": {"enum": ["format", "judgement"]}}}}}
+                 "input_identity": {"type": "object", "description": "2.24: {size, mtime_ns} of the file graded, the same shape as a writing tool's output_identity"},
+                 "notes": {"type": "array", "items": {"type": "string"}, "description": "present when no --platform was named (youtube was assumed and judgement rows are WARN), or when --compare paired a different number of frames than the reference has"},
+                 "checks": {"type": "array", "items": {"type": "object", "properties": {"check": {"type": "string", "description": "row name; --content adds black, frozen and silence; --compare adds similarity; --expect-identity adds identity; audio FAILs when the track is present but silent (peak <= -50 dBFS)"}, "status": {"enum": ["PASS", "WARN", "FAIL"]}, "value": {}, "expected": {}, "fix": {"type": "string"}, "kind": {"enum": ["format", "judgement"]}}}}}
     elif name == "caption":
         extra = {"transcription": {"type": "object", "description": "with --transcribe: {engine, model, language, routing (why auto chose it), detected_language when auto detected it}; when auto ran the English-only Parakeet model on English assumed, notes says so"},
                  "caption": {"type": "object", "description": "cue layout: shifted / wrapped / rebalanced / split / extended / dropped counts, plus wrap ('phrase' or 'measured'), phrase_breaks (1.16), broken_inside_word -- atoms hard-sliced at the column edge because they did not fit alone even at the size floor (1.18.4) -- and overlong -- now residual: a single character alone wider than the column. Burn mode: cues_burned -- non-blank cues that overlap [0, duration] and are drawn -- and cues_outside -- non-blank cues wholly outside it (2.2.6); a burn with no visible cue is refused, kind input. --ass: ass_malformed -- Dialogue lines that could not be read and are not counted -- and with --offset ass_shifted / ass_unshifted -- lines moved, and lines whose timestamps could not be read, copied as they were (each count above 0 is also a diagnostics.warnings entry)"},
@@ -613,7 +618,8 @@ def output_schema(name: str, meta: Dict[str, Any], hw: bool = False) -> Dict[str
     elif name == "report":
         extra = {"report": {"type": "string"}, "check": {"type": ["object", "null"], "description": "check.py's document for --platform; null when it was not asked for or could not run (a failing compliance verification step then says why)"}}
     elif name == "export":
-        extra = {"loudness": {"type": "object", "description": "platform presets with audio: the written file's lufs/tp against the platform's target_lufs/target_tp, ok true when inside the spec; normalized true when --normalize ran loudness.py on the file; silent true (lufs \"-inf\", ok false, no normalize) when the written audio is silent; measured false (lufs, tp and ok null, reason not_measured) when the measurement could not be read"},
+        extra = {"repair": {"type": "object", "description": "--repair (2.24): {before, after}, each {timestamp_warnings, decode_errors} counted over one full decode at -v warning"},
+                 "loudness": {"type": "object", "description": "platform presets with audio: the written file's lufs/tp against the platform's target_lufs/target_tp, ok true when inside the spec; normalized true when --normalize ran loudness.py on the file; silent true (lufs \"-inf\", ok false, no normalize) when the written audio is silent; measured false (lufs, tp and ok null, reason not_measured) when the measurement could not be read"},
                  "trimmed": {"type": "object", "description": "present when the source was longer than the preset's platform maximum and the export was cut to it (no --allow-long): {from, to} in seconds; notes says so too"},
                  "non_av_streams": {"type": "object", "description": "the source's subtitle and data streams: {kept: [{type: subtitle|data, index (n-th of its type), codec, language (subtitle), output_codec}], dropped: [{type, index, codec, language, reason: write_failed|gif|not_in_output}]}, read back from the written file (under --dry-run, as planned). A subtitle track is copied when the container holds it as it is, else converted to the container's text track (mov_text in .mp4/.mov, SubRip in .mkv); a trimmed export cuts it at the same point. Both lists are empty for a source with neither"},
                  "dropped_non_av_streams": {"type": "boolean", "description": "true when non_av_streams.dropped is not empty; a notes line says so too"},

package/scripts/batch.py

@@ -49,7 +49,7 @@ ALLOWED_STEP_SCRIPTS = {p.name for p in HERE.glob("*.py") if not p.name.startswi
 
 def file_key(path: Path) -> str:
     st = path.stat()
-    h = hashlib.sha1()
+    h = hashlib.sha1(usedforsecurity=False)  # a change detector, not a secret
     h.update(f"{path.name}|{st.st_size}|{int(st.st_mtime)}".encode())
     with open(path, "rb") as fh:  # first and last MB: cheap and good enough to detect changes
         h.update(fh.read(1 << 20))
@@ -75,7 +75,7 @@ def recipe_key(recipe: Dict[str, Any]) -> str:
             # content, so say so (each item that needs it then fails on reading it)
             warn(f"recipe.project {recipe['project']} could not be read ({exc}); the cache key covers the recipe only",
                  "recipe_project_unreadable", path=str(recipe["project"]), error=str(exc))
-    return hashlib.sha1((json.dumps(recipe, sort_keys=True) + "\0" + project_content).encode()).hexdigest()[:12]
+    return hashlib.sha1((json.dumps(recipe, sort_keys=True) + "\0" + project_content).encode(), usedforsecurity=False).hexdigest()[:12]
 
 
 def gpu_key() -> str:

package/scripts/check.py

@@ -38,7 +38,9 @@ from fractions import Fraction
 from typing import Any, Dict, List, Optional
 
 from _platforms import PLATFORMS, PLATFORM_CHOICES, TP_TOLERANCE_DB, spec_of, resolve as resolve_platform
-from _common.probe import CONTENT_FLAT_STD, SILENT_PEAK_DBFS, detect_content, detect_flashes, first_frame_stats, presentation_times, measured_level_dbfs
+from _common.probe import (COMPARE_SSIM_WARN, CONTENT_FLAT_STD, SILENT_PEAK_DBFS, compare_pictures, detect_content,
+                           detect_flashes, first_frame_stats, presentation_times, measured_level_dbfs)
+from _common.emit import file_identity
 from _common import STATE, add_common, apply_common, die, emit, info, probe, require_tool, run, run_analysis, dry_run_input_pending
 
 # The one delivery table (scripts/_platforms.py): check.py's rows, export.py's presets and the
@@ -71,20 +73,26 @@ FROZEN_FAIL = 0.95                         # a frozen span covering this share =
 SILENCE_WARN, SILENCE_FAIL = 0.50, 0.95    # share of the duration below SILENT_PEAK_DBFS
 
 
+STATE_NOTES: List[str] = []
+
+
 def flash_times(flashes: List[int], v: Dict[str, Any], meta: Dict[str, Any], pts: "Optional[List[float]]" = None) -> str:
     """The flashes row's first five times on look.py's timeline (seconds from the container
     start). A flash frame's own presentation time when `pts` has it (presentation_times(): exact,
     gaps in the timestamps included); otherwise index / average rate plus the video's lead over
     the container, marked "~" (approximate), and `frame N` when there is no frame rate either."""
+    return ", ".join(frame_time(n, v, meta, pts) for n in flashes[:5]) + (" ..." if len(flashes) > 5 else "")
+
+
+def frame_time(n: int, v: Dict[str, Any], meta: Dict[str, Any], pts: "Optional[List[float]]" = None) -> str:
+    """Frame `n` (decode order) on look.py's timeline, as flash_times() describes: exact from
+    `pts`, else "~" index / average rate plus the video's lead, else `frame N`."""
     origin = float(meta.get("start_time") or 0.0)
+    if pts and n < len(pts):
+        return f"{max(0.0, pts[n] - origin):.2f}s"
     fps = float(v.get("fps") or 0.0)
     lead = max(0.0, float(v.get("start_time") or 0.0) - origin)
-
-    def one(n: int) -> str:
-        if pts and n < len(pts):
-            return f"{max(0.0, pts[n] - origin):.2f}s"
-        return f"~{lead + n / fps:.2f}s" if fps else f"frame {n}"
-    return ", ".join(one(n) for n in flashes[:5]) + (" ..." if len(flashes) > 5 else "")
+    return f"~{lead + n / fps:.2f}s" if fps else f"frame {n}"
 
 
 def _span_total(spans: Any) -> float:
@@ -100,6 +108,7 @@ def aspect_name(w: int, h: int) -> str:
 
 
 def main() -> int:
+    STATE_NOTES.clear()  # module state: a second main() in one process starts without the first's notes
     ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
     ap.add_argument("input")
     ap.add_argument("--platform", choices=PLATFORM_CHOICES, default=None, help="delivery spec to check against (default: youtube, with judgement rows reported as WARN because no platform was named)")
@@ -111,6 +120,12 @@ def main() -> int:
     ap.add_argument("--no-loudness", action="store_true", help="skip the loudness measurement (faster)")
     ap.add_argument("--content", action="store_true",
                     help="also check the content (one decode pass, plus an audio-only and a video-only one): black, frozen, silence, dropouts, flashes and first frame rows")
+    ap.add_argument("--compare", metavar="REFERENCE",
+                    help="also measure the picture against REFERENCE, frame by frame (SSIM and PSNR, one pass): a "
+                         "similarity row, WARN when the mean SSIM is under 0.98, with the worst frame's time")
+    ap.add_argument("--expect-identity", metavar="SIZE:MTIME_NS",
+                    help="the output_identity a writing tool reported ({size}:{mtime_ns}): an identity row FAILs when "
+                         "the file at INPUT is not that file (a later run replaced it, or a failed run left an older one)")
     add_common(ap)
     args = ap.parse_args()
     apply_common(args)
@@ -132,6 +147,8 @@ def main() -> int:
     if args.max_mb is not None:
         spec["max_bytes"] = int(args.max_mb * 1024 * 1024)
 
+    # read before anything is measured and again after: a file replaced mid-check is not the one graded
+    identity_before = None if dry_run_input_pending(args.input) else file_identity(args.input)
     meta = probe(args.input)
     v, a = meta.get("video") or {}, meta.get("audio") or {}
     rows: List[Dict[str, Any]] = []
@@ -345,11 +362,65 @@ def main() -> int:
                 "again); a noise gate makes these on purpose" if gaps else "",
                 reason="a short stretch of digital silence in the middle of the sound is usually lost audio at a join or a glitched decode" if gaps else "")
 
+    if args.compare and not dry_run_input_pending(args.input) and not dry_run_input_pending(args.compare):
+        ref = probe(args.compare)
+        rv = ref.get("video") or {}
+        if not v or not rv:
+            row("similarity", "WARN", "not compared", "two pictures", "--compare needs a video stream in both files",
+                reason="one of the files has no picture")
+        elif (v.get("width"), v.get("height")) != (rv.get("width"), rv.get("height")):
+            row("similarity", "WARN", f"not compared: {v.get('width')}x{v.get('height')} vs {rv.get('width')}x{rv.get('height')}",
+                "the same frame size", "compare against a reference at the delivery size (fit.py or export.py it first)",
+                reason="SSIM and PSNR compare pixels, so the two frames must be the same size")
+        else:
+            cmp_ = compare_pictures(args.input, args.compare)
+            if cmp_ is None:
+                row("similarity", "WARN", "could not measure", f"mean SSIM >= {COMPARE_SSIM_WARN:g}",
+                    "look.py --compare at a few times", reason="the comparison pass failed")
+            else:
+                pts = presentation_times(args.input, float(v.get("start_time") or 0.0))
+                at = frame_time(cmp_["worst_frame"], v, meta, pts)
+                seek = "0" if at.startswith("frame") else at.lstrip("~").rstrip("s")
+                psnr = "identical" if cmp_["psnr"] == float("inf") else (f"{cmp_['psnr']:.1f} dB" if cmp_["psnr"] is not None else "?")
+                low = cmp_["ssim"] < COMPARE_SSIM_WARN
+                row("similarity", "WARN" if low else "PASS",
+                    f"SSIM {cmp_['ssim']:.4f} (worst {cmp_['worst_ssim']:.4f} at {at}), PSNR {psnr}, {cmp_['frames']} frames",
+                    f"mean SSIM >= {COMPARE_SSIM_WARN:g}",
+                    f"look.py --compare {args.compare} --at {seek}" if low else "",
+                    reason="the picture differs visibly from the reference somewhere" if low else "")
+                # frames counted from the packets when a container does not state nb_frames
+                nout = v.get("nb_frames") or len(pts) or None
+                nref = rv.get("nb_frames") or len(presentation_times(args.compare, float(rv.get("start_time") or 0.0))) or None
+                if nref and nout and int(nref) != int(nout):
+                    STATE_NOTES.append(f"--compare: {nout} frames against the reference's {nref}; frames were paired in "
+                                       f"order, so an added or dropped frame shifts every later pair")
+    # read after every measurement, --compare's included, so a replacement at any point shows
+    identity = None if dry_run_input_pending(args.input) else file_identity(args.input)
+    if identity is not None and identity != identity_before and not args.expect_identity:
+        STATE_NOTES.append("the file changed while it was being checked: input_identity is the file as it was "
+                           "at the end, and the rows may describe an earlier one; check it again")
+    if args.expect_identity:
+        try:
+            size, mtime_ns = (int(x) for x in args.expect_identity.split(":"))
+        except ValueError:
+            die(f"--expect-identity takes SIZE:MTIME_NS from a tool's output_identity, got {args.expect_identity!r}")
+        if identity is not None:
+            changed = identity != identity_before
+            same = identity == {"size": size, "mtime_ns": mtime_ns} and not changed
+            row("identity", "PASS" if same else "FAIL", f"{identity['size']}:{identity['mtime_ns']}", f"{size}:{mtime_ns}",
+                "" if same else "re-run the tool that writes this file and check what it reports, not what the path holds",
+                reason="" if same else ("the file changed while it was being checked" if changed
+                                        else "the file at this path is not the one the run being checked wrote"))
+        elif not dry_run_input_pending(args.input):
+            row("identity", "FAIL", "no regular file at this path", f"{size}:{mtime_ns}",
+                "re-run the tool that writes this file and check what it reports, not what the path holds",
+                reason="the file at this path is not the one the run being checked wrote")
     failed = [r for r in rows if r["status"] == "FAIL"]
     warned = [r for r in rows if r["status"] == "WARN"]
     notes: List[str] = []
     if not named:
         notes.append("no --platform given: youtube's spec was assumed, so judgement rows (duration, aspect, fps, resolution, loudness, true peak) are WARN, not FAIL; name a platform to enforce them")
+    notes += STATE_NOTES
     if not args.json:
         width = max(len(r["check"]) for r in rows)
         print(f"{args.input} — {args.platform}" + ("" if named else " (assumed)"))
@@ -366,6 +437,8 @@ def main() -> int:
         for n in notes:
             print(f"  note: {n}")
     extra: Dict[str, Any] = {"notes": notes} if notes else {}
+    if identity is not None:
+        extra["input_identity"] = identity
     if failed:
         die(f"{len(failed)} of {len(rows)} {args.platform} checks failed: {', '.join(r['check'] for r in failed)}",
             kind="verification", output=None, dry_run=STATE.dry_run,

package/scripts/color.py

@@ -26,7 +26,9 @@ from typing import List, Optional
 from _common import STATE, add_common, analyze_levels, apply_common, emit, aac_args, cfr_args, default_output, die, escape_filter_path, ffmpeg_base, ffmpeg_bsfs, info, probe, require_tool, run, run_analysis, run_keeping_subtitles, x264_args, X264_PRESETS, fmt_secs
 from _common.emit import warn
 
-TONEMAPS = ["hable", "mobius", "reinhard", "bt2390", "clip", "linear", "gamma"]
+# the curves of FFmpeg's CPU `tonemap` filter (every version from 5.1 to 9). bt2390 was listed
+# until 2.24 but exists only in tonemap_opencl/libplacebo, so choosing it always failed.
+TONEMAPS = ["hable", "mobius", "reinhard", "clip", "linear", "gamma"]
 
 # Typed primary correction: each flag is one option of one real, always-available libavfilter filter
 # (never a caller-supplied filter string). Range is this script's own safe subset of what the filter
@@ -164,7 +166,7 @@ def hdr_to_sdr_chain(meta: dict, tonemap: str, peak: float, desat: float) -> str
         f"zscale=tin={trc}:pin={prim}:min={space}:rin={v.get('color_range') or 'tv'}:t=linear:npl={peak:g}",
         "format=gbrpf32le",
         "zscale=p=bt709",
-        f"tonemap=tonemap={tonemap}:desat={desat:g}" + (":peak=%g" % (peak / 100.0) if tonemap in ("bt2390",) else ""),
+        f"tonemap=tonemap={tonemap}:desat={desat:g}",
         "zscale=t=bt709:m=bt709:r=tv",
         "format=yuv420p",
     ]

package/scripts/cut.py

@@ -57,6 +57,7 @@ from typing import List, Tuple
 
 from _common import (beat_grid, snap_points, decode_pcm_mono, rms_envelope, BEAT_MIN_CONFIDENCE)
 from _common import require_tool
+from _common.probe import TS_EXTS, signatures_match, stream_signature
 from _common import sibling_temp, source_codec_video_args, STATE, add_common, apply_common, audio_codec_for, emit, aac_args, cfr_args, default_output, die, ffmpeg_base, info, is_audio_output, time_arg, probe, run, X264_PRESETS, keyframes_near, MissingFpsError, concat_list_line, place_output, refuse_output_is_input, fmt_secs, measure_frame_timing, dry_run_input_pending
 
 # outputs whose re-encode dropped a subtitle/data stream (reported as dropped_non_av_streams)
@@ -73,7 +74,28 @@ def _t(value: str, fps, flag: str = "--segments") -> float:
     return time_arg(value, flag, fps)
 
 
-def parse_segments(spec: str, fps=None) -> List[Tuple[float, float]]:
+def keep_ranges_after_removing(removed: List[Tuple[float, float]], total: float, fps=None) -> List[Tuple[float, float]]:
+    """--remove: what is left of [0, total] once `removed` is dropped (merged where they overlap,
+    clamped to the file). A leftover shorter than one frame (10 ms without a frame rate) is not a
+    piece anyone asked to keep. Refuses a removal that would leave nothing."""
+    least = 1.0 / fps if fps else 0.01
+    keep, cursor = [], 0.0
+    for s, e in sorted((max(0.0, s), min(total, e)) for s, e in removed):
+        if e <= cursor:
+            continue
+        if s - cursor >= least:
+            keep.append((cursor, s))
+        cursor = max(cursor, e)
+    if total - cursor >= least:
+        keep.append((cursor, total))
+    if not keep:
+        die(f"--remove drops the whole file ({total:.3f}s); nothing would be left to keep")
+    return keep
+
+
+def parse_segments(spec: str, fps=None, clamp: bool = False) -> List[Tuple[float, float]]:
+    """START-END,... as seconds. `clamp` (--remove) lets a range start before 0, since the file
+    clamps it; --segments refuses that."""
     segs = []
     for raw in spec.split(","):
         raw = raw.strip()
@@ -83,7 +105,7 @@ def parse_segments(spec: str, fps=None) -> List[Tuple[float, float]]:
             die(f"segment '{raw}' must look like START-END (e.g. 0:05-0:12)")
         a, b = raw.rsplit("-", 1)
         start, end = _t(a, fps), _t(b, fps)
-        if start < 0:
+        if start < 0 and not clamp:
             die(f"segment '{raw}': start must be >= 0")
         if end <= start:
             die(f"segment '{raw}': end must be after start")
@@ -927,13 +949,6 @@ DECLICK_S = 0.02
 # keyframe still decodes from the keyframe BEFORE it
 SEEK_MARGIN = 1.0
 # codecs whose configuration travels in the packets, so a part with no extradata is not missing any
-IN_BAND_CONFIG = ("mp3", "mp2")
-TS_EXTS = (".ts", ".m2ts", ".mts")
-_SIG_FIELDS = {"video": ("codec_name", "profile", "pix_fmt", "width", "height", "r_frame_rate", "time_base",
-                         "sample_aspect_ratio", "color_transfer", "color_primaries"),
-               "audio": ("codec_name", "profile", "sample_rate", "channels", "time_base")}
-
-
 _PARTS: dict = {}
 
 
@@ -952,21 +967,10 @@ def _part_probe(path: str):
 
 
 def join_signature(path: str):
-    """What a stream-copy join needs to be identical across parts: each stream's type, codec
-    parameters, rotation and extradata hash. None when ffprobe cannot read the part."""
+    """What a stream-copy join needs to be identical across parts (_common.probe.stream_signature).
+    None when ffprobe cannot read the part."""
     doc = _part_probe(path)
-    if doc is None:
-        return None
-    sig = []
-    for st in doc["streams"]:
-        kind = st.get("codec_type")
-        entry = {"type": kind, "extradata_hash": st.get("extradata_hash")}
-        entry.update({f: st.get(f) for f in _SIG_FIELDS.get(kind, ("codec_name",))})
-        if kind == "video":
-            entry["rotation"] = next((sd.get("rotation") for sd in st.get("side_data_list") or []
-                                      if "rotation" in sd), None)
-        sig.append(entry)
-    return sig
+    return None if doc is None else stream_signature(doc)
 
 
 # how far a measured join step may be from the predicted one: ffprobe prints the parts' times
@@ -995,30 +999,6 @@ def predicted_join_steps(parts) -> list:
     return [None if a is None or b is None else (a[1] - a[3]) + (b[2] - b[0]) for a, b in zip(spans, spans[1:])]
 
 
-def signatures_match(sigs: list, ext: str) -> bool:
-    """True when every part can be joined by stream copy: same streams in the same order with the
-    same parameters. A missing extradata hash only matches another missing one where the codec (or
-    an MPEG-TS output) carries its configuration in-band -- PCM has none at all."""
-    if not sigs or any(sig is None for sig in sigs):
-        return False
-    first = sigs[0]
-    for sig in sigs[1:]:
-        if len(sig) != len(first):
-            return False
-        for a, b in zip(first, sig):
-            if {k: v for k, v in a.items() if k != "extradata_hash"} != {k: v for k, v in b.items() if k != "extradata_hash"}:
-                return False
-            ha, hb = a.get("extradata_hash"), b.get("extradata_hash")
-            if ha or hb:
-                if ha != hb:
-                    return False
-            else:
-                codec = str(a.get("codec_name") or "")
-                if not (codec.startswith("pcm_") or codec in IN_BAND_CONFIG or ext in TS_EXTS):
-                    return False
-    return True
-
-
 def _fraction_rate(text: str) -> float:
     """"30000/1001" or "30" as a number."""
     num, _, den = str(text).partition("/")
@@ -1316,10 +1296,13 @@ def main() -> int:
     ap.add_argument("input")
     ap.add_argument("-o", "--output", help="output file (default: <name>_cut.<ext>)")
     g = ap.add_argument_group("range (single segment)")
-    g.add_argument("--start", default="0", help="start time (seconds, mm:ss or hh:mm:ss.ms). default 0")
+    g.add_argument("--start", help="start time (seconds, mm:ss or hh:mm:ss.ms). default 0")
     g.add_argument("--end", help="end time")
     g.add_argument("--duration", help="duration instead of --end")
     ap.add_argument("--segments", help="comma separated START-END list, e.g. '0:05-0:12,1:00-1:20' (joined in order)")
+    ap.add_argument("--remove", metavar="START-END[,...]",
+                    help="the inverse of --segments: drop these ranges and keep the rest, joined in order "
+                         "(e.g. '1:20-2:05,10:00-10:30'); overlapping ranges merge")
     ap.add_argument("--accurate", action="store_true", help="always re-encode for frame-accurate (video) / sample-accurate (audio) cuts (default: lossless -c copy, re-encoding only when the keyframe snap exceeds --tolerance)")
     ap.add_argument("--edit-list", action="store_true", help="a single-segment .mp4/.mov stream copy keeps the MP4 edit list that hides the keyframe's pre-roll, so the picture starts at --start (default: -avoid_negative_ts make_zero, the picture starts at the keyframe); a player that ignores edit lists shows the pre-roll")
     ap.add_argument("--keep-hevc", action="store_true", help="a re-encode of an SDR HEVC source stays HEVC (x265 8-bit BT.709; VideoToolbox under --hw) instead of x264 (default: x264 for SDR, x265 Main10 for HDR); --codec overrides it")
@@ -1369,10 +1352,21 @@ def main() -> int:
         forced.append("codec")
 
     fps = (meta.get("video") or {}).get("fps")
+    if args.remove:
+        if args.segments or args.end or args.duration or args.start is not None:
+            die("--remove names what to drop from the whole file: it does not combine with --segments, --start, --end or --duration")
+        if args.snap == "beats":
+            # a snapped edge of what is kept could move into a range the caller asked to drop
+            die("--remove drops exactly the ranges given: --snap beats would move the kept edges, possibly into them; "
+                "snap the ranges first (scenes.py --beats) or cut with --segments --snap beats")
+        if not total:
+            die(f"--remove needs the input's duration, which {args.input} does not report"
+                + (" yet (it is an earlier step's output)" if dry_run_input_pending(args.input) else ""))
+        args.segments = ",".join(f"{s:.6f}-{e:.6f}" for s, e in keep_ranges_after_removing(parse_segments(args.remove, fps, clamp=True), total, fps))
     if args.segments:
         segments = parse_segments(args.segments, fps)
     else:
-        start = _t(args.start, fps, "--start")
+        start = _t(args.start or "0", fps, "--start")
         if start < 0:
             die(f"--start must not be negative, got {args.start!r}")
         if args.end and args.duration: