@runeditrun/rer 0.2.1 → 0.2.4

package/scripts/publish-check/media.py

@@ -0,0 +1,283 @@
+"""Find the pixel and sample data inside an image or video file.
+
+A file is media only when its bytes walk as one complete container, from the
+signature to the end of the file, allowing only NUL padding after it. The path
+and extension never decide, so text renamed .gif, or a valid image with text
+appended, is not media. Each walker returns where its container ends and the
+(start, end) spans of its pixel or sample data (compressed samples and colour
+tables), or None. Everything outside those spans, such as comments, XMP and
+tags, is still text to inspect.
+"""
+
+
+def gif(data):
+    if data[:6] not in (b'GIF87a', b'GIF89a') or len(data) < 13:
+        return None
+    # Colour tables are pixel data too: palette bytes can spell an address.
+    offset = 13 + ((3 << ((data[10] & 7) + 1)) if data[10] & 0x80 else 0)
+    samples = [(13, offset)]
+    while offset < len(data):
+        kind = data[offset]
+        if kind == 0x3B:
+            return offset + 1, samples
+        if kind == 0x21:
+            offset += 2
+        elif kind == 0x2C:
+            if offset + 10 > len(data):
+                return None
+            flags = data[offset + 9]
+            # Descriptor, local colour table, then the LZW minimum code size.
+            table = offset + 10 + ((3 << ((flags & 7) + 1)) if flags & 0x80 else 0)
+            samples.append((offset + 10, table))
+            offset = table + 1
+        else:
+            return None
+        # Data sub-blocks: a length byte, that many bytes, ending at length zero.
+        start = offset
+        while True:
+            if offset >= len(data):
+                return None
+            size = data[offset]
+            offset += 1 + size
+            if not size:
+                break
+        if kind == 0x2C:
+            samples.append((start, offset))
+    return None
+
+
+def png(data):
+    if not data.startswith(b'\x89PNG\r\n\x1a\n'):
+        return None
+    offset = 8
+    samples = []
+    while offset + 12 <= len(data):
+        length = int.from_bytes(data[offset:offset + 4], 'big')
+        kind = data[offset + 4:offset + 8]
+        if kind in (b'PLTE', b'IDAT', b'fdAT'):
+            samples.append((offset + 8, offset + 8 + length))
+        offset += 12 + length
+        if kind == b'IEND':
+            return offset, samples
+    return None
+
+
+def jpeg(data):
+    # A phone stores more images after the first one's end (MPF depth and
+    # gain maps): each is walked, and their samples are pixels too.
+    walked = jpeg_image(data, 0)
+    if walked is None:
+        return None
+    end, samples = walked
+    while data.startswith(b'\xff\xd8', end) and (following := jpeg_image(data, end)):
+        end, more = following
+        samples += more
+    return end, samples
+
+
+def jpeg_image(data, offset):
+    if not data.startswith(b'\xff\xd8', offset):
+        return None
+    offset += 2
+    samples = []
+    while offset + 2 <= len(data):
+        if data[offset] != 0xFF:
+            return None
+        marker = data[offset + 1]
+        if marker == 0xFF:
+            offset += 1
+            continue
+        if marker == 0xD9:
+            return offset + 2, samples
+        if 0xD0 <= marker <= 0xD7 or marker == 0x01:
+            offset += 2
+            continue
+        if offset + 4 > len(data):
+            return None
+        offset += 2 + int.from_bytes(data[offset + 2:offset + 4], 'big')
+        if marker == 0xDA:
+            # Entropy-coded data runs to the next marker other than a stuffed
+            # zero or a restart marker.
+            start = offset
+            while True:
+                offset = data.find(b'\xff', offset)
+                if offset < 0 or offset + 1 >= len(data):
+                    return None
+                if data[offset + 1] == 0 or 0xD0 <= data[offset + 1] <= 0xD7:
+                    offset += 2
+                else:
+                    break
+            samples.append((start, offset))
+    return None
+
+
+WEBP_SAMPLES = (b'VP8 ', b'VP8L', b'ALPH', b'ANMF')
+
+
+def webp(data):
+    if len(data) < 20 or data[:4] != b'RIFF' or data[8:12] != b'WEBP':
+        return None
+    end = int.from_bytes(data[4:8], 'little') + 8
+    offset = 12
+    samples = []
+    while offset + 8 <= end:
+        size = int.from_bytes(data[offset + 4:offset + 8], 'little')
+        if data[offset:offset + 4] in WEBP_SAMPLES:
+            samples.append((offset + 8, offset + 8 + size))
+        offset += 8 + size + (size & 1)
+    return (end, samples) if offset == end else None
+
+
+# Major brands of MP4, QuickTime, 3GPP and HEIF/AVIF files.
+ISO_BRANDS = {b'isom', b'iso2', b'iso4', b'iso5', b'iso6', b'mp41', b'mp42', b'avc1', b'dash',
+              b'M4V ', b'M4A ', b'M4VH', b'M4VP', b'qt  ', b'3gp4', b'3gp5', b'3gp6', b'3g2a',
+              b'mif1', b'msf1', b'avif', b'avis', b'heic', b'heix', b'heim', b'heis', b'hevc', b'hevx'}
+
+
+def iso_media(data):
+    # MP4 and AVIF share ISO base media boxes, opened by a file-type box with
+    # an explicit size and a known major brand.
+    if len(data) < 16 or data[4:8] != b'ftyp' or data[8:12] not in ISO_BRANDS:
+        return None
+    if int.from_bytes(data[:4], 'big') < 16:
+        return None
+    offset = 0
+    kinds, samples = set(), []
+    while offset + 8 <= len(data):
+        size = int.from_bytes(data[offset:offset + 4], 'big')
+        kind = data[offset + 4:offset + 8]
+        header = 8
+        if size == 1:
+            if offset + 16 > len(data):
+                return None
+            size, header = int.from_bytes(data[offset + 8:offset + 16], 'big'), 16
+        elif size == 0:
+            # Only a last box after the file type may run to the end.
+            size = len(data) - offset
+        if size < header:
+            return None
+        kinds.add(kind)
+        if kind == b'mdat':
+            samples.append((offset + header, offset + size))
+        offset += size
+    # A movie has its moov box; a still image (HEIF/AVIF) has meta.
+    if offset != len(data) or not kinds & {b'moov', b'meta'}:
+        return None
+    return offset, samples
+
+
+def ebml_number(data, offset, marker):
+    # A variable-length EBML number; the marker bit stays in IDs, not sizes.
+    if offset >= len(data) or not data[offset]:
+        return None, offset
+    length = 9 - data[offset].bit_length()
+    if offset + length > len(data):
+        return None, offset
+    value = int.from_bytes(data[offset:offset + length], 'big')
+    if not marker:
+        value &= (1 << (7 * length)) - 1
+        if value == (1 << (7 * length)) - 1:
+            value = -1
+    return value, offset + length
+
+
+SEGMENT = 0x18538067
+CLUSTER = 0x1F43B675
+# Segment children: a Cluster of unknown size ends at the next one of these.
+SEGMENT_CHILDREN = {CLUSTER, 0x114D9B74, 0x1549A966, 0x1654AE6B, 0x1C53BB6B,
+                    0x1941A469, 0x1043A770, 0x1254C367}
+
+
+def ebml_element(data, offset):
+    element, content = ebml_number(data, offset, True)
+    size, content = ebml_number(data, content, False)
+    return element, content, size
+
+
+def open_cluster(data, offset, end):
+    # A live Cluster of unknown size runs to the next Segment child.
+    start = offset
+    while offset < end:
+        element, content, size = ebml_element(data, offset)
+        if element in SEGMENT_CHILDREN:
+            break
+        if element is None or size is None or size == -1:
+            return None
+        offset = content + size
+    return offset - start
+
+
+def webm_segment(data, offset, end):
+    # Each Cluster's blocks are the samples; other Segment children are not.
+    samples = []
+    while offset < end:
+        element, content, size = ebml_element(data, offset)
+        if element is None or size is None:
+            return None
+        if size == -1:
+            size = open_cluster(data, content, end) if element == CLUSTER else None
+            if size is None:
+                return None
+        if element == CLUSTER:
+            samples.append((content, content + size))
+        offset = content + size
+    return samples if offset == end else None
+
+
+def webm(data):
+    if not data.startswith(b'\x1a\x45\xdf\xa3') or b'webm' not in data[:64]:
+        return None
+    offset, samples = 0, []
+    while offset < len(data):
+        element, content, size = ebml_element(data, offset)
+        if element is None or size is None:
+            return None
+        # Only a live Segment may leave its size unknown: it runs to the end.
+        if size == -1:
+            if element != SEGMENT:
+                return None
+            size = len(data) - content
+        if element == SEGMENT:
+            segment = webm_segment(data, content, content + size)
+            if segment is None:
+                return None
+            samples += segment
+        offset = content + size
+    return (offset, samples) if offset == len(data) else None
+
+
+def ico(data):
+    if len(data) < 6 or data[:4] != b'\0\0\1\0':
+        return None
+    count = int.from_bytes(data[4:6], 'little')
+    images = []
+    for index in range(count):
+        entry = 6 + 16 * index
+        if entry + 16 > len(data):
+            return None
+        images.append((int.from_bytes(data[entry + 12:entry + 16], 'little'),
+                       int.from_bytes(data[entry + 8:entry + 12], 'little')))
+    # Image data follows the directory with no gap. An embedded PNG keeps its
+    # own compressed samples; bitmap pixels are stored as they are.
+    offset = 6 + 16 * count
+    samples = []
+    for start, size in sorted(images):
+        if start != offset or not size:
+            return None
+        image = png(data[start:start + size])
+        if image and image[0] == size:
+            samples += [(start + a, start + b) for a, b in image[1]]
+        offset += size
+    return (offset, samples) if count else None
+
+
+FORMATS = (gif, png, jpeg, webp, iso_media, webm, ico)
+
+
+def sample_spans(data):
+    """The pixel and sample spans of a complete media file, or None."""
+    for walk in FORMATS:
+        walked = walk(data)
+        if walked and walked[0] <= len(data) and not data[walked[0]:].strip(b'\0'):
+            return [(start, min(end, walked[0])) for start, end in walked[1]]
+    return None

package/scripts/publish-check/png_text.py

@@ -0,0 +1,34 @@
+"""Separate uncompressed PNG text fields from chunk framing and CRC bytes."""
+
+
+def text_chunks(data):
+    # Detect the container by bytes, never by a publication path or extension.
+    if not data.startswith(b'\x89PNG\r\n\x1a\n'):
+        return
+    offset = 8
+    while offset + 12 <= len(data):
+        start = offset
+        length = int.from_bytes(data[offset:offset + 4], 'big')
+        end = offset + 8 + length
+        if end + 4 > len(data):
+            return
+        kind = data[offset + 4:offset + 8]
+        payload = data[offset + 8:end]
+        offset = end + 4
+        # Checksums do not confer an exemption: inspect a framed text field
+        # even if its CRC is wrong. Non-text chunks keep raw-byte inspection.
+        if kind == b'tEXt':
+            fields = payload.split(b'\0', 1)
+            if len(fields) == 2:
+                yield start, offset, [field.decode('latin-1').encode('utf-8') for field in fields]
+        elif kind == b'iTXt':
+            fields = payload.split(b'\0', 1)
+            if len(fields) != 2 or len(fields[1]) < 2:
+                continue
+            keyword, body = fields
+            fields = body[2:].split(b'\0', 2)
+            # The compression method is ignored when the flag is zero.
+            if body[0] == 0 and len(fields) == 3:
+                yield start, offset, [keyword.decode('latin-1').encode('utf-8'), *fields]
+        if kind == b'IEND':
+            return

package/scripts/publish-check/public_hygiene.py

@@ -8,6 +8,9 @@ from provenance import provenance_text
 from hygiene_config import EMAIL, LOCAL_PATH, public_email
 from font_licence import without_required_notices
 from trust_bundle import anchor_values
+from png_text import text_chunks
+from media import sample_spans
+from c2pa import certificate_spans
 URI_AUTHORITY = re.compile(r'''(?i)(?a:(?P<scheme>[a-z][a-z0-9+.-]*:))?//(?P<authority>[^/?#\s<>"']+)''')
 # Match raw, complete commit trailer lines; decoded or embedded prose cannot
 # acquire trailer privileges. Only the address span is omitted from email checks.
@@ -47,7 +50,7 @@ def email_userinfo(match):
 def private_emails(text, config):
     # URL paths, queries and fragments remain visible to email inspection.
     email_text = URI_AUTHORITY.sub(email_userinfo, without_required_notices(text))
-    email_text = re.sub(r'(?i)\b(?a:git@github\.com:)(?=[A-Za-z0-9])', '', email_text)
+    email_text = re.sub(r'(?i)\b(?a:git@github\.com:)(?=[A-Za-z0-9]|\$\{[A-Za-z_$][A-Za-z0-9_.$]*\})', '', email_text)
     # A package manager's patch filename joins the name and version with an at sign.
     version = r'(?:v?[0-9]+(?:\.[0-9]+)*(?:-[A-Za-z0-9-]+)?|[a-f0-9]{40})(?:\.patch)?(?![A-Za-z0-9_.-])'
     email_text = re.sub(r'(?m)^\s*-?\s*uses:\s*[\"\']?[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+@[^\s\"\']+', '', email_text)
@@ -92,11 +95,43 @@ def certificate_metadata(data, bundle, emails, config):
     return not Counter(emails) - inside
 
 
+def without_spans(data, spans):
+    # Each span becomes one space: text on either side stays apart, and no
+    # long blank run is left for the line-anchored patterns to rescan.
+    kept, offset = [], 0
+    for start, end in sorted(spans):
+        kept.append(data[offset:start])
+        offset = max(offset, end)
+    kept.append(data[offset:])
+    return b' '.join(kept)
+
+
 def inspect(data, path, config, resolver, *, commit_message=False):
-    return inspect_text(data, path, config, resolver, commit_message=commit_message)
+    chunks = list(text_chunks(data))
+    # Image and video samples are not text: they can spell an address or a
+    # path by chance. Comments, XMP, tags and other metadata remain text.
+    spans = sample_spans(data) or []
+    # Names in a C2PA manifest's signing certificates are certificate metadata.
+    spans += certificate_spans(data)
+    rules, provenance = set(), False
+    for start, end, fields in chunks:
+        spans.append((start, end))
+        for field in fields:
+            # A metadata value is text, not a standalone source manifest or
+            # trust bundle, even when the containing image has a JSON name.
+            field_rules, field_provenance = inspect_text(field, '@png-text', config, resolver, binary_content=False)
+            rules.update(field_rules)
+            provenance |= field_provenance
+    # Keep the original binary classification for content outside text chunks.
+    # CRC-adjacent bytes must not also create a synthetic raw mailbox match.
+    raw_rules, raw_provenance = inspect_text(without_spans(data, spans), path, config, resolver,
+                                            commit_message=commit_message, binary_content=binary(data))
+    rules.update(raw_rules)
+    provenance |= raw_provenance
+    return rules, provenance
 
 
-def inspect_text(data, path, config, resolver, *, commit_message=False):
+def inspect_text(data, path, config, resolver, *, commit_message=False, binary_content=None):
     # NUL removal covers UTF-16/32 ASCII content, matching the secret scanner.
     text = decoded_text(data)
     rules = set()
@@ -111,7 +146,7 @@ def inspect_text(data, path, config, resolver, *, commit_message=False):
         if emails and not (bundle and certificate_metadata(data, bundle, emails, config)):
             # Inside binary data a mailbox is judged by its own shape: a host
             # name with mixed-case letters inside a label is not one.
-            if not binary(data) or any(stored_host(email) for email in emails):
+            if not (binary(data) if binary_content is None else binary_content) or any(stored_host(email) for email in emails):
                 rules.add('hygiene.email')
     ref_text, provenance = provenance_text(text, path, config.get('declaredSources', frozenset()))
     stated, ambiguous = reference_sets(ref_text, path, config.get('localPaths', frozenset()), config.get('projectIdentity'))

package/scripts/publish-check/publication-config.json

@@ -22,6 +22,12 @@
     { "repository": "just-every/run-oncall", "reason": "Owner-proposed public name (13 Oct 2026), accepted ahead of the rename; created by clean-history export at the first public tag." },
     { "repository": "just-every/run-spool", "reason": "Owner-proposed public name (13 Oct 2026), accepted ahead of the rename; created by clean-history export at the first public tag." },
     { "repository": "just-every/run-happytohelp", "reason": "Owner-proposed public name (13 Oct 2026), accepted ahead of the rename; created by clean-history export at the first public tag." },
-    { "repository": "just-every/run-auth", "reason": "Owner-proposed public name (13 Oct 2026), accepted ahead of the rename; created by clean-history export at the first public tag." }
+    { "repository": "just-every/run-auth", "reason": "Owner-proposed public name (13 Oct 2026), accepted ahead of the rename; created by clean-history export at the first public tag." },
+    { "repository": "just-every/run-chest", "reason": "Owner-proposed public name (13 Oct 2026), accepted ahead of the rename; created by clean-history export at the first public tag." },
+    { "repository": "just-every/run-purse", "reason": "Owner-proposed public name (13 Oct 2026), accepted ahead of the rename; created by clean-history export at the first public tag." },
+    { "repository": "just-every/run-easel", "reason": "Owner-proposed public name (13 Oct 2026), accepted ahead of the rename; created by clean-history export at the first public tag." },
+    { "repository": "just-every/run-willow", "reason": "Owner-proposed public name (13 Oct 2026), accepted ahead of the rename; created by clean-history export at the first public tag." },
+    { "repository": "just-every/run-pier", "reason": "Owner-proposed public name (13 Oct 2026), accepted ahead of the rename; created by clean-history export at the first public tag." },
+    { "repository": "just-every/run-linen", "reason": "Owner-proposed public name (13 Oct 2026), accepted ahead of the rename; created by clean-history export at the first public tag." }
   ]
 }