@runeditrun/rer 0.2.2 → 0.2.4

package/scripts/publish-check/png_text.py

@@ -0,0 +1,34 @@
+"""Separate uncompressed PNG text fields from chunk framing and CRC bytes."""
+
+
+def text_chunks(data):
+    # Detect the container by bytes, never by a publication path or extension.
+    if not data.startswith(b'\x89PNG\r\n\x1a\n'):
+        return
+    offset = 8
+    while offset + 12 <= len(data):
+        start = offset
+        length = int.from_bytes(data[offset:offset + 4], 'big')
+        end = offset + 8 + length
+        if end + 4 > len(data):
+            return
+        kind = data[offset + 4:offset + 8]
+        payload = data[offset + 8:end]
+        offset = end + 4
+        # Checksums do not confer an exemption: inspect a framed text field
+        # even if its CRC is wrong. Non-text chunks keep raw-byte inspection.
+        if kind == b'tEXt':
+            fields = payload.split(b'\0', 1)
+            if len(fields) == 2:
+                yield start, offset, [field.decode('latin-1').encode('utf-8') for field in fields]
+        elif kind == b'iTXt':
+            fields = payload.split(b'\0', 1)
+            if len(fields) != 2 or len(fields[1]) < 2:
+                continue
+            keyword, body = fields
+            fields = body[2:].split(b'\0', 2)
+            # The compression method is ignored when the flag is zero.
+            if body[0] == 0 and len(fields) == 3:
+                yield start, offset, [keyword.decode('latin-1').encode('utf-8'), *fields]
+        if kind == b'IEND':
+            return

package/scripts/publish-check/public_hygiene.py

@@ -8,6 +8,9 @@ from provenance import provenance_text
 from hygiene_config import EMAIL, LOCAL_PATH, public_email
 from font_licence import without_required_notices
 from trust_bundle import anchor_values
+from png_text import text_chunks
+from media import sample_spans
+from c2pa import certificate_spans
 URI_AUTHORITY = re.compile(r'''(?i)(?a:(?P<scheme>[a-z][a-z0-9+.-]*:))?//(?P<authority>[^/?#\s<>"']+)''')
 # Match raw, complete commit trailer lines; decoded or embedded prose cannot
 # acquire trailer privileges. Only the address span is omitted from email checks.
@@ -47,7 +50,7 @@ def email_userinfo(match):
 def private_emails(text, config):
     # URL paths, queries and fragments remain visible to email inspection.
     email_text = URI_AUTHORITY.sub(email_userinfo, without_required_notices(text))
-    email_text = re.sub(r'(?i)\b(?a:git@github\.com:)(?=[A-Za-z0-9])', '', email_text)
+    email_text = re.sub(r'(?i)\b(?a:git@github\.com:)(?=[A-Za-z0-9]|\$\{[A-Za-z_$][A-Za-z0-9_.$]*\})', '', email_text)
     # A package manager's patch filename joins the name and version with an at sign.
     version = r'(?:v?[0-9]+(?:\.[0-9]+)*(?:-[A-Za-z0-9-]+)?|[a-f0-9]{40})(?:\.patch)?(?![A-Za-z0-9_.-])'
     email_text = re.sub(r'(?m)^\s*-?\s*uses:\s*[\"\']?[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+@[^\s\"\']+', '', email_text)
@@ -92,11 +95,43 @@ def certificate_metadata(data, bundle, emails, config):
     return not Counter(emails) - inside
 
 
+def without_spans(data, spans):
+    # Each span becomes one space: text on either side stays apart, and no
+    # long blank run is left for the line-anchored patterns to rescan.
+    kept, offset = [], 0
+    for start, end in sorted(spans):
+        kept.append(data[offset:start])
+        offset = max(offset, end)
+    kept.append(data[offset:])
+    return b' '.join(kept)
+
+
 def inspect(data, path, config, resolver, *, commit_message=False):
-    return inspect_text(data, path, config, resolver, commit_message=commit_message)
+    chunks = list(text_chunks(data))
+    # Image and video samples are not text: they can spell an address or a
+    # path by chance. Comments, XMP, tags and other metadata remain text.
+    spans = sample_spans(data) or []
+    # Names in a C2PA manifest's signing certificates are certificate metadata.
+    spans += certificate_spans(data)
+    rules, provenance = set(), False
+    for start, end, fields in chunks:
+        spans.append((start, end))
+        for field in fields:
+            # A metadata value is text, not a standalone source manifest or
+            # trust bundle, even when the containing image has a JSON name.
+            field_rules, field_provenance = inspect_text(field, '@png-text', config, resolver, binary_content=False)
+            rules.update(field_rules)
+            provenance |= field_provenance
+    # Keep the original binary classification for content outside text chunks.
+    # CRC-adjacent bytes must not also create a synthetic raw mailbox match.
+    raw_rules, raw_provenance = inspect_text(without_spans(data, spans), path, config, resolver,
+                                            commit_message=commit_message, binary_content=binary(data))
+    rules.update(raw_rules)
+    provenance |= raw_provenance
+    return rules, provenance
 
 
-def inspect_text(data, path, config, resolver, *, commit_message=False):
+def inspect_text(data, path, config, resolver, *, commit_message=False, binary_content=None):
     # NUL removal covers UTF-16/32 ASCII content, matching the secret scanner.
     text = decoded_text(data)
     rules = set()
@@ -111,7 +146,7 @@ def inspect_text(data, path, config, resolver, *, commit_message=False):
         if emails and not (bundle and certificate_metadata(data, bundle, emails, config)):
             # Inside binary data a mailbox is judged by its own shape: a host
             # name with mixed-case letters inside a label is not one.
-            if not binary(data) or any(stored_host(email) for email in emails):
+            if not (binary(data) if binary_content is None else binary_content) or any(stored_host(email) for email in emails):
                 rules.add('hygiene.email')
     ref_text, provenance = provenance_text(text, path, config.get('declaredSources', frozenset()))
     stated, ambiguous = reference_sets(ref_text, path, config.get('localPaths', frozenset()), config.get('projectIdentity'))