@runeditrun/rer 0.2.2 → 0.2.4

package/onboarding/operator-installer.mjs

@@ -19,13 +19,16 @@ export function operatorInstaller({ database, features, prepare, prepareData })
    values: ({ origin }) => ({ AUTH_LOCAL: 'true', AUTH_ORIGIN: origin, MAIL_PROVIDER: 'development', MAIL_FROM: 'auth@localhost.test' }),
   },
   build: ['pnpm', ['run', 'build:app']],
-  migrate: ['pnpm', ['exec', 'wrangler', 'd1', 'migrations', 'apply', database, '--local', '--persist-to', '.wrangler/state', '--config', 'dist/server/wrangler.json']],
+  // A demo migrates its own fresh state folder, never the checkout's.
+  migrate: ({ demo }) => ['pnpm', ['exec', 'wrangler', 'd1', 'migrations', 'apply', database, '--local', '--persist-to', demo?.state ?? '.wrangler/state', '--config', 'dist/server/wrangler.json']],
   // Setup uses preview's production runtime behind the same public proxy.
   start: previewTarget,
   owner: { existingOwnerViaSignIn: true, displayName: input => input.displayName || 'Owner' },
   signIn: { emailKey: 'OWNER_EMAIL', returnTo: '/setup' },
  });
- return {...lifecycle,localPrerequisites:async({root,vars,email})=>vars.AUTH_LOCAL==='true'&&vars.MAIL_PROVIDER==='development'&&Boolean(vars.SESSION_SECRET)&&await operatorPrerequisites(root,email),install:(root,input,store,onApp,signal)=>installOperator(root,input,store,onApp,signal,lifecycle.hooks)};
+  // A demo migrates and claims the owner in its own state, so it never opens the
+ // checkout's database: even read-only, SQLite rewrites its -shm index.
+ return {...lifecycle,localPrerequisites:async({root,vars,email,demo})=>vars.AUTH_LOCAL==='true'&&vars.MAIL_PROVIDER==='development'&&Boolean(vars.SESSION_SECRET)&&(demo||await operatorPrerequisites(root,email)),install:(root,input,store,onApp,signal)=>installOperator(root,input,store,onApp,signal,lifecycle.hooks)};
 }
 
 // Only local runtimes receive the dev key; vite.config.ts passes it to `vite dev` alone.
@@ -33,7 +36,10 @@ export function operatorInstaller({ database, features, prepare, prepareData })
 // without the key. trafficProxy also refuses /cdn-cgi/* on the public listener.
 const devBindings = (devKey, devLogin) => devKey ? { RER_DEV_KEY: devKey, RER_DEV_IDENTITY: devLogin ? 'off' : 'on' } : {};
 const internalPort = port => { if (!Number.isInteger(port) || port < 1 || port > 65535) throw new Error('A local runtime requires the traffic proxy internal port.'); return String(port); };
-const developmentEnvironment = () => { const { RER_MODELS_SERVICE, RER_MODELS_KEY, ...env } = process.env; return env; };
-export const developmentTarget = ({ root, origin, runtimePort, devKey, devLogin, modelsLive }) => [process.execPath, [projectViteBinary(root), 'dev', '--host', '127.0.0.1', '--port', internalPort(runtimePort), '--strictPort'], { env: { ...developmentEnvironment(), PATH: join(root, 'node_modules', '.bin') + delimiter + (process.env.PATH ?? ''), RER_DEV_ORIGIN: origin, WRANGLER_SEND_METRICS: 'false', X_LOCAL_EXPLORER: 'false', ...devBindings(devKey, devLogin), ...(modelsLive ? { RER_MODELS_SERVICE: modelsLive.baseUrl, RER_MODELS_KEY: modelsLive.apiKey } : {}) } }];
+const developmentEnvironment = () => { const { RER_MODELS_SERVICE, RER_MODELS_KEY, RER_DEMO, RER_DEMO_EXPIRES, RER_STATE_DIR, ...env } = process.env; return env; };
+// A demo's Vite persists to its own folder (the seed's vite.config reads RER_STATE_DIR)
+// and its Worker sees RER_DEMO=1 and the demo's end (modules/demo).
+const demoBindings = demo => demo ? { RER_DEMO: '1', RER_DEMO_EXPIRES: demo.ends, RER_STATE_DIR: demo.state } : {};
+export const developmentTarget = ({ root, origin, runtimePort, devKey, devLogin, modelsLive, demo }) => [process.execPath, [projectViteBinary(root), 'dev', '--host', '127.0.0.1', '--port', internalPort(runtimePort), '--strictPort'], { env: { ...developmentEnvironment(), PATH: join(root, 'node_modules', '.bin') + delimiter + (process.env.PATH ?? ''), RER_DEV_ORIGIN: origin, WRANGLER_SEND_METRICS: 'false', X_LOCAL_EXPLORER: 'false', ...devBindings(devKey, devLogin), ...(modelsLive ? { RER_MODELS_SERVICE: modelsLive.baseUrl, RER_MODELS_KEY: modelsLive.apiKey } : {}), ...demoBindings(demo) } }];
 
 export const previewTarget = ({ origin, runtimePort, devKey, scheduled }) => [process.execPath, [fileURLToPath(new URL('./production-runtime.mjs', import.meta.url))], { env: { ...process.env, RER_DEV_ORIGIN: origin, RER_RUNTIME_HOST: '127.0.0.1', RER_RUNTIME_PORT: internalPort(runtimePort), WRANGLER_SEND_METRICS: 'false', CLOUDFLARE_INCLUDE_PROCESS_ENV: 'false', X_LOCAL_EXPLORER: 'false', RER_RUNTIME_TRIGGERS: scheduled ? '1' : '0', ...(devKey ? { RER_DEV_KEY: devKey } : {}) } }];

package/onboarding/redaction.mjs

@@ -35,7 +35,9 @@ const DOUBLE_QUOTED = /"(?:\\[^\r\n]|[^"\\\r\n])*(?:\\(?=[\r\n]|$))?/.source;
 const SINGLE_QUOTED = /'(?:\\[^\r\n]|[^'\\\r\n])*(?:\\(?=[\r\n]|$))?/.source;
 const PRIVATE_VALUE = `(?:${DOUBLE_QUOTED}(?:"|(?=[\\r\\n]|$))|${SINGLE_QUOTED}(?:'|(?=[\\r\\n]|$))|[^\\s"'<>;&]+)`;
 const FIELD_VALUE = '["\']?\\s*[=:]\\s*' + NOT_FLAG + PRIVATE_VALUE;
-const CLI_PREFIX = '(?<![A-Za-z0-9_-])--(?:[A-Za-z0-9]+[_-])*';
+// These deploy flags take a binding name, never the secret value. Explicitly
+// registered private bytes remain redacted by the exact-value pass.
+const CLI_PREFIX = '(?<![A-Za-z0-9_-])--(?!(?:set|unset)-secret(?:\\s|$))(?:[A-Za-z0-9]+[_-])*';
 const CLI_ARGUMENT = CLI_PREFIX + '(' + FIELD_NAME + ')\\s+' + NOT_FLAG + '(' + PRIVATE_VALUE + ')';
 // A bare Bearer/Basic value must look like a credential, not the next word of
 // prose such as "Basic checks passed"; after Authorization any value is private.
@@ -57,9 +59,12 @@ const COOKIE_HEADER = '(?<![A-Za-z0-9_-])(?:Set-)?Cookie["\']?\\s*:\\s*(?:[^\\s=
 const SCHEME = '(?<![a-z0-9+.-])[a-z][a-z0-9+.-]*://';
 const USERINFO = SCHEME + '([^\\s/?#"<>`]*)@';
 const SLASHED_USERINFO = SCHEME + '([^\\s/?#:@"<>`\\[\\]]*:(?![0-9]*(?:[/?#\\s"<>`]|$))(?=(?<password>[^\\s"<>`@]{0,128}))\\k<password>)@';
+// Provider token formats recognisable by their prefix alone.
+const TOKEN_PREFIX = '(?:gh[pousr]_|github_pat_|sk-(?:proj-)?|sk_live_|sk_test_|rk_live_|rk_test_|pk_live_|whsec_|SG\\.|xox[baprs]-|AKIA|ASIA|AIza|npm_|glpat-|pypi-|hf_|re_)';
+const KNOWN_TOKEN = '(?<![A-Za-z0-9_])' + TOKEN_PREFIX + '[A-Za-z0-9_./+=-]+';
 const PATTERNS = new RegExp('(?:' + [
  AUTH_HEADER, BARE_AUTH,
- '(?<![A-Za-z0-9_])(?:gh[pousr]_|github_pat_|sk-(?:proj-)?|sk_live_|sk_test_|rk_live_|rk_test_|pk_live_|whsec_|SG\\.|xox[baprs]-|AKIA|ASIA|AIza|npm_|glpat-|pypi-|hf_|re_)[A-Za-z0-9_./+=-]+',
+ KNOWN_TOKEN,
  FIELD + FIELD_VALUE, COOKIE_HEADER,
  USERINFO, SLASHED_USERINFO,
 ].join('|') + ')', 'gi');
@@ -287,12 +292,26 @@ export function createRedaction({mask = MASK} = {}) {
  return {addSecrets,redact,stream};
 }
 
+// A variable whose name marks a credential, or whose value is a URL carrying one.
+export const credentialName = key => CREDENTIAL_NAME.test(key) || CAMEL_CREDENTIAL_NAME.test(key);
+export const credentialUrl = value => CREDENTIAL_URL.test(value);
+// A value that carries a credential whatever its variable is called: a known
+// provider token of at least 16 characters, a JWT, or a URL with userinfo or a
+// credential query or connection-string parameter (libsql://db?authToken=...,
+// ...;AccountKey=...), a private key, or an Authorization header value.
+// A token never continues a path or word (/opt/hf_models_cache is a path).
+const TOKEN_VALUE = new RegExp('(?<![A-Za-z0-9_./-])' + TOKEN_PREFIX + '[A-Za-z0-9_./+=-]{12,}');
+const CREDENTIAL_QUERY = /[?&;](?:auth[_-]?token|access[_-]?token|api[_-]?key|account[_-]?key|shared[_-]?access[_-]?key|token|key|secret|password|sig|signature)=[^&#\s]/i;
+const PRIVATE_KEY_VALUE = /-----BEGIN [A-Z0-9 ]{0,64}PRIVATE KEY/;
+const AUTHORIZATION_VALUE = new RegExp(AUTH_HEADER + '|' + BARE_AUTH, 'i');
+export const credentialValue = value => TOKEN_VALUE.test(value) || jwtRanges(value).length > 0 || CREDENTIAL_URL.test(value) || CREDENTIAL_QUERY.test(value) || PRIVATE_KEY_VALUE.test(value) || AUTHORIZATION_VALUE.test(value);
+
 export function credentialValues(input, declared = []) {
  if (!input || typeof input !== 'object') return [];
  return Object.entries(input).flatMap(([key,value]) => {
   // An explicit model declaration remains an exact secret contract, including
   // short values. Environment heuristics must not turn boolean flags into masks.
-  if (typeof value === 'string') return declared.includes(key) || (value.length>=8 && (CREDENTIAL_NAME.test(key) || CAMEL_CREDENTIAL_NAME.test(key) || CREDENTIAL_URL.test(value))) ? [value] : [];
+  if (typeof value === 'string') return declared.includes(key) || (value.length>=8 && (credentialName(key) || credentialUrl(value))) ? [value] : [];
   return value && typeof value==='object' ? credentialValues(value,declared) : [];
  });
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@runeditrun/rer",
-  "version": "0.2.2",
+  "version": "0.2.4",
   "description": "The rer command: install, set up, run and launch Run Edit Run checkouts.",
   "license": "MIT",
   "type": "module",
@@ -63,6 +63,7 @@
     "./modules/portability/migration-history": "./modules/portability/migration-history.mjs",
     "./modules/portability/storage": "./modules/portability/storage.mjs",
     "./modules/erasure/fence": "./modules/erasure/fence.mjs",
+    "./modules/deploy-kit/migration-ledger": "./modules/deploy-kit/migration-ledger.mjs",
     "./modules/deploy-kit/owner": "./modules/deploy-kit/owner.mjs",
     "./modules/deploy-kit/owner-cli": "./modules/deploy-kit/owner-cli.mjs",
     "./modules/deploy-kit/prepare": "./modules/deploy-kit/prepare.mjs",
@@ -84,5 +85,9 @@
   },
   "engines": {
     "node": ">=22"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/runeditrun/base.git"
   }
 }

package/scripts/publish-check/README.md

@@ -21,7 +21,20 @@ bounded public coauthor address rule described in the command guide; a file name
 masks only the accepted address for email inspection, leaving names, other
 addresses and every other rule intact.
 
-Binary metadata decoding, Git protocol/object/ref modelling, frozen comparison,
+`png_text.py` separates the fields of framed PNG `tEXt` and uncompressed `iTXt`
+chunks before text hygiene inspection. Container length, type and CRC bytes
+cannot extend a mailbox, and these explicit text fields do not use the binary
+mixed-case host heuristic. Raw-byte hygiene inspection retains the original
+binary classification outside those chunks; checksums do not suppress text
+inspection. Gitleaks still receives the original bytes. Compressed metadata is
+not decoded. `media.py` walks image and video containers to find their
+pixel and sample spans; only those spans skip text hygiene, never
+environment or Gitleaks inspection, and metadata around them is still text.
+`c2pa.py` finds the strictly parsed X.509 certificates inside a PNG's C2PA
+manifest store (`caBX`) and returns their Name and rfc822Name value spans,
+which skip text hygiene the same way.
+
+Other binary metadata decoding, Git protocol/object/ref modelling, frozen comparison,
 resumable inspection and their tests have been removed. Tests live in `tests.py`
 and `coverage_tests.py` and construct disposable repositories with actual Git.
 
@@ -80,3 +93,11 @@ The build that produces the artifacts (`modules/egress-service/tls/build.mjs`)
 regenerates the manifest, and its `--check` mode rejects a stale one. An
 alphabetic top-level domain label ends before a following digit, so a match
 ends at the DER string's boundary rather than absorbing the next tag byte.
+
+`controller_metadata.py` recognizes the standalone controller review manifest by
+its exact four-field structure, immutable source revision, descriptor digest,
+relative directory list and path-to-SHA-256 file map, at any JSON path. Only
+those digest value locations qualify. Extra fields, malformed paths or digests
+retain findings. The live-proof controller still independently checks all copied
+file bytes against the pinned source before execution; the scanner does not claim
+to fetch that source. Provider tokens and known environment values remain findings.

package/scripts/publish-check/c2pa.py

@@ -0,0 +1,121 @@
+"""Certificate metadata inside a PNG's C2PA manifest store (caBX chunk).
+
+Image generators sign a C2PA manifest and embed the signing certificate chain
+in it; a certificate names its holder, often by address. Only values that a
+strict X.509 parse places in an issuer or subject Name, or in a subject or
+issuer alternative name's rfc822Name, are certificate metadata. Every other
+byte of the manifest, assertions included, remains text to inspect.
+"""
+from der import OID, SEQUENCE, BIT_STRING, rdn_values, tlv
+
+INTEGER, OCTET_STRING, VERSION, EXTENSIONS, RFC822_NAME = 0x02, 0x04, 0xA0, 0xA3, 0x81
+ALTERNATIVE_NAMES = (bytes.fromhex('551d11'), bytes.fromhex('551d12'))
+
+
+def manifest_payloads(data):
+    if not data.startswith(b'\x89PNG\r\n\x1a\n'):
+        return
+    offset = 8
+    while offset + 12 <= len(data):
+        length = int.from_bytes(data[offset:offset + 4], 'big')
+        kind, start, end = data[offset + 4:offset + 8], offset + 8, offset + 8 + length
+        if end + 4 > len(data):
+            return
+        if kind == b'caBX':
+            yield start, end
+        if kind == b'IEND':
+            return
+        offset = end + 4
+
+
+def name_values(data, name):
+    # A Name is RDN SETs filling its SEQUENCE exactly.
+    values, cursor = [], name[1]
+    while cursor < name[2]:
+        rdn = rdn_values(data, cursor, name[2])
+        if not rdn:
+            return None
+        cursor, found = rdn
+        values += found
+    return values
+
+
+def alternative_names(data, extensions):
+    spans, outer = [], tlv(data, extensions[1], extensions[2])
+    if not outer or outer[0] != SEQUENCE or outer[2] != extensions[2]:
+        return None
+    cursor = outer[1]
+    while cursor < outer[2]:
+        extension = tlv(data, cursor, outer[2])
+        oid = extension and extension[0] == SEQUENCE and tlv(data, extension[1], extension[2])
+        if not oid or oid[0] != OID:
+            return None
+        value = tlv(data, oid[2], extension[2])
+        if value and value[0] == 0x01:
+            value = tlv(data, value[2], extension[2])
+        if not value or value[0] != OCTET_STRING or value[2] != extension[2]:
+            return None
+        if data[oid[1]:oid[2]] in ALTERNATIVE_NAMES:
+            general = tlv(data, value[1], value[2])
+            if not general or general[0] != SEQUENCE:
+                return None
+            entry = general[1]
+            while entry < general[2]:
+                item = tlv(data, entry, general[2])
+                if not item:
+                    return None
+                if item[0] == RFC822_NAME:
+                    spans.append((item[1], item[2]))
+                entry = item[2]
+        cursor = extension[2]
+    return spans
+
+
+def certificate(data, offset, end):
+    """Name and alternative-name value spans of one X.509 Certificate, or None."""
+    outer = tlv(data, offset, end)
+    if not outer or outer[0] != SEQUENCE:
+        return None
+    tbs = tlv(data, outer[1], outer[2])
+    algorithm = tbs and tbs[0] == SEQUENCE and tlv(data, tbs[2], outer[2])
+    signature = algorithm and algorithm[0] == SEQUENCE and tlv(data, algorithm[2], outer[2])
+    if not signature or signature[0] != BIT_STRING or signature[2] != outer[2]:
+        return None
+    fields, cursor = [], tbs[1]
+    while cursor < tbs[2]:
+        field = tlv(data, cursor, tbs[2])
+        if not field:
+            return None
+        fields.append(field)
+        cursor = field[2]
+    if fields and fields[0][0] == VERSION:
+        fields = fields[1:]
+    # serialNumber, signature, issuer, validity, subject, subjectPublicKeyInfo.
+    if len(fields) < 6 or [field[0] for field in fields[:6]] != [INTEGER] + [SEQUENCE] * 5:
+        return None
+    issuer, subject = name_values(data, fields[2]), name_values(data, fields[4])
+    if issuer is None or subject is None:
+        return None
+    spans = issuer + subject
+    for field in fields[6:]:
+        if field[0] == EXTENSIONS:
+            names = alternative_names(data, field)
+            if names is None:
+                return None
+            spans += names
+    return spans
+
+
+def certificate_spans(data):
+    """Value spans of the certificates a PNG's C2PA manifest store carries."""
+    spans = []
+    for start, end in manifest_payloads(data):
+        offset = data.find(SEQUENCE.to_bytes(1, 'big'), start, end)
+        while 0 <= offset < end:
+            found = certificate(data, offset, end)
+            if found is None:
+                offset = data.find(SEQUENCE.to_bytes(1, 'big'), offset + 1, end)
+                continue
+            spans += found
+            offset = data.find(SEQUENCE.to_bytes(1, 'big'), tlv(data, offset, end)[2], end)
+    return spans

package/scripts/publish-check/content_identities.py

@@ -2,11 +2,12 @@
 from json_identity import Document
 from provenance_pins import module_pins
 from copy_identities import copy_pointers
+from controller_metadata import controller_pointers
 from source_metadata import ARCHIVE, source_pointers
 
 
 def candidate(path):
-    return path in ('edition.json', 'sources.lock.json', 'base-copies.json', 'base/PROVENANCE.json') or bool(ARCHIVE.fullmatch(path))
+    return path in ('edition.json', 'sources.lock.json', 'base-copies.json', 'base/PROVENANCE.json') or bool(ARCHIVE.fullmatch(path)) or path.endswith('.json')
 
 
 class ContentIdentities:
@@ -22,6 +23,7 @@ class ContentIdentities:
             self.documents[path] = document
             pointers = set(module_pins(path, document.value))
             pointers.update(source_pointers(path, document.value))
+            pointers.update(controller_pointers(document.value))
             if path in ('base-copies.json', 'base/PROVENANCE.json') and read_regular is not None:
                 pointers.update(copy_pointers(document.value, read_regular))
             self.allowed[path] = pointers

package/scripts/publish-check/controller_metadata.py

@@ -0,0 +1,31 @@
+"""Exact digest locations in immutable standalone-controller review manifests.
+
+The live-proof controller verifies these digests against its pinned Git source
+before executing any copied file. This identifies the documented format; it does
+not claim to perform that remote integrity check during publication inspection.
+"""
+from copy_identities import relative_path
+from json_identity import HEX, GIT_REVISION
+
+
+def controller_pointers(value):
+    if not isinstance(value, dict) or set(value) != {
+            'sourceSha', 'descriptorSha256', 'controllerDirectories', 'controllerFiles'}:
+        return
+    if (not isinstance(value['sourceSha'], str) or not GIT_REVISION.fullmatch(value['sourceSha'])
+            or not isinstance(value['descriptorSha256'], str)
+            or not HEX.fullmatch(value['descriptorSha256'])):
+        return
+    directories, files = value['controllerDirectories'], value['controllerFiles']
+    if (not isinstance(directories, list) or not all(relative_path(p) for p in directories)
+            or len(set(directories)) != len(directories)
+            or not isinstance(files, dict) or not files
+            or not all(relative_path(p) and isinstance(digest, str) and HEX.fullmatch(digest)
+                       for p, digest in files.items())
+            or any(directory == path or directory.startswith(path + '/')
+                   for directory in directories for path in files)):
+        return
+    yield ('sourceSha',)
+    yield ('descriptorSha256',)
+    for path in files:
+        yield ('controllerFiles', path)

package/scripts/publish-check/history_scan.py

@@ -38,6 +38,9 @@ def scan(repo, ref='HEAD', public_base=None, env_file=None, project=None, reposi
         recorded = registry() + exceptions(history.selected(EXCEPTIONS))
         resolver = GitHubResolver()
         public_blobs = {e['oid'] for c in history.public_commits for e in history.entries(c)}
+        # A file byte-identical to the same path on the public base is already public.
+        public_paths = {e['path']: e['oid'] for e in history.entries(history.public_commit)
+                        if e['kind'] == 'blob'} if history.public_commit else {}
         report['history']['preExistingCommits'] = sorted(history.public_commits)
         report['history']['newlyPublicCommits'] = [c for c in history.commits if c not in history.public_commits]
         seen_trees, seen_blobs, inspected = set(), set(), {}
@@ -73,7 +76,7 @@ def scan(repo, ref='HEAD', public_base=None, env_file=None, project=None, reposi
                 if key not in inspected:
                     inspected[key] = text_rules(history.blob(oid), path, config_tree, resolver, hashes)
                 rules, provenance = inspected[key]
-                entry_scope = 'pre-existing' if scope != 'release-tree' and oid in public_blobs else scope
+                entry_scope = 'pre-existing' if (scope != 'release-tree' and oid in public_blobs) or public_paths.get(path) == oid else scope
                 for rule in rules:
                     add(rule, path, commit, oid, entry_scope)
                 if provenance:
@@ -92,8 +95,10 @@ def scan(repo, ref='HEAD', public_base=None, env_file=None, project=None, reposi
             entry = entries_by_commit[history.commit].get(path)
             oid = entry['oid'] if entry and entry['kind'] == 'blob' and history.blob(entry['oid']) == data else None
             rules, _ = text_rules(data, path, selected_context, resolver, hashes)
+            public_oid = public_paths.get(path)
+            scope = 'pre-existing' if public_oid and (public_oid == oid or history.blob(public_oid) == data) else 'working-tree'
             for rule in rules:
-                add(rule, path, history.commit, oid, 'working-tree')
+                add(rule, path, history.commit, oid, scope)
             if path == 'LICENSE':
                 for rule in license_rules(data, recorded):
                     add(rule, path, history.commit, oid, 'working-tree')
@@ -109,7 +114,12 @@ def scan(repo, ref='HEAD', public_base=None, env_file=None, project=None, reposi
                     path = Path(path).resolve().relative_to(Path(history.repo).resolve()).as_posix()
                 entry = entries_by_commit.get(commit, {}).get(path)
                 oid = entry['oid'] if entry and entry['kind'] == 'blob' else None
-                scope = 'working-tree' if mode == 'dir' else 'release-tree' if commit == history.commit else 'pre-existing' if oid in public_blobs else 'newly-public'
+                if mode == 'dir':
+                    public_oid = public_paths.get(path)
+                    held = working_regular(Path(history.repo), path) if public_oid else None
+                    scope = 'pre-existing' if held and history.blob(public_oid) == held[0] else 'working-tree'
+                else:
+                    scope = 'pre-existing' if (oid and public_paths.get(path) == oid) or (commit != history.commit and oid in public_blobs) else 'release-tree' if commit == history.commit else 'newly-public'
                 if hit['RuleID'] in ('generic-api-key', 'hex-api-key'):
                     identity_scope = 'working-tree' if mode == 'dir' else commit
                     scoped_entries = entries_by_commit.get(commit, {})

package/scripts/publish-check/media.py

@@ -0,0 +1,283 @@
+"""Find the pixel and sample data inside an image or video file.
+
+A file is media only when its bytes walk as one complete container, from the
+signature to the end of the file, allowing only NUL padding after it. The path
+and extension never decide, so text renamed .gif, or a valid image with text
+appended, is not media. Each walker returns where its container ends and the
+(start, end) spans of its pixel or sample data (compressed samples and colour
+tables), or None. Everything outside those spans, such as comments, XMP and
+tags, is still text to inspect.
+"""
+
+
+def gif(data):
+    if data[:6] not in (b'GIF87a', b'GIF89a') or len(data) < 13:
+        return None
+    # Colour tables are pixel data too: palette bytes can spell an address.
+    offset = 13 + ((3 << ((data[10] & 7) + 1)) if data[10] & 0x80 else 0)
+    samples = [(13, offset)]
+    while offset < len(data):
+        kind = data[offset]
+        if kind == 0x3B:
+            return offset + 1, samples
+        if kind == 0x21:
+            offset += 2
+        elif kind == 0x2C:
+            if offset + 10 > len(data):
+                return None
+            flags = data[offset + 9]
+            # Descriptor, local colour table, then the LZW minimum code size.
+            table = offset + 10 + ((3 << ((flags & 7) + 1)) if flags & 0x80 else 0)
+            samples.append((offset + 10, table))
+            offset = table + 1
+        else:
+            return None
+        # Data sub-blocks: a length byte, that many bytes, ending at length zero.
+        start = offset
+        while True:
+            if offset >= len(data):
+                return None
+            size = data[offset]
+            offset += 1 + size
+            if not size:
+                break
+        if kind == 0x2C:
+            samples.append((start, offset))
+    return None
+
+
+def png(data):
+    if not data.startswith(b'\x89PNG\r\n\x1a\n'):
+        return None
+    offset = 8
+    samples = []
+    while offset + 12 <= len(data):
+        length = int.from_bytes(data[offset:offset + 4], 'big')
+        kind = data[offset + 4:offset + 8]
+        if kind in (b'PLTE', b'IDAT', b'fdAT'):
+            samples.append((offset + 8, offset + 8 + length))
+        offset += 12 + length
+        if kind == b'IEND':
+            return offset, samples
+    return None
+
+
+def jpeg(data):
+    # A phone stores more images after the first one's end (MPF depth and
+    # gain maps): each is walked, and their samples are pixels too.
+    walked = jpeg_image(data, 0)
+    if walked is None:
+        return None
+    end, samples = walked
+    while data.startswith(b'\xff\xd8', end) and (following := jpeg_image(data, end)):
+        end, more = following
+        samples += more
+    return end, samples
+
+
+def jpeg_image(data, offset):
+    if not data.startswith(b'\xff\xd8', offset):
+        return None
+    offset += 2
+    samples = []
+    while offset + 2 <= len(data):
+        if data[offset] != 0xFF:
+            return None
+        marker = data[offset + 1]
+        if marker == 0xFF:
+            offset += 1
+            continue
+        if marker == 0xD9:
+            return offset + 2, samples
+        if 0xD0 <= marker <= 0xD7 or marker == 0x01:
+            offset += 2
+            continue
+        if offset + 4 > len(data):
+            return None
+        offset += 2 + int.from_bytes(data[offset + 2:offset + 4], 'big')
+        if marker == 0xDA:
+            # Entropy-coded data runs to the next marker other than a stuffed
+            # zero or a restart marker.
+            start = offset
+            while True:
+                offset = data.find(b'\xff', offset)
+                if offset < 0 or offset + 1 >= len(data):
+                    return None
+                if data[offset + 1] == 0 or 0xD0 <= data[offset + 1] <= 0xD7:
+                    offset += 2
+                else:
+                    break
+            samples.append((start, offset))
+    return None
+
+
+WEBP_SAMPLES = (b'VP8 ', b'VP8L', b'ALPH', b'ANMF')
+
+
+def webp(data):
+    if len(data) < 20 or data[:4] != b'RIFF' or data[8:12] != b'WEBP':
+        return None
+    end = int.from_bytes(data[4:8], 'little') + 8
+    offset = 12
+    samples = []
+    while offset + 8 <= end:
+        size = int.from_bytes(data[offset + 4:offset + 8], 'little')
+        if data[offset:offset + 4] in WEBP_SAMPLES:
+            samples.append((offset + 8, offset + 8 + size))
+        offset += 8 + size + (size & 1)
+    return (end, samples) if offset == end else None
+
+
+# Major brands of MP4, QuickTime, 3GPP and HEIF/AVIF files.
+ISO_BRANDS = {b'isom', b'iso2', b'iso4', b'iso5', b'iso6', b'mp41', b'mp42', b'avc1', b'dash',
+              b'M4V ', b'M4A ', b'M4VH', b'M4VP', b'qt  ', b'3gp4', b'3gp5', b'3gp6', b'3g2a',
+              b'mif1', b'msf1', b'avif', b'avis', b'heic', b'heix', b'heim', b'heis', b'hevc', b'hevx'}
+
+
+def iso_media(data):
+    # MP4 and AVIF share ISO base media boxes, opened by a file-type box with
+    # an explicit size and a known major brand.
+    if len(data) < 16 or data[4:8] != b'ftyp' or data[8:12] not in ISO_BRANDS:
+        return None
+    if int.from_bytes(data[:4], 'big') < 16:
+        return None
+    offset = 0
+    kinds, samples = set(), []
+    while offset + 8 <= len(data):
+        size = int.from_bytes(data[offset:offset + 4], 'big')
+        kind = data[offset + 4:offset + 8]
+        header = 8
+        if size == 1:
+            if offset + 16 > len(data):
+                return None
+            size, header = int.from_bytes(data[offset + 8:offset + 16], 'big'), 16
+        elif size == 0:
+            # Only a last box after the file type may run to the end.
+            size = len(data) - offset
+        if size < header:
+            return None
+        kinds.add(kind)
+        if kind == b'mdat':
+            samples.append((offset + header, offset + size))
+        offset += size
+    # A movie has its moov box; a still image (HEIF/AVIF) has meta.
+    if offset != len(data) or not kinds & {b'moov', b'meta'}:
+        return None
+    return offset, samples
+
+
+def ebml_number(data, offset, marker):
+    # A variable-length EBML number; the marker bit stays in IDs, not sizes.
+    if offset >= len(data) or not data[offset]:
+        return None, offset
+    length = 9 - data[offset].bit_length()
+    if offset + length > len(data):
+        return None, offset
+    value = int.from_bytes(data[offset:offset + length], 'big')
+    if not marker:
+        value &= (1 << (7 * length)) - 1
+        if value == (1 << (7 * length)) - 1:
+            value = -1
+    return value, offset + length
+
+
+SEGMENT = 0x18538067
+CLUSTER = 0x1F43B675
+# Segment children: a Cluster of unknown size ends at the next one of these.
+SEGMENT_CHILDREN = {CLUSTER, 0x114D9B74, 0x1549A966, 0x1654AE6B, 0x1C53BB6B,
+                    0x1941A469, 0x1043A770, 0x1254C367}
+
+
+def ebml_element(data, offset):
+    element, content = ebml_number(data, offset, True)
+    size, content = ebml_number(data, content, False)
+    return element, content, size
+
+
+def open_cluster(data, offset, end):
+    # A live Cluster of unknown size runs to the next Segment child.
+    start = offset
+    while offset < end:
+        element, content, size = ebml_element(data, offset)
+        if element in SEGMENT_CHILDREN:
+            break
+        if element is None or size is None or size == -1:
+            return None
+        offset = content + size
+    return offset - start
+
+
+def webm_segment(data, offset, end):
+    # Each Cluster's blocks are the samples; other Segment children are not.
+    samples = []
+    while offset < end:
+        element, content, size = ebml_element(data, offset)
+        if element is None or size is None:
+            return None
+        if size == -1:
+            size = open_cluster(data, content, end) if element == CLUSTER else None
+            if size is None:
+                return None
+        if element == CLUSTER:
+            samples.append((content, content + size))
+        offset = content + size
+    return samples if offset == end else None
+
+
+def webm(data):
+    if not data.startswith(b'\x1a\x45\xdf\xa3') or b'webm' not in data[:64]:
+        return None
+    offset, samples = 0, []
+    while offset < len(data):
+        element, content, size = ebml_element(data, offset)
+        if element is None or size is None:
+            return None
+        # Only a live Segment may leave its size unknown: it runs to the end.
+        if size == -1:
+            if element != SEGMENT:
+                return None
+            size = len(data) - content
+        if element == SEGMENT:
+            segment = webm_segment(data, content, content + size)
+            if segment is None:
+                return None
+            samples += segment
+        offset = content + size
+    return (offset, samples) if offset == len(data) else None
+
+
+def ico(data):
+    if len(data) < 6 or data[:4] != b'\0\0\1\0':
+        return None
+    count = int.from_bytes(data[4:6], 'little')
+    images = []
+    for index in range(count):
+        entry = 6 + 16 * index
+        if entry + 16 > len(data):
+            return None
+        images.append((int.from_bytes(data[entry + 12:entry + 16], 'little'),
+                       int.from_bytes(data[entry + 8:entry + 12], 'little')))
+    # Image data follows the directory with no gap. An embedded PNG keeps its
+    # own compressed samples; bitmap pixels are stored as they are.
+    offset = 6 + 16 * count
+    samples = []
+    for start, size in sorted(images):
+        if start != offset or not size:
+            return None
+        image = png(data[start:start + size])
+        if image and image[0] == size:
+            samples += [(start + a, start + b) for a, b in image[1]]
+        offset += size
+    return (offset, samples) if count else None
+
+
+FORMATS = (gif, png, jpeg, webp, iso_media, webm, ico)
+
+
+def sample_spans(data):
+    """The pixel and sample spans of a complete media file, or None."""
+    for walk in FORMATS:
+        walked = walk(data)
+        if walked and walked[0] <= len(data) and not data[walked[0]:].strip(b'\0'):
+            return [(start, min(end, walked[0])) for start, end in walked[1]]
+    return None