@runeditrun/rer 0.0.0-stage → 0.1.1

package/scripts/publish-check/publication-config.json

@@ -0,0 +1,12 @@
+{
+  "schemaVersion": 1,
+  "intendedPublicRepositories": [
+    { "repository": "just-every/cradle", "reason": "Owner-declared public identity; created by clean-history export at the first public tag." },
+    { "repository": "just-every/penny", "reason": "Owner-declared public identity; created by clean-history export at the first public tag." },
+    { "repository": "just-every/mango", "reason": "Owner-declared public identity; created by clean-history export at the first public tag." },
+    { "repository": "just-every/lamp", "reason": "Owner-declared public identity; created by clean-history export at the first public tag." },
+    { "repository": "just-every/dial", "reason": "Owner-declared public identity; created by clean-history export at the first public tag." },
+    { "repository": "just-every/orbit", "reason": "Owner-declared public identity; created by clean-history export at the first public tag." },
+    { "repository": "just-every/uptime-monitor", "reason": "Owner-declared public identity; created by clean-history export at the first public tag." }
+  ]
+}

package/scripts/publish-check/publication_config.py

@@ -0,0 +1,34 @@
+"""Exact owner-declared future public identities, maintained by Base only."""
+import json
+from pathlib import Path
+from github_refs import FULL
+
+
+def unique_object(pairs):
+    result = {}
+    for key, value in pairs:
+        if key in result:
+            raise ValueError('duplicate publication configuration key')
+        result[key] = value
+    return result
+
+
+def intended_public_repositories():
+    path = Path(__file__).with_name('publication-config.json')
+    document = json.loads(path.read_bytes(), object_pairs_hook=unique_object)
+    if (not isinstance(document, dict) or
+            set(document) != {'schemaVersion', 'intendedPublicRepositories'} or
+            document['schemaVersion'] != 1 or
+            not isinstance(document['intendedPublicRepositories'], list)):
+        raise ValueError('invalid Base publication configuration')
+    repositories = set()
+    for entry in document['intendedPublicRepositories']:
+        if not isinstance(entry, dict) or set(entry) != {'repository', 'reason'}:
+            raise ValueError('invalid intended public repository')
+        repository, reason = entry['repository'], entry['reason']
+        if (not isinstance(repository, str) or not FULL.fullmatch(repository) or
+                repository != repository.lower() or repository in repositories or
+                not isinstance(reason, str) or not reason.strip()):
+            raise ValueError('invalid intended public repository')
+        repositories.add(repository)
+    return frozenset(repositories)

package/scripts/publish-check/report.py

@@ -0,0 +1,52 @@
+"""Caller-compatible, value-free publication reports."""
+from policy import safe_path
+from reviewed import FORBIDDEN
+
+
+def new_report(public_base=False):
+    return dict(schema='rer.publish-check/v3', status='INCOMPLETE', complete=False, source=None,
+                scanner=dict(name='gitleaks', version='8.30.1'),
+                history=dict(scope='selected-and-public-base' if public_base else 'all-local-refs-and-selected-commit',
+                             commits=0, trees=0, blobs=0, commitMessages=0, tagMessages=0),
+                findings=[], preExisting=[], warnings=[], fixtures=[], reviewed=[], info=[],
+                guarantees=dict(historyScanned=False, visualRecognition=False, environmentChecked=False))
+
+
+class Findings:
+    def __init__(self, report, reviewed, env_hashes):
+        self.report, self.reviewed, self.env_hashes = report, reviewed, env_hashes
+        self.seen = {}
+
+    def add(self, rule, path, commit=None, oid=None, scope='newly-public', bucket=None, scanner_rule=None):
+        bucket = bucket or ('preExisting' if scope == 'pre-existing' and rule != 'secret.env-match' else 'findings')
+        if rule in ('pending-public-destination', 'provenance.declared-source'):
+            bucket = 'info'
+        if (rule, path, oid) in self.reviewed and bucket != 'info':
+            if rule in FORBIDDEN:
+                self.add('reviewed.forbidden', path, commit, oid, scope)
+            else:
+                bucket = 'reviewed'
+        key = (bucket, rule, path, oid, scanner_rule)
+        if key in self.seen:
+            item = self.seen[key]
+            if commit not in item['commits']:
+                item['commits'].append(commit)
+            if scope not in item['scopes']:
+                item['scopes'].append(scope)
+            return
+        item = dict(rule=rule, path=safe_path(path, self.env_hashes), commit=commit, commits=[commit], scope=scope, scopes=[scope])
+        if oid:
+            item['blob'] = oid
+        if scanner_rule:
+            item['scannerRule'] = scanner_rule
+        if bucket == 'info':
+            item['severity'] = 'INFO'
+        self.seen[key] = item
+        self.report[bucket].append(item)
+
+
+def finish(report):
+    report['complete'] = not any(f['rule'].startswith('scan.') for bucket in ('findings', 'preExisting') for f in report[bucket])
+    report['guarantees']['historyScanned'] = report['complete']
+    report['status'] = 'INCOMPLETE' if not report['complete'] else 'FAIL' if report['findings'] else 'PASS'
+    return report

package/scripts/publish-check/repository_commands.py

@@ -0,0 +1,75 @@
+"""Bound shell command arguments by their surrounding source/prose context."""
+import re
+from command_arguments import arguments
+from command_source_quotes import SourceQuotes
+
+COMMAND = re.compile(r'(?i)(?<![\w-])(?:git[ \t]+clone|gh[ \t]+repo[ \t]+(?:clone|view|fork))[ \t]+')
+QUOTES = re.compile(r'`+|\"{3}|\'{3}|[\"\']')
+
+
+def enclosing_quotes(text, stops):
+    """Walk quote scopes once, yielding the innermost scope at each command."""
+    delimiters, index = [], 0
+    source_quotes = SourceQuotes(text)
+    for stop in stops:
+        while index < stop:
+            if text[index] == '\\':
+                index += 2
+                continue
+            delimiter, opened = delimiters[-1] if delimiters else (None, 0)
+            if delimiter:
+                if text[index] == '\n' and delimiter in ("'", '"'):
+                    delimiters.pop()
+                    index += 1
+                    continue
+                if text.startswith(delimiter, index):
+                    index += len(delimiter)
+                    delimiters.pop()
+                    continue
+                if not delimiter.startswith('`'):
+                    index += 1
+                    continue
+                # Inline code can contain source strings. The innermost source
+                # quote bounds its command before another argument or funding URL.
+            match = QUOTES.match(text, index)
+            # An already-open quote closes above, even when it follows a word.
+            apostrophe = match and match[0] == "'" and index and text[index - 1].isalnum()
+            if apostrophe:
+                start = opened + len(delimiter) if delimiter else 0
+                end = context_end(text, start, delimiter) if delimiter else len(text)
+                apostrophe = not source_quotes.contains(index, start, end)
+            if match and not apostrophe:
+                delimiters.append((match[0], index))
+                index = match.end()
+            else:
+                index += 1
+        yield delimiters[-1][0] if delimiters else None
+
+
+def context_end(text, start, delimiter, stop=None):
+    stop = len(text) if stop is None else stop
+    if delimiter:
+        index = start
+        while index < stop:
+            if text[index] == '\\':
+                index += 2
+            elif text.startswith(delimiter, index):
+                return index
+            else:
+                index += 1
+    return stop
+
+
+def commands(text):
+    """Retain complete arguments even if later surrounding prose is malformed."""
+    text = text.replace('\\\n', ' ')
+    matches = list(COMMAND.finditer(text))
+    scopes = enclosing_quotes(text, (match.start() for match in matches))
+    for match, delimiter in zip(matches, scopes):
+        line_end = text.find('\n', match.end())
+        if line_end < 0:
+            line_end = len(text)
+        end = context_end(text, match.end(), delimiter, line_end)
+        # Options may name reference repositories too. Shell separators
+        # and the outer text delimiter bound their repository intent.
+        yield ' '.join(arguments(text[match.end():end]))

package/scripts/publish-check/repository_intent.py

@@ -0,0 +1,52 @@
+"""Repository declaration scopes independent of source language/extension."""
+import re
+from repository_commands import commands
+
+FIELD = r'''["']?(?:repository|repo|github|sourceRepository)["']?'''
+OBJECT = re.compile(r'(?i)(?<![\w$])' + FIELD + r'\s*[:=]\s*(?P<open>\{)')
+TABLE = re.compile(r'''(?m)^\s*\[\[?\s*(?P<name>[^]\n]+)\]\]?[^\n]*''')
+DECLARATION = re.compile(r'(?im)^(?P<indent>[ \t]*)(?:-\s*)?' + FIELD + r'[ \t]*:[ \t]*(?P<value>[^\n]*)')
+
+
+def declarations(text):
+    for match in OBJECT.finditer(text):
+        start = match.start('open')
+        nesting, quote, escaped = 0, None, False
+        for end in range(start, len(text)):
+            char = text[end]
+            if quote:
+                if escaped:
+                    escaped = False
+                elif char == '\\':
+                    escaped = True
+                elif char == quote:
+                    quote = None
+            elif char in ('"', "'", '`'):
+                quote = char
+            elif char == '{':
+                nesting += 1
+            elif char == '}':
+                nesting -= 1
+                if nesting == 0:
+                    yield text[start:end + 1]
+                    break
+    tables = list(TABLE.finditer(text))
+    for index, match in enumerate(tables):
+        names = [part.strip(' \t\'"').lower() for part in match['name'].split('.')]
+        if any(name in ('repository', 'repo', 'github', 'sourcerepository') for name in names):
+            end = tables[index + 1].start() if index + 1 < len(tables) else len(text)
+            yield text[match.end():end]
+    for match in DECLARATION.finditer(text):
+        if match['value'].lstrip().startswith('{'):
+            # Inline object boundaries were handled above; a sibling on the
+            # same line does not belong to the repository object.
+            continue
+        declaration = [match['value']]
+        indent = len(match['indent'].expandtabs())
+        for line in text[match.end():].splitlines()[1:]:
+            if not line.strip():
+                continue
+            if len(line.expandtabs()) - len(line.expandtabs().lstrip()) <= indent:
+                break
+            declaration.append(line)
+        yield '\n'.join(declaration)

package/scripts/publish-check/reviewed.py

@@ -0,0 +1,41 @@
+"""Exact committed finding identities; provider tokens and known environment values cannot be waived."""
+import json
+import re
+from datetime import date, datetime, timezone
+from hygiene_config import RULES
+
+REVIEWED = '.publish-check/reviewed.json'
+SECRET_RULES = {'secret.provider-token', 'secret.assignment', 'secret.private-key', 'secret.jwt',
+                'secret.url-credentials', 'secret.authorization', 'secret.shaped-file'}
+FORBIDDEN = {'secret.provider-token', 'secret.env-match', 'license.missing'}
+
+
+def unique_fields(pairs):
+    result = {}
+    for key, value in pairs:
+        if key in result:
+            raise ValueError('duplicate object field')
+        result[key] = value
+    return result
+
+
+def load_reviewed(data):
+    document = json.loads(data, object_pairs_hook=unique_fields)
+    if not isinstance(document, dict) or document.get('schemaVersion') != 1 or not isinstance(document.get('findings'), list):
+        raise ValueError('invalid reviewed document')
+    result = {}
+    for entry in document['findings']:
+        fields = {'rule', 'path', 'blob', 'reason', 'reviewer', 'date'}
+        if not isinstance(entry, dict) or set(entry) not in (fields, fields | {'fixtureValuesSha256'}, fields | {'containedFixtures'}) or entry['rule'] not in RULES | SECRET_RULES | FORBIDDEN | {'license.holder'}:
+            raise ValueError('invalid reviewed entry')
+        if not all(isinstance(entry[k], str) and entry[k].strip() for k in fields):
+            raise ValueError('invalid reviewed entry')
+        if not re.fullmatch(r'[0-9a-f]{40}|[0-9a-f]{64}', entry['blob']):
+            raise ValueError('invalid reviewed identity')
+        if date.fromisoformat(entry['date']) > datetime.now(timezone.utc).date():
+            raise ValueError('invalid review date')
+        key = (entry['rule'], entry['path'], entry['blob'])
+        # Older exports may repeat the same blob identity for distinct contained
+        # fixtures. Those fields no longer affect scanning; keep the review.
+        result[key] = entry
+    return result

package/scripts/publish-check/source_hashes.mjs

@@ -0,0 +1,6 @@
+// Reuse the maintained Source serializer; never approximate JSON.stringify in
+// another language. Input and output stay on private subprocess pipes.
+import fs from 'node:fs';
+import {hash, json} from '../../edition-authoring/files.mjs';
+const values = JSON.parse(fs.readFileSync(0, 'utf8'));
+process.stdout.write(JSON.stringify(values.map(value => hash(json(value)))));

package/scripts/publish-check/tree_scan.py

@@ -0,0 +1,88 @@
+"""Text hygiene and licence policy on immutable Git trees and present files."""
+import hashlib
+import os
+from pathlib import Path
+import re
+import stat
+from public_hygiene import inspect
+from provenance import MANIFESTS, declared_keys
+from project_identity import tree_identity
+from licenses import license_rules
+from git_history import InspectionError
+from trust_bundle import bundles
+
+DEPENDENCY_DIRS = {'.git', 'node_modules'}
+
+
+def environment_match(data, hashes):
+    # Known values are compared in memory. No value or digest enters a report.
+    candidates = [data, *re.findall(rb'[A-Za-z0-9_./+\-]{8,}(?:={1,2})?', data)]
+    return any(hashlib.sha256(value).digest() in hashes for value in candidates)
+
+
+def context(history, commit, config):
+    entries = history.entries(commit)
+    local = set()
+    for entry in entries:
+        parts = entry['path'].split('/')
+        local.update('/'.join(parts[:n]) for n in range(1, len(parts) + 1))
+    declared = set()
+    for entry in history.root_entries(commit):
+        if entry['path'] in MANIFESTS and entry['kind'] == 'blob':
+            declared.update(declared_keys(history.blob(entry['oid'])))
+    identity, invalid = tree_identity(history, history.root_entries(commit))
+    blobs = {entry['path']: entry['oid'] for entry in entries if entry['kind'] == 'blob'}
+    trust = bundles(blobs, lambda path: history.blob(blobs[path]))
+    return {**config, 'localPaths': frozenset(local), 'declaredSources': frozenset(declared), 'projectIdentity': identity,
+            'trustBundles': trust}, invalid
+
+
+def text_rules(data, path, config, resolver, env_hashes):
+    rules, provenance = inspect(data, path, config, resolver)
+    path_rules, _ = inspect(path.encode('utf-8', 'surrogateescape'), '@path', config, resolver)
+    rules.update(path_rules)
+    if env_hashes and (environment_match(data, env_hashes) or environment_match(path.encode(), env_hashes)):
+        rules.add('secret.env-match')
+    if data.startswith(b'version https://git-lfs.github.com/spec/v1'):
+        rules.add('scan.lfs-uninspected')
+    return rules, provenance
+
+
+def working_files(repo):
+    for directory, children, files in os.walk(repo, followlinks=False):
+        children[:] = [name for name in children if name not in DEPENDENCY_DIRS]
+        for name in files:
+            if name == '.git':
+                continue
+            target = Path(directory) / name
+            path = target.relative_to(repo).as_posix()
+            try:
+                data = os.fsencode(os.readlink(target)) if target.is_symlink() else target.read_bytes()
+            except OSError:
+                raise InspectionError('scan.working-tree-unreadable') from None
+            yield path, data
+
+
+def working_regular(repo, path):
+    # Anchor reads to opened directories and refuse links at every step.
+    # The file descriptor's bytes and mode prove the same regular preimage.
+    directory = None
+    try:
+        flags = os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW
+        directory = os.open(repo, flags)
+        parts = path.split('/')
+        for part in parts[:-1]:
+            child = os.open(part, flags, dir_fd=directory)
+            os.close(directory)
+            directory = child
+        descriptor = os.open(parts[-1], os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, dir_fd=directory)
+        with os.fdopen(descriptor, 'rb') as file:
+            status = os.fstat(file.fileno())
+            if not stat.S_ISREG(status.st_mode):
+                return None
+            return file.read(), '100755' if status.st_mode & 0o111 else '100644'
+    except OSError:
+        return None
+    finally:
+        if directory is not None:
+            os.close(directory)

package/scripts/publish-check/trust_bundle.py

@@ -0,0 +1,87 @@
+"""Public CA metadata inside pinned WebPKI trust-bundle artifacts.
+
+A committed trust-bundle.json, regenerated by the build that produces its
+artifacts, pins each artifact's SHA-256 and lists the SHA-256 of every compiled
+trust anchor's subject Name and SubjectPublicKeyInfo DER. Its webpki-roots
+release must be the one the same directory's Cargo.lock locks. Only attribute
+values of a subject Name parsed from such an artifact, whose anchor's subject
+and public key both occur there as a listed pair, are certificate metadata.
+"""
+from collections import namedtuple
+from functools import lru_cache
+import hashlib
+import json
+import re
+from der import names, public_keys
+
+MANIFEST = 'trust-bundle.json'
+REGISTRY = 'registry+https://github.com/rust-lang/crates.io-index'
+DIGEST = re.compile(r'[0-9a-f]{64}')
+ARTIFACT = re.compile(r'[A-Za-z0-9_][A-Za-z0-9_.-]*')
+Bundle = namedtuple('Bundle', 'digest anchors')
+
+
+def locked_packages(lock):
+    packages, current = [], None
+    for line in lock.decode('utf-8').splitlines():
+        if line == '[[package]]':
+            current = {}
+            packages.append(current)
+        elif line.startswith('['):
+            current = None
+        elif current is not None and ' = "' in line and line.endswith('"'):
+            key, value = line.split(' = "', 1)
+            current[key] = value[:-1]
+    return packages
+
+
+@lru_cache(maxsize=16)
+def parse(manifest, lock):
+    try:
+        record = json.loads(manifest)
+        roots, artifacts, anchors = record['roots'], record['artifacts'], record['anchors']
+        if (set(record) != {'schemaVersion', 'roots', 'artifacts', 'anchors'} or record['schemaVersion'] != 1 or
+                set(roots) != {'crate', 'version', 'checksum'} or roots['crate'] != 'webpki-roots' or
+                not isinstance(roots['version'], str) or not DIGEST.fullmatch(roots['checksum']) or
+                not artifacts or not all(isinstance(name, str) and ARTIFACT.fullmatch(name) and isinstance(digest, str) and DIGEST.fullmatch(digest)
+                                         for name, digest in artifacts.items()) or
+                not anchors or not all(isinstance(a, dict) and set(a) == {'subject', 'spki'} and
+                                       all(isinstance(v, str) and DIGEST.fullmatch(v) for v in a.values()) for a in anchors)):
+            return None
+        locked = [p for p in locked_packages(lock) if p.get('name') == roots['crate']]
+    except (ValueError, TypeError, KeyError, AttributeError, UnicodeError):
+        return None
+    if len(locked) != 1 or locked[0] != dict(name=roots['crate'], version=roots['version'], source=REGISTRY, checksum=roots['checksum']):
+        return None
+    pairs = frozenset((a['subject'], a['spki']) for a in anchors)
+    return tuple((name, Bundle(digest, pairs)) for name, digest in artifacts.items())
+
+
+def bundles(paths, read):
+    """{artifact path: Bundle} declared by valid manifests among a tree's paths."""
+    result = {}
+    for path in paths:
+        directory, _, name = path.rpartition('/')
+        prefix = directory + '/' if directory else ''
+        if name != MANIFEST or prefix + 'Cargo.lock' not in paths:
+            continue
+        for artifact, bundle in parse(read(path), read(prefix + 'Cargo.lock')) or ():
+            result[prefix + artifact] = bundle
+    return result
+
+
+def anchor_values(data, bundle):
+    """Attribute value spans of verified trust-anchor subjects in exact pinned bytes."""
+    if hashlib.sha256(data).hexdigest() != bundle.digest:
+        return []
+    subjects = {subject for subject, _ in bundle.anchors}
+    keys = {spki for _, spki in bundle.anchors}
+    found = {}
+    for start, end, values in names(data):
+        digest = hashlib.sha256(data[start:end]).hexdigest()
+        if digest in subjects:
+            found.setdefault(digest, []).append(values)
+    present = {digest for start, end in public_keys(data)
+               if (digest := hashlib.sha256(data[start:end]).hexdigest()) in keys}
+    return [span for subject, spki in bundle.anchors if spki in present
+            for values in found.get(subject, ()) for span in values]

package/scripts/publish-check.mjs

@@ -0,0 +1,58 @@
+#!/usr/bin/env node
+import { spawn } from 'node:child_process';
+import { mkdtempSync, readFileSync, openSync, closeSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+
+const args = process.argv.slice(2);
+let seconds = 300;
+for (let i = 0; i < args.length; i++) {
+  const raw = args[i] === '--timeout-seconds' ? args[i + 1] : args[i].startsWith('--timeout-seconds=') ? args[i].slice(18) : null;
+  if (raw !== null && Number(raw) > 0 && Number(raw) <= 7200) seconds = Number(raw);
+}
+const scratch = mkdtempSync(join(tmpdir(), 'publish-check-'));
+const output = join(scratch, 'report.json');
+const fd = openSync(output, 'wx', 0o600);
+let child, reason, timer;
+const stop = value => {
+  reason ??= value;
+  // The child remains unreaped until its close handler; descendants (native
+  // Git and Gitleaks) share the private process group and this total deadline.
+  if (child?.pid) {
+    try { process.kill(-child.pid, 'SIGKILL'); } catch { /* already stopped */ }
+  }
+};
+const interrupt = () => stop('scan.interrupted');
+process.on('SIGINT', interrupt);
+process.on('SIGTERM', interrupt);
+let report, code = 2;
+try {
+  child = spawn('python3', ['-B', new URL('./publish-check/cli.py', import.meta.url).pathname, ...args], {
+    detached: true, stdio: ['ignore', fd, 'ignore'], env: { ...process.env, TMPDIR: scratch },
+  });
+  closeSync(fd);
+  timer = setTimeout(() => stop('scan.deadline-exceeded'), seconds * 1000);
+  const result = await new Promise(resolve => {
+    child.once('error', () => resolve(null));
+    child.once('close', value => resolve(value));
+  });
+  if (!reason && [0, 1, 2].includes(result)) {
+    report = JSON.parse(readFileSync(output, 'utf8'));
+    code = result;
+  } else reason ??= 'scan.worker-incomplete';
+} catch { reason ??= 'scan.worker-incomplete'; }
+finally {
+  clearTimeout(timer);
+  process.removeListener('SIGINT', interrupt);
+  process.removeListener('SIGTERM', interrupt);
+  rmSync(scratch, { recursive: true, force: true });
+}
+if (reason) report = {
+  schema: 'rer.publish-check/v3', status: 'INCOMPLETE', complete: false, source: null,
+  findings: [{ rule: reason, path: '.', commit: null, scope: 'newly-public' }],
+  preExisting: [], warnings: [], fixtures: [], reviewed: [], info: [],
+  history: { commits: 0, trees: 0, blobs: 0 },
+  guarantees: { historyScanned: false, visualRecognition: false, environmentChecked: false },
+};
+process.stdout.write(JSON.stringify(report) + '\n');
+process.exitCode = code;