@runeditrun/rer 0.0.0-stage → 0.1.0

package/scripts/publish-check/font_licence.py

@@ -0,0 +1,23 @@
+"""The copyright notice at the top of a SIL Open Font License text."""
+import re
+
+# The licence text itself, not a path or a mention of the licence: its title
+# line followed by sections only the full text carries.
+TITLE = re.compile(r'(?im)^[ \t]*SIL OPEN FONT LICENSE[ \t]+Version[ \t]+1\.1\b.*$')
+BODY = ('PERMISSION & CONDITIONS', 'Reserved Font Name', 'Font Software')
+# An attribution line names a year after the word: "Copyright (c) 2014 Holder".
+# Other prose that starts with Copyright, such as a contact line, stays scanned.
+NOTICE = re.compile(r'(?im)^[ \t]*Copyright[ \t]+(?:(?:\(c\)|©)[ \t]*)?\d{4}\b.*$')
+
+
+def without_required_notices(text):
+    """Drop the notice lines above the licence title of a full OFL text.
+
+    The licence obliges a redistributor to keep these lines, so the address in
+    one is required attribution. Anything else in the file, and the same
+    notice in a file that is not an OFL text, stays visible.
+    """
+    title = TITLE.search(text)
+    if not title or not all(part.lower() in text[title.end():].lower() for part in BODY):
+        return text
+    return NOTICE.sub('', text[:title.start()]) + text[title.start():]

package/scripts/publish-check/git_history.py

@@ -0,0 +1,115 @@
+"""Repository plumbing delegated to the installed Git, never decoded in Python."""
+import os
+from pathlib import Path
+import subprocess
+from public_base import parse, display, PublicBaseError
+
+
+class InspectionError(Exception):
+    pass
+
+
+def git_environment():
+    env = {k: v for k, v in os.environ.items() if not k.startswith('GIT_')}
+    env.update(GIT_TERMINAL_PROMPT='0', GIT_NO_REPLACE_OBJECTS='1')
+    return env
+
+
+class History:
+    def __init__(self, repo, ref='HEAD', public_base=None):
+        self.repo = str(Path(repo).resolve())
+        self.env = git_environment()
+        self.public_commit = self.public_label = None
+        self.reader = None
+        self.commit = self.run('rev-parse', '--verify', '--end-of-options', ref + '^{commit}').decode().strip()
+        if self.run('rev-parse', '--is-shallow-repository').strip() != b'false':
+            raise InspectionError('history.shallow')
+        if public_base is not None:
+            url, branch = parse(public_base, self.repo)
+            try:
+                self.run('fetch', '--quiet', '--no-tags', '--no-recurse-submodules', '--no-write-fetch-head', '--',
+                         url, 'refs/heads/' + branch + ':refs/publish-check/public/base')
+                self.public_commit = self.run('rev-parse', 'refs/publish-check/public/base^{commit}').decode().strip()
+                self.public_label = display(url) + '#' + branch
+            except InspectionError:
+                raise PublicBaseError('public-base.fetch-failed') from None
+        self.revisions = [self.commit, self.public_commit] if public_base else ['--all', self.commit]
+        self.commits = self.run('rev-list', *self.revisions).decode().splitlines()
+        self.public_commits = set(self.run('rev-list', self.public_commit).decode().splitlines()) if self.public_commit else set()
+        self.trees = {}
+        self.tree_ids = dict(line.split() for line in self.run('log', '--format=%H %T', *self.revisions).decode().splitlines())
+        self.reader = subprocess.Popen(['git', '-C', self.repo, 'cat-file', '--batch'],
+                                       env=self.env, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
+
+    def run(self, *args):
+        try:
+            result = subprocess.run(['git', '-C', self.repo, '-c', 'core.quotePath=false', *args],
+                                    env=self.env, capture_output=True, timeout=120)
+            if result.returncode:
+                raise InspectionError('history.git-failed')
+            return result.stdout
+        except (OSError, subprocess.TimeoutExpired):
+            raise InspectionError('history.git-unavailable') from None
+
+    def commit_tree(self, commit):
+        known = getattr(self, 'tree_ids', {}).get(commit)
+        return known.encode() if known else self.run('rev-parse', commit + '^{tree}').strip()
+
+    def entries(self, commit, recursive=True):
+        tree = self.commit_tree(commit).decode()
+        key = (tree, recursive)
+        if key not in self.trees:
+            args = ['ls-tree', '-z', '-l'] + (['-r'] if recursive else []) + [tree]
+            entries = []
+            for record in self.run(*args).split(b'\0'):
+                if not record:
+                    continue
+                metadata, path = record.split(b'\t', 1)
+                mode, kind, oid, size = metadata.split()
+                entries.append(dict(path=os.fsdecode(path), mode=mode.decode(), kind=kind.decode(),
+                                    oid=oid.decode(), size=int(size) if size != b'-' else 0))
+            self.trees[key] = entries
+        return self.trees[key]
+
+    def root_entries(self, commit):
+        return self.entries(commit, False)
+
+    def blob(self, oid):
+        if not oid or any(c not in '0123456789abcdef' for c in oid):
+            raise InspectionError('history.invalid-object')
+        self.reader.stdin.write(oid.encode() + b'\n')
+        self.reader.stdin.flush()
+        header = self.reader.stdout.readline().split()
+        if len(header) != 3 or header[1] != b'blob':
+            raise InspectionError('history.missing-blob')
+        size = int(header[2])
+        data = self.reader.stdout.read(size)
+        if len(data) != size or self.reader.stdout.read(1) != b'\n':
+            raise InspectionError('history.missing-blob')
+        return data
+
+    def selected(self, path):
+        entry = next((e for e in self.entries(self.commit) if e['path'] == path), None)
+        return self.blob(entry['oid']) if entry and entry['kind'] == 'blob' else None
+
+    def messages(self):
+        records = self.run('log', '-z', '--format=%H%x00%B', *self.revisions).split(b'\0')
+        for index in range(0, len(records) - 1, 2):
+            if records[index]:
+                yield '@commit-message', records[index].decode().strip(), records[index + 1]
+        for record in self.run('for-each-ref', '--format=%(objecttype) %(objectname)', 'refs/tags').splitlines():
+            kind, oid = record.decode().split()
+            if kind == 'tag':
+                raw = self.run('cat-file', 'tag', oid)
+                yield '@tag-message', oid, raw.partition(b'\n\n')[2]
+
+    def close(self):
+        if self.reader:
+            self.reader.stdin.close()
+            self.reader.stdout.close()
+            try:
+                self.reader.wait(timeout=2)
+            except subprocess.TimeoutExpired:
+                self.reader.kill()
+                self.reader.wait()
+            self.reader = None

package/scripts/publish-check/github_refs.py

@@ -0,0 +1,217 @@
+"""Resolve publication references with gh's local credentials; retain booleans only."""
+import json
+import os
+import re
+import subprocess
+from repository_intent import declarations, commands
+
+SLUG = r'[A-Za-z0-9][A-Za-z0-9_.-]{0,99}'
+PAIR = SLUG + '/' + SLUG
+URL_BOUNDARY = r'(?<![A-Za-z0-9_./@+-])'
+TRANSPORT = r'(?:[a-z][a-z0-9+.-]*://|(?<!:)//)'
+USERINFO = r'''(?:[^/\s@"'<>`]+@)?'''
+PORT = r'(?::[0-9]+)?'
+REPOSITORY_HOSTS = {'github.com': '/', 'www.github.com': '/', 'ssh.github.com': '/',
+                    'raw.githubusercontent.com': '/', 'api.github.com': '/repos/',
+                    'codeload.github.com': '/'}
+REPOSITORY_HOST = '(?:' + '|'.join('(?a:' + re.escape(host) + ')' + PORT + route for host, route in REPOSITORY_HOSTS.items()) + ')'
+# Embedded patterns retain ASCII scheme/host folding without changing the
+# surrounding text and repository-path recognition used by privacy inspection.
+URL = re.compile(r'(?i)' + URL_BOUNDARY + r'(?:(?a:' + TRANSPORT + ')' + USERINFO + REPOSITORY_HOST +
+                 r'|(?a:git@github\.com:))(' + PAIR + r')(?![A-Za-z0-9_.-])')
+FULL = re.compile(PAIR + r'\Z')
+METADATA = re.compile(r'(?i)(?:^|/)(?:[^/]+\.(?:json|ya?ml|toml)|\.gitmodules)$')
+# GitHub documents these web auth endpoints, not repository namespaces.
+SERVICE_URL = re.compile(r'(?i)' + URL_BOUNDARY + r'(?a:https?://|(?<!:)//)' + USERINFO +
+                         r'(?a:(?:www\.)?github\.com)' + PORT +
+                         r'''/(?:login/oauth/(?:authorize|access_token)|login/device(?:/code)?)(?=[?#\s"'`<>)]|$)''')
+# Only the complete web endpoint is a service; transports and repository
+# declarations retain their original repository meaning.
+SPONSORS_URL = re.compile(URL_BOUNDARY + r'(?a:https?://|(?<!:)//)' + USERINFO +
+                          r'(?a:(?:www\.)?github\.com)' + PORT +
+                          r'''/(?-i:sponsors)/(?a:[A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?)/?(?=[?#\s"'`<>)]|$)''', re.I)
+FILE_SUFFIX = re.compile(r'(?i)\.(?:xml|json|ya?ml|toml|[cm]?[jt]sx?|md|txt|html|css|sql|png|jpe?g|svg|zip|csv|pdf|docx|xlsx|rer)$')
+DEPENDENCY = re.compile(r'(?i)\bgithub:(' + PAIR + r')(?![A-Za-z0-9_./-])')
+ACTION = re.compile(r'''(?im)^\s*(?:-\s*)?uses\s*:\s*["']?(''' + PAIR + r''')(?:/[A-Za-z0-9_./-]+)?@[^\s"']+''')
+REPOSITORY_FIELD = re.compile(r'''(?im)(?<![\w$])["'`]?(?:repository|repo|github|sourceRepository)["'`]?\s*[:=]\s*["'`]?(''' + PAIR + r''')(?![A-Za-z0-9_./-])''')
+# A code span such as `dist/client` names a path in the project, not an owner/repo.
+LOCAL_DIRECTORIES = frozenset({'dist', 'src', 'scripts', 'public', 'tests', 'docs', 'exports', 'node_modules', '.rer', '.seed'})
+SOURCE = re.compile(r'(?i)\.[cm]?[jt]sx?$')
+TYPED_REPOSITORY = re.compile(r'''(?im)(?<![\w$])(?:repository|repo|github|sourceRepository)\s*:\s*string\s*=\s*["'`](''' + PAIR + r''')["'`]''')
+DECLARED_SHORTHAND = re.compile(r'''(?im)(?<![\w$])["'`]?(?:url|fullURL)["'`]?\s*[:=]\s*["'`](''' + PAIR + r''')["'`]''')
+REPOSITORY_ROUTE = re.compile(r'/(?:tree|blob|issues|pull|pulls|commit|commits|releases|actions|wiki|discussions|compare|tags)(?:/|$)')
+SHORTHAND = PAIR + r'(?:[/?#][^\s`<>]*)?'
+REPOSITORY_CONTEXT = r'github|repository|repo|source|upstream|downstream|fork(?:ed)?(?:\s+(?:from|of|into))?|clone(?:d)?(?:\s+from)?|gh\s+repo\s+(?:clone|view|fork)'
+MARKUP = r'[`*_~\[\]]*'
+MARKDOWN_REFERENCE = re.compile(r'`(?P<code>' + SHORTHAND + r')`|(?<![A-Za-z])(?P<label>(?i:' + REPOSITORY_CONTEXT + r'))(?:\s+(?:repository|repo))?' + MARKUP + r'(?P<separator>\s*[:=|]\s*|\s+)(?P<markup>' + MARKUP + r')\s*(?P<reference>' + SHORTHAND + r')(?![A-Za-z0-9_./-])')
+
+EXPLICIT_URL = re.compile(r'''(?im)["']?(?:repository|repo|github|sourceRepository)["']?\s*[:=]\s*(?:\{\s*["']?(?:url|fullURL)["']?\s*:\s*)?[`*_~\["']*\s*(''' + URL.pattern.replace('(?i)', '') + ')', re.I)
+CONTEXT_URL = re.compile(r'(?i)(?<![A-Za-z])(?:' + REPOSITORY_CONTEXT + r')'
+                         r'(?:\s+(?:repository|repo))?' + MARKUP + r'(?:\s*[:=|]\s*|\s+)'
+                         + r'''[`*_~\[\]\s"']*(?:''' + URL.pattern.replace('(?i)', '') + ')')
+LINK_URL = re.compile(r'(?i)\[(?P<label>[^]\n]+)\]\(\s*(?P<url>' + URL.pattern.replace('(?i)', '') + ')')
+REPOSITORY_LABEL = re.compile(r'(?i)\b(?:' + REPOSITORY_CONTEXT + r')\b')
+
+
+def explicit_references(text, path):
+    found = {normalize(m.group(2)) for m in EXPLICIT_URL.finditer(text)}
+    found.update(normalize(m.group(1)) for m in CONTEXT_URL.finditer(text))
+    for match in LINK_URL.finditer(text):
+        if REPOSITORY_LABEL.search(match['label']):
+            found.update(normalize(m.group(1)) for m in URL.finditer(match['url']))
+    for declaration in (*declarations(text), *commands(text)):
+        found.update(normalize(m.group(1)) for m in URL.finditer(declaration))
+    return found
+
+
+def normalize(value):
+    return value.lower().rstrip('.').removesuffix('.git')
+
+
+class RepositoryObject(dict):
+    """Retain duplicate JSON fields when checking explicit reference context."""
+    def __init__(self, pairs):
+        super().__init__(pairs)
+        self.pairs = pairs
+
+
+def references(text, path, local_paths=frozenset(), project_identity=None):
+    repository_text = SERVICE_URL.sub('', text)
+    found = {normalize(m.group(1)) for m in URL.finditer(repository_text)
+             if not SPONSORS_URL.match(repository_text, m.start())}
+    # A URL in an explicit repository declaration has stronger context than
+    # its coincident service spelling. Never erase the original text first.
+    found.update(explicit_references(repository_text, path))
+    found.update(normalize(m.group(1)) for m in DEPENDENCY.finditer(text))
+    if path.lower().endswith(('.yml', '.yaml', '.md', '.markdown')):
+        found.update(normalize(m.group(1)) for m in ACTION.finditer(text))
+    # Repository fields avoid mistaking module IDs for repository references.
+    # Markdown slash prose needs an explicit code span or repository context.
+    if path.lower().endswith(('.md', '.markdown')) or path in ('@commit-message', '@tag-message'):
+        markdown = URL.sub('', text)
+        markdown = re.sub(r'\]\((?![A-Za-z][A-Za-z0-9+.-]*://)[^)]*\)', ']()', markdown)
+        for match in MARKDOWN_REFERENCE.finditer(markdown):
+            explicit = match['reference'] is not None
+            value = re.split(r'[?#]', match['reference'] or match['code'], maxsplit=1)[0]
+            # Underscores are also legal slug characters; remove a closing
+            # emphasis marker only when the reference has its matching opener.
+            emphasis = re.search(r'_+$', match['markup'] or '')
+            if emphasis and value.endswith(emphasis.group()):
+                value = value[:-len(emphasis.group())]
+            repository = '/'.join(value.split('/')[:2])
+            suffix = value[len(repository):]
+            if not explicit and (value in local_paths or repository in local_paths or FILE_SUFFIX.search(repository)
+                                 or repository.split('/')[0] in LOCAL_DIRECTORIES):
+                continue
+            # A plain multi-segment path is ambiguous; GitHub's repository
+            # routes, a labeled field or a GitHub command disambiguate it.
+            labeled = explicit and (re.search(r'[:=|]', match['separator']) or re.match(r'(?i)github|gh\s', match['label']))
+            if not suffix or REPOSITORY_ROUTE.match(suffix) or labeled:
+                found.add(normalize(repository))
+    elif METADATA.search(path) or SOURCE.search(path):
+        if path.lower().endswith('.json'):
+            try:
+                document = json.loads(text, object_pairs_hook=RepositoryObject)
+            except ValueError:
+                document = None
+            identity_field = {'edition.json': 'id', 'sources.lock.json': 'edition', 'rer-project.json': 'projectId'}.get(path.rsplit('/', 1)[-1])
+            identity = document.get(identity_field) if isinstance(document, dict) and identity_field else None
+            logical = path in ('edition.json', 'sources.lock.json', 'rer-project.json') and identity == project_identity
+            if isinstance(identity, str) and FULL.fullmatch(identity) and not logical:
+                found.add(normalize(identity))
+            def visit(value):
+                if isinstance(value, dict):
+                    for key, item in value.pairs:
+                        if key.lower() in ('repository', 'repo', 'github', 'sourcerepository'):
+                            candidate = item.get('url') if isinstance(item, dict) else item
+                            if isinstance(candidate, str) and FULL.fullmatch(candidate):
+                                found.add(normalize(candidate))
+                            def declared(item):
+                                if isinstance(item, str):
+                                    found.update(normalize(m.group(1)) for m in URL.finditer(SERVICE_URL.sub('', item)))
+                                elif isinstance(item, dict):
+                                    for _, child in item.pairs:
+                                        declared(child)
+                                elif isinstance(item, list):
+                                    for child in item:
+                                        declared(child)
+                            declared(item)
+                        visit(item)
+                elif isinstance(value, list):
+                    for item in value:
+                        visit(item)
+            visit(document)
+        found.update(normalize(m.group(1)) for m in REPOSITORY_FIELD.finditer(text))
+        if SOURCE.search(path):
+            found.update(normalize(m.group(1)) for m in TYPED_REPOSITORY.finditer(text))
+            for declaration in declarations(text):
+                found.update(normalize(m.group(1)) for m in DECLARED_SHORTHAND.finditer(declaration))
+    return found
+
+
+def local_dependencies(data, path):
+    """Cache only local-path memberships that can change this blob's meaning."""
+    if not path.lower().endswith(('.md', '.markdown')):
+        return frozenset()
+    from hygiene_text import decoded_text
+    result = set()
+    for match in MARKDOWN_REFERENCE.finditer(decoded_text(data)):
+        value = re.split(r'[?#]', match['reference'] or match['code'], maxsplit=1)[0]
+        result.update((value, '/'.join(value.split('/')[:2])))
+    return frozenset(result)
+
+
+def response_object(pairs):
+    document = {}
+    for key, value in pairs:
+        if key in document:
+            raise ValueError('duplicate API response key')
+        document[key] = value
+    return document
+
+
+def response_constant(value):
+    raise ValueError('nonstandard API JSON constant')
+
+
+def visibility(output, exit_code):
+    headers, body = re.split(r'\r?\n\r?\n', output, maxsplit=1)
+    status = re.fullmatch(r'HTTP/(?:1\.[01]|2(?:\.0)?|3(?:\.0)?) (\d{3})(?: [^\r\n]*)?',
+                          headers.splitlines()[0])
+    if not status:
+        return 'incomplete'
+    code = int(status.group(1))
+    document = json.loads(body, object_pairs_hook=response_object, parse_constant=response_constant)
+    if not isinstance(document, dict):
+        return 'incomplete'
+    if code == 200 and exit_code == 0 and type(document.get('private')) is bool:
+        return 'private' if document['private'] else 'public'
+    # gh exits 1 for HTTP errors. A header alone or an arbitrary error document
+    # is not evidence of a missing repository, even at the intended destination.
+    if (code == 404 and exit_code == 1 and document.get('message') == 'Not Found' and
+            document.get('status') == '404' and isinstance(document.get('documentation_url'), str) and
+            re.fullmatch(r'https://docs\.github\.com/rest/[^\s]+', document['documentation_url'])):
+        return 'missing'
+    return 'incomplete'
+
+
+class GitHubResolver:
+    def __init__(self, spool=None):
+        self.cache = {}
+
+    def resolve(self, repository):
+        if repository in self.cache:
+            return self.cache[repository]
+        # No disk cache, auth arguments, diagnostic output or response metadata.
+        # Disable gh debug logging even if inherited from the calling shell.
+        env = {k: v for k, v in os.environ.items() if k not in ('GH_DEBUG', 'DEBUG')}
+        env['GH_PROMPT_DISABLED'] = '1'
+        try:
+            response = subprocess.run(['gh', 'api', '--hostname', 'github.com', '--include', 'repos/' + repository],
+                                      env=env, capture_output=True, timeout=60)
+            output = response.stdout.decode('utf-8')
+            result = visibility(output, response.returncode)
+        except (OSError, subprocess.TimeoutExpired, ValueError, IndexError):
+            result = 'incomplete'
+        self.cache[repository] = result
+        return result

package/scripts/publish-check/gitleaks.py

@@ -0,0 +1,72 @@
+"""Pinned Gitleaks is the only secret detection engine."""
+import json
+import os
+from pathlib import Path
+import shutil
+import subprocess
+import tempfile
+from git_history import InspectionError
+
+VERSION = '8.30.1'
+CONFIG = Path(__file__).with_name('gitleaks.toml')
+
+
+def executable():
+    binary = shutil.which('gitleaks') or str(Path.home() / '.local/bin/gitleaks')
+    try:
+        version = subprocess.run([binary, 'version'], capture_output=True, timeout=10)
+        if version.returncode or version.stdout.decode().strip() != VERSION:
+            raise InspectionError('scan.gitleaks-version')
+    except OSError:
+        raise InspectionError('scan.gitleaks-unavailable') from None
+    return binary
+
+
+def inspect(binary, mode, repo=None, revisions=None, data=None, timeout=300, redact=True):
+    with tempfile.TemporaryDirectory(prefix='publish-gitleaks-') as directory:
+        report = Path(directory) / 'report.json'
+        report.touch(mode=0o600)
+        config = CONFIG
+        if mode == 'dir':
+            config = Path(directory) / 'directory.toml'
+            # Extending a config and defining a global allowlist replaces its
+            # global rules. Keep the maintained policy intact in both modes;
+            # append only the directory exclusions already used here.
+            config.write_text(CONFIG.read_text() + '\n[[allowlists]]\npaths = [\"(^|/)(\\\\.git|node_modules)(/|$)\"]\n')
+        args = [binary, mode, '--config', str(config), '--redact=' + ('100' if redact else '0'), '--no-banner', '--no-color',
+                '--ignore-gitleaks-allow', '--gitleaks-ignore-path', directory, '--report-format', 'json',
+                '--report-path', str(report), '--timeout', str(max(1, int(timeout)))]
+        if mode == 'git':
+            args += ['--log-opts=' + ' '.join(['--full-history', '-m', '--no-ext-diff', '--no-textconv', *revisions])]
+        if repo is not None:
+            args.append(repo)
+        env = {k: v for k, v in os.environ.items() if not k.startswith(('GITLEAKS_', 'GIT_'))}
+        env.update(GIT_NO_REPLACE_OBJECTS='1', GIT_TERMINAL_PROMPT='0')
+        try:
+            result = subprocess.run(args, input=data, capture_output=True, env=env, timeout=timeout)
+            if result.returncode not in (0, 1):
+                raise InspectionError('scan.gitleaks-failed')
+            # Native reports may contain raw matches for location-bound proof.
+            # The enclosing temporary directory is private; values never enter
+            # the caller-facing report or logs. Gitleaks replaces file mode.
+            report.chmod(0o600)
+            findings = json.loads(report.read_text())
+            if not isinstance(findings, list):
+                raise ValueError()
+            return findings
+        except subprocess.TimeoutExpired:
+            raise InspectionError('scan.deadline-exceeded') from None
+        except (OSError, ValueError):
+            raise InspectionError('scan.gitleaks-failed') from None
+
+
+def rule(identifier):
+    if identifier in ('generic-api-key', 'hex-api-key'):
+        return 'secret.assignment'
+    if 'private-key' in identifier:
+        return 'secret.private-key'
+    if 'jwt' in identifier:
+        return 'secret.jwt'
+    if 'password' in identifier or 'connection-string' in identifier:
+        return 'secret.url-credentials'
+    return 'secret.provider-token'

package/scripts/publish-check/gitleaks.toml

@@ -0,0 +1,40 @@
+title = "Base publication scan"
+[extend]
+useDefault = true
+
+[[allowlists]]
+description = "Explicit synthetic credential values; no exemption by file or directory"
+regexes = ['''(?i)^(?:test|fixture|fake|example|dummy|placeholder|your)[_-]''', '''(?i)^(?:gh[pousr]_|sk-(?:proj-|ant-)?|glpat-|npm_|whsec_|rk_live_|sk_live_|hf_|xapp-)(?:test|fixture|fake|example|dummy|placeholder)[_-]''']
+
+[[allowlists]]
+description = "Explicit synthetic values used by redaction fixtures"
+regexes = ['''(?i)^synthetic[_-]''']
+
+[[rules]]
+id = "private-key"
+[[rules.allowlists]]
+description = "A PEM-shaped fixture starts its body with a synthetic marker"
+regexes = ['''(?i)^-----BEGIN[ A-Z0-9_-]{0,100}PRIVATE KEY(?: BLOCK)?-----(?:\s|\\[nr])+(?:test|fixture|fake|example|dummy|placeholder|synthetic)[_-]''']
+
+[[rules.allowlists]]
+description = "A private-key header literal without an attached body is not a stored key"
+regexTarget = "match"
+regexes = ['''(?i)^-----BEGIN[ A-Z0-9_-]{0,100}PRIVATE KEY(?: BLOCK)?-----(?:\s|\\[nr])*['"]''']
+
+[[rules]]
+id = "jwt"
+[[rules.allowlists]]
+description = "A JWT-shaped fixture explicitly marks the signature synthetic"
+regexes = ['''\.(?:test|fixture|fake|example|dummy|placeholder|synthetic)[_-][a-zA-Z0-9_-]+$''']
+
+# SHA-shaped credentials must not disappear because their random bytes contain
+# an English stopword (for example dead/feed) or fall below generic entropy.
+# Source/module identities still require the location-bound proof in Python.
+[[rules]]
+id = "hex-api-key"
+description = "Hexadecimal credential assignment"
+regex = '''(?i)[\w.-]{0,50}?(?:access|auth|(?-i:[Aa]pi|API)|credential|creds|key|passw(?:or)?d|secret|token)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-f0-9]{64}|[a-f0-9]{40})(?:[\x60'"\s;]|\\[nr]|$)'''
+keywords = ["access", "auth", "api", "credential", "creds", "key", "passwd", "password", "secret", "token"]
+
+[[rules]]
+id = "generic-api-key"

package/scripts/publish-check/history_scan.py

@@ -0,0 +1,176 @@
+"""Scan current files and full history using native Git and pinned Gitleaks."""
+from pathlib import Path
+from git_history import History, InspectionError
+from public_base import PublicBaseError
+from gitleaks import executable, inspect as secrets, rule as secret_rule
+from hygiene_config import CONFIG, load_config
+from github_refs import GitHubResolver, local_dependencies
+from destination import destination, INVALID
+from env_values import load_hashes
+from licenses import EXCEPTIONS, exceptions, license_rules, registry
+from reviewed import REVIEWED, load_reviewed
+from report import new_report, Findings, finish
+from tree_scan import context, text_rules, working_files, working_regular
+from content_identities import ContentIdentities, candidate as identity_candidate
+from checksum_maps import ChecksumMaps
+
+
+def scan(repo, ref='HEAD', public_base=None, env_file=None, project=None, repository=None, *, cache_dir=None, timeout=300):
+    report = new_report(public_base is not None)
+    history = None
+    add = Findings(report, {}, frozenset()).add
+    try:
+        hashes = load_hashes(env_file)
+        report['guarantees']['environmentChecked'] = env_file is not None
+        binary = executable()
+        history = History(repo, ref, public_base)
+        report['source'] = dict(commit=history.commit, tree=history.commit_tree(history.commit).decode())
+        report['publicBase'] = dict(ref=history.public_label, commit=history.public_commit) if public_base else None
+        config = load_config(history.selected(CONFIG))
+        reviewed_bytes = history.selected(REVIEWED)
+        reviewed = load_reviewed(reviewed_bytes) if reviewed_bytes else {}
+        add = Findings(report, reviewed, hashes).add
+        try:
+            config['pendingDestination'] = destination(history, project, repository)
+        except (ValueError, TypeError, UnicodeError):
+            add(INVALID, 'edition.json', history.commit)
+            config['pendingDestination'] = None
+        recorded = registry() + exceptions(history.selected(EXCEPTIONS))
+        resolver = GitHubResolver()
+        public_blobs = {e['oid'] for c in history.public_commits for e in history.entries(c)}
+        report['history']['preExistingCommits'] = sorted(history.public_commits)
+        report['history']['newlyPublicCommits'] = [c for c in history.commits if c not in history.public_commits]
+        seen_trees, seen_blobs, inspected = set(), set(), {}
+        entries_by_commit, dependencies = {}, {}
+        release_license_good = not license_rules(history.selected('LICENSE'), recorded)
+        for commit in history.commits:
+            entries = history.entries(commit)
+            entries_by_commit[commit] = {e['path']: e for e in entries}
+            tree = history.commit_tree(commit).decode()
+            seen_trees.add(tree)
+            scope = 'release-tree' if commit == history.commit else 'pre-existing' if commit in history.public_commits else 'newly-public'
+            config_tree, invalid = context(history, commit, config)
+            for path in invalid:
+                add(INVALID if (project is not None or repository is not None) and path == 'edition.json' else 'scan.identity-invalid', path, commit, scope=scope)
+            license_entry = entries_by_commit[commit].get('LICENSE')
+            license_oid = license_entry['oid'] if license_entry and license_entry['mode'] in ('100644', '100755') else None
+            for rule in license_rules(history.blob(license_oid) if license_oid else None, recorded):
+                bucket = 'warnings' if rule == 'license.missing' and public_base and scope == 'newly-public' and release_license_good else None
+                add(rule, 'LICENSE', commit, license_oid, scope, bucket)
+            signature = (config_tree['declaredSources'], config_tree['projectIdentity'], frozenset(config_tree['trustBundles'].items()))
+            for entry in entries:
+                path, oid = entry['path'], entry['oid']
+                seen_blobs.add(oid)
+                if entry['kind'] != 'blob':
+                    add('scan.submodule-uninspected', path, commit, oid, scope)
+                    continue
+                pair = (path, oid)
+                if pair not in dependencies:
+                    data = history.blob(oid)
+                    dependencies[pair] = local_dependencies(data, path)
+                local = dependencies[pair] & config_tree['localPaths']
+                key = (path, oid, signature, local)
+                if key not in inspected:
+                    inspected[key] = text_rules(history.blob(oid), path, config_tree, resolver, hashes)
+                rules, provenance = inspected[key]
+                entry_scope = 'pre-existing' if scope != 'release-tree' and oid in public_blobs else scope
+                for rule in rules:
+                    add(rule, path, commit, oid, entry_scope)
+                if provenance:
+                    add('provenance.declared-source', path, commit, oid, entry_scope)
+        report['history'].update(commits=len(history.commits), trees=len(seen_trees), blobs=len(seen_blobs))
+        selected_context, _ = context(history, history.commit, config)
+        working_paths = []
+        working_identities = {}
+        for path, data in working_files(Path(history.repo)):
+            working_paths.append(path)
+            target = Path(history.repo) / path
+            if identity_candidate(path) and not any(p.is_symlink() for p in [target, *target.parents]):
+                working_identities[path] = data
+            # Native hash-object is unnecessary here: unchanged current bytes use
+            # their known immutable identity; new bytes have no history identity.
+            entry = entries_by_commit[history.commit].get(path)
+            oid = entry['oid'] if entry and entry['kind'] == 'blob' and history.blob(entry['oid']) == data else None
+            rules, _ = text_rules(data, path, selected_context, resolver, hashes)
+            for rule in rules:
+                add(rule, path, history.commit, oid, 'working-tree')
+            if path == 'LICENSE':
+                for rule in license_rules(data, recorded):
+                    add(rule, path, history.commit, oid, 'working-tree')
+        if not (Path(history.repo) / 'LICENSE').is_file():
+            add('license.missing', 'LICENSE', scope='working-tree')
+        identities = {'working-tree': ContentIdentities(working_identities,
+            lambda path: working_regular(Path(history.repo), path))}
+        checksums = {}
+        for mode in ('git', 'dir'):
+            for hit in secrets(binary, mode, history.repo, history.revisions, timeout=timeout, redact=False):
+                path, commit = hit.get('File', '.'), hit.get('Commit') or history.commit
+                if mode == 'dir' and Path(path).is_absolute():
+                    path = Path(path).resolve().relative_to(Path(history.repo).resolve()).as_posix()
+                entry = entries_by_commit.get(commit, {}).get(path)
+                oid = entry['oid'] if entry and entry['kind'] == 'blob' else None
+                scope = 'working-tree' if mode == 'dir' else 'release-tree' if commit == history.commit else 'pre-existing' if oid in public_blobs else 'newly-public'
+                if hit['RuleID'] in ('generic-api-key', 'hex-api-key'):
+                    identity_scope = 'working-tree' if mode == 'dir' else commit
+                    scoped_entries = entries_by_commit.get(commit, {})
+                    def read_regular(path, scoped_entries=scoped_entries):
+                        entry = scoped_entries.get(path)
+                        return (history.blob(entry['oid']), entry['mode']) if entry and entry['kind'] == 'blob' and entry['mode'] in ('100644', '100755') else None
+                    if identity_scope not in checksums:
+                        checksums[identity_scope] = ChecksumMaps(
+                            working_paths if mode == 'dir' else scoped_entries,
+                            (lambda path: working_regular(Path(history.repo), path)) if mode == 'dir' else read_regular)
+                    if identity_candidate(path):
+                        if identity_scope not in identities:
+                            identities[identity_scope] = ContentIdentities({
+                                p: history.blob(e['oid']) for p, e in scoped_entries.items()
+                                if identity_candidate(p) and e['kind'] == 'blob' and e['mode'] in ('100644', '100755')}, read_regular)
+                        if identities[identity_scope].matches(path, hit):
+                            continue
+                    else:
+                        # Content-identity documents keep their own stricter proof.
+                        held = working_regular(Path(history.repo), path) if mode == 'dir' else read_regular(path)
+                        if held and checksums[identity_scope].matches(held[0], hit):
+                            continue
+                add(secret_rule(hit['RuleID']), path, commit, oid, scope, scanner_rule=hit['RuleID'])
+        messages, lines = [], []
+        count = 1
+        all_paths = dict.fromkeys(working_paths, history.commit)
+        for commit, entries in entries_by_commit.items():
+            for path in entries:
+                all_paths.setdefault(path, commit)
+        for path, commit in sorted(all_paths.items()):
+            data = path.encode('utf-8', 'surrogateescape') + b'\n'
+            messages.append(data)
+            length = data.count(b'\n')
+            scope = 'working-tree' if path in working_paths else 'pre-existing' if commit in history.public_commits else 'newly-public'
+            lines.append((count, count + length, path, commit, scope))
+            count += length
+        for path, commit, data in history.messages():
+            report['history']['tagMessages' if path == '@tag-message' else 'commitMessages'] += 1
+            rules, _ = text_rules(data, path, selected_context, resolver, hashes)
+            scope = 'release-tree' if commit == history.commit else 'pre-existing' if commit in history.public_commits else 'newly-public'
+            for rule in rules:
+                add(rule, path, commit, scope=scope)
+            messages.append(data + b'\n')
+            length = (data + b'\n').count(b'\n')
+            lines.append((count, count + length, path, commit, scope))
+            count += length
+        for hit in secrets(binary, 'stdin', data=b''.join(messages), timeout=timeout):
+            line = hit.get('StartLine', 0)
+            location = next(((p, c, s) for start, end, p, c, s in lines if start <= line < end), ('@message', None, 'newly-public'))
+            add(secret_rule(hit['RuleID']), location[0], location[1], scope=location[2], scanner_rule=hit['RuleID'])
+    except PublicBaseError:
+        add('scan.public-base-rejected', '.')
+    except InspectionError as error:
+        if str(error).startswith('scan.'):
+            add(str(error), '.')
+        else:
+            add('scan.history-incomplete', '.')
+            report['findings'][-1]['reason'] = str(error)
+    except (OSError, ValueError, TypeError, KeyError, UnicodeError):
+        add('scan.configuration-incomplete', '.')
+    finally:
+        if history:
+            history.close()
+    return finish(report)