cctally 1.113.0 → 1.115.0

package/bin/_lib_merge_gate.py

@@ -2,12 +2,13 @@
 
 This is a reviewed component policy, not proof of every possible dependency.
 Unknown paths and shared/infrastructure changes require the full estate. The
-full background/release runs remain the cross-component backstop.
+scheduled full run and the release run remain the cross-component backstop.
 """
 from __future__ import annotations
 
 import argparse
 import hashlib
+import importlib.util
 import json
 import os
 from pathlib import Path, PurePosixPath
@@ -26,6 +27,45 @@ PRESENTATION_FILES = frozenset(
 )
 SMOKE_FILES = ('tests/test_cli_smoke.py', 'tests/test_lib_json_envelope.py')
 PRESENTATION_PYTEST = ('tests/test_dashboard_handler_static.py',)
+# The reviewed memory publication set: both topic stores and the topic metadata.
+# A private test pins it to the publication engine's own definition. Its
+# configuration is NOT memory, so a change to it is ordinary task work.
+MEMORY_PUBLICATION_SET = ('.claude-memory/**', '.agentmem/codex-memory/**', '.agentmem/topics.json')
+MEMORY_CONFIGURATION = '.agentmem/config.json'
+# Every test module that reads the memory corpus, beside the smoke files and
+# the Doc lint harness: together they certify a memory-only candidate.
+MEMORY_PYTEST = ('tests/test_cctally_memory.py', 'tests/test_agent_workflows.py',
+                 'tests/test_session_workflow_texts.py')
+# The capture publishes regular files only (a link is refused), so a memory
+# path that is anything else on either side is not a memory publication.
+REGULAR_MODES = ('100644', '100755')
+# The instruction-text class (#894): agent-instruction prose whose every reader
+# is one of INSTRUCTIONS_PYTEST. Markdown at any depth under these trees,
+# nested CLAUDE.md/AGENTS.md included.
+INSTRUCTION_TREES = ('.agent-workflows/skills/', '.claude/skills/', '.agents/skills/',
+                     '.agent-workflows/contracts/', 'docs/superpowers/')
+INSTRUCTION_NAMES = ('CLAUDE.md', 'AGENTS.md')
+# Every nested instruction file outside the trees. A new one is a structural
+# change to the instruction chain: its readers depend on where it lands, so it
+# selects the full estate until a reviewed policy change lists it here.
+NESTED_INSTRUCTION_FILES = ('bin/AGENTS.md', 'dashboard/AGENTS.md')
+# The generator inputs of the root AGENTS.md (its configuration is not one).
+INSTRUCTION_INPUTS = ('.agentmem/codex.md', '.agentmem/manifest.json')
+# Every test module that reads, validates or scans an instruction path,
+# beside the smoke files and the Doc lint harness.
+INSTRUCTIONS_PYTEST = ('tests/test_agent_workflows.py', 'tests/test_session_workflow_texts.py',
+                       'tests/test_cctally_memory.py', 'tests/test_session_issues.py',
+                       'tests/test_session_provenance.py', 'tests/test_linux_matrix_gate.py',
+                       'tests/test_release_internals.py', 'tests/test_quota_budget_error_harness.py',
+                       'tests/test_session_policy.py', 'tests/test_lib_share.py',
+                       'tests/test_dashboard_source_kernel.py', 'tests/test_share_privacy_detector.py',
+                       'tests/test_fixture_cache.py')
+# An instruction path is eligible only as a non-executable regular file.
+INSTRUCTION_MODES = ('100644',)
+# The two session execution plans are documentation: their only readers are
+# people and the Doc lint harness, so a plan refresh takes the documentation
+# focus. Exactly these two paths; any other docs/ Markdown is unclassified.
+EXECUTION_PLAN_PATHS = ('docs/cctally-claude-execution-plan.md', 'docs/cctally-execution-plan.md')
 
 
 def _git(repo, *args):
@@ -58,13 +98,113 @@ def basis(repo, base_rev, head='HEAD', *, working=True):
             'paths': sorted(committed | dirty)}
 
 
+def memory_publication_path(path):
+    """Whether `path` is in the memory publication set (the engine's glob semantics)."""
+    path = path.strip('/')
+    for pattern in MEMORY_PUBLICATION_SET:
+        if pattern.endswith('/**'):
+            prefix = pattern[:-3]
+            if path == prefix or path.startswith(prefix + '/'):
+                return True
+        elif path == pattern:
+            return True
+    return False
+
+
+def _tree_modes(repo, rev, paths):
+    """{path: git mode} of the entries `paths` name in `rev` (absent paths are omitted)."""
+    out = _git(repo, '--literal-pathspecs', 'ls-tree', '-z', '--full-tree', rev, '--', *paths)
+    modes = {}
+    for record in out.split(b'\0'):
+        if record:
+            meta, name = record.split(b'\t', 1)
+            modes[name.decode('utf-8', errors='strict')] = meta.split()[0].decode()
+    return modes
+
+
+def irregular_memory_paths(repo, base, head_oid, paths, *, working=True):
+    """The memory paths that are not a regular file (or absent) at the base, at the head,
+    or (with `working`) in the working tree: the file types the publication set does not define."""
+    irregular = set()
+    for rev in (base, head_oid):
+        irregular.update(name for name, mode in _tree_modes(repo, rev, paths).items()
+                         if mode not in REGULAR_MODES)
+    if working:
+        for path in paths:
+            target = Path(repo) / path
+            if target.is_symlink() or (target.exists() and not target.is_file()):
+                irregular.add(path)
+    return sorted(irregular & set(paths))
+
+
+def _blob_prefixes(repo, rev, paths):
+    """{path: its first two bytes} for the blobs `paths` name in `rev`, in one git call."""
+    if not paths:
+        return {}
+    request = b''.join(f'{rev}:{path}\n'.encode('utf-8') for path in paths)
+    proc = subprocess.run(['git', '--no-optional-locks', '-C', str(repo), 'cat-file', '--batch'],
+                          input=request, capture_output=True)
+    if proc.returncode:
+        raise ValueError('git cat-file failed: ' + proc.stderr.decode(errors='replace').strip())
+    out, offset, prefixes = proc.stdout, 0, {}
+    for path in paths:
+        end = out.index(b'\n', offset)
+        header = out[offset:end].split()
+        if len(header) != 3 or header[1] != b'blob':
+            raise ValueError('git cat-file returned no blob for ' + path)
+        size = int(header[2])
+        prefixes[path] = out[end + 1:end + 1 + min(size, 2)]
+        offset = end + 1 + size + 1
+    return prefixes
+
+
+def ineligible_instruction_paths(repo, base, head_oid, paths, *, working=True):
+    """The instruction paths that are not a non-executable regular file without a
+    `#!` first line at the base, at the head, or (with `working`) on disk. Absence
+    on any side is eligible. With `working=False` only the two commits are read."""
+    paths = sorted(p for p in paths if _sane(p) and _instruction_class(p) == 'instruction')
+    if not paths:
+        return []
+    ineligible = set()
+    for rev in (base, head_oid):
+        modes = _tree_modes(repo, rev, paths)
+        ineligible.update(name for name, mode in modes.items() if mode not in INSTRUCTION_MODES)
+        regular = sorted(name for name, mode in modes.items() if mode in INSTRUCTION_MODES)
+        ineligible.update(name for name, prefix in _blob_prefixes(repo, rev, regular).items()
+                          if prefix == b'#!')
+    if working:
+        for path in paths:
+            target = Path(repo) / path
+            if target.is_symlink():
+                ineligible.add(path)
+            elif target.exists():
+                if not target.is_file() or target.stat().st_mode & 0o111:
+                    ineligible.add(path)
+                else:
+                    with target.open('rb') as handle:
+                        if handle.read(2) == b'#!':
+                            ineligible.add(path)
+    return sorted(ineligible & set(paths))
+
+
+def _memory_only(paths, irregular):
+    """A memory-only candidate: every path in the publication set, each a regular file."""
+    return (irregular is not None and bool(paths) and not irregular
+            and all(memory_publication_path(p) and _sane(p) for p in paths))
+
+
+def _sane(path):
+    p = PurePosixPath(path)
+    return not (p.is_absolute() or '..' in p.parts or any(c.isspace() for c in path))
+
+
 def policy_digest(repo):
     return 'sha256:' + hashlib.sha256(
         (Path(repo) / 'bin/_lib_merge_gate.py').read_bytes()).hexdigest()
 
 
 def _document(path):
-    return path in ('README.md', 'CHANGELOG.md') or (
+    return path in ('README.md', 'CHANGELOG.md') or path in EXECUTION_PLAN_PATHS or (
         path.startswith('docs/commands/') and path.endswith('.md'))
 
 
@@ -72,7 +212,32 @@ def _frontend_test(path):
     return path.startswith('dashboard/web/src/') and path.endswith(('.test.ts', '.test.tsx'))
 
 
-def select(repo, paths):
+def _instruction_class(path):
+    """'instruction' for an instruction path, a full-selection reason for an
+    instruction-file name at an unreviewed location, or None for any other path."""
+    if path.endswith('.md') and path.startswith(INSTRUCTION_TREES):
+        return 'instruction'
+    if PurePosixPath(path).name in INSTRUCTION_NAMES:
+        if path in INSTRUCTION_NAMES or path in NESTED_INSTRUCTION_FILES:
+            return 'instruction'
+        return 'unreviewed nested instruction file: ' + path
+    if path in INSTRUCTION_INPUTS:
+        return 'instruction'
+    return None
+
+
+def select(repo, paths, *, irregular=None, instruction_evidence=None):
+    """The reviewed selection of `paths`.
+
+    `irregular` is the list of memory paths that are not regular files
+    (`irregular_memory_paths`); only a caller that computed it can have a
+    candidate selected as memory-only. Any other candidate is selected over its
+    whole diff, where a memory path is unclassified.
+
+    `instruction_evidence` is the list of instruction paths that are not
+    non-executable regular files (`ineligible_instruction_paths`); only a caller
+    that computed it can have an instruction path selected focused.
+    """
     repo = Path(repo)
     manifest = json.loads((repo / 'tests/authoritative-test-manifest.json').read_text())
     private = (repo / '.mirror-allowlist').exists()
@@ -82,10 +247,28 @@ def select(repo, paths):
         raise ValueError('invalid harness manifest')
     paths = sorted(set(paths))
     reasons, components = [], set()
-    for path in paths:
+    memory = _memory_only(paths, irregular)
+    if memory:
+        components.add('memory')
+    for path in ([] if memory else paths):
         p = PurePosixPath(path)
-        if p.is_absolute() or '..' in p.parts or any(c.isspace() for c in path):
+        if path in (irregular or ()):
+            reasons.append('memory path that is not a regular file: ' + path)
+        elif p.is_absolute() or '..' in p.parts or any(c.isspace() for c in path):
             reasons.append('unclassified path: ' + path)
+        elif '__pycache__' in p.parts:
+            # Terminal: the skill comparison never reads cache content, so no
+            # class may admit it (the memory-only shortcut is decided above).
+            reasons.append('runtime cache path: ' + path)
+        elif (instruction := _instruction_class(path)) is not None:
+            if instruction != 'instruction':
+                reasons.append(instruction)
+            elif instruction_evidence is None:
+                reasons.append('instruction path without file evidence: ' + path)
+            elif path in instruction_evidence:
+                reasons.append('instruction path that is not a non-executable regular file: ' + path)
+            else:
+                components.add('instructions')
         elif _document(path):
             components.add('documentation')
         elif path in PRESENTATION_FILES or _frontend_test(path):
@@ -96,11 +279,17 @@ def select(repo, paths):
             components.add('presentation')
         else:
             reasons.append('shared, high-risk, or unclassified path: ' + path)
+    if 'instructions' in components and 'presentation' in components:
+        reasons.append('instruction text and presentation are not certified together')
     harnesses = {'doc-lint'}
     files = set(SMOKE_FILES)
     if 'presentation' in components:
         harnesses.add('frontend')
         files.update(PRESENTATION_PYTEST)
+    if 'memory' in components:
+        files.update(MEMORY_PYTEST)
+    if 'instructions' in components:
+        files.update(INSTRUCTIONS_PYTEST)
     if not harnesses <= set(names):
         reasons.append('required merge harness absent from manifest')
     if any(not (repo / p).is_file() for p in files):
@@ -113,19 +302,32 @@ def select(repo, paths):
         'omittedHarnesses': [n for n in names if not full and n not in harnesses],
         'selectedPytestFiles': [] if full else sorted(files),
         'components': sorted(components), 'reasons': reasons,
-        'policyDigest': policy_digest(repo), 'maxWallSeconds': MAX_SECONDS,
+        'policyDigest': policy_digest(repo),
+        'maxWallSeconds': MAX_SECONDS,
     }
 
 
 def plan(repo, base_rev, head='HEAD', *, working=True):
     b = basis(repo, base_rev, head, working=working)
-    result = select(repo, b['paths'])
+    irregular = None
+    if b['paths'] and all(memory_publication_path(p) for p in b['paths']):
+        irregular = irregular_memory_paths(repo, b['baseOid'], b['headOid'], b['paths'], working=working)
+    evidence = None
+    if any(_sane(p) and _instruction_class(p) == 'instruction' for p in b['paths']):
+        evidence = ineligible_instruction_paths(repo, b['baseOid'], b['headOid'], b['paths'],
+                                                working=working)
+    result = select(repo, b['paths'], irregular=irregular, instruction_evidence=evidence)
     result['mergeBasis'] = b
     return result
 
 
-def filter_legs(plan_record, directory):
-    """Narrow both expected-node legs to selected files; retain serial ownership."""
+def filter_legs(plan_record, directory, quarantined=()):
+    """Narrow both expected-node legs to selected files; retain serial ownership.
+
+    bin/cctally-test-all applies tests/quarantine.json to the legs first, so a
+    selected file whose every node is quarantined has none left. Such a file
+    (one that ``quarantined``, the list's node ids, names) contributes no
+    targets; any other selected file without a recorded node refuses."""
     directory = Path(directory)
     selected = set(plan_record['selectedPytestFiles'])
     found = set()
@@ -134,78 +336,20 @@ def filter_legs(plan_record, directory):
         nodes = [n for n in path.read_text().splitlines() if n.split('::')[0] in selected]
         found.update(n.split('::')[0] for n in nodes)
         path.write_text(''.join(n + '\n' for n in nodes))
-    if found != selected:
-        raise ValueError('selected files absent from recorded pytest estate: ' + ', '.join(sorted(selected - found)))
-
-
-def background_health(rows):
-    """Newest completed full verification wins; running work hides nothing."""
-    for row in rows:
-        if row.get('head_branch') != 'main' or row.get('status') != 'completed':
-            continue
-        if row.get('event') not in ('schedule', 'workflow_dispatch') and row.get('fullSuite') is not True:
-            continue
-        passed = row.get('conclusion') == 'success'
-        return {'state': 'passed' if passed else 'failed', 'allowFocused': passed,
-                'runId': row.get('id'), 'headSha': row.get('head_sha')}
-    return {'state': 'unavailable', 'allowFocused': False, 'runId': None, 'headSha': None}
+    absent = selected - found - {node.split('::')[0] for node in quarantined}
+    if absent:
+        raise ValueError('selected files absent from recorded pytest estate: ' + ', '.join(sorted(absent)))
 
 
-def read_background_health(repo):
-    """Read actual full test-macos job results, never a receipt-discharge skip.
+def quarantine_ids(repo):
+    """The pytest node ids tests/quarantine.json lists; none when the tree carries no list.
 
-    Existing full CI jobs provide a bootstrap before the new schedule lands.
-    A successful workflow with a skipped full job is not full verification.
-    """
-    def gh(*args):
-        process = subprocess.run(['gh', *args], cwd=repo, capture_output=True,
-                                 text=True, timeout=20)
-        if process.returncode:
-            raise ValueError('GitHub full-verification evidence is unavailable')
-        return json.loads(process.stdout)
-    try:
-        name = gh('repo', 'view', '--json', 'nameWithOwner')['nameWithOwner']
-        rows = gh('api', f'repos/{name}/actions/workflows/ci.yml/runs?branch=main&per_page=20')['workflow_runs']
-        for row in rows:
-            if row.get('status') != 'completed':
-                # GitHub reuses the run ID on retry: the earlier failed
-                # attempt disappears from this listing while the retry runs.
-                # Do not let that mutation reveal an older successful run.
-                if row.get('run_attempt', 1) != 1:
-                    return background_health([])
-                continue
-            jobs = gh('api', f'repos/{name}/actions/runs/{row["id"]}/jobs?per_page=100')['jobs']
-            full = next((j for j in jobs if j.get('name') == 'test-macos'
-                         and j.get('status') == 'completed'
-                         and j.get('conclusion') != 'skipped'), None)
-            if full is not None:
-                record = dict(row, fullSuite=True, conclusion=full.get('conclusion'))
-                return background_health([record])
-            if row.get('event') in ('schedule', 'workflow_dispatch'):
-                # A scheduled/manual attempt must actually run the full job.
-                # Only receipt-discharged ordinary pushes may be skipped while
-                # looking for the last full verification.
-                return background_health([])
-            # ci.yml can skip its declared full job on a successful private
-            # main push only through a successful release/receipt gate. The
-            # skipped job plus successful workflow is evidence of discharge;
-            # a merely successful classifier job alone is not (it may emit
-            # discharged=false). Missing jobs, cancellation, and failures
-            # must not silently inherit an older pass.
-            discharged = (
-                row.get('event') == 'push' and row.get('conclusion') == 'success'
-                and any(j.get('name') == 'test-macos'
-                        and j.get('status') == 'completed'
-                        and j.get('conclusion') == 'skipped' for j in jobs)
-                and any(j.get('name') in ('receipt-gate', 'release-stamp-gate')
-                        and j.get('status') == 'completed'
-                        and j.get('conclusion') == 'success' for j in jobs)
-            )
-            if not discharged:
-                return background_health([])
-        return background_health([])
-    except (OSError, ValueError, KeyError, TypeError, subprocess.TimeoutExpired):
-        return background_health([])
+    Parsed by the shared bin/_lib_quarantine.py; harness cases are not pytest
+    nodes and are left to their harness. bin/cctally-test-all validated the
+    list at admission, before this runs."""
+    sys.path.insert(0, str(Path(__file__).resolve().parent))
+    import _lib_quarantine
+    return _lib_quarantine.pytest_ids(_lib_quarantine.load(repo))
 
 
 def supervise(command, seconds):
@@ -264,7 +408,6 @@ def main(argv=None):
     parser = argparse.ArgumentParser(description=__doc__)
     parser.add_argument('--repo', default=str(Path(__file__).resolve().parents[1]))
     parser.add_argument('--base')
-    parser.add_argument('--health', action='store_true')
     parser.add_argument('--required-mode', action='store_true')
     parser.add_argument('--shell', action='store_true')
     parser.add_argument('--filter-legs')
@@ -275,18 +418,20 @@ def main(argv=None):
         if len(args.deadline) < 2:
             parser.error('deadline requires seconds and a command')
         return supervise(args.deadline[1:], int(args.deadline[0]))
-    if args.health:
-        health = read_background_health(args.repo)
-        print(json.dumps(health, sort_keys=True))
-        return 0 if health['allowFocused'] else 3
     if not args.base:
-        parser.error('--base is required outside --health')
+        parser.error('--base is required')
     try:
         result = plan(args.repo, args.base)
         if args.required_mode:
-            print(result['mode'])
+            # `<mode> <reason>`: the wrapper decides on the mode alone; the
+            # reason names the focused components for whoever reads the log.
+            if result['mode'] == 'full':
+                reason = 'reviewed-selection'
+            else:
+                reason = '+'.join(result['components']) or 'none'
+            print(f"{result['mode']} {reason}")
         elif args.filter_legs:
-            filter_legs(result, args.filter_legs)
+            filter_legs(result, args.filter_legs, quarantined=quarantine_ids(args.repo))
         elif args.shell:
             print('mode\t' + result['mode'])
             print('harnesses\t' + ' '.join(result['selectedHarnesses']))

package/bin/_lib_meter_rate_change.py

@@ -246,6 +246,12 @@ def _adjacent_pairs(regimes) -> dict:
     `now` with `status: "stale"` and appends a fresh regime starting at the
     analysis start, so the two are neither adjacent nor a rate transition. The
     status check is the second, independent guard.
+
+    #978 spec 4.5: a pair also requires one `fingerprint` and one
+    `coefficientEra`. Units-per-point from two eras measure two different
+    weightings, so their difference is a coefficient change and never a
+    metering-rate change. The comparison uses `.get()`, so a legacy pair in
+    which neither regime names an era keeps its treatment.
     """
     closed: dict = {}
     for regime in regimes:
@@ -266,6 +272,10 @@ def _adjacent_pairs(regimes) -> dict:
         predecessor = closed.get(start)
         if predecessor is None or predecessor is regime:
             continue
+        if predecessor.get("fingerprint") != regime.get("fingerprint") \
+                or predecessor.get("coefficientEra") \
+                != regime.get("coefficientEra"):
+            continue
         pairs[start] = (predecessor, regime)
     return pairs