cctally 1.112.0 → 1.114.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. package/CHANGELOG.md +27 -0
  2. package/README.md +4 -5
  3. package/bin/_cctally_cache.py +1683 -485
  4. package/bin/_cctally_core.py +223 -1
  5. package/bin/_cctally_dashboard.py +96 -18
  6. package/bin/_cctally_dashboard_perf.py +128 -9
  7. package/bin/_cctally_dashboard_sources.py +267 -204
  8. package/bin/_cctally_db.py +861 -18
  9. package/bin/_cctally_diagnosis_sources.py +22 -4
  10. package/bin/_cctally_doctor.py +136 -18
  11. package/bin/_cctally_five_hour.py +2 -2
  12. package/bin/_cctally_forecast.py +27 -6
  13. package/bin/_cctally_journal.py +67 -5
  14. package/bin/_cctally_milestone_history.py +82 -9
  15. package/bin/_cctally_percent_breakdown.py +3 -2
  16. package/bin/_cctally_quota.py +201 -33
  17. package/bin/_cctally_record.py +14 -0
  18. package/bin/_cctally_rederive.py +23 -9
  19. package/bin/_cctally_source_analytics.py +60 -15
  20. package/bin/_cctally_statusline.py +17 -0
  21. package/bin/_cctally_store.py +80 -5
  22. package/bin/_cctally_tui.py +4 -3
  23. package/bin/_cctally_weekrefs.py +128 -21
  24. package/bin/_lib_cache_report.py +35 -6
  25. package/bin/_lib_codex_conversation.py +32 -0
  26. package/bin/_lib_codex_conversation_query.py +164 -30
  27. package/bin/_lib_conversation_query.py +95 -56
  28. package/bin/_lib_conversation_retention.py +1605 -640
  29. package/bin/_lib_cost_provenance.py +53 -2
  30. package/bin/_lib_doctor.py +263 -69
  31. package/bin/_lib_fmt.py +11 -0
  32. package/bin/_lib_merge_gate.py +469 -51
  33. package/bin/_lib_perf.py +27 -1
  34. package/bin/_lib_pricing.py +209 -8
  35. package/bin/_lib_pricing_check.py +20 -4
  36. package/bin/_lib_reclaim_planner.py +792 -0
  37. package/bin/_lib_rederive.py +225 -77
  38. package/bin/_lib_snapshot_cache.py +21 -0
  39. package/bin/_lib_sqlite_close.py +81 -0
  40. package/bin/_lib_subscription_weeks.py +100 -17
  41. package/bin/_lib_tick_stats.py +217 -17
  42. package/bin/_lib_wal_checkpoint.py +1077 -0
  43. package/bin/_lib_write_budget.py +483 -0
  44. package/bin/_lib_write_io.py +703 -0
  45. package/bin/cctally +8 -0
  46. package/dashboard/static/assets/{ConversationsView-CEZ-BUyi.js → ConversationsView-Bc8LCOAZ.js} +1 -1
  47. package/dashboard/static/assets/{DoctorModal-CE1oH_Zy.js → DoctorModal-D--qVBwe.js} +1 -1
  48. package/dashboard/static/assets/ModalRoot-DLPgyAM_.js +1 -0
  49. package/dashboard/static/assets/{ProjectsDrillPanel-B3DZoCeD.js → ProjectsDrillPanel-z53a4v4M.js} +1 -1
  50. package/dashboard/static/assets/{SourceDetailModal-CHp9EgLP.js → SourceDetailModal-Bw0T8W-W.js} +1 -1
  51. package/dashboard/static/assets/{UpdateModal-5iKST6jo.js → UpdateModal-D5AKx4Yw.js} +1 -1
  52. package/dashboard/static/assets/index-BKITk1CT.css +1 -0
  53. package/dashboard/static/assets/{index-Cm0VTxJC.js → index-BXDOzSng.js} +3 -3
  54. package/dashboard/static/assets/{outlineNavigation-DkXGL8J6.js → outlineNavigation-JObY7cUY.js} +1 -1
  55. package/dashboard/static/dashboard.html +2 -2
  56. package/package.json +6 -1
  57. package/dashboard/static/assets/ModalRoot-D5Y8IFUG.js +0 -1
  58. package/dashboard/static/assets/index-Dt4tqf43.css +0 -1
@@ -7,13 +7,18 @@ full background/release runs remain the cross-component backstop.
7
7
  from __future__ import annotations
8
8
 
9
9
  import argparse
10
+ import base64
11
+ from datetime import datetime, timezone
10
12
  import hashlib
13
+ import importlib.util
11
14
  import json
12
15
  import os
13
16
  from pathlib import Path, PurePosixPath
17
+ import re
14
18
  import signal
15
19
  import subprocess
16
20
  import sys
21
+ import time
17
22
 
18
23
  GATE_ID = 'cctally-test-remote/merge-focused@1'
19
24
  MAX_SECONDS = 300
@@ -26,6 +31,45 @@ PRESENTATION_FILES = frozenset(
26
31
  )
27
32
  SMOKE_FILES = ('tests/test_cli_smoke.py', 'tests/test_lib_json_envelope.py')
28
33
  PRESENTATION_PYTEST = ('tests/test_dashboard_handler_static.py',)
34
+ # The reviewed memory publication set: both topic stores and the topic metadata.
35
+ # A private test pins it to the publication engine's own definition. Its
36
+ # configuration is NOT memory, so a change to it is ordinary task work.
37
+ MEMORY_PUBLICATION_SET = ('.claude-memory/**', '.agentmem/codex-memory/**', '.agentmem/topics.json')
38
+ MEMORY_CONFIGURATION = '.agentmem/config.json'
39
+ # Every test module that reads the memory corpus, beside the smoke files and
40
+ # the Doc lint harness: together they certify a memory-only candidate.
41
+ MEMORY_PYTEST = ('tests/test_cctally_memory.py', 'tests/test_agent_workflows.py',
42
+ 'tests/test_session_workflow_texts.py')
43
+ # The capture publishes regular files only (a link is refused), so a memory
44
+ # path that is anything else on either side is not a memory publication.
45
+ REGULAR_MODES = ('100644', '100755')
46
+ # The instruction-text class (#894): agent-instruction prose whose every reader
47
+ # is one of INSTRUCTIONS_PYTEST. Markdown at any depth under these trees,
48
+ # nested CLAUDE.md/AGENTS.md included.
49
+ INSTRUCTION_TREES = ('.agent-workflows/skills/', '.claude/skills/', '.agents/skills/',
50
+ '.agent-workflows/contracts/', 'docs/superpowers/')
51
+ INSTRUCTION_NAMES = ('CLAUDE.md', 'AGENTS.md')
52
+ # Every nested instruction file outside the trees. A new one is a structural
53
+ # change to the instruction chain: its readers depend on where it lands, so it
54
+ # selects the full estate until a reviewed policy change lists it here.
55
+ NESTED_INSTRUCTION_FILES = ('bin/AGENTS.md', 'dashboard/AGENTS.md')
56
+ # The generator inputs of the root AGENTS.md (its configuration is not one).
57
+ INSTRUCTION_INPUTS = ('.agentmem/codex.md', '.agentmem/manifest.json')
58
+ # Every test module that reads, validates or scans an instruction path,
59
+ # beside the smoke files and the Doc lint harness.
60
+ INSTRUCTIONS_PYTEST = ('tests/test_agent_workflows.py', 'tests/test_session_workflow_texts.py',
61
+ 'tests/test_cctally_memory.py', 'tests/test_session_issues.py',
62
+ 'tests/test_session_provenance.py', 'tests/test_linux_matrix_gate.py',
63
+ 'tests/test_release_internals.py', 'tests/test_quota_budget_error_harness.py',
64
+ 'tests/test_session_policy.py', 'tests/test_lib_share.py',
65
+ 'tests/test_dashboard_source_kernel.py', 'tests/test_share_privacy_detector.py',
66
+ 'tests/test_fixture_cache.py')
67
+ # An instruction path is eligible only as a non-executable regular file.
68
+ INSTRUCTION_MODES = ('100644',)
69
+ # The two session execution plans are documentation: their only readers are
70
+ # people and the Doc lint harness, so a plan refresh takes the documentation
71
+ # focus. Exactly these two paths; any other docs/ Markdown is unclassified.
72
+ EXECUTION_PLAN_PATHS = ('docs/cctally-claude-execution-plan.md', 'docs/cctally-execution-plan.md')
29
73
 
30
74
 
31
75
  def _git(repo, *args):
@@ -58,13 +102,113 @@ def basis(repo, base_rev, head='HEAD', *, working=True):
58
102
  'paths': sorted(committed | dirty)}
59
103
 
60
104
 
105
+ def memory_publication_path(path):
106
+ """Whether `path` is in the memory publication set (the engine's glob semantics)."""
107
+ path = path.strip('/')
108
+ for pattern in MEMORY_PUBLICATION_SET:
109
+ if pattern.endswith('/**'):
110
+ prefix = pattern[:-3]
111
+ if path == prefix or path.startswith(prefix + '/'):
112
+ return True
113
+ elif path == pattern:
114
+ return True
115
+ return False
116
+
117
+
118
+ def _tree_modes(repo, rev, paths):
119
+ """{path: git mode} of the entries `paths` name in `rev` (absent paths are omitted)."""
120
+ out = _git(repo, '--literal-pathspecs', 'ls-tree', '-z', '--full-tree', rev, '--', *paths)
121
+ modes = {}
122
+ for record in out.split(b'\0'):
123
+ if record:
124
+ meta, name = record.split(b'\t', 1)
125
+ modes[name.decode('utf-8', errors='strict')] = meta.split()[0].decode()
126
+ return modes
127
+
128
+
129
+ def irregular_memory_paths(repo, base, head_oid, paths, *, working=True):
130
+ """The memory paths that are not a regular file (or absent) at the base, at the head,
131
+ or (with `working`) in the working tree: the file types the publication set does not define."""
132
+ irregular = set()
133
+ for rev in (base, head_oid):
134
+ irregular.update(name for name, mode in _tree_modes(repo, rev, paths).items()
135
+ if mode not in REGULAR_MODES)
136
+ if working:
137
+ for path in paths:
138
+ target = Path(repo) / path
139
+ if target.is_symlink() or (target.exists() and not target.is_file()):
140
+ irregular.add(path)
141
+ return sorted(irregular & set(paths))
142
+
143
+
144
+ def _blob_prefixes(repo, rev, paths):
145
+ """{path: its first two bytes} for the blobs `paths` name in `rev`, in one git call."""
146
+ if not paths:
147
+ return {}
148
+ request = b''.join(f'{rev}:{path}\n'.encode('utf-8') for path in paths)
149
+ proc = subprocess.run(['git', '--no-optional-locks', '-C', str(repo), 'cat-file', '--batch'],
150
+ input=request, capture_output=True)
151
+ if proc.returncode:
152
+ raise ValueError('git cat-file failed: ' + proc.stderr.decode(errors='replace').strip())
153
+ out, offset, prefixes = proc.stdout, 0, {}
154
+ for path in paths:
155
+ end = out.index(b'\n', offset)
156
+ header = out[offset:end].split()
157
+ if len(header) != 3 or header[1] != b'blob':
158
+ raise ValueError('git cat-file returned no blob for ' + path)
159
+ size = int(header[2])
160
+ prefixes[path] = out[end + 1:end + 1 + min(size, 2)]
161
+ offset = end + 1 + size + 1
162
+ return prefixes
163
+
164
+
165
+ def ineligible_instruction_paths(repo, base, head_oid, paths, *, working=True):
166
+ """The instruction paths that are not a non-executable regular file without a
167
+ `#!` first line at the base, at the head, or (with `working`) on disk. Absence
168
+ on any side is eligible. With `working=False` only the two commits are read."""
169
+ paths = sorted(p for p in paths if _sane(p) and _instruction_class(p) == 'instruction')
170
+ if not paths:
171
+ return []
172
+ ineligible = set()
173
+ for rev in (base, head_oid):
174
+ modes = _tree_modes(repo, rev, paths)
175
+ ineligible.update(name for name, mode in modes.items() if mode not in INSTRUCTION_MODES)
176
+ regular = sorted(name for name, mode in modes.items() if mode in INSTRUCTION_MODES)
177
+ ineligible.update(name for name, prefix in _blob_prefixes(repo, rev, regular).items()
178
+ if prefix == b'#!')
179
+ if working:
180
+ for path in paths:
181
+ target = Path(repo) / path
182
+ if target.is_symlink():
183
+ ineligible.add(path)
184
+ elif target.exists():
185
+ if not target.is_file() or target.stat().st_mode & 0o111:
186
+ ineligible.add(path)
187
+ else:
188
+ with target.open('rb') as handle:
189
+ if handle.read(2) == b'#!':
190
+ ineligible.add(path)
191
+ return sorted(ineligible & set(paths))
192
+
193
+
194
+ def _memory_only(paths, irregular):
195
+ """A memory-only candidate: every path in the publication set, each a regular file."""
196
+ return (irregular is not None and bool(paths) and not irregular
197
+ and all(memory_publication_path(p) and _sane(p) for p in paths))
198
+
199
+
200
+ def _sane(path):
201
+ p = PurePosixPath(path)
202
+ return not (p.is_absolute() or '..' in p.parts or any(c.isspace() for c in path))
203
+
204
+
61
205
  def policy_digest(repo):
62
206
  return 'sha256:' + hashlib.sha256(
63
207
  (Path(repo) / 'bin/_lib_merge_gate.py').read_bytes()).hexdigest()
64
208
 
65
209
 
66
210
  def _document(path):
67
- return path in ('README.md', 'CHANGELOG.md') or (
211
+ return path in ('README.md', 'CHANGELOG.md') or path in EXECUTION_PLAN_PATHS or (
68
212
  path.startswith('docs/commands/') and path.endswith('.md'))
69
213
 
70
214
 
@@ -72,7 +216,32 @@ def _frontend_test(path):
72
216
  return path.startswith('dashboard/web/src/') and path.endswith(('.test.ts', '.test.tsx'))
73
217
 
74
218
 
75
- def select(repo, paths):
219
+ def _instruction_class(path):
220
+ """'instruction' for an instruction path, a full-selection reason for an
221
+ instruction-file name at an unreviewed location, or None for any other path."""
222
+ if path.endswith('.md') and path.startswith(INSTRUCTION_TREES):
223
+ return 'instruction'
224
+ if PurePosixPath(path).name in INSTRUCTION_NAMES:
225
+ if path in INSTRUCTION_NAMES or path in NESTED_INSTRUCTION_FILES:
226
+ return 'instruction'
227
+ return 'unreviewed nested instruction file: ' + path
228
+ if path in INSTRUCTION_INPUTS:
229
+ return 'instruction'
230
+ return None
231
+
232
+
233
+ def select(repo, paths, *, irregular=None, instruction_evidence=None):
234
+ """The reviewed selection of `paths`.
235
+
236
+ `irregular` is the list of memory paths that are not regular files
237
+ (`irregular_memory_paths`); only a caller that computed it can have a
238
+ candidate selected as memory-only. Any other candidate is selected over its
239
+ whole diff, where a memory path is unclassified.
240
+
241
+ `instruction_evidence` is the list of instruction paths that are not
242
+ non-executable regular files (`ineligible_instruction_paths`); only a caller
243
+ that computed it can have an instruction path selected focused.
244
+ """
76
245
  repo = Path(repo)
77
246
  manifest = json.loads((repo / 'tests/authoritative-test-manifest.json').read_text())
78
247
  private = (repo / '.mirror-allowlist').exists()
@@ -82,10 +251,28 @@ def select(repo, paths):
82
251
  raise ValueError('invalid harness manifest')
83
252
  paths = sorted(set(paths))
84
253
  reasons, components = [], set()
85
- for path in paths:
254
+ memory = _memory_only(paths, irregular)
255
+ if memory:
256
+ components.add('memory')
257
+ for path in ([] if memory else paths):
86
258
  p = PurePosixPath(path)
87
- if p.is_absolute() or '..' in p.parts or any(c.isspace() for c in path):
259
+ if path in (irregular or ()):
260
+ reasons.append('memory path that is not a regular file: ' + path)
261
+ elif p.is_absolute() or '..' in p.parts or any(c.isspace() for c in path):
88
262
  reasons.append('unclassified path: ' + path)
263
+ elif '__pycache__' in p.parts:
264
+ # Terminal: the skill comparison never reads cache content, so no
265
+ # class may admit it (the memory-only shortcut is decided above).
266
+ reasons.append('runtime cache path: ' + path)
267
+ elif (instruction := _instruction_class(path)) is not None:
268
+ if instruction != 'instruction':
269
+ reasons.append(instruction)
270
+ elif instruction_evidence is None:
271
+ reasons.append('instruction path without file evidence: ' + path)
272
+ elif path in instruction_evidence:
273
+ reasons.append('instruction path that is not a non-executable regular file: ' + path)
274
+ else:
275
+ components.add('instructions')
89
276
  elif _document(path):
90
277
  components.add('documentation')
91
278
  elif path in PRESENTATION_FILES or _frontend_test(path):
@@ -96,16 +283,33 @@ def select(repo, paths):
96
283
  components.add('presentation')
97
284
  else:
98
285
  reasons.append('shared, high-risk, or unclassified path: ' + path)
286
+ if 'instructions' in components and 'presentation' in components:
287
+ reasons.append('instruction text and presentation are not certified together')
99
288
  harnesses = {'doc-lint'}
100
289
  files = set(SMOKE_FILES)
101
290
  if 'presentation' in components:
102
291
  harnesses.add('frontend')
103
292
  files.update(PRESENTATION_PYTEST)
293
+ if 'memory' in components:
294
+ files.update(MEMORY_PYTEST)
295
+ if 'instructions' in components:
296
+ files.update(INSTRUCTIONS_PYTEST)
104
297
  if not harnesses <= set(names):
105
298
  reasons.append('required merge harness absent from manifest')
106
299
  if any(not (repo / p).is_file() for p in files):
107
300
  reasons.append('required merge pytest file absent')
108
301
  full = bool(reasons)
302
+ # Whether the wrapper must read background full-verification health before
303
+ # trusting this focus. Only a memory-only selection derived with its
304
+ # irregular-file evidence is exempt: memory publication is certified by the
305
+ # memory focus whatever the background state. Containing memory is not
306
+ # enough, and a full selection has nothing to read health for.
307
+ if full:
308
+ background_health = 'not-applicable'
309
+ elif sorted(components) == ['memory'] and memory:
310
+ background_health = 'exempt'
311
+ else:
312
+ background_health = 'required'
109
313
  return {
110
314
  'mode': 'full' if full else 'merge-focused',
111
315
  'pytest': 'full' if full else 'selected',
@@ -113,13 +317,21 @@ def select(repo, paths):
113
317
  'omittedHarnesses': [n for n in names if not full and n not in harnesses],
114
318
  'selectedPytestFiles': [] if full else sorted(files),
115
319
  'components': sorted(components), 'reasons': reasons,
116
- 'policyDigest': policy_digest(repo), 'maxWallSeconds': MAX_SECONDS,
320
+ 'policyDigest': policy_digest(repo), 'backgroundHealth': background_health,
321
+ 'maxWallSeconds': MAX_SECONDS,
117
322
  }
118
323
 
119
324
 
120
325
  def plan(repo, base_rev, head='HEAD', *, working=True):
121
326
  b = basis(repo, base_rev, head, working=working)
122
- result = select(repo, b['paths'])
327
+ irregular = None
328
+ if b['paths'] and all(memory_publication_path(p) for p in b['paths']):
329
+ irregular = irregular_memory_paths(repo, b['baseOid'], b['headOid'], b['paths'], working=working)
330
+ evidence = None
331
+ if any(_sane(p) and _instruction_class(p) == 'instruction' for p in b['paths']):
332
+ evidence = ineligible_instruction_paths(repo, b['baseOid'], b['headOid'], b['paths'],
333
+ working=working)
334
+ result = select(repo, b['paths'], irregular=irregular, instruction_evidence=evidence)
123
335
  result['mergeBasis'] = b
124
336
  return result
125
337
 
@@ -138,6 +350,12 @@ def filter_legs(plan_record, directory):
138
350
  raise ValueError('selected files absent from recorded pytest estate: ' + ', '.join(sorted(selected - found)))
139
351
 
140
352
 
353
+ # Health reads one unfiltered ci.yml page and selects main on the client:
354
+ # GitHub's `branch=` index can lag the unfiltered listing by weeks (#892).
355
+ _HEALTH_LISTED_RUNS = 100 # one page at GitHub's maximum size; never a second
356
+ _HEALTH_MAIN_DEPTH = 20 # the retired branch-filtered read's depth: no older evidence
357
+
358
+
141
359
  def background_health(rows):
142
360
  """Newest completed full verification wins; running work hides nothing."""
143
361
  for row in rows:
@@ -151,11 +369,179 @@ def background_health(rows):
151
369
  return {'state': 'unavailable', 'allowFocused': False, 'runId': None, 'headSha': None}
152
370
 
153
371
 
154
- def read_background_health(repo):
155
- """Read actual full test-macos job results, never a receipt-discharge skip.
372
+ # D1(b): a scheduled test-macos job whose suite failed only on the runtime
373
+ # budget. ci.yml runs the marker step when the suite's outcome record says so;
374
+ # the reader also requires the suite step to be the job's only failed step.
375
+ _SUITE_STEP = 'Run cctally test suite'
376
+ _BUDGET_MARKER_STEP = 'Runtime budget is the only failure'
377
+ # D1(a): how many first-parent main commits are searched for a receipt.
378
+ _HEALTH_RECEIPT_COMMITS = 20
379
+ # D1(a) receipt discovery's overall time budget (#899 S2): once spent, the CI-derived
380
+ # result stands; it never waits out twenty commits of slow reads.
381
+ _HEALTH_RECEIPT_BUDGET_S = 60
382
+ _RECEIPT_RUN_ID = re.compile(r'^(\d{8}T\d{6}Z)-')
383
+
384
+
385
+ def _attempt_start(value):
386
+ """The UTC instant of a GitHub ISO-8601 timestamp; ValueError when absent or malformed."""
387
+ if not isinstance(value, str) or not value:
388
+ raise ValueError('a run has no attempt start time')
389
+ moment = datetime.fromisoformat(value.replace('Z', '+00:00'))
390
+ if moment.tzinfo is None:
391
+ raise ValueError('a run attempt start time carries no timezone')
392
+ return moment.astimezone(timezone.utc)
393
+
394
+
395
+ def _receipt_start(run_id):
396
+ """The UTC start a receipt's run ID embeds, or None."""
397
+ match = _RECEIPT_RUN_ID.match(run_id) if isinstance(run_id, str) else None
398
+ if match is None:
399
+ return None
400
+ try:
401
+ return datetime.strptime(match.group(1), '%Y%m%dT%H%M%SZ').replace(tzinfo=timezone.utc)
402
+ except ValueError:
403
+ return None
404
+
405
+
406
+ def _budget_only(row, job):
407
+ """D1(b): a scheduled run's failed test-macos whose only failure is the runtime budget."""
408
+ if row.get('event') != 'schedule' or job.get('status') != 'completed' or job.get('conclusion') != 'failure':
409
+ return False
410
+ steps = [step for step in (job.get('steps') or []) if isinstance(step, dict)]
411
+ markers = [step for step in steps if step.get('name') == _BUDGET_MARKER_STEP]
412
+ failed = {step.get('name') for step in steps if step.get('conclusion') == 'failure'}
413
+ return (len(markers) == 1 and markers[0].get('conclusion') == 'success'
414
+ and failed == {_SUITE_STEP})
415
+
416
+
417
+ def _run_item(row, jobs_of):
418
+ """The full-verification item one main run contributes: 'pass', 'fail', 'unknown'
419
+ (decisive, never a pass), or None when the run is skipped."""
420
+ if row.get('status') != 'completed':
421
+ # GitHub reuses the run ID on retry: the earlier failed attempt
422
+ # disappears from this listing while the retry runs. Do not let that
423
+ # mutation reveal an older successful run. A first attempt that is
424
+ # still running hides nothing.
425
+ return 'unknown' if row.get('run_attempt', 1) != 1 else None
426
+ jobs = jobs_of(row)
427
+ full = next((j for j in jobs if j.get('name') == 'test-macos'
428
+ and j.get('status') == 'completed'
429
+ and j.get('conclusion') != 'skipped'), None)
430
+ if full is not None:
431
+ if full.get('conclusion') == 'success' or _budget_only(row, full):
432
+ return 'pass'
433
+ return 'fail'
434
+ if row.get('event') in ('schedule', 'workflow_dispatch'):
435
+ # A scheduled/manual attempt must actually run the full job. Only
436
+ # receipt-discharged ordinary pushes may be skipped while looking for
437
+ # the last full verification.
438
+ return 'unknown'
439
+ # ci.yml can skip its declared full job on a successful private main push
440
+ # only through a successful release/receipt gate. The skipped job plus
441
+ # successful workflow is evidence of discharge; a merely successful
442
+ # classifier job alone is not (it may emit discharged=false). Missing
443
+ # jobs, cancellation, and failures must not silently inherit an older pass.
444
+ discharged = (
445
+ row.get('event') == 'push' and row.get('conclusion') == 'success'
446
+ and any(j.get('name') == 'test-macos'
447
+ and j.get('status') == 'completed'
448
+ and j.get('conclusion') == 'skipped' for j in jobs)
449
+ and any(j.get('name') in ('receipt-gate', 'release-stamp-gate')
450
+ and j.get('status') == 'completed'
451
+ and j.get('conclusion') == 'success' for j in jobs)
452
+ )
453
+ return None if discharged else 'unknown'
454
+
455
+
456
+ def _ci_item(rows, jobs_of):
457
+ """(kind, start, row) of the newest decisive run, or None.
458
+
459
+ The window is sorted by attempt start, newest first, BEFORE the walk, so a
460
+ retry of an older-created run is read in its true place. Equal starts are
461
+ decided together and a non-pass among them wins.
462
+ """
463
+ dated = [(_attempt_start(row.get('run_started_at')), index, row) for index, row in enumerate(rows)]
464
+ dated.sort(key=lambda item: (-item[0].timestamp(), item[1]))
465
+ index = 0
466
+ while index < len(dated):
467
+ start = dated[index][0]
468
+ group = []
469
+ while index < len(dated) and dated[index][0] == start:
470
+ group.append(dated[index][2])
471
+ index += 1
472
+ items = [(kind, row) for row in group if (kind := _run_item(row, jobs_of)) is not None]
473
+ if items:
474
+ kind, row = next((item for item in items if item[0] != 'pass'), items[0])
475
+ return kind, start, row
476
+ return None
477
+
478
+
479
+ def _receipt_pass(gh, name, repo, newer_than, digest_of=None):
480
+ """D1(a): (runId, commit sha) of the first valid passing full-suite receipt for
481
+ the exact tree of a recent first-parent main commit, newer than `newer_than`
482
+ (any age when None); None when there is none. Raises on any read error.
483
+
484
+ `digest_of(sha)`, when given, reconstructs a commit's tree-manifest digest in
485
+ its caller's stead (bounded by the caller's budget); by default it is built
486
+ in-process."""
487
+ # The validator of this reader's own tree, the one receipt-gate runs.
488
+ spec = importlib.util.spec_from_file_location(
489
+ '_merge_gate_receipts',
490
+ Path(__file__).resolve().parents[1] / '.github' / 'scripts' / 'classify_receipt.py')
491
+ receipts = importlib.util.module_from_spec(spec)
492
+ spec.loader.exec_module(receipts)
493
+ toolchain = receipts.toolchain_digest(str(repo))
494
+ if toolchain is None:
495
+ return None
496
+ listing = gh('api', f'repos/{name}/commits?sha=main&per_page=100')
497
+ by_sha = {commit['sha']: commit for commit in listing}
498
+ commit = listing[0] if listing else None
499
+ for _ in range(_HEALTH_RECEIPT_COMMITS):
500
+ if commit is None:
501
+ break
502
+ sha, tree = commit['sha'], commit['commit']['tree']['sha']
503
+ digest = None
504
+ refs = gh('api', f'repos/{name}/git/matching-refs/cctally/receipts/{tree}/')
505
+ for ref in sorted(refs, key=lambda r: r['ref'], reverse=True):
506
+ run_id = ref['ref'].rsplit('/', 1)[-1]
507
+ when = _receipt_start(run_id)
508
+ if when is None or (newer_than is not None and when <= newer_than):
509
+ continue
510
+ blob = gh('api', f'repos/{name}/git/blobs/{ref["object"]["sha"]}')
511
+ receipt = json.loads(base64.b64decode(blob['content']))
512
+ if not isinstance(receipt, dict) or receipt.get('runId') != run_id:
513
+ continue # dated by the run it records, and filed under that run
514
+ if digest is None:
515
+ digest = (digest_of(sha) if digest_of is not None
516
+ else receipts.manifest_digest(receipts.reconstruct_manifest(sha, str(repo))))
517
+ if receipts.verify_candidate(receipt, tree_oid=tree, ref_namespace=tree,
518
+ reconstructed_digest=digest,
519
+ toolchain_digest=toolchain) is None:
520
+ return run_id, sha
521
+ parents = commit.get('parents') or []
522
+ commit = by_sha.get(parents[0]['sha']) if parents else None
523
+ return None
524
+
156
525
 
157
- Existing full CI jobs provide a bootstrap before the new schedule lands.
158
- A successful workflow with a skipped full job is not full verification.
526
+ def read_background_health(repo):
527
+ """The state of the newest completed full-verification item.
528
+
529
+ Items are actual full test-macos job results, never a receipt-discharge
530
+ skip (a successful workflow with a skipped full job is not full
531
+ verification), with two additions (operator decision D1 of #899):
532
+
533
+ (b) a scheduled run whose suite failed only on the runtime budget, proven by
534
+ its marker step and its final step conclusions, is a pass;
535
+ (a) a valid passing full-suite receipt for the exact tree of a recent
536
+ first-parent main commit, newer than the newest CI item, is a pass.
537
+
538
+ Runs are ordered by attempt start. A missing start, an unreadable listing
539
+ and a malformed row are missing evidence; a failed receipt read leaves the
540
+ CI-derived result unchanged and never manufactures a pass, and so does a
541
+ receipt discovery that outlives its overall budget (_HEALTH_RECEIPT_BUDGET_S).
542
+ That budget bounds the whole discovery, the reconstruction of a receipt's
543
+ tree manifest included: it runs as a subprocess stopped when the budget is
544
+ spent, after which nothing further is discovered (#899 S2).
159
545
  """
160
546
  def gh(*args):
161
547
  process = subprocess.run(['gh', *args], cwd=repo, capture_output=True,
@@ -163,49 +549,81 @@ def read_background_health(repo):
163
549
  if process.returncode:
164
550
  raise ValueError('GitHub full-verification evidence is unavailable')
165
551
  return json.loads(process.stdout)
552
+
553
+ def jobs_of(row):
554
+ return gh('api', f'repos/{name}/actions/runs/{row["id"]}/jobs?per_page=100')['jobs']
166
555
  try:
167
556
  name = gh('repo', 'view', '--json', 'nameWithOwner')['nameWithOwner']
168
- rows = gh('api', f'repos/{name}/actions/workflows/ci.yml/runs?branch=main&per_page=20')['workflow_runs']
169
- for row in rows:
170
- if row.get('status') != 'completed':
171
- # GitHub reuses the run ID on retry: the earlier failed
172
- # attempt disappears from this listing while the retry runs.
173
- # Do not let that mutation reveal an older successful run.
174
- if row.get('run_attempt', 1) != 1:
175
- return background_health([])
176
- continue
177
- jobs = gh('api', f'repos/{name}/actions/runs/{row["id"]}/jobs?per_page=100')['jobs']
178
- full = next((j for j in jobs if j.get('name') == 'test-macos'
179
- and j.get('status') == 'completed'
180
- and j.get('conclusion') != 'skipped'), None)
181
- if full is not None:
182
- record = dict(row, fullSuite=True, conclusion=full.get('conclusion'))
183
- return background_health([record])
184
- if row.get('event') in ('schedule', 'workflow_dispatch'):
185
- # A scheduled/manual attempt must actually run the full job.
186
- # Only receipt-discharged ordinary pushes may be skipped while
187
- # looking for the last full verification.
188
- return background_health([])
189
- # ci.yml can skip its declared full job on a successful private
190
- # main push only through a successful release/receipt gate. The
191
- # skipped job plus successful workflow is evidence of discharge;
192
- # a merely successful classifier job alone is not (it may emit
193
- # discharged=false). Missing jobs, cancellation, and failures
194
- # must not silently inherit an older pass.
195
- discharged = (
196
- row.get('event') == 'push' and row.get('conclusion') == 'success'
197
- and any(j.get('name') == 'test-macos'
198
- and j.get('status') == 'completed'
199
- and j.get('conclusion') == 'skipped' for j in jobs)
200
- and any(j.get('name') in ('receipt-gate', 'release-stamp-gate')
201
- and j.get('status') == 'completed'
202
- and j.get('conclusion') == 'success' for j in jobs)
203
- )
204
- if not discharged:
205
- return background_health([])
206
- return background_health([])
207
- except (OSError, ValueError, KeyError, TypeError, subprocess.TimeoutExpired):
557
+ listed = gh('api', f'repos/{name}/actions/workflows/ci.yml/runs'
558
+ f'?per_page={_HEALTH_LISTED_RUNS}')['workflow_runs']
559
+ # Other branches are never main's evidence, so they are dropped before
560
+ # any rule below can decide on them or count toward the depth.
561
+ rows = [row for row in listed if row.get('head_branch') == 'main']
562
+ item = _ci_item(rows[:_HEALTH_MAIN_DEPTH], jobs_of)
563
+ except (OSError, ValueError, KeyError, TypeError, AttributeError,
564
+ subprocess.TimeoutExpired):
565
+ # A malformed listing (a null or non-object row) is missing evidence.
208
566
  return background_health([])
567
+ if item is not None and item[0] == 'pass':
568
+ return background_health([dict(item[2], fullSuite=True, conclusion='success')])
569
+ if item is not None and item[0] == 'fail':
570
+ result = background_health([dict(item[2], fullSuite=True, conclusion='failure')])
571
+ else:
572
+ result = background_health([])
573
+ budget_end = time.monotonic() + _HEALTH_RECEIPT_BUDGET_S
574
+
575
+ def budgeted_gh(*args):
576
+ # Every discovery read gets only what is left of the overall budget, and a read
577
+ # that ends past it voids the discovery: the CI-derived result stands.
578
+ left = budget_end - time.monotonic()
579
+ if left <= 0:
580
+ raise ValueError('the receipt discovery budget is spent')
581
+ process = subprocess.run(['gh', *args], cwd=repo, capture_output=True, text=True,
582
+ timeout=min(20.0, left))
583
+ if process.returncode:
584
+ raise ValueError('GitHub full-verification evidence is unavailable')
585
+ if time.monotonic() > budget_end:
586
+ raise ValueError('the receipt discovery budget is spent')
587
+ return json.loads(process.stdout)
588
+
589
+ def budgeted_digest(sha):
590
+ # The tree-manifest reconstruction runs as the classifier's own `--self-test-manifest`
591
+ # in a separate session, stopped (with every process it started) when the budget is spent.
592
+ left = budget_end - time.monotonic()
593
+ if left <= 0:
594
+ raise ValueError('the receipt discovery budget is spent')
595
+ script = Path(__file__).resolve().parents[1] / '.github' / 'scripts' / 'classify_receipt.py'
596
+ process = subprocess.Popen([sys.executable, str(script), '--sha', sha, '--repo', str(repo),
597
+ '--self-test-manifest'], stdout=subprocess.PIPE,
598
+ stderr=subprocess.DEVNULL, text=True, start_new_session=True)
599
+ try:
600
+ out, _ = process.communicate(timeout=left)
601
+ except subprocess.TimeoutExpired:
602
+ try:
603
+ os.killpg(process.pid, signal.SIGKILL)
604
+ except (ProcessLookupError, PermissionError):
605
+ pass
606
+ try:
607
+ process.communicate(timeout=5)
608
+ except subprocess.TimeoutExpired:
609
+ pass
610
+ raise ValueError('the receipt discovery budget is spent') from None
611
+ if process.returncode:
612
+ raise ValueError('the tree manifest could not be reconstructed')
613
+ if time.monotonic() > budget_end:
614
+ raise ValueError('the receipt discovery budget is spent')
615
+ return out.strip()
616
+ try:
617
+ found = _receipt_pass(budgeted_gh, name, repo, item[1] if item is not None else None,
618
+ digest_of=budgeted_digest)
619
+ if found is not None and time.monotonic() > budget_end:
620
+ found = None # validated after the budget: the CI-derived result stands
621
+ except Exception: # any read error, or a spent budget, leaves the CI-derived result unchanged
622
+ found = None
623
+ if found is None:
624
+ return result
625
+ run_id, sha = found
626
+ return {'state': 'passed', 'allowFocused': True, 'runId': run_id, 'headSha': sha}
209
627
 
210
628
 
211
629
  def supervise(command, seconds):
@@ -284,7 +702,7 @@ def main(argv=None):
284
702
  try:
285
703
  result = plan(args.repo, args.base)
286
704
  if args.required_mode:
287
- print(result['mode'])
705
+ print(f"{result['mode']} {result['backgroundHealth']}")
288
706
  elif args.filter_legs:
289
707
  filter_legs(result, args.filter_legs)
290
708
  elif args.shell: