cctally 1.114.0 → 1.116.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (86) hide show
  1. package/CHANGELOG.md +41 -0
  2. package/bin/_cctally_account.py +32 -8
  3. package/bin/_cctally_cache.py +4491 -295
  4. package/bin/_cctally_completion.py +1355 -0
  5. package/bin/_cctally_config.py +37 -0
  6. package/bin/_cctally_core.py +282 -9
  7. package/bin/_cctally_dashboard.py +8 -11
  8. package/bin/_cctally_dashboard_conversation.py +292 -23
  9. package/bin/_cctally_dashboard_envelope.py +42 -35
  10. package/bin/_cctally_dashboard_sources.py +18 -9
  11. package/bin/_cctally_db.py +401 -9
  12. package/bin/_cctally_doctor.py +469 -27
  13. package/bin/_cctally_five_hour.py +166 -1049
  14. package/bin/_cctally_forecast.py +81 -9
  15. package/bin/_cctally_journal.py +5076 -177
  16. package/bin/_cctally_journal_repair.py +110 -19
  17. package/bin/_cctally_oauth_samples.py +280 -0
  18. package/bin/_cctally_parser.py +47 -0
  19. package/bin/_cctally_project.py +8 -2
  20. package/bin/_cctally_quarantine.py +662 -0
  21. package/bin/_cctally_quota.py +96 -12
  22. package/bin/_cctally_quota_model.py +391 -83
  23. package/bin/_cctally_record.py +1088 -269
  24. package/bin/_cctally_rederive.py +854 -192
  25. package/bin/_cctally_refresh.py +113 -13
  26. package/bin/_cctally_source_analytics.py +23 -2
  27. package/bin/_cctally_statusline.py +7062 -505
  28. package/bin/_cctally_store.py +1244 -91
  29. package/bin/_cctally_transcript.py +8 -7
  30. package/bin/_cctally_tui.py +119 -60
  31. package/bin/_cctally_update.py +25 -0
  32. package/bin/_lib_account_binding.py +152 -0
  33. package/bin/_lib_account_protocol.py +569 -0
  34. package/bin/_lib_aggregators.py +4 -0
  35. package/bin/_lib_cache_coverage.py +13 -1
  36. package/bin/_lib_cache_writer_lock.py +19 -3
  37. package/bin/_lib_clone_copy.py +177 -0
  38. package/bin/_lib_codex_conversation_query.py +133 -19
  39. package/bin/_lib_codex_conversation_watch.py +59 -14
  40. package/bin/_lib_codex_drain_admission.py +214 -0
  41. package/bin/_lib_codex_metadata.py +18 -1
  42. package/bin/_lib_codex_publication_barrier.py +277 -0
  43. package/bin/_lib_codex_rollout_container.py +502 -0
  44. package/bin/_lib_conversation_dispatch.py +3 -0
  45. package/bin/_lib_conversation_query.py +11 -3
  46. package/bin/_lib_dashboard_sources.py +57 -15
  47. package/bin/_lib_detector_access_registry.py +4038 -0
  48. package/bin/_lib_doctor.py +753 -1
  49. package/bin/_lib_five_hour.py +126 -0
  50. package/bin/_lib_ingest_frontier.py +414 -20
  51. package/bin/_lib_journal.py +261 -10
  52. package/bin/_lib_journal_router.py +166 -48
  53. package/bin/_lib_jsonl.py +5 -4
  54. package/bin/_lib_legacy_spool.py +126 -0
  55. package/bin/_lib_legacy_writer_evidence.py +136 -0
  56. package/bin/_lib_meter_rate_change.py +10 -0
  57. package/bin/_lib_oauth_samples.py +238 -0
  58. package/bin/_lib_outcome_quarantine.py +690 -0
  59. package/bin/_lib_pricing.py +7 -2
  60. package/bin/_lib_pricing_check.py +0 -12
  61. package/bin/_lib_provider_activity.py +770 -0
  62. package/bin/_lib_quarantine.py +155 -0
  63. package/bin/_lib_quota_calibration.py +53 -8
  64. package/bin/_lib_quota_copy.py +2 -1
  65. package/bin/_lib_quota_model.py +384 -52
  66. package/bin/_lib_rebuild_progress.py +146 -0
  67. package/bin/_lib_record.py +22 -0
  68. package/bin/_lib_rederive.py +393 -8
  69. package/bin/_lib_segment_summary.py +35 -2
  70. package/bin/_lib_snapshot_cache.py +43 -1
  71. package/bin/_lib_stale_proof.py +708 -0
  72. package/bin/_lib_statusline_candidates.py +356 -27
  73. package/bin/_lib_worker_launches.py +315 -0
  74. package/bin/cctally +29 -12
  75. package/dashboard/static/assets/{ConversationsView-Bc8LCOAZ.js → ConversationsView-Dflb_myA.js} +1 -1
  76. package/dashboard/static/assets/DoctorModal-DFX5CfIs.js +1 -0
  77. package/dashboard/static/assets/{ModalRoot-DLPgyAM_.js → ModalRoot-HlxoagIa.js} +1 -1
  78. package/dashboard/static/assets/{ProjectsDrillPanel-z53a4v4M.js → ProjectsDrillPanel-C3PJy1xx.js} +1 -1
  79. package/dashboard/static/assets/{SourceDetailModal-Bw0T8W-W.js → SourceDetailModal-BczsWKhD.js} +1 -1
  80. package/dashboard/static/assets/{UpdateModal-D5AKx4Yw.js → UpdateModal-Os-Lo5cs.js} +1 -1
  81. package/dashboard/static/assets/{index-BXDOzSng.js → index-CYZ3NLqF.js} +3 -3
  82. package/dashboard/static/assets/{outlineNavigation-JObY7cUY.js → outlineNavigation-k-zIif6W.js} +1 -1
  83. package/dashboard/static/dashboard.html +1 -1
  84. package/package.json +20 -2
  85. package/bin/_lib_merge_gate.py +0 -725
  86. package/dashboard/static/assets/DoctorModal-D--qVBwe.js +0 -1
@@ -1,725 +0,0 @@
1
- """Conservative impact selection for merge certification (stdlib only).
2
-
3
- This is a reviewed component policy, not proof of every possible dependency.
4
- Unknown paths and shared/infrastructure changes require the full estate. The
5
- full background/release runs remain the cross-component backstop.
6
- """
7
- from __future__ import annotations
8
-
9
- import argparse
10
- import base64
11
- from datetime import datetime, timezone
12
- import hashlib
13
- import importlib.util
14
- import json
15
- import os
16
- from pathlib import Path, PurePosixPath
17
- import re
18
- import signal
19
- import subprocess
20
- import sys
21
- import time
22
-
23
- GATE_ID = 'cctally-test-remote/merge-focused@1'
24
- MAX_SECONDS = 300
25
- # All of these are leaf presentation components. Their entire frontend estate
26
- # runs, including their consumers, instead of relying on a TS import heuristic.
27
- PRESENTATION_FILES = frozenset(
28
- 'dashboard/web/src/components/' + name + '.tsx'
29
- for name in ('ZoneTag', 'ModelLegend', 'SortableHeader', 'HelpOverlay',
30
- 'AlertsEmptyGauge', 'SessionsControls', 'BasketChip', 'Toast')
31
- )
32
- SMOKE_FILES = ('tests/test_cli_smoke.py', 'tests/test_lib_json_envelope.py')
33
- PRESENTATION_PYTEST = ('tests/test_dashboard_handler_static.py',)
34
- # The reviewed memory publication set: both topic stores and the topic metadata.
35
- # A private test pins it to the publication engine's own definition. Its
36
- # configuration is NOT memory, so a change to it is ordinary task work.
37
- MEMORY_PUBLICATION_SET = ('.claude-memory/**', '.agentmem/codex-memory/**', '.agentmem/topics.json')
38
- MEMORY_CONFIGURATION = '.agentmem/config.json'
39
- # Every test module that reads the memory corpus, beside the smoke files and
40
- # the Doc lint harness: together they certify a memory-only candidate.
41
- MEMORY_PYTEST = ('tests/test_cctally_memory.py', 'tests/test_agent_workflows.py',
42
- 'tests/test_session_workflow_texts.py')
43
- # The capture publishes regular files only (a link is refused), so a memory
44
- # path that is anything else on either side is not a memory publication.
45
- REGULAR_MODES = ('100644', '100755')
46
- # The instruction-text class (#894): agent-instruction prose whose every reader
47
- # is one of INSTRUCTIONS_PYTEST. Markdown at any depth under these trees,
48
- # nested CLAUDE.md/AGENTS.md included.
49
- INSTRUCTION_TREES = ('.agent-workflows/skills/', '.claude/skills/', '.agents/skills/',
50
- '.agent-workflows/contracts/', 'docs/superpowers/')
51
- INSTRUCTION_NAMES = ('CLAUDE.md', 'AGENTS.md')
52
- # Every nested instruction file outside the trees. A new one is a structural
53
- # change to the instruction chain: its readers depend on where it lands, so it
54
- # selects the full estate until a reviewed policy change lists it here.
55
- NESTED_INSTRUCTION_FILES = ('bin/AGENTS.md', 'dashboard/AGENTS.md')
56
- # The generator inputs of the root AGENTS.md (its configuration is not one).
57
- INSTRUCTION_INPUTS = ('.agentmem/codex.md', '.agentmem/manifest.json')
58
- # Every test module that reads, validates or scans an instruction path,
59
- # beside the smoke files and the Doc lint harness.
60
- INSTRUCTIONS_PYTEST = ('tests/test_agent_workflows.py', 'tests/test_session_workflow_texts.py',
61
- 'tests/test_cctally_memory.py', 'tests/test_session_issues.py',
62
- 'tests/test_session_provenance.py', 'tests/test_linux_matrix_gate.py',
63
- 'tests/test_release_internals.py', 'tests/test_quota_budget_error_harness.py',
64
- 'tests/test_session_policy.py', 'tests/test_lib_share.py',
65
- 'tests/test_dashboard_source_kernel.py', 'tests/test_share_privacy_detector.py',
66
- 'tests/test_fixture_cache.py')
67
- # An instruction path is eligible only as a non-executable regular file.
68
- INSTRUCTION_MODES = ('100644',)
69
- # The two session execution plans are documentation: their only readers are
70
- # people and the Doc lint harness, so a plan refresh takes the documentation
71
- # focus. Exactly these two paths; any other docs/ Markdown is unclassified.
72
- EXECUTION_PLAN_PATHS = ('docs/cctally-claude-execution-plan.md', 'docs/cctally-execution-plan.md')
73
-
74
-
75
- def _git(repo, *args):
76
- proc = subprocess.run(['git', '--no-optional-locks', '-C', str(repo), *args],
77
- capture_output=True)
78
- if proc.returncode:
79
- raise ValueError('git revision/diff failed: ' + proc.stderr.decode(errors='replace').strip())
80
- return proc.stdout
81
-
82
-
83
- def _changed_names(data):
84
- """--name-only -z lists both endpoints when rename detection is disabled."""
85
- return [p.decode('utf-8', errors='strict') for p in data.split(b'\0') if p]
86
-
87
-
88
- def basis(repo, base_rev, head='HEAD', *, working=True):
89
- if not base_rev or base_rev.startswith('-'):
90
- raise ValueError('base revision must be nonempty and must not be an option')
91
- base = _git(repo, 'merge-base', base_rev, head).decode().strip()
92
- head_oid = _git(repo, 'rev-parse', '--verify', head + '^{commit}').decode().strip()
93
- committed = set(_changed_names(_git(repo, 'diff', '--no-renames', '--name-only', '-z', base, head_oid)))
94
- dirty = set()
95
- if working:
96
- dirty.update(_changed_names(_git(repo, 'diff', '--no-renames', '--name-only', '-z', head_oid)))
97
- dirty.update(_changed_names(_git(repo, 'ls-files', '--others', '--exclude-standard', '-z')))
98
- if _git(repo, 'ls-files', '-u'):
99
- raise ValueError('the index has unmerged entries')
100
- return {'baseOid': base, 'headOid': head_oid,
101
- 'committedPaths': len(committed), 'worktreePaths': len(dirty),
102
- 'paths': sorted(committed | dirty)}
103
-
104
-
105
- def memory_publication_path(path):
106
- """Whether `path` is in the memory publication set (the engine's glob semantics)."""
107
- path = path.strip('/')
108
- for pattern in MEMORY_PUBLICATION_SET:
109
- if pattern.endswith('/**'):
110
- prefix = pattern[:-3]
111
- if path == prefix or path.startswith(prefix + '/'):
112
- return True
113
- elif path == pattern:
114
- return True
115
- return False
116
-
117
-
118
- def _tree_modes(repo, rev, paths):
119
- """{path: git mode} of the entries `paths` name in `rev` (absent paths are omitted)."""
120
- out = _git(repo, '--literal-pathspecs', 'ls-tree', '-z', '--full-tree', rev, '--', *paths)
121
- modes = {}
122
- for record in out.split(b'\0'):
123
- if record:
124
- meta, name = record.split(b'\t', 1)
125
- modes[name.decode('utf-8', errors='strict')] = meta.split()[0].decode()
126
- return modes
127
-
128
-
129
- def irregular_memory_paths(repo, base, head_oid, paths, *, working=True):
130
- """The memory paths that are not a regular file (or absent) at the base, at the head,
131
- or (with `working`) in the working tree: the file types the publication set does not define."""
132
- irregular = set()
133
- for rev in (base, head_oid):
134
- irregular.update(name for name, mode in _tree_modes(repo, rev, paths).items()
135
- if mode not in REGULAR_MODES)
136
- if working:
137
- for path in paths:
138
- target = Path(repo) / path
139
- if target.is_symlink() or (target.exists() and not target.is_file()):
140
- irregular.add(path)
141
- return sorted(irregular & set(paths))
142
-
143
-
144
- def _blob_prefixes(repo, rev, paths):
145
- """{path: its first two bytes} for the blobs `paths` name in `rev`, in one git call."""
146
- if not paths:
147
- return {}
148
- request = b''.join(f'{rev}:{path}\n'.encode('utf-8') for path in paths)
149
- proc = subprocess.run(['git', '--no-optional-locks', '-C', str(repo), 'cat-file', '--batch'],
150
- input=request, capture_output=True)
151
- if proc.returncode:
152
- raise ValueError('git cat-file failed: ' + proc.stderr.decode(errors='replace').strip())
153
- out, offset, prefixes = proc.stdout, 0, {}
154
- for path in paths:
155
- end = out.index(b'\n', offset)
156
- header = out[offset:end].split()
157
- if len(header) != 3 or header[1] != b'blob':
158
- raise ValueError('git cat-file returned no blob for ' + path)
159
- size = int(header[2])
160
- prefixes[path] = out[end + 1:end + 1 + min(size, 2)]
161
- offset = end + 1 + size + 1
162
- return prefixes
163
-
164
-
165
- def ineligible_instruction_paths(repo, base, head_oid, paths, *, working=True):
166
- """The instruction paths that are not a non-executable regular file without a
167
- `#!` first line at the base, at the head, or (with `working`) on disk. Absence
168
- on any side is eligible. With `working=False` only the two commits are read."""
169
- paths = sorted(p for p in paths if _sane(p) and _instruction_class(p) == 'instruction')
170
- if not paths:
171
- return []
172
- ineligible = set()
173
- for rev in (base, head_oid):
174
- modes = _tree_modes(repo, rev, paths)
175
- ineligible.update(name for name, mode in modes.items() if mode not in INSTRUCTION_MODES)
176
- regular = sorted(name for name, mode in modes.items() if mode in INSTRUCTION_MODES)
177
- ineligible.update(name for name, prefix in _blob_prefixes(repo, rev, regular).items()
178
- if prefix == b'#!')
179
- if working:
180
- for path in paths:
181
- target = Path(repo) / path
182
- if target.is_symlink():
183
- ineligible.add(path)
184
- elif target.exists():
185
- if not target.is_file() or target.stat().st_mode & 0o111:
186
- ineligible.add(path)
187
- else:
188
- with target.open('rb') as handle:
189
- if handle.read(2) == b'#!':
190
- ineligible.add(path)
191
- return sorted(ineligible & set(paths))
192
-
193
-
194
- def _memory_only(paths, irregular):
195
- """A memory-only candidate: every path in the publication set, each a regular file."""
196
- return (irregular is not None and bool(paths) and not irregular
197
- and all(memory_publication_path(p) and _sane(p) for p in paths))
198
-
199
-
200
- def _sane(path):
201
- p = PurePosixPath(path)
202
- return not (p.is_absolute() or '..' in p.parts or any(c.isspace() for c in path))
203
-
204
-
205
- def policy_digest(repo):
206
- return 'sha256:' + hashlib.sha256(
207
- (Path(repo) / 'bin/_lib_merge_gate.py').read_bytes()).hexdigest()
208
-
209
-
210
- def _document(path):
211
- return path in ('README.md', 'CHANGELOG.md') or path in EXECUTION_PLAN_PATHS or (
212
- path.startswith('docs/commands/') and path.endswith('.md'))
213
-
214
-
215
- def _frontend_test(path):
216
- return path.startswith('dashboard/web/src/') and path.endswith(('.test.ts', '.test.tsx'))
217
-
218
-
219
- def _instruction_class(path):
220
- """'instruction' for an instruction path, a full-selection reason for an
221
- instruction-file name at an unreviewed location, or None for any other path."""
222
- if path.endswith('.md') and path.startswith(INSTRUCTION_TREES):
223
- return 'instruction'
224
- if PurePosixPath(path).name in INSTRUCTION_NAMES:
225
- if path in INSTRUCTION_NAMES or path in NESTED_INSTRUCTION_FILES:
226
- return 'instruction'
227
- return 'unreviewed nested instruction file: ' + path
228
- if path in INSTRUCTION_INPUTS:
229
- return 'instruction'
230
- return None
231
-
232
-
233
- def select(repo, paths, *, irregular=None, instruction_evidence=None):
234
- """The reviewed selection of `paths`.
235
-
236
- `irregular` is the list of memory paths that are not regular files
237
- (`irregular_memory_paths`); only a caller that computed it can have a
238
- candidate selected as memory-only. Any other candidate is selected over its
239
- whole diff, where a memory path is unclassified.
240
-
241
- `instruction_evidence` is the list of instruction paths that are not
242
- non-executable regular files (`ineligible_instruction_paths`); only a caller
243
- that computed it can have an instruction path selected focused.
244
- """
245
- repo = Path(repo)
246
- manifest = json.loads((repo / 'tests/authoritative-test-manifest.json').read_text())
247
- private = (repo / '.mirror-allowlist').exists()
248
- names = [r['name'] for r in manifest['harnesses']
249
- if private or r.get('visibility', 'public') == 'public']
250
- if not names or len(names) != len(set(names)):
251
- raise ValueError('invalid harness manifest')
252
- paths = sorted(set(paths))
253
- reasons, components = [], set()
254
- memory = _memory_only(paths, irregular)
255
- if memory:
256
- components.add('memory')
257
- for path in ([] if memory else paths):
258
- p = PurePosixPath(path)
259
- if path in (irregular or ()):
260
- reasons.append('memory path that is not a regular file: ' + path)
261
- elif p.is_absolute() or '..' in p.parts or any(c.isspace() for c in path):
262
- reasons.append('unclassified path: ' + path)
263
- elif '__pycache__' in p.parts:
264
- # Terminal: the skill comparison never reads cache content, so no
265
- # class may admit it (the memory-only shortcut is decided above).
266
- reasons.append('runtime cache path: ' + path)
267
- elif (instruction := _instruction_class(path)) is not None:
268
- if instruction != 'instruction':
269
- reasons.append(instruction)
270
- elif instruction_evidence is None:
271
- reasons.append('instruction path without file evidence: ' + path)
272
- elif path in instruction_evidence:
273
- reasons.append('instruction path that is not a non-executable regular file: ' + path)
274
- else:
275
- components.add('instructions')
276
- elif _document(path):
277
- components.add('documentation')
278
- elif path in PRESENTATION_FILES or _frontend_test(path):
279
- components.add('presentation')
280
- elif path.startswith('dashboard/static/'):
281
- components.add('presentation')
282
- elif path in PRESENTATION_PYTEST:
283
- components.add('presentation')
284
- else:
285
- reasons.append('shared, high-risk, or unclassified path: ' + path)
286
- if 'instructions' in components and 'presentation' in components:
287
- reasons.append('instruction text and presentation are not certified together')
288
- harnesses = {'doc-lint'}
289
- files = set(SMOKE_FILES)
290
- if 'presentation' in components:
291
- harnesses.add('frontend')
292
- files.update(PRESENTATION_PYTEST)
293
- if 'memory' in components:
294
- files.update(MEMORY_PYTEST)
295
- if 'instructions' in components:
296
- files.update(INSTRUCTIONS_PYTEST)
297
- if not harnesses <= set(names):
298
- reasons.append('required merge harness absent from manifest')
299
- if any(not (repo / p).is_file() for p in files):
300
- reasons.append('required merge pytest file absent')
301
- full = bool(reasons)
302
- # Whether the wrapper must read background full-verification health before
303
- # trusting this focus. Only a memory-only selection derived with its
304
- # irregular-file evidence is exempt: memory publication is certified by the
305
- # memory focus whatever the background state. Containing memory is not
306
- # enough, and a full selection has nothing to read health for.
307
- if full:
308
- background_health = 'not-applicable'
309
- elif sorted(components) == ['memory'] and memory:
310
- background_health = 'exempt'
311
- else:
312
- background_health = 'required'
313
- return {
314
- 'mode': 'full' if full else 'merge-focused',
315
- 'pytest': 'full' if full else 'selected',
316
- 'selectedHarnesses': [n for n in names if full or n in harnesses],
317
- 'omittedHarnesses': [n for n in names if not full and n not in harnesses],
318
- 'selectedPytestFiles': [] if full else sorted(files),
319
- 'components': sorted(components), 'reasons': reasons,
320
- 'policyDigest': policy_digest(repo), 'backgroundHealth': background_health,
321
- 'maxWallSeconds': MAX_SECONDS,
322
- }
323
-
324
-
325
- def plan(repo, base_rev, head='HEAD', *, working=True):
326
- b = basis(repo, base_rev, head, working=working)
327
- irregular = None
328
- if b['paths'] and all(memory_publication_path(p) for p in b['paths']):
329
- irregular = irregular_memory_paths(repo, b['baseOid'], b['headOid'], b['paths'], working=working)
330
- evidence = None
331
- if any(_sane(p) and _instruction_class(p) == 'instruction' for p in b['paths']):
332
- evidence = ineligible_instruction_paths(repo, b['baseOid'], b['headOid'], b['paths'],
333
- working=working)
334
- result = select(repo, b['paths'], irregular=irregular, instruction_evidence=evidence)
335
- result['mergeBasis'] = b
336
- return result
337
-
338
-
339
- def filter_legs(plan_record, directory):
340
- """Narrow both expected-node legs to selected files; retain serial ownership."""
341
- directory = Path(directory)
342
- selected = set(plan_record['selectedPytestFiles'])
343
- found = set()
344
- for name in ('pytest', 'benchmark'):
345
- path = directory / (name + '.txt')
346
- nodes = [n for n in path.read_text().splitlines() if n.split('::')[0] in selected]
347
- found.update(n.split('::')[0] for n in nodes)
348
- path.write_text(''.join(n + '\n' for n in nodes))
349
- if found != selected:
350
- raise ValueError('selected files absent from recorded pytest estate: ' + ', '.join(sorted(selected - found)))
351
-
352
-
353
- # Health reads one unfiltered ci.yml page and selects main on the client:
354
- # GitHub's `branch=` index can lag the unfiltered listing by weeks (#892).
355
- _HEALTH_LISTED_RUNS = 100 # one page at GitHub's maximum size; never a second
356
- _HEALTH_MAIN_DEPTH = 20 # the retired branch-filtered read's depth: no older evidence
357
-
358
-
359
- def background_health(rows):
360
- """Newest completed full verification wins; running work hides nothing."""
361
- for row in rows:
362
- if row.get('head_branch') != 'main' or row.get('status') != 'completed':
363
- continue
364
- if row.get('event') not in ('schedule', 'workflow_dispatch') and row.get('fullSuite') is not True:
365
- continue
366
- passed = row.get('conclusion') == 'success'
367
- return {'state': 'passed' if passed else 'failed', 'allowFocused': passed,
368
- 'runId': row.get('id'), 'headSha': row.get('head_sha')}
369
- return {'state': 'unavailable', 'allowFocused': False, 'runId': None, 'headSha': None}
370
-
371
-
372
- # D1(b): a scheduled test-macos job whose suite failed only on the runtime
373
- # budget. ci.yml runs the marker step when the suite's outcome record says so;
374
- # the reader also requires the suite step to be the job's only failed step.
375
- _SUITE_STEP = 'Run cctally test suite'
376
- _BUDGET_MARKER_STEP = 'Runtime budget is the only failure'
377
- # D1(a): how many first-parent main commits are searched for a receipt.
378
- _HEALTH_RECEIPT_COMMITS = 20
379
- # D1(a) receipt discovery's overall time budget (#899 S2): once spent, the CI-derived
380
- # result stands; it never waits out twenty commits of slow reads.
381
- _HEALTH_RECEIPT_BUDGET_S = 60
382
- _RECEIPT_RUN_ID = re.compile(r'^(\d{8}T\d{6}Z)-')
383
-
384
-
385
- def _attempt_start(value):
386
- """The UTC instant of a GitHub ISO-8601 timestamp; ValueError when absent or malformed."""
387
- if not isinstance(value, str) or not value:
388
- raise ValueError('a run has no attempt start time')
389
- moment = datetime.fromisoformat(value.replace('Z', '+00:00'))
390
- if moment.tzinfo is None:
391
- raise ValueError('a run attempt start time carries no timezone')
392
- return moment.astimezone(timezone.utc)
393
-
394
-
395
- def _receipt_start(run_id):
396
- """The UTC start a receipt's run ID embeds, or None."""
397
- match = _RECEIPT_RUN_ID.match(run_id) if isinstance(run_id, str) else None
398
- if match is None:
399
- return None
400
- try:
401
- return datetime.strptime(match.group(1), '%Y%m%dT%H%M%SZ').replace(tzinfo=timezone.utc)
402
- except ValueError:
403
- return None
404
-
405
-
406
- def _budget_only(row, job):
407
- """D1(b): a scheduled run's failed test-macos whose only failure is the runtime budget."""
408
- if row.get('event') != 'schedule' or job.get('status') != 'completed' or job.get('conclusion') != 'failure':
409
- return False
410
- steps = [step for step in (job.get('steps') or []) if isinstance(step, dict)]
411
- markers = [step for step in steps if step.get('name') == _BUDGET_MARKER_STEP]
412
- failed = {step.get('name') for step in steps if step.get('conclusion') == 'failure'}
413
- return (len(markers) == 1 and markers[0].get('conclusion') == 'success'
414
- and failed == {_SUITE_STEP})
415
-
416
-
417
- def _run_item(row, jobs_of):
418
- """The full-verification item one main run contributes: 'pass', 'fail', 'unknown'
419
- (decisive, never a pass), or None when the run is skipped."""
420
- if row.get('status') != 'completed':
421
- # GitHub reuses the run ID on retry: the earlier failed attempt
422
- # disappears from this listing while the retry runs. Do not let that
423
- # mutation reveal an older successful run. A first attempt that is
424
- # still running hides nothing.
425
- return 'unknown' if row.get('run_attempt', 1) != 1 else None
426
- jobs = jobs_of(row)
427
- full = next((j for j in jobs if j.get('name') == 'test-macos'
428
- and j.get('status') == 'completed'
429
- and j.get('conclusion') != 'skipped'), None)
430
- if full is not None:
431
- if full.get('conclusion') == 'success' or _budget_only(row, full):
432
- return 'pass'
433
- return 'fail'
434
- if row.get('event') in ('schedule', 'workflow_dispatch'):
435
- # A scheduled/manual attempt must actually run the full job. Only
436
- # receipt-discharged ordinary pushes may be skipped while looking for
437
- # the last full verification.
438
- return 'unknown'
439
- # ci.yml can skip its declared full job on a successful private main push
440
- # only through a successful release/receipt gate. The skipped job plus
441
- # successful workflow is evidence of discharge; a merely successful
442
- # classifier job alone is not (it may emit discharged=false). Missing
443
- # jobs, cancellation, and failures must not silently inherit an older pass.
444
- discharged = (
445
- row.get('event') == 'push' and row.get('conclusion') == 'success'
446
- and any(j.get('name') == 'test-macos'
447
- and j.get('status') == 'completed'
448
- and j.get('conclusion') == 'skipped' for j in jobs)
449
- and any(j.get('name') in ('receipt-gate', 'release-stamp-gate')
450
- and j.get('status') == 'completed'
451
- and j.get('conclusion') == 'success' for j in jobs)
452
- )
453
- return None if discharged else 'unknown'
454
-
455
-
456
- def _ci_item(rows, jobs_of):
457
- """(kind, start, row) of the newest decisive run, or None.
458
-
459
- The window is sorted by attempt start, newest first, BEFORE the walk, so a
460
- retry of an older-created run is read in its true place. Equal starts are
461
- decided together and a non-pass among them wins.
462
- """
463
- dated = [(_attempt_start(row.get('run_started_at')), index, row) for index, row in enumerate(rows)]
464
- dated.sort(key=lambda item: (-item[0].timestamp(), item[1]))
465
- index = 0
466
- while index < len(dated):
467
- start = dated[index][0]
468
- group = []
469
- while index < len(dated) and dated[index][0] == start:
470
- group.append(dated[index][2])
471
- index += 1
472
- items = [(kind, row) for row in group if (kind := _run_item(row, jobs_of)) is not None]
473
- if items:
474
- kind, row = next((item for item in items if item[0] != 'pass'), items[0])
475
- return kind, start, row
476
- return None
477
-
478
-
479
- def _receipt_pass(gh, name, repo, newer_than, digest_of=None):
480
- """D1(a): (runId, commit sha) of the first valid passing full-suite receipt for
481
- the exact tree of a recent first-parent main commit, newer than `newer_than`
482
- (any age when None); None when there is none. Raises on any read error.
483
-
484
- `digest_of(sha)`, when given, reconstructs a commit's tree-manifest digest in
485
- its caller's stead (bounded by the caller's budget); by default it is built
486
- in-process."""
487
- # The validator of this reader's own tree, the one receipt-gate runs.
488
- spec = importlib.util.spec_from_file_location(
489
- '_merge_gate_receipts',
490
- Path(__file__).resolve().parents[1] / '.github' / 'scripts' / 'classify_receipt.py')
491
- receipts = importlib.util.module_from_spec(spec)
492
- spec.loader.exec_module(receipts)
493
- toolchain = receipts.toolchain_digest(str(repo))
494
- if toolchain is None:
495
- return None
496
- listing = gh('api', f'repos/{name}/commits?sha=main&per_page=100')
497
- by_sha = {commit['sha']: commit for commit in listing}
498
- commit = listing[0] if listing else None
499
- for _ in range(_HEALTH_RECEIPT_COMMITS):
500
- if commit is None:
501
- break
502
- sha, tree = commit['sha'], commit['commit']['tree']['sha']
503
- digest = None
504
- refs = gh('api', f'repos/{name}/git/matching-refs/cctally/receipts/{tree}/')
505
- for ref in sorted(refs, key=lambda r: r['ref'], reverse=True):
506
- run_id = ref['ref'].rsplit('/', 1)[-1]
507
- when = _receipt_start(run_id)
508
- if when is None or (newer_than is not None and when <= newer_than):
509
- continue
510
- blob = gh('api', f'repos/{name}/git/blobs/{ref["object"]["sha"]}')
511
- receipt = json.loads(base64.b64decode(blob['content']))
512
- if not isinstance(receipt, dict) or receipt.get('runId') != run_id:
513
- continue # dated by the run it records, and filed under that run
514
- if digest is None:
515
- digest = (digest_of(sha) if digest_of is not None
516
- else receipts.manifest_digest(receipts.reconstruct_manifest(sha, str(repo))))
517
- if receipts.verify_candidate(receipt, tree_oid=tree, ref_namespace=tree,
518
- reconstructed_digest=digest,
519
- toolchain_digest=toolchain) is None:
520
- return run_id, sha
521
- parents = commit.get('parents') or []
522
- commit = by_sha.get(parents[0]['sha']) if parents else None
523
- return None
524
-
525
-
526
- def read_background_health(repo):
527
- """The state of the newest completed full-verification item.
528
-
529
- Items are actual full test-macos job results, never a receipt-discharge
530
- skip (a successful workflow with a skipped full job is not full
531
- verification), with two additions (operator decision D1 of #899):
532
-
533
- (b) a scheduled run whose suite failed only on the runtime budget, proven by
534
- its marker step and its final step conclusions, is a pass;
535
- (a) a valid passing full-suite receipt for the exact tree of a recent
536
- first-parent main commit, newer than the newest CI item, is a pass.
537
-
538
- Runs are ordered by attempt start. A missing start, an unreadable listing
539
- and a malformed row are missing evidence; a failed receipt read leaves the
540
- CI-derived result unchanged and never manufactures a pass, and so does a
541
- receipt discovery that outlives its overall budget (_HEALTH_RECEIPT_BUDGET_S).
542
- That budget bounds the whole discovery, the reconstruction of a receipt's
543
- tree manifest included: it runs as a subprocess stopped when the budget is
544
- spent, after which nothing further is discovered (#899 S2).
545
- """
546
- def gh(*args):
547
- process = subprocess.run(['gh', *args], cwd=repo, capture_output=True,
548
- text=True, timeout=20)
549
- if process.returncode:
550
- raise ValueError('GitHub full-verification evidence is unavailable')
551
- return json.loads(process.stdout)
552
-
553
- def jobs_of(row):
554
- return gh('api', f'repos/{name}/actions/runs/{row["id"]}/jobs?per_page=100')['jobs']
555
- try:
556
- name = gh('repo', 'view', '--json', 'nameWithOwner')['nameWithOwner']
557
- listed = gh('api', f'repos/{name}/actions/workflows/ci.yml/runs'
558
- f'?per_page={_HEALTH_LISTED_RUNS}')['workflow_runs']
559
- # Other branches are never main's evidence, so they are dropped before
560
- # any rule below can decide on them or count toward the depth.
561
- rows = [row for row in listed if row.get('head_branch') == 'main']
562
- item = _ci_item(rows[:_HEALTH_MAIN_DEPTH], jobs_of)
563
- except (OSError, ValueError, KeyError, TypeError, AttributeError,
564
- subprocess.TimeoutExpired):
565
- # A malformed listing (a null or non-object row) is missing evidence.
566
- return background_health([])
567
- if item is not None and item[0] == 'pass':
568
- return background_health([dict(item[2], fullSuite=True, conclusion='success')])
569
- if item is not None and item[0] == 'fail':
570
- result = background_health([dict(item[2], fullSuite=True, conclusion='failure')])
571
- else:
572
- result = background_health([])
573
- budget_end = time.monotonic() + _HEALTH_RECEIPT_BUDGET_S
574
-
575
- def budgeted_gh(*args):
576
- # Every discovery read gets only what is left of the overall budget, and a read
577
- # that ends past it voids the discovery: the CI-derived result stands.
578
- left = budget_end - time.monotonic()
579
- if left <= 0:
580
- raise ValueError('the receipt discovery budget is spent')
581
- process = subprocess.run(['gh', *args], cwd=repo, capture_output=True, text=True,
582
- timeout=min(20.0, left))
583
- if process.returncode:
584
- raise ValueError('GitHub full-verification evidence is unavailable')
585
- if time.monotonic() > budget_end:
586
- raise ValueError('the receipt discovery budget is spent')
587
- return json.loads(process.stdout)
588
-
589
- def budgeted_digest(sha):
590
- # The tree-manifest reconstruction runs as the classifier's own `--self-test-manifest`
591
- # in a separate session, stopped (with every process it started) when the budget is spent.
592
- left = budget_end - time.monotonic()
593
- if left <= 0:
594
- raise ValueError('the receipt discovery budget is spent')
595
- script = Path(__file__).resolve().parents[1] / '.github' / 'scripts' / 'classify_receipt.py'
596
- process = subprocess.Popen([sys.executable, str(script), '--sha', sha, '--repo', str(repo),
597
- '--self-test-manifest'], stdout=subprocess.PIPE,
598
- stderr=subprocess.DEVNULL, text=True, start_new_session=True)
599
- try:
600
- out, _ = process.communicate(timeout=left)
601
- except subprocess.TimeoutExpired:
602
- try:
603
- os.killpg(process.pid, signal.SIGKILL)
604
- except (ProcessLookupError, PermissionError):
605
- pass
606
- try:
607
- process.communicate(timeout=5)
608
- except subprocess.TimeoutExpired:
609
- pass
610
- raise ValueError('the receipt discovery budget is spent') from None
611
- if process.returncode:
612
- raise ValueError('the tree manifest could not be reconstructed')
613
- if time.monotonic() > budget_end:
614
- raise ValueError('the receipt discovery budget is spent')
615
- return out.strip()
616
- try:
617
- found = _receipt_pass(budgeted_gh, name, repo, item[1] if item is not None else None,
618
- digest_of=budgeted_digest)
619
- if found is not None and time.monotonic() > budget_end:
620
- found = None # validated after the budget: the CI-derived result stands
621
- except Exception: # any read error, or a spent budget, leaves the CI-derived result unchanged
622
- found = None
623
- if found is None:
624
- return result
625
- run_id, sha = found
626
- return {'state': 'passed', 'allowFocused': True, 'runId': run_id, 'headSha': sha}
627
-
628
-
629
- def supervise(command, seconds):
630
- """Bound the complete gate, including foreground admission subprocesses.
631
-
632
- A fresh POSIX session contains the aggregator's separate pool/pytest
633
- process groups. Terminating that session also reaches children when bash
634
- is waiting on an admission command and cannot yet run its signal trap.
635
- The caller writes the ordinary contract's timeout verdict after teardown.
636
- """
637
- process = subprocess.Popen(command, start_new_session=True)
638
-
639
- def members():
640
- listing = subprocess.check_output(['ps', '-A', '-o', 'pid='], text=True)
641
- result = []
642
- for token in listing.split():
643
- pid = int(token)
644
- try:
645
- if os.getsid(pid) == process.pid:
646
- result.append(pid)
647
- except (ProcessLookupError, PermissionError):
648
- pass
649
- return result
650
-
651
- def terminate():
652
- for sig in (signal.SIGTERM, signal.SIGKILL):
653
- for pid in members():
654
- try:
655
- os.kill(pid, sig)
656
- except ProcessLookupError:
657
- pass
658
- if sig == signal.SIGTERM:
659
- try:
660
- process.wait(timeout=2)
661
- except subprocess.TimeoutExpired:
662
- pass
663
- process.wait()
664
-
665
- def interrupted(signum, _frame):
666
- terminate()
667
- raise SystemExit(128 + signum)
668
-
669
- old = {sig: signal.signal(sig, interrupted) for sig in (signal.SIGTERM, signal.SIGINT)}
670
- try:
671
- try:
672
- return process.wait(timeout=max(0, seconds))
673
- except subprocess.TimeoutExpired:
674
- terminate()
675
- return 124
676
- finally:
677
- for sig, handler in old.items():
678
- signal.signal(sig, handler)
679
-
680
-
681
- def main(argv=None):
682
- parser = argparse.ArgumentParser(description=__doc__)
683
- parser.add_argument('--repo', default=str(Path(__file__).resolve().parents[1]))
684
- parser.add_argument('--base')
685
- parser.add_argument('--health', action='store_true')
686
- parser.add_argument('--required-mode', action='store_true')
687
- parser.add_argument('--shell', action='store_true')
688
- parser.add_argument('--filter-legs')
689
- parser.add_argument('--deadline', nargs=argparse.REMAINDER,
690
- help=argparse.SUPPRESS)
691
- args = parser.parse_args(argv)
692
- if args.deadline is not None:
693
- if len(args.deadline) < 2:
694
- parser.error('deadline requires seconds and a command')
695
- return supervise(args.deadline[1:], int(args.deadline[0]))
696
- if args.health:
697
- health = read_background_health(args.repo)
698
- print(json.dumps(health, sort_keys=True))
699
- return 0 if health['allowFocused'] else 3
700
- if not args.base:
701
- parser.error('--base is required outside --health')
702
- try:
703
- result = plan(args.repo, args.base)
704
- if args.required_mode:
705
- print(f"{result['mode']} {result['backgroundHealth']}")
706
- elif args.filter_legs:
707
- filter_legs(result, args.filter_legs)
708
- elif args.shell:
709
- print('mode\t' + result['mode'])
710
- print('harnesses\t' + ' '.join(result['selectedHarnesses']))
711
- print('omitted\t' + ' '.join(result['omittedHarnesses']))
712
- print('pytest\t' + result['pytest'])
713
- print('files\t' + ' '.join(result['selectedPytestFiles']))
714
- print('seconds\t' + str(result['maxWallSeconds']))
715
- print('record\t' + json.dumps(result, sort_keys=True, separators=(',', ':')))
716
- else:
717
- print(json.dumps(result, sort_keys=True, indent=2))
718
- except (ValueError, OSError, KeyError, UnicodeError) as exc:
719
- print('merge-gate: ' + str(exc), file=sys.stderr)
720
- return 2
721
- return 0
722
-
723
-
724
- if __name__ == '__main__':
725
- raise SystemExit(main())