cctally 1.112.0 → 1.114.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +27 -0
- package/README.md +4 -5
- package/bin/_cctally_cache.py +1683 -485
- package/bin/_cctally_core.py +223 -1
- package/bin/_cctally_dashboard.py +96 -18
- package/bin/_cctally_dashboard_perf.py +128 -9
- package/bin/_cctally_dashboard_sources.py +267 -204
- package/bin/_cctally_db.py +861 -18
- package/bin/_cctally_diagnosis_sources.py +22 -4
- package/bin/_cctally_doctor.py +136 -18
- package/bin/_cctally_five_hour.py +2 -2
- package/bin/_cctally_forecast.py +27 -6
- package/bin/_cctally_journal.py +67 -5
- package/bin/_cctally_milestone_history.py +82 -9
- package/bin/_cctally_percent_breakdown.py +3 -2
- package/bin/_cctally_quota.py +201 -33
- package/bin/_cctally_record.py +14 -0
- package/bin/_cctally_rederive.py +23 -9
- package/bin/_cctally_source_analytics.py +60 -15
- package/bin/_cctally_statusline.py +17 -0
- package/bin/_cctally_store.py +80 -5
- package/bin/_cctally_tui.py +4 -3
- package/bin/_cctally_weekrefs.py +128 -21
- package/bin/_lib_cache_report.py +35 -6
- package/bin/_lib_codex_conversation.py +32 -0
- package/bin/_lib_codex_conversation_query.py +164 -30
- package/bin/_lib_conversation_query.py +95 -56
- package/bin/_lib_conversation_retention.py +1605 -640
- package/bin/_lib_cost_provenance.py +53 -2
- package/bin/_lib_doctor.py +263 -69
- package/bin/_lib_fmt.py +11 -0
- package/bin/_lib_merge_gate.py +469 -51
- package/bin/_lib_perf.py +27 -1
- package/bin/_lib_pricing.py +209 -8
- package/bin/_lib_pricing_check.py +20 -4
- package/bin/_lib_reclaim_planner.py +792 -0
- package/bin/_lib_rederive.py +225 -77
- package/bin/_lib_snapshot_cache.py +21 -0
- package/bin/_lib_sqlite_close.py +81 -0
- package/bin/_lib_subscription_weeks.py +100 -17
- package/bin/_lib_tick_stats.py +217 -17
- package/bin/_lib_wal_checkpoint.py +1077 -0
- package/bin/_lib_write_budget.py +483 -0
- package/bin/_lib_write_io.py +703 -0
- package/bin/cctally +8 -0
- package/dashboard/static/assets/{ConversationsView-CEZ-BUyi.js → ConversationsView-Bc8LCOAZ.js} +1 -1
- package/dashboard/static/assets/{DoctorModal-CE1oH_Zy.js → DoctorModal-D--qVBwe.js} +1 -1
- package/dashboard/static/assets/ModalRoot-DLPgyAM_.js +1 -0
- package/dashboard/static/assets/{ProjectsDrillPanel-B3DZoCeD.js → ProjectsDrillPanel-z53a4v4M.js} +1 -1
- package/dashboard/static/assets/{SourceDetailModal-CHp9EgLP.js → SourceDetailModal-Bw0T8W-W.js} +1 -1
- package/dashboard/static/assets/{UpdateModal-5iKST6jo.js → UpdateModal-D5AKx4Yw.js} +1 -1
- package/dashboard/static/assets/index-BKITk1CT.css +1 -0
- package/dashboard/static/assets/{index-Cm0VTxJC.js → index-BXDOzSng.js} +3 -3
- package/dashboard/static/assets/{outlineNavigation-DkXGL8J6.js → outlineNavigation-JObY7cUY.js} +1 -1
- package/dashboard/static/dashboard.html +2 -2
- package/package.json +6 -1
- package/dashboard/static/assets/ModalRoot-D5Y8IFUG.js +0 -1
- package/dashboard/static/assets/index-Dt4tqf43.css +0 -1
package/bin/_lib_merge_gate.py
CHANGED
|
@@ -7,13 +7,18 @@ full background/release runs remain the cross-component backstop.
|
|
|
7
7
|
from __future__ import annotations
|
|
8
8
|
|
|
9
9
|
import argparse
|
|
10
|
+
import base64
|
|
11
|
+
from datetime import datetime, timezone
|
|
10
12
|
import hashlib
|
|
13
|
+
import importlib.util
|
|
11
14
|
import json
|
|
12
15
|
import os
|
|
13
16
|
from pathlib import Path, PurePosixPath
|
|
17
|
+
import re
|
|
14
18
|
import signal
|
|
15
19
|
import subprocess
|
|
16
20
|
import sys
|
|
21
|
+
import time
|
|
17
22
|
|
|
18
23
|
GATE_ID = 'cctally-test-remote/merge-focused@1'
|
|
19
24
|
MAX_SECONDS = 300
|
|
@@ -26,6 +31,45 @@ PRESENTATION_FILES = frozenset(
|
|
|
26
31
|
)
|
|
27
32
|
SMOKE_FILES = ('tests/test_cli_smoke.py', 'tests/test_lib_json_envelope.py')
|
|
28
33
|
PRESENTATION_PYTEST = ('tests/test_dashboard_handler_static.py',)
|
|
34
|
+
# The reviewed memory publication set: both topic stores and the topic metadata.
|
|
35
|
+
# A private test pins it to the publication engine's own definition. Its
|
|
36
|
+
# configuration is NOT memory, so a change to it is ordinary task work.
|
|
37
|
+
MEMORY_PUBLICATION_SET = ('.claude-memory/**', '.agentmem/codex-memory/**', '.agentmem/topics.json')
|
|
38
|
+
MEMORY_CONFIGURATION = '.agentmem/config.json'
|
|
39
|
+
# Every test module that reads the memory corpus, beside the smoke files and
|
|
40
|
+
# the Doc lint harness: together they certify a memory-only candidate.
|
|
41
|
+
MEMORY_PYTEST = ('tests/test_cctally_memory.py', 'tests/test_agent_workflows.py',
|
|
42
|
+
'tests/test_session_workflow_texts.py')
|
|
43
|
+
# The capture publishes regular files only (a link is refused), so a memory
|
|
44
|
+
# path that is anything else on either side is not a memory publication.
|
|
45
|
+
REGULAR_MODES = ('100644', '100755')
|
|
46
|
+
# The instruction-text class (#894): agent-instruction prose whose every reader
|
|
47
|
+
# is one of INSTRUCTIONS_PYTEST. Markdown at any depth under these trees,
|
|
48
|
+
# nested CLAUDE.md/AGENTS.md included.
|
|
49
|
+
INSTRUCTION_TREES = ('.agent-workflows/skills/', '.claude/skills/', '.agents/skills/',
|
|
50
|
+
'.agent-workflows/contracts/', 'docs/superpowers/')
|
|
51
|
+
INSTRUCTION_NAMES = ('CLAUDE.md', 'AGENTS.md')
|
|
52
|
+
# Every nested instruction file outside the trees. A new one is a structural
|
|
53
|
+
# change to the instruction chain: its readers depend on where it lands, so it
|
|
54
|
+
# selects the full estate until a reviewed policy change lists it here.
|
|
55
|
+
NESTED_INSTRUCTION_FILES = ('bin/AGENTS.md', 'dashboard/AGENTS.md')
|
|
56
|
+
# The generator inputs of the root AGENTS.md (its configuration is not one).
|
|
57
|
+
INSTRUCTION_INPUTS = ('.agentmem/codex.md', '.agentmem/manifest.json')
|
|
58
|
+
# Every test module that reads, validates or scans an instruction path,
|
|
59
|
+
# beside the smoke files and the Doc lint harness.
|
|
60
|
+
INSTRUCTIONS_PYTEST = ('tests/test_agent_workflows.py', 'tests/test_session_workflow_texts.py',
|
|
61
|
+
'tests/test_cctally_memory.py', 'tests/test_session_issues.py',
|
|
62
|
+
'tests/test_session_provenance.py', 'tests/test_linux_matrix_gate.py',
|
|
63
|
+
'tests/test_release_internals.py', 'tests/test_quota_budget_error_harness.py',
|
|
64
|
+
'tests/test_session_policy.py', 'tests/test_lib_share.py',
|
|
65
|
+
'tests/test_dashboard_source_kernel.py', 'tests/test_share_privacy_detector.py',
|
|
66
|
+
'tests/test_fixture_cache.py')
|
|
67
|
+
# An instruction path is eligible only as a non-executable regular file.
|
|
68
|
+
INSTRUCTION_MODES = ('100644',)
|
|
69
|
+
# The two session execution plans are documentation: their only readers are
|
|
70
|
+
# people and the Doc lint harness, so a plan refresh takes the documentation
|
|
71
|
+
# focus. Exactly these two paths; any other docs/ Markdown is unclassified.
|
|
72
|
+
EXECUTION_PLAN_PATHS = ('docs/cctally-claude-execution-plan.md', 'docs/cctally-execution-plan.md')
|
|
29
73
|
|
|
30
74
|
|
|
31
75
|
def _git(repo, *args):
|
|
@@ -58,13 +102,113 @@ def basis(repo, base_rev, head='HEAD', *, working=True):
|
|
|
58
102
|
'paths': sorted(committed | dirty)}
|
|
59
103
|
|
|
60
104
|
|
|
105
|
+
def memory_publication_path(path):
|
|
106
|
+
"""Whether `path` is in the memory publication set (the engine's glob semantics)."""
|
|
107
|
+
path = path.strip('/')
|
|
108
|
+
for pattern in MEMORY_PUBLICATION_SET:
|
|
109
|
+
if pattern.endswith('/**'):
|
|
110
|
+
prefix = pattern[:-3]
|
|
111
|
+
if path == prefix or path.startswith(prefix + '/'):
|
|
112
|
+
return True
|
|
113
|
+
elif path == pattern:
|
|
114
|
+
return True
|
|
115
|
+
return False
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def _tree_modes(repo, rev, paths):
|
|
119
|
+
"""{path: git mode} of the entries `paths` name in `rev` (absent paths are omitted)."""
|
|
120
|
+
out = _git(repo, '--literal-pathspecs', 'ls-tree', '-z', '--full-tree', rev, '--', *paths)
|
|
121
|
+
modes = {}
|
|
122
|
+
for record in out.split(b'\0'):
|
|
123
|
+
if record:
|
|
124
|
+
meta, name = record.split(b'\t', 1)
|
|
125
|
+
modes[name.decode('utf-8', errors='strict')] = meta.split()[0].decode()
|
|
126
|
+
return modes
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def irregular_memory_paths(repo, base, head_oid, paths, *, working=True):
|
|
130
|
+
"""The memory paths that are not a regular file (or absent) at the base, at the head,
|
|
131
|
+
or (with `working`) in the working tree: the file types the publication set does not define."""
|
|
132
|
+
irregular = set()
|
|
133
|
+
for rev in (base, head_oid):
|
|
134
|
+
irregular.update(name for name, mode in _tree_modes(repo, rev, paths).items()
|
|
135
|
+
if mode not in REGULAR_MODES)
|
|
136
|
+
if working:
|
|
137
|
+
for path in paths:
|
|
138
|
+
target = Path(repo) / path
|
|
139
|
+
if target.is_symlink() or (target.exists() and not target.is_file()):
|
|
140
|
+
irregular.add(path)
|
|
141
|
+
return sorted(irregular & set(paths))
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def _blob_prefixes(repo, rev, paths):
|
|
145
|
+
"""{path: its first two bytes} for the blobs `paths` name in `rev`, in one git call."""
|
|
146
|
+
if not paths:
|
|
147
|
+
return {}
|
|
148
|
+
request = b''.join(f'{rev}:{path}\n'.encode('utf-8') for path in paths)
|
|
149
|
+
proc = subprocess.run(['git', '--no-optional-locks', '-C', str(repo), 'cat-file', '--batch'],
|
|
150
|
+
input=request, capture_output=True)
|
|
151
|
+
if proc.returncode:
|
|
152
|
+
raise ValueError('git cat-file failed: ' + proc.stderr.decode(errors='replace').strip())
|
|
153
|
+
out, offset, prefixes = proc.stdout, 0, {}
|
|
154
|
+
for path in paths:
|
|
155
|
+
end = out.index(b'\n', offset)
|
|
156
|
+
header = out[offset:end].split()
|
|
157
|
+
if len(header) != 3 or header[1] != b'blob':
|
|
158
|
+
raise ValueError('git cat-file returned no blob for ' + path)
|
|
159
|
+
size = int(header[2])
|
|
160
|
+
prefixes[path] = out[end + 1:end + 1 + min(size, 2)]
|
|
161
|
+
offset = end + 1 + size + 1
|
|
162
|
+
return prefixes
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def ineligible_instruction_paths(repo, base, head_oid, paths, *, working=True):
|
|
166
|
+
"""The instruction paths that are not a non-executable regular file without a
|
|
167
|
+
`#!` first line at the base, at the head, or (with `working`) on disk. Absence
|
|
168
|
+
on any side is eligible. With `working=False` only the two commits are read."""
|
|
169
|
+
paths = sorted(p for p in paths if _sane(p) and _instruction_class(p) == 'instruction')
|
|
170
|
+
if not paths:
|
|
171
|
+
return []
|
|
172
|
+
ineligible = set()
|
|
173
|
+
for rev in (base, head_oid):
|
|
174
|
+
modes = _tree_modes(repo, rev, paths)
|
|
175
|
+
ineligible.update(name for name, mode in modes.items() if mode not in INSTRUCTION_MODES)
|
|
176
|
+
regular = sorted(name for name, mode in modes.items() if mode in INSTRUCTION_MODES)
|
|
177
|
+
ineligible.update(name for name, prefix in _blob_prefixes(repo, rev, regular).items()
|
|
178
|
+
if prefix == b'#!')
|
|
179
|
+
if working:
|
|
180
|
+
for path in paths:
|
|
181
|
+
target = Path(repo) / path
|
|
182
|
+
if target.is_symlink():
|
|
183
|
+
ineligible.add(path)
|
|
184
|
+
elif target.exists():
|
|
185
|
+
if not target.is_file() or target.stat().st_mode & 0o111:
|
|
186
|
+
ineligible.add(path)
|
|
187
|
+
else:
|
|
188
|
+
with target.open('rb') as handle:
|
|
189
|
+
if handle.read(2) == b'#!':
|
|
190
|
+
ineligible.add(path)
|
|
191
|
+
return sorted(ineligible & set(paths))
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
def _memory_only(paths, irregular):
|
|
195
|
+
"""A memory-only candidate: every path in the publication set, each a regular file."""
|
|
196
|
+
return (irregular is not None and bool(paths) and not irregular
|
|
197
|
+
and all(memory_publication_path(p) and _sane(p) for p in paths))
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
def _sane(path):
|
|
201
|
+
p = PurePosixPath(path)
|
|
202
|
+
return not (p.is_absolute() or '..' in p.parts or any(c.isspace() for c in path))
|
|
203
|
+
|
|
204
|
+
|
|
61
205
|
def policy_digest(repo):
|
|
62
206
|
return 'sha256:' + hashlib.sha256(
|
|
63
207
|
(Path(repo) / 'bin/_lib_merge_gate.py').read_bytes()).hexdigest()
|
|
64
208
|
|
|
65
209
|
|
|
66
210
|
def _document(path):
|
|
67
|
-
return path in ('README.md', 'CHANGELOG.md') or (
|
|
211
|
+
return path in ('README.md', 'CHANGELOG.md') or path in EXECUTION_PLAN_PATHS or (
|
|
68
212
|
path.startswith('docs/commands/') and path.endswith('.md'))
|
|
69
213
|
|
|
70
214
|
|
|
@@ -72,7 +216,32 @@ def _frontend_test(path):
|
|
|
72
216
|
return path.startswith('dashboard/web/src/') and path.endswith(('.test.ts', '.test.tsx'))
|
|
73
217
|
|
|
74
218
|
|
|
75
|
-
def
|
|
219
|
+
def _instruction_class(path):
|
|
220
|
+
"""'instruction' for an instruction path, a full-selection reason for an
|
|
221
|
+
instruction-file name at an unreviewed location, or None for any other path."""
|
|
222
|
+
if path.endswith('.md') and path.startswith(INSTRUCTION_TREES):
|
|
223
|
+
return 'instruction'
|
|
224
|
+
if PurePosixPath(path).name in INSTRUCTION_NAMES:
|
|
225
|
+
if path in INSTRUCTION_NAMES or path in NESTED_INSTRUCTION_FILES:
|
|
226
|
+
return 'instruction'
|
|
227
|
+
return 'unreviewed nested instruction file: ' + path
|
|
228
|
+
if path in INSTRUCTION_INPUTS:
|
|
229
|
+
return 'instruction'
|
|
230
|
+
return None
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
def select(repo, paths, *, irregular=None, instruction_evidence=None):
|
|
234
|
+
"""The reviewed selection of `paths`.
|
|
235
|
+
|
|
236
|
+
`irregular` is the list of memory paths that are not regular files
|
|
237
|
+
(`irregular_memory_paths`); only a caller that computed it can have a
|
|
238
|
+
candidate selected as memory-only. Any other candidate is selected over its
|
|
239
|
+
whole diff, where a memory path is unclassified.
|
|
240
|
+
|
|
241
|
+
`instruction_evidence` is the list of instruction paths that are not
|
|
242
|
+
non-executable regular files (`ineligible_instruction_paths`); only a caller
|
|
243
|
+
that computed it can have an instruction path selected focused.
|
|
244
|
+
"""
|
|
76
245
|
repo = Path(repo)
|
|
77
246
|
manifest = json.loads((repo / 'tests/authoritative-test-manifest.json').read_text())
|
|
78
247
|
private = (repo / '.mirror-allowlist').exists()
|
|
@@ -82,10 +251,28 @@ def select(repo, paths):
|
|
|
82
251
|
raise ValueError('invalid harness manifest')
|
|
83
252
|
paths = sorted(set(paths))
|
|
84
253
|
reasons, components = [], set()
|
|
85
|
-
|
|
254
|
+
memory = _memory_only(paths, irregular)
|
|
255
|
+
if memory:
|
|
256
|
+
components.add('memory')
|
|
257
|
+
for path in ([] if memory else paths):
|
|
86
258
|
p = PurePosixPath(path)
|
|
87
|
-
if
|
|
259
|
+
if path in (irregular or ()):
|
|
260
|
+
reasons.append('memory path that is not a regular file: ' + path)
|
|
261
|
+
elif p.is_absolute() or '..' in p.parts or any(c.isspace() for c in path):
|
|
88
262
|
reasons.append('unclassified path: ' + path)
|
|
263
|
+
elif '__pycache__' in p.parts:
|
|
264
|
+
# Terminal: the skill comparison never reads cache content, so no
|
|
265
|
+
# class may admit it (the memory-only shortcut is decided above).
|
|
266
|
+
reasons.append('runtime cache path: ' + path)
|
|
267
|
+
elif (instruction := _instruction_class(path)) is not None:
|
|
268
|
+
if instruction != 'instruction':
|
|
269
|
+
reasons.append(instruction)
|
|
270
|
+
elif instruction_evidence is None:
|
|
271
|
+
reasons.append('instruction path without file evidence: ' + path)
|
|
272
|
+
elif path in instruction_evidence:
|
|
273
|
+
reasons.append('instruction path that is not a non-executable regular file: ' + path)
|
|
274
|
+
else:
|
|
275
|
+
components.add('instructions')
|
|
89
276
|
elif _document(path):
|
|
90
277
|
components.add('documentation')
|
|
91
278
|
elif path in PRESENTATION_FILES or _frontend_test(path):
|
|
@@ -96,16 +283,33 @@ def select(repo, paths):
|
|
|
96
283
|
components.add('presentation')
|
|
97
284
|
else:
|
|
98
285
|
reasons.append('shared, high-risk, or unclassified path: ' + path)
|
|
286
|
+
if 'instructions' in components and 'presentation' in components:
|
|
287
|
+
reasons.append('instruction text and presentation are not certified together')
|
|
99
288
|
harnesses = {'doc-lint'}
|
|
100
289
|
files = set(SMOKE_FILES)
|
|
101
290
|
if 'presentation' in components:
|
|
102
291
|
harnesses.add('frontend')
|
|
103
292
|
files.update(PRESENTATION_PYTEST)
|
|
293
|
+
if 'memory' in components:
|
|
294
|
+
files.update(MEMORY_PYTEST)
|
|
295
|
+
if 'instructions' in components:
|
|
296
|
+
files.update(INSTRUCTIONS_PYTEST)
|
|
104
297
|
if not harnesses <= set(names):
|
|
105
298
|
reasons.append('required merge harness absent from manifest')
|
|
106
299
|
if any(not (repo / p).is_file() for p in files):
|
|
107
300
|
reasons.append('required merge pytest file absent')
|
|
108
301
|
full = bool(reasons)
|
|
302
|
+
# Whether the wrapper must read background full-verification health before
|
|
303
|
+
# trusting this focus. Only a memory-only selection derived with its
|
|
304
|
+
# irregular-file evidence is exempt: memory publication is certified by the
|
|
305
|
+
# memory focus whatever the background state. Containing memory is not
|
|
306
|
+
# enough, and a full selection has nothing to read health for.
|
|
307
|
+
if full:
|
|
308
|
+
background_health = 'not-applicable'
|
|
309
|
+
elif sorted(components) == ['memory'] and memory:
|
|
310
|
+
background_health = 'exempt'
|
|
311
|
+
else:
|
|
312
|
+
background_health = 'required'
|
|
109
313
|
return {
|
|
110
314
|
'mode': 'full' if full else 'merge-focused',
|
|
111
315
|
'pytest': 'full' if full else 'selected',
|
|
@@ -113,13 +317,21 @@ def select(repo, paths):
|
|
|
113
317
|
'omittedHarnesses': [n for n in names if not full and n not in harnesses],
|
|
114
318
|
'selectedPytestFiles': [] if full else sorted(files),
|
|
115
319
|
'components': sorted(components), 'reasons': reasons,
|
|
116
|
-
'policyDigest': policy_digest(repo), '
|
|
320
|
+
'policyDigest': policy_digest(repo), 'backgroundHealth': background_health,
|
|
321
|
+
'maxWallSeconds': MAX_SECONDS,
|
|
117
322
|
}
|
|
118
323
|
|
|
119
324
|
|
|
120
325
|
def plan(repo, base_rev, head='HEAD', *, working=True):
|
|
121
326
|
b = basis(repo, base_rev, head, working=working)
|
|
122
|
-
|
|
327
|
+
irregular = None
|
|
328
|
+
if b['paths'] and all(memory_publication_path(p) for p in b['paths']):
|
|
329
|
+
irregular = irregular_memory_paths(repo, b['baseOid'], b['headOid'], b['paths'], working=working)
|
|
330
|
+
evidence = None
|
|
331
|
+
if any(_sane(p) and _instruction_class(p) == 'instruction' for p in b['paths']):
|
|
332
|
+
evidence = ineligible_instruction_paths(repo, b['baseOid'], b['headOid'], b['paths'],
|
|
333
|
+
working=working)
|
|
334
|
+
result = select(repo, b['paths'], irregular=irregular, instruction_evidence=evidence)
|
|
123
335
|
result['mergeBasis'] = b
|
|
124
336
|
return result
|
|
125
337
|
|
|
@@ -138,6 +350,12 @@ def filter_legs(plan_record, directory):
|
|
|
138
350
|
raise ValueError('selected files absent from recorded pytest estate: ' + ', '.join(sorted(selected - found)))
|
|
139
351
|
|
|
140
352
|
|
|
353
|
+
# Health reads one unfiltered ci.yml page and selects main on the client:
|
|
354
|
+
# GitHub's `branch=` index can lag the unfiltered listing by weeks (#892).
|
|
355
|
+
_HEALTH_LISTED_RUNS = 100 # one page at GitHub's maximum size; never a second
|
|
356
|
+
_HEALTH_MAIN_DEPTH = 20 # the retired branch-filtered read's depth: no older evidence
|
|
357
|
+
|
|
358
|
+
|
|
141
359
|
def background_health(rows):
|
|
142
360
|
"""Newest completed full verification wins; running work hides nothing."""
|
|
143
361
|
for row in rows:
|
|
@@ -151,11 +369,179 @@ def background_health(rows):
|
|
|
151
369
|
return {'state': 'unavailable', 'allowFocused': False, 'runId': None, 'headSha': None}
|
|
152
370
|
|
|
153
371
|
|
|
154
|
-
|
|
155
|
-
|
|
372
|
+
# D1(b): a scheduled test-macos job whose suite failed only on the runtime
|
|
373
|
+
# budget. ci.yml runs the marker step when the suite's outcome record says so;
|
|
374
|
+
# the reader also requires the suite step to be the job's only failed step.
|
|
375
|
+
_SUITE_STEP = 'Run cctally test suite'
|
|
376
|
+
_BUDGET_MARKER_STEP = 'Runtime budget is the only failure'
|
|
377
|
+
# D1(a): how many first-parent main commits are searched for a receipt.
|
|
378
|
+
_HEALTH_RECEIPT_COMMITS = 20
|
|
379
|
+
# D1(a) receipt discovery's overall time budget (#899 S2): once spent, the CI-derived
|
|
380
|
+
# result stands; it never waits out twenty commits of slow reads.
|
|
381
|
+
_HEALTH_RECEIPT_BUDGET_S = 60
|
|
382
|
+
_RECEIPT_RUN_ID = re.compile(r'^(\d{8}T\d{6}Z)-')
|
|
383
|
+
|
|
384
|
+
|
|
385
|
+
def _attempt_start(value):
|
|
386
|
+
"""The UTC instant of a GitHub ISO-8601 timestamp; ValueError when absent or malformed."""
|
|
387
|
+
if not isinstance(value, str) or not value:
|
|
388
|
+
raise ValueError('a run has no attempt start time')
|
|
389
|
+
moment = datetime.fromisoformat(value.replace('Z', '+00:00'))
|
|
390
|
+
if moment.tzinfo is None:
|
|
391
|
+
raise ValueError('a run attempt start time carries no timezone')
|
|
392
|
+
return moment.astimezone(timezone.utc)
|
|
393
|
+
|
|
394
|
+
|
|
395
|
+
def _receipt_start(run_id):
|
|
396
|
+
"""The UTC start a receipt's run ID embeds, or None."""
|
|
397
|
+
match = _RECEIPT_RUN_ID.match(run_id) if isinstance(run_id, str) else None
|
|
398
|
+
if match is None:
|
|
399
|
+
return None
|
|
400
|
+
try:
|
|
401
|
+
return datetime.strptime(match.group(1), '%Y%m%dT%H%M%SZ').replace(tzinfo=timezone.utc)
|
|
402
|
+
except ValueError:
|
|
403
|
+
return None
|
|
404
|
+
|
|
405
|
+
|
|
406
|
+
def _budget_only(row, job):
|
|
407
|
+
"""D1(b): a scheduled run's failed test-macos whose only failure is the runtime budget."""
|
|
408
|
+
if row.get('event') != 'schedule' or job.get('status') != 'completed' or job.get('conclusion') != 'failure':
|
|
409
|
+
return False
|
|
410
|
+
steps = [step for step in (job.get('steps') or []) if isinstance(step, dict)]
|
|
411
|
+
markers = [step for step in steps if step.get('name') == _BUDGET_MARKER_STEP]
|
|
412
|
+
failed = {step.get('name') for step in steps if step.get('conclusion') == 'failure'}
|
|
413
|
+
return (len(markers) == 1 and markers[0].get('conclusion') == 'success'
|
|
414
|
+
and failed == {_SUITE_STEP})
|
|
415
|
+
|
|
416
|
+
|
|
417
|
+
def _run_item(row, jobs_of):
|
|
418
|
+
"""The full-verification item one main run contributes: 'pass', 'fail', 'unknown'
|
|
419
|
+
(decisive, never a pass), or None when the run is skipped."""
|
|
420
|
+
if row.get('status') != 'completed':
|
|
421
|
+
# GitHub reuses the run ID on retry: the earlier failed attempt
|
|
422
|
+
# disappears from this listing while the retry runs. Do not let that
|
|
423
|
+
# mutation reveal an older successful run. A first attempt that is
|
|
424
|
+
# still running hides nothing.
|
|
425
|
+
return 'unknown' if row.get('run_attempt', 1) != 1 else None
|
|
426
|
+
jobs = jobs_of(row)
|
|
427
|
+
full = next((j for j in jobs if j.get('name') == 'test-macos'
|
|
428
|
+
and j.get('status') == 'completed'
|
|
429
|
+
and j.get('conclusion') != 'skipped'), None)
|
|
430
|
+
if full is not None:
|
|
431
|
+
if full.get('conclusion') == 'success' or _budget_only(row, full):
|
|
432
|
+
return 'pass'
|
|
433
|
+
return 'fail'
|
|
434
|
+
if row.get('event') in ('schedule', 'workflow_dispatch'):
|
|
435
|
+
# A scheduled/manual attempt must actually run the full job. Only
|
|
436
|
+
# receipt-discharged ordinary pushes may be skipped while looking for
|
|
437
|
+
# the last full verification.
|
|
438
|
+
return 'unknown'
|
|
439
|
+
# ci.yml can skip its declared full job on a successful private main push
|
|
440
|
+
# only through a successful release/receipt gate. The skipped job plus
|
|
441
|
+
# successful workflow is evidence of discharge; a merely successful
|
|
442
|
+
# classifier job alone is not (it may emit discharged=false). Missing
|
|
443
|
+
# jobs, cancellation, and failures must not silently inherit an older pass.
|
|
444
|
+
discharged = (
|
|
445
|
+
row.get('event') == 'push' and row.get('conclusion') == 'success'
|
|
446
|
+
and any(j.get('name') == 'test-macos'
|
|
447
|
+
and j.get('status') == 'completed'
|
|
448
|
+
and j.get('conclusion') == 'skipped' for j in jobs)
|
|
449
|
+
and any(j.get('name') in ('receipt-gate', 'release-stamp-gate')
|
|
450
|
+
and j.get('status') == 'completed'
|
|
451
|
+
and j.get('conclusion') == 'success' for j in jobs)
|
|
452
|
+
)
|
|
453
|
+
return None if discharged else 'unknown'
|
|
454
|
+
|
|
455
|
+
|
|
456
|
+
def _ci_item(rows, jobs_of):
|
|
457
|
+
"""(kind, start, row) of the newest decisive run, or None.
|
|
458
|
+
|
|
459
|
+
The window is sorted by attempt start, newest first, BEFORE the walk, so a
|
|
460
|
+
retry of an older-created run is read in its true place. Equal starts are
|
|
461
|
+
decided together and a non-pass among them wins.
|
|
462
|
+
"""
|
|
463
|
+
dated = [(_attempt_start(row.get('run_started_at')), index, row) for index, row in enumerate(rows)]
|
|
464
|
+
dated.sort(key=lambda item: (-item[0].timestamp(), item[1]))
|
|
465
|
+
index = 0
|
|
466
|
+
while index < len(dated):
|
|
467
|
+
start = dated[index][0]
|
|
468
|
+
group = []
|
|
469
|
+
while index < len(dated) and dated[index][0] == start:
|
|
470
|
+
group.append(dated[index][2])
|
|
471
|
+
index += 1
|
|
472
|
+
items = [(kind, row) for row in group if (kind := _run_item(row, jobs_of)) is not None]
|
|
473
|
+
if items:
|
|
474
|
+
kind, row = next((item for item in items if item[0] != 'pass'), items[0])
|
|
475
|
+
return kind, start, row
|
|
476
|
+
return None
|
|
477
|
+
|
|
478
|
+
|
|
479
|
+
def _receipt_pass(gh, name, repo, newer_than, digest_of=None):
|
|
480
|
+
"""D1(a): (runId, commit sha) of the first valid passing full-suite receipt for
|
|
481
|
+
the exact tree of a recent first-parent main commit, newer than `newer_than`
|
|
482
|
+
(any age when None); None when there is none. Raises on any read error.
|
|
483
|
+
|
|
484
|
+
`digest_of(sha)`, when given, reconstructs a commit's tree-manifest digest in
|
|
485
|
+
its caller's stead (bounded by the caller's budget); by default it is built
|
|
486
|
+
in-process."""
|
|
487
|
+
# The validator of this reader's own tree, the one receipt-gate runs.
|
|
488
|
+
spec = importlib.util.spec_from_file_location(
|
|
489
|
+
'_merge_gate_receipts',
|
|
490
|
+
Path(__file__).resolve().parents[1] / '.github' / 'scripts' / 'classify_receipt.py')
|
|
491
|
+
receipts = importlib.util.module_from_spec(spec)
|
|
492
|
+
spec.loader.exec_module(receipts)
|
|
493
|
+
toolchain = receipts.toolchain_digest(str(repo))
|
|
494
|
+
if toolchain is None:
|
|
495
|
+
return None
|
|
496
|
+
listing = gh('api', f'repos/{name}/commits?sha=main&per_page=100')
|
|
497
|
+
by_sha = {commit['sha']: commit for commit in listing}
|
|
498
|
+
commit = listing[0] if listing else None
|
|
499
|
+
for _ in range(_HEALTH_RECEIPT_COMMITS):
|
|
500
|
+
if commit is None:
|
|
501
|
+
break
|
|
502
|
+
sha, tree = commit['sha'], commit['commit']['tree']['sha']
|
|
503
|
+
digest = None
|
|
504
|
+
refs = gh('api', f'repos/{name}/git/matching-refs/cctally/receipts/{tree}/')
|
|
505
|
+
for ref in sorted(refs, key=lambda r: r['ref'], reverse=True):
|
|
506
|
+
run_id = ref['ref'].rsplit('/', 1)[-1]
|
|
507
|
+
when = _receipt_start(run_id)
|
|
508
|
+
if when is None or (newer_than is not None and when <= newer_than):
|
|
509
|
+
continue
|
|
510
|
+
blob = gh('api', f'repos/{name}/git/blobs/{ref["object"]["sha"]}')
|
|
511
|
+
receipt = json.loads(base64.b64decode(blob['content']))
|
|
512
|
+
if not isinstance(receipt, dict) or receipt.get('runId') != run_id:
|
|
513
|
+
continue # dated by the run it records, and filed under that run
|
|
514
|
+
if digest is None:
|
|
515
|
+
digest = (digest_of(sha) if digest_of is not None
|
|
516
|
+
else receipts.manifest_digest(receipts.reconstruct_manifest(sha, str(repo))))
|
|
517
|
+
if receipts.verify_candidate(receipt, tree_oid=tree, ref_namespace=tree,
|
|
518
|
+
reconstructed_digest=digest,
|
|
519
|
+
toolchain_digest=toolchain) is None:
|
|
520
|
+
return run_id, sha
|
|
521
|
+
parents = commit.get('parents') or []
|
|
522
|
+
commit = by_sha.get(parents[0]['sha']) if parents else None
|
|
523
|
+
return None
|
|
524
|
+
|
|
156
525
|
|
|
157
|
-
|
|
158
|
-
|
|
526
|
+
def read_background_health(repo):
|
|
527
|
+
"""The state of the newest completed full-verification item.
|
|
528
|
+
|
|
529
|
+
Items are actual full test-macos job results, never a receipt-discharge
|
|
530
|
+
skip (a successful workflow with a skipped full job is not full
|
|
531
|
+
verification), with two additions (operator decision D1 of #899):
|
|
532
|
+
|
|
533
|
+
(b) a scheduled run whose suite failed only on the runtime budget, proven by
|
|
534
|
+
its marker step and its final step conclusions, is a pass;
|
|
535
|
+
(a) a valid passing full-suite receipt for the exact tree of a recent
|
|
536
|
+
first-parent main commit, newer than the newest CI item, is a pass.
|
|
537
|
+
|
|
538
|
+
Runs are ordered by attempt start. A missing start, an unreadable listing
|
|
539
|
+
and a malformed row are missing evidence; a failed receipt read leaves the
|
|
540
|
+
CI-derived result unchanged and never manufactures a pass, and so does a
|
|
541
|
+
receipt discovery that outlives its overall budget (_HEALTH_RECEIPT_BUDGET_S).
|
|
542
|
+
That budget bounds the whole discovery, the reconstruction of a receipt's
|
|
543
|
+
tree manifest included: it runs as a subprocess stopped when the budget is
|
|
544
|
+
spent, after which nothing further is discovered (#899 S2).
|
|
159
545
|
"""
|
|
160
546
|
def gh(*args):
|
|
161
547
|
process = subprocess.run(['gh', *args], cwd=repo, capture_output=True,
|
|
@@ -163,49 +549,81 @@ def read_background_health(repo):
|
|
|
163
549
|
if process.returncode:
|
|
164
550
|
raise ValueError('GitHub full-verification evidence is unavailable')
|
|
165
551
|
return json.loads(process.stdout)
|
|
552
|
+
|
|
553
|
+
def jobs_of(row):
|
|
554
|
+
return gh('api', f'repos/{name}/actions/runs/{row["id"]}/jobs?per_page=100')['jobs']
|
|
166
555
|
try:
|
|
167
556
|
name = gh('repo', 'view', '--json', 'nameWithOwner')['nameWithOwner']
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
jobs = gh('api', f'repos/{name}/actions/runs/{row["id"]}/jobs?per_page=100')['jobs']
|
|
178
|
-
full = next((j for j in jobs if j.get('name') == 'test-macos'
|
|
179
|
-
and j.get('status') == 'completed'
|
|
180
|
-
and j.get('conclusion') != 'skipped'), None)
|
|
181
|
-
if full is not None:
|
|
182
|
-
record = dict(row, fullSuite=True, conclusion=full.get('conclusion'))
|
|
183
|
-
return background_health([record])
|
|
184
|
-
if row.get('event') in ('schedule', 'workflow_dispatch'):
|
|
185
|
-
# A scheduled/manual attempt must actually run the full job.
|
|
186
|
-
# Only receipt-discharged ordinary pushes may be skipped while
|
|
187
|
-
# looking for the last full verification.
|
|
188
|
-
return background_health([])
|
|
189
|
-
# ci.yml can skip its declared full job on a successful private
|
|
190
|
-
# main push only through a successful release/receipt gate. The
|
|
191
|
-
# skipped job plus successful workflow is evidence of discharge;
|
|
192
|
-
# a merely successful classifier job alone is not (it may emit
|
|
193
|
-
# discharged=false). Missing jobs, cancellation, and failures
|
|
194
|
-
# must not silently inherit an older pass.
|
|
195
|
-
discharged = (
|
|
196
|
-
row.get('event') == 'push' and row.get('conclusion') == 'success'
|
|
197
|
-
and any(j.get('name') == 'test-macos'
|
|
198
|
-
and j.get('status') == 'completed'
|
|
199
|
-
and j.get('conclusion') == 'skipped' for j in jobs)
|
|
200
|
-
and any(j.get('name') in ('receipt-gate', 'release-stamp-gate')
|
|
201
|
-
and j.get('status') == 'completed'
|
|
202
|
-
and j.get('conclusion') == 'success' for j in jobs)
|
|
203
|
-
)
|
|
204
|
-
if not discharged:
|
|
205
|
-
return background_health([])
|
|
206
|
-
return background_health([])
|
|
207
|
-
except (OSError, ValueError, KeyError, TypeError, subprocess.TimeoutExpired):
|
|
557
|
+
listed = gh('api', f'repos/{name}/actions/workflows/ci.yml/runs'
|
|
558
|
+
f'?per_page={_HEALTH_LISTED_RUNS}')['workflow_runs']
|
|
559
|
+
# Other branches are never main's evidence, so they are dropped before
|
|
560
|
+
# any rule below can decide on them or count toward the depth.
|
|
561
|
+
rows = [row for row in listed if row.get('head_branch') == 'main']
|
|
562
|
+
item = _ci_item(rows[:_HEALTH_MAIN_DEPTH], jobs_of)
|
|
563
|
+
except (OSError, ValueError, KeyError, TypeError, AttributeError,
|
|
564
|
+
subprocess.TimeoutExpired):
|
|
565
|
+
# A malformed listing (a null or non-object row) is missing evidence.
|
|
208
566
|
return background_health([])
|
|
567
|
+
if item is not None and item[0] == 'pass':
|
|
568
|
+
return background_health([dict(item[2], fullSuite=True, conclusion='success')])
|
|
569
|
+
if item is not None and item[0] == 'fail':
|
|
570
|
+
result = background_health([dict(item[2], fullSuite=True, conclusion='failure')])
|
|
571
|
+
else:
|
|
572
|
+
result = background_health([])
|
|
573
|
+
budget_end = time.monotonic() + _HEALTH_RECEIPT_BUDGET_S
|
|
574
|
+
|
|
575
|
+
def budgeted_gh(*args):
|
|
576
|
+
# Every discovery read gets only what is left of the overall budget, and a read
|
|
577
|
+
# that ends past it voids the discovery: the CI-derived result stands.
|
|
578
|
+
left = budget_end - time.monotonic()
|
|
579
|
+
if left <= 0:
|
|
580
|
+
raise ValueError('the receipt discovery budget is spent')
|
|
581
|
+
process = subprocess.run(['gh', *args], cwd=repo, capture_output=True, text=True,
|
|
582
|
+
timeout=min(20.0, left))
|
|
583
|
+
if process.returncode:
|
|
584
|
+
raise ValueError('GitHub full-verification evidence is unavailable')
|
|
585
|
+
if time.monotonic() > budget_end:
|
|
586
|
+
raise ValueError('the receipt discovery budget is spent')
|
|
587
|
+
return json.loads(process.stdout)
|
|
588
|
+
|
|
589
|
+
def budgeted_digest(sha):
|
|
590
|
+
# The tree-manifest reconstruction runs as the classifier's own `--self-test-manifest`
|
|
591
|
+
# in a separate session, stopped (with every process it started) when the budget is spent.
|
|
592
|
+
left = budget_end - time.monotonic()
|
|
593
|
+
if left <= 0:
|
|
594
|
+
raise ValueError('the receipt discovery budget is spent')
|
|
595
|
+
script = Path(__file__).resolve().parents[1] / '.github' / 'scripts' / 'classify_receipt.py'
|
|
596
|
+
process = subprocess.Popen([sys.executable, str(script), '--sha', sha, '--repo', str(repo),
|
|
597
|
+
'--self-test-manifest'], stdout=subprocess.PIPE,
|
|
598
|
+
stderr=subprocess.DEVNULL, text=True, start_new_session=True)
|
|
599
|
+
try:
|
|
600
|
+
out, _ = process.communicate(timeout=left)
|
|
601
|
+
except subprocess.TimeoutExpired:
|
|
602
|
+
try:
|
|
603
|
+
os.killpg(process.pid, signal.SIGKILL)
|
|
604
|
+
except (ProcessLookupError, PermissionError):
|
|
605
|
+
pass
|
|
606
|
+
try:
|
|
607
|
+
process.communicate(timeout=5)
|
|
608
|
+
except subprocess.TimeoutExpired:
|
|
609
|
+
pass
|
|
610
|
+
raise ValueError('the receipt discovery budget is spent') from None
|
|
611
|
+
if process.returncode:
|
|
612
|
+
raise ValueError('the tree manifest could not be reconstructed')
|
|
613
|
+
if time.monotonic() > budget_end:
|
|
614
|
+
raise ValueError('the receipt discovery budget is spent')
|
|
615
|
+
return out.strip()
|
|
616
|
+
try:
|
|
617
|
+
found = _receipt_pass(budgeted_gh, name, repo, item[1] if item is not None else None,
|
|
618
|
+
digest_of=budgeted_digest)
|
|
619
|
+
if found is not None and time.monotonic() > budget_end:
|
|
620
|
+
found = None # validated after the budget: the CI-derived result stands
|
|
621
|
+
except Exception: # any read error, or a spent budget, leaves the CI-derived result unchanged
|
|
622
|
+
found = None
|
|
623
|
+
if found is None:
|
|
624
|
+
return result
|
|
625
|
+
run_id, sha = found
|
|
626
|
+
return {'state': 'passed', 'allowFocused': True, 'runId': run_id, 'headSha': sha}
|
|
209
627
|
|
|
210
628
|
|
|
211
629
|
def supervise(command, seconds):
|
|
@@ -284,7 +702,7 @@ def main(argv=None):
|
|
|
284
702
|
try:
|
|
285
703
|
result = plan(args.repo, args.base)
|
|
286
704
|
if args.required_mode:
|
|
287
|
-
print(result['mode'])
|
|
705
|
+
print(f"{result['mode']} {result['backgroundHealth']}")
|
|
288
706
|
elif args.filter_legs:
|
|
289
707
|
filter_legs(result, args.filter_legs)
|
|
290
708
|
elif args.shell:
|