cctally 1.108.0 → 1.110.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (85) hide show
  1. package/CHANGELOG.md +147 -0
  2. package/README.md +6 -6
  3. package/bin/_cctally_account.py +4 -0
  4. package/bin/_cctally_alerts.py +6 -2
  5. package/bin/_cctally_cache.py +1652 -166
  6. package/bin/_cctally_config.py +3 -2
  7. package/bin/_cctally_core.py +594 -33
  8. package/bin/_cctally_dashboard.py +1585 -201
  9. package/bin/_cctally_dashboard_conversation.py +494 -48
  10. package/bin/_cctally_dashboard_envelope.py +167 -11
  11. package/bin/_cctally_dashboard_share.py +154 -36
  12. package/bin/_cctally_dashboard_sources.py +2615 -422
  13. package/bin/_cctally_db.py +678 -5
  14. package/bin/_cctally_diagnosis_sources.py +238 -26
  15. package/bin/_cctally_doctor.py +99 -18
  16. package/bin/_cctally_five_hour.py +1295 -40
  17. package/bin/_cctally_forecast.py +124 -28
  18. package/bin/_cctally_journal.py +408 -38
  19. package/bin/_cctally_milestone_history.py +150 -14
  20. package/bin/_cctally_parser.py +5 -0
  21. package/bin/_cctally_percent_breakdown.py +6 -1
  22. package/bin/_cctally_project.py +48 -76
  23. package/bin/_cctally_quota.py +39 -7
  24. package/bin/_cctally_quota_model.py +17 -2
  25. package/bin/_cctally_record.py +1647 -443
  26. package/bin/_cctally_rederive.py +899 -30
  27. package/bin/_cctally_refresh.py +2 -1
  28. package/bin/_cctally_share.py +24 -5
  29. package/bin/_cctally_source_analytics.py +1035 -78
  30. package/bin/_cctally_statusline.py +83 -19
  31. package/bin/_cctally_store.py +61 -4
  32. package/bin/_cctally_transcript.py +27 -2
  33. package/bin/_cctally_tui.py +264 -76
  34. package/bin/_cctally_weekrefs.py +28 -5
  35. package/bin/_lib_blocks.py +127 -20
  36. package/bin/_lib_cache_report.py +20 -5
  37. package/bin/_lib_codex_conversation_query.py +297 -64
  38. package/bin/_lib_codex_hooks.py +102 -0
  39. package/bin/_lib_codex_metadata.py +207 -0
  40. package/bin/_lib_conversation.py +119 -0
  41. package/bin/_lib_conversation_dispatch.py +31 -1
  42. package/bin/_lib_conversation_query.py +315 -51
  43. package/bin/_lib_conversation_retention.py +413 -25
  44. package/bin/_lib_dashboard_sources.py +267 -7
  45. package/bin/_lib_diagnosis.py +10 -0
  46. package/bin/_lib_diff_kernel.py +12 -1
  47. package/bin/_lib_doctor.py +230 -8
  48. package/bin/_lib_ingest_frontier.py +1366 -49
  49. package/bin/_lib_journal.py +19 -2
  50. package/bin/_lib_merge_gate.py +307 -0
  51. package/bin/_lib_pricing.py +588 -29
  52. package/bin/_lib_record.py +130 -12
  53. package/bin/_lib_rederive.py +903 -2
  54. package/bin/_lib_render.py +9 -3
  55. package/bin/_lib_retained_size.py +27 -2
  56. package/bin/_lib_segment_summary.py +66 -8
  57. package/bin/_lib_share_templates.py +28 -18
  58. package/bin/_lib_snapshot_cache.py +115 -1
  59. package/bin/_lib_source_retry.py +320 -0
  60. package/bin/_lib_statusline_candidates.py +400 -36
  61. package/bin/_lib_tick_stats.py +119 -0
  62. package/bin/_lib_view_models.py +256 -76
  63. package/bin/cctally +69 -0
  64. package/dashboard/static/assets/ConversationsView-BB9vUiL6.js +72 -0
  65. package/dashboard/static/assets/{DoctorModal-CWn3U4Wl.js → DoctorModal-DFY2iPDf.js} +1 -1
  66. package/dashboard/static/assets/ModalRoot-DvRKHbQE.js +1 -0
  67. package/dashboard/static/assets/ProjectsDrillPanel-DAv39uSo.js +1 -0
  68. package/dashboard/static/assets/SourceDetailModal-DSAWo2Fp.js +1 -0
  69. package/dashboard/static/assets/{UpdateModal-D3m8GG6V.js → UpdateModal-R_9dOVnh.js} +3 -3
  70. package/dashboard/static/assets/dashboardStream.shared-worker-DbG6Ef0T.js +1 -0
  71. package/dashboard/static/assets/index-COhxe3vU.css +1 -0
  72. package/dashboard/static/assets/index-DdU62sov.js +13 -0
  73. package/dashboard/static/assets/outlineNavigation-BOCLcRa6.js +9 -0
  74. package/dashboard/static/assets/useKeymap-CW7Up0Ay.js +1 -0
  75. package/dashboard/static/dashboard.html +3 -3
  76. package/package.json +4 -1
  77. package/dashboard/static/assets/ConversationsView-BOSaBRtu.js +0 -72
  78. package/dashboard/static/assets/ModalRoot-SR070V6c.js +0 -1
  79. package/dashboard/static/assets/ProjectsDrillPanel-QLI9i5mZ.js +0 -1
  80. package/dashboard/static/assets/SourceDetailModal-pv0wlFxl.js +0 -1
  81. package/dashboard/static/assets/dashboardStream.shared-worker-1XTMV3nr.js +0 -1
  82. package/dashboard/static/assets/index-BHu4mxd8.css +0 -1
  83. package/dashboard/static/assets/index-CIWsbux3.js +0 -13
  84. package/dashboard/static/assets/outlineNavigation-CJvKqmLV.js +0 -9
  85. package/dashboard/static/assets/useKeymap-ffqsS5G0.js +0 -1
@@ -333,7 +333,7 @@ def week_reset_identity_parts(account_key, old_week_end_at, new_week_end_at,
333
333
  return (account_key, old_week_end_at, new_week_end_at)
334
334
 
335
335
 
336
- def effects_payload_digest(payload: dict) -> str:
336
+ def effects_payload_digest(payload: dict, refs: dict | None = None) -> str:
337
337
  """A 16-hex digest over an effects-only evt payload, for its id.
338
338
 
339
339
  An evt id must discriminate every payload a family can emit under it,
@@ -348,14 +348,31 @@ def effects_payload_digest(payload: dict) -> str:
348
348
  own event while a genuine byte-identical replay still collapses to the
349
349
  duplicate the journal already tolerates.
350
350
 
351
+ The digest covers the EFFECTIVE payload, formed the way `emit_model_a`
352
+ forms the line it writes (`_cctally_journal.py:4476`): a copy of `columns`
353
+ updated with `refs` when `refs` is non-empty. Digesting `columns` alone
354
+ would let two events differing only in their refs collide on one id, and
355
+ the second would then classify as a conflict and be withheld — leaving its
356
+ destructive effect applied inline and undone by the next rebuild.
357
+
358
+ The hashed bytes are unchanged when `refs` is absent or empty, which is what
359
+ makes this a straight edit rather than a versioned digest: the only
360
+ production caller passes no refs (`_cctally_record.py:3575`), so no retained
361
+ identity moves. An unconditional `{columns, refs}` envelope was rejected for
362
+ exactly that reason — it would change every existing digest.
363
+
351
364
  The digest is over the canonical JSON, so it is stable under key order.
352
365
  Reproducing it from a decoded line takes one step: the input is the payload
353
366
  the emitter passes as `columns`, and `make_evt` adds the `kind` fold
354
367
  discriminator to the payload AFTERWARDS, so a reader must drop `kind`
355
368
  before re-digesting.
356
369
  """
370
+ effective = payload
371
+ if refs:
372
+ effective = dict(payload)
373
+ effective.update(refs)
357
374
  return hashlib.sha256(
358
- _canonical_json(payload).encode("utf-8")).hexdigest()[:16]
375
+ _canonical_json(effective).encode("utf-8")).hexdigest()[:16]
359
376
 
360
377
 
361
378
  # --------------------------------------------------------------------------
@@ -0,0 +1,307 @@
1
+ """Conservative impact selection for merge certification (stdlib only).
2
+
3
+ This is a reviewed component policy, not proof of every possible dependency.
4
+ Unknown paths and shared/infrastructure changes require the full estate. The
5
+ full background/release runs remain the cross-component backstop.
6
+ """
7
+ from __future__ import annotations
8
+
9
+ import argparse
10
+ import hashlib
11
+ import json
12
+ import os
13
+ from pathlib import Path, PurePosixPath
14
+ import signal
15
+ import subprocess
16
+ import sys
17
+
18
+ GATE_ID = 'cctally-test-remote/merge-focused@1'
19
+ MAX_SECONDS = 300
20
+ # All of these are leaf presentation components. Their entire frontend estate
21
+ # runs, including their consumers, instead of relying on a TS import heuristic.
22
+ PRESENTATION_FILES = frozenset(
23
+ 'dashboard/web/src/components/' + name + '.tsx'
24
+ for name in ('ZoneTag', 'ModelLegend', 'SortableHeader', 'HelpOverlay',
25
+ 'AlertsEmptyGauge', 'SessionsControls', 'BasketChip', 'Toast')
26
+ )
27
+ SMOKE_FILES = ('tests/test_cli_smoke.py', 'tests/test_lib_json_envelope.py')
28
+ PRESENTATION_PYTEST = ('tests/test_dashboard_handler_static.py',)
29
+
30
+
31
+ def _git(repo, *args):
32
+ proc = subprocess.run(['git', '--no-optional-locks', '-C', str(repo), *args],
33
+ capture_output=True)
34
+ if proc.returncode:
35
+ raise ValueError('git revision/diff failed: ' + proc.stderr.decode(errors='replace').strip())
36
+ return proc.stdout
37
+
38
+
39
+ def _changed_names(data):
40
+ """--name-only -z lists both endpoints when rename detection is disabled."""
41
+ return [p.decode('utf-8', errors='strict') for p in data.split(b'\0') if p]
42
+
43
+
44
+ def basis(repo, base_rev, head='HEAD', *, working=True):
45
+ if not base_rev or base_rev.startswith('-'):
46
+ raise ValueError('base revision must be nonempty and must not be an option')
47
+ base = _git(repo, 'merge-base', base_rev, head).decode().strip()
48
+ head_oid = _git(repo, 'rev-parse', '--verify', head + '^{commit}').decode().strip()
49
+ committed = set(_changed_names(_git(repo, 'diff', '--no-renames', '--name-only', '-z', base, head_oid)))
50
+ dirty = set()
51
+ if working:
52
+ dirty.update(_changed_names(_git(repo, 'diff', '--no-renames', '--name-only', '-z', head_oid)))
53
+ dirty.update(_changed_names(_git(repo, 'ls-files', '--others', '--exclude-standard', '-z')))
54
+ if _git(repo, 'ls-files', '-u'):
55
+ raise ValueError('the index has unmerged entries')
56
+ return {'baseOid': base, 'headOid': head_oid,
57
+ 'committedPaths': len(committed), 'worktreePaths': len(dirty),
58
+ 'paths': sorted(committed | dirty)}
59
+
60
+
61
+ def policy_digest(repo):
62
+ return 'sha256:' + hashlib.sha256(
63
+ (Path(repo) / 'bin/_lib_merge_gate.py').read_bytes()).hexdigest()
64
+
65
+
66
+ def _document(path):
67
+ return path in ('README.md', 'CHANGELOG.md') or (
68
+ path.startswith('docs/commands/') and path.endswith('.md'))
69
+
70
+
71
+ def _frontend_test(path):
72
+ return path.startswith('dashboard/web/src/') and path.endswith(('.test.ts', '.test.tsx'))
73
+
74
+
75
+ def select(repo, paths):
76
+ repo = Path(repo)
77
+ manifest = json.loads((repo / 'tests/authoritative-test-manifest.json').read_text())
78
+ private = (repo / '.mirror-allowlist').exists()
79
+ names = [r['name'] for r in manifest['harnesses']
80
+ if private or r.get('visibility', 'public') == 'public']
81
+ if not names or len(names) != len(set(names)):
82
+ raise ValueError('invalid harness manifest')
83
+ paths = sorted(set(paths))
84
+ reasons, components = [], set()
85
+ for path in paths:
86
+ p = PurePosixPath(path)
87
+ if p.is_absolute() or '..' in p.parts or any(c.isspace() for c in path):
88
+ reasons.append('unclassified path: ' + path)
89
+ elif _document(path):
90
+ components.add('documentation')
91
+ elif path in PRESENTATION_FILES or _frontend_test(path):
92
+ components.add('presentation')
93
+ elif path.startswith('dashboard/static/'):
94
+ components.add('presentation')
95
+ elif path in PRESENTATION_PYTEST:
96
+ components.add('presentation')
97
+ else:
98
+ reasons.append('shared, high-risk, or unclassified path: ' + path)
99
+ harnesses = {'doc-lint'}
100
+ files = set(SMOKE_FILES)
101
+ if 'presentation' in components:
102
+ harnesses.add('frontend')
103
+ files.update(PRESENTATION_PYTEST)
104
+ if not harnesses <= set(names):
105
+ reasons.append('required merge harness absent from manifest')
106
+ if any(not (repo / p).is_file() for p in files):
107
+ reasons.append('required merge pytest file absent')
108
+ full = bool(reasons)
109
+ return {
110
+ 'mode': 'full' if full else 'merge-focused',
111
+ 'pytest': 'full' if full else 'selected',
112
+ 'selectedHarnesses': [n for n in names if full or n in harnesses],
113
+ 'omittedHarnesses': [n for n in names if not full and n not in harnesses],
114
+ 'selectedPytestFiles': [] if full else sorted(files),
115
+ 'components': sorted(components), 'reasons': reasons,
116
+ 'policyDigest': policy_digest(repo), 'maxWallSeconds': MAX_SECONDS,
117
+ }
118
+
119
+
120
+ def plan(repo, base_rev, head='HEAD', *, working=True):
121
+ b = basis(repo, base_rev, head, working=working)
122
+ result = select(repo, b['paths'])
123
+ result['mergeBasis'] = b
124
+ return result
125
+
126
+
127
+ def filter_legs(plan_record, directory):
128
+ """Narrow both expected-node legs to selected files; retain serial ownership."""
129
+ directory = Path(directory)
130
+ selected = set(plan_record['selectedPytestFiles'])
131
+ found = set()
132
+ for name in ('pytest', 'benchmark'):
133
+ path = directory / (name + '.txt')
134
+ nodes = [n for n in path.read_text().splitlines() if n.split('::')[0] in selected]
135
+ found.update(n.split('::')[0] for n in nodes)
136
+ path.write_text(''.join(n + '\n' for n in nodes))
137
+ if found != selected:
138
+ raise ValueError('selected files absent from recorded pytest estate: ' + ', '.join(sorted(selected - found)))
139
+
140
+
141
+ def background_health(rows):
142
+ """Newest completed full verification wins; running work hides nothing."""
143
+ for row in rows:
144
+ if row.get('head_branch') != 'main' or row.get('status') != 'completed':
145
+ continue
146
+ if row.get('event') not in ('schedule', 'workflow_dispatch') and row.get('fullSuite') is not True:
147
+ continue
148
+ passed = row.get('conclusion') == 'success'
149
+ return {'state': 'passed' if passed else 'failed', 'allowFocused': passed,
150
+ 'runId': row.get('id'), 'headSha': row.get('head_sha')}
151
+ return {'state': 'unavailable', 'allowFocused': False, 'runId': None, 'headSha': None}
152
+
153
+
154
+ def read_background_health(repo):
155
+ """Read actual full test-macos job results, never a receipt-discharge skip.
156
+
157
+ Existing full CI jobs provide a bootstrap before the new schedule lands.
158
+ A successful workflow with a skipped full job is not full verification.
159
+ """
160
+ def gh(*args):
161
+ process = subprocess.run(['gh', *args], cwd=repo, capture_output=True,
162
+ text=True, timeout=20)
163
+ if process.returncode:
164
+ raise ValueError('GitHub full-verification evidence is unavailable')
165
+ return json.loads(process.stdout)
166
+ try:
167
+ name = gh('repo', 'view', '--json', 'nameWithOwner')['nameWithOwner']
168
+ rows = gh('api', f'repos/{name}/actions/workflows/ci.yml/runs?branch=main&per_page=20')['workflow_runs']
169
+ for row in rows:
170
+ if row.get('status') != 'completed':
171
+ # GitHub reuses the run ID on retry: the earlier failed
172
+ # attempt disappears from this listing while the retry runs.
173
+ # Do not let that mutation reveal an older successful run.
174
+ if row.get('run_attempt', 1) != 1:
175
+ return background_health([])
176
+ continue
177
+ jobs = gh('api', f'repos/{name}/actions/runs/{row["id"]}/jobs?per_page=100')['jobs']
178
+ full = next((j for j in jobs if j.get('name') == 'test-macos'
179
+ and j.get('status') == 'completed'
180
+ and j.get('conclusion') != 'skipped'), None)
181
+ if full is not None:
182
+ record = dict(row, fullSuite=True, conclusion=full.get('conclusion'))
183
+ return background_health([record])
184
+ if row.get('event') in ('schedule', 'workflow_dispatch'):
185
+ # A scheduled/manual attempt must actually run the full job.
186
+ # Only receipt-discharged ordinary pushes may be skipped while
187
+ # looking for the last full verification.
188
+ return background_health([])
189
+ # ci.yml can skip its declared full job on a successful private
190
+ # main push only through a successful release/receipt gate. The
191
+ # skipped job plus successful workflow is evidence of discharge;
192
+ # a merely successful classifier job alone is not (it may emit
193
+ # discharged=false). Missing jobs, cancellation, and failures
194
+ # must not silently inherit an older pass.
195
+ discharged = (
196
+ row.get('event') == 'push' and row.get('conclusion') == 'success'
197
+ and any(j.get('name') == 'test-macos'
198
+ and j.get('status') == 'completed'
199
+ and j.get('conclusion') == 'skipped' for j in jobs)
200
+ and any(j.get('name') in ('receipt-gate', 'release-stamp-gate')
201
+ and j.get('status') == 'completed'
202
+ and j.get('conclusion') == 'success' for j in jobs)
203
+ )
204
+ if not discharged:
205
+ return background_health([])
206
+ return background_health([])
207
+ except (OSError, ValueError, KeyError, TypeError, subprocess.TimeoutExpired):
208
+ return background_health([])
209
+
210
+
211
+ def supervise(command, seconds):
212
+ """Bound the complete gate, including foreground admission subprocesses.
213
+
214
+ A fresh POSIX session contains the aggregator's separate pool/pytest
215
+ process groups. Terminating that session also reaches children when bash
216
+ is waiting on an admission command and cannot yet run its signal trap.
217
+ The caller writes the ordinary contract's timeout verdict after teardown.
218
+ """
219
+ process = subprocess.Popen(command, start_new_session=True)
220
+
221
+ def members():
222
+ listing = subprocess.check_output(['ps', '-A', '-o', 'pid='], text=True)
223
+ result = []
224
+ for token in listing.split():
225
+ pid = int(token)
226
+ try:
227
+ if os.getsid(pid) == process.pid:
228
+ result.append(pid)
229
+ except (ProcessLookupError, PermissionError):
230
+ pass
231
+ return result
232
+
233
+ def terminate():
234
+ for sig in (signal.SIGTERM, signal.SIGKILL):
235
+ for pid in members():
236
+ try:
237
+ os.kill(pid, sig)
238
+ except ProcessLookupError:
239
+ pass
240
+ if sig == signal.SIGTERM:
241
+ try:
242
+ process.wait(timeout=2)
243
+ except subprocess.TimeoutExpired:
244
+ pass
245
+ process.wait()
246
+
247
+ def interrupted(signum, _frame):
248
+ terminate()
249
+ raise SystemExit(128 + signum)
250
+
251
+ old = {sig: signal.signal(sig, interrupted) for sig in (signal.SIGTERM, signal.SIGINT)}
252
+ try:
253
+ try:
254
+ return process.wait(timeout=max(0, seconds))
255
+ except subprocess.TimeoutExpired:
256
+ terminate()
257
+ return 124
258
+ finally:
259
+ for sig, handler in old.items():
260
+ signal.signal(sig, handler)
261
+
262
+
263
+ def main(argv=None):
264
+ parser = argparse.ArgumentParser(description=__doc__)
265
+ parser.add_argument('--repo', default=str(Path(__file__).resolve().parents[1]))
266
+ parser.add_argument('--base')
267
+ parser.add_argument('--health', action='store_true')
268
+ parser.add_argument('--required-mode', action='store_true')
269
+ parser.add_argument('--shell', action='store_true')
270
+ parser.add_argument('--filter-legs')
271
+ parser.add_argument('--deadline', nargs=argparse.REMAINDER,
272
+ help=argparse.SUPPRESS)
273
+ args = parser.parse_args(argv)
274
+ if args.deadline is not None:
275
+ if len(args.deadline) < 2:
276
+ parser.error('deadline requires seconds and a command')
277
+ return supervise(args.deadline[1:], int(args.deadline[0]))
278
+ if args.health:
279
+ health = read_background_health(args.repo)
280
+ print(json.dumps(health, sort_keys=True))
281
+ return 0 if health['allowFocused'] else 3
282
+ if not args.base:
283
+ parser.error('--base is required outside --health')
284
+ try:
285
+ result = plan(args.repo, args.base)
286
+ if args.required_mode:
287
+ print(result['mode'])
288
+ elif args.filter_legs:
289
+ filter_legs(result, args.filter_legs)
290
+ elif args.shell:
291
+ print('mode\t' + result['mode'])
292
+ print('harnesses\t' + ' '.join(result['selectedHarnesses']))
293
+ print('omitted\t' + ' '.join(result['omittedHarnesses']))
294
+ print('pytest\t' + result['pytest'])
295
+ print('files\t' + ' '.join(result['selectedPytestFiles']))
296
+ print('seconds\t' + str(result['maxWallSeconds']))
297
+ print('record\t' + json.dumps(result, sort_keys=True, separators=(',', ':')))
298
+ else:
299
+ print(json.dumps(result, sort_keys=True, indent=2))
300
+ except (ValueError, OSError, KeyError, UnicodeError) as exc:
301
+ print('merge-gate: ' + str(exc), file=sys.stderr)
302
+ return 2
303
+ return 0
304
+
305
+
306
+ if __name__ == '__main__':
307
+ raise SystemExit(main())