cctally 1.117.0 → 1.118.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. package/CHANGELOG.md +46 -0
  2. package/README.md +7 -3
  3. package/bin/_cctally_cache.py +2783 -175
  4. package/bin/_cctally_completion.py +106 -33
  5. package/bin/_cctally_core.py +69 -4
  6. package/bin/_cctally_dashboard.py +227 -23
  7. package/bin/_cctally_dashboard_conversation.py +5 -1
  8. package/bin/_cctally_dashboard_envelope.py +60 -19
  9. package/bin/_cctally_dashboard_sources.py +23 -1
  10. package/bin/_cctally_db.py +193 -153
  11. package/bin/_cctally_diagnosis_sources.py +63 -7
  12. package/bin/_cctally_doctor.py +865 -93
  13. package/bin/_cctally_five_hour.py +124 -54
  14. package/bin/_cctally_forecast.py +301 -7
  15. package/bin/_cctally_journal.py +186 -18
  16. package/bin/_cctally_milestone_history.py +1193 -108
  17. package/bin/_cctally_oauth_samples.py +286 -15
  18. package/bin/_cctally_parser.py +2 -2
  19. package/bin/_cctally_quota.py +104 -12
  20. package/bin/_cctally_record.py +153 -29
  21. package/bin/_cctally_rederive.py +608 -68
  22. package/bin/_cctally_statusline.py +108 -81
  23. package/bin/_cctally_store.py +179 -38
  24. package/bin/_cctally_tui.py +1210 -170
  25. package/bin/_cctally_update.py +32 -13
  26. package/bin/_lib_codex_conversation_query.py +52 -14
  27. package/bin/_lib_codex_drain_admission.py +224 -6
  28. package/bin/_lib_codex_metadata.py +39 -0
  29. package/bin/_lib_conversation_anon.py +15 -8
  30. package/bin/_lib_conversation_dispatch.py +7 -0
  31. package/bin/_lib_conversation_query.py +289 -70
  32. package/bin/_lib_dashboard_sources.py +131 -9
  33. package/bin/_lib_detector_access_registry.py +497 -26
  34. package/bin/_lib_doctor.py +85 -20
  35. package/bin/_lib_doctor_scan_index.py +321 -0
  36. package/bin/_lib_ingest_frontier.py +20 -1
  37. package/bin/_lib_oauth_samples.py +23 -7
  38. package/bin/_lib_pricing.py +146 -16
  39. package/bin/_lib_quota.py +19 -0
  40. package/bin/_lib_quota_model.py +29 -11
  41. package/bin/_lib_rederive.py +11 -6
  42. package/bin/_lib_retained_records.py +516 -0
  43. package/bin/_lib_snapshot_cache.py +18 -0
  44. package/bin/_lib_stale_proof.py +272 -43
  45. package/bin/_lib_stats_content_revision.py +831 -0
  46. package/bin/_lib_tick_stats.py +10 -5
  47. package/bin/_lib_write_io.py +12 -5
  48. package/bin/cctally +8 -1
  49. package/dashboard/static/assets/ConversationsView-D4Iv4ku0.js +72 -0
  50. package/dashboard/static/assets/{DoctorModal-BqmXbnhn.js → DoctorModal-D4QSq6uz.js} +1 -1
  51. package/dashboard/static/assets/ModalRoot-C9F5dhZi.js +1 -0
  52. package/dashboard/static/assets/{ProjectsDrillPanel-B-HZkDnT.js → ProjectsDrillPanel-WxAdRDMt.js} +1 -1
  53. package/dashboard/static/assets/{SourceDetailModal-QhJkHB7i.js → SourceDetailModal-NyyEfXmc.js} +1 -1
  54. package/dashboard/static/assets/{UpdateModal-CkXEIQcK.js → UpdateModal-CFLq9pT1.js} +1 -1
  55. package/dashboard/static/assets/{index-HKS1s56o.js → index-BKTWH9JE.js} +2 -2
  56. package/dashboard/static/assets/{outlineNavigation-BNGaNPCp.js → outlineNavigation-BnGUwCS3.js} +3 -3
  57. package/dashboard/static/dashboard.html +1 -1
  58. package/package.json +4 -1
  59. package/dashboard/static/assets/ConversationsView-Dcfn8T6f.js +0 -72
  60. package/dashboard/static/assets/ModalRoot-Ck3WJK2V.js +0 -1
@@ -55,6 +55,7 @@ import _lib_rebuild_progress
55
55
  import _lib_record
56
56
  import _lib_segment_summary
57
57
  import _lib_selector_state
58
+ import _lib_stats_content_revision
58
59
 
59
60
 
60
61
  # Torn-tail scan window (spec §4.3). A single line must fit inside it — a
@@ -1974,6 +1975,67 @@ def _read_framed_line(fh, segment, offset, length) -> bytes:
1974
1975
  return data[lead:lead + length]
1975
1976
 
1976
1977
 
1978
+ class _RetainedLineReader:
1979
+ """`_lib_retained_records.LineReader` for the final rebuild's and
1980
+ derivation completion's streamed records (#1004 D4): the framed read
1981
+ `QuotaLineSource` uses, through the one physical segment read boundary,
1982
+ with one open handle (the current segment's), so an in-order pass opens
1983
+ each segment once. A line that is gone or no longer framed raises
1984
+ `JournalError`, the error those readers already raise for a journal
1985
+ that changed under them."""
1986
+
1987
+ __slots__ = ("_segment", "_fh", "__weakref__")
1988
+
1989
+ def __init__(self) -> None:
1990
+ self._segment = None
1991
+ self._fh = None
1992
+
1993
+ def read(self, segment: str, start: int, length: int) -> bytes:
1994
+ if segment != self._segment:
1995
+ self.close()
1996
+ try:
1997
+ self._fh = _open_segment_for_read(
1998
+ _cctally_core.JOURNAL_DIR / segment)
1999
+ except OSError as exc:
2000
+ raise _retained_line_changed(
2001
+ f"{segment}: {type(exc).__name__}: {exc}") from exc
2002
+ self._segment = segment
2003
+ try:
2004
+ return _read_framed_line(self._fh, segment, start, length)
2005
+ except (QuotaLineSourceError, OSError) as exc:
2006
+ raise _retained_line_changed(str(exc)) from exc
2007
+
2008
+ def close(self) -> None:
2009
+ fh, self._fh = self._fh, None
2010
+ self._segment = None
2011
+ if fh is not None:
2012
+ fh.close()
2013
+
2014
+
2015
+ def _retained_line_changed(message: str) -> Exception:
2016
+ """The integrity error of a streamed record whose line changed after
2017
+ the read pass (framing or digest): a `JournalError`."""
2018
+ return JournalError(f"journal changed since it was read: {message}")
2019
+
2020
+
2021
+ def _decode_retained_line(line: bytes):
2022
+ """#1004 D4: a streamed slot's record exactly as the rebuild and
2023
+ completion retained it before: the strict decode, then #997 D15's
2024
+ compaction."""
2025
+ record = _lib_journal.decode_line(line)
2026
+ if record is None:
2027
+ return None
2028
+ return _lib_journal.compact_retained(record)
2029
+
2030
+
2031
+ def _new_retained_records():
2032
+ """An empty coordinate-backed `RetainedRecords` over this journal."""
2033
+ import _lib_retained_records
2034
+ return _lib_retained_records.RetainedRecords(
2035
+ _lib_retained_records.LineCoordinates(), _RetainedLineReader,
2036
+ _decode_retained_line, error=_retained_line_changed)
2037
+
2038
+
1977
2039
  def _quota_line_sizes(quota_raw):
1978
2040
  """Encoded size per observation (line plus newline), for chunk planning."""
1979
2041
  if isinstance(quota_raw, QuotaLineSource):
@@ -4303,7 +4365,7 @@ def _resolve_obs_anchor(resolver, rec: dict) -> "str | None":
4303
4365
 
4304
4366
 
4305
4367
  def _apply_quota_records(
4306
- cache, records, *, reported_conflicts=None, quiet=False,
4368
+ cache, records, *, reported_conflicts=None, quiet=False, anchors=None,
4307
4369
  ) -> None:
4308
4370
  """Materialize Codex quota obs into an OPEN cache.db transaction, applying
4309
4371
  the §3.5 precedence rule. Callers must apply the batch's file-account
@@ -4322,7 +4384,13 @@ def _apply_quota_records(
4322
4384
  reachable from the open-time reconciliation, so `cache-sync` and the
4323
4385
  dashboard would emit a line they never emitted before, and acceptance
4324
4386
  criterion 10 requires no new stderr on an ordinary command. Its sibling was
4325
- guarded and this one was not."""
4387
+ guarded and this one was not.
4388
+
4389
+ ``anchors`` (#975, spec 4.7) replaces the per-call reset-anchor resolver
4390
+ when given. The history provenance drain passes one built from its
4391
+ committed-evidence seed, so a pass seeds each anchor group once instead of
4392
+ once per identity; every other caller omits it and gets a new resolver per
4393
+ call, as before."""
4326
4394
  oracle = _CodexAttributionOracle(cache)
4327
4395
  # One line per FILE PER BATCH, not per record. A mid-file account switch
4328
4396
  # legitimately produces a run of observations whose first-stamp-wins account
@@ -4342,7 +4410,10 @@ def _apply_quota_records(
4342
4410
  # NULL anchor, a later walk's `INSERT OR IGNORE` cannot correct it, and that
4343
4411
  # window stays fragmented forever.
4344
4412
  has_anchor = _cache_has_anchor_column(cache)
4345
- anchors = _codex_anchor_resolver(cache) if has_anchor else None
4413
+ if not has_anchor:
4414
+ anchors = None
4415
+ elif anchors is None:
4416
+ anchors = _codex_anchor_resolver(cache)
4346
4417
  insert_sql = _QUOTA_SNAPSHOT_INSERT if has_anchor else _QUOTA_SNAPSHOT_INSERT_LEGACY
4347
4418
  upsert_sql = _QUOTA_SNAPSHOT_UPSERT if has_anchor else _QUOTA_SNAPSHOT_UPSERT_LEGACY
4348
4419
  # #980 D3: a span per root covering every quota record this call wrote
@@ -6325,6 +6396,10 @@ def _replace_block_children(
6325
6396
  convergence replace both sets rather than relying on natural-key
6326
6397
  ``INSERT OR IGNORE``. This removes stale children and restores missing
6327
6398
  children while preserving the parent rowid used by milestone FKs.
6399
+
6400
+ A stored set that already equals the replacement is left in place
6401
+ (#1050): a DELETE and re-INSERT of identical rows would advance the
6402
+ content revision on every repeated fold or convergence.
6328
6403
  """
6329
6404
  for payload_key, child_table in _BLOCK_CHILDREN:
6330
6405
  if parent_account is None:
@@ -6333,14 +6408,21 @@ def _replace_block_children(
6333
6408
  else:
6334
6409
  predicate = "account_key = ? AND five_hour_window_key = ?"
6335
6410
  params = (parent_account, int(parent_window))
6336
- conn.execute(
6337
- f"DELETE FROM {child_table} WHERE {predicate}", params
6338
- )
6411
+ rows = []
6339
6412
  for child in children.get(payload_key, []):
6340
6413
  cols = dict(child)
6341
6414
  cols["block_id"] = int(block_id)
6342
6415
  if parent_account is not None:
6343
6416
  cols["account_key"] = parent_account
6417
+ rows.append(cols)
6418
+ if _lib_stats_content_revision.child_set_unchanged(
6419
+ conn, child_table, five_hour_window_key=int(parent_window),
6420
+ account_key=parent_account, replacement=rows):
6421
+ continue
6422
+ conn.execute(
6423
+ f"DELETE FROM {child_table} WHERE {predicate}", params
6424
+ )
6425
+ for cols in rows:
6344
6426
  _insert_or_ignore(conn, child_table, cols, strict=True)
6345
6427
 
6346
6428
 
@@ -11706,6 +11788,10 @@ _REBUILD_REQUIRED_TABLES = frozenset(
11706
11788
  "schema_migrations",
11707
11789
  "schema_migrations_skipped",
11708
11790
  "stats_open_fixups",
11791
+ # #1050 §4.3: the content-revision singleton and its scope table
11792
+ # (`_lib_stats_content_revision.KERNEL_TABLES`).
11793
+ "stats_content_revision",
11794
+ "stats_content_revision_scopes",
11709
11795
  "stats_publication_stamp",
11710
11796
  "stats_quota_projection_state",
11711
11797
  "usage_snapshot_suppressions",
@@ -11716,11 +11802,54 @@ _REBUILD_REQUIRED_TABLES = frozenset(
11716
11802
  "weekly_usage_snapshots",
11717
11803
  }
11718
11804
  )
11719
- #: #825/#832 Task 4.0. The one stats trigger. A scratch index without it would
11805
+ #: #825/#832 Task 4.0. The suppression trigger. A scratch index without it would
11720
11806
  #: publish a store whose live stale-replica removals leave dangling milestone
11721
11807
  #: references a rebuild cannot reproduce, so its absence fails validation like a
11722
- #: missing table does.
11723
- _REBUILD_REQUIRED_TRIGGERS = frozenset({"trg_usage_snapshot_suppression"})
11808
+ #: missing table does. #1050 §4.3 adds the content-revision kernel's triggers:
11809
+ #: a generation published without them would leave every in-place rewrite
11810
+ #: invisible to the dashboard until the next open reinstalled them.
11811
+ _REBUILD_REQUIRED_TRIGGERS = frozenset(
11812
+ {
11813
+ # `_lib_stats_content_revision.KERNEL_TRIGGERS`
11814
+ "trg_content_revision_budget_milestones_delete",
11815
+ "trg_content_revision_budget_milestones_update",
11816
+ "trg_content_revision_claude_weekly_observation_index_delete",
11817
+ "trg_content_revision_claude_weekly_observation_index_update",
11818
+ "trg_content_revision_five_hour_block_models_delete",
11819
+ "trg_content_revision_five_hour_block_models_insert",
11820
+ "trg_content_revision_five_hour_block_models_update",
11821
+ "trg_content_revision_five_hour_block_projects_delete",
11822
+ "trg_content_revision_five_hour_block_projects_insert",
11823
+ "trg_content_revision_five_hour_block_projects_update",
11824
+ "trg_content_revision_five_hour_blocks_delete",
11825
+ "trg_content_revision_five_hour_blocks_insert",
11826
+ "trg_content_revision_five_hour_blocks_update",
11827
+ "trg_content_revision_five_hour_milestones_delete",
11828
+ "trg_content_revision_five_hour_milestones_insert",
11829
+ "trg_content_revision_five_hour_milestones_update",
11830
+ "trg_content_revision_five_hour_reset_events_delete",
11831
+ "trg_content_revision_five_hour_reset_events_insert",
11832
+ "trg_content_revision_five_hour_reset_events_update",
11833
+ "trg_content_revision_percent_milestones_delete",
11834
+ "trg_content_revision_percent_milestones_insert",
11835
+ "trg_content_revision_percent_milestones_update",
11836
+ "trg_content_revision_project_budget_milestones_delete",
11837
+ "trg_content_revision_project_budget_milestones_update",
11838
+ "trg_content_revision_projected_milestones_delete",
11839
+ "trg_content_revision_projected_milestones_update",
11840
+ "trg_content_revision_week_reset_events_delete",
11841
+ "trg_content_revision_week_reset_events_update",
11842
+ "trg_content_revision_weekly_cost_snapshots_delete",
11843
+ "trg_content_revision_weekly_cost_snapshots_insert",
11844
+ "trg_content_revision_weekly_cost_snapshots_update",
11845
+ "trg_content_revision_weekly_credit_floors_delete",
11846
+ "trg_content_revision_weekly_credit_floors_update",
11847
+ "trg_content_revision_weekly_usage_snapshots_delete",
11848
+ "trg_content_revision_weekly_usage_snapshots_insert",
11849
+ "trg_content_revision_weekly_usage_snapshots_update",
11850
+ "trg_usage_snapshot_suppression",
11851
+ }
11852
+ )
11724
11853
  _REBUILD_REQUIRED_INDEXES = frozenset(
11725
11854
  {
11726
11855
  "idx_budget_milestones_alert_digest",
@@ -11729,6 +11858,7 @@ _REBUILD_REQUIRED_INDEXES = frozenset(
11729
11858
  "idx_budget_milestones_journal_id_null",
11730
11859
  "idx_claude_oauth_samples_five_hour",
11731
11860
  "idx_claude_oauth_samples_weekly",
11861
+ "idx_claude_weekly_observation_index_freshness",
11732
11862
  "idx_claude_weekly_observation_index_window",
11733
11863
  "idx_cost_week_start_at_time",
11734
11864
  "idx_cost_week_time",
@@ -11747,6 +11877,9 @@ _REBUILD_REQUIRED_INDEXES = frozenset(
11747
11877
  "idx_five_hour_milestones_block",
11748
11878
  "idx_five_hour_milestones_journal_id",
11749
11879
  "idx_five_hour_milestones_journal_id_null",
11880
+ # #1050 §4.2: the snapshot trigger's block lookup
11881
+ # (`_lib_stats_content_revision.USAGE_SNAPSHOT_INDEX`).
11882
+ "idx_five_hour_milestones_usage_snapshot_id",
11750
11883
  "idx_five_hour_reset_events_journal_id",
11751
11884
  "idx_five_hour_reset_events_journal_id_null",
11752
11885
  "idx_journal_protocol_violations_batch",
@@ -11807,8 +11940,12 @@ _REBUILD_REQUIRED_INDEXES = frozenset(
11807
11940
  # #948 epoch 1023: `outcome_quarantine_envelope` and
11808
11941
  # `outcome_quarantine_certificates` dropped; `outcome_quarantine_episodes`,
11809
11942
  # `outcome_quarantine_bounds` and `outcome_quarantine_axis_state` added.
11943
+ # #1018 twin epoch 1024: the four enriched columns of
11944
+ # `claude_weekly_observation_index` and its partial freshness index.
11945
+ # #1050 (same epoch, no bump): the content-revision kernel's two tables, its
11946
+ # triggers and `idx_five_hour_milestones_usage_snapshot_id`.
11810
11947
  _REBUILD_SCHEMA_FINGERPRINT = (
11811
- "b7fab856ea97a64c43b3d24669212925d21348c92948f57fba50787aa966995a"
11948
+ "1cc565ce0908c80a72884d5e76d815776f57a297fc4d928d0735ba179da4a6f8"
11812
11949
  )
11813
11950
 
11814
11951
 
@@ -15038,7 +15175,17 @@ def _rebuild_stats_index_locked(
15038
15175
  # dropped as soon as it has contributed (#496 S4 §4). #997 D7: the
15039
15176
  # quota observations are now retained as COORDINATES (16 bytes each)
15040
15177
  # and the cache leg re-reads them one chunk at a time.
15041
- decoded: list = []
15178
+ #
15179
+ # #1004 D4: `decoded` is a `RetainedRecords`. A retained record is a
15180
+ # streamed slot (its coordinates and the digest of its line), re-read,
15181
+ # digest-checked and decoded on each access, so every pass sees a
15182
+ # fresh record equal to the one retained before; every other position
15183
+ # keeps its `None` slot. With `MATERIALIZE_FOR_TESTS` set the pass
15184
+ # retains the decoded records and hands on today's list (a test
15185
+ # switch).
15186
+ import _lib_retained_records
15187
+ materialize = _lib_retained_records.MATERIALIZE_FOR_TESTS
15188
+ decoded = _new_retained_records()
15042
15189
  quota_raw = QuotaLineSource()
15043
15190
  protocol_evidence = []
15044
15191
  prior_high_water = None
@@ -15100,7 +15247,10 @@ def _rebuild_stats_index_locked(
15100
15247
  # fingerprint that `journal_protocol_resolution` ops reference by
15101
15248
  # name, so contributing the wrong count makes an acknowledged
15102
15249
  # violation unresolvable (#496 S5b §5.4).
15103
- decoded.extend([None] * int(summary.decoded_entry_count))
15250
+ for _ in range(int(summary.decoded_entry_count)):
15251
+ # An elided line is never read back; its coordinate is a
15252
+ # zero-length stand-in in its segment.
15253
+ decoded.append_retained(None, name, 0, 0)
15104
15254
  last_seen.merge(
15105
15255
  summary.last_seen_stamped,
15106
15256
  summary.last_seen_legacy_claude_at,
@@ -15162,7 +15312,7 @@ def _rebuild_stats_index_locked(
15162
15312
  {"t": rec.get("t"), "id": rec.get("id"),
15163
15313
  "rev": rec.get("rev", 0)})
15164
15314
  summaries.decoded(False)
15165
- decoded.append(None)
15315
+ decoded.append_retained(None, segment, offset, len(raw))
15166
15316
  prior_high_water = (segment, offset + len(raw) + 1)
15167
15317
  continue
15168
15318
  rebuild_quarantine.on_record(
@@ -15231,7 +15381,12 @@ def _rebuild_stats_index_locked(
15231
15381
  rec.get("t") in _lib_journal_router.RETAINED_RECORD_TYPES)
15232
15382
  summaries.decoded(retained, _is_codex_quota_obs(rec))
15233
15383
  if retained:
15234
- decoded.append(rec)
15384
+ if materialize:
15385
+ decoded.append_retained(
15386
+ rec, segment, offset, len(raw))
15387
+ else:
15388
+ decoded.append_streamed(
15389
+ raw, segment, offset, len(raw))
15235
15390
  else:
15236
15391
  if update_quota_cache and _is_codex_quota_obs(rec):
15237
15392
  quota_raw.append(segment, offset, len(raw))
@@ -15247,7 +15402,7 @@ def _rebuild_stats_index_locked(
15247
15402
  # and raise on every later rebuild. The selector skips a non-dict
15248
15403
  # element, so this costs one pointer and keeps every sequence
15249
15404
  # identical to the pre-change numbering (#496 S4; corrects §4.7).
15250
- decoded.append(None)
15405
+ decoded.append_retained(None, segment, offset, len(raw))
15251
15406
  prior_high_water = (
15252
15407
  segment,
15253
15408
  offset + len(raw) + 1,
@@ -15295,9 +15450,22 @@ def _rebuild_stats_index_locked(
15295
15450
  # is dropped before the inventory check and publication.
15296
15451
  _cctally_oauth_samples.attribute_staged_observations(
15297
15452
  conn, cutover_claude)
15298
- for rec in decoded:
15299
- if rec is not None:
15300
- _normalize_legacy_account_stamp(rec, cutover_claude)
15453
+ if materialize:
15454
+ # Every slot is a retained object here, so this decodes nothing.
15455
+ listed = list(decoded)
15456
+ decoded.close()
15457
+ decoded = listed
15458
+ for rec in decoded:
15459
+ if rec is not None:
15460
+ _normalize_legacy_account_stamp(rec, cutover_claude)
15461
+ else:
15462
+ # #1004 D4: normalization moves to decode time. The hook is
15463
+ # installed before the first consumer reads a record (the staged
15464
+ # attribution above reads only projected tables), so every pass
15465
+ # below sees each record normalized, as the in-place loop left it.
15466
+ decoded.set_normalizer(
15467
+ lambda rec: _normalize_legacy_account_stamp(
15468
+ rec, cutover_claude))
15301
15469
 
15302
15470
  # Resolve corrections BEFORE either disposable index is mutated. A
15303
15471
  # malformed revision, divergent same-revision candidate, or invalid