cctally 1.108.0 → 1.110.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +147 -0
- package/README.md +6 -6
- package/bin/_cctally_account.py +4 -0
- package/bin/_cctally_alerts.py +6 -2
- package/bin/_cctally_cache.py +1652 -166
- package/bin/_cctally_config.py +3 -2
- package/bin/_cctally_core.py +594 -33
- package/bin/_cctally_dashboard.py +1585 -201
- package/bin/_cctally_dashboard_conversation.py +494 -48
- package/bin/_cctally_dashboard_envelope.py +167 -11
- package/bin/_cctally_dashboard_share.py +154 -36
- package/bin/_cctally_dashboard_sources.py +2615 -422
- package/bin/_cctally_db.py +678 -5
- package/bin/_cctally_diagnosis_sources.py +238 -26
- package/bin/_cctally_doctor.py +99 -18
- package/bin/_cctally_five_hour.py +1295 -40
- package/bin/_cctally_forecast.py +124 -28
- package/bin/_cctally_journal.py +408 -38
- package/bin/_cctally_milestone_history.py +150 -14
- package/bin/_cctally_parser.py +5 -0
- package/bin/_cctally_percent_breakdown.py +6 -1
- package/bin/_cctally_project.py +48 -76
- package/bin/_cctally_quota.py +39 -7
- package/bin/_cctally_quota_model.py +17 -2
- package/bin/_cctally_record.py +1647 -443
- package/bin/_cctally_rederive.py +899 -30
- package/bin/_cctally_refresh.py +2 -1
- package/bin/_cctally_share.py +24 -5
- package/bin/_cctally_source_analytics.py +1035 -78
- package/bin/_cctally_statusline.py +83 -19
- package/bin/_cctally_store.py +61 -4
- package/bin/_cctally_transcript.py +27 -2
- package/bin/_cctally_tui.py +264 -76
- package/bin/_cctally_weekrefs.py +28 -5
- package/bin/_lib_blocks.py +127 -20
- package/bin/_lib_cache_report.py +20 -5
- package/bin/_lib_codex_conversation_query.py +297 -64
- package/bin/_lib_codex_hooks.py +102 -0
- package/bin/_lib_codex_metadata.py +207 -0
- package/bin/_lib_conversation.py +119 -0
- package/bin/_lib_conversation_dispatch.py +31 -1
- package/bin/_lib_conversation_query.py +315 -51
- package/bin/_lib_conversation_retention.py +413 -25
- package/bin/_lib_dashboard_sources.py +267 -7
- package/bin/_lib_diagnosis.py +10 -0
- package/bin/_lib_diff_kernel.py +12 -1
- package/bin/_lib_doctor.py +230 -8
- package/bin/_lib_ingest_frontier.py +1366 -49
- package/bin/_lib_journal.py +19 -2
- package/bin/_lib_merge_gate.py +307 -0
- package/bin/_lib_pricing.py +588 -29
- package/bin/_lib_record.py +130 -12
- package/bin/_lib_rederive.py +903 -2
- package/bin/_lib_render.py +9 -3
- package/bin/_lib_retained_size.py +27 -2
- package/bin/_lib_segment_summary.py +66 -8
- package/bin/_lib_share_templates.py +28 -18
- package/bin/_lib_snapshot_cache.py +115 -1
- package/bin/_lib_source_retry.py +320 -0
- package/bin/_lib_statusline_candidates.py +400 -36
- package/bin/_lib_tick_stats.py +119 -0
- package/bin/_lib_view_models.py +256 -76
- package/bin/cctally +69 -0
- package/dashboard/static/assets/ConversationsView-BB9vUiL6.js +72 -0
- package/dashboard/static/assets/{DoctorModal-CWn3U4Wl.js → DoctorModal-DFY2iPDf.js} +1 -1
- package/dashboard/static/assets/ModalRoot-DvRKHbQE.js +1 -0
- package/dashboard/static/assets/ProjectsDrillPanel-DAv39uSo.js +1 -0
- package/dashboard/static/assets/SourceDetailModal-DSAWo2Fp.js +1 -0
- package/dashboard/static/assets/{UpdateModal-D3m8GG6V.js → UpdateModal-R_9dOVnh.js} +3 -3
- package/dashboard/static/assets/dashboardStream.shared-worker-DbG6Ef0T.js +1 -0
- package/dashboard/static/assets/index-COhxe3vU.css +1 -0
- package/dashboard/static/assets/index-DdU62sov.js +13 -0
- package/dashboard/static/assets/outlineNavigation-BOCLcRa6.js +9 -0
- package/dashboard/static/assets/useKeymap-CW7Up0Ay.js +1 -0
- package/dashboard/static/dashboard.html +3 -3
- package/package.json +4 -1
- package/dashboard/static/assets/ConversationsView-BOSaBRtu.js +0 -72
- package/dashboard/static/assets/ModalRoot-SR070V6c.js +0 -1
- package/dashboard/static/assets/ProjectsDrillPanel-QLI9i5mZ.js +0 -1
- package/dashboard/static/assets/SourceDetailModal-pv0wlFxl.js +0 -1
- package/dashboard/static/assets/dashboardStream.shared-worker-1XTMV3nr.js +0 -1
- package/dashboard/static/assets/index-BHu4mxd8.css +0 -1
- package/dashboard/static/assets/index-CIWsbux3.js +0 -13
- package/dashboard/static/assets/outlineNavigation-CJvKqmLV.js +0 -9
- package/dashboard/static/assets/useKeymap-ffqsS5G0.js +0 -1
package/bin/_cctally_db.py
CHANGED
|
@@ -138,6 +138,66 @@ def _compute_block_totals(*args, **kwargs):
|
|
|
138
138
|
return sys.modules["cctally"]._compute_block_totals(*args, **kwargs)
|
|
139
139
|
|
|
140
140
|
|
|
141
|
+
def _ownership_windows_by_account(rows):
|
|
142
|
+
"""Group `five_hour_blocks` rows into per-account ownership contexts.
|
|
143
|
+
|
|
144
|
+
The two child backfills walk every block row, so the row set they
|
|
145
|
+
already hold IS the competing-window context each of those blocks
|
|
146
|
+
needs (issue #751a). Returns ``{account_key: [OwnedWindow, ...]}``.
|
|
147
|
+
|
|
148
|
+
A row this function cannot interpret is REPORTED rather than skipped.
|
|
149
|
+
It used to be skipped, on the reasoning that defaulting to an interval
|
|
150
|
+
would silently reassign entries between two windows — which is right,
|
|
151
|
+
but skipping it silently is not the alternative. The caller goes on to
|
|
152
|
+
price that same row and `_compute_block_totals` then raises `owner_key
|
|
153
|
+
... is absent from the competing-window context`, a diagnostic that
|
|
154
|
+
names the wrong cause: the window is absent because this row could not
|
|
155
|
+
be read, not because the caller asked about a window nobody declared
|
|
156
|
+
(#769 S2 review P3-1). Both call sites already fail on an unparseable
|
|
157
|
+
`block_start_at` of their own, so this fails no run that previously
|
|
158
|
+
succeeded; it only says why.
|
|
159
|
+
"""
|
|
160
|
+
import _lib_blocks
|
|
161
|
+
|
|
162
|
+
def _field(row, name):
|
|
163
|
+
try:
|
|
164
|
+
return row[name]
|
|
165
|
+
except (KeyError, IndexError):
|
|
166
|
+
return None
|
|
167
|
+
|
|
168
|
+
grouped: dict = {}
|
|
169
|
+
for row in rows:
|
|
170
|
+
row_id = _field(row, "id")
|
|
171
|
+
parsed = {}
|
|
172
|
+
for field in ("block_start_at", "five_hour_resets_at"):
|
|
173
|
+
raw = _field(row, field)
|
|
174
|
+
try:
|
|
175
|
+
parsed[field] = parse_iso_datetime(
|
|
176
|
+
raw, f"five_hour_blocks.{field}",
|
|
177
|
+
).astimezone(dt.timezone.utc)
|
|
178
|
+
except (ValueError, TypeError) as exc:
|
|
179
|
+
raise ValueError(
|
|
180
|
+
f"_ownership_windows_by_account: five_hour_blocks row "
|
|
181
|
+
f"id={row_id!r} has an unusable {field}: {raw!r}"
|
|
182
|
+
) from exc
|
|
183
|
+
try:
|
|
184
|
+
window_key = int(_field(row, "five_hour_window_key"))
|
|
185
|
+
except (ValueError, TypeError) as exc:
|
|
186
|
+
raise ValueError(
|
|
187
|
+
f"_ownership_windows_by_account: five_hour_blocks row "
|
|
188
|
+
f"id={row_id!r} has an unusable five_hour_window_key: "
|
|
189
|
+
f"{_field(row, 'five_hour_window_key')!r}"
|
|
190
|
+
) from exc
|
|
191
|
+
grouped.setdefault(_field(row, "account_key"), []).append(
|
|
192
|
+
_lib_blocks.OwnedWindow(
|
|
193
|
+
key=window_key,
|
|
194
|
+
start=parsed["block_start_at"],
|
|
195
|
+
reset=parsed["five_hour_resets_at"],
|
|
196
|
+
)
|
|
197
|
+
)
|
|
198
|
+
return grouped
|
|
199
|
+
|
|
200
|
+
|
|
141
201
|
# === BEGIN MOVED REGIONS ===
|
|
142
202
|
# Regions below are inserted verbatim from bin/cctally. Bare-name
|
|
143
203
|
# references to `now_utc_iso(...)`, `parse_iso_datetime(...)`,
|
|
@@ -2023,6 +2083,8 @@ def _run_pending_migrations(
|
|
|
2023
2083
|
db_label, db_version=cur_version, max_known=len(registry),
|
|
2024
2084
|
)
|
|
2025
2085
|
if cur_version == len(registry):
|
|
2086
|
+
if db_label == "conversations.db":
|
|
2087
|
+
_ensure_codex_session_meta_provenance_index(conn)
|
|
2026
2088
|
# When the registry is currently empty (today's cache.db case),
|
|
2027
2089
|
# still leave the schema_migrations table behind so a later
|
|
2028
2090
|
# transition to len(registry) >= 1 can distinguish populated
|
|
@@ -2064,6 +2126,9 @@ def _run_pending_migrations(
|
|
|
2064
2126
|
db_label, _first_pending_migration_name(conn, registry, cur_version)
|
|
2065
2127
|
)
|
|
2066
2128
|
|
|
2129
|
+
if db_label == "conversations.db":
|
|
2130
|
+
_ensure_codex_session_meta_provenance_index(conn)
|
|
2131
|
+
|
|
2067
2132
|
# Track whether schema_migrations existed before this open so we can
|
|
2068
2133
|
# detect the fresh-install path. After bootstrap, even a "first time
|
|
2069
2134
|
# opened with framework code" DB might have rows from the legacy
|
|
@@ -2335,9 +2400,13 @@ def _backfill_five_hour_block_models(conn: sqlite3.Connection) -> None:
|
|
|
2335
2400
|
|
|
2336
2401
|
rows = conn.execute(
|
|
2337
2402
|
"SELECT id, five_hour_window_key, block_start_at, "
|
|
2338
|
-
" last_observed_at_utc "
|
|
2403
|
+
" five_hour_resets_at, last_observed_at_utc, account_key "
|
|
2339
2404
|
" FROM five_hour_blocks"
|
|
2340
2405
|
).fetchall()
|
|
2406
|
+
# #751a: the row set is the competing-window context. Without it
|
|
2407
|
+
# each block re-sums its own raw interval and prices an adjacent
|
|
2408
|
+
# window's overlap entries a second time.
|
|
2409
|
+
ownership = _ownership_windows_by_account(rows)
|
|
2341
2410
|
for row in rows:
|
|
2342
2411
|
block_start_dt = parse_iso_datetime(
|
|
2343
2412
|
row["block_start_at"],
|
|
@@ -2357,6 +2426,8 @@ def _backfill_five_hour_block_models(conn: sqlite3.Connection) -> None:
|
|
|
2357
2426
|
# so there is no recursion risk.
|
|
2358
2427
|
totals = _compute_block_totals(
|
|
2359
2428
|
block_start_dt, last_obs_dt, skip_sync=False,
|
|
2429
|
+
owner_key=int(row["five_hour_window_key"]),
|
|
2430
|
+
windows=ownership.get(row["account_key"], []),
|
|
2360
2431
|
)
|
|
2361
2432
|
if totals.get("by_model"):
|
|
2362
2433
|
conn.executemany(
|
|
@@ -2418,9 +2489,13 @@ def _backfill_five_hour_block_projects(conn: sqlite3.Connection) -> None:
|
|
|
2418
2489
|
|
|
2419
2490
|
rows = conn.execute(
|
|
2420
2491
|
"SELECT id, five_hour_window_key, block_start_at, "
|
|
2421
|
-
" last_observed_at_utc "
|
|
2492
|
+
" five_hour_resets_at, last_observed_at_utc, account_key "
|
|
2422
2493
|
" FROM five_hour_blocks"
|
|
2423
2494
|
).fetchall()
|
|
2495
|
+
# #751a: the row set is the competing-window context. Without it
|
|
2496
|
+
# each block re-sums its own raw interval and prices an adjacent
|
|
2497
|
+
# window's overlap entries a second time.
|
|
2498
|
+
ownership = _ownership_windows_by_account(rows)
|
|
2424
2499
|
for row in rows:
|
|
2425
2500
|
block_start_dt = parse_iso_datetime(
|
|
2426
2501
|
row["block_start_at"],
|
|
@@ -2437,6 +2512,8 @@ def _backfill_five_hour_block_projects(conn: sqlite3.Connection) -> None:
|
|
|
2437
2512
|
# so there is no open_db() recursion risk.
|
|
2438
2513
|
totals = _compute_block_totals(
|
|
2439
2514
|
block_start_dt, last_obs_dt, skip_sync=False,
|
|
2515
|
+
owner_key=int(row["five_hour_window_key"]),
|
|
2516
|
+
windows=ownership.get(row["account_key"], []),
|
|
2440
2517
|
)
|
|
2441
2518
|
if totals.get("by_project"):
|
|
2442
2519
|
conn.executemany(
|
|
@@ -4386,6 +4463,14 @@ CACHE_REDERIVABLE_OBJECTS: "tuple[SchemaDeliveryObject, ...]" = (
|
|
|
4386
4463
|
SchemaDeliveryObject(
|
|
4387
4464
|
"index", "idx_codex_window_attributions_root", None,
|
|
4388
4465
|
"043_codex_window_attributions"),
|
|
4466
|
+
SchemaDeliveryObject(
|
|
4467
|
+
"column", "codex_session_files.device_id",
|
|
4468
|
+
"_apply_codex_session_file_identity",
|
|
4469
|
+
"046_codex_source_file_identity"),
|
|
4470
|
+
SchemaDeliveryObject(
|
|
4471
|
+
"column", "codex_session_files.inode",
|
|
4472
|
+
"_apply_codex_session_file_identity",
|
|
4473
|
+
"046_codex_source_file_identity"),
|
|
4389
4474
|
SchemaDeliveryObject(
|
|
4390
4475
|
"index", "idx_codex_accounting_change_mutation",
|
|
4391
4476
|
"_apply_codex_accounting_change_ledger",
|
|
@@ -4455,6 +4540,33 @@ CACHE_REDERIVABLE_OBJECTS: "tuple[SchemaDeliveryObject, ...]" = (
|
|
|
4455
4540
|
)
|
|
4456
4541
|
|
|
4457
4542
|
|
|
4543
|
+
def _apply_codex_session_file_identity(conn: sqlite3.Connection) -> None:
|
|
4544
|
+
"""Deliver the Codex accounting cursor's file identity (#769 S6, #716 A).
|
|
4545
|
+
|
|
4546
|
+
``codex_session_files`` records a size, an mtime and a byte offset but no
|
|
4547
|
+
file identity, so a replacement that lands at the same size and the same
|
|
4548
|
+
mtime leaves the retained cursor indistinguishable from a cursor into the
|
|
4549
|
+
file that now occupies the pathname. ``conversation_source_files`` carries
|
|
4550
|
+
exactly these two columns for the same reason (#777); this is the Codex
|
|
4551
|
+
half of that identity.
|
|
4552
|
+
|
|
4553
|
+
NULLABLE with NO DEFAULT, and deliberately NOT backfilled. NULL is the
|
|
4554
|
+
"identity unknown" state an existing row produces for free, and a reader
|
|
4555
|
+
treats it as no evidence, which is the behavior that store already has.
|
|
4556
|
+
Stamping a CURRENT stat onto a cursor written by an earlier pass would
|
|
4557
|
+
assert that the pass read this inode, which the migration cannot know: the
|
|
4558
|
+
file may have been replaced between that pass and the upgrade. The value
|
|
4559
|
+
arrives on the next ingest of that path, which is the only moment the
|
|
4560
|
+
identity and the offset are observed together.
|
|
4561
|
+
|
|
4562
|
+
``add_column_if_missing`` ONLY, never in the CREATE TABLE body: ALTER TABLE
|
|
4563
|
+
ADD COLUMN appends, so a column in both places sits at a different ordinal
|
|
4564
|
+
on a fresh store than on a migrated one (the #195 hazard).
|
|
4565
|
+
"""
|
|
4566
|
+
add_column_if_missing(conn, "codex_session_files", "device_id", "INTEGER")
|
|
4567
|
+
add_column_if_missing(conn, "codex_session_files", "inode", "INTEGER")
|
|
4568
|
+
|
|
4569
|
+
|
|
4458
4570
|
def _apply_cache_schema(conn: sqlite3.Connection) -> None:
|
|
4459
4571
|
"""Single source of cache.db's schema (cctally-dev#93, spec D4).
|
|
4460
4572
|
|
|
@@ -4998,6 +5110,7 @@ def _apply_cache_schema(conn: sqlite3.Connection) -> None:
|
|
|
4998
5110
|
add_column_if_missing(
|
|
4999
5111
|
conn, "codex_session_files", "ingest_complete",
|
|
5000
5112
|
"INTEGER NOT NULL DEFAULT 1")
|
|
5113
|
+
_apply_codex_session_file_identity(conn)
|
|
5001
5114
|
add_column_if_missing(conn, "quota_window_snapshots", "account_key", "TEXT")
|
|
5002
5115
|
# #195: the cache-write TTL split. NULLable with NO DEFAULT — NULL is the
|
|
5003
5116
|
# "split unknown" sentinel a pre-#195 row produces for free, and a real
|
|
@@ -5279,6 +5392,8 @@ def _apply_conversations_schema(conn: sqlite3.Connection) -> None:
|
|
|
5279
5392
|
current = None
|
|
5280
5393
|
if current is not None and current[0] == "2":
|
|
5281
5394
|
_apply_codex_find_projection_schema(conn)
|
|
5395
|
+
_apply_conversation_generation_schema(conn)
|
|
5396
|
+
_apply_codex_conversation_source_identity(conn)
|
|
5282
5397
|
return
|
|
5283
5398
|
|
|
5284
5399
|
_apply_cache_schema(conn)
|
|
@@ -5361,6 +5476,182 @@ def _apply_conversations_schema(conn: sqlite3.Connection) -> None:
|
|
|
5361
5476
|
"ON CONFLICT(key) DO UPDATE SET value=excluded.value"
|
|
5362
5477
|
)
|
|
5363
5478
|
_apply_codex_find_projection_schema(conn)
|
|
5479
|
+
_apply_conversation_generation_schema(conn)
|
|
5480
|
+
_apply_codex_conversation_source_identity(conn)
|
|
5481
|
+
|
|
5482
|
+
|
|
5483
|
+
def _ensure_codex_session_meta_provenance_index(
|
|
5484
|
+
conn: sqlite3.Connection,
|
|
5485
|
+
) -> bool:
|
|
5486
|
+
"""Ensure the account-scoped anonymization provenance index exists.
|
|
5487
|
+
|
|
5488
|
+
``codex_conversation_events`` is an append-only physical log and can be
|
|
5489
|
+
much larger than the normalized transcript. The scoped anonymizer reads
|
|
5490
|
+
only ``session_meta`` payloads, so an index restricted to that record type
|
|
5491
|
+
makes the read proportional to the retained provenance rows rather than
|
|
5492
|
+
every event. This is an idempotent, data-free schema ensure run by the
|
|
5493
|
+
conversations dispatcher on both its fast and pending paths; keeping the
|
|
5494
|
+
helper here also gives the schema migration layer one stable delivery
|
|
5495
|
+
point for fresh and already-current stores.
|
|
5496
|
+
|
|
5497
|
+
A pre-conversations store may not have the event table yet. That is a
|
|
5498
|
+
valid no-op; the schema apply creates it before the dispatcher runs.
|
|
5499
|
+
Other SQLite errors propagate so a damaged store cannot silently fall back
|
|
5500
|
+
to a full physical-event scan.
|
|
5501
|
+
"""
|
|
5502
|
+
if conn.execute(
|
|
5503
|
+
"SELECT 1 FROM sqlite_master "
|
|
5504
|
+
"WHERE type='table' AND name='codex_conversation_events'"
|
|
5505
|
+
).fetchone() is None:
|
|
5506
|
+
return False
|
|
5507
|
+
if conn.execute(
|
|
5508
|
+
"SELECT 1 FROM sqlite_master "
|
|
5509
|
+
"WHERE type='index' AND name='idx_codex_events_session_meta_provenance'"
|
|
5510
|
+
).fetchone() is not None:
|
|
5511
|
+
return True
|
|
5512
|
+
conn.execute(
|
|
5513
|
+
"CREATE INDEX IF NOT EXISTS idx_codex_events_session_meta_provenance "
|
|
5514
|
+
"ON codex_conversation_events("
|
|
5515
|
+
"source_root_key, account_key, conversation_key, payload_json) "
|
|
5516
|
+
"WHERE record_type='session_meta'"
|
|
5517
|
+
)
|
|
5518
|
+
conn.commit()
|
|
5519
|
+
return True
|
|
5520
|
+
|
|
5521
|
+
|
|
5522
|
+
#: `cache_meta` keys the stamp backfill uses. The cursor is the durable resume
|
|
5523
|
+
#: point (the last source path whose rows are fully stamped); the completion
|
|
5524
|
+
#: marker is what a rebuild's publication gate reads. Both are values, not
|
|
5525
|
+
#: schema, so a store that has finished the backfill carries only the marker.
|
|
5526
|
+
CLAUDE_STAMP_BACKFILL_CURSOR_KEY = "claude_account_stamp_backfill_cursor"
|
|
5527
|
+
CLAUDE_STAMP_COVERAGE_COMPLETE_KEY = "claude_account_stamp_coverage_complete"
|
|
5528
|
+
|
|
5529
|
+
#: Column shapes for the two staging tables, mirroring their live twins. Kept
|
|
5530
|
+
#: as literals rather than derived from `PRAGMA table_info` at runtime: a
|
|
5531
|
+
#: staging table whose shape is read off the live one at open time would follow
|
|
5532
|
+
#: the live table silently through a future column addition, and the publish
|
|
5533
|
+
#: `INSERT ... SELECT *` would then succeed against a shape nobody reviewed.
|
|
5534
|
+
#: A future column addition must edit BOTH, which is the point.
|
|
5535
|
+
_CONVERSATION_GENERATION_DDL = """
|
|
5536
|
+
CREATE TABLE IF NOT EXISTS conversation_ai_titles_staging (
|
|
5537
|
+
session_id TEXT NOT NULL PRIMARY KEY,
|
|
5538
|
+
ai_title TEXT NOT NULL,
|
|
5539
|
+
source_path TEXT,
|
|
5540
|
+
byte_offset INTEGER NOT NULL
|
|
5541
|
+
);
|
|
5542
|
+
CREATE TABLE IF NOT EXISTS conversation_sessions_staging (
|
|
5543
|
+
session_id TEXT NOT NULL PRIMARY KEY,
|
|
5544
|
+
msg_count INTEGER NOT NULL DEFAULT 0,
|
|
5545
|
+
started_utc TEXT,
|
|
5546
|
+
last_activity_utc TEXT,
|
|
5547
|
+
project_label TEXT,
|
|
5548
|
+
cost_usd REAL NOT NULL DEFAULT 0,
|
|
5549
|
+
cache_rebuild_count INTEGER NOT NULL DEFAULT 0,
|
|
5550
|
+
git_branch TEXT,
|
|
5551
|
+
models_json TEXT,
|
|
5552
|
+
title TEXT,
|
|
5553
|
+
render_revision INTEGER NOT NULL DEFAULT 0
|
|
5554
|
+
);
|
|
5555
|
+
CREATE TABLE IF NOT EXISTS claude_conversation_account_stamps (
|
|
5556
|
+
canonical_source_path TEXT NOT NULL,
|
|
5557
|
+
source_incarnation_id TEXT NOT NULL,
|
|
5558
|
+
byte_offset INTEGER NOT NULL,
|
|
5559
|
+
record_sha256 TEXT NOT NULL,
|
|
5560
|
+
account_key TEXT,
|
|
5561
|
+
PRIMARY KEY(canonical_source_path, source_incarnation_id,
|
|
5562
|
+
byte_offset, record_sha256)
|
|
5563
|
+
) WITHOUT ROWID;
|
|
5564
|
+
CREATE TABLE IF NOT EXISTS claude_conversation_account_stamp_gaps (
|
|
5565
|
+
source_path TEXT NOT NULL,
|
|
5566
|
+
byte_offset INTEGER NOT NULL,
|
|
5567
|
+
cause TEXT NOT NULL,
|
|
5568
|
+
account_key TEXT,
|
|
5569
|
+
PRIMARY KEY(source_path, byte_offset)
|
|
5570
|
+
) WITHOUT ROWID;
|
|
5571
|
+
"""
|
|
5572
|
+
|
|
5573
|
+
|
|
5574
|
+
def _apply_conversation_generation_schema(conn: sqlite3.Connection) -> None:
|
|
5575
|
+
"""Create #752's staging tables and #777's durable account stamps.
|
|
5576
|
+
|
|
5577
|
+
Called from BOTH branches of `_apply_conversations_schema` — the
|
|
5578
|
+
version-current early return and the full apply — exactly like
|
|
5579
|
+
`_apply_codex_find_projection_schema`, so migration 009 can deliver the
|
|
5580
|
+
objects to an existing store without replaying the much larger historical
|
|
5581
|
+
transcript schema.
|
|
5582
|
+
|
|
5583
|
+
**The staging tables carry no FTS and no triggers.** `conversation_title_fts`
|
|
5584
|
+
is an external-content FTS5 index bound BY NAME to `conversation_ai_titles`,
|
|
5585
|
+
and it follows the live table through the existing `conv_title_fts_ai` /
|
|
5586
|
+
`_ad` / `_au` triggers. A staging table with its own index would have to be
|
|
5587
|
+
swapped too, which is the reader-routing problem the narrowed #752 design
|
|
5588
|
+
exists to avoid.
|
|
5589
|
+
|
|
5590
|
+
**`claude_conversation_account_stamps` is the one table in this store that
|
|
5591
|
+
a replay cannot reconstruct.** Everything else in `conversations.db` can be
|
|
5592
|
+
rebuilt from provider JSONL; an observation-time account attribution cannot,
|
|
5593
|
+
because the transcript does not record which account was active when the
|
|
5594
|
+
bytes arrived.
|
|
5595
|
+
That is why a rebuild reads attribution back from here instead of stamping
|
|
5596
|
+
every replayed row with the currently active identity.
|
|
5597
|
+
|
|
5598
|
+
All four identity columns are NOT NULL and form the primary key;
|
|
5599
|
+
`account_key` is nullable, and NULL is the stored spelling of
|
|
5600
|
+
`unattributed`, matching the rest of the store. `WITHOUT ROWID` because the
|
|
5601
|
+
table is all key and one small value, and every access is by the full key.
|
|
5602
|
+
|
|
5603
|
+
`claude_conversation_account_stamp_gaps` is what "explicitly classified"
|
|
5604
|
+
means for a record whose bytes cannot be digested — a source file deleted
|
|
5605
|
+
since ingestion, or one truncated below the offset. Such a row still holds
|
|
5606
|
+
the attribution the pre-rebuild `conversation_messages` row carried, which
|
|
5607
|
+
is the same evidence a stamp holds, minus the digest that would let it
|
|
5608
|
+
survive the file being rewritten. Recording it is strictly better than
|
|
5609
|
+
discarding it: without the gaps table, every orphaned path's attribution
|
|
5610
|
+
would be lost at the first rebuild, which is the loss the backfill exists
|
|
5611
|
+
to prevent.
|
|
5612
|
+
"""
|
|
5613
|
+
conn.executescript(_CONVERSATION_GENERATION_DDL)
|
|
5614
|
+
# #777: the source-incarnation identity. `size_bytes`/`mtime_ns` stay
|
|
5615
|
+
# observations; these four are identity. `_conversation_target_risk`
|
|
5616
|
+
# decides the size-preserving case on mtime and is defeated by a `touch`,
|
|
5617
|
+
# so the committed-prefix digest is what makes the promise absolute.
|
|
5618
|
+
add_column_if_missing(conn, "conversation_source_files", "device_id",
|
|
5619
|
+
"INTEGER")
|
|
5620
|
+
add_column_if_missing(conn, "conversation_source_files", "inode",
|
|
5621
|
+
"INTEGER")
|
|
5622
|
+
add_column_if_missing(conn, "conversation_source_files",
|
|
5623
|
+
"source_incarnation_id", "TEXT")
|
|
5624
|
+
add_column_if_missing(conn, "conversation_source_files",
|
|
5625
|
+
"committed_prefix_sha256", "TEXT")
|
|
5626
|
+
# A store with no retained messages has nothing to backfill, so its stamp
|
|
5627
|
+
# coverage is complete by construction and the rebuild gate must let it
|
|
5628
|
+
# through. Without this a FRESH install would be refused every rebuild,
|
|
5629
|
+
# because the completion marker is written only by migration 009 — which a
|
|
5630
|
+
# fresh install stamps without running. Same shape as the Codex find
|
|
5631
|
+
# projection's fresh-store completion beside it.
|
|
5632
|
+
if conn.execute(
|
|
5633
|
+
"SELECT 1 FROM cache_meta WHERE key=?",
|
|
5634
|
+
(CLAUDE_STAMP_COVERAGE_COMPLETE_KEY,),
|
|
5635
|
+
).fetchone() is None and conn.execute(
|
|
5636
|
+
"SELECT 1 FROM conversation_messages LIMIT 1"
|
|
5637
|
+
).fetchone() is None:
|
|
5638
|
+
_set_cache_meta(conn, CLAUDE_STAMP_COVERAGE_COMPLETE_KEY, "1")
|
|
5639
|
+
|
|
5640
|
+
|
|
5641
|
+
def _apply_codex_conversation_source_identity(conn: sqlite3.Connection) -> None:
|
|
5642
|
+
"""Deliver the Codex transcript cursor's file identity (#769 S6, #716 A).
|
|
5643
|
+
|
|
5644
|
+
The twin of :func:`_apply_codex_session_file_identity` for
|
|
5645
|
+
``codex_conversation_source_files``. Same nullable shape, same reason for
|
|
5646
|
+
not backfilling, same ``add_column_if_missing``-only delivery. The two
|
|
5647
|
+
tables are one frontier root set, so an identity added to only one half
|
|
5648
|
+
would let the accounting frontier and the conversation frontier disagree
|
|
5649
|
+
about whether the pathname still names the file each of them read.
|
|
5650
|
+
"""
|
|
5651
|
+
add_column_if_missing(
|
|
5652
|
+
conn, "codex_conversation_source_files", "device_id", "INTEGER")
|
|
5653
|
+
add_column_if_missing(
|
|
5654
|
+
conn, "codex_conversation_source_files", "inode", "INTEGER")
|
|
5364
5655
|
|
|
5365
5656
|
|
|
5366
5657
|
def _apply_codex_find_projection_schema(conn: sqlite3.Connection) -> None:
|
|
@@ -5502,6 +5793,46 @@ CONVERSATIONS_REDERIVABLE_OBJECTS: "tuple[SchemaDeliveryObject, ...]" = (
|
|
|
5502
5793
|
SchemaDeliveryObject(
|
|
5503
5794
|
"trigger", "codex_find_projection_message_ad",
|
|
5504
5795
|
"_apply_codex_find_projection_schema", "004_codex_find_projection"),
|
|
5796
|
+
SchemaDeliveryObject(
|
|
5797
|
+
"column", "conversation_source_files.committed_prefix_sha256",
|
|
5798
|
+
"_apply_conversation_generation_schema",
|
|
5799
|
+
"009_conversation_title_staging_and_account_stamps"),
|
|
5800
|
+
SchemaDeliveryObject(
|
|
5801
|
+
"column", "conversation_source_files.device_id",
|
|
5802
|
+
"_apply_conversation_generation_schema",
|
|
5803
|
+
"009_conversation_title_staging_and_account_stamps"),
|
|
5804
|
+
SchemaDeliveryObject(
|
|
5805
|
+
"column", "conversation_source_files.inode",
|
|
5806
|
+
"_apply_conversation_generation_schema",
|
|
5807
|
+
"009_conversation_title_staging_and_account_stamps"),
|
|
5808
|
+
SchemaDeliveryObject(
|
|
5809
|
+
"column", "conversation_source_files.source_incarnation_id",
|
|
5810
|
+
"_apply_conversation_generation_schema",
|
|
5811
|
+
"009_conversation_title_staging_and_account_stamps"),
|
|
5812
|
+
SchemaDeliveryObject(
|
|
5813
|
+
"column", "codex_conversation_source_files.device_id",
|
|
5814
|
+
"_apply_codex_conversation_source_identity",
|
|
5815
|
+
"010_codex_conversation_source_file_identity"),
|
|
5816
|
+
SchemaDeliveryObject(
|
|
5817
|
+
"column", "codex_conversation_source_files.inode",
|
|
5818
|
+
"_apply_codex_conversation_source_identity",
|
|
5819
|
+
"010_codex_conversation_source_file_identity"),
|
|
5820
|
+
SchemaDeliveryObject(
|
|
5821
|
+
"table", "claude_conversation_account_stamp_gaps",
|
|
5822
|
+
"_apply_conversation_generation_schema",
|
|
5823
|
+
"009_conversation_title_staging_and_account_stamps"),
|
|
5824
|
+
SchemaDeliveryObject(
|
|
5825
|
+
"table", "claude_conversation_account_stamps",
|
|
5826
|
+
"_apply_conversation_generation_schema",
|
|
5827
|
+
"009_conversation_title_staging_and_account_stamps"),
|
|
5828
|
+
SchemaDeliveryObject(
|
|
5829
|
+
"table", "conversation_ai_titles_staging",
|
|
5830
|
+
"_apply_conversation_generation_schema",
|
|
5831
|
+
"009_conversation_title_staging_and_account_stamps"),
|
|
5832
|
+
SchemaDeliveryObject(
|
|
5833
|
+
"table", "conversation_sessions_staging",
|
|
5834
|
+
"_apply_conversation_generation_schema",
|
|
5835
|
+
"009_conversation_title_staging_and_account_stamps"),
|
|
5505
5836
|
)
|
|
5506
5837
|
|
|
5507
5838
|
|
|
@@ -5828,6 +6159,300 @@ def _conv_008_conversation_render_revision(conn: sqlite3.Connection) -> None:
|
|
|
5828
6159
|
conn.commit()
|
|
5829
6160
|
|
|
5830
6161
|
|
|
6162
|
+
#: How many source paths one backfill transaction covers. Small enough that an
|
|
6163
|
+
#: interrupted migration loses at most this much work, large enough that the
|
|
6164
|
+
#: per-transaction overhead does not dominate a 12,000-file store.
|
|
6165
|
+
_STAMP_BACKFILL_PATHS_PER_TXN = 25
|
|
6166
|
+
|
|
6167
|
+
|
|
6168
|
+
def _stamp_backfill_read_records(path_str, offsets, committed_offset):
|
|
6169
|
+
"""Read one source file's record spans AND the identity a stamp is keyed to.
|
|
6170
|
+
|
|
6171
|
+
Returns ``(spans, cause, identity, offset_causes)`` — a
|
|
6172
|
+
``{offset: raw_bytes}`` map, a classification string when the file could
|
|
6173
|
+
not be used at all, the ``(device_id, inode, committed_prefix_sha256)``
|
|
6174
|
+
triple for ``conversation_source_files``, and a per-offset cause for each
|
|
6175
|
+
offset that produced no span.
|
|
6176
|
+
|
|
6177
|
+
THE IDENTITY IS NOT OPTIONAL, and leaving it out was a real defect this
|
|
6178
|
+
function's first version shipped. A stamp is keyed by the incarnation, and
|
|
6179
|
+
`_resolve_source_incarnation` only keeps an incarnation when the stored
|
|
6180
|
+
inode and committed-prefix digest still match (the device is recorded as
|
|
6181
|
+
corroborating evidence but does not decide — #814). A backfill that
|
|
6182
|
+
wrote the incarnation but none of those left every one of its stamps
|
|
6183
|
+
unreachable: the first rebuild rejected continuity, minted a fresh
|
|
6184
|
+
incarnation, missed every stamp and re-attributed the whole store to
|
|
6185
|
+
whichever account was active. Measured on a copy-on-write clone of the real
|
|
6186
|
+
store: 43,810 of 43,810 sampled messages restamped, with the coverage
|
|
6187
|
+
marker reporting complete throughout.
|
|
6188
|
+
|
|
6189
|
+
Opens the file ONCE and slices it, because one open per offset would
|
|
6190
|
+
multiply a 334,239-row backfill by 334,239 syscalls. The `fstat` pair
|
|
6191
|
+
around the read is the same time-of-check/time-of-use guard the ingest
|
|
6192
|
+
boundary uses: a file replaced mid-read yields an identity that describes
|
|
6193
|
+
neither version, so the whole path is classified rather than stamped
|
|
6194
|
+
against bytes that are already gone.
|
|
6195
|
+
|
|
6196
|
+
An offset past the end of the file, and an offset that does not begin a
|
|
6197
|
+
record, are both absent from the map and carry a per-offset cause the
|
|
6198
|
+
caller records as a gap. The record-start check is what makes the second of
|
|
6199
|
+
those true: without it a mid-record offset yields the tail of a record, and
|
|
6200
|
+
a stamp written over that partial span has a digest no record will ever
|
|
6201
|
+
reproduce, so it is written and then permanently unreachable. The backfill
|
|
6202
|
+
never guesses a digest.
|
|
6203
|
+
"""
|
|
6204
|
+
import hashlib # noqa: PLC0415 — stdlib, matching the kernel beside it
|
|
6205
|
+
try:
|
|
6206
|
+
with open(path_str, "rb") as fh:
|
|
6207
|
+
before = os.fstat(fh.fileno())
|
|
6208
|
+
data = fh.read()
|
|
6209
|
+
after = os.fstat(fh.fileno())
|
|
6210
|
+
except OSError:
|
|
6211
|
+
return {}, "source_unreadable", None, {}
|
|
6212
|
+
if (before.st_dev != after.st_dev or before.st_ino != after.st_ino
|
|
6213
|
+
or before.st_size != after.st_size
|
|
6214
|
+
or before.st_mtime_ns != after.st_mtime_ns):
|
|
6215
|
+
return {}, "source_changed_during_backfill", None, {}
|
|
6216
|
+
size = len(data)
|
|
6217
|
+
if committed_offset > size:
|
|
6218
|
+
# The cursor describes bytes the file no longer holds, so no identity
|
|
6219
|
+
# this backfill could record would survive the continuity check anyway.
|
|
6220
|
+
return {}, "source_shorter_than_cursor", None, {}
|
|
6221
|
+
identity = (
|
|
6222
|
+
after.st_dev,
|
|
6223
|
+
after.st_ino,
|
|
6224
|
+
hashlib.sha256(data[:max(0, committed_offset)]).hexdigest(),
|
|
6225
|
+
)
|
|
6226
|
+
strip = _lib_conversation_kernel().strip_record_terminator
|
|
6227
|
+
spans = {}
|
|
6228
|
+
offset_causes = {}
|
|
6229
|
+
for offset in offsets:
|
|
6230
|
+
if offset < 0 or offset >= size:
|
|
6231
|
+
offset_causes[offset] = "offset_past_end_of_file"
|
|
6232
|
+
continue
|
|
6233
|
+
if offset and data[offset - 1:offset] != b"\n":
|
|
6234
|
+
# Not a record start. The walker only ever records record-start
|
|
6235
|
+
# offsets, so this means the file's records moved without the
|
|
6236
|
+
# `fstat` pair or the identity check catching it. Slicing from here
|
|
6237
|
+
# would digest a record's tail and write a stamp nothing can find.
|
|
6238
|
+
offset_causes[offset] = "offset_not_a_record_start"
|
|
6239
|
+
continue
|
|
6240
|
+
end = data.find(b"\n", offset)
|
|
6241
|
+
# Include the terminator in the slice and let the shared kernel remove
|
|
6242
|
+
# it, so the `\r\n` rule lives in exactly one place. A last line with no
|
|
6243
|
+
# terminator takes the same path with nothing to strip.
|
|
6244
|
+
span = data[offset:] if end < 0 else data[offset:end + 1]
|
|
6245
|
+
spans[offset] = strip(span)
|
|
6246
|
+
return spans, None, identity, offset_causes
|
|
6247
|
+
|
|
6248
|
+
|
|
6249
|
+
def _lib_conversation_kernel():
|
|
6250
|
+
"""The pure conversation kernel, imported lazily.
|
|
6251
|
+
|
|
6252
|
+
`_cctally_db` is loaded by every command, and this kernel is reached only
|
|
6253
|
+
by the transcript backfill, so a module-level import would add its cost to
|
|
6254
|
+
`statusline`.
|
|
6255
|
+
"""
|
|
6256
|
+
import _lib_conversation # noqa: PLC0415 — lazy sibling, like _lib_pricing
|
|
6257
|
+
return _lib_conversation
|
|
6258
|
+
|
|
6259
|
+
|
|
6260
|
+
def backfill_claude_account_stamps(conn: sqlite3.Connection) -> bool:
|
|
6261
|
+
"""Stamp every retained `conversation_messages` row with the account it was
|
|
6262
|
+
ingested under (#777). Resumable, chunked, and idempotent.
|
|
6263
|
+
|
|
6264
|
+
THE ORDERING THIS PROTECTS. A rebuild replays every source file from byte
|
|
6265
|
+
zero and must decide, per record, which account it belongs to. The rule
|
|
6266
|
+
#777 introduces is that a record below the published high-water mark with
|
|
6267
|
+
no stamp is historical-and-unknown and gets NULL, never the currently
|
|
6268
|
+
active account. Production holds 334,239 attributed rows and zero NULLs
|
|
6269
|
+
against an empty stamps table, so shipping that rule without this backfill
|
|
6270
|
+
converts a latent defect into the immediate loss of every historical
|
|
6271
|
+
attribution. This runs INSIDE migration 009, before any rebuild can reach
|
|
6272
|
+
the new rule.
|
|
6273
|
+
|
|
6274
|
+
RESUMABILITY. Progress is a durable cursor — the last source path whose
|
|
6275
|
+
rows are fully stamped — advanced and committed every
|
|
6276
|
+
`_STAMP_BACKFILL_PATHS_PER_TXN` paths. A killed migration re-runs the
|
|
6277
|
+
handler (the dispatcher stamps its marker in a separate transaction, so a
|
|
6278
|
+
crash between the two re-invokes the handler), and it resumes from the
|
|
6279
|
+
cursor rather than restarting. Paths are processed in sorted order, which
|
|
6280
|
+
is what makes a single scalar cursor sufficient.
|
|
6281
|
+
|
|
6282
|
+
IDEMPOTENCE. Stamps and gaps are written with `INSERT OR IGNORE` against
|
|
6283
|
+
their full primary keys, and an incarnation is minted only for a source
|
|
6284
|
+
file that does not already carry one. A second run over the same store
|
|
6285
|
+
therefore reproduces the same rows, which the per-migration golden pins.
|
|
6286
|
+
|
|
6287
|
+
GAPS ARE CLASSIFIED, NOT DISCARDED. A record whose source file has been
|
|
6288
|
+
deleted, or truncated below its offset, has no bytes to digest and cannot
|
|
6289
|
+
get a stamp. Its attribution is still real, and it is recorded in
|
|
6290
|
+
`claude_conversation_account_stamp_gaps` with the cause. Writing NULL there
|
|
6291
|
+
instead would lose exactly the attribution this function exists to keep,
|
|
6292
|
+
for the files most likely to be gone: orphans from removed worktrees.
|
|
6293
|
+
|
|
6294
|
+
Returns True when coverage is complete.
|
|
6295
|
+
"""
|
|
6296
|
+
kernel = _lib_conversation_kernel()
|
|
6297
|
+
if conn.execute(
|
|
6298
|
+
"SELECT 1 FROM cache_meta WHERE key=?",
|
|
6299
|
+
(CLAUDE_STAMP_COVERAGE_COMPLETE_KEY,),
|
|
6300
|
+
).fetchone() is not None:
|
|
6301
|
+
return True
|
|
6302
|
+
row = conn.execute(
|
|
6303
|
+
"SELECT value FROM cache_meta WHERE key=?",
|
|
6304
|
+
(CLAUDE_STAMP_BACKFILL_CURSOR_KEY,),
|
|
6305
|
+
).fetchone()
|
|
6306
|
+
cursor = row[0] if row and row[0] is not None else ""
|
|
6307
|
+
cursors = {
|
|
6308
|
+
r[0]: (r[1], int(r[2] or 0)) for r in conn.execute(
|
|
6309
|
+
"SELECT path,source_incarnation_id,last_byte_offset "
|
|
6310
|
+
"FROM conversation_source_files")
|
|
6311
|
+
}
|
|
6312
|
+
# Driven by the MESSAGE rows, not by `conversation_source_files`: a message
|
|
6313
|
+
# whose source-file row was pruned still carries an attribution worth
|
|
6314
|
+
# keeping, and the source-file table is the derived half of the pair.
|
|
6315
|
+
paths = [
|
|
6316
|
+
r[0] for r in conn.execute(
|
|
6317
|
+
"SELECT DISTINCT source_path FROM conversation_messages "
|
|
6318
|
+
"WHERE source_path IS NOT NULL AND source_path > ? "
|
|
6319
|
+
"ORDER BY source_path",
|
|
6320
|
+
(cursor,),
|
|
6321
|
+
)
|
|
6322
|
+
]
|
|
6323
|
+
pending = 0
|
|
6324
|
+
for path_str in paths:
|
|
6325
|
+
rows = conn.execute(
|
|
6326
|
+
"SELECT byte_offset,account_key FROM conversation_messages "
|
|
6327
|
+
"WHERE source_path=? ORDER BY byte_offset",
|
|
6328
|
+
(path_str,),
|
|
6329
|
+
).fetchall()
|
|
6330
|
+
attribution = {int(offset): key for offset, key in rows}
|
|
6331
|
+
incarnation, committed = cursors.get(path_str, (None, 0))
|
|
6332
|
+
spans, cause, identity, offset_causes = _stamp_backfill_read_records(
|
|
6333
|
+
path_str, sorted(attribution), committed)
|
|
6334
|
+
if identity is None:
|
|
6335
|
+
# Without a trustworthy identity a stamp cannot be found again, so
|
|
6336
|
+
# this path takes the classified-gap branch for every offset — its
|
|
6337
|
+
# attribution survives, keyed by path and offset alone.
|
|
6338
|
+
spans = {}
|
|
6339
|
+
elif path_str in cursors:
|
|
6340
|
+
# Record the WHOLE identity, not just the incarnation. Continuity
|
|
6341
|
+
# is checked on the inode and the committed-prefix digest together
|
|
6342
|
+
# — the device is recorded but does not decide (#814) — so writing
|
|
6343
|
+
# one of the four leaves the other three NULL and the first rebuild
|
|
6344
|
+
# rejects the incarnation it just minted.
|
|
6345
|
+
if incarnation is None:
|
|
6346
|
+
incarnation = kernel.new_source_incarnation_id()
|
|
6347
|
+
conn.execute(
|
|
6348
|
+
"UPDATE conversation_source_files "
|
|
6349
|
+
"SET source_incarnation_id=?, device_id=?, inode=?, "
|
|
6350
|
+
"committed_prefix_sha256=? WHERE path=?",
|
|
6351
|
+
(incarnation, identity[0], identity[1], identity[2], path_str),
|
|
6352
|
+
)
|
|
6353
|
+
cursors[path_str] = (incarnation, committed)
|
|
6354
|
+
elif spans:
|
|
6355
|
+
# A message whose source-file row was pruned. Its bytes are
|
|
6356
|
+
# readable, but no cursor row exists to carry an identity, so a
|
|
6357
|
+
# rebuild would have nothing to check continuity against. Keep the
|
|
6358
|
+
# attribution as a classified gap rather than as a stamp nothing
|
|
6359
|
+
# can reach.
|
|
6360
|
+
spans = {}
|
|
6361
|
+
cause = cause or "no_source_file_cursor"
|
|
6362
|
+
for offset, account_key in attribution.items():
|
|
6363
|
+
span = spans.get(offset)
|
|
6364
|
+
if span is None or incarnation is None:
|
|
6365
|
+
conn.execute(
|
|
6366
|
+
"INSERT OR IGNORE INTO "
|
|
6367
|
+
"claude_conversation_account_stamp_gaps"
|
|
6368
|
+
"(source_path,byte_offset,cause,account_key) "
|
|
6369
|
+
"VALUES(?,?,?,?)",
|
|
6370
|
+
(path_str, offset,
|
|
6371
|
+
cause or offset_causes.get(offset)
|
|
6372
|
+
or "offset_past_end_of_file", account_key),
|
|
6373
|
+
)
|
|
6374
|
+
continue
|
|
6375
|
+
conn.execute(
|
|
6376
|
+
"INSERT OR IGNORE INTO claude_conversation_account_stamps"
|
|
6377
|
+
"(canonical_source_path,source_incarnation_id,byte_offset,"
|
|
6378
|
+
"record_sha256,account_key) VALUES(?,?,?,?,?)",
|
|
6379
|
+
(path_str, incarnation, offset,
|
|
6380
|
+
kernel.record_sha256(span), account_key),
|
|
6381
|
+
)
|
|
6382
|
+
pending += 1
|
|
6383
|
+
if pending >= _STAMP_BACKFILL_PATHS_PER_TXN:
|
|
6384
|
+
_set_cache_meta(conn, CLAUDE_STAMP_BACKFILL_CURSOR_KEY, path_str)
|
|
6385
|
+
conn.commit()
|
|
6386
|
+
pending = 0
|
|
6387
|
+
_set_cache_meta(conn, CLAUDE_STAMP_COVERAGE_COMPLETE_KEY, "1")
|
|
6388
|
+
conn.execute(
|
|
6389
|
+
"DELETE FROM cache_meta WHERE key=?",
|
|
6390
|
+
(CLAUDE_STAMP_BACKFILL_CURSOR_KEY,),
|
|
6391
|
+
)
|
|
6392
|
+
conn.commit()
|
|
6393
|
+
return True
|
|
6394
|
+
|
|
6395
|
+
|
|
6396
|
+
@conversations_migration("009_conversation_title_staging_and_account_stamps")
|
|
6397
|
+
def _conv_009_conversation_title_staging_and_account_stamps(
|
|
6398
|
+
conn: sqlite3.Connection,
|
|
6399
|
+
) -> None:
|
|
6400
|
+
"""Deliver #752's staging tables and #777's durable stamps, then backfill.
|
|
6401
|
+
|
|
6402
|
+
Takes the Claude provider flock and defers cleanly on contention, matching
|
|
6403
|
+
conversations migrations 003 and 005: the dispatcher holds only
|
|
6404
|
+
maintenance-shared admission, and this handler mutates staging, stamp and
|
|
6405
|
+
incarnation state that Claude ingestion writes concurrently.
|
|
6406
|
+
|
|
6407
|
+
The DDL is owned by `_apply_conversation_generation_schema`, the same
|
|
6408
|
+
helper the fresh-install schema apply calls, so a fresh store and a
|
|
6409
|
+
migrated 008 store converge on one shape and the object's delivery path is
|
|
6410
|
+
declared rather than incidental (#580).
|
|
6411
|
+
|
|
6412
|
+
It must not assume `conversation_title_fts` exists. Staging carries no FTS,
|
|
6413
|
+
so nothing here touches the index; the `fts5_unavailable` topology and the
|
|
6414
|
+
migration-018 pending-backfill window are therefore both no-ops for this
|
|
6415
|
+
handler rather than special cases.
|
|
6416
|
+
"""
|
|
6417
|
+
held = _acquire_conversations_db_claude_provider_flock(
|
|
6418
|
+
conn, migration="conversations 009 title staging and account stamps"
|
|
6419
|
+
)
|
|
6420
|
+
try:
|
|
6421
|
+
_apply_conversation_generation_schema(conn)
|
|
6422
|
+
conn.commit()
|
|
6423
|
+
backfill_claude_account_stamps(conn)
|
|
6424
|
+
finally:
|
|
6425
|
+
_release_cache_db_writer_flocks(held)
|
|
6426
|
+
|
|
6427
|
+
|
|
6428
|
+
@conversations_migration("010_codex_conversation_source_file_identity")
|
|
6429
|
+
def _conv_010_codex_conversation_source_file_identity(
|
|
6430
|
+
conn: sqlite3.Connection,
|
|
6431
|
+
) -> None:
|
|
6432
|
+
"""Deliver ``codex_conversation_source_files.device_id``/``.inode``.
|
|
6433
|
+
|
|
6434
|
+
#769 S6, #716 Task A. The conversations twin of cache migration
|
|
6435
|
+
``046_codex_source_file_identity``, and the same three facts apply: the
|
|
6436
|
+
schema apply that declares the columns is version-gated, so only a
|
|
6437
|
+
registered migration bumps the head and reaches an already-current store;
|
|
6438
|
+
the DDL is owned by ``_apply_codex_conversation_source_identity``, the
|
|
6439
|
+
helper the fresh-install apply also calls, so both shapes converge; and
|
|
6440
|
+
there is deliberately NO backfill, because a stat taken now cannot prove it
|
|
6441
|
+
describes the file the last transcript pass read.
|
|
6442
|
+
|
|
6443
|
+
Takes no provider flock. Unlike conversations 009 beside it, this handler
|
|
6444
|
+
only adds two columns, and ``ALTER TABLE ADD COLUMN`` with no default is a
|
|
6445
|
+
metadata-only change that rewrites no rows, so there is no window in which
|
|
6446
|
+
a concurrent Codex transcript writer could observe a partial rewrite.
|
|
6447
|
+
|
|
6448
|
+
Re-running is a no-op. NO self-stamp — the dispatcher central-stamps on a
|
|
6449
|
+
clean return (#140).
|
|
6450
|
+
"""
|
|
6451
|
+
if _table_exists(conn, "codex_conversation_source_files"):
|
|
6452
|
+
_apply_codex_conversation_source_identity(conn)
|
|
6453
|
+
conn.commit()
|
|
6454
|
+
|
|
6455
|
+
|
|
5831
6456
|
# #177 S6: the consolidated multi-column external-content FTS5 table that
|
|
5832
6457
|
# replaces the old conversation_fts(text) + conversation_fts_aux(search_aux)
|
|
5833
6458
|
# pair. The three column names MUST match the conversation_messages columns BY
|
|
@@ -8620,6 +9245,38 @@ def _045_conversation_render_revision_columns(conn: sqlite3.Connection) -> None:
|
|
|
8620
9245
|
conn.commit()
|
|
8621
9246
|
|
|
8622
9247
|
|
|
9248
|
+
@cache_migration("046_codex_source_file_identity")
|
|
9249
|
+
def _046_codex_source_file_identity(conn: sqlite3.Connection) -> None:
|
|
9250
|
+
"""Deliver ``codex_session_files.device_id``/``.inode`` (#769 S6, #716 A).
|
|
9251
|
+
|
|
9252
|
+
``_apply_cache_schema`` declares both columns through
|
|
9253
|
+
``_apply_codex_session_file_identity``, but that pass is VERSION-GATED:
|
|
9254
|
+
``open_cache_db`` runs it only when ``PRAGMA user_version`` differs from
|
|
9255
|
+
``len(_CACHE_MIGRATIONS)``. Registering here bumps the head, which is the
|
|
9256
|
+
mechanism migrations 029, 031, 044 and 045 rely on and is stated there in
|
|
9257
|
+
the same words. Without it an already-current store would never gain the
|
|
9258
|
+
two columns and the Codex ingest commit would fail on every tick.
|
|
9259
|
+
|
|
9260
|
+
NO BACKFILL, and that is a decision rather than an omission. The identity
|
|
9261
|
+
means "the pass that wrote this offset read this inode". A migration
|
|
9262
|
+
running now can stat the pathname, but it cannot know that the file it
|
|
9263
|
+
stats is the file the last ingest read, so writing that stat here would
|
|
9264
|
+
assert something unverified. NULL is the honest value: it says the
|
|
9265
|
+
identity is unknown, which is exactly the evidence every existing store
|
|
9266
|
+
has today, and the real value arrives on the next ingest of that path,
|
|
9267
|
+
where identity and offset are observed together.
|
|
9268
|
+
|
|
9269
|
+
``codex_session_files`` is checked for existence first for the same reason
|
|
9270
|
+
migration 045 checks its two tables: ``add_column_if_missing`` raises on a
|
|
9271
|
+
missing table, and a table created later is created by the schema apply,
|
|
9272
|
+
whose own body already carries the columns. Re-running is a no-op. NO
|
|
9273
|
+
self-stamp — the dispatcher central-stamps on a clean return (#140).
|
|
9274
|
+
"""
|
|
9275
|
+
if _table_exists(conn, "codex_session_files"):
|
|
9276
|
+
_apply_codex_session_file_identity(conn)
|
|
9277
|
+
conn.commit()
|
|
9278
|
+
|
|
9279
|
+
|
|
8623
9280
|
# === Region 7d: Stats migration 008_recompute_weekly_cost_snapshots_dedup_fix ===
|
|
8624
9281
|
|
|
8625
9282
|
@stats_migration("008_recompute_weekly_cost_snapshots_dedup_fix")
|
|
@@ -9036,6 +9693,19 @@ def _009_recompute_five_hour_blocks_dedup_fix(
|
|
|
9036
9693
|
block_id, window_key, block_start_at,
|
|
9037
9694
|
last_observed_at_utc,
|
|
9038
9695
|
) in block_rows:
|
|
9696
|
+
# #751a exemption, recorded deliberately. This walk is a
|
|
9697
|
+
# raw interval and is NOT routed through
|
|
9698
|
+
# `resolve_owning_window`, unlike the live writer and the
|
|
9699
|
+
# two child backfills. It is a one-shot historical
|
|
9700
|
+
# migration whose per-migration goldens are committed, and
|
|
9701
|
+
# the spec's route for correcting history already written
|
|
9702
|
+
# under the raw-interval rule is `claude-usage`
|
|
9703
|
+
# rederivation, not a re-run of this handler. Its
|
|
9704
|
+
# `[block_start, last_observed]` bounds are also the
|
|
9705
|
+
# closed-interval contract
|
|
9706
|
+
# `tests/test_migration_009_boundary_inclusive.py` pins,
|
|
9707
|
+
# which the ownership change must leave alone.
|
|
9708
|
+
#
|
|
9039
9709
|
# Walk session_entries over [block_start, last_observed]
|
|
9040
9710
|
# joined to session_files for project_path attribution.
|
|
9041
9711
|
# NULL session_files.project_path collapses to
|
|
@@ -9746,7 +10416,9 @@ def _db_status_for(
|
|
|
9746
10416
|
import _cctally_store
|
|
9747
10417
|
conn = _cctally_store.stats_open_guarded(
|
|
9748
10418
|
db_path,
|
|
9749
|
-
|
|
10419
|
+
# #778: forward the opener's `cached_statements=0`.
|
|
10420
|
+
connect=lambda p, **kw: sqlite3.connect(
|
|
10421
|
+
f"file:{p}?mode=rw", uri=True, **kw),
|
|
9750
10422
|
recover_interruptions=recover_interrupted_stats,
|
|
9751
10423
|
)
|
|
9752
10424
|
else:
|
|
@@ -11196,10 +11868,11 @@ def cmd_db_backup(args: argparse.Namespace) -> int:
|
|
|
11196
11868
|
# a long-lived reader, so it participates in the replacement
|
|
11197
11869
|
# protocol. The `mode=ro` open is preserved verbatim through the
|
|
11198
11870
|
# `connect` seam.
|
|
11199
|
-
|
|
11871
|
+
# #778: `**kw` carries the opener's `cached_statements=0`.
|
|
11872
|
+
def _backup_source_connect(_p, _timeout_ms=timeout_ms, **kw):
|
|
11200
11873
|
return sqlite3.connect(
|
|
11201
11874
|
f"file:{_p}?mode=ro", uri=True,
|
|
11202
|
-
timeout=max(_timeout_ms, 0) / 1000,
|
|
11875
|
+
timeout=max(_timeout_ms, 0) / 1000, **kw,
|
|
11203
11876
|
)
|
|
11204
11877
|
|
|
11205
11878
|
if which == "stats":
|