cctally 1.109.0 → 1.111.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +52 -0
- package/README.md +6 -6
- package/bin/_cctally_cache.py +62 -13
- package/bin/_cctally_core.py +8 -1
- package/bin/_cctally_dashboard.py +422 -195
- package/bin/_cctally_dashboard_conversation.py +231 -40
- package/bin/_cctally_dashboard_envelope.py +48 -32
- package/bin/_cctally_dashboard_sources.py +503 -109
- package/bin/_cctally_db.py +44 -0
- package/bin/_cctally_diagnosis_sources.py +181 -15
- package/bin/_cctally_doctor.py +15 -3
- package/bin/_cctally_forecast.py +25 -12
- package/bin/_cctally_journal.py +59 -32
- package/bin/_cctally_milestone_history.py +65 -9
- package/bin/_cctally_parser.py +5 -0
- package/bin/_cctally_record.py +50 -12
- package/bin/_cctally_rederive.py +899 -30
- package/bin/_cctally_source_analytics.py +567 -133
- package/bin/_cctally_transcript.py +27 -2
- package/bin/_cctally_tui.py +39 -6
- package/bin/_cctally_update.py +420 -14
- package/bin/_lib_cache_report.py +7 -10
- package/bin/_lib_codex_conversation_query.py +286 -63
- package/bin/_lib_codex_metadata.py +207 -0
- package/bin/_lib_conversation_query.py +254 -37
- package/bin/_lib_doctor.py +16 -2
- package/bin/_lib_pricing.py +80 -276
- package/bin/_lib_rederive.py +903 -2
- package/bin/_lib_view_models.py +5 -0
- package/bin/cctally +18 -5
- package/dashboard/static/assets/ConversationsView-BB9vUiL6.js +72 -0
- package/dashboard/static/assets/{DoctorModal-BZGXOlc2.js → DoctorModal-DFY2iPDf.js} +1 -1
- package/dashboard/static/assets/ModalRoot-DvRKHbQE.js +1 -0
- package/dashboard/static/assets/{ProjectsDrillPanel-KbO_KWa7.js → ProjectsDrillPanel-DAv39uSo.js} +1 -1
- package/dashboard/static/assets/{SourceDetailModal-BrOPLVRL.js → SourceDetailModal-DSAWo2Fp.js} +1 -1
- package/dashboard/static/assets/{UpdateModal-CYgEFAi4.js → UpdateModal-R_9dOVnh.js} +3 -3
- package/dashboard/static/assets/index-COhxe3vU.css +1 -0
- package/dashboard/static/assets/index-DdU62sov.js +13 -0
- package/dashboard/static/assets/outlineNavigation-BOCLcRa6.js +9 -0
- package/dashboard/static/assets/{useKeymap-DQx_A1MB.js → useKeymap-CW7Up0Ay.js} +1 -1
- package/dashboard/static/dashboard.html +3 -3
- package/package.json +2 -1
- package/dashboard/static/assets/ConversationsView-TO5bpIyc.js +0 -72
- package/dashboard/static/assets/ModalRoot-C_fvugTl.js +0 -1
- package/dashboard/static/assets/index-Cj_l7T7u.css +0 -1
- package/dashboard/static/assets/index-DLVwACUM.js +0 -13
- package/dashboard/static/assets/outlineNavigation-DXX-RjKl.js +0 -9
package/bin/_cctally_rederive.py
CHANGED
|
@@ -62,6 +62,9 @@ class RederivePreview:
|
|
|
62
62
|
# #374: the quarantined same-revision groups this plan will resolve by
|
|
63
63
|
# forcing a revision advance. Additive; empty on a clean journal.
|
|
64
64
|
journal_conflicts: tuple = ()
|
|
65
|
+
reviewed_decision_id: "str | None" = None
|
|
66
|
+
baseline_plan: "_lib_rederive.RederivePlan | None" = None
|
|
67
|
+
baseline_plan_guard: "dict | None" = None
|
|
65
68
|
|
|
66
69
|
|
|
67
70
|
@dataclass(frozen=True)
|
|
@@ -259,11 +262,407 @@ def _rederivable_raw_records(records: list[dict]) -> list[dict]:
|
|
|
259
262
|
continue
|
|
260
263
|
kind = (record.get("payload") or {}).get("kind")
|
|
261
264
|
if kind in {"weekly_credit_floor", "account_observe",
|
|
262
|
-
"account_label", "accounts_cutover", "sync_week"
|
|
265
|
+
"account_label", "accounts_cutover", "sync_week",
|
|
266
|
+
"claude_weekly_observation_decision"}:
|
|
263
267
|
out.append(record)
|
|
264
268
|
return out
|
|
265
269
|
|
|
266
270
|
|
|
271
|
+
_REVIEWED_WEEKLY_KIND = "claude_weekly_observation_decision"
|
|
272
|
+
_REVIEWED_MANIFEST_KEYS_V1 = frozenset({
|
|
273
|
+
"schemaVersion", "journalHighWater", "journalPrefixHash",
|
|
274
|
+
"reviewedAt", "reason", "decisions",
|
|
275
|
+
})
|
|
276
|
+
_REVIEWED_MANIFEST_KEYS_V2 = frozenset({
|
|
277
|
+
"schemaVersion", "journalHighWater", "journalPrefixHash",
|
|
278
|
+
"reviewedAt", "reason", "weeklyAxisDecisions",
|
|
279
|
+
"snapshotIdentityDecisions",
|
|
280
|
+
})
|
|
281
|
+
_REVIEWED_MANIFEST_EXPECTATION_KEYS = frozenset({
|
|
282
|
+
"expectedBaselinePlanHash", "expectedDecisionPlanHash",
|
|
283
|
+
})
|
|
284
|
+
_REVIEWED_OP_EXPECTATION_KEYS = frozenset({
|
|
285
|
+
"expected_baseline_plan_hash", "expected_decision_plan_hash",
|
|
286
|
+
})
|
|
287
|
+
_REVIEWED_OP_KEYS_V1 = frozenset({
|
|
288
|
+
"kind", "schema_version", "journal_high_water",
|
|
289
|
+
"journal_prefix_hash", "reason", "decisions",
|
|
290
|
+
}) | _REVIEWED_OP_EXPECTATION_KEYS
|
|
291
|
+
_REVIEWED_OP_KEYS_V2 = frozenset({
|
|
292
|
+
"kind", "schema_version", "journal_high_water",
|
|
293
|
+
"journal_prefix_hash", "reason", "weekly_axis_decisions",
|
|
294
|
+
"snapshot_identity_decisions",
|
|
295
|
+
}) | _REVIEWED_OP_EXPECTATION_KEYS
|
|
296
|
+
|
|
297
|
+
|
|
298
|
+
def _reviewed_weekly_op_from_manifest(path) -> dict:
|
|
299
|
+
try:
|
|
300
|
+
manifest = json.loads(Path(path).read_text(encoding="utf-8"))
|
|
301
|
+
except (ValueError, UnicodeError) as exc:
|
|
302
|
+
raise _lib_rederive.RederiveConflict(
|
|
303
|
+
f"invalid reviewed weekly decision manifest: {exc}"
|
|
304
|
+
) from exc
|
|
305
|
+
if not isinstance(manifest, dict):
|
|
306
|
+
raise _lib_rederive.RederiveConflict(
|
|
307
|
+
"reviewed weekly decision manifest has unexpected fields"
|
|
308
|
+
)
|
|
309
|
+
version = manifest.get("schemaVersion")
|
|
310
|
+
expected_keys = (
|
|
311
|
+
_REVIEWED_MANIFEST_KEYS_V1 if version == 1
|
|
312
|
+
else _REVIEWED_MANIFEST_KEYS_V2 if version == 2
|
|
313
|
+
else None
|
|
314
|
+
)
|
|
315
|
+
actual_keys = set(manifest)
|
|
316
|
+
if (expected_keys is None
|
|
317
|
+
or actual_keys not in (
|
|
318
|
+
expected_keys,
|
|
319
|
+
expected_keys | _REVIEWED_MANIFEST_EXPECTATION_KEYS,
|
|
320
|
+
)):
|
|
321
|
+
raise _lib_rederive.RederiveConflict(
|
|
322
|
+
"reviewed weekly decision manifest requires schemaVersion 1 or 2"
|
|
323
|
+
)
|
|
324
|
+
for key in _REVIEWED_MANIFEST_EXPECTATION_KEYS & actual_keys:
|
|
325
|
+
value = manifest[key]
|
|
326
|
+
if (not isinstance(value, str) or len(value) != 71
|
|
327
|
+
or not value.startswith("sha256:")
|
|
328
|
+
or any(ch not in "0123456789abcdef" for ch in value[7:])):
|
|
329
|
+
raise _lib_rederive.RederiveConflict(
|
|
330
|
+
f"reviewed weekly decision manifest has invalid {key}"
|
|
331
|
+
)
|
|
332
|
+
hw = manifest["journalHighWater"]
|
|
333
|
+
if (not isinstance(hw, dict) or set(hw) != {"segment", "offset"}
|
|
334
|
+
or not isinstance(hw["segment"], str) or not hw["segment"]
|
|
335
|
+
or type(hw["offset"]) is not int or hw["offset"] < 0):
|
|
336
|
+
raise _lib_rederive.RederiveConflict(
|
|
337
|
+
"reviewed weekly decision manifest has invalid journalHighWater"
|
|
338
|
+
)
|
|
339
|
+
digest = manifest["journalPrefixHash"]
|
|
340
|
+
if (not isinstance(digest, str) or len(digest) != 71
|
|
341
|
+
or not digest.startswith("sha256:")
|
|
342
|
+
or any(ch not in "0123456789abcdef" for ch in digest[7:])):
|
|
343
|
+
raise _lib_rederive.RederiveConflict(
|
|
344
|
+
"reviewed weekly decision manifest has invalid journalPrefixHash"
|
|
345
|
+
)
|
|
346
|
+
reviewed_at = manifest["reviewedAt"]
|
|
347
|
+
if not isinstance(reviewed_at, str):
|
|
348
|
+
raise _lib_rederive.RederiveConflict("reviewedAt must be UTC ISO-Z")
|
|
349
|
+
try:
|
|
350
|
+
parsed = dt.datetime.fromisoformat(reviewed_at.replace("Z", "+00:00"))
|
|
351
|
+
except ValueError as exc:
|
|
352
|
+
raise _lib_rederive.RederiveConflict(
|
|
353
|
+
"reviewedAt must be UTC ISO-Z"
|
|
354
|
+
) from exc
|
|
355
|
+
if (parsed.tzinfo is None or parsed.utcoffset() != dt.timedelta(0)
|
|
356
|
+
or parsed.isoformat(timespec="seconds").replace("+00:00", "Z")
|
|
357
|
+
!= reviewed_at):
|
|
358
|
+
raise _lib_rederive.RederiveConflict("reviewedAt must be UTC ISO-Z")
|
|
359
|
+
reason = manifest["reason"]
|
|
360
|
+
if not isinstance(reason, str) or not reason.strip() or reason != reason.strip():
|
|
361
|
+
raise _lib_rederive.RederiveConflict(
|
|
362
|
+
"reviewed weekly decision reason must be nonempty"
|
|
363
|
+
)
|
|
364
|
+
decisions = (
|
|
365
|
+
manifest["decisions"] if version == 1
|
|
366
|
+
else manifest["weeklyAxisDecisions"]
|
|
367
|
+
)
|
|
368
|
+
identity_decisions = (
|
|
369
|
+
[] if version == 1 else manifest["snapshotIdentityDecisions"]
|
|
370
|
+
)
|
|
371
|
+
if (not isinstance(decisions, list)
|
|
372
|
+
or not isinstance(identity_decisions, list)
|
|
373
|
+
or not decisions and not identity_decisions):
|
|
374
|
+
raise _lib_rederive.RederiveConflict(
|
|
375
|
+
"reviewed weekly decisions must select at least one decision"
|
|
376
|
+
)
|
|
377
|
+
seen = set()
|
|
378
|
+
normalized = []
|
|
379
|
+
for item in decisions:
|
|
380
|
+
if (not isinstance(item, dict)
|
|
381
|
+
or set(item) != {"observationId", "disposition"}
|
|
382
|
+
or not isinstance(item["observationId"], str)
|
|
383
|
+
or not item["observationId"]
|
|
384
|
+
or item["disposition"] not in ("hold", "accept")):
|
|
385
|
+
raise _lib_rederive.RederiveConflict(
|
|
386
|
+
"reviewed weekly decision entry is malformed"
|
|
387
|
+
)
|
|
388
|
+
obs_id = item["observationId"]
|
|
389
|
+
if obs_id in seen:
|
|
390
|
+
raise _lib_rederive.RederiveConflict(
|
|
391
|
+
f"duplicate reviewed weekly observation: {obs_id}"
|
|
392
|
+
)
|
|
393
|
+
seen.add(obs_id)
|
|
394
|
+
normalized.append({
|
|
395
|
+
"observationId": obs_id,
|
|
396
|
+
"disposition": item["disposition"],
|
|
397
|
+
})
|
|
398
|
+
identity_ids = set()
|
|
399
|
+
normalized_identity = []
|
|
400
|
+
for item in identity_decisions:
|
|
401
|
+
if (not isinstance(item, dict)
|
|
402
|
+
or set(item) != {
|
|
403
|
+
"acceptedObservationId", "replayObservationId",
|
|
404
|
+
"disposition",
|
|
405
|
+
}
|
|
406
|
+
or not all(
|
|
407
|
+
isinstance(item[key], str) and item[key]
|
|
408
|
+
for key in (
|
|
409
|
+
"acceptedObservationId", "replayObservationId",
|
|
410
|
+
)
|
|
411
|
+
)
|
|
412
|
+
or item["disposition"] not in ("preserve", "rederive")
|
|
413
|
+
or item["acceptedObservationId"] == item["replayObservationId"]
|
|
414
|
+
or item["acceptedObservationId"] in identity_ids
|
|
415
|
+
or item["replayObservationId"] in identity_ids):
|
|
416
|
+
raise _lib_rederive.RederiveConflict(
|
|
417
|
+
"snapshot identity decision is malformed, repeated, or ambiguous"
|
|
418
|
+
)
|
|
419
|
+
identity_ids.update((
|
|
420
|
+
item["acceptedObservationId"], item["replayObservationId"],
|
|
421
|
+
))
|
|
422
|
+
normalized_identity.append(dict(item))
|
|
423
|
+
payload = {
|
|
424
|
+
"kind": _REVIEWED_WEEKLY_KIND,
|
|
425
|
+
"schema_version": version,
|
|
426
|
+
"journal_high_water": dict(hw),
|
|
427
|
+
"journal_prefix_hash": digest,
|
|
428
|
+
"reason": reason,
|
|
429
|
+
}
|
|
430
|
+
if version == 1:
|
|
431
|
+
payload["decisions"] = sorted(
|
|
432
|
+
normalized, key=lambda item: item["observationId"])
|
|
433
|
+
else:
|
|
434
|
+
payload["weekly_axis_decisions"] = sorted(
|
|
435
|
+
normalized, key=lambda item: item["observationId"])
|
|
436
|
+
payload["snapshot_identity_decisions"] = sorted(
|
|
437
|
+
normalized_identity,
|
|
438
|
+
key=lambda item: (
|
|
439
|
+
item["acceptedObservationId"], item["replayObservationId"],
|
|
440
|
+
),
|
|
441
|
+
)
|
|
442
|
+
if _REVIEWED_MANIFEST_EXPECTATION_KEYS <= actual_keys:
|
|
443
|
+
payload.update({
|
|
444
|
+
"expected_baseline_plan_hash": (
|
|
445
|
+
manifest["expectedBaselinePlanHash"]
|
|
446
|
+
),
|
|
447
|
+
"expected_decision_plan_hash": (
|
|
448
|
+
manifest["expectedDecisionPlanHash"]
|
|
449
|
+
),
|
|
450
|
+
})
|
|
451
|
+
op = _lib_journal.make_op(
|
|
452
|
+
at=reviewed_at,
|
|
453
|
+
src="rederive",
|
|
454
|
+
payload=payload,
|
|
455
|
+
)
|
|
456
|
+
if len(_lib_journal.encode_line(op)) > _journal._MAX_LINE_BYTES:
|
|
457
|
+
raise _lib_rederive.RederiveConflict(
|
|
458
|
+
"reviewed weekly decision exceeds journal line limit"
|
|
459
|
+
)
|
|
460
|
+
return op
|
|
461
|
+
|
|
462
|
+
|
|
463
|
+
def _reviewed_weekly_expected_hashes(path, *, required: bool):
|
|
464
|
+
"""Read plan pins that a successful apply also writes into its op."""
|
|
465
|
+
try:
|
|
466
|
+
manifest = json.loads(Path(path).read_text(encoding="utf-8"))
|
|
467
|
+
except (ValueError, UnicodeError) as exc:
|
|
468
|
+
raise _lib_rederive.RederiveConflict(
|
|
469
|
+
f"invalid reviewed weekly decision manifest: {exc}"
|
|
470
|
+
) from exc
|
|
471
|
+
if not isinstance(manifest, dict):
|
|
472
|
+
raise _lib_rederive.RederiveConflict(
|
|
473
|
+
"reviewed weekly decision manifest has unexpected fields"
|
|
474
|
+
)
|
|
475
|
+
version = manifest.get("schemaVersion")
|
|
476
|
+
if version not in (1, 2):
|
|
477
|
+
return None
|
|
478
|
+
present = _REVIEWED_MANIFEST_EXPECTATION_KEYS & set(manifest)
|
|
479
|
+
if not present:
|
|
480
|
+
if required:
|
|
481
|
+
raise _lib_rederive.RederiveConflict(
|
|
482
|
+
f"schemaVersion {version} apply requires "
|
|
483
|
+
"expectedBaselinePlanHash and expectedDecisionPlanHash "
|
|
484
|
+
"from a reviewed preview"
|
|
485
|
+
)
|
|
486
|
+
return None
|
|
487
|
+
if present != _REVIEWED_MANIFEST_EXPECTATION_KEYS:
|
|
488
|
+
raise _lib_rederive.RederiveConflict(
|
|
489
|
+
f"schemaVersion {version} apply requires both reviewed plan hashes"
|
|
490
|
+
)
|
|
491
|
+
for key in present:
|
|
492
|
+
value = manifest[key]
|
|
493
|
+
if (not isinstance(value, str) or len(value) != 71
|
|
494
|
+
or not value.startswith("sha256:")
|
|
495
|
+
or any(ch not in "0123456789abcdef" for ch in value[7:])):
|
|
496
|
+
raise _lib_rederive.RederiveConflict(
|
|
497
|
+
f"reviewed weekly decision manifest has invalid {key}"
|
|
498
|
+
)
|
|
499
|
+
return (
|
|
500
|
+
manifest["expectedBaselinePlanHash"],
|
|
501
|
+
manifest["expectedDecisionPlanHash"],
|
|
502
|
+
)
|
|
503
|
+
|
|
504
|
+
|
|
505
|
+
def _reviewed_op_expected_hashes(record, *, required: bool):
|
|
506
|
+
payload = record.get("payload") or {}
|
|
507
|
+
present = _REVIEWED_OP_EXPECTATION_KEYS & set(payload)
|
|
508
|
+
if not present:
|
|
509
|
+
if required:
|
|
510
|
+
raise _lib_rederive.RederiveConflict(
|
|
511
|
+
"retained reviewed weekly decision lacks durable plan hashes"
|
|
512
|
+
)
|
|
513
|
+
return None
|
|
514
|
+
if present != _REVIEWED_OP_EXPECTATION_KEYS:
|
|
515
|
+
raise _lib_rederive.RederiveConflict(
|
|
516
|
+
"retained reviewed weekly decision has incomplete plan hashes"
|
|
517
|
+
)
|
|
518
|
+
values = (
|
|
519
|
+
payload["expected_baseline_plan_hash"],
|
|
520
|
+
payload["expected_decision_plan_hash"],
|
|
521
|
+
)
|
|
522
|
+
if any(
|
|
523
|
+
not isinstance(value, str)
|
|
524
|
+
or len(value) != 71
|
|
525
|
+
or not value.startswith("sha256:")
|
|
526
|
+
or any(ch not in "0123456789abcdef" for ch in value[7:])
|
|
527
|
+
for value in values
|
|
528
|
+
):
|
|
529
|
+
raise _lib_rederive.RederiveConflict(
|
|
530
|
+
"retained reviewed weekly decision has invalid plan hashes"
|
|
531
|
+
)
|
|
532
|
+
return values
|
|
533
|
+
|
|
534
|
+
|
|
535
|
+
def _reviewed_weekly_state(records):
|
|
536
|
+
raw_ids = {
|
|
537
|
+
record.get("id") for record in records
|
|
538
|
+
if record.get("t") == "obs"
|
|
539
|
+
and record.get("provider") == "claude"
|
|
540
|
+
and record.get("src") in _record._CLAUDE_OBS_SRCS
|
|
541
|
+
and "weekly_percent" in (record.get("payload") or {})
|
|
542
|
+
and "resets_at" in (record.get("payload") or {})
|
|
543
|
+
}
|
|
544
|
+
effective = {}
|
|
545
|
+
ops = []
|
|
546
|
+
for record in records:
|
|
547
|
+
if (record.get("t") != "op" or
|
|
548
|
+
(record.get("payload") or {}).get("kind") != _REVIEWED_WEEKLY_KIND):
|
|
549
|
+
continue
|
|
550
|
+
payload = record["payload"]
|
|
551
|
+
version = payload.get("schema_version")
|
|
552
|
+
decisions = (
|
|
553
|
+
payload.get("decisions") if version == 1
|
|
554
|
+
else payload.get("weekly_axis_decisions") if version == 2
|
|
555
|
+
else None
|
|
556
|
+
)
|
|
557
|
+
identity_decisions = (
|
|
558
|
+
[] if version == 1 else payload.get("snapshot_identity_decisions")
|
|
559
|
+
)
|
|
560
|
+
if (not isinstance(decisions, list)
|
|
561
|
+
or not isinstance(identity_decisions, list)
|
|
562
|
+
or not decisions and not identity_decisions):
|
|
563
|
+
raise _lib_rederive.RederiveConflict(
|
|
564
|
+
"malformed retained reviewed weekly decision op"
|
|
565
|
+
)
|
|
566
|
+
seen = set()
|
|
567
|
+
for item in decisions:
|
|
568
|
+
if (not isinstance(item, dict)
|
|
569
|
+
or set(item) != {"observationId", "disposition"}
|
|
570
|
+
or not isinstance(item["observationId"], str)
|
|
571
|
+
or item["disposition"] not in ("hold", "accept")):
|
|
572
|
+
raise _lib_rederive.RederiveConflict(
|
|
573
|
+
"malformed retained reviewed weekly decision entry"
|
|
574
|
+
)
|
|
575
|
+
obs_id = item["observationId"]
|
|
576
|
+
if obs_id not in raw_ids or obs_id in seen:
|
|
577
|
+
raise _lib_rederive.RederiveConflict(
|
|
578
|
+
f"unknown or duplicate reviewed weekly observation: {obs_id}"
|
|
579
|
+
)
|
|
580
|
+
seen.add(obs_id)
|
|
581
|
+
effective[obs_id] = item["disposition"]
|
|
582
|
+
identity_ids = set()
|
|
583
|
+
for item in identity_decisions:
|
|
584
|
+
if (not isinstance(item, dict)
|
|
585
|
+
or set(item) != {
|
|
586
|
+
"acceptedObservationId", "replayObservationId",
|
|
587
|
+
"disposition",
|
|
588
|
+
}
|
|
589
|
+
or not all(
|
|
590
|
+
isinstance(item.get(key), str) and item[key]
|
|
591
|
+
for key in (
|
|
592
|
+
"acceptedObservationId", "replayObservationId",
|
|
593
|
+
)
|
|
594
|
+
)
|
|
595
|
+
or item.get("disposition") not in (
|
|
596
|
+
"preserve", "rederive",
|
|
597
|
+
)
|
|
598
|
+
or item["acceptedObservationId"]
|
|
599
|
+
== item["replayObservationId"]
|
|
600
|
+
or item["acceptedObservationId"] in identity_ids
|
|
601
|
+
or item["replayObservationId"] in identity_ids):
|
|
602
|
+
raise _lib_rederive.RederiveConflict(
|
|
603
|
+
"malformed retained snapshot identity decision"
|
|
604
|
+
)
|
|
605
|
+
identity_ids.update((
|
|
606
|
+
item["acceptedObservationId"], item["replayObservationId"],
|
|
607
|
+
))
|
|
608
|
+
ops.append(record)
|
|
609
|
+
identity_effective = {}
|
|
610
|
+
for op in ops:
|
|
611
|
+
payload = op["payload"]
|
|
612
|
+
for item in payload.get("snapshot_identity_decisions", []):
|
|
613
|
+
identity_effective[
|
|
614
|
+
(item["acceptedObservationId"], item["replayObservationId"])
|
|
615
|
+
] = item["disposition"]
|
|
616
|
+
return (frozenset(
|
|
617
|
+
obs_id for obs_id, disposition in effective.items()
|
|
618
|
+
if disposition == "hold"
|
|
619
|
+
), frozenset(
|
|
620
|
+
obs_id for obs_id, disposition in effective.items()
|
|
621
|
+
if disposition == "accept"
|
|
622
|
+
), tuple(
|
|
623
|
+
pair for pair, disposition in identity_effective.items()
|
|
624
|
+
if disposition == "preserve"
|
|
625
|
+
), ops)
|
|
626
|
+
|
|
627
|
+
|
|
628
|
+
def _validate_retained_reviewed_op(record, prior_high_water, hasher) -> None:
|
|
629
|
+
"""Verify an operator decision against the bytes before its journal line."""
|
|
630
|
+
payload = record.get("payload")
|
|
631
|
+
if (not isinstance(payload, dict)
|
|
632
|
+
or payload.get("kind") != _REVIEWED_WEEKLY_KIND):
|
|
633
|
+
return
|
|
634
|
+
bound = payload.get("journal_high_water")
|
|
635
|
+
version = payload.get("schema_version")
|
|
636
|
+
expected_keys = (
|
|
637
|
+
_REVIEWED_OP_KEYS_V1 if version == 1
|
|
638
|
+
else _REVIEWED_OP_KEYS_V2 if version == 2
|
|
639
|
+
else None
|
|
640
|
+
)
|
|
641
|
+
if (expected_keys is None or set(payload) != expected_keys
|
|
642
|
+
or not isinstance(bound, dict)
|
|
643
|
+
or set(bound) != {"segment", "offset"}
|
|
644
|
+
or type(bound.get("segment")) is not str
|
|
645
|
+
or type(bound.get("offset")) is not int
|
|
646
|
+
or not isinstance(payload.get("journal_prefix_hash"), str)
|
|
647
|
+
or not isinstance(payload.get("reason"), str)
|
|
648
|
+
or not payload["reason"].strip()
|
|
649
|
+
or record != _lib_journal.make_op(
|
|
650
|
+
at=record.get("at"), src=record.get("src"), payload=payload,
|
|
651
|
+
)):
|
|
652
|
+
raise _lib_rederive.RederiveConflict(
|
|
653
|
+
"malformed retained reviewed weekly decision op"
|
|
654
|
+
)
|
|
655
|
+
_reviewed_op_expected_hashes(record, required=True)
|
|
656
|
+
expected = (bound["segment"], bound["offset"])
|
|
657
|
+
if prior_high_water != expected or hasher.digest_at(expected) != (
|
|
658
|
+
payload["journal_prefix_hash"]
|
|
659
|
+
):
|
|
660
|
+
raise _lib_rederive.RederiveConflict(
|
|
661
|
+
"retained reviewed weekly decision prefix does not match its "
|
|
662
|
+
"preceding journal bytes"
|
|
663
|
+
)
|
|
664
|
+
|
|
665
|
+
|
|
267
666
|
def _normalize_legacy_accounts(records: list[dict]) -> None:
|
|
268
667
|
import _lib_accounts
|
|
269
668
|
|
|
@@ -291,7 +690,8 @@ def _normalize_legacy_accounts(records: list[dict]) -> None:
|
|
|
291
690
|
|
|
292
691
|
|
|
293
692
|
def _derive_desired_events(records: list[dict], cache_conn: sqlite3.Connection,
|
|
294
|
-
scratch_dir: Path
|
|
693
|
+
scratch_dir: Path, *,
|
|
694
|
+
held_weekly_observation_ids=frozenset()) -> list[dict]:
|
|
295
695
|
# #386: this replays the whole ingest pipeline into a PRIVATE scratch index
|
|
296
696
|
# and is reached from `db rederive`'s PREVIEW, which takes no locks by
|
|
297
697
|
# design (its contract is zero persistent writes to the live family). The
|
|
@@ -304,12 +704,31 @@ def _derive_desired_events(records: list[dict], cache_conn: sqlite3.Connection,
|
|
|
304
704
|
scratch_path = scratch_dir / "stats.rederive.db"
|
|
305
705
|
with _cctally_store.stats_write_scope("rederive-derive"):
|
|
306
706
|
return _derive_desired_events_into(
|
|
307
|
-
records, cache_conn, str(scratch_path)
|
|
707
|
+
records, cache_conn, str(scratch_path),
|
|
708
|
+
held_weekly_observation_ids=held_weekly_observation_ids)
|
|
308
709
|
|
|
309
710
|
|
|
310
|
-
def _derive_desired_events_into(
|
|
711
|
+
def _derive_desired_events_into(
|
|
712
|
+
records, cache_conn, scratch_path, *,
|
|
713
|
+
held_weekly_observation_ids=frozenset(),
|
|
714
|
+
) -> list[dict]:
|
|
715
|
+
raw_records = _rederivable_raw_records(records)
|
|
716
|
+
held_ids = frozenset(held_weekly_observation_ids)
|
|
717
|
+
valid_ids = {
|
|
718
|
+
record.get("id") for record in raw_records
|
|
719
|
+
if record.get("t") == "obs"
|
|
720
|
+
and "weekly_percent" in (record.get("payload") or {})
|
|
721
|
+
and "resets_at" in (record.get("payload") or {})
|
|
722
|
+
}
|
|
723
|
+
unknown_ids = held_ids - valid_ids
|
|
724
|
+
if unknown_ids:
|
|
725
|
+
raise _lib_rederive.RederiveConflict(
|
|
726
|
+
"reviewed weekly hold names no Claude raw observation: "
|
|
727
|
+
+ ", ".join(sorted(unknown_ids))
|
|
728
|
+
)
|
|
311
729
|
conn = _cctally_core.open_db(_target_path=str(scratch_path))
|
|
312
730
|
events: list[dict] = []
|
|
731
|
+
reviewed_weekly_basis_by_account = {}
|
|
313
732
|
hooks = (
|
|
314
733
|
_journal._pipeline_op_fold,
|
|
315
734
|
_record._pipeline_claude_usage,
|
|
@@ -318,13 +737,16 @@ def _derive_desired_events_into(records, cache_conn, scratch_path) -> list[dict]
|
|
|
318
737
|
)
|
|
319
738
|
try:
|
|
320
739
|
with _scratch_read_adapters(cache_conn):
|
|
321
|
-
for record in
|
|
740
|
+
for record in raw_records:
|
|
322
741
|
ctx = _journal.IngestContext(
|
|
323
742
|
conn=conn,
|
|
324
743
|
batch=[record],
|
|
325
744
|
config={},
|
|
326
745
|
event_sink=events,
|
|
327
746
|
projection_writes=False,
|
|
747
|
+
held_weekly_observation_ids=held_ids,
|
|
748
|
+
reviewed_weekly_basis_by_account=(
|
|
749
|
+
reviewed_weekly_basis_by_account),
|
|
328
750
|
)
|
|
329
751
|
conn.execute("BEGIN IMMEDIATE")
|
|
330
752
|
try:
|
|
@@ -365,6 +787,7 @@ def plan_claude_usage(
|
|
|
365
787
|
cache_conn: sqlite3.Connection,
|
|
366
788
|
journal_high_water: "tuple[str, int] | None",
|
|
367
789
|
protocol_prefix_evidence=(),
|
|
790
|
+
enforce_guard=True,
|
|
368
791
|
):
|
|
369
792
|
"""Produce a deterministic Task-A-compatible plan without durable writes.
|
|
370
793
|
|
|
@@ -389,10 +812,33 @@ def plan_claude_usage(
|
|
|
389
812
|
raw_records = _rederivable_raw_records(records)
|
|
390
813
|
_validate_cache_rows(cache_conn, raw_records)
|
|
391
814
|
cache_fingerprint = _cache_fingerprint(cache_conn)
|
|
392
|
-
|
|
815
|
+
held_ids, _accepted_ids, identity_pairs, reviewed_ops = (
|
|
816
|
+
_reviewed_weekly_state(records)
|
|
817
|
+
)
|
|
818
|
+
config_basis = {
|
|
393
819
|
"historicalConfig": "not-retained",
|
|
394
820
|
"projectionPolicy": "retire-and-rematerialize",
|
|
395
|
-
}
|
|
821
|
+
}
|
|
822
|
+
if reviewed_ops:
|
|
823
|
+
config_basis["reviewedWeeklyDecisionOps"] = []
|
|
824
|
+
for op in reviewed_ops:
|
|
825
|
+
# Durable plan pins authenticate this derived plan. Feeding them
|
|
826
|
+
# back into its own config fingerprint would make the plan hash
|
|
827
|
+
# self-referential, so planning uses the decision's pin-free
|
|
828
|
+
# logical identity while the complete journal record remains
|
|
829
|
+
# content-addressed and validates the pins on recovery.
|
|
830
|
+
payload = {
|
|
831
|
+
key: value for key, value in op["payload"].items()
|
|
832
|
+
if key not in _REVIEWED_OP_EXPECTATION_KEYS
|
|
833
|
+
}
|
|
834
|
+
logical = _lib_journal.make_op(
|
|
835
|
+
at=op["at"], src=op["src"], payload=payload,
|
|
836
|
+
)
|
|
837
|
+
config_basis["reviewedWeeklyDecisionOps"].append({
|
|
838
|
+
"id": logical["id"], "at": logical["at"],
|
|
839
|
+
"payload": logical["payload"],
|
|
840
|
+
})
|
|
841
|
+
config_fingerprint = _fingerprint(config_basis)
|
|
396
842
|
selection = _lib_journal.resolve_effective_events(
|
|
397
843
|
records,
|
|
398
844
|
protocol_prefix_evidence=protocol_prefix_evidence,
|
|
@@ -410,7 +856,10 @@ def plan_claude_usage(
|
|
|
410
856
|
"journal contains tainted correction batch(es): " + summary
|
|
411
857
|
)
|
|
412
858
|
with tempfile.TemporaryDirectory(prefix="cctally-rederive-") as tmp:
|
|
413
|
-
desired = _derive_desired_events(
|
|
859
|
+
desired = _derive_desired_events(
|
|
860
|
+
records, cache_conn, Path(tmp),
|
|
861
|
+
held_weekly_observation_ids=held_ids,
|
|
862
|
+
)
|
|
414
863
|
# #426: the scratch replay only sees RETAINED sources, so it can never
|
|
415
864
|
# reproduce the pre-cutover rows the journal exported as `b:<table>:<rowid>`
|
|
416
865
|
# evt lines. Hold them out of the diff instead of retiring them. Evidence is
|
|
@@ -431,6 +880,10 @@ def plan_claude_usage(
|
|
|
431
880
|
config_fingerprint=config_fingerprint,
|
|
432
881
|
preserved_events=preserved.values(),
|
|
433
882
|
conflicted_event_ids=owned_conflicted_event_ids(selection),
|
|
883
|
+
raw_observations=raw_records,
|
|
884
|
+
snapshot_identity_decisions=identity_pairs,
|
|
885
|
+
reviewed_weekly_hold_ids=held_ids,
|
|
886
|
+
enforce_guard=enforce_guard,
|
|
434
887
|
)
|
|
435
888
|
|
|
436
889
|
|
|
@@ -503,6 +956,9 @@ def read_rederive_journal_prefix(
|
|
|
503
956
|
prior_high_water = record_end
|
|
504
957
|
continue
|
|
505
958
|
if record.get("t") == "op":
|
|
959
|
+
_validate_retained_reviewed_op(
|
|
960
|
+
record, prior_high_water, hasher,
|
|
961
|
+
)
|
|
506
962
|
_journal._capture_protocol_prefix_evidence(
|
|
507
963
|
record,
|
|
508
964
|
prior_high_water,
|
|
@@ -643,6 +1099,7 @@ def _preview_from_snapshot(
|
|
|
643
1099
|
family: str,
|
|
644
1100
|
*,
|
|
645
1101
|
journal_high_water: "tuple[str, int] | None" = None,
|
|
1102
|
+
reviewed_op: "dict | None" = None,
|
|
646
1103
|
) -> RederivePreview:
|
|
647
1104
|
if family != _lib_rederive.FAMILY:
|
|
648
1105
|
raise _lib_rederive.RederiveConflict(
|
|
@@ -656,18 +1113,116 @@ def _preview_from_snapshot(
|
|
|
656
1113
|
records, high_water, record_ends, protocol_evidence = (
|
|
657
1114
|
read_rederive_journal_prefix(journal_high_water)
|
|
658
1115
|
)
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
1116
|
+
planning_records = records
|
|
1117
|
+
baseline_records = None
|
|
1118
|
+
decision_op = reviewed_op
|
|
1119
|
+
decision_from_journal = False
|
|
1120
|
+
decision_batch_completed = False
|
|
1121
|
+
retained_decision = None
|
|
1122
|
+
retained_decision_index = None
|
|
1123
|
+
if reviewed_op is not None:
|
|
1124
|
+
bound = reviewed_op["payload"]["journal_high_water"]
|
|
1125
|
+
expected = (bound["segment"], bound["offset"])
|
|
1126
|
+
if high_water != expected:
|
|
1127
|
+
raise _lib_rederive.RederiveConflict(
|
|
1128
|
+
"reviewed weekly decision journalHighWater drifted"
|
|
1129
|
+
)
|
|
1130
|
+
if _journal.journal_prefix_hash(high_water) != (
|
|
1131
|
+
reviewed_op["payload"]["journal_prefix_hash"]
|
|
1132
|
+
):
|
|
1133
|
+
raise _lib_rederive.RederiveConflict(
|
|
1134
|
+
"reviewed weekly decision journalPrefixHash drifted"
|
|
1135
|
+
)
|
|
1136
|
+
planning_records = [*records, reviewed_op]
|
|
1137
|
+
baseline_records = records
|
|
1138
|
+
else:
|
|
1139
|
+
decision_indexes = [
|
|
1140
|
+
index for index, record in enumerate(records)
|
|
1141
|
+
if record.get("t") == "op"
|
|
1142
|
+
and (record.get("payload") or {}).get("kind")
|
|
1143
|
+
== _REVIEWED_WEEKLY_KIND
|
|
1144
|
+
]
|
|
1145
|
+
if decision_indexes:
|
|
1146
|
+
index = decision_indexes[-1]
|
|
1147
|
+
retained_decision = records[index]
|
|
1148
|
+
retained_decision_index = index
|
|
667
1149
|
selection = _lib_journal.resolve_effective_events(
|
|
668
1150
|
records,
|
|
669
1151
|
protocol_prefix_evidence=protocol_evidence,
|
|
670
1152
|
)
|
|
1153
|
+
if retained_decision is not None:
|
|
1154
|
+
suffix = records[retained_decision_index + 1:]
|
|
1155
|
+
first = suffix[0] if suffix else None
|
|
1156
|
+
completed_decision_batch = (
|
|
1157
|
+
first is not None
|
|
1158
|
+
and first.get("t") == "correction_batch"
|
|
1159
|
+
and first.get("phase") == "begin"
|
|
1160
|
+
and first.get("family") == family
|
|
1161
|
+
and first.get("id") in selection.completed_batches
|
|
1162
|
+
)
|
|
1163
|
+
correction_only_suffix = all(
|
|
1164
|
+
record.get("t") in {"correction", "correction_batch"}
|
|
1165
|
+
for record in suffix
|
|
1166
|
+
)
|
|
1167
|
+
if correction_only_suffix:
|
|
1168
|
+
decision_op = retained_decision
|
|
1169
|
+
decision_from_journal = not completed_decision_batch
|
|
1170
|
+
decision_batch_completed = completed_decision_batch
|
|
1171
|
+
baseline_records = [
|
|
1172
|
+
*records[:retained_decision_index],
|
|
1173
|
+
*records[retained_decision_index + 1:],
|
|
1174
|
+
]
|
|
1175
|
+
elif not completed_decision_batch:
|
|
1176
|
+
raise _lib_rederive.RederiveConflict(
|
|
1177
|
+
"unfinished reviewed weekly decision has new journal records "
|
|
1178
|
+
"after it; review a fresh manifest"
|
|
1179
|
+
)
|
|
1180
|
+
current_events = {
|
|
1181
|
+
event_id: selected.record
|
|
1182
|
+
for event_id, selected in selection.by_id.items()
|
|
1183
|
+
if selected.status == "active" and selected.record is not None
|
|
1184
|
+
}
|
|
1185
|
+
plan_high_water = high_water
|
|
1186
|
+
if decision_op is not None and not decision_batch_completed:
|
|
1187
|
+
bound = decision_op["payload"]["journal_high_water"]
|
|
1188
|
+
plan_high_water = (bound["segment"], bound["offset"])
|
|
1189
|
+
baseline_plan = None
|
|
1190
|
+
baseline_plan_guard = None
|
|
1191
|
+
with _open_cache_read_view() as cache:
|
|
1192
|
+
if decision_op is None:
|
|
1193
|
+
plan = plan_claude_usage(
|
|
1194
|
+
planning_records,
|
|
1195
|
+
cache_conn=cache,
|
|
1196
|
+
journal_high_water=high_water,
|
|
1197
|
+
protocol_prefix_evidence=protocol_evidence,
|
|
1198
|
+
)
|
|
1199
|
+
else:
|
|
1200
|
+
baseline_plan = plan_claude_usage(
|
|
1201
|
+
baseline_records,
|
|
1202
|
+
cache_conn=cache,
|
|
1203
|
+
journal_high_water=plan_high_water,
|
|
1204
|
+
protocol_prefix_evidence=protocol_evidence,
|
|
1205
|
+
enforce_guard=False,
|
|
1206
|
+
)
|
|
1207
|
+
baseline_plan_guard = _lib_rederive.week_reset_add_burst_guard(
|
|
1208
|
+
baseline_plan)
|
|
1209
|
+
reviewed_plan = plan_claude_usage(
|
|
1210
|
+
planning_records,
|
|
1211
|
+
cache_conn=cache,
|
|
1212
|
+
journal_high_water=plan_high_water,
|
|
1213
|
+
protocol_prefix_evidence=protocol_evidence,
|
|
1214
|
+
enforce_guard=False,
|
|
1215
|
+
)
|
|
1216
|
+
_held_ids, accepted_ids, _identity_pairs, _reviewed_ops = (
|
|
1217
|
+
_reviewed_weekly_state(planning_records)
|
|
1218
|
+
)
|
|
1219
|
+
plan = _lib_rederive.causal_delta_plan(
|
|
1220
|
+
baseline_plan, reviewed_plan,
|
|
1221
|
+
current_events=current_events,
|
|
1222
|
+
authorized_week_reset_origins=accepted_ids,
|
|
1223
|
+
)
|
|
1224
|
+
_lib_rederive.enforce_week_reset_add_burst_guard(plan)
|
|
1225
|
+
|
|
671
1226
|
batch_id = _batch_id_for_plan(plan)
|
|
672
1227
|
generated_at = _iso_now()
|
|
673
1228
|
incomplete = False
|
|
@@ -695,7 +1250,7 @@ def _preview_from_snapshot(
|
|
|
695
1250
|
conflict for conflict in selection.conflicts
|
|
696
1251
|
if conflict.event_id in owned_conflicts
|
|
697
1252
|
)
|
|
698
|
-
|
|
1253
|
+
preview = RederivePreview(
|
|
699
1254
|
journal_conflicts=journal_conflicts,
|
|
700
1255
|
plan=plan,
|
|
701
1256
|
records=tuple(records),
|
|
@@ -707,7 +1262,18 @@ def _preview_from_snapshot(
|
|
|
707
1262
|
latest_completed_batch=latest_id,
|
|
708
1263
|
latest_completed_high_water=latest_high_water,
|
|
709
1264
|
recovery_required=recovery_required,
|
|
1265
|
+
reviewed_decision_id=(
|
|
1266
|
+
None if decision_op is None else decision_op["id"]),
|
|
1267
|
+
baseline_plan=baseline_plan,
|
|
1268
|
+
baseline_plan_guard=baseline_plan_guard,
|
|
710
1269
|
)
|
|
1270
|
+
if decision_op is not None and not decision_batch_completed:
|
|
1271
|
+
expected_hashes = _reviewed_op_expected_hashes(
|
|
1272
|
+
decision_op, required=decision_from_journal,
|
|
1273
|
+
)
|
|
1274
|
+
if expected_hashes is not None:
|
|
1275
|
+
_assert_reviewed_plan_hashes(preview, expected_hashes)
|
|
1276
|
+
return preview
|
|
711
1277
|
|
|
712
1278
|
|
|
713
1279
|
@contextlib.contextmanager
|
|
@@ -891,12 +1457,13 @@ def preview_db_rederive(
|
|
|
891
1457
|
family: str,
|
|
892
1458
|
*,
|
|
893
1459
|
lock_timeout: float = _REDERIVE_LOCK_TIMEOUT_SECONDS,
|
|
1460
|
+
reviewed_op: "dict | None" = None,
|
|
894
1461
|
) -> RederivePreview:
|
|
895
1462
|
# Preview has a literal zero-persistent-write contract. A fixed append-only
|
|
896
1463
|
# journal prefix and a read-only SQLite transaction are stable inputs
|
|
897
1464
|
# without creating any coordination files.
|
|
898
1465
|
del lock_timeout
|
|
899
|
-
return _preview_from_snapshot(family)
|
|
1466
|
+
return _preview_from_snapshot(family, reviewed_op=reviewed_op)
|
|
900
1467
|
|
|
901
1468
|
|
|
902
1469
|
def _stats_has_batch(batch_id: str) -> bool:
|
|
@@ -928,13 +1495,214 @@ def _call_crash_hook(stage: str) -> None:
|
|
|
928
1495
|
os.kill(os.getpid(), signal.SIGKILL)
|
|
929
1496
|
|
|
930
1497
|
|
|
1498
|
+
def _reviewed_op_is_exact_retry(preview: RederivePreview, op: dict) -> bool:
|
|
1499
|
+
bound = op["payload"]["journal_high_water"]
|
|
1500
|
+
expected = (bound["segment"], bound["offset"])
|
|
1501
|
+
if _journal.journal_prefix_hash(expected) != (
|
|
1502
|
+
op["payload"]["journal_prefix_hash"]
|
|
1503
|
+
):
|
|
1504
|
+
return False
|
|
1505
|
+
matches = [
|
|
1506
|
+
index for index, record in enumerate(preview.records)
|
|
1507
|
+
if record.get("t") == "op" and record.get("id") == op["id"]
|
|
1508
|
+
]
|
|
1509
|
+
if len(matches) != 1:
|
|
1510
|
+
return False
|
|
1511
|
+
index = matches[0]
|
|
1512
|
+
return preview.records[index] == op and (
|
|
1513
|
+
preview.record_ends[index - 1] if index else None
|
|
1514
|
+
) == expected
|
|
1515
|
+
|
|
1516
|
+
|
|
1517
|
+
def _reviewed_op_needs_materialization(preview: RederivePreview) -> bool:
|
|
1518
|
+
last_index = None
|
|
1519
|
+
for index, record in enumerate(preview.records):
|
|
1520
|
+
if (record.get("t") == "op" and
|
|
1521
|
+
(record.get("payload") or {}).get("kind") == _REVIEWED_WEEKLY_KIND):
|
|
1522
|
+
last_index = index
|
|
1523
|
+
if last_index is None:
|
|
1524
|
+
return False
|
|
1525
|
+
try:
|
|
1526
|
+
with _open_sqlite_snapshot(
|
|
1527
|
+
_cctally_core.DB_PATH, prefix="cctally-rederive-stats-",
|
|
1528
|
+
) as conn:
|
|
1529
|
+
row = conn.execute(
|
|
1530
|
+
"SELECT segment, offset FROM journal_cursor WHERE id=1"
|
|
1531
|
+
).fetchone()
|
|
1532
|
+
except (OSError, sqlite3.Error):
|
|
1533
|
+
row = None
|
|
1534
|
+
if row is None:
|
|
1535
|
+
return True
|
|
1536
|
+
cursor = (str(row[0]), int(row[1]))
|
|
1537
|
+
if cursor == preview.journal_high_water:
|
|
1538
|
+
return False
|
|
1539
|
+
try:
|
|
1540
|
+
return preview.record_ends.index(cursor) < last_index
|
|
1541
|
+
except ValueError:
|
|
1542
|
+
return True
|
|
1543
|
+
|
|
1544
|
+
|
|
1545
|
+
def _refuse_earlier_baseline_actions(
|
|
1546
|
+
family: str, preview: RederivePreview, reviewed_op: dict,
|
|
1547
|
+
) -> None:
|
|
1548
|
+
"""Keep pre-existing corrections outside the exact decision's history.
|
|
1549
|
+
|
|
1550
|
+
A reviewed weekly decision can remove a reset-burst guard without causing
|
|
1551
|
+
older rederive drift. Such drift must be resolved separately before the
|
|
1552
|
+
reviewed operation is appended; it is not part of the operator's selected
|
|
1553
|
+
observations. The baseline plan may itself trip the burst guard, but that
|
|
1554
|
+
exception still carries its complete proposed action set.
|
|
1555
|
+
"""
|
|
1556
|
+
selected_ids = {
|
|
1557
|
+
item["observationId"]
|
|
1558
|
+
for item in reviewed_op["payload"]["decisions"]
|
|
1559
|
+
}
|
|
1560
|
+
selected_at = [
|
|
1561
|
+
dt.datetime.fromisoformat(record["at"].replace("Z", "+00:00"))
|
|
1562
|
+
for record in preview.records
|
|
1563
|
+
if record.get("t") == "obs" and record.get("id") in selected_ids
|
|
1564
|
+
]
|
|
1565
|
+
if len(selected_at) != len(selected_ids):
|
|
1566
|
+
raise _lib_rederive.RederiveConflict(
|
|
1567
|
+
"reviewed weekly decision has unknown selected observations"
|
|
1568
|
+
)
|
|
1569
|
+
first_selected_at = min(selected_at)
|
|
1570
|
+
try:
|
|
1571
|
+
baseline = _preview_from_snapshot(family)
|
|
1572
|
+
baseline_plan = baseline.plan
|
|
1573
|
+
except _lib_rederive.RederivePlanGuardConflict as exc:
|
|
1574
|
+
baseline_plan = exc.plan
|
|
1575
|
+
baseline_actions = {
|
|
1576
|
+
action.event_id: action.payload_hash
|
|
1577
|
+
for action in baseline_plan.actions
|
|
1578
|
+
}
|
|
1579
|
+
unchanged_earlier = [
|
|
1580
|
+
action for action in preview.plan.actions
|
|
1581
|
+
if action.disposition != "add"
|
|
1582
|
+
and baseline_actions.get(action.event_id) == action.payload_hash
|
|
1583
|
+
and dt.datetime.fromisoformat(action.at.replace("Z", "+00:00"))
|
|
1584
|
+
< first_selected_at
|
|
1585
|
+
]
|
|
1586
|
+
if unchanged_earlier:
|
|
1587
|
+
raise _lib_rederive.RederiveConflict(
|
|
1588
|
+
"reviewed weekly correction contains pre-existing actions "
|
|
1589
|
+
"earlier than its selected observations; resolve baseline "
|
|
1590
|
+
"rederive drift first"
|
|
1591
|
+
)
|
|
1592
|
+
|
|
1593
|
+
|
|
1594
|
+
def _assert_reviewed_plan_hashes(preview, expected_plan_hashes) -> None:
|
|
1595
|
+
if expected_plan_hashes is None:
|
|
1596
|
+
return
|
|
1597
|
+
if preview.baseline_plan is None:
|
|
1598
|
+
raise _lib_rederive.RederiveConflict(
|
|
1599
|
+
"reviewed weekly decision has no baseline plan to verify"
|
|
1600
|
+
)
|
|
1601
|
+
expected_baseline, expected_decision = expected_plan_hashes
|
|
1602
|
+
if preview.baseline_plan.plan_hash != expected_baseline:
|
|
1603
|
+
raise _lib_rederive.RederiveConflict(
|
|
1604
|
+
"reviewed weekly baseline plan hash drifted: expected "
|
|
1605
|
+
f"{expected_baseline}, got {preview.baseline_plan.plan_hash}"
|
|
1606
|
+
)
|
|
1607
|
+
if preview.plan.plan_hash != expected_decision:
|
|
1608
|
+
raise _lib_rederive.RederiveConflict(
|
|
1609
|
+
"reviewed weekly decision plan hash drifted: expected "
|
|
1610
|
+
f"{expected_decision}, got {preview.plan.plan_hash}"
|
|
1611
|
+
)
|
|
1612
|
+
|
|
1613
|
+
|
|
931
1614
|
def apply_db_rederive(
|
|
932
1615
|
family: str,
|
|
933
1616
|
*,
|
|
934
1617
|
lock_timeout: float = _REDERIVE_LOCK_TIMEOUT_SECONDS,
|
|
1618
|
+
reviewed_op: "dict | None" = None,
|
|
1619
|
+
expected_plan_hashes: "tuple[str, str] | None" = None,
|
|
935
1620
|
) -> RederiveCommandResult:
|
|
936
1621
|
with _rederive_locks(apply=True, timeout=lock_timeout):
|
|
937
|
-
|
|
1622
|
+
appended_reviewed_op = False
|
|
1623
|
+
if reviewed_op is not None:
|
|
1624
|
+
durable_hashes = _reviewed_op_expected_hashes(
|
|
1625
|
+
reviewed_op, required=True,
|
|
1626
|
+
)
|
|
1627
|
+
if expected_plan_hashes != durable_hashes:
|
|
1628
|
+
raise _lib_rederive.RederiveConflict(
|
|
1629
|
+
"reviewed weekly apply hashes do not match its durable op"
|
|
1630
|
+
)
|
|
1631
|
+
if reviewed_op is None:
|
|
1632
|
+
preview = _preview_from_snapshot(family)
|
|
1633
|
+
report_preview = preview
|
|
1634
|
+
else:
|
|
1635
|
+
try:
|
|
1636
|
+
report_preview = _preview_from_snapshot(
|
|
1637
|
+
family, reviewed_op=reviewed_op)
|
|
1638
|
+
except _lib_rederive.RederiveConflict as exc:
|
|
1639
|
+
if "journalHighWater drifted" not in str(exc):
|
|
1640
|
+
raise
|
|
1641
|
+
preview = _preview_from_snapshot(family)
|
|
1642
|
+
if not _reviewed_op_is_exact_retry(preview, reviewed_op):
|
|
1643
|
+
raise exc
|
|
1644
|
+
bound = reviewed_op["payload"]["journal_high_water"]
|
|
1645
|
+
original = _preview_from_snapshot(
|
|
1646
|
+
family,
|
|
1647
|
+
journal_high_water=(bound["segment"], bound["offset"]),
|
|
1648
|
+
reviewed_op=reviewed_op,
|
|
1649
|
+
)
|
|
1650
|
+
op_index = next(
|
|
1651
|
+
index for index, record in enumerate(preview.records)
|
|
1652
|
+
if record.get("t") == "op"
|
|
1653
|
+
and record.get("id") == reviewed_op["id"]
|
|
1654
|
+
)
|
|
1655
|
+
if any(
|
|
1656
|
+
not (
|
|
1657
|
+
(record.get("t") == "correction_batch"
|
|
1658
|
+
and record.get("id") == original.batch_id)
|
|
1659
|
+
or (record.get("t") == "correction"
|
|
1660
|
+
and record.get("batch") == original.batch_id)
|
|
1661
|
+
)
|
|
1662
|
+
for record in preview.records[op_index + 1:]
|
|
1663
|
+
):
|
|
1664
|
+
raise _lib_rederive.RederiveConflict(
|
|
1665
|
+
"reviewed weekly decision retry has new journal records "
|
|
1666
|
+
"after its decision; review a fresh manifest"
|
|
1667
|
+
)
|
|
1668
|
+
original_actions = original.plan.to_correction_actions()
|
|
1669
|
+
current_actions = preview.plan.to_correction_actions()
|
|
1670
|
+
if current_actions != original_actions and not (
|
|
1671
|
+
not current_actions
|
|
1672
|
+
and original.batch_id is not None
|
|
1673
|
+
and preview.latest_completed_batch == original.batch_id
|
|
1674
|
+
):
|
|
1675
|
+
raise _lib_rederive.RederiveConflict(
|
|
1676
|
+
"reviewed weekly decision retry changed correction "
|
|
1677
|
+
"actions"
|
|
1678
|
+
)
|
|
1679
|
+
report_preview = original
|
|
1680
|
+
_assert_reviewed_plan_hashes(
|
|
1681
|
+
report_preview, expected_plan_hashes)
|
|
1682
|
+
else:
|
|
1683
|
+
_assert_reviewed_plan_hashes(
|
|
1684
|
+
report_preview, expected_plan_hashes)
|
|
1685
|
+
try:
|
|
1686
|
+
op_high_water = _journal.append_records(
|
|
1687
|
+
[reviewed_op],
|
|
1688
|
+
expected_high_water=report_preview.journal_high_water,
|
|
1689
|
+
)
|
|
1690
|
+
except Exception as exc:
|
|
1691
|
+
raise RederiveApplyError(
|
|
1692
|
+
"reviewed decision append", report_preview,
|
|
1693
|
+
report_preview.batch_id, exc,
|
|
1694
|
+
) from exc
|
|
1695
|
+
appended_reviewed_op = True
|
|
1696
|
+
_call_crash_hook("after-reviewed-decision-op")
|
|
1697
|
+
preview = _preview_from_snapshot(
|
|
1698
|
+
family, journal_high_water=op_high_water)
|
|
1699
|
+
if (preview.plan.to_correction_actions()
|
|
1700
|
+
!= report_preview.plan.to_correction_actions()):
|
|
1701
|
+
raise RederiveApplyError(
|
|
1702
|
+
"reviewed decision revalidation", report_preview,
|
|
1703
|
+
report_preview.batch_id,
|
|
1704
|
+
"scratch actions changed after decision append",
|
|
1705
|
+
)
|
|
938
1706
|
plan = preview.plan
|
|
939
1707
|
recovering_prior = (
|
|
940
1708
|
preview.latest_completed_batch is not None
|
|
@@ -981,7 +1749,7 @@ def apply_db_rederive(
|
|
|
981
1749
|
else "applied"
|
|
982
1750
|
)
|
|
983
1751
|
return RederiveCommandResult(
|
|
984
|
-
preview=
|
|
1752
|
+
preview=report_preview,
|
|
985
1753
|
status=status,
|
|
986
1754
|
batch_id=preview.batch_id,
|
|
987
1755
|
rebuild=result,
|
|
@@ -1010,13 +1778,31 @@ def apply_db_rederive(
|
|
|
1010
1778
|
"stats recovery", preview, latest, exc
|
|
1011
1779
|
) from exc
|
|
1012
1780
|
return RederiveCommandResult(
|
|
1013
|
-
preview=
|
|
1781
|
+
preview=report_preview,
|
|
1014
1782
|
status="recovered",
|
|
1015
1783
|
batch_id=latest,
|
|
1016
1784
|
rebuild=result,
|
|
1017
1785
|
)
|
|
1786
|
+
if appended_reviewed_op or _reviewed_op_needs_materialization(preview):
|
|
1787
|
+
try:
|
|
1788
|
+
result = _journal.rebuild_stats_index(
|
|
1789
|
+
context=_journal.RebuildContext(trigger="rederive-recovery"),
|
|
1790
|
+
high_water=preview.journal_high_water,
|
|
1791
|
+
update_quota_cache=False,
|
|
1792
|
+
)
|
|
1793
|
+
except Exception as exc:
|
|
1794
|
+
raise RederiveApplyError(
|
|
1795
|
+
"reviewed decision materialization", report_preview,
|
|
1796
|
+
latest, exc,
|
|
1797
|
+
) from exc
|
|
1798
|
+
return RederiveCommandResult(
|
|
1799
|
+
preview=report_preview,
|
|
1800
|
+
status="applied" if appended_reviewed_op else "recovered",
|
|
1801
|
+
batch_id=latest,
|
|
1802
|
+
rebuild=result,
|
|
1803
|
+
)
|
|
1018
1804
|
return RederiveCommandResult(
|
|
1019
|
-
preview=
|
|
1805
|
+
preview=report_preview,
|
|
1020
1806
|
status="no-op",
|
|
1021
1807
|
batch_id=latest,
|
|
1022
1808
|
rebuild=None,
|
|
@@ -1052,8 +1838,11 @@ def _command_payload(
|
|
|
1052
1838
|
errors=(),
|
|
1053
1839
|
family: str = _lib_rederive.FAMILY,
|
|
1054
1840
|
journal_high_water=None,
|
|
1841
|
+
guarded_plan: "_lib_rederive.RederivePlan | None" = None,
|
|
1842
|
+
plan_guard=None,
|
|
1843
|
+
reviewed_decision_id=None,
|
|
1055
1844
|
):
|
|
1056
|
-
plan =
|
|
1845
|
+
plan = guarded_plan if preview is None else preview.plan
|
|
1057
1846
|
counts = (
|
|
1058
1847
|
{"retain": 0, "supersede": 0, "tombstone": 0, "add": 0}
|
|
1059
1848
|
if plan is None else dict(plan.counts)
|
|
@@ -1069,6 +1858,14 @@ def _command_payload(
|
|
|
1069
1858
|
"batchId": batch_id,
|
|
1070
1859
|
"planHash": None if plan is None else plan.plan_hash,
|
|
1071
1860
|
"actionCounts": counts,
|
|
1861
|
+
"actionCountsByEventKind": (
|
|
1862
|
+
{
|
|
1863
|
+
kind: {name: 0 for name in counts}
|
|
1864
|
+
for kind in sorted(_lib_rederive._EVT_CLASSIFICATIONS)
|
|
1865
|
+
}
|
|
1866
|
+
if plan is None else plan.action_counts_by_event_kind
|
|
1867
|
+
),
|
|
1868
|
+
"planGuard": plan_guard,
|
|
1072
1869
|
# #426: how many owned events the plan held OUT of the diff because no
|
|
1073
1870
|
# retained source can re-derive them (pre-cutover exported history).
|
|
1074
1871
|
"preservedEventCount": 0 if plan is None else plan.preserved_event_count,
|
|
@@ -1086,6 +1883,14 @@ def _command_payload(
|
|
|
1086
1883
|
"rebuild": _rebuild_dict(rebuild),
|
|
1087
1884
|
"noOp": status == "no-op",
|
|
1088
1885
|
}
|
|
1886
|
+
if reviewed_decision_id is not None:
|
|
1887
|
+
body["reviewedWeeklyDecisionId"] = reviewed_decision_id
|
|
1888
|
+
if preview is not None and preview.baseline_plan is not None:
|
|
1889
|
+
body["baselinePlanHash"] = preview.baseline_plan.plan_hash
|
|
1890
|
+
body["baselineActionCounts"] = dict(preview.baseline_plan.counts)
|
|
1891
|
+
body["baselinePlanGuard"] = preview.baseline_plan_guard
|
|
1892
|
+
body["decisionPlanHash"] = preview.plan.plan_hash
|
|
1893
|
+
body["decisionActionCounts"] = dict(preview.plan.counts)
|
|
1089
1894
|
return _lib_json_envelope.stamp_schema_version(body, version=1)
|
|
1090
1895
|
|
|
1091
1896
|
|
|
@@ -1111,10 +1916,16 @@ def _emit_command_payload(payload: dict, *, as_json: bool) -> None:
|
|
|
1111
1916
|
f"{actions} correction action(s); no changes written."
|
|
1112
1917
|
)
|
|
1113
1918
|
elif status == "applied":
|
|
1114
|
-
|
|
1115
|
-
|
|
1116
|
-
|
|
1117
|
-
|
|
1919
|
+
if payload["batchId"] is None:
|
|
1920
|
+
print(
|
|
1921
|
+
f"cctally: applied {payload['family']} reviewed decision "
|
|
1922
|
+
"and rebuilt stats.db."
|
|
1923
|
+
)
|
|
1924
|
+
else:
|
|
1925
|
+
print(
|
|
1926
|
+
f"cctally: applied {payload['family']} correction batch "
|
|
1927
|
+
f"{payload['batchId']} and rebuilt stats.db."
|
|
1928
|
+
)
|
|
1118
1929
|
elif status == "recovered":
|
|
1119
1930
|
print(
|
|
1120
1931
|
f"cctally: recovered {payload['family']} correction batch "
|
|
@@ -1125,6 +1936,30 @@ def _emit_command_payload(payload: dict, *, as_json: bool) -> None:
|
|
|
1125
1936
|
f"cctally: {payload['family']} is already current; "
|
|
1126
1937
|
"no correction batch was appended."
|
|
1127
1938
|
)
|
|
1939
|
+
if "baselinePlanHash" in payload:
|
|
1940
|
+
def _counts_text(counts):
|
|
1941
|
+
return ", ".join(
|
|
1942
|
+
f"{name}={counts[name]}"
|
|
1943
|
+
for name in ("retain", "supersede", "tombstone", "add")
|
|
1944
|
+
)
|
|
1945
|
+
print(f"baseline plan {payload['baselinePlanHash']}")
|
|
1946
|
+
print(
|
|
1947
|
+
"baseline action counts: "
|
|
1948
|
+
+ _counts_text(payload["baselineActionCounts"])
|
|
1949
|
+
)
|
|
1950
|
+
if payload.get("baselinePlanGuard") is not None:
|
|
1951
|
+
print(
|
|
1952
|
+
"baseline plan guard: "
|
|
1953
|
+
+ json.dumps(
|
|
1954
|
+
payload["baselinePlanGuard"],
|
|
1955
|
+
sort_keys=True, separators=(",", ":"),
|
|
1956
|
+
)
|
|
1957
|
+
)
|
|
1958
|
+
print(f"decision plan {payload['decisionPlanHash']}")
|
|
1959
|
+
print(
|
|
1960
|
+
"decision action counts: "
|
|
1961
|
+
+ _counts_text(payload["decisionActionCounts"])
|
|
1962
|
+
)
|
|
1128
1963
|
|
|
1129
1964
|
|
|
1130
1965
|
def cmd_db_rederive(args) -> int:
|
|
@@ -1179,19 +2014,37 @@ def cmd_db_rederive(args) -> int:
|
|
|
1179
2014
|
print(f"cctally: db rederive: {message}", file=sys.stderr)
|
|
1180
2015
|
return 2
|
|
1181
2016
|
try:
|
|
2017
|
+
reviewed_path = getattr(args, "reviewed_weekly_decisions", None)
|
|
2018
|
+
reviewed_op = (
|
|
2019
|
+
None if reviewed_path is None
|
|
2020
|
+
else _reviewed_weekly_op_from_manifest(reviewed_path)
|
|
2021
|
+
)
|
|
2022
|
+
expected_plan_hashes = (
|
|
2023
|
+
None if reviewed_path is None
|
|
2024
|
+
else _reviewed_weekly_expected_hashes(
|
|
2025
|
+
reviewed_path, required=apply,
|
|
2026
|
+
)
|
|
2027
|
+
)
|
|
1182
2028
|
if apply:
|
|
1183
|
-
result = apply_db_rederive(
|
|
2029
|
+
result = apply_db_rederive(
|
|
2030
|
+
family,
|
|
2031
|
+
reviewed_op=reviewed_op,
|
|
2032
|
+
expected_plan_hashes=expected_plan_hashes,
|
|
2033
|
+
)
|
|
1184
2034
|
payload = _command_payload(
|
|
1185
2035
|
status=result.status,
|
|
1186
2036
|
preview=result.preview,
|
|
1187
2037
|
batch_id=result.batch_id,
|
|
1188
2038
|
rebuild=result.rebuild,
|
|
2039
|
+
reviewed_decision_id=(
|
|
2040
|
+
None if reviewed_op is None else reviewed_op["id"]),
|
|
1189
2041
|
)
|
|
1190
2042
|
else:
|
|
1191
|
-
preview = preview_db_rederive(family)
|
|
2043
|
+
preview = preview_db_rederive(family, reviewed_op=reviewed_op)
|
|
1192
2044
|
status = (
|
|
1193
2045
|
"preview"
|
|
1194
|
-
if preview.plan.actions
|
|
2046
|
+
if reviewed_op is not None or preview.plan.actions
|
|
2047
|
+
or preview.recovery_required
|
|
1195
2048
|
else "no-op"
|
|
1196
2049
|
)
|
|
1197
2050
|
payload = _command_payload(
|
|
@@ -1202,6 +2055,7 @@ def cmd_db_rederive(args) -> int:
|
|
|
1202
2055
|
if status == "no-op" or preview.recovery_required
|
|
1203
2056
|
else preview.batch_id
|
|
1204
2057
|
),
|
|
2058
|
+
reviewed_decision_id=preview.reviewed_decision_id,
|
|
1205
2059
|
)
|
|
1206
2060
|
except _lib_rederive.RederiveDataGap as exc:
|
|
1207
2061
|
payload = _command_payload(
|
|
@@ -1215,6 +2069,21 @@ def cmd_db_rederive(args) -> int:
|
|
|
1215
2069
|
else:
|
|
1216
2070
|
print(f"cctally: db rederive missing source: {exc}", file=sys.stderr)
|
|
1217
2071
|
return 2
|
|
2072
|
+
except _lib_rederive.RederivePlanGuardConflict as exc:
|
|
2073
|
+
payload = _command_payload(
|
|
2074
|
+
status="conflict",
|
|
2075
|
+
family=family,
|
|
2076
|
+
journal_high_water=exc.plan.journal_high_water,
|
|
2077
|
+
batch_id=_batch_id_for_plan(exc.plan),
|
|
2078
|
+
guarded_plan=exc.plan,
|
|
2079
|
+
plan_guard=exc.plan_guard,
|
|
2080
|
+
conflicts=(str(exc),),
|
|
2081
|
+
)
|
|
2082
|
+
if as_json:
|
|
2083
|
+
_emit_command_payload(payload, as_json=True)
|
|
2084
|
+
else:
|
|
2085
|
+
print(f"cctally: db rederive conflict: {exc}", file=sys.stderr)
|
|
2086
|
+
return 2
|
|
1218
2087
|
except (_lib_rederive.RederiveConflict,
|
|
1219
2088
|
_lib_journal.JournalProtocolError) as exc:
|
|
1220
2089
|
payload = _command_payload(
|