cctally 1.115.0 → 1.116.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. package/CHANGELOG.md +16 -0
  2. package/bin/_cctally_cache.py +85 -4
  3. package/bin/_cctally_completion.py +80 -12
  4. package/bin/_cctally_core.py +106 -9
  5. package/bin/_cctally_dashboard.py +6 -9
  6. package/bin/_cctally_dashboard_envelope.py +26 -32
  7. package/bin/_cctally_db.py +85 -4
  8. package/bin/_cctally_doctor.py +38 -5
  9. package/bin/_cctally_five_hour.py +83 -1017
  10. package/bin/_cctally_journal.py +1307 -180
  11. package/bin/_cctally_journal_repair.py +9 -7
  12. package/bin/_cctally_oauth_samples.py +280 -0
  13. package/bin/_cctally_parser.py +5 -0
  14. package/bin/_cctally_quarantine.py +14 -4
  15. package/bin/_cctally_quota.py +70 -8
  16. package/bin/_cctally_quota_model.py +35 -4
  17. package/bin/_cctally_record.py +414 -123
  18. package/bin/_cctally_rederive.py +632 -235
  19. package/bin/_cctally_refresh.py +29 -0
  20. package/bin/_cctally_statusline.py +984 -47
  21. package/bin/_cctally_store.py +972 -58
  22. package/bin/_cctally_tui.py +62 -7
  23. package/bin/_cctally_update.py +25 -0
  24. package/bin/_lib_account_protocol.py +201 -18
  25. package/bin/_lib_clone_copy.py +46 -11
  26. package/bin/_lib_dashboard_sources.py +57 -15
  27. package/bin/_lib_detector_access_registry.py +273 -52
  28. package/bin/_lib_doctor.py +302 -1
  29. package/bin/_lib_journal.py +145 -25
  30. package/bin/_lib_journal_router.py +166 -48
  31. package/bin/_lib_oauth_samples.py +238 -0
  32. package/bin/_lib_pricing_check.py +0 -12
  33. package/bin/_lib_provider_activity.py +770 -0
  34. package/bin/_lib_quarantine.py +3 -2
  35. package/bin/_lib_quota_model.py +96 -6
  36. package/bin/_lib_rebuild_progress.py +146 -0
  37. package/bin/_lib_record.py +22 -0
  38. package/bin/_lib_rederive.py +299 -3
  39. package/bin/_lib_snapshot_cache.py +43 -1
  40. package/bin/_lib_stale_proof.py +708 -0
  41. package/bin/_lib_statusline_candidates.py +65 -5
  42. package/bin/_lib_worker_launches.py +315 -0
  43. package/bin/cctally +6 -11
  44. package/dashboard/static/assets/{ConversationsView-Bc8LCOAZ.js → ConversationsView-Dflb_myA.js} +1 -1
  45. package/dashboard/static/assets/DoctorModal-DFX5CfIs.js +1 -0
  46. package/dashboard/static/assets/{ModalRoot-DLPgyAM_.js → ModalRoot-HlxoagIa.js} +1 -1
  47. package/dashboard/static/assets/{ProjectsDrillPanel-z53a4v4M.js → ProjectsDrillPanel-C3PJy1xx.js} +1 -1
  48. package/dashboard/static/assets/{SourceDetailModal-Bw0T8W-W.js → SourceDetailModal-BczsWKhD.js} +1 -1
  49. package/dashboard/static/assets/{UpdateModal-D5AKx4Yw.js → UpdateModal-Os-Lo5cs.js} +1 -1
  50. package/dashboard/static/assets/{index-BXDOzSng.js → index-CYZ3NLqF.js} +3 -3
  51. package/dashboard/static/assets/{outlineNavigation-JObY7cUY.js → outlineNavigation-k-zIif6W.js} +1 -1
  52. package/dashboard/static/dashboard.html +1 -1
  53. package/package.json +7 -2
  54. package/bin/_lib_merge_gate.py +0 -452
  55. package/dashboard/static/assets/DoctorModal-D--qVBwe.js +0 -1
@@ -5919,7 +5919,8 @@ def _tui_precompute_doctor_payload(
5919
5919
  """Precompute the dashboard envelope's small doctor block, via the
5920
5920
  short-TTL memo (#268 M4, spec §6).
5921
5921
 
5922
- Returns the SAME ``{severity, counts, generated_at, fingerprint}`` dict
5922
+ Returns the SAME ``{severity, counts, generated_at, fingerprint,
5923
+ rebuild_worker}`` dict (``_lib_doctor.envelope_block``)
5923
5924
  ``snapshot_to_envelope`` used to build inline (or a synthetic FAIL block
5924
5925
  with ``_error`` on a doctor gather/checks failure), so moving the
5925
5926
  computation onto the snapshot is byte-identical for a given
@@ -5938,12 +5939,7 @@ def _tui_precompute_doctor_payload(
5938
5939
  now_utc=now, runtime_bind=bind)
5939
5940
  with _perf.phase("doctor.checks"):
5940
5941
  _doc_report = _ld.run_checks(_doc_state)
5941
- return {
5942
- "severity": _doc_report.overall_severity,
5943
- "counts": dict(_doc_report.counts),
5944
- "generated_at": _ld._iso_z(_doc_report.generated_at),
5945
- "fingerprint": _ld.fingerprint(_doc_report),
5946
- }
5942
+ return _ld.envelope_block(_doc_report)
5947
5943
  except Exception as exc: # noqa: BLE001 — never crash the rebuild
5948
5944
  # Mirror `snapshot_to_envelope`'s inline FAIL fallback (dashboard
5949
5945
  # `_iso_z`: strftime, no microseconds) so a doctor failure serves
@@ -5955,6 +5951,7 @@ def _tui_precompute_doctor_payload(
5955
5951
  "%Y-%m-%dT%H:%M:%SZ"
5956
5952
  ),
5957
5953
  "fingerprint": "sha1:" + ("0" * 40),
5954
+ "rebuild_worker": None,
5958
5955
  "_error": f"{type(exc).__name__}: {exc}",
5959
5956
  }
5960
5957
 
@@ -5963,6 +5960,51 @@ def _tui_precompute_doctor_payload(
5963
5960
  )
5964
5961
 
5965
5962
 
5963
+ def _deferred_doctor_placeholder(now_utc: dt.datetime) -> dict:
5964
+ """The schema-valid, privacy-safe doctor block a deferred frame carries
5965
+ while the real doctor gather cannot run (it opens stats repeatedly); the
5966
+ first successful snapshot replaces it with a real doctor payload."""
5967
+ return {
5968
+ "severity": "warn",
5969
+ "counts": {"ok": 0, "warn": 1, "fail": 0},
5970
+ "generated_at": now_utc.astimezone(dt.timezone.utc).strftime(
5971
+ "%Y-%m-%dT%H:%M:%SZ"
5972
+ ),
5973
+ "fingerprint": "sha1:" + ("0" * 40),
5974
+ "rebuild_worker": None,
5975
+ }
5976
+
5977
+
5978
+ def _tui_rebuild_worker_value(now_utc: dt.datetime, fallback=None):
5979
+ """#997 D10: the doctor block's ``rebuild_worker`` value for a deferred
5980
+ frame (``{severity, summary}`` or ``None``), via a short-TTL memo.
5981
+
5982
+ While an epoch rebuild is pending or running every live stats open
5983
+ defers, so neither the deferred startup frame nor a deferred tick can run
5984
+ the doctor gather; without this the chip would show only its placeholder
5985
+ for the whole run. The value comes from the worker snapshots alone
5986
+ (`_cctally_store.rebuild_worker_snapshot`: file reads, the stats.db
5987
+ header bytes and a `ps` probe; it never opens stats.db and takes no
5988
+ cctally lock, because the detached worker owns maintenance-exclusive)
5989
+ through the same `_lib_doctor` assessment as the `db.rebuild_worker`
5990
+ check. Never raises: a failure returns ``fallback``.
5991
+ """
5992
+ try:
5993
+ c = _cctally()
5994
+ sc = c._load_sibling("_lib_snapshot_cache")
5995
+
5996
+ def _compute(now: dt.datetime):
5997
+ _ld = c._load_sibling("_lib_doctor")
5998
+ _cd = c._load_sibling("_cctally_doctor")
5999
+ return _ld.rebuild_worker_envelope_value(
6000
+ _cd._gather_rebuild_workers(), now)
6001
+
6002
+ return sc.rebuild_worker_memo(
6003
+ now_utc, ttl_s=sc.REBUILD_WORKER_MEMO_TTL_S, compute=_compute)
6004
+ except Exception: # noqa: BLE001 — a frame must never fail on this
6005
+ return fallback
6006
+
6007
+
5966
6008
  def _tui_precompute_envelope_config(raw_config: dict) -> dict:
5967
6009
  """Precompute the config / update-state reads `snapshot_to_envelope`
5968
6010
  needs (#268 M4, spec §6).
@@ -9201,8 +9243,21 @@ def _make_run_sync_now_locked(*, ref, hub, pinned_now, display_tz_pref_override,
9201
9243
  # an index that could not be read, and reporting that as
9202
9244
  # non-corruption makes the envelope name the wrong fault.
9203
9245
  prev = ref.get()
9246
+ # #997 D10: the doctor gather cannot run during the deferral, so
9247
+ # refresh only the rebuild-worker aggregate the chip leads with.
9248
+ deferred_now = pinned_now or dt.datetime.now(dt.timezone.utc)
9249
+ prev_doctor = getattr(prev, "doctor_payload", None)
9250
+ base_doctor = (prev_doctor if isinstance(prev_doctor, dict)
9251
+ else _deferred_doctor_placeholder(deferred_now))
9252
+ doctor_payload = {
9253
+ **base_doctor,
9254
+ "rebuild_worker": _tui_rebuild_worker_value(
9255
+ deferred_now,
9256
+ fallback=base_doctor.get("rebuild_worker")),
9257
+ }
9204
9258
  pending = dataclasses.replace(
9205
9259
  prev,
9260
+ doctor_payload=doctor_payload,
9206
9261
  last_sync_error=f"stats-open: {exc}",
9207
9262
  sync_failures=(
9208
9263
  SyncFailureAttribution(
@@ -1475,6 +1475,31 @@ def _spawn_detached(command: str) -> bool:
1475
1475
  return False
1476
1476
 
1477
1477
 
1478
+ def _spawn_detached_with_pid(
1479
+ command: str, *, env: "dict | None" = None,
1480
+ ) -> "int | None":
1481
+ """`_spawn_detached`'s launch shape, returning the child's pid.
1482
+
1483
+ The detached stats rebuild workers (#997 D8) record each launch's pid and
1484
+ start identity and hand the child its launch id through ``env``, so they
1485
+ need the pid that `_spawn_detached`'s bool contract (kept for its other
1486
+ callers) does not carry. ``None`` when the spawn raised.
1487
+ """
1488
+ try:
1489
+ proc = subprocess.Popen(
1490
+ [sys.executable, os.path.realpath(sys.argv[0]), command],
1491
+ stdin=subprocess.DEVNULL,
1492
+ stdout=subprocess.DEVNULL,
1493
+ stderr=subprocess.DEVNULL,
1494
+ start_new_session=True,
1495
+ close_fds=True,
1496
+ env=env,
1497
+ )
1498
+ except Exception:
1499
+ return None
1500
+ return proc.pid
1501
+
1502
+
1478
1503
  def _spawn_background_update_check() -> None:
1479
1504
  """Fire-and-forget the hidden ``_update-check`` worker (spec §3.6)."""
1480
1505
  _spawn_detached("_update-check")
@@ -24,6 +24,15 @@ post-boundary record without the current writer generation is admitted only
24
24
  when an explicit field its real producer writes proves it Codex-only. An
25
25
  unknown or unclassifiable record is excluded, never admitted by omission.
26
26
 
27
+ #797/#823/#840 (spec section 4.6, B1-B3): revision 2 of the fence. ``wpg``
28
+ keeps its value (the previous release's exact-equality ``is_stamped`` must
29
+ keep admitting what the current binary writes), and a second reserved
30
+ top-level field ``wpr`` carries the integer protocol revision. A durable
31
+ ``account_bound_revision`` op records a second physical boundary with the same
32
+ codec; after it, a Claude-scoped record without ``wpr == 2`` is excluded
33
+ exactly as an unstamped record is after the original cutover. The revision
34
+ boundary never re-admits a record the original rule excludes.
35
+
27
36
  Pure: no I/O, no clock, no SQLite.
28
37
  """
29
38
  from __future__ import annotations
@@ -39,6 +48,12 @@ WRITER_GENERATION_KEY = "wpg"
39
48
  CUTOVER_OP_KIND = "account_bound_cutover"
40
49
  CUTOVER_OP_ID = "account-bound-cutover-v1"
41
50
  LEGACY_SPOOL_ACTIVITY_KIND = "legacy_spool_activity"
51
+ #: #797/#823/#840 (spec 4.6 B1, B2): the protocol revision stamp and the
52
+ #: durable op that records where revision 2 begins.
53
+ REVISION_KEY = "wpr"
54
+ PROTOCOL_REVISION = 2
55
+ REVISION_OP_KIND = "account_bound_revision"
56
+ REVISION_OP_ID = "account-bound-revision-2"
42
57
  #: Spec revision 5 (section 7, operator decision on Codex design-fix-round-5
43
58
  #: Q4): the `legacy_spool_activity` reason of an incomplete inventory. Such an
44
59
  #: op records typed failure evidence and no digest, so it anchors no baseline.
@@ -59,6 +74,11 @@ INVENTORY_OPERATIONS = ("list", "stat", "read")
59
74
  #: previous-release validators require exact keys (spec 5.1). The previous
60
75
  #: release ignores an unknown op kind, so the current binary emits only these.
61
76
  PROTOCOL_RESOLUTION_V2_KIND = "journal_protocol_resolution_v2"
77
+ #: #797/#823/#840 (spec 4.6 B5): the previous release's strict validator of
78
+ #: the v2 acknowledgement rejects the extra ``wpr`` field, so the current
79
+ #: binary emits repair acknowledgements under this successor kind, which the
80
+ #: previous release ignores as an unknown op.
81
+ PROTOCOL_RESOLUTION_V3_KIND = "journal_protocol_resolution_v3"
62
82
  WEEKLY_OBSERVATION_DECISION_V2_KIND = "claude_weekly_observation_decision_v2"
63
83
 
64
84
  #: Admission families. ``codex`` is the only family admitted without a stamp.
@@ -73,7 +93,9 @@ FAMILIES = (
73
93
  )
74
94
 
75
95
  ADMISSION_REASONS = ("no-cutover", "historical", "stamped", "codex-only",
76
- "excluded")
96
+ "excluded", "revision-unstamped")
97
+ #: The reasons that exclude a record.
98
+ EXCLUDED_REASONS = frozenset(("excluded", "revision-unstamped"))
77
99
 
78
100
  _DIGEST_RE = re.compile(r"^sha256:[0-9a-f]{64}$")
79
101
 
@@ -87,9 +109,11 @@ _CODEX_OP_KINDS = frozenset((
87
109
  ))
88
110
  _ADMIN_OP_KINDS = frozenset((
89
111
  "journal_protocol_resolution", "claude_weekly_observation_decision",
90
- PROTOCOL_RESOLUTION_V2_KIND, WEEKLY_OBSERVATION_DECISION_V2_KIND,
112
+ PROTOCOL_RESOLUTION_V2_KIND, PROTOCOL_RESOLUTION_V3_KIND,
113
+ WEEKLY_OBSERVATION_DECISION_V2_KIND,
91
114
  ))
92
- _PROTOCOL_OP_KINDS = frozenset((CUTOVER_OP_KIND, LEGACY_SPOOL_ACTIVITY_KIND))
115
+ _PROTOCOL_OP_KINDS = frozenset((CUTOVER_OP_KIND, LEGACY_SPOOL_ACTIVITY_KIND,
116
+ REVISION_OP_KIND))
93
117
  _ADMIN_RECORD_TYPES = frozenset(("correction", "correction_batch"))
94
118
  _CLAUDE_EVT_KINDS = frozenset((
95
119
  "snapshot_accept", "weekly_cost_snapshot", "week_reset", "five_hour_credit",
@@ -133,6 +157,43 @@ class CutoverBoundary:
133
157
  frozen_extents: Mapping[str, int]
134
158
  legacy_spool_baseline_digest: "str | None"
135
159
  legacy_spool_inventory_failure: "Mapping[str, object] | None" = None
160
+ #: #797/#823/#840 (spec 4.6 B3): the revision-2 boundary a lookup found
161
+ #: after this cutover, carried beside it so every admission consumer
162
+ #: applies both fences through the one boundary object it already
163
+ #: threads. Not part of the cutover op's codec, and not part of equality:
164
+ #: two lookups of one cutover op name the same original boundary, and
165
+ #: the admission summary records the revision separately.
166
+ revision: "RevisionBoundary | None" = dataclasses.field(
167
+ default=None, compare=False)
168
+
169
+
170
+ @dataclasses.dataclass(frozen=True)
171
+ class RevisionBoundary:
172
+ """The revision-2 boundary recorded by the ``account_bound_revision`` op
173
+ (spec 4.6 B2): the same physical-position codec as `CutoverBoundary`,
174
+ plus the original cutover it extends (``cutover_segment`` and
175
+ ``cutover_offset``, the coordinate of the cutover op).
176
+
177
+ ``legacy_spool_baseline_digest`` is the EXPANDED baseline digest — the
178
+ pre-#825 legacy paths and the retired v1 directory's coordination state
179
+ (spec 4.6 B4) — or None when the establishment could not complete that
180
+ inventory and records ``legacy_spool_inventory_failure`` instead, exactly
181
+ as a cutover does (#825 spec revision 5).
182
+
183
+ ``continued_inventory_episode`` is the coordinate of the uncertainty
184
+ record whose failure episode was still open when an incomplete revision
185
+ inventory was recorded: the failure belongs to that episode, which
186
+ already has its trigger, so the revision op raises none of its own (one
187
+ trigger per failure episode, #825 spec revision 5). None otherwise."""
188
+
189
+ segment: str
190
+ end_offset: int
191
+ frozen_extents: Mapping[str, int]
192
+ legacy_spool_baseline_digest: "str | None"
193
+ cutover_segment: str
194
+ cutover_offset: int
195
+ legacy_spool_inventory_failure: "Mapping[str, object] | None" = None
196
+ continued_inventory_episode: "tuple[str, int] | None" = None
136
197
 
137
198
 
138
199
  @dataclasses.dataclass(frozen=True)
@@ -148,17 +209,23 @@ class Admission:
148
209
 
149
210
  def stamp(record: dict) -> dict:
150
211
  """Return a shallow copy of ``record`` carrying the current writer
151
- generation. Idempotent; refuses a record that already names a DIFFERENT
152
- generation (a caller must never relabel another protocol's record). The
153
- payload is never touched: the stamp is transport metadata."""
212
+ generation and protocol revision (spec 4.6 B1). Idempotent; refuses a
213
+ record that already names a DIFFERENT generation or revision (a caller
214
+ must never relabel another protocol's record). The payload is never
215
+ touched: both stamps are transport metadata."""
154
216
  if not isinstance(record, dict):
155
217
  raise ValueError("journal record must be a dict")
156
218
  existing = record.get(WRITER_GENERATION_KEY)
157
219
  if existing is not None and existing != PROTOCOL_GENERATION:
158
220
  raise ValueError(
159
221
  f"record already carries writer generation {existing!r}")
222
+ if REVISION_KEY in record and not has_current_revision(record):
223
+ raise ValueError(
224
+ f"record already carries protocol revision "
225
+ f"{record[REVISION_KEY]!r}")
160
226
  out = dict(record)
161
227
  out[WRITER_GENERATION_KEY] = PROTOCOL_GENERATION
228
+ out[REVISION_KEY] = PROTOCOL_REVISION
162
229
  return out
163
230
 
164
231
 
@@ -167,12 +234,26 @@ def is_stamped(record: dict) -> bool:
167
234
  and record.get(WRITER_GENERATION_KEY) == PROTOCOL_GENERATION)
168
235
 
169
236
 
237
+ def has_current_revision(record: dict) -> bool:
238
+ """Whether ``record`` carries exactly protocol revision 2 (spec 4.6 B1).
239
+ Only an ``int`` equal to 2: never ``True``, ``"2"``, ``2.0`` or a later
240
+ revision. A future revision is admitted only through an explicit
241
+ compatibility rule added here, never an open-ended ``>= 2``."""
242
+ if not isinstance(record, dict):
243
+ return False
244
+ value = record.get(REVISION_KEY)
245
+ return type(value) is int and value == PROTOCOL_REVISION
246
+
247
+
170
248
  def strip_stamp(record: dict) -> dict:
171
- """``record`` without its writer-generation field — the comparison shape
172
- for semantic event duplicate/conflict classification."""
173
- if not isinstance(record, dict) or WRITER_GENERATION_KEY not in record:
249
+ """``record`` without its writer-generation and protocol-revision fields
250
+ — the comparison shape for semantic event duplicate/conflict
251
+ classification."""
252
+ if not isinstance(record, dict) or (
253
+ WRITER_GENERATION_KEY not in record and REVISION_KEY not in record):
174
254
  return record
175
- return {k: v for k, v in record.items() if k != WRITER_GENERATION_KEY}
255
+ return {k: v for k, v in record.items()
256
+ if k not in (WRITER_GENERATION_KEY, REVISION_KEY)}
176
257
 
177
258
 
178
259
  # --------------------------------------------------------------------------
@@ -272,6 +353,92 @@ def boundary_from_op(op_record: dict) -> CutoverBoundary:
272
353
  )
273
354
 
274
355
 
356
+ def with_revision(boundary: "CutoverBoundary | None",
357
+ revision: "RevisionBoundary | None"):
358
+ """``boundary`` carrying ``revision`` (spec 4.6 B3), or None when there is
359
+ no cutover. A revision that names a different cutover is malformed and
360
+ never attached."""
361
+ if boundary is None:
362
+ return None
363
+ if revision is not None and (
364
+ (revision.cutover_segment, revision.cutover_offset)
365
+ != (boundary.segment, boundary.end_offset)):
366
+ raise ValueError("revision boundary names a different cutover")
367
+ return dataclasses.replace(boundary, revision=revision)
368
+
369
+
370
+ def revision_boundary_payload(boundary: RevisionBoundary) -> dict:
371
+ """The camelCase payload fields of the revision op (without ``kind``):
372
+ the `boundary_payload` codec, the revision number and the original
373
+ cutover's coordinate."""
374
+ payload = {
375
+ "revision": PROTOCOL_REVISION,
376
+ "cutover": {"segment": boundary.cutover_segment,
377
+ "offset": boundary.cutover_offset},
378
+ "segment": boundary.segment,
379
+ "endOffset": boundary.end_offset,
380
+ "frozenExtents": {k: boundary.frozen_extents[k]
381
+ for k in sorted(boundary.frozen_extents)},
382
+ "legacySpoolBaselineDigest": boundary.legacy_spool_baseline_digest,
383
+ }
384
+ if boundary.legacy_spool_inventory_failure is not None:
385
+ payload["legacySpoolInventoryFailure"] = dict(
386
+ boundary.legacy_spool_inventory_failure)
387
+ if boundary.continued_inventory_episode is not None:
388
+ segment, offset = boundary.continued_inventory_episode
389
+ payload["inventoryEpisode"] = {"segment": segment,
390
+ "offset": offset}
391
+ return payload
392
+
393
+
394
+ def revision_boundary_from_op(op_record: dict) -> RevisionBoundary:
395
+ """Decode the revision boundary from an ``account_bound_revision`` op.
396
+ Raises ``ValueError`` on any shape failure, including a revision other
397
+ than exactly 2 and an op that does not itself carry ``wpr == 2``."""
398
+ if not isinstance(op_record, dict) or op_record.get("t") != "op":
399
+ raise ValueError("revision boundary must come from an op record")
400
+ payload = op_record.get("payload")
401
+ if not isinstance(payload, dict) or payload.get("kind") != REVISION_OP_KIND:
402
+ raise ValueError("record is not an account_bound_revision op")
403
+ if type(payload.get("revision")) is not int or (
404
+ payload["revision"] != PROTOCOL_REVISION):
405
+ raise ValueError("revision boundary revision is invalid")
406
+ cutover = payload.get("cutover")
407
+ if (not isinstance(cutover, dict)
408
+ or set(cutover) != {"segment", "offset"}
409
+ or not isinstance(cutover.get("segment"), str)
410
+ or not cutover["segment"]
411
+ or not _nonneg_int(cutover.get("offset"))):
412
+ raise ValueError("revision boundary cutover coordinate is invalid")
413
+ episode = payload.get("inventoryEpisode")
414
+ if episode is not None:
415
+ if (payload.get("legacySpoolInventoryFailure") is None
416
+ or not isinstance(episode, dict)
417
+ or set(episode) != {"segment", "offset"}
418
+ or not isinstance(episode.get("segment"), str)
419
+ or not episode["segment"]
420
+ or not _nonneg_int(episode.get("offset"))):
421
+ raise ValueError("revision boundary inventory episode is invalid")
422
+ episode = (episode["segment"], episode["offset"])
423
+ base = boundary_from_op({
424
+ "t": "op",
425
+ "payload": {**{k: v for k, v in payload.items()
426
+ if k not in ("revision", "cutover",
427
+ "inventoryEpisode")},
428
+ "kind": CUTOVER_OP_KIND},
429
+ })
430
+ return RevisionBoundary(
431
+ segment=base.segment,
432
+ end_offset=base.end_offset,
433
+ frozen_extents=base.frozen_extents,
434
+ legacy_spool_baseline_digest=base.legacy_spool_baseline_digest,
435
+ cutover_segment=cutover["segment"],
436
+ cutover_offset=cutover["offset"],
437
+ legacy_spool_inventory_failure=base.legacy_spool_inventory_failure,
438
+ continued_inventory_episode=episode,
439
+ )
440
+
441
+
275
442
  # --------------------------------------------------------------------------
276
443
  # admission
277
444
  # --------------------------------------------------------------------------
@@ -329,20 +496,36 @@ def record_family(record: dict) -> str:
329
496
 
330
497
 
331
498
  def classify_admission(record: dict, coord: Coordinate,
332
- boundary: CutoverBoundary | None) -> Admission:
333
- """The single admission predicate (spec 5.3).
334
-
335
- Admitted when there is no cutover yet, when the record lies inside the
336
- historical boundary, when it carries the current writer generation, or when
337
- its family is proven Codex-only. Every other post-boundary record is
338
- excluded."""
499
+ boundary: CutoverBoundary | None,
500
+ revision_boundary: "RevisionBoundary | None" = None,
501
+ ) -> Admission:
502
+ """The single admission predicate (spec 5.3; #797/#823/#840 spec 4.6 B3).
503
+
504
+ The original rule: admitted when there is no cutover yet, when the
505
+ record lies inside the historical boundary, when it carries the current
506
+ writer generation, or when its family is proven Codex-only; every other
507
+ post-boundary record is excluded.
508
+
509
+ Composed with revision 2: ``revision_boundary`` (default: the revision a
510
+ lookup attached to ``boundary``) excludes, after its own frozen extents,
511
+ a record the original rule admitted only by its writer generation unless
512
+ it also carries ``wpr == 2`` (``revision-unstamped``). The revision never
513
+ re-admits what the original rule excludes, and the Codex-only exemptions
514
+ are unchanged."""
339
515
  family, codex_only = _family(record)
340
516
  if boundary is None:
341
517
  return Admission(True, family, "no-cutover")
342
518
  if is_historical(coord, boundary):
343
519
  return Admission(True, family, "historical")
344
520
  if is_stamped(record):
345
- return Admission(True, family, "stamped")
521
+ revision = (revision_boundary if revision_boundary is not None
522
+ else boundary.revision)
523
+ if (revision is None or is_historical(coord, revision)
524
+ or has_current_revision(record)):
525
+ return Admission(True, family, "stamped")
526
+ if codex_only:
527
+ return Admission(True, family, "codex-only")
528
+ return Admission(False, family, "revision-unstamped")
346
529
  if codex_only:
347
530
  return Admission(True, family, "codex-only")
348
531
  return Admission(False, family, "excluded")
@@ -12,6 +12,9 @@ for:
12
12
  existing destination is overwritten in place (same inode), which a clone
13
13
  cannot do, so that case is a real copy. Metadata is ``shutil.copystat``'s,
14
14
  applied after the clone as after a copy.
15
+ - :func:`try_clone` is the clone alone: True when `dst` was created as a
16
+ clone, False (nothing created) when the platform or filesystem refuses, so
17
+ a caller can choose where its real copy goes (#997 D13).
15
18
  - :func:`copytree` is ``shutil.copytree`` with :func:`copy2` as its file
16
19
  copier, so symlink policy, ignore lists, dangling-link errors and
17
20
  destination collisions are the stdlib's own code.
@@ -26,11 +29,10 @@ The optional ``counts`` mapping is incremented under ``"clone"`` or
26
29
  ``"copy"`` per file, so a caller or a measurement can see which path ran.
27
30
  This module is public and imports nothing outside the standard library.
28
31
 
29
- It binds ``fclonefileat`` itself rather than importing the binding in
30
- ``bin/_lib_remote_staging.py`` (#940 D-2 F17): that module is private, so it
31
- is absent from the public tree that this module and its public callers ship
32
- in, and a public module may not import it. The staging module may import
33
- from this one instead (spec §6.7).
32
+ It binds ``fclonefileat`` itself rather than importing the binding from the
33
+ retired private staging module (#940 D-2 F17): a private module is absent
34
+ from the public tree that this module and its public callers ship in, and a
35
+ public module may not import it.
34
36
  """
35
37
 
36
38
  from __future__ import annotations
@@ -85,6 +87,43 @@ def _clone_fd(source_fd: int, destination: str) -> None:
85
87
  raise OSError(code, os.strerror(code), destination)
86
88
 
87
89
 
90
+ def _clone_open_file(fd: int, src, dst, *, follow_symlinks: bool = True) -> bool:
91
+ """Clone the open regular file `fd` (opened from `src`) to the new path
92
+ `dst` and apply `src`'s metadata. False, with nothing created, when the
93
+ filesystem or the pair of volumes refuses the clone (`_FALLBACK_ERRNOS`);
94
+ any other error propagates."""
95
+ try:
96
+ _clone_fd(fd, dst)
97
+ except OSError as exc:
98
+ if exc.errno not in _FALLBACK_ERRNOS:
99
+ raise
100
+ return False
101
+ shutil.copystat(src, dst, follow_symlinks=follow_symlinks)
102
+ return True
103
+
104
+
105
+ def try_clone(src, dst) -> bool:
106
+ """Clone the regular file `src` to the NEW path `dst` (#997 D13).
107
+
108
+ True: `dst` is a clone sharing `src`'s blocks, with ``shutil.copystat``
109
+ metadata applied. False, with `dst` not created: the platform is not
110
+ Darwin (or has no ``fclonefileat``), `src` is not a regular file, or the
111
+ filesystem refused with a fallback errno (an unsupported filesystem, a
112
+ cross-volume destination, an existing `dst`). Any other error, including
113
+ a `src` that cannot be opened, propagates. A caller that needs a copy
114
+ either way makes the real copy itself on False.
115
+ """
116
+ if _fclonefileat() is None:
117
+ return False
118
+ fd = os.open(src, os.O_RDONLY | getattr(os, "O_CLOEXEC", 0))
119
+ try:
120
+ if not stat.S_ISREG(os.fstat(fd).st_mode):
121
+ return False
122
+ return _clone_open_file(fd, src, dst)
123
+ finally:
124
+ os.close(fd)
125
+
126
+
88
127
  def _count(counts, method: str) -> None:
89
128
  if counts is not None:
90
129
  counts[method] = counts.get(method, 0) + 1
@@ -113,15 +152,11 @@ def copy2(src, dst, *, follow_symlinks: bool = True, counts=None):
113
152
  os.close(fd)
114
153
  fd = -1
115
154
  return shutil.copy2(src, dst, follow_symlinks=follow_symlinks)
116
- try:
117
- _clone_fd(fd, dst)
118
- except OSError as exc:
119
- if exc.errno not in _FALLBACK_ERRNOS:
120
- raise
155
+ if not _clone_open_file(fd, src, dst,
156
+ follow_symlinks=follow_symlinks):
121
157
  result = shutil.copy2(src, dst, follow_symlinks=follow_symlinks)
122
158
  _count(counts, "copy")
123
159
  return result
124
- shutil.copystat(src, dst, follow_symlinks=follow_symlinks)
125
160
  _count(counts, "clone")
126
161
  return dst
127
162
  finally:
@@ -1079,27 +1079,61 @@ def alerted_at_order_key(value: object) -> str:
1079
1079
  return _parse_alerted_at(value).strftime("%Y-%m-%dT%H:%M:%S.%fZ")
1080
1080
 
1081
1081
 
1082
- def _alerted_at_fraction_sql(column: str) -> str:
1083
- """The six-digit sub-second fraction of an ISO-8601 value, in SQLite.
1084
-
1085
- SQLite's date functions keep only milliseconds, so the fraction is read
1086
- from the text itself. A UTC offset is a whole number of minutes, so the
1087
- source fraction is also the UTC instant's fraction.
1088
- """
1082
+ def _alerted_at_fraction_digits_sql(column: str) -> str:
1083
+ """Every digit after the seconds' ``.``, up to the zone designator."""
1089
1084
  tail = f"substr({column}, 21)"
1090
- digits = (
1085
+ return (
1091
1086
  f"substr({tail}, 1, CASE "
1092
1087
  f"WHEN instr({tail}, 'Z') > 0 THEN instr({tail}, 'Z') - 1 "
1093
1088
  f"WHEN instr({tail}, '+') > 0 THEN instr({tail}, '+') - 1 "
1094
1089
  f"WHEN instr({tail}, '-') > 0 THEN instr({tail}, '-') - 1 "
1095
1090
  f"ELSE length({tail}) END)"
1096
1091
  )
1092
+
1093
+
1094
+ def _alerted_at_fraction_sql(column: str) -> str:
1095
+ """The six-digit sub-second fraction of an ISO-8601 value, in SQLite.
1096
+
1097
+ SQLite's date functions keep only milliseconds, so the fraction is read
1098
+ from the text itself. A UTC offset is a whole number of minutes, so the
1099
+ source fraction is also the UTC instant's fraction.
1100
+ """
1101
+ digits = _alerted_at_fraction_digits_sql(column)
1097
1102
  return (
1098
1103
  f"(CASE WHEN substr({column}, 20, 1) = '.' "
1099
1104
  f"THEN substr({digits} || '000000', 1, 6) ELSE '000000' END)"
1100
1105
  )
1101
1106
 
1102
1107
 
1108
+ def _alerted_at_whole_second_sql(column: str) -> str:
1109
+ """The value with its sub-second fraction removed, zone kept.
1110
+
1111
+ SQLite before 3.46 rounds a parsed fraction to the nearest millisecond, so
1112
+ ``…00.999999`` formats as second ``01`` there and as ``00`` on later
1113
+ versions. The seconds part must come from the whole second alone; the
1114
+ fraction is supplied from the text by :func:`_alerted_at_fraction_sql`.
1115
+ """
1116
+ digits = _alerted_at_fraction_digits_sql(column)
1117
+ return (
1118
+ f"(CASE WHEN substr({column}, 20, 1) = '.' "
1119
+ f"THEN substr({column}, 1, 19) || "
1120
+ f"substr({column}, 21 + length({digits})) ELSE {column} END)"
1121
+ )
1122
+
1123
+
1124
+ def _alerted_at_malformed_fraction_sql(column: str) -> str:
1125
+ """True when the value has a ``.`` fraction that is not all digits.
1126
+
1127
+ :func:`_alerted_at_whole_second_sql` cuts the fraction out before
1128
+ ``strftime`` parses the rest, so ``strftime`` alone no longer rejects
1129
+ ``…00.abcZ``; this guard keeps such a value ``NULL``, as Python rejects it.
1130
+ """
1131
+ digits = _alerted_at_fraction_digits_sql(column)
1132
+ return (
1133
+ f"(substr({column}, 20, 1) = '.' AND {digits} GLOB '*[^0-9]*')"
1134
+ )
1135
+
1136
+
1103
1137
  def alerted_at_order_key_sql(column: str = "alerted_at") -> str:
1104
1138
  """The SQL twin of :func:`alerted_at_order_key`, for ``ORDER BY … LIMIT``.
1105
1139
 
@@ -1111,12 +1145,18 @@ def alerted_at_order_key_sql(column: str = "alerted_at") -> str:
1111
1145
  every aware spelling this returns the Python helper's string —
1112
1146
  ``tests/test_556_s3_alert_ordering.py`` pins that agreement. A naive value
1113
1147
  is read as UTC here where the Python helper raises; an unparseable value
1114
- yields ``NULL``, which sorts last under ``DESC``. The expression names
1115
- ``column`` several times, so a bound parameter must be numbered (``?1``).
1148
+ yields ``NULL``, which sorts last under ``DESC``. The two disagree on
1149
+ spellings Python 3.11+ also accepts (``+05``, ``+0500``, a ``,`` fraction),
1150
+ but every ``alerted_at`` writer stores Python ``isoformat()`` UTC text
1151
+ (``…SS[.ffffff]Z`` or ``…SS[.ffffff]+00:00``), on which they agree. The
1152
+ expression names ``column`` several times, so a bound parameter must be
1153
+ numbered (``?1``).
1116
1154
  """
1155
+ whole = _alerted_at_whole_second_sql(column)
1117
1156
  return (
1118
- f"(strftime('%Y-%m-%dT%H:%M:%S', {column}) || '.' || "
1119
- f"{_alerted_at_fraction_sql(column)} || 'Z')"
1157
+ f"(CASE WHEN {_alerted_at_malformed_fraction_sql(column)} THEN NULL "
1158
+ f"ELSE strftime('%Y-%m-%dT%H:%M:%S', {whole}) || '.' || "
1159
+ f"{_alerted_at_fraction_sql(column)} || 'Z' END)"
1120
1160
  )
1121
1161
 
1122
1162
 
@@ -1130,10 +1170,12 @@ def canonical_alerted_at_sql(column: str = "alerted_at") -> str:
1130
1170
  :func:`alerted_at_order_key_sql`'s job.
1131
1171
  """
1132
1172
  fraction = _alerted_at_fraction_sql(column)
1173
+ whole = _alerted_at_whole_second_sql(column)
1133
1174
  return (
1134
- f"(CASE WHEN {fraction} = '000000' "
1135
- f"THEN strftime('%Y-%m-%dT%H:%M:%SZ', {column}) "
1136
- f"ELSE strftime('%Y-%m-%dT%H:%M:%S', {column}) || '.' || {fraction} "
1175
+ f"(CASE WHEN {_alerted_at_malformed_fraction_sql(column)} THEN NULL "
1176
+ f"WHEN {fraction} = '000000' "
1177
+ f"THEN strftime('%Y-%m-%dT%H:%M:%SZ', {whole}) "
1178
+ f"ELSE strftime('%Y-%m-%dT%H:%M:%S', {whole}) || '.' || {fraction} "
1137
1179
  f"|| 'Z' END)"
1138
1180
  )
1139
1181