@matt82198/aesop 0.2.0 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +28 -0
- package/README.md +50 -260
- package/bin/cli.js +7 -3
- package/docs/HOOK-INSTALL.md +15 -56
- package/docs/INSTALL.md +4 -3
- package/docs/PORTING.md +166 -0
- package/docs/TEAM-STATE.md +16 -184
- package/docs/reproduce.md +33 -3
- package/driver/CLAUDE.md +50 -48
- package/driver/codex_driver.py +59 -12
- package/driver/openai_transport.py +33 -0
- package/driver/wave_bridge.py +9 -1
- package/driver/wave_loop.py +437 -70
- package/driver/wave_scheduler.py +890 -0
- package/hooks/pre-push-policy.sh +22 -5
- package/monitor/collect-signals.mjs +69 -0
- package/package.json +4 -4
- package/state_store/__init__.py +2 -1
- package/state_store/projections.py +63 -0
- package/state_store/read_api.py +156 -0
- package/state_store/write_api.py +462 -0
- package/tools/cost_ceiling.py +106 -15
- package/tools/cost_projection.py +559 -0
- package/tools/crossos_drift.py +394 -0
- package/tools/eod_sweep.py +137 -33
- package/tools/mutation_test.py +130 -8
- package/tools/proposals.mjs +47 -2
- package/tools/reproduce.js +405 -0
- package/tools/secret_scan.py +73 -18
- package/tools/stall_check.py +247 -16
- package/tools/stateapi_lint.py +325 -0
- package/tools/test_battery.py +173 -0
- package/tools/verify_cost_panel.py +345 -0
- package/tools/wave_preflight.py +296 -29
- package/tools/wave_templates.py +170 -45
- package/ui/collectors.py +81 -0
- package/ui/handler.py +230 -85
- package/ui/sse.py +3 -3
- package/ui/wave_telemetry.py +24 -18
- package/ui/web/dist/assets/{index-DqZLgwNg.css → index-CNQxaiOW.css} +1 -1
- package/ui/web/dist/assets/index-CP68RIh3.js +9 -0
- package/ui/web/dist/index.html +2 -2
- package/docs/QUICKSTART.md +0 -80
- package/ui/web/dist/assets/index-BGbgw2Nh.js +0 -9
|
@@ -0,0 +1,462 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""
|
|
3
|
+
state_store.write_api — Typed write facade for tracker mutations (state consolidation).
|
|
4
|
+
|
|
5
|
+
Consolidates write patterns for tracker mutations: status updates and item creation.
|
|
6
|
+
This facade allows the underlying write implementation to change (immediate projection
|
|
7
|
+
→ queued render → event store publishing) without altering caller code.
|
|
8
|
+
|
|
9
|
+
Mirrors the read_api.py facade pattern: callers use WriteAPI only; backend
|
|
10
|
+
implementation (EventStore + projection rendering) is hidden.
|
|
11
|
+
|
|
12
|
+
Callers use:
|
|
13
|
+
api = WriteAPI(state_dir)
|
|
14
|
+
api.tracker_update_status(item_id, new_status, note="optional note")
|
|
15
|
+
api.tracker_append_item({"title": "...", "priority": "P1", ...})
|
|
16
|
+
|
|
17
|
+
Both operations are fail-closed: event append failure → no projection write.
|
|
18
|
+
Projection write conflicts raise WriteConflict (honest failure, no silent data loss).
|
|
19
|
+
"""
|
|
20
|
+
from __future__ import annotations
|
|
21
|
+
|
|
22
|
+
import hashlib
|
|
23
|
+
import json
|
|
24
|
+
import os
|
|
25
|
+
import sys
|
|
26
|
+
import tempfile
|
|
27
|
+
from datetime import datetime, timezone
|
|
28
|
+
from pathlib import Path
|
|
29
|
+
|
|
30
|
+
# Ensure tools and state_store modules are importable
|
|
31
|
+
repo_root = Path(__file__).parent.parent
|
|
32
|
+
if str(repo_root) not in sys.path:
|
|
33
|
+
sys.path.insert(0, str(repo_root))
|
|
34
|
+
|
|
35
|
+
try:
|
|
36
|
+
from state_store import EventStore, ConcurrencyConflict
|
|
37
|
+
except ImportError:
|
|
38
|
+
from state_store.store import EventStore, ConcurrencyConflict
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class WriteConflict(Exception):
|
|
42
|
+
"""Raised when a projection write conflicts (content-hash mismatch).
|
|
43
|
+
|
|
44
|
+
Signifies that the tracker.json file's content hash does not match the
|
|
45
|
+
expected value, indicating concurrent modification. The event was appended
|
|
46
|
+
(durable in EventStore), but the projection write was skipped to prevent
|
|
47
|
+
silent data loss. Caller should re-read tracker.json, extract new version,
|
|
48
|
+
and retry.
|
|
49
|
+
|
|
50
|
+
Attributes:
|
|
51
|
+
expected_hash: The content hash caller expected for tracker.json
|
|
52
|
+
actual_hash: The content hash found on disk
|
|
53
|
+
reason: Human-readable description of the conflict
|
|
54
|
+
"""
|
|
55
|
+
|
|
56
|
+
def __init__(self, expected_hash: str | None, actual_hash: str | None, reason: str = ""):
|
|
57
|
+
self.expected_hash = expected_hash
|
|
58
|
+
self.actual_hash = actual_hash
|
|
59
|
+
self.reason = reason
|
|
60
|
+
super().__init__(
|
|
61
|
+
f"Projection write conflict: {reason} "
|
|
62
|
+
f"(expected hash {expected_hash}, found {actual_hash})"
|
|
63
|
+
)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
class WriteAPI:
|
|
67
|
+
"""Write facade for tracker mutations, backed by EventStore + atomic projection rendering.
|
|
68
|
+
|
|
69
|
+
Designed to be swappable: write backend can change (immediate render → event sourcing)
|
|
70
|
+
without altering call sites. Current implementation appends to event log and re-renders
|
|
71
|
+
tracker.json atomically (tempfile + os.replace).
|
|
72
|
+
|
|
73
|
+
All write operations are fail-closed: if event append fails, projection is not written.
|
|
74
|
+
If projection write fails due to concurrent modification, raises WriteConflict (event
|
|
75
|
+
is safely in the log, caller must retry).
|
|
76
|
+
|
|
77
|
+
OCC (Optimistic Concurrency Control): Each WriteAPI instance tracks the last projection
|
|
78
|
+
hash it wrote (or read at init). Before atomic write, if on-disk hash differs from
|
|
79
|
+
both tracked_hash and new_hash, raises WriteConflict to prevent overwriting concurrent
|
|
80
|
+
modifications. First-write case: hash is captured at operation start (before event append).
|
|
81
|
+
"""
|
|
82
|
+
|
|
83
|
+
def __init__(self, state_dir: str | Path):
|
|
84
|
+
"""Initialize the write API with a state directory.
|
|
85
|
+
|
|
86
|
+
Args:
|
|
87
|
+
state_dir: Path to the state directory (e.g., "state" or "/absolute/path/state")
|
|
88
|
+
"""
|
|
89
|
+
self.state_dir = Path(state_dir)
|
|
90
|
+
self.state_dir.mkdir(parents=True, exist_ok=True)
|
|
91
|
+
self.db_path = str(self.state_dir / "tracker_events.db")
|
|
92
|
+
self.tracker_file = self.state_dir / "tracker.json"
|
|
93
|
+
|
|
94
|
+
def tracker_update_status(
|
|
95
|
+
self,
|
|
96
|
+
item_id: str,
|
|
97
|
+
new_status: str,
|
|
98
|
+
note: str | None = None,
|
|
99
|
+
actor: str = "api",
|
|
100
|
+
) -> dict:
|
|
101
|
+
"""Update an existing tracker item's status and optionally add a note.
|
|
102
|
+
|
|
103
|
+
Appends an item_updated event to the event log, then re-renders tracker.json
|
|
104
|
+
atomically. Fail-closed: event append failure blocks projection write.
|
|
105
|
+
|
|
106
|
+
Args:
|
|
107
|
+
item_id: The item UUID to update
|
|
108
|
+
new_status: New status (e.g., "todo", "in-progress", "done", "archived")
|
|
109
|
+
note: Optional note to append to the item's notes field
|
|
110
|
+
actor: Actor performing the update (default "api")
|
|
111
|
+
|
|
112
|
+
Returns:
|
|
113
|
+
dict: The updated item from the tracker projection
|
|
114
|
+
|
|
115
|
+
Raises:
|
|
116
|
+
ValueError: If item_id not found or other validation failure
|
|
117
|
+
WriteConflict: If projection write fails due to concurrent modification
|
|
118
|
+
ConcurrencyConflict: If EventStore append hits OCC mismatch (should not happen
|
|
119
|
+
in this phase, but reserved for future use)
|
|
120
|
+
"""
|
|
121
|
+
store = EventStore(self.db_path)
|
|
122
|
+
|
|
123
|
+
# Read current tracker to find the item
|
|
124
|
+
current_tracker = self._load_tracker_safe()
|
|
125
|
+
current_items = {item["id"]: item for item in current_tracker.get("items", [])}
|
|
126
|
+
|
|
127
|
+
if item_id not in current_items:
|
|
128
|
+
raise ValueError(f"Item not found: {item_id}")
|
|
129
|
+
|
|
130
|
+
current_item = current_items[item_id]
|
|
131
|
+
|
|
132
|
+
# Build the update payload
|
|
133
|
+
update_payload = {"id": item_id, "status": new_status}
|
|
134
|
+
|
|
135
|
+
# If adding a note, append to the item's notes field
|
|
136
|
+
if note:
|
|
137
|
+
existing_notes = current_item.get("notes", "")
|
|
138
|
+
if existing_notes:
|
|
139
|
+
update_payload["notes"] = f"{existing_notes}\n{note}"
|
|
140
|
+
else:
|
|
141
|
+
update_payload["notes"] = note
|
|
142
|
+
|
|
143
|
+
# CRITICAL: Capture on-disk hash at operation START (before event append)
|
|
144
|
+
# This is our baseline for OCC conflict detection.
|
|
145
|
+
start_disk_hash = None
|
|
146
|
+
if self.tracker_file.exists():
|
|
147
|
+
try:
|
|
148
|
+
current_on_disk = json.loads(
|
|
149
|
+
self.tracker_file.read_text(encoding="utf-8")
|
|
150
|
+
)
|
|
151
|
+
start_disk_hash = self._compute_content_hash(current_on_disk)
|
|
152
|
+
except Exception:
|
|
153
|
+
# Corrupt file on disk at START: treat as conflict (fail-closed)
|
|
154
|
+
pass
|
|
155
|
+
|
|
156
|
+
# Append the event (fail-closed: if this fails, no projection write)
|
|
157
|
+
try:
|
|
158
|
+
store.append("tracker", "item_updated", update_payload, actor)
|
|
159
|
+
except Exception as e:
|
|
160
|
+
raise ValueError(f"Failed to append update event: {e}") from e
|
|
161
|
+
|
|
162
|
+
# Now re-render the projection atomically
|
|
163
|
+
self._render_tracker_atomic(store, start_disk_hash=start_disk_hash)
|
|
164
|
+
|
|
165
|
+
# Return the updated item from the freshly projected state
|
|
166
|
+
updated_tracker = self._load_tracker_safe()
|
|
167
|
+
updated_items = {item["id"]: item for item in updated_tracker.get("items", [])}
|
|
168
|
+
|
|
169
|
+
if item_id not in updated_items:
|
|
170
|
+
# Should not happen if projection is consistent, but defend against it
|
|
171
|
+
raise ValueError(f"Item disappeared after update: {item_id}")
|
|
172
|
+
|
|
173
|
+
return updated_items[item_id]
|
|
174
|
+
|
|
175
|
+
def tracker_append_item(
|
|
176
|
+
self,
|
|
177
|
+
item_dict: dict,
|
|
178
|
+
actor: str = "api",
|
|
179
|
+
) -> dict:
|
|
180
|
+
"""Create a new tracker item.
|
|
181
|
+
|
|
182
|
+
Validates the item dict, appends an item_created event to the event log,
|
|
183
|
+
then re-renders tracker.json atomically. Fail-closed: event append failure
|
|
184
|
+
blocks projection write.
|
|
185
|
+
|
|
186
|
+
Args:
|
|
187
|
+
item_dict: Item dict with fields: id (optional, auto-generated if missing),
|
|
188
|
+
title, priority (optional, defaults to "P1"), status (optional,
|
|
189
|
+
defaults to "todo"), lane (optional, defaults to "proposed"),
|
|
190
|
+
source (optional, defaults to "api"), tags, notes, pr_link, etc.
|
|
191
|
+
actor: Actor performing the create (default "api")
|
|
192
|
+
|
|
193
|
+
Returns:
|
|
194
|
+
dict: The created item from the tracker projection
|
|
195
|
+
|
|
196
|
+
Raises:
|
|
197
|
+
ValueError: If item_dict is invalid, missing required fields, or explicit ID
|
|
198
|
+
already exists in projection
|
|
199
|
+
WriteConflict: If projection write fails due to concurrent modification
|
|
200
|
+
ConcurrencyConflict: If EventStore append hits OCC mismatch
|
|
201
|
+
"""
|
|
202
|
+
if not isinstance(item_dict, dict):
|
|
203
|
+
raise ValueError("item_dict must be a dict")
|
|
204
|
+
|
|
205
|
+
title = item_dict.get("title", "").strip()
|
|
206
|
+
if not title:
|
|
207
|
+
raise ValueError("item_dict must have a non-empty 'title' field")
|
|
208
|
+
|
|
209
|
+
# Build the canonical item structure
|
|
210
|
+
import secrets
|
|
211
|
+
item_id = item_dict.get("id") or secrets.token_hex(6)
|
|
212
|
+
|
|
213
|
+
# CRITICAL: Capture on-disk hash at operation START (before any other reads)
|
|
214
|
+
# This is our baseline for OCC conflict detection. The check window covers
|
|
215
|
+
# the entire operation (read baseline → load tracker → append event → render).
|
|
216
|
+
start_disk_hash = None
|
|
217
|
+
if self.tracker_file.exists():
|
|
218
|
+
try:
|
|
219
|
+
current_on_disk = json.loads(
|
|
220
|
+
self.tracker_file.read_text(encoding="utf-8")
|
|
221
|
+
)
|
|
222
|
+
start_disk_hash = self._compute_content_hash(current_on_disk)
|
|
223
|
+
except Exception:
|
|
224
|
+
# Corrupt file on disk at START of operation
|
|
225
|
+
# Treat as a conflict (fail-closed)
|
|
226
|
+
pass
|
|
227
|
+
|
|
228
|
+
# Check for duplicate explicit ID: if caller provided an ID, verify it's not
|
|
229
|
+
# already in the current projection. This is fail-closed: reject the operation
|
|
230
|
+
# before appending to event store.
|
|
231
|
+
if "id" in item_dict:
|
|
232
|
+
current_tracker = self._load_tracker_safe()
|
|
233
|
+
existing_ids = {item["id"] for item in current_tracker.get("items", [])}
|
|
234
|
+
if item_id in existing_ids:
|
|
235
|
+
raise ValueError(f"Item with id '{item_id}' already exists in projection")
|
|
236
|
+
|
|
237
|
+
created_item = {
|
|
238
|
+
"id": item_id,
|
|
239
|
+
"title": title,
|
|
240
|
+
"priority": item_dict.get("priority", "P1"),
|
|
241
|
+
"status": item_dict.get("status", "todo"),
|
|
242
|
+
"lane": item_dict.get("lane", "proposed"),
|
|
243
|
+
"source": item_dict.get("source", actor),
|
|
244
|
+
"tags": item_dict.get("tags", []) if isinstance(item_dict.get("tags"), list) else [],
|
|
245
|
+
"notes": item_dict.get("notes"),
|
|
246
|
+
"pr_link": item_dict.get("pr_link"),
|
|
247
|
+
"created_at": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"),
|
|
248
|
+
"completed_at": None,
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
store = EventStore(self.db_path)
|
|
252
|
+
|
|
253
|
+
# Append the event (fail-closed: if this fails, no projection write)
|
|
254
|
+
try:
|
|
255
|
+
store.append("tracker", "item_created", created_item, actor)
|
|
256
|
+
except Exception as e:
|
|
257
|
+
raise ValueError(f"Failed to append create event: {e}") from e
|
|
258
|
+
|
|
259
|
+
# Now re-render the projection atomically
|
|
260
|
+
self._render_tracker_atomic(store, start_disk_hash=start_disk_hash)
|
|
261
|
+
|
|
262
|
+
# Return the created item from the freshly projected state
|
|
263
|
+
created_tracker = self._load_tracker_safe()
|
|
264
|
+
created_items = {item["id"]: item for item in created_tracker.get("items", [])}
|
|
265
|
+
|
|
266
|
+
if item_id not in created_items:
|
|
267
|
+
# Should not happen if projection is consistent, but defend against it
|
|
268
|
+
raise ValueError(f"Item disappeared after create: {item_id}")
|
|
269
|
+
|
|
270
|
+
return created_items[item_id]
|
|
271
|
+
|
|
272
|
+
def rebuild_projection(self) -> None:
|
|
273
|
+
"""Rebuild tracker.json from the event store.
|
|
274
|
+
|
|
275
|
+
Renders the projection from events, bypassing OCC conflict detection.
|
|
276
|
+
Use this to recover from orphaned events (event in store, missing from projection).
|
|
277
|
+
|
|
278
|
+
The projection is derived from the event store, so rebuilding naturally recovers
|
|
279
|
+
prior events. This is the self-healing recovery contract: if projection becomes
|
|
280
|
+
stale or corrupted, rebuild_projection() restores consistency.
|
|
281
|
+
|
|
282
|
+
Raises:
|
|
283
|
+
WriteConflict: If atomic write fails (disk write error, not conflict).
|
|
284
|
+
"""
|
|
285
|
+
store = EventStore(self.db_path)
|
|
286
|
+
# Bypass OCC check with force=True (recovery always bypasses conflict detection)
|
|
287
|
+
self._render_tracker_atomic(store, start_disk_hash=None, force=True)
|
|
288
|
+
|
|
289
|
+
# --- Private helpers ---
|
|
290
|
+
|
|
291
|
+
def _load_tracker_safe(self) -> dict:
|
|
292
|
+
"""Load tracker.json, return empty tracker if missing or corrupt.
|
|
293
|
+
|
|
294
|
+
Returns:
|
|
295
|
+
dict: Tracker snapshot ({"version": 1, "items": [...]}) or empty dict
|
|
296
|
+
"""
|
|
297
|
+
if not self.tracker_file.exists():
|
|
298
|
+
return {"version": 1, "items": []}
|
|
299
|
+
|
|
300
|
+
try:
|
|
301
|
+
content = self.tracker_file.read_text(encoding="utf-8")
|
|
302
|
+
data = json.loads(content)
|
|
303
|
+
if not isinstance(data, dict) or "version" not in data:
|
|
304
|
+
return {"version": 1, "items": []}
|
|
305
|
+
return data
|
|
306
|
+
except Exception:
|
|
307
|
+
# Corrupt or unreadable; return empty tracker
|
|
308
|
+
return {"version": 1, "items": []}
|
|
309
|
+
|
|
310
|
+
def _project_tracker(self, store: EventStore) -> dict:
|
|
311
|
+
"""Project the tracker state from the event log.
|
|
312
|
+
|
|
313
|
+
Reads all events from the "tracker" stream and folds them into the
|
|
314
|
+
current tracker state using the standard projection rules.
|
|
315
|
+
|
|
316
|
+
Args:
|
|
317
|
+
store: EventStore instance to read events from
|
|
318
|
+
|
|
319
|
+
Returns:
|
|
320
|
+
dict: Tracker projection ({"version": 1, "items": [...]})
|
|
321
|
+
"""
|
|
322
|
+
try:
|
|
323
|
+
from state_store import project_tracker
|
|
324
|
+
except ImportError:
|
|
325
|
+
from state_store.projections import project_tracker
|
|
326
|
+
|
|
327
|
+
events = store.read("tracker")
|
|
328
|
+
return project_tracker(events)
|
|
329
|
+
|
|
330
|
+
def _compute_content_hash(self, tracker_dict: dict) -> str:
|
|
331
|
+
"""Compute a stable SHA256 hash of the tracker content.
|
|
332
|
+
|
|
333
|
+
Used for conflict detection: if the hash doesn't match expected, a concurrent
|
|
334
|
+
writer has changed the file (either tracker.json directly or another WriteAPI
|
|
335
|
+
caller's projection render).
|
|
336
|
+
|
|
337
|
+
Args:
|
|
338
|
+
tracker_dict: The tracker dict to hash
|
|
339
|
+
|
|
340
|
+
Returns:
|
|
341
|
+
str: Hex-encoded SHA256 hash
|
|
342
|
+
"""
|
|
343
|
+
# Normalize to a canonical JSON form for hashing (sorted keys, no whitespace)
|
|
344
|
+
content = json.dumps(tracker_dict, sort_keys=True, separators=(",", ":"))
|
|
345
|
+
return hashlib.sha256(content.encode("utf-8")).hexdigest()
|
|
346
|
+
|
|
347
|
+
def _render_tracker_atomic(
|
|
348
|
+
self, store: EventStore, start_disk_hash: str | None = None, force: bool = False
|
|
349
|
+
) -> None:
|
|
350
|
+
"""Render the tracker projection to tracker.json atomically.
|
|
351
|
+
|
|
352
|
+
Projects the event log, writes to a temp file, then renames atomically.
|
|
353
|
+
Includes OCC (Optimistic Concurrency Control): before write, detects concurrent
|
|
354
|
+
modification by comparing on-disk state against our projection.
|
|
355
|
+
|
|
356
|
+
Fail-closed: if on-disk content differs from our computed projection AND it
|
|
357
|
+
didn't exist at operation start, raises WriteConflict (another writer has
|
|
358
|
+
modified the file). If on-disk JSON is corrupt, raises WriteConflict (fail-closed).
|
|
359
|
+
|
|
360
|
+
Args:
|
|
361
|
+
store: EventStore instance to project from
|
|
362
|
+
start_disk_hash: Hash of tracker.json at operation START (before event append).
|
|
363
|
+
Used for OCC baseline. If None, assume file didn't exist at start.
|
|
364
|
+
force: If True, bypass OCC check (recovery-only, use rebuild_projection).
|
|
365
|
+
|
|
366
|
+
Raises:
|
|
367
|
+
WriteConflict: If concurrent modification detected or on-disk corruption.
|
|
368
|
+
Event is safely appended; caller must retry.
|
|
369
|
+
"""
|
|
370
|
+
# Project the current state
|
|
371
|
+
projection = self._project_tracker(store)
|
|
372
|
+
new_hash = self._compute_content_hash(projection)
|
|
373
|
+
|
|
374
|
+
# OCC Conflict Detection
|
|
375
|
+
# Only perform if not forcing (recovery use case bypasses check)
|
|
376
|
+
if not force and self.tracker_file.exists():
|
|
377
|
+
try:
|
|
378
|
+
current_on_disk = json.loads(
|
|
379
|
+
self.tracker_file.read_text(encoding="utf-8")
|
|
380
|
+
)
|
|
381
|
+
disk_hash = self._compute_content_hash(current_on_disk)
|
|
382
|
+
|
|
383
|
+
# Conflict detection: fail-closed if disk state is unexplained
|
|
384
|
+
# Check 1: If disk has items not in our projection, it's a conflict
|
|
385
|
+
# (external write or divergent event store)
|
|
386
|
+
disk_item_ids = {item.get("id") for item in current_on_disk.get("items", [])}
|
|
387
|
+
projection_item_ids = {item.get("id") for item in projection.get("items", [])}
|
|
388
|
+
unexplained_items = disk_item_ids - projection_item_ids
|
|
389
|
+
|
|
390
|
+
if unexplained_items:
|
|
391
|
+
raise WriteConflict(
|
|
392
|
+
expected_hash=start_disk_hash,
|
|
393
|
+
actual_hash=disk_hash,
|
|
394
|
+
reason=f"Unexplained disk state: items {unexplained_items} "
|
|
395
|
+
f"on disk but not in event store (divergent state or external write)",
|
|
396
|
+
)
|
|
397
|
+
|
|
398
|
+
# Check 2: If disk differs from both start and projection, it's a concurrent modification
|
|
399
|
+
if disk_hash != new_hash and disk_hash != start_disk_hash:
|
|
400
|
+
raise WriteConflict(
|
|
401
|
+
expected_hash=start_disk_hash,
|
|
402
|
+
actual_hash=disk_hash,
|
|
403
|
+
reason=f"Concurrent modification detected: disk hash {disk_hash[:8]} "
|
|
404
|
+
f"differs from operation start {start_disk_hash[:8] if start_disk_hash else 'N/A'} "
|
|
405
|
+
f"and projection {new_hash[:8]}",
|
|
406
|
+
)
|
|
407
|
+
|
|
408
|
+
except json.JSONDecodeError as e:
|
|
409
|
+
# Corrupt JSON on disk: fail-closed
|
|
410
|
+
raise WriteConflict(
|
|
411
|
+
expected_hash=start_disk_hash,
|
|
412
|
+
actual_hash=None,
|
|
413
|
+
reason=f"Corrupt JSON on disk (cannot detect conflict safely): {e}",
|
|
414
|
+
) from e
|
|
415
|
+
except WriteConflict:
|
|
416
|
+
# Re-raise conflict (don't catch our own exception)
|
|
417
|
+
raise
|
|
418
|
+
except Exception as e:
|
|
419
|
+
# Other read errors (permissions, etc.): fail-closed
|
|
420
|
+
raise WriteConflict(
|
|
421
|
+
expected_hash=start_disk_hash,
|
|
422
|
+
actual_hash=None,
|
|
423
|
+
reason=f"Failed to read tracker.json for conflict check: {e}",
|
|
424
|
+
) from e
|
|
425
|
+
|
|
426
|
+
# Write atomically via tempfile + os.replace
|
|
427
|
+
# Use POSIX-safe temp file creation (works on Windows too via Python's tempfile)
|
|
428
|
+
try:
|
|
429
|
+
fd, temp_path = tempfile.mkstemp(
|
|
430
|
+
suffix=".json",
|
|
431
|
+
prefix=".tracker-",
|
|
432
|
+
dir=str(self.state_dir),
|
|
433
|
+
text=False, # Binary mode for explicit encoding control
|
|
434
|
+
)
|
|
435
|
+
try:
|
|
436
|
+
# Write projection as JSON (indent for git diffability)
|
|
437
|
+
content = json.dumps(projection, indent=2, ensure_ascii=False)
|
|
438
|
+
os.write(fd, content.encode("utf-8"))
|
|
439
|
+
os.close(fd)
|
|
440
|
+
|
|
441
|
+
# Atomic rename (fails if target exists on some systems, but Python's
|
|
442
|
+
# os.replace is cross-platform atomic where the OS supports it)
|
|
443
|
+
os.replace(str(temp_path), str(self.tracker_file))
|
|
444
|
+
|
|
445
|
+
except Exception:
|
|
446
|
+
# Ensure fd is closed on error
|
|
447
|
+
try:
|
|
448
|
+
os.close(fd)
|
|
449
|
+
except Exception:
|
|
450
|
+
pass
|
|
451
|
+
# Clean up temp file
|
|
452
|
+
try:
|
|
453
|
+
os.unlink(temp_path)
|
|
454
|
+
except Exception:
|
|
455
|
+
pass
|
|
456
|
+
raise
|
|
457
|
+
except Exception as e:
|
|
458
|
+
raise WriteConflict(
|
|
459
|
+
expected_hash=start_disk_hash,
|
|
460
|
+
actual_hash=None,
|
|
461
|
+
reason=f"Failed to write tracker.json atomically: {e}",
|
|
462
|
+
) from e
|
package/tools/cost_ceiling.py
CHANGED
|
@@ -14,19 +14,33 @@ Configuration (aesop.config.json):
|
|
|
14
14
|
|
|
15
15
|
Spend source: an explicit --spent/spent= figure always wins. Otherwise spend
|
|
16
16
|
is read from the cost ledger (tools/fleet_ledger.py's OUTCOMES-LEDGER.md under
|
|
17
|
-
the resolved state dir): sum of tokens_in + tokens_out across all ledger rows
|
|
17
|
+
the resolved state dir): sum of tokens_in + tokens_out across all ledger rows
|
|
18
|
+
within the specified window (or all rows if no window specified).
|
|
18
19
|
Missing/unreadable ledger -> spend of 0 (never trips on a ledger that doesn't
|
|
19
20
|
exist yet).
|
|
20
21
|
|
|
22
|
+
WINDOW CONTRACT (shared with cost_projection.py):
|
|
23
|
+
Both tools filter ledger rows by the SAME window parameters to ensure
|
|
24
|
+
consistent spend calculations. The helper calculate_window_bounds() is
|
|
25
|
+
imported by both modules to ensure identical window calculations.
|
|
26
|
+
|
|
27
|
+
When both cost_projection.project(window_minutes=W) and cost_ceiling.check(window_minutes=W)
|
|
28
|
+
are called with the same W value, they will produce identical spend figures.
|
|
29
|
+
|
|
21
30
|
API:
|
|
22
|
-
check(spent=None, period="wave", config=None, state_dir=None, trip=True
|
|
31
|
+
check(spent=None, period="wave", config=None, state_dir=None, trip=True,
|
|
32
|
+
window_minutes=None) -> dict
|
|
23
33
|
Returns {"period", "ceiling", "spent", "exceeded", "tripped"}.
|
|
24
34
|
When exceeded and trip=True, calls tools/halt.py's halt() with a reason
|
|
25
35
|
describing the breach, and "tripped" is True. When ceiling is None
|
|
26
36
|
(unconfigured), exceeded is always False and nothing is ever tripped.
|
|
27
37
|
|
|
38
|
+
window_minutes: Optional time window in minutes for ledger filtering.
|
|
39
|
+
If None, uses all ledger rows (backward compat). If specified, only rows
|
|
40
|
+
within the window contribute to spend calculation.
|
|
41
|
+
|
|
28
42
|
CLI:
|
|
29
|
-
python tools/cost_ceiling.py --check --spent N [--period wave|daily]
|
|
43
|
+
python tools/cost_ceiling.py --check --spent N [--period wave|daily] [--window MINUTES]
|
|
30
44
|
Exit 0 if not exceeded (or ceiling unconfigured), exit 1 if exceeded
|
|
31
45
|
(and thus tripped, unless already halted).
|
|
32
46
|
"""
|
|
@@ -35,7 +49,7 @@ import argparse
|
|
|
35
49
|
import json
|
|
36
50
|
import sys
|
|
37
51
|
from pathlib import Path
|
|
38
|
-
from datetime import datetime, timezone
|
|
52
|
+
from datetime import datetime, timezone, timedelta
|
|
39
53
|
|
|
40
54
|
try:
|
|
41
55
|
import halt
|
|
@@ -45,6 +59,26 @@ except ImportError:
|
|
|
45
59
|
from tools import fleet_ledger
|
|
46
60
|
|
|
47
61
|
|
|
62
|
+
def calculate_window_bounds(window_minutes):
|
|
63
|
+
"""Calculate UTC window start and end times for ledger filtering.
|
|
64
|
+
|
|
65
|
+
Shared contract between cost_projection.py and cost_ceiling.py to ensure
|
|
66
|
+
consistent ledger windowing. Both tools import and use this helper to
|
|
67
|
+
guarantee agreement on spend figures when using the same window.
|
|
68
|
+
|
|
69
|
+
Args:
|
|
70
|
+
window_minutes: Time window in minutes (e.g., 30 = last 30 minutes)
|
|
71
|
+
|
|
72
|
+
Returns:
|
|
73
|
+
Tuple of (window_start_utc, window_end_utc) as datetime objects with UTC timezone.
|
|
74
|
+
window_end_utc is always datetime.now(timezone.utc).
|
|
75
|
+
window_start_utc is window_end_utc minus window_minutes.
|
|
76
|
+
"""
|
|
77
|
+
now_utc = datetime.now(timezone.utc)
|
|
78
|
+
window_start = now_utc - timedelta(minutes=window_minutes)
|
|
79
|
+
return window_start, now_utc
|
|
80
|
+
|
|
81
|
+
|
|
48
82
|
def load_config():
|
|
49
83
|
"""Load aesop.config.json from current directory, return dict (or {} if absent/bad)."""
|
|
50
84
|
config_file = Path("aesop.config.json")
|
|
@@ -73,15 +107,22 @@ def get_ceiling(config, period):
|
|
|
73
107
|
return None
|
|
74
108
|
|
|
75
109
|
|
|
76
|
-
def read_ledger_total_tokens(state_dir, period="wave"):
|
|
77
|
-
"""Sum tokens_in + tokens_out from OUTCOMES-LEDGER.md.
|
|
110
|
+
def read_ledger_total_tokens(state_dir, period="wave", window_minutes=None):
|
|
111
|
+
"""Sum tokens_in + tokens_out from OUTCOMES-LEDGER.md with optional windowing.
|
|
78
112
|
|
|
79
113
|
Args:
|
|
80
114
|
state_dir: path to state directory
|
|
81
115
|
period: "wave" (all rows) or "daily" (today's rows only, filtered by UTC date)
|
|
116
|
+
window_minutes: Optional time window in minutes. If specified, only rows
|
|
117
|
+
within the last window_minutes are included. Default None
|
|
118
|
+
means include all rows (backward compat).
|
|
82
119
|
|
|
83
120
|
Returns 0 if the ledger doesn't exist or is unreadable/empty.
|
|
84
121
|
Uses fleet_ledger.py's shared parser (single source of truth).
|
|
122
|
+
|
|
123
|
+
Window contract: When both cost_projection and cost_ceiling use the same
|
|
124
|
+
window_minutes value, they produce identical spend figures (single source
|
|
125
|
+
of truth is the ledger parse result, and the same filtering is applied).
|
|
85
126
|
"""
|
|
86
127
|
# Set the state root temporarily so fleet_ledger can find the ledger
|
|
87
128
|
import os
|
|
@@ -98,6 +139,12 @@ def read_ledger_total_tokens(state_dir, period="wave"):
|
|
|
98
139
|
if not rows:
|
|
99
140
|
return 0
|
|
100
141
|
|
|
142
|
+
# Calculate window bounds if windowing is requested
|
|
143
|
+
if window_minutes is not None:
|
|
144
|
+
window_start, window_end = calculate_window_bounds(window_minutes)
|
|
145
|
+
else:
|
|
146
|
+
window_start = None # No window filtering
|
|
147
|
+
|
|
101
148
|
total = 0
|
|
102
149
|
|
|
103
150
|
if period == "daily":
|
|
@@ -107,26 +154,57 @@ def read_ledger_total_tokens(state_dir, period="wave"):
|
|
|
107
154
|
# Extract date from ISO timestamp (format: YYYY-MM-DDTHH:MM:SSZ or similar)
|
|
108
155
|
try:
|
|
109
156
|
iso_ts = row['iso_ts']
|
|
110
|
-
# Parse
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
157
|
+
# Parse ISO timestamp
|
|
158
|
+
ts_clean = iso_ts.replace('Z', '+00:00') if 'Z' in iso_ts else iso_ts
|
|
159
|
+
row_dt = datetime.fromisoformat(ts_clean)
|
|
160
|
+
if row_dt.tzinfo is None:
|
|
161
|
+
row_dt = row_dt.replace(tzinfo=timezone.utc)
|
|
162
|
+
|
|
163
|
+
# Check date filter
|
|
164
|
+
row_date = row_dt.date()
|
|
165
|
+
if row_date != today_utc:
|
|
166
|
+
continue
|
|
167
|
+
|
|
168
|
+
# Check window filter (if specified)
|
|
169
|
+
if window_start is not None and row_dt < window_start:
|
|
170
|
+
continue
|
|
171
|
+
|
|
172
|
+
total += row['tokens_in'] + row['tokens_out']
|
|
173
|
+
except (ValueError, IndexError, KeyError, AttributeError):
|
|
115
174
|
# Skip malformed timestamps
|
|
116
175
|
continue
|
|
117
176
|
else:
|
|
118
|
-
# period == "wave": sum all
|
|
177
|
+
# period == "wave": sum rows within window (or all if no window)
|
|
119
178
|
for row in rows:
|
|
179
|
+
# Check window filter (if specified)
|
|
180
|
+
if window_start is not None:
|
|
181
|
+
try:
|
|
182
|
+
iso_ts = row['iso_ts']
|
|
183
|
+
ts_clean = iso_ts.replace('Z', '+00:00') if 'Z' in iso_ts else iso_ts
|
|
184
|
+
row_dt = datetime.fromisoformat(ts_clean)
|
|
185
|
+
if row_dt.tzinfo is None:
|
|
186
|
+
row_dt = row_dt.replace(tzinfo=timezone.utc)
|
|
187
|
+
|
|
188
|
+
if row_dt < window_start:
|
|
189
|
+
continue
|
|
190
|
+
except (ValueError, AttributeError):
|
|
191
|
+
# Skip malformed timestamps
|
|
192
|
+
continue
|
|
193
|
+
|
|
120
194
|
total += row['tokens_in'] + row['tokens_out']
|
|
121
195
|
|
|
122
196
|
return total
|
|
123
197
|
|
|
124
198
|
|
|
125
|
-
def check(spent=None, period="wave", config=None, state_dir=None, trip=True):
|
|
199
|
+
def check(spent=None, period="wave", config=None, state_dir=None, trip=True, window_minutes=None):
|
|
126
200
|
"""Check spend against the configured ceiling for `period`.
|
|
127
201
|
|
|
128
202
|
Returns a dict: {"period", "ceiling", "spent", "exceeded", "tripped", "reason"}.
|
|
129
203
|
|
|
204
|
+
Window contract: when cost_ceiling and cost_projection both pass the same
|
|
205
|
+
window_minutes value, they will compute identical spend figures from the
|
|
206
|
+
ledger, ensuring agreement on cost tracking.
|
|
207
|
+
|
|
130
208
|
Distinctions:
|
|
131
209
|
- Genuine ceiling breach (ceiling is configured, spent >= ceiling): when trip=True,
|
|
132
210
|
writes persistent .HALT sentinel via halt.halt() and sets tripped=True.
|
|
@@ -134,6 +212,18 @@ def check(spent=None, period="wave", config=None, state_dir=None, trip=True):
|
|
|
134
212
|
abort of current wave (exceeded=True) but does NOT write persistent sentinel
|
|
135
213
|
(tripped=False). This preserves fleet availability across transient I/O errors
|
|
136
214
|
(e.g., momentary file lock, disk full) while still aborting the current wave.
|
|
215
|
+
|
|
216
|
+
Args:
|
|
217
|
+
spent: Optional explicit spend figure in tokens. If provided, overrides ledger.
|
|
218
|
+
period: "wave" (all rows) or "daily" (today's rows only)
|
|
219
|
+
config: aesop.config.json dict, or None to load from disk
|
|
220
|
+
state_dir: path to state directory, or None to resolve from config
|
|
221
|
+
trip: If True and exceeded, trip the .HALT sentinel
|
|
222
|
+
window_minutes: Optional time window in minutes for ledger filtering.
|
|
223
|
+
If None, all ledger rows are included (backward compat).
|
|
224
|
+
When specified, only rows within the last window_minutes
|
|
225
|
+
are included. MUST match the window_minutes used in
|
|
226
|
+
cost_projection.project() to ensure agreement.
|
|
137
227
|
"""
|
|
138
228
|
try:
|
|
139
229
|
if config is None:
|
|
@@ -145,7 +235,7 @@ def check(spent=None, period="wave", config=None, state_dir=None, trip=True):
|
|
|
145
235
|
ceiling = get_ceiling(config, period)
|
|
146
236
|
|
|
147
237
|
if spent is None:
|
|
148
|
-
spent = read_ledger_total_tokens(state_dir, period=period)
|
|
238
|
+
spent = read_ledger_total_tokens(state_dir, period=period, window_minutes=window_minutes)
|
|
149
239
|
spent = int(spent)
|
|
150
240
|
|
|
151
241
|
exceeded = ceiling is not None and spent >= ceiling
|
|
@@ -190,13 +280,14 @@ def main(argv=None):
|
|
|
190
280
|
parser.add_argument("--check", action="store_true", help="Run the ceiling check (required).")
|
|
191
281
|
parser.add_argument("--spent", type=int, default=None, help="Explicit spend figure in tokens; defaults to ledger total.")
|
|
192
282
|
parser.add_argument("--period", choices=("wave", "daily"), default="wave", help="Which ceiling to check against (default: wave).")
|
|
283
|
+
parser.add_argument("--window", type=int, default=None, help="Optional time window in minutes for ledger filtering; None means all rows (default, backward compat).")
|
|
193
284
|
args = parser.parse_args(argv)
|
|
194
285
|
|
|
195
286
|
if not args.check:
|
|
196
287
|
parser.print_usage(sys.stderr)
|
|
197
288
|
return 2
|
|
198
289
|
|
|
199
|
-
result = check(spent=args.spent, period=args.period)
|
|
290
|
+
result = check(spent=args.spent, period=args.period, window_minutes=args.window)
|
|
200
291
|
|
|
201
292
|
# Check for errors FIRST (fail-closed): exception during check() means abort the wave
|
|
202
293
|
if "error" in result:
|