@matt82198/aesop 0.1.0 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +117 -2
- package/README.md +59 -218
- package/bin/cli.js +168 -41
- package/daemons/run-watchdog.sh +16 -4
- package/daemons/selfheal.sh +231 -0
- package/docs/ANY-REPO.md +427 -0
- package/docs/CONTRIBUTING.md +72 -0
- package/docs/DEMO.md +334 -0
- package/docs/HOOK-INSTALL.md +15 -56
- package/docs/INSTALL.md +74 -4
- package/docs/PORTING.md +166 -0
- package/docs/README.md +33 -3
- package/docs/TEAM-STATE.md +372 -0
- package/docs/reproduce.md +33 -3
- package/driver/CLAUDE.md +150 -0
- package/driver/README.md +383 -0
- package/driver/aesop.config.example.json +80 -0
- package/driver/agent_driver.py +355 -0
- package/driver/backend_config.py +253 -0
- package/driver/claude_code_driver.py +198 -0
- package/driver/codex_driver.py +673 -0
- package/driver/openai_compatible_driver.py +249 -0
- package/driver/openai_transport.py +179 -0
- package/driver/verification_policy.py +75 -0
- package/driver/wave_bridge.py +254 -0
- package/driver/wave_loop.py +1408 -0
- package/driver/wave_scheduler.py +890 -0
- package/hooks/pre-push-policy.sh +131 -33
- package/mcp/server.mjs +320 -4
- package/monitor/collect-signals.mjs +69 -0
- package/package.json +22 -14
- package/skills/CLAUDE.md +132 -2
- package/skills/buildsystem/SKILL.md +330 -0
- package/skills/buildsystem/wave-flat-dispatch.template.mjs +658 -0
- package/skills/fleet/SKILL.md +113 -0
- package/skills/power/SKILL.md +246 -131
- package/state_store/__init__.py +4 -2
- package/state_store/api.py +19 -4
- package/state_store/coordination.py +209 -0
- package/state_store/identity.py +51 -0
- package/state_store/projections.py +63 -0
- package/state_store/read_api.py +156 -0
- package/state_store/store.py +185 -73
- package/state_store/write_api.py +462 -0
- package/templates/wave-presets/data.json +65 -0
- package/templates/wave-presets/library.json +65 -0
- package/templates/wave-presets/saas.json +64 -0
- package/tools/audit_report.py +388 -0
- package/tools/bench_runner.py +100 -3
- package/tools/ci_merge_wait.py +256 -35
- package/tools/ci_workflow_lint.py +430 -0
- package/tools/claudemd_drift.py +394 -0
- package/tools/claudemd_lint.py +359 -0
- package/tools/common.py +39 -3
- package/tools/cost_ceiling.py +166 -43
- package/tools/cost_econ.py +480 -0
- package/tools/cost_projection.py +559 -0
- package/tools/crossos_drift.py +394 -0
- package/tools/defect_escape.py +252 -0
- package/tools/doctor.js +1 -1
- package/tools/eod_sweep.py +188 -26
- package/tools/fleet.js +260 -0
- package/tools/fleet_ledger.py +209 -7
- package/tools/git_identity_check.py +315 -0
- package/tools/health-score.js +40 -0
- package/tools/health_score.py +361 -0
- package/tools/metrics_gate.py +13 -4
- package/tools/mutation_test.py +523 -0
- package/tools/portability_check.py +206 -0
- package/tools/proposals.mjs +47 -2
- package/tools/reconcile.py +7 -4
- package/tools/reproduce.js +405 -0
- package/tools/secret_scan.py +207 -65
- package/tools/self_stats.py +20 -0
- package/tools/stall_check.py +247 -16
- package/tools/stateapi_lint.py +325 -0
- package/tools/test_battery.py +173 -0
- package/tools/transcript_digest.py +380 -0
- package/tools/verify_activity_filter.py +437 -0
- package/tools/verify_agent_inspector.py +2 -0
- package/tools/verify_cost_panel.py +345 -0
- package/tools/verify_dash.py +2 -0
- package/tools/verify_dispatch_panel.py +301 -0
- package/tools/verify_failure_drilldown.py +188 -0
- package/tools/verify_prboard.py +2 -0
- package/tools/verify_scorecards.py +281 -0
- package/tools/verify_submit_encoding.py +2 -0
- package/tools/verify_ui_trio.py +409 -0
- package/tools/verify_wave_telemetry.py +268 -0
- package/tools/wave_backlog_analyzer.py +490 -0
- package/tools/wave_ledger_hook.py +150 -0
- package/tools/wave_preflight.py +779 -0
- package/tools/wave_resume.py +215 -0
- package/tools/wave_templates.py +340 -0
- package/ui/agents.py +68 -14
- package/ui/collectors.py +81 -55
- package/ui/config.py +7 -2
- package/ui/cost.py +231 -12
- package/ui/handler.py +383 -55
- package/ui/quality_scorecard.py +232 -0
- package/ui/sse.py +3 -3
- package/ui/wave_audit_tail.py +213 -0
- package/ui/wave_dispatch.py +280 -0
- package/ui/wave_failure.py +288 -0
- package/ui/wave_gantt.py +152 -0
- package/ui/wave_reasoning_tail.py +176 -0
- package/ui/wave_telemetry.py +383 -0
- package/ui/web/dist/assets/index-CNQxaiOW.css +1 -0
- package/ui/web/dist/assets/index-CP68RIh3.js +9 -0
- package/ui/web/dist/index.html +2 -2
- package/bin/CLAUDE.md +0 -76
- package/daemons/CLAUDE.md +0 -36
- package/dash/CLAUDE.md +0 -32
- package/docs/archive/README.md +0 -3
- package/docs/archive/spikes/tiered-cognition/ACTIVATION.md +0 -125
- package/docs/archive/spikes/tiered-cognition/DESIGN.md +0 -287
- package/docs/archive/spikes/tiered-cognition/FINDINGS.md +0 -113
- package/docs/archive/spikes/tiered-cognition/README.md +0 -27
- package/docs/archive/spikes/tiered-cognition/aesop-cognition.example.md +0 -32
- package/docs/archive/spikes/tiered-cognition/force-model-policy.merged.mjs +0 -673
- package/docs/archive/spikes/tiered-cognition/strip-tools-hook.mjs +0 -434
- package/hooks/CLAUDE.md +0 -89
- package/mcp/CLAUDE.md +0 -213
- package/monitor/CLAUDE.md +0 -40
- package/scan/CLAUDE.md +0 -30
- package/state_store/CLAUDE.md +0 -39
- package/tools/CLAUDE.md +0 -79
- package/ui/CLAUDE.md +0 -127
- package/ui/web/dist/assets/index-0qQYnvMC.js +0 -9
- package/ui/web/dist/assets/index-BdIlFieV.css +0 -1
|
@@ -0,0 +1,462 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""
|
|
3
|
+
state_store.write_api — Typed write facade for tracker mutations (state consolidation).
|
|
4
|
+
|
|
5
|
+
Consolidates write patterns for tracker mutations: status updates and item creation.
|
|
6
|
+
This facade allows the underlying write implementation to change (immediate projection
|
|
7
|
+
→ queued render → event store publishing) without altering caller code.
|
|
8
|
+
|
|
9
|
+
Mirrors the read_api.py facade pattern: callers use WriteAPI only; backend
|
|
10
|
+
implementation (EventStore + projection rendering) is hidden.
|
|
11
|
+
|
|
12
|
+
Callers use:
|
|
13
|
+
api = WriteAPI(state_dir)
|
|
14
|
+
api.tracker_update_status(item_id, new_status, note="optional note")
|
|
15
|
+
api.tracker_append_item({"title": "...", "priority": "P1", ...})
|
|
16
|
+
|
|
17
|
+
Both operations are fail-closed: event append failure → no projection write.
|
|
18
|
+
Projection write conflicts raise WriteConflict (honest failure, no silent data loss).
|
|
19
|
+
"""
|
|
20
|
+
from __future__ import annotations
|
|
21
|
+
|
|
22
|
+
import hashlib
|
|
23
|
+
import json
|
|
24
|
+
import os
|
|
25
|
+
import sys
|
|
26
|
+
import tempfile
|
|
27
|
+
from datetime import datetime, timezone
|
|
28
|
+
from pathlib import Path
|
|
29
|
+
|
|
30
|
+
# Ensure tools and state_store modules are importable
|
|
31
|
+
repo_root = Path(__file__).parent.parent
|
|
32
|
+
if str(repo_root) not in sys.path:
|
|
33
|
+
sys.path.insert(0, str(repo_root))
|
|
34
|
+
|
|
35
|
+
try:
|
|
36
|
+
from state_store import EventStore, ConcurrencyConflict
|
|
37
|
+
except ImportError:
|
|
38
|
+
from state_store.store import EventStore, ConcurrencyConflict
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class WriteConflict(Exception):
|
|
42
|
+
"""Raised when a projection write conflicts (content-hash mismatch).
|
|
43
|
+
|
|
44
|
+
Signifies that the tracker.json file's content hash does not match the
|
|
45
|
+
expected value, indicating concurrent modification. The event was appended
|
|
46
|
+
(durable in EventStore), but the projection write was skipped to prevent
|
|
47
|
+
silent data loss. Caller should re-read tracker.json, extract new version,
|
|
48
|
+
and retry.
|
|
49
|
+
|
|
50
|
+
Attributes:
|
|
51
|
+
expected_hash: The content hash caller expected for tracker.json
|
|
52
|
+
actual_hash: The content hash found on disk
|
|
53
|
+
reason: Human-readable description of the conflict
|
|
54
|
+
"""
|
|
55
|
+
|
|
56
|
+
def __init__(self, expected_hash: str | None, actual_hash: str | None, reason: str = ""):
|
|
57
|
+
self.expected_hash = expected_hash
|
|
58
|
+
self.actual_hash = actual_hash
|
|
59
|
+
self.reason = reason
|
|
60
|
+
super().__init__(
|
|
61
|
+
f"Projection write conflict: {reason} "
|
|
62
|
+
f"(expected hash {expected_hash}, found {actual_hash})"
|
|
63
|
+
)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
class WriteAPI:
|
|
67
|
+
"""Write facade for tracker mutations, backed by EventStore + atomic projection rendering.
|
|
68
|
+
|
|
69
|
+
Designed to be swappable: write backend can change (immediate render → event sourcing)
|
|
70
|
+
without altering call sites. Current implementation appends to event log and re-renders
|
|
71
|
+
tracker.json atomically (tempfile + os.replace).
|
|
72
|
+
|
|
73
|
+
All write operations are fail-closed: if event append fails, projection is not written.
|
|
74
|
+
If projection write fails due to concurrent modification, raises WriteConflict (event
|
|
75
|
+
is safely in the log, caller must retry).
|
|
76
|
+
|
|
77
|
+
OCC (Optimistic Concurrency Control): Each WriteAPI instance tracks the last projection
|
|
78
|
+
hash it wrote (or read at init). Before atomic write, if on-disk hash differs from
|
|
79
|
+
both tracked_hash and new_hash, raises WriteConflict to prevent overwriting concurrent
|
|
80
|
+
modifications. First-write case: hash is captured at operation start (before event append).
|
|
81
|
+
"""
|
|
82
|
+
|
|
83
|
+
def __init__(self, state_dir: str | Path):
|
|
84
|
+
"""Initialize the write API with a state directory.
|
|
85
|
+
|
|
86
|
+
Args:
|
|
87
|
+
state_dir: Path to the state directory (e.g., "state" or "/absolute/path/state")
|
|
88
|
+
"""
|
|
89
|
+
self.state_dir = Path(state_dir)
|
|
90
|
+
self.state_dir.mkdir(parents=True, exist_ok=True)
|
|
91
|
+
self.db_path = str(self.state_dir / "tracker_events.db")
|
|
92
|
+
self.tracker_file = self.state_dir / "tracker.json"
|
|
93
|
+
|
|
94
|
+
def tracker_update_status(
|
|
95
|
+
self,
|
|
96
|
+
item_id: str,
|
|
97
|
+
new_status: str,
|
|
98
|
+
note: str | None = None,
|
|
99
|
+
actor: str = "api",
|
|
100
|
+
) -> dict:
|
|
101
|
+
"""Update an existing tracker item's status and optionally add a note.
|
|
102
|
+
|
|
103
|
+
Appends an item_updated event to the event log, then re-renders tracker.json
|
|
104
|
+
atomically. Fail-closed: event append failure blocks projection write.
|
|
105
|
+
|
|
106
|
+
Args:
|
|
107
|
+
item_id: The item UUID to update
|
|
108
|
+
new_status: New status (e.g., "todo", "in-progress", "done", "archived")
|
|
109
|
+
note: Optional note to append to the item's notes field
|
|
110
|
+
actor: Actor performing the update (default "api")
|
|
111
|
+
|
|
112
|
+
Returns:
|
|
113
|
+
dict: The updated item from the tracker projection
|
|
114
|
+
|
|
115
|
+
Raises:
|
|
116
|
+
ValueError: If item_id not found or other validation failure
|
|
117
|
+
WriteConflict: If projection write fails due to concurrent modification
|
|
118
|
+
ConcurrencyConflict: If EventStore append hits OCC mismatch (should not happen
|
|
119
|
+
in this phase, but reserved for future use)
|
|
120
|
+
"""
|
|
121
|
+
store = EventStore(self.db_path)
|
|
122
|
+
|
|
123
|
+
# Read current tracker to find the item
|
|
124
|
+
current_tracker = self._load_tracker_safe()
|
|
125
|
+
current_items = {item["id"]: item for item in current_tracker.get("items", [])}
|
|
126
|
+
|
|
127
|
+
if item_id not in current_items:
|
|
128
|
+
raise ValueError(f"Item not found: {item_id}")
|
|
129
|
+
|
|
130
|
+
current_item = current_items[item_id]
|
|
131
|
+
|
|
132
|
+
# Build the update payload
|
|
133
|
+
update_payload = {"id": item_id, "status": new_status}
|
|
134
|
+
|
|
135
|
+
# If adding a note, append to the item's notes field
|
|
136
|
+
if note:
|
|
137
|
+
existing_notes = current_item.get("notes", "")
|
|
138
|
+
if existing_notes:
|
|
139
|
+
update_payload["notes"] = f"{existing_notes}\n{note}"
|
|
140
|
+
else:
|
|
141
|
+
update_payload["notes"] = note
|
|
142
|
+
|
|
143
|
+
# CRITICAL: Capture on-disk hash at operation START (before event append)
|
|
144
|
+
# This is our baseline for OCC conflict detection.
|
|
145
|
+
start_disk_hash = None
|
|
146
|
+
if self.tracker_file.exists():
|
|
147
|
+
try:
|
|
148
|
+
current_on_disk = json.loads(
|
|
149
|
+
self.tracker_file.read_text(encoding="utf-8")
|
|
150
|
+
)
|
|
151
|
+
start_disk_hash = self._compute_content_hash(current_on_disk)
|
|
152
|
+
except Exception:
|
|
153
|
+
# Corrupt file on disk at START: treat as conflict (fail-closed)
|
|
154
|
+
pass
|
|
155
|
+
|
|
156
|
+
# Append the event (fail-closed: if this fails, no projection write)
|
|
157
|
+
try:
|
|
158
|
+
store.append("tracker", "item_updated", update_payload, actor)
|
|
159
|
+
except Exception as e:
|
|
160
|
+
raise ValueError(f"Failed to append update event: {e}") from e
|
|
161
|
+
|
|
162
|
+
# Now re-render the projection atomically
|
|
163
|
+
self._render_tracker_atomic(store, start_disk_hash=start_disk_hash)
|
|
164
|
+
|
|
165
|
+
# Return the updated item from the freshly projected state
|
|
166
|
+
updated_tracker = self._load_tracker_safe()
|
|
167
|
+
updated_items = {item["id"]: item for item in updated_tracker.get("items", [])}
|
|
168
|
+
|
|
169
|
+
if item_id not in updated_items:
|
|
170
|
+
# Should not happen if projection is consistent, but defend against it
|
|
171
|
+
raise ValueError(f"Item disappeared after update: {item_id}")
|
|
172
|
+
|
|
173
|
+
return updated_items[item_id]
|
|
174
|
+
|
|
175
|
+
def tracker_append_item(
|
|
176
|
+
self,
|
|
177
|
+
item_dict: dict,
|
|
178
|
+
actor: str = "api",
|
|
179
|
+
) -> dict:
|
|
180
|
+
"""Create a new tracker item.
|
|
181
|
+
|
|
182
|
+
Validates the item dict, appends an item_created event to the event log,
|
|
183
|
+
then re-renders tracker.json atomically. Fail-closed: event append failure
|
|
184
|
+
blocks projection write.
|
|
185
|
+
|
|
186
|
+
Args:
|
|
187
|
+
item_dict: Item dict with fields: id (optional, auto-generated if missing),
|
|
188
|
+
title, priority (optional, defaults to "P1"), status (optional,
|
|
189
|
+
defaults to "todo"), lane (optional, defaults to "proposed"),
|
|
190
|
+
source (optional, defaults to "api"), tags, notes, pr_link, etc.
|
|
191
|
+
actor: Actor performing the create (default "api")
|
|
192
|
+
|
|
193
|
+
Returns:
|
|
194
|
+
dict: The created item from the tracker projection
|
|
195
|
+
|
|
196
|
+
Raises:
|
|
197
|
+
ValueError: If item_dict is invalid, missing required fields, or explicit ID
|
|
198
|
+
already exists in projection
|
|
199
|
+
WriteConflict: If projection write fails due to concurrent modification
|
|
200
|
+
ConcurrencyConflict: If EventStore append hits OCC mismatch
|
|
201
|
+
"""
|
|
202
|
+
if not isinstance(item_dict, dict):
|
|
203
|
+
raise ValueError("item_dict must be a dict")
|
|
204
|
+
|
|
205
|
+
title = item_dict.get("title", "").strip()
|
|
206
|
+
if not title:
|
|
207
|
+
raise ValueError("item_dict must have a non-empty 'title' field")
|
|
208
|
+
|
|
209
|
+
# Build the canonical item structure
|
|
210
|
+
import secrets
|
|
211
|
+
item_id = item_dict.get("id") or secrets.token_hex(6)
|
|
212
|
+
|
|
213
|
+
# CRITICAL: Capture on-disk hash at operation START (before any other reads)
|
|
214
|
+
# This is our baseline for OCC conflict detection. The check window covers
|
|
215
|
+
# the entire operation (read baseline → load tracker → append event → render).
|
|
216
|
+
start_disk_hash = None
|
|
217
|
+
if self.tracker_file.exists():
|
|
218
|
+
try:
|
|
219
|
+
current_on_disk = json.loads(
|
|
220
|
+
self.tracker_file.read_text(encoding="utf-8")
|
|
221
|
+
)
|
|
222
|
+
start_disk_hash = self._compute_content_hash(current_on_disk)
|
|
223
|
+
except Exception:
|
|
224
|
+
# Corrupt file on disk at START of operation
|
|
225
|
+
# Treat as a conflict (fail-closed)
|
|
226
|
+
pass
|
|
227
|
+
|
|
228
|
+
# Check for duplicate explicit ID: if caller provided an ID, verify it's not
|
|
229
|
+
# already in the current projection. This is fail-closed: reject the operation
|
|
230
|
+
# before appending to event store.
|
|
231
|
+
if "id" in item_dict:
|
|
232
|
+
current_tracker = self._load_tracker_safe()
|
|
233
|
+
existing_ids = {item["id"] for item in current_tracker.get("items", [])}
|
|
234
|
+
if item_id in existing_ids:
|
|
235
|
+
raise ValueError(f"Item with id '{item_id}' already exists in projection")
|
|
236
|
+
|
|
237
|
+
created_item = {
|
|
238
|
+
"id": item_id,
|
|
239
|
+
"title": title,
|
|
240
|
+
"priority": item_dict.get("priority", "P1"),
|
|
241
|
+
"status": item_dict.get("status", "todo"),
|
|
242
|
+
"lane": item_dict.get("lane", "proposed"),
|
|
243
|
+
"source": item_dict.get("source", actor),
|
|
244
|
+
"tags": item_dict.get("tags", []) if isinstance(item_dict.get("tags"), list) else [],
|
|
245
|
+
"notes": item_dict.get("notes"),
|
|
246
|
+
"pr_link": item_dict.get("pr_link"),
|
|
247
|
+
"created_at": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"),
|
|
248
|
+
"completed_at": None,
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
store = EventStore(self.db_path)
|
|
252
|
+
|
|
253
|
+
# Append the event (fail-closed: if this fails, no projection write)
|
|
254
|
+
try:
|
|
255
|
+
store.append("tracker", "item_created", created_item, actor)
|
|
256
|
+
except Exception as e:
|
|
257
|
+
raise ValueError(f"Failed to append create event: {e}") from e
|
|
258
|
+
|
|
259
|
+
# Now re-render the projection atomically
|
|
260
|
+
self._render_tracker_atomic(store, start_disk_hash=start_disk_hash)
|
|
261
|
+
|
|
262
|
+
# Return the created item from the freshly projected state
|
|
263
|
+
created_tracker = self._load_tracker_safe()
|
|
264
|
+
created_items = {item["id"]: item for item in created_tracker.get("items", [])}
|
|
265
|
+
|
|
266
|
+
if item_id not in created_items:
|
|
267
|
+
# Should not happen if projection is consistent, but defend against it
|
|
268
|
+
raise ValueError(f"Item disappeared after create: {item_id}")
|
|
269
|
+
|
|
270
|
+
return created_items[item_id]
|
|
271
|
+
|
|
272
|
+
def rebuild_projection(self) -> None:
|
|
273
|
+
"""Rebuild tracker.json from the event store.
|
|
274
|
+
|
|
275
|
+
Renders the projection from events, bypassing OCC conflict detection.
|
|
276
|
+
Use this to recover from orphaned events (event in store, missing from projection).
|
|
277
|
+
|
|
278
|
+
The projection is derived from the event store, so rebuilding naturally recovers
|
|
279
|
+
prior events. This is the self-healing recovery contract: if projection becomes
|
|
280
|
+
stale or corrupted, rebuild_projection() restores consistency.
|
|
281
|
+
|
|
282
|
+
Raises:
|
|
283
|
+
WriteConflict: If atomic write fails (disk write error, not conflict).
|
|
284
|
+
"""
|
|
285
|
+
store = EventStore(self.db_path)
|
|
286
|
+
# Bypass OCC check with force=True (recovery always bypasses conflict detection)
|
|
287
|
+
self._render_tracker_atomic(store, start_disk_hash=None, force=True)
|
|
288
|
+
|
|
289
|
+
# --- Private helpers ---
|
|
290
|
+
|
|
291
|
+
def _load_tracker_safe(self) -> dict:
|
|
292
|
+
"""Load tracker.json, return empty tracker if missing or corrupt.
|
|
293
|
+
|
|
294
|
+
Returns:
|
|
295
|
+
dict: Tracker snapshot ({"version": 1, "items": [...]}) or empty dict
|
|
296
|
+
"""
|
|
297
|
+
if not self.tracker_file.exists():
|
|
298
|
+
return {"version": 1, "items": []}
|
|
299
|
+
|
|
300
|
+
try:
|
|
301
|
+
content = self.tracker_file.read_text(encoding="utf-8")
|
|
302
|
+
data = json.loads(content)
|
|
303
|
+
if not isinstance(data, dict) or "version" not in data:
|
|
304
|
+
return {"version": 1, "items": []}
|
|
305
|
+
return data
|
|
306
|
+
except Exception:
|
|
307
|
+
# Corrupt or unreadable; return empty tracker
|
|
308
|
+
return {"version": 1, "items": []}
|
|
309
|
+
|
|
310
|
+
def _project_tracker(self, store: EventStore) -> dict:
|
|
311
|
+
"""Project the tracker state from the event log.
|
|
312
|
+
|
|
313
|
+
Reads all events from the "tracker" stream and folds them into the
|
|
314
|
+
current tracker state using the standard projection rules.
|
|
315
|
+
|
|
316
|
+
Args:
|
|
317
|
+
store: EventStore instance to read events from
|
|
318
|
+
|
|
319
|
+
Returns:
|
|
320
|
+
dict: Tracker projection ({"version": 1, "items": [...]})
|
|
321
|
+
"""
|
|
322
|
+
try:
|
|
323
|
+
from state_store import project_tracker
|
|
324
|
+
except ImportError:
|
|
325
|
+
from state_store.projections import project_tracker
|
|
326
|
+
|
|
327
|
+
events = store.read("tracker")
|
|
328
|
+
return project_tracker(events)
|
|
329
|
+
|
|
330
|
+
def _compute_content_hash(self, tracker_dict: dict) -> str:
|
|
331
|
+
"""Compute a stable SHA256 hash of the tracker content.
|
|
332
|
+
|
|
333
|
+
Used for conflict detection: if the hash doesn't match expected, a concurrent
|
|
334
|
+
writer has changed the file (either tracker.json directly or another WriteAPI
|
|
335
|
+
caller's projection render).
|
|
336
|
+
|
|
337
|
+
Args:
|
|
338
|
+
tracker_dict: The tracker dict to hash
|
|
339
|
+
|
|
340
|
+
Returns:
|
|
341
|
+
str: Hex-encoded SHA256 hash
|
|
342
|
+
"""
|
|
343
|
+
# Normalize to a canonical JSON form for hashing (sorted keys, no whitespace)
|
|
344
|
+
content = json.dumps(tracker_dict, sort_keys=True, separators=(",", ":"))
|
|
345
|
+
return hashlib.sha256(content.encode("utf-8")).hexdigest()
|
|
346
|
+
|
|
347
|
+
def _render_tracker_atomic(
|
|
348
|
+
self, store: EventStore, start_disk_hash: str | None = None, force: bool = False
|
|
349
|
+
) -> None:
|
|
350
|
+
"""Render the tracker projection to tracker.json atomically.
|
|
351
|
+
|
|
352
|
+
Projects the event log, writes to a temp file, then renames atomically.
|
|
353
|
+
Includes OCC (Optimistic Concurrency Control): before write, detects concurrent
|
|
354
|
+
modification by comparing on-disk state against our projection.
|
|
355
|
+
|
|
356
|
+
Fail-closed: if on-disk content differs from our computed projection AND it
|
|
357
|
+
didn't exist at operation start, raises WriteConflict (another writer has
|
|
358
|
+
modified the file). If on-disk JSON is corrupt, raises WriteConflict (fail-closed).
|
|
359
|
+
|
|
360
|
+
Args:
|
|
361
|
+
store: EventStore instance to project from
|
|
362
|
+
start_disk_hash: Hash of tracker.json at operation START (before event append).
|
|
363
|
+
Used for OCC baseline. If None, assume file didn't exist at start.
|
|
364
|
+
force: If True, bypass OCC check (recovery-only, use rebuild_projection).
|
|
365
|
+
|
|
366
|
+
Raises:
|
|
367
|
+
WriteConflict: If concurrent modification detected or on-disk corruption.
|
|
368
|
+
Event is safely appended; caller must retry.
|
|
369
|
+
"""
|
|
370
|
+
# Project the current state
|
|
371
|
+
projection = self._project_tracker(store)
|
|
372
|
+
new_hash = self._compute_content_hash(projection)
|
|
373
|
+
|
|
374
|
+
# OCC Conflict Detection
|
|
375
|
+
# Only perform if not forcing (recovery use case bypasses check)
|
|
376
|
+
if not force and self.tracker_file.exists():
|
|
377
|
+
try:
|
|
378
|
+
current_on_disk = json.loads(
|
|
379
|
+
self.tracker_file.read_text(encoding="utf-8")
|
|
380
|
+
)
|
|
381
|
+
disk_hash = self._compute_content_hash(current_on_disk)
|
|
382
|
+
|
|
383
|
+
# Conflict detection: fail-closed if disk state is unexplained
|
|
384
|
+
# Check 1: If disk has items not in our projection, it's a conflict
|
|
385
|
+
# (external write or divergent event store)
|
|
386
|
+
disk_item_ids = {item.get("id") for item in current_on_disk.get("items", [])}
|
|
387
|
+
projection_item_ids = {item.get("id") for item in projection.get("items", [])}
|
|
388
|
+
unexplained_items = disk_item_ids - projection_item_ids
|
|
389
|
+
|
|
390
|
+
if unexplained_items:
|
|
391
|
+
raise WriteConflict(
|
|
392
|
+
expected_hash=start_disk_hash,
|
|
393
|
+
actual_hash=disk_hash,
|
|
394
|
+
reason=f"Unexplained disk state: items {unexplained_items} "
|
|
395
|
+
f"on disk but not in event store (divergent state or external write)",
|
|
396
|
+
)
|
|
397
|
+
|
|
398
|
+
# Check 2: If disk differs from both start and projection, it's a concurrent modification
|
|
399
|
+
if disk_hash != new_hash and disk_hash != start_disk_hash:
|
|
400
|
+
raise WriteConflict(
|
|
401
|
+
expected_hash=start_disk_hash,
|
|
402
|
+
actual_hash=disk_hash,
|
|
403
|
+
reason=f"Concurrent modification detected: disk hash {disk_hash[:8]} "
|
|
404
|
+
f"differs from operation start {start_disk_hash[:8] if start_disk_hash else 'N/A'} "
|
|
405
|
+
f"and projection {new_hash[:8]}",
|
|
406
|
+
)
|
|
407
|
+
|
|
408
|
+
except json.JSONDecodeError as e:
|
|
409
|
+
# Corrupt JSON on disk: fail-closed
|
|
410
|
+
raise WriteConflict(
|
|
411
|
+
expected_hash=start_disk_hash,
|
|
412
|
+
actual_hash=None,
|
|
413
|
+
reason=f"Corrupt JSON on disk (cannot detect conflict safely): {e}",
|
|
414
|
+
) from e
|
|
415
|
+
except WriteConflict:
|
|
416
|
+
# Re-raise conflict (don't catch our own exception)
|
|
417
|
+
raise
|
|
418
|
+
except Exception as e:
|
|
419
|
+
# Other read errors (permissions, etc.): fail-closed
|
|
420
|
+
raise WriteConflict(
|
|
421
|
+
expected_hash=start_disk_hash,
|
|
422
|
+
actual_hash=None,
|
|
423
|
+
reason=f"Failed to read tracker.json for conflict check: {e}",
|
|
424
|
+
) from e
|
|
425
|
+
|
|
426
|
+
# Write atomically via tempfile + os.replace
|
|
427
|
+
# Use POSIX-safe temp file creation (works on Windows too via Python's tempfile)
|
|
428
|
+
try:
|
|
429
|
+
fd, temp_path = tempfile.mkstemp(
|
|
430
|
+
suffix=".json",
|
|
431
|
+
prefix=".tracker-",
|
|
432
|
+
dir=str(self.state_dir),
|
|
433
|
+
text=False, # Binary mode for explicit encoding control
|
|
434
|
+
)
|
|
435
|
+
try:
|
|
436
|
+
# Write projection as JSON (indent for git diffability)
|
|
437
|
+
content = json.dumps(projection, indent=2, ensure_ascii=False)
|
|
438
|
+
os.write(fd, content.encode("utf-8"))
|
|
439
|
+
os.close(fd)
|
|
440
|
+
|
|
441
|
+
# Atomic rename (fails if target exists on some systems, but Python's
|
|
442
|
+
# os.replace is cross-platform atomic where the OS supports it)
|
|
443
|
+
os.replace(str(temp_path), str(self.tracker_file))
|
|
444
|
+
|
|
445
|
+
except Exception:
|
|
446
|
+
# Ensure fd is closed on error
|
|
447
|
+
try:
|
|
448
|
+
os.close(fd)
|
|
449
|
+
except Exception:
|
|
450
|
+
pass
|
|
451
|
+
# Clean up temp file
|
|
452
|
+
try:
|
|
453
|
+
os.unlink(temp_path)
|
|
454
|
+
except Exception:
|
|
455
|
+
pass
|
|
456
|
+
raise
|
|
457
|
+
except Exception as e:
|
|
458
|
+
raise WriteConflict(
|
|
459
|
+
expected_hash=start_disk_hash,
|
|
460
|
+
actual_hash=None,
|
|
461
|
+
reason=f"Failed to write tracker.json atomically: {e}",
|
|
462
|
+
) from e
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "Data Pipeline & Analytics",
|
|
3
|
+
"description": "Bootstrap a data project wave with ETL pipeline, data warehouse, and analytics components",
|
|
4
|
+
"categories": ["data-engineering", "analytics", "infrastructure"],
|
|
5
|
+
"items": [
|
|
6
|
+
{
|
|
7
|
+
"slug": "etl-pipeline",
|
|
8
|
+
"ownsFiles": [
|
|
9
|
+
"src/pipeline/extract.py",
|
|
10
|
+
"src/pipeline/transform.py",
|
|
11
|
+
"src/pipeline/load.py",
|
|
12
|
+
"src/pipeline/main.py",
|
|
13
|
+
"src/pipeline/config.yaml",
|
|
14
|
+
"tests/pipeline/",
|
|
15
|
+
"dags/",
|
|
16
|
+
"requirements.txt"
|
|
17
|
+
],
|
|
18
|
+
"prompt": "Build the {project_name} ETL pipeline. Implement data extraction from CSV/APIs, transformation logic for cleaning and aggregation, and loading into a data warehouse. Use Apache Airflow or Python scheduler for orchestration. Include error handling, logging, and retry logic. Create sample DAGs that run daily. Tests must validate transform outputs.",
|
|
19
|
+
"testCmd": "python -m pytest tests/pipeline/ -v --cov=src/pipeline",
|
|
20
|
+
"workDir": "{base_dir}"
|
|
21
|
+
},
|
|
22
|
+
{
|
|
23
|
+
"slug": "data-warehouse",
|
|
24
|
+
"ownsFiles": [
|
|
25
|
+
"warehouse/schema/",
|
|
26
|
+
"warehouse/migrations/",
|
|
27
|
+
"warehouse/seeds/",
|
|
28
|
+
"warehouse/macros/",
|
|
29
|
+
"dbt_project.yml",
|
|
30
|
+
"warehouse/models/",
|
|
31
|
+
"tests/warehouse/"
|
|
32
|
+
],
|
|
33
|
+
"prompt": "Set up the {project_name} data warehouse using dbt or SQL-based approach. Design fact and dimension tables for analytics. Create data models, transformations, and data quality tests. Build marts for reporting (daily sales, user cohorts, product performance). Use {base_dir}/warehouse as root. Ensure idempotent, incremental loading strategies.",
|
|
34
|
+
"testCmd": "dbt test --select +state:modified --state ./target",
|
|
35
|
+
"workDir": "{base_dir}"
|
|
36
|
+
},
|
|
37
|
+
{
|
|
38
|
+
"slug": "analytics-dashboard",
|
|
39
|
+
"ownsFiles": [
|
|
40
|
+
"src/dashboard/app.py",
|
|
41
|
+
"src/dashboard/pages/",
|
|
42
|
+
"src/dashboard/components/",
|
|
43
|
+
"src/dashboard/utils/",
|
|
44
|
+
"tests/dashboard/",
|
|
45
|
+
"requirements-dash.txt"
|
|
46
|
+
],
|
|
47
|
+
"prompt": "Build the {project_name} analytics dashboard using Streamlit or Plotly Dash. Create interactive visualizations for: daily revenue trends, user acquisition cohorts, product performance leaderboard. Connect to the data warehouse for live queries. Add filters and date pickers. Deploy instructions for local and cloud. Responsive and accessible design.",
|
|
48
|
+
"testCmd": "python -m pytest tests/dashboard/ -v && streamlit run src/dashboard/app.py --logger.level=debug 2>&1 | head -50",
|
|
49
|
+
"workDir": "{base_dir}"
|
|
50
|
+
},
|
|
51
|
+
{
|
|
52
|
+
"slug": "data-quality",
|
|
53
|
+
"ownsFiles": [
|
|
54
|
+
"src/quality/checks.py",
|
|
55
|
+
"src/quality/validations.py",
|
|
56
|
+
"src/quality/reports.py",
|
|
57
|
+
"tests/quality/",
|
|
58
|
+
"config/quality.yaml"
|
|
59
|
+
],
|
|
60
|
+
"prompt": "Implement data quality framework for {project_name}. Create validation rules for data freshness, null percentages, value ranges. Build automated checks that run after ETL. Generate quality reports (pass/fail/anomalies). Integrate with monitoring (alerting on failures). Include documentation on data lineage and SLAs.",
|
|
61
|
+
"testCmd": "python -m pytest tests/quality/ -v",
|
|
62
|
+
"workDir": "{base_dir}"
|
|
63
|
+
}
|
|
64
|
+
]
|
|
65
|
+
}
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "Reusable Library",
|
|
3
|
+
"description": "Bootstrap a reusable library wave with core functionality, comprehensive tests, and documentation",
|
|
4
|
+
"categories": ["library", "sdk", "tooling"],
|
|
5
|
+
"items": [
|
|
6
|
+
{
|
|
7
|
+
"slug": "core-module",
|
|
8
|
+
"ownsFiles": [
|
|
9
|
+
"src/{project_name}/__init__.py",
|
|
10
|
+
"src/{project_name}/core.py",
|
|
11
|
+
"src/{project_name}/utils.py",
|
|
12
|
+
"src/{project_name}/types.py"
|
|
13
|
+
],
|
|
14
|
+
"prompt": "Build the core module for the {project_name} library. Implement the primary API/functions with clear interfaces. Export public functions in __init__.py. Include type hints using Python typing module. Add docstrings to all public functions (Google/NumPy style). Ensure the module is importable and has zero external dependencies (only stdlib or pinned minimal deps).",
|
|
15
|
+
"testCmd": "python -c \"import {project_name}; print(dir({project_name}))\" && python -m pytest tests/unit/ -v",
|
|
16
|
+
"workDir": "{base_dir}"
|
|
17
|
+
},
|
|
18
|
+
{
|
|
19
|
+
"slug": "comprehensive-tests",
|
|
20
|
+
"ownsFiles": [
|
|
21
|
+
"tests/unit/",
|
|
22
|
+
"tests/integration/",
|
|
23
|
+
"tests/conftest.py",
|
|
24
|
+
"tests/fixtures/",
|
|
25
|
+
"coverage.xml",
|
|
26
|
+
".coveragerc",
|
|
27
|
+
"tests/README.md"
|
|
28
|
+
],
|
|
29
|
+
"prompt": "Write comprehensive test suite for {project_name}. Create unit tests for all core functions (happy path, edge cases, error conditions). Write integration tests for end-to-end flows. Aim for >90% code coverage. Use pytest framework with fixtures for reusable test data. Include performance benchmarks. Generate coverage reports in XML and HTML formats. Document test patterns.",
|
|
30
|
+
"testCmd": "python -m pytest tests/ -v --cov={project_name} --cov-report=term-missing --cov-report=html",
|
|
31
|
+
"workDir": "{base_dir}"
|
|
32
|
+
},
|
|
33
|
+
{
|
|
34
|
+
"slug": "documentation",
|
|
35
|
+
"ownsFiles": [
|
|
36
|
+
"docs/",
|
|
37
|
+
"README.md",
|
|
38
|
+
"docs/conf.py",
|
|
39
|
+
"docs/source/",
|
|
40
|
+
"docs/Makefile",
|
|
41
|
+
"CONTRIBUTING.md",
|
|
42
|
+
"CHANGELOG.md",
|
|
43
|
+
"LICENSE"
|
|
44
|
+
],
|
|
45
|
+
"prompt": "Create comprehensive documentation for {project_name}. Write a clear README with quick-start examples. Build Sphinx docs with: getting-started guide, API reference (auto-generated from docstrings), design docs, troubleshooting. Add contribution guidelines and changelog. Include code examples. Make docs buildable locally (HTML/PDF). Add documentation to the build CI pipeline.",
|
|
46
|
+
"testCmd": "cd docs && sphinx-build -W -b html source _build/html && ls -la _build/html/index.html",
|
|
47
|
+
"workDir": "{base_dir}"
|
|
48
|
+
},
|
|
49
|
+
{
|
|
50
|
+
"slug": "packaging-release",
|
|
51
|
+
"ownsFiles": [
|
|
52
|
+
"pyproject.toml",
|
|
53
|
+
"setup.cfg",
|
|
54
|
+
"setup.py",
|
|
55
|
+
"MANIFEST.in",
|
|
56
|
+
".github/workflows/publish.yml",
|
|
57
|
+
"PKG-INFO",
|
|
58
|
+
"VERSION"
|
|
59
|
+
],
|
|
60
|
+
"prompt": "Set up packaging and release infrastructure for {project_name}. Configure pyproject.toml (PEP 517/518) with metadata, dependencies, and entry points. Create GitHub Actions workflow for publishing to PyPI (semantic versioning, release tags). Write instructions for local testing with pip install -e. Include security scanning (audit for vulnerabilities). Ensure the package is installable and validated before release.",
|
|
61
|
+
"testCmd": "python -m build --sdist --wheel . && python -m twine check dist/*",
|
|
62
|
+
"workDir": "{base_dir}"
|
|
63
|
+
}
|
|
64
|
+
]
|
|
65
|
+
}
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "SaaS Web Application",
|
|
3
|
+
"description": "Bootstrap a SaaS application wave with backend API, frontend, and infrastructure components",
|
|
4
|
+
"categories": ["backend", "frontend", "infrastructure"],
|
|
5
|
+
"items": [
|
|
6
|
+
{
|
|
7
|
+
"slug": "backend-api",
|
|
8
|
+
"ownsFiles": [
|
|
9
|
+
"src/api/main.py",
|
|
10
|
+
"src/api/models.py",
|
|
11
|
+
"src/api/routes/",
|
|
12
|
+
"tests/api/",
|
|
13
|
+
"requirements.txt",
|
|
14
|
+
"pyproject.toml"
|
|
15
|
+
],
|
|
16
|
+
"prompt": "Build the {project_name} backend API service. Create a FastAPI/Flask application with basic CRUD routes, database models (users, products), and authentication scaffolding. Implement endpoints for /api/users, /api/products. Include database connection pooling and schema migrations. Tests must cover happy path and error cases. Use {base_dir}/src/api as the root.",
|
|
17
|
+
"testCmd": "python -m pytest tests/api/ -v",
|
|
18
|
+
"workDir": "{base_dir}"
|
|
19
|
+
},
|
|
20
|
+
{
|
|
21
|
+
"slug": "frontend-web",
|
|
22
|
+
"ownsFiles": [
|
|
23
|
+
"src/web/index.html",
|
|
24
|
+
"src/web/styles.css",
|
|
25
|
+
"src/web/components/",
|
|
26
|
+
"src/web/pages/",
|
|
27
|
+
"src/web/lib/",
|
|
28
|
+
"tests/web/",
|
|
29
|
+
"package.json"
|
|
30
|
+
],
|
|
31
|
+
"prompt": "Build the {project_name} web frontend. Create a React or vanilla-JS SPA with pages for dashboard, user management, and product catalog. Implement client-side routing, form validation, and API client library. Use {base_dir}/src/web as root. Ensure responsive design and accessibility (WCAG AA).",
|
|
32
|
+
"testCmd": "npm test -- --coverage",
|
|
33
|
+
"workDir": "{base_dir}"
|
|
34
|
+
},
|
|
35
|
+
{
|
|
36
|
+
"slug": "infrastructure-deploy",
|
|
37
|
+
"ownsFiles": [
|
|
38
|
+
"infra/docker-compose.yml",
|
|
39
|
+
"infra/Dockerfile.api",
|
|
40
|
+
"infra/Dockerfile.web",
|
|
41
|
+
"infra/kubernetes/",
|
|
42
|
+
"infra/terraform/",
|
|
43
|
+
".github/workflows/deploy.yml",
|
|
44
|
+
"infra/README.md"
|
|
45
|
+
],
|
|
46
|
+
"prompt": "Set up {project_name} deployment infrastructure. Create Docker images for both API and web services, write a docker-compose.yml for local development, and scaffold Kubernetes manifests (deployment, service, ingress) for production. Add a GitHub Actions workflow for CI/CD. Include health checks and resource limits.",
|
|
47
|
+
"testCmd": "docker-compose config > /dev/null && ls -la infra/kubernetes/*.yaml",
|
|
48
|
+
"workDir": "{base_dir}"
|
|
49
|
+
},
|
|
50
|
+
{
|
|
51
|
+
"slug": "database-schema",
|
|
52
|
+
"ownsFiles": [
|
|
53
|
+
"migrations/",
|
|
54
|
+
"src/api/database.py",
|
|
55
|
+
"schema.sql",
|
|
56
|
+
"seeds/initial.sql",
|
|
57
|
+
"tests/database/"
|
|
58
|
+
],
|
|
59
|
+
"prompt": "Design and implement the {project_name} database schema. Create SQL migrations for core tables: users (id, email, created_at), products (id, name, price), and orders (id, user_id, total). Add indexes on foreign keys and email. Provide seed data for development. Write Python ORM models (SQLAlchemy) that correspond to tables.",
|
|
60
|
+
"testCmd": "python -m pytest tests/database/ -v --tb=short",
|
|
61
|
+
"workDir": "{base_dir}"
|
|
62
|
+
}
|
|
63
|
+
]
|
|
64
|
+
}
|