@0dai-dev/cli 4.3.5 → 4.3.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +12 -11
- package/bin/0dai.js +214 -40
- package/lib/ai/manifest/mcp-exposure-contract.json +121 -0
- package/lib/ai/meta/manifest/mcp-tool-tiers.json +435 -0
- package/lib/ai/registry/mcp-catalog.json +98 -0
- package/lib/commands/auth.js +55 -1
- package/lib/commands/compliance.js +1 -1
- package/lib/commands/detect.js +10 -4
- package/lib/commands/doctor.js +545 -26
- package/lib/commands/experience.js +40 -5
- package/lib/commands/export.js +73 -0
- package/lib/commands/feedback.js +157 -15
- package/lib/commands/gh.js +26 -0
- package/lib/commands/graph.js +9 -4
- package/lib/commands/heatmap.js +1 -1
- package/lib/commands/init.js +222 -30
- package/lib/commands/mcp.js +129 -21
- package/lib/commands/models.js +138 -41
- package/lib/commands/provider.js +30 -59
- package/lib/commands/quota.js +1 -1
- package/lib/commands/receipt.js +1 -1
- package/lib/commands/run.js +18 -7
- package/lib/commands/runner.js +31 -1
- package/lib/commands/status.js +44 -11
- package/lib/commands/swarm.js +130 -12
- package/lib/commands/trust.js +286 -0
- package/lib/commands/update.js +184 -38
- package/lib/commands/usage.js +1 -1
- package/lib/commands/validate.js +32 -3
- package/lib/commands/vault.js +46 -9
- package/lib/python/__init__.py +0 -0
- package/lib/python/agent_quotas.py +525 -0
- package/lib/python/anomaly_alert.py +397 -0
- package/lib/python/anti_pattern_detector.py +799 -0
- package/lib/python/auth.py +443 -0
- package/lib/python/capi_profile_guard.py +477 -0
- package/lib/python/compliance_report.py +581 -0
- package/lib/python/drift_detector.py +388 -0
- package/lib/python/experience_pipeline.py +1130 -0
- package/lib/python/graph.py +19 -0
- package/lib/python/graph_core.py +293 -0
- package/lib/python/graph_io.py +179 -0
- package/lib/python/graph_legacy.py +2052 -0
- package/lib/python/graph_legacy_helpers.py +221 -0
- package/lib/python/graph_outcomes_core.py +85 -0
- package/lib/python/graph_queries.py +171 -0
- package/lib/python/graph_slice.py +198 -0
- package/lib/python/graph_slicer.py +576 -0
- package/lib/python/graph_slicer_cli.py +60 -0
- package/lib/python/graph_validation.py +64 -0
- package/lib/python/heatmap.py +934 -0
- package/lib/python/json_utils.py +193 -0
- package/lib/python/mcp_exposure_check.py +247 -0
- package/lib/python/model_router.py +1434 -0
- package/lib/python/project_manager.py +621 -0
- package/lib/python/provider_profiles.py +1618 -0
- package/lib/python/provider_registry.py +1211 -0
- package/lib/python/provider_registry_cli.py +125 -0
- package/lib/python/receipt_png.py +727 -0
- package/lib/python/structural_memory.py +325 -0
- package/lib/python/swarm_cost.py +177 -0
- package/lib/python/usage_ledger.py +569 -0
- package/lib/scripts/mcp_tier_config.py +240 -0
- package/lib/shared.js +97 -14
- package/lib/tui/index.mjs +35174 -0
- package/lib/utils/activation_telemetry.js +230 -11
- package/lib/utils/constants.js +7 -1
- package/lib/utils/export-bundler.js +285 -0
- package/lib/utils/identity.js +198 -1
- package/lib/utils/mcp-auth.js +81 -15
- package/lib/utils/plan.js +1 -1
- package/lib/vault/index.js +19 -3
- package/lib/vault/storage.js +21 -2
- package/lib/wizard.js +5 -2
- package/package.json +9 -3
- package/scripts/build-python-bundle.js +106 -0
- package/scripts/build-tui.js +14 -1
- package/scripts/harvest_experience.py +523 -0
- package/scripts/postinstall.js +15 -9
|
@@ -0,0 +1,799 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Anti-pattern detection from experience events.
|
|
3
|
+
|
|
4
|
+
Analyzes structured experience events to detect recurring failure patterns
|
|
5
|
+
and produces actionable warnings for users.
|
|
6
|
+
"""
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import argparse
|
|
10
|
+
import datetime as dt
|
|
11
|
+
import hashlib
|
|
12
|
+
import json
|
|
13
|
+
import pathlib
|
|
14
|
+
import sys
|
|
15
|
+
import time
|
|
16
|
+
|
|
17
|
+
import experience_pipeline as exp
|
|
18
|
+
from json_utils import load_json, save_json
|
|
19
|
+
|
|
20
|
+
CACHE_TTL = 600 # seconds (10 minutes)
|
|
21
|
+
WARNING_VERSION = 1
|
|
22
|
+
|
|
23
|
+
SEVERITY_ORDER = {"critical": 0, "warning": 1, "info": 2}
|
|
24
|
+
UTC = getattr(dt, "UTC", dt.timezone.utc)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
# ---------------------------------------------------------------------------
|
|
28
|
+
# Helpers
|
|
29
|
+
# ---------------------------------------------------------------------------
|
|
30
|
+
|
|
31
|
+
def _now_dt() -> dt.datetime:
|
|
32
|
+
return dt.datetime.now(UTC).replace(microsecond=0)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _now() -> str:
|
|
36
|
+
return _now_dt().isoformat().replace("+00:00", "Z")
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _dt_from_iso(value: str | None) -> dt.datetime | None:
|
|
40
|
+
if not value:
|
|
41
|
+
return None
|
|
42
|
+
try:
|
|
43
|
+
parsed = dt.datetime.fromisoformat(str(value).replace("Z", "+00:00"))
|
|
44
|
+
except ValueError:
|
|
45
|
+
return None
|
|
46
|
+
if parsed.tzinfo is None:
|
|
47
|
+
parsed = parsed.replace(tzinfo=UTC)
|
|
48
|
+
return parsed.astimezone(UTC)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def _safe_div(num: float, den: float) -> float:
|
|
52
|
+
return round(num / den, 4) if den else 0.0
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _detect_target(target: str | pathlib.Path = ".") -> pathlib.Path:
|
|
56
|
+
return pathlib.Path(target).resolve()
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def _warnings_path(target: pathlib.Path) -> pathlib.Path:
|
|
60
|
+
return target / "ai" / "experience" / "warnings.json"
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def _evidence_hash(evidence: dict) -> str:
|
|
64
|
+
raw = json.dumps(evidence, sort_keys=True, default=str)
|
|
65
|
+
return hashlib.sha256(raw.encode()).hexdigest()[:12]
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _task_events(events: list[dict]) -> list[dict]:
|
|
69
|
+
return [e for e in events if str(e.get("event_type") or "").startswith("task_")]
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def _task_result(event: dict) -> str:
|
|
73
|
+
return str((event.get("task") or {}).get("result") or "unknown")
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def _task_type(event: dict) -> str:
|
|
77
|
+
return str((event.get("task") or {}).get("task_type") or "unknown")
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def _task_cost(event: dict) -> float:
|
|
81
|
+
try:
|
|
82
|
+
return float((event.get("task") or {}).get("cost_usd") or 0)
|
|
83
|
+
except (TypeError, ValueError):
|
|
84
|
+
return 0.0
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def _task_elapsed(event: dict) -> float:
|
|
88
|
+
try:
|
|
89
|
+
return float((event.get("task") or {}).get("elapsed_seconds") or 0)
|
|
90
|
+
except (TypeError, ValueError):
|
|
91
|
+
return 0.0
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def _event_ts(event: dict) -> dt.datetime | None:
|
|
95
|
+
return _dt_from_iso(str(event.get("timestamp") or ""))
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def _make_warning(
|
|
99
|
+
pattern_id: str,
|
|
100
|
+
pattern_type: str,
|
|
101
|
+
severity: str,
|
|
102
|
+
title: str,
|
|
103
|
+
detail: str,
|
|
104
|
+
evidence: dict,
|
|
105
|
+
) -> dict:
|
|
106
|
+
return {
|
|
107
|
+
"pattern_id": pattern_id,
|
|
108
|
+
"pattern_type": pattern_type,
|
|
109
|
+
"severity": severity,
|
|
110
|
+
"title": title,
|
|
111
|
+
"detail": detail,
|
|
112
|
+
"evidence": evidence,
|
|
113
|
+
"detected_at": _now(),
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
# ---------------------------------------------------------------------------
|
|
118
|
+
# Detector 1: Agent-task mismatch
|
|
119
|
+
# ---------------------------------------------------------------------------
|
|
120
|
+
|
|
121
|
+
def detect_agent_task_mismatch(
|
|
122
|
+
events: list[dict],
|
|
123
|
+
*,
|
|
124
|
+
min_attempts: int = 3,
|
|
125
|
+
fail_threshold: float = 0.5,
|
|
126
|
+
) -> list[dict]:
|
|
127
|
+
tasks = _task_events(events)
|
|
128
|
+
buckets: dict[tuple[str, str], dict] = {}
|
|
129
|
+
for e in tasks:
|
|
130
|
+
agent = str(e.get("agent") or "unknown")
|
|
131
|
+
ttype = _task_type(e)
|
|
132
|
+
key = (agent, ttype)
|
|
133
|
+
b = buckets.setdefault(key, {"total": 0, "fail": 0})
|
|
134
|
+
b["total"] += 1
|
|
135
|
+
if _task_result(e) in exp.FAIL_RESULTS:
|
|
136
|
+
b["fail"] += 1
|
|
137
|
+
|
|
138
|
+
# Find best alternative agents per task_type
|
|
139
|
+
type_stats: dict[str, dict[str, dict]] = {}
|
|
140
|
+
for (agent, ttype), b in buckets.items():
|
|
141
|
+
type_stats.setdefault(ttype, {})[agent] = b
|
|
142
|
+
|
|
143
|
+
warnings: list[dict] = []
|
|
144
|
+
for (agent, ttype), b in buckets.items():
|
|
145
|
+
if b["total"] < min_attempts:
|
|
146
|
+
continue
|
|
147
|
+
fail_rate = _safe_div(b["fail"], b["total"])
|
|
148
|
+
if fail_rate <= fail_threshold:
|
|
149
|
+
continue
|
|
150
|
+
|
|
151
|
+
# Find best alternative
|
|
152
|
+
alt_msg = ""
|
|
153
|
+
alts = type_stats.get(ttype, {})
|
|
154
|
+
best_alt = None
|
|
155
|
+
best_rate = fail_rate
|
|
156
|
+
for other_agent, ob in alts.items():
|
|
157
|
+
if other_agent == agent or ob["total"] < min_attempts:
|
|
158
|
+
continue
|
|
159
|
+
other_fail = _safe_div(ob["fail"], ob["total"])
|
|
160
|
+
if other_fail < best_rate:
|
|
161
|
+
best_rate = other_fail
|
|
162
|
+
best_alt = other_agent
|
|
163
|
+
if best_alt:
|
|
164
|
+
alt_success = round((1 - best_rate) * 100)
|
|
165
|
+
alt_msg = f"Consider using {best_alt} for {ttype} tasks ({alt_success}% success rate)"
|
|
166
|
+
else:
|
|
167
|
+
alt_msg = f"Consider trying a different agent for {ttype} tasks"
|
|
168
|
+
|
|
169
|
+
warnings.append(_make_warning(
|
|
170
|
+
pattern_id=f"agent_task_mismatch:{agent}:{ttype}",
|
|
171
|
+
pattern_type="agent_task_mismatch",
|
|
172
|
+
severity="warning",
|
|
173
|
+
title=f"{agent} fails on {ttype} tasks {round(fail_rate * 100)}% ({b['fail']}/{b['total']})",
|
|
174
|
+
detail=alt_msg,
|
|
175
|
+
evidence={
|
|
176
|
+
"agent": agent,
|
|
177
|
+
"task_type": ttype,
|
|
178
|
+
"attempts": b["total"],
|
|
179
|
+
"failures": b["fail"],
|
|
180
|
+
"fail_rate": fail_rate,
|
|
181
|
+
},
|
|
182
|
+
))
|
|
183
|
+
return warnings
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
# ---------------------------------------------------------------------------
|
|
187
|
+
# Detector 2: Cost escalation
|
|
188
|
+
# ---------------------------------------------------------------------------
|
|
189
|
+
|
|
190
|
+
def detect_cost_escalation(
|
|
191
|
+
events: list[dict],
|
|
192
|
+
*,
|
|
193
|
+
growth_threshold: float = 0.3,
|
|
194
|
+
min_tasks: int = 3,
|
|
195
|
+
window_days: int = 7,
|
|
196
|
+
) -> list[dict]:
|
|
197
|
+
now = _now_dt()
|
|
198
|
+
recent_cutoff = now - dt.timedelta(days=window_days)
|
|
199
|
+
prev_cutoff = now - dt.timedelta(days=window_days * 2)
|
|
200
|
+
|
|
201
|
+
recent_costs: list[float] = []
|
|
202
|
+
prev_costs: list[float] = []
|
|
203
|
+
for e in _task_events(events):
|
|
204
|
+
ts = _event_ts(e)
|
|
205
|
+
if not ts:
|
|
206
|
+
continue
|
|
207
|
+
cost = _task_cost(e)
|
|
208
|
+
if ts >= recent_cutoff:
|
|
209
|
+
recent_costs.append(cost)
|
|
210
|
+
elif ts >= prev_cutoff:
|
|
211
|
+
prev_costs.append(cost)
|
|
212
|
+
|
|
213
|
+
if len(recent_costs) < min_tasks or len(prev_costs) < min_tasks:
|
|
214
|
+
return []
|
|
215
|
+
|
|
216
|
+
recent_avg = _safe_div(sum(recent_costs), len(recent_costs))
|
|
217
|
+
prev_avg = _safe_div(sum(prev_costs), len(prev_costs))
|
|
218
|
+
if prev_avg <= 0:
|
|
219
|
+
return []
|
|
220
|
+
|
|
221
|
+
growth = _safe_div(recent_avg - prev_avg, prev_avg)
|
|
222
|
+
if growth <= growth_threshold:
|
|
223
|
+
return []
|
|
224
|
+
|
|
225
|
+
return [_make_warning(
|
|
226
|
+
pattern_id="cost_escalation",
|
|
227
|
+
pattern_type="cost_escalation",
|
|
228
|
+
severity="info",
|
|
229
|
+
title=f"Average task cost increased {round(growth * 100)}%: ${prev_avg:.2f} -> ${recent_avg:.2f}",
|
|
230
|
+
detail="Consider using cheaper models for simple tasks. Run: 0dai experience stats --by model",
|
|
231
|
+
evidence={
|
|
232
|
+
"recent_avg": recent_avg,
|
|
233
|
+
"previous_avg": prev_avg,
|
|
234
|
+
"growth_pct": growth,
|
|
235
|
+
"recent_count": len(recent_costs),
|
|
236
|
+
"previous_count": len(prev_costs),
|
|
237
|
+
},
|
|
238
|
+
)]
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
# ---------------------------------------------------------------------------
|
|
242
|
+
# Detector 3: File churn
|
|
243
|
+
# ---------------------------------------------------------------------------
|
|
244
|
+
|
|
245
|
+
def detect_file_churn(
|
|
246
|
+
target: pathlib.Path,
|
|
247
|
+
*,
|
|
248
|
+
max_touches: int = 3,
|
|
249
|
+
window_hours: int = 24,
|
|
250
|
+
) -> list[dict]:
|
|
251
|
+
done_dir = target / "ai" / "swarm" / "done"
|
|
252
|
+
if not done_dir.is_dir():
|
|
253
|
+
return []
|
|
254
|
+
|
|
255
|
+
cutoff = _now_dt() - dt.timedelta(hours=window_hours)
|
|
256
|
+
file_tasks: dict[str, set[str]] = {} # file_path -> set of task_ids
|
|
257
|
+
|
|
258
|
+
for task_file in done_dir.glob("*.json"):
|
|
259
|
+
task = load_json(task_file)
|
|
260
|
+
completed = _dt_from_iso(str(task.get("completed_at") or task.get("done_at") or ""))
|
|
261
|
+
if not completed or completed < cutoff:
|
|
262
|
+
continue
|
|
263
|
+
task_id = str(task.get("id") or task_file.stem)
|
|
264
|
+
files = task.get("files") or task.get("context", {}).get("files") or []
|
|
265
|
+
if isinstance(files, str):
|
|
266
|
+
files = [f.strip() for f in files.split(",") if f.strip()]
|
|
267
|
+
for f in files:
|
|
268
|
+
f_str = str(f).strip()
|
|
269
|
+
if f_str:
|
|
270
|
+
file_tasks.setdefault(f_str, set()).add(task_id)
|
|
271
|
+
|
|
272
|
+
warnings: list[dict] = []
|
|
273
|
+
for fpath, task_ids in file_tasks.items():
|
|
274
|
+
if len(task_ids) <= max_touches:
|
|
275
|
+
continue
|
|
276
|
+
fhash = hashlib.sha256(fpath.encode()).hexdigest()[:8]
|
|
277
|
+
warnings.append(_make_warning(
|
|
278
|
+
pattern_id=f"file_churn:{fhash}",
|
|
279
|
+
pattern_type="file_churn",
|
|
280
|
+
severity="warning",
|
|
281
|
+
title=f"{fpath} modified {len(task_ids)} times in {window_hours}h",
|
|
282
|
+
detail="This file may need architectural review. Multiple agents are fighting over it.",
|
|
283
|
+
evidence={
|
|
284
|
+
"file": fpath,
|
|
285
|
+
"touches": len(task_ids),
|
|
286
|
+
"task_ids": sorted(task_ids),
|
|
287
|
+
"window_hours": window_hours,
|
|
288
|
+
},
|
|
289
|
+
))
|
|
290
|
+
return warnings
|
|
291
|
+
|
|
292
|
+
|
|
293
|
+
# ---------------------------------------------------------------------------
|
|
294
|
+
# Detector 4: Stuck agent
|
|
295
|
+
# ---------------------------------------------------------------------------
|
|
296
|
+
|
|
297
|
+
def detect_stuck_agent(
|
|
298
|
+
target: pathlib.Path,
|
|
299
|
+
events: list[dict],
|
|
300
|
+
*,
|
|
301
|
+
multiplier: float = 2.0,
|
|
302
|
+
min_history: int = 3,
|
|
303
|
+
) -> list[dict]:
|
|
304
|
+
# Compute historical averages per task_type
|
|
305
|
+
type_elapsed: dict[str, list[float]] = {}
|
|
306
|
+
for e in _task_events(events):
|
|
307
|
+
ttype = _task_type(e)
|
|
308
|
+
elapsed = _task_elapsed(e)
|
|
309
|
+
if elapsed > 0:
|
|
310
|
+
type_elapsed.setdefault(ttype, []).append(elapsed)
|
|
311
|
+
|
|
312
|
+
type_avg: dict[str, float] = {}
|
|
313
|
+
for ttype, times in type_elapsed.items():
|
|
314
|
+
if len(times) >= min_history:
|
|
315
|
+
type_avg[ttype] = sum(times) / len(times)
|
|
316
|
+
|
|
317
|
+
if not type_avg:
|
|
318
|
+
return []
|
|
319
|
+
|
|
320
|
+
# Check active tasks
|
|
321
|
+
active_dir = target / "ai" / "swarm" / "active"
|
|
322
|
+
if not active_dir.is_dir():
|
|
323
|
+
return []
|
|
324
|
+
|
|
325
|
+
now_ts = time.time()
|
|
326
|
+
warnings: list[dict] = []
|
|
327
|
+
for task_file in active_dir.glob("*.json"):
|
|
328
|
+
task = load_json(task_file)
|
|
329
|
+
if not task:
|
|
330
|
+
continue
|
|
331
|
+
task_id = str(task.get("id") or task_file.stem)
|
|
332
|
+
ttype = str(task.get("type") or "feat")
|
|
333
|
+
started = _dt_from_iso(str(task.get("started_at") or task.get("created_at") or ""))
|
|
334
|
+
if not started:
|
|
335
|
+
continue
|
|
336
|
+
elapsed = now_ts - started.timestamp()
|
|
337
|
+
avg = type_avg.get(ttype)
|
|
338
|
+
if avg and elapsed > avg * multiplier:
|
|
339
|
+
agent = str(task.get("assigned_to") or "unknown")
|
|
340
|
+
warnings.append(_make_warning(
|
|
341
|
+
pattern_id=f"stuck_agent:{task_id}",
|
|
342
|
+
pattern_type="stuck_agent",
|
|
343
|
+
severity="warning",
|
|
344
|
+
title=f"{agent} task running {int(elapsed)}s (avg for {ttype}: {int(avg)}s)",
|
|
345
|
+
detail="Agent may be stuck. Consider: 0dai swarm kill + retry with different model",
|
|
346
|
+
evidence={
|
|
347
|
+
"task_id": task_id,
|
|
348
|
+
"agent": agent,
|
|
349
|
+
"task_type": ttype,
|
|
350
|
+
"elapsed": int(elapsed),
|
|
351
|
+
"avg_elapsed": int(avg),
|
|
352
|
+
"multiplier": multiplier,
|
|
353
|
+
},
|
|
354
|
+
))
|
|
355
|
+
return warnings
|
|
356
|
+
|
|
357
|
+
|
|
358
|
+
# ---------------------------------------------------------------------------
|
|
359
|
+
# Detector 5: Repeated failure
|
|
360
|
+
# ---------------------------------------------------------------------------
|
|
361
|
+
|
|
362
|
+
def detect_repeated_failure(
|
|
363
|
+
events: list[dict],
|
|
364
|
+
*,
|
|
365
|
+
streak_threshold: int = 3,
|
|
366
|
+
) -> list[dict]:
|
|
367
|
+
tasks = _task_events(events)
|
|
368
|
+
# Sort by timestamp descending (most recent first)
|
|
369
|
+
tasks.sort(key=lambda e: str(e.get("timestamp") or ""), reverse=True)
|
|
370
|
+
|
|
371
|
+
streak = 0
|
|
372
|
+
streak_ids: list[str] = []
|
|
373
|
+
for e in tasks:
|
|
374
|
+
result = _task_result(e)
|
|
375
|
+
if result in exp.FAIL_RESULTS:
|
|
376
|
+
streak += 1
|
|
377
|
+
streak_ids.append(str((e.get("task") or {}).get("task_id") or "?"))
|
|
378
|
+
else:
|
|
379
|
+
break # streak broken
|
|
380
|
+
|
|
381
|
+
if streak < streak_threshold:
|
|
382
|
+
return []
|
|
383
|
+
|
|
384
|
+
return [_make_warning(
|
|
385
|
+
pattern_id="repeated_failure",
|
|
386
|
+
pattern_type="repeated_failure",
|
|
387
|
+
severity="critical",
|
|
388
|
+
title=f"{streak} tasks failed in a row ({', '.join(streak_ids[:5])})",
|
|
389
|
+
detail="Something systematic is wrong. Check: test suite broken? API down? Auth expired?",
|
|
390
|
+
evidence={
|
|
391
|
+
"streak": streak,
|
|
392
|
+
"task_ids": streak_ids[:10],
|
|
393
|
+
},
|
|
394
|
+
)]
|
|
395
|
+
|
|
396
|
+
|
|
397
|
+
# ---------------------------------------------------------------------------
|
|
398
|
+
# Detector 6: Model overspend
|
|
399
|
+
# ---------------------------------------------------------------------------
|
|
400
|
+
|
|
401
|
+
def detect_model_overspend(
|
|
402
|
+
events: list[dict],
|
|
403
|
+
*,
|
|
404
|
+
usage_threshold: float = 0.8,
|
|
405
|
+
min_tasks: int = 5,
|
|
406
|
+
success_tolerance: float = 0.05,
|
|
407
|
+
) -> list[dict]:
|
|
408
|
+
tasks = _task_events(events)
|
|
409
|
+
total = len(tasks)
|
|
410
|
+
if total < min_tasks:
|
|
411
|
+
return []
|
|
412
|
+
|
|
413
|
+
# Group by model
|
|
414
|
+
model_stats: dict[str, dict] = {}
|
|
415
|
+
for e in tasks:
|
|
416
|
+
model = str(e.get("model") or "unknown")
|
|
417
|
+
b = model_stats.setdefault(model, {"count": 0, "success": 0, "cost": 0.0})
|
|
418
|
+
b["count"] += 1
|
|
419
|
+
if _task_result(e) in exp.SUCCESS_RESULTS:
|
|
420
|
+
b["success"] += 1
|
|
421
|
+
b["cost"] += _task_cost(e)
|
|
422
|
+
|
|
423
|
+
# Sort by avg cost descending
|
|
424
|
+
ranked = sorted(
|
|
425
|
+
model_stats.items(),
|
|
426
|
+
key=lambda item: _safe_div(item[1]["cost"], item[1]["count"]),
|
|
427
|
+
reverse=True,
|
|
428
|
+
)
|
|
429
|
+
|
|
430
|
+
warnings: list[dict] = []
|
|
431
|
+
for model, stats in ranked:
|
|
432
|
+
usage_pct = _safe_div(stats["count"], total)
|
|
433
|
+
if usage_pct < usage_threshold:
|
|
434
|
+
continue
|
|
435
|
+
success_rate = _safe_div(stats["success"], stats["count"])
|
|
436
|
+
avg_cost = _safe_div(stats["cost"], stats["count"])
|
|
437
|
+
|
|
438
|
+
# Find cheaper alternative with comparable success rate
|
|
439
|
+
for alt_model, alt_stats in ranked:
|
|
440
|
+
if alt_model == model or alt_stats["count"] < 2:
|
|
441
|
+
continue
|
|
442
|
+
alt_avg_cost = _safe_div(alt_stats["cost"], alt_stats["count"])
|
|
443
|
+
if alt_avg_cost >= avg_cost:
|
|
444
|
+
continue
|
|
445
|
+
alt_success = _safe_div(alt_stats["success"], alt_stats["count"])
|
|
446
|
+
if alt_success >= success_rate - success_tolerance:
|
|
447
|
+
cost_ratio = _safe_div(avg_cost, alt_avg_cost) if alt_avg_cost > 0 else 0
|
|
448
|
+
warnings.append(_make_warning(
|
|
449
|
+
pattern_id=f"model_overspend:{model}",
|
|
450
|
+
pattern_type="model_overspend",
|
|
451
|
+
severity="info",
|
|
452
|
+
title=f"{model} used for {round(usage_pct * 100)}% of tasks, "
|
|
453
|
+
f"but {alt_model} has similar success at lower cost",
|
|
454
|
+
detail=f"Switch to {alt_model}: save ~${stats['cost'] - alt_stats['cost']:.2f}/period",
|
|
455
|
+
evidence={
|
|
456
|
+
"model": model,
|
|
457
|
+
"usage_pct": usage_pct,
|
|
458
|
+
"success_rate": success_rate,
|
|
459
|
+
"avg_cost": avg_cost,
|
|
460
|
+
"cheaper_alt": alt_model,
|
|
461
|
+
"alt_success_rate": alt_success,
|
|
462
|
+
"alt_avg_cost": alt_avg_cost,
|
|
463
|
+
"cost_ratio": cost_ratio,
|
|
464
|
+
},
|
|
465
|
+
))
|
|
466
|
+
break # one recommendation per model
|
|
467
|
+
return warnings
|
|
468
|
+
|
|
469
|
+
|
|
470
|
+
# ---------------------------------------------------------------------------
|
|
471
|
+
# Detector 7: Abandoned sessions
|
|
472
|
+
# ---------------------------------------------------------------------------
|
|
473
|
+
|
|
474
|
+
def detect_abandoned_session(
|
|
475
|
+
target: pathlib.Path,
|
|
476
|
+
*,
|
|
477
|
+
stale_hours: int = 48,
|
|
478
|
+
) -> list[dict]:
|
|
479
|
+
try:
|
|
480
|
+
import session_roaming as sr
|
|
481
|
+
sessions = sr.list_saved_sessions(target)
|
|
482
|
+
except (ImportError, OSError, json.JSONDecodeError, ValueError, AttributeError):
|
|
483
|
+
# session_roaming is optional (ImportError); list_saved_sessions
|
|
484
|
+
# reads JSON files (OSError/JSONDecodeError/ValueError); shape drift
|
|
485
|
+
# would surface as AttributeError. All non-fatal — return [].
|
|
486
|
+
return []
|
|
487
|
+
|
|
488
|
+
cutoff = _now_dt() - dt.timedelta(hours=stale_hours)
|
|
489
|
+
warnings: list[dict] = []
|
|
490
|
+
for sess in sessions:
|
|
491
|
+
status = str(sess.get("status") or "").lower()
|
|
492
|
+
if status in ("completed", "done", "closed"):
|
|
493
|
+
continue
|
|
494
|
+
saved_at = _dt_from_iso(str(sess.get("saved_at") or sess.get("updated") or ""))
|
|
495
|
+
if not saved_at or saved_at > cutoff:
|
|
496
|
+
continue
|
|
497
|
+
hours_stale = int((_now_dt() - saved_at).total_seconds() / 3600)
|
|
498
|
+
sess_id = str(sess.get("session_id") or sess.get("id") or "?")
|
|
499
|
+
goal = str((sess.get("goal") or {}).get("refined")
|
|
500
|
+
or (sess.get("goal") or {}).get("original")
|
|
501
|
+
or sess.get("goal") or "?")
|
|
502
|
+
if isinstance(goal, dict):
|
|
503
|
+
goal = str(goal.get("refined") or goal.get("original") or "?")
|
|
504
|
+
warnings.append(_make_warning(
|
|
505
|
+
pattern_id=f"abandoned_session:{sess_id}",
|
|
506
|
+
pattern_type="abandoned_session",
|
|
507
|
+
severity="info",
|
|
508
|
+
title=f"Session {sess_id[:16]} ({goal[:40]}) saved {hours_stale}h ago, never resumed",
|
|
509
|
+
detail=f"Resume or close: 0dai session resume --session {sess_id}",
|
|
510
|
+
evidence={
|
|
511
|
+
"session_id": sess_id,
|
|
512
|
+
"goal": goal[:80],
|
|
513
|
+
"saved_at": str(saved_at.isoformat()),
|
|
514
|
+
"hours_stale": hours_stale,
|
|
515
|
+
},
|
|
516
|
+
))
|
|
517
|
+
return warnings
|
|
518
|
+
|
|
519
|
+
|
|
520
|
+
# ---------------------------------------------------------------------------
|
|
521
|
+
# Detector 8: Graph stale
|
|
522
|
+
# ---------------------------------------------------------------------------
|
|
523
|
+
|
|
524
|
+
def detect_graph_stale(
|
|
525
|
+
target: pathlib.Path,
|
|
526
|
+
events: list[dict],
|
|
527
|
+
*,
|
|
528
|
+
stale_days: int = 7,
|
|
529
|
+
) -> list[dict]:
|
|
530
|
+
graph_path = target / "ai" / "manifest" / "project_graph.json"
|
|
531
|
+
graph = load_json(graph_path)
|
|
532
|
+
synced_at = _dt_from_iso(str(
|
|
533
|
+
graph.get("_synced_at")
|
|
534
|
+
or graph.get("meta", {}).get("updated_at")
|
|
535
|
+
or ""
|
|
536
|
+
))
|
|
537
|
+
|
|
538
|
+
# Also check experience events for recent graph_synced
|
|
539
|
+
for e in events:
|
|
540
|
+
if str(e.get("event_type") or "") == "graph_synced":
|
|
541
|
+
ev_ts = _event_ts(e)
|
|
542
|
+
if ev_ts and (not synced_at or ev_ts > synced_at):
|
|
543
|
+
synced_at = ev_ts
|
|
544
|
+
break # events are sorted newest first
|
|
545
|
+
|
|
546
|
+
if not synced_at:
|
|
547
|
+
return [] # no graph data at all, nothing to warn about
|
|
548
|
+
|
|
549
|
+
cutoff = _now_dt() - dt.timedelta(days=stale_days)
|
|
550
|
+
if synced_at > cutoff:
|
|
551
|
+
return []
|
|
552
|
+
|
|
553
|
+
# Check for active swarm tasks
|
|
554
|
+
active_dir = target / "ai" / "swarm" / "active"
|
|
555
|
+
has_active = active_dir.is_dir() and any(active_dir.glob("*.json"))
|
|
556
|
+
# Also count recent task events as activity
|
|
557
|
+
recent_tasks = len(_task_events(events[:20]))
|
|
558
|
+
|
|
559
|
+
if not has_active and recent_tasks == 0:
|
|
560
|
+
return []
|
|
561
|
+
|
|
562
|
+
days_stale = int((_now_dt() - synced_at).total_seconds() / 86400)
|
|
563
|
+
return [_make_warning(
|
|
564
|
+
pattern_id="graph_stale",
|
|
565
|
+
pattern_type="graph_stale",
|
|
566
|
+
severity="warning",
|
|
567
|
+
title=f"Graph last synced {days_stale} days ago with active swarm tasks",
|
|
568
|
+
detail="Graph is out of date. Run: 0dai graph push",
|
|
569
|
+
evidence={
|
|
570
|
+
"last_sync": synced_at.isoformat(),
|
|
571
|
+
"days_stale": days_stale,
|
|
572
|
+
"has_active_tasks": has_active,
|
|
573
|
+
},
|
|
574
|
+
)]
|
|
575
|
+
|
|
576
|
+
|
|
577
|
+
# ---------------------------------------------------------------------------
|
|
578
|
+
# Orchestrator
|
|
579
|
+
# ---------------------------------------------------------------------------
|
|
580
|
+
|
|
581
|
+
def run_all_detectors(target: pathlib.Path) -> list[dict]:
|
|
582
|
+
events = exp.load_events(target, since="30d", limit=10000)
|
|
583
|
+
warnings: list[dict] = []
|
|
584
|
+
|
|
585
|
+
detectors = [
|
|
586
|
+
lambda: detect_agent_task_mismatch(events),
|
|
587
|
+
lambda: detect_cost_escalation(events),
|
|
588
|
+
lambda: detect_file_churn(target),
|
|
589
|
+
lambda: detect_stuck_agent(target, events),
|
|
590
|
+
lambda: detect_repeated_failure(events),
|
|
591
|
+
lambda: detect_model_overspend(events),
|
|
592
|
+
lambda: detect_abandoned_session(target),
|
|
593
|
+
lambda: detect_graph_stale(target, events),
|
|
594
|
+
]
|
|
595
|
+
|
|
596
|
+
for detector in detectors:
|
|
597
|
+
try:
|
|
598
|
+
warnings.extend(detector())
|
|
599
|
+
except Exception as exc: # noqa: BLE001 — defensive dispatch loop
|
|
600
|
+
# Intentional broad catch: each detector is independently fallible
|
|
601
|
+
# (filesystem, JSON parse, third-party shape drift) and one failing
|
|
602
|
+
# detector must not silence the others. Surface to stderr so
|
|
603
|
+
# operators can still see which detector misbehaved instead of the
|
|
604
|
+
# previous fully-silent `pass`.
|
|
605
|
+
print(
|
|
606
|
+
f"anti_pattern_detector: {detector!r} failed: {exc!r}",
|
|
607
|
+
file=sys.stderr,
|
|
608
|
+
)
|
|
609
|
+
|
|
610
|
+
# Sort: critical first, then warning, then info
|
|
611
|
+
warnings.sort(key=lambda w: SEVERITY_ORDER.get(w.get("severity", "info"), 9))
|
|
612
|
+
return warnings
|
|
613
|
+
|
|
614
|
+
|
|
615
|
+
# ---------------------------------------------------------------------------
|
|
616
|
+
# Cache layer
|
|
617
|
+
# ---------------------------------------------------------------------------
|
|
618
|
+
|
|
619
|
+
def load_warnings(
|
|
620
|
+
target: pathlib.Path | str,
|
|
621
|
+
*,
|
|
622
|
+
force_refresh: bool = False,
|
|
623
|
+
) -> dict:
|
|
624
|
+
target = _detect_target(target)
|
|
625
|
+
path = _warnings_path(target)
|
|
626
|
+
cached = load_json(path)
|
|
627
|
+
|
|
628
|
+
if not force_refresh and cached.get("refreshed_at"):
|
|
629
|
+
refreshed = _dt_from_iso(cached["refreshed_at"])
|
|
630
|
+
if refreshed and (_now_dt() - refreshed).total_seconds() < CACHE_TTL:
|
|
631
|
+
return cached
|
|
632
|
+
|
|
633
|
+
return refresh_warnings(target, cached)
|
|
634
|
+
|
|
635
|
+
|
|
636
|
+
def refresh_warnings(target: pathlib.Path, previous: dict | None = None) -> dict:
|
|
637
|
+
previous = previous or {}
|
|
638
|
+
dismissed: dict[str, str] = dict(previous.get("dismissed") or {})
|
|
639
|
+
new_warnings = run_all_detectors(target)
|
|
640
|
+
|
|
641
|
+
# Filter out dismissed warnings with same evidence
|
|
642
|
+
active: list[dict] = []
|
|
643
|
+
for w in new_warnings:
|
|
644
|
+
pid = w["pattern_id"]
|
|
645
|
+
ehash = _evidence_hash(w["evidence"])
|
|
646
|
+
w["evidence_hash"] = ehash
|
|
647
|
+
if pid in dismissed and dismissed[pid] == ehash:
|
|
648
|
+
continue # still dismissed, same evidence
|
|
649
|
+
active.append(w)
|
|
650
|
+
|
|
651
|
+
result = {
|
|
652
|
+
"version": WARNING_VERSION,
|
|
653
|
+
"refreshed_at": _now(),
|
|
654
|
+
"warnings": active,
|
|
655
|
+
"dismissed": dismissed,
|
|
656
|
+
}
|
|
657
|
+
save_json(_warnings_path(target), result)
|
|
658
|
+
return result
|
|
659
|
+
|
|
660
|
+
|
|
661
|
+
# ---------------------------------------------------------------------------
|
|
662
|
+
# Dismiss
|
|
663
|
+
# ---------------------------------------------------------------------------
|
|
664
|
+
|
|
665
|
+
def dismiss_warning(target: pathlib.Path | str, pattern_id: str) -> dict:
|
|
666
|
+
target = _detect_target(target)
|
|
667
|
+
data = load_warnings(target, force_refresh=True)
|
|
668
|
+
for w in data.get("warnings", []):
|
|
669
|
+
if w["pattern_id"] == pattern_id:
|
|
670
|
+
data["dismissed"][pattern_id] = w.get("evidence_hash", _evidence_hash(w["evidence"]))
|
|
671
|
+
data["warnings"] = [x for x in data["warnings"] if x["pattern_id"] != pattern_id]
|
|
672
|
+
save_json(_warnings_path(target), data)
|
|
673
|
+
return {"ok": True, "dismissed": pattern_id}
|
|
674
|
+
return {"ok": False, "error": f"warning not found: {pattern_id}"}
|
|
675
|
+
|
|
676
|
+
|
|
677
|
+
def get_active_warnings(target: pathlib.Path | str) -> list[dict]:
|
|
678
|
+
data = load_warnings(target)
|
|
679
|
+
return data.get("warnings", [])
|
|
680
|
+
|
|
681
|
+
|
|
682
|
+
# ---------------------------------------------------------------------------
|
|
683
|
+
# CLI formatting
|
|
684
|
+
# ---------------------------------------------------------------------------
|
|
685
|
+
|
|
686
|
+
_SEVERITY_ICON = {"critical": "\u0052\u0045\u0044", "warning": "\u0059\u0045\u004c", "info": "\u0049\u004e\u0046"}
|
|
687
|
+
|
|
688
|
+
def _severity_prefix(severity: str) -> str:
|
|
689
|
+
if severity == "critical":
|
|
690
|
+
return " CRITICAL"
|
|
691
|
+
if severity == "warning":
|
|
692
|
+
return " WARNING"
|
|
693
|
+
return " INFO"
|
|
694
|
+
|
|
695
|
+
|
|
696
|
+
def format_warnings(warnings: list[dict], *, verbose: bool = False) -> str:
|
|
697
|
+
if not warnings:
|
|
698
|
+
return "No active warnings."
|
|
699
|
+
|
|
700
|
+
lines = [f"Anti-pattern warnings ({len(warnings)}):"]
|
|
701
|
+
lines.append("")
|
|
702
|
+
for w in warnings:
|
|
703
|
+
prefix = _severity_prefix(w.get("severity", "info"))
|
|
704
|
+
lines.append(f"{prefix}: {w.get('title', '')}")
|
|
705
|
+
lines.append(f" -> {w.get('detail', '')}")
|
|
706
|
+
if verbose:
|
|
707
|
+
lines.append(f" id: {w.get('pattern_id', '')}")
|
|
708
|
+
for k, v in (w.get("evidence") or {}).items():
|
|
709
|
+
lines.append(f" {k}: {v}")
|
|
710
|
+
lines.append("")
|
|
711
|
+
|
|
712
|
+
lines.append("Dismiss: 0dai experience dismiss <pattern_id>")
|
|
713
|
+
return "\n".join(lines)
|
|
714
|
+
|
|
715
|
+
|
|
716
|
+
# ---------------------------------------------------------------------------
|
|
717
|
+
# CLI commands
|
|
718
|
+
# ---------------------------------------------------------------------------
|
|
719
|
+
|
|
720
|
+
def cmd_warnings(args: argparse.Namespace) -> int:
|
|
721
|
+
target = _detect_target(args.target)
|
|
722
|
+
data = load_warnings(target, force_refresh=getattr(args, "refresh", False))
|
|
723
|
+
active = data.get("warnings", [])
|
|
724
|
+
|
|
725
|
+
if getattr(args, "severity", None):
|
|
726
|
+
active = [w for w in active if w.get("severity") == args.severity]
|
|
727
|
+
|
|
728
|
+
if getattr(args, "json", False):
|
|
729
|
+
print(json.dumps({"warnings": active, "count": len(active)}, indent=2))
|
|
730
|
+
else:
|
|
731
|
+
print(format_warnings(active, verbose=getattr(args, "verbose", False)))
|
|
732
|
+
return 0
|
|
733
|
+
|
|
734
|
+
|
|
735
|
+
def cmd_dismiss(args: argparse.Namespace) -> int:
|
|
736
|
+
target = _detect_target(args.target)
|
|
737
|
+
result = dismiss_warning(target, args.pattern_id)
|
|
738
|
+
if getattr(args, "json", False):
|
|
739
|
+
print(json.dumps(result, indent=2))
|
|
740
|
+
elif result["ok"]:
|
|
741
|
+
print(f"Dismissed {args.pattern_id}. Will not show again unless pattern reoccurs with new data.")
|
|
742
|
+
else:
|
|
743
|
+
print(result["error"])
|
|
744
|
+
return 0 if result["ok"] else 1
|
|
745
|
+
|
|
746
|
+
|
|
747
|
+
def cmd_count(args: argparse.Namespace) -> int:
|
|
748
|
+
target = _detect_target(args.target)
|
|
749
|
+
data = load_warnings(target)
|
|
750
|
+
active = data.get("warnings", [])
|
|
751
|
+
by_severity: dict[str, int] = {}
|
|
752
|
+
for w in active:
|
|
753
|
+
s = w.get("severity", "info")
|
|
754
|
+
by_severity[s] = by_severity.get(s, 0) + 1
|
|
755
|
+
print(json.dumps({"count": len(active), "by_severity": by_severity}))
|
|
756
|
+
return 0
|
|
757
|
+
|
|
758
|
+
|
|
759
|
+
# ---------------------------------------------------------------------------
|
|
760
|
+
# Argparse
|
|
761
|
+
# ---------------------------------------------------------------------------
|
|
762
|
+
|
|
763
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
764
|
+
parser = argparse.ArgumentParser(prog="anti_pattern_detector")
|
|
765
|
+
sub = parser.add_subparsers(dest="command")
|
|
766
|
+
|
|
767
|
+
w = sub.add_parser("warnings", help="Show active anti-pattern warnings")
|
|
768
|
+
w.add_argument("--target", default=".")
|
|
769
|
+
w.add_argument("--json", action="store_true")
|
|
770
|
+
w.add_argument("--refresh", action="store_true")
|
|
771
|
+
w.add_argument("--verbose", action="store_true")
|
|
772
|
+
w.add_argument("--severity", choices=["critical", "warning", "info"])
|
|
773
|
+
w.set_defaults(func=cmd_warnings)
|
|
774
|
+
|
|
775
|
+
d = sub.add_parser("dismiss", help="Dismiss a warning by pattern_id")
|
|
776
|
+
d.add_argument("pattern_id")
|
|
777
|
+
d.add_argument("--target", default=".")
|
|
778
|
+
d.add_argument("--json", action="store_true")
|
|
779
|
+
d.set_defaults(func=cmd_dismiss)
|
|
780
|
+
|
|
781
|
+
c = sub.add_parser("count", help="Output warning count as JSON")
|
|
782
|
+
c.add_argument("--target", default=".")
|
|
783
|
+
c.set_defaults(func=cmd_count)
|
|
784
|
+
|
|
785
|
+
return parser
|
|
786
|
+
|
|
787
|
+
|
|
788
|
+
def main() -> None:
|
|
789
|
+
parser = build_parser()
|
|
790
|
+
args = parser.parse_args()
|
|
791
|
+
if hasattr(args, "func"):
|
|
792
|
+
sys.exit(args.func(args))
|
|
793
|
+
else:
|
|
794
|
+
parser.print_help()
|
|
795
|
+
sys.exit(0)
|
|
796
|
+
|
|
797
|
+
|
|
798
|
+
if __name__ == "__main__":
|
|
799
|
+
main()
|