loki-mode 7.79.0 → 7.80.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/SKILL.md +2 -2
- package/VERSION +1 -1
- package/autonomy/loki +130 -0
- package/autonomy/run.sh +3 -0
- package/dashboard/__init__.py +1 -1
- package/dashboard/registry.py +212 -0
- package/dashboard/server.py +236 -0
- package/dashboard/static/index.html +383 -150
- package/docs/ENTERPRISE-IDENTITY-ROADMAP.md +206 -0
- package/docs/INSTALLATION.md +2 -2
- package/loki-ts/dist/loki.js +2 -2
- package/lokistore/__init__.py +65 -0
- package/lokistore/base.py +172 -0
- package/lokistore/cloud.py +305 -0
- package/lokistore/factory.py +187 -0
- package/lokistore/local.py +219 -0
- package/mcp/__init__.py +1 -1
- package/memory/retrieval.py +147 -0
- package/memory/tree_index.py +499 -0
- package/memory/tree_search.py +305 -0
- package/package.json +2 -1
- package/plugins/loki-mode/.claude-plugin/plugin.json +1 -1
package/SKILL.md
CHANGED
|
@@ -3,7 +3,7 @@ name: loki-mode
|
|
|
3
3
|
description: Autonomous spec-driven build system with a built-in trust layer. It does not call work done until it is verified (RARV-C closure loop, 8 quality gates, completion council, verified-completion evidence gate). Triggers on "Loki Mode". Takes a spec (PRD, GitHub issue, OpenAPI doc, etc.) to deployed product with minimal human intervention. Provider-agnostic. Requires --dangerously-skip-permissions flag.
|
|
4
4
|
---
|
|
5
5
|
|
|
6
|
-
# Loki Mode v7.
|
|
6
|
+
# Loki Mode v7.80.0
|
|
7
7
|
|
|
8
8
|
**You are an autonomous agent. You make decisions. You do not ask questions. You do not stop.**
|
|
9
9
|
|
|
@@ -406,4 +406,4 @@ See `CHANGELOG.md` entries [7.5.7], [7.5.8], [7.5.13] for the per-fix list and r
|
|
|
406
406
|
|
|
407
407
|
---
|
|
408
408
|
|
|
409
|
-
**v7.
|
|
409
|
+
**v7.80.0 | [Autonomi](https://www.autonomi.dev/) flagship product | ~260 lines core**
|
package/VERSION
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
7.
|
|
1
|
+
7.80.0
|
package/autonomy/loki
CHANGED
|
@@ -2927,6 +2927,133 @@ cmd_resume() {
|
|
|
2927
2927
|
fi
|
|
2928
2928
|
}
|
|
2929
2929
|
|
|
2930
|
+
# loki why -- actionable failure/outcome diagnosis (B5).
|
|
2931
|
+
# Reads the already-captured run artifacts (no new state): the terminal run state
|
|
2932
|
+
# (.loki/<autonomy-state>.json: status, lastExitCode, iterationCount), the durable
|
|
2933
|
+
# completion record (.loki/state/completion.json: outcome, branch, files, pr_url),
|
|
2934
|
+
# and the latest structured handoff (.loki/memory/handoffs/*.md). Produces an
|
|
2935
|
+
# honest "what happened + what to do" report. Read-only; never fabricates -- if a
|
|
2936
|
+
# field was not captured it says so. --json emits the machine-readable record.
|
|
2937
|
+
cmd_why() {
|
|
2938
|
+
local as_json=0
|
|
2939
|
+
case "${1:-}" in
|
|
2940
|
+
--json) as_json=1 ;;
|
|
2941
|
+
--help|-h) echo "Usage: loki why [--json] -- explain the last build's outcome and what to do next"; return 0 ;;
|
|
2942
|
+
"" ) : ;;
|
|
2943
|
+
*) echo -e "${RED}Unknown flag: $1${NC}"; echo "Usage: loki why [--json]"; return 1 ;;
|
|
2944
|
+
esac
|
|
2945
|
+
|
|
2946
|
+
local loki_dir="${LOKI_DIR:-.loki}"
|
|
2947
|
+
# Session-namespaced state when LOKI_SESSION_ID is set (mirrors run.sh A6).
|
|
2948
|
+
local state_file="$loki_dir/autonomy-state.json"
|
|
2949
|
+
if [ -n "${LOKI_SESSION_ID:-}" ] && [ -f "$loki_dir/sessions/${LOKI_SESSION_ID}/autonomy-state.json" ]; then
|
|
2950
|
+
state_file="$loki_dir/sessions/${LOKI_SESSION_ID}/autonomy-state.json"
|
|
2951
|
+
fi
|
|
2952
|
+
local completion_file="$loki_dir/state/completion.json"
|
|
2953
|
+
|
|
2954
|
+
if [ ! -f "$state_file" ] && [ ! -f "$completion_file" ]; then
|
|
2955
|
+
echo "loki why: no run found here yet (no $state_file or $completion_file)." >&2
|
|
2956
|
+
echo "Run a build first: loki start <spec>" >&2
|
|
2957
|
+
return 1
|
|
2958
|
+
fi
|
|
2959
|
+
|
|
2960
|
+
if [ "$as_json" = "1" ]; then
|
|
2961
|
+
_LOKI_WHY_STATE="$state_file" _LOKI_WHY_COMPLETION="$completion_file" python3 - <<'WHYJSON'
|
|
2962
|
+
import json, os
|
|
2963
|
+
def load(p):
|
|
2964
|
+
try:
|
|
2965
|
+
with open(p) as f: return json.load(f)
|
|
2966
|
+
except Exception: return {}
|
|
2967
|
+
state = load(os.environ.get("_LOKI_WHY_STATE", ""))
|
|
2968
|
+
comp = load(os.environ.get("_LOKI_WHY_COMPLETION", ""))
|
|
2969
|
+
print(json.dumps({"state": state, "completion": comp}, indent=2))
|
|
2970
|
+
WHYJSON
|
|
2971
|
+
return 0
|
|
2972
|
+
fi
|
|
2973
|
+
|
|
2974
|
+
# Human-readable report. The diagnosis maps the terminal status to a plain
|
|
2975
|
+
# explanation + a concrete next action; everything is sourced from the files,
|
|
2976
|
+
# nothing is invented.
|
|
2977
|
+
_LOKI_WHY_STATE="$state_file" _LOKI_WHY_COMPLETION="$completion_file" \
|
|
2978
|
+
_LOKI_WHY_HANDOFFS="$loki_dir/memory/handoffs" python3 - <<'WHYTXT'
|
|
2979
|
+
import json, os, glob
|
|
2980
|
+
def load(p):
|
|
2981
|
+
try:
|
|
2982
|
+
with open(p) as f: return json.load(f)
|
|
2983
|
+
except Exception: return {}
|
|
2984
|
+
state = load(os.environ.get("_LOKI_WHY_STATE", ""))
|
|
2985
|
+
comp = load(os.environ.get("_LOKI_WHY_COMPLETION", ""))
|
|
2986
|
+
status = state.get("status") or comp.get("outcome") or "unknown"
|
|
2987
|
+
exit_code = state.get("lastExitCode")
|
|
2988
|
+
iters = state.get("iterationCount")
|
|
2989
|
+
|
|
2990
|
+
# status -> (one-line meaning, suggested next action). Honest + specific.
|
|
2991
|
+
GUIDE = {
|
|
2992
|
+
"council_approved": ("The completion council agreed the work is done and verified.",
|
|
2993
|
+
"Review the diff and open a PR (git push + gh pr create, or LOKI_AUTO_PR=1)."),
|
|
2994
|
+
"council_force_approved": ("Completion was force-approved (council could not fully converge).",
|
|
2995
|
+
"Review the diff carefully before merging -- convergence was not unanimous."),
|
|
2996
|
+
"completion_promise_fulfilled": ("The agent declared its explicit completion promise fulfilled.",
|
|
2997
|
+
"Verify the promised outcome, then review and PR."),
|
|
2998
|
+
"max_iterations_reached": ("The build hit the iteration cap before the council approved it.",
|
|
2999
|
+
"Inspect what is left (loki status), raise LOKI_MAX_ITERATIONS or narrow the spec, and resume."),
|
|
3000
|
+
"max_retries_exceeded": ("The build exhausted its retry budget on a repeating failure.",
|
|
3001
|
+
"Read .loki/logs for the recurring error (rate limit? failing test?), fix the root cause, then re-run."),
|
|
3002
|
+
"failed": ("The build ended in a failure state.",
|
|
3003
|
+
"Read .loki/logs/ + the handoff below for the failure, fix it, then re-run."),
|
|
3004
|
+
"policy_blocked": ("A policy/trust gate blocked completion.",
|
|
3005
|
+
"Review the blocking finding; address it or use the documented override path."),
|
|
3006
|
+
"budget_exceeded": ("The cost budget breaker paused the build.",
|
|
3007
|
+
"Raise LOKI_BUDGET_LIMIT or accept the partial result, then resume."),
|
|
3008
|
+
"paused": ("The build is paused (human-intervention signal).",
|
|
3009
|
+
"Resume with: loki resume."),
|
|
3010
|
+
"interrupted": ("The build was interrupted before a terminal state.",
|
|
3011
|
+
"Resume with: loki resume."),
|
|
3012
|
+
"stopped": ("The build was stopped by the operator.",
|
|
3013
|
+
"Start a new build with loki start, or resume if you meant to continue."),
|
|
3014
|
+
"force_stopped": ("The build was force-stopped.",
|
|
3015
|
+
"Start a new build, or investigate why a force-stop was needed."),
|
|
3016
|
+
"running": ("The recorded state says a build is still running (or crashed mid-run).",
|
|
3017
|
+
"If no build is active it likely crashed; in durable mode (LOKI_DURABLE_STATE=1) a restart resumes, else loki start re-runs."),
|
|
3018
|
+
}
|
|
3019
|
+
meaning, action = GUIDE.get(status, ("No diagnosis mapping for this status; see the raw fields below.",
|
|
3020
|
+
"Check loki status and .loki/logs/ for detail."))
|
|
3021
|
+
|
|
3022
|
+
print("Loki: why")
|
|
3023
|
+
print("=" * 60)
|
|
3024
|
+
print(f" Outcome : {status}")
|
|
3025
|
+
if exit_code is not None:
|
|
3026
|
+
print(f" Exit code : {exit_code}")
|
|
3027
|
+
if iters is not None:
|
|
3028
|
+
print(f" Iterations : {iters}")
|
|
3029
|
+
if comp.get("branch"):
|
|
3030
|
+
print(f" Branch : {comp['branch']}")
|
|
3031
|
+
if comp.get("files_changed") is not None:
|
|
3032
|
+
print(f" Changes : {comp.get('files_changed',0)} files (+{comp.get('insertions',0)}/-{comp.get('deletions',0)})")
|
|
3033
|
+
if comp.get("pr_url"):
|
|
3034
|
+
print(f" PR : {comp['pr_url']}")
|
|
3035
|
+
print()
|
|
3036
|
+
print(f" What happened: {meaning}")
|
|
3037
|
+
print(f" What to do : {action}")
|
|
3038
|
+
|
|
3039
|
+
# Surface the latest structured handoff (already-captured context), honestly.
|
|
3040
|
+
hd = sorted(glob.glob(os.path.join(os.environ.get("_LOKI_WHY_HANDOFFS",""), "*.md")))
|
|
3041
|
+
if hd:
|
|
3042
|
+
print()
|
|
3043
|
+
print(f" Latest handoff: {hd[-1]}")
|
|
3044
|
+
try:
|
|
3045
|
+
with open(hd[-1]) as f:
|
|
3046
|
+
head = "".join(f.readlines()[:8]).rstrip()
|
|
3047
|
+
for line in head.splitlines():
|
|
3048
|
+
print(f" {line}")
|
|
3049
|
+
except Exception:
|
|
3050
|
+
pass
|
|
3051
|
+
print()
|
|
3052
|
+
print(" (loki why --json for the raw record; loki status for live state.)")
|
|
3053
|
+
WHYTXT
|
|
3054
|
+
return 0
|
|
3055
|
+
}
|
|
3056
|
+
|
|
2930
3057
|
# Show current status
|
|
2931
3058
|
cmd_status() {
|
|
2932
3059
|
# Check for flags
|
|
@@ -15481,6 +15608,9 @@ main() {
|
|
|
15481
15608
|
status)
|
|
15482
15609
|
cmd_status "$@"
|
|
15483
15610
|
;;
|
|
15611
|
+
why)
|
|
15612
|
+
cmd_why "$@"
|
|
15613
|
+
;;
|
|
15484
15614
|
stats)
|
|
15485
15615
|
# CLI consolidation (Phase A): 'stats' is a deprecated alias of
|
|
15486
15616
|
# 'report session'. On the Bun route this arm is never reached
|
package/autonomy/run.sh
CHANGED
|
@@ -5253,6 +5253,9 @@ print_ttfv_next_steps() {
|
|
|
5253
5253
|
echo " loki start ./prd.md # build from a full PRD"
|
|
5254
5254
|
echo " loki start \"<one line>\" # fast first pass from a brief"
|
|
5255
5255
|
fi
|
|
5256
|
+
# Frictionless discovery: surface 'loki why' so users learn the
|
|
5257
|
+
# what-happened/what-to-do diagnosis without having to know it exists.
|
|
5258
|
+
echo " loki why # explain this outcome + what to do next"
|
|
5256
5259
|
echo ""
|
|
5257
5260
|
return 0
|
|
5258
5261
|
}
|
package/dashboard/__init__.py
CHANGED
package/dashboard/registry.py
CHANGED
|
@@ -525,6 +525,218 @@ def get_cross_project_tasks(project_ids: Optional[list[str]] = None) -> list[dic
|
|
|
525
525
|
return all_tasks
|
|
526
526
|
|
|
527
527
|
|
|
528
|
+
def _pid_alive(pid) -> bool:
|
|
529
|
+
"""Return True if pid is a positive int naming a live process.
|
|
530
|
+
|
|
531
|
+
Mirrors the liveness probe used by the dashboard's running-projects view:
|
|
532
|
+
signal 0 delivered -> alive; EPERM (owned by another user) -> alive; ESRCH
|
|
533
|
+
-> dead. Never raises.
|
|
534
|
+
"""
|
|
535
|
+
if not isinstance(pid, int) or pid <= 0:
|
|
536
|
+
return False
|
|
537
|
+
try:
|
|
538
|
+
os.kill(pid, 0)
|
|
539
|
+
return True
|
|
540
|
+
except PermissionError:
|
|
541
|
+
return True
|
|
542
|
+
except (ProcessLookupError, OSError):
|
|
543
|
+
return False
|
|
544
|
+
|
|
545
|
+
|
|
546
|
+
def _read_project_run_snapshot(path: str) -> dict:
|
|
547
|
+
"""Read a single project's live run snapshot from its .loki/ state files.
|
|
548
|
+
|
|
549
|
+
HONEST SCOPE: this polls the shared on-disk metadata that `loki start`
|
|
550
|
+
already writes per project (no new store, no controller). It reads:
|
|
551
|
+
- .loki/dashboard-state.json (phase, iteration; written ~every 2s)
|
|
552
|
+
- .loki/session.json (status, startedAt fallback)
|
|
553
|
+
- .loki/metrics/efficiency/*.json (summed cost_usd) with a
|
|
554
|
+
.loki/context/tracking.json fallback (totals.total_cost_usd)
|
|
555
|
+
|
|
556
|
+
Returns a dict with phase, iteration, cost_usd, started_at, and ended_at
|
|
557
|
+
(best-effort; missing values default to None/0). Never raises: any file
|
|
558
|
+
problem degrades the affected field to its default.
|
|
559
|
+
"""
|
|
560
|
+
snap = {
|
|
561
|
+
"phase": "",
|
|
562
|
+
"iteration": 0,
|
|
563
|
+
"cost_usd": 0.0,
|
|
564
|
+
"started_at": None,
|
|
565
|
+
"ended_at": None,
|
|
566
|
+
}
|
|
567
|
+
if not path:
|
|
568
|
+
return snap
|
|
569
|
+
loki_dir = Path(path) / ".loki"
|
|
570
|
+
|
|
571
|
+
# Phase + iteration from dashboard-state.json (the live writer).
|
|
572
|
+
state_file = loki_dir / "dashboard-state.json"
|
|
573
|
+
if state_file.exists():
|
|
574
|
+
try:
|
|
575
|
+
state = json.loads(state_file.read_text())
|
|
576
|
+
if isinstance(state, dict):
|
|
577
|
+
_p = state.get("phase", "")
|
|
578
|
+
snap["phase"] = _p if isinstance(_p, str) else ""
|
|
579
|
+
_i = state.get("iteration", 0)
|
|
580
|
+
snap["iteration"] = _i if isinstance(_i, int) else 0
|
|
581
|
+
except (json.JSONDecodeError, OSError, ValueError):
|
|
582
|
+
pass
|
|
583
|
+
|
|
584
|
+
# Timestamps from session.json (startedAt; endedAt when present).
|
|
585
|
+
session_file = loki_dir / "session.json"
|
|
586
|
+
if session_file.exists():
|
|
587
|
+
try:
|
|
588
|
+
sd = json.loads(session_file.read_text())
|
|
589
|
+
if isinstance(sd, dict):
|
|
590
|
+
_sa = sd.get("startedAt") or sd.get("started_at")
|
|
591
|
+
snap["started_at"] = _sa if isinstance(_sa, str) else None
|
|
592
|
+
_ea = sd.get("endedAt") or sd.get("ended_at")
|
|
593
|
+
snap["ended_at"] = _ea if isinstance(_ea, str) else None
|
|
594
|
+
except (json.JSONDecodeError, OSError, ValueError):
|
|
595
|
+
pass
|
|
596
|
+
|
|
597
|
+
# Cost: sum per-iteration efficiency files; fall back to context tracking.
|
|
598
|
+
cost = 0.0
|
|
599
|
+
found_cost = False
|
|
600
|
+
eff_dir = loki_dir / "metrics" / "efficiency"
|
|
601
|
+
if eff_dir.is_dir():
|
|
602
|
+
try:
|
|
603
|
+
for eff_file in eff_dir.glob("*.json"):
|
|
604
|
+
try:
|
|
605
|
+
data = json.loads(eff_file.read_text())
|
|
606
|
+
if not isinstance(data, dict):
|
|
607
|
+
continue
|
|
608
|
+
c = data.get("cost_usd")
|
|
609
|
+
if isinstance(c, (int, float)):
|
|
610
|
+
cost += float(c)
|
|
611
|
+
found_cost = True
|
|
612
|
+
except (json.JSONDecodeError, OSError, ValueError):
|
|
613
|
+
continue
|
|
614
|
+
except OSError:
|
|
615
|
+
pass
|
|
616
|
+
if not found_cost:
|
|
617
|
+
ctx_file = loki_dir / "context" / "tracking.json"
|
|
618
|
+
if ctx_file.exists():
|
|
619
|
+
try:
|
|
620
|
+
ctx = json.loads(ctx_file.read_text())
|
|
621
|
+
if isinstance(ctx, dict):
|
|
622
|
+
totals = ctx.get("totals", {})
|
|
623
|
+
if isinstance(totals, dict):
|
|
624
|
+
tc = totals.get("total_cost_usd")
|
|
625
|
+
if isinstance(tc, (int, float)):
|
|
626
|
+
cost = float(tc)
|
|
627
|
+
except (json.JSONDecodeError, OSError, ValueError):
|
|
628
|
+
pass
|
|
629
|
+
snap["cost_usd"] = round(cost, 6)
|
|
630
|
+
return snap
|
|
631
|
+
|
|
632
|
+
|
|
633
|
+
def get_fleet_runs(include_inactive: bool = True) -> list[dict]:
|
|
634
|
+
"""Build a fleet-wide view of builds across ALL registered projects.
|
|
635
|
+
|
|
636
|
+
v1 SCOPE (honest): this polls the shared metadata store that `loki start`
|
|
637
|
+
already maintains -- the machine-global registry (~/.loki/dashboard/
|
|
638
|
+
projects.json) plus each project's own .loki/ state files. There is NO
|
|
639
|
+
controller, CRD, or Job-watcher here; a real k8s operator watching Jobs is
|
|
640
|
+
future work. One registered project maps to one "run" entry (its current /
|
|
641
|
+
most-recent build), which is the granularity the registry tracks.
|
|
642
|
+
|
|
643
|
+
Each entry carries: id, name, path, status (running|stopped|<registry
|
|
644
|
+
status>), running (live pid probe), phase, iteration, cost_usd, started_at,
|
|
645
|
+
duration_seconds, port. Never raises: registry problems degrade to an empty
|
|
646
|
+
list and per-project read problems degrade that entry's fields.
|
|
647
|
+
"""
|
|
648
|
+
try:
|
|
649
|
+
projects = list_projects(include_inactive=include_inactive)
|
|
650
|
+
except Exception:
|
|
651
|
+
return []
|
|
652
|
+
|
|
653
|
+
out = []
|
|
654
|
+
for p in projects:
|
|
655
|
+
path = p.get("path", "")
|
|
656
|
+
pid = p.get("pid")
|
|
657
|
+
running = _pid_alive(pid)
|
|
658
|
+
snap = _read_project_run_snapshot(path)
|
|
659
|
+
|
|
660
|
+
# A live pid is authoritative for "running"; otherwise reflect the
|
|
661
|
+
# registry status (stopped/active/missing). This mirrors the
|
|
662
|
+
# running-projects endpoint's pid-first precedence.
|
|
663
|
+
if running:
|
|
664
|
+
status = "running"
|
|
665
|
+
else:
|
|
666
|
+
status = p.get("status") or "unknown"
|
|
667
|
+
|
|
668
|
+
# Duration: wall time from started_at to ended_at (or now if running).
|
|
669
|
+
duration_seconds = None
|
|
670
|
+
started_at = snap.get("started_at")
|
|
671
|
+
if started_at:
|
|
672
|
+
try:
|
|
673
|
+
st = datetime.fromisoformat(str(started_at).replace("Z", "+00:00"))
|
|
674
|
+
if st.tzinfo is None:
|
|
675
|
+
st = st.replace(tzinfo=timezone.utc)
|
|
676
|
+
end_ref = None
|
|
677
|
+
ended_at = snap.get("ended_at")
|
|
678
|
+
if ended_at and not running:
|
|
679
|
+
try:
|
|
680
|
+
end_ref = datetime.fromisoformat(
|
|
681
|
+
str(ended_at).replace("Z", "+00:00")
|
|
682
|
+
)
|
|
683
|
+
if end_ref.tzinfo is None:
|
|
684
|
+
end_ref = end_ref.replace(tzinfo=timezone.utc)
|
|
685
|
+
except (ValueError, TypeError):
|
|
686
|
+
end_ref = None
|
|
687
|
+
if end_ref is None:
|
|
688
|
+
end_ref = datetime.now(timezone.utc)
|
|
689
|
+
duration_seconds = max(0, int((end_ref - st).total_seconds()))
|
|
690
|
+
except (ValueError, TypeError):
|
|
691
|
+
duration_seconds = None
|
|
692
|
+
|
|
693
|
+
out.append({
|
|
694
|
+
"id": p.get("id"),
|
|
695
|
+
"name": p.get("name") or (os.path.basename(path) if path else "project"),
|
|
696
|
+
"path": path,
|
|
697
|
+
"status": status,
|
|
698
|
+
"running": running,
|
|
699
|
+
"phase": snap.get("phase", ""),
|
|
700
|
+
"iteration": snap.get("iteration", 0),
|
|
701
|
+
"cost_usd": snap.get("cost_usd", 0.0),
|
|
702
|
+
"started_at": started_at,
|
|
703
|
+
"duration_seconds": duration_seconds,
|
|
704
|
+
"port": p.get("port"),
|
|
705
|
+
})
|
|
706
|
+
|
|
707
|
+
# Running builds first, then by most-recent start time.
|
|
708
|
+
out.sort(
|
|
709
|
+
key=lambda r: (
|
|
710
|
+
0 if r.get("running") else 1,
|
|
711
|
+
r.get("started_at") or "",
|
|
712
|
+
),
|
|
713
|
+
reverse=False,
|
|
714
|
+
)
|
|
715
|
+
# Within the same running-group, most recent start first.
|
|
716
|
+
out.sort(key=lambda r: r.get("started_at") or "", reverse=True)
|
|
717
|
+
out.sort(key=lambda r: 0 if r.get("running") else 1)
|
|
718
|
+
return out
|
|
719
|
+
|
|
720
|
+
|
|
721
|
+
def get_fleet_summary(include_inactive: bool = True) -> dict:
|
|
722
|
+
"""Aggregate fleet-wide totals from get_fleet_runs().
|
|
723
|
+
|
|
724
|
+
Returns counts (total / running / stopped) and a summed cost across all
|
|
725
|
+
registered projects. v1 polls the shared metadata store (see
|
|
726
|
+
get_fleet_runs); not a controller. Never raises.
|
|
727
|
+
"""
|
|
728
|
+
runs = get_fleet_runs(include_inactive=include_inactive)
|
|
729
|
+
total = len(runs)
|
|
730
|
+
running = sum(1 for r in runs if r.get("running"))
|
|
731
|
+
total_cost = round(sum(float(r.get("cost_usd") or 0.0) for r in runs), 6)
|
|
732
|
+
return {
|
|
733
|
+
"total_runs": total,
|
|
734
|
+
"running_runs": running,
|
|
735
|
+
"stopped_runs": total - running,
|
|
736
|
+
"total_cost_usd": total_cost,
|
|
737
|
+
}
|
|
738
|
+
|
|
739
|
+
|
|
528
740
|
def get_cross_project_learnings() -> dict:
|
|
529
741
|
"""
|
|
530
742
|
Get learnings from the global learnings database.
|
package/dashboard/server.py
CHANGED
|
@@ -2389,6 +2389,118 @@ async def get_cross_project_learnings():
|
|
|
2389
2389
|
return learnings
|
|
2390
2390
|
|
|
2391
2391
|
|
|
2392
|
+
# =============================================================================
|
|
2393
|
+
# Fleet Observability (v1: poll the shared metadata store)
|
|
2394
|
+
# =============================================================================
|
|
2395
|
+
#
|
|
2396
|
+
# HONEST SCOPE: these endpoints aggregate the data `loki start` already writes
|
|
2397
|
+
# -- the machine-global registry (~/.loki/dashboard/projects.json) plus each
|
|
2398
|
+
# project's own .loki/ state files. There is NO controller, CRD, or Kubernetes
|
|
2399
|
+
# Job-watcher; a real operator watching Jobs is future work. One registered
|
|
2400
|
+
# project maps to one fleet "run" (its current / most-recent build), which is
|
|
2401
|
+
# the granularity the registry tracks. Cancel reuses the same STOP-file + pid
|
|
2402
|
+
# teardown as the per-project switcher Stop. Retry is intentionally NOT exposed
|
|
2403
|
+
# here: there is no clean cross-project re-launch primitive in the registry
|
|
2404
|
+
# path (the original spec source lives only in each project's CWD), so retry is
|
|
2405
|
+
# a follow-up.
|
|
2406
|
+
|
|
2407
|
+
|
|
2408
|
+
class FleetRunResponse(BaseModel):
|
|
2409
|
+
"""One fleet run = one registered project's current/most-recent build."""
|
|
2410
|
+
id: Optional[str] = None
|
|
2411
|
+
name: str
|
|
2412
|
+
path: str
|
|
2413
|
+
status: str
|
|
2414
|
+
running: bool
|
|
2415
|
+
phase: str = ""
|
|
2416
|
+
iteration: int = 0
|
|
2417
|
+
cost_usd: float = 0.0
|
|
2418
|
+
started_at: Optional[str] = None
|
|
2419
|
+
duration_seconds: Optional[int] = None
|
|
2420
|
+
port: Optional[int] = None
|
|
2421
|
+
|
|
2422
|
+
|
|
2423
|
+
class FleetSummaryResponse(BaseModel):
|
|
2424
|
+
"""Fleet-wide totals across all registered projects."""
|
|
2425
|
+
total_runs: int
|
|
2426
|
+
running_runs: int
|
|
2427
|
+
stopped_runs: int
|
|
2428
|
+
total_cost_usd: float
|
|
2429
|
+
|
|
2430
|
+
|
|
2431
|
+
@app.get(
|
|
2432
|
+
"/api/fleet/runs",
|
|
2433
|
+
response_model=list[FleetRunResponse],
|
|
2434
|
+
dependencies=[Depends(auth.require_scope("read"))],
|
|
2435
|
+
)
|
|
2436
|
+
async def list_fleet_runs(include_inactive: bool = True):
|
|
2437
|
+
"""List all builds across every registered project (fleet view).
|
|
2438
|
+
|
|
2439
|
+
v1 polls the shared metadata store (registry + per-project .loki/ state);
|
|
2440
|
+
it is not a controller. Never raises: registry problems degrade to [].
|
|
2441
|
+
"""
|
|
2442
|
+
if not _read_limiter.check("fleet_runs"):
|
|
2443
|
+
raise HTTPException(status_code=429, detail="Rate limit exceeded")
|
|
2444
|
+
return await asyncio.to_thread(registry.get_fleet_runs, include_inactive)
|
|
2445
|
+
|
|
2446
|
+
|
|
2447
|
+
@app.get(
|
|
2448
|
+
"/api/fleet/summary",
|
|
2449
|
+
response_model=FleetSummaryResponse,
|
|
2450
|
+
dependencies=[Depends(auth.require_scope("read"))],
|
|
2451
|
+
)
|
|
2452
|
+
async def get_fleet_summary(include_inactive: bool = True):
|
|
2453
|
+
"""Fleet-wide totals (counts + summed cost) across registered projects."""
|
|
2454
|
+
if not _read_limiter.check("fleet_summary"):
|
|
2455
|
+
raise HTTPException(status_code=429, detail="Rate limit exceeded")
|
|
2456
|
+
return await asyncio.to_thread(registry.get_fleet_summary, include_inactive)
|
|
2457
|
+
|
|
2458
|
+
|
|
2459
|
+
@app.get(
|
|
2460
|
+
"/api/fleet/runs/{identifier}",
|
|
2461
|
+
response_model=FleetRunResponse,
|
|
2462
|
+
dependencies=[Depends(auth.require_scope("read"))],
|
|
2463
|
+
)
|
|
2464
|
+
async def get_fleet_run(identifier: str):
|
|
2465
|
+
"""Get a single fleet run by registry id / path / alias.
|
|
2466
|
+
|
|
2467
|
+
Resolves the identifier through the registry (never a caller-supplied
|
|
2468
|
+
arbitrary path), then returns that project's current build snapshot.
|
|
2469
|
+
"""
|
|
2470
|
+
project = await asyncio.to_thread(registry.get_project, identifier)
|
|
2471
|
+
if not project:
|
|
2472
|
+
raise HTTPException(status_code=404, detail="Run not found in fleet")
|
|
2473
|
+
pid = project.get("pid")
|
|
2474
|
+
running = registry._pid_alive(pid)
|
|
2475
|
+
snap = registry._read_project_run_snapshot(project.get("path", ""))
|
|
2476
|
+
duration_seconds = None
|
|
2477
|
+
started_at = snap.get("started_at")
|
|
2478
|
+
if started_at:
|
|
2479
|
+
try:
|
|
2480
|
+
st = datetime.fromisoformat(str(started_at).replace("Z", "+00:00"))
|
|
2481
|
+
if st.tzinfo is None:
|
|
2482
|
+
st = st.replace(tzinfo=timezone.utc)
|
|
2483
|
+
duration_seconds = max(
|
|
2484
|
+
0, int((datetime.now(timezone.utc) - st).total_seconds())
|
|
2485
|
+
)
|
|
2486
|
+
except (ValueError, TypeError):
|
|
2487
|
+
duration_seconds = None
|
|
2488
|
+
path = project.get("path", "")
|
|
2489
|
+
return FleetRunResponse(
|
|
2490
|
+
id=project.get("id"),
|
|
2491
|
+
name=project.get("name") or (os.path.basename(path) if path else "project"),
|
|
2492
|
+
path=path,
|
|
2493
|
+
status="running" if running else (project.get("status") or "unknown"),
|
|
2494
|
+
running=running,
|
|
2495
|
+
phase=snap.get("phase", ""),
|
|
2496
|
+
iteration=snap.get("iteration", 0),
|
|
2497
|
+
cost_usd=snap.get("cost_usd", 0.0),
|
|
2498
|
+
started_at=started_at,
|
|
2499
|
+
duration_seconds=duration_seconds,
|
|
2500
|
+
port=project.get("port"),
|
|
2501
|
+
)
|
|
2502
|
+
|
|
2503
|
+
|
|
2392
2504
|
# =============================================================================
|
|
2393
2505
|
# Active Project Focus (for AI Chat / cross-directory usage)
|
|
2394
2506
|
# =============================================================================
|
|
@@ -3204,6 +3316,130 @@ async def stop_running_project(request: Request, body: RunningProjectStopRequest
|
|
|
3204
3316
|
}
|
|
3205
3317
|
|
|
3206
3318
|
|
|
3319
|
+
@app.post(
|
|
3320
|
+
"/api/fleet/runs/{identifier}/cancel",
|
|
3321
|
+
dependencies=[Depends(auth.require_scope("control"))],
|
|
3322
|
+
)
|
|
3323
|
+
async def cancel_fleet_run(request: Request, identifier: str):
|
|
3324
|
+
"""Cancel ONE build in the fleet view.
|
|
3325
|
+
|
|
3326
|
+
Resolves the run via the registry (by id / path / alias), then runs the
|
|
3327
|
+
same teardown the per-project switcher Stop uses: write a STOP file into the
|
|
3328
|
+
registry-resolved .loki dir (the only path ever written -- never a
|
|
3329
|
+
caller-supplied one) for a clean runner exit, SIGTERM->poll->SIGKILL the
|
|
3330
|
+
recorded orchestrator pid, group-kill + cwd-scoped reap as backstop, then
|
|
3331
|
+
mark the registry/session stopped.
|
|
3332
|
+
|
|
3333
|
+
Retry is intentionally NOT exposed: there is no clean cross-project
|
|
3334
|
+
re-launch primitive in the registry path (the original spec source lives
|
|
3335
|
+
only in each project's CWD). Retry is a documented follow-up.
|
|
3336
|
+
"""
|
|
3337
|
+
if not _control_limiter.check("control"):
|
|
3338
|
+
raise HTTPException(status_code=429, detail="Rate limit exceeded")
|
|
3339
|
+
|
|
3340
|
+
project = registry.get_project(identifier)
|
|
3341
|
+
if not project:
|
|
3342
|
+
raise HTTPException(status_code=404, detail="Run not found in fleet")
|
|
3343
|
+
|
|
3344
|
+
project_id = project.get("id")
|
|
3345
|
+
audit.log_event(
|
|
3346
|
+
action="cancel",
|
|
3347
|
+
resource_type="fleet_run",
|
|
3348
|
+
details={"source": "api", "project_id": project_id},
|
|
3349
|
+
ip_address=request.client.host if request.client else None,
|
|
3350
|
+
)
|
|
3351
|
+
|
|
3352
|
+
# Only ever operate on the registry-stored path.
|
|
3353
|
+
path = project.get("path", "")
|
|
3354
|
+
loki_dir = None
|
|
3355
|
+
if path:
|
|
3356
|
+
p = _Path(path)
|
|
3357
|
+
if p.is_dir() and (p / ".loki").is_dir():
|
|
3358
|
+
loki_dir = p / ".loki"
|
|
3359
|
+
|
|
3360
|
+
# STOP file: clean runner teardown (also stops a containerized `loki docker`
|
|
3361
|
+
# build polling the bind-mounted .loki/STOP).
|
|
3362
|
+
stop_signaled = False
|
|
3363
|
+
if loki_dir is not None:
|
|
3364
|
+
try:
|
|
3365
|
+
(loki_dir / "STOP").write_text(datetime.now(timezone.utc).isoformat())
|
|
3366
|
+
stop_signaled = True
|
|
3367
|
+
except OSError:
|
|
3368
|
+
pass
|
|
3369
|
+
|
|
3370
|
+
pid = project.get("pid")
|
|
3371
|
+
stopped = False
|
|
3372
|
+
# Ownership guard (hardening): only direct-kill the registry pid if it is STILL
|
|
3373
|
+
# a process whose cwd is this project's path. A stale pid reused by an unrelated
|
|
3374
|
+
# host process after a crash must NOT be SIGKILLed. If ownership cannot be
|
|
3375
|
+
# confirmed, skip the direct kill and let the cwd-scoped reaper below handle it.
|
|
3376
|
+
if isinstance(pid, int) and pid > 0:
|
|
3377
|
+
_owned = True
|
|
3378
|
+
try:
|
|
3379
|
+
_cwd = _pid_cwd(pid)
|
|
3380
|
+
if _cwd is not None and path:
|
|
3381
|
+
_owned = os.path.realpath(_cwd) == os.path.realpath(path)
|
|
3382
|
+
except Exception:
|
|
3383
|
+
_owned = True # best-effort: if we cannot tell, preserve prior behavior
|
|
3384
|
+
if not _owned:
|
|
3385
|
+
pid = None # do not direct-kill a pid that is not this project's
|
|
3386
|
+
if isinstance(pid, int) and pid > 0:
|
|
3387
|
+
try:
|
|
3388
|
+
os.kill(pid, 15) # SIGTERM
|
|
3389
|
+
for _ in range(10):
|
|
3390
|
+
await asyncio.sleep(0.5)
|
|
3391
|
+
try:
|
|
3392
|
+
os.kill(pid, 0)
|
|
3393
|
+
except OSError:
|
|
3394
|
+
stopped = True
|
|
3395
|
+
break
|
|
3396
|
+
if not stopped:
|
|
3397
|
+
try:
|
|
3398
|
+
os.kill(pid, 9) # SIGKILL
|
|
3399
|
+
stopped = True
|
|
3400
|
+
except (OSError, ProcessLookupError):
|
|
3401
|
+
stopped = True
|
|
3402
|
+
except (ValueError, OSError, ProcessLookupError):
|
|
3403
|
+
stopped = True
|
|
3404
|
+
else:
|
|
3405
|
+
# No host pid (genuinely stopped, or a docker build): the STOP write is
|
|
3406
|
+
# the cancel signal. Treat a successful STOP write as cancelled.
|
|
3407
|
+
stopped = stop_signaled
|
|
3408
|
+
|
|
3409
|
+
# Group-kill + cwd-scoped reaper backstop against a stale pid.
|
|
3410
|
+
if loki_dir is not None:
|
|
3411
|
+
proj_dir = loki_dir.parent
|
|
3412
|
+
_pgid = _read_pgid(loki_dir)
|
|
3413
|
+
if _pgid is not None:
|
|
3414
|
+
await asyncio.to_thread(
|
|
3415
|
+
_killpg_project, _pgid, _collect_protected_pids(loki_dir)
|
|
3416
|
+
)
|
|
3417
|
+
found_any, all_gone = await asyncio.to_thread(
|
|
3418
|
+
_reap_orchestrators_until_clear, proj_dir, str(proj_dir)
|
|
3419
|
+
)
|
|
3420
|
+
if found_any:
|
|
3421
|
+
stopped = all_gone
|
|
3422
|
+
|
|
3423
|
+
# Mark session.json stopped.
|
|
3424
|
+
session_file = loki_dir / "session.json"
|
|
3425
|
+
if session_file.exists():
|
|
3426
|
+
try:
|
|
3427
|
+
sd = json.loads(session_file.read_text())
|
|
3428
|
+
sd["status"] = "stopped"
|
|
3429
|
+
atomic_write_json(session_file, sd, use_lock=True)
|
|
3430
|
+
except Exception:
|
|
3431
|
+
pass
|
|
3432
|
+
|
|
3433
|
+
registry.mark_project_stopped(project_id)
|
|
3434
|
+
|
|
3435
|
+
return {
|
|
3436
|
+
"success": True,
|
|
3437
|
+
"project_id": project_id,
|
|
3438
|
+
"cancelled": stopped,
|
|
3439
|
+
"stop_signaled": stop_signaled,
|
|
3440
|
+
}
|
|
3441
|
+
|
|
3442
|
+
|
|
3207
3443
|
# =============================================================================
|
|
3208
3444
|
# Enterprise Features (Optional - enabled via environment variables)
|
|
3209
3445
|
# =============================================================================
|