loki-mode 7.104.5 → 7.106.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/SKILL.md +2 -2
- package/VERSION +1 -1
- package/autonomy/completion-council.sh +183 -1
- package/autonomy/run.sh +24 -1
- package/dashboard/__init__.py +1 -1
- package/docs/FOUNDER-STATUS-AND-DECISIONS-2026-07-01.md +106 -0
- package/docs/INSTALLATION.md +1 -1
- package/docs/SPEED-DIAGNOSIS-2026-07-01.md +101 -0
- package/docs/TOP-100-BACKLOG.md +122 -0
- package/docs/research-2026-07/GAP-ANALYSIS-BACKLOG.md +232 -0
- package/docs/research-2026-07/OBSERVABILITY-TOOL-SPEC.md +96 -0
- package/loki-ts/dist/loki.js +2 -2
- package/mcp/__init__.py +1 -1
- package/package.json +1 -1
- package/plugins/loki-mode/.claude-plugin/plugin.json +1 -1
package/SKILL.md
CHANGED
|
@@ -3,7 +3,7 @@ name: loki-mode
|
|
|
3
3
|
description: Autonomous spec-driven build system with a built-in trust layer. It does not call work done until it is verified (RARV-C closure loop, 8 quality gates, completion council, verified-completion evidence gate). Triggers on "Loki Mode". Takes a spec (PRD, GitHub issue, OpenAPI doc, etc.) to deployed product with minimal human intervention. Provider-agnostic. Requires --dangerously-skip-permissions flag.
|
|
4
4
|
---
|
|
5
5
|
|
|
6
|
-
# Loki Mode v7.
|
|
6
|
+
# Loki Mode v7.106.0
|
|
7
7
|
|
|
8
8
|
**You are an autonomous agent. You make decisions. You do not ask questions. You do not stop.**
|
|
9
9
|
|
|
@@ -408,4 +408,4 @@ See `CHANGELOG.md` entries [7.5.7], [7.5.8], [7.5.13] for the per-fix list and r
|
|
|
408
408
|
|
|
409
409
|
---
|
|
410
410
|
|
|
411
|
-
**v7.
|
|
411
|
+
**v7.106.0 | [Autonomi](https://www.autonomi.dev/) flagship product | ~260 lines core**
|
package/VERSION
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
7.
|
|
1
|
+
7.106.0
|
|
@@ -1517,6 +1517,112 @@ HELDOUT_EOF
|
|
|
1517
1517
|
return 0
|
|
1518
1518
|
}
|
|
1519
1519
|
|
|
1520
|
+
# v7.106.0: reverse-classical test provenance.
|
|
1521
|
+
# FrontierCode insight: a NEW test only proves the change if it FAILS on the
|
|
1522
|
+
# pre-change base and PASSES on HEAD. A test that passes on BOTH is tautological
|
|
1523
|
+
# (or was written to pass) and proves nothing -- it must NOT count as affirmative
|
|
1524
|
+
# "tests green" evidence. This helper runs the run's NEW/CHANGED test files
|
|
1525
|
+
# against the run-start base in an ISOLATED git worktree (NEVER touching the live
|
|
1526
|
+
# tree -- the self-delete/self-mutate hazard) and classifies the provenance.
|
|
1527
|
+
#
|
|
1528
|
+
# INVARIANT (safety): this only ever DOWNGRADES unearned affirmative -> inconclusive.
|
|
1529
|
+
# It NEVER grants affirmative, and NEVER turns inconclusive into a block. Any
|
|
1530
|
+
# failure to establish provenance (no base, no git, no test files, scratch
|
|
1531
|
+
# materialization error, runner error) returns "unknown" and the caller keeps the
|
|
1532
|
+
# existing behavior -- fail OPEN toward the current gate, never a false block.
|
|
1533
|
+
#
|
|
1534
|
+
# Echoes exactly one token:
|
|
1535
|
+
# tautological-> the FULL suite (incl. the new/changed tests) PASSES on base, so
|
|
1536
|
+
# the new tests added no failing signal on the old code = they
|
|
1537
|
+
# prove nothing = downgrade to inconclusive.
|
|
1538
|
+
# unknown -> cannot determine, so caller no-ops (keeps existing behavior).
|
|
1539
|
+
# Covers: no base/git/tests, materialization/runner error, AND a
|
|
1540
|
+
# base-suite FAILURE (rc != 0) -- because a whole-suite failure
|
|
1541
|
+
# cannot be attributed to the new test specifically, we do NOT
|
|
1542
|
+
# claim "confirmed provenance" from it (that would be a false
|
|
1543
|
+
# signal). So there is deliberately NO affirmative-granting token:
|
|
1544
|
+
# this helper can only ever DOWNGRADE, never confirm.
|
|
1545
|
+
# Opt out with LOKI_TEST_PROVENANCE=0 (returns unknown immediately).
|
|
1546
|
+
_loki_test_provenance() {
|
|
1547
|
+
[ "${LOKI_TEST_PROVENANCE:-1}" = "0" ] && { echo "unknown"; return 0; }
|
|
1548
|
+
local base_sha="$1" test_cmd="$2"
|
|
1549
|
+
local target="${TARGET_DIR:-.}"
|
|
1550
|
+
[ -n "$base_sha" ] || { echo "unknown"; return 0; }
|
|
1551
|
+
command -v git >/dev/null 2>&1 || { echo "unknown"; return 0; }
|
|
1552
|
+
( cd "$target" 2>/dev/null && git rev-parse --is-inside-work-tree >/dev/null 2>&1 ) || { echo "unknown"; return 0; }
|
|
1553
|
+
( cd "$target" 2>/dev/null && git cat-file -e "${base_sha}^{commit}" 2>/dev/null ) || { echo "unknown"; return 0; }
|
|
1554
|
+
|
|
1555
|
+
# Identify NEW/CHANGED test files in the diff vs base (committed + working tree).
|
|
1556
|
+
# Heuristic test-path match across common ecosystems; conservative (only files
|
|
1557
|
+
# that look like tests). If none, provenance is not applicable -> unknown.
|
|
1558
|
+
local test_files
|
|
1559
|
+
test_files=$( cd "$target" 2>/dev/null && {
|
|
1560
|
+
git diff --name-only "$base_sha" -- 2>/dev/null
|
|
1561
|
+
git ls-files --others --exclude-standard 2>/dev/null
|
|
1562
|
+
} | LC_ALL=C sort -u | grep -iE '(^|/)(test|tests|spec|__tests__)/|(\.|_|-)(test|spec)\.[a-z0-9]+$|_test\.[a-z0-9]+$|test_[^/]*\.py$' || true )
|
|
1563
|
+
[ -n "$test_files" ] || { echo "unknown"; return 0; }
|
|
1564
|
+
|
|
1565
|
+
# Materialize base in an ISOLATED worktree; never checkout in the live tree.
|
|
1566
|
+
local scratch; scratch="$(mktemp -d "${TMPDIR:-/tmp}/loki-prov-XXXXXX" 2>/dev/null)" || { echo "unknown"; return 0; }
|
|
1567
|
+
local wt="$scratch/base"
|
|
1568
|
+
if ! ( cd "$target" && git worktree add --detach --quiet "$wt" "$base_sha" 2>/dev/null ); then
|
|
1569
|
+
rm -rf "$scratch" 2>/dev/null; echo "unknown"; return 0
|
|
1570
|
+
fi
|
|
1571
|
+
|
|
1572
|
+
# Inject the current NEW/CHANGED test files into the base worktree so we run
|
|
1573
|
+
# the AGENT'S tests against the OLD code. Copy from the live tree (HEAD/working
|
|
1574
|
+
# version of each test file). Missing source (deleted test) is skipped.
|
|
1575
|
+
local f
|
|
1576
|
+
while IFS= read -r f; do
|
|
1577
|
+
[ -n "$f" ] || continue
|
|
1578
|
+
[ -f "$target/$f" ] || continue
|
|
1579
|
+
mkdir -p "$wt/$(dirname "$f")" 2>/dev/null || true
|
|
1580
|
+
cp -f "$target/$f" "$wt/$f" 2>/dev/null || true
|
|
1581
|
+
done <<< "$test_files"
|
|
1582
|
+
|
|
1583
|
+
# Run the recorded test command in the base worktree, bounded. If the runner
|
|
1584
|
+
# cannot run at all (missing deps on base, command error distinct from a test
|
|
1585
|
+
# failure), we cannot conclude tautological -> unknown (fail open).
|
|
1586
|
+
#
|
|
1587
|
+
# LIVENESS SAFETY: the base run MUST be bounded, else a base suite that hangs
|
|
1588
|
+
# (e.g. waits on a service/dep the new code introduced) would stall the
|
|
1589
|
+
# completion-time evidence gate. If no timeout binary is available (stock
|
|
1590
|
+
# macOS with no coreutils), we do NOT run the base suite unbounded -- we skip
|
|
1591
|
+
# and return unknown (a safe no-op that keeps the existing affirmative-or-
|
|
1592
|
+
# council behavior). This trades a provenance check we cannot run safely for
|
|
1593
|
+
# guaranteed liveness; it can never cause false-green or false-block.
|
|
1594
|
+
local rc timeout_bin=""
|
|
1595
|
+
command -v gtimeout >/dev/null 2>&1 && timeout_bin="gtimeout"
|
|
1596
|
+
[ -z "$timeout_bin" ] && command -v timeout >/dev/null 2>&1 && timeout_bin="timeout"
|
|
1597
|
+
if [ -z "$timeout_bin" ]; then
|
|
1598
|
+
( cd "$target" && git worktree remove --force "$wt" 2>/dev/null ); rm -rf "$scratch" 2>/dev/null
|
|
1599
|
+
echo "unknown"; return 0
|
|
1600
|
+
fi
|
|
1601
|
+
( cd "$wt" && eval "$timeout_bin 300 $test_cmd" >/dev/null 2>&1 )
|
|
1602
|
+
rc=$?
|
|
1603
|
+
|
|
1604
|
+
( cd "$target" && git worktree remove --force "$wt" 2>/dev/null ); rm -rf "$scratch" 2>/dev/null
|
|
1605
|
+
|
|
1606
|
+
# Interpretation (deliberately conservative -- the runner runs the WHOLE test
|
|
1607
|
+
# command, not only the injected tests, so a base failure CANNOT be attributed
|
|
1608
|
+
# to the new test specifically: the base suite may fail for unrelated reasons,
|
|
1609
|
+
# or because the feature's non-test code is absent):
|
|
1610
|
+
# rc == 0 -> the FULL suite (incl. the injected new tests) passes on the
|
|
1611
|
+
# OLD code. The new tests therefore added NO failing signal on
|
|
1612
|
+
# base = tautological. This is the ONLY safe, actionable
|
|
1613
|
+
# conclusion, and it only ever DOWNGRADES affirmative.
|
|
1614
|
+
# rc != 0 -> UNKNOWN. We will not claim "confirmed provenance" from a
|
|
1615
|
+
# whole-suite failure we cannot attribute to the new test (that
|
|
1616
|
+
# would be a meaningless/false signal). Caller no-ops (keeps the
|
|
1617
|
+
# existing affirmative-or-council behavior). Timeout (124) is
|
|
1618
|
+
# likewise unknown.
|
|
1619
|
+
# This keeps the feature strictly a safe downgrade: it can turn an unearned
|
|
1620
|
+
# green into inconclusive, and can never fabricate a "provenance-confirmed"
|
|
1621
|
+
# affirmative.
|
|
1622
|
+
if [ "$rc" = "0" ]; then echo "tautological"; else echo "unknown"; fi
|
|
1623
|
+
return 0
|
|
1624
|
+
}
|
|
1625
|
+
|
|
1520
1626
|
#===============================================================================
|
|
1521
1627
|
# Council Evidence Hard Gate (v7.19.1) - "verified completion"
|
|
1522
1628
|
#===============================================================================
|
|
@@ -1687,6 +1793,56 @@ else:
|
|
|
1687
1793
|
fi
|
|
1688
1794
|
fi
|
|
1689
1795
|
|
|
1796
|
+
# --- v7.106.0: reverse-classical test provenance ---------------------------
|
|
1797
|
+
# A green test suite is only affirmative evidence if the run's NEW/CHANGED
|
|
1798
|
+
# tests actually FAIL on the pre-change base (they exercise the change). Tests
|
|
1799
|
+
# that pass on both base and HEAD are tautological and prove nothing. When the
|
|
1800
|
+
# tests are an affirmative candidate (they ran and passed, and are not already
|
|
1801
|
+
# inconclusive), verify provenance against the base in an ISOLATED worktree.
|
|
1802
|
+
# SAFETY: this can only DOWNGRADE affirmative -> inconclusive; "confirmed" and
|
|
1803
|
+
# "unknown" both leave the existing state untouched, and inconclusive is
|
|
1804
|
+
# pass-through (never a block). No-op on greenfield (unknown: no base/tests)
|
|
1805
|
+
# and when LOKI_TEST_PROVENANCE=0.
|
|
1806
|
+
if [ "$test_fails" != "true" ] && [ "$test_inconclusive" != "true" ] \
|
|
1807
|
+
&& [ "$test_pass" = "true" ] && [ "$test_runner" != "none" ] \
|
|
1808
|
+
&& [ -n "$base_sha" ]; then
|
|
1809
|
+
local _prov_cmd="${LOKI_TEST_COMMAND:-}"
|
|
1810
|
+
# Fall back to a sensible per-runner default when no explicit command was set.
|
|
1811
|
+
# For pytest, prefer python3 (many machines have only python3, not python);
|
|
1812
|
+
# a missing interpreter just makes the base run error -> unknown -> safe no-op.
|
|
1813
|
+
if [ -z "$_prov_cmd" ]; then
|
|
1814
|
+
case "$test_runner" in
|
|
1815
|
+
pytest)
|
|
1816
|
+
if command -v python3 >/dev/null 2>&1; then _prov_cmd="python3 -m pytest -q"
|
|
1817
|
+
else _prov_cmd="python -m pytest -q"; fi ;;
|
|
1818
|
+
node-test) _prov_cmd="node --test" ;;
|
|
1819
|
+
*) _prov_cmd="npm test" ;;
|
|
1820
|
+
esac
|
|
1821
|
+
fi
|
|
1822
|
+
local _prov
|
|
1823
|
+
_prov="$(_loki_test_provenance "$base_sha" "$_prov_cmd")"
|
|
1824
|
+
if [ "$_prov" = "tautological" ]; then
|
|
1825
|
+
# The new/changed tests pass on the OLD code too -> they prove nothing
|
|
1826
|
+
# about the change. DOWNGRADE the test signal from affirmative to
|
|
1827
|
+
# inconclusive (pass-through; the diff axis + council still decide, so
|
|
1828
|
+
# a legitimate refactor/greenfield-in-existing-repo with a non-empty
|
|
1829
|
+
# diff still completes -- inconclusive never blocks).
|
|
1830
|
+
test_inconclusive="true"
|
|
1831
|
+
test_inconclusive_reason="test_provenance_unconfirmed"
|
|
1832
|
+
log_info "[Evidence] New/changed tests pass on the run-start base too (tautological); test signal downgraded to inconclusive, not affirmative"
|
|
1833
|
+
mkdir -p "${TARGET_DIR:-.}/.loki/state" 2>/dev/null || true
|
|
1834
|
+
printf '{"status":"tautological","reason":"new/changed tests pass on the run-start base; not affirmative provenance","base_sha":"%s","runner":"%s"}' \
|
|
1835
|
+
"$base_sha" "$test_runner" > "${TARGET_DIR:-.}/.loki/state/test-provenance.json" 2>/dev/null || true
|
|
1836
|
+
if command -v emit_event_json >/dev/null 2>&1; then
|
|
1837
|
+
emit_event_json "test_provenance" "status" "tautological" "runner" "$test_runner" 2>/dev/null || true
|
|
1838
|
+
fi
|
|
1839
|
+
fi
|
|
1840
|
+
# _prov == "unknown" -> no-op: greenfield (no base/tests), or a base-suite
|
|
1841
|
+
# failure we cannot attribute to the new test. We keep the existing
|
|
1842
|
+
# affirmative-or-council behavior; we never fabricate a "confirmed" from an
|
|
1843
|
+
# unattributable base failure.
|
|
1844
|
+
fi
|
|
1845
|
+
|
|
1690
1846
|
# --- v7.28.0: inconclusive-baseline lifecycle -------------------------------
|
|
1691
1847
|
# When the gate cannot establish a diff baseline (no git repo, or no run-start
|
|
1692
1848
|
# SHA) it does NOT block (would break non-git projects), but completion is no
|
|
@@ -3226,10 +3382,36 @@ council_should_stop() {
|
|
|
3226
3382
|
circuit_triggered=true
|
|
3227
3383
|
fi
|
|
3228
3384
|
|
|
3229
|
-
# Only run council at check intervals OR if circuit breaker triggered
|
|
3385
|
+
# Only run council at check intervals OR if circuit breaker triggered OR the
|
|
3386
|
+
# agent has EXPLICITLY claimed completion this iteration (v7.105.0 convergence
|
|
3387
|
+
# fix). Rationale, measured from real build telemetry (docs/SPEED-DIAGNOSIS):
|
|
3388
|
+
# a build could be verifiably done at iteration 1 (reviews non-blocking, tests
|
|
3389
|
+
# green, checklist complete) yet the council was not even allowed to evaluate
|
|
3390
|
+
# until the next 5-iteration boundary -- so the loop ground out "next
|
|
3391
|
+
# improvement" iterations the user never asked for (14 iterations for a job
|
|
3392
|
+
# done in ~1). An explicit completion CLAIM is the strongest possible signal
|
|
3393
|
+
# that it is worth asking the council NOW. This changes only WHEN the council
|
|
3394
|
+
# evaluates, never WHETHER it must approve: MIN_ITERATIONS below, the hard
|
|
3395
|
+
# checklist gate, the evidence gate, the aggregate vote, and the devil's
|
|
3396
|
+
# advocate all still run unchanged. A premature or unearned claim is still
|
|
3397
|
+
# rejected (returns 1, loop continues); it is never a forced green.
|
|
3398
|
+
#
|
|
3399
|
+
# The claim signal is the structured one (loki_complete_task MCP tool ->
|
|
3400
|
+
# .loki/signals/COMPLETION_REQUESTED, or the runner exporting
|
|
3401
|
+
# LOKI_COMPLETION_CLAIMED=1 for this iteration), NOT the fuzzy log-grep the
|
|
3402
|
+
# circuit breaker uses -- so it fires on a real, intentional claim only.
|
|
3403
|
+
local completion_claimed=false
|
|
3404
|
+
if [ "${LOKI_COMPLETION_CLAIMED:-0}" = "1" ] \
|
|
3405
|
+
|| [ -f "${TARGET_DIR:-.}/.loki/signals/COMPLETION_REQUESTED" ]; then
|
|
3406
|
+
completion_claimed=true
|
|
3407
|
+
fi
|
|
3408
|
+
|
|
3230
3409
|
local should_check=false
|
|
3231
3410
|
if [ "$circuit_triggered" = "true" ]; then
|
|
3232
3411
|
should_check=true
|
|
3412
|
+
elif [ "$completion_claimed" = "true" ]; then
|
|
3413
|
+
should_check=true
|
|
3414
|
+
log_info "[Council] Completion claimed this iteration -- evaluating now (not deferring to the ${COUNCIL_CHECK_INTERVAL}-iteration interval)"
|
|
3233
3415
|
elif [ $((ITERATION_COUNT % COUNCIL_CHECK_INTERVAL)) -eq 0 ]; then
|
|
3234
3416
|
should_check=true
|
|
3235
3417
|
fi
|
package/autonomy/run.sh
CHANGED
|
@@ -17530,7 +17530,30 @@ else:
|
|
|
17530
17530
|
if type ensure_completion_test_evidence &>/dev/null; then
|
|
17531
17531
|
ensure_completion_test_evidence || true
|
|
17532
17532
|
fi
|
|
17533
|
-
if
|
|
17533
|
+
# v7.105.0 convergence: if the agent EXPLICITLY claimed completion this
|
|
17534
|
+
# iteration (structured loki_complete_task / COMPLETION_REQUESTED
|
|
17535
|
+
# signal), tell the council to evaluate NOW instead of deferring to the
|
|
17536
|
+
# 5-iteration check interval. The council still must approve (MIN_
|
|
17537
|
+
# ITERATIONS + hard checklist + evidence gate + vote + devil's advocate
|
|
17538
|
+
# all unchanged); this only stops an already-verified build from
|
|
17539
|
+
# grinding needless "next improvement" iterations. Scoped to this call.
|
|
17540
|
+
#
|
|
17541
|
+
# CRITICAL (council-review finding): we must PEEK at the claim signal
|
|
17542
|
+
# NON-DESTRUCTIVELY. check_task_completion_signal() CONSUMES the signal
|
|
17543
|
+
# (rm -f on read), and the DEFAULT completion-promise route below
|
|
17544
|
+
# (v6.82.0, the live gate-based acceptance path) also consumes it -- so
|
|
17545
|
+
# calling the consuming detector here would drop the claim before that
|
|
17546
|
+
# route sees it, re-introducing the v7.28 "claim drop" bug. Instead we
|
|
17547
|
+
# only test the signal FILES' existence (the same two paths
|
|
17548
|
+
# check_task_completion_signal reads), leaving consumption to the
|
|
17549
|
+
# existing single owner. The council side likewise peeks
|
|
17550
|
+
# ($TARGET_DIR/.loki/signals/COMPLETION_REQUESTED) without consuming.
|
|
17551
|
+
local _loki_completion_claimed=0
|
|
17552
|
+
if [ -f "${TARGET_DIR:-.}/.loki/signals/TASK_COMPLETION_CLAIMED" ] \
|
|
17553
|
+
|| [ -f "${TARGET_DIR:-.}/.loki/signals/COMPLETION_REQUESTED" ]; then
|
|
17554
|
+
_loki_completion_claimed=1
|
|
17555
|
+
fi
|
|
17556
|
+
if type council_should_stop &>/dev/null && LOKI_COMPLETION_CLAIMED="$_loki_completion_claimed" council_should_stop; then
|
|
17534
17557
|
# bash-F1: council_should_stop returns 0 from a genuine approval
|
|
17535
17558
|
# AND from two force-stop safety valves (stagnation flood /
|
|
17536
17559
|
# repeated done-signals). A force-stop is NOT a verified-complete
|
package/dashboard/__init__.py
CHANGED
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
# Founder Status + Decision List (2026-07-01)
|
|
2
|
+
|
|
3
|
+
The consolidated "where everything stands + the decisions only you can make" report.
|
|
4
|
+
Written after: 5 patch releases shipped, top-100 backlog enumerated + triaged, ecosystem
|
|
5
|
+
currency assessed, no-HITL product principle recorded.
|
|
6
|
+
|
|
7
|
+
## 1. Shipped this session (all live on npm/Docker/GH, council-gated)
|
|
8
|
+
|
|
9
|
+
| Version | What | Verified |
|
|
10
|
+
|---|---|---|
|
|
11
|
+
| v7.104.2 | Native/Keychain Claude login recognized; `loki start` no longer false-blocks a logged-in user; `loki doctor` confirms login (bash+Bun parity) | reproduced + 8-case test + live Docker |
|
|
12
|
+
| v7.104.3 | Task-list accuracy: honest done column, no empty cards / dups / fake-green (failed iters honestly labeled) | live vs real anonima, browser screenshots, council 3/3 |
|
|
13
|
+
| v7.104.4 | `/api/tasks` never 500s on malformed dashboard-state.json | RED/GREEN-proven test |
|
|
14
|
+
| v7.104.5 | Memory panel correct + self-consistent on JSON-backed projects (completes PR #178, @iizotov co-authored, PR closed with thank-you) | live vs anonima, parity tests |
|
|
15
|
+
| (earlier) v7.104.2 auth + v7.104.3 task-list | (same as above; two-patch train) | |
|
|
16
|
+
|
|
17
|
+
Also: closed issue #177 (spam advisory). PR #178 merged-in-substance + contributor thanked.
|
|
18
|
+
|
|
19
|
+
## 2. Backlog reality (honest)
|
|
20
|
+
|
|
21
|
+
The top-100 was enumerated (116 raw -> 100 ranked) and then TRIAGED against current source.
|
|
22
|
+
Key finding: **the autonomous critical/high queue is essentially ALREADY DONE.**
|
|
23
|
+
|
|
24
|
+
| Rank | Item | Triage verdict |
|
|
25
|
+
|---|---|---|
|
|
26
|
+
| 1 | run.sh self-delete EXIT trap | already fixed (guarded to /tmp/loki-run-*; verified safe) |
|
|
27
|
+
| 2 | saas cross-tenant leak + stuck-Building | CLOSED (commits a7667d0/44f7c29/8b05b36; 187/187 tests) |
|
|
28
|
+
| 9 | anti-fake-green + gate parity | shipped (verify 282 tests; coherentStatus guard) |
|
|
29
|
+
| 10 | VS-7 worker cutover | CLOSED (commit e6b96ed; BFF 181 + worker 102 tests) |
|
|
30
|
+
| 17 | base_unreachable suppresses empty_diff | already wired (completion-council.sh:1616) |
|
|
31
|
+
| 22 | 3 parallel reviewers | already implemented (run.sh:10461-10954) |
|
|
32
|
+
|
|
33
|
+
The autonomous well is mostly dry BECAUSE the real remaining leverage is founder-gated
|
|
34
|
+
(the trust-as-product / enterprise surface). That is the honest headline: the next 10x is
|
|
35
|
+
not another patch, it is the decisions below.
|
|
36
|
+
|
|
37
|
+
## 3. Ecosystem currency (what loki CONSUMES from the Claude ecosystem)
|
|
38
|
+
|
|
39
|
+
CLI invocation contract (claude 2.1.198): STABLE, no drift. Auth: fixed this session.
|
|
40
|
+
Model catalog: current (opus-4-8, sonnet-5, haiku-4-5, fable-5). Real deltas are small and
|
|
41
|
+
tracked in docs/ECOSYSTEM-CURRENCY-PLAN.json (EC-1..EC-4). Adjacent Anthropic PRODUCTS
|
|
42
|
+
(Managed Agents, Claude Tag, loop-engineering, agent identity) assessed as peer/inspiration
|
|
43
|
+
categories, NOT dependencies -- declined with reason (the CTO triage move).
|
|
44
|
+
|
|
45
|
+
The one real net-new build from this: **EC-1, the external-tool contract adapter + drift
|
|
46
|
+
test** -- the keychain incident this session proves it would catch a user-facing break
|
|
47
|
+
before a user does. Building autonomously now.
|
|
48
|
+
|
|
49
|
+
## 4. DECISIONS ONLY YOU CAN MAKE (founder-gated, prepped to one-approval-away)
|
|
50
|
+
|
|
51
|
+
These are the real blockers to "easiest to integrate with anyone" + the trust-as-product moat.
|
|
52
|
+
Per the no-HITL principle, these are the LEGITIMATE exceptions: irreversible / strategic /
|
|
53
|
+
brand / legal / spend decisions a human must own. Each is ready for a one-word go.
|
|
54
|
+
|
|
55
|
+
### A. Autonomi Verify productization (the trust-as-product core)
|
|
56
|
+
1. **License decision (rank 5, S, BLOCKS everything commercial).** BUSL-1.1 vs open for
|
|
57
|
+
@autonomi/verify. No hosted/commercial launch until decided. DECISION: which license?
|
|
58
|
+
2. **npm registry + package name (rank 4, M).** @autonomi/verify is currently unpublishable
|
|
59
|
+
(private:true, no exports/types/build, .ts bin). And a naming collision exists
|
|
60
|
+
(loki-vaas vs loki-verify bin vs `loki verify` subcommand, rank 13). DECISION: publish to
|
|
61
|
+
public npm as `@autonomi/verify`? Confirm the brand/bin name to resolve the collision.
|
|
62
|
+
-> Once decided, I can make it genuinely npm-installable + fix the collision autonomously.
|
|
63
|
+
3. **Hosted /verify deploy (rank 6, M + rank 36/42 sandbox).** The signed neutral REST
|
|
64
|
+
endpoint is the category-defining asset but not deployed (Bun.serve-only, ephemeral key).
|
|
65
|
+
DECISION: deploy target (Fly/Render/AWS/your infra) + signing-key storage (secret manager)?
|
|
66
|
+
-> I can stage the Node entry + Dockerfile + serve script + 12-factor key guard to
|
|
67
|
+
one-command-deploy; I will NOT deploy or spend without your go.
|
|
68
|
+
4. **wire verifyCompletion() pipeline (rank 3, XL).** The SDK/MCP verdict path is stubbed;
|
|
69
|
+
the whole value prop is unreachable until gate->integrity->council->cost->sign is wired.
|
|
70
|
+
Depends on (2)/(3) decisions. Large; I can start once the license + registry are set.
|
|
71
|
+
|
|
72
|
+
### B. Enterprise scale (needed for any enterprise sale)
|
|
73
|
+
5. **RBAC / SSO / API-key auth / metering (rank 15, XL)** + **on-prem persistence backend
|
|
74
|
+
(rank 16, L, Postgres/SQLite)** + **sandbox backend (rank 14, L, gVisor/Firecracker/Kata)**.
|
|
75
|
+
DECISION: which enterprise deploy shape do we target first (hosted multi-tenant vs on-prem)?
|
|
76
|
+
That choice sequences 14/15/16/36/42. -> I can build the PersistenceBackend interface +
|
|
77
|
+
RBAC scaffolding once you pick the shape.
|
|
78
|
+
6. **GitHub App verification integration (rank 46, L, greenfield).** DECISION: build now or later?
|
|
79
|
+
|
|
80
|
+
### C. Benchmark spend
|
|
81
|
+
7. **SWE-bench Pro 119 resume (issue #174, paused 35/119).** Resume harness exists. DECISION:
|
|
82
|
+
spend the compute to finish? (This is a $ + time decision, hence gated.)
|
|
83
|
+
|
|
84
|
+
### D. Outreach (rank/issue #172, low-risk but external-posting)
|
|
85
|
+
8. MCP registry submission + Glama claim + awesome-list PRs. I can prepare the submissions;
|
|
86
|
+
the actual external POSTS need your nod (per action rules). DECISION: proceed with posts?
|
|
87
|
+
|
|
88
|
+
## 5. What I am doing autonomously RIGHT NOW (no decision needed)
|
|
89
|
+
- EC-1 external-contract adapter + drift test (the one real net-new build).
|
|
90
|
+
- Small no-HITL audit: find every place the PRODUCT hard-stops/prompts a human; fix genuine
|
|
91
|
+
gaps so it decides from model+memory+cross-project context (degrading to honest-inconclusive,
|
|
92
|
+
never fake-green). Not a speculative refactor -- a triage + targeted fixes.
|
|
93
|
+
- NOT building: dashboard steering (#44) -- it is a human-in-the-loop feature, deprioritized by
|
|
94
|
+
the no-HITL mandate; architect plan is durable and waits. NOT building "autonomous-everywhere"
|
|
95
|
+
as a speculative refactor.
|
|
96
|
+
|
|
97
|
+
## 6. The one-line asks
|
|
98
|
+
Reply with any subset:
|
|
99
|
+
- **License:** BUSL / MIT / Apache / other for @autonomi/verify
|
|
100
|
+
- **npm:** publish `@autonomi/verify` publicly? + confirm bin name (resolve loki-verify collision)
|
|
101
|
+
- **Hosted deploy:** target + key storage (or "not yet")
|
|
102
|
+
- **Enterprise shape:** hosted-multitenant-first vs on-prem-first (sequences the enterprise stories)
|
|
103
|
+
- **SWE-bench:** spend to finish 119? (yes/no)
|
|
104
|
+
- **Outreach:** post the MCP-registry/Glama/awesome-list submissions? (yes/no)
|
|
105
|
+
|
|
106
|
+
Everything else, I continue autonomously.
|
package/docs/INSTALLATION.md
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
The flagship product of [Autonomi](https://www.autonomi.dev/). Loki Mode is a spec-driven autonomous builder with a built-in trust layer that takes any spec to a deployed product and verifies completion with evidence (quality gates plus a completion council), not just a "done" claim. Complete installation instructions for all platforms and use cases.
|
|
4
4
|
|
|
5
|
-
**Version:** v7.
|
|
5
|
+
**Version:** v7.106.0
|
|
6
6
|
|
|
7
7
|
---
|
|
8
8
|
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
# Loki Build Speed Diagnosis (2026-07-01)
|
|
2
|
+
|
|
3
|
+
Measured from real build telemetry (anonima `.loki/events.jsonl`, stage_complete +
|
|
4
|
+
iteration timeline), NOT from reading run.sh. Method per advisor: measure before optimizing.
|
|
5
|
+
|
|
6
|
+
## Data hygiene
|
|
7
|
+
anonima's events.jsonl contained **9 interleaved builds/sessions** (9 session_starts) plus
|
|
8
|
+
session-long manual poking. Naive sums ("5 min/iter x 14") are contaminated. Split on
|
|
9
|
+
session_start and analyzed the real ones.
|
|
10
|
+
|
|
11
|
+
## The pathological build (session 3): the smoking gun
|
|
12
|
+
A simple topic-based chat app. 14 ACT iterations, ~97 min of real work (idle gaps excluded).
|
|
13
|
+
|
|
14
|
+
| iteration | real work |
|
|
15
|
+
|---|---|
|
|
16
|
+
| 1 | 2.1 min |
|
|
17
|
+
| 6 | 9.9 min |
|
|
18
|
+
| 10 | 15.3 min |
|
|
19
|
+
| 12 | **29.6 min** |
|
|
20
|
+
| (others) | 3-6 min each |
|
|
21
|
+
|
|
22
|
+
**13 completion claims across 14 iterations.** The FIRST claim already stated:
|
|
23
|
+
"All PRD requirements implemented and tests passing. Evidence: npm test => 19/19 pass...
|
|
24
|
+
PRD checklist (14 items) fully implemented." Yet the loop ran 13 more iterations.
|
|
25
|
+
|
|
26
|
+
## Root cause: NON-CONVERGENCE, not per-stage slowness
|
|
27
|
+
- **Code review / 3-reviewer council is NOT the bottleneck** (0.1-2.8 min per iteration,
|
|
28
|
+
mostly <1 min). Cutting the council would harm accuracy for ~no speed gain. Ruled out.
|
|
29
|
+
- **The bottleneck is iteration COUNT + a few pathological iterations.** The app was
|
|
30
|
+
essentially done early (real passing tests, checklist complete) but the RARV loop's
|
|
31
|
+
"there is NEVER a finished state -- always find the next improvement" behavior kept it
|
|
32
|
+
grinding polish iterations. For a user who wants a fullstack app fast, this is THE
|
|
33
|
+
frustration: loki keeps going after done.
|
|
34
|
+
- iter 10 (15 min) and iter 12 (30 min) did disproportionate work -- candidate over-scoping
|
|
35
|
+
or re-analysis; needs per-iteration content inspection.
|
|
36
|
+
|
|
37
|
+
## Buckets (advisor framework)
|
|
38
|
+
1. **WASTE (cut freely):** polish iterations after genuine completion; the PRD-reuse
|
|
39
|
+
spurious-update re-reconciliation (already diagnosed, design fix pending). These add
|
|
40
|
+
wall-clock with no trust value.
|
|
41
|
+
2. **VERIFICATION COST (keep -- this is the product):** council, gates, completion vote.
|
|
42
|
+
Fast already. The moat. Do NOT cut for speed.
|
|
43
|
+
3. **STALE SCAFFOLDING (test per-stage):** machinery built for weaker models may be dead
|
|
44
|
+
weight on Opus 4.8 / Sonnet 5 (cf. Anthropic's "context resets became dead weight on
|
|
45
|
+
Opus 4.5"). Hypothesis -- must be tested against the benchmark, not assumed.
|
|
46
|
+
|
|
47
|
+
## The fix direction (speed AND accuracy, never speed-by-cutting-verification)
|
|
48
|
+
**Convergence:** stop as soon as the completion council genuinely agrees it is done
|
|
49
|
+
(honest verified completion), instead of running "next improvement" iterations. This is a
|
|
50
|
+
SPEED win that INCREASES trust alignment (stop when verified-done = the moat working), not a
|
|
51
|
+
verification cut. The user's mandate is "accuracy AND speed"; convergence delivers both.
|
|
52
|
+
|
|
53
|
+
## HARD PREREQUISITE (before any council/RARV-C change): the benchmark
|
|
54
|
+
Both the speed directive and the "update council/RARV-C to latest research, show before/after
|
|
55
|
+
accuracy" directive require a **clean reproducible benchmark on a fixed spec** emitting
|
|
56
|
+
wall-clock + iteration count + completion verdict + pass/fail vs known acceptance criteria.
|
|
57
|
+
Without it, changing the verification core is unmeasurable churn on the moat (architecture-
|
|
58
|
+
level fake-green). Build the benchmark FIRST, capture before-numbers, fix the clearest waste
|
|
59
|
+
(convergence), re-run, show real before->after. Then gate any council/RARV-C change on that
|
|
60
|
+
benchmark, one named change at a time, kept only if accuracy goes up and wall-clock does not
|
|
61
|
+
regress.
|
|
62
|
+
|
|
63
|
+
## MEASURED before/after (2026-07-01, same greet-CLI spec, isolated, council+gates ON)
|
|
64
|
+
|
|
65
|
+
| metric | before v7.104.1 (no fix) | after v7.105.0 (convergence fix) |
|
|
66
|
+
|---|---|---|
|
|
67
|
+
| wall clock | 28.6 min | 7.4 min |
|
|
68
|
+
| ACT iterations | 13 | 1 |
|
|
69
|
+
| completion claims | 11 | 1 |
|
|
70
|
+
| engine_completed | true | true |
|
|
71
|
+
| greet.js built correctly | yes | yes |
|
|
72
|
+
|
|
73
|
+
First after-run: 3.9x faster (-74% wall clock), 13->1 iterations, SAME correct output (accuracy
|
|
74
|
+
preserved -- not speed-by-cutting-verification; the council still approved).
|
|
75
|
+
|
|
76
|
+
HONEST CAVEATS (do not overclaim): (1) n=1 per side; LLM builds are stochastic, so a single
|
|
77
|
+
1-iteration after-run could be a low draw. Running 2-3 more after-runs for a real range before
|
|
78
|
+
presenting a headline multiplier. (2) before=v7.104.1 vs after=v7.105.0 differ by 7 commits;
|
|
79
|
+
convergence is the only one plausibly affecting iteration count, so it is the likely driver, but
|
|
80
|
+
the clean claim is "v7.105.0 vs v7.104.1 on this spec", not "convergence alone". (3) this is one
|
|
81
|
+
trivial spec; larger specs may show a smaller relative gain.
|
|
82
|
+
|
|
83
|
+
## UPDATE: after-run consistency (2 of 3 repeats in; n>1)
|
|
84
|
+
- after #1: 7.4 min, 1 iteration, correct
|
|
85
|
+
- after #2: 7.0 min, 1 iteration, correct
|
|
86
|
+
Both after-runs converged at 1 ACT iteration / ~7 min / correct output -- consistent, NOT a
|
|
87
|
+
single lucky draw. The convergence fix reliably stops at verified-done. Avg ~7.2 min vs 28.6 min
|
|
88
|
+
before = ~4x faster, 13x fewer iterations, accuracy preserved (council still approves). The
|
|
89
|
+
residual ~7 min is the single build+verify iteration's real inference cost (the honest floor for
|
|
90
|
+
this spec WITH verified-completion). Next speed lever: per-iteration cost on larger specs (the
|
|
91
|
+
fat-iteration investigation), measured via this same benchmark.
|
|
92
|
+
|
|
93
|
+
## CONFIRMED (n=3 after-runs): the convergence fix delivers 4.0x, reliably
|
|
94
|
+
- before (v7.104.1): 28.6 min, 13 iterations
|
|
95
|
+
- after (v7.105.0): 7.4 / 7.0 / 7.1 min -> avg 7.2 min, EVERY run exactly 1 iteration, all correct
|
|
96
|
+
- Result: 4.0x faster wall-clock, 13 -> 1 iterations, 100% correct across all 3 runs.
|
|
97
|
+
The tight range (7.0-7.4 min, always 1 iteration) confirms this is the reliable behavior, not a
|
|
98
|
+
single lucky draw. Attribution: v7.105.0 vs v7.104.1 on the greet-CLI spec; the convergence fix
|
|
99
|
+
(completion-claim triggers immediate council evaluation) is the sole plausible driver of the
|
|
100
|
+
13->1 iteration collapse. Residual ~7.2 min = one build+verify iteration's real inference cost
|
|
101
|
+
(the honest floor WITH verified-completion). NEXT speed lever: per-iteration cost on larger specs.
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
# Loki + Autonomi: Top-100 Backlog (Master Plan)
|
|
2
|
+
|
|
3
|
+
Generated 2026-07-01 by the top100-backlog-enumeration ultracode workflow (7 agents, 116 raw items -> 100 ranked). North star: easiest to integrate with anyone; ranked by leverage toward zero-friction adoption, modularization, enterprise-grade, and scale.
|
|
4
|
+
|
|
5
|
+
## Themes
|
|
6
|
+
- Trust and correctness bugs (data-loss, cross-tenant, fake-green safety)
|
|
7
|
+
- Autonomi Verify productization (stub -> shipped: gate/council/cost/sign wiring)
|
|
8
|
+
- Zero-friction adoption (reduce user setup and error surface)
|
|
9
|
+
- Modularization (extract libraries, break monoliths, fix import boundaries)
|
|
10
|
+
- Enterprise-scale and multi-tenant (RBAC/SSO/persistence/sandbox, mostly founder-gated)
|
|
11
|
+
- autonomi-saas evidence and proof pipeline (receipt phases, correlation, worker cutover)
|
|
12
|
+
- CI, test, and distribution hardening (coverage, unreachable tests, publish gates)
|
|
13
|
+
- Runtime-agnostic and Bun migration (bash sunset, write-path port)
|
|
14
|
+
|
|
15
|
+
## Legend
|
|
16
|
+
- **gate**: autonomous (I execute) | founder-gated (prep to one-approval-away) | community
|
|
17
|
+
- **effort**: S/M/L/XL | **value**: critical/high/medium/low
|
|
18
|
+
|
|
19
|
+
## Full ranked list
|
|
20
|
+
|
|
21
|
+
| # | value | eff | gate | area | item |
|
|
22
|
+
|---|---|---|---|---|---|
|
|
23
|
+
| 1 | critical | S | autonomous | engine | run.sh self-delete EXIT trap nukes canonical source on LOKI_RUNNING_FROM_TEMP |
|
|
24
|
+
| 2 | critical | M | autonomous | saas | autonomi-saas proof correlation: cross-tenant leak + stuck-Building fix (per-build run_id) |
|
|
25
|
+
| 3 | critical | XL | founder-gated | verify | Verify: verifyCompletion() stub throws not-implemented (wire gate->integrity->council->cost->sign pipeline) |
|
|
26
|
+
| 4 | critical | M | founder-gated | verify | Verify: npm @autonomi/verify unpublishable (private:true, no exports/types/build, .ts bin, undeclared zod) |
|
|
27
|
+
| 5 | critical | S | founder-gated | saas | Verify: BUSL-1.1 vs open licensing undecided (blocks hosted/commercial launch) |
|
|
28
|
+
| 6 | critical | M | founder-gated | saas | Verify: hosted /verify REST API not deployed (Node entry + Dockerfile + serve script + persistent signing key) |
|
|
29
|
+
| 7 | critical | M | autonomous | modularization | Verify: extract verification-core library (port council_evidence_gate to TS) |
|
|
30
|
+
| 8 | critical | S | autonomous | verify | Verify: 5 deterministic gates parity with completion-council.sh (staged, verify + ship) |
|
|
31
|
+
| 9 | critical | S | autonomous | saas | autonomi-saas anti-fake-green safety: verdict strictly from engine honesty.headline (verify + guard) |
|
|
32
|
+
| 10 | critical | M | autonomous | saas | autonomi-saas VS-7 worker cutover: BFF enqueues, worker sole executor (verify live e2e) |
|
|
33
|
+
| 11 | high | M | founder-gated | verify | Signed neutral records on SDK/MCP path (verifyCompletion returns UNSIGNED_SENTINEL; only REST signs) |
|
|
34
|
+
| 12 | medium | S | autonomous | verify | Verify: fix embed import boundary (review/* drags LLM council into offline core; refresh regresses) |
|
|
35
|
+
| 13 | high | S | founder-gated | verify | Verify: resolve naming collision (loki-vaas vs loki-verify bin vs loki verify subcommand) |
|
|
36
|
+
| 14 | critical | L | founder-gated | engine | Sandbox backend implementation (createSandbox throws; choose gVisor/Firecracker/Kata) |
|
|
37
|
+
| 15 | critical | XL | founder-gated | enterprise | Enterprise RBAC / SSO / API-key auth / metering (hosted + on-prem control plane) |
|
|
38
|
+
| 16 | high | L | founder-gated | enterprise | On-prem persistence backend interface + first prod store (Postgres/SQLite) |
|
|
39
|
+
| 17 | high | S | autonomous | engine | base_unreachable signal in ObservedDiff (suppress empty_diff false-block once sandbox lands) |
|
|
40
|
+
| 18 | high | M | founder-gated | engine | Real Anthropic LLM provider wiring for production verify (council against live model) |
|
|
41
|
+
| 19 | high | M | founder-gated | verify | Verify: LLM council wired into default verdict path (module exists, verifyCompletion stub does not invoke) |
|
|
42
|
+
| 20 | high | S | autonomous | engine | Signed verification records + ed25519 audit trail (staged, verify + ship) |
|
|
43
|
+
| 21 | high | M | autonomous | verify | Golden-output conformance corpus for byte-identical CLI-invariance across implementations |
|
|
44
|
+
| 22 | high | M | autonomous | engine | Code review gate: run 3 parallel reviewers (currently single-reviewer path) |
|
|
45
|
+
| 23 | high | S | founder-gated | saas | HTTP /verify endpoint production wiring (handler complete, README lists as planned) |
|
|
46
|
+
| 24 | high | S | autonomous | saas | MCP verify tools (verify_completion, run_evidence_gate) wiring + distribution (staged) |
|
|
47
|
+
| 25 | high | M | community | test | Memory failure-loop (FAILURE_MEMORY) persistence bug (strict-xfail; recency read incomplete) |
|
|
48
|
+
| 26 | high | XL | autonomous | engine | Bun runtime Phase 2: migrate write-path commands (feat/bun-migration) |
|
|
49
|
+
| 27 | high | M | autonomous | cli | Bash runtime sunset (loki run-shell removal, Phase 6, after clean soak) |
|
|
50
|
+
| 28 | high | M | community | saas | autonomi-saas Evidence Receipt Phase 2: screenshot capture + BFF stream route (in progress) |
|
|
51
|
+
| 29 | high | M | community | saas | autonomi-saas Evidence Receipt Phase 3: test results from engine workspace |
|
|
52
|
+
| 30 | high | L | autonomous | saas | autonomi-saas Cost tab + budget-breaker UX (web + enforcement audit) |
|
|
53
|
+
| 31 | high | S | community | saas | autonomi-saas BFF overlays stored saas_evidence onto engine proof (verify) |
|
|
54
|
+
| 32 | medium | M | founder-gated | saas | E4 no-server run output capture for CLI/library builds (bounded inline + artifact) |
|
|
55
|
+
| 33 | high | M | autonomous | dashboard | Prompt Optimizer: wire real LLM-as-judge (currently heuristic-only) |
|
|
56
|
+
| 34 | high | M | community | engine | Cost-honesty check implementation (normalizeModel + quote==dispatched) |
|
|
57
|
+
| 35 | high | S | community | saas | POST /cost-honesty API handler (blocked on cost core) |
|
|
58
|
+
| 36 | critical | L | founder-gated | saas | Hosted metered runner with per-verification container sandbox |
|
|
59
|
+
| 37 | high | M | community | ci | CI parity-matrix flake root-cause (state cooldown after test-cli-commands.sh) |
|
|
60
|
+
| 38 | high | M | community | ci | Post-release smoke sign-off gate before npm/Docker go public |
|
|
61
|
+
| 39 | high | M | community | test | Document the 3 pre-existing failing shell tests (no public known-failures list) |
|
|
62
|
+
| 40 | medium | S | autonomous | dashboard | PR #178: dashboard memory summary reports 0 on JSON-backed projects |
|
|
63
|
+
| 41 | high | M | community | test | ARM64 runtime verification (emulation-only in CI; needs real Apple Silicon run) |
|
|
64
|
+
| 42 | high | XL | founder-gated | saas | Multi-tenant sandbox backend (gVisor/Firecracker/Kata) selection spike |
|
|
65
|
+
| 43 | critical | L | community | test | Real provider end-to-end tests (cost + secrets + duration make CI-unreachable) |
|
|
66
|
+
| 44 | high | XL | founder-gated | enterprise | Enterprise deploy: object-store sync + queue fleet modes + #691 --config loader |
|
|
67
|
+
| 45 | high | M | founder-gated | enterprise | Compliance scanner (loki-verify) converged as module in trust-layer SKU |
|
|
68
|
+
| 46 | high | L | founder-gated | saas | GitHub App verification integration (webhook wrapper, greenfield) |
|
|
69
|
+
| 47 | high | L | community | test | Long-duration stress tests (leak/queue-churn) unreachable in CI |
|
|
70
|
+
| 48 | medium | L | community | saas | autonomi-saas Evidence Receipt Phase 4: clip + full test-log streaming + build_artifacts table |
|
|
71
|
+
| 49 | high | XL | community | test | SWE-bench Pro 119 resume (paused 35/119) |
|
|
72
|
+
| 50 | medium | M | autonomous | engine | Multi-reviewer completion council option in hosted verify (council:true dropped today) |
|
|
73
|
+
| 51 | medium | M | autonomous | engine | Integrity detectors (mock-only, tautological, test-weakening) wired into evidence block |
|
|
74
|
+
| 52 | medium | S | community | test | Bun test coverage threshold enforcement on branches (70% baseline exists) |
|
|
75
|
+
| 53 | medium | L | autonomous | dashboard | Dashboard mid-run steering / builder control (mid-flight model switch exists, broader control deferred) |
|
|
76
|
+
| 54 | medium | S | community | test | Dashboard static-asset freshness gate in local-ci (like loki-ts/dist) |
|
|
77
|
+
| 55 | medium | L | community | test | Windows/WSL runtime verification (no Windows CI runner) |
|
|
78
|
+
| 56 | low | M | autonomous | cli | KPI reporting (loki report kpis) on bash legacy route (Bun-only today) |
|
|
79
|
+
| 57 | medium | M | autonomous | cli | Voice mode (Issue #85) |
|
|
80
|
+
| 58 | medium | M | community | test | Mutation testing (stryker) workflow re-enable + fix stale 5-provider assertions |
|
|
81
|
+
| 59 | medium | XL | founder-gated | engine | Managed Agents multiagent path (LOKI_EXPERIMENTAL_MANAGED_*, research preview) |
|
|
82
|
+
| 60 | low | M | autonomous | engine | Sentrux iteration-loop auto-gating (manual-only today) |
|
|
83
|
+
| 61 | medium | S | founder-gated | adoption | npm registry publish prep for @autonomi/verify SDK (distribution wiring) |
|
|
84
|
+
| 62 | low | M | founder-gated | adoption | Disclosed default-on PostHog telemetry (Phase L, blocked + deferred) |
|
|
85
|
+
| 63 | medium | S | community | saas | MCP check_cost_honesty tool registration (deferred until cost core lands) |
|
|
86
|
+
| 64 | medium | L | founder-gated | dashboard | Hosted dashboard with shareable verification badges |
|
|
87
|
+
| 65 | medium | M | founder-gated | adoption | MCP registry submission + Glama claim + awesome-list PRs |
|
|
88
|
+
| 66 | medium | M | autonomous | saas | autonomi-saas UI/UX pass: build-watching pane + finished cues from trust verdict |
|
|
89
|
+
| 67 | medium | XL | autonomous | dashboard | Server.py route modularization (~10K-line monolith, no domain separation) |
|
|
90
|
+
| 68 | medium | L | autonomous | dashboard | Large dashboard-ui components modularization (5 files 1K-1.9K lines) |
|
|
91
|
+
| 69 | medium | L | founder-gated | engine | Wave-13 deferred trust fixes + PRD-reuse spurious-update design fix |
|
|
92
|
+
| 70 | medium | M | community | dashboard | Dashboard dark-toggle iframe theme not following SPA (harness/product parity) |
|
|
93
|
+
| 71 | medium | M | community | ci | Sonnet-5 calibration follow-ups: test-verify --hosted + gate missing-tool handling |
|
|
94
|
+
| 72 | medium | S | community | ci | Integrity audit (SBOM/provenance) blocking on publish (currently informational) |
|
|
95
|
+
| 73 | medium | S | community | distribution | Python SDK version pre-check before publish (mismatch risk if publish fails post-tag) |
|
|
96
|
+
| 74 | medium | S | community | ci | Docker buildx early-detection for uncommitted loki-ts/dist (silent COPY failure risk) |
|
|
97
|
+
| 75 | medium | S | community | ci | loki-enterprise workflow enabled in main CI (dispatch-only today) |
|
|
98
|
+
| 76 | medium | S | community | distribution | Brew tap real-install verification on clean macOS host post-release |
|
|
99
|
+
| 77 | low | L | founder-gated | perf | Horizontal scaling / distributed rate-limit + persistence (in-memory today) |
|
|
100
|
+
| 78 | low | S | community | saas | Optional build_artifacts table for enumeration/lifecycle/reaping (deferred) |
|
|
101
|
+
| 79 | medium | S | community | saas | Headless browser (playwright-core) dep + Docker image size cap for screenshot capture |
|
|
102
|
+
| 80 | low | M | community | engine | Wave-8/9 deferred tail: spawn_timeout removal, heredoc guards, MED/LOW findings |
|
|
103
|
+
| 81 | medium | L | community | adoption | v7.19.1 traction: uncertainty-gated escalation + shareable proof + integrate 8 tutorials |
|
|
104
|
+
| 82 | medium | M | autonomous | verify | Benchmarks HumanEval/SWE-bench: publish results (runners + datasets exist) |
|
|
105
|
+
| 83 | low | S | autonomous | cli | loki run removal (deprecated alias for loki start, next major) |
|
|
106
|
+
| 84 | low | S | autonomous | cli | Fable model tier wiring (collapses to Opus until API-available) |
|
|
107
|
+
| 85 | low | S | community | distribution | Homepage/blog release update automation (Slack-notify-only today) |
|
|
108
|
+
| 86 | low | S | community | distribution | Wiki-sync scheduled trigger (manual/dispatch only, skips without API key) |
|
|
109
|
+
| 87 | low | L | autonomous | dashboard | Dashboard React app: implement four stub views (experimental app) |
|
|
110
|
+
| 88 | low | S | community | test | Embeddings edge-case tests run without numpy (skip today) |
|
|
111
|
+
| 89 | low | S | community | test | Hybrid-search test runs without chromadb skip |
|
|
112
|
+
| 90 | low | S | community | test | PID-recycling test platform-portability (ps lstart skip for pid 1) |
|
|
113
|
+
| 91 | low | S | community | test | Cloud integration tests (aider-cloud) run in a credentialed gated job |
|
|
114
|
+
| 92 | low | S | community | test | Sentrux real-binary nightly promoted from best-effort to tracked |
|
|
115
|
+
| 93 | low | S | community | test | Model-catalog probe: surface staleness in tests (issue-create skipped without admin scope) |
|
|
116
|
+
| 94 | low | S | community | test | Phase 6 readiness check re-enable (cron disabled; issues disabled on repo) |
|
|
117
|
+
| 95 | low | S | community | distribution | TS/Bun SDK (@loki-mode/sdk) discoverability from main package/docs |
|
|
118
|
+
| 96 | low | S | community | distribution | Docker Hub description update resilience (PAT scope failures non-blocking) |
|
|
119
|
+
| 97 | low | S | community | distribution | VS Code extension: keep source marked deprecated, not silently removed |
|
|
120
|
+
| 98 | low | S | autonomous | dashboard | Web-app docs: fill video placeholder (coming-soon text) |
|
|
121
|
+
| 99 | low | M | autonomous | cli | C# provider support (roslyn analyzers + dotnet build, deferred) |
|
|
122
|
+
| 100 | low | XL | founder-gated | engine | BMAD-METHOD v6 integration (net-new, needs founder planning) |
|
|
@@ -0,0 +1,232 @@
|
|
|
1
|
+
# Loki + Autonomi: Competitive Gap-Analysis Backlog (2026-07-01)
|
|
2
|
+
|
|
3
|
+
From studying ToolHive, brood-box, FrontierCode, SWE-1.6/Model-UX, Devin verification/async/productivity, and Anthropic's Code Modernization Playbook, gap-analyzed against loki/autonomi's real current source. Accuracy is the moat; zero adoption friction is the top need. Research-first: each item has acceptance criteria + a validation plan. Full data: gap-analysis-backlog.json.
|
|
4
|
+
|
|
5
|
+
## Themes
|
|
6
|
+
- Accuracy is the moat: the highest-leverage items make loki's core 'tests green' and 'checklist passed' signals actually mean something (test-provenance, source-grounded checklists, annotate-before-act, golden-master equivalence, runtime-boot proof). These are what verified-completion / trust-as-product cashes out to.
|
|
7
|
+
- Adoption friction: an honest, calibrated engineering-hours/ROI estimate on the trust rail (proof.json) is the enterprise-sales lever; deterministic per-repo setup recipes reduce the top user need (reliably reaching a runnable state).
|
|
8
|
+
- Anti-overthinking is a prompt-orchestration problem, not a training problem: loki can stop telling the model 'never finished' and can lower the no-claim council floor; these complement the in-flight completion-claim convergence fix.
|
|
9
|
+
- Hosting (server-side sandbox backend, durable session, multi-tenant isolation, egress firewall) is real but scoped to autonomi-saas and already founder-gated; it does not outrank the accuracy moat despite source 'critical' labels.
|
|
10
|
+
- Measurement instruments (FrontierCode-style mergeability benchmark, estimator calibration set) are prerequisites, not vanity: they are the validation_plan for the accuracy and ROI items and must exist before any headline number is claimed.
|
|
11
|
+
- Honesty over count: five source candidates are adjacent product categories or unconsumable model-lab techniques and are dropped, not padded; the ROI chain and golden-master chain are merged to their load-bearing primitives.
|
|
12
|
+
|
|
13
|
+
## Ranked backlog
|
|
14
|
+
|
|
15
|
+
| # | value | eff | product | axis | item |
|
|
16
|
+
|---|---|---|---|---|---|
|
|
17
|
+
| 1 | critical | L | loki-mode | verification | Reverse-classical test-provenance gate (change-mode: issue/verify/healing) |
|
|
18
|
+
| 2 | critical | L | loki-mode | accuracy | Ground the acceptance checklist in source reality, not spec prose alone |
|
|
19
|
+
| 3 | high | L | loki-mode | accuracy | Annotate-before-act: pre-committed expected-outcome ledger, diffed against reality at verify time |
|
|
20
|
+
| 4 | critical | L | loki-mode | accuracy | Golden-master parallel-run equivalence harness + per-boundary characterization (heal/migrate) |
|
|
21
|
+
| 5 | high | L | loki-mode | verification | Runtime/boot smoke gate in loki verify (shipped evidence includes 'it actually runs') |
|
|
22
|
+
| 6 | high | L | loki-mode | adoption-friction | LLM-calibrated engineering-hours estimator emitted into proof.json per build (+ calibration gate) |
|
|
23
|
+
| 7 | high | M | loki-mode | adoption-friction | Persist a deterministic per-repo setup/run recipe (reusable setup skill) consumed by verify |
|
|
24
|
+
| 8 | high | M | loki-mode | speed | Make rarv_instruction mode-aware + add 'stop-when-verified, gates are authority' directive |
|
|
25
|
+
| 9 | high | M | loki-mode | accuracy | Add a mergeability rubric dimension to run_code_review (weighted, tech-lead framing) |
|
|
26
|
+
| 10 | medium | M | loki-mode | accuracy | Code-scope / locality gate in autonomy/verify.sh (advisory-first) |
|
|
27
|
+
| 11 | medium | M | loki-mode | accuracy | Wire grill findings into the verification/completion gate |
|
|
28
|
+
| 12 | high | L | loki-mode | accuracy | FrontierCode-style scored mergeability benchmark harness for loki change-mode output |
|
|
29
|
+
| 13 | medium | M | loki-mode | adoption-friction | Modernization readiness + target-selection triage (loki heal --assess, pre-flight) |
|
|
30
|
+
| 14 | medium | L | loki-mode | verification | Route loki heal modernize/validate through the RARV-C loop + completion council |
|
|
31
|
+
| 15 | medium | M | loki-mode | speed | Lower the no-claim council convergence floor so genuinely-finished no-promise runs stop sooner |
|
|
32
|
+
| 16 | low | S | loki-mode | speed | Instruct the inner dev agent to batch independent tool calls in a single message |
|
|
33
|
+
| 17 | medium | L | autonomi-saas | adoption-friction | autonomi-saas: hours-saved / ROI receipt overlay consuming loki-mode proof.json effort_estimate |
|
|
34
|
+
| 18 | low | S | loki-mode | hosting | Add optional bearer-token auth + explicit loopback bind to loki mcp --transport http |
|
|
35
|
+
| 19 | medium | XL | autonomi-saas | hosting | autonomi-saas: hosted sandbox backend abstraction (gVisor-first) behind a SandboxBackend interface |
|
|
36
|
+
| 20 | medium | L | autonomi-saas | hosting | autonomi-saas: externalize the session as a durable append-only log decoupled from the sandbox |
|
|
37
|
+
|
|
38
|
+
## Assessed NOT applicable (CTO triage - declined with reason)
|
|
39
|
+
|
|
40
|
+
- **ToolHive-style per-MCP-server container isolation / curated registry / secrets vault (source: ToolHive)**: Presupposes running UNTRUSTED third-party MCP servers. Loki ships only first-party servers and uses --strict-mcp-config (providers/claude.sh:209) to actively EXCLUDE arbitrary third-party servers. Building a registry/containment layer adds adoption friction and code loki does not need for its threat model (it runs code it wrote). Adjacent product category. Source itself marked applicable=false (effort XL).
|
|
41
|
+
- **ToolHive as inspiration for autonomi hosting IF/when it hosts third-party MCP servers (source: ToolHive)**: Cannot be verified from this tree (autonomi is a separate private repo) and is gated on autonomi actually hosting untrusted third-party MCP servers, which is not a current requirement. Recorded as autonomi-hosting inspiration only, not a loki-mode backlog item. Source marked applicable=false. (Note: the gateway-auth primitive that IS directly consumable is captured as rank 18 for loki's own mcp http command.)
|
|
42
|
+
- **Integrate brood-box or Anthropic Managed Agents as a runtime dependency (source: brood-box / Managed Agents)**: brood-box is a local-developer agent sandbox CLI (adjacent product category loki's own three-tier sandbox already covers for local use); Managed Agents is a hosted Anthropic PLATFORM loki competes with, not an API loki consumes. The repo already made this triage (docs/ECOSYSTEM-CURRENCY-PLAN.json:77). The correct move is borrowing the architectural PATTERNS (decoupling, disposable untrusted sandbox, diff-then-flush) into autonomi's own founder-gated backend (ranks 19-20), not bolting on either tool. Source marked applicable=false.
|
|
43
|
+
- **mutagent-style adaptive test grading (source: FrontierCode)**: An internal benchmark-authoring tool for a third-party eval grader: LLMs surgically adapt a hidden-canonical-answer test env so open-ended solutions grade deterministically. Loki does not grade external agents' solutions against hidden canonical answers, so there is no consumer inside loki/autonomi. Adjacent product category. Source marked applicable=false (effort XL).
|
|
44
|
+
- **Training-time length penalty (source: SWE-1.6 Model UX)**: Cognition's headline fix was a length penalty during model POST-TRAINING. Loki is an orchestration layer over a hosted model (claude -p) and cannot retrain or add a training-time penalty. Adjacent (model-lab technique). The consumable analogs are delivered structurally via ranks 8, 15, 16 (stop telling the model never to stop; lower the no-claim floor; batch tool calls). Source marked applicable=false (effort XL).
|
|
45
|
+
|
|
46
|
+
## Detail (acceptance criteria + validation plan per item)
|
|
47
|
+
|
|
48
|
+
### #1 Reverse-classical test-provenance gate (change-mode: issue/verify/healing) (loki-mode, critical/L)
|
|
49
|
+
Before counting an agent's NEW tests as affirmative 'green' evidence in council_evidence_gate, run them against the pre-change base (VERIFY_MERGE_BASE / _LOKI_RUN_START_SHA) and require they FAIL on base and PASS on HEAD. Tests that pass on both base and HEAD are tautological and marked inconclusive, not affirmative. No-op on greenfield (no base) and when LOKI_TEST_PROVENANCE=0. This is the single highest-value accuracy-moat item: it makes loki's strongest verification signal (tests-green) actually mean something and directly attacks the 'test written to pass, proving nothing' failure mode.
|
|
50
|
+
Acceptance criteria:
|
|
51
|
+
- Given a change-mode run with a resolvable base, test files added/modified in the diff are identified
|
|
52
|
+
- New/changed tests are run against base (checkout or reverse-patch); which FAIL on base is recorded
|
|
53
|
+
- In council_evidence_gate (completion-council.sh:1621-1688) a completion claim whose only test evidence is provenance-UNconfirmed (passes on base) is marked inconclusive, never affirmative VERIFIED
|
|
54
|
+
- A trust event + .loki record lists each new test with base-fail/head-pass status
|
|
55
|
+
- Byte-identical no-op on greenfield and when LOKI_TEST_PROVENANCE=0
|
|
56
|
+
Validation: Fixture harness with two change-mode tasks: (A) a genuine fix with a test that fails on base + passes on HEAD, (B) a tautological test that passes on both. Assert: A is counted affirmative, B is downgraded to inconclusive and surfaced as a finding. Prove no-op via byte-diff of build_prompt/gate output on a greenfield run before/after. Cross-validate against the FrontierCode-style mergeability benchmark (rank 12): provenance-confirmed tasks should score higher on the reverse-classical dimension.
|
|
57
|
+
|
|
58
|
+
### #2 Ground the acceptance checklist in source reality, not spec prose alone (loki-mode, critical/L)
|
|
59
|
+
Extend checklist_oracle_triangulate beyond datastore to triangulate the checklist against the actual codebase: discovered HTTP routes/handlers (does each spec'd endpoint map to a real handler?), CLI commands, exported public API symbols (via the LSP proxy lsp_workspace_symbols / lsp_check_exists), and >=1 high-value domain invariant (passwords hashed not plaintext; money not float). Auto-generate deterministic checks (http_check, lsp_check_exists) instead of relying on the same LLM that wrote the code to author acceptance checks from the same prose. Attacks the single-oracle weakness the code itself flags as deferred (prd-checklist.sh).
|
|
60
|
+
Acceptance criteria:
|
|
61
|
+
- For a web project, every endpoint named in the spec maps to a real route/handler in source; unmatched -> High finding (mirrors existing oracle-datastore-conflict shape)
|
|
62
|
+
- Spec'd public API symbols verified to exist via the LSP proxy, not LLM-authored grep_codebase checks
|
|
63
|
+
- At least one domain-invariant check implemented as a deterministic gate (e.g. password fields hashed)
|
|
64
|
+
- Greenfield/absent (nothing wired yet) does NOT fire a finding, matching existing non-conflict guards
|
|
65
|
+
Validation: Fixture repos: (A) spec names endpoint /orders that exists -> no finding; (B) spec names /orders but no handler -> High finding; (C) plaintext password store -> invariant finding. Measure false-positive rate on 3 clean real repos (must be zero). Prove accuracy lift on the FrontierCode-style benchmark (rank 12): source-grounded checklists should catch wrong-but-internally-consistent implementations that prose-only checklists pass.
|
|
66
|
+
|
|
67
|
+
### #3 Annotate-before-act: pre-committed expected-outcome ledger, diffed against reality at verify time (loki-mode, high/L)
|
|
68
|
+
Before running verification for a change/checklist-item, the runtime writes a tamper-evident ledger of EXPECTED observable outcomes (e.g. 'GET /health -> 200 {status:ok}', 'test X fails before fix, passes after', 'endpoint Y should now exist'). At verify time, actual results are compared to the pre-committed prediction; any expectation silently dropped or contradicted becomes a finding. Reuse the proof-verify canonical-hash + drift pattern so the ledger is tamper-evident and edits-after-the-fact are detectable. Forces a falsifiable prediction, cutting self-serving post-hoc rationalization.
|
|
69
|
+
Acceptance criteria:
|
|
70
|
+
- .loki/expectations/<iter>.json written BEFORE the verify/checklist run, each entry {id, statement, check_ref, expected}, canonicalized and hashed like proof-generator
|
|
71
|
+
- loki verify (or the completion gate) reads the ledger and emits a finding for any expectation with no matching executed check, or whose actual contradicts expected
|
|
72
|
+
- An unevaluable expectation maps to inconclusive->CONCERNS, never VERIFIED (reuse verify.sh Entanglement-2)
|
|
73
|
+
- The ledger hash is embedded in evidence.json/proof.json so an edited-after-the-fact expectation is detectable
|
|
74
|
+
Validation: Test: seed a ledger with 3 expectations, run verify, assert (1) a met expectation passes, (2) a contradicted one is a finding, (3) a dropped/unexecuted one is a finding, (4) editing the ledger after write breaks the embedded hash. Before/after: same build with ledger disabled vs enabled must surface >=1 contradiction that post-hoc-only grading missed on a deliberately-mismatched fixture.
|
|
75
|
+
|
|
76
|
+
### #4 Golden-master parallel-run equivalence harness + per-boundary characterization (heal/migrate) (loki-mode, critical/L)
|
|
77
|
+
MERGED (harness + boundary-characterization). During archaeology/guardrail, capture outputs at natural system boundaries (CLI stdout/exit, HTTP responses, DB state deltas, file/queue outputs) into .loki/healing/behavioral-baseline/ as golden masters. During modernize/validate, re-run the SAME boundary probes against modernized code and diff. Replace hook_post_healing_modify's current whole-suite pass/fail (migration-hooks.sh:580) with per-boundary comparison so verification is implementation-agnostic and cannot be satisfied by overfit unit tests. Any divergence blocks the phase gate unless documented as intentional. Turns the playbook's 'side-by-side identical outputs' from a prompt suggestion into machine-verified evidence. The behavioral-baseline/ dir is currently mkdir'd (loki:14323) but never populated or compared.
|
|
78
|
+
Acceptance criteria:
|
|
79
|
+
- cmd_heal archaeology (and migrate guardrail) writes >=1 golden-master artifact per detected boundary into .loki/healing/behavioral-baseline/
|
|
80
|
+
- hook_healing_phase_gate for modernize->validate fails closed unless boundary outputs match baseline OR a documented-intentional-change record exists
|
|
81
|
+
- hook_post_healing_modify compares boundary outputs to baseline, not only whole-suite exit code; a change keeping unit tests green but altering a CLI/API boundary is BLOCKED
|
|
82
|
+
- diff report enumerates every changed boundary (old vs new) in loki heal --report and structured evidence JSON
|
|
83
|
+
- Degrades honestly ('no boundaries detected') rather than false-green
|
|
84
|
+
Validation: Fixture: a Python2->Python3 (or C->Java) sample repo. Capture baseline, apply (1) a behavior-preserving refactor -> gate passes; (2) a transform that keeps unit tests green but changes a CLI output -> gate BLOCKS. Assert baseline artifacts exist post-archaeology (grep count > 0, currently 0). Matches references/legacy-healing-patterns.md:156-181 prescribed system-boundary approach.
|
|
85
|
+
|
|
86
|
+
### #5 Runtime/boot smoke gate in loki verify (shipped evidence includes 'it actually runs') (loki-mode, high/L)
|
|
87
|
+
Promote the build-loop's playwright/http smoke into loki verify as a deterministic runtime gate: detect the start command (or use the persisted setup recipe from rank 9), boot the app with a timeout, hit a health/root path, capture status + a screenshot artifact, tear down, record in evidence.json as reproducible. Closes the biggest trust gap vs Devin (compiles + green-tests != works); the screenshot becomes a proof artifact. Runtime-not-reachable -> inconclusive->CONCERNS, never VERIFIED. Fail-open: without a detectable app, output is byte-identical to today.
|
|
88
|
+
Acceptance criteria:
|
|
89
|
+
- verify_gate_runtime added to verify_main gate list, skipped (not failed) only when no start command/setup recipe is detectable
|
|
90
|
+
- A successful boot records HTTP status + artifact path in evidence.json with reproducible=true
|
|
91
|
+
- Boot failure -> Critical/High finding; boot-not-attempted-but-app-detected -> inconclusive->CONCERNS
|
|
92
|
+
- No regression: without a detectable app, output byte-identical to today (mirrors --hosted fail-open pattern)
|
|
93
|
+
Validation: Fixture web app: (A) healthy boot -> evidence.json has status 200 + screenshot path + reproducible=true; (B) broken start command -> Critical finding, verdict not VERIFIED; (C) library repo with no app -> gate skipped, evidence.json byte-identical to pre-change baseline (diff must be empty).
|
|
94
|
+
|
|
95
|
+
### #6 LLM-calibrated engineering-hours estimator emitted into proof.json per build (+ calibration gate) (loki-mode, high/L)
|
|
96
|
+
MERGED (estimator + calibration). Replace the iterations x 15min heuristic (loki:23478) with an LLM estimator that reads the actual work (diff stat, files changed, tests written, task/PRD scope, iteration transcript) and predicts equivalent human-engineering-hours plus a low/high band. Persist as an effort_estimate object in per-build proof.json so it rides the trust/evidence rail. Fall back to the heuristic when no LLM is available, labeled 'heuristic (uncalibrated)' vs 'estimated' so the number is never silently fabricated. Ship a validation harness (ground-truth engineer-hour set) that computes an error metric (r_log or MAE-in-log-space); estimates may only be labeled 'calibrated' once the metric clears a threshold. This is the load-bearing primitive both the dollar ROI surface and any guarantee depend on, and it directly serves the 'accuracy is the moat, no fabrication' rule.
|
|
97
|
+
Acceptance criteria:
|
|
98
|
+
- proof.json gains effort_estimate: {hours, low, high, method:'llm'|'heuristic', model, inputs_hash}
|
|
99
|
+
- loki metrics TIME SAVED reads per-build estimates when present, falls back to iterations x 15min only when absent, labeling the method
|
|
100
|
+
- Estimator input is real work (diff stat + files + tests + scope), NOT iteration count alone; unit test feeds two builds of different scope and asserts different hours
|
|
101
|
+
- A validation dataset of builds with engineer-hour ground truth exists in benchmarks/; estimator error metric computed and reported; threshold gates 'calibrated' vs 'uncalibrated' labeling
|
|
102
|
+
- No LLM -> heuristic path clearly labeled 'heuristic (uncalibrated)', never presented as validated
|
|
103
|
+
Validation: Unit test: two builds of clearly different scope (2-iter schema tweak vs full-stack feature) must yield materially different hours (heuristic gives both 30min; assert estimator does not). Calibration: score LLM estimates against the ground-truth set, report r_log/MAE; assert copy stays 'uncalibrated' until threshold cleared. This IS the validation gate for the entire ROI chain.
|
|
104
|
+
|
|
105
|
+
### #7 Persist a deterministic per-repo setup/run recipe (reusable setup skill) consumed by verify (loki-mode, high/M)
|
|
106
|
+
On first successful build/boot, save .loki/setup-recipe.json capturing deterministic steps to install, seed, auth, and start the app (commands + env var NAMES + health path). Replay on subsequent verify/e2e runs instead of re-detecting heuristically. This is Devin's 'saved login/setup skill'; it reduces the top adoption need (reliably reaching a runnable state) and enables the runtime gate (rank 5) for repos needing auth/seed. Distinct from cmd_setup_skill (which installs the Loki skill, not a repo recipe). Secrets never persisted (only env var NAMES; reuse proof_redact.py).
|
|
107
|
+
Acceptance criteria:
|
|
108
|
+
- .loki/setup-recipe.json written after a verified boot with {install, seed, env_keys(no secret values), start, health_path}
|
|
109
|
+
- verify_gate_runtime prefers the recipe over heuristic detection when present
|
|
110
|
+
- Secrets never persisted into the recipe (only env var NAMES); reuse proof_redact.py patterns
|
|
111
|
+
- Recipe replayed idempotently and its use recorded in evidence.json
|
|
112
|
+
Validation: Fixture repo needing seed+auth: first run writes recipe; second run reaches runnable state via recipe (assert no heuristic re-detection ran, e.g. log marker). Assert recipe contains env var NAMES but zero secret VALUES (grep for known secret pattern must return nothing). Measure: time-to-runnable second run < first run.
|
|
113
|
+
|
|
114
|
+
### #8 Make rarv_instruction mode-aware + add 'stop-when-verified, gates are authority' directive (loki-mode, high/M)
|
|
115
|
+
MERGED (mode-aware rarv + stop directive). build_prompt currently emits the same 'There is NEVER a finished state - always find the next improvement' rarv_instruction (run.sh:13770) in every mode, including PRD/checkpoint runs just auto-switched to finite scope (a direct self-contradiction). Gate it on AUTONOMY_MODE/PERPETUAL_MODE: for finite runs replace with 'When the PRD requirements are implemented and completion gates pass, claim done via loki_complete_task and STOP; do not add unrequested improvements. Verify once -- the completion gates (tests, checklist, evidence) are the authority on done; do not re-verify redundantly.' Keep never-finished text only for genuine perpetual mode. Delete the unverifiable '2-3x quality improvement' clause (biases toward redundant self-verification). This is the orchestration analog of Cognition's length penalty and the prompt root of the 14-iter convergence bug. COMPLEMENTARY to the in-flight 'council evaluates on completion claim' fix (that changes when the council checks; this stops the prompt telling the model never to stop) -- not a duplicate.
|
|
116
|
+
Acceptance criteria:
|
|
117
|
+
- In a PRD run (auto-switched to checkpoint) the emitted build_prompt no longer contains 'There is NEVER a finished' or 'always find the next improvement'
|
|
118
|
+
- In a perpetual/no-promise run the never-finished text still appears (behavior preserved)
|
|
119
|
+
- The '2-3x quality improvement' clause removed from all modes; finite-mode prompt contains the concise 'stop when verified-done, gates are the authority' directive
|
|
120
|
+
- Deterministic completion gates (council_evidence_gate/checklist/heldout/assumption) unchanged and still block on failure
|
|
121
|
+
- loki-ts/src/runner/build_prompt.ts parity string updated (parity-locked)
|
|
122
|
+
Validation: String assertions on emitted build_prompt for each mode (finite lacks 'NEVER finished', perpetual retains it; '2-3x' absent everywhere). Behavioral: a spec satisfiable in ~1 iteration claims completion within a small bounded iteration count on a benchmark spec, with all completion gates still passing (verification not weakened -- run completion-council tests, assert no coverage regression). Confirm dual-route (bash + Bun) parity.
|
|
123
|
+
|
|
124
|
+
### #9 Add a mergeability rubric dimension to run_code_review (weighted, tech-lead framing) (loki-mode, high/M)
|
|
125
|
+
Extend (do not replace) the existing severity mechanism with a 'maintainer-mergeability' reviewer whose rubric covers scope creep, dead/duplicated code, and conformance to the surrounding code's conventions -- and have the aggregator produce a weighted quality SCORE alongside the binary block verdict (FrontierCode-style: 0 on any blocker, else weighted non-blocker sum). Reuse has_blocking (run.sh:10770) for the blocker layer; add the weighted score as a reported quality metric, not a new hard gate initially. Upgrades loki's review from 'CI + architecture' toward 'tech lead' on the axes it currently misses (the specialist pool is security/test/perf/dependency/architecture; none carries a 'would a maintainer merge this' mandate).
|
|
126
|
+
Acceptance criteria:
|
|
127
|
+
- A mergeability-focused reviewer added to the pool with a rubric prompt covering scope, dead code, and convention-conformance
|
|
128
|
+
- Aggregator emits a numeric quality score (0 if any blocker, else weighted non-blocker sum) in aggregate.json
|
|
129
|
+
- Existing Critical/High = block, Medium/Low = non-blocking behavior preserved unchanged
|
|
130
|
+
- Score reported/surfaced but not a new hard gate without explicit opt-in flag
|
|
131
|
+
- No emojis/em-dashes in prompt text; dual-route (bash + Bun) parity maintained
|
|
132
|
+
Validation: Fixture PRs: (A) tight 3-file fix -> high score, no scope blocker; (B) 40-file sprawling diff touching unrelated code -> scope blocker, score 0. Assert existing block behavior unchanged on a known-Critical fixture. Prove the score is discriminating on the FrontierCode-style benchmark (rank 12): mergeable vs unmergeable tasks should separate on the weighted score.
|
|
133
|
+
|
|
134
|
+
### #10 Code-scope / locality gate in autonomy/verify.sh (advisory-first) (loki-mode, medium/M)
|
|
135
|
+
Turn the already-computed VERIFY_DIFF_FILES/INS/DEL (verify.sh:154-164) into an enforced gate: configurable caps on total files changed and net line growth, plus an optional LLM semantic-locality check that flagged edits stay within the files/functions the task requires. Emit a 'scope' record on violation. Advisory-with-warning by default (record, don't hard-block) to avoid adoption friction; LOKI_SCOPE_ENFORCE=1 makes it blocking. Over-broad diffs are the #1 human merge-rejection reason FrontierCode targets, and loki currently measures the numbers but never uses them as a gate. N/A to greenfield full builds.
|
|
136
|
+
Acceptance criteria:
|
|
137
|
+
- verify.sh emits a structured scope record: files_changed, net_lines, verdict (ok|warn|block) with thresholds used
|
|
138
|
+
- Thresholds configurable via env (LOKI_SCOPE_MAX_FILES, LOKI_SCOPE_MAX_NET_LINES) with sane defaults
|
|
139
|
+
- Advisory by default (surfaces in verify output + SARIF), blocking only under LOKI_SCOPE_ENFORCE=1
|
|
140
|
+
- Optional semantic-locality LLM check gated behind its own flag (off by default, zero added latency/cost)
|
|
141
|
+
- No behavior change on greenfield loki start
|
|
142
|
+
Validation: Fixture change-mode diffs: (A) 3 files within scope -> verdict ok; (B) 40 files -> verdict warn (default) and block under LOKI_SCOPE_ENFORCE=1. Assert greenfield loki start output byte-identical before/after. Assert the scope record appears in evidence.json + SARIF.
|
|
143
|
+
|
|
144
|
+
### #11 Wire grill findings into the verification/completion gate (loki-mode, medium/M)
|
|
145
|
+
Feed the pre-build Devil's-Advocate grill's surfaced ambiguities/missing-acceptance-criteria into checklist generation and the completion gate: unresolved High grill findings become required checks or block completion until acknowledged (reuse the existing assumption-ledger gate in completion-council.sh). grill.sh currently writes a standalone report never consumed by verify/council. Makes the Devil's-Advocate posture actually harden what gets verified rather than being advisory-only.
|
|
146
|
+
Acceptance criteria:
|
|
147
|
+
- grill High findings converted into checklist items or assumption-ledger entries
|
|
148
|
+
- Unresolved High grill findings block completion via an existing gate path (not a new bespoke one)
|
|
149
|
+
- Opt-out knob preserves current standalone behavior
|
|
150
|
+
- No double-firing when grill was not run (gate inert, like the held-out gate NONE branch)
|
|
151
|
+
Validation: Fixture spec with a known ambiguity: run grill -> High finding -> assert it becomes a required check and blocks completion until acknowledged. Assert gate is inert (no finding) when grill was not run. Confirm reuse of the assumption-ledger path (no new bespoke gate) via code inspection + existing completion-council test suite passing.
|
|
152
|
+
|
|
153
|
+
### #12 FrontierCode-style scored mergeability benchmark harness for loki change-mode output (loki-mode, high/L)
|
|
154
|
+
Not a runtime feature -- the measurement instrument. Assemble a small suite of change-mode tasks with maintainer rubrics (blocker + weighted non-blocker) and reverse-classical tests, run loki verify / issue-mode against them, and produce a per-task mergeability score. This is what lets loki claim an accuracy/mergeability number and is the validation_plan backbone for ranks 1, 2, 9, 10. Aligns with pending task #49 (reproducible speed+accuracy benchmark) and #51 (council/RARV-C research update, gated on benchmark).
|
|
155
|
+
Acceptance criteria:
|
|
156
|
+
- A handful of tasks with base repo + rubric (blocker/non-blocker) + reverse-classical tests
|
|
157
|
+
- Harness runs loki change-mode and produces a per-task mergeability score
|
|
158
|
+
- Reproducible, documented, wired into the existing benchmarks/ conventions
|
|
159
|
+
- Reports a headline mergeability % comparable across loki versions
|
|
160
|
+
Validation: Self-validating: run harness on current loki, record baseline mergeability %. Re-run after ranks 1/2/9 land; assert the % moves (this IS the before/after proof for the accuracy items). Reproducibility check: two runs on the same loki version produce the same score within tolerance.
|
|
161
|
+
|
|
162
|
+
### #13 Modernization readiness + target-selection triage (loki heal --assess, pre-flight) (loki-mode, medium/M)
|
|
163
|
+
Add loki heal --assess (or a readiness step auto-run before archaeology) that scans a codebase read-only and outputs: estimated LOC/language mix, technical-debt signals, maintenance-risk indicators, placement on the playbook's 4-level maturity model, and a RANKED list of low-risk high-visibility targets to start with (playbook weeks 1-2). Reduces the top adoption-friction question 'where do I start' to one command, reusing loki's existing complexity-detection machinery. Human-readable + structured JSON for the dashboard.
|
|
164
|
+
Acceptance criteria:
|
|
165
|
+
- Single command emits a readiness report with maturity-level placement + ranked candidate targets + rationale
|
|
166
|
+
- Ranking prefers isolated/low-blast-radius modules (matches playbook weeks 1-2)
|
|
167
|
+
- Runs read-only (no modification), zero required flags
|
|
168
|
+
- Output is both human-readable and structured JSON for the dashboard
|
|
169
|
+
Validation: Fixture legacy repo with a mix of isolated and highly-coupled modules: assert the ranking places the isolated module above the coupled one, assert zero filesystem modifications (git status clean after run), assert valid JSON output parses.
|
|
170
|
+
|
|
171
|
+
### #14 Route loki heal modernize/validate through the RARV-C loop + completion council (loki-mode, medium/L)
|
|
172
|
+
loki heal is single-shot per phase (claude -p, loki:14465-14502), bypassing the RARV-C loop, completion council, and 8 quality gates that make loki start/migrate trustworthy. For behavior-changing phases (stabilize/isolate/modernize/validate), execute via run_autonomous with LOKI_INJECT_FINDINGS/LOKI_OVERRIDE_COUNCIL/LOKI_AUTO_LEARNINGS auto-enabled and the legacy-healing-auditor in the review pool. Archaeology-only can stay single-shot. Gives modernization builds the same iterative verification and override-council trust loki start already provides.
|
|
173
|
+
Acceptance criteria:
|
|
174
|
+
- cmd_heal for phases stabilize|isolate|modernize|validate invokes the RARV loop (run.sh path), not a bare claude -p call
|
|
175
|
+
- RARV-C closure flags default-on within heal (documented in skills/healing.md, verified by grep of cmd_heal)
|
|
176
|
+
- legacy-healing-auditor fires and can BLOCK on behavioral change without characterization update
|
|
177
|
+
- A deliberately-broken transform is caught + reverted by the loop, proven on a fixture
|
|
178
|
+
Validation: Fixture: run heal modernize with a deliberately-broken transform; assert the RARV loop catches + reverts it (vs current single-shot which would ship it). Grep cmd_heal to confirm run_autonomous path taken for the four phases. Assert legacy-healing-auditor appears in the review pool during heal. Pairs with rank 4 golden-master for the behavioral-change block proof.
|
|
179
|
+
|
|
180
|
+
### #15 Lower the no-claim council convergence floor so genuinely-finished no-promise runs stop sooner (loki-mode, medium/M)
|
|
181
|
+
For the convergence-detection (NO explicit completion claim) path only, reduce the effective floor: allow council_should_stop to evaluate as early as MIN_ITERATIONS=1 when tests+checklist evidence already indicate done, or check every iteration once evidence is green instead of every 5th (currently CHECK_INTERVAL=5, MIN_ITERATIONS=3). The explicit-claim path already bypasses this (run.sh:17418), so this only helps analysis-mode / no-promise runs -- COMPLEMENTARY to the in-flight 'council evaluates on completion claim' fix (which is the explicit-claim path), not a duplicate. Keep stagnation and evidence gates intact -- changes WHEN the stop check runs, not WHETHER work is verified.
|
|
182
|
+
Acceptance criteria:
|
|
183
|
+
- A no-promise run that is genuinely complete can stop before iteration 5 when evidence gates pass
|
|
184
|
+
- Stagnation and evidence gates (completion-council.sh) still block premature/unverified stops
|
|
185
|
+
- Explicit-claim completion path behavior unchanged
|
|
186
|
+
- Gated behind existing LOKI_COUNCIL_CHECK_INTERVAL/MIN_ITERATIONS so operators can restore old behavior
|
|
187
|
+
Validation: Fixture no-promise/analysis run that is complete by iteration 2: assert it can stop before iteration 5 with evidence gates green. Negative test: an unverified/stagnating run still does NOT stop early (evidence gate blocks). Assert explicit-claim path timing unchanged via existing completion-council tests.
|
|
188
|
+
|
|
189
|
+
### #16 Instruct the inner dev agent to batch independent tool calls in a single message (loki-mode, low/S)
|
|
190
|
+
Add a short build_prompt directive (all modes): 'When issuing independent read-only operations (reads, greps, file lookups, LSP checks) that do not depend on each other, issue them in a single message so they run in parallel; do not serialize independent lookups.' The direct, cheap analog of Cognition's parallel-tool-call fix at the exact layer loki controls (the prompt to claude -p). Zero adoption friction (internal prompt string), no verification impact. Parity mirror in loki-ts.
|
|
191
|
+
Acceptance criteria:
|
|
192
|
+
- build_prompt emits a parallel/batch tool-call directive in every mode
|
|
193
|
+
- Parity mirror added to loki-ts/src/runner/build_prompt.ts
|
|
194
|
+
- Directive is <= 2 sentences and adds no new env knob
|
|
195
|
+
- A representative run shows the inner agent issuing at least one multi-tool-call message for independent lookups (spot-checked in agent.log / stream-json)
|
|
196
|
+
Validation: String assertion: directive present in all build_prompt mode variants and in loki-ts parity string. Behavioral spot-check: on a representative run, grep agent.log/stream-json for at least one message issuing multiple independent tool calls. Confirm no new env knob added.
|
|
197
|
+
|
|
198
|
+
### #17 autonomi-saas: hours-saved / ROI receipt overlay consuming loki-mode proof.json effort_estimate (autonomi-saas, medium/L)
|
|
199
|
+
In autonomi-saas, overlay the per-build effort_estimate from loki's proof.json (rank 6) onto the Evidence Receipt and aggregate into an account-level ROI dashboard ('this month: ~N engineer-hours, ~$M at a configurable blended rate'). Aggregation matches the level where the estimator is honest (Devin's aggregation-honest framing). $ requires an operator-set rate; unset shows hours only, never a fabricated dollar figure. The natural home for a Devin-style productivity guarantee (SaaS carries billing/tenancy); the guarantee is founder-gated and blocked until the estimator is calibrated (rank 6). No cross-tenant leakage (respects the per-build two-read split used for #17).
|
|
200
|
+
Acceptance criteria:
|
|
201
|
+
- BFF reads proof.json effort_estimate per build and stores it against the tenant/build
|
|
202
|
+
- Account view aggregates hours (+ $ if rate set) across builds, labeled with the aggregation-honest caveat
|
|
203
|
+
- Guarantee logic (if pursued) founder-gated and blocked until estimator has a validation number
|
|
204
|
+
- No cross-tenant leakage in the aggregate (respects the per-build two-read split for #17)
|
|
205
|
+
Validation: Two-tenant fixture: assert tenant A's aggregate excludes tenant B's builds (cross-tenant leak test, reuse #17 harness). Assert $ absent when no rate configured. Assert guarantee copy is blocked/hidden while estimator method='heuristic' or label='uncalibrated'. Depends on rank 6 shipping first.
|
|
206
|
+
|
|
207
|
+
### #18 Add optional bearer-token auth + explicit loopback bind to loki mcp --transport http (loki-mode, low/S)
|
|
208
|
+
The one place loki's surface maps directly onto ToolHive's gateway. mcp/server.py:2639 calls mcp.run(transport='http', port=args.port) with no explicit host and no auth. Pass an explicit host (default 127.0.0.1) and support an optional LOKI_MCP_AUTH_TOKEN that the server requires on every request. Low-cost hardening of an already-shipping command; loopback default unchanged (zero adoption friction, token opt-in). Latent-not-exposed today (FastMCP loopback default) but the explicit bind + opt-in token close it honestly.
|
|
209
|
+
Acceptance criteria:
|
|
210
|
+
- loki mcp --transport http binds 127.0.0.1 explicitly (verified via ss/lsof, not 0.0.0.0)
|
|
211
|
+
- When LOKI_MCP_AUTH_TOKEN is set, requests without a matching bearer are rejected 401; when unset, behavior is unchanged
|
|
212
|
+
- Docs note the token env var; default local stdio path untouched
|
|
213
|
+
Validation: Start loki mcp --transport http, verify bind address via lsof/ss is 127.0.0.1 not 0.0.0.0. With LOKI_MCP_AUTH_TOKEN set: request without bearer -> 401, with matching bearer -> 200. With it unset: behavior byte-identical to today. Confirm stdio path unaffected.
|
|
214
|
+
|
|
215
|
+
### #19 autonomi-saas: hosted sandbox backend abstraction (gVisor-first) behind a SandboxBackend interface (autonomi-saas, medium/XL)
|
|
216
|
+
FOUNDER-GATED. Define one SandboxBackend interface (create/exec/collect-diff/flush/destroy) and implement gVisor as the first backend for per-build/per-verification isolation, mirroring Managed Agents' disposable untrusted-sandbox. Concrete unblock for founder-gated backlog ranks 14/36/42. Borrow brood-box's diff-then-flush and Managed Agents' decoupling as PATTERNS, not dependencies; libkrun-microVM is a strong second backend to evaluate in the selection spike. Sandbox treated as untrusted: no tenant secrets/signing keys reachable from inside. This is real but scoped to autonomi and already triaged/founder-gated in the repo docs; it does NOT outrank the accuracy moat despite the source 'critical' label.
|
|
217
|
+
Acceptance criteria:
|
|
218
|
+
- A SandboxBackend interface exists with create/exec/collect-diff/flush/destroy and at least one working backend (gVisor)
|
|
219
|
+
- Each hosted build/verification runs in its own disposable sandbox destroyed on completion
|
|
220
|
+
- Sandbox is treated as untrusted: no tenant secrets or signing keys reachable from inside
|
|
221
|
+
- Selection spike documents gVisor vs Firecracker vs Kata vs libkrun with a chosen default + rationale (satisfies rank 42)
|
|
222
|
+
- Pre-flush diff review gates change persistence; per-domain egress allowlist (default provider API + git host, deny-by-default option); secret paths (.env*, keys) non-overridably excluded from any flushed change set
|
|
223
|
+
Validation: Isolation test: attempt to read a mounted tenant secret from inside the sandbox -> must fail. Disposability: assert sandbox is destroyed post-build (no lingering container). Egress: request to a non-allowlisted domain blocked, provider API + git host allowed. Diff-then-flush: an unapproved change is not persisted to the tenant workspace. Founder-gated: do not start until the founder unblocks; kept below the accuracy moat.
|
|
224
|
+
|
|
225
|
+
### #20 autonomi-saas: externalize the session as a durable append-only log decoupled from the sandbox (autonomi-saas, medium/L)
|
|
226
|
+
FOUNDER-GATED. Move build state off local .loki/ files into a durable session store (the persistence backend at backlog rank 16) so the harness/sandbox can crash or scale horizontally without losing progress -- Managed Agents' session/harness/sandbox decoupling. Enables resume-across-workers and starting reasoning before the sandbox is fully provisioned. NOT applicable to loki-mode local (its .loki/ file state is correct for single-user). Scoped to autonomi, founder-gated; below the accuracy moat.
|
|
227
|
+
Acceptance criteria:
|
|
228
|
+
- A PersistenceBackend interface with a first prod store (Postgres or SQLite) exists (satisfies backlog rank 16)
|
|
229
|
+
- A hosted build survives a worker/sandbox restart and resumes from the durable session
|
|
230
|
+
- The harness reads session state independently of any specific sandbox instance
|
|
231
|
+
Validation: Crash-resume test: kill the worker/sandbox mid-build, restart, assert the build resumes from the durable session with no lost progress. Horizontal test: two workers read the same session store without corruption. Assert loki-mode local .loki/ path is untouched (scoped to autonomi only).
|
|
232
|
+
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
# Founder Observability Tool - Spec
|
|
2
|
+
|
|
3
|
+
## Goal
|
|
4
|
+
ONE local, Docker-hosted dashboard for the founder to navigate the whole program's history at a
|
|
5
|
+
glance: releases, before/after metrics, what worked, what did NOT work, mistakes made + fixes,
|
|
6
|
+
and current status - across loki-mode + autonomi-verify + autonomi-saas. Simplest possible form,
|
|
7
|
+
in Autonomi's design language. Runs in Docker locally; opens in the browser when done.
|
|
8
|
+
|
|
9
|
+
## Absolute rules (never violate)
|
|
10
|
+
- **HONEST DATA ONLY.** Every number/claim must come from a REAL source file (below). NEVER
|
|
11
|
+
fabricate a metric, a release, or an outcome. Accuracy is the moat; a fabricated founder
|
|
12
|
+
dashboard destroys it. If a datum is unknown, show "not measured" - never invent it.
|
|
13
|
+
- Include the UGLY: "what didn't work" and "mistakes" are REQUIRED sections and must show real
|
|
14
|
+
ones (e.g. the reverted PRD-reuse denylist fix; the convergence double-consume bug two Opus
|
|
15
|
+
reviewers caught before ship; the benchmark harness dying at turn boundaries). Honesty about
|
|
16
|
+
misses is the point of the tool.
|
|
17
|
+
- No emojis, no em-dashes/en-dashes anywhere (project rule).
|
|
18
|
+
|
|
19
|
+
## Data sources (read these; do not invent)
|
|
20
|
+
- Releases: `CHANGELOG.md` (parse `## [x.y.z] - date` + the entry body).
|
|
21
|
+
- Before/after metrics: `benchmarks/results/*.json` (speed-before/after-*.json = wall_clock_min,
|
|
22
|
+
act_iterations, completion_claims, engine_completed, acceptance, per_iteration_work_s).
|
|
23
|
+
- Diagnoses / decisions / plans: `docs/SPEED-DIAGNOSIS-2026-07-01.md`,
|
|
24
|
+
`docs/FOUNDER-STATUS-AND-DECISIONS-2026-07-01.md`, `docs/TOP-100-BACKLOG.md`,
|
|
25
|
+
`docs/research-2026-07/GAP-ANALYSIS-BACKLOG.md`.
|
|
26
|
+
- What worked / didn't / mistakes: parse the above docs' honest sections + git log on origin/main
|
|
27
|
+
(`git log --oneline` for shipped commits; look for "revert" and "fix(...council-review finding)"
|
|
28
|
+
as mistake->fix signals).
|
|
29
|
+
- Founder-gated decisions (open): section 4/6 of FOUNDER-STATUS-AND-DECISIONS.
|
|
30
|
+
|
|
31
|
+
## REAL adoption + efficiency metrics (founder wants graphs, real numbers, NO fakes)
|
|
32
|
+
Fetch these live at build/refresh time. If any endpoint fails, show "unavailable" - NEVER fake.
|
|
33
|
+
Verified working 2026-07-01 (sample values shown so you know the shape; re-fetch for current):
|
|
34
|
+
- **GitHub stars/forks/issues**: `gh api repos/asklokesh/loki-mode` (.stargazers_count=998,
|
|
35
|
+
.forks_count=193, .open_issues_count, .created_at=2025-12-26).
|
|
36
|
+
- **Per-release download counts + dates**: `gh api repos/asklokesh/loki-mode/releases`
|
|
37
|
+
(per tag: .tag_name, .published_at, sum of .assets[].download_count). Real per-release adoption.
|
|
38
|
+
- **npm downloads (last-month total + 30-day DAILY series for a line graph)**:
|
|
39
|
+
`curl -s https://api.npmjs.org/downloads/range/last-month/loki-mode` -> .downloads[] is
|
|
40
|
+
[{day, downloads}] (30 points; last month total ~23,798; peak 2,598 on 2026-06-17). GRAPH THIS.
|
|
41
|
+
Also per-version: `https://api.npmjs.org/versions/loki-mode/last-week` (downloads by version).
|
|
42
|
+
- **Docker Hub pulls**: `curl -s https://hub.docker.com/v2/repositories/asklokesh/loki-mode/`
|
|
43
|
+
-> .pull_count (=56,232), .last_updated. (Cumulative total only; no time-series from Hub.)
|
|
44
|
+
- **Homebrew custom-tap installs**: NOT publicly available for a custom tap (only homebrew/core
|
|
45
|
+
taps have analytics). Show "not available (custom tap has no public install API)" - do NOT fake.
|
|
46
|
+
- **Efficiency per release**: from `benchmarks/results/speed-*.json` (wall_clock_min,
|
|
47
|
+
act_iterations) + git-diff churn per release from CHANGELOG. Where a release has no benchmark,
|
|
48
|
+
show "not benchmarked" honestly. As more before/after runs accumulate, chart wall-clock and
|
|
49
|
+
iteration-count per version to show the efficiency trend.
|
|
50
|
+
|
|
51
|
+
GRAPHS REQUIRED (real data only): npm daily-downloads line (30d), per-release GH-download bars,
|
|
52
|
+
cumulative Docker pulls (single stat + timestamp), stars (single stat), and a
|
|
53
|
+
speed/efficiency-per-release chart (wall-clock + iterations) that grows as benchmarks accumulate.
|
|
54
|
+
Use a tiny dependency-light chart approach (inline SVG or a single vendored lib) so it stays
|
|
55
|
+
Dockerizable and offline-safe. A daily/refresh fetch keeps the numbers current.
|
|
56
|
+
|
|
57
|
+
## Design language (Autonomi)
|
|
58
|
+
Match the shipped Autonomi look. Reference files: `artifacts/whitepaper/Autonomi-Whitepaper.html`
|
|
59
|
+
and `dashboard/static/trust.html`. Palette: dark bg (#0F0B1A / #0d1117), Autonomi purple accent
|
|
60
|
+
(#553DE9 / #7B6BF0), verified teal (#2ED8B6 / #1AAF95), dim ink (#B8B0C8 / #8A857C), warning amber
|
|
61
|
+
(#C4922E), reject red (#C04848). Semantic status colors: verified=teal, reported/pending=amber,
|
|
62
|
+
mistake/reverted=red, roadmap=purple. Clean, calm, founder-readable; big numbers, short labels,
|
|
63
|
+
before->after arrows. Reuse the whitepaper's CSS-variable pattern (--bg, --accent, --verified,
|
|
64
|
+
--ink, --panel, --line) so it feels native.
|
|
65
|
+
|
|
66
|
+
## Sections (the dashboard)
|
|
67
|
+
1. **At a glance**: current live version (VERSION), # releases this cycle, latest before/after
|
|
68
|
+
speed number (with an honest "n=1, pending repeat" caveat if only one after-run exists).
|
|
69
|
+
2. **Release timeline**: each version, date, one-line what+why, and its outcome (live/verified).
|
|
70
|
+
3. **Before/After metrics**: the speed benchmark(s) as before->after cards (wall-clock, iterations,
|
|
71
|
+
completion-claims); mark clearly what engine version each ran on and that attribution is
|
|
72
|
+
"vX vs vY" not single-cause. If only a before exists, show before + "after pending".
|
|
73
|
+
4. **What worked**: shipped, council-approved, verified fixes (with the evidence: tests, live check).
|
|
74
|
+
5. **What did NOT work / reverted**: honest list (PRD-reuse denylist reverted as unsound; any
|
|
75
|
+
dropped approach) with the reason.
|
|
76
|
+
6. **Mistakes -> Fixes**: bugs I introduced that review/CI caught and I fixed (convergence
|
|
77
|
+
double-consume; server.py payload edge; benchmark cross-spec contamination avoided). Each:
|
|
78
|
+
what went wrong -> how caught -> how fixed. This is the founder's audit trail.
|
|
79
|
+
7. **Open founder decisions**: the 6 one-line asks from FOUNDER-STATUS-AND-DECISIONS.
|
|
80
|
+
8. **Backlog / roadmap**: top ranked items from the gap-analysis + top-100 (accuracy-first).
|
|
81
|
+
|
|
82
|
+
## Delivery
|
|
83
|
+
- Static-first is fine (a single index.html + a small data.json generated from the sources by a
|
|
84
|
+
build script), OR a tiny server. Whatever is simplest and robust.
|
|
85
|
+
- Dockerized: a `Dockerfile` + a one-command run (e.g. `docker build -t loki-observability . &&
|
|
86
|
+
docker run -p <port>:<port> loki-observability`) serving the dashboard. Pick a free port
|
|
87
|
+
(NOT 57374/57375 which loki uses); e.g. 58080.
|
|
88
|
+
- A build/refresh script that regenerates data.json from the real sources so it stays current.
|
|
89
|
+
- Place under `artifacts/observability/` (or `tools/observability/`).
|
|
90
|
+
- OPEN it in the browser when up (macOS `open http://127.0.0.1:<port>`).
|
|
91
|
+
|
|
92
|
+
## Validation (before calling done)
|
|
93
|
+
- Every displayed number traces to a source file (spot-check 3).
|
|
94
|
+
- The "mistakes" and "didn't work" sections are NON-EMPTY and real.
|
|
95
|
+
- `docker run` serves the page; the page renders in the Autonomi look; no console errors.
|
|
96
|
+
- No emojis / dashes. Screenshot the running page as proof.
|
package/loki-ts/dist/loki.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
// @bun
|
|
2
|
-
var X8=Object.defineProperty;var K8=($)=>$;function q8($,Q){this[$]=K8.bind(null,Q)}var b=($,Q)=>{for(var Z in Q)X8($,Z,{get:Q[Z],enumerable:!0,configurable:!0,set:q8.bind(Q,Z)})};var L=($,Q)=>()=>($&&(Q=$($=0)),Q);var J$=import.meta.require;var v1={};b(v1,{lokiDir:()=>P,homeLokiDir:()=>T$,findRepoRootForVersion:()=>e$,REPO_ROOT:()=>h});import{resolve as r,dirname as i$}from"path";import{fileURLToPath as J8}from"url";import{existsSync as x$}from"fs";import{homedir as V8}from"os";function W8(){let $=y1;for(let Q=0;Q<6;Q++){if(x$(r($,"VERSION"))&&x$(r($,"autonomy/run.sh")))return $;let Z=i$($);if(Z===$)break;$=Z}return r(y1,"..","..","..")}function e$($){let Q=$;for(let Z=0;Z<6;Z++){if(x$(r(Q,"VERSION"))&&x$(r(Q,"autonomy/run.sh")))return Q;let z=i$(Q);if(z===Q)break;Q=z}return r($,"..","..","..")}function P(){return process.env.LOKI_DIR??r(process.cwd(),".loki")}function T$(){return r(V8(),".loki")}var y1,h;var C=L(()=>{y1=i$(J8(import.meta.url));h=W8()});import{readFileSync as U8}from"fs";import{resolve as H8,dirname as G8}from"path";import{fileURLToPath as B8}from"url";function N$(){if(Q$!==null)return Q$;let $="7.
|
|
2
|
+
var X8=Object.defineProperty;var K8=($)=>$;function q8($,Q){this[$]=K8.bind(null,Q)}var b=($,Q)=>{for(var Z in Q)X8($,Z,{get:Q[Z],enumerable:!0,configurable:!0,set:q8.bind(Q,Z)})};var L=($,Q)=>()=>($&&(Q=$($=0)),Q);var J$=import.meta.require;var v1={};b(v1,{lokiDir:()=>P,homeLokiDir:()=>T$,findRepoRootForVersion:()=>e$,REPO_ROOT:()=>h});import{resolve as r,dirname as i$}from"path";import{fileURLToPath as J8}from"url";import{existsSync as x$}from"fs";import{homedir as V8}from"os";function W8(){let $=y1;for(let Q=0;Q<6;Q++){if(x$(r($,"VERSION"))&&x$(r($,"autonomy/run.sh")))return $;let Z=i$($);if(Z===$)break;$=Z}return r(y1,"..","..","..")}function e$($){let Q=$;for(let Z=0;Z<6;Z++){if(x$(r(Q,"VERSION"))&&x$(r(Q,"autonomy/run.sh")))return Q;let z=i$(Q);if(z===Q)break;Q=z}return r($,"..","..","..")}function P(){return process.env.LOKI_DIR??r(process.cwd(),".loki")}function T$(){return r(V8(),".loki")}var y1,h;var C=L(()=>{y1=i$(J8(import.meta.url));h=W8()});import{readFileSync as U8}from"fs";import{resolve as H8,dirname as G8}from"path";import{fileURLToPath as B8}from"url";function N$(){if(Q$!==null)return Q$;let $="7.106.0";if(typeof $==="string"&&$.length>0)return Q$=$,Q$;try{let Q=G8(B8(import.meta.url)),Z=e$(Q);Q$=U8(H8(Z,"VERSION"),"utf-8").trim()}catch{Q$="unknown"}return Q$}var Q$=null;var $1=L(()=>{C()});var c1={};b(c1,{runOrThrow:()=>x8,run:()=>F,readStreamCapped:()=>u1,commandVersion:()=>S8,commandExists:()=>f,ShellError:()=>Q1,MAX_STDOUT_BYTES:()=>f1});async function u1($,Q=f1){let Z=$.getReader(),z=new TextDecoder,X="",q=0;try{while(q<Q){let{done:K,value:W}=await Z.read();if(K)break;if(!W)continue;if(q+=W.byteLength,q>Q){let V=W.byteLength-(q-Q);X+=z.decode(W.subarray(0,V),{stream:!0});break}X+=z.decode(W,{stream:!0})}X+=z.decode()}finally{try{await Z.cancel()}catch{}Z.releaseLock()}return X}async function F($,Q={}){let Z=Bun.spawn({cmd:[...$],stdout:"pipe",stderr:"pipe",env:Q.env?{...process.env,...Q.env}:process.env,cwd:Q.cwd}),z,X;if(Q.timeoutMs&&Q.timeoutMs>0)z=setTimeout(()=>{try{Z.kill("SIGTERM")}catch{}X=setTimeout(()=>{try{Z.kill("SIGKILL")}catch{}},2000)},Q.timeoutMs);try{let[q,K,W]=await Promise.all([u1(Z.stdout),new Response(Z.stderr).text(),Z.exited]);return{stdout:q,stderr:K,exitCode:W}}finally{if(z)clearTimeout(z);if(X)clearTimeout(X)}}async function x8($,Q={}){let Z=await F($,Q);if(Z.exitCode!==0)throw new Q1(`command failed (${Z.exitCode}): ${$.join(" ")}`,Z.exitCode,Z.stdout,Z.stderr);return Z}async function f($){let Q=N8($),Z=await F(["sh","-c",`command -v ${Q}`],{timeoutMs:5000});if(Z.exitCode===0)return Z.stdout.trim()||null;return null}function N8($){if(!/^[A-Za-z0-9._/-]+$/.test($))throw Error(`refused to shell-escape suspect token: ${$}`);return $}async function S8($,Q="--version"){if(!await f($))return null;let z=await F([$,Q],{timeoutMs:5000});if(z.exitCode!==0)return null;return((z.stdout||z.stderr).split(/\r?\n/)[0]?.trim()??"")||null}var f1=16777216,Q1;var d=L(()=>{Q1=class Q1 extends Error{message;exitCode;stdout;stderr;constructor($,Q,Z,z){super($);this.message=$;this.exitCode=Q;this.stdout=Z;this.stderr=z;this.name="ShellError"}}});function t($){return D8?"":$}var D8,T,S,_,nZ,I,k,y,J;var c=L(()=>{D8=(process.env.NO_COLOR??"").length>0;T=t("\x1B[0;31m"),S=t("\x1B[0;32m"),_=t("\x1B[1;33m"),nZ=t("\x1B[0;34m"),I=t("\x1B[0;36m"),k=t("\x1B[1m"),y=t("\x1B[2m"),J=t("\x1B[0m")});import{existsSync as c8}from"fs";async function Z$(){if(A$!==void 0)return A$;let $="/opt/homebrew/bin/python3.12";if(c8($))return A$=$,$;let Q=await f("python3.12");if(Q)return A$=Q,Q;let Z=await f("python3");return A$=Z,Z}async function z$($,Q={}){let Z=await Z$();if(!Z)return{stdout:"",stderr:"python3 not found",exitCode:127};return F([Z,"-c",$],Q)}var A$;var V$=L(()=>{d()});var J0={};b(J0,{runStatus:()=>U3});import{existsSync as v,readFileSync as U$,readdirSync as e1,statSync as $0}from"fs";import{resolve as D,basename as Q3}from"path";import{homedir as Z3}from"os";function Q0($){let Q=Math.trunc($);if(Q>=1e6)return`${(Math.trunc(Q/1e6*10)/10).toFixed(1)}M`;if(Q>=1000)return`${(Math.trunc(Q/1000*10)/10).toFixed(1)}K`;return String(Q)}function Z0($,Q,Z){if(Q===0)return null;let z=Math.trunc($*100/Q),X=Math.trunc($*D$/Q);if(X>D$)X=D$;let q=D$-X,K=S;if(z>=80)K=T;else if(z>=50)K=_;let W="=".repeat(Math.max(0,X))+" ".repeat(Math.max(0,q)),V=Q0($),U=Q0(Q);return` ${k}${Z}${J} ${K}[${W}]${J} ${z}% (${V} / ${U})`}async function X3(){if(await f("jq"))return!0;return process.stdout.write(`${T}Error: jq is required but not installed.${J}
|
|
3
3
|
`),process.stdout.write(`Install with:
|
|
4
4
|
`),process.stdout.write(` brew install jq (macOS)
|
|
5
5
|
`),process.stdout.write(` apt install jq (Debian/Ubuntu)
|
|
@@ -805,4 +805,4 @@ Set LOKI_LEGACY_BASH=1 to force the bash CLI for every command.
|
|
|
805
805
|
`),2}default:return process.stderr.write(`Unknown command: ${Q}
|
|
806
806
|
`),process.stderr.write(z8),2}}i1();process.on("SIGINT",()=>process.exit(130));process.on("SIGTERM",()=>process.exit(143));var EZ=await RZ(Bun.argv.slice(2));process.exit(EZ);
|
|
807
807
|
|
|
808
|
-
//# debugId=
|
|
808
|
+
//# debugId=FD93D5FC3483E3A564756E2164756E21
|
package/mcp/__init__.py
CHANGED
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "loki-mode",
|
|
3
3
|
"mcpName": "io.github.asklokesh/loki-mode",
|
|
4
|
-
"version": "7.
|
|
4
|
+
"version": "7.106.0",
|
|
5
5
|
"description": "Loki Mode by Autonomi. Autonomous spec-to-product system: takes a PRD, GitHub issue, OpenAPI/JSON/YAML, or one-line brief to a deployed app via the RARV-C closure loop with 8 quality gates. Provider-agnostic (Claude Code, OpenAI Codex, Cline, Aider).",
|
|
6
6
|
"keywords": [
|
|
7
7
|
"agent",
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
"$schema": "https://json.schemastore.org/claude-code-plugin-manifest.json",
|
|
3
3
|
"name": "loki-mode",
|
|
4
4
|
"displayName": "Loki Mode",
|
|
5
|
-
"version": "7.
|
|
5
|
+
"version": "7.106.0",
|
|
6
6
|
"description": "Autonomous spec-to-product build system with a built-in trust layer (RARV-C closure loop, 8 quality gates, completion council). Ships Loki's spec-hardening, drift-detection, and deterministic PR verification commands plus the Loki MCP server.",
|
|
7
7
|
"author": {
|
|
8
8
|
"name": "Autonomi",
|