loki-mode 9.16.0 → 9.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/SKILL.md +2 -2
- package/VERSION +1 -1
- package/autonomy/issue-providers.sh +12 -9
- package/autonomy/lib/agent_readiness.py +79 -1
- package/autonomy/lib/gate_policy.py +166 -0
- package/autonomy/lib/outcome_ledger.py +122 -0
- package/autonomy/lib/proof-generator.py +71 -4
- package/autonomy/loki +83 -12
- package/autonomy/notify.sh +70 -1
- package/autonomy/queue-consumer.sh +290 -18
- package/autonomy/run.sh +11 -1
- package/completions/_loki +1 -0
- package/completions/loki.bash +2 -2
- package/dashboard/__init__.py +1 -1
- package/dashboard/server.py +168 -1
- package/dashboard/static/index.html +248 -55
- package/docs/QUEUE-OPERATIONS.md +107 -0
- package/docs/VERIFICATION-COST.md +170 -0
- package/loki-ts/dist/loki.js +287 -293
- package/mcp/__init__.py +1 -1
- package/package.json +1 -1
- package/plugins/loki-mode/.claude-plugin/plugin.json +1 -1
package/autonomy/loki
CHANGED
|
@@ -1040,7 +1040,7 @@ show_help() {
|
|
|
1040
1040
|
echo " agent analyze api assets audit bench checkpoint (cp) ci cleanup"
|
|
1041
1041
|
echo " cluster cockpit code compliance completions compound config context (ctx)"
|
|
1042
1042
|
echo " cost council crash dashboard demo deploy docker docs doctor dogfood"
|
|
1043
|
-
echo " enterprise estimate explain export failover github grill heal help"
|
|
1043
|
+
echo " enterprise estimate explain export failover gates github grill heal help"
|
|
1044
1044
|
echo " import init intent"
|
|
1045
1045
|
echo " issue kpis logs magic mcp memory metrics migrate modernize monitor"
|
|
1046
1046
|
echo " next notify onboard open optimize otel outcomes own (handoff) pause plan preview"
|
|
@@ -13821,46 +13821,60 @@ set_ttfv_lightweight_profile() {
|
|
|
13821
13821
|
# honest about being shallow -- it turns stated nouns into checkable lines and
|
|
13822
13822
|
# claims nothing about intent it cannot see. Anything cleverer belongs in the
|
|
13823
13823
|
# spec-interrogation grill, which already runs after this and does call a model.
|
|
13824
|
+
#
|
|
13825
|
+
# STABLE IDs. Each criterion is emitted as "AC-<AXIS>-NNN: <text>" rather than a
|
|
13826
|
+
# bare bullet. An anonymous bullet cannot be referred to: a receipt can say "3 of
|
|
13827
|
+
# 8 gates passed" but never "AC-PERSIST-001 is satisfied by this test", drift
|
|
13828
|
+
# cannot be tracked per criterion, and two runs of the same spec produce lists
|
|
13829
|
+
# nothing can diff. The ID is what turns a criterion into a citable claim, which
|
|
13830
|
+
# is the whole point of shipping a receipt someone can check.
|
|
13831
|
+
#
|
|
13832
|
+
# The axis is derived from WHICH obligation fired, not from the criterion's
|
|
13833
|
+
# position, so IDs are stable across runs: adding a payment criterion never
|
|
13834
|
+
# renumbers the persistence one. Same reason we do not use a running counter.
|
|
13824
13835
|
_brief_acceptance_criteria() {
|
|
13825
13836
|
local t
|
|
13826
13837
|
t="$(printf '%s' "${1:-}" | tr '[:upper:]' '[:lower:]')"
|
|
13827
13838
|
local out=""
|
|
13828
|
-
_bac
|
|
13839
|
+
# _bac <AXIS> <text>. The sequence is per-axis and always 001 today because
|
|
13840
|
+
# each axis fires at most once; the NNN slot exists so a second criterion on
|
|
13841
|
+
# the same axis can be added later without renumbering the first.
|
|
13842
|
+
_bac() { out="${out}- AC-${1}-001: ${2}"$'\n'; }
|
|
13829
13843
|
|
|
13830
13844
|
# Persistence. The single most common churn report is "I submitted the form
|
|
13831
13845
|
# and nothing happened", so a stated store or form becomes an explicit
|
|
13832
13846
|
# survives-a-reload obligation rather than an implied one.
|
|
13833
13847
|
case "$t" in
|
|
13834
13848
|
*save*|*persist*|*store*|*databas*|*crud*|*todo*|*note*|*task*|*record*)
|
|
13835
|
-
_bac "Data the user creates survives a page reload and a server restart (it is written to a real store, not held in memory)." ;;
|
|
13849
|
+
_bac PERSIST "Data the user creates survives a page reload and a server restart (it is written to a real store, not held in memory)." ;;
|
|
13836
13850
|
esac
|
|
13837
13851
|
case "$t" in
|
|
13838
13852
|
*form*|*submit*|*signup*|*"sign up"*|*contact*|*upload*|*checkout*)
|
|
13839
|
-
_bac "Every form actually submits: the happy path writes real data and the user sees a confirmation, and a validation failure shows an inline error." ;;
|
|
13853
|
+
_bac FORM "Every form actually submits: the happy path writes real data and the user sees a confirmation, and a validation failure shows an inline error." ;;
|
|
13840
13854
|
esac
|
|
13841
13855
|
case "$t" in
|
|
13842
13856
|
*auth*|*login*|*"log in"*|*"sign in"*|*account*|*user*|*password*|*session*)
|
|
13843
|
-
_bac "Authentication works end to end: a real signup, a real login, and a protected route that returns 401 when logged out." ;;
|
|
13857
|
+
_bac AUTH "Authentication works end to end: a real signup, a real login, and a protected route that returns 401 when logged out." ;;
|
|
13844
13858
|
esac
|
|
13845
13859
|
case "$t" in
|
|
13846
13860
|
*api*|*endpoint*|*rest*|*graphql*|*backend*|*server*)
|
|
13847
|
-
_bac "Each endpoint returns real data with correct status codes, and is callable with curl without a browser." ;;
|
|
13861
|
+
_bac API "Each endpoint returns real data with correct status codes, and is callable with curl without a browser." ;;
|
|
13848
13862
|
esac
|
|
13849
13863
|
case "$t" in
|
|
13850
13864
|
*payment*|*stripe*|*billing*|*subscription*|*checkout*|*invoice*)
|
|
13851
|
-
_bac "The payment path is wired to the provider's test mode and a test transaction completes; no mocked charge stands in for the integration." ;;
|
|
13865
|
+
_bac PAY "The payment path is wired to the provider's test mode and a test transaction completes; no mocked charge stands in for the integration." ;;
|
|
13852
13866
|
esac
|
|
13853
13867
|
case "$t" in
|
|
13854
13868
|
*search*|*filter*|*sort*)
|
|
13855
|
-
_bac "Search or filtering queries the real dataset and returns different results for different inputs." ;;
|
|
13869
|
+
_bac SEARCH "Search or filtering queries the real dataset and returns different results for different inputs." ;;
|
|
13856
13870
|
esac
|
|
13857
13871
|
case "$t" in
|
|
13858
13872
|
*dashboard*|*chart*|*graph*|*analytic*|*report*|*metric*)
|
|
13859
|
-
_bac "Every figure shown traces to a real query. No hardcoded sample numbers." ;;
|
|
13873
|
+
_bac DATA "Every figure shown traces to a real query. No hardcoded sample numbers." ;;
|
|
13860
13874
|
esac
|
|
13861
13875
|
case "$t" in
|
|
13862
13876
|
*page*|*landing*|*site*|*website*|*ui*|*app*|*frontend*)
|
|
13863
|
-
_bac "The page renders with real content, no lorem ipsum, and no placeholder image where a real asset belongs." ;;
|
|
13877
|
+
_bac UI "The page renders with real content, no lorem ipsum, and no placeholder image where a real asset belongs." ;;
|
|
13864
13878
|
esac
|
|
13865
13879
|
printf '%s' "$out"
|
|
13866
13880
|
}
|
|
@@ -19484,6 +19498,11 @@ main() {
|
|
|
19484
19498
|
# Can an agent verify its own work here? Measured, not LLM-scored.
|
|
19485
19499
|
cmd_readiness "$@"
|
|
19486
19500
|
;;
|
|
19501
|
+
gates)
|
|
19502
|
+
# What blocks here vs only advises, and what promoting a gate would
|
|
19503
|
+
# have cost. Reports only; never promotes.
|
|
19504
|
+
cmd_gates "$@"
|
|
19505
|
+
;;
|
|
19487
19506
|
secure)
|
|
19488
19507
|
# Secure-by-default gate surface: inspect findings + manage waivers.
|
|
19489
19508
|
cmd_secure "$@"
|
|
@@ -31012,6 +31031,47 @@ cmd_verdict() {
|
|
|
31012
31031
|
# is no test command" tells you what to do, "readiness 62%" does not. Criteria
|
|
31013
31032
|
# that cannot be determined report UNKNOWN by name rather than counting as
|
|
31014
31033
|
# failures. See autonomy/lib/agent_readiness.py.
|
|
31034
|
+
cmd_gates() {
|
|
31035
|
+
local lib="${_LOKI_SCRIPT_DIR}/lib/gate_policy.py"
|
|
31036
|
+
if [ ! -f "$lib" ]; then
|
|
31037
|
+
echo "gate policy reporter is not installed at $lib" >&2
|
|
31038
|
+
return 2
|
|
31039
|
+
fi
|
|
31040
|
+
case "${1:-}" in
|
|
31041
|
+
--help|-h|help)
|
|
31042
|
+
echo -e "${BOLD}loki gates${NC} - what blocks here, and what only advises"
|
|
31043
|
+
echo ""
|
|
31044
|
+
echo "Usage: loki gates [.loki-dir] [--json]"
|
|
31045
|
+
echo ""
|
|
31046
|
+
echo "Ona's Veto Exec ships an audit-first ladder: start in audit mode,"
|
|
31047
|
+
echo "review what matched, then promote the confirmed rules to block. The"
|
|
31048
|
+
echo "middle step is the load-bearing one -- a policy you cannot safely"
|
|
31049
|
+
echo "turn on is a policy nobody turns on."
|
|
31050
|
+
echo ""
|
|
31051
|
+
echo "We had both ends and nothing between them: gates are advisory or"
|
|
31052
|
+
echo "blocking, three promotion knobs exist, and the failure ledger has"
|
|
31053
|
+
echo "counted per-gate hits all along. Nothing joined them, so deciding"
|
|
31054
|
+
echo "whether to promote a gate meant guessing."
|
|
31055
|
+
echo ""
|
|
31056
|
+
echo "For each gate this prints its mode, how many times it has fired,"
|
|
31057
|
+
echo "and -- for an advisory one -- the exact variable that promotes it."
|
|
31058
|
+
echo ""
|
|
31059
|
+
echo "Deterministic: reads two files and the environment. No model, no"
|
|
31060
|
+
echo "key, no spend. Counts come from"
|
|
31061
|
+
echo ".loki/quality/gate-failure-count.json; open it and count them"
|
|
31062
|
+
echo "yourself."
|
|
31063
|
+
echo ""
|
|
31064
|
+
echo "A gate with no ledger entry reports 'not measured', never 0: an"
|
|
31065
|
+
echo "absent measurement is not evidence a gate never fired."
|
|
31066
|
+
echo ""
|
|
31067
|
+
echo "This command NEVER promotes a gate. Promotion stays an explicit"
|
|
31068
|
+
echo "operator act via the named variable."
|
|
31069
|
+
return 0
|
|
31070
|
+
;;
|
|
31071
|
+
esac
|
|
31072
|
+
python3 "$lib" "$@"
|
|
31073
|
+
}
|
|
31074
|
+
|
|
31015
31075
|
cmd_readiness() {
|
|
31016
31076
|
local lib="${_LOKI_SCRIPT_DIR}/lib/agent_readiness.py"
|
|
31017
31077
|
if [ ! -f "$lib" ]; then
|
|
@@ -31022,7 +31082,7 @@ cmd_readiness() {
|
|
|
31022
31082
|
--help|-h|help)
|
|
31023
31083
|
echo -e "${BOLD}loki readiness${NC} - can an agent verify its own work in this repo?"
|
|
31024
31084
|
echo ""
|
|
31025
|
-
echo "Usage: loki readiness [path] [--json]"
|
|
31085
|
+
echo "Usage: loki readiness [path] [--json] [--fix]"
|
|
31026
31086
|
echo ""
|
|
31027
31087
|
echo "Measures whether this repo gives an agent a way to check itself: a"
|
|
31028
31088
|
echo "test command, a build, CI config, typed sources, a lockfile. Not"
|
|
@@ -31030,7 +31090,18 @@ cmd_readiness() {
|
|
|
31030
31090
|
echo ""
|
|
31031
31091
|
echo "Deterministic: no model, no key, no spend. Same commit, same answer."
|
|
31032
31092
|
echo "Criteria that cannot be determined report UNKNOWN rather than failing."
|
|
31033
|
-
echo "
|
|
31093
|
+
echo ""
|
|
31094
|
+
echo -e "${BOLD}--fix${NC} writes the missing files whose content can be derived"
|
|
31095
|
+
echo "honestly (README.md, AGENTS.md, .gitignore) as TODO stubs, then"
|
|
31096
|
+
echo "re-measures and reports what is true AFTER the change."
|
|
31097
|
+
echo ""
|
|
31098
|
+
echo "It deliberately REFUSES to generate a test command, a lockfile or a"
|
|
31099
|
+
echo "CI config. Guessing one writes a line that lies: an invented"
|
|
31100
|
+
echo "'npm test' in a repo with no runner fails forever, and this check"
|
|
31101
|
+
echo "would then report the criterion present for something that does not"
|
|
31102
|
+
echo "work. Those stay reported, never generated."
|
|
31103
|
+
echo ""
|
|
31104
|
+
echo "Without --fix it is read-only and never writes to the repo."
|
|
31034
31105
|
return 0
|
|
31035
31106
|
;;
|
|
31036
31107
|
esac
|
package/autonomy/notify.sh
CHANGED
|
@@ -86,6 +86,68 @@ _get_discord_color() {
|
|
|
86
86
|
# Slack Notifications
|
|
87
87
|
#===============================================================================
|
|
88
88
|
|
|
89
|
+
# Verdict fields for a Slack card, from the newest Evidence Receipt.
|
|
90
|
+
#
|
|
91
|
+
# Roadmap item 9: "put the receipt where review already happens". Slack
|
|
92
|
+
# notifications carried the event name and the project name and nothing else --
|
|
93
|
+
# grep for receipt/verdict/proof in this file returned 0. A team watching a
|
|
94
|
+
# channel saw "build finished" and had to go somewhere else to learn whether it
|
|
95
|
+
# was verified, which is the surface problem item 9 names: verification nobody
|
|
96
|
+
# sees does not build trust.
|
|
97
|
+
#
|
|
98
|
+
# Emits Slack `fields` entries (the payload already has that array), so this
|
|
99
|
+
# adds to the card rather than restructuring it.
|
|
100
|
+
#
|
|
101
|
+
# SILENT AND EMPTY ON ANY DOUBT. No receipt, unreadable JSON, no python -- emit
|
|
102
|
+
# nothing. A notification is a side channel; it must never fail a build, and it
|
|
103
|
+
# must never guess a verdict. An absent receipt yields no verdict field at all
|
|
104
|
+
# rather than a reassuring default.
|
|
105
|
+
_slack_verdict_fields() {
|
|
106
|
+
local loki_dir="${LOKI_DIR:-.loki}"
|
|
107
|
+
[ -d "$loki_dir/proofs" ] || return 0
|
|
108
|
+
command -v python3 >/dev/null 2>&1 || return 0
|
|
109
|
+
python3 - "$loki_dir" <<'PY' 2>/dev/null || true
|
|
110
|
+
import glob, json, os, sys
|
|
111
|
+
d = sys.argv[1]
|
|
112
|
+
paths = sorted(glob.glob(os.path.join(d, "proofs", "*", "proof.json")))
|
|
113
|
+
if not paths:
|
|
114
|
+
sys.exit(0)
|
|
115
|
+
try:
|
|
116
|
+
r = json.load(open(paths[-1]))
|
|
117
|
+
except (OSError, ValueError):
|
|
118
|
+
sys.exit(0)
|
|
119
|
+
|
|
120
|
+
def esc(s):
|
|
121
|
+
return json.dumps(str(s))[1:-1]
|
|
122
|
+
|
|
123
|
+
out = []
|
|
124
|
+
qg = r.get("quality_gates") or {}
|
|
125
|
+
gates = qg.get("gates") or []
|
|
126
|
+
if gates:
|
|
127
|
+
passed = sum(1 for g in gates if str(g.get("status", "")).startswith("pass"))
|
|
128
|
+
out.append(("Gates", f"{passed}/{len(gates)} passed"))
|
|
129
|
+
|
|
130
|
+
git = (r.get("facts") or {}).get("git") or {}
|
|
131
|
+
# base_sha empty means the receipt cannot be anchored, so it cannot be
|
|
132
|
+
# verified. Say that plainly rather than showing a diff count that implies it.
|
|
133
|
+
if not git.get("base_sha"):
|
|
134
|
+
out.append(("Verified", "no (unanchored receipt)"))
|
|
135
|
+
else:
|
|
136
|
+
diff = git.get("diff") or {}
|
|
137
|
+
if diff.get("count") is not None:
|
|
138
|
+
out.append(("Files changed", str(diff["count"])))
|
|
139
|
+
|
|
140
|
+
run_id = r.get("run_id")
|
|
141
|
+
if run_id:
|
|
142
|
+
out.append(("Receipt", esc(run_id)))
|
|
143
|
+
|
|
144
|
+
print(",".join(
|
|
145
|
+
'{"title": "%s", "value": "%s", "short": true}' % (esc(t), esc(v))
|
|
146
|
+
for t, v in out
|
|
147
|
+
))
|
|
148
|
+
PY
|
|
149
|
+
}
|
|
150
|
+
|
|
89
151
|
_notify_slack() {
|
|
90
152
|
local event="$1"
|
|
91
153
|
local title="$2"
|
|
@@ -106,6 +168,13 @@ _notify_slack() {
|
|
|
106
168
|
escaped_event="$(_json_escape "$event")"
|
|
107
169
|
escaped_project="$(_json_escape "$project")"
|
|
108
170
|
|
|
171
|
+
# Receipt-derived fields. Empty on any doubt, and the ${var:+,...} expansion
|
|
172
|
+
# below means an empty value adds no trailing comma -- a malformed payload
|
|
173
|
+
# would make Slack reject the whole card, so the failure mode of "no receipt"
|
|
174
|
+
# must be a card without verdict fields, never a card that does not send.
|
|
175
|
+
local verdict_fields
|
|
176
|
+
verdict_fields="$(_slack_verdict_fields 2>/dev/null || true)"
|
|
177
|
+
|
|
109
178
|
# Build Slack payload with attachment
|
|
110
179
|
local payload
|
|
111
180
|
payload=$(cat <<PAYLOAD
|
|
@@ -116,7 +185,7 @@ _notify_slack() {
|
|
|
116
185
|
"text": "$escaped_message",
|
|
117
186
|
"fields": [
|
|
118
187
|
{"title": "Event", "value": "$escaped_event", "short": true},
|
|
119
|
-
{"title": "Project", "value": "$escaped_project", "short": true}
|
|
188
|
+
{"title": "Project", "value": "$escaped_project", "short": true}${verdict_fields:+,$verdict_fields}
|
|
120
189
|
],
|
|
121
190
|
"footer": "Loki Mode",
|
|
122
191
|
"ts": $(date +%s)
|
|
@@ -30,11 +30,30 @@
|
|
|
30
30
|
# - Only redis and file are shipped. SQS, Pub/Sub, RabbitMQ, Kafka, etc. are
|
|
31
31
|
# BRING-YOUR-OWN: override queue.command in values.yaml with your own
|
|
32
32
|
# consumer. They are documented, not implemented here.
|
|
33
|
-
# - The redis backend is
|
|
34
|
-
#
|
|
35
|
-
#
|
|
36
|
-
#
|
|
37
|
-
#
|
|
33
|
+
# - The redis backend is AT-LEAST-ONCE by default. LMOVE (RPOPLPUSH pre-6.2)
|
|
34
|
+
# pops and records the item in <key>:processing atomically, and the ack
|
|
35
|
+
# (LREM) runs only after a zero-exit build. A crashed worker leaves its item
|
|
36
|
+
# in <key>:processing for re-drive -- the same shape as the file backend's
|
|
37
|
+
# processing/ directory.
|
|
38
|
+
# It was at-most-once (LPOP-then-run) until this was fixed; a crash after
|
|
39
|
+
# the pop lost the build with no record anywhere.
|
|
40
|
+
# A VISIBILITY TIMEOUT now exists: `queue-consumer.sh --reap` requeues
|
|
41
|
+
# in-flight items whose claim is older than LOKI_QUEUE_VISIBILITY_SEC
|
|
42
|
+
# (default 7200). Run it on a schedule (a CronJob) or by hand. It is a
|
|
43
|
+
# separate mode, not folded into the consume loop, because a consumer that
|
|
44
|
+
# reaped on every poll would race its peers on a busy queue.
|
|
45
|
+
# `--reap` works on BOTH backends. It was redis-only until this was fixed,
|
|
46
|
+
# which left the file backend with no crash recovery at all: a dead worker's
|
|
47
|
+
# item sat in processing/ forever and an unattended fleet stalled on it.
|
|
48
|
+
# A FRESH claim is never reaped -- requeuing a live item would build the
|
|
49
|
+
# user's work twice -- and an in-flight item with NO claim is treated as
|
|
50
|
+
# infinitely old, since that means the worker died between the LMOVE and the
|
|
51
|
+
# claim stamp, which is exactly the case that must be recoverable.
|
|
52
|
+
# LOKI_QUEUE_ACK=0 restores the legacy lossy path.
|
|
53
|
+
# The file backend's claim time is the item file's MTIME, stamped on claim,
|
|
54
|
+
# since a flat directory carries no claim hash. The limit is real and worth
|
|
55
|
+
# stating: a consumer that rewrote its own item file would reset that clock
|
|
56
|
+
# and look fresh forever. See file_reap.
|
|
38
57
|
# - The file backend's atomicity relies on `mv` being atomic within a single
|
|
39
58
|
# filesystem (true for a normal PVC). Two consumers racing the same pending
|
|
40
59
|
# dir is safe (mv either wins or fails-and-skips), but is not load-balanced.
|
|
@@ -55,6 +74,8 @@
|
|
|
55
74
|
# LOKI_QUEUE_POLL_SEC loop-mode empty-poll wait, seconds (default: 5)
|
|
56
75
|
# LOKI_QUEUE_BLOCK_SEC redis BLPOP block timeout, seconds (default: 5)
|
|
57
76
|
# LOKI_TERMINAL_EXIT run.sh terminal-failure exit code (default: 20)
|
|
77
|
+
# LOKI_QUEUE_VISIBILITY_SEC --reap requeues in-flight items older than this,
|
|
78
|
+
# both backends, seconds (default: 7200)
|
|
58
79
|
#===============================================================================
|
|
59
80
|
|
|
60
81
|
set -uo pipefail
|
|
@@ -176,17 +197,136 @@ redis_cli() {
|
|
|
176
197
|
redis-cli -u "$QUEUE_URL" "$@"
|
|
177
198
|
}
|
|
178
199
|
|
|
179
|
-
# Pop one item from the redis list
|
|
180
|
-
#
|
|
181
|
-
#
|
|
182
|
-
#
|
|
200
|
+
# Pop one item from the redis list, ATOMICALLY RECORDING IT AS IN-FLIGHT.
|
|
201
|
+
#
|
|
202
|
+
# THE DEFECT THIS FIXES, proven on a real redis 8.6.3 before it was written:
|
|
203
|
+
#
|
|
204
|
+
# RPUSH loki-builds '{"spec":"build a todo app"}' -> LLEN 1
|
|
205
|
+
# LPOP loki-builds -> worker holds the item
|
|
206
|
+
# <worker is OOM-killed / node evicted / kill -9>
|
|
207
|
+
# LLEN loki-builds -> 0
|
|
208
|
+
#
|
|
209
|
+
# The user's build is gone and NOTHING anywhere records that it existed. For a
|
|
210
|
+
# product whose entire thesis is "we hand you a receipt you can check", silently
|
|
211
|
+
# losing the work is the worst failure mode available: there is no receipt, no
|
|
212
|
+
# error, and no queue entry to retry.
|
|
213
|
+
#
|
|
214
|
+
# LMOVE (redis 6.2+) pops and pushes to a processing list in ONE atomic step, so
|
|
215
|
+
# a crash between the two is impossible. A dead worker leaves its item in
|
|
216
|
+
# <key>:processing where `loki queue reap` (or an operator) can re-drive it.
|
|
217
|
+
# RPOPLPUSH is the pre-6.2 equivalent and is tried automatically.
|
|
218
|
+
#
|
|
219
|
+
# Kept OPT-OUT rather than opt-in (LOKI_QUEUE_ACK=0 restores LPOP). At-least-once
|
|
220
|
+
# is the safer default: its failure mode is a duplicate build, which the
|
|
221
|
+
# idempotence work already handles, versus silent data loss. But an operator
|
|
222
|
+
# running a broker that already guarantees delivery should be able to turn it
|
|
223
|
+
# off rather than maintain two in-flight records.
|
|
224
|
+
#
|
|
225
|
+
# In loop mode with acking we use a short-poll LMOVE plus a sleep instead of
|
|
226
|
+
# BLMOVE, because BLMOVE blocks the connection and the stop-flag check has to
|
|
227
|
+
# stay responsive; the empty-poll backoff below already bounds the cost.
|
|
183
228
|
redis_pop() {
|
|
184
|
-
if [ "$
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
229
|
+
if [ "${LOKI_QUEUE_ACK:-1}" = "0" ]; then
|
|
230
|
+
# Legacy at-most-once path, explicitly requested.
|
|
231
|
+
if [ "$ONESHOT" = "1" ]; then
|
|
232
|
+
redis-cli -u "$QUEUE_URL" --no-raw LPOP "$QUEUE_KEY" 2>/dev/null | _redis_unquote
|
|
233
|
+
else
|
|
234
|
+
redis-cli -u "$QUEUE_URL" BLPOP "$QUEUE_KEY" "$BLOCK_SEC" 2>/dev/null | sed -n '2p'
|
|
235
|
+
fi
|
|
236
|
+
return 0
|
|
237
|
+
fi
|
|
238
|
+
|
|
239
|
+
# RAW output, not --no-raw. `--no-raw` escapes the INNER quotes of a JSON
|
|
240
|
+
# payload ({"spec":"x"} comes back as "{\"spec\":\"x\"}"), and
|
|
241
|
+
# _redis_unquote only strips the outer pair -- so the item reaching
|
|
242
|
+
# extract_spec was mangled AND the string handed to LREM no longer matched
|
|
243
|
+
# what redis stored, silently acking nothing. Both were caught by running
|
|
244
|
+
# against a real redis 8.6.3; neither is visible in a mock.
|
|
245
|
+
local out
|
|
246
|
+
out="$(redis-cli -u "$QUEUE_URL" LMOVE "$QUEUE_KEY" "${QUEUE_KEY}:processing" LEFT RIGHT 2>/dev/null)"
|
|
247
|
+
# Pre-6.2 servers reject LMOVE as an unknown command (stderr, empty stdout),
|
|
248
|
+
# so fall back rather than reporting an empty queue.
|
|
249
|
+
if [ -z "$out" ]; then
|
|
250
|
+
out="$(redis-cli -u "$QUEUE_URL" RPOPLPUSH "$QUEUE_KEY" "${QUEUE_KEY}:processing" 2>/dev/null)"
|
|
251
|
+
fi
|
|
252
|
+
# Record WHEN this item was claimed. A bare list carries no time, so nothing
|
|
253
|
+
# could tell a stale entry (dead worker) from a running one (long build) --
|
|
254
|
+
# and requeuing a live item duplicates the user's build. The claim hash is
|
|
255
|
+
# what makes the reaper below safe to run at all.
|
|
256
|
+
#
|
|
257
|
+
# Written AFTER the move, so a crash between them leaves the item in-flight
|
|
258
|
+
# with no claim time. The reaper treats a missing claim as INFINITELY OLD
|
|
259
|
+
# rather than infinitely young: an item nothing is tracking is exactly the
|
|
260
|
+
# case that must be recoverable.
|
|
261
|
+
if [ -n "$out" ]; then
|
|
262
|
+
redis-cli -u "$QUEUE_URL" HSET "${QUEUE_KEY}:claims" "$out" "$(date +%s)" >/dev/null 2>&1 || true
|
|
189
263
|
fi
|
|
264
|
+
printf '%s' "$out"
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
# Remove a completed item from the in-flight list. Called ONLY after the build
|
|
268
|
+
# finished; until then the item stays recoverable.
|
|
269
|
+
#
|
|
270
|
+
# LREM with count 0 removes every equal element. Two identical specs queued twice
|
|
271
|
+
# would both clear on the first ack -- accepted deliberately: the alternative is
|
|
272
|
+
# a per-item token, which needs a producer change, and the failure mode here is
|
|
273
|
+
# one duplicate re-drive rather than a lost build.
|
|
274
|
+
redis_ack() {
|
|
275
|
+
[ "${LOKI_QUEUE_ACK:-1}" = "0" ] && return 0
|
|
276
|
+
[ -n "${1:-}" ] || return 0
|
|
277
|
+
redis-cli -u "$QUEUE_URL" LREM "${QUEUE_KEY}:processing" 0 "$1" >/dev/null 2>&1 || true
|
|
278
|
+
# Drop the claim too, or the hash grows without bound and every completed
|
|
279
|
+
# item looks like a candidate for reaping forever.
|
|
280
|
+
redis-cli -u "$QUEUE_URL" HDEL "${QUEUE_KEY}:claims" "$1" >/dev/null 2>&1 || true
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
# Requeue in-flight items whose claim is older than the visibility timeout.
|
|
284
|
+
#
|
|
285
|
+
# THE GAP THIS CLOSES. LMOVE made a crashed worker's item RECOVERABLE, but
|
|
286
|
+
# nothing recovered it: re-drive was an operator action, so an unattended fleet
|
|
287
|
+
# still stalled on every dead worker. That was documented rather than claimed;
|
|
288
|
+
# this closes it.
|
|
289
|
+
#
|
|
290
|
+
# SAFETY IS THE WHOLE DESIGN. Requeuing an item a worker is still building
|
|
291
|
+
# duplicates the user's build, so the timeout must exceed the longest legitimate
|
|
292
|
+
# build. Default 2h (LOKI_MAX_DURATION territory), tunable.
|
|
293
|
+
#
|
|
294
|
+
# An item with NO claim entry is treated as INFINITELY OLD, not young: it means
|
|
295
|
+
# the worker died between the LMOVE and the HSET, which is precisely the case
|
|
296
|
+
# that must be recoverable. Erring the other way would strand exactly the items
|
|
297
|
+
# this exists to rescue.
|
|
298
|
+
#
|
|
299
|
+
# Reports what it did on stdout. Requeues nothing when the timeout has not
|
|
300
|
+
# elapsed, and says so rather than printing a silent zero.
|
|
301
|
+
redis_reap() {
|
|
302
|
+
[ "${LOKI_QUEUE_ACK:-1}" = "0" ] && { echo "reap: acking disabled, nothing to reap"; return 0; }
|
|
303
|
+
local timeout now items requeued=0 kept=0
|
|
304
|
+
timeout="${LOKI_QUEUE_VISIBILITY_SEC:-7200}"
|
|
305
|
+
now="$(date +%s)"
|
|
306
|
+
items="$(redis-cli -u "$QUEUE_URL" LRANGE "${QUEUE_KEY}:processing" 0 -1 2>/dev/null)"
|
|
307
|
+
[ -n "$items" ] || { echo "reap: nothing in flight"; return 0; }
|
|
308
|
+
while IFS= read -r it; do
|
|
309
|
+
[ -n "$it" ] || continue
|
|
310
|
+
local claimed age
|
|
311
|
+
claimed="$(redis-cli -u "$QUEUE_URL" HGET "${QUEUE_KEY}:claims" "$it" 2>/dev/null)"
|
|
312
|
+
if [ -z "$claimed" ]; then
|
|
313
|
+
age="$((timeout + 1))" # unclaimed == infinitely old, see above
|
|
314
|
+
else
|
|
315
|
+
age="$((now - claimed))"
|
|
316
|
+
fi
|
|
317
|
+
if [ "$age" -gt "$timeout" ]; then
|
|
318
|
+
# LREM then RPUSH, not LMOVE: the item may appear more than once and
|
|
319
|
+
# LREM 0 clears every copy, so the queue cannot gain duplicates.
|
|
320
|
+
redis-cli -u "$QUEUE_URL" LREM "${QUEUE_KEY}:processing" 0 "$it" >/dev/null 2>&1 || true
|
|
321
|
+
redis-cli -u "$QUEUE_URL" HDEL "${QUEUE_KEY}:claims" "$it" >/dev/null 2>&1 || true
|
|
322
|
+
redis-cli -u "$QUEUE_URL" RPUSH "$QUEUE_KEY" "$it" >/dev/null 2>&1 || true
|
|
323
|
+
requeued=$((requeued + 1))
|
|
324
|
+
log "reaped stale item (age ${age}s > ${timeout}s), requeued"
|
|
325
|
+
else
|
|
326
|
+
kept=$((kept + 1))
|
|
327
|
+
fi
|
|
328
|
+
done <<< "$items"
|
|
329
|
+
echo "reap: requeued $requeued, left $kept in flight (timeout ${timeout}s)"
|
|
190
330
|
}
|
|
191
331
|
|
|
192
332
|
# --no-raw LPOP wraps the value in quotes; strip a single surrounding pair and
|
|
@@ -207,10 +347,21 @@ redis_consume_one() {
|
|
|
207
347
|
if [ -z "$item" ]; then
|
|
208
348
|
return 100 # sentinel: queue empty
|
|
209
349
|
fi
|
|
210
|
-
local spec
|
|
350
|
+
local spec rc
|
|
211
351
|
spec="$(extract_spec "$item")"
|
|
212
352
|
run_build "$spec"
|
|
213
|
-
|
|
353
|
+
rc=$?
|
|
354
|
+
# ACK ONLY ON SUCCESS, and the order is load-bearing: acking before the
|
|
355
|
+
# build would reintroduce the exact data loss this exists to prevent, and
|
|
356
|
+
# acking a FAILED build would discard work that a re-drive could complete.
|
|
357
|
+
# A failed item stays in <key>:processing so `loki queue reap` can requeue
|
|
358
|
+
# it -- the same shape as the file backend leaving it in processing/.
|
|
359
|
+
if [ "$rc" -eq 0 ]; then
|
|
360
|
+
redis_ack "$item"
|
|
361
|
+
else
|
|
362
|
+
log "item left in ${QUEUE_KEY}:processing for re-drive (exit $rc)"
|
|
363
|
+
fi
|
|
364
|
+
return "$rc"
|
|
214
365
|
}
|
|
215
366
|
|
|
216
367
|
# =============================================================================
|
|
@@ -241,6 +392,18 @@ file_claim_oldest() {
|
|
|
241
392
|
base="$(basename "$f")"
|
|
242
393
|
dest="$QUEUE_DIR/processing/$base"
|
|
243
394
|
if mv "$f" "$dest" 2>/dev/null; then
|
|
395
|
+
# Stamp the CLAIM TIME. This is the file-backend analogue of the
|
|
396
|
+
# `HSET <key>:claims` that redis_pop does right after its LMOVE, and
|
|
397
|
+
# file_reap below depends on it being here.
|
|
398
|
+
#
|
|
399
|
+
# Load-bearing: `mv` is a rename(2), which PRESERVES mtime. An item
|
|
400
|
+
# that sat in pending/ behind a backlog for longer than the
|
|
401
|
+
# visibility timeout would arrive in processing/ already looking
|
|
402
|
+
# stale, and the reaper would requeue it while a worker was actively
|
|
403
|
+
# building it -- the exact duplicate build the timeout exists to
|
|
404
|
+
# prevent, firing on precisely the busy queues that need a reaper.
|
|
405
|
+
# Verified on darwin: mv of a file stamped 2020 kept the 2020 mtime.
|
|
406
|
+
touch "$dest" 2>/dev/null || true
|
|
244
407
|
printf '%s' "$dest"
|
|
245
408
|
return 0
|
|
246
409
|
fi
|
|
@@ -272,13 +435,110 @@ file_consume_one() {
|
|
|
272
435
|
mv "$claimed" "$QUEUE_DIR/failed/$base" 2>/dev/null || log "WARN: could not move $base to failed/"
|
|
273
436
|
log "item $base TERMINAL-FAILED (exit $rc); moved to failed/, not acked"
|
|
274
437
|
else
|
|
275
|
-
# Transient crash: leave it in processing/ for
|
|
276
|
-
#
|
|
438
|
+
# Transient crash: leave it in processing/ for re-drive. We do not
|
|
439
|
+
# requeue INLINE (no retry counter in a flat dir, so an instant retry
|
|
440
|
+
# could hot-loop a poison item). `--reap` picks it up once the
|
|
441
|
+
# visibility timeout has elapsed; see file_reap.
|
|
277
442
|
log "item $base crashed (exit $rc); left in processing/ for re-drive"
|
|
278
443
|
fi
|
|
279
444
|
return "$rc"
|
|
280
445
|
}
|
|
281
446
|
|
|
447
|
+
# Requeue in-flight items whose claim is older than the visibility timeout.
|
|
448
|
+
#
|
|
449
|
+
# THE GAP THIS CLOSES. The redis backend got a reaper; the file backend did not,
|
|
450
|
+
# so an item left in processing/ by a dead worker stayed there FOREVER and an
|
|
451
|
+
# unattended fleet on the file backend stalled on every crash. `--reap` answered
|
|
452
|
+
# "redis-only" and changed nothing.
|
|
453
|
+
#
|
|
454
|
+
# CLAIM TIME IS THE FILE MTIME. The file backend has no claim hash, so the mtime
|
|
455
|
+
# stamped by file_claim_oldest stands in for one.
|
|
456
|
+
#
|
|
457
|
+
# THE LIMIT, stated plainly: mtime is whatever last WROTE the file. A build that
|
|
458
|
+
# rewrites its own item file resets the clock and would look fresh forever, so
|
|
459
|
+
# it would never be reaped. Nothing here writes back to a claimed item, but a
|
|
460
|
+
# custom consumer that did would silently opt out of recovery. The redis claim
|
|
461
|
+
# hash does not have this weakness; the file backend accepts it rather than
|
|
462
|
+
# maintain a second sidecar file whose own staleness would need reaping too.
|
|
463
|
+
#
|
|
464
|
+
# There is also no retry counter (a flat dir carries none), so a poison item can
|
|
465
|
+
# be reaped repeatedly. That is the same shape the backend already documents for
|
|
466
|
+
# manual re-drive, not a new failure introduced here.
|
|
467
|
+
#
|
|
468
|
+
# SAFETY IS THE WHOLE DESIGN, mirroring redis_reap:
|
|
469
|
+
# - a FRESH item is NEVER requeued; a live build requeued is the user's work
|
|
470
|
+
# built twice, which is worse than the stranding this fixes
|
|
471
|
+
# - it NEVER deletes an item -- a name collision in pending/ leaves the item
|
|
472
|
+
# in processing/ and says so, rather than clobbering the pending copy
|
|
473
|
+
# - it reports counts on stdout; per-item detail goes to stderr via log()
|
|
474
|
+
file_reap() {
|
|
475
|
+
local timeout now requeued=0 kept=0 seen=0 f base mt age
|
|
476
|
+
timeout="${LOKI_QUEUE_VISIBILITY_SEC:-7200}"
|
|
477
|
+
now="$(date +%s)"
|
|
478
|
+
if [ ! -d "$QUEUE_DIR/processing" ]; then
|
|
479
|
+
echo "reap: nothing in flight"
|
|
480
|
+
return 0
|
|
481
|
+
fi
|
|
482
|
+
mkdir -p "$QUEUE_DIR/pending" 2>/dev/null || true
|
|
483
|
+
# Glob, not `ls`: a filename with a space or newline survives it.
|
|
484
|
+
for f in "$QUEUE_DIR"/processing/*; do
|
|
485
|
+
[ -e "$f" ] || continue # unmatched glob stays literal
|
|
486
|
+
[ -f "$f" ] || continue
|
|
487
|
+
seen=$((seen + 1))
|
|
488
|
+
base="$(basename "$f")"
|
|
489
|
+
# GNU FIRST, and the order is load-bearing. `stat -f` means
|
|
490
|
+
# --file-system on GNU coreutils and EXITS 0, printing a mount point
|
|
491
|
+
# rather than an mtime -- so a `-f`-first probe never falls through on
|
|
492
|
+
# Linux and hands back garbage. BSD `stat -c` has no such ambiguity: it
|
|
493
|
+
# is an illegal option and exits 1 (verified on darwin), so `-f %m` is
|
|
494
|
+
# reached only where it means what we want.
|
|
495
|
+
# `find -printf` and `-mmin` are GNU-only / minute-granular, and the
|
|
496
|
+
# fresh-vs-stale distinction needs seconds.
|
|
497
|
+
mt="$(stat -c %Y "$f" 2>/dev/null || stat -f %m "$f" 2>/dev/null)"
|
|
498
|
+
case "$mt" in
|
|
499
|
+
''|*[!0-9]*)
|
|
500
|
+
# Unreadable or non-numeric claim time. KEEP the item, and note
|
|
501
|
+
# the ASYMMETRY with redis_reap deliberately: a missing redis
|
|
502
|
+
# claim is DIAGNOSTIC (the worker died between LMOVE and HSET),
|
|
503
|
+
# but a file that exists ALWAYS has an mtime, so failing to read
|
|
504
|
+
# one means our stat invocation is wrong -- a portability bug,
|
|
505
|
+
# not a dead worker.
|
|
506
|
+
# MEASURED, not assumed: with a `stat` that prints a non-number
|
|
507
|
+
# and exits 0, the un-guarded arithmetic aborts the reaper under
|
|
508
|
+
# `set -uo pipefail` (rc=1, nothing requeued). So the observed
|
|
509
|
+
# failure is a DEAD REAPER on the affected platform, not a
|
|
510
|
+
# duplicate build. Keeping the item is correct either way, and
|
|
511
|
+
# the WARN names the reason instead of exiting silently.
|
|
512
|
+
kept=$((kept + 1))
|
|
513
|
+
log "WARN: cannot read mtime for $base (got '$mt'); left in processing/ rather than guess a claim time"
|
|
514
|
+
continue
|
|
515
|
+
;;
|
|
516
|
+
esac
|
|
517
|
+
age="$((now - mt))"
|
|
518
|
+
if [ "$age" -le "$timeout" ]; then
|
|
519
|
+
kept=$((kept + 1))
|
|
520
|
+
continue
|
|
521
|
+
fi
|
|
522
|
+
if [ -e "$QUEUE_DIR/pending/$base" ]; then
|
|
523
|
+
kept=$((kept + 1))
|
|
524
|
+
log "stale item $base NOT requeued: pending/$base already exists (never overwrite an item)"
|
|
525
|
+
continue
|
|
526
|
+
fi
|
|
527
|
+
if mv "$f" "$QUEUE_DIR/pending/$base" 2>/dev/null; then
|
|
528
|
+
requeued=$((requeued + 1))
|
|
529
|
+
log "reaped stale item $base (age ${age}s > ${timeout}s), requeued"
|
|
530
|
+
else
|
|
531
|
+
kept=$((kept + 1))
|
|
532
|
+
log "WARN: could not requeue $base; left in processing/"
|
|
533
|
+
fi
|
|
534
|
+
done
|
|
535
|
+
if [ "$seen" -eq 0 ]; then
|
|
536
|
+
echo "reap: nothing in flight"
|
|
537
|
+
return 0
|
|
538
|
+
fi
|
|
539
|
+
echo "reap: requeued $requeued, left $kept in flight (timeout ${timeout}s)"
|
|
540
|
+
}
|
|
541
|
+
|
|
282
542
|
# =============================================================================
|
|
283
543
|
# Driver
|
|
284
544
|
# =============================================================================
|
|
@@ -307,6 +567,18 @@ main() {
|
|
|
307
567
|
return 2
|
|
308
568
|
fi
|
|
309
569
|
|
|
570
|
+
# --reap requeues stale in-flight items and exits. Kept a SEPARATE mode
|
|
571
|
+
# rather than folded into the consume loop: a reaper wants its own cadence
|
|
572
|
+
# (a CronJob, or an operator hand-running it), and a consumer that reaped on
|
|
573
|
+
# every poll would race its own peers on a busy queue.
|
|
574
|
+
if [ "${1:-}" = "--reap" ] || [ "${LOKI_QUEUE_REAP:-0}" = "1" ]; then
|
|
575
|
+
case "$backend" in
|
|
576
|
+
redis) redis_reap ;;
|
|
577
|
+
file) file_init_dirs && file_reap ;;
|
|
578
|
+
esac
|
|
579
|
+
return $?
|
|
580
|
+
fi
|
|
581
|
+
|
|
310
582
|
if [ "$ONESHOT" = "1" ]; then
|
|
311
583
|
log "mode=oneshot backend=$backend (serverless: process one item then exit)"
|
|
312
584
|
consume_one "$backend"
|
package/autonomy/run.sh
CHANGED
|
@@ -24889,7 +24889,17 @@ check_human_intervention() {
|
|
|
24889
24889
|
# Security: Check it's a regular file (not symlink) to prevent symlink attacks
|
|
24890
24890
|
if [ -f "$loki_dir/HUMAN_INPUT.md" ] && [ ! -L "$loki_dir/HUMAN_INPUT.md" ]; then
|
|
24891
24891
|
# Security: Prompt injection disabled by default for enterprise security
|
|
24892
|
-
|
|
24892
|
+
# Accepts "1" AND "true". It used to accept only "true", while
|
|
24893
|
+
# `loki steer` prints "Enable it: export LOKI_PROMPT_INJECTION=1" and
|
|
24894
|
+
# then "The next iteration will read and apply it." A user who followed
|
|
24895
|
+
# that instruction exactly had their steering note moved to
|
|
24896
|
+
# logs/human-input-REJECTED-*.md and never read -- the CLI reported
|
|
24897
|
+
# success, the runner silently discarded it, and the only warning went
|
|
24898
|
+
# to the runner log rather than to the person who typed the command.
|
|
24899
|
+
#
|
|
24900
|
+
# Both forms is the convention here (LOKI_GATE_SEMANTIC_TESTS_BLOCK and
|
|
24901
|
+
# LOKI_DURABLE_STATE each test for both); this flag was the outlier.
|
|
24902
|
+
if [ "${LOKI_PROMPT_INJECTION:-false}" != "true" ] && [ "${LOKI_PROMPT_INJECTION:-false}" != "1" ]; then
|
|
24893
24903
|
log_warn "HUMAN_INPUT.md detected but prompt injection is DISABLED"
|
|
24894
24904
|
log_warn "To enable, set LOKI_PROMPT_INJECTION=true (only in trusted environments)"
|
|
24895
24905
|
# Move to rejected instead of processed
|
package/completions/_loki
CHANGED
|
@@ -120,6 +120,7 @@ function _loki {
|
|
|
120
120
|
function _loki_commands {
|
|
121
121
|
local -a commands
|
|
122
122
|
commands=(
|
|
123
|
+
'gates:What blocks here vs only advises, and the promotion knob'
|
|
123
124
|
'start:Start Loki Mode'
|
|
124
125
|
'quick:Quick single-task mode'
|
|
125
126
|
'quickstart:Guided first build from your idea'
|