loki-mode 9.16.0 → 9.17.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/autonomy/loki CHANGED
@@ -1040,7 +1040,7 @@ show_help() {
1040
1040
  echo " agent analyze api assets audit bench checkpoint (cp) ci cleanup"
1041
1041
  echo " cluster cockpit code compliance completions compound config context (ctx)"
1042
1042
  echo " cost council crash dashboard demo deploy docker docs doctor dogfood"
1043
- echo " enterprise estimate explain export failover github grill heal help"
1043
+ echo " enterprise estimate explain export failover gates github grill heal help"
1044
1044
  echo " import init intent"
1045
1045
  echo " issue kpis logs magic mcp memory metrics migrate modernize monitor"
1046
1046
  echo " next notify onboard open optimize otel outcomes own (handoff) pause plan preview"
@@ -13821,46 +13821,60 @@ set_ttfv_lightweight_profile() {
13821
13821
  # honest about being shallow -- it turns stated nouns into checkable lines and
13822
13822
  # claims nothing about intent it cannot see. Anything cleverer belongs in the
13823
13823
  # spec-interrogation grill, which already runs after this and does call a model.
13824
+ #
13825
+ # STABLE IDs. Each criterion is emitted as "AC-<AXIS>-NNN: <text>" rather than a
13826
+ # bare bullet. An anonymous bullet cannot be referred to: a receipt can say "3 of
13827
+ # 8 gates passed" but never "AC-PERSIST-001 is satisfied by this test", drift
13828
+ # cannot be tracked per criterion, and two runs of the same spec produce lists
13829
+ # nothing can diff. The ID is what turns a criterion into a citable claim, which
13830
+ # is the whole point of shipping a receipt someone can check.
13831
+ #
13832
+ # The axis is derived from WHICH obligation fired, not from the criterion's
13833
+ # position, so IDs are stable across runs: adding a payment criterion never
13834
+ # renumbers the persistence one. Same reason we do not use a running counter.
13824
13835
  _brief_acceptance_criteria() {
13825
13836
  local t
13826
13837
  t="$(printf '%s' "${1:-}" | tr '[:upper:]' '[:lower:]')"
13827
13838
  local out=""
13828
- _bac() { out="${out}- ${1}"$'\n'; }
13839
+ # _bac <AXIS> <text>. The sequence is per-axis and always 001 today because
13840
+ # each axis fires at most once; the NNN slot exists so a second criterion on
13841
+ # the same axis can be added later without renumbering the first.
13842
+ _bac() { out="${out}- AC-${1}-001: ${2}"$'\n'; }
13829
13843
 
13830
13844
  # Persistence. The single most common churn report is "I submitted the form
13831
13845
  # and nothing happened", so a stated store or form becomes an explicit
13832
13846
  # survives-a-reload obligation rather than an implied one.
13833
13847
  case "$t" in
13834
13848
  *save*|*persist*|*store*|*databas*|*crud*|*todo*|*note*|*task*|*record*)
13835
- _bac "Data the user creates survives a page reload and a server restart (it is written to a real store, not held in memory)." ;;
13849
+ _bac PERSIST "Data the user creates survives a page reload and a server restart (it is written to a real store, not held in memory)." ;;
13836
13850
  esac
13837
13851
  case "$t" in
13838
13852
  *form*|*submit*|*signup*|*"sign up"*|*contact*|*upload*|*checkout*)
13839
- _bac "Every form actually submits: the happy path writes real data and the user sees a confirmation, and a validation failure shows an inline error." ;;
13853
+ _bac FORM "Every form actually submits: the happy path writes real data and the user sees a confirmation, and a validation failure shows an inline error." ;;
13840
13854
  esac
13841
13855
  case "$t" in
13842
13856
  *auth*|*login*|*"log in"*|*"sign in"*|*account*|*user*|*password*|*session*)
13843
- _bac "Authentication works end to end: a real signup, a real login, and a protected route that returns 401 when logged out." ;;
13857
+ _bac AUTH "Authentication works end to end: a real signup, a real login, and a protected route that returns 401 when logged out." ;;
13844
13858
  esac
13845
13859
  case "$t" in
13846
13860
  *api*|*endpoint*|*rest*|*graphql*|*backend*|*server*)
13847
- _bac "Each endpoint returns real data with correct status codes, and is callable with curl without a browser." ;;
13861
+ _bac API "Each endpoint returns real data with correct status codes, and is callable with curl without a browser." ;;
13848
13862
  esac
13849
13863
  case "$t" in
13850
13864
  *payment*|*stripe*|*billing*|*subscription*|*checkout*|*invoice*)
13851
- _bac "The payment path is wired to the provider's test mode and a test transaction completes; no mocked charge stands in for the integration." ;;
13865
+ _bac PAY "The payment path is wired to the provider's test mode and a test transaction completes; no mocked charge stands in for the integration." ;;
13852
13866
  esac
13853
13867
  case "$t" in
13854
13868
  *search*|*filter*|*sort*)
13855
- _bac "Search or filtering queries the real dataset and returns different results for different inputs." ;;
13869
+ _bac SEARCH "Search or filtering queries the real dataset and returns different results for different inputs." ;;
13856
13870
  esac
13857
13871
  case "$t" in
13858
13872
  *dashboard*|*chart*|*graph*|*analytic*|*report*|*metric*)
13859
- _bac "Every figure shown traces to a real query. No hardcoded sample numbers." ;;
13873
+ _bac DATA "Every figure shown traces to a real query. No hardcoded sample numbers." ;;
13860
13874
  esac
13861
13875
  case "$t" in
13862
13876
  *page*|*landing*|*site*|*website*|*ui*|*app*|*frontend*)
13863
- _bac "The page renders with real content, no lorem ipsum, and no placeholder image where a real asset belongs." ;;
13877
+ _bac UI "The page renders with real content, no lorem ipsum, and no placeholder image where a real asset belongs." ;;
13864
13878
  esac
13865
13879
  printf '%s' "$out"
13866
13880
  }
@@ -19484,6 +19498,11 @@ main() {
19484
19498
  # Can an agent verify its own work here? Measured, not LLM-scored.
19485
19499
  cmd_readiness "$@"
19486
19500
  ;;
19501
+ gates)
19502
+ # What blocks here vs only advises, and what promoting a gate would
19503
+ # have cost. Reports only; never promotes.
19504
+ cmd_gates "$@"
19505
+ ;;
19487
19506
  secure)
19488
19507
  # Secure-by-default gate surface: inspect findings + manage waivers.
19489
19508
  cmd_secure "$@"
@@ -31012,6 +31031,47 @@ cmd_verdict() {
31012
31031
  # is no test command" tells you what to do, "readiness 62%" does not. Criteria
31013
31032
  # that cannot be determined report UNKNOWN by name rather than counting as
31014
31033
  # failures. See autonomy/lib/agent_readiness.py.
31034
+ cmd_gates() {
31035
+ local lib="${_LOKI_SCRIPT_DIR}/lib/gate_policy.py"
31036
+ if [ ! -f "$lib" ]; then
31037
+ echo "gate policy reporter is not installed at $lib" >&2
31038
+ return 2
31039
+ fi
31040
+ case "${1:-}" in
31041
+ --help|-h|help)
31042
+ echo -e "${BOLD}loki gates${NC} - what blocks here, and what only advises"
31043
+ echo ""
31044
+ echo "Usage: loki gates [.loki-dir] [--json]"
31045
+ echo ""
31046
+ echo "Ona's Veto Exec ships an audit-first ladder: start in audit mode,"
31047
+ echo "review what matched, then promote the confirmed rules to block. The"
31048
+ echo "middle step is the load-bearing one -- a policy you cannot safely"
31049
+ echo "turn on is a policy nobody turns on."
31050
+ echo ""
31051
+ echo "We had both ends and nothing between them: gates are advisory or"
31052
+ echo "blocking, three promotion knobs exist, and the failure ledger has"
31053
+ echo "counted per-gate hits all along. Nothing joined them, so deciding"
31054
+ echo "whether to promote a gate meant guessing."
31055
+ echo ""
31056
+ echo "For each gate this prints its mode, how many times it has fired,"
31057
+ echo "and -- for an advisory one -- the exact variable that promotes it."
31058
+ echo ""
31059
+ echo "Deterministic: reads two files and the environment. No model, no"
31060
+ echo "key, no spend. Counts come from"
31061
+ echo ".loki/quality/gate-failure-count.json; open it and count them"
31062
+ echo "yourself."
31063
+ echo ""
31064
+ echo "A gate with no ledger entry reports 'not measured', never 0: an"
31065
+ echo "absent measurement is not evidence a gate never fired."
31066
+ echo ""
31067
+ echo "This command NEVER promotes a gate. Promotion stays an explicit"
31068
+ echo "operator act via the named variable."
31069
+ return 0
31070
+ ;;
31071
+ esac
31072
+ python3 "$lib" "$@"
31073
+ }
31074
+
31015
31075
  cmd_readiness() {
31016
31076
  local lib="${_LOKI_SCRIPT_DIR}/lib/agent_readiness.py"
31017
31077
  if [ ! -f "$lib" ]; then
@@ -31022,7 +31082,7 @@ cmd_readiness() {
31022
31082
  --help|-h|help)
31023
31083
  echo -e "${BOLD}loki readiness${NC} - can an agent verify its own work in this repo?"
31024
31084
  echo ""
31025
- echo "Usage: loki readiness [path] [--json]"
31085
+ echo "Usage: loki readiness [path] [--json] [--fix]"
31026
31086
  echo ""
31027
31087
  echo "Measures whether this repo gives an agent a way to check itself: a"
31028
31088
  echo "test command, a build, CI config, typed sources, a lockfile. Not"
@@ -31030,7 +31090,18 @@ cmd_readiness() {
31030
31090
  echo ""
31031
31091
  echo "Deterministic: no model, no key, no spend. Same commit, same answer."
31032
31092
  echo "Criteria that cannot be determined report UNKNOWN rather than failing."
31033
- echo "Read-only: it never writes to the repo it analyses."
31093
+ echo ""
31094
+ echo -e "${BOLD}--fix${NC} writes the missing files whose content can be derived"
31095
+ echo "honestly (README.md, AGENTS.md, .gitignore) as TODO stubs, then"
31096
+ echo "re-measures and reports what is true AFTER the change."
31097
+ echo ""
31098
+ echo "It deliberately REFUSES to generate a test command, a lockfile or a"
31099
+ echo "CI config. Guessing one writes a line that lies: an invented"
31100
+ echo "'npm test' in a repo with no runner fails forever, and this check"
31101
+ echo "would then report the criterion present for something that does not"
31102
+ echo "work. Those stay reported, never generated."
31103
+ echo ""
31104
+ echo "Without --fix it is read-only and never writes to the repo."
31034
31105
  return 0
31035
31106
  ;;
31036
31107
  esac
@@ -86,6 +86,68 @@ _get_discord_color() {
86
86
  # Slack Notifications
87
87
  #===============================================================================
88
88
 
89
+ # Verdict fields for a Slack card, from the newest Evidence Receipt.
90
+ #
91
+ # Roadmap item 9: "put the receipt where review already happens". Slack
92
+ # notifications carried the event name and the project name and nothing else --
93
+ # grep for receipt/verdict/proof in this file returned 0. A team watching a
94
+ # channel saw "build finished" and had to go somewhere else to learn whether it
95
+ # was verified, which is the surface problem item 9 names: verification nobody
96
+ # sees does not build trust.
97
+ #
98
+ # Emits Slack `fields` entries (the payload already has that array), so this
99
+ # adds to the card rather than restructuring it.
100
+ #
101
+ # SILENT AND EMPTY ON ANY DOUBT. No receipt, unreadable JSON, no python -- emit
102
+ # nothing. A notification is a side channel; it must never fail a build, and it
103
+ # must never guess a verdict. An absent receipt yields no verdict field at all
104
+ # rather than a reassuring default.
105
+ _slack_verdict_fields() {
106
+ local loki_dir="${LOKI_DIR:-.loki}"
107
+ [ -d "$loki_dir/proofs" ] || return 0
108
+ command -v python3 >/dev/null 2>&1 || return 0
109
+ python3 - "$loki_dir" <<'PY' 2>/dev/null || true
110
+ import glob, json, os, sys
111
+ d = sys.argv[1]
112
+ paths = sorted(glob.glob(os.path.join(d, "proofs", "*", "proof.json")))
113
+ if not paths:
114
+ sys.exit(0)
115
+ try:
116
+ r = json.load(open(paths[-1]))
117
+ except (OSError, ValueError):
118
+ sys.exit(0)
119
+
120
+ def esc(s):
121
+ return json.dumps(str(s))[1:-1]
122
+
123
+ out = []
124
+ qg = r.get("quality_gates") or {}
125
+ gates = qg.get("gates") or []
126
+ if gates:
127
+ passed = sum(1 for g in gates if str(g.get("status", "")).startswith("pass"))
128
+ out.append(("Gates", f"{passed}/{len(gates)} passed"))
129
+
130
+ git = (r.get("facts") or {}).get("git") or {}
131
+ # base_sha empty means the receipt cannot be anchored, so it cannot be
132
+ # verified. Say that plainly rather than showing a diff count that implies it.
133
+ if not git.get("base_sha"):
134
+ out.append(("Verified", "no (unanchored receipt)"))
135
+ else:
136
+ diff = git.get("diff") or {}
137
+ if diff.get("count") is not None:
138
+ out.append(("Files changed", str(diff["count"])))
139
+
140
+ run_id = r.get("run_id")
141
+ if run_id:
142
+ out.append(("Receipt", esc(run_id)))
143
+
144
+ print(",".join(
145
+ '{"title": "%s", "value": "%s", "short": true}' % (esc(t), esc(v))
146
+ for t, v in out
147
+ ))
148
+ PY
149
+ }
150
+
89
151
  _notify_slack() {
90
152
  local event="$1"
91
153
  local title="$2"
@@ -106,6 +168,13 @@ _notify_slack() {
106
168
  escaped_event="$(_json_escape "$event")"
107
169
  escaped_project="$(_json_escape "$project")"
108
170
 
171
+ # Receipt-derived fields. Empty on any doubt, and the ${var:+,...} expansion
172
+ # below means an empty value adds no trailing comma -- a malformed payload
173
+ # would make Slack reject the whole card, so the failure mode of "no receipt"
174
+ # must be a card without verdict fields, never a card that does not send.
175
+ local verdict_fields
176
+ verdict_fields="$(_slack_verdict_fields 2>/dev/null || true)"
177
+
109
178
  # Build Slack payload with attachment
110
179
  local payload
111
180
  payload=$(cat <<PAYLOAD
@@ -116,7 +185,7 @@ _notify_slack() {
116
185
  "text": "$escaped_message",
117
186
  "fields": [
118
187
  {"title": "Event", "value": "$escaped_event", "short": true},
119
- {"title": "Project", "value": "$escaped_project", "short": true}
188
+ {"title": "Project", "value": "$escaped_project", "short": true}${verdict_fields:+,$verdict_fields}
120
189
  ],
121
190
  "footer": "Loki Mode",
122
191
  "ts": $(date +%s)
@@ -30,11 +30,30 @@
30
30
  # - Only redis and file are shipped. SQS, Pub/Sub, RabbitMQ, Kafka, etc. are
31
31
  # BRING-YOUR-OWN: override queue.command in values.yaml with your own
32
32
  # consumer. They are documented, not implemented here.
33
- # - The redis backend is at-most-once (LPOP-then-run). It has no visibility
34
- # timeout / dead-letter requeue. If a build crashes after the item is popped,
35
- # that item is lost from the queue. For at-least-once delivery use the file
36
- # backend (a crashed build leaves the item in processing/ for manual
37
- # re-drive) or bring a real broker.
33
+ # - The redis backend is AT-LEAST-ONCE by default. LMOVE (RPOPLPUSH pre-6.2)
34
+ # pops and records the item in <key>:processing atomically, and the ack
35
+ # (LREM) runs only after a zero-exit build. A crashed worker leaves its item
36
+ # in <key>:processing for re-drive -- the same shape as the file backend's
37
+ # processing/ directory.
38
+ # It was at-most-once (LPOP-then-run) until this was fixed; a crash after
39
+ # the pop lost the build with no record anywhere.
40
+ # A VISIBILITY TIMEOUT now exists: `queue-consumer.sh --reap` requeues
41
+ # in-flight items whose claim is older than LOKI_QUEUE_VISIBILITY_SEC
42
+ # (default 7200). Run it on a schedule (a CronJob) or by hand. It is a
43
+ # separate mode, not folded into the consume loop, because a consumer that
44
+ # reaped on every poll would race its peers on a busy queue.
45
+ # `--reap` works on BOTH backends. It was redis-only until this was fixed,
46
+ # which left the file backend with no crash recovery at all: a dead worker's
47
+ # item sat in processing/ forever and an unattended fleet stalled on it.
48
+ # A FRESH claim is never reaped -- requeuing a live item would build the
49
+ # user's work twice -- and an in-flight item with NO claim is treated as
50
+ # infinitely old, since that means the worker died between the LMOVE and the
51
+ # claim stamp, which is exactly the case that must be recoverable.
52
+ # LOKI_QUEUE_ACK=0 restores the legacy lossy path.
53
+ # The file backend's claim time is the item file's MTIME, stamped on claim,
54
+ # since a flat directory carries no claim hash. The limit is real and worth
55
+ # stating: a consumer that rewrote its own item file would reset that clock
56
+ # and look fresh forever. See file_reap.
38
57
  # - The file backend's atomicity relies on `mv` being atomic within a single
39
58
  # filesystem (true for a normal PVC). Two consumers racing the same pending
40
59
  # dir is safe (mv either wins or fails-and-skips), but is not load-balanced.
@@ -55,6 +74,8 @@
55
74
  # LOKI_QUEUE_POLL_SEC loop-mode empty-poll wait, seconds (default: 5)
56
75
  # LOKI_QUEUE_BLOCK_SEC redis BLPOP block timeout, seconds (default: 5)
57
76
  # LOKI_TERMINAL_EXIT run.sh terminal-failure exit code (default: 20)
77
+ # LOKI_QUEUE_VISIBILITY_SEC --reap requeues in-flight items older than this,
78
+ # both backends, seconds (default: 7200)
58
79
  #===============================================================================
59
80
 
60
81
  set -uo pipefail
@@ -176,17 +197,136 @@ redis_cli() {
176
197
  redis-cli -u "$QUEUE_URL" "$@"
177
198
  }
178
199
 
179
- # Pop one item from the redis list. In loop mode use BLPOP (blocks up to
180
- # BLOCK_SEC, then returns empty so we can check the stop flag); in one-shot use
181
- # LPOP (non-blocking, exits immediately on an empty queue).
182
- # Prints the popped item to stdout, or nothing if the queue was empty.
200
+ # Pop one item from the redis list, ATOMICALLY RECORDING IT AS IN-FLIGHT.
201
+ #
202
+ # THE DEFECT THIS FIXES, proven on a real redis 8.6.3 before it was written:
203
+ #
204
+ # RPUSH loki-builds '{"spec":"build a todo app"}' -> LLEN 1
205
+ # LPOP loki-builds -> worker holds the item
206
+ # <worker is OOM-killed / node evicted / kill -9>
207
+ # LLEN loki-builds -> 0
208
+ #
209
+ # The user's build is gone and NOTHING anywhere records that it existed. For a
210
+ # product whose entire thesis is "we hand you a receipt you can check", silently
211
+ # losing the work is the worst failure mode available: there is no receipt, no
212
+ # error, and no queue entry to retry.
213
+ #
214
+ # LMOVE (redis 6.2+) pops and pushes to a processing list in ONE atomic step, so
215
+ # a crash between the two is impossible. A dead worker leaves its item in
216
+ # <key>:processing where `loki queue reap` (or an operator) can re-drive it.
217
+ # RPOPLPUSH is the pre-6.2 equivalent and is tried automatically.
218
+ #
219
+ # Kept OPT-OUT rather than opt-in (LOKI_QUEUE_ACK=0 restores LPOP). At-least-once
220
+ # is the safer default: its failure mode is a duplicate build, which the
221
+ # idempotence work already handles, versus silent data loss. But an operator
222
+ # running a broker that already guarantees delivery should be able to turn it
223
+ # off rather than maintain two in-flight records.
224
+ #
225
+ # In loop mode with acking we use a short-poll LMOVE plus a sleep instead of
226
+ # BLMOVE, because BLMOVE blocks the connection and the stop-flag check has to
227
+ # stay responsive; the empty-poll backoff below already bounds the cost.
183
228
  redis_pop() {
184
- if [ "$ONESHOT" = "1" ]; then
185
- redis-cli -u "$QUEUE_URL" --no-raw LPOP "$QUEUE_KEY" 2>/dev/null | _redis_unquote
186
- else
187
- # BLPOP returns two lines: the key name, then the value. Take the value.
188
- redis-cli -u "$QUEUE_URL" BLPOP "$QUEUE_KEY" "$BLOCK_SEC" 2>/dev/null | sed -n '2p'
229
+ if [ "${LOKI_QUEUE_ACK:-1}" = "0" ]; then
230
+ # Legacy at-most-once path, explicitly requested.
231
+ if [ "$ONESHOT" = "1" ]; then
232
+ redis-cli -u "$QUEUE_URL" --no-raw LPOP "$QUEUE_KEY" 2>/dev/null | _redis_unquote
233
+ else
234
+ redis-cli -u "$QUEUE_URL" BLPOP "$QUEUE_KEY" "$BLOCK_SEC" 2>/dev/null | sed -n '2p'
235
+ fi
236
+ return 0
237
+ fi
238
+
239
+ # RAW output, not --no-raw. `--no-raw` escapes the INNER quotes of a JSON
240
+ # payload ({"spec":"x"} comes back as "{\"spec\":\"x\"}"), and
241
+ # _redis_unquote only strips the outer pair -- so the item reaching
242
+ # extract_spec was mangled AND the string handed to LREM no longer matched
243
+ # what redis stored, silently acking nothing. Both were caught by running
244
+ # against a real redis 8.6.3; neither is visible in a mock.
245
+ local out
246
+ out="$(redis-cli -u "$QUEUE_URL" LMOVE "$QUEUE_KEY" "${QUEUE_KEY}:processing" LEFT RIGHT 2>/dev/null)"
247
+ # Pre-6.2 servers reject LMOVE as an unknown command (stderr, empty stdout),
248
+ # so fall back rather than reporting an empty queue.
249
+ if [ -z "$out" ]; then
250
+ out="$(redis-cli -u "$QUEUE_URL" RPOPLPUSH "$QUEUE_KEY" "${QUEUE_KEY}:processing" 2>/dev/null)"
251
+ fi
252
+ # Record WHEN this item was claimed. A bare list carries no time, so nothing
253
+ # could tell a stale entry (dead worker) from a running one (long build) --
254
+ # and requeuing a live item duplicates the user's build. The claim hash is
255
+ # what makes the reaper below safe to run at all.
256
+ #
257
+ # Written AFTER the move, so a crash between them leaves the item in-flight
258
+ # with no claim time. The reaper treats a missing claim as INFINITELY OLD
259
+ # rather than infinitely young: an item nothing is tracking is exactly the
260
+ # case that must be recoverable.
261
+ if [ -n "$out" ]; then
262
+ redis-cli -u "$QUEUE_URL" HSET "${QUEUE_KEY}:claims" "$out" "$(date +%s)" >/dev/null 2>&1 || true
189
263
  fi
264
+ printf '%s' "$out"
265
+ }
266
+
267
+ # Remove a completed item from the in-flight list. Called ONLY after the build
268
+ # finished; until then the item stays recoverable.
269
+ #
270
+ # LREM with count 0 removes every equal element. Two identical specs queued twice
271
+ # would both clear on the first ack -- accepted deliberately: the alternative is
272
+ # a per-item token, which needs a producer change, and the failure mode here is
273
+ # one duplicate re-drive rather than a lost build.
274
+ redis_ack() {
275
+ [ "${LOKI_QUEUE_ACK:-1}" = "0" ] && return 0
276
+ [ -n "${1:-}" ] || return 0
277
+ redis-cli -u "$QUEUE_URL" LREM "${QUEUE_KEY}:processing" 0 "$1" >/dev/null 2>&1 || true
278
+ # Drop the claim too, or the hash grows without bound and every completed
279
+ # item looks like a candidate for reaping forever.
280
+ redis-cli -u "$QUEUE_URL" HDEL "${QUEUE_KEY}:claims" "$1" >/dev/null 2>&1 || true
281
+ }
282
+
283
+ # Requeue in-flight items whose claim is older than the visibility timeout.
284
+ #
285
+ # THE GAP THIS CLOSES. LMOVE made a crashed worker's item RECOVERABLE, but
286
+ # nothing recovered it: re-drive was an operator action, so an unattended fleet
287
+ # still stalled on every dead worker. That was documented rather than claimed;
288
+ # this closes it.
289
+ #
290
+ # SAFETY IS THE WHOLE DESIGN. Requeuing an item a worker is still building
291
+ # duplicates the user's build, so the timeout must exceed the longest legitimate
292
+ # build. Default 2h (LOKI_MAX_DURATION territory), tunable.
293
+ #
294
+ # An item with NO claim entry is treated as INFINITELY OLD, not young: it means
295
+ # the worker died between the LMOVE and the HSET, which is precisely the case
296
+ # that must be recoverable. Erring the other way would strand exactly the items
297
+ # this exists to rescue.
298
+ #
299
+ # Reports what it did on stdout. Requeues nothing when the timeout has not
300
+ # elapsed, and says so rather than printing a silent zero.
301
+ redis_reap() {
302
+ [ "${LOKI_QUEUE_ACK:-1}" = "0" ] && { echo "reap: acking disabled, nothing to reap"; return 0; }
303
+ local timeout now items requeued=0 kept=0
304
+ timeout="${LOKI_QUEUE_VISIBILITY_SEC:-7200}"
305
+ now="$(date +%s)"
306
+ items="$(redis-cli -u "$QUEUE_URL" LRANGE "${QUEUE_KEY}:processing" 0 -1 2>/dev/null)"
307
+ [ -n "$items" ] || { echo "reap: nothing in flight"; return 0; }
308
+ while IFS= read -r it; do
309
+ [ -n "$it" ] || continue
310
+ local claimed age
311
+ claimed="$(redis-cli -u "$QUEUE_URL" HGET "${QUEUE_KEY}:claims" "$it" 2>/dev/null)"
312
+ if [ -z "$claimed" ]; then
313
+ age="$((timeout + 1))" # unclaimed == infinitely old, see above
314
+ else
315
+ age="$((now - claimed))"
316
+ fi
317
+ if [ "$age" -gt "$timeout" ]; then
318
+ # LREM then RPUSH, not LMOVE: the item may appear more than once and
319
+ # LREM 0 clears every copy, so the queue cannot gain duplicates.
320
+ redis-cli -u "$QUEUE_URL" LREM "${QUEUE_KEY}:processing" 0 "$it" >/dev/null 2>&1 || true
321
+ redis-cli -u "$QUEUE_URL" HDEL "${QUEUE_KEY}:claims" "$it" >/dev/null 2>&1 || true
322
+ redis-cli -u "$QUEUE_URL" RPUSH "$QUEUE_KEY" "$it" >/dev/null 2>&1 || true
323
+ requeued=$((requeued + 1))
324
+ log "reaped stale item (age ${age}s > ${timeout}s), requeued"
325
+ else
326
+ kept=$((kept + 1))
327
+ fi
328
+ done <<< "$items"
329
+ echo "reap: requeued $requeued, left $kept in flight (timeout ${timeout}s)"
190
330
  }
191
331
 
192
332
  # --no-raw LPOP wraps the value in quotes; strip a single surrounding pair and
@@ -207,10 +347,21 @@ redis_consume_one() {
207
347
  if [ -z "$item" ]; then
208
348
  return 100 # sentinel: queue empty
209
349
  fi
210
- local spec
350
+ local spec rc
211
351
  spec="$(extract_spec "$item")"
212
352
  run_build "$spec"
213
- return $?
353
+ rc=$?
354
+ # ACK ONLY ON SUCCESS, and the order is load-bearing: acking before the
355
+ # build would reintroduce the exact data loss this exists to prevent, and
356
+ # acking a FAILED build would discard work that a re-drive could complete.
357
+ # A failed item stays in <key>:processing so `loki queue reap` can requeue
358
+ # it -- the same shape as the file backend leaving it in processing/.
359
+ if [ "$rc" -eq 0 ]; then
360
+ redis_ack "$item"
361
+ else
362
+ log "item left in ${QUEUE_KEY}:processing for re-drive (exit $rc)"
363
+ fi
364
+ return "$rc"
214
365
  }
215
366
 
216
367
  # =============================================================================
@@ -241,6 +392,18 @@ file_claim_oldest() {
241
392
  base="$(basename "$f")"
242
393
  dest="$QUEUE_DIR/processing/$base"
243
394
  if mv "$f" "$dest" 2>/dev/null; then
395
+ # Stamp the CLAIM TIME. This is the file-backend analogue of the
396
+ # `HSET <key>:claims` that redis_pop does right after its LMOVE, and
397
+ # file_reap below depends on it being here.
398
+ #
399
+ # Load-bearing: `mv` is a rename(2), which PRESERVES mtime. An item
400
+ # that sat in pending/ behind a backlog for longer than the
401
+ # visibility timeout would arrive in processing/ already looking
402
+ # stale, and the reaper would requeue it while a worker was actively
403
+ # building it -- the exact duplicate build the timeout exists to
404
+ # prevent, firing on precisely the busy queues that need a reaper.
405
+ # Verified on darwin: mv of a file stamped 2020 kept the 2020 mtime.
406
+ touch "$dest" 2>/dev/null || true
244
407
  printf '%s' "$dest"
245
408
  return 0
246
409
  fi
@@ -272,13 +435,110 @@ file_consume_one() {
272
435
  mv "$claimed" "$QUEUE_DIR/failed/$base" 2>/dev/null || log "WARN: could not move $base to failed/"
273
436
  log "item $base TERMINAL-FAILED (exit $rc); moved to failed/, not acked"
274
437
  else
275
- # Transient crash: leave it in processing/ for manual re-drive. We do NOT
276
- # auto-requeue (no retry counter in a flat dir); honest at-least-once.
438
+ # Transient crash: leave it in processing/ for re-drive. We do not
439
+ # requeue INLINE (no retry counter in a flat dir, so an instant retry
440
+ # could hot-loop a poison item). `--reap` picks it up once the
441
+ # visibility timeout has elapsed; see file_reap.
277
442
  log "item $base crashed (exit $rc); left in processing/ for re-drive"
278
443
  fi
279
444
  return "$rc"
280
445
  }
281
446
 
447
+ # Requeue in-flight items whose claim is older than the visibility timeout.
448
+ #
449
+ # THE GAP THIS CLOSES. The redis backend got a reaper; the file backend did not,
450
+ # so an item left in processing/ by a dead worker stayed there FOREVER and an
451
+ # unattended fleet on the file backend stalled on every crash. `--reap` answered
452
+ # "redis-only" and changed nothing.
453
+ #
454
+ # CLAIM TIME IS THE FILE MTIME. The file backend has no claim hash, so the mtime
455
+ # stamped by file_claim_oldest stands in for one.
456
+ #
457
+ # THE LIMIT, stated plainly: mtime is whatever last WROTE the file. A build that
458
+ # rewrites its own item file resets the clock and would look fresh forever, so
459
+ # it would never be reaped. Nothing here writes back to a claimed item, but a
460
+ # custom consumer that did would silently opt out of recovery. The redis claim
461
+ # hash does not have this weakness; the file backend accepts it rather than
462
+ # maintain a second sidecar file whose own staleness would need reaping too.
463
+ #
464
+ # There is also no retry counter (a flat dir carries none), so a poison item can
465
+ # be reaped repeatedly. That is the same shape the backend already documents for
466
+ # manual re-drive, not a new failure introduced here.
467
+ #
468
+ # SAFETY IS THE WHOLE DESIGN, mirroring redis_reap:
469
+ # - a FRESH item is NEVER requeued; a live build requeued is the user's work
470
+ # built twice, which is worse than the stranding this fixes
471
+ # - it NEVER deletes an item -- a name collision in pending/ leaves the item
472
+ # in processing/ and says so, rather than clobbering the pending copy
473
+ # - it reports counts on stdout; per-item detail goes to stderr via log()
474
+ file_reap() {
475
+ local timeout now requeued=0 kept=0 seen=0 f base mt age
476
+ timeout="${LOKI_QUEUE_VISIBILITY_SEC:-7200}"
477
+ now="$(date +%s)"
478
+ if [ ! -d "$QUEUE_DIR/processing" ]; then
479
+ echo "reap: nothing in flight"
480
+ return 0
481
+ fi
482
+ mkdir -p "$QUEUE_DIR/pending" 2>/dev/null || true
483
+ # Glob, not `ls`: a filename with a space or newline survives it.
484
+ for f in "$QUEUE_DIR"/processing/*; do
485
+ [ -e "$f" ] || continue # unmatched glob stays literal
486
+ [ -f "$f" ] || continue
487
+ seen=$((seen + 1))
488
+ base="$(basename "$f")"
489
+ # GNU FIRST, and the order is load-bearing. `stat -f` means
490
+ # --file-system on GNU coreutils and EXITS 0, printing a mount point
491
+ # rather than an mtime -- so a `-f`-first probe never falls through on
492
+ # Linux and hands back garbage. BSD `stat -c` has no such ambiguity: it
493
+ # is an illegal option and exits 1 (verified on darwin), so `-f %m` is
494
+ # reached only where it means what we want.
495
+ # `find -printf` and `-mmin` are GNU-only / minute-granular, and the
496
+ # fresh-vs-stale distinction needs seconds.
497
+ mt="$(stat -c %Y "$f" 2>/dev/null || stat -f %m "$f" 2>/dev/null)"
498
+ case "$mt" in
499
+ ''|*[!0-9]*)
500
+ # Unreadable or non-numeric claim time. KEEP the item, and note
501
+ # the ASYMMETRY with redis_reap deliberately: a missing redis
502
+ # claim is DIAGNOSTIC (the worker died between LMOVE and HSET),
503
+ # but a file that exists ALWAYS has an mtime, so failing to read
504
+ # one means our stat invocation is wrong -- a portability bug,
505
+ # not a dead worker.
506
+ # MEASURED, not assumed: with a `stat` that prints a non-number
507
+ # and exits 0, the un-guarded arithmetic aborts the reaper under
508
+ # `set -uo pipefail` (rc=1, nothing requeued). So the observed
509
+ # failure is a DEAD REAPER on the affected platform, not a
510
+ # duplicate build. Keeping the item is correct either way, and
511
+ # the WARN names the reason instead of exiting silently.
512
+ kept=$((kept + 1))
513
+ log "WARN: cannot read mtime for $base (got '$mt'); left in processing/ rather than guess a claim time"
514
+ continue
515
+ ;;
516
+ esac
517
+ age="$((now - mt))"
518
+ if [ "$age" -le "$timeout" ]; then
519
+ kept=$((kept + 1))
520
+ continue
521
+ fi
522
+ if [ -e "$QUEUE_DIR/pending/$base" ]; then
523
+ kept=$((kept + 1))
524
+ log "stale item $base NOT requeued: pending/$base already exists (never overwrite an item)"
525
+ continue
526
+ fi
527
+ if mv "$f" "$QUEUE_DIR/pending/$base" 2>/dev/null; then
528
+ requeued=$((requeued + 1))
529
+ log "reaped stale item $base (age ${age}s > ${timeout}s), requeued"
530
+ else
531
+ kept=$((kept + 1))
532
+ log "WARN: could not requeue $base; left in processing/"
533
+ fi
534
+ done
535
+ if [ "$seen" -eq 0 ]; then
536
+ echo "reap: nothing in flight"
537
+ return 0
538
+ fi
539
+ echo "reap: requeued $requeued, left $kept in flight (timeout ${timeout}s)"
540
+ }
541
+
282
542
  # =============================================================================
283
543
  # Driver
284
544
  # =============================================================================
@@ -307,6 +567,18 @@ main() {
307
567
  return 2
308
568
  fi
309
569
 
570
+ # --reap requeues stale in-flight items and exits. Kept a SEPARATE mode
571
+ # rather than folded into the consume loop: a reaper wants its own cadence
572
+ # (a CronJob, or an operator hand-running it), and a consumer that reaped on
573
+ # every poll would race its own peers on a busy queue.
574
+ if [ "${1:-}" = "--reap" ] || [ "${LOKI_QUEUE_REAP:-0}" = "1" ]; then
575
+ case "$backend" in
576
+ redis) redis_reap ;;
577
+ file) file_init_dirs && file_reap ;;
578
+ esac
579
+ return $?
580
+ fi
581
+
310
582
  if [ "$ONESHOT" = "1" ]; then
311
583
  log "mode=oneshot backend=$backend (serverless: process one item then exit)"
312
584
  consume_one "$backend"
package/autonomy/run.sh CHANGED
@@ -24889,7 +24889,17 @@ check_human_intervention() {
24889
24889
  # Security: Check it's a regular file (not symlink) to prevent symlink attacks
24890
24890
  if [ -f "$loki_dir/HUMAN_INPUT.md" ] && [ ! -L "$loki_dir/HUMAN_INPUT.md" ]; then
24891
24891
  # Security: Prompt injection disabled by default for enterprise security
24892
- if [ "${LOKI_PROMPT_INJECTION:-false}" != "true" ]; then
24892
+ # Accepts "1" AND "true". It used to accept only "true", while
24893
+ # `loki steer` prints "Enable it: export LOKI_PROMPT_INJECTION=1" and
24894
+ # then "The next iteration will read and apply it." A user who followed
24895
+ # that instruction exactly had their steering note moved to
24896
+ # logs/human-input-REJECTED-*.md and never read -- the CLI reported
24897
+ # success, the runner silently discarded it, and the only warning went
24898
+ # to the runner log rather than to the person who typed the command.
24899
+ #
24900
+ # Both forms is the convention here (LOKI_GATE_SEMANTIC_TESTS_BLOCK and
24901
+ # LOKI_DURABLE_STATE each test for both); this flag was the outlier.
24902
+ if [ "${LOKI_PROMPT_INJECTION:-false}" != "true" ] && [ "${LOKI_PROMPT_INJECTION:-false}" != "1" ]; then
24893
24903
  log_warn "HUMAN_INPUT.md detected but prompt injection is DISABLED"
24894
24904
  log_warn "To enable, set LOKI_PROMPT_INJECTION=true (only in trusted environments)"
24895
24905
  # Move to rejected instead of processed
package/completions/_loki CHANGED
@@ -120,6 +120,7 @@ function _loki {
120
120
  function _loki_commands {
121
121
  local -a commands
122
122
  commands=(
123
+ 'gates:What blocks here vs only advises, and the promotion knob'
123
124
  'start:Start Loki Mode'
124
125
  'quick:Quick single-task mode'
125
126
  'quickstart:Guided first build from your idea'