loki-mode 9.17.0 → 9.18.2

package/autonomy/notify.sh

@@ -86,68 +86,6 @@ _get_discord_color() {
 # Slack Notifications
 #===============================================================================
 
-# Verdict fields for a Slack card, from the newest Evidence Receipt.
-#
-# Roadmap item 9: "put the receipt where review already happens". Slack
-# notifications carried the event name and the project name and nothing else --
-# grep for receipt/verdict/proof in this file returned 0. A team watching a
-# channel saw "build finished" and had to go somewhere else to learn whether it
-# was verified, which is the surface problem item 9 names: verification nobody
-# sees does not build trust.
-#
-# Emits Slack `fields` entries (the payload already has that array), so this
-# adds to the card rather than restructuring it.
-#
-# SILENT AND EMPTY ON ANY DOUBT. No receipt, unreadable JSON, no python -- emit
-# nothing. A notification is a side channel; it must never fail a build, and it
-# must never guess a verdict. An absent receipt yields no verdict field at all
-# rather than a reassuring default.
-_slack_verdict_fields() {
-    local loki_dir="${LOKI_DIR:-.loki}"
-    [ -d "$loki_dir/proofs" ] || return 0
-    command -v python3 >/dev/null 2>&1 || return 0
-    python3 - "$loki_dir" <<'PY' 2>/dev/null || true
-import glob, json, os, sys
-d = sys.argv[1]
-paths = sorted(glob.glob(os.path.join(d, "proofs", "*", "proof.json")))
-if not paths:
-    sys.exit(0)
-try:
-    r = json.load(open(paths[-1]))
-except (OSError, ValueError):
-    sys.exit(0)
-
-def esc(s):
-    return json.dumps(str(s))[1:-1]
-
-out = []
-qg = r.get("quality_gates") or {}
-gates = qg.get("gates") or []
-if gates:
-    passed = sum(1 for g in gates if str(g.get("status", "")).startswith("pass"))
-    out.append(("Gates", f"{passed}/{len(gates)} passed"))
-
-git = (r.get("facts") or {}).get("git") or {}
-# base_sha empty means the receipt cannot be anchored, so it cannot be
-# verified. Say that plainly rather than showing a diff count that implies it.
-if not git.get("base_sha"):
-    out.append(("Verified", "no (unanchored receipt)"))
-else:
-    diff = git.get("diff") or {}
-    if diff.get("count") is not None:
-        out.append(("Files changed", str(diff["count"])))
-
-run_id = r.get("run_id")
-if run_id:
-    out.append(("Receipt", esc(run_id)))
-
-print(",".join(
-    '{"title": "%s", "value": "%s", "short": true}' % (esc(t), esc(v))
-    for t, v in out
-))
-PY
-}
-
 _notify_slack() {
     local event="$1"
     local title="$2"
@@ -168,13 +106,6 @@ _notify_slack() {
     escaped_event="$(_json_escape "$event")"
     escaped_project="$(_json_escape "$project")"
 
-    # Receipt-derived fields. Empty on any doubt, and the ${var:+,...} expansion
-    # below means an empty value adds no trailing comma -- a malformed payload
-    # would make Slack reject the whole card, so the failure mode of "no receipt"
-    # must be a card without verdict fields, never a card that does not send.
-    local verdict_fields
-    verdict_fields="$(_slack_verdict_fields 2>/dev/null || true)"
-
     # Build Slack payload with attachment
     local payload
     payload=$(cat <<PAYLOAD
@@ -185,7 +116,7 @@ _notify_slack() {
         "text": "$escaped_message",
         "fields": [
             {"title": "Event", "value": "$escaped_event", "short": true},
-            {"title": "Project", "value": "$escaped_project", "short": true}${verdict_fields:+,$verdict_fields}
+            {"title": "Project", "value": "$escaped_project", "short": true}
         ],
         "footer": "Loki Mode",
         "ts": $(date +%s)

package/autonomy/queue-consumer.sh

@@ -30,30 +30,11 @@
 #   - Only redis and file are shipped. SQS, Pub/Sub, RabbitMQ, Kafka, etc. are
 #     BRING-YOUR-OWN: override queue.command in values.yaml with your own
 #     consumer. They are documented, not implemented here.
-#   - The redis backend is AT-LEAST-ONCE by default. LMOVE (RPOPLPUSH pre-6.2)
-#     pops and records the item in <key>:processing atomically, and the ack
-#     (LREM) runs only after a zero-exit build. A crashed worker leaves its item
-#     in <key>:processing for re-drive -- the same shape as the file backend's
-#     processing/ directory.
-#     It was at-most-once (LPOP-then-run) until this was fixed; a crash after
-#     the pop lost the build with no record anywhere.
-#     A VISIBILITY TIMEOUT now exists: `queue-consumer.sh --reap` requeues
-#     in-flight items whose claim is older than LOKI_QUEUE_VISIBILITY_SEC
-#     (default 7200). Run it on a schedule (a CronJob) or by hand. It is a
-#     separate mode, not folded into the consume loop, because a consumer that
-#     reaped on every poll would race its peers on a busy queue.
-#     `--reap` works on BOTH backends. It was redis-only until this was fixed,
-#     which left the file backend with no crash recovery at all: a dead worker's
-#     item sat in processing/ forever and an unattended fleet stalled on it.
-#     A FRESH claim is never reaped -- requeuing a live item would build the
-#     user's work twice -- and an in-flight item with NO claim is treated as
-#     infinitely old, since that means the worker died between the LMOVE and the
-#     claim stamp, which is exactly the case that must be recoverable.
-#     LOKI_QUEUE_ACK=0 restores the legacy lossy path.
-#     The file backend's claim time is the item file's MTIME, stamped on claim,
-#     since a flat directory carries no claim hash. The limit is real and worth
-#     stating: a consumer that rewrote its own item file would reset that clock
-#     and look fresh forever. See file_reap.
+#   - The redis backend is at-most-once (LPOP-then-run). It has no visibility
+#     timeout / dead-letter requeue. If a build crashes after the item is popped,
+#     that item is lost from the queue. For at-least-once delivery use the file
+#     backend (a crashed build leaves the item in processing/ for manual
+#     re-drive) or bring a real broker.
 #   - The file backend's atomicity relies on `mv` being atomic within a single
 #     filesystem (true for a normal PVC). Two consumers racing the same pending
 #     dir is safe (mv either wins or fails-and-skips), but is not load-balanced.
@@ -74,8 +55,6 @@
 #   LOKI_QUEUE_POLL_SEC  loop-mode empty-poll wait, seconds (default: 5)
 #   LOKI_QUEUE_BLOCK_SEC redis BLPOP block timeout, seconds (default: 5)
 #   LOKI_TERMINAL_EXIT   run.sh terminal-failure exit code (default: 20)
-#   LOKI_QUEUE_VISIBILITY_SEC  --reap requeues in-flight items older than this,
-#                        both backends, seconds (default: 7200)
 #===============================================================================
 
 set -uo pipefail
@@ -197,136 +176,17 @@ redis_cli() {
     redis-cli -u "$QUEUE_URL" "$@"
 }
 
-# Pop one item from the redis list, ATOMICALLY RECORDING IT AS IN-FLIGHT.
-#
-# THE DEFECT THIS FIXES, proven on a real redis 8.6.3 before it was written:
-#
-#   RPUSH loki-builds '{"spec":"build a todo app"}'   -> LLEN 1
-#   LPOP  loki-builds                                 -> worker holds the item
-#   <worker is OOM-killed / node evicted / kill -9>
-#   LLEN  loki-builds                                 -> 0
-#
-# The user's build is gone and NOTHING anywhere records that it existed. For a
-# product whose entire thesis is "we hand you a receipt you can check", silently
-# losing the work is the worst failure mode available: there is no receipt, no
-# error, and no queue entry to retry.
-#
-# LMOVE (redis 6.2+) pops and pushes to a processing list in ONE atomic step, so
-# a crash between the two is impossible. A dead worker leaves its item in
-# <key>:processing where `loki queue reap` (or an operator) can re-drive it.
-# RPOPLPUSH is the pre-6.2 equivalent and is tried automatically.
-#
-# Kept OPT-OUT rather than opt-in (LOKI_QUEUE_ACK=0 restores LPOP). At-least-once
-# is the safer default: its failure mode is a duplicate build, which the
-# idempotence work already handles, versus silent data loss. But an operator
-# running a broker that already guarantees delivery should be able to turn it
-# off rather than maintain two in-flight records.
-#
-# In loop mode with acking we use a short-poll LMOVE plus a sleep instead of
-# BLMOVE, because BLMOVE blocks the connection and the stop-flag check has to
-# stay responsive; the empty-poll backoff below already bounds the cost.
+# Pop one item from the redis list. In loop mode use BLPOP (blocks up to
+# BLOCK_SEC, then returns empty so we can check the stop flag); in one-shot use
+# LPOP (non-blocking, exits immediately on an empty queue).
+# Prints the popped item to stdout, or nothing if the queue was empty.
 redis_pop() {
-    if [ "${LOKI_QUEUE_ACK:-1}" = "0" ]; then
-        # Legacy at-most-once path, explicitly requested.
-        if [ "$ONESHOT" = "1" ]; then
-            redis-cli -u "$QUEUE_URL" --no-raw LPOP "$QUEUE_KEY" 2>/dev/null | _redis_unquote
-        else
-            redis-cli -u "$QUEUE_URL" BLPOP "$QUEUE_KEY" "$BLOCK_SEC" 2>/dev/null | sed -n '2p'
-        fi
-        return 0
-    fi
-
-    # RAW output, not --no-raw. `--no-raw` escapes the INNER quotes of a JSON
-    # payload ({"spec":"x"} comes back as "{\"spec\":\"x\"}"), and
-    # _redis_unquote only strips the outer pair -- so the item reaching
-    # extract_spec was mangled AND the string handed to LREM no longer matched
-    # what redis stored, silently acking nothing. Both were caught by running
-    # against a real redis 8.6.3; neither is visible in a mock.
-    local out
-    out="$(redis-cli -u "$QUEUE_URL" LMOVE "$QUEUE_KEY" "${QUEUE_KEY}:processing" LEFT RIGHT 2>/dev/null)"
-    # Pre-6.2 servers reject LMOVE as an unknown command (stderr, empty stdout),
-    # so fall back rather than reporting an empty queue.
-    if [ -z "$out" ]; then
-        out="$(redis-cli -u "$QUEUE_URL" RPOPLPUSH "$QUEUE_KEY" "${QUEUE_KEY}:processing" 2>/dev/null)"
-    fi
-    # Record WHEN this item was claimed. A bare list carries no time, so nothing
-    # could tell a stale entry (dead worker) from a running one (long build) --
-    # and requeuing a live item duplicates the user's build. The claim hash is
-    # what makes the reaper below safe to run at all.
-    #
-    # Written AFTER the move, so a crash between them leaves the item in-flight
-    # with no claim time. The reaper treats a missing claim as INFINITELY OLD
-    # rather than infinitely young: an item nothing is tracking is exactly the
-    # case that must be recoverable.
-    if [ -n "$out" ]; then
-        redis-cli -u "$QUEUE_URL" HSET "${QUEUE_KEY}:claims" "$out" "$(date +%s)" >/dev/null 2>&1 || true
+    if [ "$ONESHOT" = "1" ]; then
+        redis-cli -u "$QUEUE_URL" --no-raw LPOP "$QUEUE_KEY" 2>/dev/null | _redis_unquote
+    else
+        # BLPOP returns two lines: the key name, then the value. Take the value.
+        redis-cli -u "$QUEUE_URL" BLPOP "$QUEUE_KEY" "$BLOCK_SEC" 2>/dev/null | sed -n '2p'
     fi
-    printf '%s' "$out"
-}
-
-# Remove a completed item from the in-flight list. Called ONLY after the build
-# finished; until then the item stays recoverable.
-#
-# LREM with count 0 removes every equal element. Two identical specs queued twice
-# would both clear on the first ack -- accepted deliberately: the alternative is
-# a per-item token, which needs a producer change, and the failure mode here is
-# one duplicate re-drive rather than a lost build.
-redis_ack() {
-    [ "${LOKI_QUEUE_ACK:-1}" = "0" ] && return 0
-    [ -n "${1:-}" ] || return 0
-    redis-cli -u "$QUEUE_URL" LREM "${QUEUE_KEY}:processing" 0 "$1" >/dev/null 2>&1 || true
-    # Drop the claim too, or the hash grows without bound and every completed
-    # item looks like a candidate for reaping forever.
-    redis-cli -u "$QUEUE_URL" HDEL "${QUEUE_KEY}:claims" "$1" >/dev/null 2>&1 || true
-}
-
-# Requeue in-flight items whose claim is older than the visibility timeout.
-#
-# THE GAP THIS CLOSES. LMOVE made a crashed worker's item RECOVERABLE, but
-# nothing recovered it: re-drive was an operator action, so an unattended fleet
-# still stalled on every dead worker. That was documented rather than claimed;
-# this closes it.
-#
-# SAFETY IS THE WHOLE DESIGN. Requeuing an item a worker is still building
-# duplicates the user's build, so the timeout must exceed the longest legitimate
-# build. Default 2h (LOKI_MAX_DURATION territory), tunable.
-#
-# An item with NO claim entry is treated as INFINITELY OLD, not young: it means
-# the worker died between the LMOVE and the HSET, which is precisely the case
-# that must be recoverable. Erring the other way would strand exactly the items
-# this exists to rescue.
-#
-# Reports what it did on stdout. Requeues nothing when the timeout has not
-# elapsed, and says so rather than printing a silent zero.
-redis_reap() {
-    [ "${LOKI_QUEUE_ACK:-1}" = "0" ] && { echo "reap: acking disabled, nothing to reap"; return 0; }
-    local timeout now items requeued=0 kept=0
-    timeout="${LOKI_QUEUE_VISIBILITY_SEC:-7200}"
-    now="$(date +%s)"
-    items="$(redis-cli -u "$QUEUE_URL" LRANGE "${QUEUE_KEY}:processing" 0 -1 2>/dev/null)"
-    [ -n "$items" ] || { echo "reap: nothing in flight"; return 0; }
-    while IFS= read -r it; do
-        [ -n "$it" ] || continue
-        local claimed age
-        claimed="$(redis-cli -u "$QUEUE_URL" HGET "${QUEUE_KEY}:claims" "$it" 2>/dev/null)"
-        if [ -z "$claimed" ]; then
-            age="$((timeout + 1))"   # unclaimed == infinitely old, see above
-        else
-            age="$((now - claimed))"
-        fi
-        if [ "$age" -gt "$timeout" ]; then
-            # LREM then RPUSH, not LMOVE: the item may appear more than once and
-            # LREM 0 clears every copy, so the queue cannot gain duplicates.
-            redis-cli -u "$QUEUE_URL" LREM "${QUEUE_KEY}:processing" 0 "$it" >/dev/null 2>&1 || true
-            redis-cli -u "$QUEUE_URL" HDEL "${QUEUE_KEY}:claims" "$it" >/dev/null 2>&1 || true
-            redis-cli -u "$QUEUE_URL" RPUSH "$QUEUE_KEY" "$it" >/dev/null 2>&1 || true
-            requeued=$((requeued + 1))
-            log "reaped stale item (age ${age}s > ${timeout}s), requeued"
-        else
-            kept=$((kept + 1))
-        fi
-    done <<< "$items"
-    echo "reap: requeued $requeued, left $kept in flight (timeout ${timeout}s)"
 }
 
 # --no-raw LPOP wraps the value in quotes; strip a single surrounding pair and
@@ -347,21 +207,10 @@ redis_consume_one() {
     if [ -z "$item" ]; then
         return 100  # sentinel: queue empty
     fi
-    local spec rc
+    local spec
     spec="$(extract_spec "$item")"
     run_build "$spec"
-    rc=$?
-    # ACK ONLY ON SUCCESS, and the order is load-bearing: acking before the
-    # build would reintroduce the exact data loss this exists to prevent, and
-    # acking a FAILED build would discard work that a re-drive could complete.
-    # A failed item stays in <key>:processing so `loki queue reap` can requeue
-    # it -- the same shape as the file backend leaving it in processing/.
-    if [ "$rc" -eq 0 ]; then
-        redis_ack "$item"
-    else
-        log "item left in ${QUEUE_KEY}:processing for re-drive (exit $rc)"
-    fi
-    return "$rc"
+    return $?
 }
 
 # =============================================================================
@@ -392,18 +241,6 @@ file_claim_oldest() {
         base="$(basename "$f")"
         dest="$QUEUE_DIR/processing/$base"
         if mv "$f" "$dest" 2>/dev/null; then
-            # Stamp the CLAIM TIME. This is the file-backend analogue of the
-            # `HSET <key>:claims` that redis_pop does right after its LMOVE, and
-            # file_reap below depends on it being here.
-            #
-            # Load-bearing: `mv` is a rename(2), which PRESERVES mtime. An item
-            # that sat in pending/ behind a backlog for longer than the
-            # visibility timeout would arrive in processing/ already looking
-            # stale, and the reaper would requeue it while a worker was actively
-            # building it -- the exact duplicate build the timeout exists to
-            # prevent, firing on precisely the busy queues that need a reaper.
-            # Verified on darwin: mv of a file stamped 2020 kept the 2020 mtime.
-            touch "$dest" 2>/dev/null || true
             printf '%s' "$dest"
             return 0
         fi
@@ -435,110 +272,13 @@ file_consume_one() {
         mv "$claimed" "$QUEUE_DIR/failed/$base" 2>/dev/null || log "WARN: could not move $base to failed/"
         log "item $base TERMINAL-FAILED (exit $rc); moved to failed/, not acked"
     else
-        # Transient crash: leave it in processing/ for re-drive. We do not
-        # requeue INLINE (no retry counter in a flat dir, so an instant retry
-        # could hot-loop a poison item). `--reap` picks it up once the
-        # visibility timeout has elapsed; see file_reap.
+        # Transient crash: leave it in processing/ for manual re-drive. We do NOT
+        # auto-requeue (no retry counter in a flat dir); honest at-least-once.
         log "item $base crashed (exit $rc); left in processing/ for re-drive"
     fi
     return "$rc"
 }
 
-# Requeue in-flight items whose claim is older than the visibility timeout.
-#
-# THE GAP THIS CLOSES. The redis backend got a reaper; the file backend did not,
-# so an item left in processing/ by a dead worker stayed there FOREVER and an
-# unattended fleet on the file backend stalled on every crash. `--reap` answered
-# "redis-only" and changed nothing.
-#
-# CLAIM TIME IS THE FILE MTIME. The file backend has no claim hash, so the mtime
-# stamped by file_claim_oldest stands in for one.
-#
-# THE LIMIT, stated plainly: mtime is whatever last WROTE the file. A build that
-# rewrites its own item file resets the clock and would look fresh forever, so
-# it would never be reaped. Nothing here writes back to a claimed item, but a
-# custom consumer that did would silently opt out of recovery. The redis claim
-# hash does not have this weakness; the file backend accepts it rather than
-# maintain a second sidecar file whose own staleness would need reaping too.
-#
-# There is also no retry counter (a flat dir carries none), so a poison item can
-# be reaped repeatedly. That is the same shape the backend already documents for
-# manual re-drive, not a new failure introduced here.
-#
-# SAFETY IS THE WHOLE DESIGN, mirroring redis_reap:
-#   - a FRESH item is NEVER requeued; a live build requeued is the user's work
-#     built twice, which is worse than the stranding this fixes
-#   - it NEVER deletes an item -- a name collision in pending/ leaves the item
-#     in processing/ and says so, rather than clobbering the pending copy
-#   - it reports counts on stdout; per-item detail goes to stderr via log()
-file_reap() {
-    local timeout now requeued=0 kept=0 seen=0 f base mt age
-    timeout="${LOKI_QUEUE_VISIBILITY_SEC:-7200}"
-    now="$(date +%s)"
-    if [ ! -d "$QUEUE_DIR/processing" ]; then
-        echo "reap: nothing in flight"
-        return 0
-    fi
-    mkdir -p "$QUEUE_DIR/pending" 2>/dev/null || true
-    # Glob, not `ls`: a filename with a space or newline survives it.
-    for f in "$QUEUE_DIR"/processing/*; do
-        [ -e "$f" ] || continue   # unmatched glob stays literal
-        [ -f "$f" ] || continue
-        seen=$((seen + 1))
-        base="$(basename "$f")"
-        # GNU FIRST, and the order is load-bearing. `stat -f` means
-        # --file-system on GNU coreutils and EXITS 0, printing a mount point
-        # rather than an mtime -- so a `-f`-first probe never falls through on
-        # Linux and hands back garbage. BSD `stat -c` has no such ambiguity: it
-        # is an illegal option and exits 1 (verified on darwin), so `-f %m` is
-        # reached only where it means what we want.
-        # `find -printf` and `-mmin` are GNU-only / minute-granular, and the
-        # fresh-vs-stale distinction needs seconds.
-        mt="$(stat -c %Y "$f" 2>/dev/null || stat -f %m "$f" 2>/dev/null)"
-        case "$mt" in
-            ''|*[!0-9]*)
-                # Unreadable or non-numeric claim time. KEEP the item, and note
-                # the ASYMMETRY with redis_reap deliberately: a missing redis
-                # claim is DIAGNOSTIC (the worker died between LMOVE and HSET),
-                # but a file that exists ALWAYS has an mtime, so failing to read
-                # one means our stat invocation is wrong -- a portability bug,
-                # not a dead worker.
-                # MEASURED, not assumed: with a `stat` that prints a non-number
-                # and exits 0, the un-guarded arithmetic aborts the reaper under
-                # `set -uo pipefail` (rc=1, nothing requeued). So the observed
-                # failure is a DEAD REAPER on the affected platform, not a
-                # duplicate build. Keeping the item is correct either way, and
-                # the WARN names the reason instead of exiting silently.
-                kept=$((kept + 1))
-                log "WARN: cannot read mtime for $base (got '$mt'); left in processing/ rather than guess a claim time"
-                continue
-                ;;
-        esac
-        age="$((now - mt))"
-        if [ "$age" -le "$timeout" ]; then
-            kept=$((kept + 1))
-            continue
-        fi
-        if [ -e "$QUEUE_DIR/pending/$base" ]; then
-            kept=$((kept + 1))
-            log "stale item $base NOT requeued: pending/$base already exists (never overwrite an item)"
-            continue
-        fi
-        if mv "$f" "$QUEUE_DIR/pending/$base" 2>/dev/null; then
-            requeued=$((requeued + 1))
-            log "reaped stale item $base (age ${age}s > ${timeout}s), requeued"
-        else
-            kept=$((kept + 1))
-            log "WARN: could not requeue $base; left in processing/"
-        fi
-    done
-    if [ "$seen" -eq 0 ]; then
-        echo "reap: nothing in flight"
-        return 0
-    fi
-    echo "reap: requeued $requeued, left $kept in flight (timeout ${timeout}s)"
-}
-
 # =============================================================================
 # Driver
 # =============================================================================
@@ -567,18 +307,6 @@ main() {
         return 2
     fi
 
-    # --reap requeues stale in-flight items and exits. Kept a SEPARATE mode
-    # rather than folded into the consume loop: a reaper wants its own cadence
-    # (a CronJob, or an operator hand-running it), and a consumer that reaped on
-    # every poll would race its own peers on a busy queue.
-    if [ "${1:-}" = "--reap" ] || [ "${LOKI_QUEUE_REAP:-0}" = "1" ]; then
-        case "$backend" in
-            redis) redis_reap ;;
-            file)  file_init_dirs && file_reap ;;
-        esac
-        return $?
-    fi
-
     if [ "$ONESHOT" = "1" ]; then
         log "mode=oneshot backend=$backend (serverless: process one item then exit)"
         consume_one "$backend"