opencode-thrum-pro 0.1.5 → 0.1.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/commands/thrum-compact-extended.md +367 -0
- package/assets/commands/thrum-compact.md +301 -0
- package/assets/commands/thrum-inbox.md +1 -1
- package/assets/commands/thrum-restart-extended.md +85 -16
- package/assets/commands/thrum-restart.md +106 -14
- package/assets/commands/thrum-send.md +4 -1
- package/assets/commands/thrum-sleep-extended.md +68 -51
- package/assets/commands/thrum-sleep.md +68 -63
- package/assets/commands/thrum-update-project.md +1 -1
- package/assets/skills/adversarial-critique/SKILL.md +5 -8
- package/assets/skills/brainstorm-steward/SKILL.md +65 -11
- package/assets/skills/brainstormer-staying-current/SKILL.md +1 -4
- package/assets/skills/choosing-subagent-models/SKILL.md +200 -70
- package/assets/skills/closing-findings-while-warm/SKILL.md +1 -2
- package/assets/skills/configure-roles/SKILL.md +2 -2
- package/assets/skills/conversation-only-context/SKILL.md +11 -11
- package/assets/skills/conversation-only-context/resources/driver-subagent-pattern.md +1 -1
- package/assets/skills/conversation-only-context/resources/remote-environment-scenarios.md +1 -1
- package/assets/skills/coordinator-assessing-agent-completion/SKILL.md +98 -6
- package/assets/skills/coordinator-branch-split-on-block/SKILL.md +1 -2
- package/assets/skills/coordinator-context-monitoring/SKILL.md +29 -19
- package/assets/skills/coordinator-deploying-a-node/SKILL.md +124 -0
- package/assets/skills/coordinator-dispatching-work/SKILL.md +26 -14
- package/assets/skills/coordinator-hotpath-merge-gate/SKILL.md +65 -21
- package/assets/skills/coordinator-interacting-with-the-owner/SKILL.md +1 -2
- package/assets/skills/coordinator-maintaining-memory/SKILL.md +1 -1
- package/assets/skills/coordinator-managing-state-and-lifecycle/SKILL.md +7 -8
- package/assets/skills/coordinator-merging-code/SKILL.md +43 -22
- package/assets/skills/coordinator-philosophy-merge-gate/SKILL.md +29 -9
- package/assets/skills/coordinator-plan-reconcile-gate/SKILL.md +4 -5
- package/assets/skills/coordinator-rolling-out-to-nodes/SKILL.md +92 -0
- package/assets/skills/coordinator-running-autonomous-feature-development/SKILL.md +3 -3
- package/assets/skills/coordinator-running-brainstorm-cycles/SKILL.md +25 -25
- package/assets/skills/coordinator-running-review-cycles/SKILL.md +16 -6
- package/assets/{commands/thrum-_deviations-protocol.md → skills/deviations-protocol/SKILL.md} +6 -12
- package/assets/skills/diagnosing-fleet-and-daemon/SKILL.md +0 -2
- package/assets/skills/efficient-multi-agent-research/SKILL.md +3 -3
- package/assets/skills/generator-idempotency-protocol/SKILL.md +9 -0
- package/assets/skills/implementer-maintaining-memory/SKILL.md +3 -4
- package/assets/skills/implementer-orchestrating-execution/SKILL.md +12 -16
- package/assets/skills/implementer-receiving-dispatch/SKILL.md +15 -17
- package/assets/skills/implementer-receiving-review-feedback/SKILL.md +3 -4
- package/assets/skills/implementer-status-and-handoff/SKILL.md +1 -14
- package/assets/skills/implementer-tdd-and-quality/SKILL.md +2 -2
- package/assets/skills/memory-curate/SKILL.md +1 -1
- package/assets/skills/memory-write-discipline/SKILL.md +1 -1
- package/assets/skills/orchestrate/SKILL.md +77 -36
- package/assets/skills/persistent-watcher-archetype/SKILL.md +172 -25
- package/assets/skills/persistent-watcher-archetype/resources/capture_archive.py +618 -0
- package/assets/skills/persistent-watcher-archetype/resources/jev_watcher_filter.py +837 -0
- package/assets/skills/persistent-watcher-archetype/resources/roster-watch.sh +1140 -0
- package/assets/skills/persistent-watcher-archetype/resources/roster-watch_test.sh +291 -0
- package/assets/skills/persistent-watcher-archetype/resources/thrum-capture-fallback.sh +2 -2
- package/assets/skills/persistent-watcher-archetype/resources/thrum-watch-pane-capture.sh +20 -8
- package/assets/skills/prime-agent/SKILL.md +10 -0
- package/assets/skills/project-hotpath-gate/SKILL.md +1 -2
- package/assets/skills/project-philosophy/SKILL.md +1 -2
- package/assets/skills/project-rules-protocol/SKILL.md +12 -0
- package/assets/skills/project-setup/SKILL.md +27 -22
- package/assets/skills/project-setup/resources/implementation-agent.md +55 -35
- package/assets/skills/researcher-answering-queries/SKILL.md +1 -2
- package/assets/skills/researcher-investigating/SKILL.md +8 -13
- package/assets/skills/researcher-maintaining-memory/SKILL.md +1 -1
- package/assets/{commands/thrum-_snapshot-protocol.md → skills/snapshot-protocol/SKILL.md} +27 -21
- package/assets/{commands/thrum-_stamp-protocol.md → skills/stamp-protocol/SKILL.md} +14 -8
- package/assets/skills/thrum/SKILL.md +14 -7
- package/assets/skills/thrum/resources/ANTI_PATTERNS.md +16 -18
- package/assets/skills/thrum/resources/CLI_REFERENCE.md +17 -11
- package/assets/skills/thrum/resources/LISTENER_PATTERN.md +9 -3
- package/assets/skills/thrum/resources/MESSAGING.md +18 -11
- package/assets/skills/thrum/resources/WORKTREES.md +1 -1
- package/assets/skills/thrum-memory/SKILL.md +1 -1
- package/assets/skills/trace-spec-to-plan/resources/sweep-agent-prompt.md +1 -1
- package/assets/skills/update-queue/SKILL.md +6 -8
- package/assets/skills/using-the-dashboard/SKILL.md +30 -0
- package/assets/skills/using-the-queue/SKILL.md +76 -5
- package/assets/skills/using-thrum-state/SKILL.md +85 -2
- package/assets/skills/verify-against-plan/SKILL.md +122 -7
- package/assets/skills/watcher-archetype/SKILL.md +18 -0
- package/assets/skills/writing-durable-markdown/SKILL.md +5 -0
- package/dist/installer-assets.test.js +41 -0
- package/dist/installer.js +4 -4
- package/dist/permission-fixture.integration.test.js +4 -4
- package/package.json +1 -1
- package/assets/commands/thrum-_generator-idempotency-protocol.md +0 -15
- package/assets/commands/thrum-_project-rules-protocol.md +0 -18
- package/assets/commands/thrum-_verify-before-write-protocol.md +0 -27
- package/assets/skills/coordinator-deploying-the-fleet/SKILL.md +0 -396
- package/dist/allowlist-sync.test.js +0 -44
- /package/dist/{allowlist-sync.test.d.ts → installer-assets.test.d.ts} +0 -0
|
@@ -0,0 +1,1140 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# roster-watch.sh — reconciled persistent-watcher/steward roster capture
|
|
3
|
+
# script.
|
|
4
|
+
#
|
|
5
|
+
# PROVENANCE: this file reconciles two live, independently-evolved,
|
|
6
|
+
# never-merged personal-branch scripts (`origin/agent/watcher-primary` and
|
|
7
|
+
# `origin/agent/brainstorm-steward`, reconciled against a full byte-level
|
|
8
|
+
# diff account of both) into one canonical, git-tracked script.
|
|
9
|
+
# Neither branch is an ancestor of thrum-agents; this is the first time
|
|
10
|
+
# either design lands in the shared tree.
|
|
11
|
+
#
|
|
12
|
+
# EXECUTION MODEL: single-pass / cron-driven by default (ported from
|
|
13
|
+
# brainstorm-steward, the LATER of the two designs, Sep 10 vs Sep 6). Its
|
|
14
|
+
# own header explained why it moved off a resident loop: "the earlier
|
|
15
|
+
# persistent-loop version had its sleep reset every time the monitor
|
|
16
|
+
# supervisor restarted the script during docker-bridge-flap capture
|
|
17
|
+
# failures, so the 2h overnight cadence never held." Cron + single-pass is
|
|
18
|
+
# immune to that class of bug because the NEXT wake is `thrum monitor`'s
|
|
19
|
+
# cron schedule, not an internal `sleep` that a supervisor restart can
|
|
20
|
+
# reset. This is now the RECOMMENDED mode for any deployment.
|
|
21
|
+
#
|
|
22
|
+
# LEGACY LOOP MODE: set ROSTER_WATCH_MODE=loop to opt into the OLDER
|
|
23
|
+
# watcher-primary persistent-loop model (a resident `while true; do
|
|
24
|
+
# ...; sleep "${INTERVAL_SECS:-600}"; done`, plus its self-submit-Enter
|
|
25
|
+
# block — see run_self_submit_enter below). This mode is kept, not
|
|
26
|
+
# dropped, because some live deployments may still depend on it until
|
|
27
|
+
# they are migrated to a `thrum monitor --schedule` cron registration;
|
|
28
|
+
# migrating any such deployment is an operational follow-up, NOT done by
|
|
29
|
+
# this script. Do not add new loop-mode deployments — use cron/single-pass
|
|
30
|
+
# (the unset/default mode) instead.
|
|
31
|
+
#
|
|
32
|
+
# GHOST-TIP DETECTION: Claude Code's TUI renders an unaccepted inline
|
|
33
|
+
# autocomplete suggestion in the input box using the literal SGR
|
|
34
|
+
# "dim/faint" escape code (ESC[2m ... ESC[0m) — confirmed by direct byte
|
|
35
|
+
# inspection (2026-08-17, a reconciler agent's "Stop the sweep loop
|
|
36
|
+
# for now" tip). Real typed/submitted text never carries that code. `thrum
|
|
37
|
+
# tmux capture` returns plain text with all styling stripped, so a script
|
|
38
|
+
# parsing ONLY that output cannot tell a live ghost-tip apart from real
|
|
39
|
+
# unsubmitted input sitting in the box — both render as identical plain
|
|
40
|
+
# text. This script does a SECOND, raw pass per agent (tmux capture-pane
|
|
41
|
+
# -e, escape codes intact) purely to check for that one signature and
|
|
42
|
+
# annotate it, so the agent reading the capture file knows NOT to treat a
|
|
43
|
+
# flagged line as real pending input (never nudge/escalate on it) or as an
|
|
44
|
+
# agent's own submitted text. watcher-primary's committed copy had this;
|
|
45
|
+
# brainstorm-steward's did not (dropped, not deliberately removed per any
|
|
46
|
+
# commit message found) — kept here because the underlying TUI behavior it
|
|
47
|
+
# guards against is not mode-specific, and dropping it silently would
|
|
48
|
+
# regress a real, previously-fixed false-escalation risk for every
|
|
49
|
+
# steward-mode deployment.
|
|
50
|
+
#
|
|
51
|
+
# REDIRECT FIX (the actual bug this reconciliation targets):
|
|
52
|
+
# `watch_params.json` lives under the SHARED, redirect-resolved `agents/`
|
|
53
|
+
# tree (`internal/paths/paths.go` AgentDir: "the agents/ tree is shared,
|
|
54
|
+
# not per-worktree"), not under this script's own worktree. Both source
|
|
55
|
+
# scripts read it as if it were always reachable via a hardcoded/relative
|
|
56
|
+
# main-repo path (`cd /Users/you/dev/thrum` +
|
|
57
|
+
# `.thrum/agents/<agent>/watch_params.json`) — correct only by accident of
|
|
58
|
+
# always invoking from the main repo, and silently wrong (reads a stale,
|
|
59
|
+
# worktree-local copy) the moment this script or its cwd ever moves to a
|
|
60
|
+
# feature worktree. `resolve_watch_params()` below replaces that implicit
|
|
61
|
+
# assumption with the same explicit `.thrum/redirect`-following idiom
|
|
62
|
+
# already used by `scripts/heartbeat-lib.sh:225-232` and
|
|
63
|
+
# `scripts/thrum-check-inbox.sh:25-41`.
|
|
64
|
+
#
|
|
65
|
+
# PORTABILITY FIX: both source scripts also hardcoded a single operator's
|
|
66
|
+
# absolute paths (`cd /Users/you/dev/thrum`,
|
|
67
|
+
# `OUTDIR=/Users/you/.thrum/worktrees/thrum/<agent>/.thrum-watch`). This
|
|
68
|
+
# script derives every path from its own location instead, so the exact
|
|
69
|
+
# same file works when copied into ANY worktree for ANY agent.
|
|
70
|
+
#
|
|
71
|
+
# DEPLOYMENT: per prior operational research into this exact redirect-
|
|
72
|
+
# resolution issue, this script is hand-copied to
|
|
73
|
+
# `<worktree>/.thrum-watch/roster-watch.sh` — a directory SIBLING to
|
|
74
|
+
# `.thrum/`, NOT inside `.thrum/agents/<you>/` (unlike this skill's other
|
|
75
|
+
# resource, thrum-watch-pane-capture.sh, which IS deployed under
|
|
76
|
+
# `.thrum/agents/<you>/`). Because of that different deployment shape,
|
|
77
|
+
# SCRIPT_DIR here is `<worktree>/.thrum-watch`, so WORKTREE_ROOT is
|
|
78
|
+
# `dirname(SCRIPT_DIR)` — one level up, not three. Register with:
|
|
79
|
+
# thrum monitor start --name roster-watch-<you> \
|
|
80
|
+
# --match "^(roster capture ready:|ROSTER INCIDENT:)" --to @<you> --notify-on-success \
|
|
81
|
+
# --schedule '*/10 * * * *' -- <worktree>/.thrum-watch/roster-watch.sh
|
|
82
|
+
# (loop mode does not need --schedule; it stays resident under Monitor.)
|
|
83
|
+
# Keep --notify-on-success; the daemon honors the terminal JEV quiet marker per run.
|
|
84
|
+
#
|
|
85
|
+
# AGENT IDENTITY: neither source script derived its own agent name — both
|
|
86
|
+
# hardcoded it (watcher_primary, brainstorm_steward) throughout. This
|
|
87
|
+
# script defaults AGENT_NAME to the worktree directory's basename with
|
|
88
|
+
# hyphens folded to underscores (matches both known deployments:
|
|
89
|
+
# worktree `watcher-primary` -> agent `watcher_primary`; worktree
|
|
90
|
+
# `brainstorm_steward` -> agent `brainstorm_steward`, a no-op fold). Set
|
|
91
|
+
# the AGENT_NAME env var explicitly if a deployment's worktree/agent-name
|
|
92
|
+
# pair doesn't follow that convention.
|
|
93
|
+
#
|
|
94
|
+
# TESTABILITY: every function below is safe to `source` on its own (no
|
|
95
|
+
# top-level side effects, no network calls, no loop) — only `main` (run
|
|
96
|
+
# at the bottom, guarded so `source`-ing this file does not execute it)
|
|
97
|
+
# performs I/O. This lets `resolve_watch_params` and friends be unit
|
|
98
|
+
# tested by sourcing this file and calling them directly.
|
|
99
|
+
set -uo pipefail
|
|
100
|
+
|
|
101
|
+
# ---------------------------------------------------------------------------
|
|
102
|
+
# resolve_thrum_dir — follow .thrum/redirect (factored out
|
|
103
|
+
# of resolve_watch_params/resolve_capture_outdir per dual-review,
|
|
104
|
+
# both of which were independently reimplementing this same body).
|
|
105
|
+
# ---------------------------------------------------------------------------
|
|
106
|
+
# Args: worktree_root. Prints the redirect-resolved absolute .thrum/ dir.
|
|
107
|
+
# Matches the idiom already used by scripts/heartbeat-lib.sh's
|
|
108
|
+
# hb_sessions_dir() and scripts/thrum-check-inbox.sh's SPOOL_THRUM
|
|
109
|
+
# resolution: no redirect file present (main repo) -> use the local .thrum/
|
|
110
|
+
# as-is; redirect file present (feature worktree) -> follow it to the main
|
|
111
|
+
# repo's .thrum/.
|
|
112
|
+
resolve_thrum_dir() {
|
|
113
|
+
local worktree_root="$1" thrum
|
|
114
|
+
thrum="${worktree_root}/.thrum"
|
|
115
|
+
[[ -f "${thrum}/redirect" ]] && thrum="$(tr -d '[:space:]' < "${thrum}/redirect")"
|
|
116
|
+
printf '%s\n' "${thrum}"
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
# ---------------------------------------------------------------------------
|
|
120
|
+
# resolve_watch_params
|
|
121
|
+
# ---------------------------------------------------------------------------
|
|
122
|
+
# Args: worktree_root, agent. Prints the redirect-resolved absolute path
|
|
123
|
+
# to that agent's watch_params.json.
|
|
124
|
+
resolve_watch_params() {
|
|
125
|
+
local worktree_root="$1" agent="$2"
|
|
126
|
+
printf '%s/agents/%s/watch_params.json\n' "$(resolve_thrum_dir "${worktree_root}")" "${agent}"
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
# ---------------------------------------------------------------------------
|
|
130
|
+
# resolve_capture_outdir — same redirect-following idiom as
|
|
131
|
+
# resolve_watch_params, but for roster-capture output.
|
|
132
|
+
# outdir must NEVER be derived from this script's own on-disk location
|
|
133
|
+
# (BASH_SOURCE/SCRIPT_DIR) — that resolves into plugin source whenever the
|
|
134
|
+
# script is invoked directly from its skill/resources directory instead of
|
|
135
|
+
# its deployed <worktree>/.thrum-watch copy. Anchor it instead to the
|
|
136
|
+
# shared, redirect-resolved .thrum/agents/<agent>/ tree, matching where
|
|
137
|
+
# watch_params.json itself already lives.
|
|
138
|
+
# ---------------------------------------------------------------------------
|
|
139
|
+
resolve_capture_outdir() {
|
|
140
|
+
local worktree_root="$1" agent="$2"
|
|
141
|
+
printf '%s/agents/%s/watch-captures\n' "$(resolve_thrum_dir "${worktree_root}")" "${agent}"
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
# ---------------------------------------------------------------------------
|
|
145
|
+
# prune_roster_captures — intentionally preserve roster-capture-*.txt
|
|
146
|
+
# history for watcher review and later training/export.
|
|
147
|
+
# ---------------------------------------------------------------------------
|
|
148
|
+
prune_roster_captures() {
|
|
149
|
+
:
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
utc_now_iso() {
|
|
153
|
+
python3 -c 'import datetime; print(datetime.datetime.now(datetime.timezone.utc).isoformat(timespec="microseconds").replace("+00:00","Z"))'
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
utc_now_file_stamp() {
|
|
157
|
+
python3 -c 'import datetime; print(datetime.datetime.now(datetime.timezone.utc).strftime("%Y%m%d-%H%M%S-%f"))'
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
safe_capture_name() {
|
|
161
|
+
python3 -c 'import re,sys; print(re.sub(r"[^A-Za-z0-9_.-]+", "_", sys.argv[1])[:80] or "agent")' "$1"
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
measure_geometry_json() {
|
|
165
|
+
local session="$1" raw
|
|
166
|
+
[ -n "$session" ] || return 0
|
|
167
|
+
raw=$(tmux display-message -p -t "$session" '#{pane_width} #{pane_height} #{cursor_x} #{cursor_y} #{pane_in_mode}' 2>/dev/null) || return 0
|
|
168
|
+
python3 -c '
|
|
169
|
+
import json, sys
|
|
170
|
+
parts=sys.argv[1].split("\t")
|
|
171
|
+
if len(parts) != 5:
|
|
172
|
+
sys.exit(0)
|
|
173
|
+
try:
|
|
174
|
+
width=int(parts[0]); height=int(parts[1]); cursor_x=int(parts[2]); cursor_y=int(parts[3])
|
|
175
|
+
except Exception:
|
|
176
|
+
sys.exit(0)
|
|
177
|
+
print(json.dumps({
|
|
178
|
+
"pane_width": width,
|
|
179
|
+
"pane_height": height,
|
|
180
|
+
"cursor_x": cursor_x,
|
|
181
|
+
"cursor_y": cursor_y,
|
|
182
|
+
"pane_in_mode": parts[4],
|
|
183
|
+
"source": "tmux display-message",
|
|
184
|
+
}, sort_keys=True))
|
|
185
|
+
' "$raw" 2>/dev/null
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
agent_is_local() {
|
|
189
|
+
local agent="$1"
|
|
190
|
+
thrum team "@$agent" --offline --json 2>/dev/null | python3 -c '
|
|
191
|
+
import json, sys
|
|
192
|
+
agent=sys.argv[1]
|
|
193
|
+
try:
|
|
194
|
+
data=json.load(sys.stdin)
|
|
195
|
+
except Exception:
|
|
196
|
+
sys.exit(1)
|
|
197
|
+
for member in data.get("team", {}).get("members", []):
|
|
198
|
+
if member.get("agent_id") == agent:
|
|
199
|
+
sys.exit(0 if member.get("is_local") is True else 1)
|
|
200
|
+
sys.exit(1)
|
|
201
|
+
' "$agent" 2>/dev/null
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
record_archive_degraded() {
|
|
205
|
+
local outdir="$1" capture_id="$2" component="$3" agent="$4" message="$5"
|
|
206
|
+
local archive_dir="${outdir}/archive" health_dir="${archive_dir}/health"
|
|
207
|
+
mkdir -p "$health_dir" 2>/dev/null || true
|
|
208
|
+
chmod 700 "$archive_dir" "$health_dir" 2>/dev/null || true
|
|
209
|
+
ARCHIVE_DEGRADED=1
|
|
210
|
+
ARCHIVE_DEGRADED_MESSAGES+=("${component}${agent:+:$agent}")
|
|
211
|
+
python3 -c '
|
|
212
|
+
import json, os, sys, datetime
|
|
213
|
+
out=sys.argv[1]
|
|
214
|
+
payload={
|
|
215
|
+
"schema_version": "roster-capture-archive-health-v1",
|
|
216
|
+
"status": "archive_degraded",
|
|
217
|
+
"capture_id": sys.argv[2],
|
|
218
|
+
"component": sys.argv[3],
|
|
219
|
+
"agent": sys.argv[4] or None,
|
|
220
|
+
"message": sys.argv[5],
|
|
221
|
+
"recorded_at_utc": datetime.datetime.now(datetime.timezone.utc).isoformat(timespec="microseconds").replace("+00:00","Z"),
|
|
222
|
+
}
|
|
223
|
+
stamp = payload["recorded_at_utc"].replace(":","").replace("-","")
|
|
224
|
+
agent = payload["agent"] or "source"
|
|
225
|
+
component = payload["component"]
|
|
226
|
+
name = f"{stamp}_{component}_{agent}.json"
|
|
227
|
+
path=os.path.join(out, name)
|
|
228
|
+
tmp=path+f".{os.getpid()}.tmp"
|
|
229
|
+
with open(tmp, "w", encoding="utf-8") as fh:
|
|
230
|
+
fh.write(json.dumps(payload, sort_keys=True, separators=(",", ":"))+"\n")
|
|
231
|
+
fh.flush(); os.fsync(fh.fileno())
|
|
232
|
+
os.chmod(tmp, 0o600)
|
|
233
|
+
os.replace(tmp, path)
|
|
234
|
+
' "$health_dir" "$capture_id" "$component" "$agent" "$message" 2>/dev/null || true
|
|
235
|
+
printf '%s\t%s\t%s\t%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$capture_id" "$component${agent:+:$agent}" "$message" >>"${archive_dir}/archive-health.log" 2>/dev/null || true
|
|
236
|
+
chmod 600 "${archive_dir}/archive-health.log" 2>/dev/null || true
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
archive_source_file() {
|
|
240
|
+
local outdir="$1" file="$2" capture_id="$3" ts="$4"
|
|
241
|
+
local archive_py="${SCRIPT_DIR}/capture_archive.py"
|
|
242
|
+
local archive_dir="${outdir}/archive"
|
|
243
|
+
if [ ! -f "$archive_py" ]; then
|
|
244
|
+
echo "capture archive warning: missing helper $archive_py; source archive skipped" >&2
|
|
245
|
+
record_archive_degraded "$outdir" "$capture_id" "archive_source_missing_helper" "" "missing helper $archive_py"
|
|
246
|
+
return 0
|
|
247
|
+
fi
|
|
248
|
+
mkdir -p "$archive_dir" 2>/dev/null || true
|
|
249
|
+
chmod 700 "$archive_dir" 2>/dev/null || true
|
|
250
|
+
python3 "$archive_py" archive-source \
|
|
251
|
+
--archive-dir "$archive_dir" \
|
|
252
|
+
--source "$file" \
|
|
253
|
+
--capture-id "$capture_id" \
|
|
254
|
+
--capture-timestamp-utc "$ts" >/dev/null 2>>"${archive_dir}/archive-errors.log" || \
|
|
255
|
+
{ echo "capture archive warning: source archive failed for $file" >&2; record_archive_degraded "$outdir" "$capture_id" "archive_source_failed" "" "source archive failed for $file"; }
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
archive_agent_file() {
|
|
259
|
+
local outdir="$1" agent="$2" raw_file="$3" source_file="$4" capture_id="$5" ts="$6" cap_rc="$7" requested_lines="$8" runtime="${9-}" geometry_json="${10-}" capture_route="${11-}" primary_rc="${12-}" ssh_rc="${13-}" final_rc="${14-}"
|
|
260
|
+
local archive_py="${SCRIPT_DIR}/capture_archive.py"
|
|
261
|
+
local archive_dir="${outdir}/archive"
|
|
262
|
+
if [ ! -f "$archive_py" ]; then
|
|
263
|
+
echo "capture archive warning: missing helper $archive_py; agent archive skipped for $agent" >&2
|
|
264
|
+
record_archive_degraded "$outdir" "$capture_id" "archive_agent_missing_helper" "$agent" "missing helper $archive_py"
|
|
265
|
+
return 0
|
|
266
|
+
fi
|
|
267
|
+
mkdir -p "$archive_dir" 2>/dev/null || true
|
|
268
|
+
chmod 700 "$archive_dir" 2>/dev/null || true
|
|
269
|
+
local cmd=(python3 "$archive_py" archive-agent
|
|
270
|
+
--archive-dir "$archive_dir"
|
|
271
|
+
--agent "$agent"
|
|
272
|
+
--raw-file "$raw_file"
|
|
273
|
+
--source-file "$source_file"
|
|
274
|
+
--capture-id "$capture_id"
|
|
275
|
+
--capture-timestamp-utc "$ts"
|
|
276
|
+
--capture-rc "$cap_rc"
|
|
277
|
+
--requested-lines "$requested_lines")
|
|
278
|
+
[ -n "$runtime" ] && cmd+=(--runtime "$runtime")
|
|
279
|
+
[ -n "$geometry_json" ] && cmd+=(--geometry-json "$geometry_json")
|
|
280
|
+
[ -n "$capture_route" ] && cmd+=(--capture-route "$capture_route")
|
|
281
|
+
[ -n "$primary_rc" ] && cmd+=(--primary-rc "$primary_rc")
|
|
282
|
+
[ -n "$ssh_rc" ] && cmd+=(--ssh-rc "$ssh_rc")
|
|
283
|
+
[ -n "$final_rc" ] && cmd+=(--final-rc "$final_rc")
|
|
284
|
+
"${cmd[@]}" >/dev/null 2>>"${archive_dir}/archive-errors.log" || \
|
|
285
|
+
{ echo "capture archive warning: agent archive failed for $agent in $capture_id" >&2; record_archive_degraded "$outdir" "$capture_id" "archive_agent_failed" "$agent" "agent archive failed for $agent in $capture_id"; }
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
write_raw_from_capture_json() {
|
|
289
|
+
local json_file="$1" raw_file="$2"
|
|
290
|
+
python3 -c '
|
|
291
|
+
import json, pathlib, sys
|
|
292
|
+
doc=json.load(open(sys.argv[1], encoding="utf-8"))
|
|
293
|
+
if not doc.get("ok"):
|
|
294
|
+
raise SystemExit(1)
|
|
295
|
+
lines=doc.get("lines")
|
|
296
|
+
if not isinstance(lines, list) or any(not isinstance(line, str) for line in lines):
|
|
297
|
+
raise SystemExit(1)
|
|
298
|
+
ghost_lines=doc.get("ghost_lines") or []
|
|
299
|
+
if not isinstance(ghost_lines, list) or any(not isinstance(i, int) for i in ghost_lines):
|
|
300
|
+
raise SystemExit(1)
|
|
301
|
+
out=[]
|
|
302
|
+
out.extend(lines)
|
|
303
|
+
if ghost_lines or doc.get("has_ghost") or doc.get("composer_text"):
|
|
304
|
+
out.append("--- GHOST METADATA (structured; non-submitted suggestion text, not pane input) ---")
|
|
305
|
+
if ghost_lines:
|
|
306
|
+
out.append("ghost_lines: " + ",".join(str(i) for i in ghost_lines))
|
|
307
|
+
if doc.get("composer_text"):
|
|
308
|
+
out.append("ghost_text: " + str(doc.get("composer_text")))
|
|
309
|
+
text="\n".join(out)
|
|
310
|
+
if text:
|
|
311
|
+
text += "\n"
|
|
312
|
+
path=pathlib.Path(sys.argv[2])
|
|
313
|
+
path.write_text(text, encoding="utf-8")
|
|
314
|
+
' "$json_file" "$raw_file"
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
write_filter_manifest_row() {
|
|
318
|
+
local manifest="$1" agent="$2" raw_file="$3" source_file="$4" capture_id="$5" ts="$6" cap_rc="$7" route="$8" provider_input_file="${9-}" provider_input_source="${10-}" capture_json_file="${11-}"
|
|
319
|
+
python3 -c '
|
|
320
|
+
import json, sys
|
|
321
|
+
row={
|
|
322
|
+
"agent": sys.argv[1],
|
|
323
|
+
"raw_file": sys.argv[2],
|
|
324
|
+
"source_file": sys.argv[3],
|
|
325
|
+
"capture_id": sys.argv[4],
|
|
326
|
+
"capture_timestamp_utc": sys.argv[5],
|
|
327
|
+
"capture_rc": int(sys.argv[6]),
|
|
328
|
+
"capture_route": sys.argv[7],
|
|
329
|
+
}
|
|
330
|
+
if sys.argv[8]:
|
|
331
|
+
row["provider_input_file"] = sys.argv[8]
|
|
332
|
+
if sys.argv[9]:
|
|
333
|
+
row["provider_input_source"] = sys.argv[9]
|
|
334
|
+
if sys.argv[10]:
|
|
335
|
+
row["capture_json_file"] = sys.argv[10]
|
|
336
|
+
print(json.dumps(row, sort_keys=True, separators=(",", ":")))
|
|
337
|
+
' "$agent" "$raw_file" "$source_file" "$capture_id" "$ts" "$cap_rc" "$route" "$provider_input_file" "$provider_input_source" "$capture_json_file" >>"$manifest"
|
|
338
|
+
}
|
|
339
|
+
|
|
340
|
+
# jev_key_present — true (rc 0) iff a JEV provider key is actually available:
|
|
341
|
+
# checks THRUM_TYPESAFE_KEY/OPENROUTER_API_KEY in the process environment
|
|
342
|
+
# first, then falls back to parsing the resolved env file ($1) as DATA
|
|
343
|
+
# (never sourced/eval'd) using the same key-matching + quote-stripping
|
|
344
|
+
# convention as jev_watcher_filter.py's read_env_keys().
|
|
345
|
+
jev_key_present() {
|
|
346
|
+
local env_file="$1"
|
|
347
|
+
if [ -n "${THRUM_TYPESAFE_KEY:-}" ] || [ -n "${OPENROUTER_API_KEY:-}" ]; then
|
|
348
|
+
return 0
|
|
349
|
+
fi
|
|
350
|
+
[ -n "$env_file" ] && [ -f "$env_file" ] || return 1
|
|
351
|
+
python3 -c '
|
|
352
|
+
import sys
|
|
353
|
+
env_file = sys.argv[1]
|
|
354
|
+
names = ("THRUM_TYPESAFE_KEY", "OPENROUTER_API_KEY")
|
|
355
|
+
try:
|
|
356
|
+
with open(env_file, encoding="utf-8", errors="replace") as fh:
|
|
357
|
+
text = fh.read()
|
|
358
|
+
except OSError:
|
|
359
|
+
sys.exit(1)
|
|
360
|
+
for line in text.splitlines():
|
|
361
|
+
if "=" not in line:
|
|
362
|
+
continue
|
|
363
|
+
name, value = line.split("=", 1)
|
|
364
|
+
if name not in names:
|
|
365
|
+
continue
|
|
366
|
+
value = value.strip().strip("\"").strip("'"'"'")
|
|
367
|
+
if value:
|
|
368
|
+
sys.exit(0)
|
|
369
|
+
sys.exit(1)
|
|
370
|
+
' "$env_file"
|
|
371
|
+
}
|
|
372
|
+
|
|
373
|
+
run_jev_filter() {
|
|
374
|
+
local outdir="$1" manifest="$2" capture_id="$3"
|
|
375
|
+
local filter_py="${SCRIPT_DIR}/jev_watcher_filter.py"
|
|
376
|
+
local decisions_dir="${outdir}/archive/jev-decisions/${capture_id}"
|
|
377
|
+
JEV_FILTER_MODE=fallback
|
|
378
|
+
JEV_FILTER_ATTENTION=0
|
|
379
|
+
if [ ! -f "$filter_py" ]; then
|
|
380
|
+
echo "jev filter warning: missing helper; using watcher fallback" >&2
|
|
381
|
+
return 0
|
|
382
|
+
fi
|
|
383
|
+
mkdir -p "$decisions_dir" 2>/dev/null || true
|
|
384
|
+
chmod 700 "${outdir}/archive" "${outdir}/archive/jev-decisions" "$decisions_dir" 2>/dev/null || true
|
|
385
|
+
local env_file="${ROSTER_JEV_ENV_FILE:-${REPO_ROOT}/.env}"
|
|
386
|
+
local cmd=(python3 "$filter_py" --manifest "$manifest" --out-dir "$decisions_dir" --env-file "$env_file"
|
|
387
|
+
--timeout "${ROSTER_JEV_TIMEOUT:-8}" --retries "${ROSTER_JEV_RETRIES:-1}" --concurrency "${ROSTER_JEV_CONCURRENCY:-4}")
|
|
388
|
+
[ -n "${ROSTER_JEV_EGRESS_AUDIT_DIR:-}" ] && cmd+=(--egress-audit-dir "$ROSTER_JEV_EGRESS_AUDIT_DIR")
|
|
389
|
+
if ! "${cmd[@]}" >"${decisions_dir}/filter-output.json" 2>>"${decisions_dir}/filter-errors.log"; then
|
|
390
|
+
echo "jev filter warning: execution failed; using watcher fallback" >&2
|
|
391
|
+
return 0
|
|
392
|
+
fi
|
|
393
|
+
local parsed
|
|
394
|
+
if ! parsed=$(python3 - "${decisions_dir}/summary.json" "${ROSTER[@]}" <<'PY'
|
|
395
|
+
import json, sys
|
|
396
|
+
try:
|
|
397
|
+
summary = json.load(open(sys.argv[1], encoding="utf-8"))
|
|
398
|
+
expected = sys.argv[2:]
|
|
399
|
+
rows = summary["results"]
|
|
400
|
+
if not isinstance(rows, list) or len(rows) != len(expected):
|
|
401
|
+
raise ValueError("incomplete results")
|
|
402
|
+
if sorted(row["agent"] for row in rows) != sorted(expected):
|
|
403
|
+
raise ValueError("roster mismatch")
|
|
404
|
+
attention = fallback = 0
|
|
405
|
+
for row in rows:
|
|
406
|
+
status, route = row.get("filter_status"), row.get("route")
|
|
407
|
+
if status == "ok" and route == "archive_only" and row.get("reason") == "confident_ordinary":
|
|
408
|
+
continue
|
|
409
|
+
if status == "ok" and route == "notify_watcher" and row.get("reason") in ("active_permission_prompt", "blocking_user_question_tui"):
|
|
410
|
+
attention += 1
|
|
411
|
+
else:
|
|
412
|
+
fallback += 1
|
|
413
|
+
print(f"{attention}|{fallback}")
|
|
414
|
+
except (OSError, ValueError, TypeError, KeyError, json.JSONDecodeError):
|
|
415
|
+
raise SystemExit(1)
|
|
416
|
+
PY
|
|
417
|
+
); then
|
|
418
|
+
echo "jev filter warning: malformed or incomplete summary; using watcher fallback" >&2
|
|
419
|
+
return 0
|
|
420
|
+
fi
|
|
421
|
+
local fallback_count
|
|
422
|
+
IFS='|' read -r JEV_FILTER_ATTENTION fallback_count <<<"$parsed"
|
|
423
|
+
if [ "$fallback_count" -gt 0 ]; then
|
|
424
|
+
echo "jev filter warning: uncertain classification; using watcher fallback" >&2
|
|
425
|
+
return 0
|
|
426
|
+
fi
|
|
427
|
+
JEV_FILTER_MODE=ok
|
|
428
|
+
}
|
|
429
|
+
|
|
430
|
+
# ---------------------------------------------------------------------------
|
|
431
|
+
# resolve_session — resolve an agent name to its tmux session name via
|
|
432
|
+
# `thrum tmux status --json` (ported from watcher-primary; needed only for
|
|
433
|
+
# the ghost-tip raw-capture pass below, which reads the tmux pane
|
|
434
|
+
# directly rather than through `thrum tmux capture`). Falls back to the
|
|
435
|
+
# agent name itself if resolution fails (session name sometimes equals
|
|
436
|
+
# agent name).
|
|
437
|
+
# ---------------------------------------------------------------------------
|
|
438
|
+
resolve_session() {
|
|
439
|
+
local agent="$1"
|
|
440
|
+
local session
|
|
441
|
+
session=$(thrum tmux status --json 2>/dev/null | python3 -c "
|
|
442
|
+
import json, sys
|
|
443
|
+
try:
|
|
444
|
+
d = json.load(sys.stdin)
|
|
445
|
+
except Exception:
|
|
446
|
+
sys.exit(1)
|
|
447
|
+
for s in d.get('sessions', []):
|
|
448
|
+
if s.get('agent') == '$agent':
|
|
449
|
+
print(s.get('name', ''))
|
|
450
|
+
break
|
|
451
|
+
" 2>/dev/null)
|
|
452
|
+
if [ -z "$session" ]; then
|
|
453
|
+
session="$agent"
|
|
454
|
+
fi
|
|
455
|
+
printf '%s' "$session"
|
|
456
|
+
}
|
|
457
|
+
|
|
458
|
+
# ---------------------------------------------------------------------------
|
|
459
|
+
# check_ghost_tip — see the GHOST-TIP DETECTION header comment above.
|
|
460
|
+
# Skips cleanly (no-op) if the sibling detect_ghost_tip.py hasn't been
|
|
461
|
+
# deployed alongside this script (it ships separately, per the
|
|
462
|
+
# persistent-watcher-archetype skill's "copy BOTH ..." setup recipe).
|
|
463
|
+
# ---------------------------------------------------------------------------
|
|
464
|
+
check_ghost_tip() {
|
|
465
|
+
local session="$1"
|
|
466
|
+
[ -n "${DETECTOR:-}" ] && [ -f "${DETECTOR}" ] || return 0
|
|
467
|
+
local raw
|
|
468
|
+
raw=$(tmux -L default capture-pane -e -p -t "$session" -S -6 2>/dev/null)
|
|
469
|
+
if [ -z "$raw" ]; then
|
|
470
|
+
local sock
|
|
471
|
+
for sock in $(ls "${TMUX_TMPDIR:-/tmp}/tmux-$(id -u)/" 2>/dev/null | grep -v '^te-' | head -20); do
|
|
472
|
+
raw=$(tmux -L "$sock" capture-pane -e -p -t "$session" -S -6 2>/dev/null)
|
|
473
|
+
[ -n "$raw" ] && break
|
|
474
|
+
done
|
|
475
|
+
fi
|
|
476
|
+
[ -z "$raw" ] && return 0
|
|
477
|
+
|
|
478
|
+
local tip
|
|
479
|
+
tip=$(printf '%s' "$raw" | python3 "$DETECTOR" 2>/dev/null | head -1)
|
|
480
|
+
if [ -n "$tip" ]; then
|
|
481
|
+
echo "--- GHOST-TIP (dim/unaccepted autocomplete on the input line — NOT real typed input; do NOT nudge/escalate on it): ${tip#GHOST-TIP: } ---"
|
|
482
|
+
fi
|
|
483
|
+
}
|
|
484
|
+
|
|
485
|
+
# ---------------------------------------------------------------------------
|
|
486
|
+
# quote_shell / resolve_capture_target / ssh_capture_pane — SSH fallback,
|
|
487
|
+
# ported verbatim from brainstorm-steward (watcher-primary had none).
|
|
488
|
+
# ---------------------------------------------------------------------------
|
|
489
|
+
quote_shell() {
|
|
490
|
+
python3 -c 'import shlex,sys; print(shlex.quote(sys.argv[1]))' "$1"
|
|
491
|
+
}
|
|
492
|
+
|
|
493
|
+
resolve_capture_target() {
|
|
494
|
+
local agent="$1" team_json host worktree
|
|
495
|
+
# Scoped member read: this fallback runs per failed pane, so `team
|
|
496
|
+
# --all` would repeatedly pull the entire fleet and fan out under an
|
|
497
|
+
# outage. The scoped form carries the same member fields we need below.
|
|
498
|
+
if ! team_json="$(thrum team "@$agent" --offline --json 2>/dev/null)"; then
|
|
499
|
+
return 1
|
|
500
|
+
fi
|
|
501
|
+
host="$(printf '%s' "$team_json" | python3 -c '
|
|
502
|
+
import json,sys
|
|
503
|
+
agent=sys.argv[1]
|
|
504
|
+
data=json.load(sys.stdin)
|
|
505
|
+
for member in data.get("team", {}).get("members", []):
|
|
506
|
+
if member.get("agent_id") == agent:
|
|
507
|
+
print(member.get("hostname") or "")
|
|
508
|
+
break
|
|
509
|
+
' "$agent")"
|
|
510
|
+
worktree="$(printf '%s' "$team_json" | python3 -c '
|
|
511
|
+
import json,sys
|
|
512
|
+
agent=sys.argv[1]
|
|
513
|
+
data=json.load(sys.stdin)
|
|
514
|
+
for member in data.get("team", {}).get("members", []):
|
|
515
|
+
if member.get("agent_id") == agent:
|
|
516
|
+
print(member.get("worktree") or "")
|
|
517
|
+
break
|
|
518
|
+
' "$agent")"
|
|
519
|
+
# Thrum session IDs (ses_...) are not tmux targets. Persistent panes use
|
|
520
|
+
# both conventions in the fleet: some use the agent ID, while others use
|
|
521
|
+
# the worktree basename. Return both and let the remote tmux server
|
|
522
|
+
# choose the first existing session; this remains useful while daemon
|
|
523
|
+
# routing is circuit-open and `thrum tmux status` cannot resolve a
|
|
524
|
+
# canonical name.
|
|
525
|
+
[ -n "$host" ] && [ -n "$agent" ] || return 1
|
|
526
|
+
printf '%s|%s|%s\n' "$host" "$agent" "${worktree##*/}"
|
|
527
|
+
}
|
|
528
|
+
|
|
529
|
+
ssh_capture_pane() {
|
|
530
|
+
local agent="$1" declared_host="${2-}" declared_session="${3-}"
|
|
531
|
+
local host agent_session worktree_session worktree_normalized
|
|
532
|
+
local agent_quoted worktree_quoted normalized_quoted remote_cmd
|
|
533
|
+
local target
|
|
534
|
+
if [ -n "$declared_host" ] || [ -n "$declared_session" ]; then
|
|
535
|
+
[ -n "$declared_host" ] && [ -n "$declared_session" ] || return 1
|
|
536
|
+
host="$declared_host"
|
|
537
|
+
agent_quoted="$(quote_shell "$declared_session")"
|
|
538
|
+
remote_cmd="tmux has-session -t ${agent_quoted} 2>/dev/null || exit 1; tmux capture-pane -p -J -t ${agent_quoted} -S -30"
|
|
539
|
+
ssh -n -o BatchMode=yes -o ConnectTimeout=5 -o ServerAliveInterval=5 -o ServerAliveCountMax=1 "$host" "$remote_cmd" 2>&1
|
|
540
|
+
return $?
|
|
541
|
+
fi
|
|
542
|
+
if ! target="$(resolve_capture_target "$agent")"; then
|
|
543
|
+
return 1
|
|
544
|
+
fi
|
|
545
|
+
host="${target%%|*}"
|
|
546
|
+
target="${target#*|}"
|
|
547
|
+
agent_session="${target%%|*}"
|
|
548
|
+
worktree_session="${target#*|}"
|
|
549
|
+
[ -n "$host" ] && [ -n "$agent_session" ] || return 1
|
|
550
|
+
# Worktree basenames can contain dots while tmux launch normalizes them
|
|
551
|
+
# to hyphens (for example my.thrum.team -> my-thrum-team). Try both
|
|
552
|
+
# forms.
|
|
553
|
+
worktree_normalized="${worktree_session//./-}"
|
|
554
|
+
agent_quoted="$(quote_shell "$agent_session")"
|
|
555
|
+
worktree_quoted="$(quote_shell "$worktree_session")"
|
|
556
|
+
normalized_quoted="$(quote_shell "$worktree_normalized")"
|
|
557
|
+
remote_cmd="for session in ${agent_quoted} ${worktree_quoted} ${normalized_quoted}; do if tmux has-session -t \"\$session\" 2>/dev/null; then tmux capture-pane -p -J -t \"\$session\" -S -30; exit \$?; fi; done; exit 1"
|
|
558
|
+
ssh -n -o BatchMode=yes -o ConnectTimeout=5 -o ServerAliveInterval=5 -o ServerAliveCountMax=1 "$host" "$remote_cmd" 2>&1
|
|
559
|
+
}
|
|
560
|
+
|
|
561
|
+
# ---------------------------------------------------------------------------
|
|
562
|
+
# probe_agent — SHARED DEGRADED-KNOWN / hard-FAIL PID-liveness
|
|
563
|
+
# classification. Both source scripts had this exact logic (the same
|
|
564
|
+
# two-proof design: independently-verified-live-
|
|
565
|
+
# PID OR fresh direct-message/session-artifact) — watcher-primary as an
|
|
566
|
+
# extracted function, brainstorm-steward inlined a second, drifted copy.
|
|
567
|
+
# Factored into ONE function here so there is exactly one place to fix it.
|
|
568
|
+
#
|
|
569
|
+
# Args: agent, first-line-of-the-failed-primary-capture-output (for the
|
|
570
|
+
# message only). Prints the classification line to stdout. Side effect:
|
|
571
|
+
# appends the agent id to the global DEGRADED or FAILED array (both must
|
|
572
|
+
# be declared by the caller before use).
|
|
573
|
+
# ---------------------------------------------------------------------------
|
|
574
|
+
probe_agent() {
|
|
575
|
+
local a="$1" firstline="$2"
|
|
576
|
+
local pid="" hostname="" last_seen="" is_local="" session_id="" inbox_total="0" state=""
|
|
577
|
+
# --offline: the online-only read omits offline agents entirely (empty
|
|
578
|
+
# members[]), which false-fails the proof; the offline record carries
|
|
579
|
+
# the same fields plus last-known state.
|
|
580
|
+
eval "$(thrum team "@$a" --offline --json 2>/dev/null | python3 -c "
|
|
581
|
+
import json,sys
|
|
582
|
+
agent=sys.argv[1]
|
|
583
|
+
try:
|
|
584
|
+
d=json.load(sys.stdin)
|
|
585
|
+
except: sys.exit(0)
|
|
586
|
+
for m in d.get('team',{}).get('members',[]):
|
|
587
|
+
if m.get('agent_id')==agent:
|
|
588
|
+
print(f\"pid={m.get('agent_pid') or 0}\")
|
|
589
|
+
print(f\"hostname={m.get('hostname') or ''}\")
|
|
590
|
+
print(f\"last_seen={m.get('last_seen') or ''}\")
|
|
591
|
+
print(f\"is_local={str(m.get('is_local')).lower()}\")
|
|
592
|
+
print(f\"state={m.get('state') or ''}\")
|
|
593
|
+
print(f\"session_id={m.get('session_id') or ''}\")
|
|
594
|
+
print(f\"inbox_total={m.get('inbox_total') or 0}\")
|
|
595
|
+
break
|
|
596
|
+
" "$a" 2>/dev/null)"
|
|
597
|
+
local live_pid_verified=0 fresh_msg_verified=0
|
|
598
|
+
# Case 1: independently verified live PID (ps on the owning host, not
|
|
599
|
+
# just team pid != 0).
|
|
600
|
+
if [ -n "$pid" ] && [ "$pid" != "0" ] && [ -n "$hostname" ] && { [ -z "$state" ] || [ "$state" = "alive" ]; }; then
|
|
601
|
+
if [ "$is_local" = "true" ] || [ "$hostname" = "$(hostname)" ] || [ "$hostname" = "$(hostname).local" ]; then
|
|
602
|
+
if ps -p "$pid" >/dev/null 2>&1; then live_pid_verified=1; fi
|
|
603
|
+
else
|
|
604
|
+
if ssh -n -o ConnectTimeout=5 -o BatchMode=yes "$hostname" "ps -p $pid >/dev/null 2>&1" 2>/dev/null; then live_pid_verified=1; fi
|
|
605
|
+
if [ $live_pid_verified -eq 0 ]; then
|
|
606
|
+
sess=$(thrum tmux status --json 2>/dev/null | python3 -c "import json,sys; d=json.load(sys.stdin); [print(s.get('name','')) for s in d.get('sessions',[]) if s.get('agent')==sys.argv[1]]" "$a" 2>/dev/null | head -1)
|
|
607
|
+
if [ -n "$sess" ]; then
|
|
608
|
+
sess_quoted="$(quote_shell "$sess")"
|
|
609
|
+
if ssh -n -o ConnectTimeout=5 -o BatchMode=yes "$hostname" "tmux has-session -t ${sess_quoted} 2>/dev/null" 2>/dev/null; then live_pid_verified=1; fi
|
|
610
|
+
fi
|
|
611
|
+
fi
|
|
612
|
+
fi
|
|
613
|
+
fi
|
|
614
|
+
# Case 2: fresh direct-message/session-artifact (last_seen within 15m,
|
|
615
|
+
# single snapshot carries session_id/inbox_total). The monitor runs
|
|
616
|
+
# every 10m, so a 5m window falsely fails healthy agents between
|
|
617
|
+
# captures.
|
|
618
|
+
if [ -n "$last_seen" ] && { [ -z "$state" ] || [ "$state" = "alive" ]; }; then
|
|
619
|
+
fresh_msg_verified=$(python3 -c "
|
|
620
|
+
import sys, datetime
|
|
621
|
+
s=sys.argv[1]
|
|
622
|
+
try:
|
|
623
|
+
if s.endswith('Z'): s=s[:-1]+'+00:00'
|
|
624
|
+
dt=datetime.datetime.fromisoformat(s)
|
|
625
|
+
now=datetime.datetime.now(datetime.timezone.utc)
|
|
626
|
+
delta=(now-dt).total_seconds()
|
|
627
|
+
has_artifact=bool(sys.argv[2]) or int(sys.argv[3]) > 0
|
|
628
|
+
print(1 if 0 <= delta < 900 and has_artifact else 0)
|
|
629
|
+
except: print(0)
|
|
630
|
+
" "$last_seen" "$session_id" "$inbox_total" 2>/dev/null)
|
|
631
|
+
fi
|
|
632
|
+
# Negative controls: stale last_seen (>15m) and dead/reused PID (ps
|
|
633
|
+
# fails) remain hard FAIL unless other proof.
|
|
634
|
+
if [ "${live_pid_verified:-0}" -eq 1 ] || [ "${fresh_msg_verified:-0}" -eq 1 ]; then
|
|
635
|
+
proof=""
|
|
636
|
+
[ "${live_pid_verified:-0}" -eq 1 ] && proof="independently verified live PID ${pid:-unknown} on ${hostname}"
|
|
637
|
+
[ "${fresh_msg_verified:-0}" -eq 1 ] && proof="${proof:+$proof + }fresh direct-message/session-artifact (last_seen ${last_seen})"
|
|
638
|
+
echo "--- DEGRADED-KNOWN for $a: thrum tmux capture failed (capability-denied/empty binding: ${firstline}) but LIVE by fallback proof ($proof) — visible warning; use direct-message/session-artifact fallback, not repeated hard FAIL ---"
|
|
639
|
+
DEGRADED+=("$a")
|
|
640
|
+
else
|
|
641
|
+
echo "--- CAPTURE FAILED for $a (capability-denied/empty binding — no independent live proof: true pid0/dead PID or stale last_seen) — hard FAIL ---"
|
|
642
|
+
FAILED+=("$a")
|
|
643
|
+
fi
|
|
644
|
+
}
|
|
645
|
+
|
|
646
|
+
# ---------------------------------------------------------------------------
|
|
647
|
+
# roster_self_check — ported from watcher-primary: verify every ROSTER
|
|
648
|
+
# entry resolves to a real agent_id via `thrum team --offline --json`, and
|
|
649
|
+
# log ROSTER-CHECK OK/FAIL. Cheap and valuable regardless of execution
|
|
650
|
+
# model (catches the exact "bare `<box>`-as-hostname" class of stale-
|
|
651
|
+
# roster-entry bug watcher-primary hit 2026-07-24..2026-09-06). Runs AFTER
|
|
652
|
+
# the dynamic roster is loaded from watch_params.json, not against a
|
|
653
|
+
# hardcoded array.
|
|
654
|
+
# ---------------------------------------------------------------------------
|
|
655
|
+
roster_self_check() {
|
|
656
|
+
local outdir="$1"; shift
|
|
657
|
+
local roster=("$@")
|
|
658
|
+
local bad="" r
|
|
659
|
+
for r in "${roster[@]}"; do
|
|
660
|
+
if ! thrum team "@$r" --offline --json 2>/dev/null | python3 -c "
|
|
661
|
+
import json,sys
|
|
662
|
+
agent=sys.argv[1]
|
|
663
|
+
try: ms=json.load(sys.stdin)['team']['members']
|
|
664
|
+
except: sys.exit(1)
|
|
665
|
+
sys.exit(0 if any(m.get('agent_id')==agent for m in ms) else 1)
|
|
666
|
+
" "$r" 2>/dev/null; then
|
|
667
|
+
bad="$bad $r"
|
|
668
|
+
fi
|
|
669
|
+
done
|
|
670
|
+
if [ -n "$bad" ]; then
|
|
671
|
+
ROSTER_SELF_CHECK_FAILED=1
|
|
672
|
+
echo "ROSTER-CHECK FAIL: no such agent(s):$bad — remove or fix before trusting this run's coverage" | tee "$outdir/roster-check.log"
|
|
673
|
+
else
|
|
674
|
+
echo "ROSTER-CHECK OK: ${#roster[@]} entries resolve" > "$outdir/roster-check.log"
|
|
675
|
+
fi
|
|
676
|
+
}
|
|
677
|
+
|
|
678
|
+
# ---------------------------------------------------------------------------
|
|
679
|
+
# run_self_submit_enter — LOOP MODE ONLY (watcher-primary): 10s after the
|
|
680
|
+
# ready-line notice, submit the watcher's OWN pane with Enter so a message
|
|
681
|
+
# queued while the loop was mid-capture gets processed. GUARDED: never
|
|
682
|
+
# send Enter while a permission modal is open in the watcher's own pane
|
|
683
|
+
# (Enter would approve the default option unseen); only submit when a
|
|
684
|
+
# message-waiting notice is visible.
|
|
685
|
+
# ---------------------------------------------------------------------------
|
|
686
|
+
run_self_submit_enter() {
|
|
687
|
+
local self_agent="$1"
|
|
688
|
+
sleep 10
|
|
689
|
+
local me_out
|
|
690
|
+
me_out=$(thrum tmux capture "$self_agent" --lines 15 2>&1)
|
|
691
|
+
if echo "$me_out" | grep -q "run 'thrum inbox --unread' to read"; then
|
|
692
|
+
if echo "$me_out" | grep -qiE "Do you want to proceed\?|requires confirmation|Permission rule"; then
|
|
693
|
+
echo "self-submit skipped: modal open in ${self_agent} pane" >&2
|
|
694
|
+
else
|
|
695
|
+
thrum tmux send "$self_agent" --keys "Enter" >/dev/null 2>&1 || true
|
|
696
|
+
fi
|
|
697
|
+
fi
|
|
698
|
+
}
|
|
699
|
+
|
|
700
|
+
# ---------------------------------------------------------------------------
|
|
701
|
+
# load_roster — populate the global ROSTER / ROSTER_HOSTNAME /
|
|
702
|
+
# ROSTER_TMUX_SESSION / ROSTER_RUNTIME / ROSTER_DAEMON_OVERRIDE parallel
|
|
703
|
+
# arrays from watch_params.json (bash 3.2
|
|
704
|
+
# portable — no mapfile, no associative arrays). Entries may remain legacy
|
|
705
|
+
# plain strings, or may declare the exact remote capture target as
|
|
706
|
+
# {"agent_id","hostname","tmux_session"}; the declared form is
|
|
707
|
+
# authoritative for SSH fallback (never rediscover its host or guess a
|
|
708
|
+
# pane). This dynamic read is brainstorm-steward's design and already
|
|
709
|
+
# fixes the OTHER bug (hardcoded ROSTER array, drifted from
|
|
710
|
+
# watch_params.json in both directions) that watcher-primary's copy still
|
|
711
|
+
# has in full — do not reintroduce a hardcoded array here.
|
|
712
|
+
#
|
|
713
|
+
# ROSTER_DAEMON_OVERRIDE is sourced from the OPTIONAL top-level
|
|
714
|
+
# "capture_daemon_overrides" map: {"<agent>": "<daemon_id>"}. Default
|
|
715
|
+
# empty/absent -> every entry resolves to "" -> plain `thrum tmux capture`
|
|
716
|
+
# for everyone (inert). Set an entry only for an agent whose capture must
|
|
717
|
+
# route through a specific daemon (a real, currently rare, cross-daemon
|
|
718
|
+
# deployment need) — see run_capture_cycle below for where it's applied.
|
|
719
|
+
# ---------------------------------------------------------------------------
|
|
720
|
+
load_roster() {
|
|
721
|
+
local params_file="$1"
|
|
722
|
+
ROSTER=()
|
|
723
|
+
ROSTER_HOSTNAME=()
|
|
724
|
+
ROSTER_TMUX_SESSION=()
|
|
725
|
+
ROSTER_RUNTIME=()
|
|
726
|
+
ROSTER_DAEMON_OVERRIDE=()
|
|
727
|
+
while IFS='|' read -r _a _host _tmux _runtime _daemon_override; do
|
|
728
|
+
[ -n "$_a" ] || continue
|
|
729
|
+
ROSTER+=("$_a")
|
|
730
|
+
ROSTER_HOSTNAME+=("$_host")
|
|
731
|
+
ROSTER_TMUX_SESSION+=("$_tmux")
|
|
732
|
+
ROSTER_RUNTIME+=("$_runtime")
|
|
733
|
+
ROSTER_DAEMON_OVERRIDE+=("$_daemon_override")
|
|
734
|
+
done < <(python3 -c '
|
|
735
|
+
import json, sys
|
|
736
|
+
data = json.load(open(sys.argv[1]))
|
|
737
|
+
overrides = data.get("capture_daemon_overrides") or {}
|
|
738
|
+
for entry in data.get("roster", []):
|
|
739
|
+
if isinstance(entry, str):
|
|
740
|
+
agent = entry
|
|
741
|
+
print(f"{agent}||||{overrides.get(agent) or '"'"''"'"'}")
|
|
742
|
+
continue
|
|
743
|
+
agent = entry.get("agent_id") or entry.get("agent") or entry.get("name") or ""
|
|
744
|
+
print(f"{agent}|{entry.get('"'"'hostname'"'"') or '"'"''"'"'}|{entry.get('"'"'tmux_session'"'"') or '"'"''"'"'}|{entry.get('"'"'runtime'"'"') or '"'"''"'"'}|{overrides.get(agent) or '"'"''"'"'}")
|
|
745
|
+
' "$params_file" 2>/dev/null)
|
|
746
|
+
}
|
|
747
|
+
|
|
748
|
+
# ---------------------------------------------------------------------------
|
|
749
|
+
# run_capture_cycle — one full roster pass: local capture first (always;
|
|
750
|
+
# `thrum tmux capture <agent>` already reaches remote agents by name via
|
|
751
|
+
# the normal proxy), SSH fallback only on a known capability-denied/proxy-
|
|
752
|
+
# failure signature, DEGRADED-KNOWN/hard-FAIL classification via the
|
|
753
|
+
# shared probe_agent, and a GHOST-TIP annotation pass per agent. Writes
|
|
754
|
+
# the whole roster to $FILE. Populates the global FAILED/DEGRADED/CAPTURED
|
|
755
|
+
# arrays (caller must reset them to empty before calling).
|
|
756
|
+
# ---------------------------------------------------------------------------
|
|
757
|
+
run_capture_cycle() {
|
|
758
|
+
local file="$1" capture_id="$2" ts="$3" requested_lines="${4:-30}"
|
|
759
|
+
local roster_idx=0
|
|
760
|
+
local a session
|
|
761
|
+
ARCHIVE_AGENTS=()
|
|
762
|
+
ARCHIVE_RAW_FILES=()
|
|
763
|
+
ARCHIVE_CAPTURE_RCS=()
|
|
764
|
+
ARCHIVE_RUNTIMES=()
|
|
765
|
+
ARCHIVE_GEOMETRIES=()
|
|
766
|
+
ARCHIVE_ROUTES=()
|
|
767
|
+
ARCHIVE_PRIMARY_RCS=()
|
|
768
|
+
ARCHIVE_SSH_RCS=()
|
|
769
|
+
ARCHIVE_FINAL_RCS=()
|
|
770
|
+
local raw_root="${outdir}/archive/raw-ticks/${capture_id}"
|
|
771
|
+
mkdir -p "$raw_root" 2>/dev/null || true
|
|
772
|
+
chmod 700 "${outdir}/archive" "${outdir}/archive/raw-ticks" "$raw_root" 2>/dev/null || true
|
|
773
|
+
if [ "${USE_JEV:-0}" = "1" ]; then
|
|
774
|
+
JEV_FILTER_MANIFEST="${raw_root}/jev-filter-manifest.jsonl"
|
|
775
|
+
: >"$JEV_FILTER_MANIFEST"
|
|
776
|
+
fi
|
|
777
|
+
: >"$file"
|
|
778
|
+
for a in "${ROSTER[@]}"; do
|
|
779
|
+
local safe raw_file primary_file primary_json_file provider_input_file ssh_file primary_rc=0 primary_json_rc="" ssh_rc="" cap_rc=0 runtime geometry_json capture_route provider_input_source capture_json_file daemon_override primary_capture_file capture_format
|
|
780
|
+
safe="$(safe_capture_name "$a")"
|
|
781
|
+
raw_file="${raw_root}/$(printf '%04d' "$roster_idx")-${safe}.txt"
|
|
782
|
+
primary_file="${raw_root}/$(printf '%04d' "$roster_idx")-${safe}.primary.txt"
|
|
783
|
+
primary_json_file="${raw_root}/$(printf '%04d' "$roster_idx")-${safe}.primary.json"
|
|
784
|
+
provider_input_file="${raw_root}/$(printf '%04d' "$roster_idx")-${safe}.provider-input.txt"
|
|
785
|
+
ssh_file="${raw_root}/$(printf '%04d' "$roster_idx")-${safe}.ssh.txt"
|
|
786
|
+
runtime="${ROSTER_RUNTIME[$roster_idx]}"
|
|
787
|
+
daemon_override="${ROSTER_DAEMON_OVERRIDE[$roster_idx]}"
|
|
788
|
+
geometry_json=""
|
|
789
|
+
capture_route="primary"
|
|
790
|
+
provider_input_source=""
|
|
791
|
+
capture_json_file=""
|
|
792
|
+
session=$(resolve_session "$a")
|
|
793
|
+
# capture_daemon_overrides (watch_params.json, optional, default empty
|
|
794
|
+
# -> inert): routes this agent's capture through a specific daemon
|
|
795
|
+
# instead of the normal proxy-by-name resolution, for the rare
|
|
796
|
+
# cross-daemon deployment where that's required. See load_roster.
|
|
797
|
+
if [ "${USE_JEV:-0}" = "1" ]; then
|
|
798
|
+
primary_capture_file="$primary_json_file"
|
|
799
|
+
capture_format=json
|
|
800
|
+
else
|
|
801
|
+
primary_capture_file="$primary_file"
|
|
802
|
+
capture_format=annotated
|
|
803
|
+
fi
|
|
804
|
+
if [ -n "$daemon_override" ]; then
|
|
805
|
+
thrum tmux capture "$a" --daemon-id "$daemon_override" --format="$capture_format" --lines "$requested_lines" >"$primary_capture_file" 2>&1
|
|
806
|
+
else
|
|
807
|
+
thrum tmux capture "$a" --format="$capture_format" --lines "$requested_lines" >"$primary_capture_file" 2>&1
|
|
808
|
+
fi
|
|
809
|
+
primary_rc=$?
|
|
810
|
+
if [ "${USE_JEV:-0}" = "1" ] && [ "$primary_rc" -eq 0 ] && [ -s "$primary_json_file" ] && write_raw_from_capture_json "$primary_json_file" "$raw_file" 2>/dev/null; then
|
|
811
|
+
cap_rc=0
|
|
812
|
+
primary_json_rc=0
|
|
813
|
+
capture_json_file="$primary_json_file"
|
|
814
|
+
provider_input_source="thrum_tmux_capture_json_structured_state"
|
|
815
|
+
if agent_is_local "$a"; then
|
|
816
|
+
geometry_json=$(measure_geometry_json "$session" 2>/dev/null || true)
|
|
817
|
+
fi
|
|
818
|
+
CAPTURED+=("$a")
|
|
819
|
+
elif [ "${USE_JEV:-0}" != "1" ] && [ "$primary_rc" -eq 0 ] && [ -s "$primary_file" ]; then
|
|
820
|
+
cp "$primary_file" "$raw_file"
|
|
821
|
+
cap_rc=0
|
|
822
|
+
if agent_is_local "$a"; then
|
|
823
|
+
geometry_json=$(measure_geometry_json "$session" 2>/dev/null || true)
|
|
824
|
+
fi
|
|
825
|
+
CAPTURED+=("$a")
|
|
826
|
+
else
|
|
827
|
+
ssh_capture_pane "$a" "${ROSTER_HOSTNAME[$roster_idx]}" "${ROSTER_TMUX_SESSION[$roster_idx]}" >"$ssh_file" 2>&1
|
|
828
|
+
ssh_rc=$?
|
|
829
|
+
if [ "$ssh_rc" -eq 0 ] && [ -s "$ssh_file" ]; then
|
|
830
|
+
cp "$ssh_file" "$raw_file"
|
|
831
|
+
if [ "${USE_JEV:-0}" = "1" ]; then
|
|
832
|
+
cp "$ssh_file" "$provider_input_file"
|
|
833
|
+
chmod 600 "$provider_input_file" 2>/dev/null || true
|
|
834
|
+
provider_input_source="ssh_capture_pane_text"
|
|
835
|
+
fi
|
|
836
|
+
cap_rc=0
|
|
837
|
+
capture_route="ssh_fallback"
|
|
838
|
+
geometry_json=""
|
|
839
|
+
CAPTURED+=("$a")
|
|
840
|
+
else
|
|
841
|
+
cat "$primary_capture_file" >"$raw_file"
|
|
842
|
+
printf '\n--- SSH fallback failed for %s (exit %s) ---\n' "$a" "$ssh_rc" >>"$raw_file"
|
|
843
|
+
cat "$ssh_file" >>"$raw_file"
|
|
844
|
+
cap_rc=$ssh_rc
|
|
845
|
+
capture_route="both_failed"
|
|
846
|
+
geometry_json=""
|
|
847
|
+
fi
|
|
848
|
+
# A successful SSH capture is a real pane capture, not a degraded
|
|
849
|
+
# liveness proof. Only classify when both capture routes failed.
|
|
850
|
+
if [ "$cap_rc" -ne 0 ]; then
|
|
851
|
+
if grep -qiE "local agent not found|lacks required capability|empty.*binding|proxy.*capability|capability.*denied|peer.*unreachable|circuit open|dial skipped" "$primary_capture_file"; then
|
|
852
|
+
probe_agent "$a" "$(head -1 "$primary_capture_file" | tr -d '\n' | cut -c1-120)" >>"$raw_file"
|
|
853
|
+
else
|
|
854
|
+
echo "--- CAPTURE FAILED for $a (nonzero exit — NOT an empty pane) ---" >>"$raw_file"
|
|
855
|
+
FAILED+=("$a")
|
|
856
|
+
fi
|
|
857
|
+
fi
|
|
858
|
+
fi
|
|
859
|
+
if [ "${USE_JEV:-0}" != "1" ] || [ "$capture_route" != "primary" ]; then
|
|
860
|
+
check_ghost_tip "$session" >>"$raw_file"
|
|
861
|
+
fi
|
|
862
|
+
{
|
|
863
|
+
echo "=== $a ==="
|
|
864
|
+
cat "$raw_file"
|
|
865
|
+
echo
|
|
866
|
+
} >>"$file"
|
|
867
|
+
ARCHIVE_AGENTS+=("$a")
|
|
868
|
+
ARCHIVE_RAW_FILES+=("$raw_file")
|
|
869
|
+
ARCHIVE_CAPTURE_RCS+=("$cap_rc")
|
|
870
|
+
ARCHIVE_RUNTIMES+=("$runtime")
|
|
871
|
+
ARCHIVE_GEOMETRIES+=("$geometry_json")
|
|
872
|
+
ARCHIVE_ROUTES+=("$capture_route")
|
|
873
|
+
ARCHIVE_PRIMARY_RCS+=("$primary_rc")
|
|
874
|
+
ARCHIVE_SSH_RCS+=("$ssh_rc")
|
|
875
|
+
ARCHIVE_FINAL_RCS+=("$cap_rc")
|
|
876
|
+
if [ "${USE_JEV:-0}" = "1" ]; then
|
|
877
|
+
write_filter_manifest_row "$JEV_FILTER_MANIFEST" "$a" "$raw_file" "$file" "$capture_id" "$ts" "$cap_rc" "$capture_route" "${provider_input_source:+$provider_input_file}" "$provider_input_source" "$capture_json_file"
|
|
878
|
+
fi
|
|
879
|
+
roster_idx=$((roster_idx + 1))
|
|
880
|
+
done
|
|
881
|
+
archive_source_file "$outdir" "$file" "$capture_id" "$ts"
|
|
882
|
+
local archive_idx=0
|
|
883
|
+
while [ "$archive_idx" -lt "${#ARCHIVE_AGENTS[@]}" ]; do
|
|
884
|
+
archive_agent_file \
|
|
885
|
+
"$outdir" \
|
|
886
|
+
"${ARCHIVE_AGENTS[$archive_idx]}" \
|
|
887
|
+
"${ARCHIVE_RAW_FILES[$archive_idx]}" \
|
|
888
|
+
"$file" \
|
|
889
|
+
"$capture_id" \
|
|
890
|
+
"$ts" \
|
|
891
|
+
"${ARCHIVE_CAPTURE_RCS[$archive_idx]}" \
|
|
892
|
+
"$requested_lines" \
|
|
893
|
+
"${ARCHIVE_RUNTIMES[$archive_idx]}" \
|
|
894
|
+
"${ARCHIVE_GEOMETRIES[$archive_idx]}" \
|
|
895
|
+
"${ARCHIVE_ROUTES[$archive_idx]}" \
|
|
896
|
+
"${ARCHIVE_PRIMARY_RCS[$archive_idx]}" \
|
|
897
|
+
"${ARCHIVE_SSH_RCS[$archive_idx]}" \
|
|
898
|
+
"${ARCHIVE_FINAL_RCS[$archive_idx]}"
|
|
899
|
+
archive_idx=$((archive_idx + 1))
|
|
900
|
+
done
|
|
901
|
+
if [ "${USE_JEV:-0}" = "1" ]; then
|
|
902
|
+
run_jev_filter "$outdir" "$JEV_FILTER_MANIFEST" "$capture_id"
|
|
903
|
+
fi
|
|
904
|
+
}
|
|
905
|
+
|
|
906
|
+
# ---------------------------------------------------------------------------
|
|
907
|
+
# emit_wake_line — overnight wake-gating (brainstorm-steward; TZ fix:
|
|
908
|
+
# the monitor process runs in UTC, so a bare `date +%H` gives the UTC
|
|
909
|
+
# hour and the overnight gate never matches; the gate is meant to compare
|
|
910
|
+
# against LOCAL time by design).
|
|
911
|
+
# Daytime (local 08:00-23:59) every run emits the matchable wake line.
|
|
912
|
+
# Overnight (local 00:00-07:59) it still captures every run, but only
|
|
913
|
+
# EMITS the wake line once ~2h has passed (tracked via $stamp), printing a
|
|
914
|
+
# non-matching line otherwise. Reverts automatically at 08:00.
|
|
915
|
+
#
|
|
916
|
+
# Test seam: ROSTER_WATCH_FAKE_HOUR, when set, is validated as an integer
|
|
917
|
+
# 0-23 (one or two digits, optional leading zero) and used in place of the
|
|
918
|
+
# real America/Los_Angeles wall-clock hour, so tests can force the
|
|
919
|
+
# daytime/overnight branch deterministically. An invalid value (non-numeric,
|
|
920
|
+
# or numeric but outside 0-23) prints one diagnostic to stderr naming the
|
|
921
|
+
# bad value and the accepted range, then skips wake-line emission for this
|
|
922
|
+
# tick entirely -- it never falls back to the real clock, so a bad test
|
|
923
|
+
# value fails loudly instead of silently passing as if unset. When unset
|
|
924
|
+
# (the default, and every real production invocation), behavior is
|
|
925
|
+
# unchanged: the hour is still read from the real America/Los_Angeles
|
|
926
|
+
# wall clock.
|
|
927
|
+
# ---------------------------------------------------------------------------
|
|
928
|
+
emit_wake_line() {
|
|
929
|
+
local file="$1" ts="$2" stamp="$3"
|
|
930
|
+
local hour now last emit=1
|
|
931
|
+
if [ -n "${ROSTER_WATCH_FAKE_HOUR:-}" ]; then
|
|
932
|
+
if [[ "${ROSTER_WATCH_FAKE_HOUR}" =~ ^([0-9]|[01][0-9]|2[0-3])$ ]]; then
|
|
933
|
+
hour=$((10#${ROSTER_WATCH_FAKE_HOUR}))
|
|
934
|
+
else
|
|
935
|
+
echo "roster-watch: ROSTER_WATCH_FAKE_HOUR='${ROSTER_WATCH_FAKE_HOUR}' is invalid (expected an integer 0-23) -- skipping wake-line emission this tick" >&2
|
|
936
|
+
return 1
|
|
937
|
+
fi
|
|
938
|
+
else
|
|
939
|
+
hour=$(TZ=America/Los_Angeles date +%H); hour=$((10#$hour)) # 10# avoids octal parse of 08/09
|
|
940
|
+
fi
|
|
941
|
+
now=$(date +%s)
|
|
942
|
+
if [ "$hour" -ge 0 ] && [ "$hour" -lt 8 ]; then
|
|
943
|
+
last=$(cat "$stamp" 2>/dev/null || echo 0)
|
|
944
|
+
[ $((now - last)) -lt 7000 ] && emit=0 # 7000s ~ 2h with cron fudge
|
|
945
|
+
fi
|
|
946
|
+
if [ "$emit" -eq 1 ]; then
|
|
947
|
+
echo "$now" > "$stamp"
|
|
948
|
+
echo "roster capture ready: $file ($ts) — reconcile roster state and linked queue waits"
|
|
949
|
+
else
|
|
950
|
+
echo "roster capture taken, wake suppressed (overnight 2h cadence): $file ($ts)"
|
|
951
|
+
fi
|
|
952
|
+
}
|
|
953
|
+
|
|
954
|
+
# ---------------------------------------------------------------------------
|
|
955
|
+
# main
|
|
956
|
+
# ---------------------------------------------------------------------------
|
|
957
|
+
main() {
|
|
958
|
+
# --- locate paths --------------------------------------------------------
|
|
959
|
+
local script_dir worktree_root agent_name
|
|
960
|
+
script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
|
|
961
|
+
SCRIPT_DIR="$script_dir"
|
|
962
|
+
|
|
963
|
+
# --prune-roster-captures passthrough: usable standalone (e.g. from a
|
|
964
|
+
# cron/monitor cleanup step) without touching the network. Checked FIRST,
|
|
965
|
+
# before the source-tree refusal guard below -- this
|
|
966
|
+
# mode never resolves worktree_root/outdir/watch_params and never writes
|
|
967
|
+
# anywhere but the explicit $2 directory argument, so script_dir alone
|
|
968
|
+
# (always the resources/ dir for this standalone invocation shape) must
|
|
969
|
+
# not disqualify it the way it correctly disqualifies the main
|
|
970
|
+
# capture-cycle path below.
|
|
971
|
+
if [ "${1-}" = "--prune-roster-captures" ]; then
|
|
972
|
+
[ "$#" -eq 2 ] || exit 2
|
|
973
|
+
prune_roster_captures "$2"
|
|
974
|
+
exit 0
|
|
975
|
+
fi
|
|
976
|
+
|
|
977
|
+
# Deployment shape: <worktree>/.thrum-watch/roster-watch.sh -- SCRIPT_DIR
|
|
978
|
+
# is .thrum-watch, so WORKTREE_ROOT is one level up, NOT the
|
|
979
|
+
# .thrum/agents/<you>/-relative three-levels-up thrum-watch-pane-
|
|
980
|
+
# capture.sh uses (that script deploys under .thrum/agents/<you>/, this
|
|
981
|
+
# one deploys under a worktree-root-sibling .thrum-watch/).
|
|
982
|
+
# Refuse to run in place from a plugin skill source tree.
|
|
983
|
+
# Any deployed copy lives at <worktree>/.thrum-watch/roster-watch.sh
|
|
984
|
+
# (see SKILL.md); this exact suffix is only ever the plugin skill
|
|
985
|
+
# resources dir itself. Failing loud here -- before worktree_root/outdir
|
|
986
|
+
# are ever computed -- is what keeps a mis-invocation from writing
|
|
987
|
+
# anything at all under claude-plugin/ or any mirror plugin tree, rather
|
|
988
|
+
# than resolving worktree_root to a bogus plugin-adjacent path and
|
|
989
|
+
# silently creating a new .thrum/ sibling there.
|
|
990
|
+
#
|
|
991
|
+
# NOTE (dual-review): this glob match is a fail-fast
|
|
992
|
+
# nicety for the one known mis-invocation shape, not the real barrier --
|
|
993
|
+
# it only ever catches THIS exact source layout. The actual fix is below:
|
|
994
|
+
# outdir no longer derives from script_dir/BASH_SOURCE at all, so even an
|
|
995
|
+
# invocation shape this glob fails to recognize still cannot write under
|
|
996
|
+
# claude-plugin/ (or any mirror). Do not extend this case statement and
|
|
997
|
+
# assume that alone closes a new variant -- verify outdir's derivation
|
|
998
|
+
# instead.
|
|
999
|
+
case "${script_dir}" in
|
|
1000
|
+
*/skills/persistent-watcher-archetype/resources)
|
|
1001
|
+
echo "roster-watch.sh: refusing to run from a plugin skill source tree (${script_dir}); deploy to <worktree>/.thrum-watch/roster-watch.sh first (see SKILL.md)" >&2
|
|
1002
|
+
exit 1
|
|
1003
|
+
;;
|
|
1004
|
+
esac
|
|
1005
|
+
|
|
1006
|
+
worktree_root="$(cd -- "${script_dir}/.." >/dev/null 2>&1 && pwd)"
|
|
1007
|
+
agent_name="${AGENT_NAME:-$(basename -- "${worktree_root}" | tr '-' '_')}"
|
|
1008
|
+
|
|
1009
|
+
# outdir must resolve via the shared, redirect-followed .thrum/agents/
|
|
1010
|
+
# tree -- NOT from script_dir/BASH_SOURCE, which
|
|
1011
|
+
# resolves into plugin source whenever this script runs from its
|
|
1012
|
+
# skill/resources location instead of its deployed .thrum-watch copy.
|
|
1013
|
+
local outdir
|
|
1014
|
+
outdir="$(resolve_capture_outdir "${worktree_root}" "${agent_name}")"
|
|
1015
|
+
mkdir -p "${outdir}"
|
|
1016
|
+
|
|
1017
|
+
# One-time, idempotent capture-history migration (per dual-review).
|
|
1018
|
+
# Capture output moved from <worktree>/.thrum-watch/ (script_dir,
|
|
1019
|
+
# the pre-fix outdir) to the new outdir above. A live deployment's
|
|
1020
|
+
# existing .thrum-watch/archive/ history must not be silently orphaned on
|
|
1021
|
+
# redeploy. Runs on every invocation but is a no-op after the first pass:
|
|
1022
|
+
# only fires when the OLD archive exists and the NEW one does not, and
|
|
1023
|
+
# never overwrites an existing new archive (mv fails loudly instead, per
|
|
1024
|
+
# the project's never-clobber-with-a-move convention).
|
|
1025
|
+
local legacy_archive="${script_dir}/archive" new_archive="${outdir}/archive"
|
|
1026
|
+
if [ -d "${legacy_archive}" ] && [ ! -e "${new_archive}" ]; then
|
|
1027
|
+
if mv "${legacy_archive}" "${new_archive}"; then
|
|
1028
|
+
echo "roster-watch.sh: migrated capture archive history ${legacy_archive} -> ${new_archive}" >&2
|
|
1029
|
+
else
|
|
1030
|
+
echo "roster-watch.sh: capture archive migration failed (${legacy_archive} -> ${new_archive}); leaving legacy archive in place" >&2
|
|
1031
|
+
fi
|
|
1032
|
+
fi
|
|
1033
|
+
|
|
1034
|
+
local watch_params
|
|
1035
|
+
watch_params="$(resolve_watch_params "${worktree_root}" "${agent_name}")"
|
|
1036
|
+
|
|
1037
|
+
local detector="${script_dir}/detect_ghost_tip.py"
|
|
1038
|
+
DETECTOR="${detector}"
|
|
1039
|
+
|
|
1040
|
+
local interval_secs="${INTERVAL_SECS:-600}"
|
|
1041
|
+
local stamp="${outdir}/.last-emit"
|
|
1042
|
+
local fail_state="${outdir}/.last-failed-roster"
|
|
1043
|
+
|
|
1044
|
+
load_roster "${watch_params}"
|
|
1045
|
+
REPO_ROOT="$(git -C "$worktree_root" rev-parse --show-toplevel 2>/dev/null || printf '%s' "$worktree_root")"
|
|
1046
|
+
USE_JEV_REQUESTED=$(python3 -c 'import json,sys; print(int(json.load(open(sys.argv[1])).get("use_jev") is True))' "$watch_params" 2>/dev/null || echo 0)
|
|
1047
|
+
USE_JEV=0
|
|
1048
|
+
if [ "$USE_JEV_REQUESTED" = "1" ]; then
|
|
1049
|
+
if jev_key_present "${ROSTER_JEV_ENV_FILE:-${REPO_ROOT}/.env}"; then
|
|
1050
|
+
USE_JEV=1
|
|
1051
|
+
else
|
|
1052
|
+
echo "JEV requested but no key found — skipped" >&2
|
|
1053
|
+
fi
|
|
1054
|
+
fi
|
|
1055
|
+
local ts file capture_stamp capture_id
|
|
1056
|
+
ts=$(utc_now_iso)
|
|
1057
|
+
capture_stamp=$(utc_now_file_stamp)
|
|
1058
|
+
file="${outdir}/roster-capture-${capture_stamp}.txt"
|
|
1059
|
+
if [ "${#ROSTER[@]}" -eq 0 ]; then
|
|
1060
|
+
echo "roster capture ready: $file ($ts) — WARNING empty/unreadable roster in $watch_params" | tee "$file"
|
|
1061
|
+
exit 0
|
|
1062
|
+
fi
|
|
1063
|
+
|
|
1064
|
+
# Startup self-check runs once, against the dynamically-loaded roster,
|
|
1065
|
+
# regardless of execution mode.
|
|
1066
|
+
ROSTER_SELF_CHECK_FAILED=0
|
|
1067
|
+
roster_self_check "${outdir}" "${ROSTER[@]}"
|
|
1068
|
+
|
|
1069
|
+
run_one_cycle() {
|
|
1070
|
+
FAILED=(); DEGRADED=(); CAPTURED=()
|
|
1071
|
+
ROSTER_INCIDENT_CLEARED=0
|
|
1072
|
+
ARCHIVE_DEGRADED=0; ARCHIVE_DEGRADED_MESSAGES=()
|
|
1073
|
+
ts=$(utc_now_iso)
|
|
1074
|
+
capture_stamp=$(utc_now_file_stamp)
|
|
1075
|
+
capture_id="roster-capture-${capture_stamp}-$$"
|
|
1076
|
+
file="${outdir}/roster-capture-${capture_stamp}.txt"
|
|
1077
|
+
|
|
1078
|
+
run_capture_cycle "${file}" "${capture_id}" "${ts}" 30
|
|
1079
|
+
prune_roster_captures "${outdir}"
|
|
1080
|
+
|
|
1081
|
+
if [ "${#DEGRADED[@]}" -gt 0 ]; then
|
|
1082
|
+
echo "DEGRADED-KNOWN: tmux capture capability-denied/empty binding for ${DEGRADED[*]} but LIVE by independent fallback proof; inspect $file (direct-message/session-artifact fallback)"
|
|
1083
|
+
fi
|
|
1084
|
+
if [ "${#FAILED[@]}" -gt 0 ]; then
|
|
1085
|
+
local current_failures="${FAILED[*]}" previous_failures
|
|
1086
|
+
previous_failures=$(cat "$fail_state" 2>/dev/null || true)
|
|
1087
|
+
printf '%s\n' "$current_failures" > "$fail_state"
|
|
1088
|
+
if [ "$current_failures" != "$previous_failures" ]; then
|
|
1089
|
+
echo "ROSTER INCIDENT: tmux capture failed for $current_failures; inspect $file"
|
|
1090
|
+
else
|
|
1091
|
+
echo "roster incident persists unchanged for $current_failures; inspect $file"
|
|
1092
|
+
fi
|
|
1093
|
+
elif [ -f "$fail_state" ]; then
|
|
1094
|
+
local previous_failures
|
|
1095
|
+
previous_failures=$(cat "$fail_state" 2>/dev/null || true)
|
|
1096
|
+
rm -f "$fail_state"
|
|
1097
|
+
ROSTER_INCIDENT_CLEARED=1
|
|
1098
|
+
echo "roster incident cleared for $previous_failures; inspect $file"
|
|
1099
|
+
fi
|
|
1100
|
+
|
|
1101
|
+
if [ "${ARCHIVE_DEGRADED:-0}" -ne 0 ]; then
|
|
1102
|
+
echo "roster capture ready: ${file} (${ts}) — ARCHIVE-DEGRADED: ${ARCHIVE_DEGRADED_MESSAGES[*]}; inspect archive/health and archive-errors.log"
|
|
1103
|
+
elif [ "${USE_JEV:-0}" = "1" ]; then
|
|
1104
|
+
if [ "${ROSTER_SELF_CHECK_FAILED:-0}" -ne 0 ]; then
|
|
1105
|
+
echo "roster capture ready: $file ($ts) - ROSTER-CHECK FAIL; inspect $outdir/roster-check.log"
|
|
1106
|
+
elif [ "${ROSTER_INCIDENT_CLEARED:-0}" -ne 0 ]; then
|
|
1107
|
+
echo "roster capture ready: $file ($ts) - roster incident cleared; inspect $file"
|
|
1108
|
+
elif [ "${JEV_FILTER_MODE:-fallback}" != "ok" ]; then
|
|
1109
|
+
emit_wake_line "${file}" "${ts}" "${stamp}"
|
|
1110
|
+
elif [ "${JEV_FILTER_ATTENTION:-0}" -gt 0 ]; then
|
|
1111
|
+
echo "roster capture ready: $file ($ts) — active permission prompt or blocking user choice (${JEV_FILTER_ATTENTION} agent(s)); decisions archived"
|
|
1112
|
+
elif [ "${#FAILED[@]}" -gt 0 ] || [ "${#DEGRADED[@]}" -gt 0 ]; then
|
|
1113
|
+
emit_wake_line "${file}" "${ts}" "${stamp}"
|
|
1114
|
+
else
|
|
1115
|
+
echo "roster capture archived, jev filter found no watcher-needed panes: $file ($ts)"
|
|
1116
|
+
if [ "${ROSTER_WATCH_MODE:-}" != "loop" ]; then
|
|
1117
|
+
echo "THRUM_ROSTER_JEV_ARCHIVE_ONLY_V1"
|
|
1118
|
+
fi
|
|
1119
|
+
fi
|
|
1120
|
+
else
|
|
1121
|
+
emit_wake_line "${file}" "${ts}" "${stamp}"
|
|
1122
|
+
fi
|
|
1123
|
+
}
|
|
1124
|
+
|
|
1125
|
+
if [ "${ROSTER_WATCH_MODE:-}" = "loop" ]; then
|
|
1126
|
+
# LEGACY LOOP MODE — see header comment for why this exists and why
|
|
1127
|
+
# cron/single-pass (the default) is preferred.
|
|
1128
|
+
while true; do
|
|
1129
|
+
run_one_cycle
|
|
1130
|
+
run_self_submit_enter "${agent_name}"
|
|
1131
|
+
sleep "${interval_secs}"
|
|
1132
|
+
done
|
|
1133
|
+
else
|
|
1134
|
+
run_one_cycle
|
|
1135
|
+
fi
|
|
1136
|
+
}
|
|
1137
|
+
|
|
1138
|
+
if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
|
|
1139
|
+
main "$@"
|
|
1140
|
+
fi
|