@ouro.bot/cli 0.1.0-alpha.741 → 0.1.0-alpha.743
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/changelog.json +42 -0
- package/deploy/unraid/Dockerfile +7 -2
- package/deploy/unraid/README.txt +717 -106
- package/deploy/unraid/migrate-sanctuary-bundle.mjs +21 -0
- package/deploy/unraid/ouro-events/bootstrap-spool.sh +92 -0
- package/deploy/unraid/ouro-events/emit-event.mjs +310 -0
- package/deploy/unraid/ouro-events/emit-usenet-event.sh +54 -0
- package/deploy/unraid/ouro-events/install-usenet-guard.sh +434 -0
- package/deploy/unraid/ouro-events/usenet-health.sh +178 -0
- package/deploy/unraid/sanctuary-acceptance-contract.json +4 -3
- package/deploy/unraid/sanctuary-unit16-host-broker.mjs +5 -1
- package/deploy/unraid/sanctuary.ouro/agent.json +2 -2
- package/deploy/unraid/sanctuary.ouro/bundle-meta.json +2 -2
- package/deploy/unraid/sanctuary.ouro/habits/sanctuary-health.md +1 -1
- package/deploy/unraid/sanctuary.ouro/provider-readiness.json +3 -10
- package/deploy/unraid/sanctuary.ouro/psyche/ASPIRATIONS.md +3 -1
- package/deploy/unraid/sanctuary.ouro/psyche/IDENTITY.md +3 -1
- package/deploy/unraid/sanctuary.ouro/psyche/LORE.md +4 -0
- package/deploy/unraid/sanctuary.ouro/psyche/SOUL.md +6 -2
- package/deploy/unraid/sanctuary.ouro/psyche/TACIT.md +4 -2
- package/deploy/unraid/sanctuary.ouro/state/policy/steward.json +7 -0
- package/deploy/unraid/sanctuary.ouro/tool-profiles.json +19 -17
- package/deploy/unraid/sanctuary.xml +12 -9
- package/dist/arc/cares.js +171 -37
- package/dist/arc/obligations.js +38 -0
- package/dist/heart/awaiting/await-alert.js +4 -2
- package/dist/heart/awaiting/await-expiry.js +39 -4
- package/dist/heart/awaiting/await-parser.js +8 -0
- package/dist/heart/context-loss-gauntlet.js +31 -2
- package/dist/heart/context-loss-sentinel.js +2 -0
- package/dist/heart/core.js +37 -7
- package/dist/heart/cross-chat-delivery.js +1 -0
- package/dist/heart/daemon/container-healthcheck.js +3 -1
- package/dist/heart/daemon/container-runtime.js +13 -0
- package/dist/heart/daemon/container-spec-auditor-main.js +11 -7
- package/dist/heart/daemon/container-spec-auditor.js +38 -11
- package/dist/heart/daemon/daemon-entry.js +19 -5
- package/dist/heart/daemon/daemon.js +154 -22
- package/dist/heart/daemon/sanctuary-acceptance-adapter.js +151 -87
- package/dist/heart/daemon/sanctuary-acceptance-harness.js +26 -29
- package/dist/heart/daemon/sanctuary-acceptance-scenarios.js +42 -51
- package/dist/heart/daemon/sanctuary-bundle-migration.js +398 -0
- package/dist/heart/daemon/sanctuary-scheduler-origin.js +11 -5
- package/dist/heart/daemon/supercronic-supervisor.js +16 -1
- package/dist/heart/external-events/router.js +1025 -43
- package/dist/heart/provider-models.js +10 -3
- package/dist/heart/session-events.js +23 -5
- package/dist/heart/steward-policy.js +302 -0
- package/dist/heart/tool-approval.js +3 -1
- package/dist/heart/turn-context.js +1 -0
- package/dist/mind/context.js +8 -3
- package/dist/mind/pending.js +25 -0
- package/dist/mind/prompt.js +49 -1
- package/dist/repertoire/guardrails.js +2 -1
- package/dist/repertoire/relationship-authorization.js +194 -0
- package/dist/repertoire/tools-awaiting.js +229 -26
- package/dist/repertoire/tools-base.js +15 -0
- package/dist/repertoire/tools-continuity.js +259 -4
- package/dist/repertoire/tools-notes.js +29 -3
- package/dist/repertoire/tools-session.js +219 -4
- package/dist/repertoire/tools-steward-policy.js +88 -0
- package/dist/repertoire/tools-telegram-contacts.js +37 -0
- package/dist/repertoire/tools-unraid.js +113 -6
- package/dist/repertoire/tools.js +104 -9
- package/dist/repertoire/unraid-restart.js +86 -4
- package/dist/senses/private-runtime-worker.js +36 -3
- package/dist/senses/private-runtime.js +310 -63
- package/dist/senses/sanctuary-health-acceptance-probe.js +8 -6
- package/dist/senses/sanctuary-health-runner.js +51 -96
- package/dist/senses/sanctuary-health.js +54 -93
- package/dist/senses/sanctuary-runtime.js +36 -2
- package/dist/senses/sanctuary-sab.js +80 -0
- package/dist/senses/shared-turn.js +64 -9
- package/dist/senses/telegram-admission.js +684 -0
- package/dist/senses/telegram-approval-runtime.js +46 -6
- package/dist/senses/telegram-client.js +236 -72
- package/dist/senses/telegram-effect-adapter.js +787 -0
- package/dist/senses/telegram-entry.js +4 -1
- package/dist/senses/telegram.js +728 -48
- package/npm-shrinkwrap.json +6 -6
- package/package.json +2 -2
|
@@ -0,0 +1,434 @@
|
|
|
1
|
+
#!/bin/bash
|
|
2
|
+
set -euo pipefail
|
|
3
|
+
|
|
4
|
+
SOURCE_ROOT="$(cd "$(dirname "$0")" && pwd -P)"
|
|
5
|
+
INSTALL_ROOT="/boot/config/custom"
|
|
6
|
+
GO_FILE="/boot/config/go"
|
|
7
|
+
CRONTAB_FILE=""
|
|
8
|
+
ACTION="install"
|
|
9
|
+
RESTORE_ROOT=""
|
|
10
|
+
STAGE_PATH=""
|
|
11
|
+
SYSTEM_CRON_PRESENT=false
|
|
12
|
+
|
|
13
|
+
while (($# > 0)); do
|
|
14
|
+
case "$1" in
|
|
15
|
+
--install-only) ACTION="install"; shift ;;
|
|
16
|
+
--boot) ACTION="boot"; shift ;;
|
|
17
|
+
--restore-root) ACTION="restore"; RESTORE_ROOT="${2:?missing --restore-root value}"; shift 2 ;;
|
|
18
|
+
--source-root) SOURCE_ROOT="${2:?missing --source-root value}"; shift 2 ;;
|
|
19
|
+
--install-root) INSTALL_ROOT="${2:?missing --install-root value}"; shift 2 ;;
|
|
20
|
+
--go-file) GO_FILE="${2:?missing --go-file value}"; shift 2 ;;
|
|
21
|
+
--crontab-file) CRONTAB_FILE="${2:?missing --crontab-file value}"; shift 2 ;;
|
|
22
|
+
*) echo "usenet guard installer: invalid argument: $1" >&2; exit 2 ;;
|
|
23
|
+
esac
|
|
24
|
+
done
|
|
25
|
+
|
|
26
|
+
validate_persisted_path() {
|
|
27
|
+
local value="$1" label="$2"
|
|
28
|
+
[[ "$value" =~ ^/[A-Za-z0-9._/-]+$ ]] && [[ "$value" != */ ]] && [[ "$value" != *"//"* ]] && [[ "$value" != *"/../"* ]] && [[ "$value" != */.. ]] && [[ "$value" != *"/./"* ]] && [[ "$value" != */. ]] || {
|
|
29
|
+
echo "usenet guard installer: $label must be a canonical absolute path" >&2
|
|
30
|
+
exit 2
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
validate_persisted_path "$INSTALL_ROOT" "install root"
|
|
35
|
+
if [ -n "$CRONTAB_FILE" ]; then validate_persisted_path "$CRONTAB_FILE" "crontab file"; fi
|
|
36
|
+
if [ -n "$RESTORE_ROOT" ]; then validate_persisted_path "$RESTORE_ROOT" "restore root"; fi
|
|
37
|
+
|
|
38
|
+
EVENT_ROOT="$INSTALL_ROOT/ouro-events"
|
|
39
|
+
printf -v EVENT_INSTALLER_Q '%q' "$EVENT_ROOT/install-usenet-guard.sh"
|
|
40
|
+
printf -v INSTALL_ROOT_Q '%q' "$INSTALL_ROOT"
|
|
41
|
+
BOOT_HOOK="/bin/bash $EVENT_INSTALLER_Q --boot --install-root $INSTALL_ROOT_Q"
|
|
42
|
+
PREVIOUS_INSTALLER_BOOT_HOOK="/bin/bash $EVENT_INSTALLER_Q --boot"
|
|
43
|
+
if [ -n "$CRONTAB_FILE" ]; then
|
|
44
|
+
printf -v CRONTAB_FILE_Q '%q' "$CRONTAB_FILE"
|
|
45
|
+
BOOT_HOOK+=" --crontab-file $CRONTAB_FILE_Q"
|
|
46
|
+
PREVIOUS_INSTALLER_BOOT_HOOK+=" --crontab-file $CRONTAB_FILE_Q"
|
|
47
|
+
fi
|
|
48
|
+
LEGACY_BOOT_HOOK="/boot/config/custom/ouro-events/bootstrap-spool.sh --mount"
|
|
49
|
+
LEGACY_BASH_BOOT_HOOK="/bin/bash /boot/config/custom/ouro-events/bootstrap-spool.sh --mount"
|
|
50
|
+
CANONICAL_PREVIOUS_BOOT_HOOK="/bin/bash /boot/config/custom/ouro-events/install-usenet-guard.sh --boot"
|
|
51
|
+
CANONICAL_BOOT_HOOK="$CANONICAL_PREVIOUS_BOOT_HOOK --install-root /boot/config/custom"
|
|
52
|
+
printf -v HEALTH_SCRIPT_Q '%q' "$INSTALL_ROOT/usenet_health.sh"
|
|
53
|
+
CRON_LINE="*/15 * * * * /bin/bash $HEALTH_SCRIPT_Q # ouro:usenet-health"
|
|
54
|
+
LEGACY_CRON_LINE="*/15 * * * * /bin/bash $HEALTH_SCRIPT_Q"
|
|
55
|
+
CANONICAL_CRON_LINE="*/15 * * * * /bin/bash /boot/config/custom/usenet_health.sh # ouro:usenet-health"
|
|
56
|
+
CANONICAL_LEGACY_CRON_LINE="*/15 * * * * /bin/bash /boot/config/custom/usenet_health.sh"
|
|
57
|
+
ASSET_NAMES=(usenet-health.sh bootstrap-spool.sh emit-event.mjs emit-usenet-event.sh install-usenet-guard.sh)
|
|
58
|
+
TARGET_PATHS=("$INSTALL_ROOT/usenet_health.sh" "$EVENT_ROOT/bootstrap-spool.sh" "$EVENT_ROOT/emit-event.mjs" "$EVENT_ROOT/emit-usenet-event.sh" "$EVENT_ROOT/install-usenet-guard.sh")
|
|
59
|
+
|
|
60
|
+
atomic_install() {
|
|
61
|
+
local source="$1" target="$2" mode="$3" temporary="${2}.ouro-next.$$" status
|
|
62
|
+
/bin/mkdir -p "$(dirname "$target")"
|
|
63
|
+
install -m "$mode" "$source" "$temporary" || { status=$?; /bin/rm -f "$temporary"; return "$status"; }
|
|
64
|
+
mv "$temporary" "$target" || { status=$?; /bin/rm -f "$temporary"; return "$status"; }
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
private_transaction_workspace() {
|
|
68
|
+
local kind="$1" workspace identity status
|
|
69
|
+
workspace="$(/usr/bin/mktemp -d "/tmp/ouro-usenet-$kind.XXXXXX")" || return $?
|
|
70
|
+
[ -d "$workspace" ] && [ ! -L "$workspace" ] || { status=$?; /bin/rmdir "$workspace" 2>/dev/null || true; return "$status"; }
|
|
71
|
+
identity="$(/usr/bin/stat -c '%u:%a' "$workspace" 2>/dev/null || /usr/bin/stat -f '%u:%Lp' "$workspace")" || { status=$?; /bin/rmdir "$workspace"; return "$status"; }
|
|
72
|
+
[ "$identity" = "$(id -u):700" ] || { /bin/rmdir "$workspace"; return 1; }
|
|
73
|
+
printf '%s\n' "$workspace"
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
cleanup_stale_atomic_residue() {
|
|
77
|
+
local target parent base
|
|
78
|
+
for target in "${TARGET_PATHS[@]}" "$GO_FILE"; do
|
|
79
|
+
parent="$(dirname "$target")"
|
|
80
|
+
base="$(basename "$target")"
|
|
81
|
+
[ ! -d "$parent" ] || find "$parent" -maxdepth 1 \( -type f -o -type l \) \( -name "$base.ouro-next.*" -o -name "$base.ouro-restore.*" \) -delete
|
|
82
|
+
done
|
|
83
|
+
if [ -n "$CRONTAB_FILE" ]; then
|
|
84
|
+
parent="$(dirname "$CRONTAB_FILE")"
|
|
85
|
+
base="$(basename "$CRONTAB_FILE")"
|
|
86
|
+
[ ! -d "$parent" ] || find "$parent" -maxdepth 1 \( -type f -o -type l \) \( -name "$base.ouro-next.*" -o -name "$base.ouro-restore.*" \) -delete
|
|
87
|
+
fi
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
is_canonical_butler_cron_line() {
|
|
91
|
+
case "$1" in
|
|
92
|
+
"$CRON_LINE"|"$LEGACY_CRON_LINE"|"$CANONICAL_CRON_LINE"|"$CANONICAL_LEGACY_CRON_LINE") return 0 ;;
|
|
93
|
+
*) return 1 ;;
|
|
94
|
+
esac
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
render_cron_without_owned() {
|
|
98
|
+
local source="$1" target="$2" line
|
|
99
|
+
: > "$target"
|
|
100
|
+
while IFS= read -r line || [ -n "$line" ]; do
|
|
101
|
+
if ! is_canonical_butler_cron_line "$line"; then printf '%s\n' "$line" >> "$target"; fi
|
|
102
|
+
done < "$source"
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
cron_contains_owned() {
|
|
106
|
+
local source="$1" line
|
|
107
|
+
while IFS= read -r line || [ -n "$line" ]; do
|
|
108
|
+
if is_canonical_butler_cron_line "$line"; then return 0; fi
|
|
109
|
+
done < "$source"
|
|
110
|
+
return 1
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
render_cron() {
|
|
114
|
+
local source="$1" target="$2"
|
|
115
|
+
render_cron_without_owned "$source" "$target"
|
|
116
|
+
printf '%s\n' "$CRON_LINE" >> "$target"
|
|
117
|
+
/bin/chmod 0600 "$target"
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
render_inactive_cron() {
|
|
121
|
+
local source="$1" target="$2"
|
|
122
|
+
render_cron_without_owned "$source" "$target"
|
|
123
|
+
/bin/chmod 0600 "$target"
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
is_canonical_butler_go_hook() {
|
|
127
|
+
case "$1" in
|
|
128
|
+
"$LEGACY_BOOT_HOOK"|"$LEGACY_BASH_BOOT_HOOK"|"$CANONICAL_PREVIOUS_BOOT_HOOK"|"$CANONICAL_BOOT_HOOK") return 0 ;;
|
|
129
|
+
*) return 1 ;;
|
|
130
|
+
esac
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
render_go_without_owned() {
|
|
134
|
+
local source="$1" target="$2" line
|
|
135
|
+
: > "$target"
|
|
136
|
+
while IFS= read -r line || [ -n "$line" ]; do
|
|
137
|
+
if ! is_canonical_butler_go_hook "$line"; then printf '%s\n' "$line" >> "$target"; fi
|
|
138
|
+
done < "$source"
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
read_system_cron() {
|
|
142
|
+
local error_file="${1}.error" status current_user
|
|
143
|
+
current_user="$(id -un)" || return $?
|
|
144
|
+
if crontab -l > "$1" 2>"$error_file"; then
|
|
145
|
+
SYSTEM_CRON_PRESENT=true
|
|
146
|
+
else
|
|
147
|
+
status=$?
|
|
148
|
+
if [ "$status" -eq 1 ] && grep -Fxq "no crontab for $current_user" "$error_file"; then
|
|
149
|
+
SYSTEM_CRON_PRESENT=false
|
|
150
|
+
: > "$1"
|
|
151
|
+
else
|
|
152
|
+
/bin/rm -f "$1" "$error_file"
|
|
153
|
+
return "$status"
|
|
154
|
+
fi
|
|
155
|
+
fi
|
|
156
|
+
/bin/rm -f "$error_file"
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
activate_cron_file() {
|
|
160
|
+
if [ -n "$CRONTAB_FILE" ]; then
|
|
161
|
+
atomic_install "$1" "$CRONTAB_FILE" 0600
|
|
162
|
+
else
|
|
163
|
+
crontab "$1"
|
|
164
|
+
fi
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
preflight_sources() {
|
|
168
|
+
local name
|
|
169
|
+
for name in "${ASSET_NAMES[@]}"; do
|
|
170
|
+
[ -f "$SOURCE_ROOT/$name" ] && [ ! -L "$SOURCE_ROOT/$name" ] && [ -s "$SOURCE_ROOT/$name" ] || { echo "usenet guard installer: source asset is missing or unsafe: $name" >&2; return 1; }
|
|
171
|
+
done
|
|
172
|
+
/bin/bash -n "$SOURCE_ROOT/usenet-health.sh" "$SOURCE_ROOT/bootstrap-spool.sh" "$SOURCE_ROOT/emit-usenet-event.sh" "$SOURCE_ROOT/install-usenet-guard.sh"
|
|
173
|
+
"$(command -v node)" --check "$SOURCE_ROOT/emit-event.mjs"
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
restore_target() {
|
|
177
|
+
local target="$1" backup="$2" temporary="${1}.ouro-next.$$"
|
|
178
|
+
if [ -f "$backup" ]; then
|
|
179
|
+
/usr/bin/install -m "$(/bin/stat -c '%a' "$backup" 2>/dev/null || /usr/bin/stat -f '%Lp' "$backup")" "$backup" "$temporary"
|
|
180
|
+
/bin/mv "$temporary" "$target"
|
|
181
|
+
else
|
|
182
|
+
/bin/rm -f "$target" "$temporary"
|
|
183
|
+
fi
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
rollback_host_state() {
|
|
187
|
+
local backup="$1" index rollback_status=0 attempt_status
|
|
188
|
+
for index in "${!TARGET_PATHS[@]}"; do
|
|
189
|
+
restore_target "${TARGET_PATHS[$index]}" "$backup/asset-$index"
|
|
190
|
+
attempt_status=$?
|
|
191
|
+
if [ "$attempt_status" -ne 0 ] && [ "$rollback_status" -eq 0 ]; then rollback_status=$attempt_status; fi
|
|
192
|
+
done
|
|
193
|
+
restore_target "$GO_FILE" "$backup/go"
|
|
194
|
+
attempt_status=$?
|
|
195
|
+
if [ "$attempt_status" -ne 0 ] && [ "$rollback_status" -eq 0 ]; then rollback_status=$attempt_status; fi
|
|
196
|
+
if [ -n "$CRONTAB_FILE" ]; then
|
|
197
|
+
restore_target "$CRONTAB_FILE" "$backup/cron"
|
|
198
|
+
elif [ "$SYSTEM_CRON_PRESENT" = true ]; then
|
|
199
|
+
crontab "$backup/cron"
|
|
200
|
+
else
|
|
201
|
+
crontab -r >/dev/null 2>&1
|
|
202
|
+
fi
|
|
203
|
+
attempt_status=$?
|
|
204
|
+
if [ "$attempt_status" -ne 0 ] && [ "$rollback_status" -eq 0 ]; then rollback_status=$attempt_status; fi
|
|
205
|
+
return "$rollback_status"
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
install_transaction() {
|
|
209
|
+
preflight_sources
|
|
210
|
+
/bin/mkdir -p "$INSTALL_ROOT"
|
|
211
|
+
cleanup_stale_atomic_residue
|
|
212
|
+
local stage backup cron_original cron_inactive cron_candidate go_candidate status rollback_status index name target
|
|
213
|
+
stage="$(private_transaction_workspace install)"
|
|
214
|
+
STAGE_PATH="$stage"
|
|
215
|
+
backup="$stage/backup"
|
|
216
|
+
/bin/mkdir "$backup"
|
|
217
|
+
cleanup() { [ -z "$STAGE_PATH" ] || /bin/rm -rf "$STAGE_PATH"; }
|
|
218
|
+
trap cleanup EXIT
|
|
219
|
+
|
|
220
|
+
for index in "${!ASSET_NAMES[@]}"; do
|
|
221
|
+
name="${ASSET_NAMES[$index]}"
|
|
222
|
+
/usr/bin/install -m 0700 "$SOURCE_ROOT/$name" "$stage/$name"
|
|
223
|
+
done
|
|
224
|
+
/bin/bash "$stage/bootstrap-spool.sh" --mount
|
|
225
|
+
/bin/bash "$stage/bootstrap-spool.sh" --self-test
|
|
226
|
+
/bin/bash "$stage/emit-usenet-event.sh" --self-test "$stage/emit-event.mjs"
|
|
227
|
+
|
|
228
|
+
if [ -f "$GO_FILE" ]; then
|
|
229
|
+
/bin/cp -p "$GO_FILE" "$backup/go"
|
|
230
|
+
/bin/cp -p "$GO_FILE" "$stage/go.base"
|
|
231
|
+
else
|
|
232
|
+
printf '#!/bin/bash\n' > "$stage/go.base"
|
|
233
|
+
fi
|
|
234
|
+
go_candidate="$stage/go.candidate"
|
|
235
|
+
grep -Fvx -e "$LEGACY_BOOT_HOOK" -e "$LEGACY_BASH_BOOT_HOOK" -e "$PREVIOUS_INSTALLER_BOOT_HOOK" -e "$BOOT_HOOK" "$stage/go.base" > "$stage/go.filtered" || true
|
|
236
|
+
if [ -s "$stage/go.filtered" ]; then
|
|
237
|
+
awk -v hook="$BOOT_HOOK" 'NR == 1 && /^#!/ { print; print hook; next } NR == 1 { print hook } { print }' "$stage/go.filtered" > "$go_candidate"
|
|
238
|
+
else
|
|
239
|
+
printf '#!/bin/bash\n%s\n' "$BOOT_HOOK" > "$go_candidate"
|
|
240
|
+
fi
|
|
241
|
+
/bin/chmod 0700 "$go_candidate"
|
|
242
|
+
|
|
243
|
+
cron_original="$stage/cron.original"
|
|
244
|
+
if [ -n "$CRONTAB_FILE" ]; then
|
|
245
|
+
if [ -f "$CRONTAB_FILE" ]; then /bin/cp -p "$CRONTAB_FILE" "$backup/cron"; /bin/cp "$CRONTAB_FILE" "$cron_original"; else : > "$cron_original"; fi
|
|
246
|
+
else
|
|
247
|
+
read_system_cron "$cron_original"
|
|
248
|
+
/bin/cp "$cron_original" "$backup/cron"
|
|
249
|
+
fi
|
|
250
|
+
cron_candidate="$stage/cron.candidate"
|
|
251
|
+
render_cron "$cron_original" "$cron_candidate"
|
|
252
|
+
cron_inactive="$stage/cron.inactive"
|
|
253
|
+
render_inactive_cron "$cron_original" "$cron_inactive"
|
|
254
|
+
|
|
255
|
+
for index in "${!TARGET_PATHS[@]}"; do
|
|
256
|
+
target="${TARGET_PATHS[$index]}"
|
|
257
|
+
if [ -e "$target" ]; then
|
|
258
|
+
[ -f "$target" ] && [ ! -L "$target" ] || { echo "usenet guard installer: live asset is unsafe" >&2; return 1; }
|
|
259
|
+
/bin/cp -p "$target" "$backup/asset-$index"
|
|
260
|
+
fi
|
|
261
|
+
done
|
|
262
|
+
|
|
263
|
+
set +e
|
|
264
|
+
status=0
|
|
265
|
+
if cron_contains_owned "$cron_original"; then activate_cron_file "$cron_inactive" || status=$?; fi
|
|
266
|
+
if [ "$status" -eq 0 ]; then
|
|
267
|
+
for index in "${!TARGET_PATHS[@]}"; do
|
|
268
|
+
atomic_install "$stage/${ASSET_NAMES[$index]}" "${TARGET_PATHS[$index]}" 0700 || { status=$?; break; }
|
|
269
|
+
done
|
|
270
|
+
fi
|
|
271
|
+
if [ "$status" -eq 0 ]; then atomic_install "$go_candidate" "$GO_FILE" 0700 || status=$?; fi
|
|
272
|
+
if [ "$status" -eq 0 ]; then activate_cron_file "$cron_candidate" || status=$?; fi
|
|
273
|
+
set -e
|
|
274
|
+
if [ "$status" -ne 0 ]; then
|
|
275
|
+
set +e
|
|
276
|
+
rollback_host_state "$backup"
|
|
277
|
+
rollback_status=$?
|
|
278
|
+
set -e
|
|
279
|
+
if [ "$rollback_status" -ne 0 ]; then return "$rollback_status"; fi
|
|
280
|
+
return "$status"
|
|
281
|
+
fi
|
|
282
|
+
cleanup
|
|
283
|
+
STAGE_PATH=""
|
|
284
|
+
trap - EXIT
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
boot_activate() {
|
|
288
|
+
/bin/bash "$EVENT_ROOT/bootstrap-spool.sh" --mount
|
|
289
|
+
/bin/bash "$EVENT_ROOT/bootstrap-spool.sh" --self-test
|
|
290
|
+
/bin/bash "$EVENT_ROOT/emit-usenet-event.sh" --self-test "$EVENT_ROOT/emit-event.mjs"
|
|
291
|
+
local workspace current candidate
|
|
292
|
+
workspace="$(private_transaction_workspace boot)"
|
|
293
|
+
STAGE_PATH="$workspace"
|
|
294
|
+
cleanup() { [ -z "$STAGE_PATH" ] || /bin/rm -rf "$STAGE_PATH"; }
|
|
295
|
+
trap cleanup EXIT
|
|
296
|
+
current="$workspace/current"
|
|
297
|
+
candidate="$workspace/candidate"
|
|
298
|
+
/usr/bin/install -m 0600 /dev/null "$current"
|
|
299
|
+
/usr/bin/install -m 0600 /dev/null "$current.error"
|
|
300
|
+
/usr/bin/install -m 0600 /dev/null "$candidate"
|
|
301
|
+
if [ -n "$CRONTAB_FILE" ]; then
|
|
302
|
+
if [ -f "$CRONTAB_FILE" ]; then /bin/cp "$CRONTAB_FILE" "$current"; else : > "$current"; fi
|
|
303
|
+
else
|
|
304
|
+
read_system_cron "$current"
|
|
305
|
+
fi
|
|
306
|
+
render_cron "$current" "$candidate"
|
|
307
|
+
activate_cron_file "$candidate"
|
|
308
|
+
cleanup
|
|
309
|
+
STAGE_PATH=""
|
|
310
|
+
trap - EXIT
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
restore_snapshot() {
|
|
314
|
+
local expected_inventory stage backup cron_original cron_inactive cron_candidate go_original go_filtered go_candidate status rollback_status index target relative source state global_name global_state global_digest global_count go_state cron_state
|
|
315
|
+
expected_inventory=$'present\tcustom/usenet_health.sh\npresent\tcustom/ouro-events/bootstrap-spool.sh\npresent\tcustom/ouro-events/emit-event.mjs\npresent\tcustom/ouro-events/emit-usenet-event.sh\npresent\tcustom/ouro-events/install-usenet-guard.sh\npresent\tgo.butler-lines\npresent\tcrontab.butler-lines\npresent\tglobal-state'
|
|
316
|
+
[ -d "$RESTORE_ROOT" ] && [ ! -L "$RESTORE_ROOT" ] || return 1
|
|
317
|
+
[ -f "$RESTORE_ROOT/inventory" ] && [ ! -L "$RESTORE_ROOT/inventory" ] || return 1
|
|
318
|
+
while IFS=$'\t' read -r state relative; do
|
|
319
|
+
case "$state:$relative" in
|
|
320
|
+
present:custom/usenet_health.sh|absent:custom/usenet_health.sh|present:custom/ouro-events/bootstrap-spool.sh|absent:custom/ouro-events/bootstrap-spool.sh|present:custom/ouro-events/emit-event.mjs|absent:custom/ouro-events/emit-event.mjs|present:custom/ouro-events/emit-usenet-event.sh|absent:custom/ouro-events/emit-usenet-event.sh|present:custom/ouro-events/install-usenet-guard.sh|absent:custom/ouro-events/install-usenet-guard.sh|present:go.butler-lines|present:crontab.butler-lines|present:global-state) ;;
|
|
321
|
+
*) return 1 ;;
|
|
322
|
+
esac
|
|
323
|
+
if [ "$state" = present ]; then [ -f "$RESTORE_ROOT/$relative" ] && [ ! -L "$RESTORE_ROOT/$relative" ] || return 1; else [ ! -e "$RESTORE_ROOT/$relative" ] && [ ! -L "$RESTORE_ROOT/$relative" ] || return 1; fi
|
|
324
|
+
done < "$RESTORE_ROOT/inventory"
|
|
325
|
+
test "$(sed 's/^absent/present/' "$RESTORE_ROOT/inventory")" = "$expected_inventory" || return 1
|
|
326
|
+
index=0
|
|
327
|
+
while IFS=$'\t' read -r global_name global_state global_digest global_count; do
|
|
328
|
+
case "$index:$global_name:$global_state" in
|
|
329
|
+
0:go:present|0:go:absent) go_state="$global_state" ;;
|
|
330
|
+
1:crontab:present|1:crontab:absent) cron_state="$global_state" ;;
|
|
331
|
+
*) return 1 ;;
|
|
332
|
+
esac
|
|
333
|
+
if [ "$global_state" = absent ]; then
|
|
334
|
+
[ "$global_digest" = - ] || return 1
|
|
335
|
+
else
|
|
336
|
+
[ "${#global_digest}" -eq 64 ] || return 1
|
|
337
|
+
case "$global_digest" in *[!0-9a-f]*) return 1 ;; esac
|
|
338
|
+
fi
|
|
339
|
+
case "$global_count" in ''|*[!0-9]*) return 1 ;; esac
|
|
340
|
+
[ "$(awk 'END { print NR + 0 }' "$RESTORE_ROOT/$global_name.butler-lines")" = "$global_count" ] || return 1
|
|
341
|
+
index=$((index + 1))
|
|
342
|
+
done < "$RESTORE_ROOT/global-state"
|
|
343
|
+
[ "$index" -eq 2 ] || return 1
|
|
344
|
+
while IFS= read -r source || [ -n "$source" ]; do is_canonical_butler_go_hook "$source" || return 1; done < "$RESTORE_ROOT/go.butler-lines"
|
|
345
|
+
while IFS= read -r source || [ -n "$source" ]; do
|
|
346
|
+
case "$source" in
|
|
347
|
+
"*/15 * * * * /bin/bash /boot/config/custom/usenet_health.sh"|"*/15 * * * * /bin/bash /boot/config/custom/usenet_health.sh # ouro:usenet-health") ;;
|
|
348
|
+
*) return 1 ;;
|
|
349
|
+
esac
|
|
350
|
+
done < "$RESTORE_ROOT/crontab.butler-lines"
|
|
351
|
+
|
|
352
|
+
/bin/mkdir -p "$INSTALL_ROOT"
|
|
353
|
+
cleanup_stale_atomic_residue
|
|
354
|
+
stage="$(private_transaction_workspace restore)"
|
|
355
|
+
STAGE_PATH="$stage"
|
|
356
|
+
backup="$stage/backup"
|
|
357
|
+
/bin/mkdir "$backup"
|
|
358
|
+
cleanup() { [ -z "$STAGE_PATH" ] || /bin/rm -rf "$STAGE_PATH"; }
|
|
359
|
+
trap cleanup EXIT
|
|
360
|
+
for index in "${!TARGET_PATHS[@]}"; do
|
|
361
|
+
target="${TARGET_PATHS[$index]}"
|
|
362
|
+
if [ -e "$target" ]; then [ -f "$target" ] && [ ! -L "$target" ] || return 1; /bin/cp -p "$target" "$backup/asset-$index"; fi
|
|
363
|
+
done
|
|
364
|
+
if [ -e "$GO_FILE" ]; then [ -f "$GO_FILE" ] && [ ! -L "$GO_FILE" ] || return 1; /bin/cp -p "$GO_FILE" "$backup/go"; fi
|
|
365
|
+
go_original="$stage/go.original"
|
|
366
|
+
if [ -f "$GO_FILE" ]; then /bin/cp "$GO_FILE" "$go_original"; else printf '#!/bin/bash\n' > "$go_original"; fi
|
|
367
|
+
go_filtered="$stage/go.filtered"
|
|
368
|
+
render_go_without_owned "$go_original" "$go_filtered"
|
|
369
|
+
go_candidate="$stage/go.candidate"
|
|
370
|
+
if [ -s "$RESTORE_ROOT/go.butler-lines" ]; then
|
|
371
|
+
awk -v fragments="$RESTORE_ROOT/go.butler-lines" 'NR == 1 && /^#!/ { print; while ((getline line < fragments) > 0) print line; close(fragments); next } NR == 1 { while ((getline line < fragments) > 0) print line; close(fragments) } { print }' "$go_filtered" > "$go_candidate"
|
|
372
|
+
else
|
|
373
|
+
/bin/cp "$go_filtered" "$go_candidate"
|
|
374
|
+
fi
|
|
375
|
+
/bin/chmod 0700 "$go_candidate"
|
|
376
|
+
cron_original="$stage/cron.original"
|
|
377
|
+
if [ -n "$CRONTAB_FILE" ]; then
|
|
378
|
+
if [ -f "$CRONTAB_FILE" ]; then /bin/cp -p "$CRONTAB_FILE" "$backup/cron"; /bin/cp "$CRONTAB_FILE" "$cron_original"; else : > "$cron_original"; fi
|
|
379
|
+
else
|
|
380
|
+
read_system_cron "$cron_original"
|
|
381
|
+
/bin/cp "$cron_original" "$backup/cron"
|
|
382
|
+
fi
|
|
383
|
+
cron_inactive="$stage/cron.inactive"
|
|
384
|
+
render_inactive_cron "$cron_original" "$cron_inactive"
|
|
385
|
+
cron_candidate="$stage/cron.candidate"
|
|
386
|
+
/bin/cp "$cron_inactive" "$cron_candidate"
|
|
387
|
+
/bin/cat "$RESTORE_ROOT/crontab.butler-lines" >> "$cron_candidate"
|
|
388
|
+
/bin/chmod 0600 "$cron_candidate"
|
|
389
|
+
|
|
390
|
+
set +e
|
|
391
|
+
status=0
|
|
392
|
+
if cron_contains_owned "$cron_original"; then activate_cron_file "$cron_inactive" || status=$?; fi
|
|
393
|
+
for index in "${!TARGET_PATHS[@]}"; do
|
|
394
|
+
[ "$status" -eq 0 ] || break
|
|
395
|
+
target="${TARGET_PATHS[$index]}"
|
|
396
|
+
case "$index" in
|
|
397
|
+
0) relative=custom/usenet_health.sh ;;
|
|
398
|
+
1) relative=custom/ouro-events/bootstrap-spool.sh ;;
|
|
399
|
+
2) relative=custom/ouro-events/emit-event.mjs ;;
|
|
400
|
+
3) relative=custom/ouro-events/emit-usenet-event.sh ;;
|
|
401
|
+
4) relative=custom/ouro-events/install-usenet-guard.sh ;;
|
|
402
|
+
esac
|
|
403
|
+
source="$RESTORE_ROOT/$relative"
|
|
404
|
+
if [ -f "$source" ]; then atomic_install "$source" "$target" 0700 || status=$?; else /bin/rm -f "$target" || status=$?; fi
|
|
405
|
+
done
|
|
406
|
+
if [ "$status" -eq 0 ]; then
|
|
407
|
+
if [ "$go_state" = absent ] && [ "$(awk 'NF && $0 !~ /^#!/ { count++ } END { print count + 0 }' "$go_candidate")" -eq 0 ]; then /bin/rm -f "$GO_FILE" || status=$?; else atomic_install "$go_candidate" "$GO_FILE" 0700 || status=$?; fi
|
|
408
|
+
fi
|
|
409
|
+
if [ "$status" -eq 0 ]; then
|
|
410
|
+
if [ "$cron_state" = absent ] && [ ! -s "$cron_candidate" ]; then
|
|
411
|
+
if [ -n "$CRONTAB_FILE" ]; then /bin/rm -f "$CRONTAB_FILE" || status=$?; else crontab -r >/dev/null 2>&1 || status=$?; fi
|
|
412
|
+
else
|
|
413
|
+
activate_cron_file "$cron_candidate" || status=$?
|
|
414
|
+
fi
|
|
415
|
+
fi
|
|
416
|
+
set -e
|
|
417
|
+
if [ "$status" -ne 0 ]; then
|
|
418
|
+
set +e
|
|
419
|
+
rollback_host_state "$backup"
|
|
420
|
+
rollback_status=$?
|
|
421
|
+
set -e
|
|
422
|
+
if [ "$rollback_status" -ne 0 ]; then return "$rollback_status"; fi
|
|
423
|
+
return "$status"
|
|
424
|
+
fi
|
|
425
|
+
cleanup
|
|
426
|
+
STAGE_PATH=""
|
|
427
|
+
trap - EXIT
|
|
428
|
+
}
|
|
429
|
+
|
|
430
|
+
case "$ACTION" in
|
|
431
|
+
install) install_transaction ;;
|
|
432
|
+
boot) boot_activate ;;
|
|
433
|
+
restore) restore_snapshot ;;
|
|
434
|
+
esac
|
|
@@ -0,0 +1,178 @@
|
|
|
1
|
+
#!/bin/bash
|
|
2
|
+
set -uo pipefail
|
|
3
|
+
|
|
4
|
+
LOG="/var/log/usenet_health.log"
|
|
5
|
+
SAB_INI="/mnt/user/appdata/sabnzbd/sabnzbd.ini"
|
|
6
|
+
EVENT_PRODUCER="/boot/config/custom/ouro-events/emit-event.mjs"
|
|
7
|
+
EVENT_ADAPTER="/boot/config/custom/ouro-events/emit-usenet-event.sh"
|
|
8
|
+
BASELINE_STATE="/boot/config/custom/usenet_health_baseline.json"
|
|
9
|
+
BASELINE_STATE_EXPLICIT=0
|
|
10
|
+
|
|
11
|
+
while (($# > 0)); do
|
|
12
|
+
case "$1" in
|
|
13
|
+
--sab-ini) SAB_INI="${2:?missing --sab-ini value}"; shift 2 ;;
|
|
14
|
+
--log) LOG="${2:?missing --log value}"; shift 2 ;;
|
|
15
|
+
--producer) EVENT_PRODUCER="${2:?missing --producer value}"; shift 2 ;;
|
|
16
|
+
--adapter) EVENT_ADAPTER="${2:?missing --adapter value}"; shift 2 ;;
|
|
17
|
+
--state) BASELINE_STATE="${2:?missing --state value}"; BASELINE_STATE_EXPLICIT=1; shift 2 ;;
|
|
18
|
+
*) echo "usenet health guard: invalid argument: $1" >&2; exit 2 ;;
|
|
19
|
+
esac
|
|
20
|
+
done
|
|
21
|
+
if [ "$BASELINE_STATE_EXPLICIT" -eq 0 ] && [ "$LOG" != "/var/log/usenet_health.log" ]; then BASELINE_STATE="${LOG}.baseline.json"; fi
|
|
22
|
+
|
|
23
|
+
log() { echo "$(date '+%Y-%m-%d %H:%M:%S') $1" >> "$LOG"; }
|
|
24
|
+
|
|
25
|
+
emit_transition() {
|
|
26
|
+
if ! /bin/bash "$EVENT_ADAPTER" "$@" >> "$LOG" 2>&1; then
|
|
27
|
+
log "[EVENT] Canonical Butler event emission failed; Unraid UI and this log retain the observation or verified action."
|
|
28
|
+
fi
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
emit_blind_observation() {
|
|
32
|
+
local reason="$1"
|
|
33
|
+
local observed_slot verified_at verification_digest
|
|
34
|
+
observed_slot=$(date -u '+%Y%m%dT%H%M%SZ')
|
|
35
|
+
verified_at=$(date -u '+%Y-%m-%dT%H:%M:%S.000Z')
|
|
36
|
+
verification_digest=$(printf 'sabnzbd.health_check=indeterminate\nreason=%s' "$reason" | sha256sum | cut -d' ' -f1)
|
|
37
|
+
log "[USENET] CRITICAL — the SABnzbd health check is blind (${reason}); no healthy state was inferred."
|
|
38
|
+
emit_transition "usenet.observe" "provider-health" "indeterminate:${observed_slot}" "usenet:provider-health:${observed_slot}:indeterminate" "The Usenet health check could not verify current state." "A required SABnzbd ${reason} check failed; treat provider health as unknown and investigate the local API path." "false" "$verification_digest" "$verified_at"
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
sab_request() {
|
|
42
|
+
local timeout="$1"
|
|
43
|
+
local url="$2"
|
|
44
|
+
printf 'get\ndata-urlencode = "apikey=%s"\n' "$SAB_KEY" \
|
|
45
|
+
| curl --silent --fail --max-time "$timeout" --config - "$url" 2>/dev/null
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
fetch_sab_json() {
|
|
49
|
+
local target="$1"
|
|
50
|
+
local timeout="$2"
|
|
51
|
+
local url="$3"
|
|
52
|
+
local schema="$4"
|
|
53
|
+
local surface="$5"
|
|
54
|
+
local payload
|
|
55
|
+
if ! payload=$(sab_request "$timeout" "$url"); then
|
|
56
|
+
emit_blind_observation "${surface}-transport"
|
|
57
|
+
return 1
|
|
58
|
+
fi
|
|
59
|
+
if ! printf '%s' "$payload" | jq -e 'type == "object" and (.status != false) and ((has("error") | not) or .error == null or .error == "")' >/dev/null 2>&1 \
|
|
60
|
+
|| ! printf '%s' "$payload" | jq -e "$schema" >/dev/null 2>&1; then
|
|
61
|
+
emit_blind_observation "${surface}-response"
|
|
62
|
+
return 1
|
|
63
|
+
fi
|
|
64
|
+
printf -v "$target" '%s' "$payload"
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/usr/local/bin/node "$EVENT_PRODUCER" --maintain >> "$LOG" 2>&1 || log "[EVENT] Spool maintenance failed; unsafe or unparseable artifacts were preserved."
|
|
68
|
+
|
|
69
|
+
SAB_KEY=$(grep -m1 '^api_key' "$SAB_INI" 2>/dev/null | cut -d= -f2 | tr -d ' ')
|
|
70
|
+
if [ -z "$SAB_KEY" ]; then
|
|
71
|
+
emit_blind_observation "credential-unavailable"
|
|
72
|
+
exit 1
|
|
73
|
+
fi
|
|
74
|
+
if [[ ! "$SAB_KEY" =~ ^[[:alnum:]_-]+$ ]]; then
|
|
75
|
+
emit_blind_observation "credential-invalid"
|
|
76
|
+
exit 1
|
|
77
|
+
fi
|
|
78
|
+
|
|
79
|
+
fetch_sab_json WARN 15 "http://localhost:8090/api?mode=warnings&output=json" '(.warnings | type) == "array"' "warnings" || exit 1
|
|
80
|
+
|
|
81
|
+
fetch_sab_json Q 15 "http://localhost:8090/api?mode=queue&output=json" '(.queue | type) == "object" and (.queue.status | type) == "string" and (.queue.paused | type) == "boolean" and (((.queue.noofslots | type) == "number") or ((.queue.noofslots | type) == "string"))' "queue" || exit 1
|
|
82
|
+
Q_STATUS=$(echo "$Q" | jq -r '.queue.status // "?"' 2>/dev/null)
|
|
83
|
+
Q_SLOTS=$(echo "$Q" | jq -r '.queue.noofslots // 0' 2>/dev/null)
|
|
84
|
+
STALLED=0
|
|
85
|
+
if [ "$Q_STATUS" = "Idle" ] && [ "${Q_SLOTS:-0}" -gt 0 ]; then STALLED=1; fi
|
|
86
|
+
|
|
87
|
+
# Protected spend-guard logic: volume is allowed; sustained article failure is not.
|
|
88
|
+
MIN_ARTICLES=50000
|
|
89
|
+
MIN_RATE=30
|
|
90
|
+
TODAY=$(date +%Y-%m-%d)
|
|
91
|
+
fetch_sab_json STATS 20 "http://localhost:8090/api?mode=server_stats&output=json" '(.servers | type) == "array"' "server-stats" || exit 1
|
|
92
|
+
TRIED=$(echo "$STATS" | jq -r --arg d "$TODAY" '[.servers[].articles_tried[$d] // 0] | add // 0' 2>/dev/null)
|
|
93
|
+
OKAY=$(echo "$STATS" | jq -r --arg d "$TODAY" '[.servers[].articles_success[$d] // 0] | add // 0' 2>/dev/null)
|
|
94
|
+
BYTES=$(echo "$STATS" | jq -r --arg d "$TODAY" '[.servers[].daily[$d] // 0] | add // 0' 2>/dev/null)
|
|
95
|
+
case "$TRIED" in ''|*[!0-9]*) TRIED=0 ;; esac
|
|
96
|
+
case "$OKAY" in ''|*[!0-9]*) OKAY=0 ;; esac
|
|
97
|
+
case "$BYTES" in ''|*[!0-9]*) BYTES=0 ;; esac
|
|
98
|
+
GB=$(( ${BYTES:-0} / 1000000000 ))
|
|
99
|
+
|
|
100
|
+
BASE_DAY=""
|
|
101
|
+
BASE_TRIED=0
|
|
102
|
+
BASE_OKAY=0
|
|
103
|
+
BASE_VALID=0
|
|
104
|
+
if [ -f "$BASELINE_STATE" ] && [ ! -L "$BASELINE_STATE" ]; then
|
|
105
|
+
BASE_DAY=$(/usr/bin/jq -r '.day // ""' "$BASELINE_STATE" 2>/dev/null)
|
|
106
|
+
BASE_TRIED=$(/usr/bin/jq -r '.tried // 0' "$BASELINE_STATE" 2>/dev/null)
|
|
107
|
+
BASE_OKAY=$(/usr/bin/jq -r '.okay // 0' "$BASELINE_STATE" 2>/dev/null)
|
|
108
|
+
fi
|
|
109
|
+
case "$BASE_TRIED" in ''|*[!0-9]*) BASE_TRIED=0 ;; esac
|
|
110
|
+
case "$BASE_OKAY" in ''|*[!0-9]*) BASE_OKAY=0 ;; esac
|
|
111
|
+
if [ "$BASE_DAY" = "$TODAY" ] && [ "$TRIED" -ge "$BASE_TRIED" ] && [ "$OKAY" -ge "$BASE_OKAY" ]; then BASE_VALID=1; fi
|
|
112
|
+
write_baseline() {
|
|
113
|
+
BASELINE_TMP=$(mktemp "${BASELINE_STATE}.tmp.XXXXXX") || return 1
|
|
114
|
+
printf '{"day":"%s","tried":%s,"okay":%s}\n' "$TODAY" "$TRIED" "$OKAY" > "$BASELINE_TMP" && chmod 600 "$BASELINE_TMP" && mv "$BASELINE_TMP" "$BASELINE_STATE"
|
|
115
|
+
}
|
|
116
|
+
if [ "$BASE_VALID" -eq 0 ]; then BASE_TRIED=$TRIED; BASE_OKAY=$OKAY; write_baseline || { log "[SPEND] CRITICAL — could not persist the spend baseline."; exit 1; }; fi
|
|
117
|
+
DELTA_TRIED=$(( TRIED - BASE_TRIED ))
|
|
118
|
+
DELTA_OKAY=$(( OKAY - BASE_OKAY ))
|
|
119
|
+
|
|
120
|
+
# A cleared warning is not recovery. Require bounded evidence that articles succeeded
|
|
121
|
+
# and one download reached Completed during the last two guard intervals.
|
|
122
|
+
fetch_sab_json HISTORY 15 "http://localhost:8090/api?mode=history&limit=20&start=0&output=json" '(.history | type) == "object" and (.history.slots | type) == "array"' "history" || exit 1
|
|
123
|
+
RECENT_SINCE=$(( $(date -u '+%s') - 1800 ))
|
|
124
|
+
RECENT_COMPLETED=$(echo "$HISTORY" | jq -r --argjson since "$RECENT_SINCE" '[.history.slots[]? | select(.status == "Completed" and ((.completed // 0) | tonumber) >= $since)] | length' 2>/dev/null)
|
|
125
|
+
case "$RECENT_COMPLETED" in ''|*[!0-9]*) RECENT_COMPLETED=0 ;; esac
|
|
126
|
+
AUTH_FAIL=$(echo "$WARN" | jq -r --argjson since "$RECENT_SINCE" '[.warnings[]? | select((.text | test("Authentication Failed|Failed login"; "i")) and ((.time // 0) | tonumber) >= $since)] | length' 2>/dev/null)
|
|
127
|
+
case "$AUTH_FAIL" in ''|*[!0-9]*) AUTH_FAIL=0 ;; esac
|
|
128
|
+
|
|
129
|
+
if [ "${DELTA_TRIED:-0}" -ge "$MIN_ARTICLES" ]; then
|
|
130
|
+
RATE=$(( DELTA_OKAY * 100 / DELTA_TRIED ))
|
|
131
|
+
if [ "$RATE" -lt "$MIN_RATE" ]; then
|
|
132
|
+
fetch_sab_json PAUSE_RESPONSE 15 "http://localhost:8090/api?mode=pause&output=json" '(.status == true)' "pause" || exit 1
|
|
133
|
+
fetch_sab_json PAUSE_Q 15 "http://localhost:8090/api?mode=queue&output=json" '(.queue | type) == "object" and (.queue.paused | type) == "boolean"' "pause-verification" || exit 1
|
|
134
|
+
PAUSED_AFTER=$(printf '%s' "$PAUSE_Q" | jq -r '.queue.paused')
|
|
135
|
+
VERIFIED_AT=$(date -u '+%Y-%m-%dT%H:%M:%S.000Z')
|
|
136
|
+
VERIFICATION_DIGEST=$(printf 'sabnzbd.queue.paused=%s' "$PAUSED_AFTER" | sha256sum | cut -d' ' -f1)
|
|
137
|
+
if [ "$PAUSED_AFTER" = "true" ]; then
|
|
138
|
+
log "[SPEND] CRITICAL — verified the download queue is PAUSED. ${GB} GB fetched today at ${RATE}% article success (threshold ${MIN_RATE}%)."
|
|
139
|
+
emit_transition "sabnzbd.pause" "spend-guard" "spend-pause:${TODAY}:verified" "sabnzbd:pause:${TODAY}:spend-guard" "Downloads were paused after article success fell below the spend guard." "At least 50,000 articles were attempted at less than 30% success; an independent queue read verified paused=true." "true" "$VERIFICATION_DIGEST" "$VERIFIED_AT"
|
|
140
|
+
else
|
|
141
|
+
log "[SPEND] CRITICAL — pause request was not independently verified; only a verification-failure event was emitted."
|
|
142
|
+
emit_transition "sabnzbd.pause" "spend-guard" "spend-pause:${TODAY}:unverified" "sabnzbd:pause-request:${TODAY}:spend-guard" "The spend guard requested a download pause but could not verify it." "An independent SAB queue read returned paused=false; treat the protective action as unverified and investigate." "false" "$VERIFICATION_DIGEST" "$VERIFIED_AT"
|
|
143
|
+
fi
|
|
144
|
+
else
|
|
145
|
+
log "[SPEND] OK — ${GB} GB today at ${RATE}% article success."
|
|
146
|
+
fi
|
|
147
|
+
write_baseline || { log "[SPEND] CRITICAL — could not advance the spend baseline."; exit 1; }
|
|
148
|
+
fi
|
|
149
|
+
|
|
150
|
+
VERIFIED_AT=$(date -u '+%Y-%m-%dT%H:%M:%S.000Z')
|
|
151
|
+
OBSERVED_SLOT=$(date -u '+%Y%m%dT%H%M%SZ')
|
|
152
|
+
fetch_sab_json FINAL_Q 15 "http://localhost:8090/api?mode=queue&output=json" '(.queue | type) == "object" and (.queue.paused | type) == "boolean"' "final-queue" || exit 1
|
|
153
|
+
FINAL_PAUSED=$(echo "$FINAL_Q" | jq -r '.queue.paused // false' 2>/dev/null)
|
|
154
|
+
if [ "$AUTH_FAIL" -eq 0 ] && [ "$STALLED" -eq 0 ] && [ "$FINAL_PAUSED" = "false" ] && [ "${DELTA_OKAY:-0}" -gt 0 ] && [ "${RECENT_COMPLETED:-0}" -gt 0 ]; then
|
|
155
|
+
VERIFICATION_DIGEST=$(printf 'sabnzbd.auth_fail=0\nsabnzbd.stalled=0\nsabnzbd.queue.paused=false\nsabnzbd.interval_articles_tried=%s\nsabnzbd.interval_articles_success=%s\nsabnzbd.recent_completed=%s' "$DELTA_TRIED" "$DELTA_OKAY" "$RECENT_COMPLETED" | sha256sum | cut -d' ' -f1)
|
|
156
|
+
emit_transition "usenet.observe" "provider-health" "recovered:${OBSERVED_SLOT}" "usenet:provider-health:${OBSERVED_SLOT}:recovered" "The download path completed a real job again." "SABnzbd reports ${DELTA_OKAY} successful articles out of ${DELTA_TRIED} newly attempted since the prior check and ${RECENT_COMPLETED} recent completed downloads; a final independent queue read verified paused=false." "true" "$VERIFICATION_DIGEST" "$VERIFIED_AT"
|
|
157
|
+
tail -300 "$LOG" > "${LOG}.tmp" 2>/dev/null && mv "${LOG}.tmp" "$LOG"
|
|
158
|
+
exit 0
|
|
159
|
+
fi
|
|
160
|
+
|
|
161
|
+
if [ "$AUTH_FAIL" -eq 0 ] && [ "$STALLED" -eq 0 ]; then
|
|
162
|
+
log "[USENET] OK — there is not yet whole-path recovery evidence (${DELTA_OKAY} newly successful articles; ${RECENT_COMPLETED} recent completed downloads; paused=${FINAL_PAUSED})."
|
|
163
|
+
tail -300 "$LOG" > "${LOG}.tmp" 2>/dev/null && mv "${LOG}.tmp" "$LOG"
|
|
164
|
+
exit 0
|
|
165
|
+
fi
|
|
166
|
+
|
|
167
|
+
if [ "$AUTH_FAIL" -gt 0 ]; then
|
|
168
|
+
SAMPLE=$(echo "$WARN" | jq -r --argjson since "$RECENT_SINCE" '[.warnings[]? | select((.text | test("Authentication Failed|Failed login"; "i")) and ((.time // 0) | tonumber) >= $since)] | last | .text' 2>/dev/null)
|
|
169
|
+
log "[USENET] CRITICAL — SABnzbd cannot authenticate to the news server: ${SAMPLE:0:120}"
|
|
170
|
+
VERIFICATION_DIGEST=$(printf 'sabnzbd.auth_fail=1' | sha256sum | cut -d' ' -f1)
|
|
171
|
+
emit_transition "usenet.observe" "provider-health" "auth-failed:${OBSERVED_SLOT}" "usenet:provider-health:${OBSERVED_SLOT}:auth-failed" "The news provider rejected SABnzbd authentication." "SAB warning history contains an authentication failure; the Butler should investigate credentials, provider status, and next steps." "true" "$VERIFICATION_DIGEST" "$VERIFIED_AT"
|
|
172
|
+
else
|
|
173
|
+
log "[USENET] CRITICAL — SABnzbd is Idle with $Q_SLOTS job(s) queued and nothing downloading."
|
|
174
|
+
VERIFICATION_DIGEST=$(printf 'sabnzbd.queue.status=Idle\nsabnzbd.queue.slots=%s' "$Q_SLOTS" | sha256sum | cut -d' ' -f1)
|
|
175
|
+
emit_transition "usenet.observe" "provider-health" "stalled:${OBSERVED_SLOT}" "usenet:provider-health:${OBSERVED_SLOT}:stalled" "Usenet downloads appear stalled." "SABnzbd is idle with queued jobs; the Butler should investigate the provider and download path before deciding what to do." "true" "$VERIFICATION_DIGEST" "$VERIFIED_AT"
|
|
176
|
+
fi
|
|
177
|
+
|
|
178
|
+
tail -300 "$LOG" > "${LOG}.tmp" 2>/dev/null && mv "${LOG}.tmp" "$LOG"
|
|
@@ -71,13 +71,13 @@
|
|
|
71
71
|
"restart-attempt-ledger": { "kind": "fixed-ndjson", "path": "/home/ouro/AgentBundles/sanctuary.ouro/state/acceptance/restart-attempts.ndjson", "redaction": "scenario-action-target-digests-and-state-only" },
|
|
72
72
|
"telegram-turn-receipts": { "kind": "fixed-ndjson", "path": "/home/ouro/AgentBundles/sanctuary.ouro/state/acceptance/telegram-turns.ndjson", "redaction": "scenario-update-tool-response-and-delivery-digests-only" },
|
|
73
73
|
"read-only-denial-receipt": { "kind": "persisted-one-shot-live-graphql-denial", "schemaVersion": "sanctuary-read-only-denial-receipt-v1", "path": "/home/ouro/AgentBundles/sanctuary.ouro/state/acceptance/denial-receipts/<scenario-handle-digest>.json", "attemptPath": "/home/ouro/AgentBundles/sanctuary.ouro/state/acceptance/denial-attempts/<scenario-handle-digest>.json", "endpoint": "http://127.0.0.1/graphql", "targetSnapshotOperation": "denial_target_snapshot", "resumePolicy": "attempted-or-indeterminate-inspect-before-retry", "redaction": "schema-binding-api-classification-counts-restart-count-and-seven-boundary-digests-only" },
|
|
74
|
-
"containment-audit": { "kind": "effective-container-key-tool-and-audit-boundary", "schemaVersion": "sanctuary-containment-audit-v1", "path": "/
|
|
74
|
+
"containment-audit": { "kind": "effective-container-key-tool-and-audit-boundary", "schemaVersion": "sanctuary-containment-audit-v1", "path": "/home/ouro/AgentBundles/sanctuary.ouro/tool-profiles.json", "auditPath": "/home/ouro/AgentBundles/sanctuary.ouro/state/acceptance/telegram-audit-chain.ndjson", "producers": ["container_snapshot", "inventory_keys", "live-bundle-relationship-capability-registry", "runtime-tool-resolver", "telegram-audit-mac-chain"], "redaction": "counts-static-policy-values-and-sha256-digests-only" },
|
|
75
75
|
"container-inspect": { "kind": "fixed-host-snapshot", "path": "/run/ouro-acceptance/container-inspect.json", "redaction": "typed-facts-and-digests-only" },
|
|
76
76
|
"provider-live-check": { "kind": "fixed-runtime-api", "operation": "sanctuary-provider-readiness", "redaction": "booleans-and-distinctness-only" },
|
|
77
77
|
"live-grounding-read": { "kind": "fixed-runtime-api", "operation": "sanctuary-independent-system-or-storage-read", "redaction": "bounded-system-or-storage-facts-and-digest-only" },
|
|
78
78
|
"cron-runtime": { "kind": "fixed-runtime-api", "operation": "sanctuary-cron-readiness", "redaction": "typed-counters-and-digests-only" },
|
|
79
79
|
"health-runtime": { "kind": "fixed-runtime-api", "operation": "sanctuary-health-transition", "redaction": "typed-counters-and-digests-only" },
|
|
80
|
-
"digest-runtime": { "kind": "fixed-runtime-api", "operation": "sanctuary-
|
|
80
|
+
"digest-runtime": { "kind": "fixed-runtime-api", "operation": "sanctuary-acceptance-delivery-probe", "redaction": "typed-counters-and-digests-only" },
|
|
81
81
|
"health-probe-receipt": { "kind": "fixed-private-json", "path": "/home/ouro/AgentBundles/sanctuary.ouro/state/acceptance/health-probe-receipts/<scenarioHandleDigest>.json", "redaction": "typed-phase-counters-owner-state-and-cleanup-digests-only" },
|
|
82
82
|
"scheduler-liveness-receipt": { "kind": "fixed-private-json", "path": "/home/ouro/AgentBundles/sanctuary.ouro/state/acceptance/scheduler-liveness-receipts/<scenarioHandleDigest>.json", "redaction": "daemon-runner-supervisor-manifest-and-cursor-attestation-only" },
|
|
83
83
|
"interactive-driver-receipt": { "kind": "fixed-private-json", "path": "/home/ouro/AgentBundles/sanctuary.ouro/state/acceptance/interactive-driver-receipts/<scenarioHandleDigest>.json", "redaction": "digested-approval-checkpoint-session-callback-and-owner-coordinates-only" },
|
|
@@ -150,7 +150,8 @@
|
|
|
150
150
|
"timeoutMs": 5610000,
|
|
151
151
|
"intervalMs": 2000
|
|
152
152
|
},
|
|
153
|
-
"timing": { "execution": "sequential", "cleanupReservePerScenarioMs": 5000, "approvalReconciliationJitterMs": 1000, "approvalTerminalEditMaxMs": 30000, "noneMs": 35000, "externalGateMs": 305000, "cronLivenessMs": 1025000, "noCallbackMs": 365000, "
|
|
153
|
+
"timing": { "execution": "sequential", "cleanupReservePerScenarioMs": 5000, "approvalReconciliationJitterMs": 1000, "approvalTerminalEditMaxMs": 30000, "noneMs": 35000, "externalGateMs": 305000, "cronLivenessMs": 1025000, "noCallbackMs": 365000, "acceptanceDeliveryProbeMs": 1025000, "delayedApprovalMs": 185000, "timeoutStaleMs": 875000, "timeoutStaleBreakdown": { "promptCreationAdapterMs": 210000, "approvalTtlMs": 300000, "staleCallbackInjectionMs": 120000, "pollIntervalReserveMs": 30000, "reconciliationPollAdapterMs": 210000, "cleanupReserveMs": 5000 }, "scenarioSumMs": 5585000, "totalMs": 5610000 },
|
|
154
|
+
"scenarioSemantics": { "unit-16d-2-unknown-admission": "A genuinely distinct private sender remains pending and quarantined after the fixed acknowledgement and owner card; no Friend, session, provider, tool, approval, restart, or agent turn may exist.", "unit-16h-acceptance-delivery-probe": "An acceptance-only delivery-path probe restores exact health state and cron bytes and must not enable or change any production digest schedule." }
|
|
154
155
|
},
|
|
155
156
|
"reboot-request": {
|
|
156
157
|
"fixed": {
|