instar 1.3.102 → 1.3.103

package/src/templates/scripts/imessage-reply.sh

@@ -42,9 +42,13 @@ fi
 
 PORT="${INSTAR_PORT:-4042}"
 
-AUTH_TOKEN=""
-if [ -f ".instar/config.json" ]; then
-  AUTH_TOKEN=$(python3 -c "import json; print(json.load(open('.instar/config.json')).get('authToken',''))" 2>/dev/null)
+# Auth: env first (SessionManager injects it per spawned session and it
+# survives secret-externalization), legacy plaintext-config fallback with
+# string-type guard so the { "secret": true } placeholder produced by
+# SecretMigrator cannot leak through as a bogus Bearer.
+AUTH_TOKEN="${INSTAR_AUTH_TOKEN:-}"
+if [ -z "$AUTH_TOKEN" ] && [ -f ".instar/config.json" ]; then
+  AUTH_TOKEN=$(python3 -c "import json; v=json.load(open('.instar/config.json')).get('authToken',''); print(v if isinstance(v, str) else '')" 2>/dev/null)
 fi
 
 # Escape message for JSON

package/src/templates/scripts/instar-watchdog.sh

@@ -324,7 +324,11 @@ escalate_via_peer() {
     node_bin=$(resolve_node || true)
     [ -z "$node_bin" ] && continue
     local peer_meta
-    peer_meta=$("$node_bin" -e "const c=JSON.parse(require('fs').readFileSync(process.argv[1],'utf8'));process.stdout.write((c.port||'')+'\t'+(c.authToken||''))" "$peer_dir/.instar/config.json" 2>/dev/null || true)
+    # The string-type guard on authToken protects against the { secret: true }
+    # placeholder that SecretMigrator writes after externalizing the auth token:
+    # `c.authToken || ''` would otherwise return the object itself (truthy)
+    # which serializes to "[object Object]" — never a valid Bearer.
+    peer_meta=$("$node_bin" -e "const c=JSON.parse(require('fs').readFileSync(process.argv[1],'utf8'));const t=typeof c.authToken==='string'?c.authToken:'';process.stdout.write((c.port||'')+'\t'+t)" "$peer_dir/.instar/config.json" 2>/dev/null || true)
     peer_port="${peer_meta%%	*}"
     peer_auth="${peer_meta##*	}"
     [ -z "$peer_port" ] && continue
@@ -434,11 +438,16 @@ probe_server_identity() {
   node_bin=$(resolve_node || true)
   [ -z "$node_bin" ] && return 0
 
-  # Read port + authToken in one node invocation. The authToken is required
-  # because /health gates the `project` field behind authentication — without
-  # it the probe can't verify identity and would silently fail-open.
+  # Read port + authToken in one node invocation. The authToken is needed
+  # because /health gates the `project` field behind authentication; without
+  # it the probe can still hit /health for a basic alive check but loses the
+  # identity disambiguation. INSTAR_AUTH_TOKEN env is preferred when set
+  # (preserves cross-version compatibility). The string-type guard rejects the
+  # { secret: true } placeholder SecretMigrator writes after externalizing the
+  # auth token — `String(c.authToken || '')` would otherwise yield
+  # "[object Object]" and the Bearer would be invalid.
   local meta
-  meta=$("$node_bin" -e "const c=JSON.parse(require('fs').readFileSync(process.argv[1],'utf8'));process.stdout.write(String(c.port||'')+'\t'+String(c.authToken||''))" "$config_file" 2>/dev/null || true)
+  meta=$("$node_bin" -e "const c=JSON.parse(require('fs').readFileSync(process.argv[1],'utf8'));const t=process.env.INSTAR_AUTH_TOKEN||(typeof c.authToken==='string'?c.authToken:'');process.stdout.write(String(c.port||'')+'\t'+t)" "$config_file" 2>/dev/null || true)
   port="${meta%%	*}"
   auth_token="${meta##*	}"
   # Lifeline-only agents have no port; treat as healthy from probe POV.

package/src/templates/scripts/secret-drop-retrieve.mjs

@@ -47,15 +47,23 @@ if (!token || !mode || mode === '--consume') {
   process.exit(2);
 }
 
-let authToken;
+// Auth-token resolution: INSTAR_AUTH_TOKEN env first (SessionManager and
+// JobScheduler inject it for every spawned context; survives the
+// secret-externalization refactor that moved authToken out of config.json into
+// the encrypted store), legacy plaintext-config fallback with a string-type
+// guard so the { "secret": true } placeholder produced by SecretMigrator
+// cannot leak through as a bogus Bearer.
+let authToken = process.env.INSTAR_AUTH_TOKEN || '';
 let port;
 try {
   const config = JSON.parse(fs.readFileSync('.instar/config.json', 'utf-8'));
-  authToken = config.authToken;
+  if (!authToken && typeof config.authToken === 'string') authToken = config.authToken;
   port = config.port;
 } catch (e) {
-  process.stderr.write('cannot read .instar/config.json: ' + e.message + '\n');
-  process.exit(2);
+  if (!authToken) {
+    process.stderr.write('cannot read .instar/config.json: ' + e.message + '\n');
+    process.exit(2);
+  }
 }
 
 // Port resolution: env > config. Mirrors the telegram-reply.sh precedence

package/src/templates/scripts/serendipity-capture.sh

@@ -294,15 +294,21 @@ if patch_file or artifact_type:
 # Key derivation: HMAC-SHA256(authToken, "serendipity-v1:" + sessionId)
 signing_key = os.environ.get("SERENDIPITY_SIGNING_KEY", "")
 if not signing_key:
-    # Derive from auth token + session
-    config_path = os.environ.get("CONFIG_FILE", "")
-    auth_token = ""
-    if config_path and os.path.exists(config_path):
-        try:
-            cfg = json.load(open(config_path))
-            auth_token = cfg.get("authToken", "")
-        except:
-            pass
+    # Derive from auth token + session. Token: INSTAR_AUTH_TOKEN env first
+    # (SessionManager injects it; survives secret-externalization), legacy
+    # plaintext-config fallback with a string-type guard so the
+    # { "secret": true } placeholder produced by SecretMigrator cannot leak.
+    auth_token = os.environ.get("INSTAR_AUTH_TOKEN", "")
+    if not auth_token:
+        config_path = os.environ.get("CONFIG_FILE", "")
+        if config_path and os.path.exists(config_path):
+            try:
+                cfg = json.load(open(config_path))
+                v = cfg.get("authToken", "")
+                if isinstance(v, str):
+                    auth_token = v
+            except:
+                pass
     session_id = finding["source"]["sessionId"]
     key_material = f"serendipity-v1:{session_id}"
     signing_key = hmac.new(

package/src/templates/scripts/slack-reply.sh

@@ -44,7 +44,14 @@ if [ -f ".instar/config.json" ]; then
     CONFIG_PORT=$(python3 -c "import json; print(json.load(open('.instar/config.json')).get('port',''))" 2>/dev/null)
     [ -n "$CONFIG_PORT" ] && PORT="$CONFIG_PORT"
   fi
-  AUTH=$(python3 -c "import json; print(json.load(open('.instar/config.json')).get('authToken',''))" 2>/dev/null)
+  # Auth: env first (SessionManager injects it into spawned sessions; survives
+  # secret-externalization), legacy plaintext-config fallback with string-type
+  # guard so the { "secret": true } placeholder produced by SecretMigrator
+  # cannot leak as a bogus Bearer.
+  AUTH="${INSTAR_AUTH_TOKEN:-}"
+  if [ -z "$AUTH" ]; then
+    AUTH=$(python3 -c "import json; v=json.load(open('.instar/config.json')).get('authToken',''); print(v if isinstance(v, str) else '')" 2>/dev/null)
+  fi
 fi
 
 PORT="${PORT:-4042}"

package/src/templates/scripts/telegram-reply.sh

@@ -75,9 +75,13 @@ if [ -z "$MSG" ]; then
 fi
 
 # Resolve config-derived values from .instar/config.json (single python3
-# invocation). Env > config > 4040-warn for port; config-only for authToken
-# and agentId.
-AUTH_TOKEN=""
+# invocation). Env > config > 4040-warn for port. Auth: INSTAR_AUTH_TOKEN env
+# first (SessionManager injects it per spawned session; survives the
+# secret-externalization refactor that moved authToken out of config.json into
+# the encrypted store), legacy plaintext-config fallback with a string-type
+# guard so the { "secret": true } placeholder produced by SecretMigrator
+# cannot leak through as a bogus Bearer.
+AUTH_TOKEN="${INSTAR_AUTH_TOKEN:-}"
 AGENT_ID=""
 CONFIG_PORT=""
 if [ -f ".instar/config.json" ]; then
@@ -87,11 +91,13 @@ try:
     c = json.load(open('.instar/config.json'))
 except Exception:
     sys.exit(0)
-print(c.get('authToken', ''))
+v = c.get('authToken', '')
+print(v if isinstance(v, str) else '')
 print(c.get('projectName', ''))
 print(c.get('port', ''))
 " 2>/dev/null)
-  AUTH_TOKEN=$(printf '%s\n' "$CONFIG_VALUES" | sed -n '1p')
+  CONFIG_AUTH=$(printf '%s\n' "$CONFIG_VALUES" | sed -n '1p')
+  [ -z "$AUTH_TOKEN" ] && AUTH_TOKEN="$CONFIG_AUTH"
   AGENT_ID=$(printf '%s\n' "$CONFIG_VALUES" | sed -n '2p')
   CONFIG_PORT=$(printf '%s\n' "$CONFIG_VALUES" | sed -n '3p')
 fi

package/src/templates/scripts/whatsapp-reply.sh

@@ -33,10 +33,13 @@ fi
 
 PORT="${INSTAR_PORT:-4040}"
 
-# Read auth token from config (if present)
-AUTH_TOKEN=""
-if [ -f ".instar/config.json" ]; then
-  AUTH_TOKEN=$(python3 -c "import json; print(json.load(open('.instar/config.json')).get('authToken',''))" 2>/dev/null)
+# Read auth token: env first (SessionManager injects it per spawned session and
+# it survives secret-externalization), legacy plaintext-config fallback with
+# string-type guard so the { "secret": true } placeholder produced by
+# SecretMigrator cannot leak through as a bogus Bearer.
+AUTH_TOKEN="${INSTAR_AUTH_TOKEN:-}"
+if [ -z "$AUTH_TOKEN" ] && [ -f ".instar/config.json" ]; then
+  AUTH_TOKEN=$(python3 -c "import json; v=json.load(open('.instar/config.json')).get('authToken',''); print(v if isinstance(v, str) else '')" 2>/dev/null)
 fi
 
 # Escape for JSON

package/upgrades/1.3.103.md

@@ -0,0 +1,77 @@
+---
+review-convergence: complete
+approved: true
+approved-by: justin (verbal, topic 2169: "yes! then we need to do a full, robust audit to fix this whole class of issues since it keeps coming up")
+---
+
+# Upgrade Guide — vNEXT
+
+<!-- bump: patch -->
+
+## What Changed
+
+**Fleet-wide fix for the silent-403 class.** Every shipped hook, script, Node
+helper, and migrator-emitted template that resolves the agent's `authToken`
+now survives the secret-externalization refactor. Two related bugs landed
+together because both produced the same user-visible symptom (a hook that
+stops emitting any output):
+
+- **Auth-token resolution: env-first with a string-type guard.**
+  `INSTAR_AUTH_TOKEN` (set by `SessionManager` per spawned session and by
+  `JobScheduler` per scheduled job) is checked first. The disk fallback now
+  guards `cfg.authToken` against non-string values, so the literal
+  `{ "secret": true }` placeholder produced by `SecretMigrator` after
+  multi-machine pairing can never leak through as a Bearer token. (Previously,
+  scripts sent `[object Object]` or `{'secret': True}` as the Authorization
+  header, the server 403'd, and the script's downstream Python parser
+  silently exited on the error JSON — no output at all.)
+
+- **Port-parse tolerates whitespace.** The `grep -o '"port":[0-9]*'` pattern
+  used in every hook required no whitespace between the colon and the
+  number, but our prettified `config.json` writes `"port": 4042` (with
+  space). The pattern silently produced empty `$PORT`, the hook exited
+  early, and not a single HTTP request reached the server. Replaced with
+  `grep -oE '"port"[[:space:]]*:[[:space:]]*[0-9]+'` + digit-only extraction.
+
+The structural cure includes a unit-tier lint that fails any future re-
+introduction of the broken pattern (positive + negative-verified) and a new
+migration pass (`migrateSecretExternalizationSurvivability`) that upgrades
+deployed auxiliary scripts (`imessage-reply.sh`, `serendipity-capture.sh`,
+`slack-channel-context.sh`) without touching custom forks. The canonical
+always-overwrite hooks (`session-start.sh`, `compaction-recovery.sh`,
+`telegram-topic-context.sh`) heal on next auto-update by the existing
+`migrateHooks` path.
+
+## What to Tell Your User
+
+If your agent recently went silent after compaction — emitting only the
+wall-clock-time block and then nothing else, despite a healthy server — this
+is the fix. Topic-history injection is back. No configuration needed.
+
+After auto-update, send yourself a Telegram message in any forum topic.
+The agent's response should reference what you actually said — not greet
+you like it's the first message. If it doesn't, something else is wrong;
+please flag.
+
+## Summary of New Capabilities
+
+| Capability | How to Use |
+|-----------|-----------|
+| Auth-token resolution survives secret-externalization | Automatic. Every shipped script + hook now reads `INSTAR_AUTH_TOKEN` env first, then falls back to `config.json` with a string-type guard. |
+| Port-parse tolerates JSON whitespace | Automatic. Hooks no longer exit-early when `config.json` is prettified. |
+| Lint blocks future regressions | `tests/unit/secret-externalization-hook-resolver-lint.test.ts` greps every shell + Node read for the broken pattern and fails on any reintroduction. |
+| Auxiliary-script migration for deployed agents | Existing agents with the old `imessage-reply.sh` / `serendipity-capture.sh` / `slack-channel-context.sh` get the fix on next auto-update via `migrateSecretExternalizationSurvivability`. Custom forks (no shipped marker) are untouched. |
+
+## Evidence
+
+- 13 new tests across 3 tiers (unit + integration-grade + e2e). All green on a
+  full `vitest` run.
+- 35 PostUpdateMigrator-* unit test files (232 tests) verified clean (no
+  regression from migrator extensions).
+- Manual smoke test against a real externalized agent: the fixed
+  `getTelegramTopicContextHook()` content correctly fetches topic history
+  with INSTAR_AUTH_TOKEN env alone (config.json holds the placeholder), and
+  correctly degrades to the unauth branch without leaking the placeholder
+  when env is missing.
+- Side-effects review:
+  `upgrades/side-effects/secret-externalization-hook-resolver-audit.md`.

package/upgrades/NEXT.eli16.md

@@ -0,0 +1,71 @@
+# vNEXT — plain English overview
+
+## What this change is
+
+Your agent has a bunch of helper scripts that need to talk to its own local
+server. They authenticate using an "auth token" — basically a password the
+server expects in every request.
+
+Recently we added a feature that **moves that password out of the plain
+config file into an encrypted store**, so the password no longer sits in
+plain text on disk. That's a good security improvement.
+
+But when we did that, we forgot to update *every* place the helpers read
+the password from. So a bunch of helpers kept reading the plain config
+file, found the placeholder marker (a tiny note that says "the real
+password lives in the encrypted store now"), and used **that marker** as
+the password. The server rejected every request. The helpers didn't
+notice the rejection — they just quietly emitted nothing.
+
+The user-visible result: your agent comes back from a memory compaction
+and has no idea what conversation you were in. So it greets you like a
+stranger.
+
+## What already exists
+
+- The `INSTAR_AUTH_TOKEN` environment variable. Every time the agent
+  spawns a new session, it sets this env var with the real password.
+- The encrypted secret store at `.instar/secrets/config.secrets.enc`,
+  managed by `SecretMigrator` / `SecretStore`.
+- A migration system (`PostUpdateMigrator`) that rewrites helper scripts
+  on every auto-update. Some helpers were already on this track; others
+  weren't.
+
+## What's new
+
+- **Every helper now reads `INSTAR_AUTH_TOKEN` env first.** The env var
+  is always present in a session, so it's the canonical path.
+- **The disk-fallback now has a "is this actually a password?" check.**
+  If the config file says `{ "secret": true }` (the placeholder), the
+  guard rejects it and falls through to no-auth instead of sending the
+  placeholder as a password.
+- **A second, unrelated bug** also got fixed in the same change: the
+  helpers parsed the server port from the config file with a regex that
+  refused to allow whitespace, so `"port": 4042` (the format we
+  actually ship) was unparseable. The helpers exited silently with no
+  port, hitting nothing. Replaced with a whitespace-tolerant pattern.
+- **A new automated check** scans every helper for this broken pattern
+  and refuses to commit any future change that reintroduces it.
+- **An update routine** that fixes the helpers on existing deployed
+  agents (without overwriting any custom scripts the user wrote
+  themselves).
+
+## What you need to decide
+
+Nothing. This is a bug fix, no configuration involved. Auto-update will
+heal every deployed agent on next version bump.
+
+## How to verify it worked after deploy
+
+After your agent updates past the version that includes this fix, send
+yourself a Telegram message in any forum topic. The agent's response
+should reference what you actually said — not greet you like it's the
+first message. If it doesn't, something else is wrong; please flag.
+
+## Why this matters more than it might look
+
+This is a "silent failure" class — the helpers were broken but emitted
+no error, so the bug only surfaced through the symptom (incoherent
+agent replies after compaction). That makes the failure invisible until
+a user notices the symptom. The structural lint added in this change
+makes the class itself impossible to reintroduce silently.

package/upgrades/side-effects/secret-externalization-hook-resolver-audit.md

@@ -0,0 +1,48 @@
+# Side-effects review — secret-externalization hook resolver audit
+
+## What changed
+
+Two related fleet-wide classes are fixed in one PR.
+
+1. **Auth-token reads survive secret-externalization.** Every shipped shell hook, shell script, Node helper, and migrator-emitted template string that resolves the instar server's `authToken` now:
+   - Prefers `INSTAR_AUTH_TOKEN` env (set by `SessionManager` per spawned session and by `JobScheduler` per scheduled job — always available inside the Claude Code session).
+   - Falls back to reading `config.json` with a string-type guard. When `SecretMigrator` has externalized the token, on-disk `authToken` is the literal placeholder `{ "secret": true }`; the guard rejects it and yields empty so the placeholder cannot leak as a Bearer.
+2. **Port-parse regex tolerates whitespace.** `grep -o '"port":[0-9]*'` is replaced by `grep -oE '"port"[[:space:]]*:[[:space:]]*[0-9]+'` + digit-only extraction. The previous form silently produced empty PORT whenever `config.json` had `"port": 4042` (with space) — and our prettified configs all do — making every history-injection hook exit early on a healthy install. Both bugs together produced the 2026-05-29 silent-history failure.
+
+## Why
+
+The 2026-05-29 incident: Justin's `telegram-topic-context.sh` stopped injecting topic history after the agent compacted. The hook silently 403'd because `authToken` had been moved to the encrypted secret store and the hook was still reading the literal `{ secret: true }` placeholder out of `config.json` as a Bearer token. The agent came back with no topic context, sent an incoherent reply, and Justin had to manually trace the root cause back. This is the SECOND time a script in this class has bitten a deployed agent (the 2026-05-28 `telegram-reply.sh` 403 was the first), so the fix is structural: env-first everywhere + string-type guard + lint that catches the broken pattern + migrator that upgrades stale-on-disk copies.
+
+## Risk surface
+
+- **Hook scripts (always-overwritten):** `session-start.sh`, `compaction-recovery.sh`, `telegram-topic-context.sh`. Existing agents get the fix on first auto-update. The canonical content lives inside `PostUpdateMigrator.getHookContent()`; the parallel `src/templates/hooks/*.sh` files are kept in sync (the manifest tracks their SHA256). Risk: low — these are already overwritten on every migration tick.
+- **Reply scripts (SHA-based migration):** `telegram-reply.sh` upgrades via `migrateReplyScriptToPortConfig` (pre-PR SHA `371d7e8f…` is already in the prior-shipped allowlist; agents get the fix on first auto-update). `slack-reply.sh` and `whatsapp-reply.sh` upgrade via `migrateReplyScriptTo408`, extended to also fire when `INSTAR_AUTH_TOKEN` is missing — previously the function skipped any script with HTTP 408 handling, which would have stranded a 408-aware-but-auth-broken copy forever. Risk: low — the marker-string check still protects custom forks.
+- **Auxiliary scripts (new migration):** `imessage-reply.sh`, `serendipity-capture.sh`, `slack-channel-context.sh` get a new `migrateSecretExternalizationSurvivability` pass that upgrades shipped copies and skips custom forks (shipped-marker check). Risk: low — guarded by both the marker check and the negative `INSTAR_AUTH_TOKEN` check.
+- **CLI/in-process callers:** `gate.ts` and `nuke.ts` did raw `JSON.parse(fs.readFileSync('config.json'))` then `cfg.authToken` directly. Patched to env-first + string-type-guard. The other CLI commands go through `loadConfig()` (which calls `mergeConfigWithSecrets`) and are already safe. Risk: low — the change is the same env-first/string-guard pattern.
+- **Daemons:** `listener-daemon.ts` (HMAC-derives a signing key from `authToken`), `mcp-stdio-entry.ts` (uses `authToken` as the inbound Bearer). Both now env-first + string-type-guard. Risk: low — the env-first path is preferred when the daemon is launched by the spawn pipeline (which exports the var); the disk fallback still works for plaintext-config installs.
+- **CI / lint:** new unit test `secret-externalization-hook-resolver-lint.test.ts` greps every shell + Node read of `authToken`. The lint MUST go green on the fixed tree (verified) and MUST fail on any re-introduction of the broken pattern (verified via destructive negative test). Risk: low — the lint is allowlisted for in-process `loadConfig` callers; future authors won't trip on the safe path.
+
+## Bug surfaces eliminated
+
+- Topic-history injection silently 403s after secret externalization. (Direct cause of the 2026-05-29 incident.)
+- `telegram-reply.sh` 403s when invoked from a session whose env is missing — partially mitigated in the 2026-05-28 fix but the audit confirmed the disk-fallback path was still broken; this PR closes both halves.
+- `session-start.sh` could not fetch working-memory or shared-state after externalization (same root cause as the topic-history bug).
+- `compaction-recovery.sh` lost its post-compaction memory query after externalization.
+- A standby-machine daemon whose key derivation went through `authToken` would silently derive a key from `{ secret: true }` and lose peer connectivity. (Speculative — no incident observed, but the path was broken.)
+
+## Migration footprint
+
+- `migrateHooks()` already always-overwrites the three canonical hooks → existing agents heal automatically on next auto-update.
+- `migrateScripts()` already SHA-migrates `telegram-reply.sh` and now (with the extended INSTAR_AUTH_TOKEN check) auto-migrates `slack-reply.sh` + `whatsapp-reply.sh` when the previously-shipped marker is present.
+- `migrateSecretExternalizationSurvivability()` (NEW) auto-migrates `imessage-reply.sh`, `serendipity-capture.sh`, `slack-channel-context.sh`. Idempotent on the post-fix shape.
+
+## Testing
+
+- Unit (lint): `tests/unit/secret-externalization-hook-resolver-lint.test.ts` — 3 tests. Greps `src/templates/`, `src/`, `scripts/` for the broken pattern; verified negative (broken pattern caught) and positive (clean pattern passes).
+- Unit (migrator): `tests/unit/secret-externalization-survivability-migrator.test.ts` — 5 tests. Seeds broken state, runs `migrate()`, asserts upgrade. Custom-fork test verifies the marker check protects user-modified scripts.
+- Integration-grade (in `tests/unit/`): `tests/unit/secret-externalization-hook-injection.test.ts` — 2 tests. Spins an in-process stub server (bound to all interfaces; ASYNC `spawn` is required because `execFileSync` blocks Node's event loop and freezes the stub). Seeds `config.json` with the placeholder, runs the canonical migrator-emitted hook, asserts injection works with env-first auth and never leaks the placeholder as a Bearer.
+- E2E: `tests/e2e/secret-externalization-hook-lifecycle.test.ts` — 3 tests. Simulates pre-fix agent → `migrate()` → asserts every shipped script carries the env-first canary; verifies idempotency on a second pass; verifies the canonical migrator-emitted hooks always carry the canary.
+
+## Test count
+
+13 new tests across 3 tiers (per Testing Integrity Standard). 29 existing related tests (`PostUpdateMigrator-time-injection.test.ts`, `compaction-telegram-context.test.ts`) remain green.