instar 1.3.309 → 1.3.311

package/src/templates/scripts/secret-get.mjs

@@ -0,0 +1,143 @@
+#!/usr/bin/env node
+/**
+ * secret-get.mjs — read ONE secret value from the agent's encrypted SecretStore
+ * vault and stream it to stdout for piping. The retrieval affordance promised by
+ * the Session Boot Self-Knowledge block (docs/specs/session-boot-self-knowledge.md):
+ * a session that boots knowing `github_token` is in the vault uses THIS script to
+ * fetch it — never `node -e` ad-hoc reads, never asking the user to re-send it.
+ *
+ * Containment contract (sibling of secret-drop-retrieve.mjs, same rules):
+ *   - The VALUE goes to stdout ONLY — single write, no trailing newline — so it
+ *     can pipe straight into the consuming command without ever being echoed
+ *     into a terminal, chat, or transcript.
+ *   - ALL diagnostics go to stderr and are limited to key NAMES, lengths, and
+ *     error categories — never values.
+ *   - On ANY error, stdout receives ZERO value bytes (stderr-only, non-zero exit).
+ *
+ * Usage:
+ *   node .instar/scripts/secret-get.mjs <keyPath>
+ *     → prints the value to stdout. Pipe it: `... github_token | gh auth login --with-token`
+ *
+ *   node .instar/scripts/secret-get.mjs --names
+ *     → prints vault key paths + value lengths to stderr; nothing to stdout.
+ *
+ *   node .instar/scripts/secret-get.mjs <keyPath> --run -- <cmd...>
+ *     → runs <cmd> with the value piped to its stdin (atomic handoff — the value
+ *     never touches a shell variable or the transcript). Exits with <cmd>'s code.
+ *
+ * Vault access:
+ *   Loads the instar SecretStore implementation from the local install and reads
+ *   the vault at `.instar/secrets/config.secrets.enc` relative to cwd (run from
+ *   the agent home). Keychain-backed master key by default — the same read path
+ *   the server uses.
+ *
+ * Exit codes:
+ *   0 — value printed (or --run command succeeded, or --names listed)
+ *   1 — key not found, vault absent/undecryptable, or --run command failed
+ *   2 — usage error (missing args, cannot resolve the instar dist)
+ */
+
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import { spawnSync } from 'node:child_process';
+import { createRequire } from 'node:module';
+
+const args = process.argv.slice(2);
+
+// `<keyPath> --run -- <cmd...>`: everything after the first `--` (which must
+// follow `--run`) is the command to receive the value on stdin.
+const runIdx = args.indexOf('--run');
+const dashIdx = args.indexOf('--');
+let runCmd = null;
+if (runIdx !== -1) {
+  if (dashIdx === -1 || dashIdx < runIdx || dashIdx === args.length - 1) {
+    process.stderr.write('usage: secret-get.mjs <keyPath> --run -- <cmd...>\n');
+    process.exit(2);
+  }
+  runCmd = args.slice(dashIdx + 1);
+}
+const positional = (dashIdx === -1 ? args : args.slice(0, dashIdx)).filter((a) => !a.startsWith('--'));
+const namesOnly = args.includes('--names');
+const keyPath = positional[0];
+
+if (!namesOnly && !keyPath) {
+  process.stderr.write('usage: secret-get.mjs <keyPath> | secret-get.mjs --names\n');
+  process.exit(2);
+}
+
+// Resolve the instar dist: deployed agents run from the shadow-install; the dev
+// checkout falls back to its own dist. Never print resolution paths on success.
+const require = createRequire(import.meta.url);
+const candidates = [
+  path.resolve('.instar/shadow-install/node_modules/instar/dist/core/SecretStore.js'),
+  path.resolve('dist/core/SecretStore.js'),
+];
+let SecretStore = null;
+for (const c of candidates) {
+  if (fs.existsSync(c)) {
+    try {
+      ({ SecretStore } = require(c));
+      break;
+    } catch {
+      // try the next candidate; details are not value-bearing but stay off stdout
+    }
+  }
+}
+if (!SecretStore) {
+  process.stderr.write('secret-get: cannot resolve the instar SecretStore module (run from the agent home)\n');
+  process.exit(2);
+}
+
+const stateDir = path.resolve('.instar');
+if (!fs.existsSync(path.join(stateDir, 'secrets', 'config.secrets.enc'))) {
+  process.stderr.write('secret-get: no vault on this machine (.instar/secrets/config.secrets.enc absent)\n');
+  process.exit(1);
+}
+
+let secrets;
+try {
+  secrets = new SecretStore({ stateDir }).read();
+} catch {
+  process.stderr.write(
+    'secret-get: vault exists but could not be decrypted (master-key mismatch?). ' +
+      'Do NOT repair/rotate/delete — surface to the operator.\n',
+  );
+  process.exit(1);
+}
+
+// Flatten to dot-notation leaves — names + lengths only, mirroring the vault's
+// own get/set addressing. Values never leave this function except via stdout.
+function leaves(obj, prefix = '') {
+  const out = [];
+  for (const [k, v] of Object.entries(obj)) {
+    const p = prefix ? `${prefix}.${k}` : k;
+    if (v && typeof v === 'object' && !Array.isArray(v)) out.push(...leaves(v, p));
+    else out.push([p, v]);
+  }
+  return out;
+}
+const entries = leaves(secrets);
+
+if (namesOnly) {
+  for (const [name, value] of entries) {
+    process.stderr.write(`${name} (${String(value ?? '').length} chars)\n`);
+  }
+  if (entries.length === 0) process.stderr.write('(vault is empty)\n');
+  process.exit(0);
+}
+
+const hit = entries.find(([name]) => name === keyPath);
+if (!hit) {
+  process.stderr.write(`secret-get: no key "${keyPath}" in the vault. Known keys:\n`);
+  for (const [name] of entries) process.stderr.write(`  ${name}\n`);
+  process.exit(1);
+}
+const value = typeof hit[1] === 'string' ? hit[1] : JSON.stringify(hit[1]);
+
+if (runCmd) {
+  const r = spawnSync(runCmd[0], runCmd.slice(1), { input: value, stdio: ['pipe', 'inherit', 'inherit'] });
+  process.exit(r.status ?? 1);
+}
+
+process.stdout.write(value);
+process.exit(0);

package/upgrades/1.3.310.md

@@ -0,0 +1,20 @@
+# Upgrade Guide — vNEXT
+
+<!-- assembled-by: assemble-next-md -->
+<!-- bump: patch -->
+
+## What Changed
+
+DeliveryFailureSentinel.start() performs an immediate awaited first drain (startup tick) before relying on the 5-minute watchdog interval. Watchdog cadence, event-driven ticks, per-topic pacing, backoff, and purge rules unchanged.
+
+## What to Tell Your User
+
+Messages that queued up while your agent's server was restarting now deliver the moment it comes back, instead of waiting up to five extra minutes — which under back-to-back update restarts could mean never, as the next restart landed first.
+
+## Summary of New Capabilities
+
+- A started delivery sentinel means a drained backlog: recovery no longer depends on the watchdog timer surviving the up-window.
+
+## Evidence
+
+Live (2026-06-05 ~15:40-16:50Z, echo): a release-train restart cascade produced up-windows shorter than boot-time + the 5-min first tick; five queued operator messages survived FOUR up-windows undelivered while fresh sends in the same windows succeeded. Pinned by a new integration test (pre-existing http_code-0 backlog delivered by start() alone, no explicit tick) plus the re-pinned recovery happy path (start drains; subsequent tick finds zero) — 71/71 across the sentinel suites.

package/upgrades/1.3.311.md

@@ -0,0 +1,20 @@
+# Upgrade Guide — vNEXT
+
+<!-- assembled-by: assemble-next-md -->
+<!-- bump: patch -->
+
+## What Changed
+
+New `src/core/BootSelfKnowledge.ts` (block builder: sanitization, depth-2 collapse, alphabetical + capped + byte-bounded rendering, mtime+size-keyed module cache); three routes in `routes.ts`; one fetch block in `getSessionStartHook()`; `selfKnowledge` config surface (defaults backfilled by `migrateConfig`, `enabled` left unset for the developmentAgent gate); CLAUDE.md template section + `migrateClaudeMd` parity; `secret-get.mjs` shipped via `migrateScripts` + init. Spec: `docs/specs/session-boot-self-knowledge.md` (converged, 3 iterations, cross-model codex-cli:gpt-5.5).
+
+## What to Tell Your User
+
+Nothing user-visible yet — the feature ships dark on the fleet (live on the development agent for the bake). When the fleet flip lands it gets its own note: "your agent now remembers what credentials it holds across sessions — it won't ask you to re-send a key it already has."
+
+## Summary of New Capabilities
+
+- `GET /self-knowledge/session-context` (Bearer; `enabled ?? !!developmentAgent`): a bounded, sanitized `<session-self-knowledge>` block with the agent's vault secret NAMES (never values; same `secretKeyPaths()` derivation as `/secrets/sync-status`, depth-capped) + self-asserted operational facts. `?full=1` bypasses the display caps. A vault that exists but won't decrypt is reported honestly as DECRYPT-FAILED with hands-off guidance — never as an empty vault.
+- `POST/DELETE /self-knowledge/facts`: agent-driven writer for durable per-machine operational facts (auto-stamped `{fact, updatedAt, machine}`; duplicate/cap/ambiguity guarded; atomic temp+rename config write).
+- `.instar/scripts/secret-get.mjs` (always-overwrite installed): hardened vault retrieval — value streams to stdout for piping straight into the consuming command, names/diagnostics to stderr, zero value bytes on any error path.
+- The session-start hook injects the block at every boot (fail-open: dark/unreachable/version-skew → silent skip), placed after the org-intent and preferences blocks.
+- Structural test guard: `MasterKeyManager` forces the file key under vitest — no test can ever read or overwrite the machine-global OS keychain master key again (the 2026-06-05 bifurcated-key incident class is closed).

package/upgrades/sentinel-immediate-first-tick.eli16.md

@@ -0,0 +1,7 @@
+# Queued messages now deliver the moment the server comes back — not five minutes later
+
+The delivery recovery engine drains the queue of undelivered outgoing messages on a five-minute timer. The timer's FIRST firing is five minutes after the engine starts — and the engine restarts with the server. That was fine when restarts were rare. Today the fleet shipped ten-plus releases, the server restarted roughly every ten minutes, and each up-window between restarts was shorter than boot time plus five minutes. The result, observed live for over an hour: five queued messages to the operator — including the run summary he had just asked for — survived four separate up-windows undelivered, while freshly-sent messages in those same windows went through fine. The queue protected the messages perfectly (that part of the week's fixes worked) and then never got a chance to deliver them.
+
+The fix is one step in the engine's startup: drain the queue immediately when the engine starts, before the timer ever matters. A booted engine now means a drained backlog — the user's queued messages go out the moment the server is reachable, instead of betting on the up-window outlasting a timer. The startup drain is awaited, so "started" deterministically means "backlog handled once," which also makes the behavior directly testable.
+
+Nothing else changes: the five-minute watchdog, the event-driven kicks, per-topic pacing, retry backoff, and the loud-purge rules all behave exactly as before. Three integration tests pin the new contract — a pre-existing backlog of connection-refused messages (the cascade signature) is delivered by startup alone, with the existing recovery and mismatch paths re-pinned around it.

package/upgrades/side-effects/sentinel-immediate-first-tick.md

@@ -0,0 +1,38 @@
+# Side-Effects Review — DeliveryFailureSentinel startup tick
+
+**Version / slug:** `sentinel-immediate-first-tick`
+**Date:** `2026-06-05`
+**Author:** `instar-echo`
+
+## Summary
+
+start() runs one awaited tick() after arming the watchdog (and after the existing one-shot restore-purge and event-listener registration). The tick is the same code path the watchdog and delivery_failed events already invoke; its own try/catch contains failures (logged, start() still completes).
+
+## Decision-point inventory
+
+- start() — modified — awaited first drain; 'sentinel:started' now emitted AFTER the initial drain completes.
+- tick(), watchdog cadence, event kicks, per-topic rate limit, backoff, stampede digest, purge — untouched.
+
+## Direction of failure
+
+- Old: recovery waited ≥5 min after every boot; under restart cascades (up-window < boot + 5 min) the queue never drained — live 70+ min user-facing outage of QUEUED (safe but stuck) messages.
+- New: backlog drains at boot. A failing startup tick logs and start() completes — degraded to exactly the old behavior (watchdog picks it up).
+- Conservative direction: messages deliver sooner; nothing new is dropped or retried beyond existing budgets.
+
+## Side-effects checklist
+
+1. **Boot latency:** start() now includes one drain pass. Per-delivery I/O is already bounded (postReply timeouts); a large backlog is paced by the existing per-topic rate limit — residual items wait for the watchdog exactly as before. Boot-blocking risk bounded by the same budgets every tick obeys.
+2. **Ordering with restore-purge:** unchanged — purge (one-shot) still precedes the first drain, so the startup tick never delivers rows the purge policy retires.
+3. **Double-tick race:** the in-flight/lease machinery already serializes concurrent ticks (watchdog + event kicks coexist today); the startup tick is just one more entrant.
+4. **Test contract change:** "start() then explicit tick()" tests now see the backlog consumed by start — the recovery happy path was re-pinned to assert the NEW contract explicitly (start drains; next tick returns 0).
+5. **External surfaces:** none — no routes/config/schema.
+6. **Rollback cost:** revert the commit.
+
+## Scope not taken
+
+- No watchdog-interval retune (5 min stays right for steady state).
+- No restart-cascade dampener changes (separate system; its own review queued from today's observations).
+
+## Rollback
+
+Revert. Recovery returns to first-drain-at-+5min.

package/upgrades/side-effects/session-boot-self-knowledge.md

@@ -0,0 +1,78 @@
+# Side-Effects Review — Session Boot Self-Knowledge
+
+**Version / slug:** `session-boot-self-knowledge`
+**Date:** `2026-06-05`
+**Author:** `echo`
+**Second-pass reviewer:** `spec-converge multi-reviewer panel (3 rounds: security/adversarial/integration/scalability/lessons-aware internal + Standards-Conformance Gate + codex-cli:gpt-5.5 external each round)`
+
+## Summary of the change
+
+Adds the boot self-knowledge block: a bounded, sanitized `<session-self-knowledge>` context block (vault secret NAMES — never values — plus self-asserted operational facts) built server-side by the new `src/core/BootSelfKnowledge.ts`, served by `GET /self-knowledge/session-context` (with `?full=1`), written by `POST/DELETE /self-knowledge/facts`, injected by a new fetch block in `getSessionStartHook()`, retrieved-from by the new hardened `secret-get.mjs` script, and configured by the new `InstarConfig.selfKnowledge` surface (defaults via `ConfigDefaults` + `migrateConfig`; CLAUDE.md template + `migrateClaudeMd`; script via `migrateScripts` + init). Includes the structural `MasterKeyManager` VITEST constructor guard. Files: `src/core/BootSelfKnowledge.ts` (new), `src/core/SecretStore.ts`, `src/server/routes.ts`, `src/core/PostUpdateMigrator.ts`, `src/core/types.ts`, `src/config/ConfigDefaults.ts`, `src/scaffold/templates.ts`, `src/commands/init.ts`, `src/templates/scripts/secret-get.mjs` (new), plus 4 test files, the spec + ELI16 + convergence report, and the release fragment. Spec: `docs/specs/session-boot-self-knowledge.md` (converged + approved).
+
+## Decision-point inventory
+
+This change adds NO decision point with blocking authority — it is a pure signal producer (read-only context injection; per `docs/signal-vs-authority.md`). Conditionals it adds are availability switches and writer-input validation, not behavior gates:
+
+- `GET /self-knowledge/session-context` enabled-resolution (`enabled ?? !!developmentAgent`) — add — availability switch (graduated rollout), not a behavior gate.
+- Facts writer validation (400 empty/oversize; 409 duplicate/cap/ambiguous/expect-mismatch) — add — input validation on an agent-driven write surface.
+- Pass-through: SecretStore read path (read-only), session-start hook (additive fetch block, fail-open), config write path (new atomic helper for one array).
+
+---
+
+## 1. Over-block
+
+The enabled-resolution 503s the route on fleet agents (flag unset, `developmentAgent` false) — by design (graduated rollout), not an over-block: the hook fail-opens silently. The facts writer rejects: empty/oversize facts (legitimate long facts >256 chars must be split — accepted cost, keeps the boot block bounded), exact duplicates, adds past the 50-fact cap, ambiguous `match` deletes, and stale `index+expect` deletes. Each 4xx carries an actionable message. No legitimate session-context READ is ever rejected beyond auth + the availability switch. No other block/allow surface.
+
+## 2. Under-block
+
+- A fact that is misleading-but-validly-shaped (≤256 chars, unique) is stored and injected every boot — mitigated by the self-asserted/unverified labeling, per-index render for one-call removal, and the per-serve audit line; residual risk accepted for v1 and explicitly watched during the bake (spec §Threat model).
+- The last-writer-wins window between the facts writer and the pre-existing NON-atomic config writers (PATCH /config, telemetry) remains — bounded to the handler's microseconds by re-read-before-write, pinned by the interleaving migration test; accepted and documented.
+- Names already written into past transcripts are not retroactively scrubbed if the feature is later disabled.
+
+## 3. Level-of-abstraction fit
+
+Right layer. The names derivation stays in `secretKeyPaths()` (shared with `/secrets/sync-status` — no logic fork); the block-building/presentation is a new module rather than overloading sync-status (which 503s when secret-sync is dark — wrong availability semantics for a boot surface) or the SelfKnowledgeTree (LLM search over AGENT.md — different system, noted in code comments on both). The hook injection rides the existing org-intent/preferences pattern rather than inventing a new injection mechanism. Rejected alternatives (pull surfaces: /capabilities, MCP resources, memory files) are analyzed in the spec — the failure class is "agent doesn't know to look," which only push-at-boot removes.
+
+## 4. Signal vs authority compliance
+
+Compliant — pure signal producer. The block is wrapped in an envelope that explicitly subordinates it to org-intent constraints, safety rules, and user instructions. The guidance line is signal-shaped ("retrieve rather than re-ask, unless you have evidence it is invalid") not absolute. A deterministic block on credential re-asks was considered and rejected as brittle-authority (spec §Why guidance stays a signal); the designed escalation if the bake shows non-compliance is a smart-gate signal feed, not a regex block. The VITEST keychain guard is a test-environment safety rail, not a runtime decision point.
+
+## 5. Interactions
+
+- Coexists with the org-intent and preferences injections: placed after both in the hook (authoritative contract first); envelope states precedence. No shadowing — different routes, different envelopes.
+- `/self-knowledge/*` namespace shared with the SelfKnowledgeTree routes (search/validate/health) — no path collision; comments mark which system serves which path.
+- The names cache keys on the vault file path + (mtimeMs,size); secret-sync writes go through the same in-process server, so a peer-pushed secret invalidates the cache on its atomic write. No double-fire: the route is the only consumer.
+- `migrateConfig`'s recursive add-missing merge interacts with operator-set `selfKnowledge` values — partial-override case pinned by test.
+- The VITEST guard interacts with every existing test that constructs SecretStores — it can only make them SAFER (file-key instead of keychain); tests that explicitly pass `forceFileKey: true` (e.g. SecretMigrator's) are unchanged.
+
+## 6. External surfaces
+
+- Vault key NAMES become visible in: the Bearer-gated route response, the agent's session context, and therefore on-disk session transcripts (which can travel further than vaults — debug bundles, provider retention; spec §Threat model). This is the feature's one genuinely new exposure and the reason it ships dark-fleet with the live flip as an explicit follow-up decision.
+- No cross-agent or cross-machine surface changes: facts are per-machine (config doesn't sync); names reflect the local vault (which secret-sync may populate). Nothing here changes timing-sensitive behavior visible to other systems; the hook fetch is fail-open with `--max-time 4 --connect-timeout 1`.
+
+## 7. Rollback cost
+
+Low. Per-agent: `selfKnowledge.sessionContext.enabled: false` (route 503s, hook silently skips — no restart needed; the flag is fresh-read). Fleet: revert the PR — no data formats change; the only state this feature writes is the `operationalFacts` array via explicit calls, which survives or is hand-removable. The VITEST guard's rollback is one constructor line. Names already in transcripts are the only non-revertible residue (documented).
+
+## Deferred / follow-ups (all tracked)
+
+- Live-fleet flip (`enabled: true` in ConfigDefaults) — rides PR #800's merge or explicit approver direction (spec §Availability Resolution rule; the approver was asked directly in the approval request).
+- Session-start hook's pre-existing uncapped sibling curls — framework-issues ledger `session-start-hook-uncapped-curls`.
+- `/secrets/sync-status` rendering decrypt-failure as an empty vault — framework-issues ledger `sync-status-decrypt-fail-reads-empty`.
+- Per-agent keychain accounts + key-id header + dual-key read fallback — pre-existing commitment from the 2026-06-05 incident (CMT lineage in topic 13481), unchanged by this PR.
+
+## Post-review fix round (fresh-eyes code review, 2026-06-05)
+
+The independent code review of the feature commit found ONE real bug, fixed before PR: the names cache (keyed on the vault file) cached the `decrypt-failed` outcome — but a decrypt failure is almost always a MASTER-KEY problem (a separate file the cache key cannot see), so a recovered key kept serving the stale hands-off warning until a restart. Fix: only the healthy outcome is cached; a failed state is re-tried on every read (cheap relative to lying about recovery). Plus hardening: backticks are stripped from rendered names/facts (a hostile name can no longer break the inline-code span). Two regression tests added (decrypt-recovery-heals-without-restart; backtick-inertness).
+
+## CI-fix round (post-PR, 2026-06-05)
+
+Three CI failures owned per Zero-Failure: (1) `ConversationStore.test.ts` time-bomb — the test anchored retirement at fixed `2026-05-30`, and its 25h-stale entries crossed the store's 7-day expiry on 2026-06-05 (main's last CI run squeaked under the boundary by an hour; this PR's run detonated it) — re-anchored at real now, semantics unchanged; (2) no-silent-fallbacks ratchet — the four new BootSelfKnowledge catch blocks annotated `@silent-fallback-ok` with per-catch justifications (never a baseline bump); (3) docs-coverage route floor — the three new routes documented in the site API reference + a new features page (which also satisfies the route ratchet for the existing tree routes' namespace).
+
+## Compaction-parity round (approver design review, 2026-06-05)
+
+Justin's review surfaced the long-session gap: the block injected at session start survives compaction only if the summary carries it. Fixed in-PR: the compaction-recovery hook now carries the same fail-open fetch (re-injection after every compaction — refreshed, not merely preserved), with a Phase-3 e2e running the real compact-hook block against a live server. Collateral: org-intent + preferences share the boot-only gap (filed as `session-context-injectors-lack-compaction-parity`); a "Compaction Parity" constitution amendment is proposed separately. His scale concern is answered by the existing hard 2KB byte-cap (pointer-not-payload design); the AGGREGATE boot-budget concern across all injectors filed as `boot-context-aggregate-budget`.
+
+## Post-merge-conflict round (2026-06-05 AM)
+
+Rebase onto the post-#848 main + CI surfaced the feature-delivery-completeness registry: the new CLAUDE.md section is now tracked in `featureSections` AND mirrored to the framework-shadow markers (`migrateFrameworkShadowCapabilities`) — Codex/Gemini agents learn the capability too (the Secret Drop lesson: an unshadowed capability gets improvised around). Local pre-push had skipped the smoke ("CI is the authority"), which is why CI caught it.