@rubytech/create-realagent 1.0.652 → 1.0.653

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rubytech/create-realagent",
-  "version": "1.0.652",
+  "version": "1.0.653",
   "description": "Install Real Agent — Built for agents. By agents.",
   "bin": {
     "create-realagent": "./dist/index.js"

package/payload/platform/plugins/cloudflare/scripts/_stream-log.sh

@@ -6,13 +6,28 @@
 # helpers to emit phase lines and tee subprocess output into the same
 # per-conversation file the chat UI's server-side tailer reads.
 #
-# Contract (read by platform/ui/app/api/admin/chat/route.ts tailer and
+# Contract (read by platform/ui/app/lib/script-stream-tailer.ts tailer and
 # .docs/platform.md):
 #   [<ISO-ts>] [<scope>] <kv …>
 #   [<ISO-ts>] [<scope>:<subprocess-tag>] <raw line>
-# where <scope> ∈ {setup-tunnel, reset-tunnel}. The tailer regex is
-#   ^\[[^]]+\] \[(setup-tunnel|reset-tunnel)(:[^]]+)?\]
-# so any prefix change must be made on both sides atomically.
+# Canonical regex — SCRIPT_STREAM_RE at platform/ui/app/lib/script-stream-tailer.ts:51:
+#   ^\[([^\]]+)\] \[([a-z][a-z0-9-]*)((?::[a-z0-9:_-]+)?)\] (.*)$
+# <scope> is any `[a-z][a-z0-9-]*` token (Task 592 generalised from the
+# pre-592 enum `setup-tunnel|reset-tunnel`, which silently filtered out the
+# `[list-cf-domains]` lines Task 589 emitted). <subprocess-tag> may contain
+# lowercase, digits, `-`, `_`, `:`. Adding a new <scope> requires no edit to
+# the regex — the shape is the only contract. Any prefix-shape change must
+# be made on both sides (this helper + script-stream-tailer.ts) atomically.
+#
+# Inner layers (e.g. a node/python helper a .sh wrapper spawns — Task 598's
+# `list-cf-domains.sh` → `list-cf-domains.ts` pattern) must write phase lines
+# directly to STREAM_LOG_PATH with the same prefix shape: stderr alone is
+# silently discarded by runFormSpawn on exit 0. The build-gate
+# `platform/ui/scripts/check-stream-log-contract.mjs` (Task 600) enforces this
+# by rejecting any .sh under `platform/plugins/*/scripts/` that sources this
+# helper and invokes an interpreter subprocess whose target does not pair a
+# STREAM_LOG_PATH env read with an append/write call. Opt out per invocation
+# with `# stream-log-contract: stderr-only (reason: <prose>)`.
 
 # Exit 1 loudly with the variable name and the invoking scope so direct-SSH
 # invocations fail fast and the operator reads exactly what to set. No

package/payload/platform/plugins/cloudflare/scripts/list-cf-domains.ts

@@ -17,6 +17,23 @@
 //     routing and cannot drift without breaking their entire dashboard; it
 //     is the most stable surface to key off.
 //
+// Why href-only (no page-text fallback): Task 599 removed an earlier
+// <main>-wide FQDN text walker ("Source B") after a reproduction returned
+// `count=1` with a bogus string (`leg.interest` — a static footer FQDN-shape
+// that hit the regex). The walker's non-empty result short-circuited the
+// poll-to-success branch before zones hydrated 500 ms later, silently
+// returning wrong data with `reason=ok`. The walker's intended role —
+// "defend against CF redesign removing href links" — is already served by
+// the existing `empty-or-drift` body-dump branch when href scraping returns
+// nothing; a silent fallback that produces wrong answers is strictly worse
+// than a loud drift signal with a snapshotted HTML dump for diagnosis.
+//
+// Why poll-to-stable: the CF dashboard lazy-loads its zone list after the
+// initial document-ready signal. A naive "return on first non-empty poll"
+// can race into a partial result (e.g. first zone rendered at t=500 ms, all
+// zones at t=1000 ms). The `scrapeDomains` poll waits for the observed count
+// to stabilise across two consecutive iterations before returning.
+//
 // Contract with the caller (list-cf-domains.sh → route → form):
 //   stdout on exit 0: JSON `string[]`  (empty array means signed-in-but-empty)
 //   stderr on any path: phase_line-formatted lines, `[list-cf-domains] …`
@@ -29,6 +46,7 @@ import { appendFileSync } from "node:fs";
 import { writeFile } from "node:fs/promises";
 import { resolve } from "node:path";
 import { homedir } from "node:os";
+import { fileURLToPath } from "node:url";
 
 // CDP host/port default to the VNC Chromium's localhost bind; env overrides
 // exist solely so the vitest regression under platform/ui/__tests__ can force
@@ -272,86 +290,154 @@ async function waitForSignedIn(cdp: CdpClient): Promise<string> {
   die("not-signed-in", "no /<accountId>/… path reached within budget");
 }
 
-// Extract every domain mentioned on the Domains Overview page via URL-
-// pattern match, then merge with any FQDN-shaped text found in table cells.
-// Two sources: (a) `/<accountId>/<hostname>` URL hrefs on the page; (b) any
-// text node on the page matching a valid FQDN pattern — the overview page
-// renders the zone name as both a link and a table cell, so either source
-// catches it. The merge is conservative: we require the string look like a
-// domain (2+ labels, last label 2-63 alpha) AND not be a cloudflare-owned
-// marketing host.
-const SCRAPE_EXPRESSION = `(function() {
-  const accountIdMatch = location.pathname.match(/^\\/([a-f0-9]{32})/);
-  if (!accountIdMatch) return { reason: 'no-account-id', domains: [] };
+export interface ScrapeOutcome {
+  reason: "ok" | "no-account-id";
+  domains: string[];
+}
+
+// Pure scrape logic — extracts the zones on the current Cloudflare dashboard
+// page by matching `<a href="/<accountId>/<zone>…">` links. Exported for
+// direct in-process testing under JSDOM (see `list-cf-domains-scrape.test.ts`),
+// and serialised into `SCRAPE_EXPRESSION` below for execution inside the
+// operator's VNC Chromium via CDP `Runtime.evaluate`. The function takes its
+// document and location as arguments (rather than reading globals) so the
+// JSDOM test can pass in its window's document+location directly — test and
+// prod share the same implementation, with no duplication and no eval.
+//
+// Note: this function body is serialised via `Function.prototype.toString()`,
+// so it must be fully self-contained — no imports, no captures from outer
+// scope. Node-side helpers belong outside.
+export function scrapeCurrentPage(
+  document: Document,
+  location: Location,
+): ScrapeOutcome {
+  const accountIdMatch = location.pathname.match(/^\/([a-f0-9]{32})/);
+  if (!accountIdMatch) return { reason: "no-account-id", domains: [] };
   const accountId = accountIdMatch[1];
 
-  const out = new Set();
-  const pushIfDomain = (s) => {
-    if (typeof s !== 'string') return;
+  const out = new Set<string>();
+  const pushIfDomain = (s: unknown): void => {
+    if (typeof s !== "string") return;
     const t = s.trim().toLowerCase();
-    if (!/^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?(?:\\.[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?)+$/.test(t)) return;
-    if (t.endsWith('.cloudflare.com') || t === 'cloudflare.com') return;
+    if (
+      !/^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?(?:\.[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?)+$/.test(
+        t,
+      )
+    )
+      return;
+    if (t.endsWith(".cloudflare.com") || t === "cloudflare.com") return;
     out.add(t);
   };
 
-  // Source A: /<accountId>/<host> hrefs — the canonical CF routing pattern
-  // that the overview page uses to link each zone to its management screens.
-  const hrefPrefix = '/' + accountId + '/';
-  for (const a of document.querySelectorAll('a[href]')) {
-    const href = a.getAttribute('href') || '';
+  // The only scrape source: `/<accountId>/<host>` hrefs — the canonical CF
+  // routing pattern the overview page uses to link each zone to its
+  // management screens. Task 599 removed a full-<main> FQDN text walker
+  // after it returned a bogus FQDN-shaped string from static page copy
+  // (see module header). When this source returns empty, the caller's
+  // `empty-or-drift` branch snapshots the page HTML for diagnosis — loud
+  // drift signal, not silent wrong data.
+  const hrefPrefix = "/" + accountId + "/";
+  for (const a of document.querySelectorAll("a[href]")) {
+    const href = a.getAttribute("href") || "";
     if (!href.startsWith(hrefPrefix)) continue;
-    const tail = href.slice(hrefPrefix.length).split('?')[0].split('#')[0];
-    const firstSegment = tail.split('/')[0];
+    const tail = href.slice(hrefPrefix.length).split("?")[0].split("#")[0];
+    const firstSegment = tail.split("/")[0];
     pushIfDomain(firstSegment);
   }
 
-  // Source B: explicit FQDN-shaped text in the main content region. Guards
-  // against a hypothetical CF redesign that drops the link tree — the zone
-  // name still appears as rendered text in the table row.
-  const main = document.querySelector('main') || document.body;
-  if (main) {
-    const treeWalker = document.createTreeWalker(main, NodeFilter.SHOW_TEXT);
-    let node;
-    while ((node = treeWalker.nextNode())) {
-      const text = (node.nodeValue || '').trim();
-      if (text.length < 4 || text.length > 253) continue;
-      pushIfDomain(text);
-    }
-  }
-
-  return { reason: 'ok', domains: Array.from(out).sort() };
-})()`;
-
-interface ScrapeOutcome {
-  reason: "ok" | "no-account-id";
-  domains: string[];
+  return { reason: "ok", domains: Array.from(out).sort() };
 }
 
-async function scrapeDomains(cdp: CdpClient): Promise<string[]> {
+// The serialised form the CDP Runtime.evaluate executes inside the operator's
+// VNC Chromium. Deriving it from `scrapeCurrentPage.toString()` means the
+// browser-side and the test-side share one implementation — drift between
+// them is impossible by construction.
+export const SCRAPE_EXPRESSION = `(${scrapeCurrentPage.toString()})(document, location)`;
+
+// Minimum surface `scrapeDomains` needs from the CDP client: execute a JS
+// expression in the target page and return the serialised value. Typing the
+// dependency at this width lets the vitest regression inject a mock without
+// constructing a full CdpClient + WebSocket.
+export type CdpEvaluator = (expression: string) => Promise<unknown>;
+
+// Number of consecutive polls returning an identical non-empty count that
+// proves the SPA zone-list has finished hydrating. Two iterations at 500 ms
+// each = 1 s of observed stability, which is the empirical ceiling on the
+// CF dashboard's lazy zone-list fetch post-document-ready. Shorter thresholds
+// risk returning a partial list; longer thresholds add user-visible latency
+// without proportional robustness.
+const STABLE_POLL_THRESHOLD = 2;
+
+export async function scrapeDomains(evaluator: CdpEvaluator): Promise<string[]> {
   const deadline = Date.now() + SCRAPE_POLL_MS;
   let lastOutcome: ScrapeOutcome | null = null;
+  // Track the most recent non-empty observation so deadline-reached without
+  // stability returns the caller's best evidence rather than falling through
+  // to the drift-dump (which is reserved for the always-empty case).
+  let lastNonEmptyDomains: string[] = [];
+  let stableCount = -1;
+  let stableIterations = 0;
+  let polls = 0;
+
   while (Date.now() < deadline) {
+    polls += 1;
     try {
-      const outcome = await evaluate<ScrapeOutcome>(cdp, SCRAPE_EXPRESSION);
+      const outcome = (await evaluator(SCRAPE_EXPRESSION)) as ScrapeOutcome;
       lastOutcome = outcome;
+
       if (outcome.reason === "ok" && outcome.domains.length > 0) {
-        logPhase(`phase=dom-scrape-complete result=ok count=${outcome.domains.length}`);
-        return outcome.domains;
+        lastNonEmptyDomains = outcome.domains;
+        if (outcome.domains.length === stableCount) {
+          stableIterations += 1;
+          if (stableIterations >= STABLE_POLL_THRESHOLD) {
+            logPhase(
+              `phase=dom-scrape-complete result=ok count=${outcome.domains.length} polls=${polls} stable_polls=${stableIterations} unstable=false`,
+            );
+            return outcome.domains;
+          }
+        } else {
+          stableCount = outcome.domains.length;
+          stableIterations = 1;
+        }
+      } else {
+        // `ok` with zero domains OR a non-ok reason: treat as still-hydrating.
+        // Keep polling; the final iteration's zero result is the empty-account
+        // signal. Reset stability tracking so a 0 → N → 0 sequence doesn't
+        // falsely satisfy the threshold.
+        stableCount = -1;
+        stableIterations = 0;
       }
-      // `ok` with zero domains is ambiguous during SPA hydration — keep
-      // polling in case the table renders after a data fetch. The final
-      // iteration's zero result is the empty-account signal.
     } catch (err) {
-      logPhase(`phase=scrape-retry err="${(err instanceof Error ? err.message : String(err)).slice(0, 120)}"`);
+      logPhase(
+        `phase=scrape-retry err="${(err instanceof Error ? err.message : String(err)).slice(0, 120)}"`,
+      );
     }
     await new Promise((r) => setTimeout(r, POLL_INTERVAL_MS));
   }
-  // Poll exhausted. Distinguish genuine empty account from selector drift by
-  // dumping the body HTML (bounded) for manual inspection, then reporting
-  // empty with the diagnostic enum. The script still exits 0 — the caller
-  // treats empty as a valid signal and shows the "add a domain" empty state.
+
+  // Deadline reached. Two distinct branches:
+  //
+  // (i) We saw non-empty results but they never stabilised across two
+  //     consecutive polls — the page is either paginating or re-rendering
+  //     the zone list. Return the last non-empty observation and flag
+  //     `unstable=true` on the phase line so the operator can see the race
+  //     in the stream log. This is NOT drift — the HTML dump would be
+  //     misleading noise, so we suppress it.
+  //
+  // (ii) We never saw non-empty results — this is either a genuinely empty
+  //      account OR CF has drifted the href URL shape so Source A yields
+  //      nothing. The `empty-or-drift` body-dump branch snapshots the page
+  //      HTML so the operator can distinguish the two by inspecting the
+  //      dump file.
+  if (lastNonEmptyDomains.length > 0) {
+    logPhase(
+      `phase=dom-scrape-complete result=ok count=${lastNonEmptyDomains.length} polls=${polls} stable_polls=${stableIterations} unstable=true`,
+    );
+    return lastNonEmptyDomains;
+  }
+
   try {
-    const html = await evaluate<string>(cdp, "document.documentElement.outerHTML.slice(0, 100000)");
+    const html = (await evaluator("document.documentElement.outerHTML.slice(0, 100000)")) as string;
     // CONFIG_DIR is set by list-cf-domains.sh before the spawn. A silent
     // fallback would dump logs into the wrong brand's directory on a Real
     // Agent install — a silent-miswrite masking the wrapper-side break it
@@ -359,15 +445,21 @@ async function scrapeDomains(cdp: CdpClient): Promise<string[]> {
     // values.
     const configDir = process.env.CONFIG_DIR;
     if (!configDir) {
-      throw new Error("CONFIG_DIR env var not set by wrapper — refusing to guess brand log directory");
+      throw new Error(
+        "CONFIG_DIR env var not set by wrapper — refusing to guess brand log directory",
+      );
     }
     const logDir = resolve(homedir(), configDir, "logs");
     const ts = new Date().toISOString().replace(/[:.]/g, "-");
     const dumpPath = resolve(logDir, `list-cf-domains-${ts}.html`);
     await writeFile(dumpPath, typeof html === "string" ? html : String(html), "utf-8");
-    logPhase(`phase=dom-scrape-complete result=empty-or-drift dump=${dumpPath} lastReason=${lastOutcome?.reason ?? "unknown"}`);
+    logPhase(
+      `phase=dom-scrape-complete result=empty-or-drift dump=${dumpPath} lastReason=${lastOutcome?.reason ?? "unknown"} polls=${polls}`,
+    );
   } catch (err) {
-    logPhase(`phase=dom-scrape-complete result=empty-or-drift dump=failed lastReason=${lastOutcome?.reason ?? "unknown"} err="${(err instanceof Error ? err.message : String(err)).slice(0, 120)}"`);
+    logPhase(
+      `phase=dom-scrape-complete result=empty-or-drift dump=failed lastReason=${lastOutcome?.reason ?? "unknown"} polls=${polls} err="${(err instanceof Error ? err.message : String(err)).slice(0, 120)}"`,
+    );
   }
   return [];
 }
@@ -419,7 +511,7 @@ async function main(): Promise<void> {
     await navigate(cdp, overviewUrl);
     await waitForDocumentReady(cdp);
 
-    const domains = await scrapeDomains(cdp);
+    const domains = await scrapeDomains((expr) => evaluate(cdp, expr));
 
     logPhase(`phase=target-closed`);
     process.stdout.write(JSON.stringify(domains) + "\n");
@@ -433,6 +525,13 @@ async function main(): Promise<void> {
   }
 }
 
-main().catch((err: unknown) => {
-  die("runtime-error", err instanceof Error ? err.message : String(err));
-});
+// Entry-point gate: only run `main()` when this module is the script Node was
+// invoked with, not when imported by the vitest regressions that exercise
+// `scrapeCurrentPage` / `scrapeDomains` directly. Without the gate, importing
+// this module would trigger a live CDP connection attempt and process.exit(1)
+// inside the test process.
+if (process.argv[1] && process.argv[1] === fileURLToPath(import.meta.url)) {
+  main().catch((err: unknown) => {
+    die("runtime-error", err instanceof Error ? err.message : String(err));
+  });
+}

package/payload/platform/plugins/docs/references/platform.md

@@ -66,7 +66,9 @@ The chat input auto-grows as you type — it expands to fit your message and shr
 
 The admin UI includes a live terminal surface that opens a real shell on your Pi in the browser, reached via the Software Update modal. Under the hood it's a WebSocket (`/admin/terminal/ws`) attached through `@xterm/xterm` to `ttyd` on `127.0.0.1:7681`, backed by a persistent tmux session named `maxy-pty`.
 
-The tmux session outlives admin-server restarts — running an upgrade inside this terminal means you see the live shell output continuously, even through the admin server's own restart mid-upgrade. Closing the browser tab does not kill the running work; re-opening the Software Update window reattaches to the same session and scrollback shows everything that happened in the meantime. Password-protected `sudo` prompts appear natively inside the terminal, and the password you type never leaves the Pi — the admin-server proxy is a raw byte pipe that never inspects frame payloads.
+The tmux session outlives admin-server restarts — running an upgrade inside this terminal means you see the live shell output continuously, even through the admin server's own restart mid-upgrade. Closing the browser tab does not kill the running work; re-opening the Software Update window reattaches to the same session during an active upgrade and scrollback shows everything that happened in the meantime. Password-protected `sudo` prompts appear natively inside the terminal, and the password you type never leaves the Pi — the admin-server proxy is a raw byte pipe that never inspects frame payloads.
+
+The Software Update window mounts the terminal lazily: the WebSocket is opened on the first Upgrade click, not when the window opens. Until you click Upgrade, the terminal area shows "Ready to upgrade." and no network traffic flows. If the admin server cannot reach `ttyd`, the window renders an inline "Admin terminal not available" message with the exact re-install command and a Try again button — no silent reconnect loops, no empty black rectangle. The scrollback-across-reopen behaviour above still applies during an active upgrade (a sessionStorage flag remembers that an upgrade is in flight so reopening the window re-mounts the terminal and reattaches).
 
 ## AI Content Provenance
 

package/payload/platform/plugins/docs/references/plugins-guide.md

@@ -103,7 +103,7 @@ After this, every `console.error("[your-tool] ...")` from any tool in the plugin
 
 **How the tee decides which file to write to (Task 532):** the platform sets `STREAM_LOG_PATH` as an environment variable on every MCP server spawn, pointing to the conversation-scoped stream log. The MCP server does not know about conversations — it just trusts `STREAM_LOG_PATH`. Multiple concurrent conversations produce multiple concurrent MCP server processes, each teeing to its own file; no cross-conversation leakage.
 
-**`STREAM_LOG_PATH` reaches every Claude Code child (Task 556).** The platform now sets `STREAM_LOG_PATH` on the parent `claude` spawn env itself (not only on MCP server envs), so the bundled Bun runtime inherits it and every Bash-tool subprocess the CLI spawns sees it too. Opt-in shell scripts — currently `setup-tunnel.sh` and `reset-tunnel.sh` under `platform/plugins/cloudflare/scripts/` — read the variable, guard against a missing value with a loud exit, and tee subprocess output line-by-line into the same per-conversation file. Each spawn writes one `[spawn-env] STREAM_LOG_PATH=set pid=… conversationId=… site=…` line so the env-propagation is auditable per session. The chat UI tails the same file for lines matching `^\[[^]]+\] \[(setup-tunnel|reset-tunnel)(:[^]]+)?\] ` and emits them as `script_stream` SSE events; see `.docs/web-chat.md` for the contract.
+**`STREAM_LOG_PATH` reaches every Claude Code child (Task 556).** The platform now sets `STREAM_LOG_PATH` on the parent `claude` spawn env itself (not only on MCP server envs), so the bundled Bun runtime inherits it and every Bash-tool subprocess the CLI spawns sees it too. Opt-in shell scripts — currently `setup-tunnel.sh`, `reset-tunnel.sh`, and `list-cf-domains.sh` under `platform/plugins/cloudflare/scripts/` — read the variable, guard against a missing value with a loud exit, and tee subprocess output line-by-line into the same per-conversation file. Each spawn writes one `[spawn-env] STREAM_LOG_PATH=set pid=… conversationId=… site=…` line so the env-propagation is auditable per session. The chat UI tails the same file for lines matching `^\[([^\]]+)\] \[([a-z][a-z0-9-]*)((?::[a-z0-9:_-]+)?)\] ` — any lowercase scope shape participates on first write (Task 592 generalised from the pre-592 enum `setup-tunnel|reset-tunnel`) — and emits them as `script_stream` SSE events; see `.docs/web-chat.md` for the contract. Inner-layer helpers that a .sh wrapper spawns (e.g. `list-cf-domains.ts` via `node --experimental-strip-types`) must write phase lines directly to `STREAM_LOG_PATH` rather than relying on stderr propagation (Task 598); the build-gate `platform/ui/scripts/check-stream-log-contract.mjs` (Task 600) enforces this and is the definitive reference for the three allowed patterns (tee-wrapped, direct-write, or explicit stderr-only marker).
 
 **Retrieve MCP diagnostic lines for a conversation:**
 

package/payload/platform/plugins/docs/references/troubleshooting.md

@@ -109,6 +109,22 @@ This is safe — the tmux session survives the ttyd restart because `tmux new-se
 
 **Terminal unit missing entirely?** If `sudo systemctl --user status maxy-ttyd` reports `unit not found`, the installer's ttyd provisioning step failed — typically on a Bookworm device where `ttyd` is not available via apt and the upstream download (or SHA256 check) failed at install time. Re-run `npx -y @rubytech/create-maxy@latest`; the operator-visible remediation command is also printed at install time in `~/.maxy/logs/install-*.log`.
 
+---
+
+## "Admin terminal not available" in the Software Update window
+
+**Symptom:** The Software Update window displays "Admin terminal not available. Re-run the installer from a shell: `npx -y @rubytech/create-maxy@latest`" with a Try again button, instead of the usual terminal area.
+
+**What it means:** The admin server could not reach `ttyd` on `127.0.0.1:7681`. Either `maxy-ttyd.service` is not running, or it failed to install during setup.
+
+**Fix:** Run the exact command shown in the error message from a shell on the device:
+
+```bash
+npx -y @rubytech/create-maxy@latest
+```
+
+Then return to the upgrade window and click **Try again**. The window re-probes `/api/health` and, once ttyd is listening, the terminal area mounts as normal. If the problem persists, check the boot log for `[ttyd] upstream NOT reachable on 127.0.0.1:7681` and follow the `maxy-ttyd` restart steps above.
+
 ## Orphan Account Directory Archived to `.trash/`
 
 **What happened:** During upgrade, the installer detected multiple account directories under `~/maxy/data/accounts/` and identified one as live (its `admins` list matches the device's `users.json`). Non-matching siblings are archived — not deleted — under `~/maxy/data/accounts/.trash/<uuid>-<ISO8601-ts>/`.