humanish 0.104.3 → 0.106.0

package/dist/comms/sandbox-catch.js

@@ -0,0 +1,451 @@
+// Deploy the vendor-neutral email catch INSIDE the subject E2B sandbox and bridge captured sends back
+// to the host bus (#297 config-block core). The host `startEmailCatchServer` binds the HOST's loopback,
+// which a sandboxed app cannot reach — `127.0.0.1:PORT` from the app is the SANDBOX's loopback. So the
+// listener must live in the sandbox: we write a tiny self-contained capture server (no deps, no host
+// import) into the sandbox, launch it detached (the same substrate that serves the subject app), and
+// each poll `cat` its append-only NDJSON of captured sends back to the host, where the real profiles
+// parse them and route into the CommsChannel. A FIXED loopback port is chosen up front so the app's
+// injected base-URL env (`http://127.0.0.1:<port>`) is known before the sandbox is created.
+import { FakeInbox } from "./fake-inbox.js";
+import { buildCommsThreadArtifact } from "./thread-evidence.js";
+import { SANDBOX_CATCH_SCRIPT } from "./sandbox-catch-script.js";
+import { buildInboxSurface } from "./capture-surface.js";
+import { DEFAULT_EMAIL_PROFILES } from "./email-catch.js";
+import { startDetachedProcess } from "../substrates/detached.js";
+import { runOrThrow, shellQuote } from "../substrates/shell.js";
+/** The default in-sandbox loopback port for the catch. Fixed (not ephemeral) so the injected base-URL
+ *  env is known before the sandbox is created. 8025 is the conventional local-mail-UI port and is
+ *  unlikely to collide with a subject app; override via config if it does. */
+export const DEFAULT_SANDBOX_CATCH_PORT = 8025;
+const DEFAULT_CATCH_DIR = "/tmp/humanish-comms";
+/** Readiness probe that asserts OUR service marker in the /health body (not merely any 2xx) — so a
+ *  process squatting on the fixed port cannot produce a false "ready" while the app's sends bypass us. */
+async function catchHealthy(shell, port, options) {
+    const now = options.now ?? Date.now;
+    const sleep = options.sleep ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms)));
+    const deadline = now() + options.timeoutMs;
+    for (;;) {
+        const result = await shell
+            .run(`curl -s --max-time 5 http://127.0.0.1:${port}/health 2>/dev/null || true`, {
+            requestTimeoutMs: options.requestTimeoutMs,
+        })
+            .catch(() => ({ stdout: "" }));
+        if (result.stdout.includes("humanish-comms-catch"))
+            return true;
+        if (now() >= deadline)
+            return false;
+        await sleep(1000);
+    }
+}
+/**
+ * Write + launch the in-sandbox catch (detached), then probe it ready. Call AFTER the subject sandbox
+ * is created and BEFORE the subject app's serve.start, so the base URL resolves at the app's boot.
+ */
+export async function deployCommsCatch(shell, options = {}) {
+    // Validate the port to an integer before it reaches the shell command (defense-in-depth: a future
+    // caller might cast a config value; the value is typed `number` but this makes injection impossible).
+    const port = Math.trunc(Number(options.port ?? DEFAULT_SANDBOX_CATCH_PORT));
+    if (!Number.isInteger(port) || port <= 0 || port > 65_535) {
+        throw new Error(`deployCommsCatch: invalid port ${JSON.stringify(options.port)}`);
+    }
+    const inboxPort = options.inboxPort === undefined ? undefined : Math.trunc(Number(options.inboxPort));
+    if (inboxPort !== undefined &&
+        (!Number.isInteger(inboxPort) || inboxPort <= 0 || inboxPort > 65_535 || inboxPort === port)) {
+        throw new Error(`deployCommsCatch: invalid inboxPort ${JSON.stringify(options.inboxPort)}`);
+    }
+    const smtpPort = options.smtpPort === undefined ? undefined : Math.trunc(Number(options.smtpPort));
+    if (smtpPort !== undefined &&
+        (!Number.isInteger(smtpPort) ||
+            smtpPort <= 0 ||
+            smtpPort > 65_535 ||
+            smtpPort === port ||
+            smtpPort === inboxPort)) {
+        throw new Error(`deployCommsCatch: invalid smtpPort ${JSON.stringify(options.smtpPort)}`);
+    }
+    const dir = options.dir ?? DEFAULT_CATCH_DIR;
+    const name = options.name ?? "comms-catch";
+    const requestTimeoutMs = options.requestTimeoutMs ?? 30_000;
+    const scriptPath = `${dir}/catch.py`;
+    const deliveriesPath = `${dir}/deliveries.ndjson`;
+    const surfaceDir = `${dir}/surface`;
+    await runOrThrow(shell, `mkdir -p ${shellQuote(dir)} ${shellQuote(surfaceDir)}`, {
+        requestTimeoutMs,
+    });
+    await shell.writeFile(scriptPath, SANDBOX_CATCH_SCRIPT);
+    await startDetachedProcess(shell, {
+        name,
+        command: [
+            "python3",
+            shellQuote(scriptPath),
+            String(port),
+            shellQuote(deliveriesPath),
+            shellQuote(surfaceDir),
+            ...(inboxPort === undefined && smtpPort === undefined ? [] : [String(inboxPort ?? 0)]),
+            // The token slot is positional: an SMTP port cannot be reached without filling it. In-sandbox
+            // the capture listener is loopback-only, so an empty token is the same posture as before.
+            ...(smtpPort === undefined ? [] : ['""', String(smtpPort)]),
+        ].join(" "),
+        requestTimeoutMs,
+    });
+    const probe = {
+        timeoutMs: options.readyTimeoutMs ?? 15_000,
+        requestTimeoutMs,
+        ...options.timers,
+    };
+    const ready = await catchHealthy(shell, port, probe);
+    // Confirm the read-only inbox listener bound too (loopback-reachable at its own port), when requested —
+    // else a getHost-exposed inbox would 502. Fail closed by folding it into `ready`.
+    const inboxReady = inboxPort === undefined ? true : await catchHealthy(shell, inboxPort, probe);
+    return {
+        port,
+        baseUrl: `http://127.0.0.1:${port}`,
+        deliveriesPath,
+        surfaceDir,
+        ...(inboxPort === undefined ? {} : { inboxPort }),
+        ...(smtpPort === undefined ? {} : { smtpPort }),
+        ready: ready && inboxReady,
+    };
+}
+/**
+ * Drain new captured sends from the in-sandbox NDJSON since `cursor` (a line count). Returns the fresh
+ * sends and the new cursor. Cheap `cat` over commands.run; NDJSON is small for a run.
+ */
+export async function drainCommsCatch(shell, deployed, cursor = 0, requestTimeoutMs = 30_000) {
+    const result = await runOrThrow(shell, `cat ${shellQuote(deployed.deliveriesPath)} 2>/dev/null || true`, { requestTimeoutMs });
+    const stdout = result.stdout;
+    let lines = stdout.split("\n").filter((line) => line.trim().length > 0);
+    // If the file doesn't end in a newline, the last line may be a PARTIAL append (the host `cat` raced
+    // an in-sandbox append of a large body). Drop it and don't advance the cursor past it — it re-reads
+    // complete on the next poll, so a captured send is never lost to the race (the script only ever emits
+    // valid JSON, so an incomplete line is the only cause of a parse miss).
+    if (!stdout.endsWith("\n") && lines.length > 0)
+        lines = lines.slice(0, -1);
+    const sends = [];
+    for (const line of lines.slice(cursor)) {
+        try {
+            const parsed = JSON.parse(line);
+            if (typeof parsed.path === "string" && typeof parsed.body === "string") {
+                sends.push({
+                    path: parsed.path,
+                    body: parsed.body,
+                    t: typeof parsed.t === "number" ? parsed.t : 0,
+                });
+            }
+        }
+        catch {
+            // skip a malformed line
+        }
+    }
+    return { sends, cursor: lines.length };
+}
+/**
+ * Parse an append-only deliveries NDJSON blob into raw sends. Split out of drainCommsCatch (#380) so
+ * the SAME parsing serves a sandbox we own (read over the E2B command channel) and a catch running on
+ * a plane we do not own (read from the local filesystem by `humanish comms catch`).
+ *
+ * A file that does not end in a newline may have a PARTIAL last line — a reader racing an append of a
+ * large body. Dropping it is never lossy: the script only ever emits valid JSON lines, so an incomplete
+ * line re-reads complete on the next pass.
+ */
+export function parseDeliveriesNdjson(text) {
+    let lines = text.split("\n").filter((line) => line.trim().length > 0);
+    if (!text.endsWith("\n") && lines.length > 0)
+        lines = lines.slice(0, -1);
+    const sends = [];
+    for (const line of lines) {
+        try {
+            const parsed = JSON.parse(line);
+            if (typeof parsed.path === "string" && typeof parsed.body === "string") {
+                sends.push({
+                    path: parsed.path,
+                    body: parsed.body,
+                    t: typeof parsed.t === "number" ? parsed.t : 0,
+                });
+            }
+        }
+        catch {
+            // skip a malformed line
+        }
+    }
+    return sends;
+}
+/**
+ * The distinct `to` addresses the captured mail was actually sent to, parsed with the SAME profiles
+ * that route it. A lab run knows its recipients from the declared roster; a standalone catch does not,
+ * so it discovers them from the mail itself — otherwise an operator who forgot to name an address gets
+ * a technically-healthy catch rendering an empty inbox forever, which is the false-green class #380 is
+ * about.
+ */
+export function capturedRecipientAddresses(sends, profiles = DEFAULT_EMAIL_PROFILES) {
+    const addresses = new Set();
+    for (const send of sends) {
+        let parsed;
+        try {
+            parsed = JSON.parse(send.body.length > 0 ? send.body : "{}");
+        }
+        catch {
+            continue;
+        }
+        const profile = profiles.find((candidate) => candidate.sendPaths.includes(send.path)) ?? profiles[0];
+        if (profile === undefined)
+            continue;
+        for (const normalized of profile.parse(send.path, parsed)) {
+            for (const address of normalized.to) {
+                if (address.trim().length > 0)
+                    addresses.add(address);
+            }
+        }
+    }
+    return [...addresses];
+}
+/**
+ * Route raw sends into a FRESH FakeInbox and return the deduped, delivery-ordered messages. Split out
+ * of refreshInboxSurface (#380) so the rendering pipeline is shared by every transport; the freshness
+ * is what makes a full rebuild idempotent (a send is never routed twice, so no duplicate emails).
+ */
+export async function inboxMessagesFrom(sends, recipients) {
+    const channel = new FakeInbox();
+    const inboxes = [];
+    for (const recipient of recipients)
+        inboxes.push(await channel.provisionAddress(recipient.participantId, recipient.address));
+    await routeCapturedSends([...sends], channel);
+    const seen = new Set();
+    const messages = [];
+    for (const inbox of inboxes) {
+        for (const message of await channel.poll(inbox, 0)) {
+            if (seen.has(message.id))
+                continue;
+            seen.add(message.id);
+            messages.push(message);
+        }
+    }
+    messages.sort((a, b) => a.deliveredAt - b.deliveredAt || a.id.localeCompare(b.id));
+    return messages;
+}
+/**
+ * Parse drained raw sends with the host profiles and route them into the CommsChannel (the host-side
+ * FakeInbox). Returns the number of inbox deliveries made. Same profiles as the host catch, so the
+ * in-sandbox and in-process routes normalize identically.
+ */
+export async function routeCapturedSends(sends, channel, profiles = DEFAULT_EMAIL_PROFILES) {
+    let delivered = 0;
+    for (const send of sends) {
+        let parsed;
+        try {
+            parsed = JSON.parse(send.body.length > 0 ? send.body : "{}");
+        }
+        catch {
+            continue;
+        }
+        const profile = profiles.find((candidate) => candidate.sendPaths.includes(send.path)) ?? profiles[0];
+        if (profile === undefined)
+            continue;
+        for (const normalized of profile.parse(send.path, parsed)) {
+            const messages = await channel.deliverRaw({
+                from: normalized.from,
+                to: normalized.to,
+                ...(normalized.subject === undefined ? {} : { subject: normalized.subject }),
+                body: normalized.body,
+                ...(normalized.inlineImages ? { inlineImages: normalized.inlineImages } : {}),
+            });
+            delivered += messages.length;
+        }
+    }
+    return delivered;
+}
+/**
+ * End of a run's comms funnel: drain everything the in-sandbox catch captured, route it into the
+ * host `channel`, poll the provisioned `inboxes`, and build the digest-only thread artifact. The
+ * `artifact` is omitted when nothing was captured OR nothing matched a provisioned inbox (an empty
+ * file would be a false claim of a delivered thread) — but `captured`/`matched` are always reported so
+ * the caller can warn on captured-but-unevidenced mail rather than lose it silently. Composes the
+ * tested drain/route/build pieces so the CUA and shared-world routes collect evidence identically. The
+ * full NDJSON is drained from cursor 0 (a whole-run collect), so it is idempotent to call once at
+ * teardown.
+ */
+export async function collectCommsThread(args) {
+    const { sends } = await drainCommsCatch(args.shell, args.deployed, 0, args.requestTimeoutMs);
+    if (sends.length === 0)
+        return { captured: 0, matched: 0 };
+    await routeCapturedSends(sends, args.channel, args.profiles);
+    const seen = new Set();
+    const messages = [];
+    for (const inbox of args.inboxes) {
+        for (const message of await args.channel.poll(inbox, 0)) {
+            // A single send addressed to several provisioned inboxes must appear once in the thread.
+            if (seen.has(message.id))
+                continue;
+            seen.add(message.id);
+            messages.push(message);
+        }
+    }
+    if (messages.length === 0)
+        return { captured: sends.length, matched: 0 };
+    messages.sort((a, b) => a.deliveredAt - b.deliveredAt || a.id.localeCompare(b.id));
+    return {
+        artifact: buildCommsThreadArtifact(messages),
+        captured: sends.length,
+        matched: messages.length,
+    };
+}
+/** Trim one trailing slash so `${base}/deliveries` never becomes a double slash. */
+function baseOf(url) {
+    return url.replace(/\/+$/, "");
+}
+/** The URL a persona is told to open to read its mail on an adopter-hosted plane. */
+export function externalInboxUrl(external) {
+    return `${baseOf(external.inboxBaseUrl ?? external.catchBaseUrl)}/inbox`;
+}
+/**
+ * Probe an adopter-hosted catch the way the in-sandbox one is probed: assert OUR service marker in
+ * /health, not merely any 2xx — an adopter's reverse proxy or a captive portal will happily return
+ * 200 for anything, and a comms lab whose catch is not actually there collects nothing while
+ * looking fine. Fail-closed callers treat `false` as a hard stop before spending on a run.
+ */
+export async function externalCatchHealthy(external, options = {}) {
+    const fetchFn = options.fetchFn ?? fetch;
+    try {
+        const bases = new Set([external.catchBaseUrl, external.inboxBaseUrl ?? external.catchBaseUrl].map(baseOf));
+        const health = await Promise.all([...bases].map(async (base) => {
+            const response = await fetchFn(`${base}/health`, {
+                signal: AbortSignal.timeout(options.timeoutMs ?? 15_000),
+            });
+            if (!response.ok)
+                return false;
+            const body = await response.json();
+            if (!body || typeof body !== "object")
+                return false;
+            const value = body;
+            return (value.ok === true &&
+                value.service === "humanish-comms-catch" &&
+                Array.isArray(value.capabilities) &&
+                value.capabilities.includes("recipient-inbox-v1"));
+        }));
+        return health.every(Boolean);
+    }
+    catch {
+        return false;
+    }
+}
+/**
+ * Drain an adopter-hosted catch over HTTP. Same NDJSON contract and same partial-line discipline as
+ * the in-sandbox `cat` drain: a body that does not end in a newline may have a torn final append, so
+ * that line is dropped rather than parsed into a half-message.
+ */
+export async function drainExternalCommsCatch(external, cursor = 0, options = {}) {
+    const fetchFn = options.fetchFn ?? fetch;
+    const response = await fetchFn(`${baseOf(external.catchBaseUrl)}/deliveries`, {
+        signal: AbortSignal.timeout(options.timeoutMs ?? 30_000),
+        ...(external.authToken ? { headers: { authorization: `Bearer ${external.authToken}` } } : {}),
+    });
+    if (!response.ok) {
+        throw new Error(`comms catch GET /deliveries returned ${response.status}`);
+    }
+    const body = await response.text();
+    let lines = body.split("\n").filter((line) => line.trim().length > 0);
+    if (!body.endsWith("\n") && lines.length > 0)
+        lines = lines.slice(0, -1);
+    const sends = [];
+    for (const line of lines.slice(cursor)) {
+        try {
+            const parsed = JSON.parse(line);
+            if (typeof parsed.path === "string" && typeof parsed.body === "string") {
+                sends.push({
+                    path: parsed.path,
+                    body: parsed.body,
+                    t: typeof parsed.t === "number" ? parsed.t : 0,
+                });
+            }
+        }
+        catch {
+            // skip a malformed line
+        }
+    }
+    return { sends, cursor: lines.length };
+}
+/**
+ * The adopter-hosted analogue of collectCommsThread: drain over HTTP, route into the host inbox bus,
+ * and build the SAME digest-only humanish.comms-thread.v1 artifact. Evidence shape does not depend
+ * on who hosted the catch — only the transport does.
+ */
+export async function collectExternalCommsThread(args) {
+    const drainOptions = {
+        ...(args.timeoutMs === undefined ? {} : { timeoutMs: args.timeoutMs }),
+        ...(args.fetchFn ? { fetchFn: args.fetchFn } : {}),
+    };
+    const { sends } = await drainExternalCommsCatch(args.external, 0, drainOptions);
+    if (sends.length === 0)
+        return { captured: 0, matched: 0 };
+    await routeCapturedSends(sends, args.channel, args.profiles);
+    const seen = new Set();
+    const messages = [];
+    for (const inbox of args.inboxes) {
+        for (const message of await args.channel.poll(inbox, 0)) {
+            if (seen.has(message.id))
+                continue;
+            seen.add(message.id);
+            messages.push(message);
+        }
+    }
+    if (messages.length === 0)
+        return { captured: sends.length, matched: 0 };
+    messages.sort((a, b) => a.deliveredAt - b.deliveredAt || a.id.localeCompare(b.id));
+    return {
+        artifact: buildCommsThreadArtifact(messages),
+        captured: sends.length,
+        matched: messages.length,
+    };
+}
+/**
+ * Render the persona-facing inbox surface (host-side, typed — see capture-surface.ts) and write the files
+ * into the sandbox's served dir, so the catch serves a LIVE inbox the persona opens and clicks. Creates
+ * the nested route dirs first; overwrites idempotently, so call it whenever the message set changes
+ * (e.g. after a mid-run drain). Returns the number of files written. Raw content is written INTO the
+ * sandbox only (runtime-only, served to the in-sandbox browser); nothing here persists to the bundle.
+ */
+export async function writeInboxSurface(shell, surfaceDir, messages, options = {}) {
+    const files = buildInboxSurface(messages, options);
+    // Create the union of parent dirs (inbox/<id>/synth etc.) in one mkdir before writing.
+    const dirs = new Set([surfaceDir]);
+    for (const file of files) {
+        const slash = file.path.lastIndexOf("/");
+        if (slash > 0)
+            dirs.add(`${surfaceDir}/${file.path.slice(0, slash)}`);
+    }
+    await runOrThrow(shell, `mkdir -p ${[...dirs].map(shellQuote).join(" ")}`, {
+        requestTimeoutMs: options.requestTimeoutMs ?? 30_000,
+    });
+    for (const file of files) {
+        await shell.writeFile(`${surfaceDir}/${file.path}`, file.body);
+    }
+    return files.length;
+}
+/**
+ * One mid-run inbox-surface refresh cycle: FULL rebuild from the append-only NDJSON (drain from cursor 0)
+ * into a FRESH FakeInbox each call, provisioning the declared `recipients`, then (re)render the surface
+ * so the persona sees new mail while the session is live. Returns the total captured-send `count` + whether
+ * it rendered.
+ *
+ * The full rebuild is deliberate — it is IDEMPOTENT and RETRY-SAFE: a transient writeInboxSurface failure
+ * PROPAGATES (the caller retries next tick without advancing its `sinceCount`), and because each rebuild
+ * starts from a clean channel, a send is never routed twice, so the persona never sees duplicate emails.
+ * Pass `sinceCount` (the last SUCCESSFULLY-rendered send count) to skip the (N-file) render when nothing
+ * new has arrived. This is independent of the teardown collectCommsThread drain (its own fresh channel,
+ * also cursor 0) — no evidence is lost or altered. The NDJSON is small for a run, so re-reading it is cheap.
+ */
+export async function refreshInboxSurface(args) {
+    const { sends } = await drainCommsCatch(args.shell, args.deployed, 0, args.requestTimeoutMs);
+    if (sends.length === 0)
+        return { count: 0, rendered: false };
+    if (args.sinceCount !== undefined && sends.length <= args.sinceCount)
+        return { count: sends.length, rendered: false };
+    const messages = await inboxMessagesFrom(sends, args.recipients);
+    if (messages.length === 0)
+        return { count: sends.length, rendered: false };
+    await writeInboxSurface(args.shell, args.deployed.surfaceDir, messages, {
+        recipients: args.recipients.map((recipient) => recipient.address),
+        ...(args.originMap === undefined ? {} : { originMap: args.originMap }),
+        ...(args.requestTimeoutMs === undefined ? {} : { requestTimeoutMs: args.requestTimeoutMs }),
+    });
+    return { count: sends.length, rendered: true };
+}

package/dist/comms/setup.d.ts

@@ -0,0 +1,65 @@
+import { type KeyResolutionDeps } from "../keys/key-resolution.js";
+import { COMMS_PROVIDERS, type ReceivingProviderId } from "./providers.js";
+import { type ReceivingAdapter } from "./receiving-types.js";
+export interface CommsCheckResult {
+    schema: "humanish.comms-check.v1";
+    ok: boolean;
+    connection: string;
+    online: boolean;
+    credentialPresent: boolean;
+    authenticated: boolean | null;
+    ready: boolean | null;
+    permissions: "unknown";
+    capacity: "unknown";
+    checkedAt: string | null;
+    code: string;
+    message: string;
+}
+export interface CommsSetupStatus {
+    authentication?: CommsCheckResult;
+    schema: "humanish.comms-setup.v1";
+    ok: boolean;
+    configPath: string;
+    providers: typeof COMMS_PROVIDERS;
+    connections: {
+        name: string;
+        provider: ReceivingProviderId;
+        apiKeyEnv: string;
+    }[];
+    credential: {
+        present: boolean;
+        source: string | null;
+        stored: boolean;
+        strict: boolean;
+        explicitlyEmpty: boolean;
+    };
+    message: string;
+}
+export declare function checkCommsConnection(args: {
+    cwd: string;
+    connection?: string;
+    env: NodeJS.ProcessEnv;
+    online?: boolean;
+    makeAdapter?: (apiKey: string) => ReceivingAdapter;
+    /** Where key discovery looks for the vendor stores; tests point it at a temp home. */
+    keyDeps?: KeyResolutionDeps;
+}): Promise<CommsCheckResult>;
+export declare function receivingRequiredKey(cwd: string, connectionName: string): Promise<string | null>;
+export interface CommsConfigureResult {
+    schema: "humanish.comms-configure.v1";
+    ok: boolean;
+    applied: boolean;
+    message: string;
+    path?: string;
+    planToken?: string;
+    connection?: string;
+}
+/** Save a local copy, never rewrite a committed manifest or silently shadow it by handle. */
+export declare function configureCommsLab(args: {
+    cwd: string;
+    lab: string;
+    connection: string;
+    apply?: boolean;
+    planToken?: string;
+}): Promise<CommsConfigureResult>;
+export declare function readCommsSetup(cwd: string, env: NodeJS.ProcessEnv, deps?: KeyResolutionDeps): Promise<CommsSetupStatus>;