@agenthippo-ai/agenthippo 0.2.3-dataplane → 1.110.11

package/out/evidence/sessionShip.js

@@ -0,0 +1,643 @@
+"use strict";
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) AgentHippo.ai. All rights reserved.
+ *  Licensed under the MIT License. See LICENSE.txt in the project root for license information.
+ *  In-process session evidence shipper — drains Layer 2 JSONL to evidence-server.
+ *--------------------------------------------------------------------------------------------*/
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getEvidenceShipStatus = getEvidenceShipStatus;
+exports.validateEvidenceUrl = validateEvidenceUrl;
+exports.resolveSessionShipConfig = resolveSessionShipConfig;
+exports.redactSessionLine = redactSessionLine;
+exports.startSessionShipper = startSessionShipper;
+const path = __importStar(require("path"));
+const fs_1 = require("fs");
+const crypto_1 = require("crypto");
+const zlib_1 = require("zlib");
+const identity_1 = require("../analytics/identity");
+const sessionEvents_1 = require("../analytics/sessionEvents");
+const DISABLE_ENV = 'AGENTHIPPO_EVIDENCE_SHIP';
+const URL_ENV = 'AGENTHIPPO_EVIDENCE_URL';
+const TOKEN_ENV = 'AGENTHIPPO_EVIDENCE_SHIP_TOKEN';
+const INTERVAL_ENV = 'AGENTHIPPO_EVIDENCE_SHIP_INTERVAL_SEC';
+const SINCE_DAYS_ENV = 'AGENTHIPPO_EVIDENCE_SHIP_SINCE_DAYS';
+const DRAIN_TIMEOUT_ENV = 'AGENTHIPPO_EVIDENCE_SHIP_DRAIN_TIMEOUT_SEC';
+const ALLOW_INSECURE_ENV = 'AGENTHIPPO_EVIDENCE_ALLOW_INSECURE_URL';
+/**
+ * Default ship window, in days.
+ *
+ * Was hard-coded to 7. An outage longer than the window makes the backlog PERMANENTLY
+ * unshippable — the shipper skips those files forever, so a two-week incident silently deletes
+ * the two weeks of evidence that matter most. 30 days is a working default; set
+ * AGENTHIPPO_EVIDENCE_SHIP_SINCE_DAYS higher for a longer guaranteed recovery window.
+ */
+const DEFAULT_SINCE_DAYS = 30;
+/** Was 5s, which is shorter than a single in-flight ship attempt with its retries. */
+const DEFAULT_DRAIN_TIMEOUT_SEC = 30;
+/** Session-event payload keys that can carry prompt/tool CONTENT rather than metadata. */
+const CONTENT_KEY_RE = /(^|_)(input|output|result|prompt|completion|content|text|messages?)(_|$)|preview$/i;
+let lastStatus = {
+    enabled: false,
+    disabledReason: 'not-configured',
+    consecutiveFailures: 0,
+    recordsStaged: 0,
+    recordsShipped: 0,
+    recordsSkippedTooOld: 0,
+    filesSkippedTooOld: 0,
+    writeFailures: 0,
+    contentCapture: false,
+};
+/**
+ * Evidence shipping status for /health.
+ *
+ * Deliberately available even when shipping is OFF: "disabled" must be a visible state, not the
+ * absence of one, or a deployment that silently stopped shipping looks identical to a quiet one.
+ */
+function getEvidenceShipStatus() {
+    return { ...lastStatus, writeFailures: (0, sessionEvents_1.getSessionEventWriterStats)().failed };
+}
+function setStatus(patch) {
+    lastStatus = { ...lastStatus, ...patch };
+}
+function isTruthy(value) {
+    return ['1', 'true', 'yes', 'on'].includes((value ?? '').trim().toLowerCase());
+}
+/** evidence-ship reads EVIDENCE_ALLOW_INSECURE_URL; serve documents AGENTHIPPO_EVIDENCE_ALLOW_INSECURE_URL. */
+function resolveAllowInsecure(env) {
+    return isTruthy(env[ALLOW_INSECURE_ENV]) || isTruthy(env.EVIDENCE_ALLOW_INSECURE_URL);
+}
+const LOOPBACK_HOSTS = new Set(['localhost', '127.0.0.1', '::1', '[::1]', '0.0.0.0']);
+/**
+ * Validate the ingest URL.
+ *
+ * This is a bearer token plus the organisation's audit trail; plaintext http on anything but
+ * loopback puts both on the wire. Private-network deployments (`http://evidence:8787` on a
+ * compose network) can opt back in with AGENTHIPPO_EVIDENCE_ALLOW_INSECURE_URL=1.
+ */
+function validateEvidenceUrl(raw, env = process.env) {
+    let parsed;
+    try {
+        parsed = new URL(raw);
+    }
+    catch {
+        return { error: `${URL_ENV} is not a valid URL: ${JSON.stringify(raw)}` };
+    }
+    if (parsed.protocol === 'https:') {
+        return { url: parsed.toString().replace(/\/$/, '') };
+    }
+    if (parsed.protocol !== 'http:') {
+        return { error: `${URL_ENV} must be https (got ${parsed.protocol.replace(':', '')})` };
+    }
+    if (LOOPBACK_HOSTS.has(parsed.hostname)) {
+        return { url: parsed.toString().replace(/\/$/, '') };
+    }
+    if (resolveAllowInsecure(env)) {
+        const reason = isTruthy(env[ALLOW_INSECURE_ENV])
+            ? ALLOW_INSECURE_ENV
+            : 'EVIDENCE_ALLOW_INSECURE_URL';
+        console.warn(`[evidence-ship] INSECURE: shipping evidence and the ingest token over plaintext http ` +
+            `to ${parsed.host} because ${reason} is set. Use https unless this network ` +
+            `is genuinely private.`);
+        return { url: parsed.toString().replace(/\/$/, '') };
+    }
+    return {
+        error: `${URL_ENV} must use https — plaintext http to ${parsed.host} would put the ingest ` +
+            `token and the audit trail on the wire. Use https, a loopback address, or set ` +
+            `${ALLOW_INSECURE_ENV}=1 (or EVIDENCE_ALLOW_INSECURE_URL=1) if the network is genuinely private.`,
+    };
+}
+/** Resolve config from env; returns undefined when shipping is disabled or misconfigured. */
+function resolveSessionShipConfig(instanceId, sessionsDir, stateDir, env = process.env) {
+    if (env[DISABLE_ENV] === '0' || env[DISABLE_ENV]?.toLowerCase() === 'false') {
+        // Disabling evidence shipping is legitimate, but it must never be quiet: this is the one
+        // setting that turns the compliance record off, and it was a single character in an env
+        // file that produced no output at all.
+        console.warn('\n' +
+            '  ############################################################\n' +
+            '  ##  EVIDENCE SHIPPING IS DISABLED                         ##\n' +
+            `  ##  ${DISABLE_ENV}=${env[DISABLE_ENV]}\n` +
+            '  ##  Session evidence stays on local disk and is lost when  ##\n' +
+            '  ##  this instance goes away. Unset to re-enable.           ##\n' +
+            '  ############################################################\n');
+        setStatus({ enabled: false, disabledReason: 'disabled-by-env' });
+        return undefined;
+    }
+    const rawUrl = env[URL_ENV]?.trim();
+    const shipToken = env[TOKEN_ENV]?.trim();
+    if (!rawUrl || !shipToken) {
+        // Half-configured is a mistake, not a choice: an operator who set one of the pair meant to
+        // ship. Silence here made "misconfigured" look exactly like "deliberately off" — and
+        // /health only grows evidence_ship_last_ok_at after the first SUCCESS, so nothing else
+        // distinguishes them either.
+        if (rawUrl || shipToken) {
+            const missing = rawUrl ? TOKEN_ENV : URL_ENV;
+            console.warn(`[evidence-ship] disabled — ${missing} is not set (the other half of the pair is)`);
+        }
+        setStatus({ enabled: false, disabledReason: 'not-configured' });
+        return undefined;
+    }
+    const checked = validateEvidenceUrl(rawUrl, env);
+    if ('error' in checked) {
+        console.error(`[evidence-ship] DISABLED — ${checked.error}`);
+        setStatus({ enabled: false, disabledReason: 'invalid-url' });
+        return undefined;
+    }
+    const intervalSec = Math.max(15, Number(env[INTERVAL_ENV] ?? 60) || 60);
+    const sinceDays = Math.max(1, Number(env[SINCE_DAYS_ENV] ?? DEFAULT_SINCE_DAYS) || DEFAULT_SINCE_DAYS);
+    const drainTimeoutSec = Math.max(1, Number(env[DRAIN_TIMEOUT_ENV] ?? DEFAULT_DRAIN_TIMEOUT_SEC) || DEFAULT_DRAIN_TIMEOUT_SEC);
+    return {
+        evidenceUrl: checked.url,
+        shipToken,
+        instanceId,
+        sessionsDir,
+        stateDir,
+        intervalSec,
+        sinceDays,
+        drainTimeoutSec,
+        allowInsecure: resolveAllowInsecure(env),
+    };
+}
+// -----------------------------------------------------------------------------
+// Redaction (all lanes) — see fix note 3 at the top of this file
+// -----------------------------------------------------------------------------
+function digest(text) {
+    return {
+        sha256: (0, crypto_1.createHash)('sha256').update(text, 'utf8').digest('hex'),
+        length: Buffer.byteLength(text, 'utf8'),
+    };
+}
+/**
+ * Strip content-bearing payload fields from one session-event line, leaving a sha256 + length.
+ *
+ * Applied on EVERY lane. The upstream shipper only redacts when `source === 'seat'`, so serve —
+ * the multi-tenant lane, the one with other people's data in it — shipped raw. A line we cannot
+ * parse is passed through unchanged: it is already not a record we produced, and dropping it
+ * would be exactly the silent evidence loss this file exists to prevent.
+ */
+function redactSessionLine(line, captureContent) {
+    if (captureContent) {
+        return line;
+    }
+    let doc;
+    try {
+        doc = JSON.parse(line);
+    }
+    catch {
+        return line;
+    }
+    if (!doc || typeof doc !== 'object') {
+        return line;
+    }
+    const record = doc;
+    if (!record.payload || typeof record.payload !== 'object') {
+        return line;
+    }
+    const payload = record.payload;
+    let changed = false;
+    for (const key of Object.keys(payload)) {
+        if (!CONTENT_KEY_RE.test(key) || key.endsWith('_sha256') || key.endsWith('_length')) {
+            continue;
+        }
+        const value = payload[key];
+        if (value === undefined || value === null) {
+            continue;
+        }
+        const text = typeof value === 'string' ? value : JSON.stringify(value);
+        const d = digest(text);
+        delete payload[key];
+        payload[`${key}_sha256`] = d.sha256;
+        payload[`${key}_length`] = d.length;
+        payload[`${key}_redacted`] = true;
+        changed = true;
+    }
+    return changed ? JSON.stringify(record) : line;
+}
+function emptyCursor() {
+    return { version: 1, files: {}, pending: { staged: 0, shipped: 0, skippedRecords: 0, skippedFiles: 0 } };
+}
+function countRecords(text) {
+    let n = 0;
+    for (const line of text.split('\n')) {
+        if (line.trim()) {
+            n++;
+        }
+    }
+    return n;
+}
+/** Start in-process session shipper: one drain on boot, then interval loop. */
+function startSessionShipper(config) {
+    const cursorFile = path.join(config.stateDir, '.evidence-ship-cursor.json');
+    const stageCursorFile = path.join(config.stateDir, '.evidence-stage-cursor.json');
+    // Redacted copies live outside sessionsDir so the shipper never re-reads its own output and
+    // DuckDB queries keep pointing at the unredacted local store.
+    const stagingDir = path.join(config.stateDir, 'evidence-staging', 'sessions');
+    const log = config.logger?.info ?? (() => undefined);
+    const warn = config.logger?.warn ?? (() => undefined);
+    const sinceDays = config.sinceDays ?? DEFAULT_SINCE_DAYS;
+    const intervalSec = config.intervalSec ?? 60;
+    const drainTimeoutMs = (config.drainTimeoutSec ?? DEFAULT_DRAIN_TIMEOUT_SEC) * 1000;
+    const captureContent = (0, identity_1.isContentCaptureEnabled)();
+    const allowInsecure = config.allowInsecure === true;
+    let stopped = false;
+    let inFlight = false;
+    let lastOkAt;
+    let consecutiveFailures = 0;
+    let timer;
+    let shipperPromise;
+    let cursor;
+    /** Records staged but not yet confirmed shipped — what `flush()` reports as abandoned. */
+    let unshippedRecords = 0;
+    const totals = { staged: 0, shipped: 0, skippedRecords: 0, skippedFiles: 0 };
+    setStatus({
+        enabled: true,
+        disabledReason: undefined,
+        evidenceUrl: config.evidenceUrl,
+        intervalSec,
+        sinceDays,
+        consecutiveFailures: 0,
+        recordsStaged: 0,
+        recordsShipped: 0,
+        recordsSkippedTooOld: 0,
+        filesSkippedTooOld: 0,
+        contentCapture: captureContent,
+    });
+    const getShipper = () => {
+        if (!shipperPromise) {
+            shipperPromise = import('@agenthippo-ai/evidence-ship').then(({ createShipper }) => createShipper({
+                serverUrl: config.evidenceUrl,
+                token: config.shipToken,
+                instanceId: config.instanceId,
+                source: 'serve',
+                sinceDays,
+                cursorFile,
+                // evidence-ship >=0.2.0 strips content on every lane by default. serve already
+                // redacts before staging and that redaction is idempotent, so leaving this unset
+                // is safe — but it would also strip content on a deployment that deliberately
+                // opted in, turning the opt-in into a no-op. Pass our own decision through.
+                captureContent,
+                // Compose stacks use http://evidence:8787 on a private network. serve accepts that
+                // via AGENTHIPPO_EVIDENCE_ALLOW_INSECURE_URL; evidence-ship needs allowInsecure here
+                // (or EVIDENCE_ALLOW_INSECURE_URL in the process env).
+                allowInsecure,
+                // Heartbeat is posted here instead: the upstream one is the literal '{}', which
+                // proves the process was alive and nothing about whether records are complete.
+                heartbeat: false,
+                log: (m) => log(m),
+                error: (m) => warn(m),
+            }));
+        }
+        return shipperPromise;
+    };
+    const loadCursor = async () => {
+        if (cursor) {
+            return cursor;
+        }
+        try {
+            const raw = await fs_1.promises.readFile(stageCursorFile, 'utf8');
+            const parsed = JSON.parse(raw);
+            cursor = {
+                version: 1,
+                files: parsed.files ?? {},
+                windowFrom: parsed.windowFrom,
+                pending: parsed.pending ?? emptyCursor().pending,
+            };
+        }
+        catch {
+            cursor = emptyCursor();
+        }
+        return cursor;
+    };
+    const saveCursor = async () => {
+        if (!cursor) {
+            return;
+        }
+        try {
+            await fs_1.promises.mkdir(path.dirname(stageCursorFile), { recursive: true });
+            await fs_1.promises.writeFile(stageCursorFile, JSON.stringify(cursor, null, 1));
+        }
+        catch (err) {
+            warn(`[evidence-ship] could not persist stage cursor: ${err instanceof Error ? err.message : String(err)}`);
+        }
+    };
+    /**
+     * Redact + stage changed session files, and record permanent losses.
+     *
+     * Returns the number of records staged this pass. Staged files get a strictly increasing
+     * mtime so the upstream mtime-equality cursor always re-ships them; the decision about
+     * WHETHER to restage is ours and is made on content (size + sha256), which an editor that
+     * preserves mtime cannot fool.
+     */
+    const stage = async () => {
+        const state = await loadCursor();
+        const now = Date.now();
+        const sinceMs = now - sinceDays * 86400_000;
+        let staged = 0;
+        let names;
+        try {
+            names = (await fs_1.promises.readdir(config.sessionsDir)).filter((f) => f.endsWith('.jsonl'));
+        }
+        catch {
+            return 0; // no sessions dir yet
+        }
+        await fs_1.promises.mkdir(stagingDir, { recursive: true });
+        const losses = [];
+        for (const name of names) {
+            if (name.startsWith('_evidence-loss')) {
+                continue;
+            }
+            const src = path.join(config.sessionsDir, name);
+            let stat;
+            try {
+                stat = await fs_1.promises.stat(src);
+            }
+            catch {
+                continue;
+            }
+            if (stat.mtimeMs < sinceMs) {
+                // PERMANENT loss, not a deferral: the shipper will skip this file on every future
+                // tick, so these records are never going to arrive. Say so, once, loudly, and put
+                // a record of the loss into the evidence stream itself — a gap the server can see
+                // beats a gap only the disk knows about.
+                const entry = state.files[name];
+                if (!entry?.lostReported) {
+                    let records = 0;
+                    try {
+                        records = countRecords(await fs_1.promises.readFile(src, 'utf8'));
+                    }
+                    catch { /* size-only report below */ }
+                    totals.skippedFiles++;
+                    totals.skippedRecords += records;
+                    state.pending.skippedFiles++;
+                    state.pending.skippedRecords += records;
+                    state.files[name] = {
+                        size: stat.size,
+                        sha256: '',
+                        stagedAt: 0,
+                        stagedMtimeMs: entry?.stagedMtimeMs ?? 0,
+                        lostReported: true,
+                    };
+                    warn(`[evidence-ship] EVIDENCE PERMANENTLY SKIPPED: ${name} last changed ` +
+                        `${new Date(stat.mtimeMs).toISOString()}, older than the ${sinceDays}-day ship ` +
+                        `window — ${records} record(s) will never be shipped. Raise ` +
+                        `${SINCE_DAYS_ENV} before the next restart to recover a backlog this old.`);
+                    losses.push(JSON.stringify({
+                        session_id: '_evidence-loss',
+                        request_id: name,
+                        timestamp: now,
+                        event_type: 'evidence_skipped',
+                        payload: {
+                            reason: 'older_than_ship_window',
+                            file: name,
+                            file_mtime_ms: stat.mtimeMs,
+                            file_bytes: stat.size,
+                            records: records,
+                            since_days: sinceDays,
+                            instance_id: config.instanceId,
+                            permanent: true,
+                        },
+                    }));
+                }
+                continue;
+            }
+            let content;
+            try {
+                content = await fs_1.promises.readFile(src, 'utf8');
+            }
+            catch (err) {
+                warn(`[evidence-ship] cannot read ${name}: ${err instanceof Error ? err.message : String(err)}`);
+                continue;
+            }
+            const sha256 = (0, crypto_1.createHash)('sha256').update(content, 'utf8').digest('hex');
+            const prev = state.files[name];
+            if (prev && prev.sha256 === sha256 && prev.size === content.length) {
+                continue; // genuinely unchanged — content, not a timestamp, says so
+            }
+            const redacted = content
+                .split('\n')
+                .map((line) => (line.trim() ? redactSessionLine(line, captureContent) : line))
+                .join('\n');
+            const dest = path.join(stagingDir, name);
+            await fs_1.promises.writeFile(dest, redacted);
+            // Strictly increasing: the upstream cursor skips a file whose mtime it has already
+            // seen, so a same-millisecond restage would otherwise be dropped.
+            const stagedMtimeMs = Math.max(Date.now(), (prev?.stagedMtimeMs ?? 0) + 1000);
+            const seconds = stagedMtimeMs / 1000;
+            await fs_1.promises.utimes(dest, seconds, seconds);
+            const records = countRecords(redacted);
+            staged += records;
+            state.files[name] = { size: content.length, sha256, stagedAt: now, stagedMtimeMs };
+        }
+        if (losses.length > 0) {
+            // Appended, never truncated: a loss record must outlive the tick that noticed it.
+            //
+            // The NAME matters and cannot be `_evidence-loss.jsonl`: @agenthippo-ai/evidence-ship's
+            // shipSessions() skips every file whose name starts with `_evidence-loss` (it posts its
+            // OWN losses under that name from its cursor, and refuses to re-ingest one found on
+            // disk). Staging ours under that name made it a dead letter — the warning printed, the
+            // counters incremented, and the record never left the machine, which is precisely the
+            // silent gap this feature exists to close.
+            const lossFile = path.join(stagingDir, 'evidence-loss.jsonl');
+            await fs_1.promises.appendFile(lossFile, `${losses.join('\n')}\n`);
+            const seconds = Date.now() / 1000;
+            await fs_1.promises.utimes(lossFile, seconds, seconds);
+            staged += losses.length;
+        }
+        totals.staged += staged;
+        state.pending.staged += staged;
+        if (state.windowFrom === undefined) {
+            state.windowFrom = now;
+        }
+        await saveCursor();
+        return staged;
+    };
+    /**
+     * Heartbeat with substance (R-13).
+     *
+     * The upstream shipper posts the literal `'{}'`. That proves a process was alive; it says
+     * nothing about whether the window it covers is complete, which is the entire question gap
+     * detection asks. Window bounds plus counts let the server spot both a missing interval and an
+     * interval that shipped fewer records than were produced.
+     */
+    const postHeartbeat = async () => {
+        const state = await loadCursor();
+        const now = Date.now();
+        const writeStats = (0, sessionEvents_1.getSessionEventWriterStats)();
+        const body = {
+            kind: 'session_ship_heartbeat',
+            instance_id: config.instanceId,
+            wire: 'agenthippo/session-ship/1',
+            window_from_ms: state.windowFrom ?? now,
+            window_to_ms: now,
+            interval_sec: intervalSec,
+            since_days: sinceDays,
+            records_staged: state.pending.staged,
+            records_shipped: state.pending.shipped,
+            records_skipped_too_old: state.pending.skippedRecords,
+            files_skipped_too_old: state.pending.skippedFiles,
+            evidence_write_failures: writeStats.failed,
+            evidence_writes: writeStats.written,
+            consecutive_ship_failures: consecutiveFailures,
+            content_capture: captureContent,
+            totals: { ...totals },
+        };
+        let INGEST_WIRE_VERSION = 1;
+        try {
+            ({ INGEST_WIRE_VERSION } = await import('@agenthippo-ai/evidence-ship'));
+        }
+        catch { /* keep the default */ }
+        const text = JSON.stringify(body);
+        const res = await fetch(`${config.evidenceUrl.replace(/\/$/, '')}/ingest/batch`, {
+            method: 'POST',
+            headers: {
+                authorization: `Bearer ${config.shipToken}`,
+                'content-type': 'application/json',
+                'X-Evidence-Ingest-Version': String(INGEST_WIRE_VERSION),
+            },
+            body: JSON.stringify({
+                instance_id: config.instanceId,
+                kind: 'heartbeat',
+                name: 'heartbeat.json',
+                content_gzip_b64: (0, zlib_1.gzipSync)(Buffer.from(text)).toString('base64'),
+            }),
+        });
+        if (!res.ok) {
+            throw new Error(`heartbeat HTTP ${res.status}`);
+        }
+        // Only now is the window closed: a failed heartbeat must keep accumulating, or the counts
+        // for the lost interval disappear along with it.
+        state.windowFrom = now;
+        state.pending = { staged: 0, shipped: 0, skippedRecords: 0, skippedFiles: 0 };
+        setStatus({ lastHeartbeatAt: now });
+        await saveCursor();
+        return true;
+    };
+    const tick = async () => {
+        if (stopped || inFlight) {
+            return;
+        }
+        inFlight = true;
+        try {
+            const staged = await stage();
+            unshippedRecords += staged;
+            try {
+                const shipper = await getShipper();
+                const ok = await shipper.drain({ sessions: stagingDir, heartbeat: false });
+                if (ok) {
+                    const state = await loadCursor();
+                    state.pending.shipped += unshippedRecords;
+                    totals.shipped += unshippedRecords;
+                    unshippedRecords = 0;
+                    lastOkAt = Date.now();
+                    consecutiveFailures = 0;
+                }
+                else {
+                    consecutiveFailures++;
+                    warn(`[evidence-ship] drain failed (${consecutiveFailures} in a row); ` +
+                        `${unshippedRecords} record(s) still unshipped`);
+                }
+            }
+            catch (shipErr) {
+                consecutiveFailures++;
+                warn(`[evidence-ship] ${shipErr instanceof Error ? shipErr.message : String(shipErr)}`);
+            }
+        }
+        catch (err) {
+            consecutiveFailures++;
+            warn(`[evidence-ship] ${err instanceof Error ? err.message : String(err)}`);
+        }
+        finally {
+            // R-13: heartbeat runs every tick even when drain/shipper fails, so gap detection
+            // sees window bounds and staged-vs-shipped counts instead of silence.
+            try {
+                await postHeartbeat();
+            }
+            catch (hbErr) {
+                warn(`[evidence-ship] heartbeat failed: ${hbErr instanceof Error ? hbErr.message : String(hbErr)}`);
+            }
+            inFlight = false;
+            setStatus({
+                lastOkAt,
+                consecutiveFailures,
+                recordsStaged: totals.staged,
+                recordsShipped: totals.shipped,
+                recordsSkippedTooOld: totals.skippedRecords,
+                filesSkippedTooOld: totals.skippedFiles,
+            });
+        }
+    };
+    void tick();
+    timer = setInterval(() => void tick(), intervalSec * 1000);
+    timer.unref?.();
+    log(`[evidence-ship] enabled → ${config.evidenceUrl} (every ${intervalSec}s, ` +
+        `${sinceDays}-day window, content capture ${captureContent ? 'ON' : 'off'})`);
+    const stop = () => {
+        stopped = true;
+        if (timer) {
+            clearInterval(timer);
+        }
+    };
+    return {
+        stop,
+        async flush() {
+            stop();
+            // A tick already running holds the cursor; let it finish rather than racing it, then
+            // ship the tail. `stopped` is cleared only for this final pass.
+            const deadline = Date.now() + drainTimeoutMs;
+            while (inFlight && Date.now() < deadline) {
+                await new Promise((r) => setTimeout(r, 50));
+            }
+            if (inFlight) {
+                warn(`[evidence-ship] FINAL DRAIN ABANDONED after ${Math.round(drainTimeoutMs / 1000)}s ` +
+                    `— a ship was still in flight. ${unshippedRecords} staged record(s) plus anything ` +
+                    `written since are lost if this state dir is ephemeral. Raise ` +
+                    `${DRAIN_TIMEOUT_ENV} if this recurs.`);
+                return;
+            }
+            stopped = false;
+            try {
+                await tick();
+            }
+            finally {
+                stopped = true;
+            }
+            if (unshippedRecords > 0) {
+                warn(`[evidence-ship] final drain finished with ${unshippedRecords} record(s) still unshipped`);
+            }
+        },
+        lastOkAt: () => lastOkAt,
+        status: () => getEvidenceShipStatus(),
+    };
+}

package/out/extension.js

@@ -272,6 +272,15 @@ async function activate(context) {
     const config = {
         storagePath: storageBase.fsPath,
     };
+    // Spotlight setup generates the local proxy's key into ~/.agent-hippo/.env (already merged
+    // into process.env above). Adopt it so a first-run user never has to copy a key by hand;
+    // a key the user enters in Settings arrives later via setParticipantApiKey and wins.
+    const generatedLiteLLMKey = process.env.AGENTHIPPO_LITELLM_API_KEY?.trim() ||
+        process.env.LITELLM_MASTER_KEY?.trim();
+    if (generatedLiteLLMKey) {
+        config.litellmApiKey = generatedLiteLLMKey;
+        log(outputChannel, "[AgentIDE] Using local LiteLLM key from ~/.agent-hippo/.env");
+    }
     for (const engine of Object.values(types_1.AGENT_ENGINES)) {
         const dirPath = path.join(storageBase.fsPath, engine.id);
         try {
@@ -842,7 +851,7 @@ function registerCommands(context, state) {
         const configuredCliPath = cliConfig.get("path", "");
         const installSource = cliConfig.get("installSource", "").trim();
         if (installSource && /[;`|$&\n\r"']/.test(installSource)) {
-            showError(`${branding_1.AGENTIDE_BRAND_NAME}: Invalid CLI install source. Remove shell control characters from 'agentide.cli.installSource'.`);
+            showError(`${branding_1.AGENTIDE_BRAND_NAME}: Invalid CLI install source. Remove shell control characters from '${(0, branding_1.describeSettingKey)("agentide.cli.installSource")}'.`);
             return { success: false, error: "Invalid install source" };
         }
         const health = await (0, runtime_1.getCliHealth)({
@@ -873,7 +882,7 @@ function registerCommands(context, state) {
             }
         });
         if (!scriptPath) {
-            showError(`${branding_1.AGENTIDE_BRAND_NAME}: CLI setup script not found. Build CLI via extensions/agentide/scripts/build-cli.sh and set agentide.cli.path.`);
+            showError(`${branding_1.AGENTIDE_BRAND_NAME}: CLI setup script not found. Build CLI via extensions/agentide/scripts/build-cli.sh and set ${(0, branding_1.describeSettingKey)("agentide.cli.path")}.`);
             return { success: false, error: "Script not found" };
         }
         const setupResult = await runCliSetup(scriptPath, outputChannel, installSource);