@tangle-network/agent-app 0.44.2 → 0.44.4

package/dist/turn-health/index.d.ts

@@ -0,0 +1,398 @@
+/**
+ * The classifier for turns that FAIL BY RETURNING SUCCESS.
+ *
+ * Every failure this module names shipped to a customer with HTTP 200, no
+ * thrown error, and no log line anyone read. Three were measured in production
+ * in a single week:
+ *
+ *   - a turn settled `{"outcome":{"type":"completed"},"finalText":"",
+ *     "tokenUsage":{"outputTokens":0}}` — the customer saw a blank bubble;
+ *   - six `submit_proposal` tool calls collapsed into ONE whose arguments were
+ *     a 1,652-character non-JSON string, so zero proposals persisted and
+ *     nothing errored (agent-runtime #626);
+ *   - a thread took 255 user messages over 17 days and produced 2 replies,
+ *     both of them error text.
+ *
+ * A conventional health check cannot see any of these, because it probes
+ * DEPENDENCIES (is the sandbox reachable, is the router up) and every one of
+ * these failures happens with all dependencies green. This classifier probes
+ * the OUTCOME instead.
+ *
+ * It is deliberately pure and structural: it reads a settled turn's own
+ * projection, so the SAME function judges a live turn through the
+ * `/chat-routes` lifecycle seam and a historical row read back out of the
+ * store during a sweep. One definition of "silently broken", two call sites.
+ */
+/** How loudly a reason should be routed. `critical` means a customer got
+ *  nothing usable; `warning` means the turn degraded but still produced
+ *  something a human could read. */
+type TurnHealthSeverity = 'critical' | 'warning';
+/** One specific way a turn returned success while failing.
+ *
+ *  Each variant carries the evidence that identified it, so an alert can name
+ *  the offending value instead of asserting a verdict the reader has to take
+ *  on faith. */
+type TurnHealthReason = 
+/** Settled without error and produced nothing a user can read: no text, and
+ *  no artifact part (file/image/work-product/plan/interaction). This is the
+ *  verbatim blank-completion capture. */
+{
+    kind: 'empty_completion';
+    outputTokens: number | null;
+    partCount: number;
+    durationMs?: number;
+}
+/** A tool call whose arguments never parsed. The engine surfaces unparseable
+ *  arguments as a RAW STRING rather than throwing, so the call is neither
+ *  dropped nor errored — it silently does nothing. Detecting a string-typed
+ *  tool input that fails `JSON.parse` is the exact fingerprint of the
+ *  index-less parallel-tool-call collapse. */
+ | {
+    kind: 'malformed_tool_call';
+    tool: string;
+    inputLength: number;
+    /** Leading characters of the offending input, for the alert body. */
+    sample: string;
+}
+/** A tool call that never reached a terminal state carrying output. The call
+ *  was issued and then simply produced no effect. */
+ | {
+    kind: 'tool_call_no_effect';
+    tool: string;
+    status: string;
+}
+/** The turn failed outright. Not silent by itself — but it becomes silent
+ *  the moment nothing is watching, which is how 16 days of
+ *  `TANGLE_HUB_URL is required` reached customers unnoticed. */
+ | {
+    kind: 'turn_failed';
+    reason: string;
+};
+/** A settled turn, in the narrowest shape both call sites can supply.
+ *
+ *  Structural on purpose: the lifecycle seam supplies `finalText`/`usage`, a
+ *  store sweep supplies `content`/`parts` read back from a row, and neither
+ *  has to import the other's types. */
+interface TurnOutcomeInput {
+    /** The turn's final assistant text. */
+    finalText?: string | null;
+    /** The persisted assistant parts. Untyped by design — a sweep reads these
+     *  out of a JSON column and must not be forced to validate them first. */
+    parts?: readonly unknown[] | null;
+    /** Output tokens, when the caller has usage. `null`/absent is unknown, which
+     *  is NOT the same as zero and is never treated as evidence. */
+    outputTokens?: number | null;
+    /** Set when the turn surfaced a terminal error event. */
+    failed?: boolean;
+    failureReason?: string | null;
+    durationMs?: number;
+}
+/** The verdict for one turn. `healthy` is exactly `reasons.length === 0`, kept
+ *  as a field so callers read intent rather than an array length. */
+interface TurnHealthVerdict {
+    healthy: boolean;
+    severity: TurnHealthSeverity | null;
+    reasons: TurnHealthReason[];
+}
+/**
+ * Judge one settled turn.
+ *
+ * Never throws: a malformed `parts` blob is a thing this function REPORTS on,
+ * so it must not be a thing it dies on. Telemetry that can crash the turn it
+ * measures is worse than no telemetry.
+ */
+declare function classifyTurnOutcome(input: TurnOutcomeInput): TurnHealthVerdict;
+/** One-line human summary of a reason, for an alert body. */
+declare function describeReason(reason: TurnHealthReason): string;
+
+/**
+ * Where a silent-failure verdict GOES.
+ *
+ * The detection half is worthless without this half. Every failure this module
+ * finds was already visible in the database the whole time — 255 unanswered
+ * messages sat in a table for 17 days. What was missing was not the data, it
+ * was delivery to a human who had not thought to look.
+ *
+ * So the sink is a seam, not a channel: the product supplies the transport,
+ * and agent-app ships the two shapes the fleet already has credentials for
+ * (an ops webhook, and stderr). No new channel is invented here.
+ */
+
+/** One deliverable alert. */
+interface TurnHealthAlert {
+    /** Which product raised it (`projectId`). Alerts from four products land in
+     *  one channel, so this is what makes the message actionable. */
+    product: string;
+    severity: TurnHealthSeverity;
+    /** Stable grouping key. Throttling is keyed on this, so it must NOT contain
+     *  a turn id or a timestamp or every alert is unique and nothing dedupes. */
+    key: string;
+    title: string;
+    /** Human-readable lines. */
+    details: string[];
+    /** Structured payload for a machine consumer. */
+    data?: Record<string, unknown>;
+    at: number;
+}
+/** Deliver an alert. Implementations MUST NOT throw — see
+ *  {@link createGuardedAlertSink}. */
+interface AlertSink {
+    deliver(alert: TurnHealthAlert): Promise<void>;
+}
+/** Build the alert for a set of reasons found on one turn. */
+declare function turnAlert(input: {
+    product: string;
+    severity: TurnHealthSeverity;
+    reasons: TurnHealthReason[];
+    threadId?: string;
+    turnId?: string;
+    model?: string;
+    at?: number;
+}): TurnHealthAlert;
+/** Minimal structural fetch, so this module has no lib-dom dependency and can
+ *  be driven by a fake in tests. */
+type FetchLike = (url: string, init: {
+    method: string;
+    headers: Record<string, string>;
+    body: string;
+}) => Promise<{
+    ok: boolean;
+    status: number;
+    text?(): Promise<string>;
+}>;
+/**
+ * POST to an incoming webhook in the Slack message format.
+ *
+ * Chosen because the org already runs one (`SLACK_OPS_WEBHOOK_URL`) and
+ * gtm-agent's outbound webhook code already speaks this exact shape — the
+ * instruction was to route somewhere humans already look, not to stand up a
+ * new channel. Discord and most log drains accept the same `{text}` body.
+ */
+declare function createWebhookAlertSink(options: {
+    webhookUrl: string;
+    fetchImpl?: FetchLike;
+}): AlertSink;
+/** stderr sink. The zero-config fallback so a product that has not yet been
+ *  given a webhook still emits something a log search can find. */
+declare function createConsoleAlertSink(log?: (message: string) => void): AlertSink;
+/** Fan out to several sinks. One failing transport must not stop the others. */
+declare function createMultiAlertSink(sinks: readonly AlertSink[]): AlertSink;
+/** Records the last time a key was alerted on. A product backs this with KV,
+ *  D1, or a Durable Object; the in-memory default is correct for a sweep that
+ *  runs as a single cron invocation. */
+interface AlertThrottleStore {
+    lastSentAt(key: string): Promise<number | null>;
+    markSent(key: string, at: number): Promise<void>;
+}
+/** Process-local throttle store. */
+declare function createMemoryThrottleStore(): AlertThrottleStore;
+/**
+ * Collapse repeats of the same `key` inside `windowMs`.
+ *
+ * Deliberately re-alerts once per window rather than going silent after the
+ * first: an incident that is still burning must keep saying so. Going quiet
+ * after one message is how a 17-day outage stays invisible after someone
+ * dismisses the first notification.
+ */
+declare function createThrottledAlertSink(inner: AlertSink, options: {
+    windowMs: number;
+    store?: AlertThrottleStore;
+}): AlertSink;
+/**
+ * Swallow transport errors so telemetry can never fail the turn it measures.
+ *
+ * Use this at the LIVE lifecycle call site only. A sweep should let the error
+ * surface, because a sweep that cannot deliver has done nothing at all and its
+ * cron run should go red.
+ */
+declare function createGuardedAlertSink(inner: AlertSink, onError?: (error: unknown) => void): AlertSink;
+
+/**
+ * The LIVE half: judge each turn the moment it settles.
+ *
+ * This adds no control flow. `createChatTurnRoutes` already exposes a
+ * `lifecycle` seam that fires exactly one of `onTurnComplete`/`onTurnError`
+ * after a turn settles, and already swallows hook errors so telemetry cannot
+ * fail a turn. That seam was shipped and then wired by nobody, which is a fair
+ * description of why the outage lasted 17 days. This function fills it.
+ *
+ * The shape is declared structurally rather than imported from
+ * `/chat-routes`, so `/turn-health` stays free of the server chat vertical and
+ * can be used by any turn driver that reports the same three moments.
+ */
+
+/** Structural mirror of `/chat-routes`' `ChatTurnLifecycle` complete payload. */
+interface TurnHealthCompleteInfo {
+    finalText: string;
+    usage?: {
+        outputTokens?: number | null;
+    } | null;
+    durationMs: number;
+    threadId?: string;
+    turnStreamId?: string;
+    executionId?: string;
+}
+/** Structural mirror of the lifecycle error payload. */
+interface TurnHealthErrorInfo {
+    error: unknown;
+    durationMs: number;
+    threadId?: string;
+    turnStreamId?: string;
+    executionId?: string;
+}
+/** What {@link createTurnHealthLifecycle} returns — assignable to
+ *  `createChatTurnRoutes`' `lifecycle` option. */
+interface TurnHealthLifecycle {
+    onTurnComplete(info: TurnHealthCompleteInfo): Promise<void>;
+    onTurnError(info: TurnHealthErrorInfo): Promise<void>;
+}
+interface TurnHealthLifecycleOptions {
+    /** Names the product in every alert. */
+    product: string;
+    sink: AlertSink;
+    /** Called for every verdict, healthy or not — the hook for a counter or a
+     *  metrics push. Alerts are for humans; this is for graphs. */
+    onVerdict?(verdict: {
+        product: string;
+        healthy: boolean;
+        kinds: string[];
+        durationMs: number;
+    }): void;
+}
+/**
+ * Build the lifecycle hooks that page on a turn which succeeded at nothing.
+ *
+ * The live lane sees `finalText` and usage but not the persisted parts, so it
+ * catches the blank-completion and hard-failure shapes immediately. The
+ * parts-dependent shapes (a tool call whose arguments never parsed, a tool
+ * call that left no effect) are caught by {@link sweepSilentFailures}, which
+ * reads what was actually written to the store — the honest place to ask
+ * whether an effect persisted.
+ */
+declare function createTurnHealthLifecycle(options: TurnHealthLifecycleOptions): TurnHealthLifecycle;
+
+/**
+ * The SWEEP half: ask the store what it has been quietly accumulating.
+ *
+ * A live per-turn hook cannot see the failure that matters most, because the
+ * worst outage produced NO turns at all to hook: gtm-agent took 9–21 user
+ * messages a day for sixteen straight days and wrote zero real assistant
+ * replies. Nothing crashed on a schedule; the product simply stopped
+ * answering. The only thing that could have noticed is something that
+ * periodically counts what arrived against what was answered.
+ *
+ * That is this. It runs on a cron, reads the shared `/chat-store` schema, and
+ * pages when the ratio breaks.
+ *
+ * The queries live HERE and not in each product because all four products
+ * (gtm, tax, legal, workcomp) persist to the same `message`/`thread` tables —
+ * four copies of this cron is exactly the duplication the repo's engine/shell
+ * rule exists to prevent.
+ */
+
+/** A thread that has taken user messages with no reply since. */
+interface UnansweredThread {
+    threadId: string;
+    /** User messages newer than the newest real assistant reply. */
+    pendingMessages: number;
+    /** Age of the OLDEST unanswered user message, in ms. */
+    oldestAgeMs: number;
+}
+/** A persisted assistant row, as the sweep needs to judge it. */
+interface PersistedTurnRow {
+    id: string;
+    threadId: string;
+    content: string;
+    /** Raw `parts` column. A JSON string or an already-parsed array; the sweep
+     *  accepts both because D1 drivers differ. */
+    parts: unknown;
+    outputTokens?: number | null;
+    model?: string | null;
+    createdAt: number;
+}
+/** What the sweep needs from a store. A product on a non-standard schema
+ *  implements these two reads; everything else is shared. */
+interface TurnHealthSource {
+    findUnansweredThreads(input: {
+        minAgeMs: number;
+        now: number;
+    }): Promise<UnansweredThread[]>;
+    listRecentAssistantTurns(input: {
+        sinceMs: number;
+        now: number;
+        limit: number;
+    }): Promise<PersistedTurnRow[]>;
+}
+interface SweepOptions {
+    product: string;
+    source: TurnHealthSource;
+    sink: AlertSink;
+    /** A user message must go unanswered this long before it counts. Guards
+     *  against alerting on a turn that is simply still streaming. Default 15 min. */
+    minAgeMs?: number;
+    /** How far back to judge settled turns. Default 24 h. */
+    lookbackMs?: number;
+    /** Cap on rows judged per sweep. Default 500. */
+    limit?: number;
+    /** Fraction of recent turns allowed to be silently broken before paging.
+     *  Default 0.05 — the measured blank-completion rate on the tax tool surface
+     *  was 12.2%, so 5% separates a real regression from noise. */
+    emptyRateThreshold?: number;
+    /** Absolute floor: never page on a rate computed from fewer turns than this. */
+    minTurnsForRate?: number;
+    now?: number;
+}
+/** What the sweep found. Returned as well as alerted, so a cron can log it and
+ *  a test can assert on it. */
+interface SweepResult {
+    product: string;
+    unansweredThreads: number;
+    pendingUserMessages: number;
+    oldestUnansweredMs: number;
+    turnsJudged: number;
+    unhealthyTurns: number;
+    emptyCompletions: number;
+    malformedToolCalls: number;
+    toolCallsWithoutEffect: number;
+    alerts: TurnHealthAlert[];
+}
+/**
+ * Run one sweep and deliver whatever it finds.
+ *
+ * Errors from the sink are NOT swallowed here (unlike the live lane): a sweep
+ * that could not deliver has accomplished nothing, and its cron invocation
+ * should go red rather than report a clean run.
+ */
+declare function sweepSilentFailures(options: SweepOptions): Promise<SweepResult>;
+/** Minimal structural D1 contract (Cloudflare's `D1Database` satisfies it). */
+interface D1LikeForHealth {
+    prepare(sql: string): {
+        bind(...values: unknown[]): {
+            all<T = Record<string, unknown>>(): Promise<{
+                results: T[];
+            }>;
+        };
+    };
+}
+/**
+ * The sweep source for products on the canonical `/chat-store` tables.
+ *
+ * "Answered" deliberately means an assistant row with NON-EMPTY content. A
+ * blank assistant row is what a broken turn writes, so counting it as an
+ * answer would let the exact failure being hunted mark itself resolved. That
+ * single predicate is the difference between this catching the gtm outage and
+ * sleeping through it — during those sixteen days the table was NOT empty.
+ *
+ * The query deliberately does NOT join the thread table. Products do not all
+ * keep one: tax-agent's `thread` table holds zero rows because it groups by
+ * its own `tax_sessions`, and an inner join against it silently reported "0
+ * unanswered threads, healthy" while 18 real messages sat unanswered. A
+ * detector that reports healthy because its join found nothing is the same
+ * bug class it was built to catch.
+ */
+declare function createD1TurnHealthSource(db: D1LikeForHealth, options?: {
+    messageTable?: string;
+    threadTable?: string;
+}): TurnHealthSource;
+
+export { type AlertSink, type AlertThrottleStore, type D1LikeForHealth, type FetchLike, type PersistedTurnRow, type SweepOptions, type SweepResult, type TurnHealthAlert, type TurnHealthCompleteInfo, type TurnHealthErrorInfo, type TurnHealthLifecycle, type TurnHealthLifecycleOptions, type TurnHealthReason, type TurnHealthSeverity, type TurnHealthSource, type TurnHealthVerdict, type TurnOutcomeInput, type UnansweredThread, classifyTurnOutcome, createConsoleAlertSink, createD1TurnHealthSource, createGuardedAlertSink, createMemoryThrottleStore, createMultiAlertSink, createThrottledAlertSink, createTurnHealthLifecycle, createWebhookAlertSink, describeReason, sweepSilentFailures, turnAlert };