@agent-native/core 0.168.13 → 0.169.0

package/dist/jobs/background-automation-runner.js

@@ -2,7 +2,7 @@ import { collectFinalResponseTextFromAgentEvents } from "../a2a/response-text.js
 import { getStoredModelForEngine, normalizeModelForEngine, resolveEngine, } from "../agent/engine/index.js";
 import { actionsToEngineTools, filterInitialEngineTools, getOwnerActiveApiKey, } from "../agent/production-agent.js";
 import { runAgentLoopDirectWithSoftTimeout } from "../agent/run-loop-with-resume.js";
-import { resolveRunSoftTimeoutMs, startRun, } from "../agent/run-manager.js";
+import { abortRun, resolveBackgroundAutomationSoftTimeoutMs, resolveBackgroundRunHardTimeoutMs, startRun, } from "../agent/run-manager.js";
 import { claimBackgroundRun, insertRun } from "../agent/run-store.js";
 import { buildAssistantMessage, buildUserMessage, extractThreadMeta, foldAssistantTurn, upsertUserMessage, } from "../agent/thread-data-builder.js";
 import { attachToolSearch } from "../agent/tool-search.js";
@@ -13,8 +13,27 @@ import { organizationIdFromResourceOwner, organizationResourceOwner, } from "../
 import { captureError } from "../server/capture-error.js";
 import { runWithRequestContext, } from "../server/request-context.js";
 import { attachAutomationRunThread, finishAutomationRun, startAutomationRun, } from "./run-history.js";
-const BACKGROUND_RUN_STUCK_MS = 10 * 60_000;
+/**
+ * Default hard abort for one in-process automation run. Read through
+ * `resolveBackgroundRunHardTimeoutMs()` at the use site — this is the host's
+ * real function budget for scheduled work, and it differs by deployment.
+ */
 export const BACKGROUND_RUN_HARD_TIMEOUT_MS = 10 * 60_000;
+/**
+ * Terminal failure of a background automation, carrying the machine-readable
+ * code the failure taxonomy already computes.
+ *
+ * The code used to be produced and then dropped, so "how often are runs cut
+ * off?" was a `LIKE '%no_progress%'` over an English sentence.
+ */
+export class BackgroundAutomationRunError extends Error {
+    errorCode;
+    constructor(message, errorCode) {
+        super(message);
+        this.name = "BackgroundAutomationRunError";
+        this.errorCode = errorCode;
+    }
+}
 /**
  * A persisted background run must not outlive its execution identity.
  * Organization runs fail closed when membership state cannot be read. A
@@ -112,8 +131,11 @@ export function isBackgroundAutomationRunActive(meta, now = new Date()) {
     if (!meta.lastRun)
         return false;
     const startedAt = new Date(meta.lastRun).getTime();
+    // Tracks the hard abort: past it no run of this automation is still alive, so
+    // a deployment that raises the abort must not have its live runs treated as
+    // stuck and re-dispatched underneath themselves.
     return (Number.isFinite(startedAt) &&
-        now.getTime() - startedAt < BACKGROUND_RUN_STUCK_MS);
+        now.getTime() - startedAt < resolveBackgroundRunHardTimeoutMs());
 }
 /**
  * A soft-timeout/no-progress checkpoint is a continuation boundary, not a
@@ -186,6 +208,9 @@ export async function runBackgroundAutomation(options, deps) {
     }
     catch (err) {
         const message = err instanceof Error ? err.message : String(err);
+        const errorCode = err instanceof BackgroundAutomationRunError
+            ? err.errorCode
+            : "background_automation_failed";
         // Both callers (recurring-jobs scheduler, trigger dispatcher) record this
         // onto the automation's own metadata and console.error it, and neither
         // reports it. A failure visible only in a resource field and stdout is not
@@ -204,7 +229,7 @@ export async function runBackgroundAutomation(options, deps) {
             },
             ...(runIdRef.current ? { aiTraceId: runIdRef.current } : {}),
         });
-        await recordRunOutcome(historyId, "error", `${message}. No delivery was confirmed.`);
+        await recordRunOutcome(historyId, "error", `${message}. No delivery was confirmed.`, errorCode);
         throw err;
     }
     // Outside the try: history is bookkeeping about the run, so a failure to
@@ -229,8 +254,9 @@ async function recordRunThread(historyId, threadId, runId) {
 }
 function backgroundAutomationPersistFailure(input) {
     if (input.hardTimedOut) {
+        const minutes = Math.round((input.hardTimeoutMs ?? BACKGROUND_RUN_HARD_TIMEOUT_MS) / 60_000);
         return {
-            message: `Background automation timed out after ${BACKGROUND_RUN_HARD_TIMEOUT_MS / 60_000} minutes`,
+            message: `Background automation timed out after ${minutes} minutes`,
             errorCode: "background_automation_hard_timeout",
         };
     }
@@ -301,11 +327,11 @@ async function persistBackgroundAutomationTurn(input) {
         await updateThreadData(input.threadId, JSON.stringify(repo), input.threadTitle || row.title, meta.preview || row.preview, Array.isArray(messages) ? messages.length : 0);
     });
 }
-async function recordRunOutcome(historyId, status, error) {
+async function recordRunOutcome(historyId, status, error, errorCode) {
     if (!historyId)
         return;
     try {
-        await finishAutomationRun(historyId, status, error);
+        await finishAutomationRun(historyId, status, error, errorCode);
     }
     catch (err) {
         console.error(`[automations] Could not record run ${historyId} as ${status}:`, err);
@@ -372,12 +398,15 @@ async function executeBackgroundAutomation(options, deps, historyId, runIdRef) {
         // Hardcoded rather than `isInBackgroundFunctionRuntime()` (what
         // webhook-handler.ts uses): a webhook can arrive on either runtime, but
         // a scheduler tick never serves a synchronous request, so the
-        // interactive clamp never applies to it. The wider soft ceiling stays
-        // bounded by this runner's own BACKGROUND_RUN_HARD_TIMEOUT_MS abort.
-        const softTimeoutMs = resolveRunSoftTimeoutMs(undefined, {
-            useHostedDefault: true,
-            backgroundFunction: true,
-        });
+        // interactive clamp never applies to it.
+        //
+        // Derived from this runner's OWN hard abort, not from the durable-chat
+        // background ceiling: that ceiling is 13 minutes and this process is
+        // killed at 10, so taking it left the recoverable soft-timeout boundary
+        // as dead code and the terminal no-progress backstop as the only
+        // boundary an automation could reach.
+        const hardTimeoutMs = resolveBackgroundRunHardTimeoutMs();
+        const softTimeoutMs = resolveBackgroundAutomationSoftTimeoutMs();
         const usageRef = { current: null };
         let responseText = "";
         let hardAbortTimer = null;
@@ -400,8 +429,8 @@ async function executeBackgroundAutomation(options, deps, historyId, runIdRef) {
             throw new Error(`Background automation "${automation.name}" (run "${runId}") could not claim its own freshly-inserted run row`);
         }
         await new Promise((resolve, reject) => {
-            const activeRun = startRun(runId, thread.id, async (send, signal) => {
-                usageRef.current = await runAgentLoopDirectWithSoftTimeout({
+            const activeRun = startRun(runId, thread.id, async (send, signal, control) => {
+                const loopOpts = {
                     engine,
                     model,
                     systemPrompt,
@@ -425,7 +454,45 @@ async function executeBackgroundAutomation(options, deps, historyId, runIdRef) {
                     runId,
                     maxIterations: automation.meta.maxIterations,
                     maxRunInputTokens: automation.meta.maxRunInputTokens,
-                }, softTimeoutMs, { backgroundFunction: true });
+                };
+                // Same adapter A2A uses: bridge this runner's multi-argument shape
+                // to the single-argument `runAgentLoop` `instrumentAgentLoop`
+                // expects. `control` is what lets a chunk boundary be recovered
+                // here instead of ending the turn.
+                const execute = (o = loopOpts) => runAgentLoopDirectWithSoftTimeout(o, softTimeoutMs, { backgroundFunction: true }, control);
+                let instrumented = false;
+                try {
+                    const { getObservabilityConfig, instrumentAgentLoop } = await import("../observability/traces.js");
+                    const config = await getObservabilityConfig();
+                    if (config.enabled) {
+                        instrumented = true;
+                        usageRef.current = await instrumentAgentLoop({
+                            runAgentLoop: (o) => execute(o),
+                            loopOpts,
+                            runId,
+                            threadId: thread.id,
+                            // A scheduled run is NOT anonymous. Passing the owner is what
+                            // makes it visible to per-user observability reads.
+                            userId: ownerEmail,
+                            config,
+                            spanName: "background_automation_run",
+                            metadata: {
+                                automation: automation.name,
+                                trigger: "background_automation",
+                                scope: orgId ? "organization" : "personal",
+                            },
+                        });
+                        return;
+                    }
+                }
+                catch (error) {
+                    // Match A2A and interactive chat: a setup failure falls through
+                    // to an uninstrumented run, but a failure from INSIDE the
+                    // instrumented loop is the real run failure and must rethrow.
+                    if (instrumented)
+                        throw error;
+                }
+                usageRef.current = await execute();
             }, async (run) => {
                 if (hardAbortTimer) {
                     clearTimeout(hardAbortTimer);
@@ -434,6 +501,7 @@ async function executeBackgroundAutomation(options, deps, historyId, runIdRef) {
                 const persistFailure = backgroundAutomationPersistFailure({
                     run,
                     hardTimedOut,
+                    hardTimeoutMs,
                 });
                 try {
                     await persistBackgroundAutomationTurn({
@@ -453,11 +521,11 @@ async function executeBackgroundAutomation(options, deps, historyId, runIdRef) {
                 if (hardTimedOut)
                     return;
                 if (persistFailure) {
-                    reject(new Error(persistFailure.message));
+                    reject(new BackgroundAutomationRunError(persistFailure.message, persistFailure.errorCode));
                     return;
                 }
                 if (run.status !== "completed") {
-                    reject(new Error(`Background automation ended with status: ${run.status}`));
+                    reject(new BackgroundAutomationRunError(`Background automation ended with status: ${run.status}`, `background_automation_${run.status}`));
                     return;
                 }
                 responseText = collectFinalResponseTextFromAgentEvents((run.events ?? []).map((entry) => entry.event));
@@ -465,22 +533,37 @@ async function executeBackgroundAutomation(options, deps, historyId, runIdRef) {
             }, {
                 softTimeoutMs,
                 backgroundFunction: true,
+                // This runner owns continuation in-process: there is no HTTP body
+                // to re-POST and no `chainServerDrivenContinuation` behind it, so a
+                // checkpoint must end the CHUNK and let the loop above recover it.
+                recoverChunkBoundaries: true,
+                // Matches the `dispatch_mode` this runner already writes onto the
+                // run row at insert. Without it the terminal and boundary analytics
+                // events reported every scheduled run as foreground.
+                dispatchMode: "background",
+                noProgressTimeoutMs: options.noProgressTimeoutMs,
+                backgroundNoProgressTimeoutMs: options.backgroundNoProgressTimeoutMs,
                 model,
                 engineName: engine.name,
+                userId: ownerEmail,
             });
             hardAbortTimer = setTimeout(() => {
                 hardAbortTimer = null;
                 if (activeRun.status !== "running")
                     return;
                 hardTimedOut = true;
-                activeRun.abort.abort("background_automation_hard_timeout");
-                const timeoutError = new Error(`Background automation timed out after ${BACKGROUND_RUN_HARD_TIMEOUT_MS / 60_000} minutes`);
+                // `abortRun`, not `activeRun.abort.abort`: the controller alone
+                // carries no reason the run manager can see, so finalization fell
+                // through to `aborted:user` and a hard timeout was recorded as a
+                // person pressing Stop.
+                abortRun(runId, "background_automation_hard_timeout");
+                const timeoutError = new BackgroundAutomationRunError(`Background automation timed out after ${Math.round(hardTimeoutMs / 60_000)} minutes`, "background_automation_hard_timeout");
                 void activeRun.finalized
                     .catch(() => { })
                     .then(() => {
                     reject(timeoutError);
                 });
-            }, BACKGROUND_RUN_HARD_TIMEOUT_MS);
+            }, hardTimeoutMs);
         }).finally(() => {
             if (hardAbortTimer) {
                 clearTimeout(hardAbortTimer);

package/dist/jobs/run-history.d.ts

@@ -20,6 +20,12 @@ export interface AutomationRun {
     startedAt: number;
     finishedAt: number | null;
     error: string | null;
+    /**
+     * Machine-readable failure code, so "how often are runs cut off?" is a
+     * GROUP BY instead of a LIKE over an English sentence. Null on success and
+     * on rows written before the column existed.
+     */
+    errorCode: string | null;
 }
 export interface StartAutomationRunInput {
     owner: string;
@@ -56,7 +62,7 @@ export declare function listUnclaimedAutomationRuns(options?: {
     olderThanMs?: number;
     limit?: number;
 }): Promise<AutomationRun[]>;
-export declare function finishAutomationRun(id: string, status: Exclude<AutomationRunStatus, "running">, error?: string): Promise<void>;
+export declare function finishAutomationRun(id: string, status: Exclude<AutomationRunStatus, "running">, error?: string, errorCode?: string): Promise<void>;
 /**
  * Attach the agent thread once it exists. The thread is created after the run
  * row so the history survives a crash between the two.

package/dist/jobs/run-history.js

@@ -1,5 +1,6 @@
 import { randomUUID } from "node:crypto";
 import { z } from "zod";
+import { resolveBackgroundRunHardTimeoutMs } from "../agent/run-manager.js";
 import { getDbExec, intType, isPostgres } from "../db/client.js";
 import { ensureColumnExists, ensureIndexExists, ensureTableExists, } from "../db/ddl-guard.js";
 import { runMigrations } from "../db/migrations.js";
@@ -17,20 +18,38 @@ registerEvent({
         threadId: z.string().nullable(),
         status: z.enum(["success", "error", "interrupted"]),
         error: z.string().nullable(),
+        errorCode: z.string().nullable(),
+        /** Wall-clock from `started_at` to now. Null when the row predates the
+         *  start timestamp being readable, so "not measured" stays distinct from
+         *  "took no time". */
+        durationMs: z.number().nullable(),
     }),
 });
 const TABLE = "automation_runs";
 const MAX_ERROR_LENGTH = 500;
+const MAX_ERROR_CODE_LENGTH = 100;
 /**
- * Generous multiple of the runner's own 10 minute hard abort
- * (BACKGROUND_RUN_HARD_TIMEOUT_MS). Past this, no run is still alive.
+ * Past this, no run of this automation is still alive.
+ *
+ * DERIVED from the runner's own hard abort rather than pinned, because that
+ * abort became configurable: a fixed 15 minutes against a longer configured
+ * timeout would report a still-executing run as `interrupted` and expire its
+ * claim lease, redispatching it on top of itself. Half again the abort leaves
+ * room for wind-down and the terminal write without ever preceding them.
  */
-const RUN_LIVENESS_CEILING_MS = 15 * 60_000;
+function resolveRunLivenessCeilingMs() {
+    return Math.ceil(resolveBackgroundRunHardTimeoutMs() * 1.5);
+}
 const INTERRUPTED_RUN_MESSAGE = "Worker stopped before a terminal result was recorded. The serverless worker may have timed out or been recycled. No delivery was confirmed.";
-// The background worker has a shorter hard timeout than this lease. A worker
-// that dies after claiming can therefore be redelivered without overlapping a
-// still-live execution under normal runtime limits.
-const CLAIM_LEASE_MS = RUN_LIVENESS_CEILING_MS;
+/**
+ * Derived at read time alongside `INTERRUPTED_RUN_MESSAGE`: a process killed
+ * mid-run cannot write its own code any more than it can write its own message.
+ */
+const INTERRUPTED_RUN_ERROR_CODE = "background_automation_interrupted";
+// The background worker's hard timeout is always shorter than this lease, by
+// construction above. A worker that dies after claiming can therefore be
+// redelivered without overlapping a still-live execution.
+const claimLeaseMs = () => resolveRunLivenessCeilingMs();
 /** Rows kept per automation, so a per-minute schedule cannot grow forever. */
 const RUNS_RETAINED_PER_AUTOMATION = 50;
 /** Authoritative release-time schema for durable automation history. */
@@ -76,6 +95,11 @@ export const AUTOMATION_RUN_MIGRATIONS = [
         name: "automation-runs-app-id",
         sql: `ALTER TABLE ${TABLE} ADD COLUMN IF NOT EXISTS app_id TEXT`,
     },
+    {
+        version: 5,
+        name: "automation-runs-error-code",
+        sql: `ALTER TABLE ${TABLE} ADD COLUMN IF NOT EXISTS error_code TEXT`,
+    },
 ];
 export async function runAutomationRunMigrations(nitroApp) {
     await runMigrations(AUTOMATION_RUN_MIGRATIONS, {
@@ -102,6 +126,7 @@ export async function ensureTable() {
           started_at ${intType()} NOT NULL,
           finished_at ${intType()},
           error TEXT,
+          error_code TEXT,
           claimed_at ${intType()},
           dispatch_pending ${intType()} NOT NULL DEFAULT 0
         )
@@ -111,6 +136,7 @@ export async function ensureTable() {
                 await ensureTableExists(TABLE, createSql);
                 await ensureColumnExists(TABLE, "claimed_at", `ALTER TABLE ${TABLE} ADD COLUMN IF NOT EXISTS claimed_at ${intType()}`);
                 await ensureColumnExists(TABLE, "dispatch_pending", `ALTER TABLE ${TABLE} ADD COLUMN IF NOT EXISTS dispatch_pending ${intType()} NOT NULL DEFAULT 0`);
+                await ensureColumnExists(TABLE, "error_code", `ALTER TABLE ${TABLE} ADD COLUMN IF NOT EXISTS error_code TEXT`);
                 await ensureIndexExists(`idx_${TABLE}_owner_automation`, indexSql);
                 return;
             }
@@ -121,6 +147,7 @@ export async function ensureTable() {
                 ["claimed_at", `${intType()}`],
                 ["dispatch_pending", `${intType()} NOT NULL DEFAULT 0`],
                 ["app_id", "TEXT"],
+                ["error_code", "TEXT"],
             ]) {
                 if (columns.has(name))
                     continue;
@@ -146,7 +173,7 @@ export async function ensureTable() {
 function toRun(row, now) {
     const stored = String(row.status);
     const startedAt = Number(row.started_at);
-    const status = stored === "running" && now - startedAt > RUN_LIVENESS_CEILING_MS
+    const status = stored === "running" && now - startedAt > resolveRunLivenessCeilingMs()
         ? "interrupted"
         : stored;
     return {
@@ -167,6 +194,11 @@ function toRun(row, now) {
             : row.error == null
                 ? null
                 : String(row.error),
+        errorCode: row.error_code == null && status === "interrupted"
+            ? INTERRUPTED_RUN_ERROR_CODE
+            : row.error_code == null
+                ? null
+                : String(row.error_code),
     };
 }
 /**
@@ -212,7 +244,7 @@ export async function claimAutomationRun(id) {
     const now = Date.now();
     const result = await getDbExec().execute({
         sql: `UPDATE ${TABLE} SET claimed_at = ? WHERE id = ? AND dispatch_pending = 1 AND (claimed_at IS NULL OR claimed_at <= ?) AND status = 'running'`,
-        args: [now, id, now - CLAIM_LEASE_MS],
+        args: [now, id, now - claimLeaseMs()],
     });
     return Number(result.rowsAffected ?? 0) > 0;
 }
@@ -234,7 +266,7 @@ export async function listUnclaimedAutomationRuns(options) {
             AND started_at <= ?
           ORDER BY started_at ASC LIMIT ${limit}`,
         args: [
-            Date.now() - CLAIM_LEASE_MS,
+            Date.now() - claimLeaseMs(),
             ...(appId ? [appId] : []),
             Date.now() - olderThanMs,
         ],
@@ -263,19 +295,28 @@ async function pruneAutomationRuns(owner, automation) {
         args: [owner, automation, owner, automation],
     });
 }
-export async function finishAutomationRun(id, status, error) {
+export async function finishAutomationRun(id, status, error, errorCode) {
     await ensureTable();
     const existing = await getDbExec().execute({
-        sql: `SELECT owner, automation, path, org_id, run_id, thread_id FROM ${TABLE} WHERE id = ? LIMIT 1`,
+        sql: `SELECT owner, automation, path, org_id, run_id, thread_id, started_at FROM ${TABLE} WHERE id = ? LIMIT 1`,
         args: [id],
     });
     const row = existing.rows?.[0];
+    const finishedAt = Date.now();
     await getDbExec().execute({
-        sql: `UPDATE ${TABLE} SET status = ?, finished_at = ?, error = ? WHERE id = ?`,
-        args: [status, Date.now(), error?.slice(0, MAX_ERROR_LENGTH) ?? null, id],
+        sql: `UPDATE ${TABLE} SET status = ?, finished_at = ?, error = ?, error_code = ? WHERE id = ?`,
+        args: [
+            status,
+            finishedAt,
+            error?.slice(0, MAX_ERROR_LENGTH) ?? null,
+            errorCode?.slice(0, MAX_ERROR_CODE_LENGTH) ?? null,
+            id,
+        ],
     });
     if (!row)
         return;
+    const rawStartedAt = Number(row.started_at);
+    const startedAt = Number.isFinite(rawStartedAt) ? rawStartedAt : null;
     try {
         emitBusEvent("automation.run.finished", {
             automationRunId: id,
@@ -287,6 +328,8 @@ export async function finishAutomationRun(id, status, error) {
             threadId: row.thread_id == null ? null : String(row.thread_id),
             status,
             error: error?.slice(0, MAX_ERROR_LENGTH) ?? null,
+            errorCode: errorCode?.slice(0, MAX_ERROR_CODE_LENGTH) ?? null,
+            durationMs: startedAt === null ? null : Math.max(0, finishedAt - startedAt),
         }, { owner: String(row.owner) });
     }
     catch (eventError) {

package/dist/observability/traces.d.ts

@@ -44,6 +44,19 @@ export declare function instrumentAgentLoop(opts: {
      *  reads. */
     userId: string | null;
     config: ObservabilityConfig;
+    /**
+     * Name for this run's root span, in the local trace store and in PostHog LLM
+     * analytics. Defaults to `"agent_run"`. Without it every path emits the same
+     * name and a scheduled automation is indistinguishable from a chat turn in
+     * the one view where telling them apart is the whole question.
+     */
+    spanName?: string;
+    /**
+     * Free-form run context. Persisted onto the local store's parent span AND
+     * forwarded to PostHog as trace properties — a channel that reached only the
+     * SQL store was a channel that could not answer "which automation was this?"
+     * in LLM analytics.
+     */
     metadata?: Record<string, unknown> | null;
     experimentAssignments?: Array<{
         experimentId: string;