@agent-native/core 0.168.12 → 0.169.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. package/corpus/templates/clips/app/lib/capture-install-options.ts +20 -2
  2. package/corpus/templates/dispatch/app/root.tsx +9 -1
  3. package/dist/agent/engine/first-event-timeout.d.ts +8 -0
  4. package/dist/agent/engine/first-event-timeout.js +8 -0
  5. package/dist/agent/production-agent.d.ts +0 -30
  6. package/dist/agent/production-agent.js +17 -38
  7. package/dist/agent/run-loop-with-resume.d.ts +38 -25
  8. package/dist/agent/run-loop-with-resume.js +140 -55
  9. package/dist/agent/run-manager.d.ts +83 -68
  10. package/dist/agent/run-manager.js +280 -94
  11. package/dist/agent/run-store.d.ts +31 -0
  12. package/dist/agent/run-store.js +42 -12
  13. package/dist/app-config/agent.d.ts +2 -0
  14. package/dist/app-config/agent.js +33 -0
  15. package/dist/app-config/run-lifecycle-invariants.d.ts +248 -0
  16. package/dist/app-config/run-lifecycle-invariants.js +342 -0
  17. package/dist/app-config/schema.d.ts +2 -0
  18. package/dist/app-config/store.js +9 -1
  19. package/dist/client/EnvironmentBadge.d.ts +5 -4
  20. package/dist/client/EnvironmentBadge.js +19 -8
  21. package/dist/client/agent-chat-adapter.d.ts +0 -2
  22. package/dist/client/agent-chat-adapter.js +7 -23
  23. package/dist/client/app-providers.d.ts +3 -2
  24. package/dist/client/app-providers.js +3 -2
  25. package/dist/collab/awareness.d.ts +2 -2
  26. package/dist/collab/routes.d.ts +1 -1
  27. package/dist/jobs/background-automation-runner.d.ts +25 -0
  28. package/dist/jobs/background-automation-runner.js +104 -21
  29. package/dist/jobs/run-history.d.ts +7 -1
  30. package/dist/jobs/run-history.js +57 -14
  31. package/dist/notifications/routes.d.ts +3 -3
  32. package/dist/observability/traces.d.ts +13 -0
  33. package/dist/observability/traces.js +369 -317
  34. package/dist/resources/handlers.d.ts +1 -1
  35. package/dist/server/agent-chat-plugin.js +2 -4
  36. package/dist/server/beta-opt-out-html.js +3 -2
  37. package/dist/server/onboarding-html.js +3 -2
  38. package/dist/server/realtime-token.d.ts +1 -1
  39. package/package.json +1 -1
@@ -16,7 +16,9 @@ export function isAgentNativeDesktopUserAgent(userAgent) {
16
16
  }
17
17
  export function resolveEnvironmentChannel(config, hostname) {
18
18
  const configured = config.deployment?.environment;
19
- if (configured === "beta" || configured === "production") {
19
+ if (configured === "local" ||
20
+ configured === "beta" ||
21
+ configured === "production") {
20
22
  return configured;
21
23
  }
22
24
  const targets = resolveEnvironmentTargets(hostname);
@@ -74,6 +76,7 @@ function consumeBetaOptOutQueryParam(sourceHref, now = Date.now()) {
74
76
  }
75
77
  return active;
76
78
  }
79
+ const environmentBadgePlacementClasses = "fixed bottom-3 left-3 z-[100] h-6 min-w-0 rounded-xl px-2 text-[11px] font-semibold uppercase tracking-[0.5px] shadow-sm backdrop-blur-sm";
77
80
  function EnvironmentLink({ label, href }) {
78
81
  return (_jsx(Button, { asChild: true, className: "w-full justify-center", size: "sm", variant: "outline", children: _jsx("a", { href: href, children: label }) }));
79
82
  }
@@ -89,9 +92,12 @@ function EnvironmentBadgeContent({ environment, targets, }) {
89
92
  const title = environment === "beta"
90
93
  ? "You're on Agent Native Beta"
91
94
  : "You're on Agent Native Production";
92
- return (_jsxs(Popover, { children: [_jsx(PopoverTrigger, { asChild: true, children: _jsx(Button, { "aria-label": `Open ${title.toLowerCase()} switcher`, className: cn("fixed bottom-3 right-3 z-[100] h-6 min-w-0 rounded-xl px-2 text-[11px] font-semibold uppercase tracking-[0.5px] shadow-sm backdrop-blur-sm", environment === "beta"
95
+ return (_jsxs(Popover, { children: [_jsx(PopoverTrigger, { asChild: true, children: _jsx(Button, { "aria-label": `Open ${title.toLowerCase()} switcher`, className: cn(environmentBadgePlacementClasses, environment === "beta"
93
96
  ? "border-primary/80"
94
- : "border-border/80 bg-background/95 text-foreground"), size: "sm", variant: environment === "beta" ? "default" : "outline", children: label }) }), _jsxs(PopoverContent, { align: "center", className: "w-[280px] p-5", side: "top", sideOffset: 8, children: [_jsx("div", { className: "mb-1 text-sm font-semibold leading-5", children: title }), _jsx("div", { className: "mb-4 text-sm text-muted-foreground", children: "Choose where you want to continue." }), _jsx("div", { className: "grid gap-2", children: environment === "beta" ? (_jsx(EnvironmentLink, { href: productionHref, label: "Switch to production" })) : (_jsx(EnvironmentLink, { href: betaHref, label: "Go to beta" })) })] })] }));
97
+ : "border-border/80 bg-background/95 text-foreground"), size: "sm", variant: environment === "beta" ? "default" : "outline", children: label }) }), _jsxs(PopoverContent, { align: "start", className: "w-[280px] p-5", side: "top", sideOffset: 8, children: [_jsx("div", { className: "mb-1 text-sm font-semibold leading-5", children: title }), _jsx("div", { className: "mb-4 text-sm text-muted-foreground", children: "Choose where you want to continue." }), _jsx("div", { className: "grid gap-2", children: environment === "beta" ? (_jsx(EnvironmentLink, { href: productionHref, label: "Switch to production" })) : (_jsx(EnvironmentLink, { href: betaHref, label: "Go to beta" })) })] })] }));
98
+ }
99
+ function LocalEnvironmentBadge() {
100
+ return (_jsx("div", { "aria-label": "Local development environment", className: cn(environmentBadgePlacementClasses, "inline-flex items-center justify-center border border-border/80 bg-background/95 text-foreground"), role: "status", children: "dev" }));
95
101
  }
96
102
  function ProductionEnvironmentBadge({ targets, }) {
97
103
  const { session, status } = useSession();
@@ -128,9 +134,10 @@ function ProductionEnvironmentBadge({ targets, }) {
128
134
  return _jsx(EnvironmentBadgeContent, { environment: "production", targets: targets });
129
135
  }
130
136
  /**
131
- * First-party hosted lane switcher. Beta is intentionally visible before
132
- * authentication so a visitor can always leave beta from the sign-in page.
133
- * Production remains an internal auto-redirect lane for authenticated staff.
137
+ * Environment indicator and first-party hosted lane switcher. Beta is
138
+ * intentionally visible before authentication so a visitor can always leave
139
+ * beta from the sign-in page. Production remains an internal auto-redirect
140
+ * lane for authenticated staff.
134
141
  */
135
142
  export function EnvironmentBadge({ showProduction = true, } = {}) {
136
143
  const config = useMemo(injectedAgentNativeConfig, []);
@@ -139,10 +146,14 @@ export function EnvironmentBadge({ showProduction = true, } = {}) {
139
146
  const targets = resolveEnvironmentTargets(hostname);
140
147
  if (typeof window === "undefined" ||
141
148
  window.parent !== window ||
142
- !environment ||
143
- !targets) {
149
+ !environment) {
144
150
  return null;
145
151
  }
152
+ if (environment === "local") {
153
+ return _jsx(LocalEnvironmentBadge, {});
154
+ }
155
+ if (!targets)
156
+ return null;
146
157
  if (environment === "beta") {
147
158
  return _jsx(EnvironmentBadgeContent, { environment: "beta", targets: targets });
148
159
  }
@@ -15,8 +15,6 @@ export type AgentChatSurfaceKind =
15
15
  | "desktop";
16
16
  export declare const BACKGROUND_FOLLOW_ATTACH_WATCHDOG_MS = 90000;
17
17
  export declare const BACKGROUND_FOLLOW_IDLE_TIMEOUT_MS = 210000;
18
- export declare const MAX_FOLLOWED_BACKGROUND_RUNS = 24;
19
- export declare const MAX_BACKGROUND_FOLLOW_WALL_TIME_MS: number;
20
18
  /**
21
19
  * True when an action was streamed but never returned a result yet — i.e. a
22
20
  * `tool_start` with no matching `tool_done`. The server is still executing it,
@@ -170,29 +170,13 @@ export const BACKGROUND_FOLLOW_IDLE_TIMEOUT_MS = 210_000;
170
170
  // 22 minutes, all error:stale_run, user watching a spinner). These bound the
171
171
  // whole turn instead, and an identical repeated failure counts as no progress.
172
172
  //
173
- // CLIENT-ABOVE-SERVER INVARIANT (asserted in agent-chat-adapter.spec.ts).
174
- // These are a backstop for a server that has gone silent in a way the idle
175
- // timeout missesNOT the primary limit. They must stay ABOVE the server's
176
- // own ceilings so the server, which can actually tell progress from looping,
177
- // always terminates a turn first and writes a truthful terminal reason:
178
- // BACKGROUND_SOFT_TIMEOUT_CEILING_MS (13 min, run-manager.ts) — one chunk
179
- // MAX_TURN_WALL_CLOCK_MS (90 min, production-agent.ts) one turn
180
- // MAX_BACKGROUND_RUN_CONTINUATIONS (20, production-agent.ts)
181
- //
182
- // They were originally set to 10 min / 6 runs, which put the whole-turn client
183
- // budget BELOW the 13-minute ceiling of a single legal chunk. Any turn needing
184
- // a second full-length chunk was killed by the client while the server was
185
- // healthy and had 80 minutes left — measured in prod as turns dying at 11-25
186
- // minutes with `last_progress_at` tracking the abort, i.e. still streaming
187
- // tokens and completing tools when the client gave up. That is the top
188
- // non-auth cause of "the chat just stopped" reports.
189
- //
190
- // Killing a turn that is NOT progressing is already covered twice over, by
191
- // mechanisms that read progress rather than a clock: the 210s idle timeout
192
- // above, and MAX_REPEATED_BACKGROUND_TERMINAL_REASONS below. Do not re-tighten
193
- // these two to catch a stuck turn — fix the progress signal instead.
194
- export const MAX_FOLLOWED_BACKGROUND_RUNS = 24;
195
- export const MAX_BACKGROUND_FOLLOW_WALL_TIME_MS = 95 * 60_000;
173
+ // Defined in `app-config/run-lifecycle-invariants.ts`, not here, because the
174
+ // CLIENT-ABOVE-SERVER relationship they belong to is checked there against the
175
+ // RESOLVED server configuration — the server bounds are runtime-configurable
176
+ // now, so a spec pinning them against module constants stopped enforcing
177
+ // anything. Re-exported under the same names so importers are unchanged; that
178
+ // module has no runtime imports, so the bundle pays nothing for it.
179
+ import { MAX_BACKGROUND_FOLLOW_WALL_TIME_MS, MAX_FOLLOWED_BACKGROUND_RUNS, } from "../app-config/run-lifecycle-invariants.js";
196
180
  const MAX_REPEATED_BACKGROUND_TERMINAL_REASONS = 3;
197
181
  // A re-observed terminal run whose outcome would be an ERROR (never a
198
182
  // genuine "done" success) gets a short extra grace window before the follow
@@ -39,7 +39,8 @@
39
39
  * toaster — custom Toaster element rendered after children.
40
40
  * Pass `null` to suppress the built-in Toaster when
41
41
  * children already include a styled one.
42
- * Defaults to `<Toaster richColors position="bottom-left" />`.
42
+ * Defaults to a rich-color bottom-left toaster raised above
43
+ * the environment badge.
43
44
  * disableThemeTransitions — passed to next-themes ThemeProvider
44
45
  * `disableTransitionOnChange`. Defaults to `true`
45
46
  * (suppresses CSS transitions during theme switches,
@@ -75,7 +76,7 @@ export interface AppProvidersProps {
75
76
  * Custom Toaster element rendered after children inside TooltipProvider.
76
77
  * Pass `null` to suppress the built-in Toaster when children already
77
78
  * include a styled one.
78
- * Defaults to `<Toaster richColors position="bottom-left" />`.
79
+ * Defaults to a rich-color bottom-left toaster raised above the environment badge.
79
80
  */
80
81
  toaster?: React.ReactNode | null;
81
82
  /**
@@ -40,7 +40,8 @@ import { Fragment as _Fragment, jsx as _jsx, jsxs as _jsxs } from "react/jsx-run
40
40
  * toaster — custom Toaster element rendered after children.
41
41
  * Pass `null` to suppress the built-in Toaster when
42
42
  * children already include a styled one.
43
- * Defaults to `<Toaster richColors position="bottom-left" />`.
43
+ * Defaults to a rich-color bottom-left toaster raised above
44
+ * the environment badge.
44
45
  * disableThemeTransitions — passed to next-themes ThemeProvider
45
46
  * `disableTransitionOnChange`. Defaults to `true`
46
47
  * (suppresses CSS transitions during theme switches,
@@ -65,7 +66,7 @@ import { AgentNativeRouteWarmup } from "./route-warmup.js";
65
66
  import { RouteTransitionIndicator } from "./RouteTransitionIndicator.js";
66
67
  import { RuntimeConfigNotice } from "./RuntimeConfigNotice.js";
67
68
  import { EMBEDDED_THEME_CHANGE_EVENT, applyEmbeddedThemeUpdate, parseEmbeddedThemeUpdate, } from "./theme.js";
68
- const DEFAULT_TOASTER = _jsx(Toaster, { richColors: true, position: "bottom-left" });
69
+ const DEFAULT_TOASTER = (_jsx(Toaster, { richColors: true, position: "bottom-left", offset: { bottom: 44, left: 32 }, mobileOffset: { bottom: 44, left: 16 } }));
69
70
  function RoutedAppEnhancements() {
70
71
  const isInRouter = useInRouterContext();
71
72
  if (!isInRouter)
@@ -62,11 +62,11 @@ export declare const postAwareness: import("h3").EventHandlerWithFetch<import("h
62
62
  error: string;
63
63
  states?: undefined;
64
64
  } | {
65
- error?: undefined;
66
65
  states: {
67
66
  clientId: number;
68
67
  state: string;
69
68
  }[];
69
+ error?: undefined;
70
70
  }>>;
71
71
  /**
72
72
  * GET /_agent-native/collab/:docId/users
@@ -77,9 +77,9 @@ export declare const getActiveUsers: import("h3").EventHandlerWithFetch<import("
77
77
  error: string;
78
78
  users?: undefined;
79
79
  } | {
80
- error?: undefined;
81
80
  users: {
82
81
  clientId: number;
83
82
  lastSeen: number;
84
83
  }[];
84
+ error?: undefined;
85
85
  }>>;
@@ -26,8 +26,8 @@ export declare const getCollabState: import("h3").EventHandlerWithFetch<import("
26
26
  * Body: { update: string (base64), requestSource?: string }
27
27
  */
28
28
  export declare const postCollabUpdate: import("h3").EventHandlerWithFetch<import("h3").EventHandlerRequest, Promise<{
29
- error: string;
30
29
  ok?: undefined;
30
+ error: string;
31
31
  } | {
32
32
  error?: undefined;
33
33
  ok: boolean;
@@ -5,7 +5,23 @@ import { type AutomationExecutionIdentity } from "../automations/service.js";
5
5
  import { type Resource } from "../resources/store.js";
6
6
  import { type RequestContext } from "../server/request-context.js";
7
7
  import type { JobFrontmatter } from "./frontmatter.js";
8
+ /**
9
+ * Default hard abort for one in-process automation run. Read through
10
+ * `resolveBackgroundRunHardTimeoutMs()` at the use site — this is the host's
11
+ * real function budget for scheduled work, and it differs by deployment.
12
+ */
8
13
  export declare const BACKGROUND_RUN_HARD_TIMEOUT_MS: number;
14
+ /**
15
+ * Terminal failure of a background automation, carrying the machine-readable
16
+ * code the failure taxonomy already computes.
17
+ *
18
+ * The code used to be produced and then dropped, so "how often are runs cut
19
+ * off?" was a `LIKE '%no_progress%'` over an English sentence.
20
+ */
21
+ export declare class BackgroundAutomationRunError extends Error {
22
+ readonly errorCode: string;
23
+ constructor(message: string, errorCode: string);
24
+ }
9
25
  export interface BackgroundAutomationContext {
10
26
  name: string;
11
27
  meta: JobFrontmatter;
@@ -35,6 +51,15 @@ export interface BackgroundAutomationRunOptions {
35
51
  actionAutomation?: ActionAutomationContext;
36
52
  /** Reuse a history row created by a durable run-now enqueue. */
37
53
  historyId?: string;
54
+ /**
55
+ * Per-run overrides for the run-manager no-progress backstop. `startRun` has
56
+ * always accepted these; the automation path had no way to reach them, and
57
+ * the one indirect route (zeroing `agent.runSoftTimeoutMs`) is global and
58
+ * would strip foreground chat of its chunk boundary. Additive: unset means
59
+ * the configured/default behaviour, unchanged.
60
+ */
61
+ noProgressTimeoutMs?: number;
62
+ backgroundNoProgressTimeoutMs?: number;
38
63
  }
39
64
  export interface BackgroundAutomationRunResult {
40
65
  responseText: string;
@@ -2,7 +2,7 @@ import { collectFinalResponseTextFromAgentEvents } from "../a2a/response-text.js
2
2
  import { getStoredModelForEngine, normalizeModelForEngine, resolveEngine, } from "../agent/engine/index.js";
3
3
  import { actionsToEngineTools, filterInitialEngineTools, getOwnerActiveApiKey, } from "../agent/production-agent.js";
4
4
  import { runAgentLoopDirectWithSoftTimeout } from "../agent/run-loop-with-resume.js";
5
- import { resolveRunSoftTimeoutMs, startRun, } from "../agent/run-manager.js";
5
+ import { abortRun, resolveBackgroundAutomationSoftTimeoutMs, resolveBackgroundRunHardTimeoutMs, startRun, } from "../agent/run-manager.js";
6
6
  import { claimBackgroundRun, insertRun } from "../agent/run-store.js";
7
7
  import { buildAssistantMessage, buildUserMessage, extractThreadMeta, foldAssistantTurn, upsertUserMessage, } from "../agent/thread-data-builder.js";
8
8
  import { attachToolSearch } from "../agent/tool-search.js";
@@ -13,8 +13,27 @@ import { organizationIdFromResourceOwner, organizationResourceOwner, } from "../
13
13
  import { captureError } from "../server/capture-error.js";
14
14
  import { runWithRequestContext, } from "../server/request-context.js";
15
15
  import { attachAutomationRunThread, finishAutomationRun, startAutomationRun, } from "./run-history.js";
16
- const BACKGROUND_RUN_STUCK_MS = 10 * 60_000;
16
+ /**
17
+ * Default hard abort for one in-process automation run. Read through
18
+ * `resolveBackgroundRunHardTimeoutMs()` at the use site — this is the host's
19
+ * real function budget for scheduled work, and it differs by deployment.
20
+ */
17
21
  export const BACKGROUND_RUN_HARD_TIMEOUT_MS = 10 * 60_000;
22
+ /**
23
+ * Terminal failure of a background automation, carrying the machine-readable
24
+ * code the failure taxonomy already computes.
25
+ *
26
+ * The code used to be produced and then dropped, so "how often are runs cut
27
+ * off?" was a `LIKE '%no_progress%'` over an English sentence.
28
+ */
29
+ export class BackgroundAutomationRunError extends Error {
30
+ errorCode;
31
+ constructor(message, errorCode) {
32
+ super(message);
33
+ this.name = "BackgroundAutomationRunError";
34
+ this.errorCode = errorCode;
35
+ }
36
+ }
18
37
  /**
19
38
  * A persisted background run must not outlive its execution identity.
20
39
  * Organization runs fail closed when membership state cannot be read. A
@@ -112,8 +131,11 @@ export function isBackgroundAutomationRunActive(meta, now = new Date()) {
112
131
  if (!meta.lastRun)
113
132
  return false;
114
133
  const startedAt = new Date(meta.lastRun).getTime();
134
+ // Tracks the hard abort: past it no run of this automation is still alive, so
135
+ // a deployment that raises the abort must not have its live runs treated as
136
+ // stuck and re-dispatched underneath themselves.
115
137
  return (Number.isFinite(startedAt) &&
116
- now.getTime() - startedAt < BACKGROUND_RUN_STUCK_MS);
138
+ now.getTime() - startedAt < resolveBackgroundRunHardTimeoutMs());
117
139
  }
118
140
  /**
119
141
  * A soft-timeout/no-progress checkpoint is a continuation boundary, not a
@@ -186,6 +208,9 @@ export async function runBackgroundAutomation(options, deps) {
186
208
  }
187
209
  catch (err) {
188
210
  const message = err instanceof Error ? err.message : String(err);
211
+ const errorCode = err instanceof BackgroundAutomationRunError
212
+ ? err.errorCode
213
+ : "background_automation_failed";
189
214
  // Both callers (recurring-jobs scheduler, trigger dispatcher) record this
190
215
  // onto the automation's own metadata and console.error it, and neither
191
216
  // reports it. A failure visible only in a resource field and stdout is not
@@ -204,7 +229,7 @@ export async function runBackgroundAutomation(options, deps) {
204
229
  },
205
230
  ...(runIdRef.current ? { aiTraceId: runIdRef.current } : {}),
206
231
  });
207
- await recordRunOutcome(historyId, "error", `${message}. No delivery was confirmed.`);
232
+ await recordRunOutcome(historyId, "error", `${message}. No delivery was confirmed.`, errorCode);
208
233
  throw err;
209
234
  }
210
235
  // Outside the try: history is bookkeeping about the run, so a failure to
@@ -229,8 +254,9 @@ async function recordRunThread(historyId, threadId, runId) {
229
254
  }
230
255
  function backgroundAutomationPersistFailure(input) {
231
256
  if (input.hardTimedOut) {
257
+ const minutes = Math.round((input.hardTimeoutMs ?? BACKGROUND_RUN_HARD_TIMEOUT_MS) / 60_000);
232
258
  return {
233
- message: `Background automation timed out after ${BACKGROUND_RUN_HARD_TIMEOUT_MS / 60_000} minutes`,
259
+ message: `Background automation timed out after ${minutes} minutes`,
234
260
  errorCode: "background_automation_hard_timeout",
235
261
  };
236
262
  }
@@ -301,11 +327,11 @@ async function persistBackgroundAutomationTurn(input) {
301
327
  await updateThreadData(input.threadId, JSON.stringify(repo), input.threadTitle || row.title, meta.preview || row.preview, Array.isArray(messages) ? messages.length : 0);
302
328
  });
303
329
  }
304
- async function recordRunOutcome(historyId, status, error) {
330
+ async function recordRunOutcome(historyId, status, error, errorCode) {
305
331
  if (!historyId)
306
332
  return;
307
333
  try {
308
- await finishAutomationRun(historyId, status, error);
334
+ await finishAutomationRun(historyId, status, error, errorCode);
309
335
  }
310
336
  catch (err) {
311
337
  console.error(`[automations] Could not record run ${historyId} as ${status}:`, err);
@@ -372,12 +398,15 @@ async function executeBackgroundAutomation(options, deps, historyId, runIdRef) {
372
398
  // Hardcoded rather than `isInBackgroundFunctionRuntime()` (what
373
399
  // webhook-handler.ts uses): a webhook can arrive on either runtime, but
374
400
  // a scheduler tick never serves a synchronous request, so the
375
- // interactive clamp never applies to it. The wider soft ceiling stays
376
- // bounded by this runner's own BACKGROUND_RUN_HARD_TIMEOUT_MS abort.
377
- const softTimeoutMs = resolveRunSoftTimeoutMs(undefined, {
378
- useHostedDefault: true,
379
- backgroundFunction: true,
380
- });
401
+ // interactive clamp never applies to it.
402
+ //
403
+ // Derived from this runner's OWN hard abort, not from the durable-chat
404
+ // background ceiling: that ceiling is 13 minutes and this process is
405
+ // killed at 10, so taking it left the recoverable soft-timeout boundary
406
+ // as dead code and the terminal no-progress backstop as the only
407
+ // boundary an automation could reach.
408
+ const hardTimeoutMs = resolveBackgroundRunHardTimeoutMs();
409
+ const softTimeoutMs = resolveBackgroundAutomationSoftTimeoutMs();
381
410
  const usageRef = { current: null };
382
411
  let responseText = "";
383
412
  let hardAbortTimer = null;
@@ -400,8 +429,8 @@ async function executeBackgroundAutomation(options, deps, historyId, runIdRef) {
400
429
  throw new Error(`Background automation "${automation.name}" (run "${runId}") could not claim its own freshly-inserted run row`);
401
430
  }
402
431
  await new Promise((resolve, reject) => {
403
- const activeRun = startRun(runId, thread.id, async (send, signal) => {
404
- usageRef.current = await runAgentLoopDirectWithSoftTimeout({
432
+ const activeRun = startRun(runId, thread.id, async (send, signal, control) => {
433
+ const loopOpts = {
405
434
  engine,
406
435
  model,
407
436
  systemPrompt,
@@ -425,7 +454,45 @@ async function executeBackgroundAutomation(options, deps, historyId, runIdRef) {
425
454
  runId,
426
455
  maxIterations: automation.meta.maxIterations,
427
456
  maxRunInputTokens: automation.meta.maxRunInputTokens,
428
- }, softTimeoutMs, { backgroundFunction: true });
457
+ };
458
+ // Same adapter A2A uses: bridge this runner's multi-argument shape
459
+ // to the single-argument `runAgentLoop` `instrumentAgentLoop`
460
+ // expects. `control` is what lets a chunk boundary be recovered
461
+ // here instead of ending the turn.
462
+ const execute = (o = loopOpts) => runAgentLoopDirectWithSoftTimeout(o, softTimeoutMs, { backgroundFunction: true }, control);
463
+ let instrumented = false;
464
+ try {
465
+ const { getObservabilityConfig, instrumentAgentLoop } = await import("../observability/traces.js");
466
+ const config = await getObservabilityConfig();
467
+ if (config.enabled) {
468
+ instrumented = true;
469
+ usageRef.current = await instrumentAgentLoop({
470
+ runAgentLoop: (o) => execute(o),
471
+ loopOpts,
472
+ runId,
473
+ threadId: thread.id,
474
+ // A scheduled run is NOT anonymous. Passing the owner is what
475
+ // makes it visible to per-user observability reads.
476
+ userId: ownerEmail,
477
+ config,
478
+ spanName: "background_automation_run",
479
+ metadata: {
480
+ automation: automation.name,
481
+ trigger: "background_automation",
482
+ scope: orgId ? "organization" : "personal",
483
+ },
484
+ });
485
+ return;
486
+ }
487
+ }
488
+ catch (error) {
489
+ // Match A2A and interactive chat: a setup failure falls through
490
+ // to an uninstrumented run, but a failure from INSIDE the
491
+ // instrumented loop is the real run failure and must rethrow.
492
+ if (instrumented)
493
+ throw error;
494
+ }
495
+ usageRef.current = await execute();
429
496
  }, async (run) => {
430
497
  if (hardAbortTimer) {
431
498
  clearTimeout(hardAbortTimer);
@@ -434,6 +501,7 @@ async function executeBackgroundAutomation(options, deps, historyId, runIdRef) {
434
501
  const persistFailure = backgroundAutomationPersistFailure({
435
502
  run,
436
503
  hardTimedOut,
504
+ hardTimeoutMs,
437
505
  });
438
506
  try {
439
507
  await persistBackgroundAutomationTurn({
@@ -453,11 +521,11 @@ async function executeBackgroundAutomation(options, deps, historyId, runIdRef) {
453
521
  if (hardTimedOut)
454
522
  return;
455
523
  if (persistFailure) {
456
- reject(new Error(persistFailure.message));
524
+ reject(new BackgroundAutomationRunError(persistFailure.message, persistFailure.errorCode));
457
525
  return;
458
526
  }
459
527
  if (run.status !== "completed") {
460
- reject(new Error(`Background automation ended with status: ${run.status}`));
528
+ reject(new BackgroundAutomationRunError(`Background automation ended with status: ${run.status}`, `background_automation_${run.status}`));
461
529
  return;
462
530
  }
463
531
  responseText = collectFinalResponseTextFromAgentEvents((run.events ?? []).map((entry) => entry.event));
@@ -465,22 +533,37 @@ async function executeBackgroundAutomation(options, deps, historyId, runIdRef) {
465
533
  }, {
466
534
  softTimeoutMs,
467
535
  backgroundFunction: true,
536
+ // This runner owns continuation in-process: there is no HTTP body
537
+ // to re-POST and no `chainServerDrivenContinuation` behind it, so a
538
+ // checkpoint must end the CHUNK and let the loop above recover it.
539
+ recoverChunkBoundaries: true,
540
+ // Matches the `dispatch_mode` this runner already writes onto the
541
+ // run row at insert. Without it the terminal and boundary analytics
542
+ // events reported every scheduled run as foreground.
543
+ dispatchMode: "background",
544
+ noProgressTimeoutMs: options.noProgressTimeoutMs,
545
+ backgroundNoProgressTimeoutMs: options.backgroundNoProgressTimeoutMs,
468
546
  model,
469
547
  engineName: engine.name,
548
+ userId: ownerEmail,
470
549
  });
471
550
  hardAbortTimer = setTimeout(() => {
472
551
  hardAbortTimer = null;
473
552
  if (activeRun.status !== "running")
474
553
  return;
475
554
  hardTimedOut = true;
476
- activeRun.abort.abort("background_automation_hard_timeout");
477
- const timeoutError = new Error(`Background automation timed out after ${BACKGROUND_RUN_HARD_TIMEOUT_MS / 60_000} minutes`);
555
+ // `abortRun`, not `activeRun.abort.abort`: the controller alone
556
+ // carries no reason the run manager can see, so finalization fell
557
+ // through to `aborted:user` and a hard timeout was recorded as a
558
+ // person pressing Stop.
559
+ abortRun(runId, "background_automation_hard_timeout");
560
+ const timeoutError = new BackgroundAutomationRunError(`Background automation timed out after ${Math.round(hardTimeoutMs / 60_000)} minutes`, "background_automation_hard_timeout");
478
561
  void activeRun.finalized
479
562
  .catch(() => { })
480
563
  .then(() => {
481
564
  reject(timeoutError);
482
565
  });
483
- }, BACKGROUND_RUN_HARD_TIMEOUT_MS);
566
+ }, hardTimeoutMs);
484
567
  }).finally(() => {
485
568
  if (hardAbortTimer) {
486
569
  clearTimeout(hardAbortTimer);
@@ -20,6 +20,12 @@ export interface AutomationRun {
20
20
  startedAt: number;
21
21
  finishedAt: number | null;
22
22
  error: string | null;
23
+ /**
24
+ * Machine-readable failure code, so "how often are runs cut off?" is a
25
+ * GROUP BY instead of a LIKE over an English sentence. Null on success and
26
+ * on rows written before the column existed.
27
+ */
28
+ errorCode: string | null;
23
29
  }
24
30
  export interface StartAutomationRunInput {
25
31
  owner: string;
@@ -56,7 +62,7 @@ export declare function listUnclaimedAutomationRuns(options?: {
56
62
  olderThanMs?: number;
57
63
  limit?: number;
58
64
  }): Promise<AutomationRun[]>;
59
- export declare function finishAutomationRun(id: string, status: Exclude<AutomationRunStatus, "running">, error?: string): Promise<void>;
65
+ export declare function finishAutomationRun(id: string, status: Exclude<AutomationRunStatus, "running">, error?: string, errorCode?: string): Promise<void>;
60
66
  /**
61
67
  * Attach the agent thread once it exists. The thread is created after the run
62
68
  * row so the history survives a crash between the two.