@agent-native/core 0.168.12 → 0.169.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/corpus/templates/clips/app/lib/capture-install-options.ts +20 -2
- package/corpus/templates/dispatch/app/root.tsx +9 -1
- package/dist/agent/engine/first-event-timeout.d.ts +8 -0
- package/dist/agent/engine/first-event-timeout.js +8 -0
- package/dist/agent/production-agent.d.ts +0 -30
- package/dist/agent/production-agent.js +17 -38
- package/dist/agent/run-loop-with-resume.d.ts +38 -25
- package/dist/agent/run-loop-with-resume.js +140 -55
- package/dist/agent/run-manager.d.ts +83 -68
- package/dist/agent/run-manager.js +280 -94
- package/dist/agent/run-store.d.ts +31 -0
- package/dist/agent/run-store.js +42 -12
- package/dist/app-config/agent.d.ts +2 -0
- package/dist/app-config/agent.js +33 -0
- package/dist/app-config/run-lifecycle-invariants.d.ts +248 -0
- package/dist/app-config/run-lifecycle-invariants.js +342 -0
- package/dist/app-config/schema.d.ts +2 -0
- package/dist/app-config/store.js +9 -1
- package/dist/client/EnvironmentBadge.d.ts +5 -4
- package/dist/client/EnvironmentBadge.js +19 -8
- package/dist/client/agent-chat-adapter.d.ts +0 -2
- package/dist/client/agent-chat-adapter.js +7 -23
- package/dist/client/app-providers.d.ts +3 -2
- package/dist/client/app-providers.js +3 -2
- package/dist/collab/awareness.d.ts +2 -2
- package/dist/collab/routes.d.ts +1 -1
- package/dist/jobs/background-automation-runner.d.ts +25 -0
- package/dist/jobs/background-automation-runner.js +104 -21
- package/dist/jobs/run-history.d.ts +7 -1
- package/dist/jobs/run-history.js +57 -14
- package/dist/notifications/routes.d.ts +3 -3
- package/dist/observability/traces.d.ts +13 -0
- package/dist/observability/traces.js +369 -317
- package/dist/resources/handlers.d.ts +1 -1
- package/dist/server/agent-chat-plugin.js +2 -4
- package/dist/server/beta-opt-out-html.js +3 -2
- package/dist/server/onboarding-html.js +3 -2
- package/dist/server/realtime-token.d.ts +1 -1
- package/package.json +1 -1
|
@@ -16,7 +16,9 @@ export function isAgentNativeDesktopUserAgent(userAgent) {
|
|
|
16
16
|
}
|
|
17
17
|
export function resolveEnvironmentChannel(config, hostname) {
|
|
18
18
|
const configured = config.deployment?.environment;
|
|
19
|
-
if (configured === "
|
|
19
|
+
if (configured === "local" ||
|
|
20
|
+
configured === "beta" ||
|
|
21
|
+
configured === "production") {
|
|
20
22
|
return configured;
|
|
21
23
|
}
|
|
22
24
|
const targets = resolveEnvironmentTargets(hostname);
|
|
@@ -74,6 +76,7 @@ function consumeBetaOptOutQueryParam(sourceHref, now = Date.now()) {
|
|
|
74
76
|
}
|
|
75
77
|
return active;
|
|
76
78
|
}
|
|
79
|
+
const environmentBadgePlacementClasses = "fixed bottom-3 left-3 z-[100] h-6 min-w-0 rounded-xl px-2 text-[11px] font-semibold uppercase tracking-[0.5px] shadow-sm backdrop-blur-sm";
|
|
77
80
|
function EnvironmentLink({ label, href }) {
|
|
78
81
|
return (_jsx(Button, { asChild: true, className: "w-full justify-center", size: "sm", variant: "outline", children: _jsx("a", { href: href, children: label }) }));
|
|
79
82
|
}
|
|
@@ -89,9 +92,12 @@ function EnvironmentBadgeContent({ environment, targets, }) {
|
|
|
89
92
|
const title = environment === "beta"
|
|
90
93
|
? "You're on Agent Native Beta"
|
|
91
94
|
: "You're on Agent Native Production";
|
|
92
|
-
return (_jsxs(Popover, { children: [_jsx(PopoverTrigger, { asChild: true, children: _jsx(Button, { "aria-label": `Open ${title.toLowerCase()} switcher`, className: cn(
|
|
95
|
+
return (_jsxs(Popover, { children: [_jsx(PopoverTrigger, { asChild: true, children: _jsx(Button, { "aria-label": `Open ${title.toLowerCase()} switcher`, className: cn(environmentBadgePlacementClasses, environment === "beta"
|
|
93
96
|
? "border-primary/80"
|
|
94
|
-
: "border-border/80 bg-background/95 text-foreground"), size: "sm", variant: environment === "beta" ? "default" : "outline", children: label }) }), _jsxs(PopoverContent, { align: "
|
|
97
|
+
: "border-border/80 bg-background/95 text-foreground"), size: "sm", variant: environment === "beta" ? "default" : "outline", children: label }) }), _jsxs(PopoverContent, { align: "start", className: "w-[280px] p-5", side: "top", sideOffset: 8, children: [_jsx("div", { className: "mb-1 text-sm font-semibold leading-5", children: title }), _jsx("div", { className: "mb-4 text-sm text-muted-foreground", children: "Choose where you want to continue." }), _jsx("div", { className: "grid gap-2", children: environment === "beta" ? (_jsx(EnvironmentLink, { href: productionHref, label: "Switch to production" })) : (_jsx(EnvironmentLink, { href: betaHref, label: "Go to beta" })) })] })] }));
|
|
98
|
+
}
|
|
99
|
+
function LocalEnvironmentBadge() {
|
|
100
|
+
return (_jsx("div", { "aria-label": "Local development environment", className: cn(environmentBadgePlacementClasses, "inline-flex items-center justify-center border border-border/80 bg-background/95 text-foreground"), role: "status", children: "dev" }));
|
|
95
101
|
}
|
|
96
102
|
function ProductionEnvironmentBadge({ targets, }) {
|
|
97
103
|
const { session, status } = useSession();
|
|
@@ -128,9 +134,10 @@ function ProductionEnvironmentBadge({ targets, }) {
|
|
|
128
134
|
return _jsx(EnvironmentBadgeContent, { environment: "production", targets: targets });
|
|
129
135
|
}
|
|
130
136
|
/**
|
|
131
|
-
*
|
|
132
|
-
* authentication so a visitor can always leave
|
|
133
|
-
* Production remains an internal auto-redirect
|
|
137
|
+
* Environment indicator and first-party hosted lane switcher. Beta is
|
|
138
|
+
* intentionally visible before authentication so a visitor can always leave
|
|
139
|
+
* beta from the sign-in page. Production remains an internal auto-redirect
|
|
140
|
+
* lane for authenticated staff.
|
|
134
141
|
*/
|
|
135
142
|
export function EnvironmentBadge({ showProduction = true, } = {}) {
|
|
136
143
|
const config = useMemo(injectedAgentNativeConfig, []);
|
|
@@ -139,10 +146,14 @@ export function EnvironmentBadge({ showProduction = true, } = {}) {
|
|
|
139
146
|
const targets = resolveEnvironmentTargets(hostname);
|
|
140
147
|
if (typeof window === "undefined" ||
|
|
141
148
|
window.parent !== window ||
|
|
142
|
-
!environment
|
|
143
|
-
!targets) {
|
|
149
|
+
!environment) {
|
|
144
150
|
return null;
|
|
145
151
|
}
|
|
152
|
+
if (environment === "local") {
|
|
153
|
+
return _jsx(LocalEnvironmentBadge, {});
|
|
154
|
+
}
|
|
155
|
+
if (!targets)
|
|
156
|
+
return null;
|
|
146
157
|
if (environment === "beta") {
|
|
147
158
|
return _jsx(EnvironmentBadgeContent, { environment: "beta", targets: targets });
|
|
148
159
|
}
|
|
@@ -15,8 +15,6 @@ export type AgentChatSurfaceKind =
|
|
|
15
15
|
| "desktop";
|
|
16
16
|
export declare const BACKGROUND_FOLLOW_ATTACH_WATCHDOG_MS = 90000;
|
|
17
17
|
export declare const BACKGROUND_FOLLOW_IDLE_TIMEOUT_MS = 210000;
|
|
18
|
-
export declare const MAX_FOLLOWED_BACKGROUND_RUNS = 24;
|
|
19
|
-
export declare const MAX_BACKGROUND_FOLLOW_WALL_TIME_MS: number;
|
|
20
18
|
/**
|
|
21
19
|
* True when an action was streamed but never returned a result yet — i.e. a
|
|
22
20
|
* `tool_start` with no matching `tool_done`. The server is still executing it,
|
|
@@ -170,29 +170,13 @@ export const BACKGROUND_FOLLOW_IDLE_TIMEOUT_MS = 210_000;
|
|
|
170
170
|
// 22 minutes, all error:stale_run, user watching a spinner). These bound the
|
|
171
171
|
// whole turn instead, and an identical repeated failure counts as no progress.
|
|
172
172
|
//
|
|
173
|
-
//
|
|
174
|
-
//
|
|
175
|
-
//
|
|
176
|
-
//
|
|
177
|
-
//
|
|
178
|
-
//
|
|
179
|
-
|
|
180
|
-
// MAX_BACKGROUND_RUN_CONTINUATIONS (20, production-agent.ts)
|
|
181
|
-
//
|
|
182
|
-
// They were originally set to 10 min / 6 runs, which put the whole-turn client
|
|
183
|
-
// budget BELOW the 13-minute ceiling of a single legal chunk. Any turn needing
|
|
184
|
-
// a second full-length chunk was killed by the client while the server was
|
|
185
|
-
// healthy and had 80 minutes left — measured in prod as turns dying at 11-25
|
|
186
|
-
// minutes with `last_progress_at` tracking the abort, i.e. still streaming
|
|
187
|
-
// tokens and completing tools when the client gave up. That is the top
|
|
188
|
-
// non-auth cause of "the chat just stopped" reports.
|
|
189
|
-
//
|
|
190
|
-
// Killing a turn that is NOT progressing is already covered twice over, by
|
|
191
|
-
// mechanisms that read progress rather than a clock: the 210s idle timeout
|
|
192
|
-
// above, and MAX_REPEATED_BACKGROUND_TERMINAL_REASONS below. Do not re-tighten
|
|
193
|
-
// these two to catch a stuck turn — fix the progress signal instead.
|
|
194
|
-
export const MAX_FOLLOWED_BACKGROUND_RUNS = 24;
|
|
195
|
-
export const MAX_BACKGROUND_FOLLOW_WALL_TIME_MS = 95 * 60_000;
|
|
173
|
+
// Defined in `app-config/run-lifecycle-invariants.ts`, not here, because the
|
|
174
|
+
// CLIENT-ABOVE-SERVER relationship they belong to is checked there against the
|
|
175
|
+
// RESOLVED server configuration — the server bounds are runtime-configurable
|
|
176
|
+
// now, so a spec pinning them against module constants stopped enforcing
|
|
177
|
+
// anything. Re-exported under the same names so importers are unchanged; that
|
|
178
|
+
// module has no runtime imports, so the bundle pays nothing for it.
|
|
179
|
+
import { MAX_BACKGROUND_FOLLOW_WALL_TIME_MS, MAX_FOLLOWED_BACKGROUND_RUNS, } from "../app-config/run-lifecycle-invariants.js";
|
|
196
180
|
const MAX_REPEATED_BACKGROUND_TERMINAL_REASONS = 3;
|
|
197
181
|
// A re-observed terminal run whose outcome would be an ERROR (never a
|
|
198
182
|
// genuine "done" success) gets a short extra grace window before the follow
|
|
@@ -39,7 +39,8 @@
|
|
|
39
39
|
* toaster — custom Toaster element rendered after children.
|
|
40
40
|
* Pass `null` to suppress the built-in Toaster when
|
|
41
41
|
* children already include a styled one.
|
|
42
|
-
* Defaults to
|
|
42
|
+
* Defaults to a rich-color bottom-left toaster raised above
|
|
43
|
+
* the environment badge.
|
|
43
44
|
* disableThemeTransitions — passed to next-themes ThemeProvider
|
|
44
45
|
* `disableTransitionOnChange`. Defaults to `true`
|
|
45
46
|
* (suppresses CSS transitions during theme switches,
|
|
@@ -75,7 +76,7 @@ export interface AppProvidersProps {
|
|
|
75
76
|
* Custom Toaster element rendered after children inside TooltipProvider.
|
|
76
77
|
* Pass `null` to suppress the built-in Toaster when children already
|
|
77
78
|
* include a styled one.
|
|
78
|
-
* Defaults to
|
|
79
|
+
* Defaults to a rich-color bottom-left toaster raised above the environment badge.
|
|
79
80
|
*/
|
|
80
81
|
toaster?: React.ReactNode | null;
|
|
81
82
|
/**
|
|
@@ -40,7 +40,8 @@ import { Fragment as _Fragment, jsx as _jsx, jsxs as _jsxs } from "react/jsx-run
|
|
|
40
40
|
* toaster — custom Toaster element rendered after children.
|
|
41
41
|
* Pass `null` to suppress the built-in Toaster when
|
|
42
42
|
* children already include a styled one.
|
|
43
|
-
* Defaults to
|
|
43
|
+
* Defaults to a rich-color bottom-left toaster raised above
|
|
44
|
+
* the environment badge.
|
|
44
45
|
* disableThemeTransitions — passed to next-themes ThemeProvider
|
|
45
46
|
* `disableTransitionOnChange`. Defaults to `true`
|
|
46
47
|
* (suppresses CSS transitions during theme switches,
|
|
@@ -65,7 +66,7 @@ import { AgentNativeRouteWarmup } from "./route-warmup.js";
|
|
|
65
66
|
import { RouteTransitionIndicator } from "./RouteTransitionIndicator.js";
|
|
66
67
|
import { RuntimeConfigNotice } from "./RuntimeConfigNotice.js";
|
|
67
68
|
import { EMBEDDED_THEME_CHANGE_EVENT, applyEmbeddedThemeUpdate, parseEmbeddedThemeUpdate, } from "./theme.js";
|
|
68
|
-
const DEFAULT_TOASTER = _jsx(Toaster, { richColors: true, position: "bottom-left" });
|
|
69
|
+
const DEFAULT_TOASTER = (_jsx(Toaster, { richColors: true, position: "bottom-left", offset: { bottom: 44, left: 32 }, mobileOffset: { bottom: 44, left: 16 } }));
|
|
69
70
|
function RoutedAppEnhancements() {
|
|
70
71
|
const isInRouter = useInRouterContext();
|
|
71
72
|
if (!isInRouter)
|
|
@@ -62,11 +62,11 @@ export declare const postAwareness: import("h3").EventHandlerWithFetch<import("h
|
|
|
62
62
|
error: string;
|
|
63
63
|
states?: undefined;
|
|
64
64
|
} | {
|
|
65
|
-
error?: undefined;
|
|
66
65
|
states: {
|
|
67
66
|
clientId: number;
|
|
68
67
|
state: string;
|
|
69
68
|
}[];
|
|
69
|
+
error?: undefined;
|
|
70
70
|
}>>;
|
|
71
71
|
/**
|
|
72
72
|
* GET /_agent-native/collab/:docId/users
|
|
@@ -77,9 +77,9 @@ export declare const getActiveUsers: import("h3").EventHandlerWithFetch<import("
|
|
|
77
77
|
error: string;
|
|
78
78
|
users?: undefined;
|
|
79
79
|
} | {
|
|
80
|
-
error?: undefined;
|
|
81
80
|
users: {
|
|
82
81
|
clientId: number;
|
|
83
82
|
lastSeen: number;
|
|
84
83
|
}[];
|
|
84
|
+
error?: undefined;
|
|
85
85
|
}>>;
|
package/dist/collab/routes.d.ts
CHANGED
|
@@ -26,8 +26,8 @@ export declare const getCollabState: import("h3").EventHandlerWithFetch<import("
|
|
|
26
26
|
* Body: { update: string (base64), requestSource?: string }
|
|
27
27
|
*/
|
|
28
28
|
export declare const postCollabUpdate: import("h3").EventHandlerWithFetch<import("h3").EventHandlerRequest, Promise<{
|
|
29
|
-
error: string;
|
|
30
29
|
ok?: undefined;
|
|
30
|
+
error: string;
|
|
31
31
|
} | {
|
|
32
32
|
error?: undefined;
|
|
33
33
|
ok: boolean;
|
|
@@ -5,7 +5,23 @@ import { type AutomationExecutionIdentity } from "../automations/service.js";
|
|
|
5
5
|
import { type Resource } from "../resources/store.js";
|
|
6
6
|
import { type RequestContext } from "../server/request-context.js";
|
|
7
7
|
import type { JobFrontmatter } from "./frontmatter.js";
|
|
8
|
+
/**
|
|
9
|
+
* Default hard abort for one in-process automation run. Read through
|
|
10
|
+
* `resolveBackgroundRunHardTimeoutMs()` at the use site — this is the host's
|
|
11
|
+
* real function budget for scheduled work, and it differs by deployment.
|
|
12
|
+
*/
|
|
8
13
|
export declare const BACKGROUND_RUN_HARD_TIMEOUT_MS: number;
|
|
14
|
+
/**
|
|
15
|
+
* Terminal failure of a background automation, carrying the machine-readable
|
|
16
|
+
* code the failure taxonomy already computes.
|
|
17
|
+
*
|
|
18
|
+
* The code used to be produced and then dropped, so "how often are runs cut
|
|
19
|
+
* off?" was a `LIKE '%no_progress%'` over an English sentence.
|
|
20
|
+
*/
|
|
21
|
+
export declare class BackgroundAutomationRunError extends Error {
|
|
22
|
+
readonly errorCode: string;
|
|
23
|
+
constructor(message: string, errorCode: string);
|
|
24
|
+
}
|
|
9
25
|
export interface BackgroundAutomationContext {
|
|
10
26
|
name: string;
|
|
11
27
|
meta: JobFrontmatter;
|
|
@@ -35,6 +51,15 @@ export interface BackgroundAutomationRunOptions {
|
|
|
35
51
|
actionAutomation?: ActionAutomationContext;
|
|
36
52
|
/** Reuse a history row created by a durable run-now enqueue. */
|
|
37
53
|
historyId?: string;
|
|
54
|
+
/**
|
|
55
|
+
* Per-run overrides for the run-manager no-progress backstop. `startRun` has
|
|
56
|
+
* always accepted these; the automation path had no way to reach them, and
|
|
57
|
+
* the one indirect route (zeroing `agent.runSoftTimeoutMs`) is global and
|
|
58
|
+
* would strip foreground chat of its chunk boundary. Additive: unset means
|
|
59
|
+
* the configured/default behaviour, unchanged.
|
|
60
|
+
*/
|
|
61
|
+
noProgressTimeoutMs?: number;
|
|
62
|
+
backgroundNoProgressTimeoutMs?: number;
|
|
38
63
|
}
|
|
39
64
|
export interface BackgroundAutomationRunResult {
|
|
40
65
|
responseText: string;
|
|
@@ -2,7 +2,7 @@ import { collectFinalResponseTextFromAgentEvents } from "../a2a/response-text.js
|
|
|
2
2
|
import { getStoredModelForEngine, normalizeModelForEngine, resolveEngine, } from "../agent/engine/index.js";
|
|
3
3
|
import { actionsToEngineTools, filterInitialEngineTools, getOwnerActiveApiKey, } from "../agent/production-agent.js";
|
|
4
4
|
import { runAgentLoopDirectWithSoftTimeout } from "../agent/run-loop-with-resume.js";
|
|
5
|
-
import {
|
|
5
|
+
import { abortRun, resolveBackgroundAutomationSoftTimeoutMs, resolveBackgroundRunHardTimeoutMs, startRun, } from "../agent/run-manager.js";
|
|
6
6
|
import { claimBackgroundRun, insertRun } from "../agent/run-store.js";
|
|
7
7
|
import { buildAssistantMessage, buildUserMessage, extractThreadMeta, foldAssistantTurn, upsertUserMessage, } from "../agent/thread-data-builder.js";
|
|
8
8
|
import { attachToolSearch } from "../agent/tool-search.js";
|
|
@@ -13,8 +13,27 @@ import { organizationIdFromResourceOwner, organizationResourceOwner, } from "../
|
|
|
13
13
|
import { captureError } from "../server/capture-error.js";
|
|
14
14
|
import { runWithRequestContext, } from "../server/request-context.js";
|
|
15
15
|
import { attachAutomationRunThread, finishAutomationRun, startAutomationRun, } from "./run-history.js";
|
|
16
|
-
|
|
16
|
+
/**
|
|
17
|
+
* Default hard abort for one in-process automation run. Read through
|
|
18
|
+
* `resolveBackgroundRunHardTimeoutMs()` at the use site — this is the host's
|
|
19
|
+
* real function budget for scheduled work, and it differs by deployment.
|
|
20
|
+
*/
|
|
17
21
|
export const BACKGROUND_RUN_HARD_TIMEOUT_MS = 10 * 60_000;
|
|
22
|
+
/**
|
|
23
|
+
* Terminal failure of a background automation, carrying the machine-readable
|
|
24
|
+
* code the failure taxonomy already computes.
|
|
25
|
+
*
|
|
26
|
+
* The code used to be produced and then dropped, so "how often are runs cut
|
|
27
|
+
* off?" was a `LIKE '%no_progress%'` over an English sentence.
|
|
28
|
+
*/
|
|
29
|
+
export class BackgroundAutomationRunError extends Error {
|
|
30
|
+
errorCode;
|
|
31
|
+
constructor(message, errorCode) {
|
|
32
|
+
super(message);
|
|
33
|
+
this.name = "BackgroundAutomationRunError";
|
|
34
|
+
this.errorCode = errorCode;
|
|
35
|
+
}
|
|
36
|
+
}
|
|
18
37
|
/**
|
|
19
38
|
* A persisted background run must not outlive its execution identity.
|
|
20
39
|
* Organization runs fail closed when membership state cannot be read. A
|
|
@@ -112,8 +131,11 @@ export function isBackgroundAutomationRunActive(meta, now = new Date()) {
|
|
|
112
131
|
if (!meta.lastRun)
|
|
113
132
|
return false;
|
|
114
133
|
const startedAt = new Date(meta.lastRun).getTime();
|
|
134
|
+
// Tracks the hard abort: past it no run of this automation is still alive, so
|
|
135
|
+
// a deployment that raises the abort must not have its live runs treated as
|
|
136
|
+
// stuck and re-dispatched underneath themselves.
|
|
115
137
|
return (Number.isFinite(startedAt) &&
|
|
116
|
-
now.getTime() - startedAt <
|
|
138
|
+
now.getTime() - startedAt < resolveBackgroundRunHardTimeoutMs());
|
|
117
139
|
}
|
|
118
140
|
/**
|
|
119
141
|
* A soft-timeout/no-progress checkpoint is a continuation boundary, not a
|
|
@@ -186,6 +208,9 @@ export async function runBackgroundAutomation(options, deps) {
|
|
|
186
208
|
}
|
|
187
209
|
catch (err) {
|
|
188
210
|
const message = err instanceof Error ? err.message : String(err);
|
|
211
|
+
const errorCode = err instanceof BackgroundAutomationRunError
|
|
212
|
+
? err.errorCode
|
|
213
|
+
: "background_automation_failed";
|
|
189
214
|
// Both callers (recurring-jobs scheduler, trigger dispatcher) record this
|
|
190
215
|
// onto the automation's own metadata and console.error it, and neither
|
|
191
216
|
// reports it. A failure visible only in a resource field and stdout is not
|
|
@@ -204,7 +229,7 @@ export async function runBackgroundAutomation(options, deps) {
|
|
|
204
229
|
},
|
|
205
230
|
...(runIdRef.current ? { aiTraceId: runIdRef.current } : {}),
|
|
206
231
|
});
|
|
207
|
-
await recordRunOutcome(historyId, "error", `${message}. No delivery was confirmed
|
|
232
|
+
await recordRunOutcome(historyId, "error", `${message}. No delivery was confirmed.`, errorCode);
|
|
208
233
|
throw err;
|
|
209
234
|
}
|
|
210
235
|
// Outside the try: history is bookkeeping about the run, so a failure to
|
|
@@ -229,8 +254,9 @@ async function recordRunThread(historyId, threadId, runId) {
|
|
|
229
254
|
}
|
|
230
255
|
function backgroundAutomationPersistFailure(input) {
|
|
231
256
|
if (input.hardTimedOut) {
|
|
257
|
+
const minutes = Math.round((input.hardTimeoutMs ?? BACKGROUND_RUN_HARD_TIMEOUT_MS) / 60_000);
|
|
232
258
|
return {
|
|
233
|
-
message: `Background automation timed out after ${
|
|
259
|
+
message: `Background automation timed out after ${minutes} minutes`,
|
|
234
260
|
errorCode: "background_automation_hard_timeout",
|
|
235
261
|
};
|
|
236
262
|
}
|
|
@@ -301,11 +327,11 @@ async function persistBackgroundAutomationTurn(input) {
|
|
|
301
327
|
await updateThreadData(input.threadId, JSON.stringify(repo), input.threadTitle || row.title, meta.preview || row.preview, Array.isArray(messages) ? messages.length : 0);
|
|
302
328
|
});
|
|
303
329
|
}
|
|
304
|
-
async function recordRunOutcome(historyId, status, error) {
|
|
330
|
+
async function recordRunOutcome(historyId, status, error, errorCode) {
|
|
305
331
|
if (!historyId)
|
|
306
332
|
return;
|
|
307
333
|
try {
|
|
308
|
-
await finishAutomationRun(historyId, status, error);
|
|
334
|
+
await finishAutomationRun(historyId, status, error, errorCode);
|
|
309
335
|
}
|
|
310
336
|
catch (err) {
|
|
311
337
|
console.error(`[automations] Could not record run ${historyId} as ${status}:`, err);
|
|
@@ -372,12 +398,15 @@ async function executeBackgroundAutomation(options, deps, historyId, runIdRef) {
|
|
|
372
398
|
// Hardcoded rather than `isInBackgroundFunctionRuntime()` (what
|
|
373
399
|
// webhook-handler.ts uses): a webhook can arrive on either runtime, but
|
|
374
400
|
// a scheduler tick never serves a synchronous request, so the
|
|
375
|
-
// interactive clamp never applies to it.
|
|
376
|
-
//
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
401
|
+
// interactive clamp never applies to it.
|
|
402
|
+
//
|
|
403
|
+
// Derived from this runner's OWN hard abort, not from the durable-chat
|
|
404
|
+
// background ceiling: that ceiling is 13 minutes and this process is
|
|
405
|
+
// killed at 10, so taking it left the recoverable soft-timeout boundary
|
|
406
|
+
// as dead code and the terminal no-progress backstop as the only
|
|
407
|
+
// boundary an automation could reach.
|
|
408
|
+
const hardTimeoutMs = resolveBackgroundRunHardTimeoutMs();
|
|
409
|
+
const softTimeoutMs = resolveBackgroundAutomationSoftTimeoutMs();
|
|
381
410
|
const usageRef = { current: null };
|
|
382
411
|
let responseText = "";
|
|
383
412
|
let hardAbortTimer = null;
|
|
@@ -400,8 +429,8 @@ async function executeBackgroundAutomation(options, deps, historyId, runIdRef) {
|
|
|
400
429
|
throw new Error(`Background automation "${automation.name}" (run "${runId}") could not claim its own freshly-inserted run row`);
|
|
401
430
|
}
|
|
402
431
|
await new Promise((resolve, reject) => {
|
|
403
|
-
const activeRun = startRun(runId, thread.id, async (send, signal) => {
|
|
404
|
-
|
|
432
|
+
const activeRun = startRun(runId, thread.id, async (send, signal, control) => {
|
|
433
|
+
const loopOpts = {
|
|
405
434
|
engine,
|
|
406
435
|
model,
|
|
407
436
|
systemPrompt,
|
|
@@ -425,7 +454,45 @@ async function executeBackgroundAutomation(options, deps, historyId, runIdRef) {
|
|
|
425
454
|
runId,
|
|
426
455
|
maxIterations: automation.meta.maxIterations,
|
|
427
456
|
maxRunInputTokens: automation.meta.maxRunInputTokens,
|
|
428
|
-
}
|
|
457
|
+
};
|
|
458
|
+
// Same adapter A2A uses: bridge this runner's multi-argument shape
|
|
459
|
+
// to the single-argument `runAgentLoop` `instrumentAgentLoop`
|
|
460
|
+
// expects. `control` is what lets a chunk boundary be recovered
|
|
461
|
+
// here instead of ending the turn.
|
|
462
|
+
const execute = (o = loopOpts) => runAgentLoopDirectWithSoftTimeout(o, softTimeoutMs, { backgroundFunction: true }, control);
|
|
463
|
+
let instrumented = false;
|
|
464
|
+
try {
|
|
465
|
+
const { getObservabilityConfig, instrumentAgentLoop } = await import("../observability/traces.js");
|
|
466
|
+
const config = await getObservabilityConfig();
|
|
467
|
+
if (config.enabled) {
|
|
468
|
+
instrumented = true;
|
|
469
|
+
usageRef.current = await instrumentAgentLoop({
|
|
470
|
+
runAgentLoop: (o) => execute(o),
|
|
471
|
+
loopOpts,
|
|
472
|
+
runId,
|
|
473
|
+
threadId: thread.id,
|
|
474
|
+
// A scheduled run is NOT anonymous. Passing the owner is what
|
|
475
|
+
// makes it visible to per-user observability reads.
|
|
476
|
+
userId: ownerEmail,
|
|
477
|
+
config,
|
|
478
|
+
spanName: "background_automation_run",
|
|
479
|
+
metadata: {
|
|
480
|
+
automation: automation.name,
|
|
481
|
+
trigger: "background_automation",
|
|
482
|
+
scope: orgId ? "organization" : "personal",
|
|
483
|
+
},
|
|
484
|
+
});
|
|
485
|
+
return;
|
|
486
|
+
}
|
|
487
|
+
}
|
|
488
|
+
catch (error) {
|
|
489
|
+
// Match A2A and interactive chat: a setup failure falls through
|
|
490
|
+
// to an uninstrumented run, but a failure from INSIDE the
|
|
491
|
+
// instrumented loop is the real run failure and must rethrow.
|
|
492
|
+
if (instrumented)
|
|
493
|
+
throw error;
|
|
494
|
+
}
|
|
495
|
+
usageRef.current = await execute();
|
|
429
496
|
}, async (run) => {
|
|
430
497
|
if (hardAbortTimer) {
|
|
431
498
|
clearTimeout(hardAbortTimer);
|
|
@@ -434,6 +501,7 @@ async function executeBackgroundAutomation(options, deps, historyId, runIdRef) {
|
|
|
434
501
|
const persistFailure = backgroundAutomationPersistFailure({
|
|
435
502
|
run,
|
|
436
503
|
hardTimedOut,
|
|
504
|
+
hardTimeoutMs,
|
|
437
505
|
});
|
|
438
506
|
try {
|
|
439
507
|
await persistBackgroundAutomationTurn({
|
|
@@ -453,11 +521,11 @@ async function executeBackgroundAutomation(options, deps, historyId, runIdRef) {
|
|
|
453
521
|
if (hardTimedOut)
|
|
454
522
|
return;
|
|
455
523
|
if (persistFailure) {
|
|
456
|
-
reject(new
|
|
524
|
+
reject(new BackgroundAutomationRunError(persistFailure.message, persistFailure.errorCode));
|
|
457
525
|
return;
|
|
458
526
|
}
|
|
459
527
|
if (run.status !== "completed") {
|
|
460
|
-
reject(new
|
|
528
|
+
reject(new BackgroundAutomationRunError(`Background automation ended with status: ${run.status}`, `background_automation_${run.status}`));
|
|
461
529
|
return;
|
|
462
530
|
}
|
|
463
531
|
responseText = collectFinalResponseTextFromAgentEvents((run.events ?? []).map((entry) => entry.event));
|
|
@@ -465,22 +533,37 @@ async function executeBackgroundAutomation(options, deps, historyId, runIdRef) {
|
|
|
465
533
|
}, {
|
|
466
534
|
softTimeoutMs,
|
|
467
535
|
backgroundFunction: true,
|
|
536
|
+
// This runner owns continuation in-process: there is no HTTP body
|
|
537
|
+
// to re-POST and no `chainServerDrivenContinuation` behind it, so a
|
|
538
|
+
// checkpoint must end the CHUNK and let the loop above recover it.
|
|
539
|
+
recoverChunkBoundaries: true,
|
|
540
|
+
// Matches the `dispatch_mode` this runner already writes onto the
|
|
541
|
+
// run row at insert. Without it the terminal and boundary analytics
|
|
542
|
+
// events reported every scheduled run as foreground.
|
|
543
|
+
dispatchMode: "background",
|
|
544
|
+
noProgressTimeoutMs: options.noProgressTimeoutMs,
|
|
545
|
+
backgroundNoProgressTimeoutMs: options.backgroundNoProgressTimeoutMs,
|
|
468
546
|
model,
|
|
469
547
|
engineName: engine.name,
|
|
548
|
+
userId: ownerEmail,
|
|
470
549
|
});
|
|
471
550
|
hardAbortTimer = setTimeout(() => {
|
|
472
551
|
hardAbortTimer = null;
|
|
473
552
|
if (activeRun.status !== "running")
|
|
474
553
|
return;
|
|
475
554
|
hardTimedOut = true;
|
|
476
|
-
activeRun.abort.abort
|
|
477
|
-
|
|
555
|
+
// `abortRun`, not `activeRun.abort.abort`: the controller alone
|
|
556
|
+
// carries no reason the run manager can see, so finalization fell
|
|
557
|
+
// through to `aborted:user` and a hard timeout was recorded as a
|
|
558
|
+
// person pressing Stop.
|
|
559
|
+
abortRun(runId, "background_automation_hard_timeout");
|
|
560
|
+
const timeoutError = new BackgroundAutomationRunError(`Background automation timed out after ${Math.round(hardTimeoutMs / 60_000)} minutes`, "background_automation_hard_timeout");
|
|
478
561
|
void activeRun.finalized
|
|
479
562
|
.catch(() => { })
|
|
480
563
|
.then(() => {
|
|
481
564
|
reject(timeoutError);
|
|
482
565
|
});
|
|
483
|
-
},
|
|
566
|
+
}, hardTimeoutMs);
|
|
484
567
|
}).finally(() => {
|
|
485
568
|
if (hardAbortTimer) {
|
|
486
569
|
clearTimeout(hardAbortTimer);
|
|
@@ -20,6 +20,12 @@ export interface AutomationRun {
|
|
|
20
20
|
startedAt: number;
|
|
21
21
|
finishedAt: number | null;
|
|
22
22
|
error: string | null;
|
|
23
|
+
/**
|
|
24
|
+
* Machine-readable failure code, so "how often are runs cut off?" is a
|
|
25
|
+
* GROUP BY instead of a LIKE over an English sentence. Null on success and
|
|
26
|
+
* on rows written before the column existed.
|
|
27
|
+
*/
|
|
28
|
+
errorCode: string | null;
|
|
23
29
|
}
|
|
24
30
|
export interface StartAutomationRunInput {
|
|
25
31
|
owner: string;
|
|
@@ -56,7 +62,7 @@ export declare function listUnclaimedAutomationRuns(options?: {
|
|
|
56
62
|
olderThanMs?: number;
|
|
57
63
|
limit?: number;
|
|
58
64
|
}): Promise<AutomationRun[]>;
|
|
59
|
-
export declare function finishAutomationRun(id: string, status: Exclude<AutomationRunStatus, "running">, error?: string): Promise<void>;
|
|
65
|
+
export declare function finishAutomationRun(id: string, status: Exclude<AutomationRunStatus, "running">, error?: string, errorCode?: string): Promise<void>;
|
|
60
66
|
/**
|
|
61
67
|
* Attach the agent thread once it exists. The thread is created after the run
|
|
62
68
|
* row so the history survives a crash between the two.
|