@aitne/daemon 0.1.4 → 0.1.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/notification-manager.d.ts +12 -0
- package/dist/adapters/notification-manager.d.ts.map +1 -1
- package/dist/adapters/notification-manager.js +39 -1
- package/dist/adapters/notification-manager.js.map +1 -1
- package/dist/api/routes/agent.d.ts.map +1 -1
- package/dist/api/routes/agent.js +7 -0
- package/dist/api/routes/agent.js.map +1 -1
- package/dist/api/routes/commands.d.ts.map +1 -1
- package/dist/api/routes/commands.js +16 -13
- package/dist/api/routes/commands.js.map +1 -1
- package/dist/api/routes/context.d.ts.map +1 -1
- package/dist/api/routes/context.js +13 -2
- package/dist/api/routes/context.js.map +1 -1
- package/dist/api/routes/dashboard.d.ts.map +1 -1
- package/dist/api/routes/dashboard.js +28 -0
- package/dist/api/routes/dashboard.js.map +1 -1
- package/dist/api/routes/fs.d.ts +23 -0
- package/dist/api/routes/fs.d.ts.map +1 -0
- package/dist/api/routes/fs.js +156 -0
- package/dist/api/routes/fs.js.map +1 -0
- package/dist/api/routes/fs.logic.d.ts +62 -0
- package/dist/api/routes/fs.logic.d.ts.map +1 -0
- package/dist/api/routes/fs.logic.js +137 -0
- package/dist/api/routes/fs.logic.js.map +1 -0
- package/dist/api/routes/health.d.ts.map +1 -1
- package/dist/api/routes/health.js +4 -2
- package/dist/api/routes/health.js.map +1 -1
- package/dist/api/routes/integrations.d.ts.map +1 -1
- package/dist/api/routes/integrations.js +8 -6
- package/dist/api/routes/integrations.js.map +1 -1
- package/dist/api/routes/metrics.d.ts +1 -0
- package/dist/api/routes/metrics.d.ts.map +1 -1
- package/dist/api/routes/metrics.js +24 -0
- package/dist/api/routes/metrics.js.map +1 -1
- package/dist/api/routes/observations.d.ts.map +1 -1
- package/dist/api/routes/observations.js +538 -25
- package/dist/api/routes/observations.js.map +1 -1
- package/dist/api/routes/skills.d.ts +9 -1
- package/dist/api/routes/skills.d.ts.map +1 -1
- package/dist/api/routes/skills.js +38 -16
- package/dist/api/routes/skills.js.map +1 -1
- package/dist/api/routes/wiki.d.ts +4 -0
- package/dist/api/routes/wiki.d.ts.map +1 -0
- package/dist/api/routes/wiki.js +1075 -0
- package/dist/api/routes/wiki.js.map +1 -0
- package/dist/api/server.d.ts +13 -0
- package/dist/api/server.d.ts.map +1 -1
- package/dist/api/server.js +27 -1
- package/dist/api/server.js.map +1 -1
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +26 -0
- package/dist/config.js.map +1 -1
- package/dist/core/agent-core.d.ts +25 -0
- package/dist/core/agent-core.d.ts.map +1 -1
- package/dist/core/agent-core.js.map +1 -1
- package/dist/core/backends/backend-router.d.ts +5 -1
- package/dist/core/backends/backend-router.d.ts.map +1 -1
- package/dist/core/backends/backend-router.js +10 -1
- package/dist/core/backends/backend-router.js.map +1 -1
- package/dist/core/backends/claude-code-core.d.ts.map +1 -1
- package/dist/core/backends/claude-code-core.js +62 -4
- package/dist/core/backends/claude-code-core.js.map +1 -1
- package/dist/core/backends/claude-tool-collection.d.ts +1 -1
- package/dist/core/backends/claude-tool-collection.d.ts.map +1 -1
- package/dist/core/backends/claude-tool-collection.js +327 -65
- package/dist/core/backends/claude-tool-collection.js.map +1 -1
- package/dist/core/backends/codex-core.d.ts.map +1 -1
- package/dist/core/backends/codex-core.js +36 -0
- package/dist/core/backends/codex-core.js.map +1 -1
- package/dist/core/backends/gemini-cli-core.d.ts +24 -5
- package/dist/core/backends/gemini-cli-core.d.ts.map +1 -1
- package/dist/core/backends/gemini-cli-core.js +62 -30
- package/dist/core/backends/gemini-cli-core.js.map +1 -1
- package/dist/core/backends/plan-presets.d.ts +3 -1
- package/dist/core/backends/plan-presets.d.ts.map +1 -1
- package/dist/core/backends/plan-presets.js +42 -2
- package/dist/core/backends/plan-presets.js.map +1 -1
- package/dist/core/bang-commands/commands-help.d.ts +5 -0
- package/dist/core/bang-commands/commands-help.d.ts.map +1 -0
- package/dist/core/bang-commands/commands-help.js +69 -0
- package/dist/core/bang-commands/commands-help.js.map +1 -0
- package/dist/core/bang-commands/commands-wiki.d.ts +75 -0
- package/dist/core/bang-commands/commands-wiki.d.ts.map +1 -0
- package/dist/core/bang-commands/commands-wiki.js +574 -0
- package/dist/core/bang-commands/commands-wiki.js.map +1 -0
- package/dist/core/bang-commands/index.d.ts +4 -2
- package/dist/core/bang-commands/index.d.ts.map +1 -1
- package/dist/core/bang-commands/index.js +15 -1
- package/dist/core/bang-commands/index.js.map +1 -1
- package/dist/core/bang-commands/registry.d.ts +47 -4
- package/dist/core/bang-commands/registry.d.ts.map +1 -1
- package/dist/core/bang-commands/registry.js +85 -15
- package/dist/core/bang-commands/registry.js.map +1 -1
- package/dist/core/context-builder.d.ts +17 -0
- package/dist/core/context-builder.d.ts.map +1 -1
- package/dist/core/context-builder.js +64 -6
- package/dist/core/context-builder.js.map +1 -1
- package/dist/core/daemon-api-cli.d.ts.map +1 -1
- package/dist/core/daemon-api-cli.js +50 -2
- package/dist/core/daemon-api-cli.js.map +1 -1
- package/dist/core/dispatcher-message-handler.d.ts.map +1 -1
- package/dist/core/dispatcher-message-handler.js +10 -0
- package/dist/core/dispatcher-message-handler.js.map +1 -1
- package/dist/core/dispatcher-morning-routine.d.ts.map +1 -1
- package/dist/core/dispatcher-morning-routine.js +17 -2
- package/dist/core/dispatcher-morning-routine.js.map +1 -1
- package/dist/core/dispatcher-result-processor.d.ts +23 -0
- package/dist/core/dispatcher-result-processor.d.ts.map +1 -1
- package/dist/core/dispatcher-result-processor.js +124 -5
- package/dist/core/dispatcher-result-processor.js.map +1 -1
- package/dist/core/dispatcher-scheduled-tasks.d.ts.map +1 -1
- package/dist/core/dispatcher-scheduled-tasks.js +114 -80
- package/dist/core/dispatcher-scheduled-tasks.js.map +1 -1
- package/dist/core/dispatcher-types.d.ts +116 -1
- package/dist/core/dispatcher-types.d.ts.map +1 -1
- package/dist/core/dispatcher-types.js.map +1 -1
- package/dist/core/dispatcher.d.ts +36 -0
- package/dist/core/dispatcher.d.ts.map +1 -1
- package/dist/core/dispatcher.js +94 -1
- package/dist/core/dispatcher.js.map +1 -1
- package/dist/core/integration-lifecycle.d.ts.map +1 -1
- package/dist/core/integration-lifecycle.js +6 -8
- package/dist/core/integration-lifecycle.js.map +1 -1
- package/dist/core/metrics.d.ts +127 -0
- package/dist/core/metrics.d.ts.map +1 -1
- package/dist/core/metrics.js +256 -1
- package/dist/core/metrics.js.map +1 -1
- package/dist/core/prompts.d.ts +2 -1
- package/dist/core/prompts.d.ts.map +1 -1
- package/dist/core/prompts.js +40 -0
- package/dist/core/prompts.js.map +1 -1
- package/dist/core/roadmap-validate.js +13 -1
- package/dist/core/roadmap-validate.js.map +1 -1
- package/dist/core/routine-acquisition-plan.d.ts +51 -0
- package/dist/core/routine-acquisition-plan.d.ts.map +1 -1
- package/dist/core/routine-acquisition-plan.js +111 -12
- package/dist/core/routine-acquisition-plan.js.map +1 -1
- package/dist/core/routine-fetch-window-retry.d.ts +109 -0
- package/dist/core/routine-fetch-window-retry.d.ts.map +1 -0
- package/dist/core/routine-fetch-window-retry.js +210 -0
- package/dist/core/routine-fetch-window-retry.js.map +1 -0
- package/dist/core/routine-fetch-window-runner.d.ts +258 -32
- package/dist/core/routine-fetch-window-runner.d.ts.map +1 -1
- package/dist/core/routine-fetch-window-runner.js +1115 -185
- package/dist/core/routine-fetch-window-runner.js.map +1 -1
- package/dist/core/routine-windows.d.ts +19 -4
- package/dist/core/routine-windows.d.ts.map +1 -1
- package/dist/core/routine-windows.js +47 -0
- package/dist/core/routine-windows.js.map +1 -1
- package/dist/core/scheduler.d.ts +50 -2
- package/dist/core/scheduler.d.ts.map +1 -1
- package/dist/core/scheduler.js +88 -7
- package/dist/core/scheduler.js.map +1 -1
- package/dist/core/skill-curation/declarations.d.ts.map +1 -1
- package/dist/core/skill-curation/declarations.js +11 -12
- package/dist/core/skill-curation/declarations.js.map +1 -1
- package/dist/core/skill-source-paths.d.ts +14 -0
- package/dist/core/skill-source-paths.d.ts.map +1 -0
- package/dist/core/skill-source-paths.js +82 -0
- package/dist/core/skill-source-paths.js.map +1 -0
- package/dist/core/skills-compiler.d.ts +18 -0
- package/dist/core/skills-compiler.d.ts.map +1 -1
- package/dist/core/skills-compiler.js +65 -18
- package/dist/core/skills-compiler.js.map +1 -1
- package/dist/core/skills-manifest.d.ts.map +1 -1
- package/dist/core/skills-manifest.js +46 -0
- package/dist/core/skills-manifest.js.map +1 -1
- package/dist/core/system-reset.d.ts +25 -0
- package/dist/core/system-reset.d.ts.map +1 -1
- package/dist/core/system-reset.js +47 -0
- package/dist/core/system-reset.js.map +1 -1
- package/dist/core/wiki/approval-queue.d.ts +31 -0
- package/dist/core/wiki/approval-queue.d.ts.map +1 -0
- package/dist/core/wiki/approval-queue.js +44 -0
- package/dist/core/wiki/approval-queue.js.map +1 -0
- package/dist/core/wiki/bridge.d.ts +74 -0
- package/dist/core/wiki/bridge.d.ts.map +1 -0
- package/dist/core/wiki/bridge.js +405 -0
- package/dist/core/wiki/bridge.js.map +1 -0
- package/dist/core/wiki/compile-lock.d.ts +42 -0
- package/dist/core/wiki/compile-lock.d.ts.map +1 -0
- package/dist/core/wiki/compile-lock.js +55 -0
- package/dist/core/wiki/compile-lock.js.map +1 -0
- package/dist/core/wiki/compile-preview.d.ts +8 -0
- package/dist/core/wiki/compile-preview.d.ts.map +1 -0
- package/dist/core/wiki/compile-preview.js +200 -0
- package/dist/core/wiki/compile-preview.js.map +1 -0
- package/dist/core/wiki/cost-estimate.d.ts +30 -0
- package/dist/core/wiki/cost-estimate.d.ts.map +1 -0
- package/dist/core/wiki/cost-estimate.js +243 -0
- package/dist/core/wiki/cost-estimate.js.map +1 -0
- package/dist/core/wiki/dispatcher.d.ts +48 -0
- package/dist/core/wiki/dispatcher.d.ts.map +1 -0
- package/dist/core/wiki/dispatcher.js +92 -0
- package/dist/core/wiki/dispatcher.js.map +1 -0
- package/dist/core/wiki/git-precompile.d.ts +86 -0
- package/dist/core/wiki/git-precompile.d.ts.map +1 -0
- package/dist/core/wiki/git-precompile.js +96 -0
- package/dist/core/wiki/git-precompile.js.map +1 -0
- package/dist/core/wiki/import-migrate.d.ts +38 -0
- package/dist/core/wiki/import-migrate.d.ts.map +1 -0
- package/dist/core/wiki/import-migrate.js +310 -0
- package/dist/core/wiki/import-migrate.js.map +1 -0
- package/dist/core/wiki/import-probe.d.ts +76 -0
- package/dist/core/wiki/import-probe.d.ts.map +1 -0
- package/dist/core/wiki/import-probe.js +245 -0
- package/dist/core/wiki/import-probe.js.map +1 -0
- package/dist/core/wiki/index-cache.d.ts +39 -0
- package/dist/core/wiki/index-cache.d.ts.map +1 -0
- package/dist/core/wiki/index-cache.js +152 -0
- package/dist/core/wiki/index-cache.js.map +1 -0
- package/dist/core/wiki/multi-url-dispatch.d.ts +52 -0
- package/dist/core/wiki/multi-url-dispatch.d.ts.map +1 -0
- package/dist/core/wiki/multi-url-dispatch.js +72 -0
- package/dist/core/wiki/multi-url-dispatch.js.map +1 -0
- package/dist/core/wiki/wiki-fts.d.ts +75 -0
- package/dist/core/wiki/wiki-fts.d.ts.map +1 -0
- package/dist/core/wiki/wiki-fts.js +265 -0
- package/dist/core/wiki/wiki-fts.js.map +1 -0
- package/dist/core/wiki/workspaces.d.ts +101 -0
- package/dist/core/wiki/workspaces.d.ts.map +1 -0
- package/dist/core/wiki/workspaces.js +352 -0
- package/dist/core/wiki/workspaces.js.map +1 -0
- package/dist/core/wiki/write-strategy.d.ts +70 -0
- package/dist/core/wiki/write-strategy.d.ts.map +1 -0
- package/dist/core/wiki/write-strategy.js +112 -0
- package/dist/core/wiki/write-strategy.js.map +1 -0
- package/dist/core/workdir.d.ts +8 -1
- package/dist/core/workdir.d.ts.map +1 -1
- package/dist/core/workdir.js +4 -1
- package/dist/core/workdir.js.map +1 -1
- package/dist/db/schema.d.ts.map +1 -1
- package/dist/db/schema.js +122 -0
- package/dist/db/schema.js.map +1 -1
- package/dist/db/wiki-store.d.ts +3 -0
- package/dist/db/wiki-store.d.ts.map +1 -0
- package/dist/db/wiki-store.js +7 -0
- package/dist/db/wiki-store.js.map +1 -0
- package/dist/index.js +80 -4
- package/dist/index.js.map +1 -1
- package/dist/messaging/url-extract.d.ts +8 -0
- package/dist/messaging/url-extract.d.ts.map +1 -0
- package/dist/messaging/url-extract.js +41 -0
- package/dist/messaging/url-extract.js.map +1 -0
- package/dist/observers/delegated-sync-worker.d.ts +33 -25
- package/dist/observers/delegated-sync-worker.d.ts.map +1 -1
- package/dist/observers/delegated-sync-worker.js +38 -31
- package/dist/observers/delegated-sync-worker.js.map +1 -1
- package/dist/observers/imminent-event-scheduler.d.ts +20 -7
- package/dist/observers/imminent-event-scheduler.d.ts.map +1 -1
- package/dist/observers/imminent-event-scheduler.js +134 -29
- package/dist/observers/imminent-event-scheduler.js.map +1 -1
- package/dist/safety/always-disallowed.d.ts +65 -0
- package/dist/safety/always-disallowed.d.ts.map +1 -1
- package/dist/safety/always-disallowed.js +106 -10
- package/dist/safety/always-disallowed.js.map +1 -1
- package/dist/safety/audit.d.ts +46 -1
- package/dist/safety/audit.d.ts.map +1 -1
- package/dist/safety/audit.js +79 -16
- package/dist/safety/audit.js.map +1 -1
- package/dist/safety/risk-classifier.d.ts.map +1 -1
- package/dist/safety/risk-classifier.js +29 -0
- package/dist/safety/risk-classifier.js.map +1 -1
- package/dist/settings/runtime-settings.d.ts +12 -1
- package/dist/settings/runtime-settings.d.ts.map +1 -1
- package/dist/settings/runtime-settings.js +59 -1
- package/dist/settings/runtime-settings.js.map +1 -1
- package/package.json +2 -2
|
@@ -1,33 +1,35 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* `RoutineFetchWindowRunner` —
|
|
2
|
+
* `RoutineFetchWindowRunner` — pre-pass fan-out coordinator.
|
|
3
3
|
*
|
|
4
|
-
* ROUTINE_DATA_ACQUISITION_DESIGN.md §6.1.1
|
|
5
|
-
* dispatcher (morning_routine, today_refresh,
|
|
6
|
-
* weekly / monthly review) calls this runner
|
|
7
|
-
* dispatching the parent session. The runner:
|
|
4
|
+
* ROUTINE_DATA_ACQUISITION_DESIGN.md §6.1.1 + PRE_PASS_FAN_OUT_DESIGN.md
|
|
5
|
+
* — every routine dispatcher (morning_routine, today_refresh,
|
|
6
|
+
* hourly_check, evening / weekly / monthly review) calls this runner
|
|
7
|
+
* immediately before dispatching the parent session. The runner:
|
|
8
8
|
*
|
|
9
9
|
* 1. Reads the per-routine plan from `ROUTINE_WINDOWS` and the current
|
|
10
|
-
* integration state, fans out per-account where applicable,
|
|
11
|
-
* each row's predicate (`direct` / `delegated-same` /
|
|
12
|
-
* / `native` / skip)
|
|
13
|
-
*
|
|
14
|
-
* 2.
|
|
15
|
-
*
|
|
16
|
-
*
|
|
17
|
-
*
|
|
18
|
-
* `
|
|
19
|
-
* `
|
|
20
|
-
*
|
|
21
|
-
*
|
|
22
|
-
* `
|
|
23
|
-
*
|
|
24
|
-
*
|
|
25
|
-
*
|
|
26
|
-
*
|
|
27
|
-
*
|
|
28
|
-
*
|
|
10
|
+
* integration state, fans rows out per-account where applicable,
|
|
11
|
+
* resolves each row's predicate (`direct` / `delegated-same` /
|
|
12
|
+
* `delegated-cross` / `native` / skip), and partitions the plan
|
|
13
|
+
* by `IntegrationKey` via `splitAcquisitionPlanByIntegration`.
|
|
14
|
+
* 2. Spawns one lite-tier `routine.fetch_window` sub-session per
|
|
15
|
+
* integration in parallel (bounded by `prePassFanOutConcurrency`).
|
|
16
|
+
* Each sub-session sees exactly one partial — the
|
|
17
|
+
* `{integration_partial}` placeholder in the
|
|
18
|
+
* `routine.fetch_window` task-flow is replaced with the body of
|
|
19
|
+
* `_partials/<integration prePassPartial>` so the lite-tier model
|
|
20
|
+
* never has to disambiguate cross-API argument names. Each
|
|
21
|
+
* sub-session runs through an independent retry loop bounded by
|
|
22
|
+
* `prePassMaxAttemptsPerIntegration`, `prePassBackoffMs`, and the
|
|
23
|
+
* per-integration / per-routine USD budget caps.
|
|
24
|
+
* 3. Merges the sub-reports into a single `FetchReport` (additive
|
|
25
|
+
* `<integration>` children on the `<fetch_report>` XML block) and
|
|
26
|
+
* returns it plus the rendered block.
|
|
27
|
+
* 4. The caller grafts the block into the **parent** routine event's
|
|
28
|
+
* `event.data.fetchReportBlock`. ContextBuilder injects it verbatim
|
|
29
|
+
* into the parent session's prompt (mirrors the `<gate_decision>`
|
|
30
|
+
* pattern used by hourly_check Stage 3).
|
|
29
31
|
*
|
|
30
|
-
* Failure-mode contract (
|
|
32
|
+
* Failure-mode contract (PRE_PASS_FAN_OUT_DESIGN.md §5):
|
|
31
33
|
*
|
|
32
34
|
* - **No applicable rows** (routine has no windows in `ROUTINE_WINDOWS`,
|
|
33
35
|
* every integration is disabled, every account list is empty) — the
|
|
@@ -35,27 +37,39 @@
|
|
|
35
37
|
* `fetched=posted=duplicates=0`. The parent routine still runs; the
|
|
36
38
|
* block is informational only.
|
|
37
39
|
* - **Pre-pass session errors** (binding resolve fails, agent throws,
|
|
38
|
-
* JSON parse fails) —
|
|
39
|
-
*
|
|
40
|
-
* Pre-pass cost gains
|
|
41
|
-
*
|
|
40
|
+
* JSON parse fails) — recorded per-attempt; the retry loop fires up
|
|
41
|
+
* to `maxAttempts` before giving up. Final per-integration status
|
|
42
|
+
* surfaces in `<integration status="failed">`. Pre-pass cost gains
|
|
43
|
+
* are forfeit for that integration; siblings are unaffected.
|
|
42
44
|
* Throwing here would otherwise propagate up and abort the parent
|
|
43
45
|
* routine — the opposite of P3 ("Lite for Fetch, Medium for Decide").
|
|
44
46
|
* - **Partial success** — the report's `errors` array carries per-row
|
|
45
|
-
* failures (`no-surface`, `fetch-failed`, `budget-exhausted
|
|
46
|
-
* block surfaces them so the
|
|
47
|
-
* treat its observations view as
|
|
47
|
+
* failures (`no-surface`, `fetch-failed`, `budget-exhausted`,
|
|
48
|
+
* `budget-cap`, `global-budget-cap`). The block surfaces them so the
|
|
49
|
+
* parent prompt can decide whether to treat its observations view as
|
|
50
|
+
* complete.
|
|
48
51
|
*/
|
|
49
52
|
import { EventPriority, INTEGRATION_KEYS, createEvent, getAgentDayDateStr, getIntegrationDescriptor, isRoutineEvent, } from "@aitne/shared";
|
|
50
53
|
import { readIntegrations } from "../db/integrations-store.js";
|
|
54
|
+
import { renderPartialForFanOut } from "./prompts.js";
|
|
51
55
|
import { ROUTINE_WINDOWS, routineHasWindows, } from "./routine-windows.js";
|
|
52
|
-
import {
|
|
56
|
+
import { buildAcquisitionTimestamps, splitAcquisitionPlanByIntegration, } from "./routine-acquisition-plan.js";
|
|
57
|
+
import { RETRY_REASONS, buildPriorAttemptHintBlock, defaultRetryDecision, } from "./routine-fetch-window-retry.js";
|
|
53
58
|
import { createLogger } from "../logging.js";
|
|
54
59
|
const logger = createLogger("routine-fetch-window-runner");
|
|
55
60
|
// ── Module helpers ────────────────────────────────────────────────────────
|
|
56
61
|
/** The ProcessKey + event type the pre-pass session always runs under. */
|
|
57
62
|
const FETCH_WINDOW_PROCESS_KEY = "routine.fetch_window";
|
|
58
63
|
const FETCH_WINDOW_EVENT_TYPE = "routine.fetch_window";
|
|
64
|
+
/**
|
|
65
|
+
* PRE_PASS_FAN_OUT_DESIGN.md §4.2 — the single placeholder the
|
|
66
|
+
* `routine.fetch_window.md` task-flow carries in place of inline
|
|
67
|
+
* integration partials. The runner substitutes this with the
|
|
68
|
+
* integration-specific partial body loaded via
|
|
69
|
+
* `renderPartialForFanOut`. Kept as a constant so the task-flow file
|
|
70
|
+
* and the substitution call cannot drift apart.
|
|
71
|
+
*/
|
|
72
|
+
const FETCH_WINDOW_INTEGRATION_PARTIAL_PLACEHOLDER = "{integration_partial}";
|
|
59
73
|
/**
|
|
60
74
|
* Daemon REST surfaces the pre-pass partials may target. Curl prefixes
|
|
61
75
|
* are constructed with the configured `apiPort` at dispatch time so a
|
|
@@ -66,6 +80,21 @@ const FETCH_WINDOW_EVENT_TYPE = "routine.fetch_window";
|
|
|
66
80
|
* the agent profile's "no notify, no context writes" guardrails (P3:
|
|
67
81
|
* Lite for Fetch — the pre-pass has zero business making decisions).
|
|
68
82
|
*
|
|
83
|
+
* Each pattern uses a wildcard `*` between `curl` and the URL so the
|
|
84
|
+
* SDK's glob matcher accepts both flag orderings the Haiku fetcher
|
|
85
|
+
* actually emits:
|
|
86
|
+
*
|
|
87
|
+
* - `curl http://localhost:.../api/observations -X POST -d @-` (URL first)
|
|
88
|
+
* - `curl -X POST -H 'Content-Type: …' http://localhost:.../api/observations -d @-`
|
|
89
|
+
* (flags first)
|
|
90
|
+
*
|
|
91
|
+
* The original prefix-anchored form (`Bash(curl http://localhost:.../api/observations*)`)
|
|
92
|
+
* silently denied the flags-first invocation, manifesting as `posted=0,
|
|
93
|
+
* fetched=N` reports — Haiku fetched via MCP, then could not POST a single
|
|
94
|
+
* observation. The curl PreToolUse hook in `claude-tool-collection.ts`
|
|
95
|
+
* remains the host/port/exfil chokepoint; this clamp now restricts only
|
|
96
|
+
* the daemon-API namespace, which is what we actually need.
|
|
97
|
+
*
|
|
69
98
|
* `jq *` stays allowed because direct-mode partials pipe curl output
|
|
70
99
|
* through jq for compact projection before posting to /api/observations.
|
|
71
100
|
*
|
|
@@ -79,16 +108,16 @@ function buildPrePassDaemonRestPatterns(apiPort) {
|
|
|
79
108
|
return [
|
|
80
109
|
// Observations — the only write surface the pre-pass touches.
|
|
81
110
|
// Catches both POST /api/observations and GET /api/observations*.
|
|
82
|
-
`Bash(curl
|
|
111
|
+
`Bash(curl *${root}/observations*)`,
|
|
83
112
|
// Direct-mode mail / calendar / notion reads.
|
|
84
|
-
`Bash(curl
|
|
85
|
-
`Bash(curl
|
|
86
|
-
`Bash(curl
|
|
113
|
+
`Bash(curl *${root}/mail/*)`,
|
|
114
|
+
`Bash(curl *${root}/calendar/*)`,
|
|
115
|
+
`Bash(curl *${root}/notion/*)`,
|
|
87
116
|
// delegated-cross proxy. Only Gmail / Google Calendar / Notion
|
|
88
117
|
// expose this; user-managed Outlook has no proxy and the runner
|
|
89
118
|
// collapses cross-backend bindings to delegated-same per
|
|
90
119
|
// routine-acquisition-plan.ts:resolveFetchMode.
|
|
91
|
-
`Bash(curl
|
|
120
|
+
`Bash(curl *${root}/integrations/*)`,
|
|
92
121
|
// Compact-projection helper used by the partials.
|
|
93
122
|
"Bash(jq *)",
|
|
94
123
|
];
|
|
@@ -150,13 +179,32 @@ function collectIntegrationToolsForBackend(integrations, backend) {
|
|
|
150
179
|
* the partial records `no-surface` and the runner's report carries
|
|
151
180
|
* the gap forward to the parent routine.
|
|
152
181
|
*
|
|
182
|
+
* `ToolSearch` is appended for Claude sessions whenever at least one
|
|
183
|
+
* descriptor-bound MCP tool is present. Claude Code 2.1+ defers large
|
|
184
|
+
* MCP tool manifests (`mcp__claude_ai_Gmail__*`,
|
|
185
|
+
* `mcp__claude_ai_Google_Calendar__*`, `mcp__claude_ai_Notion__*`, …)
|
|
186
|
+
* behind `ToolSearch`, so the model must call `ToolSearch` to load a
|
|
187
|
+
* deferred tool's schema before it can be invoked. Without `ToolSearch`
|
|
188
|
+
* allowed, the Haiku fetcher emits a denied ToolSearch call on its
|
|
189
|
+
* first turn, gives up, and returns text with no JSON — the parent
|
|
190
|
+
* routine then sees `<fetch_report status="failed" reason="no-json-object">`.
|
|
191
|
+
* Mirrors the same workaround in `claude-delegated.ts` (delegated proxy
|
|
192
|
+
* `allowedTools: [toolName, "ToolSearch"]`). Codex / Gemini have no
|
|
193
|
+
* per-spawn allowedTools surface today and ignore the override entirely
|
|
194
|
+
* (CLAUDE.md acknowledges the gap), so the `ToolSearch` widening is
|
|
195
|
+
* gated on `sessionBackend === "claude"` to keep the list minimal for
|
|
196
|
+
* other backends.
|
|
197
|
+
*
|
|
153
198
|
* Exported for unit testing — the runner consumes it via
|
|
154
199
|
* `composePrePassAllowedTools` at dispatch time.
|
|
155
200
|
*/
|
|
156
201
|
export function composePrePassAllowedTools(apiPort, integrations, sessionBackend) {
|
|
202
|
+
const integrationTools = collectIntegrationToolsForBackend(integrations, sessionBackend);
|
|
203
|
+
const needsDeferredDiscovery = sessionBackend === "claude" && integrationTools.length > 0;
|
|
157
204
|
return [
|
|
158
205
|
...buildPrePassDaemonRestPatterns(apiPort),
|
|
159
|
-
...
|
|
206
|
+
...integrationTools,
|
|
207
|
+
...(needsDeferredDiscovery ? ["ToolSearch"] : []),
|
|
160
208
|
];
|
|
161
209
|
}
|
|
162
210
|
/**
|
|
@@ -336,6 +384,304 @@ export function renderFetchReportBlock(report, meta) {
|
|
|
336
384
|
lines.push("</fetch_report>");
|
|
337
385
|
return lines.join("\n");
|
|
338
386
|
}
|
|
387
|
+
// ── Fan-out aggregation (PRE_PASS_FAN_OUT_DESIGN.md §4.5) ─────────────────
|
|
388
|
+
/**
|
|
389
|
+
* Resolve the aggregate `<fetch_report>` status from a set of
|
|
390
|
+
* sub-reports' final statuses, per §4.5:
|
|
391
|
+
*
|
|
392
|
+
* - `success` iff every non-skipped sub-report is `success`. Skipped
|
|
393
|
+
* sub-reports do not count against success.
|
|
394
|
+
* - `failed` iff every non-skipped sub-report is `failed`.
|
|
395
|
+
* - `partial` for any other mix (incl. one success + one failed).
|
|
396
|
+
* - `skipped` only when the input is empty (the caller handles
|
|
397
|
+
* "every sub-report skipped" separately — the runner short-circuits
|
|
398
|
+
* before fan-out when no integrations are active).
|
|
399
|
+
*
|
|
400
|
+
* Exported for unit testing the status-resolution branch in isolation.
|
|
401
|
+
*/
|
|
402
|
+
export function aggregateFanOutStatus(subReports) {
|
|
403
|
+
if (subReports.length === 0)
|
|
404
|
+
return "skipped";
|
|
405
|
+
const nonSkipped = subReports.filter((r) => r.status !== "skipped");
|
|
406
|
+
if (nonSkipped.length === 0)
|
|
407
|
+
return "skipped";
|
|
408
|
+
const allSuccess = nonSkipped.every((r) => r.status === "success");
|
|
409
|
+
if (allSuccess)
|
|
410
|
+
return "success";
|
|
411
|
+
const allFailed = nonSkipped.every((r) => r.status === "failed");
|
|
412
|
+
if (allFailed)
|
|
413
|
+
return "failed";
|
|
414
|
+
return "partial";
|
|
415
|
+
}
|
|
416
|
+
/**
|
|
417
|
+
* Render the additive `<integration>` children + the `<error>`
|
|
418
|
+
* grandchildren that go inside a fan-out `<fetch_report>`. The parent
|
|
419
|
+
* `<fetch_report ...>` open/close lines are produced by
|
|
420
|
+
* `renderFetchReportBlock`; this helper produces only the body lines so
|
|
421
|
+
* the two can compose cleanly.
|
|
422
|
+
*/
|
|
423
|
+
function renderPerIntegrationLines(subReports) {
|
|
424
|
+
const lines = [];
|
|
425
|
+
for (const sub of subReports) {
|
|
426
|
+
const errors = errorsForSubReport(sub);
|
|
427
|
+
const openAttrs = [
|
|
428
|
+
`key="${xmlEscape(sub.integrationKey)}"`,
|
|
429
|
+
`status="${xmlEscape(sub.status)}"`,
|
|
430
|
+
`fetched="${sub.fetched}"`,
|
|
431
|
+
`posted="${sub.posted}"`,
|
|
432
|
+
`duplicates="${sub.duplicates}"`,
|
|
433
|
+
`attempts="${sub.attempts.length}"`,
|
|
434
|
+
];
|
|
435
|
+
if (errors.length === 0) {
|
|
436
|
+
lines.push(` <integration ${openAttrs.join(" ")} />`);
|
|
437
|
+
continue;
|
|
438
|
+
}
|
|
439
|
+
lines.push(` <integration ${openAttrs.join(" ")}>`);
|
|
440
|
+
for (const err of errors) {
|
|
441
|
+
const type = typeof err.type === "string" ? err.type : "unknown";
|
|
442
|
+
const attrEntries = Object.entries(err).filter(([k, v]) => k !== "type" && (typeof v === "string" || typeof v === "number"));
|
|
443
|
+
const attrs = attrEntries
|
|
444
|
+
.map(([k, v]) => `${xmlEscape(k)}="${xmlEscape(String(v))}"`)
|
|
445
|
+
.join(" ");
|
|
446
|
+
lines.push(` <error type="${xmlEscape(type)}"${attrs ? " " + attrs : ""} />`);
|
|
447
|
+
}
|
|
448
|
+
lines.push(` </integration>`);
|
|
449
|
+
}
|
|
450
|
+
return lines;
|
|
451
|
+
}
|
|
452
|
+
/**
|
|
453
|
+
* Flatten every attempt's `errors` into a single sequence — the runner
|
|
454
|
+
* always pushes at least one record per loop iteration, so the union of
|
|
455
|
+
* attempts' errors is the canonical list (and equals `sub.errors`,
|
|
456
|
+
* which is set from `flatMap(attempts, e => e.errors)` at sub-report
|
|
457
|
+
* construction time). Kept as a helper so the rendering / merge call
|
|
458
|
+
* sites read declaratively.
|
|
459
|
+
*/
|
|
460
|
+
function errorsForSubReport(sub) {
|
|
461
|
+
return sub.attempts.flatMap((att) => att.errors);
|
|
462
|
+
}
|
|
463
|
+
function attemptDurationMs(att) {
|
|
464
|
+
const start = Date.parse(att.startedAt);
|
|
465
|
+
const end = Date.parse(att.endedAt);
|
|
466
|
+
if (!Number.isFinite(start) || !Number.isFinite(end))
|
|
467
|
+
return 0;
|
|
468
|
+
return Math.max(0, end - start);
|
|
469
|
+
}
|
|
470
|
+
function pickFinalErrorMessage(sub) {
|
|
471
|
+
if (sub.status !== "failed")
|
|
472
|
+
return undefined;
|
|
473
|
+
const finalAttempt = sub.attempts[sub.attempts.length - 1];
|
|
474
|
+
const firstError = finalAttempt?.errors?.[0];
|
|
475
|
+
if (!firstError)
|
|
476
|
+
return undefined;
|
|
477
|
+
const message = firstError.message ?? firstError.reason ?? firstError.kind;
|
|
478
|
+
return typeof message === "string" ? message : undefined;
|
|
479
|
+
}
|
|
480
|
+
export function summarizeIntegrationReport(sub) {
|
|
481
|
+
const costUsd = sub.attempts.reduce((sum, att) => sum + att.costUsd, 0);
|
|
482
|
+
const durationMs = sub.attempts.reduce((sum, att) => sum + attemptDurationMs(att), 0);
|
|
483
|
+
const finalError = pickFinalErrorMessage(sub);
|
|
484
|
+
return {
|
|
485
|
+
key: sub.integrationKey,
|
|
486
|
+
status: sub.status,
|
|
487
|
+
attempts: sub.attempts.length,
|
|
488
|
+
fetched: sub.fetched,
|
|
489
|
+
posted: sub.posted,
|
|
490
|
+
duplicates: sub.duplicates,
|
|
491
|
+
costUsd,
|
|
492
|
+
durationMs,
|
|
493
|
+
...(finalError ? { finalError } : {}),
|
|
494
|
+
};
|
|
495
|
+
}
|
|
496
|
+
export function summarizeFetchReport(report) {
|
|
497
|
+
const perIntegration = report.perIntegration ?? [];
|
|
498
|
+
const costUsd = perIntegration.reduce((sum, sub) => sum + sub.attempts.reduce((s, att) => s + att.costUsd, 0), 0);
|
|
499
|
+
return {
|
|
500
|
+
status: report.status,
|
|
501
|
+
fetched: report.fetched,
|
|
502
|
+
posted: report.posted,
|
|
503
|
+
duplicates: report.duplicates,
|
|
504
|
+
costUsd,
|
|
505
|
+
};
|
|
506
|
+
}
|
|
507
|
+
/**
|
|
508
|
+
* Merge fan-out sub-reports into a single `FetchReport` + the rendered
|
|
509
|
+
* `<fetch_report>` XML block the parent routine sees. §4.5.
|
|
510
|
+
*
|
|
511
|
+
* - Counts (`fetched`, `posted`, `duplicates`): arithmetic sum.
|
|
512
|
+
* - `errors`: concatenation, each error tagged with
|
|
513
|
+
* `integration: <key>` (and the per-attempt rows already carry
|
|
514
|
+
* `attempt: <n>` via the runner's per-attempt error-recording —
|
|
515
|
+
* `mergeSubReports` does not invent annotations beyond `integration`).
|
|
516
|
+
* - `status`: `aggregateFanOutStatus`.
|
|
517
|
+
* - `failureReason`: only when aggregate is `failed`; one-line summary
|
|
518
|
+
* listing the failed integrations and their attempt counts.
|
|
519
|
+
* - `perIntegration`: sorted by `INTEGRATION_KEYS` order regardless of
|
|
520
|
+
* completion order (deterministic — §4.6).
|
|
521
|
+
*
|
|
522
|
+
* Pure function: no side effects, no DB / clock dependencies.
|
|
523
|
+
*/
|
|
524
|
+
export function mergeSubReports(subReports, routine, agentDay) {
|
|
525
|
+
// Deterministic ordering by INTEGRATION_KEYS enumeration order so the
|
|
526
|
+
// block is stable across runs regardless of which sub-session
|
|
527
|
+
// finished first.
|
|
528
|
+
const ordered = INTEGRATION_KEYS
|
|
529
|
+
.map((k) => subReports.find((r) => r.integrationKey === k))
|
|
530
|
+
.filter((r) => r !== undefined);
|
|
531
|
+
// Defensive — preserve any non-canonical keys at the tail rather
|
|
532
|
+
// than dropping them. Today every SubReport's integrationKey comes
|
|
533
|
+
// from the canonical enum, but a future key added to the registry
|
|
534
|
+
// without the catalog catching up would otherwise be silently
|
|
535
|
+
// suppressed.
|
|
536
|
+
for (const r of subReports) {
|
|
537
|
+
if (!ordered.includes(r))
|
|
538
|
+
ordered.push(r);
|
|
539
|
+
}
|
|
540
|
+
let fetched = 0;
|
|
541
|
+
let posted = 0;
|
|
542
|
+
let duplicates = 0;
|
|
543
|
+
const errors = [];
|
|
544
|
+
for (const sub of ordered) {
|
|
545
|
+
fetched += sub.fetched;
|
|
546
|
+
posted += sub.posted;
|
|
547
|
+
duplicates += sub.duplicates;
|
|
548
|
+
for (const err of errorsForSubReport(sub)) {
|
|
549
|
+
errors.push({ ...err, integration: sub.integrationKey });
|
|
550
|
+
}
|
|
551
|
+
}
|
|
552
|
+
const status = aggregateFanOutStatus(ordered);
|
|
553
|
+
let failureReason;
|
|
554
|
+
if (status === "failed") {
|
|
555
|
+
const failedSummaries = ordered
|
|
556
|
+
.filter((r) => r.status === "failed")
|
|
557
|
+
.map((r) => `${r.integrationKey} (${r.attempts.length} attempts)`);
|
|
558
|
+
failureReason = failedSummaries.length > 0
|
|
559
|
+
? `${failedSummaries.length} integrations failed: ${failedSummaries.join(", ")}`
|
|
560
|
+
: "all sub-sessions failed";
|
|
561
|
+
}
|
|
562
|
+
const report = {
|
|
563
|
+
status,
|
|
564
|
+
fetched,
|
|
565
|
+
posted,
|
|
566
|
+
duplicates,
|
|
567
|
+
errors,
|
|
568
|
+
skipped: status === "skipped",
|
|
569
|
+
...(failureReason !== undefined ? { failureReason } : {}),
|
|
570
|
+
perIntegration: ordered,
|
|
571
|
+
};
|
|
572
|
+
// Render: open/close come from a `renderFetchReportBlock`-shaped
|
|
573
|
+
// header line; body interleaves the per-integration children. We
|
|
574
|
+
// emit a fresh string rather than calling `renderFetchReportBlock`
|
|
575
|
+
// because the aggregated block carries `<integration>` children that
|
|
576
|
+
// the short-circuit (skipped / failed) blocks emitted by
|
|
577
|
+
// `renderFetchReportBlock` do not — keeping the two render paths
|
|
578
|
+
// explicit avoids cross-contaminating their shapes.
|
|
579
|
+
const routineAttr = routine.replace(/^routine\./, "");
|
|
580
|
+
const headerAttrs = [
|
|
581
|
+
`routine="${xmlEscape(routineAttr)}"`,
|
|
582
|
+
`agent_day="${xmlEscape(agentDay)}"`,
|
|
583
|
+
`status="${xmlEscape(status)}"`,
|
|
584
|
+
`fetched="${fetched}"`,
|
|
585
|
+
`posted="${posted}"`,
|
|
586
|
+
`duplicates="${duplicates}"`,
|
|
587
|
+
];
|
|
588
|
+
const lines = [`<fetch_report ${headerAttrs.join(" ")}>`];
|
|
589
|
+
if (failureReason !== undefined) {
|
|
590
|
+
lines.push(` <failure>${xmlEscape(failureReason)}</failure>`);
|
|
591
|
+
}
|
|
592
|
+
lines.push(...renderPerIntegrationLines(ordered));
|
|
593
|
+
lines.push("</fetch_report>");
|
|
594
|
+
return { report, block: lines.join("\n") };
|
|
595
|
+
}
|
|
596
|
+
/**
|
|
597
|
+
* Race-free local budget guard for one fan-out coordinator. Reservations
|
|
598
|
+
* mutate a separate `reservedUsd` bucket before async work starts; commit
|
|
599
|
+
* releases that reservation and records the measured spend.
|
|
600
|
+
*/
|
|
601
|
+
class FanOutBudgetGuard {
|
|
602
|
+
capUsd;
|
|
603
|
+
reservedUsd = 0;
|
|
604
|
+
spentUsd = 0;
|
|
605
|
+
constructor(capUsd) {
|
|
606
|
+
this.capUsd = capUsd;
|
|
607
|
+
}
|
|
608
|
+
reserve(estimateUsd) {
|
|
609
|
+
// Defensive normalisation: a non-finite estimate (NaN / undefined coerced
|
|
610
|
+
// through Math.max) would poison `reservedUsd` permanently — every
|
|
611
|
+
// subsequent reserve() would arithmetic-NaN and report `false` for the
|
|
612
|
+
// headroom check. The binding contract today guarantees a finite number,
|
|
613
|
+
// but the guard is one strict layer below where a malformed config or a
|
|
614
|
+
// future backend shape could leak through.
|
|
615
|
+
const estimate = Number.isFinite(estimateUsd)
|
|
616
|
+
? Math.max(0, estimateUsd)
|
|
617
|
+
: 0;
|
|
618
|
+
if (!Number.isFinite(this.capUsd)) {
|
|
619
|
+
return { ok: true, remaining: Number.POSITIVE_INFINITY, estimateUsd: estimate };
|
|
620
|
+
}
|
|
621
|
+
const remaining = this.capUsd - this.spentUsd - this.reservedUsd;
|
|
622
|
+
if (estimate > remaining) {
|
|
623
|
+
return { ok: false, remaining: Math.max(0, remaining), estimateUsd: estimate };
|
|
624
|
+
}
|
|
625
|
+
this.reservedUsd += estimate;
|
|
626
|
+
return {
|
|
627
|
+
ok: true,
|
|
628
|
+
remaining: Math.max(0, this.capUsd - this.spentUsd - this.reservedUsd),
|
|
629
|
+
estimateUsd: estimate,
|
|
630
|
+
};
|
|
631
|
+
}
|
|
632
|
+
commit(reservation, actualUsd) {
|
|
633
|
+
if (!reservation.ok)
|
|
634
|
+
return;
|
|
635
|
+
this.reservedUsd = Math.max(0, this.reservedUsd - reservation.estimateUsd);
|
|
636
|
+
// Same defensive guard as reserve() — NaN actualUsd from a misbehaving
|
|
637
|
+
// backend would otherwise corrupt the spend counter and silently disable
|
|
638
|
+
// the cap for the remainder of the routine.
|
|
639
|
+
if (Number.isFinite(actualUsd)) {
|
|
640
|
+
this.spentUsd += Math.max(0, actualUsd);
|
|
641
|
+
}
|
|
642
|
+
}
|
|
643
|
+
get spent() {
|
|
644
|
+
return this.spentUsd;
|
|
645
|
+
}
|
|
646
|
+
}
|
|
647
|
+
function clampInt(value, min, max, fallback) {
|
|
648
|
+
if (typeof value !== "number" || !Number.isFinite(value))
|
|
649
|
+
return fallback;
|
|
650
|
+
return Math.min(max, Math.max(min, Math.trunc(value)));
|
|
651
|
+
}
|
|
652
|
+
function nonNegativeNumber(value, fallback) {
|
|
653
|
+
if (typeof value !== "number" || !Number.isFinite(value) || value < 0) {
|
|
654
|
+
return fallback;
|
|
655
|
+
}
|
|
656
|
+
return value;
|
|
657
|
+
}
|
|
658
|
+
function sleep(ms) {
|
|
659
|
+
if (ms <= 0)
|
|
660
|
+
return Promise.resolve();
|
|
661
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
662
|
+
}
|
|
663
|
+
async function runWithConcurrency(tasks, concurrency) {
|
|
664
|
+
if (tasks.length === 0)
|
|
665
|
+
return [];
|
|
666
|
+
if (concurrency === null || concurrency >= tasks.length) {
|
|
667
|
+
return Promise.all(tasks.map((task) => task()));
|
|
668
|
+
}
|
|
669
|
+
const limit = Math.max(1, Math.trunc(concurrency));
|
|
670
|
+
const results = new Array(tasks.length);
|
|
671
|
+
let next = 0;
|
|
672
|
+
async function worker() {
|
|
673
|
+
for (;;) {
|
|
674
|
+
const index = next;
|
|
675
|
+
next += 1;
|
|
676
|
+
if (index >= tasks.length)
|
|
677
|
+
return;
|
|
678
|
+
results[index] = await tasks[index]();
|
|
679
|
+
}
|
|
680
|
+
}
|
|
681
|
+
const workers = Array.from({ length: Math.min(limit, tasks.length) }, () => worker());
|
|
682
|
+
await Promise.all(workers);
|
|
683
|
+
return results;
|
|
684
|
+
}
|
|
339
685
|
// ── Runner ────────────────────────────────────────────────────────────────
|
|
340
686
|
export class RoutineFetchWindowRunner {
|
|
341
687
|
db;
|
|
@@ -345,6 +691,7 @@ export class RoutineFetchWindowRunner {
|
|
|
345
691
|
audit;
|
|
346
692
|
prompt;
|
|
347
693
|
getActiveMailAccounts;
|
|
694
|
+
getEventBroadcaster;
|
|
348
695
|
constructor(deps) {
|
|
349
696
|
this.db = deps.db;
|
|
350
697
|
this.config = deps.config;
|
|
@@ -353,6 +700,41 @@ export class RoutineFetchWindowRunner {
|
|
|
353
700
|
this.audit = deps.audit;
|
|
354
701
|
this.prompt = deps.prompt;
|
|
355
702
|
this.getActiveMailAccounts = deps.getActiveMailAccounts;
|
|
703
|
+
this.getEventBroadcaster = deps.getEventBroadcaster ?? null;
|
|
704
|
+
}
|
|
705
|
+
/**
|
|
706
|
+
* Broadcast a single pre-pass progress event to the dashboard SSE
|
|
707
|
+
* channel. Failure is contained — the broadcaster contract is
|
|
708
|
+
* fire-and-forget and a misbehaving writer must not affect the
|
|
709
|
+
* runner's return value or the parent routine's dispatch.
|
|
710
|
+
*
|
|
711
|
+
* Schema (default `event` SSE channel, matches existing
|
|
712
|
+
* `kind: "main_backend_changed"` / `"routine_started"` pattern):
|
|
713
|
+
* { kind, routine, source, correlationId, timestamp, status? }
|
|
714
|
+
* `status` is set on `prepass_completed` only and reflects the
|
|
715
|
+
* FetchReport.status field (success / partial / failed / skipped).
|
|
716
|
+
*/
|
|
717
|
+
broadcastPrepassProgress(kind, parentEvent, extra) {
|
|
718
|
+
const broadcaster = this.getEventBroadcaster?.();
|
|
719
|
+
if (!broadcaster)
|
|
720
|
+
return;
|
|
721
|
+
try {
|
|
722
|
+
broadcaster.broadcastEvent({
|
|
723
|
+
kind,
|
|
724
|
+
routine: isRoutineEvent(parentEvent) ? parentEvent.routine : null,
|
|
725
|
+
source: parentEvent.source,
|
|
726
|
+
correlationId: parentEvent.correlationId,
|
|
727
|
+
timestamp: new Date().toISOString(),
|
|
728
|
+
...(extra?.status ? { status: extra.status } : {}),
|
|
729
|
+
...(extra?.reason ? { reason: extra.reason } : {}),
|
|
730
|
+
...Object.fromEntries(Object.entries(extra ?? {}).filter(([key]) => key !== "status" && key !== "reason")),
|
|
731
|
+
});
|
|
732
|
+
}
|
|
733
|
+
catch {
|
|
734
|
+
// Intentionally silent — broadcaster failures are already logged at
|
|
735
|
+
// the EventBroadcaster.broadcastNamedEvent layer. Re-logging here
|
|
736
|
+
// would spam during transient SSE client churn.
|
|
737
|
+
}
|
|
356
738
|
}
|
|
357
739
|
/**
|
|
358
740
|
* Execute the pre-pass for `parentEvent`. Returns the fetch report
|
|
@@ -366,6 +748,46 @@ export class RoutineFetchWindowRunner {
|
|
|
366
748
|
* but the seam exists for future divergence).
|
|
367
749
|
*/
|
|
368
750
|
async run(parentEvent, routineKey) {
|
|
751
|
+
// B2 observability — announce pre-pass entry so the dashboard can
|
|
752
|
+
// render "Fetching your mail and Notion data…" as a sub-step of the
|
|
753
|
+
// parent routine's `routine_started` envelope. `prepass_completed`
|
|
754
|
+
// (with status) fires from the single exit point at the bottom of
|
|
755
|
+
// `runImpl` via the try/finally below. Symmetric: every started
|
|
756
|
+
// emit has a matching completed emit, even on the skipped paths
|
|
757
|
+
// (skipping is itself information the user wants to see).
|
|
758
|
+
this.broadcastPrepassProgress("prepass_started", parentEvent);
|
|
759
|
+
let outcome;
|
|
760
|
+
try {
|
|
761
|
+
outcome = await this.runImpl(parentEvent, routineKey);
|
|
762
|
+
return outcome;
|
|
763
|
+
}
|
|
764
|
+
finally {
|
|
765
|
+
// §7.2 — `prepass_completed` payload contract:
|
|
766
|
+
// { kind, routine, source, correlationId, timestamp, status, reason?,
|
|
767
|
+
// aggregate: {status, fetched, posted, duplicates, costUsd},
|
|
768
|
+
// perIntegration: [{key, status, attempts, fetched, posted,
|
|
769
|
+
// duplicates, costUsd, durationMs, finalError?}] }
|
|
770
|
+
// The aggregate + perIntegration arrays let the dashboard render
|
|
771
|
+
// the per-integration progress card the design called for without
|
|
772
|
+
// re-querying the daemon. Skipped / failed short-circuit paths
|
|
773
|
+
// (no-routine-key / no-windows / empty-plan / plan-assembly-failed)
|
|
774
|
+
// produce reports without `perIntegration`; in those cases the
|
|
775
|
+
// aggregate still carries the headline (fetched/posted = 0,
|
|
776
|
+
// costUsd = 0) and `perIntegration` is the empty array.
|
|
777
|
+
const report = outcome?.report;
|
|
778
|
+
const perIntegration = (report?.perIntegration ?? []).map(summarizeIntegrationReport);
|
|
779
|
+
const aggregate = report
|
|
780
|
+
? summarizeFetchReport(report)
|
|
781
|
+
: undefined;
|
|
782
|
+
this.broadcastPrepassProgress("prepass_completed", parentEvent, {
|
|
783
|
+
status: report?.status,
|
|
784
|
+
...(report?.failureReason ? { reason: report.failureReason } : {}),
|
|
785
|
+
...(aggregate ? { aggregate } : {}),
|
|
786
|
+
perIntegration,
|
|
787
|
+
});
|
|
788
|
+
}
|
|
789
|
+
}
|
|
790
|
+
async runImpl(parentEvent, routineKey) {
|
|
369
791
|
const key = routineKey ?? routineWindowKeyFromEvent(parentEvent);
|
|
370
792
|
const agentDay = getAgentDayDateStr(this.config.timezone || undefined, this.config.dayBoundaryHour);
|
|
371
793
|
if (!key) {
|
|
@@ -400,71 +822,21 @@ export class RoutineFetchWindowRunner {
|
|
|
400
822
|
const block = renderFetchReportBlock(report, { routine: key, agentDay });
|
|
401
823
|
return { report, block };
|
|
402
824
|
}
|
|
403
|
-
|
|
404
|
-
// active accounts. The dispatcher's session backend is the resolved
|
|
405
|
-
// main backend for the routine.fetch_window ProcessKey — resolve via
|
|
406
|
-
// the agent router so the per-(integration, mode) predicate
|
|
407
|
-
// (`delegated-same` vs `delegated-cross`) is correct for the actual
|
|
408
|
-
// run.
|
|
409
|
-
//
|
|
410
|
-
// `integrationsSnapshot` and `sessionBackend` are hoisted above the
|
|
411
|
-
// try block so the `allowedToolsOverride` composer below can re-use
|
|
412
|
-
// the same snapshot the plan was built from. Recomputing the
|
|
413
|
-
// snapshot at execute time would risk a TOCTOU drift if the
|
|
414
|
-
// operator flips an integration mid-pre-pass — the plan would have
|
|
415
|
-
// a row the override couldn't reach, manifesting as a silent MCP
|
|
416
|
-
// permission denial.
|
|
417
|
-
let fetcherEvent;
|
|
418
|
-
let sessionBackend;
|
|
419
|
-
let integrationsSnapshot;
|
|
825
|
+
let planContext;
|
|
420
826
|
try {
|
|
421
|
-
|
|
422
|
-
// backend before we synthesise the real one (the plan attribute
|
|
423
|
-
// depends on it). The placeholder borrows correlationId so audit
|
|
424
|
-
// rows correlate back to the parent.
|
|
425
|
-
const placeholder = {
|
|
426
|
-
...createEvent({
|
|
427
|
-
type: FETCH_WINDOW_EVENT_TYPE,
|
|
428
|
-
source: parentEvent.source,
|
|
429
|
-
priority: EventPriority.NORMAL,
|
|
430
|
-
correlationId: parentEvent.correlationId,
|
|
431
|
-
}),
|
|
432
|
-
routine: "fetch_window",
|
|
433
|
-
};
|
|
434
|
-
const preBinding = this.agentRouter.resolveBinding(placeholder, {
|
|
435
|
-
processKey: FETCH_WINDOW_PROCESS_KEY,
|
|
436
|
-
});
|
|
437
|
-
sessionBackend = preBinding.main.backendId;
|
|
438
|
-
integrationsSnapshot = readIntegrations(this.db);
|
|
439
|
-
const accounts = this.collectAccounts(integrationsSnapshot);
|
|
440
|
-
const timestamps = buildAcquisitionTimestamps(new Date(), this.config.timezone || undefined, this.config.dayBoundaryHour);
|
|
441
|
-
const planBlock = buildAcquisitionPlan({
|
|
442
|
-
routine: key,
|
|
443
|
-
agentDay,
|
|
444
|
-
integrations: integrationsSnapshot,
|
|
445
|
-
sessionBackend,
|
|
446
|
-
accounts,
|
|
447
|
-
timestamps,
|
|
448
|
-
});
|
|
449
|
-
fetcherEvent = {
|
|
450
|
-
...placeholder,
|
|
451
|
-
data: {
|
|
452
|
-
...placeholder.data,
|
|
453
|
-
acquisitionPlanBlock: planBlock,
|
|
454
|
-
parentRoutine: key,
|
|
455
|
-
parentCorrelationId: parentEvent.correlationId,
|
|
456
|
-
},
|
|
457
|
-
};
|
|
827
|
+
planContext = this.buildFanOutPlanContext(parentEvent, key, agentDay);
|
|
458
828
|
}
|
|
459
829
|
catch (err) {
|
|
460
830
|
return this.fail(key, agentDay, parentEvent, "plan-assembly-failed", err);
|
|
461
831
|
}
|
|
462
832
|
// The acquisition plan can resolve to zero `<fetch>` rows when every
|
|
463
833
|
// integration the routine touches is disabled / cross-backend-bound
|
|
464
|
-
// on a connector-less integration / etc.
|
|
465
|
-
//
|
|
834
|
+
// on a connector-less integration / etc. `splitAcquisitionPlanByIntegration`
|
|
835
|
+
// drops integrations with no rows, so an empty `subPlans` exactly
|
|
836
|
+
// means the routine has nothing to fetch. Treat that as a skip —
|
|
837
|
+
// we don't pay the cold-start to confirm the agent has nothing
|
|
466
838
|
// to do.
|
|
467
|
-
if (
|
|
839
|
+
if (planContext.subPlans.length === 0) {
|
|
468
840
|
const report = {
|
|
469
841
|
status: "skipped",
|
|
470
842
|
fetched: 0,
|
|
@@ -472,68 +844,647 @@ export class RoutineFetchWindowRunner {
|
|
|
472
844
|
duplicates: 0,
|
|
473
845
|
errors: [],
|
|
474
846
|
skipped: true,
|
|
475
|
-
fetcherCorrelationId:
|
|
847
|
+
fetcherCorrelationId: planContext.placeholder.correlationId,
|
|
476
848
|
};
|
|
477
849
|
const block = renderFetchReportBlock(report, { routine: key, agentDay });
|
|
478
850
|
logger.debug({
|
|
479
851
|
routine: key,
|
|
480
|
-
correlationId:
|
|
852
|
+
correlationId: planContext.placeholder.correlationId,
|
|
481
853
|
parentCorrelationId: parentEvent.correlationId,
|
|
482
854
|
}, "Routine fetch-window pre-pass skipped — acquisition plan empty");
|
|
483
855
|
return { report, block };
|
|
484
856
|
}
|
|
485
|
-
// Execute the fetcher session through the standard router pipeline.
|
|
486
|
-
let context;
|
|
487
857
|
try {
|
|
488
|
-
|
|
858
|
+
return await this.runFanOut({
|
|
859
|
+
key,
|
|
860
|
+
agentDay,
|
|
861
|
+
parentEvent,
|
|
862
|
+
subPlans: planContext.subPlans,
|
|
863
|
+
integrationsSnapshot: planContext.integrationsSnapshot,
|
|
864
|
+
accounts: planContext.accounts,
|
|
865
|
+
timestamps: planContext.timestamps,
|
|
866
|
+
});
|
|
489
867
|
}
|
|
490
868
|
catch (err) {
|
|
491
|
-
return this.fail(key, agentDay, parentEvent, "
|
|
492
|
-
fetcherCorrelationId:
|
|
869
|
+
return this.fail(key, agentDay, parentEvent, "fan-out-failed", err, {
|
|
870
|
+
fetcherCorrelationId: planContext.placeholder.correlationId,
|
|
493
871
|
});
|
|
494
872
|
}
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
873
|
+
}
|
|
874
|
+
buildFanOutPlanContext(parentEvent, key, agentDay) {
|
|
875
|
+
// Resolve binding off a placeholder event so we know the session
|
|
876
|
+
// backend before splitting the plan (the partial body's mode markers
|
|
877
|
+
// and the per-row delegated-same / delegated-cross resolution both
|
|
878
|
+
// depend on the resolved backend). The placeholder borrows the
|
|
879
|
+
// parent correlationId so the empty-plan and plan-assembly-failed
|
|
880
|
+
// short-circuit reports carry a stable correlation id; fan-out
|
|
881
|
+
// sub-sessions get fresh ids per attempt.
|
|
882
|
+
const placeholder = {
|
|
883
|
+
...createEvent({
|
|
884
|
+
type: FETCH_WINDOW_EVENT_TYPE,
|
|
885
|
+
source: parentEvent.source,
|
|
886
|
+
priority: EventPriority.NORMAL,
|
|
887
|
+
correlationId: parentEvent.correlationId,
|
|
888
|
+
}),
|
|
889
|
+
routine: "fetch_window",
|
|
890
|
+
};
|
|
891
|
+
const preBinding = this.agentRouter.resolveBinding(placeholder, {
|
|
892
|
+
processKey: FETCH_WINDOW_PROCESS_KEY,
|
|
893
|
+
});
|
|
894
|
+
const integrationsSnapshot = readIntegrations(this.db);
|
|
895
|
+
const accounts = this.collectAccounts(integrationsSnapshot);
|
|
896
|
+
const timestamps = buildAcquisitionTimestamps(new Date(), this.config.timezone || undefined, this.config.dayBoundaryHour);
|
|
897
|
+
const planInput = {
|
|
898
|
+
routine: key,
|
|
899
|
+
agentDay,
|
|
900
|
+
integrations: integrationsSnapshot,
|
|
901
|
+
sessionBackend: preBinding.main.backendId,
|
|
902
|
+
accounts,
|
|
903
|
+
timestamps,
|
|
904
|
+
};
|
|
905
|
+
return {
|
|
906
|
+
key,
|
|
907
|
+
agentDay,
|
|
908
|
+
placeholder,
|
|
909
|
+
integrationsSnapshot,
|
|
910
|
+
subPlans: splitAcquisitionPlanByIntegration(planInput),
|
|
911
|
+
accounts,
|
|
912
|
+
timestamps,
|
|
913
|
+
};
|
|
914
|
+
}
|
|
915
|
+
/**
|
|
916
|
+
* §5 BackendQuotaError row + §4.4 retryEscalationTier — re-derive the
|
|
917
|
+
* per-integration sub-plan against the CURRENT attempt's binding so a
|
|
918
|
+
* cross-backend swap (escalation tier flips main backend, or
|
|
919
|
+
* `prePassRetryEscalationTier` swaps the per-attempt tier) keeps the
|
|
920
|
+
* plan's `<fetch mode="…">` attribute aligned with the partial body's
|
|
921
|
+
* remaining mode-branch (the partial is filtered for the new backend
|
|
922
|
+
* via `renderPartialForFanOut`). Returns null only when the
|
|
923
|
+
* integration becomes unreachable on the new backend (e.g. native
|
|
924
|
+
* mode bound to A but the attempt re-resolves to B); the runner
|
|
925
|
+
* passes the call through to record the original sub-plan and let
|
|
926
|
+
* the agent emit `no-surface` errors organically.
|
|
927
|
+
*
|
|
928
|
+
* Pure: snapshot inputs (`integrationsSnapshot`, `accounts`,
|
|
929
|
+
* `timestamps`) are frozen at run() entry; only `sessionBackend`
|
|
930
|
+
* varies across attempts. Re-derivation is cheap —
|
|
931
|
+
* `splitAcquisitionPlanByIntegration` walks `ROUTINE_WINDOWS[routine]`
|
|
932
|
+
* (a small constant) without I/O.
|
|
933
|
+
*/
|
|
934
|
+
rebuildSubPlanForBackend(integrationKey, routineKey, agentDay, sessionBackend, integrationsSnapshot, accounts, timestamps) {
|
|
935
|
+
const subPlans = splitAcquisitionPlanByIntegration({
|
|
936
|
+
routine: routineKey,
|
|
937
|
+
agentDay,
|
|
938
|
+
integrations: integrationsSnapshot,
|
|
939
|
+
sessionBackend,
|
|
940
|
+
accounts,
|
|
941
|
+
timestamps,
|
|
942
|
+
});
|
|
943
|
+
return subPlans.find((p) => p.integrationKey === integrationKey) ?? null;
|
|
944
|
+
}
|
|
945
|
+
buildRetryPolicy() {
|
|
946
|
+
return {
|
|
947
|
+
maxAttempts: clampInt(this.config.prePassMaxAttemptsPerIntegration, 1, 5, 3),
|
|
948
|
+
backoffMs: Array.isArray(this.config.prePassBackoffMs)
|
|
949
|
+
? this.config.prePassBackoffMs
|
|
950
|
+
: [1000, 2000, 4000],
|
|
951
|
+
perIntegrationBudgetUsd: nonNegativeNumber(this.config.prePassMaxBudgetUsdPerIntegration, 0.6),
|
|
952
|
+
retryOnPartial: this.config.prePassRetryOnPartial !== false,
|
|
953
|
+
};
|
|
954
|
+
}
|
|
955
|
+
fanOutConcurrency() {
|
|
956
|
+
const configured = this.config.prePassFanOutConcurrency;
|
|
957
|
+
if (configured === null || configured === undefined)
|
|
958
|
+
return null;
|
|
959
|
+
// Cap at the integration roster size — fan-out spawns at most one
|
|
960
|
+
// sub-session per IntegrationKey, so a higher concurrency cannot
|
|
961
|
+
// help and a hardcoded literal would silently restrict whenever a
|
|
962
|
+
// new integration descriptor is added to `INTEGRATION_KEYS`.
|
|
963
|
+
return clampInt(configured, 1, INTEGRATION_KEYS.length, INTEGRATION_KEYS.length);
|
|
964
|
+
}
|
|
965
|
+
retryEscalationTier() {
|
|
966
|
+
const configured = this.config.prePassRetryEscalationTier;
|
|
967
|
+
return configured === "lite" || configured === "medium" || configured === "high"
|
|
968
|
+
? configured
|
|
969
|
+
: null;
|
|
970
|
+
}
|
|
971
|
+
async runFanOut(input) {
|
|
972
|
+
const policy = this.buildRetryPolicy();
|
|
973
|
+
const globalBudget = new FanOutBudgetGuard(nonNegativeNumber(this.config.prePassMaxBudgetUsdPerRoutine, 1.5));
|
|
974
|
+
const tasks = input.subPlans.map((subPlan) => async () => this.runOneIntegrationWithRetry({
|
|
975
|
+
...input,
|
|
976
|
+
subPlan,
|
|
977
|
+
policy,
|
|
978
|
+
globalBudget,
|
|
979
|
+
}));
|
|
980
|
+
const subReports = await runWithConcurrency(tasks, this.fanOutConcurrency());
|
|
981
|
+
const merged = mergeSubReports(subReports, input.key, input.agentDay);
|
|
982
|
+
// §7.1 example shape — `integrations: [{key, status, attempts, fetched,
|
|
983
|
+
// posted, duplicates, durationMs, costUsd, finalError?}]` plus an
|
|
984
|
+
// `aggregate` headline. Reuse the helpers so the SSE
|
|
985
|
+
// `prepass_completed` payload (in `run()` below) and this log line
|
|
986
|
+
// never disagree on the per-integration numbers.
|
|
987
|
+
const integrations = subReports.map(summarizeIntegrationReport);
|
|
988
|
+
logger.info({
|
|
989
|
+
routine: input.key,
|
|
990
|
+
parentCorrelationId: input.parentEvent.correlationId,
|
|
991
|
+
integrations,
|
|
992
|
+
aggregate: {
|
|
993
|
+
...summarizeFetchReport(merged.report),
|
|
994
|
+
// Use the live `globalBudget.spent` counter here so the daemon
|
|
995
|
+
// log line is the canonical readout for "what the global cap
|
|
996
|
+
// saw" — the §7.2 SSE payload mirrors `summarizeFetchReport`'s
|
|
997
|
+
// sum-over-attempts. Today the two converge (every committed
|
|
998
|
+
// cost is also reflected in an attempt's `costUsd`), but tying
|
|
999
|
+
// the log line to the guard means a future divergence (e.g. a
|
|
1000
|
+
// commit path that bypasses the per-attempt record) surfaces
|
|
1001
|
+
// here instead of going silent.
|
|
1002
|
+
costUsd: globalBudget.spent,
|
|
1003
|
+
},
|
|
1004
|
+
}, "Routine fetch-window fan-out completed");
|
|
1005
|
+
return merged;
|
|
1006
|
+
}
|
|
1007
|
+
async runOneIntegrationWithRetry(input) {
|
|
1008
|
+
const attempts = [];
|
|
1009
|
+
// Per-integration budget cap is enforced at TWO complementary layers,
|
|
1010
|
+
// BOTH driven by `policy.perIntegrationBudgetUsd`:
|
|
1011
|
+
// (1) `integrationBudget` (this guard) — HARD pre-attempt reservation
|
|
1012
|
+
// against the binding's `max_budget_usd` envelope. Trips before
|
|
1013
|
+
// the next SDK call when the upper bound on the next attempt
|
|
1014
|
+
// would exceed the cap. Surfaces as `{type:"budget-cap"}`.
|
|
1015
|
+
// (2) `defaultRetryDecision`'s cumulative-cost branch — SOFT
|
|
1016
|
+
// post-attempt check against actual `costUsd` summed across
|
|
1017
|
+
// priorAttempts. Surfaces as `decision.reason="per-integration-budget-cap"`.
|
|
1018
|
+
// Layer (1) is the pessimistic guard (envelope ≥ actual); layer (2)
|
|
1019
|
+
// catches the case where individual attempts cost more than expected.
|
|
1020
|
+
// Either fires depending on the cost shape — both are intentional.
|
|
1021
|
+
const integrationBudget = new FanOutBudgetGuard(input.policy.perIntegrationBudgetUsd);
|
|
1022
|
+
const retryOn = input.policy.retryOn ?? defaultRetryDecision;
|
|
1023
|
+
const escalationTier = this.retryEscalationTier();
|
|
1024
|
+
let retriesExhausted = false;
|
|
1025
|
+
for (let attempt = 1; attempt <= input.policy.maxAttempts; attempt++) {
|
|
1026
|
+
const startedAt = new Date().toISOString();
|
|
1027
|
+
const fetcherEvent = this.createFanOutFetcherEvent(input.parentEvent, input.key, input.subPlan, attempt, input.policy.maxAttempts);
|
|
1028
|
+
const requestedTier = attempt > 1 && escalationTier !== null ? escalationTier : undefined;
|
|
1029
|
+
// Symmetry guarantee: every iteration emits exactly one
|
|
1030
|
+
// `prepass_subsession_started` BEFORE any work, and exactly one
|
|
1031
|
+
// `prepass_subsession_completed` after the attempt's outcome is
|
|
1032
|
+
// recorded — including the binding-resolve-failed and budget-cap
|
|
1033
|
+
// short-circuits, which previously were invisible to the dashboard.
|
|
1034
|
+
this.emitSubSessionStarted(input, fetcherEvent.correlationId, attempt);
|
|
1035
|
+
let binding;
|
|
1036
|
+
try {
|
|
1037
|
+
binding = this.agentRouter.resolveBinding(fetcherEvent, {
|
|
1038
|
+
processKey: FETCH_WINDOW_PROCESS_KEY,
|
|
1039
|
+
...(requestedTier ? { requestedTier } : {}),
|
|
1040
|
+
});
|
|
1041
|
+
}
|
|
1042
|
+
catch (err) {
|
|
1043
|
+
const record = this.failedAttemptRecord(attempt, fetcherEvent.correlationId, startedAt, "binding-resolve-failed", err);
|
|
1044
|
+
attempts.push(record);
|
|
1045
|
+
const decision = retryOn(record, attempt, input.policy, attempts.slice(0, -1));
|
|
1046
|
+
this.logFanOutFailure(input, fetcherEvent, record, decision, {
|
|
1047
|
+
failureKind: "binding-resolve-failed",
|
|
1048
|
+
err,
|
|
1049
|
+
startedAt,
|
|
1050
|
+
});
|
|
1051
|
+
this.emitSubSessionCompleted(input, fetcherEvent.correlationId, attempt, record, decision);
|
|
1052
|
+
if (!decision.retry) {
|
|
1053
|
+
retriesExhausted = this.didExhaustRetries(record, decision, input.policy);
|
|
1054
|
+
break;
|
|
1055
|
+
}
|
|
1056
|
+
await sleep(this.backoffForAttempt(input.policy, attempt));
|
|
1057
|
+
continue;
|
|
1058
|
+
}
|
|
1059
|
+
// §5 BackendQuotaError row + §4.4 retryEscalationTier — the
|
|
1060
|
+
// pre-pass plan was originally rendered against
|
|
1061
|
+
// `preBinding.main.backendId` (set in `buildFanOutPlanContext`).
|
|
1062
|
+
// If THIS attempt's binding picks a different backend (escalation
|
|
1063
|
+
// tier flipped main, or the resolver picked a different default
|
|
1064
|
+
// for a higher tier), re-derive the per-integration sub-plan
|
|
1065
|
+
// against the current backend so the plan's `<fetch mode="…">`
|
|
1066
|
+
// attributes match the partial body's remaining mode-branch
|
|
1067
|
+
// (`renderPartialForFanOut` filters the partial against the
|
|
1068
|
+
// CURRENT backend; the plan must follow). The recompute uses the
|
|
1069
|
+
// frozen accounts + timestamps from `buildFanOutPlanContext` so
|
|
1070
|
+
// the windows don't drift mid-routine.
|
|
1071
|
+
const livePlan = this.rebuildSubPlanForBackend(input.subPlan.integrationKey, input.key, input.agentDay, binding.main.backendId, input.integrationsSnapshot, input.accounts, input.timestamps);
|
|
1072
|
+
// Native-mode integrations bound to a specific backend can become
|
|
1073
|
+
// unreachable after a cross-backend binding swap (the new backend
|
|
1074
|
+
// has no native connector for this integration). When that
|
|
1075
|
+
// happens, `livePlan` is null — fall back to the original
|
|
1076
|
+
// sub-plan so the agent still iterates the rows and surfaces
|
|
1077
|
+
// `no-surface` errors organically. The retry policy will then
|
|
1078
|
+
// either escalate or short-circuit per the existing matrix.
|
|
1079
|
+
if (livePlan && livePlan.block !== input.subPlan.block) {
|
|
1080
|
+
// Mutate the per-attempt fetcher event so ContextBuilder injects
|
|
1081
|
+
// the freshly-rendered plan block instead of the stale one.
|
|
1082
|
+
// Cheap — the event is a one-shot per attempt.
|
|
1083
|
+
fetcherEvent.data.acquisitionPlanBlock
|
|
1084
|
+
= livePlan.block;
|
|
1085
|
+
}
|
|
1086
|
+
const estimateUsd = binding.main.maxBudgetUsd;
|
|
1087
|
+
const globalReservation = input.globalBudget.reserve(estimateUsd);
|
|
1088
|
+
if (!globalReservation.ok) {
|
|
1089
|
+
const record = this.budgetCapAttemptRecord(attempt, fetcherEvent.correlationId, startedAt, "global-budget-cap", globalReservation.remaining);
|
|
1090
|
+
attempts.push(record);
|
|
1091
|
+
const decision = {
|
|
1092
|
+
retry: false,
|
|
1093
|
+
reason: "global-budget-cap",
|
|
1094
|
+
};
|
|
1095
|
+
this.logFanOutFailure(input, fetcherEvent, record, decision, {
|
|
1096
|
+
failureKind: "global-budget-cap",
|
|
1097
|
+
binding: binding.main,
|
|
1098
|
+
startedAt,
|
|
1099
|
+
});
|
|
1100
|
+
this.emitSubSessionCompleted(input, fetcherEvent.correlationId, attempt, record, decision);
|
|
1101
|
+
retriesExhausted = true;
|
|
1102
|
+
break;
|
|
1103
|
+
}
|
|
1104
|
+
const integrationReservation = integrationBudget.reserve(estimateUsd);
|
|
1105
|
+
if (!integrationReservation.ok) {
|
|
1106
|
+
input.globalBudget.commit(globalReservation, 0);
|
|
1107
|
+
const record = this.budgetCapAttemptRecord(attempt, fetcherEvent.correlationId, startedAt, "budget-cap", integrationReservation.remaining);
|
|
1108
|
+
attempts.push(record);
|
|
1109
|
+
const decision = {
|
|
1110
|
+
retry: false,
|
|
1111
|
+
reason: "budget-cap",
|
|
1112
|
+
};
|
|
1113
|
+
this.logFanOutFailure(input, fetcherEvent, record, decision, {
|
|
1114
|
+
failureKind: "budget-cap",
|
|
1115
|
+
binding: binding.main,
|
|
1116
|
+
startedAt,
|
|
1117
|
+
});
|
|
1118
|
+
this.emitSubSessionCompleted(input, fetcherEvent.correlationId, attempt, record, decision);
|
|
1119
|
+
retriesExhausted = true;
|
|
1120
|
+
break;
|
|
1121
|
+
}
|
|
1122
|
+
let context;
|
|
1123
|
+
try {
|
|
1124
|
+
context = await this.contextBuilder.build(fetcherEvent);
|
|
1125
|
+
}
|
|
1126
|
+
catch (err) {
|
|
1127
|
+
input.globalBudget.commit(globalReservation, 0);
|
|
1128
|
+
integrationBudget.commit(integrationReservation, 0);
|
|
1129
|
+
const record = this.failedAttemptRecord(attempt, fetcherEvent.correlationId, startedAt, "context-build-failed", err);
|
|
1130
|
+
attempts.push(record);
|
|
1131
|
+
const decision = retryOn(record, attempt, input.policy, attempts.slice(0, -1));
|
|
1132
|
+
this.logFanOutFailure(input, fetcherEvent, record, decision, {
|
|
1133
|
+
failureKind: "context-build-failed",
|
|
1134
|
+
err,
|
|
1135
|
+
binding: binding.main,
|
|
1136
|
+
startedAt,
|
|
1137
|
+
});
|
|
1138
|
+
this.emitSubSessionCompleted(input, fetcherEvent.correlationId, attempt, record, decision);
|
|
1139
|
+
if (!decision.retry) {
|
|
1140
|
+
retriesExhausted = this.didExhaustRetries(record, decision, input.policy);
|
|
1141
|
+
break;
|
|
1142
|
+
}
|
|
1143
|
+
await sleep(this.backoffForAttempt(input.policy, attempt));
|
|
1144
|
+
continue;
|
|
1145
|
+
}
|
|
1146
|
+
let result = null;
|
|
1147
|
+
let executeErr = undefined;
|
|
1148
|
+
let record;
|
|
1149
|
+
try {
|
|
1150
|
+
const priorAttemptHintBlock = buildPriorAttemptHintBlock(attempts, input.subPlan.integrationKey);
|
|
1151
|
+
// PRE_PASS_FAN_OUT_DESIGN.md §4.2 — every sub-session sees the
|
|
1152
|
+
// `routine.fetch_window` task-flow body with `{integration_partial}`
|
|
1153
|
+
// substituted for the one partial its integrationKey owns. The
|
|
1154
|
+
// integrations snapshot fed to both the partial's mode filter
|
|
1155
|
+
// and the composed allowed-tools list is sliced to a single key
|
|
1156
|
+
// so the sub-session cannot see — or call MCP tools for — any
|
|
1157
|
+
// other integration's surface (defense-in-depth on top of the
|
|
1158
|
+
// prompt isolation).
|
|
1159
|
+
const slicedIntegrations = this.sliceIntegrationSnapshot(input.integrationsSnapshot, input.subPlan.integrationKey);
|
|
1160
|
+
const partialFilename = getIntegrationDescriptor(input.subPlan.integrationKey).prePassPartial;
|
|
1161
|
+
if (!partialFilename) {
|
|
1162
|
+
throw new Error(`Integration "${input.subPlan.integrationKey}" has no prePassPartial descriptor field — cannot dispatch fan-out sub-session`);
|
|
1163
|
+
}
|
|
1164
|
+
const reassemblePrompt = (bid) => {
|
|
1165
|
+
const assembled = this.prompt.assemble(FETCH_WINDOW_EVENT_TYPE, FETCH_WINDOW_PROCESS_KEY, bid);
|
|
1166
|
+
// Re-render the partial against the resolved backend each
|
|
1167
|
+
// time the SDK reassembles (e.g. on quota-driven fallback).
|
|
1168
|
+
// Mode markers inside the partial depend on the chosen
|
|
1169
|
+
// backend, so a cross-backend fallback must regenerate the
|
|
1170
|
+
// body to match the new MCP surface — matches the failure
|
|
1171
|
+
// mode catalogue's BackendQuotaError row.
|
|
1172
|
+
const partialBody = renderPartialForFanOut(partialFilename, slicedIntegrations, bid);
|
|
1173
|
+
const filled = assembled.replaceAll(FETCH_WINDOW_INTEGRATION_PARTIAL_PLACEHOLDER, partialBody);
|
|
1174
|
+
return priorAttemptHintBlock
|
|
1175
|
+
? `${priorAttemptHintBlock}\n\n${filled}`
|
|
1176
|
+
: filled;
|
|
1177
|
+
};
|
|
1178
|
+
const prompt = reassemblePrompt(binding.main.backendId);
|
|
1179
|
+
const allowedToolsOverride = composePrePassAllowedTools(this.config.apiPort, slicedIntegrations, binding.main.backendId);
|
|
1180
|
+
result = await this.agentRouter.execute({
|
|
1181
|
+
prompt,
|
|
1182
|
+
context,
|
|
1183
|
+
event: fetcherEvent,
|
|
1184
|
+
processKey: FETCH_WINDOW_PROCESS_KEY,
|
|
1185
|
+
preResolvedBinding: binding,
|
|
1186
|
+
reassemblePrompt,
|
|
1187
|
+
allowedToolsOverride,
|
|
1188
|
+
});
|
|
1189
|
+
input.globalBudget.commit(globalReservation, result.costUsd);
|
|
1190
|
+
integrationBudget.commit(integrationReservation, result.costUsd);
|
|
1191
|
+
record = this.attemptRecordFromResult(attempt, fetcherEvent, startedAt, result);
|
|
1192
|
+
}
|
|
1193
|
+
catch (err) {
|
|
1194
|
+
input.globalBudget.commit(globalReservation, 0);
|
|
1195
|
+
integrationBudget.commit(integrationReservation, 0);
|
|
1196
|
+
executeErr = err;
|
|
1197
|
+
record = this.failedAttemptRecord(attempt, fetcherEvent.correlationId, startedAt, "agent-execute-failed", err);
|
|
1198
|
+
}
|
|
1199
|
+
attempts.push(record);
|
|
1200
|
+
const decision = retryOn(record, attempt, input.policy, attempts.slice(0, -1));
|
|
1201
|
+
if (result) {
|
|
1202
|
+
this.logFanOutAttempt(input, fetcherEvent, result, record, decision, binding.main.backendId);
|
|
1203
|
+
}
|
|
1204
|
+
else {
|
|
1205
|
+
// §7.1 — when the SDK actually invoked a backend session and
|
|
1206
|
+
// that session threw (timeout, BackendQuotaError, transport
|
|
1207
|
+
// failure, …) the audit feed must reflect it. Successful
|
|
1208
|
+
// attempts log via `logFanOutAttempt` with the AgentResult; the
|
|
1209
|
+
// throw path has no result, so we route through
|
|
1210
|
+
// `logFanOutFailure` which carries the same `detail.prePass`
|
|
1211
|
+
// payload the metrics aggregator filters on (preserving the
|
|
1212
|
+
// failure on `/metrics/pre-pass` alongside the other four
|
|
1213
|
+
// pre-execute failure modes).
|
|
1214
|
+
this.logFanOutFailure(input, fetcherEvent, record, decision, {
|
|
1215
|
+
failureKind: "agent-execute-failed",
|
|
1216
|
+
err: executeErr,
|
|
1217
|
+
binding: binding.main,
|
|
1218
|
+
startedAt,
|
|
1219
|
+
});
|
|
1220
|
+
}
|
|
1221
|
+
this.emitSubSessionCompleted(input, fetcherEvent.correlationId, attempt, record, decision);
|
|
1222
|
+
if (!decision.retry) {
|
|
1223
|
+
retriesExhausted = this.didExhaustRetries(record, decision, input.policy);
|
|
1224
|
+
break;
|
|
1225
|
+
}
|
|
1226
|
+
await sleep(this.backoffForAttempt(input.policy, attempt));
|
|
1227
|
+
}
|
|
1228
|
+
if (attempts.length === 0) {
|
|
1229
|
+
const now = new Date().toISOString();
|
|
1230
|
+
attempts.push({
|
|
1231
|
+
attempt: 0,
|
|
1232
|
+
status: "skipped",
|
|
1233
|
+
fetched: 0,
|
|
1234
|
+
posted: 0,
|
|
1235
|
+
duplicates: 0,
|
|
1236
|
+
errors: [],
|
|
1237
|
+
fetcherCorrelationId: input.parentEvent.correlationId,
|
|
1238
|
+
startedAt: now,
|
|
1239
|
+
endedAt: now,
|
|
1240
|
+
costUsd: 0,
|
|
1241
|
+
numTurns: 0,
|
|
499
1242
|
});
|
|
500
1243
|
}
|
|
501
|
-
|
|
502
|
-
|
|
1244
|
+
const final = attempts[attempts.length - 1];
|
|
1245
|
+
const allErrors = attempts.flatMap((att) => att.errors);
|
|
1246
|
+
return {
|
|
1247
|
+
...final,
|
|
1248
|
+
errors: allErrors.length > 0 ? allErrors : final.errors,
|
|
1249
|
+
integrationKey: input.subPlan.integrationKey,
|
|
1250
|
+
attempts,
|
|
1251
|
+
retriesExhausted,
|
|
1252
|
+
};
|
|
1253
|
+
}
|
|
1254
|
+
/**
|
|
1255
|
+
* Emit `prepass_subsession_started` for an attempt that is about to run.
|
|
1256
|
+
* Called at the TOP of every loop iteration so the started/completed
|
|
1257
|
+
* pair is symmetric across all paths — including binding-resolve-failed
|
|
1258
|
+
* and budget-cap short-circuits, which previously emitted neither.
|
|
1259
|
+
*/
|
|
1260
|
+
emitSubSessionStarted(input, fetcherCorrelationId, attempt) {
|
|
1261
|
+
this.broadcastPrepassProgress("prepass_subsession_started", input.parentEvent, {
|
|
1262
|
+
routine: input.key,
|
|
1263
|
+
integrationKey: input.subPlan.integrationKey,
|
|
1264
|
+
attempt,
|
|
1265
|
+
maxAttempts: input.policy.maxAttempts,
|
|
1266
|
+
fetcherCorrelationId,
|
|
1267
|
+
parentCorrelationId: input.parentEvent.correlationId,
|
|
1268
|
+
});
|
|
1269
|
+
}
|
|
1270
|
+
/**
|
|
1271
|
+
* Emit `prepass_subsession_completed` for an attempt that has just
|
|
1272
|
+
* recorded its outcome. Mirror of `emitSubSessionStarted` — invoked
|
|
1273
|
+
* once per iteration after every code path that pushes a record
|
|
1274
|
+
* (success, parse error, agent throw, binding-resolve-failed,
|
|
1275
|
+
* global-budget-cap, per-integration-budget-cap).
|
|
1276
|
+
*/
|
|
1277
|
+
emitSubSessionCompleted(input, fetcherCorrelationId, attempt, record, decision) {
|
|
1278
|
+
this.broadcastPrepassProgress("prepass_subsession_completed", input.parentEvent, {
|
|
1279
|
+
routine: input.key,
|
|
1280
|
+
integrationKey: input.subPlan.integrationKey,
|
|
1281
|
+
attempt,
|
|
1282
|
+
maxAttempts: input.policy.maxAttempts,
|
|
1283
|
+
fetcherCorrelationId,
|
|
1284
|
+
parentCorrelationId: input.parentEvent.correlationId,
|
|
1285
|
+
status: record.status,
|
|
1286
|
+
fetched: record.fetched,
|
|
1287
|
+
posted: record.posted,
|
|
1288
|
+
duplicates: record.duplicates,
|
|
1289
|
+
willRetry: decision.retry,
|
|
1290
|
+
retryReason: decision.reason,
|
|
1291
|
+
});
|
|
1292
|
+
}
|
|
1293
|
+
createFanOutFetcherEvent(parentEvent, key, subPlan, attempt, maxAttempts) {
|
|
1294
|
+
return {
|
|
1295
|
+
...createEvent({
|
|
1296
|
+
type: FETCH_WINDOW_EVENT_TYPE,
|
|
1297
|
+
source: parentEvent.source,
|
|
1298
|
+
priority: EventPriority.NORMAL,
|
|
1299
|
+
}),
|
|
1300
|
+
routine: "fetch_window",
|
|
1301
|
+
data: {
|
|
1302
|
+
acquisitionPlanBlock: subPlan.block,
|
|
1303
|
+
parentRoutine: key,
|
|
1304
|
+
parentCorrelationId: parentEvent.correlationId,
|
|
1305
|
+
prePassFanOut: {
|
|
1306
|
+
integrationKey: subPlan.integrationKey,
|
|
1307
|
+
attempt,
|
|
1308
|
+
maxAttempts,
|
|
1309
|
+
fetchRowCount: subPlan.fetchRowCount,
|
|
1310
|
+
rowsHaveAccount: subPlan.rowsHaveAccount,
|
|
1311
|
+
},
|
|
1312
|
+
},
|
|
1313
|
+
};
|
|
1314
|
+
}
|
|
1315
|
+
sliceIntegrationSnapshot(integrations, key) {
|
|
1316
|
+
const state = integrations[key];
|
|
1317
|
+
return state ? { [key]: state } : {};
|
|
1318
|
+
}
|
|
1319
|
+
attemptRecordFromResult(attempt, fetcherEvent, startedAt, result) {
|
|
1320
|
+
const parsed = parseFetchWindowOutput(result.output);
|
|
1321
|
+
const endedAt = new Date().toISOString();
|
|
1322
|
+
if ("parseError" in parsed) {
|
|
1323
|
+
return {
|
|
1324
|
+
attempt,
|
|
1325
|
+
status: "failed",
|
|
1326
|
+
fetched: 0,
|
|
1327
|
+
posted: 0,
|
|
1328
|
+
duplicates: 0,
|
|
1329
|
+
errors: [
|
|
1330
|
+
{
|
|
1331
|
+
type: "pre-pass-parse-failed",
|
|
1332
|
+
reason: parsed.parseError,
|
|
1333
|
+
attempt,
|
|
1334
|
+
},
|
|
1335
|
+
],
|
|
1336
|
+
parseError: parsed.parseError,
|
|
503
1337
|
fetcherCorrelationId: fetcherEvent.correlationId,
|
|
504
|
-
|
|
1338
|
+
startedAt,
|
|
1339
|
+
endedAt,
|
|
1340
|
+
costUsd: result.costUsd,
|
|
1341
|
+
numTurns: result.numTurns,
|
|
1342
|
+
};
|
|
1343
|
+
}
|
|
1344
|
+
return {
|
|
1345
|
+
attempt,
|
|
1346
|
+
status: parsed.status,
|
|
1347
|
+
fetched: parsed.fetched,
|
|
1348
|
+
posted: parsed.posted,
|
|
1349
|
+
duplicates: parsed.duplicates,
|
|
1350
|
+
errors: parsed.errors.map((err) => ({ ...err, attempt })),
|
|
1351
|
+
fetcherCorrelationId: fetcherEvent.correlationId,
|
|
1352
|
+
startedAt,
|
|
1353
|
+
endedAt,
|
|
1354
|
+
costUsd: result.costUsd,
|
|
1355
|
+
numTurns: result.numTurns,
|
|
1356
|
+
};
|
|
1357
|
+
}
|
|
1358
|
+
failedAttemptRecord(attempt, fetcherCorrelationId, startedAt, kind, err) {
|
|
1359
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
1360
|
+
const endedAt = new Date().toISOString();
|
|
1361
|
+
return {
|
|
1362
|
+
attempt,
|
|
1363
|
+
status: "failed",
|
|
1364
|
+
fetched: 0,
|
|
1365
|
+
posted: 0,
|
|
1366
|
+
duplicates: 0,
|
|
1367
|
+
errors: [{ type: "pre-pass-failed", kind, message, attempt }],
|
|
1368
|
+
fetcherCorrelationId,
|
|
1369
|
+
startedAt,
|
|
1370
|
+
endedAt,
|
|
1371
|
+
costUsd: 0,
|
|
1372
|
+
numTurns: 0,
|
|
1373
|
+
};
|
|
1374
|
+
}
|
|
1375
|
+
budgetCapAttemptRecord(attempt, fetcherCorrelationId, startedAt, type, remaining) {
|
|
1376
|
+
const endedAt = new Date().toISOString();
|
|
1377
|
+
return {
|
|
1378
|
+
attempt,
|
|
1379
|
+
status: "failed",
|
|
1380
|
+
fetched: 0,
|
|
1381
|
+
posted: 0,
|
|
1382
|
+
duplicates: 0,
|
|
1383
|
+
errors: [{ type, remaining, attempt }],
|
|
1384
|
+
fetcherCorrelationId,
|
|
1385
|
+
startedAt,
|
|
1386
|
+
endedAt,
|
|
1387
|
+
costUsd: 0,
|
|
1388
|
+
numTurns: 0,
|
|
1389
|
+
};
|
|
1390
|
+
}
|
|
1391
|
+
didExhaustRetries(record, decision, policy) {
|
|
1392
|
+
if (record.status === "success" || record.status === "skipped")
|
|
1393
|
+
return false;
|
|
1394
|
+
// FanOutBudgetGuard.reserve() failures surface as explicit error rows
|
|
1395
|
+
// — those are the parallel-reservation cap trips covered by §4.7.
|
|
1396
|
+
if (record.errors.some((err) => err.type === "budget-cap" || err.type === "global-budget-cap")) {
|
|
1397
|
+
return true;
|
|
1398
|
+
}
|
|
1399
|
+
// §4.3 contract: "exhausted maxAttempts (or a budget/global cap tripped)".
|
|
1400
|
+
// The per-integration cumulative-cost cap branch in defaultRetryDecision
|
|
1401
|
+
// returns reason=BUDGET_CAP without leaving a budget-cap error row on
|
|
1402
|
+
// the attempt — without this clause the sub-report would carry
|
|
1403
|
+
// retriesExhausted=false even though the loop terminated on a cap.
|
|
1404
|
+
if (decision.reason === RETRY_REASONS.MAX_ATTEMPTS
|
|
1405
|
+
|| decision.reason === RETRY_REASONS.BUDGET_CAP) {
|
|
1406
|
+
return true;
|
|
505
1407
|
}
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
1408
|
+
return record.attempt >= policy.maxAttempts;
|
|
1409
|
+
}
|
|
1410
|
+
backoffForAttempt(policy, attempt) {
|
|
1411
|
+
if (attempt >= policy.maxAttempts)
|
|
1412
|
+
return 0;
|
|
1413
|
+
const configured = policy.backoffMs[attempt - 1];
|
|
1414
|
+
if (typeof configured === "number")
|
|
1415
|
+
return Math.max(0, configured);
|
|
1416
|
+
const last = policy.backoffMs[policy.backoffMs.length - 1];
|
|
1417
|
+
return typeof last === "number" ? Math.max(0, last) : 0;
|
|
1418
|
+
}
|
|
1419
|
+
/**
|
|
1420
|
+
* Unified audit-row companion for every fan-out failure mode —
|
|
1421
|
+
* binding-resolve-failed, global-budget-cap, budget-cap (per-integration),
|
|
1422
|
+
* context-build-failed, and agent-execute-failed. Routes through
|
|
1423
|
+
* `audit.logError` (writes `result='failed'`) with a `prePass` payload
|
|
1424
|
+
* so `MetricsCollector.collectPrePassMetrics` can see every failure
|
|
1425
|
+
* mode without a parallel `result='success'` row. Before this helper
|
|
1426
|
+
* existed, the four pre-execute branches wrote nothing at all and the
|
|
1427
|
+
* agent-execute path wrote a `failureKind`-only row that the aggregator
|
|
1428
|
+
* silently skipped (it filters on `detail.prePass` being a non-null
|
|
1429
|
+
* object). Cost / tokens are intentionally NOT supplied — pre-execute
|
|
1430
|
+
* paths have zero cost, the agent-execute throw path has no usable
|
|
1431
|
+
* AgentResult, so any figure here would be a guess; the aggregator
|
|
1432
|
+
* coalesces missing `cost_usd` to 0.
|
|
1433
|
+
*/
|
|
1434
|
+
logFanOutFailure(input, fetcherEvent, record, decision, options) {
|
|
519
1435
|
try {
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
1436
|
+
const message = options.err instanceof Error
|
|
1437
|
+
? options.err.message
|
|
1438
|
+
: options.err !== undefined
|
|
1439
|
+
? String(options.err)
|
|
1440
|
+
: options.failureKind;
|
|
1441
|
+
const error = new Error(message);
|
|
1442
|
+
const startMs = Date.parse(options.startedAt);
|
|
1443
|
+
const durationMs = Number.isFinite(startMs)
|
|
1444
|
+
? Math.max(0, Date.now() - startMs)
|
|
1445
|
+
: undefined;
|
|
1446
|
+
this.audit.logError(fetcherEvent, error, "autonomous", {
|
|
1447
|
+
...(durationMs !== undefined ? { durationMs } : {}),
|
|
1448
|
+
...(options.binding ? { backendId: options.binding.backendId } : {}),
|
|
1449
|
+
...(options.binding ? { modelId: options.binding.modelId } : {}),
|
|
1450
|
+
failureKind: options.failureKind,
|
|
1451
|
+
prePass: {
|
|
1452
|
+
parentCorrelationId: input.parentEvent.correlationId,
|
|
1453
|
+
parentRoutine: input.key,
|
|
1454
|
+
integrationKey: input.subPlan.integrationKey,
|
|
1455
|
+
attempt: record.attempt,
|
|
1456
|
+
maxAttempts: input.policy.maxAttempts,
|
|
1457
|
+
retriedFromAttempt: record.attempt > 1 ? record.attempt - 1 : null,
|
|
1458
|
+
status: record.status,
|
|
1459
|
+
fetched: record.fetched,
|
|
1460
|
+
posted: record.posted,
|
|
1461
|
+
duplicates: record.duplicates,
|
|
1462
|
+
errors: record.errors,
|
|
1463
|
+
willRetry: decision.retry,
|
|
1464
|
+
retryReason: decision.reason,
|
|
1465
|
+
...(options.binding ? { requestedBackend: options.binding.backendId } : {}),
|
|
1466
|
+
},
|
|
528
1467
|
});
|
|
529
1468
|
}
|
|
530
|
-
catch (
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
1469
|
+
catch (logErr) {
|
|
1470
|
+
logger.warn({
|
|
1471
|
+
err: logErr,
|
|
1472
|
+
routine: input.key,
|
|
1473
|
+
integrationKey: input.subPlan.integrationKey,
|
|
1474
|
+
failureKind: options.failureKind,
|
|
1475
|
+
correlationId: fetcherEvent.correlationId,
|
|
1476
|
+
}, "Failed to log routine.fetch_window fan-out failure audit row");
|
|
534
1477
|
}
|
|
535
|
-
|
|
536
|
-
|
|
1478
|
+
}
|
|
1479
|
+
logFanOutAttempt(input, fetcherEvent, result, record, decision, requestedBackend) {
|
|
1480
|
+
// §5 BackendQuotaError mitigation — set `fallbackTriggered` when the
|
|
1481
|
+
// backend that actually executed differs from the binding the runner
|
|
1482
|
+
// asked for. The audit row's `backend` column carries the ACTUAL
|
|
1483
|
+
// backend (set from `result.backendId` above); `requestedBackend`
|
|
1484
|
+
// surfaces what the runner intended, so the operator can spot
|
|
1485
|
+
// recurring fallbacks by grepping `fallbackTriggered: true` rows
|
|
1486
|
+
// without reconstructing the binding state from the daemon log.
|
|
1487
|
+
const fallbackTriggered = result.backendId !== undefined && result.backendId !== requestedBackend;
|
|
537
1488
|
try {
|
|
538
1489
|
this.audit.logAction({
|
|
539
1490
|
event: fetcherEvent,
|
|
@@ -550,48 +1501,39 @@ export class RoutineFetchWindowRunner {
|
|
|
550
1501
|
...(typeof result.advisorCallCount === "number"
|
|
551
1502
|
? { advisorCallCount: result.advisorCallCount }
|
|
552
1503
|
: {}),
|
|
1504
|
+
prePass: {
|
|
1505
|
+
parentCorrelationId: input.parentEvent.correlationId,
|
|
1506
|
+
// §7.3 metric aggregation — every fan-out audit row carries
|
|
1507
|
+
// the parent routine key so `/metrics/pre-pass` can group by
|
|
1508
|
+
// routine without joining back to the parent's row.
|
|
1509
|
+
parentRoutine: input.key,
|
|
1510
|
+
integrationKey: input.subPlan.integrationKey,
|
|
1511
|
+
attempt: record.attempt,
|
|
1512
|
+
maxAttempts: input.policy.maxAttempts,
|
|
1513
|
+
// §7.1 example surfaces `retriedFromAttempt`. `null` for the
|
|
1514
|
+
// first attempt in a sub-session's chain; otherwise the prior
|
|
1515
|
+
// attempt index. Derivable but cheaper than a cross-row join.
|
|
1516
|
+
retriedFromAttempt: record.attempt > 1 ? record.attempt - 1 : null,
|
|
1517
|
+
status: record.status,
|
|
1518
|
+
fetched: record.fetched,
|
|
1519
|
+
posted: record.posted,
|
|
1520
|
+
duplicates: record.duplicates,
|
|
1521
|
+
errors: record.errors,
|
|
1522
|
+
willRetry: decision.retry,
|
|
1523
|
+
retryReason: decision.reason,
|
|
1524
|
+
...(fallbackTriggered ? { fallbackTriggered: true } : {}),
|
|
1525
|
+
requestedBackend,
|
|
1526
|
+
},
|
|
553
1527
|
});
|
|
554
1528
|
}
|
|
555
1529
|
catch (err) {
|
|
556
|
-
logger.warn({ err, routine: key, correlationId: fetcherEvent.correlationId }, "Failed to log routine.fetch_window agent_actions row");
|
|
557
|
-
}
|
|
558
|
-
const parsed = parseFetchWindowOutput(result.output);
|
|
559
|
-
if ("parseError" in parsed) {
|
|
560
|
-
const report = {
|
|
561
|
-
status: "failed",
|
|
562
|
-
fetched: 0,
|
|
563
|
-
posted: 0,
|
|
564
|
-
duplicates: 0,
|
|
565
|
-
errors: [{ type: "pre-pass-parse-failed", reason: parsed.parseError }],
|
|
566
|
-
skipped: false,
|
|
567
|
-
failureReason: parsed.parseError,
|
|
568
|
-
fetcherCorrelationId: fetcherEvent.correlationId,
|
|
569
|
-
};
|
|
570
|
-
const block = renderFetchReportBlock(report, { routine: key, agentDay });
|
|
571
1530
|
logger.warn({
|
|
572
|
-
|
|
573
|
-
|
|
1531
|
+
err,
|
|
1532
|
+
routine: input.key,
|
|
1533
|
+
integrationKey: input.subPlan.integrationKey,
|
|
574
1534
|
correlationId: fetcherEvent.correlationId,
|
|
575
|
-
|
|
576
|
-
}, "Routine fetch-window pre-pass output unparsable — parent routine will see <fetch_report status='failed'>");
|
|
577
|
-
return { report, block };
|
|
1535
|
+
}, "Failed to log routine.fetch_window fan-out agent_actions row");
|
|
578
1536
|
}
|
|
579
|
-
const report = {
|
|
580
|
-
...parsed,
|
|
581
|
-
fetcherCorrelationId: fetcherEvent.correlationId,
|
|
582
|
-
};
|
|
583
|
-
const block = renderFetchReportBlock(report, { routine: key, agentDay });
|
|
584
|
-
logger.info({
|
|
585
|
-
routine: key,
|
|
586
|
-
status: report.status,
|
|
587
|
-
fetched: report.fetched,
|
|
588
|
-
posted: report.posted,
|
|
589
|
-
duplicates: report.duplicates,
|
|
590
|
-
errorCount: report.errors.length,
|
|
591
|
-
correlationId: fetcherEvent.correlationId,
|
|
592
|
-
parentCorrelationId: parentEvent.correlationId,
|
|
593
|
-
}, "Routine fetch-window pre-pass completed");
|
|
594
|
-
return { report, block };
|
|
595
1537
|
}
|
|
596
1538
|
/**
|
|
597
1539
|
* Helper for the failure paths — renders a `<fetch_report status="failed">`
|
|
@@ -646,16 +1588,4 @@ export class RoutineFetchWindowRunner {
|
|
|
646
1588
|
return rows;
|
|
647
1589
|
}
|
|
648
1590
|
}
|
|
649
|
-
// ── Internal helpers ──────────────────────────────────────────────────────
|
|
650
|
-
/**
|
|
651
|
-
* Return true when the rendered plan carries at least one `<fetch ...>`
|
|
652
|
-
* element. The plan is well-formed even when empty (the dispatcher
|
|
653
|
-
* always emits the wrapper) — we still want to short-circuit so the
|
|
654
|
-
* pre-pass cold-start never fires for a routine with nothing to do.
|
|
655
|
-
*/
|
|
656
|
-
function fetcherPlanHasFetches(plan) {
|
|
657
|
-
if (typeof plan !== "string")
|
|
658
|
-
return false;
|
|
659
|
-
return /<fetch\s/i.test(plan);
|
|
660
|
-
}
|
|
661
1591
|
//# sourceMappingURL=routine-fetch-window-runner.js.map
|