@aitne/daemon 0.1.6 → 0.1.8

package/dist/core/routine-fetch-window-runner.js

@@ -1,7 +1,7 @@
 /**
  * `RoutineFetchWindowRunner` — pre-pass fan-out coordinator.
  *
- * ROUTINE_DATA_ACQUISITION_DESIGN.md §6.1.1 + PRE_PASS_FAN_OUT_DESIGN.md
+ * docs/design/appendices/routine-data-acquisition.md §6.1.1 + docs/design/appendices/pre-pass-fan-out.md
  * — every routine dispatcher (morning_routine, today_refresh,
  * hourly_check, evening / weekly / monthly review) calls this runner
  * immediately before dispatching the parent session. The runner:
@@ -29,7 +29,7 @@
  *     into the parent session's prompt (mirrors the `<gate_decision>`
  *     pattern used by hourly_check Stage 3).
  *
- * Failure-mode contract (PRE_PASS_FAN_OUT_DESIGN.md §5):
+ * Failure-mode contract (docs/design/appendices/pre-pass-fan-out.md §5):
  *
  *  - **No applicable rows** (routine has no windows in `ROUTINE_WINDOWS`,
  *    every integration is disabled, every account list is empty) — the
@@ -52,9 +52,12 @@
 import { EventPriority, INTEGRATION_KEYS, createEvent, getAgentDayDateStr, getIntegrationDescriptor, isRoutineEvent, } from "@aitne/shared";
 import { readIntegrations } from "../db/integrations-store.js";
 import { renderPartialForFanOut } from "./prompts.js";
-import { ROUTINE_WINDOWS, routineHasWindows, } from "./routine-windows.js";
+import { OBSERVATIONS_MCP_TOOL_NAME } from "../services/mcp/sdk-observations-server.js";
+import { ROUTINE_WINDOWS, routineHasWindows, WINDOW_QUERIES, } from "./routine-windows.js";
 import { buildAcquisitionTimestamps, splitAcquisitionPlanByIntegration, } from "./routine-acquisition-plan.js";
 import { RETRY_REASONS, buildPriorAttemptHintBlock, defaultRetryDecision, } from "./routine-fetch-window-retry.js";
+import { writeRuntimeState } from "../db/runtime-state.js";
+import { prePassLastRunRuntimeStateKey } from "./pre-pass-freshness.js";
 import { createLogger } from "../logging.js";
 const logger = createLogger("routine-fetch-window-runner");
 // ── Module helpers ────────────────────────────────────────────────────────
@@ -62,7 +65,7 @@ const logger = createLogger("routine-fetch-window-runner");
 const FETCH_WINDOW_PROCESS_KEY = "routine.fetch_window";
 const FETCH_WINDOW_EVENT_TYPE = "routine.fetch_window";
 /**
- * PRE_PASS_FAN_OUT_DESIGN.md §4.2 — the single placeholder the
+ * docs/design/appendices/pre-pass-fan-out.md §4.2 — the single placeholder the
  * `routine.fetch_window.md` task-flow carries in place of inline
  * integration partials. The runner substitutes this with the
  * integration-specific partial body loaded via
@@ -100,7 +103,7 @@ const FETCH_WINDOW_INTEGRATION_PARTIAL_PLACEHOLDER = "{integration_partial}";
  *
  * The clamp is Claude-only — Codex/Gemini have no per-spawn allowedTools
  * surface today (CLAUDE.md acknowledges the gap). The
- * `process_backend_config` envelope (`max_turns=20`, `max_budget_usd=0.20`)
+ * `process_backend_config` envelope (`max_turns=20`, `max_budget_usd=0.50`)
  * remains the floor on those backends.
  */
 function buildPrePassDaemonRestPatterns(apiPort) {
@@ -136,6 +139,57 @@ function buildPrePassDaemonRestPatterns(apiPort) {
  * the partial reaches them through `/api/integrations/<key>/exec`, not
  * via the session backend's MCP namespace.
  */
+/**
+ * Observability helper: list integrations whose `<fetch>` row will spawn
+ * on a backend OTHER than the pre-pass default `sessionBackend`. After
+ * the per-integration backend routing fix (see
+ * `routine-acquisition-plan.ts#resolveIntegrationBackend`), native
+ * bindings whose `nativeBackend !== sessionBackend` are NO LONGER
+ * silently dropped — they spawn on the integration's actual
+ * `nativeBackend` via `BackendRouter.resolveBinding({requestedBackendId})`.
+ * The previous "dropped" semantics is dead; this helper is preserved as
+ * a *cross-backend-spawn* signal so the daemon log surfaces "this
+ * routine fans out across multiple backends" up-front (useful when
+ * debugging Codex / Gemini auth issues that only manifest in pre-pass).
+ *
+ * Routine-scoped: only considers integrations that appear in
+ * `ROUTINE_WINDOWS[routine]` (via WINDOW_QUERIES per window symbol). A
+ * native-mismatched gmail row never surfaces under routine.weekly_review
+ * because weekly_review only touches calendar.
+ *
+ * Returns one entry per affected integration with `{ key, nativeBackend }`.
+ * Pure — pulls only from the integrations snapshot + the routine catalog,
+ * no side effects.
+ */
+function collectCrossBackendNativeIntegrations(routine, integrations, sessionBackend) {
+    // Collect the integrations this routine genuinely touches by walking
+    // its window specs against the WINDOW_QUERIES catalog. Using the
+    // same data sources as `collectFetchRows` in routine-acquisition-plan.ts
+    // keeps the WARN aligned with the actual filter logic.
+    const routineIntegrations = new Set();
+    for (const spec of ROUTINE_WINDOWS[routine]) {
+        const cell = WINDOW_QUERIES[spec.window];
+        if (!cell)
+            continue;
+        for (const key of Object.keys(cell)) {
+            routineIntegrations.add(key);
+        }
+    }
+    const out = [];
+    for (const key of INTEGRATION_KEYS) {
+        if (!routineIntegrations.has(key))
+            continue;
+        const state = integrations[key];
+        if (!state || state.mode !== "native")
+            continue;
+        if (!state.nativeBackend)
+            continue;
+        if (state.nativeBackend === sessionBackend)
+            continue;
+        out.push({ key, nativeBackend: state.nativeBackend });
+    }
+    return out;
+}
 function collectIntegrationToolsForBackend(integrations, backend) {
     const out = new Set();
     for (const key of INTEGRATION_KEYS) {
@@ -201,9 +255,23 @@ function collectIntegrationToolsForBackend(integrations, backend) {
 export function composePrePassAllowedTools(apiPort, integrations, sessionBackend) {
     const integrationTools = collectIntegrationToolsForBackend(integrations, sessionBackend);
     const needsDeferredDiscovery = sessionBackend === "claude" && integrationTools.length > 0;
+    // In-process MCP tool `mcp__aitne-observations__submit_observations` is
+    // the structural fix for the Unicode-whitespace-in-curl-body class of
+    // failures (2026-05-18 gmail incident: Amazon promo + NBSP/ZWS in the
+    // subject caused the SDK's `Ae6` bash preflight to mark the command
+    // `too-complex` and deny it under dontAsk, burning 3 attempts +
+    // surfacing budget-cap). Server is registered in-process by
+    // `ClaudeCodeCore.getObservationsMcpServer`; per-session exposure is
+    // controlled here. Claude-only because the SDK MCP transport is
+    // Claude-specific; codex/gemini pre-pass sessions retain the curl
+    // pattern via `buildPrePassDaemonRestPatterns` and remain at risk of
+    // the same class of denial (accepted gap until those backends gain a
+    // parallel structured channel).
+    const observationsMcpTools = sessionBackend === "claude" ? [OBSERVATIONS_MCP_TOOL_NAME] : [];
     return [
         ...buildPrePassDaemonRestPatterns(apiPort),
         ...integrationTools,
+        ...observationsMcpTools,
         ...(needsDeferredDiscovery ? ["ToolSearch"] : []),
     ];
 }
@@ -384,7 +452,7 @@ export function renderFetchReportBlock(report, meta) {
     lines.push("</fetch_report>");
     return lines.join("\n");
 }
-// ── Fan-out aggregation (PRE_PASS_FAN_OUT_DESIGN.md §4.5) ─────────────────
+// ── Fan-out aggregation (docs/design/appendices/pre-pass-fan-out.md §4.5) ─────────────────
 /**
  * Resolve the aggregate `<fetch_report>` status from a set of
  * sub-reports' final statuses, per §4.5:
@@ -742,12 +810,16 @@ export class RoutineFetchWindowRunner {
      * the parent event's `event.data.fetchReportBlock` so ContextBuilder
      * injects it into the parent prompt.
      *
-     * `routineKey` overrides the auto-derived window key — used by
-     * morning_routine to opt into `routine.morning_routine_initial`'s
-     * plan when `yesterday.md` is absent (their plans currently coincide,
-     * but the seam exists for future divergence).
+     * `routineKey` overrides the auto-derived window key. The variant
+     * collapse landed in
+     * `docs/design/appendices/morning-routine-optimization.md` Phase 4
+     * + Phase 7: morning_routine no longer toggles to a separate
+     * `routine.morning_routine_initial` (retired 2026-05-16) — both
+     * branches share the same window plan. The override seam is kept
+     * for callers that already know the canonical RoutineWindowKey and
+     * want to skip the event-shape derivation.
      */
-    async run(parentEvent, routineKey) {
+    async run(parentEvent, routineKey, options) {
         // B2 observability — announce pre-pass entry so the dashboard can
         // render "Fetching your mail and Notion data…" as a sub-step of the
         // parent routine's `routine_started` envelope. `prepass_completed`
@@ -758,7 +830,7 @@ export class RoutineFetchWindowRunner {
         this.broadcastPrepassProgress("prepass_started", parentEvent);
         let outcome;
         try {
-            outcome = await this.runImpl(parentEvent, routineKey);
+            outcome = await this.runImpl(parentEvent, routineKey, options);
             return outcome;
         }
         finally {
@@ -787,7 +859,7 @@ export class RoutineFetchWindowRunner {
             });
         }
     }
-    async runImpl(parentEvent, routineKey) {
+    async runImpl(parentEvent, routineKey, options) {
         const key = routineKey ?? routineWindowKeyFromEvent(parentEvent);
         const agentDay = getAgentDayDateStr(this.config.timezone || undefined, this.config.dayBoundaryHour);
         if (!key) {
@@ -829,6 +901,18 @@ export class RoutineFetchWindowRunner {
         catch (err) {
             return this.fail(key, agentDay, parentEvent, "plan-assembly-failed", err);
         }
+        // HOURLY_CHECK_GATE_REDESIGN_PLAN.md §3.4 — when the caller is the
+        // hourly_check coordinator, the freshness gate restricts pre-pass to
+        // a subset of integrations. `integrationKeyFilter` is honoured here
+        // (before fan-out) so the runner only spawns sub-sessions for stale
+        // integrations.
+        if (options?.integrationKeyFilter) {
+            const filter = options.integrationKeyFilter;
+            planContext = {
+                ...planContext,
+                subPlans: planContext.subPlans.filter((p) => filter.has(p.integrationKey)),
+            };
+        }
         // The acquisition plan can resolve to zero `<fetch>` rows when every
         // integration the routine touches is disabled / cross-backend-bound
         // on a connector-less integration / etc. `splitAcquisitionPlanByIntegration`
@@ -902,28 +986,59 @@ export class RoutineFetchWindowRunner {
             accounts,
             timestamps,
         };
+        const subPlans = splitAcquisitionPlanByIntegration(planInput);
+        // Observability: surface integrations whose `<fetch>` row will spawn
+        // on a backend OTHER than the pre-pass default. Per-integration
+        // backend routing (resolveIntegrationBackend +
+        // requestedBackendId) means these rows are NOT dropped — they spawn
+        // on the integration's bound backend via the BackendRouter's
+        // backend-only override path. Logged at `info` so the daemon log
+        // surfaces the cross-backend fan-out shape on every routine, which
+        // is the canonical signal when debugging Codex / Gemini auth issues
+        // that only manifest in the pre-pass (the main routine ran on
+        // claude but gmail-native sub-session is spawning on codex).
+        const crossBackendNativeIntegrations = collectCrossBackendNativeIntegrations(key, integrationsSnapshot, preBinding.main.backendId);
+        if (crossBackendNativeIntegrations.length > 0) {
+            logger.info({
+                routine: key,
+                defaultSessionBackend: preBinding.main.backendId,
+                crossBackendNativeIntegrations,
+                parentCorrelationId: parentEvent.correlationId,
+            }, "Routine fetch-window pre-pass: native bindings will spawn on "
+                + "their bound backend via per-integration backend routing "
+                + "(BackendRouter.requestedBackendId override); the default "
+                + "fetch_window backend will run direct / delegated / "
+                + "same-backend-native rows in parallel.");
+        }
         return {
             key,
             agentDay,
             placeholder,
             integrationsSnapshot,
-            subPlans: splitAcquisitionPlanByIntegration(planInput),
+            subPlans,
             accounts,
             timestamps,
         };
     }
     /**
-     * §5 BackendQuotaError row + §4.4 retryEscalationTier — re-derive the
-     * per-integration sub-plan against the CURRENT attempt's binding so a
-     * cross-backend swap (escalation tier flips main backend, or
-     * `prePassRetryEscalationTier` swaps the per-attempt tier) keeps the
-     * plan's `<fetch mode="…">` attribute aligned with the partial body's
-     * remaining mode-branch (the partial is filtered for the new backend
-     * via `renderPartialForFanOut`). Returns null only when the
-     * integration becomes unreachable on the new backend (e.g. native
-     * mode bound to A but the attempt re-resolves to B); the runner
-     * passes the call through to record the original sub-plan and let
-     * the agent emit `no-surface` errors organically.
+     * Defensive re-derivation of the per-integration sub-plan against the
+     * CURRENT attempt's binding. With per-integration backend routing in
+     * place, the attempt's binding is pinned to `subPlan.requiredBackend`
+     * via `BackendRouter.resolveBinding({ requestedBackendId })`, so
+     * `binding.main.backendId` always equals the original plan's backend
+     * and the recomputed block is bit-identical to `input.subPlan.block`
+     * in production. The defensive branch is retained for two reasons:
+     *
+     *  1. Regression test coverage — the runner unit test mocks
+     *     `resolveBinding` to return a different backend mid-loop,
+     *     exercising the rebuild mechanics so a future change that
+     *     re-introduces cross-backend swaps (e.g. dropping the
+     *     requestedBackendId pin or extending escalation to also flip
+     *     backends) cannot silently regress plan-vs-partial alignment.
+     *  2. Graceful degradation — returns null when the integration is
+     *     unreachable on the attempted backend; the caller logs the
+     *     original sub-plan and lets the agent emit `no-surface` errors
+     *     organically rather than throwing.
      *
      * Pure: snapshot inputs (`integrationsSnapshot`, `accounts`,
      * `timestamps`) are frozen at run() entry; only `sessionBackend`
@@ -970,12 +1085,51 @@ export class RoutineFetchWindowRunner {
     }
     async runFanOut(input) {
         const policy = this.buildRetryPolicy();
-        const globalBudget = new FanOutBudgetGuard(nonNegativeNumber(this.config.prePassMaxBudgetUsdPerRoutine, 1.5));
+        const preResolvedBindings = new Map();
+        let estimatedTotalUsd = 0;
+        for (const subPlan of input.subPlans) {
+            try {
+                // Use the parentEvent as the resolver event so this pre-resolve
+                // call carries no per-attempt correlationId — the inner loop
+                // creates fresh fetcherEvents for the actual sub-session
+                // execution. Attempt 1 inside `runOneIntegrationWithRetry`
+                // REUSES this binding (skip-inner-resolve) so the net
+                // resolveBinding call count is unchanged: 1 placeholder + N
+                // pre-resolves + 0 attempt-1 inner + K escalation/retry inner.
+                // Without that reuse, every test that asserts a call count
+                // would shift by N.
+                const binding = this.agentRouter.resolveBinding(input.parentEvent, {
+                    processKey: FETCH_WINDOW_PROCESS_KEY,
+                    requestedBackendId: subPlan.requiredBackend,
+                });
+                preResolvedBindings.set(subPlan.integrationKey, binding);
+                estimatedTotalUsd += binding.main.maxBudgetUsd;
+            }
+            catch {
+                preResolvedBindings.set(subPlan.integrationKey, null);
+            }
+        }
+        const configuredGlobalCap = nonNegativeNumber(this.config.prePassMaxBudgetUsdPerRoutine, 1.5);
+        const effectiveGlobalCap = Math.max(configuredGlobalCap, estimatedTotalUsd);
+        if (effectiveGlobalCap > configuredGlobalCap) {
+            logger.warn({
+                routine: input.key,
+                configuredCap: configuredGlobalCap,
+                effectiveCap: effectiveGlobalCap,
+                estimatedTotalUsd,
+                parentCorrelationId: input.parentEvent.correlationId,
+            }, "Routine fetch-window per-routine budget cap raised to floor at "
+                + "sum of binding envelopes — configured cap would block the "
+                + "first attempts. Raise prePassMaxBudgetUsdPerRoutine to "
+                + "silence this warning.");
+        }
+        const globalBudget = new FanOutBudgetGuard(effectiveGlobalCap);
         const tasks = input.subPlans.map((subPlan) => async () => this.runOneIntegrationWithRetry({
             ...input,
             subPlan,
             policy,
             globalBudget,
+            preResolvedBinding: preResolvedBindings.get(subPlan.integrationKey) ?? null,
         }));
         const subReports = await runWithConcurrency(tasks, this.fanOutConcurrency());
         const merged = mergeSubReports(subReports, input.key, input.agentDay);
@@ -1018,8 +1172,21 @@ export class RoutineFetchWindowRunner {
         // Layer (1) is the pessimistic guard (envelope ≥ actual); layer (2)
         // catches the case where individual attempts cost more than expected.
         // Either fires depending on the cost shape — both are intentional.
-        const integrationBudget = new FanOutBudgetGuard(input.policy.perIntegrationBudgetUsd);
-        const retryOn = input.policy.retryOn ?? defaultRetryDecision;
+        //
+        // The cap is floored at the pre-resolved binding's per-attempt
+        // envelope so attempt 1 always fits. Same rationale as the
+        // globalBudget floor in `runFanOut`: a static cap below a single
+        // attempt's envelope (e.g. the 0.6 default vs Codex lite's $1.25)
+        // would dead-lock every integration on attempt 1; the cap is meant
+        // to bound retry runaway, not block the first attempt. The cloned
+        // policy threads the same floor through `defaultRetryDecision`'s
+        // soft cumulative-cost check so the two layers stay aligned.
+        const floorBudget = input.preResolvedBinding?.main.maxBudgetUsd ?? 0;
+        const effectivePolicy = floorBudget > input.policy.perIntegrationBudgetUsd
+            ? { ...input.policy, perIntegrationBudgetUsd: floorBudget }
+            : input.policy;
+        const integrationBudget = new FanOutBudgetGuard(effectivePolicy.perIntegrationBudgetUsd);
+        const retryOn = effectivePolicy.retryOn ?? defaultRetryDecision;
         const escalationTier = this.retryEscalationTier();
         let retriesExhausted = false;
         for (let attempt = 1; attempt <= input.policy.maxAttempts; attempt++) {
@@ -1034,15 +1201,47 @@ export class RoutineFetchWindowRunner {
             this.emitSubSessionStarted(input, fetcherEvent.correlationId, attempt);
             let binding;
             try {
-                binding = this.agentRouter.resolveBinding(fetcherEvent, {
-                    processKey: FETCH_WINDOW_PROCESS_KEY,
-                    ...(requestedTier ? { requestedTier } : {}),
-                });
+                // Per-integration backend routing. `subPlan.requiredBackend`
+                // equals the integration's `nativeBackend` (native), the
+                // `delegatedBackend` for userManagedConnector descriptors, or
+                // the configured fetch_window default backend otherwise.
+                // Passing `requestedBackendId` triggers BackendRouter's
+                // backend-only override branch (added alongside this routing
+                // path), which preserves the `routine.fetch_window` process
+                // tier + envelope while spawning the sub-session on the
+                // integration's actual backend. This is the structural fix for
+                // the pre-existing silent-drop when a `native` binding's
+                // `nativeBackend` differed from the configured fetch_window
+                // backend.
+                //
+                // Fallback is dropped on backend-only override (router
+                // contract); the `routine.fetch_window` seed in db/schema.ts
+                // already ships without a fallback, so the override has no
+                // additional cost in the default deployment.
+                //
+                // Attempt 1 reuses the binding that `runFanOut` pre-resolved to
+                // size the budget guards — same args (no `requestedTier`,
+                // `requestedBackendId=subPlan.requiredBackend`), so resolving
+                // again would just spend an extra router call without changing
+                // the binding. Attempts > 1 resolve fresh so escalation-tier
+                // hints can flip the model. Pre-resolve failures (`null`) fall
+                // through to a fresh attempt-1 resolve, which surfaces the same
+                // `binding-resolve-failed` error path as before.
+                if (attempt === 1 && input.preResolvedBinding) {
+                    binding = input.preResolvedBinding;
+                }
+                else {
+                    binding = this.agentRouter.resolveBinding(fetcherEvent, {
+                        processKey: FETCH_WINDOW_PROCESS_KEY,
+                        ...(requestedTier ? { requestedTier } : {}),
+                        requestedBackendId: input.subPlan.requiredBackend,
+                    });
+                }
             }
             catch (err) {
                 const record = this.failedAttemptRecord(attempt, fetcherEvent.correlationId, startedAt, "binding-resolve-failed", err);
                 attempts.push(record);
-                const decision = retryOn(record, attempt, input.policy, attempts.slice(0, -1));
+                const decision = retryOn(record, attempt, effectivePolicy, attempts.slice(0, -1));
                 this.logFanOutFailure(input, fetcherEvent, record, decision, {
                     failureKind: "binding-resolve-failed",
                     err,
@@ -1128,7 +1327,7 @@ export class RoutineFetchWindowRunner {
                 integrationBudget.commit(integrationReservation, 0);
                 const record = this.failedAttemptRecord(attempt, fetcherEvent.correlationId, startedAt, "context-build-failed", err);
                 attempts.push(record);
-                const decision = retryOn(record, attempt, input.policy, attempts.slice(0, -1));
+                const decision = retryOn(record, attempt, effectivePolicy, attempts.slice(0, -1));
                 this.logFanOutFailure(input, fetcherEvent, record, decision, {
                     failureKind: "context-build-failed",
                     err,
@@ -1148,7 +1347,7 @@ export class RoutineFetchWindowRunner {
             let record;
             try {
                 const priorAttemptHintBlock = buildPriorAttemptHintBlock(attempts, input.subPlan.integrationKey);
-                // PRE_PASS_FAN_OUT_DESIGN.md §4.2 — every sub-session sees the
+                // docs/design/appendices/pre-pass-fan-out.md §4.2 — every sub-session sees the
                 // `routine.fetch_window` task-flow body with `{integration_partial}`
                 // substituted for the one partial its integrationKey owns. The
                 // integrations snapshot fed to both the partial's mode filter
@@ -1243,6 +1442,30 @@ export class RoutineFetchWindowRunner {
         }
         const final = attempts[attempts.length - 1];
         const allErrors = attempts.flatMap((att) => att.errors);
+        // HOURLY_CHECK_GATE_REDESIGN_PLAN.md §3.4 — record the freshness
+        // timestamp on every successful integration completion. The
+        // coordinator's `harvestForGate` reads this key to suppress
+        // redundant pre-pass spawns within the configured freshness window
+        // (default 30 min). Shared across morning_routine / hourly_check /
+        // evening_review / weekly_review / today_refresh by construction:
+        // the runner has no notion of parent routine.
+        //
+        // `success` only. `partial` and `failed` do NOT update the key so a
+        // partial fetch does not suppress the next tick's retry — the gate
+        // gets a fresh shot. `skipped` (no work to do) is also excluded;
+        // there is no fetched data backing the timestamp.
+        if (final.status === "success") {
+            try {
+                writeRuntimeState(this.db, prePassLastRunRuntimeStateKey(input.subPlan.integrationKey), new Date().toISOString());
+            }
+            catch (err) {
+                logger.warn({
+                    err,
+                    routine: input.key,
+                    integrationKey: input.subPlan.integrationKey,
+                }, "Failed to persist pre_pass_last_run runtime_state; freshness gate degraded");
+            }
+        }
         return {
             ...final,
             errors: allErrors.length > 0 ? allErrors : final.errors,