melchizedek-agents 1.0.3 → 1.2.0

package/dist/lib/runtime/native/agentLoop.d.ts

@@ -72,6 +72,17 @@
  *
  * DELEGATION: a call to a subagent tool runs the subagent as its own child
  * loop, as ADK's AgentTool runs it (lib/runtime/native/delegate.ts, ADR 0074).
+ * A step's delegated calls run at once through the step's DelegationGate,
+ * at most the agent's `maxConcurrency` (the syndicate's `max_concurrency`)
+ * at a time, entered in call order, their responses stored in call order
+ * (ADR 0116).
+ * A child that ends paused leaves the call open (ADR 0110): no response is
+ * stored for it, the step's other responses are, and the run ends paused on
+ * the call's id. Before each step, after the approval resume, a delegated
+ * call whose pause the latest user message answers is resumed with that
+ * answer (interrupts.ts resumedDelegations, delegate.ts resumeSubagent) and
+ * its response stored; while another open call still waits, the run ends
+ * paused again on it without a model step.
  *
  * SELF-CORRECTION (ADR 0034, ADR 0075) is ADK's reflect-and-retry plugins,
  * ported to lib/runtime/native/selfCorrection.ts: the step declares the
@@ -102,6 +113,8 @@
  * Before each step, a credential request the latest user message answers
  * runs its paused call again (interrupts.ts grantedCalls, ADK's auth
  * preprocessor), ahead of the approval resume.
+ * Inside a delegated subagent the same pause leaves the caller's call open,
+ * and the grant's answer comes down through it (ADR 0118).
  *
  * NOT HERE (later tickets): transfer_to_agent (no compiled syndicate sets
  * subAgents). The run's spans (agent.invoke, model.call,
@@ -158,10 +171,17 @@ export interface AgentLoopContext extends Omit<ModelStepOptions, 'agent' | 'adap
     /**
      * The consent step (ADR 0085): a call whose provider the user has not
      * granted asks for it (`requestCredential`, or `accessToken` by itself),
-     * and the run pauses on ADK's `adk_request_credential` call. Not passed to
-     * a delegated subagent's loop.
+     * and the run pauses on ADK's `adk_request_credential` call. A delegated
+     * subagent's loop gets it too, its flows bound to the caller's app
+     * (delegate.ts consentPinnedTo, ADR 0118): its pause leaves the call open.
      */
     consent?: Pick<OAuthConsent, 'has' | 'begin'>;
+    /**
+     * The run is a delegated subagent's, on its own child session filed under
+     * its agent path (lib/runtime/native/delegate.ts childAppName, ADR 0111):
+     * its own delegations are filed below that path. Set by runChild only.
+     */
+    delegated?: boolean;
 }
 /** How the run ended. */
 export type AgentLoopEndReason = 
@@ -181,7 +201,7 @@ export interface AgentLoopEnd {
     steps: number;
     /** The last event stored. */
     lastEvent?: TurnEvent;
-    /** For `paused`: the ids waiting on a person (the ask_user call ids, or the adk_request_confirmation call ids). */
+    /** For `paused`: the ids waiting on a person (the ask_user call ids, the adk_request_confirmation call ids, or the delegated calls whose child waits; ADR 0110). */
     pending?: string[];
     /** For `stopped`: the turn's code and message. */
     stop?: StepStop;

package/dist/lib/runtime/native/agentLoop.js

@@ -72,6 +72,17 @@
  *
  * DELEGATION: a call to a subagent tool runs the subagent as its own child
  * loop, as ADK's AgentTool runs it (lib/runtime/native/delegate.ts, ADR 0074).
+ * A step's delegated calls run at once through the step's DelegationGate,
+ * at most the agent's `maxConcurrency` (the syndicate's `max_concurrency`)
+ * at a time, entered in call order, their responses stored in call order
+ * (ADR 0116).
+ * A child that ends paused leaves the call open (ADR 0110): no response is
+ * stored for it, the step's other responses are, and the run ends paused on
+ * the call's id. Before each step, after the approval resume, a delegated
+ * call whose pause the latest user message answers is resumed with that
+ * answer (interrupts.ts resumedDelegations, delegate.ts resumeSubagent) and
+ * its response stored; while another open call still waits, the run ends
+ * paused again on it without a model step.
  *
  * SELF-CORRECTION (ADR 0034, ADR 0075) is ADK's reflect-and-retry plugins,
  * ported to lib/runtime/native/selfCorrection.ts: the step declares the
@@ -102,6 +113,8 @@
  * Before each step, a credential request the latest user message answers
  * runs its paused call again (interrupts.ts grantedCalls, ADK's auth
  * preprocessor), ahead of the approval resume.
+ * Inside a delegated subagent the same pause leaves the caller's call open,
+ * and the grant's answer comes down through it (ADR 0118).
  *
  * NOT HERE (later tickets): transfer_to_agent (no compiled syndicate sets
  * subAgents). The run's spans (agent.invoke, model.call,
@@ -125,8 +138,8 @@ import { MAX_VALUE_DEPTH, nestedDeeperThan } from '../valueDepth.js';
 import { currentTurnSignal } from '../turnControl.js';
 import { compactBeforeStep } from './compaction.js';
 import { ADK_CALL_ID_PREFIX } from './history.js';
-import { runSubagent, runWorkflowSubagent, subagentOf, workflowSubagentOf } from './delegate.js';
-import { approvedCalls, grantedCalls } from './interrupts.js';
+import { DEFAULT_MAX_CONCURRENCY, DelegationGate, SubagentPause, resumeSubagent, resumeWorkflowSubagent, runSubagent, runWorkflowSubagent, subagentOf, workflowSubagentOf } from './delegate.js';
+import { approvedCalls, grantedCalls, resumedDelegations } from './interrupts.js';
 import { SelfCorrection } from './selfCorrection.js';
 import { eitherSignal, runModelStep, stopOf } from './step.js';
 import { taskNodeRun } from './taskMode.js';
@@ -342,7 +355,7 @@ async function runOwnTool(tool, args, context) {
         throw new Error(`Error in tool '${tool.name}': ${message}`);
     }
 }
-async function runCall(scope, call, tools, correction, confirmation) {
+async function runCall(scope, call, tools, correction, confirmation, resume, ticket) {
     const context = callContext(scope, call.id || undefined, confirmation);
     const name = call.name ?? '';
     const tool = name && tools.has(name) ? tools.get(name) : undefined;
@@ -363,17 +376,26 @@ async function runCall(scope, call, tools, correction, confirmation) {
     // Delegation (WS2-6): a subagent runs as its own child loop, a nested workflow as its own walk (ADR 0098), before the generic path (delegate.ts).
     const subagent = subagentOf(tool);
     const workflow = subagent ? undefined : workflowSubagentOf(tool);
-    const delegation = { ctx: scope.ctx, stateBase: scope.stateBase, signal: scope.signal, runLoop: runAgentLoop, queue: scope };
+    const delegation = { ctx: scope.ctx, caller: scope.agent.name, stateBase: scope.stateBase, signal: scope.signal, runLoop: runAgentLoop };
     let response;
     let failure;
     try {
+        // ADR 0116: a delegated call waits for its turn in the step's gate, and frees it as soon as the child's run ends.
+        if (ticket && (subagent || workflow))
+            await ticket.ready;
         response = subagent
-            ? await runSubagent(subagent, args, context, delegation)
+            ? resume
+                ? await resumeSubagent(subagent, resume, context, delegation)
+                : await runSubagent(subagent, args, context, delegation)
             : workflow
-                ? await runWorkflowSubagent(workflow, args, context, delegation)
+                ? resume
+                    ? await resumeWorkflowSubagent(workflow, resume, context, delegation)
+                    : await runWorkflowSubagent(workflow, args, context, delegation)
                 : await runOwnTool(tool, args, context);
+        ticket?.release();
     }
     catch (e) {
+        ticket?.release();
         failure = e instanceof Error ? e.message : e;
         // Self-correction answers a thrown Error with reflection guidance in its place; a call waiting on a grant is not a failure.
         const guided = isEmptyRecord(context.actions.requestedAuthConfigs) ? await correction?.failed(toolName, args, e) : undefined;
@@ -386,6 +408,12 @@ async function runCall(scope, call, tools, correction, confirmation) {
         const provider = String(asked[0]?.credentialKey ?? '');
         return { part: { functionResponse: { id: context.functionCallId, name: toolName, response: { result: CONSENT_TEXTS.pending(provider) } } }, actions: context.actions };
     }
+    // A child that ended paused (ADR 0110): the call stays open, with no response, and the run ends paused on it.
+    if (response instanceof SubagentPause) {
+        if (context.functionCallId)
+            (scope.paused ??= []).push(context.functionCallId);
+        return isDefaultActions(context.actions) ? undefined : { actions: context.actions };
+    }
     if (failure === undefined)
         await correction?.answered(toolName, response);
     // As ADK: a long-running call with no response answers nothing, even when it threw.
@@ -401,10 +429,33 @@ async function runCall(scope, call, tools, correction, confirmation) {
  * The calls run, in parallel, as the one response event ADK stores for them
  * (not yet stored); undefined when none answered. `confirmations` holds the
  * person's answer for a pinned call an approval resumes, by call id.
+ *
+ * Delegated calls (a subagent, a nested workflow) run through the step's
+ * DelegationGate (ADR 0116): at most the agent's `maxConcurrency` at once,
+ * entered in call order, two calls to one subagent one after the other.
+ * Other calls start at once, as before. The event is built once every call
+ * has answered, its parts in call order, and the open calls a paused child
+ * left (scope.paused) are listed in call order too: neither depends on
+ * which call finished first.
  */
-async function runCalls(scope, calls, tools, confirmations) {
+async function runCalls(scope, calls, tools, confirmations, resumes) {
     const order = scope.selfCorrection?.forCalls(scope.ctx.invocationId, calls.length);
-    const outcomes = (await Promise.all(calls.map((call, i) => traceToolCall(call, tools.get(call.name ?? ''), () => runCall(scope, call, tools, order?.call(i), call.id ? confirmations?.get(call.id) : undefined)).finally(() => order?.release(i))))).filter((o) => !!o);
+    const gate = new DelegationGate(scope.agent.maxConcurrency ?? DEFAULT_MAX_CONCURRENCY);
+    const outcomes = (await Promise.all(calls.map((call, i) => {
+        const tool = tools.get(call.name ?? '');
+        // Entered here, synchronously and in call order, before any call runs (ADR 0116).
+        const delegated = subagentOf(tool) ?? workflowSubagentOf(tool);
+        const ticket = delegated ? gate.enter(delegated.name) : undefined;
+        return traceToolCall(call, tool, () => runCall(scope, call, tools, order?.call(i), call.id ? confirmations?.get(call.id) : undefined, call.id ? resumes?.get(call.id) : undefined, ticket)).finally(() => {
+            ticket?.release();
+            order?.release(i);
+        });
+    }))).filter((o) => !!o);
+    // The open calls, in call order rather than the order their children paused in.
+    if (scope.paused && scope.paused.length > 1) {
+        const at = (id) => calls.findIndex((c) => c.id === id);
+        scope.paused.sort((a, b) => at(a) - at(b));
+    }
     if (outcomes.length === 0)
         return undefined;
     const base = { invocationId: scope.ctx.invocationId, author: scope.agent.name };
@@ -503,6 +554,32 @@ async function resumeGrants(agent, ctx, stateBase, selfCorrection) {
     const auth = authEvent(scope, response);
     return { response, ...(auth ? { auth } : {}) };
 }
+/**
+ * Delegated calls the latest user message resumes (ADR 0110), before a step
+ * and after the approval resume: each open call whose child waits on what
+ * the message answers runs the child on with that answer, and their
+ * response is returned to be stored, with the open calls still paused:
+ * those the child paused again, and those the message did not answer.
+ * Undefined when the message answers none; 'stopped' when the turn stopped
+ * while they ran (nothing is stored).
+ */
+async function resumeDelegations(agent, ctx, stateBase, selfCorrection) {
+    const resumed = await resumedDelegations(agent, ctx, ctx.sessions);
+    if (!resumed)
+        return undefined;
+    const calls = resumed.calls.filter((c) => {
+        const tool = resumed.tools.get(c.name ?? '');
+        return subagentOf(tool) || workflowSubagentOf(tool);
+    });
+    if (calls.length === 0)
+        return undefined;
+    const signal = eitherSignal(ctx.signal, currentTurnSignal());
+    const scope = { agent, ctx, stateBase, selfCorrection, ...(signal ? { signal } : {}) };
+    const response = await runCalls(scope, calls, resumed.tools, undefined, resumed.answers);
+    if (signal?.aborted)
+        return 'stopped';
+    return { ...(response ? { response } : {}), paused: [...(scope.paused ?? []), ...resumed.waiting] };
+}
 // ── The model step, with partials and the fallback ───────────────────────────
 /** Runs a step, yielding its partial events as they arrive; returns the step's result. */
 async function* withPartials(start) {
@@ -640,6 +717,18 @@ async function* agentLoop(agent, ctx) {
             lastEvent = await store(resumed);
             yield lastEvent;
         }
+        // Delegation hook (WS6-2a, ADR 0110): an answer to a pause inside a delegated call resumes that call before the step.
+        const redelegated = await resumeDelegations(agent, ctx, withStateOverlay(session.state, runTemp.values()), selfCorrection);
+        if (redelegated === 'stopped')
+            return { reason: 'stopped', steps, lastEvent, stop: stopOf() };
+        if (redelegated) {
+            if (redelegated.response) {
+                lastEvent = await store(redelegated.response);
+                yield lastEvent;
+            }
+            if (redelegated.paused.length > 0)
+                return { reason: 'paused', steps, lastEvent, pending: redelegated.paused };
+        }
         // Compaction hook (WS2-9, lib/runtime/native/compaction.ts): with `context:`, the summary is stored before the step reads the history,
         // before the step budget, as ADK's request processors run before its call count.
         const compacted = await compactBeforeStep(agent, {
@@ -679,6 +768,8 @@ async function* agentLoop(agent, ctx) {
             // As ADK: a turn that stopped while the calls ran (a long subagent run, say) stores no response.
             if (scope.signal?.aborted)
                 return { reason: 'stopped', steps, lastEvent, stop: stopOf() };
+            // Delegated calls whose child ended paused (ADR 0110): they stay open, and the run ends paused on them once the step's responses are stored.
+            const delegated = scope.paused ?? [];
             if (response) {
                 const auth = authEvent(scope, response);
                 if (auth) {
@@ -690,17 +781,22 @@ async function* agentLoop(agent, ctx) {
                 if (confirmation) {
                     lastEvent = await store(confirmation);
                     yield lastEvent;
-                    return { reason: 'paused', steps, lastEvent, pending: [...(lastEvent.longRunningToolIds ?? [])] };
+                    return { reason: 'paused', steps, lastEvent, pending: [...(lastEvent.longRunningToolIds ?? []), ...delegated] };
                 }
                 lastEvent = await store(response);
                 yield lastEvent;
                 // As ADK: the response asking for auth is final, and the run waits on its credential request (ADR 0085).
                 if (authIds.length > 0)
-                    return { reason: 'paused', steps, lastEvent, pending: authIds };
+                    return { reason: 'paused', steps, lastEvent, pending: [...authIds, ...delegated] };
                 stepEnd = lastEvent;
-                if (task?.finished)
+                if (task?.finished && delegated.length === 0)
                     return { reason: 'final', steps, lastEvent, output: task.output };
             }
+            if (delegated.length > 0) {
+                const answered = new Set(getFunctionResponses(stepEnd).map((r) => r.id));
+                const longRunning = (modelEvent.longRunningToolIds ?? []).filter((id) => !answered.has(id));
+                return { reason: 'paused', steps, lastEvent, pending: [...longRunning, ...delegated] };
+            }
         }
         // ADK's runAsyncImpl: stop on a final event, unless it is an empty metadata event after tool calls.
         const emptyMetadata = stepEnd.author === agent.name && !stepEnd.partial && !hasParts(stepEnd) && isDefaultActions(stepEnd.actions);

package/dist/lib/runtime/native/checkpoint.d.ts

@@ -0,0 +1,171 @@
+/**
+ * lib/runtime/native/checkpoint.ts — durable long runs: the run's sessions
+ * snapshotted at every step boundary, and a run restored from the last
+ * snapshot taking the next step (ADR 0113, ADR 0015).
+ *
+ * WHY this file exists:
+ *   A background job (ADR 0015) runs as one turn on an in-process session
+ *   store. A worker that stopped mid-job lost everything the job had done,
+ *   and the next claim started again from the first model call. The turn
+ *   runner's signature is pinned by every surface and the boundary suite
+ *   (ADR 0024), so durability lives at the one seam the runner already
+ *   takes: the SessionService. checkpointingSessions wraps the in-process
+ *   store a run writes, hands a snapshot of every session to a sink at each
+ *   step boundary, and restores a snapshot into a fresh store so that the
+ *   runner, unchanged, continues where the snapshot stood. runDurableTurn
+ *   is runSyndicateTurn with that store and a sink.
+ *
+ * A STEP BOUNDARY is an append that carries function responses after which
+ * every function call any session of the run holds has its response (calls
+ * listed in their event's longRunningToolIds, which wait on a person, aside).
+ * So a parent waiting on a delegated subagent is never at a boundary, the
+ * subagent's own steps are not either, and a final answer never is: a
+ * resumed run redoes the step that was in flight, and the last one.
+ *
+ * WHAT IT GUARANTEES:
+ *   - A resumed run's stored events equal an uninterrupted run's, ids and
+ *     times of the steps run after the resume aside: restored events keep
+ *     their ids, times and invocation ids, and their state deltas are
+ *     applied again in order (InProcessSessionService.append, applyEvent).
+ *   - The opening message is stored once. A session whose first event is
+ *     the run's own message (the run's session, a workflow route's child)
+ *     is held back on restore: it does not exist until the runner creates
+ *     it, so `resumedSession` reads as on a fresh run, and the runner's
+ *     append of the message is answered with the stored events, the
+ *     opening one returned in its place, onto the caller's own session
+ *     object. Every other session (a delegated subagent's) is restored at
+ *     once, before the runner starts.
+ *   - A checkpoint is used only by the run that wrote it, for the message
+ *     it was written for: its runId and the sha256 of the message's parts
+ *     must match, and its events must parse (parseTurnEvents). Otherwise
+ *     the run starts fresh.
+ *   - The sink's save resolving false means the run no longer holds its
+ *     claim (the job was cancelled or leased elsewhere): runDurableTurn
+ *     aborts the run and reports lostClaim. A save that throws is reported
+ *     to onSaveError and the run goes on; a later attempt resumes from the
+ *     older checkpoint.
+ *
+ * LIMITS:
+ *   - A dispatch syndicate's route classifier runs in a throwaway store, so
+ *     its model call is not checkpointed and runs again on resume.
+ *   - A dispatch route that is an agent answers through the transcript
+ *     projection (ProjectedSessionService, lib/session/transcript.ts), which
+ *     copies the session when the route opens it. The held opening session
+ *     is empty at that open, so the copy is too; the route's append of the
+ *     message is answered with the stored events, and the projection takes
+ *     every event the real session gained, unprojected (the current turn),
+ *     so the route resumes like a single agent. A workflow route writes its
+ *     own child session and resumes. Checkpoints saved before this, which
+ *     1.1.0 set aside, restore the same way.
+ *   - A task backend stores a checkpoint only up to its cap
+ *     (TaskBackendOptions.checkpointMaxBytes, lib/tools/taskTools.ts,
+ *     default 5 MiB of JSON); a save above it keeps the previous checkpoint
+ *     and still answers whether the claim holds, so a huge run resumes from
+ *     an older boundary.
+ *   - Each resumed attempt has its own step budget (max_steps, the loop's
+ *     call ceiling); attempts are bounded by the queue's MAX_ATTEMPTS.
+ *   - `temp:` state lasts one invocation and is never stored, so it does not
+ *     survive a resume.
+ *   - A workflow walk resumes through its own rules (ADR 0094); its
+ *     checkpoint is the sessions it walks.
+ *   - A checkpoint holds the run's sessions, so it carries the user's text,
+ *     model output and tool results: it lives only where the run's record
+ *     lives, under the same owner, and is deleted with it. This module never
+ *     clears one; the caller does when the run finishes.
+ */
+import type { TurnEvent } from '../events.ts';
+import { InProcessSessionService } from '../sessions.ts';
+import type { SessionService } from '../sessions.ts';
+import type { MessagePart, SyndicateTurnOptions, SyndicateTurnResult } from '../syndicateTurn.ts';
+export declare const CHECKPOINT_VERSION = 1;
+/** One session of a run, as it stood at a step boundary. */
+export interface CheckpointSession {
+    appName: string;
+    userId: string;
+    id: string;
+    /** The state given to create(), before any event. */
+    createdState: Record<string, unknown>;
+    events: TurnEvent[];
+    /** The first event is the run's own opening user message: restore holds the session back until the runner stores that message. */
+    opening: boolean;
+}
+/** A run's sessions at its latest step boundary. */
+export interface RunCheckpoint {
+    version: 1;
+    runId: string;
+    /** The run's own session id: a resumed run uses it again. */
+    sessionId: string;
+    /** sha256 hex of JSON.stringify(parts) of the run's message (strings as `{ text }`). */
+    openingHash: string;
+    /** Step boundaries recorded so far, across every attempt. */
+    steps: number;
+    /** ISO time of the save. */
+    savedAt: string;
+    sessions: CheckpointSession[];
+}
+/** Where a run's checkpoints go. save resolves false when the run no longer holds its claim (cancelled, or re-leased): the run must stop. */
+export interface CheckpointSink {
+    load(): Promise<RunCheckpoint | null>;
+    save(checkpoint: RunCheckpoint): Promise<boolean>;
+}
+export interface CheckpointingOptions {
+    runId: string;
+    /** The run's own session id (the one runSyndicateTurn is given). */
+    sessionId: string;
+    /** The run's message. */
+    parts: MessagePart[];
+    /** The checkpoint to restore; ignored unless its runId and openingHash match and it parses. */
+    from?: RunCheckpoint | null;
+    save: (checkpoint: RunCheckpoint) => Promise<boolean>;
+    /** Called once, when a save resolves false. */
+    onLost?: () => void;
+    onSaveError?: (err: unknown) => void;
+    /** The store wrapped. Default a fresh InProcessSessionService. */
+    inner?: InProcessSessionService;
+}
+export type CheckpointingSessionService = SessionService & {
+    /** The checkpoint given was restored. */
+    readonly restored: boolean;
+    /** Step boundaries recorded, the restored checkpoint's included. */
+    readonly steps: number;
+};
+/** sha256 hex of the message's parts, as a checkpoint names its run's message. */
+export declare function openingHashOf(parts: readonly MessagePart[]): string;
+/**
+ * The checkpoint, checked, when it belongs to this run and this message;
+ * null otherwise. A stored checkpoint is data read back from a table or a
+ * file, so every field the restore reads is checked before it is trusted.
+ */
+export declare function usableCheckpoint(cp: unknown, runId: string, parts: readonly MessagePart[]): RunCheckpoint | null;
+/**
+ * A session store that checkpoints a run: the in-process store `inner`,
+ * restored from `from` when it is this run's checkpoint for this message,
+ * saving a snapshot of every session at each step boundary (see the header).
+ */
+export declare function checkpointingSessions(opts: CheckpointingOptions): CheckpointingSessionService;
+export type DurableTurnOptions = Omit<SyndicateTurnOptions, 'sessionService' | 'sessionId'> & {
+    /** Names the run: a checkpoint is restored only by the run that wrote it. */
+    runId: string;
+    checkpoints: CheckpointSink;
+    /** Called once when a save resolves false; the run is then aborted. */
+    onLost?: () => void;
+    onSaveError?: (err: unknown) => void;
+    /** The in-process store the run writes. Default a fresh one; a test passes its own to read the sessions afterwards. */
+    sessions?: InProcessSessionService;
+};
+export type DurableTurnResult = SyndicateTurnResult & {
+    /** The step boundaries the restored checkpoint had recorded; 0 on a fresh run. */
+    resumedFromStep: number;
+    /** A save resolved false: the run no longer held its claim and was aborted. */
+    lostClaim: boolean;
+    /** The run's own session id: the checkpoint's on a resume, a new UUID otherwise. */
+    sessionId: string;
+};
+/**
+ * runSyndicateTurn, durable: the run's sessions live in a fresh in-process
+ * store, restored from the sink's checkpoint when it is this run's for this
+ * message, and checkpointed at every step boundary. A save that resolves
+ * false aborts the run (`lostClaim`). The caller clears the checkpoint when
+ * the run finishes; this never does.
+ */
+export declare function runDurableTurn(opts: DurableTurnOptions): Promise<DurableTurnResult>;