@yaag/runtime 0.6.2 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/agent/agent.ts +38 -2
- package/src/agent/define-agent.ts +13 -3
- package/src/agent/spawn-request.ts +64 -0
- package/src/agent/spawn.ts +84 -63
- package/src/ask/ask-exchange-events.ts +10 -1
- package/src/ask/ask-exchange-options.ts +13 -0
- package/src/ask/ask-exchange.ts +77 -9
- package/src/ask/index.ts +1 -0
- package/src/cassette/cassette-replay.ts +23 -4
- package/src/cassette/recording-transport.ts +7 -0
- package/src/cassette/replay-divergence.ts +75 -19
- package/src/cassette/replay-transport.ts +8 -1
- package/src/cassette/resume-transport.ts +14 -1
- package/src/errors.ts +55 -2
- package/src/events.ts +19 -0
- package/src/index.ts +2 -0
- package/src/model/index.ts +18 -0
- package/src/model/model-error-history.ts +36 -0
- package/src/model/model-failure.ts +78 -0
- package/src/model/model-fallback.ts +15 -0
- package/src/model/model-match.ts +99 -0
- package/src/model/model-resolution.ts +44 -6
- package/src/model/model-swap.ts +81 -0
- package/src/model/recorded-resolution.ts +61 -0
- package/src/model/resolution-loop.ts +115 -0
- package/src/summary/index.ts +1 -0
- package/src/summary/summary-agent.ts +9 -1
- package/src/summary/summary-fallbacks.ts +50 -0
- package/src/summary/summary.ts +12 -0
- package/src/transport/fake-transport.ts +57 -1
- package/src/transport/index.ts +4 -0
- package/src/transport/live-transport.ts +37 -4
- package/src/transport/stderr-tail.ts +32 -0
- package/src/transport/transport.ts +11 -0
- package/src/types.ts +28 -5
- package/src/wire-constants.ts +3 -0
|
@@ -1,10 +1,24 @@
|
|
|
1
1
|
import { parseModelSuffix } from "./model-suffix.ts";
|
|
2
2
|
import { isThinkingLevel, type ThinkingLevel } from "./thinking-level.ts";
|
|
3
3
|
|
|
4
|
-
/**
|
|
4
|
+
/**
|
|
5
|
+
* Why one model candidate was rejected by pi (ADR-0037).
|
|
6
|
+
*
|
|
7
|
+
* `not_found`: the pattern matched no model, or pi refused the live swap.
|
|
8
|
+
* `auth`: the provider refused the credentials.
|
|
9
|
+
* `rate_limited`: the provider stayed rate-limited after pi's own retries.
|
|
10
|
+
*
|
|
11
|
+
* These three classes are the only triggers of Model Resolution. A different
|
|
12
|
+
* transport or turn failure stays an ordinary failure and starts no new attempt.
|
|
13
|
+
*/
|
|
5
14
|
export type ModelErrorReason = "not_found" | "auth" | "rate_limited";
|
|
6
15
|
|
|
7
|
-
/**
|
|
16
|
+
/**
|
|
17
|
+
* One failed Model Resolution attempt, handed back to the caller's resolver.
|
|
18
|
+
*
|
|
19
|
+
* yaag gives the full history to each resolver call, oldest attempt first. The
|
|
20
|
+
* first call gets an empty list (ADR-0037).
|
|
21
|
+
*/
|
|
8
22
|
export interface ModelError {
|
|
9
23
|
readonly reason: ModelErrorReason;
|
|
10
24
|
/** The model pattern that failed, after inline-suffix stripping. */
|
|
@@ -13,19 +27,43 @@ export interface ModelError {
|
|
|
13
27
|
readonly attempt: number;
|
|
14
28
|
}
|
|
15
29
|
|
|
16
|
-
/**
|
|
30
|
+
/**
|
|
31
|
+
* Picks the next model candidate, or `undefined` to give up (ADR-0037).
|
|
32
|
+
*
|
|
33
|
+
* When the resolver gives up, the spawn or the Ask fails with
|
|
34
|
+
* `MODEL_RESOLUTION_FAILED`, which carries the full error history. yaag also
|
|
35
|
+
* refuses a candidate that already failed with `not_found` or `auth` in the
|
|
36
|
+
* same loop, so a resolver cannot loop forever. The resolver must be pure and
|
|
37
|
+
* synchronous.
|
|
38
|
+
*/
|
|
17
39
|
export type ModelResolver = (errors: readonly ModelError[]) => string | undefined;
|
|
18
40
|
|
|
19
|
-
/**
|
|
41
|
+
/**
|
|
42
|
+
* Every accepted `model` form: one pattern, an ordered list, or a resolver.
|
|
43
|
+
*
|
|
44
|
+
* An array is the resolver that reads the candidate at the attempt index, and
|
|
45
|
+
* gives `undefined` past the end. One pattern is the one-element list. So the
|
|
46
|
+
* termination rule of `ModelResolver` covers every form (ADR-0037).
|
|
47
|
+
*/
|
|
20
48
|
export type ModelSpec = string | readonly string[] | ModelResolver;
|
|
21
49
|
|
|
22
|
-
/**
|
|
50
|
+
/**
|
|
51
|
+
* Picks the thinking level for a settled model, or `undefined` for pi's default.
|
|
52
|
+
*
|
|
53
|
+
* `undefined` means "no thinking preference". It is not a failure, and it does
|
|
54
|
+
* not stop the attempt (ADR-0037).
|
|
55
|
+
*/
|
|
23
56
|
export type ThinkingResolver = (
|
|
24
57
|
selectedModel: string,
|
|
25
58
|
errors: readonly ModelError[],
|
|
26
59
|
) => ThinkingLevel | undefined;
|
|
27
60
|
|
|
28
|
-
/**
|
|
61
|
+
/**
|
|
62
|
+
* Every accepted `thinking` form: one level or a resolver.
|
|
63
|
+
*
|
|
64
|
+
* An inline suffix on the resolved model pattern wins over this property
|
|
65
|
+
* (ADR-0037).
|
|
66
|
+
*/
|
|
29
67
|
export type ThinkingSpec = ThinkingLevel | ThinkingResolver;
|
|
30
68
|
|
|
31
69
|
/** One attempt's settled selection; `model: undefined` inherits pi's default model. */
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
import { agentError } from "../errors.ts";
|
|
2
|
+
import type { CommandResponse, Frame } from "../transport/index.ts";
|
|
3
|
+
import { matchAvailableModel, readAvailableModel, readAvailableModels } from "./model-match.ts";
|
|
4
|
+
import type { ModelSelection } from "./model-resolution.ts";
|
|
5
|
+
|
|
6
|
+
/** Sends one command frame to the live Agent. The Connection satisfies it. */
|
|
7
|
+
export type ModelSwapCommand = (frame: Frame) => Promise<CommandResponse>;
|
|
8
|
+
|
|
9
|
+
/** What one successful swap left the Agent running. */
|
|
10
|
+
export interface ModelSwapResult {
|
|
11
|
+
/** `provider/id` pi reports for the new model. */
|
|
12
|
+
readonly model: string;
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* Swaps one live Agent's model in place; the conversation context survives.
|
|
17
|
+
*
|
|
18
|
+
* pi's `set_model` takes a concrete `{provider, modelId}` pair and no pattern, so
|
|
19
|
+
* the swap is two frames: `get_available_models` to read the snapshot, then
|
|
20
|
+
* `set_model` on the entry yaag matched itself. A thinking level is a third
|
|
21
|
+
* frame, sent only when the selection names one — `undefined` means "no
|
|
22
|
+
* preference", exactly as at spawn.
|
|
23
|
+
*
|
|
24
|
+
* Rejects with an AGENT_FAILED-shaped error carrying pi's own diagnostic, so
|
|
25
|
+
* `classifyAskFailure` can re-enter Model Resolution: a refused pattern and a
|
|
26
|
+
* refused `set_model` both classify as `not_found`.
|
|
27
|
+
*/
|
|
28
|
+
export async function swapModel(options: {
|
|
29
|
+
readonly agent: string;
|
|
30
|
+
readonly command: ModelSwapCommand;
|
|
31
|
+
readonly selection: ModelSelection;
|
|
32
|
+
}): Promise<ModelSwapResult> {
|
|
33
|
+
const pattern = options.selection.model;
|
|
34
|
+
if (pattern === undefined) {
|
|
35
|
+
throw failed(options.agent, "model swap needs a model candidate");
|
|
36
|
+
}
|
|
37
|
+
const available = await options.command({ type: "get_available_models" });
|
|
38
|
+
if (!available.success) {
|
|
39
|
+
throw failed(options.agent, available.error ?? "get_available_models was rejected");
|
|
40
|
+
}
|
|
41
|
+
const models = readAvailableModels(available.data);
|
|
42
|
+
if (models === null || models.length === 0) {
|
|
43
|
+
throw failed(options.agent, "no models available");
|
|
44
|
+
}
|
|
45
|
+
const match = matchAvailableModel(pattern, models);
|
|
46
|
+
if (match.kind === "no_match") {
|
|
47
|
+
throw failed(options.agent, `No models match pattern "${pattern}"`);
|
|
48
|
+
}
|
|
49
|
+
if (match.kind === "ambiguous") {
|
|
50
|
+
throw failed(
|
|
51
|
+
options.agent,
|
|
52
|
+
`Model "${pattern}" is ambiguous across providers: ${match.candidates.join(", ")}`,
|
|
53
|
+
);
|
|
54
|
+
}
|
|
55
|
+
const response = await options.command({
|
|
56
|
+
type: "set_model",
|
|
57
|
+
provider: match.model.provider,
|
|
58
|
+
modelId: match.model.id,
|
|
59
|
+
});
|
|
60
|
+
if (!response.success) throw failed(options.agent, response.error ?? "set_model was rejected");
|
|
61
|
+
if (options.selection.thinking !== undefined) {
|
|
62
|
+
const thinking = await options.command({
|
|
63
|
+
type: "set_thinking_level",
|
|
64
|
+
level: options.selection.thinking,
|
|
65
|
+
});
|
|
66
|
+
if (!thinking.success) {
|
|
67
|
+
throw failed(options.agent, thinking.error ?? "set_thinking_level was rejected");
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
return { model: reportedModel(response.data) ?? `${match.model.provider}/${match.model.id}` };
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/** pi answers `set_model` with the full Model object it switched to. */
|
|
74
|
+
function reportedModel(data: Record<string, unknown> | null): string | undefined {
|
|
75
|
+
const model = readAvailableModel(data?.model ?? data);
|
|
76
|
+
return model === null ? undefined : `${model.provider}/${model.id}`;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
function failed(agent: string, message: string): Error {
|
|
80
|
+
return agentError(agent, "AGENT_FAILED", message);
|
|
81
|
+
}
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
import { ModelErrorHistory } from "./model-error-history.ts";
|
|
2
|
+
import type { ModelError, ModelResolution, ModelSelection } from "./model-resolution.ts";
|
|
3
|
+
import type { ThinkingLevel } from "./thinking-level.ts";
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* The model and thinking a Cassette recorded for one Agent's settled spawn.
|
|
7
|
+
*
|
|
8
|
+
* It lives in `model/` because `transport/` already depends on this directory,
|
|
9
|
+
* and never the other way round: one name for one concept (ADR-0039).
|
|
10
|
+
*/
|
|
11
|
+
export interface RecordedSpawnSelection {
|
|
12
|
+
readonly model?: string;
|
|
13
|
+
readonly thinking?: ThinkingLevel;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
/** What to replay: the declared resolution and the outcome the Cassette holds. */
|
|
17
|
+
export interface RecordedResolutionOptions {
|
|
18
|
+
readonly resolution: ModelResolution;
|
|
19
|
+
readonly recorded: RecordedSpawnSelection;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
/** The adopted selection and the attempts the declared resolution skipped to reach it. */
|
|
23
|
+
export interface RecordedResolution {
|
|
24
|
+
readonly selection: ModelSelection;
|
|
25
|
+
/**
|
|
26
|
+
* The synthetic failures of every candidate offered before the recorded one.
|
|
27
|
+
* The caller seeds the Agent's shared history with them, so a later mid-Ask
|
|
28
|
+
* fallback re-resolves from the attempt index the recording reached.
|
|
29
|
+
*/
|
|
30
|
+
readonly skipped: readonly ModelError[];
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* Replays Model Resolution against a recorded outcome, without any model call.
|
|
35
|
+
*
|
|
36
|
+
* Offers the declared resolution one candidate at a time, feeding every
|
|
37
|
+
* non-matching candidate back as a synthetic `not_found` failure, and settles on
|
|
38
|
+
* the first selection that names the recorded model. Returns `undefined` when
|
|
39
|
+
* the declared spec can no longer produce that outcome — the caller then runs
|
|
40
|
+
* the ordinary loop, and Cassette identity reports the Divergence (ADR-0039).
|
|
41
|
+
*/
|
|
42
|
+
export function resolveRecordedModel(
|
|
43
|
+
options: RecordedResolutionOptions,
|
|
44
|
+
): RecordedResolution | undefined {
|
|
45
|
+
const history = new ModelErrorHistory();
|
|
46
|
+
for (;;) {
|
|
47
|
+
const selection = options.resolution.resolve(history.errors);
|
|
48
|
+
if (selection === undefined) return undefined;
|
|
49
|
+
if (selection.model === options.recorded.model) {
|
|
50
|
+
return { selection, skipped: [...history.errors] };
|
|
51
|
+
}
|
|
52
|
+
const candidate = selection.model;
|
|
53
|
+
// No candidate means pi's default, which the resolution offers only once:
|
|
54
|
+
// it cannot be rejected and retried, so the recorded outcome is unreachable.
|
|
55
|
+
if (candidate === undefined) return undefined;
|
|
56
|
+
// A resolver that repeats a permanently failed candidate stops here, with
|
|
57
|
+
// the same termination rule the live loop uses (`resolution-loop.ts`).
|
|
58
|
+
if (history.failedPermanently(candidate)) return undefined;
|
|
59
|
+
history.record("not_found", candidate);
|
|
60
|
+
}
|
|
61
|
+
}
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
import { modelResolutionError } from "../errors.ts";
|
|
2
|
+
import { ModelErrorHistory } from "./model-error-history.ts";
|
|
3
|
+
import { classifyAskFailure, classifyModelFailure } from "./model-failure.ts";
|
|
4
|
+
import type { ModelFallbackSink } from "./model-fallback.ts";
|
|
5
|
+
import type { ModelError, ModelResolution, ModelSelection } from "./model-resolution.ts";
|
|
6
|
+
|
|
7
|
+
/** One Model Resolution loop: what to resolve, for whom, and how one attempt runs. */
|
|
8
|
+
export interface ResolutionLoopOptions<T> {
|
|
9
|
+
readonly resolution: ModelResolution;
|
|
10
|
+
readonly agent: string;
|
|
11
|
+
/** The Agent's shared attempt history; a fresh one when the caller keeps none. */
|
|
12
|
+
readonly history?: ModelErrorHistory;
|
|
13
|
+
/** Runs one attempt with the settled selection; rejects to feed classification. */
|
|
14
|
+
attempt(selection: ModelSelection): Promise<T>;
|
|
15
|
+
/** Reports each failed attempt whose resolution produced a next candidate. */
|
|
16
|
+
readonly onFallback?: ModelFallbackSink;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* Runs the Model Resolution loop: resolve → attempt → classify → re-resolve.
|
|
21
|
+
*
|
|
22
|
+
* Rejects with MODEL_RESOLUTION_FAILED (carrying the full error history) when the
|
|
23
|
+
* resolver gives up, or when it returns a candidate that already failed
|
|
24
|
+
* permanently (`not_found`/`auth`) in this loop. Any failure that is not a Model
|
|
25
|
+
* Resolution trigger is rethrown unchanged, and so is any failure of an attempt
|
|
26
|
+
* that named no candidate at all (it inherited pi's default, so there is nothing
|
|
27
|
+
* to fall back from).
|
|
28
|
+
*/
|
|
29
|
+
export async function resolveModel<T>(options: ResolutionLoopOptions<T>): Promise<T> {
|
|
30
|
+
const history = options.history ?? new ModelErrorHistory();
|
|
31
|
+
// The failed attempt still waiting for the candidate that replaces it. A
|
|
32
|
+
// resolver that gives up throws before it is ever reported, and a selection
|
|
33
|
+
// that names no candidate inherits pi's default, which is no fallback either.
|
|
34
|
+
let pending: ModelError | undefined;
|
|
35
|
+
for (;;) {
|
|
36
|
+
const selection = nextSelection(options.agent, options.resolution, history);
|
|
37
|
+
if (pending !== undefined && selection.model !== undefined) {
|
|
38
|
+
options.onFallback?.({ ...pending, resolvedModel: selection.model });
|
|
39
|
+
}
|
|
40
|
+
pending = undefined;
|
|
41
|
+
try {
|
|
42
|
+
return await options.attempt(selection);
|
|
43
|
+
} catch (error) {
|
|
44
|
+
const reason = classifyModelFailure(error);
|
|
45
|
+
if (reason === undefined || selection.model === undefined) throw error;
|
|
46
|
+
pending = history.record(reason, selection.model);
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/** One mid-Ask fallback loop: the work already runs on a model, so resolution comes second. */
|
|
52
|
+
export interface FallbackLoopOptions<T> {
|
|
53
|
+
readonly resolution: ModelResolution;
|
|
54
|
+
readonly agent: string;
|
|
55
|
+
readonly history: ModelErrorHistory;
|
|
56
|
+
/** The model the Agent runs right now; a failure is attributed to it. */
|
|
57
|
+
currentModel(): string;
|
|
58
|
+
/** Runs one attempt on the current model; rejects to feed classification. */
|
|
59
|
+
attempt(): Promise<T>;
|
|
60
|
+
/** Applies a re-resolved selection to the live Agent; rejects to feed classification. */
|
|
61
|
+
swap(selection: ModelSelection): Promise<void>;
|
|
62
|
+
/** Reports each failed attempt or swap whose resolution produced a next candidate. */
|
|
63
|
+
readonly onFallback?: ModelFallbackSink;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* Runs the mid-Ask Model Resolution loop: attempt → classify → re-resolve → swap → retry.
|
|
68
|
+
*
|
|
69
|
+
* Unlike `resolveModel`, the first attempt runs on the model already in place,
|
|
70
|
+
* and a failed swap is classified exactly like a failed attempt, so a pattern pi
|
|
71
|
+
* refuses re-enters the loop instead of ending the Ask.
|
|
72
|
+
*/
|
|
73
|
+
export async function retryOnModelFailure<T>(options: FallbackLoopOptions<T>): Promise<T> {
|
|
74
|
+
// `undefined` means "run an attempt"; a selection means "swap to it first".
|
|
75
|
+
let pending: ModelSelection | undefined;
|
|
76
|
+
for (;;) {
|
|
77
|
+
// What a failure of this step is attributed to: the candidate a swap tries
|
|
78
|
+
// to apply, or the model the Agent runs right now.
|
|
79
|
+
const failing = pending?.model ?? options.currentModel();
|
|
80
|
+
try {
|
|
81
|
+
if (pending === undefined) return await options.attempt();
|
|
82
|
+
await options.swap(pending);
|
|
83
|
+
pending = undefined;
|
|
84
|
+
} catch (error) {
|
|
85
|
+
const reason = classifyAskFailure(error);
|
|
86
|
+
if (reason === undefined) throw error;
|
|
87
|
+
const recorded = options.history.record(reason, failing);
|
|
88
|
+
pending = nextSelection(options.agent, options.resolution, options.history);
|
|
89
|
+
// A selection that names no candidate inherits pi's default, so there is
|
|
90
|
+
// nothing to swap to: the original failure stays what it was.
|
|
91
|
+
if (pending.model === undefined) throw error;
|
|
92
|
+
options.onFallback?.({
|
|
93
|
+
reason,
|
|
94
|
+
failedModel: failing,
|
|
95
|
+
attempt: recorded.attempt,
|
|
96
|
+
resolvedModel: pending.model,
|
|
97
|
+
});
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/** The next selection to try, or a rejection when resolution is over. */
|
|
103
|
+
function nextSelection(
|
|
104
|
+
agent: string,
|
|
105
|
+
resolution: ModelResolution,
|
|
106
|
+
history: ModelErrorHistory,
|
|
107
|
+
): ModelSelection {
|
|
108
|
+
const selection = resolution.resolve(history.errors);
|
|
109
|
+
if (selection === undefined) throw modelResolutionError(agent, history.errors);
|
|
110
|
+
const candidate = selection.model;
|
|
111
|
+
if (candidate !== undefined && history.failedPermanently(candidate)) {
|
|
112
|
+
throw modelResolutionError(agent, history.errors, candidate);
|
|
113
|
+
}
|
|
114
|
+
return selection;
|
|
115
|
+
}
|
package/src/summary/index.ts
CHANGED
|
@@ -1,8 +1,10 @@
|
|
|
1
1
|
import type { AgentActivity } from "../events.ts";
|
|
2
2
|
import type { TokenBreakdown, WorktreeResolution } from "../transport/index.ts";
|
|
3
|
+
import type { ModelFallbackInfo } from "./summary-fallbacks.ts";
|
|
3
4
|
import type { NodeInfo } from "./summary-nodes.ts";
|
|
4
5
|
|
|
5
6
|
export type { AgentActivity } from "../events.ts";
|
|
7
|
+
export type { ModelFallbackInfo } from "./summary-fallbacks.ts";
|
|
6
8
|
export type { NodeInfo } from "./summary-nodes.ts";
|
|
7
9
|
|
|
8
10
|
/** The observer-facing lifecycle state of an Agent. */
|
|
@@ -27,6 +29,10 @@ interface AgentInfoBase {
|
|
|
27
29
|
readonly nodes: readonly NodeInfo[];
|
|
28
30
|
/** Exited Nested Nodes dropped to keep the table bounded. */
|
|
29
31
|
readonly finishedNodesPruned: number;
|
|
32
|
+
/** This Agent's bounded Model Resolution fallback table, oldest first. */
|
|
33
|
+
readonly modelFallbacks: readonly ModelFallbackInfo[];
|
|
34
|
+
/** Fallbacks dropped to keep the table bounded. */
|
|
35
|
+
readonly modelFallbacksPruned: number;
|
|
30
36
|
}
|
|
31
37
|
|
|
32
38
|
/** An Agent between Asks; its latest settled Ask identity, if any, is retained. */
|
|
@@ -80,7 +86,7 @@ export interface AgentExitObservation {
|
|
|
80
86
|
}
|
|
81
87
|
|
|
82
88
|
/** Produces an idle placeholder for lifecycle events received before a spawn. */
|
|
83
|
-
function placeholderAgent(): IdleAgentInfo {
|
|
89
|
+
export function placeholderAgent(): IdleAgentInfo {
|
|
84
90
|
return {
|
|
85
91
|
model: null,
|
|
86
92
|
cwd: null,
|
|
@@ -98,6 +104,8 @@ function placeholderAgent(): IdleAgentInfo {
|
|
|
98
104
|
askStartedAt: null,
|
|
99
105
|
nodes: [],
|
|
100
106
|
finishedNodesPruned: 0,
|
|
107
|
+
modelFallbacks: [],
|
|
108
|
+
modelFallbacksPruned: 0,
|
|
101
109
|
};
|
|
102
110
|
}
|
|
103
111
|
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
import type { LifecycleEventBody, ModelErrorReason } from "../events.ts";
|
|
2
|
+
import { AGENT_MODEL_FALLBACK_MAX } from "../wire-constants.ts";
|
|
3
|
+
import type { AgentRecord } from "./summary-agent.ts";
|
|
4
|
+
|
|
5
|
+
/** One failed Model Resolution attempt as the Summary keeps it. */
|
|
6
|
+
export interface ModelFallbackInfo {
|
|
7
|
+
readonly failedModel: string;
|
|
8
|
+
readonly reason: ModelErrorReason;
|
|
9
|
+
readonly attempt: number;
|
|
10
|
+
readonly resolvedModel: string;
|
|
11
|
+
readonly at: number | null;
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
type ModelFallbackEvent = Extract<LifecycleEventBody, { readonly type: "model_fallback" }>;
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* Folds one `model_fallback` into an Agent's bounded fallback table.
|
|
18
|
+
*
|
|
19
|
+
* A fallback is an append-only fact, not a state replacement, so entries keep
|
|
20
|
+
* arrival order (the loop already orders attempts) and no stamped entry is
|
|
21
|
+
* rejected. Past `AGENT_MODEL_FALLBACK_MAX` entries the oldest drop into
|
|
22
|
+
* `modelFallbacksPruned`.
|
|
23
|
+
*
|
|
24
|
+
* The Agent's `model` is left alone. `resolvedModel` is a yaag pattern, which
|
|
25
|
+
* can be partial, and the event reports it before the swap is applied, while
|
|
26
|
+
* `model` is pi's resolved `provider/id` for the model the Agent really runs.
|
|
27
|
+
* `agent_spawn` is the only writer of that field: a mid-Ask swap updates the
|
|
28
|
+
* Handle alone and reports no Lifecycle Event, so the Summary keeps the
|
|
29
|
+
* spawn-time id.
|
|
30
|
+
*/
|
|
31
|
+
export function applyModelFallback(
|
|
32
|
+
agent: AgentRecord,
|
|
33
|
+
event: ModelFallbackEvent,
|
|
34
|
+
at: number | null,
|
|
35
|
+
): AgentRecord {
|
|
36
|
+
const entry: ModelFallbackInfo = {
|
|
37
|
+
failedModel: event.failedModel,
|
|
38
|
+
reason: event.reason,
|
|
39
|
+
attempt: event.attempt,
|
|
40
|
+
resolvedModel: event.resolvedModel,
|
|
41
|
+
at,
|
|
42
|
+
};
|
|
43
|
+
const appended = [...agent.modelFallbacks, entry];
|
|
44
|
+
const excess = Math.max(0, appended.length - AGENT_MODEL_FALLBACK_MAX);
|
|
45
|
+
return {
|
|
46
|
+
...agent,
|
|
47
|
+
modelFallbacks: appended.slice(excess),
|
|
48
|
+
modelFallbacksPruned: agent.modelFallbacksPruned + excess,
|
|
49
|
+
};
|
|
50
|
+
}
|
package/src/summary/summary.ts
CHANGED
|
@@ -5,12 +5,14 @@ import {
|
|
|
5
5
|
type AgentRecord,
|
|
6
6
|
endAsk,
|
|
7
7
|
exitAgent,
|
|
8
|
+
placeholderAgent,
|
|
8
9
|
setActivity,
|
|
9
10
|
setUsage,
|
|
10
11
|
spawnAgent,
|
|
11
12
|
startAsk,
|
|
12
13
|
totalsFromAgents,
|
|
13
14
|
} from "./summary-agent.ts";
|
|
15
|
+
import { applyModelFallback } from "./summary-fallbacks.ts";
|
|
14
16
|
import { applyNodeUpdate } from "./summary-nodes.ts";
|
|
15
17
|
|
|
16
18
|
export type { NodeState, NodeUsage, RunOutcome } from "../events.ts";
|
|
@@ -21,6 +23,7 @@ export type {
|
|
|
21
23
|
AskingAgentInfo,
|
|
22
24
|
ExitedAgentInfo,
|
|
23
25
|
IdleAgentInfo,
|
|
26
|
+
ModelFallbackInfo,
|
|
24
27
|
NodeInfo,
|
|
25
28
|
} from "./summary-agent.ts";
|
|
26
29
|
|
|
@@ -44,6 +47,8 @@ interface RunSummaryBase {
|
|
|
44
47
|
readonly incomplete: boolean;
|
|
45
48
|
readonly durationMs: number;
|
|
46
49
|
readonly worstFrameGapMs: number;
|
|
50
|
+
/** Run-wide Model Resolution fallbacks, pruned per-Agent entries included. */
|
|
51
|
+
readonly modelFallbacks: number;
|
|
47
52
|
}
|
|
48
53
|
|
|
49
54
|
/** A Run that has not settled; it has neither outcome nor compatibility result. */
|
|
@@ -101,6 +106,7 @@ export function initialSummary(): RunningRunSummary {
|
|
|
101
106
|
incomplete: false,
|
|
102
107
|
durationMs: 0,
|
|
103
108
|
worstFrameGapMs: 0,
|
|
109
|
+
modelFallbacks: 0,
|
|
104
110
|
ok: null,
|
|
105
111
|
};
|
|
106
112
|
}
|
|
@@ -140,6 +146,12 @@ export function applyEvent(
|
|
|
140
146
|
return updateActivity(summary, event, at);
|
|
141
147
|
case "node_update":
|
|
142
148
|
return updateNodes(summary, event, at);
|
|
149
|
+
case "model_fallback":
|
|
150
|
+
return withAgent(
|
|
151
|
+
{ ...summary, modelFallbacks: summary.modelFallbacks + 1 },
|
|
152
|
+
event.agent,
|
|
153
|
+
applyModelFallback(summary.agents[event.agent] ?? placeholderAgent(), event, at),
|
|
154
|
+
);
|
|
143
155
|
case "agent_usage":
|
|
144
156
|
return withAgent(summary, event.agent, setUsage(summary.agents[event.agent], event, at));
|
|
145
157
|
case "ask_end":
|
|
@@ -3,6 +3,7 @@ import {
|
|
|
3
3
|
isReportResultCommandFrame,
|
|
4
4
|
REPORT_RESULT_TOOL_NAME,
|
|
5
5
|
} from "../ask-contract/index.ts";
|
|
6
|
+
import type { AvailableModel } from "../model/index.ts";
|
|
6
7
|
import { FrameQueue } from "./frame-queue.ts";
|
|
7
8
|
import { parseFrame } from "./jsonl.ts";
|
|
8
9
|
import type { AgentStats, AgentTransport, AskMarker, Frame } from "./transport.ts";
|
|
@@ -51,6 +52,12 @@ export interface FakeTransportOptions extends FakePromptScript {
|
|
|
51
52
|
readonly abortError?: string;
|
|
52
53
|
/** Makes the private report_result schema command fail. */
|
|
53
54
|
readonly schemaCommandError?: string;
|
|
55
|
+
/** The snapshot answered to `get_available_models`; defaults to this fake's own model. */
|
|
56
|
+
readonly models?: readonly AvailableModel[];
|
|
57
|
+
/** Makes a `set_model` command fail, as pi does for a pair it does not know. */
|
|
58
|
+
readonly setModelError?: string;
|
|
59
|
+
/** Makes a `set_thinking_level` command fail. */
|
|
60
|
+
readonly setThinkingError?: string;
|
|
54
61
|
readonly stats?: AgentStats;
|
|
55
62
|
/** pi can answer a command after `agent_settled` — it is last among events only. */
|
|
56
63
|
readonly promptResponse?: "immediate" | "after-settle";
|
|
@@ -68,7 +75,13 @@ export interface FakeTransportOptions extends FakePromptScript {
|
|
|
68
75
|
* It never spawns or closes a real Agent process.
|
|
69
76
|
*/
|
|
70
77
|
export class FakeTransport implements AgentTransport {
|
|
71
|
-
|
|
78
|
+
#model = "test/model";
|
|
79
|
+
|
|
80
|
+
/** The model this fake currently reports, which a `set_model` swap updates. */
|
|
81
|
+
get model(): string {
|
|
82
|
+
return this.#model;
|
|
83
|
+
}
|
|
84
|
+
|
|
72
85
|
/** Everything the runtime wrote, in order. */
|
|
73
86
|
readonly sent: Frame[] = [];
|
|
74
87
|
readonly asks: AskMarker[] = [];
|
|
@@ -97,6 +110,10 @@ export class FakeTransport implements AgentTransport {
|
|
|
97
110
|
if (frame.type === "abort") void this.#answerAbort(frame);
|
|
98
111
|
if (frame.type === "get_last_assistant_text") this.#answerLastText(frame);
|
|
99
112
|
if (frame.type === "get_state") this.#answerState(frame);
|
|
113
|
+
if (frame.type === "get_available_models") this.#answerAvailableModels(frame);
|
|
114
|
+
if (frame.type === "set_model") this.#answerSetModel(frame);
|
|
115
|
+
if (frame.type === "set_thinking_level")
|
|
116
|
+
this.#answerControl(frame, this.#options.setThinkingError);
|
|
100
117
|
}
|
|
101
118
|
|
|
102
119
|
frames(): AsyncIterable<Frame> {
|
|
@@ -236,6 +253,45 @@ export class FakeTransport implements AgentTransport {
|
|
|
236
253
|
}
|
|
237
254
|
}
|
|
238
255
|
|
|
256
|
+
#answerAvailableModels(frame: Frame): void {
|
|
257
|
+
this.#queue.push({
|
|
258
|
+
type: "response",
|
|
259
|
+
command: "get_available_models",
|
|
260
|
+
id: frame.id,
|
|
261
|
+
success: true,
|
|
262
|
+
data: { models: this.#models() },
|
|
263
|
+
});
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
#models(): readonly AvailableModel[] {
|
|
267
|
+
return this.#options.models ?? [{ provider: "test", id: "model" }];
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
/**
|
|
271
|
+
* Answers pi's `set_model` and, on success, reports the new model.
|
|
272
|
+
*
|
|
273
|
+
* A swap begins a new attempt of the same Ask, so the fake re-arms the initial
|
|
274
|
+
* prompt: the retried prompt selects the next top-level script instead of a
|
|
275
|
+
* correction script. That is a yaag rule encoded in a test double on purpose
|
|
276
|
+
* — one `beginAsk` per Ask keeps the double honest.
|
|
277
|
+
*/
|
|
278
|
+
#answerSetModel(frame: Frame): void {
|
|
279
|
+
const error = this.#options.setModelError;
|
|
280
|
+
if (error === undefined) {
|
|
281
|
+
this.#model = `${String(frame.provider)}/${String(frame.modelId)}`;
|
|
282
|
+
this.#initialPromptPending = true;
|
|
283
|
+
}
|
|
284
|
+
this.#queue.push({
|
|
285
|
+
type: "response",
|
|
286
|
+
command: "set_model",
|
|
287
|
+
id: frame.id,
|
|
288
|
+
success: error === undefined,
|
|
289
|
+
...(error === undefined
|
|
290
|
+
? { data: { model: { provider: frame.provider, id: frame.modelId } } }
|
|
291
|
+
: { error }),
|
|
292
|
+
});
|
|
293
|
+
}
|
|
294
|
+
|
|
239
295
|
#answerState(frame: Frame): void {
|
|
240
296
|
if (this.#options.stateSilent === true) return;
|
|
241
297
|
this.#queue.push({
|
package/src/transport/index.ts
CHANGED
|
@@ -2,6 +2,10 @@
|
|
|
2
2
|
* Public surface of the `transport/` module: the pi process seam and its probes.
|
|
3
3
|
* Files inside this directory import each other directly.
|
|
4
4
|
*/
|
|
5
|
+
|
|
6
|
+
// One name for one concept: the recorded selection type lives in `model/`,
|
|
7
|
+
// which `transport/` already depends on (ADR-0039).
|
|
8
|
+
export type { RecordedSpawnSelection } from "../model/index.ts";
|
|
5
9
|
export { type CommandResponse, Connection } from "./connection.ts";
|
|
6
10
|
export {
|
|
7
11
|
FakeTransport,
|
|
@@ -3,6 +3,7 @@ import { FrameQueue } from "./frame-queue.ts";
|
|
|
3
3
|
import { decodeFrames } from "./jsonl.ts";
|
|
4
4
|
import { piCommand, readModel, readSessionFile, readStats } from "./pi-state.ts";
|
|
5
5
|
import { reap } from "./reap.ts";
|
|
6
|
+
import { appendStderrTail, stderrDiagnostic } from "./stderr-tail.ts";
|
|
6
7
|
import {
|
|
7
8
|
createToolProbe,
|
|
8
9
|
TOOL_PROBE_EXTENSION_PATH,
|
|
@@ -25,6 +26,8 @@ import type {
|
|
|
25
26
|
const READY_TIMEOUT_MS = 30_000;
|
|
26
27
|
/** Cost is a local read answering in ~0ms, but the kill must never block on it. */
|
|
27
28
|
const STATS_TIMEOUT_MS = 2_000;
|
|
29
|
+
/** How long an exit message waits for stderr, which grandchildren may hold open. */
|
|
30
|
+
const STDERR_DRAIN_MS = 100;
|
|
28
31
|
|
|
29
32
|
/** Agents are real `pi --mode rpc` child processes (ADR-0001). */
|
|
30
33
|
export const liveTransport: TransportFactory = {
|
|
@@ -42,7 +45,11 @@ class LiveTransport implements AgentTransport {
|
|
|
42
45
|
model = "";
|
|
43
46
|
|
|
44
47
|
readonly #name: string;
|
|
45
|
-
readonly #process: Bun.Subprocess<"pipe", "pipe", "
|
|
48
|
+
readonly #process: Bun.Subprocess<"pipe", "pipe", "pipe">;
|
|
49
|
+
/** Last bytes pi wrote to stderr; the only classification signal a spawn gets (ADR-0037). */
|
|
50
|
+
#stderrTail = "";
|
|
51
|
+
/** Settles when stderr reaches EOF, so an exit message can wait for the diagnostic. */
|
|
52
|
+
readonly #stderrDone: Promise<void>;
|
|
46
53
|
readonly #queue = new FrameQueue();
|
|
47
54
|
readonly #pending = new Map<string, (frame: Frame) => void>();
|
|
48
55
|
/** Both halves of tool verification, or null when the spawn promised nothing. */
|
|
@@ -60,12 +67,29 @@ class LiveTransport implements AgentTransport {
|
|
|
60
67
|
cwd: options.cwd,
|
|
61
68
|
stdin: "pipe",
|
|
62
69
|
stdout: "pipe",
|
|
63
|
-
stderr
|
|
70
|
+
// pi prints model and startup diagnostics to stderr and exits 1 (main() →
|
|
71
|
+
// reportDiagnostics → process.exit(1)), so stderr carries the only reason
|
|
72
|
+
// yaag can classify. The reader below drains it so pi never blocks.
|
|
73
|
+
stderr: "pipe",
|
|
64
74
|
// Its own process group, so the Agent sees stdin EOF rather than a signal
|
|
65
75
|
// when the Orchestrator dies — EOF is the path that reaps grandchildren.
|
|
66
76
|
detached: true,
|
|
67
77
|
});
|
|
68
78
|
void this.#read();
|
|
79
|
+
// A stream error must not surface as an unhandled rejection: the tail is a
|
|
80
|
+
// best-effort diagnostic, never a reason to fail an Agent that started.
|
|
81
|
+
this.#stderrDone = this.#readStderr().catch(() => undefined);
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/** Drains stderr continuously, keeping only a bounded tail. */
|
|
85
|
+
async #readStderr(): Promise<void> {
|
|
86
|
+
const decoder = new TextDecoder();
|
|
87
|
+
for await (const chunk of readChunks(this.#process.stderr)) {
|
|
88
|
+
this.#stderrTail = appendStderrTail(
|
|
89
|
+
this.#stderrTail,
|
|
90
|
+
decoder.decode(chunk, { stream: true }),
|
|
91
|
+
);
|
|
92
|
+
}
|
|
69
93
|
}
|
|
70
94
|
|
|
71
95
|
/** Confirms the Agent is up and speaking the protocol, and resolves its model. */
|
|
@@ -90,6 +114,8 @@ class LiveTransport implements AgentTransport {
|
|
|
90
114
|
// The original startup error remains the useful failure.
|
|
91
115
|
}
|
|
92
116
|
if (error instanceof YaagError) throw error;
|
|
117
|
+
// Only the exit path folds in stderr: a tool-contract or timeout failure
|
|
118
|
+
// must stay generic, whatever pi warned about earlier (ADR-0037).
|
|
93
119
|
const message = error instanceof Error ? error.message : "agent startup failed";
|
|
94
120
|
throw new YaagError("SPAWN_FAILED", message, this.#name);
|
|
95
121
|
}
|
|
@@ -174,8 +200,15 @@ class LiveTransport implements AgentTransport {
|
|
|
174
200
|
timeoutMs,
|
|
175
201
|
);
|
|
176
202
|
});
|
|
177
|
-
const death = this.#process.exited.then((code) => {
|
|
178
|
-
|
|
203
|
+
const death = this.#process.exited.then(async (code) => {
|
|
204
|
+
// Detached grandchildren can hold the pipe open, so the drain is bounded:
|
|
205
|
+
// an exit must reject fast, with whatever diagnostic already arrived.
|
|
206
|
+
await Promise.race([this.#stderrDone, Bun.sleep(STDERR_DRAIN_MS)]);
|
|
207
|
+
throw new YaagError(
|
|
208
|
+
"SPAWN_FAILED",
|
|
209
|
+
`agent "${this.#name}" exited (${code})${stderrDiagnostic(this.#stderrTail)}`,
|
|
210
|
+
this.#name,
|
|
211
|
+
);
|
|
179
212
|
});
|
|
180
213
|
return Promise.race([response, failure, death]).finally(() => clearTimeout(timer));
|
|
181
214
|
}
|