@deepstrike/wasm 0.2.39 → 0.2.40
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -13
- package/dist/harness/index.d.ts +127 -61
- package/dist/harness/index.js +212 -83
- package/dist/index.d.ts +9 -6
- package/dist/index.js +4 -2
- package/dist/memory/extraction.d.ts +4 -0
- package/dist/memory/extraction.js +48 -0
- package/dist/memory/in-memory-store.d.ts +19 -9
- package/dist/memory/in-memory-store.js +68 -23
- package/dist/memory/index.d.ts +52 -24
- package/dist/memory/ranking.d.ts +33 -0
- package/dist/memory/ranking.js +77 -0
- package/dist/memory/retention.d.ts +17 -0
- package/dist/memory/retention.js +54 -0
- package/dist/providers/base.d.ts +5 -0
- package/dist/providers/base.js +32 -0
- package/dist/runtime/context-policy.d.ts +35 -0
- package/dist/runtime/context-policy.js +66 -0
- package/dist/runtime/eval.d.ts +2 -0
- package/dist/runtime/execution-plane.d.ts +2 -1
- package/dist/runtime/execution-plane.js +3 -3
- package/dist/runtime/facade.js +2 -1
- package/dist/runtime/index.d.ts +11 -4
- package/dist/runtime/index.js +5 -1
- package/dist/runtime/kernel-event-log.js +46 -13
- package/dist/runtime/kernel-rebuild.d.ts +8 -0
- package/dist/runtime/kernel-rebuild.js +65 -0
- package/dist/runtime/kernel-step.d.ts +139 -7
- package/dist/runtime/kernel-step.js +110 -10
- package/dist/runtime/kernel-transaction-log.d.ts +61 -0
- package/dist/runtime/kernel-transaction-log.js +140 -0
- package/dist/runtime/os-profile.d.ts +9 -10
- package/dist/runtime/os-profile.js +14 -10
- package/dist/runtime/os-snapshot.d.ts +19 -0
- package/dist/runtime/os-snapshot.js +16 -3
- package/dist/runtime/runner.d.ts +73 -41
- package/dist/runtime/runner.js +562 -290
- package/dist/runtime/session-log.d.ts +55 -10
- package/dist/runtime/session-log.js +60 -3
- package/dist/runtime/session-repair.d.ts +11 -7
- package/dist/runtime/session-repair.js +11 -8
- package/dist/runtime/sub-agent-orchestrator.js +1 -1
- package/dist/runtime/types/agent.d.ts +31 -0
- package/dist/runtime/types/agent.js +35 -0
- package/dist/signals/index.d.ts +21 -1
- package/dist/signals/index.js +1 -0
- package/dist/types.d.ts +7 -2
- package/package.json +2 -2
package/README.md
CHANGED
|
@@ -79,7 +79,7 @@ src/
|
|
|
79
79
|
├── tools/ # tool() helper, executeTools (no fs/shell)
|
|
80
80
|
├── memory/ # WorkingMemory + DreamStore interfaces
|
|
81
81
|
├── knowledge/ # KnowledgeSource interface
|
|
82
|
-
├── harness/ #
|
|
82
|
+
├── harness/ # AttemptLoop body/judge/carry/stop policies
|
|
83
83
|
├── signals/ # RuntimeSignal, SignalSource, ScheduledPrompt
|
|
84
84
|
└── safety/ # PermissionManager
|
|
85
85
|
```
|
|
@@ -241,26 +241,26 @@ const runner = new RuntimeRunner({
|
|
|
241
241
|
|
|
242
242
|
---
|
|
243
243
|
|
|
244
|
-
##
|
|
244
|
+
## Attempt loop
|
|
245
245
|
|
|
246
246
|
```typescript
|
|
247
|
-
import {
|
|
247
|
+
import { AttemptLoop, RuntimeAttemptBody, LlmEvalJudge } from "@deepstrike/wasm"
|
|
248
248
|
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
const loop = new HarnessLoop(runner, evalProvider, { maxAttempts: 3 })
|
|
256
|
-
for await (const event of loop.runStreaming({
|
|
249
|
+
const loop = new AttemptLoop({
|
|
250
|
+
body: new RuntimeAttemptBody(runner),
|
|
251
|
+
judge: new LlmEvalJudge(evalProvider),
|
|
252
|
+
stop: { maxAttempts: 3 },
|
|
253
|
+
})
|
|
254
|
+
for await (const event of loop.stream({
|
|
257
255
|
goal: "Write a haiku",
|
|
258
256
|
criteria: [
|
|
259
257
|
{ text: "Exactly 3 lines", required: true },
|
|
260
258
|
{ text: "Contains a seasonal reference", required: false },
|
|
261
259
|
],
|
|
262
260
|
})) {
|
|
263
|
-
if (event.type === "
|
|
261
|
+
if (event.type === "completed") {
|
|
262
|
+
console.log(event.outcome.runStatus, event.outcome.outcome, event.outcome.verdict)
|
|
263
|
+
}
|
|
264
264
|
}
|
|
265
265
|
```
|
|
266
266
|
|
|
@@ -268,7 +268,8 @@ for await (const event of loop.runStreaming({
|
|
|
268
268
|
|
|
269
269
|
## Signals & interrupts
|
|
270
270
|
|
|
271
|
-
Delivered signals fold into Slot 3 (`turns[0]`) and are
|
|
271
|
+
Delivered signals fold into Slot 3 (`turns[0]`) and are consumed only after the correlated
|
|
272
|
+
provider result commits, so retries see the same signal exactly once.
|
|
272
273
|
|
|
273
274
|
```typescript
|
|
274
275
|
import { ScheduledPrompt } from "@deepstrike/wasm"
|
package/dist/harness/index.d.ts
CHANGED
|
@@ -1,87 +1,153 @@
|
|
|
1
1
|
import type { RuntimeRunner } from "../runtime/runner.js";
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
/** I3.3 (A4): host hint — host has a deterministic check for this criterion. */
|
|
9
|
-
machineCheckable?: boolean;
|
|
10
|
-
}
|
|
11
|
-
export interface CriterionResult {
|
|
12
|
-
criterion: string;
|
|
13
|
-
passed: boolean;
|
|
14
|
-
score: number;
|
|
15
|
-
feedback: string;
|
|
16
|
-
}
|
|
17
|
-
export interface HarnessRequest {
|
|
2
|
+
import type { Criterion, Verdict } from "../runtime/eval.js";
|
|
3
|
+
import type { LLMProvider } from "../types.js";
|
|
4
|
+
import type { WorkflowNodeSpec } from "../runtime/types/agent.js";
|
|
5
|
+
export type { Criterion, Verdict } from "../runtime/eval.js";
|
|
6
|
+
export interface AttemptRequest {
|
|
7
|
+
sessionId?: string;
|
|
18
8
|
goal: string;
|
|
19
9
|
criteria?: Criterion[];
|
|
20
10
|
extensions?: Record<string, unknown>;
|
|
11
|
+
inheritEvents?: Array<{
|
|
12
|
+
seq: number;
|
|
13
|
+
event: import("../runtime/session-log.js").SessionEvent;
|
|
14
|
+
}>;
|
|
21
15
|
}
|
|
22
|
-
export interface
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
overallScore?: number;
|
|
29
|
-
feedback?: string;
|
|
30
|
-
details?: CriterionResult[];
|
|
31
|
-
}
|
|
32
|
-
export interface Verdict {
|
|
33
|
-
passed: boolean;
|
|
34
|
-
overallScore: number;
|
|
35
|
-
feedback: string;
|
|
36
|
-
details: CriterionResult[];
|
|
37
|
-
}
|
|
38
|
-
export type HarnessEvent = {
|
|
16
|
+
export interface AttemptBodyContext extends AttemptRequest {
|
|
17
|
+
sessionId: string;
|
|
18
|
+
attempt: number;
|
|
19
|
+
contextInput?: string;
|
|
20
|
+
}
|
|
21
|
+
export type AttemptProgressEvent = {
|
|
39
22
|
type: "token";
|
|
40
23
|
text: string;
|
|
41
24
|
} | {
|
|
42
25
|
type: "tool_call";
|
|
43
26
|
id: string;
|
|
44
27
|
name: string;
|
|
28
|
+
} | {
|
|
29
|
+
type: "tool_delta";
|
|
30
|
+
callId: string;
|
|
31
|
+
delta?: string;
|
|
32
|
+
chunk?: Record<string, unknown>;
|
|
33
|
+
} | {
|
|
34
|
+
type: "tool_suspend";
|
|
35
|
+
callId: string;
|
|
36
|
+
suspensionId: string;
|
|
37
|
+
payload?: Record<string, unknown>;
|
|
45
38
|
} | {
|
|
46
39
|
type: "tool_result";
|
|
47
40
|
callId: string;
|
|
48
41
|
content: string;
|
|
49
42
|
isError: boolean;
|
|
50
43
|
} | {
|
|
51
|
-
type: "
|
|
52
|
-
|
|
53
|
-
type: "revising";
|
|
54
|
-
verdict: Verdict;
|
|
44
|
+
type: "workflow_nodes_submitted";
|
|
45
|
+
nodes: WorkflowNodeSpec[];
|
|
55
46
|
} | {
|
|
56
|
-
type: "
|
|
57
|
-
|
|
58
|
-
iterations: number;
|
|
59
|
-
totalTokens: number;
|
|
60
|
-
status: string;
|
|
61
|
-
} | {
|
|
62
|
-
type: "max_attempts_reached";
|
|
47
|
+
type: "body_error";
|
|
48
|
+
message: string;
|
|
63
49
|
};
|
|
64
|
-
export
|
|
65
|
-
|
|
50
|
+
export interface AttemptBodyTerminal {
|
|
51
|
+
type: "body_done";
|
|
52
|
+
runStatus: string;
|
|
53
|
+
result: string;
|
|
54
|
+
turns: number;
|
|
55
|
+
totalTokens: number;
|
|
56
|
+
submittedNodes?: WorkflowNodeSpec[];
|
|
57
|
+
}
|
|
58
|
+
export type AttemptBodyEvent = AttemptProgressEvent | AttemptBodyTerminal;
|
|
59
|
+
export interface AttemptBody {
|
|
60
|
+
run(context: AttemptBodyContext): AsyncIterable<AttemptBodyEvent>;
|
|
61
|
+
}
|
|
62
|
+
export declare class RuntimeAttemptBody implements AttemptBody {
|
|
63
|
+
private readonly runner;
|
|
66
64
|
constructor(runner: RuntimeRunner);
|
|
67
|
-
run(
|
|
65
|
+
run(context: AttemptBodyContext): AsyncIterable<AttemptBodyEvent>;
|
|
68
66
|
}
|
|
69
|
-
|
|
70
|
-
export type VerdictFn = (ctx: {
|
|
67
|
+
export interface JudgeContext {
|
|
71
68
|
goal: string;
|
|
72
69
|
criteria: Criterion[];
|
|
73
70
|
attempt: number;
|
|
74
71
|
result: string;
|
|
75
|
-
}
|
|
76
|
-
export interface
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
private verdictFn
|
|
85
|
-
constructor(
|
|
86
|
-
|
|
72
|
+
}
|
|
73
|
+
export interface JudgeResult {
|
|
74
|
+
verdict: Verdict;
|
|
75
|
+
}
|
|
76
|
+
export interface AttemptJudge {
|
|
77
|
+
judge(context: JudgeContext): Promise<JudgeResult | undefined>;
|
|
78
|
+
}
|
|
79
|
+
export type VerdictFn = (context: JudgeContext) => Verdict | undefined | Promise<Verdict | undefined>;
|
|
80
|
+
export declare class VerdictFnJudge implements AttemptJudge {
|
|
81
|
+
private readonly verdictFn;
|
|
82
|
+
constructor(verdictFn: VerdictFn);
|
|
83
|
+
judge(context: JudgeContext): Promise<JudgeResult | undefined>;
|
|
84
|
+
}
|
|
85
|
+
export declare class LlmEvalJudge implements AttemptJudge {
|
|
86
|
+
private readonly provider;
|
|
87
|
+
constructor(provider: LLMProvider);
|
|
88
|
+
judge(context: JudgeContext): Promise<JudgeResult>;
|
|
89
|
+
}
|
|
90
|
+
export declare class HybridJudge implements AttemptJudge {
|
|
91
|
+
private readonly primary;
|
|
92
|
+
private readonly fallback;
|
|
93
|
+
constructor(primary: AttemptJudge, fallback: AttemptJudge);
|
|
94
|
+
judge(context: JudgeContext): Promise<JudgeResult | undefined>;
|
|
95
|
+
}
|
|
96
|
+
export interface PreparedAttempt {
|
|
97
|
+
sessionId: string;
|
|
98
|
+
goal: string;
|
|
99
|
+
contextInput?: string;
|
|
100
|
+
}
|
|
101
|
+
export type CarryPolicy = (context: {
|
|
102
|
+
rootSessionId: string;
|
|
103
|
+
goal: string;
|
|
104
|
+
attempt: number;
|
|
105
|
+
previousVerdict?: Verdict;
|
|
106
|
+
}) => PreparedAttempt | Promise<PreparedAttempt>;
|
|
107
|
+
export declare const continueSession: CarryPolicy;
|
|
108
|
+
export declare const freshWithFeedback: CarryPolicy;
|
|
109
|
+
export declare function freshWithDigest(digest: (verdict: Verdict, attempt: number) => string | Promise<string>): CarryPolicy;
|
|
110
|
+
export interface StopPolicy {
|
|
111
|
+
maxAttempts: number;
|
|
112
|
+
maxTotalTokens?: number;
|
|
113
|
+
stopOnFailedVerdict?: boolean;
|
|
114
|
+
}
|
|
115
|
+
export type AttemptOutcomeKind = "passed" | "failed_judge" | "exhausted" | "run_error";
|
|
116
|
+
export interface AttemptOutcome {
|
|
117
|
+
outcome: AttemptOutcomeKind;
|
|
118
|
+
runStatus: string;
|
|
119
|
+
verdict?: Verdict;
|
|
120
|
+
result: string;
|
|
121
|
+
attempts: number;
|
|
122
|
+
turns: number;
|
|
123
|
+
totalTokens: number;
|
|
124
|
+
submittedNodes?: WorkflowNodeSpec[];
|
|
125
|
+
}
|
|
126
|
+
export type AttemptLoopEvent = AttemptProgressEvent | {
|
|
127
|
+
type: "judging";
|
|
128
|
+
attempt: number;
|
|
129
|
+
} | {
|
|
130
|
+
type: "retrying";
|
|
131
|
+
attempt: number;
|
|
132
|
+
verdict: Verdict;
|
|
133
|
+
} | {
|
|
134
|
+
type: "completed";
|
|
135
|
+
outcome: AttemptOutcome;
|
|
136
|
+
};
|
|
137
|
+
export interface AttemptLoopOptions {
|
|
138
|
+
body: AttemptBody;
|
|
139
|
+
judge: AttemptJudge;
|
|
140
|
+
carry?: CarryPolicy;
|
|
141
|
+
stop: StopPolicy;
|
|
142
|
+
onPass?: (context: {
|
|
143
|
+
outcome: AttemptOutcome;
|
|
144
|
+
judgeResult: JudgeResult;
|
|
145
|
+
}) => void | Promise<void>;
|
|
146
|
+
}
|
|
147
|
+
export declare class AttemptLoop {
|
|
148
|
+
private readonly options;
|
|
149
|
+
private readonly carry;
|
|
150
|
+
constructor(options: AttemptLoopOptions);
|
|
151
|
+
run(request: AttemptRequest): Promise<AttemptOutcome>;
|
|
152
|
+
stream(request: AttemptRequest): AsyncIterable<AttemptLoopEvent>;
|
|
87
153
|
}
|
package/dist/harness/index.js
CHANGED
|
@@ -1,101 +1,230 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
let done;
|
|
4
|
-
const sessionId = crypto.randomUUID();
|
|
5
|
-
for await (const evt of runner.run({ sessionId, goal: req.goal, criteria: req.criteria?.map(c => c.text), extensions: req.extensions })) {
|
|
6
|
-
if (evt.type === "text_delta")
|
|
7
|
-
text += evt.delta;
|
|
8
|
-
else if (evt.type === "done")
|
|
9
|
-
done = evt;
|
|
10
|
-
}
|
|
11
|
-
return { result: text, passed: false, iterations: done?.iterations ?? 0, totalTokens: done?.totalTokens ?? 0, status: done?.status ?? "error" };
|
|
12
|
-
}
|
|
13
|
-
export class SinglePassHarness {
|
|
1
|
+
import { getKernel } from "../runtime/kernel.js";
|
|
2
|
+
export class RuntimeAttemptBody {
|
|
14
3
|
runner;
|
|
15
4
|
constructor(runner) {
|
|
16
5
|
this.runner = runner;
|
|
17
6
|
}
|
|
18
|
-
async run(
|
|
19
|
-
|
|
7
|
+
async *run(context) {
|
|
8
|
+
if (context.contextInput)
|
|
9
|
+
this.runner.injectNote(context.contextInput);
|
|
10
|
+
let result = "";
|
|
11
|
+
let done;
|
|
12
|
+
const submittedNodes = [];
|
|
13
|
+
for await (const event of this.runner.run({
|
|
14
|
+
sessionId: context.sessionId,
|
|
15
|
+
goal: context.goal,
|
|
16
|
+
criteria: (context.criteria ?? []).map(criterion => criterion.text),
|
|
17
|
+
extensions: context.extensions,
|
|
18
|
+
...(context.attempt === 1 && context.inheritEvents
|
|
19
|
+
? { inheritEvents: context.inheritEvents }
|
|
20
|
+
: {}),
|
|
21
|
+
})) {
|
|
22
|
+
if (event.type === "text_delta") {
|
|
23
|
+
const text = event.delta;
|
|
24
|
+
result += text;
|
|
25
|
+
yield { type: "token", text };
|
|
26
|
+
}
|
|
27
|
+
else if (event.type === "tool_call") {
|
|
28
|
+
const call = event;
|
|
29
|
+
yield { type: "tool_call", id: call.id, name: call.name };
|
|
30
|
+
}
|
|
31
|
+
else if (event.type === "tool_delta") {
|
|
32
|
+
const delta = event;
|
|
33
|
+
yield { type: "tool_delta", callId: delta.callId, delta: delta.delta, chunk: delta.chunk };
|
|
34
|
+
}
|
|
35
|
+
else if (event.type === "tool_suspend") {
|
|
36
|
+
const suspended = event;
|
|
37
|
+
yield { type: "tool_suspend", callId: suspended.callId, suspensionId: suspended.suspensionId, payload: suspended.payload };
|
|
38
|
+
}
|
|
39
|
+
else if (event.type === "tool_result") {
|
|
40
|
+
const tool = event;
|
|
41
|
+
yield { type: "tool_result", callId: tool.callId, content: tool.content, isError: tool.isError };
|
|
42
|
+
}
|
|
43
|
+
else if (event.type === "workflow_nodes_submitted") {
|
|
44
|
+
const nodes = event.nodes;
|
|
45
|
+
submittedNodes.push(...nodes);
|
|
46
|
+
yield { type: "workflow_nodes_submitted", nodes };
|
|
47
|
+
}
|
|
48
|
+
else if (event.type === "error") {
|
|
49
|
+
yield { type: "body_error", message: String(event.message ?? "run failed") };
|
|
50
|
+
}
|
|
51
|
+
else if (event.type === "done") {
|
|
52
|
+
done = event;
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
yield {
|
|
56
|
+
type: "body_done",
|
|
57
|
+
runStatus: done?.status ?? "error",
|
|
58
|
+
result,
|
|
59
|
+
turns: done?.iterations ?? 0,
|
|
60
|
+
totalTokens: done?.totalTokens ?? 0,
|
|
61
|
+
...(submittedNodes.length ? { submittedNodes } : {}),
|
|
62
|
+
};
|
|
20
63
|
}
|
|
21
64
|
}
|
|
22
|
-
export class
|
|
23
|
-
runner;
|
|
24
|
-
evalProvider;
|
|
25
|
-
maxAttempts;
|
|
65
|
+
export class VerdictFnJudge {
|
|
26
66
|
verdictFn;
|
|
27
|
-
constructor(
|
|
28
|
-
this.
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
67
|
+
constructor(verdictFn) {
|
|
68
|
+
this.verdictFn = verdictFn;
|
|
69
|
+
}
|
|
70
|
+
async judge(context) {
|
|
71
|
+
const verdict = await this.verdictFn(context);
|
|
72
|
+
return verdict ? { verdict } : undefined;
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
export class LlmEvalJudge {
|
|
76
|
+
provider;
|
|
77
|
+
constructor(provider) {
|
|
78
|
+
this.provider = provider;
|
|
79
|
+
}
|
|
80
|
+
async judge(context) {
|
|
81
|
+
const kernel = await getKernel();
|
|
82
|
+
const messages = kernel.buildEvalMessages(context.goal, context.criteria, context.result, context.attempt, false);
|
|
83
|
+
const rendered = {
|
|
84
|
+
systemText: messages.filter(message => message.role === "system").map(message => message.content).join("\n\n"),
|
|
85
|
+
turns: messages.filter(message => message.role !== "system"),
|
|
86
|
+
};
|
|
87
|
+
let text = "";
|
|
88
|
+
for await (const event of this.provider.stream(rendered, [], undefined)) {
|
|
89
|
+
if (event.type === "text_delta")
|
|
90
|
+
text += event.delta;
|
|
91
|
+
}
|
|
92
|
+
if (!text)
|
|
93
|
+
throw new Error("attempt judge produced no text");
|
|
94
|
+
const parsed = kernel.parseVerdict(text);
|
|
95
|
+
return {
|
|
96
|
+
verdict: {
|
|
97
|
+
passed: parsed.passed,
|
|
98
|
+
overallScore: parsed.overallScore,
|
|
99
|
+
feedback: parsed.feedback,
|
|
100
|
+
details: (parsed.details ?? []),
|
|
101
|
+
},
|
|
102
|
+
};
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
export class HybridJudge {
|
|
106
|
+
primary;
|
|
107
|
+
fallback;
|
|
108
|
+
constructor(primary, fallback) {
|
|
109
|
+
this.primary = primary;
|
|
110
|
+
this.fallback = fallback;
|
|
111
|
+
}
|
|
112
|
+
async judge(context) {
|
|
113
|
+
return await this.primary.judge(context) ?? this.fallback.judge(context);
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
export const continueSession = context => ({
|
|
117
|
+
sessionId: context.rootSessionId,
|
|
118
|
+
goal: context.goal,
|
|
119
|
+
...(context.previousVerdict?.feedback ? { contextInput: context.previousVerdict.feedback } : {}),
|
|
120
|
+
});
|
|
121
|
+
export const freshWithFeedback = context => ({
|
|
122
|
+
sessionId: context.attempt === 1 ? context.rootSessionId : crypto.randomUUID(),
|
|
123
|
+
goal: context.previousVerdict?.feedback
|
|
124
|
+
? `${context.goal}\n\n[Attempt ${context.attempt - 1} feedback: ${context.previousVerdict.feedback}]`
|
|
125
|
+
: context.goal,
|
|
126
|
+
});
|
|
127
|
+
export function freshWithDigest(digest) {
|
|
128
|
+
return async (context) => ({
|
|
129
|
+
sessionId: context.attempt === 1 ? context.rootSessionId : crypto.randomUUID(),
|
|
130
|
+
goal: context.previousVerdict
|
|
131
|
+
? `${context.goal}\n\n[Prior attempt digest: ${await digest(context.previousVerdict, context.attempt - 1)}]`
|
|
132
|
+
: context.goal,
|
|
133
|
+
});
|
|
134
|
+
}
|
|
135
|
+
export class AttemptLoop {
|
|
136
|
+
options;
|
|
137
|
+
carry;
|
|
138
|
+
constructor(options) {
|
|
139
|
+
this.options = options;
|
|
140
|
+
if (!Number.isInteger(options.stop.maxAttempts) || options.stop.maxAttempts < 1) {
|
|
141
|
+
throw new Error("AttemptLoop stop.maxAttempts must be a positive integer");
|
|
142
|
+
}
|
|
143
|
+
if (options.stop.maxTotalTokens !== undefined && options.stop.maxTotalTokens < 0) {
|
|
144
|
+
throw new Error("AttemptLoop stop.maxTotalTokens must be non-negative");
|
|
145
|
+
}
|
|
146
|
+
this.carry = options.carry ?? continueSession;
|
|
147
|
+
}
|
|
148
|
+
async run(request) {
|
|
149
|
+
let outcome;
|
|
150
|
+
for await (const event of this.stream(request)) {
|
|
151
|
+
if (event.type === "completed")
|
|
152
|
+
outcome = event.outcome;
|
|
153
|
+
}
|
|
154
|
+
if (!outcome)
|
|
155
|
+
throw new Error("AttemptLoop ended without an outcome");
|
|
156
|
+
return outcome;
|
|
32
157
|
}
|
|
33
|
-
async *
|
|
34
|
-
const
|
|
158
|
+
async *stream(request) {
|
|
159
|
+
const rootSessionId = request.sessionId ?? crypto.randomUUID();
|
|
35
160
|
const criteria = request.criteria ?? [];
|
|
36
|
-
|
|
37
|
-
let
|
|
38
|
-
let
|
|
39
|
-
let
|
|
40
|
-
let
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
for await (const
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
const tr = evt;
|
|
56
|
-
yield { type: "tool_result", callId: tr.callId, content: tr.content, isError: tr.isError };
|
|
57
|
-
}
|
|
58
|
-
else if (evt.type === "done") {
|
|
59
|
-
const d = evt;
|
|
60
|
-
lastIterations = d.iterations;
|
|
61
|
-
lastTotalTokens = d.totalTokens;
|
|
62
|
-
lastStatus = d.status;
|
|
161
|
+
const submittedNodes = [];
|
|
162
|
+
let totalTokens = 0;
|
|
163
|
+
let totalTurns = 0;
|
|
164
|
+
let previousVerdict;
|
|
165
|
+
for (let attempt = 1; attempt <= this.options.stop.maxAttempts; attempt++) {
|
|
166
|
+
const prepared = await this.carry({ rootSessionId, goal: request.goal, attempt, previousVerdict });
|
|
167
|
+
let terminal;
|
|
168
|
+
for await (const event of this.options.body.run({
|
|
169
|
+
...request,
|
|
170
|
+
sessionId: prepared.sessionId,
|
|
171
|
+
goal: prepared.goal,
|
|
172
|
+
criteria,
|
|
173
|
+
attempt,
|
|
174
|
+
...(prepared.contextInput ? { contextInput: prepared.contextInput } : {}),
|
|
175
|
+
})) {
|
|
176
|
+
if (event.type === "body_done") {
|
|
177
|
+
terminal = event;
|
|
178
|
+
if (event.submittedNodes)
|
|
179
|
+
submittedNodes.push(...event.submittedNodes);
|
|
63
180
|
}
|
|
181
|
+
else
|
|
182
|
+
yield event;
|
|
64
183
|
}
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
184
|
+
if (!terminal)
|
|
185
|
+
throw new Error("AttemptBody ended without body_done");
|
|
186
|
+
totalTokens += terminal.totalTokens;
|
|
187
|
+
totalTurns += terminal.turns;
|
|
188
|
+
const base = {
|
|
189
|
+
runStatus: terminal.runStatus,
|
|
190
|
+
result: terminal.result,
|
|
191
|
+
attempts: attempt,
|
|
192
|
+
turns: totalTurns,
|
|
193
|
+
totalTokens,
|
|
194
|
+
...(submittedNodes.length ? { submittedNodes } : {}),
|
|
195
|
+
};
|
|
196
|
+
if (isRunError(terminal.runStatus)) {
|
|
197
|
+
yield { type: "completed", outcome: { outcome: "run_error", ...base } };
|
|
198
|
+
return;
|
|
70
199
|
}
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
};
|
|
79
|
-
|
|
80
|
-
if (evt.type === "text_delta")
|
|
81
|
-
evalText += evt.delta;
|
|
82
|
-
}
|
|
83
|
-
const parsed = kernel.parseVerdict(evalText);
|
|
84
|
-
verdict = {
|
|
85
|
-
passed: parsed.passed,
|
|
86
|
-
overallScore: parsed.overallScore,
|
|
87
|
-
feedback: parsed.feedback,
|
|
88
|
-
details: (parsed.details ?? []),
|
|
89
|
-
};
|
|
200
|
+
yield { type: "judging", attempt };
|
|
201
|
+
const judged = await this.options.judge.judge({ goal: request.goal, criteria, attempt, result: terminal.result });
|
|
202
|
+
if (!judged)
|
|
203
|
+
throw new Error("AttemptLoop judge produced no verdict");
|
|
204
|
+
if (judged.verdict.passed) {
|
|
205
|
+
const outcome = { outcome: "passed", ...base, verdict: judged.verdict };
|
|
206
|
+
await this.options.onPass?.({ outcome, judgeResult: judged });
|
|
207
|
+
yield { type: "completed", outcome };
|
|
208
|
+
return;
|
|
90
209
|
}
|
|
91
|
-
|
|
92
|
-
|
|
210
|
+
previousVerdict = judged.verdict;
|
|
211
|
+
const tokenLimitReached = this.options.stop.maxTotalTokens !== undefined
|
|
212
|
+
&& totalTokens >= this.options.stop.maxTotalTokens;
|
|
213
|
+
if (this.options.stop.stopOnFailedVerdict || attempt === this.options.stop.maxAttempts || tokenLimitReached) {
|
|
214
|
+
yield {
|
|
215
|
+
type: "completed",
|
|
216
|
+
outcome: {
|
|
217
|
+
outcome: this.options.stop.stopOnFailedVerdict ? "failed_judge" : "exhausted",
|
|
218
|
+
...base,
|
|
219
|
+
verdict: judged.verdict,
|
|
220
|
+
},
|
|
221
|
+
};
|
|
93
222
|
return;
|
|
94
223
|
}
|
|
95
|
-
yield { type: "
|
|
96
|
-
currentGoal = `${request.goal}\n\n[Attempt ${attempt} feedback: ${verdict.feedback}]`;
|
|
97
|
-
lastResult = "";
|
|
224
|
+
yield { type: "retrying", attempt, verdict: judged.verdict };
|
|
98
225
|
}
|
|
99
|
-
yield { type: "max_attempts_reached" };
|
|
100
226
|
}
|
|
101
227
|
}
|
|
228
|
+
function isRunError(status) {
|
|
229
|
+
return ["error", "invalid_arg", "user_abort"].includes(status.toLocaleLowerCase());
|
|
230
|
+
}
|
package/dist/index.d.ts
CHANGED
|
@@ -1,10 +1,11 @@
|
|
|
1
|
-
export { RuntimeRunner, collectText, runAgent, runFanout, InMemorySessionLog, LocalExecutionPlane,
|
|
1
|
+
export { RuntimeRunner, collectText, runAgent, runFanout, InMemorySessionLog, LocalExecutionPlane, DEFAULT_NATIVE_SIGNAL_POLICY, DEFAULT_NATIVE_GOVERNANCE_POLICY, DEFAULT_SANDBOX_POLICY, assertNativeProfile, osProfile, validateDeclarativePolicy, ReplayProvider, extractRecordedMessages, judge, buildEvalMessages, parseVerdict, verdictOutputSchema, } from "./runtime/index.js";
|
|
2
|
+
export * from "./runtime/kernel-transaction-log.js";
|
|
2
3
|
export type { ReplayProviderOpts, Criterion, Verdict, VerdictDetail, JudgeArgs, } from "./runtime/index.js";
|
|
3
|
-
export type { NativeOsProfile, OsProfileId, MemoryPolicy, MemoryWriteRateLimit, ResourceQuota, RuntimeOptions,
|
|
4
|
+
export type { NativeOsProfile, OsProfileId, MemoryPolicy, MemoryWriteRateLimit, ResourceQuota, RuntimeOptions, PromptBudget, SchedulerPolicy, SignalPolicy, SessionEvent, SessionLog, KernelTransactionEntry, RunContext, ExecutionPlane, } from "./runtime/index.js";
|
|
4
5
|
export { FilteredExecutionPlane } from "./runtime/filtered-plane.js";
|
|
5
6
|
export { SubAgentOrchestrator, defaultSubAgentOrchestrator, spawnStandalone } from "./runtime/sub-agent-orchestrator.js";
|
|
6
7
|
export type { SubAgentRunContext } from "./runtime/sub-agent-orchestrator.js";
|
|
7
|
-
export type { AgentCapabilityFilter, AgentIdentity, AgentIsolation, AgentRunSpec, AgentProcessChangedObservation, ContextInheritance, KernelAgentRole, LoopResult, MilestoneCheckResult, MilestoneContract, MilestonePhase, MilestonePolicy, SubAgentResult, TerminationReason, WorkflowSpec, WorkflowNodeSpec, WorkflowTaskSpec, WorkflowSpawnInfo, } from "./runtime/types/agent.js";
|
|
8
|
+
export type { AgentCapabilityFilter, AgentIdentity, AgentIsolation, AgentRunSpec, AgentProcessChangedObservation, ContextInheritance, KernelAgentRole, LoopResult, MilestoneCheckResult, MilestoneContract, MilestonePhase, MilestonePolicy, SubAgentResult, TerminationReason, WorkflowSpec, WorkflowNodeSpec, WorkflowDependencyPolicy, WorkflowNodeStatus, WorkflowNodeOutcome, WorkflowOutcome, WorkflowTaskSpec, WorkflowSpawnInfo, } from "./runtime/types/agent.js";
|
|
8
9
|
export { workflowSpecToKernel, workflowNodeSpecToKernel, submitWorkflowNodesToKernel, submitWorkflowToKernel, submitWorkflowNodesTool, startWorkflowTool, fanoutSynthesize, generateAndFilter, genEval, verifyRules } from "./runtime/types/agent.js";
|
|
9
10
|
export { loopInstruction, classifyInstruction, judgeGoal, extractLoopContinue, extractClassifyBranch, extractJudgeWinner, } from "./runtime/workflow-control-flow.js";
|
|
10
11
|
export { Governance } from "./governance.js";
|
|
@@ -17,10 +18,12 @@ export { safeTool, ok, fail, ToolError, formatToolError } from "./tools/errors.j
|
|
|
17
18
|
export type { ToolEnvelope, ToolEnvelopeOk, ToolEnvelopeFail } from "./tools/errors.js";
|
|
18
19
|
export { WorkingMemory } from "./memory/index.js";
|
|
19
20
|
export { InMemoryDreamStore } from "./memory/in-memory-store.js";
|
|
20
|
-
export type {
|
|
21
|
+
export type { InMemoryDreamStoreOptions } from "./memory/in-memory-store.js";
|
|
22
|
+
export { memoryRetentionScore } from "./memory/retention.js";
|
|
23
|
+
export type { DreamStore, SessionStore, SessionData, SessionMessage, MemoryRecord, MemoryRecall, MemoryRecallLifecycle, MemoryQuery, MemoryScope, MemoryProvenance, MemoryKind, MemoryAuthor, MemoryTrustLevel, } from "./memory/index.js";
|
|
21
24
|
export type { KnowledgeSource } from "./knowledge/index.js";
|
|
22
|
-
export {
|
|
23
|
-
export type {
|
|
25
|
+
export { AttemptLoop, RuntimeAttemptBody, VerdictFnJudge, LlmEvalJudge, HybridJudge, continueSession, freshWithFeedback, freshWithDigest, } from "./harness/index.js";
|
|
26
|
+
export type { AttemptBody, AttemptBodyContext, AttemptBodyEvent, AttemptBodyTerminal, AttemptJudge, AttemptLoopEvent, AttemptLoopOptions, AttemptOutcome, AttemptOutcomeKind, AttemptProgressEvent, AttemptRequest, CarryPolicy, JudgeContext, JudgeResult, PreparedAttempt, StopPolicy, VerdictFn, } from "./harness/index.js";
|
|
24
27
|
export { ScheduledPrompt } from "./signals/index.js";
|
|
25
28
|
export type { RuntimeSignal, SignalSource } from "./signals/index.js";
|
|
26
29
|
export { PermissionManager, PermissionMode } from "./safety/index.js";
|
package/dist/index.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
export { RuntimeRunner, collectText, runAgent, runFanout, InMemorySessionLog, LocalExecutionPlane,
|
|
1
|
+
export { RuntimeRunner, collectText, runAgent, runFanout, InMemorySessionLog, LocalExecutionPlane, DEFAULT_NATIVE_SIGNAL_POLICY, DEFAULT_NATIVE_GOVERNANCE_POLICY, DEFAULT_SANDBOX_POLICY, assertNativeProfile, osProfile, validateDeclarativePolicy, ReplayProvider, extractRecordedMessages, judge, buildEvalMessages, parseVerdict, verdictOutputSchema, } from "./runtime/index.js";
|
|
2
|
+
export * from "./runtime/kernel-transaction-log.js";
|
|
2
3
|
export { FilteredExecutionPlane } from "./runtime/filtered-plane.js";
|
|
3
4
|
export { SubAgentOrchestrator, defaultSubAgentOrchestrator, spawnStandalone } from "./runtime/sub-agent-orchestrator.js";
|
|
4
5
|
export { workflowSpecToKernel, workflowNodeSpecToKernel, submitWorkflowNodesToKernel, submitWorkflowToKernel, submitWorkflowNodesTool, startWorkflowTool, fanoutSynthesize, generateAndFilter, genEval, verifyRules } from "./runtime/types/agent.js";
|
|
@@ -10,6 +11,7 @@ export { tool, executeTools } from "./tools/index.js";
|
|
|
10
11
|
export { safeTool, ok, fail, ToolError, formatToolError } from "./tools/errors.js";
|
|
11
12
|
export { WorkingMemory } from "./memory/index.js";
|
|
12
13
|
export { InMemoryDreamStore } from "./memory/in-memory-store.js";
|
|
13
|
-
export {
|
|
14
|
+
export { memoryRetentionScore } from "./memory/retention.js";
|
|
15
|
+
export { AttemptLoop, RuntimeAttemptBody, VerdictFnJudge, LlmEvalJudge, HybridJudge, continueSession, freshWithFeedback, freshWithDigest, } from "./harness/index.js";
|
|
14
16
|
export { ScheduledPrompt } from "./signals/index.js";
|
|
15
17
|
export { PermissionManager, PermissionMode } from "./safety/index.js";
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
import type { LLMProvider } from "../types.js";
|
|
2
|
+
import type { MemoryRecord, MemoryScope, SessionData } from "./index.js";
|
|
3
|
+
export declare function extractSessionMemories(provider: LLMProvider, session: SessionData, scope: MemoryScope, systemPrompt?: string): Promise<MemoryRecord[]>;
|
|
4
|
+
export declare function parseExtractedMemories(output: string, session: SessionData, scope: MemoryScope): MemoryRecord[];
|