@deepstrike/wasm 0.2.18 → 0.2.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +2 -1
- package/dist/index.js +1 -1
- package/dist/runtime/eval.d.ts +45 -0
- package/dist/runtime/eval.js +51 -0
- package/dist/runtime/index.d.ts +5 -0
- package/dist/runtime/index.js +3 -0
- package/dist/runtime/kernel-step.d.ts +3 -0
- package/dist/runtime/kernel-step.js +3 -0
- package/dist/runtime/replay-fixture.d.ts +26 -0
- package/dist/runtime/replay-fixture.js +51 -0
- package/dist/runtime/replay-provider.d.ts +69 -0
- package/dist/runtime/replay-provider.js +152 -0
- package/dist/runtime/runner.d.ts +25 -0
- package/dist/runtime/runner.js +83 -3
- package/package.json +2 -2
package/dist/index.d.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
export { RuntimeRunner, collectText, InMemorySessionLog, LocalExecutionPlane, DEFAULT_NATIVE_ATTENTION_POLICY, DEFAULT_NATIVE_GOVERNANCE_POLICY, DEFAULT_SANDBOX_POLICY, assertNativeProfile, osProfile, validateDeclarativePolicy, } from "./runtime/index.js";
|
|
1
|
+
export { RuntimeRunner, collectText, InMemorySessionLog, LocalExecutionPlane, DEFAULT_NATIVE_ATTENTION_POLICY, DEFAULT_NATIVE_GOVERNANCE_POLICY, DEFAULT_SANDBOX_POLICY, assertNativeProfile, osProfile, validateDeclarativePolicy, ReplayProvider, extractRecordedMessages, judge, buildEvalMessages, parseVerdict, verdictOutputSchema, } from "./runtime/index.js";
|
|
2
|
+
export type { ReplayProviderOpts, Criterion, Verdict, VerdictDetail, JudgeArgs, } from "./runtime/index.js";
|
|
2
3
|
export type { NativeOsProfile, OsProfileId, MemoryPolicy, MemoryWriteRateLimit, ResourceQuota, RuntimeOptions, SchedulerBudget, SessionEvent, SessionLog, RunContext, ExecutionPlane, } from "./runtime/index.js";
|
|
3
4
|
export { FilteredExecutionPlane } from "./runtime/filtered-plane.js";
|
|
4
5
|
export { SubAgentOrchestrator, defaultSubAgentOrchestrator, spawnStandalone } from "./runtime/sub-agent-orchestrator.js";
|
package/dist/index.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
export { RuntimeRunner, collectText, InMemorySessionLog, LocalExecutionPlane, DEFAULT_NATIVE_ATTENTION_POLICY, DEFAULT_NATIVE_GOVERNANCE_POLICY, DEFAULT_SANDBOX_POLICY, assertNativeProfile, osProfile, validateDeclarativePolicy, } from "./runtime/index.js";
|
|
1
|
+
export { RuntimeRunner, collectText, InMemorySessionLog, LocalExecutionPlane, DEFAULT_NATIVE_ATTENTION_POLICY, DEFAULT_NATIVE_GOVERNANCE_POLICY, DEFAULT_SANDBOX_POLICY, assertNativeProfile, osProfile, validateDeclarativePolicy, ReplayProvider, extractRecordedMessages, judge, buildEvalMessages, parseVerdict, verdictOutputSchema, } from "./runtime/index.js";
|
|
2
2
|
export { FilteredExecutionPlane } from "./runtime/filtered-plane.js";
|
|
3
3
|
export { SubAgentOrchestrator, defaultSubAgentOrchestrator, spawnStandalone } from "./runtime/sub-agent-orchestrator.js";
|
|
4
4
|
export { workflowSpecToKernel, workflowNodeSpecToKernel, submitWorkflowNodesToKernel, submitWorkflowToKernel, submitWorkflowNodesTool, startWorkflowTool, fanoutSynthesize, generateAndFilter, verifyRules } from "./runtime/types/agent.js";
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `judge()` — one-shot quality scoring against a goal + criteria using the kernel's `gen_eval`.
|
|
3
|
+
*
|
|
4
|
+
* Mirrors the Node SDK's `node/src/runtime/eval.ts`. WASM port differences:
|
|
5
|
+
* - The kernel is loaded asynchronously via `getKernel()`, so the wrapper functions are async
|
|
6
|
+
* where the Node versions are sync. `judge()` was already async on both ports.
|
|
7
|
+
*
|
|
8
|
+
* See node/src/runtime/eval.ts for the full design rationale.
|
|
9
|
+
*/
|
|
10
|
+
import type { LLMProvider, Message } from "../types.js";
|
|
11
|
+
export interface Criterion {
|
|
12
|
+
text: string;
|
|
13
|
+
required?: boolean;
|
|
14
|
+
weight?: number;
|
|
15
|
+
}
|
|
16
|
+
export interface VerdictDetail {
|
|
17
|
+
criterion: string;
|
|
18
|
+
passed: boolean;
|
|
19
|
+
score: number;
|
|
20
|
+
feedback: string;
|
|
21
|
+
}
|
|
22
|
+
export interface Verdict {
|
|
23
|
+
passed: boolean;
|
|
24
|
+
overallScore: number;
|
|
25
|
+
feedback: string;
|
|
26
|
+
details: VerdictDetail[];
|
|
27
|
+
}
|
|
28
|
+
export interface JudgeArgs {
|
|
29
|
+
provider: LLMProvider;
|
|
30
|
+
goal: string;
|
|
31
|
+
criteria: Criterion[];
|
|
32
|
+
result: string;
|
|
33
|
+
signal?: AbortSignal;
|
|
34
|
+
}
|
|
35
|
+
/** Build the kernel's eval prompt for (goal, criteria, result). Async — the WASM kernel loads lazily. */
|
|
36
|
+
export declare function buildEvalMessages(goal: string, criteria: Criterion[], result: string): Promise<Message[]>;
|
|
37
|
+
/** Parse a Verdict from raw judge-LLM text. */
|
|
38
|
+
export declare function parseVerdict(text: string): Promise<Verdict>;
|
|
39
|
+
/** The JSON Schema the kernel expects judge output to conform to. */
|
|
40
|
+
export declare function verdictOutputSchema(): Promise<Record<string, unknown>>;
|
|
41
|
+
/**
|
|
42
|
+
* Run one judge pass: render the eval prompt, stream the provider, parse the verdict.
|
|
43
|
+
* Throws when the provider returns no text.
|
|
44
|
+
*/
|
|
45
|
+
export declare function judge(args: JudgeArgs): Promise<Verdict>;
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `judge()` — one-shot quality scoring against a goal + criteria using the kernel's `gen_eval`.
|
|
3
|
+
*
|
|
4
|
+
* Mirrors the Node SDK's `node/src/runtime/eval.ts`. WASM port differences:
|
|
5
|
+
* - The kernel is loaded asynchronously via `getKernel()`, so the wrapper functions are async
|
|
6
|
+
* where the Node versions are sync. `judge()` was already async on both ports.
|
|
7
|
+
*
|
|
8
|
+
* See node/src/runtime/eval.ts for the full design rationale.
|
|
9
|
+
*/
|
|
10
|
+
import { getKernel } from "./kernel.js";
|
|
11
|
+
/** Build the kernel's eval prompt for (goal, criteria, result). Async — the WASM kernel loads lazily. */
|
|
12
|
+
export async function buildEvalMessages(goal, criteria, result) {
|
|
13
|
+
const kernel = await getKernel();
|
|
14
|
+
return kernel.buildEvalMessages(goal, criteria.map(c => ({ text: c.text, required: c.required ?? true, weight: c.weight })), result, 1, false);
|
|
15
|
+
}
|
|
16
|
+
/** Parse a Verdict from raw judge-LLM text. */
|
|
17
|
+
export async function parseVerdict(text) {
|
|
18
|
+
const kernel = await getKernel();
|
|
19
|
+
const v = kernel.parseVerdict(text);
|
|
20
|
+
return {
|
|
21
|
+
passed: v.passed,
|
|
22
|
+
overallScore: v.overallScore,
|
|
23
|
+
feedback: v.feedback,
|
|
24
|
+
details: (v.details ?? []),
|
|
25
|
+
};
|
|
26
|
+
}
|
|
27
|
+
/** The JSON Schema the kernel expects judge output to conform to. */
|
|
28
|
+
export async function verdictOutputSchema() {
|
|
29
|
+
const kernel = await getKernel();
|
|
30
|
+
return JSON.parse(kernel.verdictOutputSchema(false));
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* Run one judge pass: render the eval prompt, stream the provider, parse the verdict.
|
|
34
|
+
* Throws when the provider returns no text.
|
|
35
|
+
*/
|
|
36
|
+
export async function judge(args) {
|
|
37
|
+
const msgs = await buildEvalMessages(args.goal, args.criteria, args.result);
|
|
38
|
+
const ctx = {
|
|
39
|
+
systemText: msgs.filter(m => m.role === "system").map(m => m.content).join("\n\n"),
|
|
40
|
+
turns: msgs.filter(m => m.role !== "system"),
|
|
41
|
+
};
|
|
42
|
+
let text = "";
|
|
43
|
+
for await (const evt of args.provider.stream(ctx, [], undefined, undefined, args.signal)) {
|
|
44
|
+
if (evt.type === "text_delta")
|
|
45
|
+
text += evt.delta;
|
|
46
|
+
}
|
|
47
|
+
if (!text) {
|
|
48
|
+
throw new Error("judge: provider produced no text");
|
|
49
|
+
}
|
|
50
|
+
return parseVerdict(text);
|
|
51
|
+
}
|
package/dist/runtime/index.d.ts
CHANGED
|
@@ -7,5 +7,10 @@ export { RuntimeRunner, collectText } from "./runner.js";
|
|
|
7
7
|
export { builtinReducers, resolveReducer } from "./reducers.js";
|
|
8
8
|
export type { Reducer, ReducerRegistry, ReducerInput } from "./reducers.js";
|
|
9
9
|
export { getKernel } from "./kernel.js";
|
|
10
|
+
export { ReplayProvider } from "./replay-provider.js";
|
|
11
|
+
export type { ReplayProviderOpts } from "./replay-provider.js";
|
|
12
|
+
export { extractRecordedMessages } from "./replay-fixture.js";
|
|
13
|
+
export { judge, buildEvalMessages, parseVerdict, verdictOutputSchema } from "./eval.js";
|
|
14
|
+
export type { Criterion, Verdict, VerdictDetail, JudgeArgs } from "./eval.js";
|
|
10
15
|
export { DEFAULT_NATIVE_ATTENTION_POLICY, DEFAULT_NATIVE_GOVERNANCE_POLICY, DEFAULT_SANDBOX_POLICY, assertNativeProfile, osProfile, validateDeclarativePolicy, } from "./os-profile.js";
|
|
11
16
|
export type { NativeOsProfile, OsProfileId } from "./os-profile.js";
|
package/dist/runtime/index.js
CHANGED
|
@@ -3,4 +3,7 @@ export { LocalExecutionPlane } from "./execution-plane.js";
|
|
|
3
3
|
export { RuntimeRunner, collectText } from "./runner.js";
|
|
4
4
|
export { builtinReducers, resolveReducer } from "./reducers.js";
|
|
5
5
|
export { getKernel } from "./kernel.js";
|
|
6
|
+
export { ReplayProvider } from "./replay-provider.js";
|
|
7
|
+
export { extractRecordedMessages } from "./replay-fixture.js";
|
|
8
|
+
export { judge, buildEvalMessages, parseVerdict, verdictOutputSchema } from "./eval.js";
|
|
6
9
|
export { DEFAULT_NATIVE_ATTENTION_POLICY, DEFAULT_NATIVE_GOVERNANCE_POLICY, DEFAULT_SANDBOX_POLICY, assertNativeProfile, osProfile, validateDeclarativePolicy, } from "./os-profile.js";
|
|
@@ -14,6 +14,9 @@ interface SkillMetadata {
|
|
|
14
14
|
whenToUse?: string;
|
|
15
15
|
effort?: number;
|
|
16
16
|
estimatedTokens?: number;
|
|
17
|
+
/** P1-B tool gating: tool ids this skill needs; when active the kernel narrows the toolset to
|
|
18
|
+
* `stable-core ∪ allowedTools`. Absent ⇒ no narrowing (back-compat). */
|
|
19
|
+
allowedTools?: string[];
|
|
17
20
|
}
|
|
18
21
|
export declare const KERNEL_ABI_VERSION = 1;
|
|
19
22
|
export interface KernelRuntimeHandle {
|
|
@@ -24,6 +24,9 @@ export function skillMetadataToKernel(skill) {
|
|
|
24
24
|
out.when_to_use = skill.whenToUse;
|
|
25
25
|
if (skill.effort !== undefined)
|
|
26
26
|
out.effort = skill.effort;
|
|
27
|
+
// P1-B: forward declared tool ids (additive; omitted when empty so existing skills' wire is unchanged).
|
|
28
|
+
if (skill.allowedTools?.length)
|
|
29
|
+
out.allowed_tools = skill.allowedTools;
|
|
27
30
|
return out;
|
|
28
31
|
}
|
|
29
32
|
export function messageToKernelMessage(message) {
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Fixture helpers for `ReplayProvider`.
|
|
3
|
+
*
|
|
4
|
+
* The canonical persistence shape for a recorded run is a `SessionLog` of `llm_completed` events
|
|
5
|
+
* (already written by the runner on every live run). `extractRecordedMessages` walks such a log and
|
|
6
|
+
* pulls the assistant turns in order, so the fixture is just "a prior session log + the messages
|
|
7
|
+
* the LLM produced". No new on-disk format.
|
|
8
|
+
*/
|
|
9
|
+
import type { Message } from "../types.js";
|
|
10
|
+
import type { SessionEvent } from "./session-log.js";
|
|
11
|
+
/**
|
|
12
|
+
* Extract the ordered list of assistant Messages from a recorded session log.
|
|
13
|
+
*
|
|
14
|
+
* Walks `llm_completed` events (which is what the runner appends for every LLM call) and produces
|
|
15
|
+
* one Message per event. Pass the result directly to `new ReplayProvider(messages)`.
|
|
16
|
+
*
|
|
17
|
+
* Accepts both wire shapes the SDK uses interchangeably:
|
|
18
|
+
* - in-memory: `{ toolCalls, tokenCount, providerReplay }` (camelCase)
|
|
19
|
+
* - serialised session-log: `{ tool_calls, token_count, provider_replay }` (snake_case)
|
|
20
|
+
*
|
|
21
|
+
* @param events Session events, in original order. Accepts both `{ event, seq }` (the shape
|
|
22
|
+
* `SessionLog.read()` returns) and a bare `SessionEvent[]`.
|
|
23
|
+
*/
|
|
24
|
+
export declare function extractRecordedMessages(events: Array<{
|
|
25
|
+
event: SessionEvent;
|
|
26
|
+
} | SessionEvent>): Message[];
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Fixture helpers for `ReplayProvider`.
|
|
3
|
+
*
|
|
4
|
+
* The canonical persistence shape for a recorded run is a `SessionLog` of `llm_completed` events
|
|
5
|
+
* (already written by the runner on every live run). `extractRecordedMessages` walks such a log and
|
|
6
|
+
* pulls the assistant turns in order, so the fixture is just "a prior session log + the messages
|
|
7
|
+
* the LLM produced". No new on-disk format.
|
|
8
|
+
*/
|
|
9
|
+
/**
|
|
10
|
+
* Extract the ordered list of assistant Messages from a recorded session log.
|
|
11
|
+
*
|
|
12
|
+
* Walks `llm_completed` events (which is what the runner appends for every LLM call) and produces
|
|
13
|
+
* one Message per event. Pass the result directly to `new ReplayProvider(messages)`.
|
|
14
|
+
*
|
|
15
|
+
* Accepts both wire shapes the SDK uses interchangeably:
|
|
16
|
+
* - in-memory: `{ toolCalls, tokenCount, providerReplay }` (camelCase)
|
|
17
|
+
* - serialised session-log: `{ tool_calls, token_count, provider_replay }` (snake_case)
|
|
18
|
+
*
|
|
19
|
+
* @param events Session events, in original order. Accepts both `{ event, seq }` (the shape
|
|
20
|
+
* `SessionLog.read()` returns) and a bare `SessionEvent[]`.
|
|
21
|
+
*/
|
|
22
|
+
export function extractRecordedMessages(events) {
|
|
23
|
+
const out = [];
|
|
24
|
+
for (const entry of events) {
|
|
25
|
+
const event = isWrapped(entry) ? entry.event : entry;
|
|
26
|
+
if (event.kind !== "llm_completed")
|
|
27
|
+
continue;
|
|
28
|
+
const e = event;
|
|
29
|
+
const tcRaw = (e.toolCalls ?? e.tool_calls);
|
|
30
|
+
const tokenCount = (e.tokenCount ?? e.token_count);
|
|
31
|
+
out.push({
|
|
32
|
+
role: "assistant",
|
|
33
|
+
content: typeof e.content === "string" ? e.content : "",
|
|
34
|
+
...(Array.isArray(tcRaw) && tcRaw.length > 0
|
|
35
|
+
? { toolCalls: normalizeToolCalls(tcRaw) }
|
|
36
|
+
: {}),
|
|
37
|
+
...(tokenCount !== undefined ? { tokenCount } : {}),
|
|
38
|
+
});
|
|
39
|
+
}
|
|
40
|
+
return out;
|
|
41
|
+
}
|
|
42
|
+
function isWrapped(x) {
|
|
43
|
+
return !!x && typeof x === "object" && "event" in x;
|
|
44
|
+
}
|
|
45
|
+
function normalizeToolCalls(tcs) {
|
|
46
|
+
return tcs.map(tc => ({
|
|
47
|
+
id: tc.id,
|
|
48
|
+
name: tc.name,
|
|
49
|
+
arguments: typeof tc.arguments === "string" ? tc.arguments : JSON.stringify(tc.arguments),
|
|
50
|
+
}));
|
|
51
|
+
}
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* ReplayProvider — an LLMProvider that emits previously-recorded assistant messages
|
|
3
|
+
* instead of calling a real LLM API.
|
|
4
|
+
*
|
|
5
|
+
* Purpose: deterministic re-runs for benchmarking, CI, and golden regression. Useful when you want
|
|
6
|
+
* to hold the model's behavior constant and measure something else (prompt-size cost Δ across
|
|
7
|
+
* RuntimeOptions variants, codegen of follow-on kernel work, etc.).
|
|
8
|
+
*
|
|
9
|
+
* Distinct from `provider-replay.ts`: that file's `seedProviderReplay` / `peekProviderReplay` is a
|
|
10
|
+
* session-repair *reasoning-content cache* — it preserves `reasoning_content` / `native_blocks` so
|
|
11
|
+
* the model sees its own thinking when context is re-rendered. It does NOT skip LLM calls.
|
|
12
|
+
* `ReplayProvider` is the orthogonal, request-skipping mechanism: it returns recorded responses
|
|
13
|
+
* directly, never hitting an API.
|
|
14
|
+
*
|
|
15
|
+
* Cost-accounting under replay:
|
|
16
|
+
* - `inputTokens` is ESTIMATED from the rendered context this call carries (NOT a recorded value
|
|
17
|
+
* from the original run). That's the point of replay-for-benchmarking: prompt may differ across
|
|
18
|
+
* variants, response is pinned, so a cost Δ purely reflects the prompt change.
|
|
19
|
+
* - `outputTokens` is taken from `message.tokenCount` when present; otherwise estimated from
|
|
20
|
+
* `message.content.length / 4`.
|
|
21
|
+
* - `cacheReadInputTokens` / `cacheCreationInputTokens` are emitted as 0 — replay has no real
|
|
22
|
+
* cache state. Mechanisms whose Δ depends on cache behavior must validate with a live A/B too.
|
|
23
|
+
*
|
|
24
|
+
* Tokenizer: by default a `chars/4` estimator (±20% for English; worse for code/JSON). For tighter
|
|
25
|
+
* numbers plug `opts.tokenizer = tiktokenEncoder` or similar.
|
|
26
|
+
*/
|
|
27
|
+
import type { LLMProvider, Message, ProviderDescriptor, ProviderRunState, RenderedContext, StreamEvent, ToolSchema } from "../types.js";
|
|
28
|
+
export interface ReplayProviderOpts {
|
|
29
|
+
/**
|
|
30
|
+
* Maps a rendered-context text payload to a token count. Defaults to `chars / 4`.
|
|
31
|
+
* Pass a real encoder (tiktoken etc.) for accurate cost accounting under replay.
|
|
32
|
+
*/
|
|
33
|
+
tokenizer?: (text: string) => number;
|
|
34
|
+
/**
|
|
35
|
+
* Provider descriptor advertised via `descriptor()`. Defaults to a generic
|
|
36
|
+
* `{ provider: "replay", protocol: "replay", ... }` shape. Override when a downstream consumer
|
|
37
|
+
* needs to detect the original provider (e.g., for protocol-specific decoding paths).
|
|
38
|
+
*/
|
|
39
|
+
descriptor?: ProviderDescriptor;
|
|
40
|
+
/**
|
|
41
|
+
* When true, `stream()` and `complete()` wrap to the start once the fixture is exhausted,
|
|
42
|
+
* instead of throwing. Useful for loop tests that need to keep going past the recorded length.
|
|
43
|
+
* Defaults to false.
|
|
44
|
+
*/
|
|
45
|
+
wrap?: boolean;
|
|
46
|
+
}
|
|
47
|
+
export declare class ReplayProvider implements LLMProvider {
|
|
48
|
+
private cursor;
|
|
49
|
+
private readonly messages;
|
|
50
|
+
private readonly tokenizer;
|
|
51
|
+
private readonly _descriptor;
|
|
52
|
+
private readonly wrap;
|
|
53
|
+
/**
|
|
54
|
+
* @param messages Ordered list of assistant messages to replay (one per LLM call).
|
|
55
|
+
* @param opts Optional tokenizer / descriptor / wrap-around behavior.
|
|
56
|
+
*/
|
|
57
|
+
constructor(messages: ReadonlyArray<Message>, opts?: ReplayProviderOpts);
|
|
58
|
+
descriptor(): ProviderDescriptor;
|
|
59
|
+
/** Number of messages consumed so far. */
|
|
60
|
+
consumed(): number;
|
|
61
|
+
/** Number of messages remaining in the fixture (returns 0 in wrap mode once cursor passes end). */
|
|
62
|
+
remaining(): number;
|
|
63
|
+
/** Reset the cursor — useful for re-running the same fixture in a fresh session. */
|
|
64
|
+
reset(): void;
|
|
65
|
+
complete(_context: RenderedContext, _tools: ToolSchema[]): Promise<Message>;
|
|
66
|
+
stream(context: RenderedContext, tools: ToolSchema[], _extensions?: Record<string, unknown>, _state?: ProviderRunState, _signal?: AbortSignal): AsyncIterable<StreamEvent>;
|
|
67
|
+
private pull;
|
|
68
|
+
private estimateInputTokens;
|
|
69
|
+
}
|
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* ReplayProvider — an LLMProvider that emits previously-recorded assistant messages
|
|
3
|
+
* instead of calling a real LLM API.
|
|
4
|
+
*
|
|
5
|
+
* Purpose: deterministic re-runs for benchmarking, CI, and golden regression. Useful when you want
|
|
6
|
+
* to hold the model's behavior constant and measure something else (prompt-size cost Δ across
|
|
7
|
+
* RuntimeOptions variants, codegen of follow-on kernel work, etc.).
|
|
8
|
+
*
|
|
9
|
+
* Distinct from `provider-replay.ts`: that file's `seedProviderReplay` / `peekProviderReplay` is a
|
|
10
|
+
* session-repair *reasoning-content cache* — it preserves `reasoning_content` / `native_blocks` so
|
|
11
|
+
* the model sees its own thinking when context is re-rendered. It does NOT skip LLM calls.
|
|
12
|
+
* `ReplayProvider` is the orthogonal, request-skipping mechanism: it returns recorded responses
|
|
13
|
+
* directly, never hitting an API.
|
|
14
|
+
*
|
|
15
|
+
* Cost-accounting under replay:
|
|
16
|
+
* - `inputTokens` is ESTIMATED from the rendered context this call carries (NOT a recorded value
|
|
17
|
+
* from the original run). That's the point of replay-for-benchmarking: prompt may differ across
|
|
18
|
+
* variants, response is pinned, so a cost Δ purely reflects the prompt change.
|
|
19
|
+
* - `outputTokens` is taken from `message.tokenCount` when present; otherwise estimated from
|
|
20
|
+
* `message.content.length / 4`.
|
|
21
|
+
* - `cacheReadInputTokens` / `cacheCreationInputTokens` are emitted as 0 — replay has no real
|
|
22
|
+
* cache state. Mechanisms whose Δ depends on cache behavior must validate with a live A/B too.
|
|
23
|
+
*
|
|
24
|
+
* Tokenizer: by default a `chars/4` estimator (±20% for English; worse for code/JSON). For tighter
|
|
25
|
+
* numbers plug `opts.tokenizer = tiktokenEncoder` or similar.
|
|
26
|
+
*/
|
|
27
|
+
const DEFAULT_DESCRIPTOR = {
|
|
28
|
+
provider: "replay",
|
|
29
|
+
protocol: "openai-chat",
|
|
30
|
+
model: "replay",
|
|
31
|
+
reasoning: { supported: false, preserveAcrossToolTurns: false },
|
|
32
|
+
toolCalls: { supported: true, requiresStrictPairing: false },
|
|
33
|
+
};
|
|
34
|
+
export class ReplayProvider {
|
|
35
|
+
cursor = 0;
|
|
36
|
+
messages;
|
|
37
|
+
tokenizer;
|
|
38
|
+
_descriptor;
|
|
39
|
+
wrap;
|
|
40
|
+
/**
|
|
41
|
+
* @param messages Ordered list of assistant messages to replay (one per LLM call).
|
|
42
|
+
* @param opts Optional tokenizer / descriptor / wrap-around behavior.
|
|
43
|
+
*/
|
|
44
|
+
constructor(messages, opts = {}) {
|
|
45
|
+
this.messages = messages;
|
|
46
|
+
this.tokenizer = opts.tokenizer ?? defaultTokenizer;
|
|
47
|
+
this._descriptor = opts.descriptor ?? DEFAULT_DESCRIPTOR;
|
|
48
|
+
this.wrap = !!opts.wrap;
|
|
49
|
+
}
|
|
50
|
+
descriptor() {
|
|
51
|
+
return this._descriptor;
|
|
52
|
+
}
|
|
53
|
+
/** Number of messages consumed so far. */
|
|
54
|
+
consumed() {
|
|
55
|
+
return this.cursor;
|
|
56
|
+
}
|
|
57
|
+
/** Number of messages remaining in the fixture (returns 0 in wrap mode once cursor passes end). */
|
|
58
|
+
remaining() {
|
|
59
|
+
return Math.max(0, this.messages.length - this.cursor);
|
|
60
|
+
}
|
|
61
|
+
/** Reset the cursor — useful for re-running the same fixture in a fresh session. */
|
|
62
|
+
reset() {
|
|
63
|
+
this.cursor = 0;
|
|
64
|
+
}
|
|
65
|
+
async complete(_context, _tools) {
|
|
66
|
+
const msg = this.pull();
|
|
67
|
+
return {
|
|
68
|
+
role: "assistant",
|
|
69
|
+
content: msg.content,
|
|
70
|
+
...(msg.toolCalls ? { toolCalls: msg.toolCalls } : {}),
|
|
71
|
+
...(msg.tokenCount !== undefined ? { tokenCount: msg.tokenCount } : {}),
|
|
72
|
+
};
|
|
73
|
+
}
|
|
74
|
+
async *stream(context, tools, _extensions, _state, _signal) {
|
|
75
|
+
const msg = this.pull();
|
|
76
|
+
const inputTokens = this.estimateInputTokens(context, tools);
|
|
77
|
+
const outputTokens = msg.tokenCount !== undefined ? msg.tokenCount : this.tokenizer(msg.content || "");
|
|
78
|
+
const usage = {
|
|
79
|
+
type: "usage",
|
|
80
|
+
totalTokens: inputTokens + outputTokens,
|
|
81
|
+
inputTokens,
|
|
82
|
+
outputTokens,
|
|
83
|
+
cacheReadInputTokens: 0,
|
|
84
|
+
cacheCreationInputTokens: 0,
|
|
85
|
+
};
|
|
86
|
+
yield usage;
|
|
87
|
+
if (msg.content) {
|
|
88
|
+
const delta = { type: "text_delta", delta: msg.content };
|
|
89
|
+
yield delta;
|
|
90
|
+
}
|
|
91
|
+
for (const tc of msg.toolCalls ?? []) {
|
|
92
|
+
let args = {};
|
|
93
|
+
try {
|
|
94
|
+
args = JSON.parse(tc.arguments || "{}");
|
|
95
|
+
}
|
|
96
|
+
catch {
|
|
97
|
+
// Malformed recorded arguments — pass an empty object. The runner's downstream tool
|
|
98
|
+
// execution will surface the error if the tool needs them.
|
|
99
|
+
}
|
|
100
|
+
const call = { type: "tool_call", id: tc.id, name: tc.name, arguments: args };
|
|
101
|
+
yield call;
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
pull() {
|
|
105
|
+
if (this.cursor >= this.messages.length) {
|
|
106
|
+
if (this.wrap && this.messages.length > 0) {
|
|
107
|
+
this.cursor = 0;
|
|
108
|
+
}
|
|
109
|
+
else {
|
|
110
|
+
throw new Error(`ReplayProvider: fixture exhausted (consumed=${this.cursor}, total=${this.messages.length})`);
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
return this.messages[this.cursor++];
|
|
114
|
+
}
|
|
115
|
+
estimateInputTokens(context, tools) {
|
|
116
|
+
const text = renderContextToText(context, tools);
|
|
117
|
+
return this.tokenizer(text);
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
// ── helpers ───────────────────────────────────────────────────────────────────
|
|
121
|
+
function defaultTokenizer(text) {
|
|
122
|
+
return Math.ceil(text.length / 4);
|
|
123
|
+
}
|
|
124
|
+
function renderContextToText(context, tools) {
|
|
125
|
+
const parts = [];
|
|
126
|
+
if (context.systemText)
|
|
127
|
+
parts.push(context.systemText);
|
|
128
|
+
if (context.systemStable)
|
|
129
|
+
parts.push(context.systemStable);
|
|
130
|
+
if (context.systemKnowledge)
|
|
131
|
+
parts.push(context.systemKnowledge);
|
|
132
|
+
if (context.stateTurn?.content)
|
|
133
|
+
parts.push(context.stateTurn.content);
|
|
134
|
+
for (const turn of context.turns ?? []) {
|
|
135
|
+
if (turn.content)
|
|
136
|
+
parts.push(turn.content);
|
|
137
|
+
for (const part of turn.contentParts ?? []) {
|
|
138
|
+
const p = part;
|
|
139
|
+
if (typeof p.output === "string")
|
|
140
|
+
parts.push(p.output);
|
|
141
|
+
else if (typeof p.text === "string")
|
|
142
|
+
parts.push(p.text);
|
|
143
|
+
}
|
|
144
|
+
for (const tc of turn.toolCalls ?? []) {
|
|
145
|
+
parts.push(`${tc.name} ${tc.arguments}`);
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
for (const tool of tools) {
|
|
149
|
+
parts.push(`${tool.name} ${tool.description} ${tool.parameters}`);
|
|
150
|
+
}
|
|
151
|
+
return parts.join("\n");
|
|
152
|
+
}
|
package/dist/runtime/runner.d.ts
CHANGED
|
@@ -40,6 +40,19 @@ export interface MemoryPolicy {
|
|
|
40
40
|
maxContentBytes?: number;
|
|
41
41
|
maxNameLength?: number;
|
|
42
42
|
}
|
|
43
|
+
/** P0-C tool-gating telemetry: per-LLM-turn metrics, emitted via `RuntimeOptions.onTurnMetrics`.
|
|
44
|
+
* Pure observation — no behavior change. `toolsExposed` vs `toolsCalled` quantifies over-exposure;
|
|
45
|
+
* consecutive equal `activeSkill` values measure skill dwell `D`; the cache split gives the
|
|
46
|
+
* prompt-cache hit baseline. Mirrors the node SDK. */
|
|
47
|
+
export interface TurnMetrics {
|
|
48
|
+
turn: number;
|
|
49
|
+
toolsExposed: number;
|
|
50
|
+
toolsCalled: number;
|
|
51
|
+
activeSkill?: string;
|
|
52
|
+
inputTokens: number;
|
|
53
|
+
cacheReadTokens: number;
|
|
54
|
+
cacheCreationTokens: number;
|
|
55
|
+
}
|
|
43
56
|
export interface RuntimeOptions {
|
|
44
57
|
provider: LLMProvider;
|
|
45
58
|
/** M1/G3 intelligence routing: resolve a per-node provider from a workflow node's `modelHint`.
|
|
@@ -90,6 +103,18 @@ export interface RuntimeOptions {
|
|
|
90
103
|
requiredEvidence: string[];
|
|
91
104
|
}) => Promise<MilestoneCheckResult> | MilestoneCheckResult;
|
|
92
105
|
runSpec?: AgentRunSpec;
|
|
106
|
+
/** P0-A tool gating: a static per-run tool profile — only these tool ids (plus the meta-tools)
|
|
107
|
+
* are exposed to the model each turn. Lowers to the same `capability_filter` sub-agents use;
|
|
108
|
+
* byte-stable across the run, so it never busts the prompt-cache prefix. Augments `runSpec`'s
|
|
109
|
+
* filter when both set; synthesizes a minimal spec otherwise. Omitted/empty ⇒ no gating. */
|
|
110
|
+
allowedToolIds?: string[];
|
|
111
|
+
/** P0-C: optional per-turn metrics sink for tool-gating telemetry (see `TurnMetrics`). Pure
|
|
112
|
+
* observation; invoked once per LLM turn. Never throws into the run loop (errors are swallowed). */
|
|
113
|
+
onTurnMetrics?: (metrics: TurnMetrics) => void;
|
|
114
|
+
/** P1-B/D stable-core: tool ids always exposed under skill gating. Empty/absent ⇒ skills narrow
|
|
115
|
+
* to exactly their declared tools + meta-tools. (wasm skills come from `skillContentMap`; gating
|
|
116
|
+
* engages only once that carries per-skill tool lists.) */
|
|
117
|
+
stableCoreToolIds?: string[];
|
|
93
118
|
dreamProvider?: LLMProvider;
|
|
94
119
|
dreamSummarizer?: DreamSummarizer;
|
|
95
120
|
dreamSystemPrompt?: string;
|
package/dist/runtime/runner.js
CHANGED
|
@@ -313,6 +313,13 @@ export class RuntimeRunner {
|
|
|
313
313
|
skills: metas.map(skillMetadataToKernel),
|
|
314
314
|
});
|
|
315
315
|
}
|
|
316
|
+
// P1-B/D: configure stable-core tool ids (always exposed under skill gating).
|
|
317
|
+
if (this.opts.stableCoreToolIds?.length) {
|
|
318
|
+
kernelApply(runtime, this.pendingObservations, {
|
|
319
|
+
kind: "set_stable_core_tools",
|
|
320
|
+
tool_ids: this.opts.stableCoreToolIds,
|
|
321
|
+
});
|
|
322
|
+
}
|
|
316
323
|
if (this.opts.dreamStore && this.opts.agentId) {
|
|
317
324
|
kernelApply(runtime, this.pendingObservations, { kind: "set_memory_enabled", enabled: true });
|
|
318
325
|
}
|
|
@@ -337,18 +344,45 @@ export class RuntimeRunner {
|
|
|
337
344
|
if (priorEvents && priorEvents.length > 0) {
|
|
338
345
|
const repaired = repairEventsForRecovery(priorEvents, maxBytes);
|
|
339
346
|
seedProviderReplayFromEvents(this.opts.provider, repaired);
|
|
347
|
+
const replayed = replayMessages(repaired, maxBytes);
|
|
340
348
|
kernelApply(runtime, this.pendingObservations, {
|
|
341
349
|
kind: "preload_history",
|
|
342
|
-
messages:
|
|
350
|
+
messages: replayed.map(messageToKernelMessage),
|
|
343
351
|
});
|
|
352
|
+
// P1-B B3: rebuild active-skill gating after a wake (active_skills is not snapshotted).
|
|
353
|
+
for (const m of replayed) {
|
|
354
|
+
for (const tc of m.toolCalls ?? []) {
|
|
355
|
+
if (tc.name !== "skill")
|
|
356
|
+
continue;
|
|
357
|
+
try {
|
|
358
|
+
const name = JSON.parse(tc.arguments || "{}").name;
|
|
359
|
+
if (name)
|
|
360
|
+
kernelApply(runtime, this.pendingObservations, { kind: "skill_activated", name });
|
|
361
|
+
}
|
|
362
|
+
catch { /* skip */ }
|
|
363
|
+
}
|
|
364
|
+
}
|
|
344
365
|
}
|
|
345
366
|
const sessionStart = Date.now();
|
|
346
367
|
const startPayload = {
|
|
347
368
|
kind: "start_run",
|
|
348
369
|
task: { goal, criteria },
|
|
349
370
|
};
|
|
350
|
-
|
|
351
|
-
|
|
371
|
+
// P0-A: lower an explicit `runSpec` and/or the `allowedToolIds` profile to the kernel's
|
|
372
|
+
// `capability_filter` (reuses the existing run_spec wire — no new ABI). Unset on both ⇒ no
|
|
373
|
+
// gating (铁律: no config = old behavior).
|
|
374
|
+
const allowedToolIds = this.opts.allowedToolIds;
|
|
375
|
+
const hasProfile = allowedToolIds !== undefined && allowedToolIds.length > 0;
|
|
376
|
+
if (this.opts.runSpec || hasProfile) {
|
|
377
|
+
const baseSpec = this.opts.runSpec ?? {
|
|
378
|
+
identity: { agentId: this.opts.agentId ?? "root", sessionId, isSubAgent: false },
|
|
379
|
+
role: "custom",
|
|
380
|
+
goal,
|
|
381
|
+
};
|
|
382
|
+
const spec = hasProfile
|
|
383
|
+
? { ...baseSpec, capabilityFilter: { ...baseSpec.capabilityFilter, allowedIds: allowedToolIds } }
|
|
384
|
+
: baseSpec;
|
|
385
|
+
startPayload.run_spec = agentRunSpecToKernel(spec);
|
|
352
386
|
}
|
|
353
387
|
const osProfile = assertNativeProfile(this.opts.osProfile ?? "native");
|
|
354
388
|
const attentionPolicy = this.opts.attentionPolicy ?? osProfile.attentionPolicy;
|
|
@@ -404,6 +438,8 @@ export class RuntimeRunner {
|
|
|
404
438
|
? kernelAction(runtime, this.pendingObservations, { kind: "resume" })
|
|
405
439
|
: kernelAction(runtime, this.pendingObservations, startPayload);
|
|
406
440
|
let hasAttemptedReactiveCompact = false;
|
|
441
|
+
// P0-C: the skill loaded and in effect going into the current turn → per-turn `activeSkill` metric.
|
|
442
|
+
let activeSkill;
|
|
407
443
|
while (!runtime.isTerminal()) {
|
|
408
444
|
if (action.kind === "execute_tool") {
|
|
409
445
|
await this.applyKernelPageIn(runtime, sessionId);
|
|
@@ -437,6 +473,8 @@ export class RuntimeRunner {
|
|
|
437
473
|
let turnTokens = 0;
|
|
438
474
|
let turnInputTokens = 0;
|
|
439
475
|
let turnOutputTokens = 0;
|
|
476
|
+
let turnCacheReadTokens = 0;
|
|
477
|
+
let turnCacheCreationTokens = 0;
|
|
440
478
|
let shouldRetry = false;
|
|
441
479
|
const abortSignal = this.abortController?.signal;
|
|
442
480
|
try {
|
|
@@ -449,6 +487,9 @@ export class RuntimeRunner {
|
|
|
449
487
|
turnTokens = usageEvt.totalTokens;
|
|
450
488
|
turnInputTokens = usageEvt.inputTokens ?? 0;
|
|
451
489
|
turnOutputTokens = usageEvt.outputTokens ?? 0;
|
|
490
|
+
// P0-C: capture the prompt-cache split for the tool-gating hit-rate baseline.
|
|
491
|
+
turnCacheReadTokens = usageEvt.cacheReadInputTokens ?? 0;
|
|
492
|
+
turnCacheCreationTokens = usageEvt.cacheCreationInputTokens ?? 0;
|
|
452
493
|
continue;
|
|
453
494
|
}
|
|
454
495
|
yield evt;
|
|
@@ -527,6 +568,31 @@ export class RuntimeRunner {
|
|
|
527
568
|
toolCalls: finalToolCalls,
|
|
528
569
|
providerReplay,
|
|
529
570
|
}));
|
|
571
|
+
// P0-C: per-turn tool-gating telemetry. `activeSkill` reflects the skill in effect GOING INTO
|
|
572
|
+
// this turn; a `skill` call here only takes effect next turn — emit first, then advance.
|
|
573
|
+
if (this.opts.onTurnMetrics) {
|
|
574
|
+
try {
|
|
575
|
+
this.opts.onTurnMetrics({
|
|
576
|
+
turn: runtime.turn(),
|
|
577
|
+
toolsExposed: tools.length,
|
|
578
|
+
toolsCalled: finalToolCalls.length,
|
|
579
|
+
activeSkill,
|
|
580
|
+
inputTokens: turnInputTokens,
|
|
581
|
+
cacheReadTokens: turnCacheReadTokens,
|
|
582
|
+
cacheCreationTokens: turnCacheCreationTokens,
|
|
583
|
+
});
|
|
584
|
+
}
|
|
585
|
+
catch { /* metrics must never break the run */ }
|
|
586
|
+
}
|
|
587
|
+
const skillCall = finalToolCalls.find(c => c.name === "skill");
|
|
588
|
+
if (skillCall) {
|
|
589
|
+
try {
|
|
590
|
+
const name = JSON.parse(skillCall.arguments || "{}").name;
|
|
591
|
+
if (name)
|
|
592
|
+
activeSkill = name;
|
|
593
|
+
}
|
|
594
|
+
catch { /* malformed skill args — leave activeSkill unchanged */ }
|
|
595
|
+
}
|
|
530
596
|
}
|
|
531
597
|
else if (action.kind === "execute_tool") {
|
|
532
598
|
const allCalls = action.calls;
|
|
@@ -637,6 +703,20 @@ export class RuntimeRunner {
|
|
|
637
703
|
this.pendingSpoolOutputs.set(call.id, { tool: call.name, output: result.output });
|
|
638
704
|
}
|
|
639
705
|
}
|
|
706
|
+
// P1-B B3: a successfully-resolved `skill` call activates that skill for the next turn.
|
|
707
|
+
for (const call of allCalls) {
|
|
708
|
+
if (call.name !== "skill")
|
|
709
|
+
continue;
|
|
710
|
+
const res = toolResults.find(r => r.callId === call.id);
|
|
711
|
+
if (!res || res.isError)
|
|
712
|
+
continue;
|
|
713
|
+
try {
|
|
714
|
+
const name = JSON.parse(call.arguments || "{}").name;
|
|
715
|
+
if (name)
|
|
716
|
+
kernelApply(runtime, this.pendingObservations, { kind: "skill_activated", name });
|
|
717
|
+
}
|
|
718
|
+
catch { /* skip */ }
|
|
719
|
+
}
|
|
640
720
|
action = kernelAction(runtime, this.pendingObservations, {
|
|
641
721
|
kind: "tool_results",
|
|
642
722
|
results: toolResults.map(toolResultToKernel),
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@deepstrike/wasm",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.20",
|
|
4
4
|
"description": "DeepStrike WASM SDK — browser, Cloudflare Workers, Deno Deploy",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "dist/index.js",
|
|
@@ -15,7 +15,7 @@
|
|
|
15
15
|
"test": "node --experimental-vm-modules node_modules/.bin/jest"
|
|
16
16
|
},
|
|
17
17
|
"dependencies": {
|
|
18
|
-
"@deepstrike/wasm-kernel": "0.2.
|
|
18
|
+
"@deepstrike/wasm-kernel": "0.2.20"
|
|
19
19
|
},
|
|
20
20
|
"devDependencies": {
|
|
21
21
|
"@types/jest": "^30.0.0",
|