@deepstrike/sdk 0.2.19 → 0.2.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +6 -0
- package/dist/index.js +4 -0
- package/dist/memory/in-memory-store.d.ts +37 -0
- package/dist/memory/in-memory-store.js +50 -0
- package/dist/runtime/eval.d.ts +60 -0
- package/dist/runtime/eval.js +55 -0
- package/dist/runtime/replay-fixture.d.ts +26 -0
- package/dist/runtime/replay-fixture.js +51 -0
- package/dist/runtime/replay-provider.d.ts +69 -0
- package/dist/runtime/replay-provider.js +152 -0
- package/package.json +2 -2
package/dist/index.d.ts
CHANGED
|
@@ -15,6 +15,11 @@ export { LocalExecutionPlane } from "./runtime/execution-plane.js";
|
|
|
15
15
|
export type { ExecutionPlane, RunContext } from "./runtime/execution-plane.js";
|
|
16
16
|
export { InMemorySessionLog, FileSessionLog } from "./runtime/session-log.js";
|
|
17
17
|
export type { SessionLog, SessionEvent } from "./runtime/session-log.js";
|
|
18
|
+
export { ReplayProvider } from "./runtime/replay-provider.js";
|
|
19
|
+
export type { ReplayProviderOpts } from "./runtime/replay-provider.js";
|
|
20
|
+
export { extractRecordedMessages } from "./runtime/replay-fixture.js";
|
|
21
|
+
export { judge, buildEvalMessages, parseVerdict, verdictOutputSchema } from "./runtime/eval.js";
|
|
22
|
+
export type { Criterion, Verdict, VerdictDetail, JudgeArgs } from "./runtime/eval.js";
|
|
18
23
|
export { DEFAULT_NATIVE_ATTENTION_POLICY, DEFAULT_NATIVE_GOVERNANCE_POLICY, assertNativeProfile, osProfile, } from "./runtime/os-profile.js";
|
|
19
24
|
export type { NativeOsProfile, OsProfileId } from "./runtime/os-profile.js";
|
|
20
25
|
export { rebuildOsSnapshotFromSessionEvents, sessionLogHasRequiredCategories, } from "./runtime/os-snapshot.js";
|
|
@@ -55,6 +60,7 @@ export type { RegisteredTool } from "./tools/index.js";
|
|
|
55
60
|
export { scanSkillDir, readSkillFile } from "./skills/loader.js";
|
|
56
61
|
export type { SkillMetadata } from "./skills/loader.js";
|
|
57
62
|
export { WorkingMemory } from "./memory/working.js";
|
|
63
|
+
export { InMemoryDreamStore } from "./memory/in-memory-store.js";
|
|
58
64
|
export type { DreamStore, DreamResult, SessionData, SessionMessage, MemoryEntry, CurationResult, CurationStats, MemoryWriteRequest, MemoryQuery, MemoryRetrieval, MemoryMetadata, MemoryKind, } from "./memory/protocols.js";
|
|
59
65
|
export type { KnowledgeSource } from "./knowledge/source.js";
|
|
60
66
|
export { ScheduledPrompt } from "./signals/scheduled.js";
|
package/dist/index.js
CHANGED
|
@@ -9,6 +9,9 @@ export { FilteredExecutionPlane } from "./runtime/filtered-plane.js";
|
|
|
9
9
|
export { SubAgentOrchestrator, defaultSubAgentOrchestrator, spawnStandalone } from "./runtime/sub-agent-orchestrator.js";
|
|
10
10
|
export { LocalExecutionPlane } from "./runtime/execution-plane.js";
|
|
11
11
|
export { InMemorySessionLog, FileSessionLog } from "./runtime/session-log.js";
|
|
12
|
+
export { ReplayProvider } from "./runtime/replay-provider.js";
|
|
13
|
+
export { extractRecordedMessages } from "./runtime/replay-fixture.js";
|
|
14
|
+
export { judge, buildEvalMessages, parseVerdict, verdictOutputSchema } from "./runtime/eval.js";
|
|
12
15
|
export { DEFAULT_NATIVE_ATTENTION_POLICY, DEFAULT_NATIVE_GOVERNANCE_POLICY, assertNativeProfile, osProfile, } from "./runtime/os-profile.js";
|
|
13
16
|
export { rebuildOsSnapshotFromSessionEvents, sessionLogHasRequiredCategories, } from "./runtime/os-snapshot.js";
|
|
14
17
|
export { categoryForKind, kernelObservationToSessionEvent } from "./runtime/kernel-event-log.js";
|
|
@@ -39,6 +42,7 @@ export { tool, streamingTool, executeTools, readFile, validateToolArguments } fr
|
|
|
39
42
|
export { scanSkillDir, readSkillFile } from "./skills/loader.js";
|
|
40
43
|
// ── Memory ─────────────────────────────────────────────────────────────────
|
|
41
44
|
export { WorkingMemory } from "./memory/working.js";
|
|
45
|
+
export { InMemoryDreamStore } from "./memory/in-memory-store.js";
|
|
42
46
|
export { ScheduledPrompt } from "./signals/scheduled.js";
|
|
43
47
|
export { SignalGateway } from "./signals/gateway.js";
|
|
44
48
|
// ── Safety & Governance ────────────────────────────────────────────────────
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `InMemoryDreamStore` — a lightweight `DreamStore` implementation backed by per-agent `Map`s.
|
|
3
|
+
*
|
|
4
|
+
* Originally lived as `MockDreamStore` in the SDK's test helpers; promoted here so benchmarks,
|
|
5
|
+
* examples, and downstream consumers can use it without copying the boilerplate.
|
|
6
|
+
*
|
|
7
|
+
* Use cases:
|
|
8
|
+
* - Benchmark A/B variants where memory is on/off (preload via constructor).
|
|
9
|
+
* - Unit tests that exercise `Agent.dream()` or the `memory_query` path without disk I/O.
|
|
10
|
+
* - Local development / CI where a persistent memory store isn't needed.
|
|
11
|
+
*
|
|
12
|
+
* The `search()` impl is intentionally trivial — it returns the first `topK` memories for the
|
|
13
|
+
* agent regardless of `query`. The kernel ranks by score before deciding what to surface, so the
|
|
14
|
+
* order memories were inserted is what callers see. For semantic search, plug in a real store.
|
|
15
|
+
*/
|
|
16
|
+
import type { CurationResult, DreamStore, MemoryEntry, SessionData } from "./protocols.js";
|
|
17
|
+
export declare class InMemoryDreamStore implements DreamStore {
|
|
18
|
+
private readonly initialMemories;
|
|
19
|
+
private sessions;
|
|
20
|
+
private memories;
|
|
21
|
+
/** Sessions persisted via `saveSession`; exposed for test assertions. */
|
|
22
|
+
readonly savedSessions: SessionData[];
|
|
23
|
+
/**
|
|
24
|
+
* @param initialMemories Optional seed memories applied to every agent that asks for memories
|
|
25
|
+
* for the first time. Useful for benchmark scenarios that preload a fact.
|
|
26
|
+
*/
|
|
27
|
+
constructor(initialMemories?: MemoryEntry[]);
|
|
28
|
+
/** Pre-populate sessions for a specific agent (test/benchmark setup). */
|
|
29
|
+
addSession(agentId: string, session: SessionData): void;
|
|
30
|
+
/** Pre-populate memories for a specific agent (test/benchmark setup). */
|
|
31
|
+
addMemories(agentId: string, entries: MemoryEntry[]): void;
|
|
32
|
+
loadSessions(agentId: string): Promise<SessionData[]>;
|
|
33
|
+
loadMemories(agentId: string): Promise<MemoryEntry[]>;
|
|
34
|
+
commit(agentId: string, result: CurationResult, existing: MemoryEntry[]): Promise<void>;
|
|
35
|
+
search(agentId: string, _query: string, topK?: number): Promise<MemoryEntry[]>;
|
|
36
|
+
saveSession(data: SessionData): Promise<void>;
|
|
37
|
+
}
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
export class InMemoryDreamStore {
|
|
2
|
+
initialMemories;
|
|
3
|
+
sessions = new Map();
|
|
4
|
+
memories = new Map();
|
|
5
|
+
/** Sessions persisted via `saveSession`; exposed for test assertions. */
|
|
6
|
+
savedSessions = [];
|
|
7
|
+
/**
|
|
8
|
+
* @param initialMemories Optional seed memories applied to every agent that asks for memories
|
|
9
|
+
* for the first time. Useful for benchmark scenarios that preload a fact.
|
|
10
|
+
*/
|
|
11
|
+
constructor(initialMemories = []) {
|
|
12
|
+
this.initialMemories = initialMemories;
|
|
13
|
+
}
|
|
14
|
+
/** Pre-populate sessions for a specific agent (test/benchmark setup). */
|
|
15
|
+
addSession(agentId, session) {
|
|
16
|
+
const list = this.sessions.get(agentId) ?? [];
|
|
17
|
+
list.push(session);
|
|
18
|
+
this.sessions.set(agentId, list);
|
|
19
|
+
}
|
|
20
|
+
/** Pre-populate memories for a specific agent (test/benchmark setup). */
|
|
21
|
+
addMemories(agentId, entries) {
|
|
22
|
+
this.memories.set(agentId, [...(this.memories.get(agentId) ?? []), ...entries]);
|
|
23
|
+
}
|
|
24
|
+
async loadSessions(agentId) {
|
|
25
|
+
return this.sessions.get(agentId) ?? [];
|
|
26
|
+
}
|
|
27
|
+
async loadMemories(agentId) {
|
|
28
|
+
if (this.memories.has(agentId))
|
|
29
|
+
return this.memories.get(agentId);
|
|
30
|
+
if (this.initialMemories.length > 0) {
|
|
31
|
+
this.memories.set(agentId, [...this.initialMemories]);
|
|
32
|
+
return this.memories.get(agentId);
|
|
33
|
+
}
|
|
34
|
+
return [];
|
|
35
|
+
}
|
|
36
|
+
async commit(agentId, result, existing) {
|
|
37
|
+
const kept = existing.filter((_, i) => !result.toRemoveIndices.includes(i));
|
|
38
|
+
this.memories.set(agentId, [...kept, ...result.toAdd]);
|
|
39
|
+
}
|
|
40
|
+
async search(agentId, _query, topK = 5) {
|
|
41
|
+
const all = await this.loadMemories(agentId);
|
|
42
|
+
return all.slice(0, topK);
|
|
43
|
+
}
|
|
44
|
+
async saveSession(data) {
|
|
45
|
+
this.savedSessions.push(data);
|
|
46
|
+
const list = this.sessions.get(data.agentId) ?? [];
|
|
47
|
+
list.push(data);
|
|
48
|
+
this.sessions.set(data.agentId, list);
|
|
49
|
+
}
|
|
50
|
+
}
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `judge()` — one-shot quality scoring against a goal + criteria using the kernel's `gen_eval`.
|
|
3
|
+
*
|
|
4
|
+
* Wraps the three kernel free functions `buildEvalMessages` / `parseVerdict` / `verdictOutputSchema`
|
|
5
|
+
* (folded out of the old EvalPipeline class in 0.5.0) into a small typed surface that's safe to
|
|
6
|
+
* call from a benchmark harness, a CI gate, or any caller that just wants "does this result meet
|
|
7
|
+
* the criteria?" without setting up `HarnessLoop`.
|
|
8
|
+
*
|
|
9
|
+
* The judge is a single LLM call: build the eval prompt → stream → parse verdict. No retry loop,
|
|
10
|
+
* no skill extraction, no harness state. Use `HarnessLoop` if you want the retry/refine flow.
|
|
11
|
+
*/
|
|
12
|
+
import type { LLMProvider, Message } from "../types.js";
|
|
13
|
+
export interface Criterion {
|
|
14
|
+
/** The criterion text the judge evaluates against. */
|
|
15
|
+
text: string;
|
|
16
|
+
/** When true (default), failing this criterion fails the overall verdict. */
|
|
17
|
+
required?: boolean;
|
|
18
|
+
/** Optional weight for weighted scoring (kernel-defined semantics). */
|
|
19
|
+
weight?: number;
|
|
20
|
+
}
|
|
21
|
+
export interface VerdictDetail {
|
|
22
|
+
criterion: string;
|
|
23
|
+
passed: boolean;
|
|
24
|
+
score: number;
|
|
25
|
+
feedback: string;
|
|
26
|
+
}
|
|
27
|
+
export interface Verdict {
|
|
28
|
+
passed: boolean;
|
|
29
|
+
/** 0..1 — kernel-defined aggregate score. */
|
|
30
|
+
overallScore: number;
|
|
31
|
+
feedback: string;
|
|
32
|
+
details: VerdictDetail[];
|
|
33
|
+
}
|
|
34
|
+
export interface JudgeArgs {
|
|
35
|
+
/** Provider used for the eval LLM call. Often a cheaper model than the main run. */
|
|
36
|
+
provider: LLMProvider;
|
|
37
|
+
/** The task goal the result is being evaluated against. */
|
|
38
|
+
goal: string;
|
|
39
|
+
/** The criteria the judge scores against. */
|
|
40
|
+
criteria: Criterion[];
|
|
41
|
+
/** The agent's result text (final reply, or a structured summary when the run was incomplete). */
|
|
42
|
+
result: string;
|
|
43
|
+
/** Optional abort signal forwarded to provider.stream. */
|
|
44
|
+
signal?: AbortSignal;
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* Build the kernel's eval prompt for (goal, criteria, result).
|
|
48
|
+
* Exposed in case a caller wants to render the prompt without calling the LLM (e.g., dry-run cost
|
|
49
|
+
* estimation, fixture generation). For the common case, use `judge()`.
|
|
50
|
+
*/
|
|
51
|
+
export declare function buildEvalMessages(goal: string, criteria: Criterion[], result: string): Message[];
|
|
52
|
+
/** Parse a Verdict from raw judge-LLM text. Throws on schema mismatch. */
|
|
53
|
+
export declare function parseVerdict(text: string): Verdict;
|
|
54
|
+
/** The JSON Schema the kernel expects judge output to conform to. */
|
|
55
|
+
export declare function verdictOutputSchema(): Record<string, unknown>;
|
|
56
|
+
/**
|
|
57
|
+
* Run one judge pass: render the eval prompt, stream the provider, parse the verdict.
|
|
58
|
+
* Throws when the provider returns no text or returns content that fails verdict parsing.
|
|
59
|
+
*/
|
|
60
|
+
export declare function judge(args: JudgeArgs): Promise<Verdict>;
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `judge()` — one-shot quality scoring against a goal + criteria using the kernel's `gen_eval`.
|
|
3
|
+
*
|
|
4
|
+
* Wraps the three kernel free functions `buildEvalMessages` / `parseVerdict` / `verdictOutputSchema`
|
|
5
|
+
* (folded out of the old EvalPipeline class in 0.5.0) into a small typed surface that's safe to
|
|
6
|
+
* call from a benchmark harness, a CI gate, or any caller that just wants "does this result meet
|
|
7
|
+
* the criteria?" without setting up `HarnessLoop`.
|
|
8
|
+
*
|
|
9
|
+
* The judge is a single LLM call: build the eval prompt → stream → parse verdict. No retry loop,
|
|
10
|
+
* no skill extraction, no harness state. Use `HarnessLoop` if you want the retry/refine flow.
|
|
11
|
+
*/
|
|
12
|
+
import { getKernel } from "../kernel.js";
|
|
13
|
+
/**
|
|
14
|
+
* Build the kernel's eval prompt for (goal, criteria, result).
|
|
15
|
+
* Exposed in case a caller wants to render the prompt without calling the LLM (e.g., dry-run cost
|
|
16
|
+
* estimation, fixture generation). For the common case, use `judge()`.
|
|
17
|
+
*/
|
|
18
|
+
export function buildEvalMessages(goal, criteria, result) {
|
|
19
|
+
return getKernel().buildEvalMessages(goal, criteria.map(c => ({ text: c.text, required: c.required ?? true, weight: c.weight })), result, 1, // attempt
|
|
20
|
+
false);
|
|
21
|
+
}
|
|
22
|
+
/** Parse a Verdict from raw judge-LLM text. Throws on schema mismatch. */
|
|
23
|
+
export function parseVerdict(text) {
|
|
24
|
+
const v = getKernel().parseVerdict(text);
|
|
25
|
+
return {
|
|
26
|
+
passed: v.passed,
|
|
27
|
+
overallScore: v.overallScore,
|
|
28
|
+
feedback: v.feedback,
|
|
29
|
+
details: v.details ?? [],
|
|
30
|
+
};
|
|
31
|
+
}
|
|
32
|
+
/** The JSON Schema the kernel expects judge output to conform to. */
|
|
33
|
+
export function verdictOutputSchema() {
|
|
34
|
+
return JSON.parse(getKernel().verdictOutputSchema(false));
|
|
35
|
+
}
|
|
36
|
+
/**
|
|
37
|
+
* Run one judge pass: render the eval prompt, stream the provider, parse the verdict.
|
|
38
|
+
* Throws when the provider returns no text or returns content that fails verdict parsing.
|
|
39
|
+
*/
|
|
40
|
+
export async function judge(args) {
|
|
41
|
+
const msgs = buildEvalMessages(args.goal, args.criteria, args.result);
|
|
42
|
+
const ctx = {
|
|
43
|
+
systemText: msgs.filter(m => m.role === "system").map(m => m.content).join("\n\n"),
|
|
44
|
+
turns: msgs.filter(m => m.role !== "system"),
|
|
45
|
+
};
|
|
46
|
+
let text = "";
|
|
47
|
+
for await (const evt of args.provider.stream(ctx, [], undefined, undefined, args.signal)) {
|
|
48
|
+
if (evt.type === "text_delta")
|
|
49
|
+
text += evt.delta;
|
|
50
|
+
}
|
|
51
|
+
if (!text) {
|
|
52
|
+
throw new Error("judge: provider produced no text");
|
|
53
|
+
}
|
|
54
|
+
return parseVerdict(text);
|
|
55
|
+
}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Fixture helpers for `ReplayProvider`.
|
|
3
|
+
*
|
|
4
|
+
* The canonical persistence shape for a recorded run is a `SessionLog` of `llm_completed` events
|
|
5
|
+
* (already written by the runner on every live run). `extractRecordedMessages` walks such a log and
|
|
6
|
+
* pulls the assistant turns in order, so the fixture is just "a prior session log + the messages
|
|
7
|
+
* the LLM produced". No new on-disk format.
|
|
8
|
+
*/
|
|
9
|
+
import type { Message } from "../types.js";
|
|
10
|
+
import type { SessionEvent } from "./session-log.js";
|
|
11
|
+
/**
|
|
12
|
+
* Extract the ordered list of assistant Messages from a recorded session log.
|
|
13
|
+
*
|
|
14
|
+
* Walks `llm_completed` events (which is what the runner appends for every LLM call) and produces
|
|
15
|
+
* one Message per event. Pass the result directly to `new ReplayProvider(messages)`.
|
|
16
|
+
*
|
|
17
|
+
* Accepts both wire shapes the SDK uses interchangeably:
|
|
18
|
+
* - in-memory: `{ toolCalls, tokenCount, providerReplay }` (camelCase)
|
|
19
|
+
* - serialised session-log: `{ tool_calls, token_count, provider_replay }` (snake_case)
|
|
20
|
+
*
|
|
21
|
+
* @param events Session events, in original order. Accepts both `{ event, seq }` (the shape
|
|
22
|
+
* `SessionLog.read()` returns) and a bare `SessionEvent[]`.
|
|
23
|
+
*/
|
|
24
|
+
export declare function extractRecordedMessages(events: Array<{
|
|
25
|
+
event: SessionEvent;
|
|
26
|
+
} | SessionEvent>): Message[];
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Fixture helpers for `ReplayProvider`.
|
|
3
|
+
*
|
|
4
|
+
* The canonical persistence shape for a recorded run is a `SessionLog` of `llm_completed` events
|
|
5
|
+
* (already written by the runner on every live run). `extractRecordedMessages` walks such a log and
|
|
6
|
+
* pulls the assistant turns in order, so the fixture is just "a prior session log + the messages
|
|
7
|
+
* the LLM produced". No new on-disk format.
|
|
8
|
+
*/
|
|
9
|
+
/**
|
|
10
|
+
* Extract the ordered list of assistant Messages from a recorded session log.
|
|
11
|
+
*
|
|
12
|
+
* Walks `llm_completed` events (which is what the runner appends for every LLM call) and produces
|
|
13
|
+
* one Message per event. Pass the result directly to `new ReplayProvider(messages)`.
|
|
14
|
+
*
|
|
15
|
+
* Accepts both wire shapes the SDK uses interchangeably:
|
|
16
|
+
* - in-memory: `{ toolCalls, tokenCount, providerReplay }` (camelCase)
|
|
17
|
+
* - serialised session-log: `{ tool_calls, token_count, provider_replay }` (snake_case)
|
|
18
|
+
*
|
|
19
|
+
* @param events Session events, in original order. Accepts both `{ event, seq }` (the shape
|
|
20
|
+
* `SessionLog.read()` returns) and a bare `SessionEvent[]`.
|
|
21
|
+
*/
|
|
22
|
+
export function extractRecordedMessages(events) {
|
|
23
|
+
const out = [];
|
|
24
|
+
for (const entry of events) {
|
|
25
|
+
const event = isWrapped(entry) ? entry.event : entry;
|
|
26
|
+
if (event.kind !== "llm_completed")
|
|
27
|
+
continue;
|
|
28
|
+
const e = event;
|
|
29
|
+
const tcRaw = (e.toolCalls ?? e.tool_calls);
|
|
30
|
+
const tokenCount = (e.tokenCount ?? e.token_count);
|
|
31
|
+
out.push({
|
|
32
|
+
role: "assistant",
|
|
33
|
+
content: typeof e.content === "string" ? e.content : "",
|
|
34
|
+
...(Array.isArray(tcRaw) && tcRaw.length > 0
|
|
35
|
+
? { toolCalls: normalizeToolCalls(tcRaw) }
|
|
36
|
+
: {}),
|
|
37
|
+
...(tokenCount !== undefined ? { tokenCount } : {}),
|
|
38
|
+
});
|
|
39
|
+
}
|
|
40
|
+
return out;
|
|
41
|
+
}
|
|
42
|
+
function isWrapped(x) {
|
|
43
|
+
return !!x && typeof x === "object" && "event" in x;
|
|
44
|
+
}
|
|
45
|
+
function normalizeToolCalls(tcs) {
|
|
46
|
+
return tcs.map(tc => ({
|
|
47
|
+
id: tc.id,
|
|
48
|
+
name: tc.name,
|
|
49
|
+
arguments: typeof tc.arguments === "string" ? tc.arguments : JSON.stringify(tc.arguments),
|
|
50
|
+
}));
|
|
51
|
+
}
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* ReplayProvider — an LLMProvider that emits previously-recorded assistant messages
|
|
3
|
+
* instead of calling a real LLM API.
|
|
4
|
+
*
|
|
5
|
+
* Purpose: deterministic re-runs for benchmarking, CI, and golden regression. Useful when you want
|
|
6
|
+
* to hold the model's behavior constant and measure something else (prompt-size cost Δ across
|
|
7
|
+
* RuntimeOptions variants, codegen of follow-on kernel work, etc.).
|
|
8
|
+
*
|
|
9
|
+
* Distinct from `provider-replay.ts`: that file's `seedProviderReplay` / `peekProviderReplay` is a
|
|
10
|
+
* session-repair *reasoning-content cache* — it preserves `reasoning_content` / `native_blocks` so
|
|
11
|
+
* the model sees its own thinking when context is re-rendered. It does NOT skip LLM calls.
|
|
12
|
+
* `ReplayProvider` is the orthogonal, request-skipping mechanism: it returns recorded responses
|
|
13
|
+
* directly, never hitting an API.
|
|
14
|
+
*
|
|
15
|
+
* Cost-accounting under replay:
|
|
16
|
+
* - `inputTokens` is ESTIMATED from the rendered context this call carries (NOT a recorded value
|
|
17
|
+
* from the original run). That's the point of replay-for-benchmarking: prompt may differ across
|
|
18
|
+
* variants, response is pinned, so a cost Δ purely reflects the prompt change.
|
|
19
|
+
* - `outputTokens` is taken from `message.tokenCount` when present; otherwise estimated from
|
|
20
|
+
* `message.content.length / 4`.
|
|
21
|
+
* - `cacheReadInputTokens` / `cacheCreationInputTokens` are emitted as 0 — replay has no real
|
|
22
|
+
* cache state. Mechanisms whose Δ depends on cache behavior must validate with a live A/B too.
|
|
23
|
+
*
|
|
24
|
+
* Tokenizer: by default a `chars/4` estimator (±20% for English; worse for code/JSON). For tighter
|
|
25
|
+
* numbers plug `opts.tokenizer = tiktokenEncoder` or similar.
|
|
26
|
+
*/
|
|
27
|
+
import type { LLMProvider, Message, ProviderDescriptor, ProviderRunState, RenderedContext, StreamEvent, ToolSchema } from "../types.js";
|
|
28
|
+
export interface ReplayProviderOpts {
|
|
29
|
+
/**
|
|
30
|
+
* Maps a rendered-context text payload to a token count. Defaults to `chars / 4`.
|
|
31
|
+
* Pass a real encoder (tiktoken etc.) for accurate cost accounting under replay.
|
|
32
|
+
*/
|
|
33
|
+
tokenizer?: (text: string) => number;
|
|
34
|
+
/**
|
|
35
|
+
* Provider descriptor advertised via `descriptor()`. Defaults to a generic
|
|
36
|
+
* `{ provider: "replay", protocol: "replay", ... }` shape. Override when a downstream consumer
|
|
37
|
+
* needs to detect the original provider (e.g., for protocol-specific decoding paths).
|
|
38
|
+
*/
|
|
39
|
+
descriptor?: ProviderDescriptor;
|
|
40
|
+
/**
|
|
41
|
+
* When true, `stream()` and `complete()` wrap to the start once the fixture is exhausted,
|
|
42
|
+
* instead of throwing. Useful for loop tests that need to keep going past the recorded length.
|
|
43
|
+
* Defaults to false.
|
|
44
|
+
*/
|
|
45
|
+
wrap?: boolean;
|
|
46
|
+
}
|
|
47
|
+
export declare class ReplayProvider implements LLMProvider {
|
|
48
|
+
private cursor;
|
|
49
|
+
private readonly messages;
|
|
50
|
+
private readonly tokenizer;
|
|
51
|
+
private readonly _descriptor;
|
|
52
|
+
private readonly wrap;
|
|
53
|
+
/**
|
|
54
|
+
* @param messages Ordered list of assistant messages to replay (one per LLM call).
|
|
55
|
+
* @param opts Optional tokenizer / descriptor / wrap-around behavior.
|
|
56
|
+
*/
|
|
57
|
+
constructor(messages: ReadonlyArray<Message>, opts?: ReplayProviderOpts);
|
|
58
|
+
descriptor(): ProviderDescriptor;
|
|
59
|
+
/** Number of messages consumed so far. */
|
|
60
|
+
consumed(): number;
|
|
61
|
+
/** Number of messages remaining in the fixture (returns 0 in wrap mode once cursor passes end). */
|
|
62
|
+
remaining(): number;
|
|
63
|
+
/** Reset the cursor — useful for re-running the same fixture in a fresh session. */
|
|
64
|
+
reset(): void;
|
|
65
|
+
complete(_context: RenderedContext, _tools: ToolSchema[]): Promise<Message>;
|
|
66
|
+
stream(context: RenderedContext, tools: ToolSchema[], _extensions?: Record<string, unknown>, _state?: ProviderRunState, _signal?: AbortSignal): AsyncIterable<StreamEvent>;
|
|
67
|
+
private pull;
|
|
68
|
+
private estimateInputTokens;
|
|
69
|
+
}
|
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* ReplayProvider — an LLMProvider that emits previously-recorded assistant messages
|
|
3
|
+
* instead of calling a real LLM API.
|
|
4
|
+
*
|
|
5
|
+
* Purpose: deterministic re-runs for benchmarking, CI, and golden regression. Useful when you want
|
|
6
|
+
* to hold the model's behavior constant and measure something else (prompt-size cost Δ across
|
|
7
|
+
* RuntimeOptions variants, codegen of follow-on kernel work, etc.).
|
|
8
|
+
*
|
|
9
|
+
* Distinct from `provider-replay.ts`: that file's `seedProviderReplay` / `peekProviderReplay` is a
|
|
10
|
+
* session-repair *reasoning-content cache* — it preserves `reasoning_content` / `native_blocks` so
|
|
11
|
+
* the model sees its own thinking when context is re-rendered. It does NOT skip LLM calls.
|
|
12
|
+
* `ReplayProvider` is the orthogonal, request-skipping mechanism: it returns recorded responses
|
|
13
|
+
* directly, never hitting an API.
|
|
14
|
+
*
|
|
15
|
+
* Cost-accounting under replay:
|
|
16
|
+
* - `inputTokens` is ESTIMATED from the rendered context this call carries (NOT a recorded value
|
|
17
|
+
* from the original run). That's the point of replay-for-benchmarking: prompt may differ across
|
|
18
|
+
* variants, response is pinned, so a cost Δ purely reflects the prompt change.
|
|
19
|
+
* - `outputTokens` is taken from `message.tokenCount` when present; otherwise estimated from
|
|
20
|
+
* `message.content.length / 4`.
|
|
21
|
+
* - `cacheReadInputTokens` / `cacheCreationInputTokens` are emitted as 0 — replay has no real
|
|
22
|
+
* cache state. Mechanisms whose Δ depends on cache behavior must validate with a live A/B too.
|
|
23
|
+
*
|
|
24
|
+
* Tokenizer: by default a `chars/4` estimator (±20% for English; worse for code/JSON). For tighter
|
|
25
|
+
* numbers plug `opts.tokenizer = tiktokenEncoder` or similar.
|
|
26
|
+
*/
|
|
27
|
+
const DEFAULT_DESCRIPTOR = {
|
|
28
|
+
provider: "replay",
|
|
29
|
+
protocol: "openai-chat",
|
|
30
|
+
model: "replay",
|
|
31
|
+
reasoning: { supported: false, preserveAcrossToolTurns: false },
|
|
32
|
+
toolCalls: { supported: true, requiresStrictPairing: false },
|
|
33
|
+
};
|
|
34
|
+
export class ReplayProvider {
|
|
35
|
+
cursor = 0;
|
|
36
|
+
messages;
|
|
37
|
+
tokenizer;
|
|
38
|
+
_descriptor;
|
|
39
|
+
wrap;
|
|
40
|
+
/**
|
|
41
|
+
* @param messages Ordered list of assistant messages to replay (one per LLM call).
|
|
42
|
+
* @param opts Optional tokenizer / descriptor / wrap-around behavior.
|
|
43
|
+
*/
|
|
44
|
+
constructor(messages, opts = {}) {
|
|
45
|
+
this.messages = messages;
|
|
46
|
+
this.tokenizer = opts.tokenizer ?? defaultTokenizer;
|
|
47
|
+
this._descriptor = opts.descriptor ?? DEFAULT_DESCRIPTOR;
|
|
48
|
+
this.wrap = !!opts.wrap;
|
|
49
|
+
}
|
|
50
|
+
descriptor() {
|
|
51
|
+
return this._descriptor;
|
|
52
|
+
}
|
|
53
|
+
/** Number of messages consumed so far. */
|
|
54
|
+
consumed() {
|
|
55
|
+
return this.cursor;
|
|
56
|
+
}
|
|
57
|
+
/** Number of messages remaining in the fixture (returns 0 in wrap mode once cursor passes end). */
|
|
58
|
+
remaining() {
|
|
59
|
+
return Math.max(0, this.messages.length - this.cursor);
|
|
60
|
+
}
|
|
61
|
+
/** Reset the cursor — useful for re-running the same fixture in a fresh session. */
|
|
62
|
+
reset() {
|
|
63
|
+
this.cursor = 0;
|
|
64
|
+
}
|
|
65
|
+
async complete(_context, _tools) {
|
|
66
|
+
const msg = this.pull();
|
|
67
|
+
return {
|
|
68
|
+
role: "assistant",
|
|
69
|
+
content: msg.content,
|
|
70
|
+
...(msg.toolCalls ? { toolCalls: msg.toolCalls } : {}),
|
|
71
|
+
...(msg.tokenCount !== undefined ? { tokenCount: msg.tokenCount } : {}),
|
|
72
|
+
};
|
|
73
|
+
}
|
|
74
|
+
async *stream(context, tools, _extensions, _state, _signal) {
|
|
75
|
+
const msg = this.pull();
|
|
76
|
+
const inputTokens = this.estimateInputTokens(context, tools);
|
|
77
|
+
const outputTokens = msg.tokenCount !== undefined ? msg.tokenCount : this.tokenizer(msg.content || "");
|
|
78
|
+
const usage = {
|
|
79
|
+
type: "usage",
|
|
80
|
+
totalTokens: inputTokens + outputTokens,
|
|
81
|
+
inputTokens,
|
|
82
|
+
outputTokens,
|
|
83
|
+
cacheReadInputTokens: 0,
|
|
84
|
+
cacheCreationInputTokens: 0,
|
|
85
|
+
};
|
|
86
|
+
yield usage;
|
|
87
|
+
if (msg.content) {
|
|
88
|
+
const delta = { type: "text_delta", delta: msg.content };
|
|
89
|
+
yield delta;
|
|
90
|
+
}
|
|
91
|
+
for (const tc of msg.toolCalls ?? []) {
|
|
92
|
+
let args = {};
|
|
93
|
+
try {
|
|
94
|
+
args = JSON.parse(tc.arguments || "{}");
|
|
95
|
+
}
|
|
96
|
+
catch {
|
|
97
|
+
// Malformed recorded arguments — pass an empty object. The runner's downstream tool
|
|
98
|
+
// execution will surface the error if the tool needs them.
|
|
99
|
+
}
|
|
100
|
+
const call = { type: "tool_call", id: tc.id, name: tc.name, arguments: args };
|
|
101
|
+
yield call;
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
pull() {
|
|
105
|
+
if (this.cursor >= this.messages.length) {
|
|
106
|
+
if (this.wrap && this.messages.length > 0) {
|
|
107
|
+
this.cursor = 0;
|
|
108
|
+
}
|
|
109
|
+
else {
|
|
110
|
+
throw new Error(`ReplayProvider: fixture exhausted (consumed=${this.cursor}, total=${this.messages.length})`);
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
return this.messages[this.cursor++];
|
|
114
|
+
}
|
|
115
|
+
estimateInputTokens(context, tools) {
|
|
116
|
+
const text = renderContextToText(context, tools);
|
|
117
|
+
return this.tokenizer(text);
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
// ── helpers ───────────────────────────────────────────────────────────────────
|
|
121
|
+
function defaultTokenizer(text) {
|
|
122
|
+
return Math.ceil(text.length / 4);
|
|
123
|
+
}
|
|
124
|
+
function renderContextToText(context, tools) {
|
|
125
|
+
const parts = [];
|
|
126
|
+
if (context.systemText)
|
|
127
|
+
parts.push(context.systemText);
|
|
128
|
+
if (context.systemStable)
|
|
129
|
+
parts.push(context.systemStable);
|
|
130
|
+
if (context.systemKnowledge)
|
|
131
|
+
parts.push(context.systemKnowledge);
|
|
132
|
+
if (context.stateTurn?.content)
|
|
133
|
+
parts.push(context.stateTurn.content);
|
|
134
|
+
for (const turn of context.turns ?? []) {
|
|
135
|
+
if (turn.content)
|
|
136
|
+
parts.push(turn.content);
|
|
137
|
+
for (const part of turn.contentParts ?? []) {
|
|
138
|
+
const p = part;
|
|
139
|
+
if (typeof p.output === "string")
|
|
140
|
+
parts.push(p.output);
|
|
141
|
+
else if (typeof p.text === "string")
|
|
142
|
+
parts.push(p.text);
|
|
143
|
+
}
|
|
144
|
+
for (const tc of turn.toolCalls ?? []) {
|
|
145
|
+
parts.push(`${tc.name} ${tc.arguments}`);
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
for (const tool of tools) {
|
|
149
|
+
parts.push(`${tool.name} ${tool.description} ${tool.parameters}`);
|
|
150
|
+
}
|
|
151
|
+
return parts.join("\n");
|
|
152
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@deepstrike/sdk",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.21",
|
|
4
4
|
"description": "DeepStrike Node.js SDK",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "dist/index.js",
|
|
@@ -20,7 +20,7 @@
|
|
|
20
20
|
},
|
|
21
21
|
"dependencies": {
|
|
22
22
|
"@anthropic-ai/sdk": "^0.99.0",
|
|
23
|
-
"@deepstrike/core": "0.2.
|
|
23
|
+
"@deepstrike/core": "0.2.21",
|
|
24
24
|
"@google/generative-ai": "^0.24.1",
|
|
25
25
|
"openai": "^5.23.2"
|
|
26
26
|
},
|