@kuralle-agents/core 0.8.0 → 0.8.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. package/README.md +6 -0
  2. package/dist/capabilities/validators/groundingValidator.d.ts +21 -0
  3. package/dist/capabilities/validators/groundingValidator.js +109 -0
  4. package/dist/escalation/escalation.d.ts +20 -0
  5. package/dist/escalation/escalation.js +85 -0
  6. package/dist/escalation/types.d.ts +40 -0
  7. package/dist/eval/simulation.d.ts +99 -0
  8. package/dist/eval/simulation.js +168 -0
  9. package/dist/index.d.ts +22 -2
  10. package/dist/index.js +10 -0
  11. package/dist/memory/factMemoryService.d.ts +28 -0
  12. package/dist/memory/factMemoryService.js +137 -0
  13. package/dist/memory/index.d.ts +1 -0
  14. package/dist/memory/index.js +1 -0
  15. package/dist/processors/builtin/moderationGuard.d.ts +24 -0
  16. package/dist/processors/builtin/moderationGuard.js +101 -0
  17. package/dist/processors/builtin/piiGuard.d.ts +41 -0
  18. package/dist/processors/builtin/piiGuard.js +143 -0
  19. package/dist/processors/builtin/promptInjectionGuard.d.ts +16 -0
  20. package/dist/processors/builtin/promptInjectionGuard.js +28 -0
  21. package/dist/runtime/Runtime.d.ts +53 -0
  22. package/dist/runtime/Runtime.js +184 -10
  23. package/dist/runtime/channels/TextDriver.js +6 -0
  24. package/dist/runtime/channels/VoiceDriver.js +6 -0
  25. package/dist/runtime/closeRun.js +4 -0
  26. package/dist/runtime/compaction.d.ts +51 -0
  27. package/dist/runtime/compaction.js +111 -0
  28. package/dist/runtime/hostLoop.d.ts +2 -0
  29. package/dist/runtime/hostLoop.js +1 -1
  30. package/dist/runtime/openRun.d.ts +5 -0
  31. package/dist/runtime/openRun.js +17 -0
  32. package/dist/runtime/policies/agentTurn.d.ts +4 -0
  33. package/dist/runtime/policies/agentTurn.js +35 -4
  34. package/dist/scheduler/index.d.ts +89 -0
  35. package/dist/scheduler/index.js +104 -0
  36. package/dist/types/channel.d.ts +2 -0
  37. package/dist/types/stream.d.ts +29 -0
  38. package/guides/GUARDRAILS.md +44 -0
  39. package/package.json +2 -2
package/README.md CHANGED
@@ -21,6 +21,12 @@ One tagless primitive — `defineAgent` — derives behavior from the fields you
21
21
  - **`defineTool` + `buildToolSet`** — typed effect tools wired to both the model and the executor.
22
22
  - **`createRuntime` / `Runtime`** — orchestrator: sessions, handoffs, streaming, flow state.
23
23
  - **`MemoryStore`** — in-process `SessionStore`; swap for Redis or Postgres in production.
24
+ - **`HarnessConfig.escalation` + `resumeFromEscalation`** — the human-handoff loop: handoff brief, handler, ownership claim (via engagement), resume-with-resolution.
25
+ - **`RunOptions.wake` + `Scheduler`** — agent-initiated turns (follow-ups, cart abandonment) on in-process timers or Cloudflare DO alarms (`@kuralle-agents/cf-agent`).
26
+ - **`HarnessConfig.compaction`** — automatic history summarization + context-overflow recovery for long-running threads.
27
+ - **`createFactMemoryService`** — cross-session fact memory (LLM merge-on-ingest) keyed by `userId` on any persistent block store.
28
+ - **Built-in guardrails** — `createPromptInjectionGuard`, `createPiiInputGuard`/`OutputGuard`, `createModerationGuard`, `createGroundingValidator` (see `guides/GUARDRAILS.md`).
29
+ - **Simulation eval** — `simulateConversation` + `createJudge` + `runSimulationSuite`: persona-driven simulated users scored by an LLM judge.
24
30
 
25
31
  ## Usage
26
32
 
@@ -0,0 +1,21 @@
1
+ import type { LanguageModel } from 'ai';
2
+ import type { ValidationCapability } from '../ValidationCapability.js';
3
+ export interface GroundingValidatorOptions {
4
+ /** Judge model, run at temperature 0. Use the agent's controlModel-class model. */
5
+ model: LanguageModel;
6
+ /** Extra domain rules appended to the judge prompt (e.g. "never promise refunds"). */
7
+ instructions?: string;
8
+ name?: string;
9
+ }
10
+ /**
11
+ * State-grounded output validator: checks the assistant reply for action/fact
12
+ * claims unsupported by this turn's tool calls, flow state, or citations —
13
+ * the "order placed" with no create-order call class of hallucination.
14
+ *
15
+ * Repair policy is rewrite-not-block: an ungrounded claim is rewritten out of
16
+ * the reply rather than replacing the whole turn with an apology, so the
17
+ * conversation keeps moving. If the judge flags ungrounded but cannot produce
18
+ * a rewrite, the turn is blocked (fail safe). Judge errors fail open with low
19
+ * confidence — the validator is a gate, not a single point of failure.
20
+ */
21
+ export declare function createGroundingValidator(options: GroundingValidatorOptions): ValidationCapability;
@@ -0,0 +1,109 @@
1
+ import { generateObject } from 'ai';
2
+ import { z } from 'zod';
3
+ const verdictSchema = z.object({
4
+ verdict: z.enum(['grounded', 'ungrounded']),
5
+ /** Required when ungrounded: the same reply with unsupported claims removed/softened. */
6
+ rewrittenOutput: z.union([z.string(), z.null()]),
7
+ rationale: z.union([z.string(), z.null()]),
8
+ confidence: z.number(),
9
+ });
10
+ function summarizeEvidence(input) {
11
+ const lines = [];
12
+ if (input.toolCallsMade.length === 0) {
13
+ lines.push('Tool calls this turn: NONE — no external action was performed.');
14
+ }
15
+ else {
16
+ lines.push('Tool calls this turn:');
17
+ for (const call of input.toolCallsMade) {
18
+ const result = truncateJson(call.result, 400);
19
+ lines.push(`- ${call.toolName}(${truncateJson(call.args, 200)}) → ${call.success ? result : `FAILED: ${result}`}`);
20
+ }
21
+ }
22
+ const stateKeys = Object.entries(input.state);
23
+ if (stateKeys.length > 0) {
24
+ lines.push('Flow state (evidence written by earlier steps):');
25
+ for (const [key, value] of stateKeys) {
26
+ if (key.startsWith('__'))
27
+ continue;
28
+ lines.push(`- ${key}: ${truncateJson(value, 200)}`);
29
+ }
30
+ }
31
+ if (input.knowledgeCitations.length > 0) {
32
+ lines.push(`Knowledge citations: ${input.knowledgeCitations.map((c) => c.title ?? c.id).join('; ')}`);
33
+ }
34
+ return lines.join('\n');
35
+ }
36
+ function truncateJson(value, max) {
37
+ let text;
38
+ try {
39
+ text = typeof value === 'string' ? value : JSON.stringify(value);
40
+ }
41
+ catch {
42
+ text = String(value);
43
+ }
44
+ return text.length > max ? `${text.slice(0, max)}…` : text;
45
+ }
46
+ /**
47
+ * State-grounded output validator: checks the assistant reply for action/fact
48
+ * claims unsupported by this turn's tool calls, flow state, or citations —
49
+ * the "order placed" with no create-order call class of hallucination.
50
+ *
51
+ * Repair policy is rewrite-not-block: an ungrounded claim is rewritten out of
52
+ * the reply rather than replacing the whole turn with an apology, so the
53
+ * conversation keeps moving. If the judge flags ungrounded but cannot produce
54
+ * a rewrite, the turn is blocked (fail safe). Judge errors fail open with low
55
+ * confidence — the validator is a gate, not a single point of failure.
56
+ */
57
+ export function createGroundingValidator(options) {
58
+ return {
59
+ name: options.name ?? 'grounding-validator',
60
+ async validate(input) {
61
+ let verdict;
62
+ try {
63
+ const { object } = await generateObject({
64
+ model: options.model,
65
+ schema: verdictSchema,
66
+ temperature: 0,
67
+ abortSignal: input.abortSignal,
68
+ system: [
69
+ 'You are a grounding judge for a customer-facing agent. Decide whether the assistant reply',
70
+ 'claims any COMPLETED action or specific fact that the evidence below does not support.',
71
+ 'Examples of ungrounded claims: "your order has been placed" with no order-creation evidence;',
72
+ 'inventing prices, dates, or policies not present in tool results, state, or citations.',
73
+ 'Asking questions, describing next steps, or hedged language is grounded.',
74
+ 'If ungrounded, produce rewrittenOutput: the SAME reply with unsupported claims removed or',
75
+ 'softened into honest next steps. Preserve tone and any grounded content.',
76
+ options.instructions ? `Additional rules:\n${options.instructions}` : '',
77
+ ]
78
+ .filter(Boolean)
79
+ .join('\n'),
80
+ prompt: [
81
+ `User message:\n${input.userMessage}`,
82
+ `Assistant reply:\n${input.assistantOutput}`,
83
+ `Evidence:\n${summarizeEvidence(input)}`,
84
+ ].join('\n\n'),
85
+ });
86
+ verdict = object;
87
+ }
88
+ catch {
89
+ return { decision: 'continue', confidence: 0.5, rationale: 'grounding judge unavailable' };
90
+ }
91
+ if (verdict.verdict === 'grounded') {
92
+ return { decision: 'continue', confidence: verdict.confidence };
93
+ }
94
+ if (verdict.rewrittenOutput && verdict.rewrittenOutput.trim()) {
95
+ return {
96
+ decision: 'rewrite',
97
+ confidence: verdict.confidence,
98
+ rewrittenOutput: verdict.rewrittenOutput,
99
+ rationale: verdict.rationale ?? 'ungrounded claim removed',
100
+ };
101
+ }
102
+ return {
103
+ decision: 'block',
104
+ confidence: verdict.confidence,
105
+ rationale: verdict.rationale ?? 'ungrounded claim with no safe rewrite',
106
+ };
107
+ },
108
+ };
109
+ }
@@ -0,0 +1,20 @@
1
+ import { type LanguageModel } from 'ai';
2
+ import type { Session, SessionMetadata } from '../types/session.js';
3
+ import type { RunState } from '../runtime/durable/types.js';
4
+ import type { EscalationConfig, EscalationOutcome, EscalationReason, EscalationRequest } from './types.js';
5
+ /** One-shot latch: set when the handler fired at flow-escalate pause time, so
6
+ * the post-resume terminal handoff does not notify a second time. */
7
+ export declare const ESCALATION_NOTIFIED_KEY = "__escalationNotified";
8
+ export declare function ensureSessionMetadata(session: Session): SessionMetadata;
9
+ export interface BuildEscalationRequestOptions {
10
+ session: Session;
11
+ runState: RunState;
12
+ reason: string;
13
+ category?: EscalationReason;
14
+ config: EscalationConfig;
15
+ /** Resolved summary model (config.model → controlModel → model → defaultModel). */
16
+ model?: LanguageModel;
17
+ abortSignal?: AbortSignal;
18
+ }
19
+ export declare function buildEscalationRequest(options: BuildEscalationRequestOptions): Promise<EscalationRequest>;
20
+ export declare function recordEscalationOutcome(session: Session, category: EscalationReason, outcome: EscalationOutcome): void;
@@ -0,0 +1,85 @@
1
+ import { generateText } from 'ai';
2
+ /** One-shot latch: set when the handler fired at flow-escalate pause time, so
3
+ * the post-resume terminal handoff does not notify a second time. */
4
+ export const ESCALATION_NOTIFIED_KEY = '__escalationNotified';
5
+ const SUMMARY_PROMPT = [
6
+ 'Write a concise handoff brief for a human agent taking over this conversation.',
7
+ 'Cover: who the user is (if known), what they want, what has been done or promised',
8
+ '(with exact ids/amounts), why this is being escalated, and the next action the human should take.',
9
+ 'Maximum 120 words. Do not invent details.',
10
+ ].join(' ');
11
+ function textProjection(content) {
12
+ if (typeof content === 'string')
13
+ return content;
14
+ if (!Array.isArray(content))
15
+ return '';
16
+ return content
17
+ .filter((part) => part.type === 'text' && typeof part.text === 'string')
18
+ .map((part) => part.text)
19
+ .join(' ');
20
+ }
21
+ export function ensureSessionMetadata(session) {
22
+ if (!session.metadata) {
23
+ const now = new Date();
24
+ session.metadata = {
25
+ createdAt: now,
26
+ lastActiveAt: now,
27
+ totalTokens: 0,
28
+ totalSteps: 0,
29
+ handoffHistory: [],
30
+ };
31
+ }
32
+ return session.metadata;
33
+ }
34
+ export async function buildEscalationRequest(options) {
35
+ const { session, runState, reason, category, config, model } = options;
36
+ const recentCount = config.recentMessageCount ?? 12;
37
+ const recentMessages = runState.messages
38
+ .slice(-recentCount)
39
+ .map((message) => ({ role: message.role, content: textProjection(message.content) }))
40
+ .filter((message) => message.content.trim().length > 0);
41
+ const state = {};
42
+ for (const [key, value] of Object.entries(runState.state)) {
43
+ if (!key.startsWith('__')) {
44
+ state[key] = value;
45
+ }
46
+ }
47
+ let summary;
48
+ const summarize = config.summarize ?? true;
49
+ if (summarize && model && recentMessages.length > 0) {
50
+ try {
51
+ const transcript = recentMessages
52
+ .map((message) => `${message.role}: ${message.content}`)
53
+ .join('\n');
54
+ const result = await generateText({
55
+ model,
56
+ system: SUMMARY_PROMPT,
57
+ prompt: `Escalation reason: ${reason}\n\nConversation:\n${transcript}`,
58
+ abortSignal: options.abortSignal,
59
+ });
60
+ summary = result.text.trim() || undefined;
61
+ }
62
+ catch {
63
+ summary = undefined; // the handoff proceeds without a brief — never blocks on the summarizer
64
+ }
65
+ }
66
+ return {
67
+ sessionId: session.id,
68
+ userId: session.userId,
69
+ agentId: runState.activeAgentId,
70
+ reason,
71
+ category,
72
+ summary,
73
+ state,
74
+ recentMessages,
75
+ at: new Date().toISOString(),
76
+ };
77
+ }
78
+ export function recordEscalationOutcome(session, category, outcome) {
79
+ const metadata = ensureSessionMetadata(session);
80
+ metadata.lastEscalation = {
81
+ at: new Date().toISOString(),
82
+ reason: category,
83
+ handlerOutcome: outcome.status,
84
+ };
85
+ }
@@ -1,3 +1,4 @@
1
+ import type { LanguageModel } from 'ai';
1
2
  export type EscalationReason = 'low-confidence' | 'user-request' | 'frustration' | 'tool-call' | 'safety-block';
2
3
  export type EscalationOutcome = {
3
4
  status: 'queued';
@@ -10,3 +11,42 @@ export type EscalationOutcome = {
10
11
  status: 'failed';
11
12
  error: string;
12
13
  };
14
+ /**
15
+ * The handoff package built when a conversation escalates to a human:
16
+ * everything the receiving agent needs to take over without re-asking.
17
+ */
18
+ export interface EscalationRequest {
19
+ sessionId: string;
20
+ userId?: string;
21
+ agentId: string;
22
+ /** Free-form reason from the triggering control (tool call, validator, flow transition). */
23
+ reason: string;
24
+ /** Typed category when the trigger provided one. */
25
+ category?: EscalationReason;
26
+ /** LLM-generated handoff brief (when summarization is enabled and a model is available). */
27
+ summary?: string;
28
+ /** Collected flow state snapshot (internal `__`-prefixed keys excluded). */
29
+ state: Record<string, unknown>;
30
+ /** Recent conversation tail as a text projection, oldest first. */
31
+ recentMessages: Array<{
32
+ role: string;
33
+ content: string;
34
+ }>;
35
+ /** ISO8601 timestamp. */
36
+ at: string;
37
+ }
38
+ /**
39
+ * Host-provided escalation sink: queue a ticket, claim thread ownership,
40
+ * page an operator. Must not throw for expected failures — return
41
+ * `{ status: 'failed', error }` instead (the runtime also catches).
42
+ */
43
+ export type EscalationHandler = (request: EscalationRequest) => Promise<EscalationOutcome>;
44
+ export interface EscalationConfig {
45
+ handler: EscalationHandler;
46
+ /** Generate the LLM handoff brief. Default: true. */
47
+ summarize?: boolean;
48
+ /** Summary model. Default: the active agent's controlModel → model → defaultModel. */
49
+ model?: LanguageModel;
50
+ /** Recent messages included in the request. Default: 12. */
51
+ recentMessageCount?: number;
52
+ }
@@ -0,0 +1,99 @@
1
+ import type { LanguageModel } from 'ai';
2
+ import type { TurnHandle } from '../types/stream.js';
3
+ import type { UserInputContent } from '../runtime/userInput.js';
4
+ /**
5
+ * Simulated-user evaluation: an LLM role-plays a customer persona against a
6
+ * real runtime, and an LLM judge scores the resulting transcript against a
7
+ * rubric. This is the pre-deploy gate that scripted turn assertions
8
+ * (`EvalRunner`) cannot provide — scripted turns test the happy path you
9
+ * thought of; simulated users find the conversations you didn't.
10
+ */
11
+ export interface SimulatedUserPersona {
12
+ /** Who the user is, e.g. "a busy parent in Colombo ordering a birthday cake". */
13
+ profile: string;
14
+ /** What they are trying to accomplish — the goal the judge scores against. */
15
+ goal: string;
16
+ /** Behavioral style, e.g. "impatient; sends short fragmented messages; switches topics". */
17
+ temperament?: string;
18
+ /** First user message. Generated from the persona when omitted. */
19
+ openingMessage?: string;
20
+ }
21
+ export interface SimulatedTranscriptTurn {
22
+ role: 'user' | 'assistant';
23
+ content: string;
24
+ }
25
+ export type SimulationEnd = 'goal-met' | 'user-gave-up' | 'max-turns';
26
+ export interface SimulationResult {
27
+ transcript: SimulatedTranscriptTurn[];
28
+ turns: number;
29
+ endedBy: SimulationEnd;
30
+ sessionId: string;
31
+ toolsCalled: string[];
32
+ escalated: boolean;
33
+ }
34
+ /** The runtime surface the simulator drives (satisfied by `Runtime`). */
35
+ export interface SimulatableRuntime {
36
+ run(opts: {
37
+ sessionId?: string;
38
+ input?: UserInputContent;
39
+ }): TurnHandle;
40
+ }
41
+ export interface SimulateConversationOptions {
42
+ runtime: SimulatableRuntime;
43
+ persona: SimulatedUserPersona;
44
+ /** Model that plays the user. */
45
+ userModel: LanguageModel;
46
+ /** Max user turns before the simulation stops. Default: 10. */
47
+ maxTurns?: number;
48
+ sessionId?: string;
49
+ }
50
+ export declare function simulateConversation(options: SimulateConversationOptions): Promise<SimulationResult>;
51
+ export interface JudgeDimension {
52
+ key: string;
53
+ description: string;
54
+ }
55
+ export declare const DEFAULT_JUDGE_DIMENSIONS: JudgeDimension[];
56
+ export interface JudgeVerdict {
57
+ scores: Record<string, {
58
+ score: number;
59
+ rationale: string;
60
+ }>;
61
+ /** Mean of dimension scores, 1–5. */
62
+ overall: number;
63
+ pass: boolean;
64
+ summary: string;
65
+ }
66
+ export interface CreateJudgeOptions {
67
+ model: LanguageModel;
68
+ dimensions?: JudgeDimension[];
69
+ /** Minimum mean score (1–5) to pass. Default: 3.5. */
70
+ passThreshold?: number;
71
+ /** Extra domain rules appended to the judge prompt. */
72
+ instructions?: string;
73
+ }
74
+ export interface ConversationJudge {
75
+ judge(result: SimulationResult, persona: SimulatedUserPersona): Promise<JudgeVerdict>;
76
+ }
77
+ export declare function createJudge(options: CreateJudgeOptions): ConversationJudge;
78
+ export interface SimulationScenario {
79
+ name: string;
80
+ persona: SimulatedUserPersona;
81
+ maxTurns?: number;
82
+ }
83
+ export interface SimulationSuiteResult {
84
+ scenarios: Array<{
85
+ name: string;
86
+ result: SimulationResult;
87
+ verdict: JudgeVerdict;
88
+ }>;
89
+ passed: boolean;
90
+ passRate: number;
91
+ }
92
+ export interface RunSimulationSuiteOptions {
93
+ runtime: SimulatableRuntime;
94
+ scenarios: SimulationScenario[];
95
+ userModel: LanguageModel;
96
+ judge: ConversationJudge;
97
+ }
98
+ /** Run every scenario and judge each transcript. `passed` is the CI gate. */
99
+ export declare function runSimulationSuite(options: RunSimulationSuiteOptions): Promise<SimulationSuiteResult>;
@@ -0,0 +1,168 @@
1
+ import { generateObject } from 'ai';
2
+ import { z } from 'zod';
3
+ import { newSessionId } from '../runtime/openRun.js';
4
+ const userTurnSchema = z.object({
5
+ /** Null when the user would stop talking instead of replying. */
6
+ message: z.union([z.string(), z.null()]),
7
+ status: z.enum(['continue', 'goal-met', 'give-up']),
8
+ });
9
+ function renderTranscript(transcript) {
10
+ return transcript
11
+ .map((turn) => `${turn.role === 'user' ? 'You' : 'Agent'}: ${turn.content}`)
12
+ .join('\n');
13
+ }
14
+ async function nextUserTurn(model, persona, transcript) {
15
+ const { object } = await generateObject({
16
+ model,
17
+ schema: userTurnSchema,
18
+ temperature: 0.7,
19
+ system: [
20
+ 'You are role-playing a customer talking to a business chat agent. Stay fully in character.',
21
+ `Who you are: ${persona.profile}`,
22
+ `Your goal: ${persona.goal}`,
23
+ persona.temperament ? `Your style: ${persona.temperament}` : '',
24
+ 'Write your next message as this customer would (short, natural chat messages — not essays).',
25
+ "Set status to 'goal-met' when the agent has fully accomplished your goal,",
26
+ "'give-up' if you would abandon the conversation in frustration, otherwise 'continue'.",
27
+ "When status is not 'continue', message may be null or a short closing line.",
28
+ ]
29
+ .filter(Boolean)
30
+ .join('\n'),
31
+ prompt: transcript.length === 0
32
+ ? 'Write your opening message to the agent.'
33
+ : `Conversation so far:\n${renderTranscript(transcript)}\n\nWrite your next message.`,
34
+ });
35
+ return object;
36
+ }
37
+ export async function simulateConversation(options) {
38
+ const maxTurns = options.maxTurns ?? 10;
39
+ const sessionId = options.sessionId ?? newSessionId();
40
+ const transcript = [];
41
+ const toolsCalled = [];
42
+ let escalated = false;
43
+ let endedBy = 'max-turns';
44
+ for (let turn = 0; turn < maxTurns; turn += 1) {
45
+ let userMessage;
46
+ if (turn === 0 && options.persona.openingMessage) {
47
+ userMessage = options.persona.openingMessage;
48
+ }
49
+ else {
50
+ const next = await nextUserTurn(options.userModel, options.persona, transcript);
51
+ if (next.status !== 'continue') {
52
+ if (next.message?.trim()) {
53
+ transcript.push({ role: 'user', content: next.message });
54
+ }
55
+ endedBy = next.status === 'goal-met' ? 'goal-met' : 'user-gave-up';
56
+ break;
57
+ }
58
+ if (!next.message?.trim()) {
59
+ endedBy = 'user-gave-up';
60
+ break;
61
+ }
62
+ userMessage = next.message;
63
+ }
64
+ transcript.push({ role: 'user', content: userMessage });
65
+ const handle = options.runtime.run({ sessionId, input: userMessage });
66
+ let reply = '';
67
+ for await (const part of handle.events) {
68
+ if (part.type === 'text-delta')
69
+ reply += part.delta;
70
+ if (part.type === 'tool-call')
71
+ toolsCalled.push(part.toolName);
72
+ if (part.type === 'escalation' || (part.type === 'handoff' && part.targetAgent === 'human')) {
73
+ escalated = true;
74
+ }
75
+ }
76
+ const result = await handle;
77
+ transcript.push({ role: 'assistant', content: reply || result.text });
78
+ }
79
+ return {
80
+ transcript,
81
+ turns: transcript.filter((turn) => turn.role === 'user').length,
82
+ endedBy,
83
+ sessionId,
84
+ toolsCalled,
85
+ escalated,
86
+ };
87
+ }
88
+ export const DEFAULT_JUDGE_DIMENSIONS = [
89
+ { key: 'goalCompletion', description: 'Did the agent fully accomplish what the user wanted?' },
90
+ {
91
+ key: 'grounding',
92
+ description: 'Did the agent avoid claiming actions or facts it did not perform/know (no invented orders, prices, policies)?',
93
+ },
94
+ { key: 'tone', description: 'Was the agent natural, concise, and appropriate for chat?' },
95
+ {
96
+ key: 'efficiency',
97
+ description: 'Did the agent resolve the request without unnecessary turns or repeated questions?',
98
+ },
99
+ ];
100
+ const judgeSchema = z.object({
101
+ scores: z.array(z.object({
102
+ key: z.string(),
103
+ /** 1 (poor) to 5 (excellent). */
104
+ score: z.number(),
105
+ rationale: z.string(),
106
+ })),
107
+ summary: z.string(),
108
+ });
109
+ export function createJudge(options) {
110
+ const dimensions = options.dimensions ?? DEFAULT_JUDGE_DIMENSIONS;
111
+ const passThreshold = options.passThreshold ?? 3.5;
112
+ return {
113
+ async judge(result, persona) {
114
+ const { object } = await generateObject({
115
+ model: options.model,
116
+ schema: judgeSchema,
117
+ temperature: 0,
118
+ system: [
119
+ 'You are an evaluation judge for customer-facing conversational agents.',
120
+ 'Score the AGENT (not the user) on each dimension from 1 (poor) to 5 (excellent), with a one-sentence rationale each:',
121
+ ...dimensions.map((dimension) => `- ${dimension.key}: ${dimension.description}`),
122
+ options.instructions ? `Additional rules:\n${options.instructions}` : '',
123
+ 'Be strict: claiming an action that has no evidence in the transcript is a grounding failure.',
124
+ ]
125
+ .filter(Boolean)
126
+ .join('\n'),
127
+ prompt: [
128
+ `User persona: ${persona.profile}`,
129
+ `User goal: ${persona.goal}`,
130
+ `Conversation ended by: ${result.endedBy}. Tools called: ${result.toolsCalled.join(', ') || 'none'}.`,
131
+ `Transcript:\n${renderTranscript(result.transcript)}`,
132
+ ].join('\n\n'),
133
+ });
134
+ const scores = {};
135
+ for (const entry of object.scores) {
136
+ scores[entry.key] = { score: entry.score, rationale: entry.rationale };
137
+ }
138
+ const values = Object.values(scores).map((entry) => entry.score);
139
+ const overall = values.length > 0 ? values.reduce((sum, score) => sum + score, 0) / values.length : 0;
140
+ return {
141
+ scores,
142
+ overall,
143
+ pass: overall >= passThreshold && result.endedBy !== 'user-gave-up',
144
+ summary: object.summary,
145
+ };
146
+ },
147
+ };
148
+ }
149
+ /** Run every scenario and judge each transcript. `passed` is the CI gate. */
150
+ export async function runSimulationSuite(options) {
151
+ const scenarios = [];
152
+ for (const scenario of options.scenarios) {
153
+ const result = await simulateConversation({
154
+ runtime: options.runtime,
155
+ persona: scenario.persona,
156
+ userModel: options.userModel,
157
+ maxTurns: scenario.maxTurns,
158
+ });
159
+ const verdict = await options.judge.judge(result, scenario.persona);
160
+ scenarios.push({ name: scenario.name, result, verdict });
161
+ }
162
+ const passedCount = scenarios.filter((entry) => entry.verdict.pass).length;
163
+ return {
164
+ scenarios,
165
+ passed: passedCount === scenarios.length,
166
+ passRate: scenarios.length > 0 ? passedCount / scenarios.length : 1,
167
+ };
168
+ }
package/dist/index.d.ts CHANGED
@@ -12,6 +12,8 @@ export { buildMarkOutcomeTool, OUTCOMES_MARK_TOOL_NAME } from './outcomes/index.
12
12
  export { EvalRunner } from './eval/EvalRunner.js';
13
13
  export { scoreTurn, aggregateScores } from './eval/scoring.js';
14
14
  export type { EvalScenario, EvalTurn, ScenarioScore, TurnScore } from './eval/types.js';
15
+ export { simulateConversation, createJudge, runSimulationSuite, DEFAULT_JUDGE_DIMENSIONS, } from './eval/simulation.js';
16
+ export type { SimulatedUserPersona, SimulatedTranscriptTurn, SimulationResult, SimulationEnd, SimulatableRuntime, SimulateConversationOptions, JudgeDimension, JudgeVerdict, CreateJudgeOptions, ConversationJudge, SimulationScenario, SimulationSuiteResult, RunSimulationSuiteOptions, } from './eval/simulation.js';
15
17
  export { SessionWorkingMemory } from './runtime/WorkingMemory.js';
16
18
  export type { Tool, ToolSet, ToolDefinition, ToolWithFiller, ToolExecutionOptions, ToolExecutionContext, } from './tools/Tool.js';
17
19
  export { createTool, createToolWithFiller } from './tools/Tool.js';
@@ -34,6 +36,14 @@ export type { Metrics } from './hooks/builtin/metrics.js';
34
36
  export { createObservabilityHooks } from './hooks/builtin/observability.js';
35
37
  export type { ObservabilityConfig } from './hooks/builtin/observability.js';
36
38
  export type { SessionTrace, TraceStreamEvent } from './types/telemetry.js';
39
+ export { createPromptInjectionGuard } from './processors/builtin/promptInjectionGuard.js';
40
+ export type { PromptInjectionGuardOptions } from './processors/builtin/promptInjectionGuard.js';
41
+ export { createPiiInputGuard, createPiiOutputGuard, redactPii, } from './processors/builtin/piiGuard.js';
42
+ export type { PiiDetector, PiiGuardOptions, PiiMatch, PiiScanResult, } from './processors/builtin/piiGuard.js';
43
+ export { createModerationGuard, createModerationOutputGuard, } from './processors/builtin/moderationGuard.js';
44
+ export type { ModerationGuardOptions } from './processors/builtin/moderationGuard.js';
45
+ export { createGroundingValidator } from './capabilities/validators/groundingValidator.js';
46
+ export type { GroundingValidatorOptions } from './capabilities/validators/groundingValidator.js';
37
47
  export { ToolEnforcer, createToolEnforcer } from './guards/ToolEnforcer.js';
38
48
  export * as StopConditions from './guards/StopConditions.js';
39
49
  export * from './guards/rules.js';
@@ -47,6 +57,8 @@ export type { MemoryEntry, SearchMemoryRequest, SearchMemoryResponse, MemoryInge
47
57
  export { InMemoryMemoryService } from './memory/index.js';
48
58
  export { preloadMemoryContext } from './memory/index.js';
49
59
  export { extractMemories } from './memory/index.js';
60
+ export { createFactMemoryService } from './memory/index.js';
61
+ export type { FactMemoryServiceOptions } from './memory/index.js';
50
62
  export type { PersistentMemoryStore, PersistentMemoryBlock, PersistentMemoryConfig, MemoryBlockScope, WorkingMemoryBlockSpec, WorkingMemoryConfig, } from './memory/index.js';
51
63
  export { FilePersistentMemoryStore, InMemoryPersistentMemoryStore, RoutedPersistentMemoryStore, TieredPersistentMemoryStore, scanMemoryWrite, buildMemoryBlockTool, DEFAULT_BLOCK_CHAR_LIMIT, DEFAULT_AUTO_LOAD_BLOCKS, } from './memory/index.js';
52
64
  export type { RoutedPersistentMemoryStoreConfig, MemoryRouteFn, } from './memory/index.js';
@@ -55,6 +67,10 @@ export { resolveAgentWorkspace, type ResolvedAgentWorkspace, } from './runtime/r
55
67
  export { DEFAULT_CONTEXT_BUDGET, VOICE_CONTEXT_BUDGET, computeMessageHistoryBudget, truncateToTokenBudget, formatMemoryWithBudget, estimateTokenCount, ContextBudget, } from './runtime/ContextBudget.js';
56
68
  export type { ContextBudgetConfig } from './runtime/ContextBudget.js';
57
69
  export { TokenAccumulator } from './runtime/TokenAccumulator.js';
70
+ export { compactMessages, estimateMessagesTokens, DEFAULT_COMPACTION_TRIGGER_TOKENS, DEFAULT_COMPACTION_KEEP_RECENT, } from './runtime/compaction.js';
71
+ export type { CompactionConfig, CompactionResult } from './runtime/compaction.js';
72
+ export { isContextOverflowError, recoverFromContextOverflow, } from './runtime/contextOverflow.js';
73
+ export type { OverflowRecoveryResult } from './runtime/contextOverflow.js';
58
74
  export { applyPromptCache, applyAnthropicCacheControl, buildOpenAIResponsesProviderOptions, isAnthropicLanguageModel, isOpenAIResponsesModel, } from './runtime/promptCache.js';
59
75
  export type { AnthropicCacheTtl, OpenAIResponsesCompactOptions } from './runtime/promptCache.js';
60
76
  export { handoffFilters, removeToolHistory, keepRecentMessages, removeKeys, composeFilters, } from './runtime/handoffFilters.js';
@@ -64,7 +80,10 @@ export type { ToolExecutor, ConversationState, ConversationEventLog, Conversatio
64
80
  export type * from './types/index.js';
65
81
  export { CapabilityHost, ExtractionCapability, GuardrailCapability, AutoRetrieveCapability, PassThroughRefinement, PassThroughValidation, toGeminiDeclarations, toAISDKTools, } from './capabilities/index.js';
66
82
  export type { Capability, CapabilityAction, ExtractionToolResponseEnvelope, FlowReconfigureTransition, ToolDeclaration, PromptSection as CapabilityPromptSection, GeminiFunctionDeclaration, ExtractionCapabilityConfig, RetrieveProvider, AutoRetrieveCapabilityConfig, RefinementCapability, RefineInput, RefineDecision, ValidationCapability, ValidateInput, ValidateDecision, SourceRef, } from './capabilities/index.js';
67
- export type { EscalationOutcome, EscalationReason } from './escalation/types.js';
83
+ export type { EscalationOutcome, EscalationReason, EscalationRequest, EscalationHandler, EscalationConfig, } from './escalation/types.js';
84
+ export { buildEscalationRequest, ensureSessionMetadata } from './escalation/escalation.js';
85
+ export { createInProcessScheduler, createWakeJobRunner, createScheduleFollowupTool, wakeJob, isWakeJob, WAKE_JOB_KIND, } from './scheduler/index.js';
86
+ export type { Scheduler, ScheduledJob, InjectableTimer, WakeOptions, WakeJobPayload, WakeDelivery, WakeRunnable, } from './scheduler/index.js';
68
87
  export { filterAuditEntries } from './audit/index.js';
69
88
  export type { AuditConfig, AuditEntryBase, AuditEntryType, AuditListOptions, AuditReplayOptions, ConversationAuditEntry, ConversationAuditLog, } from './audit/types.js';
70
89
  export type { RealtimeAudioClient, RealtimeSessionConfig, RealtimeAudioConfig, RealtimeToolResponse, RealtimeEventMap, RealtimeSessionHandle, } from './realtime/index.js';
@@ -95,7 +114,8 @@ export { DURABLE_RUNS_KEY } from './runtime/durable/types.js';
95
114
  export type { RunStore } from './runtime/durable/RunStore.js';
96
115
  export type { ChannelDriver, TextDriver } from './runtime/channels/index.js';
97
116
  export type { TurnResult } from './types/channel.js';
98
- export type { RunContext } from './types/run-context.js';
117
+ export type { RunContext, ToolContext, ActionContext } from './types/run-context.js';
118
+ export type { AnyTool } from './types/effectTool.js';
99
119
  export { createRuntime, Runtime, type HarnessConfig, type RunOptions, } from './runtime/Runtime.js';
100
120
  export type { RuntimeLike } from './runtime/RuntimeLike.js';
101
121
  export { userInputToText, hasMediaParts, transcribeAudioParts, type UserInputContent, } from './runtime/userInput.js';