@absolutejs/voice 0.0.22-beta.44 → 0.0.22-beta.46
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/evalRoutes.d.ts +37 -0
- package/dist/index.d.ts +4 -2
- package/dist/index.js +242 -0
- package/dist/trace.d.ts +1 -1
- package/dist/workflowContract.d.ts +85 -0
- package/package.json +1 -1
package/dist/evalRoutes.d.ts
CHANGED
|
@@ -75,6 +75,7 @@ export type VoiceScenarioEvalDefinition = {
|
|
|
75
75
|
requiredLifecycleTypes?: string[];
|
|
76
76
|
requiredPayloadPaths?: string[];
|
|
77
77
|
requiredTranscriptIncludes?: string[];
|
|
78
|
+
requiredWorkflowContracts?: string[];
|
|
78
79
|
scenarioId?: string;
|
|
79
80
|
};
|
|
80
81
|
export type VoiceScenarioEvalSessionResult = {
|
|
@@ -102,6 +103,30 @@ export type VoiceScenarioEvalReport = {
|
|
|
102
103
|
status: VoiceEvalStatus;
|
|
103
104
|
total: number;
|
|
104
105
|
};
|
|
106
|
+
export type VoiceScenarioFixture = {
|
|
107
|
+
description?: string;
|
|
108
|
+
events: StoredVoiceTraceEvent[];
|
|
109
|
+
id: string;
|
|
110
|
+
label?: string;
|
|
111
|
+
};
|
|
112
|
+
export type VoiceScenarioFixtureStore = {
|
|
113
|
+
list: () => Promise<VoiceScenarioFixture[]>;
|
|
114
|
+
};
|
|
115
|
+
export type VoiceScenarioFixtureEvalResult = {
|
|
116
|
+
description?: string;
|
|
117
|
+
fixtureId: string;
|
|
118
|
+
label: string;
|
|
119
|
+
report: VoiceScenarioEvalReport;
|
|
120
|
+
status: VoiceEvalStatus;
|
|
121
|
+
};
|
|
122
|
+
export type VoiceScenarioFixtureEvalReport = {
|
|
123
|
+
checkedAt: number;
|
|
124
|
+
failed: number;
|
|
125
|
+
fixtures: VoiceScenarioFixtureEvalResult[];
|
|
126
|
+
passed: number;
|
|
127
|
+
status: VoiceEvalStatus;
|
|
128
|
+
total: number;
|
|
129
|
+
};
|
|
105
130
|
export type VoiceEvalLink = {
|
|
106
131
|
href: string;
|
|
107
132
|
label: string;
|
|
@@ -111,6 +136,8 @@ export type VoiceEvalRoutesOptions = {
|
|
|
111
136
|
baselineComparison?: VoiceEvalBaselineComparisonOptions;
|
|
112
137
|
baselineStore?: VoiceEvalBaselineStore;
|
|
113
138
|
events?: StoredVoiceTraceEvent[];
|
|
139
|
+
fixtures?: VoiceScenarioFixture[];
|
|
140
|
+
fixtureStore?: VoiceScenarioFixtureStore;
|
|
114
141
|
headers?: HeadersInit;
|
|
115
142
|
links?: VoiceEvalLink[];
|
|
116
143
|
limit?: number;
|
|
@@ -132,8 +159,14 @@ export declare const runVoiceScenarioEvals: (options?: {
|
|
|
132
159
|
scenarios?: VoiceScenarioEvalDefinition[];
|
|
133
160
|
store?: VoiceTraceEventStore;
|
|
134
161
|
}) => Promise<VoiceScenarioEvalReport>;
|
|
162
|
+
export declare const runVoiceScenarioFixtureEvals: (options?: {
|
|
163
|
+
fixtures?: VoiceScenarioFixture[];
|
|
164
|
+
fixtureStore?: VoiceScenarioFixtureStore;
|
|
165
|
+
scenarios?: VoiceScenarioEvalDefinition[];
|
|
166
|
+
}) => Promise<VoiceScenarioFixtureEvalReport>;
|
|
135
167
|
export declare const compareVoiceEvalBaseline: (currentReport: VoiceEvalReport, baselineReport: VoiceEvalReport, options?: VoiceEvalBaselineComparisonOptions) => VoiceEvalBaselineComparison;
|
|
136
168
|
export declare const createVoiceFileEvalBaselineStore: (filePath: string) => VoiceEvalBaselineStore;
|
|
169
|
+
export declare const createVoiceFileScenarioFixtureStore: (filePath: string) => VoiceScenarioFixtureStore;
|
|
137
170
|
export declare const renderVoiceEvalHTML: (report: VoiceEvalReport, options?: {
|
|
138
171
|
links?: VoiceEvalLink[];
|
|
139
172
|
title?: string;
|
|
@@ -146,6 +179,10 @@ export declare const renderVoiceScenarioEvalHTML: (report: VoiceScenarioEvalRepo
|
|
|
146
179
|
links?: VoiceEvalLink[];
|
|
147
180
|
title?: string;
|
|
148
181
|
}) => string;
|
|
182
|
+
export declare const renderVoiceScenarioFixtureEvalHTML: (report: VoiceScenarioFixtureEvalReport, options?: {
|
|
183
|
+
links?: VoiceEvalLink[];
|
|
184
|
+
title?: string;
|
|
185
|
+
}) => string;
|
|
149
186
|
export declare const createVoiceEvalRoutes: (options: VoiceEvalRoutesOptions) => Elysia<"", {
|
|
150
187
|
decorator: {};
|
|
151
188
|
store: {};
|
package/dist/index.d.ts
CHANGED
|
@@ -2,7 +2,8 @@ export { voice } from './plugin';
|
|
|
2
2
|
export { createVoiceAssistant, createVoiceExperiment, summarizeVoiceAssistantRuns } from './assistant';
|
|
3
3
|
export { createVoiceAssistantHealthHTMLHandler, createVoiceAssistantHealthJSONHandler, createVoiceAssistantHealthRoutes, renderVoiceAssistantHealthHTML, summarizeVoiceAssistantHealth } from './assistantHealth';
|
|
4
4
|
export { buildVoiceDiagnosticsMarkdown, createVoiceDiagnosticsRoutes, resolveVoiceDiagnosticsTraceFilter } from './diagnosticsRoutes';
|
|
5
|
-
export { compareVoiceEvalBaseline, createVoiceFileEvalBaselineStore, createVoiceEvalRoutes, renderVoiceEvalBaselineHTML, renderVoiceEvalHTML, renderVoiceScenarioEvalHTML, runVoiceScenarioEvals, runVoiceSessionEvals } from './evalRoutes';
|
|
5
|
+
export { compareVoiceEvalBaseline, createVoiceFileEvalBaselineStore, createVoiceFileScenarioFixtureStore, createVoiceEvalRoutes, renderVoiceEvalBaselineHTML, renderVoiceEvalHTML, renderVoiceScenarioEvalHTML, renderVoiceScenarioFixtureEvalHTML, runVoiceScenarioEvals, runVoiceScenarioFixtureEvals, runVoiceSessionEvals } from './evalRoutes';
|
|
6
|
+
export { createVoiceWorkflowContract, createVoiceWorkflowContractHandler, createVoiceWorkflowScenario, recordVoiceWorkflowContractTrace, validateVoiceWorkflowRouteResult } from './workflowContract';
|
|
6
7
|
export { createVoiceSessionListRoutes, createVoiceSessionReplayHTMLHandler, createVoiceSessionReplayJSONHandler, createVoiceSessionReplayRoutes, createVoiceSessionsHTMLHandler, createVoiceSessionsJSONHandler, renderVoiceSessionsHTML, summarizeVoiceSessions, summarizeVoiceSessionReplay } from './sessionReplay';
|
|
7
8
|
export { createVoiceAgent, createVoiceAgentSquad, createVoiceAgentTool } from './agent';
|
|
8
9
|
export { createStoredVoiceCallReviewArtifact, createStoredVoiceExternalObjectMap, createStoredVoiceIntegrationEvent, createStoredVoiceOpsTask, createVoiceFileExternalObjectMapStore, createVoiceFileAssistantMemoryStore, createVoiceFileIntegrationEventStore, createVoiceFileReviewStore, createVoiceFileRuntimeStorage, createVoiceFileSessionStore, createVoiceFileTaskStore, createVoiceFileTraceSinkDeliveryStore, createVoiceFileTraceEventStore } from './fileStore';
|
|
@@ -40,7 +41,8 @@ export type { VoiceAssistant, VoiceAssistantArtifactPlan, VoiceAssistantExperime
|
|
|
40
41
|
export type { VoiceAssistantHealthFailure, VoiceAssistantHealthHTMLHandlerOptions, VoiceAssistantHealthRoutesOptions, VoiceAssistantHealthSummary, VoiceAssistantHealthSummaryOptions } from './assistantHealth';
|
|
41
42
|
export type { VoiceAssistantMemoryBinding, VoiceAssistantMemoryHandle, VoiceAssistantMemoryOptions, VoiceAssistantMemoryRecord, VoiceAssistantMemoryStore } from './assistantMemory';
|
|
42
43
|
export type { VoiceDiagnosticsRoutesOptions } from './diagnosticsRoutes';
|
|
43
|
-
export type { VoiceEvalBaselineComparison, VoiceEvalBaselineComparisonOptions, VoiceEvalBaselineStore, VoiceEvalBaselineSummary, VoiceEvalLink, VoiceEvalReport, VoiceEvalRoutesOptions, VoiceEvalSessionReport, VoiceEvalStatus, VoiceEvalTrendBucket, VoiceScenarioEvalDefinition, VoiceScenarioEvalReport, VoiceScenarioEvalResult, VoiceScenarioEvalSessionResult } from './evalRoutes';
|
|
44
|
+
export type { VoiceEvalBaselineComparison, VoiceEvalBaselineComparisonOptions, VoiceEvalBaselineStore, VoiceEvalBaselineSummary, VoiceEvalLink, VoiceEvalReport, VoiceEvalRoutesOptions, VoiceEvalSessionReport, VoiceEvalStatus, VoiceEvalTrendBucket, VoiceScenarioEvalDefinition, VoiceScenarioEvalReport, VoiceScenarioEvalResult, VoiceScenarioEvalSessionResult, VoiceScenarioFixture, VoiceScenarioFixtureEvalReport, VoiceScenarioFixtureEvalResult, VoiceScenarioFixtureStore } from './evalRoutes';
|
|
45
|
+
export type { VoiceWorkflowContract, VoiceWorkflowContractDefinition, VoiceWorkflowContractField, VoiceWorkflowContractFieldMatch, VoiceWorkflowContractTracePayload, VoiceWorkflowContractValidation, VoiceWorkflowContractValidationIssue, VoiceWorkflowOutcome } from './workflowContract';
|
|
44
46
|
export type { VoiceSessionListHTMLHandlerOptions, VoiceSessionListItem, VoiceSessionListOptions, VoiceSessionListRoutesOptions, VoiceSessionListStatus, VoiceSessionReplay, VoiceSessionReplayHTMLHandlerOptions, VoiceSessionReplayOptions, VoiceSessionReplayRoutesOptions, VoiceSessionReplayTurn } from './sessionReplay';
|
|
45
47
|
export type { AnthropicVoiceAssistantModelOptions, GeminiVoiceAssistantModelOptions, OpenAIVoiceAssistantModelOptions, VoiceProviderRouterEvent, VoiceProviderRouterFallbackMode, VoiceProviderRouterHealthOptions, VoiceProviderRouterOptions, VoiceProviderRouterPolicy, VoiceProviderRouterProviderHealth, VoiceProviderRouterProviderProfile, VoiceJSONAssistantModelHandler, VoiceJSONAssistantModelOptions } from './modelAdapters';
|
|
46
48
|
export type { VoiceProviderHealthStatus, VoiceProviderHealthSummary, VoiceProviderHealthSummaryOptions } from './providerHealth';
|
package/dist/index.js
CHANGED
|
@@ -8078,6 +8078,7 @@ var evaluateScenarioSession = (scenario, sessionId, events) => {
|
|
|
8078
8078
|
const turnCount = events.filter((event) => event.type === "turn.committed").length;
|
|
8079
8079
|
const sessionErrorCount = events.filter((event) => event.type === "session.error").length;
|
|
8080
8080
|
const providerErrorCount = countProviderErrors(events);
|
|
8081
|
+
const workflowContractEvents = events.filter((event) => event.type === "workflow.contract");
|
|
8081
8082
|
for (const missing of includesAll(committedText, scenario.requiredTranscriptIncludes ?? [])) {
|
|
8082
8083
|
issues.push(`Missing transcript text: ${missing}`);
|
|
8083
8084
|
}
|
|
@@ -8121,6 +8122,16 @@ var evaluateScenarioSession = (scenario, sessionId, events) => {
|
|
|
8121
8122
|
issues.push(`Missing payload path: ${path}`);
|
|
8122
8123
|
}
|
|
8123
8124
|
}
|
|
8125
|
+
for (const contractId of scenario.requiredWorkflowContracts ?? []) {
|
|
8126
|
+
const matching = workflowContractEvents.filter((event) => getString6(event.payload.contractId) === contractId);
|
|
8127
|
+
if (matching.length === 0) {
|
|
8128
|
+
issues.push(`Missing workflow contract: ${contractId}`);
|
|
8129
|
+
continue;
|
|
8130
|
+
}
|
|
8131
|
+
if (matching.some((event) => getString6(event.payload.status) !== "pass")) {
|
|
8132
|
+
issues.push(`Workflow contract failed: ${contractId}`);
|
|
8133
|
+
}
|
|
8134
|
+
}
|
|
8124
8135
|
return {
|
|
8125
8136
|
eventCount: events.length,
|
|
8126
8137
|
issues,
|
|
@@ -8167,6 +8178,33 @@ var runVoiceScenarioEvals = async (options = {}) => {
|
|
|
8167
8178
|
total: results.length
|
|
8168
8179
|
};
|
|
8169
8180
|
};
|
|
8181
|
+
var resolveScenarioFixtures = async (options) => [...options.fixtures ?? [], ...await options.fixtureStore?.list() ?? []];
|
|
8182
|
+
var runVoiceScenarioFixtureEvals = async (options = {}) => {
|
|
8183
|
+
const fixtures = await resolveScenarioFixtures(options);
|
|
8184
|
+
const results = await Promise.all(fixtures.map(async (fixture) => {
|
|
8185
|
+
const report = await runVoiceScenarioEvals({
|
|
8186
|
+
events: fixture.events,
|
|
8187
|
+
scenarios: options.scenarios
|
|
8188
|
+
});
|
|
8189
|
+
return {
|
|
8190
|
+
description: fixture.description,
|
|
8191
|
+
fixtureId: fixture.id,
|
|
8192
|
+
label: fixture.label ?? fixture.id,
|
|
8193
|
+
report,
|
|
8194
|
+
status: report.status
|
|
8195
|
+
};
|
|
8196
|
+
}));
|
|
8197
|
+
const failed = results.filter((fixture) => fixture.status === "fail").length;
|
|
8198
|
+
const passed = results.length - failed;
|
|
8199
|
+
return {
|
|
8200
|
+
checkedAt: Date.now(),
|
|
8201
|
+
failed,
|
|
8202
|
+
fixtures: results,
|
|
8203
|
+
passed,
|
|
8204
|
+
status: failed > 0 ? "fail" : "pass",
|
|
8205
|
+
total: results.length
|
|
8206
|
+
};
|
|
8207
|
+
};
|
|
8170
8208
|
var summarizeEvalBaseline = (report) => {
|
|
8171
8209
|
const failedSessionIds = report.sessions.filter((session) => session.status === "fail").map((session) => session.sessionId).sort();
|
|
8172
8210
|
return {
|
|
@@ -8228,6 +8266,20 @@ var createVoiceFileEvalBaselineStore = (filePath) => ({
|
|
|
8228
8266
|
await Bun.write(filePath, JSON.stringify(report, null, 2));
|
|
8229
8267
|
}
|
|
8230
8268
|
});
|
|
8269
|
+
var createVoiceFileScenarioFixtureStore = (filePath) => ({
|
|
8270
|
+
list: async () => {
|
|
8271
|
+
const file = Bun.file(filePath);
|
|
8272
|
+
if (!await file.exists()) {
|
|
8273
|
+
return [];
|
|
8274
|
+
}
|
|
8275
|
+
const text = await file.text();
|
|
8276
|
+
if (!text.trim()) {
|
|
8277
|
+
return [];
|
|
8278
|
+
}
|
|
8279
|
+
const parsed = JSON.parse(text);
|
|
8280
|
+
return Array.isArray(parsed) ? parsed : parsed.fixtures ?? [];
|
|
8281
|
+
}
|
|
8282
|
+
});
|
|
8231
8283
|
var formatTime = (value) => value === undefined ? "unknown" : new Date(value).toLocaleString();
|
|
8232
8284
|
var formatPercent = (value) => `${(value * 100).toFixed(2)}%`;
|
|
8233
8285
|
var renderVoiceEvalHTML = (report, options = {}) => {
|
|
@@ -8258,6 +8310,15 @@ var renderVoiceScenarioEvalHTML = (report, options = {}) => {
|
|
|
8258
8310
|
}).join("") : "<section><p>No scenarios configured.</p></section>";
|
|
8259
8311
|
return `<!doctype html><html lang="en"><head><meta charset="utf-8" /><meta name="viewport" content="width=device-width, initial-scale=1" /><title>${escapeHtml9(title)}</title><style>body{font-family:ui-sans-serif,system-ui,sans-serif;margin:2rem;background:#f8f7f2;color:#181713}main{max-width:1180px;margin:auto}nav{display:flex;gap:.5rem;flex-wrap:wrap;margin-bottom:1rem}nav a{background:#181713;border-radius:999px;color:white;padding:.35rem .7rem;text-decoration:none}.status{border-radius:999px;display:inline-flex;font-weight:800;padding:.35rem .75rem}.status.pass{background:#dcfce7;color:#166534}.status.fail{background:#fee2e2;color:#991b1b}.grid{display:grid;gap:1rem;grid-template-columns:repeat(auto-fit,minmax(160px,1fr));margin:1rem 0}.card,section{background:white;border:1px solid #e7e5e4;border-radius:1rem;padding:1rem}.card strong{display:block;font-size:2rem}section{margin:1rem 0}table{border-collapse:collapse;width:100%;margin-top:1rem}td,th{border-bottom:1px solid #eee;padding:.75rem;text-align:left}tr.fail td{border-left:4px solid #dc2626}tr.pass td{border-left:4px solid #16a34a}</style></head><body><main>${links}<h1>${escapeHtml9(title)}</h1><p class="status ${report.status}">${report.status}</p><div class="grid"><article class="card"><span>Total</span><strong>${report.total}</strong></article><article class="card"><span>Passed</span><strong>${report.passed}</strong></article><article class="card"><span>Failed</span><strong>${report.failed}</strong></article></div>${scenarios}</main></body></html>`;
|
|
8260
8312
|
};
|
|
8313
|
+
var renderVoiceScenarioFixtureEvalHTML = (report, options = {}) => {
|
|
8314
|
+
const title = options.title ?? "AbsoluteJS Voice Fixture Evals";
|
|
8315
|
+
const links = options.links?.length ? `<nav>${options.links.map((link) => `<a href="${escapeHtml9(link.href)}">${escapeHtml9(link.label)}</a>`).join("")}</nav>` : "";
|
|
8316
|
+
const fixtures = report.fixtures.length ? report.fixtures.map((fixture) => {
|
|
8317
|
+
const scenarios = fixture.report.scenarios.map((scenario) => `<tr class="${scenario.status}"><td>${escapeHtml9(scenario.label)}</td><td>${escapeHtml9(scenario.status)}</td><td>${scenario.matchedSessions}</td><td>${escapeHtml9([...scenario.issues, ...scenario.sessions.flatMap((session) => session.issues)].join(", ") || "none")}</td></tr>`).join("");
|
|
8318
|
+
return `<section class="${fixture.status}"><h2>${escapeHtml9(fixture.label)}</h2>${fixture.description ? `<p>${escapeHtml9(fixture.description)}</p>` : ""}<p class="status ${fixture.status}">${fixture.status}</p><table><thead><tr><th>Scenario</th><th>Status</th><th>Sessions</th><th>Issues</th></tr></thead><tbody>${scenarios}</tbody></table></section>`;
|
|
8319
|
+
}).join("") : "<section><p>No scenario fixtures configured.</p></section>";
|
|
8320
|
+
return `<!doctype html><html lang="en"><head><meta charset="utf-8" /><meta name="viewport" content="width=device-width, initial-scale=1" /><title>${escapeHtml9(title)}</title><style>body{font-family:ui-sans-serif,system-ui,sans-serif;margin:2rem;background:#f8f7f2;color:#181713}main{max-width:1180px;margin:auto}nav{display:flex;gap:.5rem;flex-wrap:wrap;margin-bottom:1rem}nav a{background:#181713;border-radius:999px;color:white;padding:.35rem .7rem;text-decoration:none}.status{border-radius:999px;display:inline-flex;font-weight:800;padding:.35rem .75rem}.status.pass{background:#dcfce7;color:#166534}.status.fail{background:#fee2e2;color:#991b1b}.grid{display:grid;gap:1rem;grid-template-columns:repeat(auto-fit,minmax(160px,1fr));margin:1rem 0}.card,section{background:white;border:1px solid #e7e5e4;border-radius:1rem;padding:1rem}.card strong{display:block;font-size:2rem}section{margin:1rem 0}table{border-collapse:collapse;width:100%;margin-top:1rem}td,th{border-bottom:1px solid #eee;padding:.75rem;text-align:left}tr.fail td{border-left:4px solid #dc2626}tr.pass td{border-left:4px solid #16a34a}</style></head><body><main>${links}<h1>${escapeHtml9(title)}</h1><p class="status ${report.status}">${report.status}</p><div class="grid"><article class="card"><span>Total</span><strong>${report.total}</strong></article><article class="card"><span>Passed</span><strong>${report.passed}</strong></article><article class="card"><span>Failed</span><strong>${report.failed}</strong></article></div>${fixtures}</main></body></html>`;
|
|
8321
|
+
};
|
|
8261
8322
|
var createVoiceEvalRoutes = (options) => {
|
|
8262
8323
|
const path = options.path ?? "/evals";
|
|
8263
8324
|
const routes = new Elysia7({
|
|
@@ -8279,6 +8340,11 @@ var createVoiceEvalRoutes = (options) => {
|
|
|
8279
8340
|
scenarios: options.scenarios,
|
|
8280
8341
|
store: options.store
|
|
8281
8342
|
});
|
|
8343
|
+
const getFixtureReport = () => runVoiceScenarioFixtureEvals({
|
|
8344
|
+
fixtures: options.fixtures,
|
|
8345
|
+
fixtureStore: options.fixtureStore,
|
|
8346
|
+
scenarios: options.scenarios
|
|
8347
|
+
});
|
|
8282
8348
|
routes.get(path, async () => {
|
|
8283
8349
|
const report = await getReport();
|
|
8284
8350
|
return new Response(renderVoiceEvalHTML(report, {
|
|
@@ -8366,8 +8432,176 @@ var createVoiceEvalRoutes = (options) => {
|
|
|
8366
8432
|
}
|
|
8367
8433
|
return report;
|
|
8368
8434
|
});
|
|
8435
|
+
routes.get(`${path}/fixtures`, async () => {
|
|
8436
|
+
const report = await getFixtureReport();
|
|
8437
|
+
return new Response(renderVoiceScenarioFixtureEvalHTML(report, {
|
|
8438
|
+
links: options.links,
|
|
8439
|
+
title: `${options.title ?? "AbsoluteJS Voice Evals"} Fixtures`
|
|
8440
|
+
}), {
|
|
8441
|
+
headers: {
|
|
8442
|
+
"Content-Type": "text/html; charset=utf-8",
|
|
8443
|
+
...options.headers
|
|
8444
|
+
}
|
|
8445
|
+
});
|
|
8446
|
+
});
|
|
8447
|
+
routes.get(`${path}/fixtures/json`, async () => getFixtureReport());
|
|
8448
|
+
routes.get(`${path}/fixtures/status`, async ({ set }) => {
|
|
8449
|
+
const report = await getFixtureReport();
|
|
8450
|
+
if (report.status === "fail") {
|
|
8451
|
+
set.status = 503;
|
|
8452
|
+
}
|
|
8453
|
+
return report;
|
|
8454
|
+
});
|
|
8369
8455
|
return routes;
|
|
8370
8456
|
};
|
|
8457
|
+
// src/workflowContract.ts
|
|
8458
|
+
var getObject2 = (value) => value && typeof value === "object" && !Array.isArray(value) ? value : undefined;
|
|
8459
|
+
var getPathValue2 = (value, path) => {
|
|
8460
|
+
let current = value;
|
|
8461
|
+
for (const part of path.split(".").filter(Boolean)) {
|
|
8462
|
+
const record = getObject2(current);
|
|
8463
|
+
if (!record || !(part in record)) {
|
|
8464
|
+
return;
|
|
8465
|
+
}
|
|
8466
|
+
current = record[part];
|
|
8467
|
+
}
|
|
8468
|
+
return current;
|
|
8469
|
+
};
|
|
8470
|
+
var hasValue = (value, match) => {
|
|
8471
|
+
switch (match) {
|
|
8472
|
+
case "boolean":
|
|
8473
|
+
return typeof value === "boolean";
|
|
8474
|
+
case "number":
|
|
8475
|
+
return typeof value === "number" && Number.isFinite(value);
|
|
8476
|
+
case "string":
|
|
8477
|
+
return typeof value === "string";
|
|
8478
|
+
case "truthy":
|
|
8479
|
+
return Boolean(value);
|
|
8480
|
+
case "non-empty":
|
|
8481
|
+
default:
|
|
8482
|
+
return Array.isArray(value) ? value.length > 0 : typeof value === "string" ? value.trim().length > 0 : value !== undefined && value !== null;
|
|
8483
|
+
}
|
|
8484
|
+
};
|
|
8485
|
+
var resolveOutcome2 = (routeResult) => {
|
|
8486
|
+
if (routeResult.complete)
|
|
8487
|
+
return "complete";
|
|
8488
|
+
if (routeResult.transfer)
|
|
8489
|
+
return "transfer";
|
|
8490
|
+
if (routeResult.escalate)
|
|
8491
|
+
return "escalate";
|
|
8492
|
+
if (routeResult.voicemail)
|
|
8493
|
+
return "voicemail";
|
|
8494
|
+
if (routeResult.noAnswer)
|
|
8495
|
+
return "no-answer";
|
|
8496
|
+
return;
|
|
8497
|
+
};
|
|
8498
|
+
var validateVoiceWorkflowRouteResult = (definition, routeResult) => {
|
|
8499
|
+
const issues = [];
|
|
8500
|
+
const requiredFields = (definition.fields ?? []).filter((field) => field.required !== false).map((field) => field.path);
|
|
8501
|
+
const missingFields = [];
|
|
8502
|
+
const outcome = resolveOutcome2(routeResult);
|
|
8503
|
+
if (definition.outcome && outcome !== definition.outcome) {
|
|
8504
|
+
issues.push({
|
|
8505
|
+
code: "workflow.outcome_mismatch",
|
|
8506
|
+
message: `Expected workflow outcome ${definition.outcome}, saw ${outcome ?? "none"}.`
|
|
8507
|
+
});
|
|
8508
|
+
}
|
|
8509
|
+
for (const field of definition.fields ?? []) {
|
|
8510
|
+
if (field.required === false)
|
|
8511
|
+
continue;
|
|
8512
|
+
const paths = [field.path, ...field.aliases ?? []];
|
|
8513
|
+
const present = paths.some((path) => hasValue(getPathValue2(routeResult.result, path), field.match ?? "non-empty"));
|
|
8514
|
+
if (!present) {
|
|
8515
|
+
missingFields.push(field.path);
|
|
8516
|
+
issues.push({
|
|
8517
|
+
code: "workflow.missing_field",
|
|
8518
|
+
field: field.path,
|
|
8519
|
+
message: `Missing required workflow field: ${field.label ?? field.path}.`
|
|
8520
|
+
});
|
|
8521
|
+
}
|
|
8522
|
+
}
|
|
8523
|
+
issues.push(...definition.validate?.({
|
|
8524
|
+
result: routeResult.result,
|
|
8525
|
+
routeResult
|
|
8526
|
+
}) ?? []);
|
|
8527
|
+
return {
|
|
8528
|
+
contractId: definition.id,
|
|
8529
|
+
issues,
|
|
8530
|
+
missingFields,
|
|
8531
|
+
outcome,
|
|
8532
|
+
pass: issues.length === 0,
|
|
8533
|
+
requiredFields
|
|
8534
|
+
};
|
|
8535
|
+
};
|
|
8536
|
+
var createVoiceWorkflowScenario = (definition, overrides = {}) => ({
|
|
8537
|
+
description: definition.description,
|
|
8538
|
+
forbiddenHandoffActions: definition.forbiddenHandoffActions,
|
|
8539
|
+
id: definition.id,
|
|
8540
|
+
label: definition.label,
|
|
8541
|
+
maxProviderErrors: definition.maxProviderErrors,
|
|
8542
|
+
maxSessionErrors: definition.maxSessionErrors,
|
|
8543
|
+
minSessions: definition.minSessions,
|
|
8544
|
+
minTurns: definition.minTurns,
|
|
8545
|
+
requiredAssistantIncludes: definition.requiredAssistantIncludes,
|
|
8546
|
+
requiredDisposition: definition.requiredDisposition,
|
|
8547
|
+
requiredHandoffActions: definition.requiredHandoffActions,
|
|
8548
|
+
requiredLifecycleTypes: definition.requiredLifecycleTypes,
|
|
8549
|
+
requiredTranscriptIncludes: definition.requiredTranscriptIncludes,
|
|
8550
|
+
requiredWorkflowContracts: [definition.id],
|
|
8551
|
+
scenarioId: definition.scenarioId,
|
|
8552
|
+
...overrides
|
|
8553
|
+
});
|
|
8554
|
+
var createVoiceWorkflowContract = (definition) => ({
|
|
8555
|
+
assertRouteResult: (routeResult) => {
|
|
8556
|
+
const validation = validateVoiceWorkflowRouteResult(definition, routeResult);
|
|
8557
|
+
if (!validation.pass) {
|
|
8558
|
+
throw new Error(`Voice workflow contract ${definition.id} failed: ${validation.issues.map((issue) => issue.message).join(" ")}`);
|
|
8559
|
+
}
|
|
8560
|
+
},
|
|
8561
|
+
definition,
|
|
8562
|
+
toScenarioEval: (overrides) => createVoiceWorkflowScenario(definition, overrides),
|
|
8563
|
+
validateRouteResult: (routeResult) => validateVoiceWorkflowRouteResult(definition, routeResult)
|
|
8564
|
+
});
|
|
8565
|
+
var recordVoiceWorkflowContractTrace = async (input) => input.store.append({
|
|
8566
|
+
at: input.at ?? Date.now(),
|
|
8567
|
+
payload: {
|
|
8568
|
+
contractId: input.contractId ?? input.validation.contractId,
|
|
8569
|
+
issues: input.validation.issues,
|
|
8570
|
+
missingFields: input.validation.missingFields,
|
|
8571
|
+
outcome: input.validation.outcome,
|
|
8572
|
+
requiredFields: input.validation.requiredFields,
|
|
8573
|
+
status: input.validation.pass ? "pass" : "fail"
|
|
8574
|
+
},
|
|
8575
|
+
scenarioId: input.scenarioId,
|
|
8576
|
+
sessionId: input.sessionId,
|
|
8577
|
+
traceId: input.traceId,
|
|
8578
|
+
turnId: input.turnId,
|
|
8579
|
+
type: "workflow.contract"
|
|
8580
|
+
});
|
|
8581
|
+
var createVoiceWorkflowContractHandler = (input) => {
|
|
8582
|
+
return async (session, turn, api, context) => {
|
|
8583
|
+
const legacyHandler = input.handler;
|
|
8584
|
+
const objectHandler = input.handler;
|
|
8585
|
+
const result = input.handler.length >= 4 ? await legacyHandler(session, turn, api, context) : await objectHandler({ api, context, session, turn });
|
|
8586
|
+
if (!result)
|
|
8587
|
+
return result;
|
|
8588
|
+
const resolved = input.resolveContract?.({ context, result, session, turn }) ?? input.contract;
|
|
8589
|
+
if (!resolved)
|
|
8590
|
+
return result;
|
|
8591
|
+
const contract = "validateRouteResult" in resolved ? resolved : createVoiceWorkflowContract(resolved);
|
|
8592
|
+
const validation = contract.validateRouteResult(result);
|
|
8593
|
+
if (input.store) {
|
|
8594
|
+
await recordVoiceWorkflowContractTrace({
|
|
8595
|
+
scenarioId: session.scenarioId,
|
|
8596
|
+
sessionId: session.id,
|
|
8597
|
+
store: input.store,
|
|
8598
|
+
turnId: turn.id,
|
|
8599
|
+
validation
|
|
8600
|
+
});
|
|
8601
|
+
}
|
|
8602
|
+
return result;
|
|
8603
|
+
};
|
|
8604
|
+
};
|
|
8371
8605
|
// src/sessionReplay.ts
|
|
8372
8606
|
import { Elysia as Elysia8 } from "elysia";
|
|
8373
8607
|
var getString7 = (value) => typeof value === "string" ? value : undefined;
|
|
@@ -13293,6 +13527,7 @@ export {
|
|
|
13293
13527
|
withVoiceIntegrationEventId,
|
|
13294
13528
|
voice,
|
|
13295
13529
|
verifyVoiceOpsWebhookSignature,
|
|
13530
|
+
validateVoiceWorkflowRouteResult,
|
|
13296
13531
|
transcodeTwilioInboundPayloadToPCM16,
|
|
13297
13532
|
transcodePCMToTwilioOutboundPayload,
|
|
13298
13533
|
summarizeVoiceTraceSinkDeliveries,
|
|
@@ -13312,6 +13547,7 @@ export {
|
|
|
13312
13547
|
shapeTelephonyAssistantText,
|
|
13313
13548
|
selectVoiceTraceEventsForPrune,
|
|
13314
13549
|
runVoiceSessionEvals,
|
|
13550
|
+
runVoiceScenarioFixtureEvals,
|
|
13315
13551
|
runVoiceScenarioEvals,
|
|
13316
13552
|
resolveVoiceTraceRedactionOptions,
|
|
13317
13553
|
resolveVoiceSTTRoutingStrategy,
|
|
@@ -13330,6 +13566,7 @@ export {
|
|
|
13330
13566
|
renderVoiceTraceMarkdown,
|
|
13331
13567
|
renderVoiceTraceHTML,
|
|
13332
13568
|
renderVoiceSessionsHTML,
|
|
13569
|
+
renderVoiceScenarioFixtureEvalHTML,
|
|
13333
13570
|
renderVoiceScenarioEvalHTML,
|
|
13334
13571
|
renderVoiceResilienceHTML,
|
|
13335
13572
|
renderVoiceQualityHTML,
|
|
@@ -13344,6 +13581,7 @@ export {
|
|
|
13344
13581
|
redactVoiceTraceText,
|
|
13345
13582
|
redactVoiceTraceEvents,
|
|
13346
13583
|
redactVoiceTraceEvent,
|
|
13584
|
+
recordVoiceWorkflowContractTrace,
|
|
13347
13585
|
recordVoiceRuntimeOps,
|
|
13348
13586
|
pruneVoiceTraceEvents,
|
|
13349
13587
|
matchesVoiceOpsTaskAssignmentRule,
|
|
@@ -13369,6 +13607,9 @@ export {
|
|
|
13369
13607
|
createVoiceZendeskTicketUpdateSink,
|
|
13370
13608
|
createVoiceZendeskTicketSyncSinks,
|
|
13371
13609
|
createVoiceZendeskTicketSink,
|
|
13610
|
+
createVoiceWorkflowScenario,
|
|
13611
|
+
createVoiceWorkflowContractHandler,
|
|
13612
|
+
createVoiceWorkflowContract,
|
|
13372
13613
|
createVoiceWebhookHandoffAdapter,
|
|
13373
13614
|
createVoiceWebhookDeliveryWorkerLoop,
|
|
13374
13615
|
createVoiceWebhookDeliveryWorker,
|
|
@@ -13455,6 +13696,7 @@ export {
|
|
|
13455
13696
|
createVoiceFileTraceEventStore,
|
|
13456
13697
|
createVoiceFileTaskStore,
|
|
13457
13698
|
createVoiceFileSessionStore,
|
|
13699
|
+
createVoiceFileScenarioFixtureStore,
|
|
13458
13700
|
createVoiceFileRuntimeStorage,
|
|
13459
13701
|
createVoiceFileReviewStore,
|
|
13460
13702
|
createVoiceFileIntegrationEventStore,
|
package/dist/trace.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
export type VoiceTraceEventType = 'assistant.guardrail' | 'assistant.memory' | 'assistant.run' | 'agent.handoff' | 'agent.model' | 'agent.result' | 'agent.tool' | 'call.handoff' | 'call.lifecycle' | 'session.error' | 'turn.assistant' | 'turn.committed' | 'turn.cost' | 'turn.transcript';
|
|
1
|
+
export type VoiceTraceEventType = 'assistant.guardrail' | 'assistant.memory' | 'assistant.run' | 'agent.handoff' | 'agent.model' | 'agent.result' | 'agent.tool' | 'call.handoff' | 'call.lifecycle' | 'session.error' | 'turn.assistant' | 'turn.committed' | 'turn.cost' | 'turn.transcript' | 'workflow.contract';
|
|
2
2
|
export type VoiceTraceEvent<TPayload extends Record<string, unknown> = Record<string, unknown>> = {
|
|
3
3
|
at: number;
|
|
4
4
|
id?: string;
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
import type { VoiceScenarioEvalDefinition } from './evalRoutes';
|
|
2
|
+
import type { StoredVoiceTraceEvent, VoiceTraceEventStore } from './trace';
|
|
3
|
+
import type { VoiceOnTurnHandler, VoiceRouteResult, VoiceSessionRecord, VoiceTurnRecord } from './types';
|
|
4
|
+
export type VoiceWorkflowOutcome = 'complete' | 'transfer' | 'escalate' | 'voicemail' | 'no-answer';
|
|
5
|
+
export type VoiceWorkflowContractFieldMatch = 'boolean' | 'non-empty' | 'number' | 'string' | 'truthy';
|
|
6
|
+
export type VoiceWorkflowContractField = {
|
|
7
|
+
aliases?: string[];
|
|
8
|
+
label?: string;
|
|
9
|
+
match?: VoiceWorkflowContractFieldMatch;
|
|
10
|
+
path: string;
|
|
11
|
+
required?: boolean;
|
|
12
|
+
};
|
|
13
|
+
export type VoiceWorkflowContractDefinition<TResult = unknown> = {
|
|
14
|
+
description?: string;
|
|
15
|
+
fields?: VoiceWorkflowContractField[];
|
|
16
|
+
forbiddenHandoffActions?: string[];
|
|
17
|
+
id: string;
|
|
18
|
+
label?: string;
|
|
19
|
+
maxProviderErrors?: number;
|
|
20
|
+
maxSessionErrors?: number;
|
|
21
|
+
minSessions?: number;
|
|
22
|
+
minTurns?: number;
|
|
23
|
+
outcome?: VoiceWorkflowOutcome;
|
|
24
|
+
requiredAssistantIncludes?: string[];
|
|
25
|
+
requiredDisposition?: string;
|
|
26
|
+
requiredHandoffActions?: string[];
|
|
27
|
+
requiredLifecycleTypes?: string[];
|
|
28
|
+
requiredTranscriptIncludes?: string[];
|
|
29
|
+
scenarioId?: string;
|
|
30
|
+
validate?: (input: {
|
|
31
|
+
result: TResult | undefined;
|
|
32
|
+
routeResult: VoiceRouteResult<TResult>;
|
|
33
|
+
}) => VoiceWorkflowContractValidationIssue[];
|
|
34
|
+
};
|
|
35
|
+
export type VoiceWorkflowContractValidationIssue = {
|
|
36
|
+
code: string;
|
|
37
|
+
field?: string;
|
|
38
|
+
message: string;
|
|
39
|
+
};
|
|
40
|
+
export type VoiceWorkflowContractValidation = {
|
|
41
|
+
contractId: string;
|
|
42
|
+
issues: VoiceWorkflowContractValidationIssue[];
|
|
43
|
+
missingFields: string[];
|
|
44
|
+
outcome?: VoiceWorkflowOutcome;
|
|
45
|
+
pass: boolean;
|
|
46
|
+
requiredFields: string[];
|
|
47
|
+
};
|
|
48
|
+
export type VoiceWorkflowContract<TResult = unknown> = {
|
|
49
|
+
assertRouteResult: (routeResult: VoiceRouteResult<TResult>) => void;
|
|
50
|
+
definition: VoiceWorkflowContractDefinition<TResult>;
|
|
51
|
+
toScenarioEval: (overrides?: Partial<VoiceScenarioEvalDefinition>) => VoiceScenarioEvalDefinition;
|
|
52
|
+
validateRouteResult: (routeResult: VoiceRouteResult<TResult>) => VoiceWorkflowContractValidation;
|
|
53
|
+
};
|
|
54
|
+
export type VoiceWorkflowContractTracePayload = {
|
|
55
|
+
contractId: string;
|
|
56
|
+
issues: VoiceWorkflowContractValidationIssue[];
|
|
57
|
+
missingFields: string[];
|
|
58
|
+
outcome?: VoiceWorkflowOutcome;
|
|
59
|
+
requiredFields: string[];
|
|
60
|
+
status: 'pass' | 'fail';
|
|
61
|
+
};
|
|
62
|
+
export declare const validateVoiceWorkflowRouteResult: <TResult = unknown>(definition: VoiceWorkflowContractDefinition<TResult>, routeResult: VoiceRouteResult<TResult>) => VoiceWorkflowContractValidation;
|
|
63
|
+
export declare const createVoiceWorkflowScenario: <TResult = unknown>(definition: VoiceWorkflowContractDefinition<TResult>, overrides?: Partial<VoiceScenarioEvalDefinition>) => VoiceScenarioEvalDefinition;
|
|
64
|
+
export declare const createVoiceWorkflowContract: <TResult = unknown>(definition: VoiceWorkflowContractDefinition<TResult>) => VoiceWorkflowContract<TResult>;
|
|
65
|
+
export declare const recordVoiceWorkflowContractTrace: (input: {
|
|
66
|
+
at?: number;
|
|
67
|
+
contractId?: string;
|
|
68
|
+
scenarioId?: string;
|
|
69
|
+
sessionId: string;
|
|
70
|
+
store: VoiceTraceEventStore;
|
|
71
|
+
traceId?: string;
|
|
72
|
+
turnId?: string;
|
|
73
|
+
validation: VoiceWorkflowContractValidation;
|
|
74
|
+
}) => Promise<StoredVoiceTraceEvent<VoiceWorkflowContractTracePayload>>;
|
|
75
|
+
export declare const createVoiceWorkflowContractHandler: <TContext = unknown, TSession extends VoiceSessionRecord = VoiceSessionRecord, TResult = unknown>(input: {
|
|
76
|
+
contract?: VoiceWorkflowContract<TResult> | VoiceWorkflowContractDefinition<TResult>;
|
|
77
|
+
handler: VoiceOnTurnHandler<TContext, TSession, TResult>;
|
|
78
|
+
resolveContract?: (args: {
|
|
79
|
+
context: TContext;
|
|
80
|
+
result: VoiceRouteResult<TResult>;
|
|
81
|
+
session: TSession;
|
|
82
|
+
turn: VoiceTurnRecord;
|
|
83
|
+
}) => VoiceWorkflowContract<TResult> | VoiceWorkflowContractDefinition<TResult> | undefined;
|
|
84
|
+
store?: VoiceTraceEventStore;
|
|
85
|
+
}) => VoiceOnTurnHandler<TContext, TSession, TResult>;
|