@virtualkitchenco/multiverse-sdk 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/langchain.d.ts +65 -0
- package/dist/adapters/langchain.d.ts.map +1 -0
- package/dist/adapters/langchain.js +80 -0
- package/dist/adapters/langchain.js.map +1 -0
- package/dist/ci.d.ts +31 -0
- package/dist/ci.d.ts.map +1 -0
- package/dist/ci.js +105 -0
- package/dist/ci.js.map +1 -0
- package/dist/client.d.ts +136 -0
- package/dist/client.d.ts.map +1 -0
- package/dist/client.js +92 -0
- package/dist/client.js.map +1 -0
- package/dist/github.d.ts +35 -0
- package/dist/github.d.ts.map +1 -0
- package/dist/github.js +113 -0
- package/dist/github.js.map +1 -0
- package/dist/index.d.ts +19 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +19 -0
- package/dist/index.js.map +1 -0
- package/dist/llm/anthropic.d.ts +18 -0
- package/dist/llm/anthropic.d.ts.map +1 -0
- package/dist/llm/anthropic.js +91 -0
- package/dist/llm/anthropic.js.map +1 -0
- package/dist/llm/google.d.ts +18 -0
- package/dist/llm/google.d.ts.map +1 -0
- package/dist/llm/google.js +97 -0
- package/dist/llm/google.js.map +1 -0
- package/dist/llm/index.d.ts +14 -0
- package/dist/llm/index.d.ts.map +1 -0
- package/dist/llm/index.js +29 -0
- package/dist/llm/index.js.map +1 -0
- package/dist/llm/openai.d.ts +18 -0
- package/dist/llm/openai.d.ts.map +1 -0
- package/dist/llm/openai.js +78 -0
- package/dist/llm/openai.js.map +1 -0
- package/dist/llm/provider.d.ts +42 -0
- package/dist/llm/provider.d.ts.map +1 -0
- package/dist/llm/provider.js +7 -0
- package/dist/llm/provider.js.map +1 -0
- package/dist/multiverse.d.ts +105 -0
- package/dist/multiverse.d.ts.map +1 -0
- package/dist/multiverse.js +449 -0
- package/dist/multiverse.js.map +1 -0
- package/dist/report.d.ts +30 -0
- package/dist/report.d.ts.map +1 -0
- package/dist/report.js +128 -0
- package/dist/report.js.map +1 -0
- package/dist/simulate.d.ts +49 -0
- package/dist/simulate.d.ts.map +1 -0
- package/dist/simulate.js +476 -0
- package/dist/simulate.js.map +1 -0
- package/package.json +68 -0
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Multiverse SDK - Thin Client
|
|
3
|
+
*
|
|
4
|
+
* All simulation logic runs on the server. This SDK:
|
|
5
|
+
* 1. Wraps tools and forwards calls to the API
|
|
6
|
+
* 2. Orchestrates test runs via API
|
|
7
|
+
* 3. Runs user's agent code and success function locally
|
|
8
|
+
*/
|
|
9
|
+
import type { ToolConfig, TestResults, LLMConfig, SuccessFn, EffectsFn, Invariant, OutputSchema } from '@virtualkitchenco/multiverse-types';
|
|
10
|
+
interface MultiverseConfig {
|
|
11
|
+
/** Base URL of the Multiverse API server (required) */
|
|
12
|
+
baseUrl: string;
|
|
13
|
+
/** LLM configuration passed to the API */
|
|
14
|
+
llm: LLMConfig;
|
|
15
|
+
/** Optional API key for authentication */
|
|
16
|
+
apiKey?: string;
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* Configure Multiverse
|
|
20
|
+
*/
|
|
21
|
+
declare function configure(options: MultiverseConfig): void;
|
|
22
|
+
/**
|
|
23
|
+
* Seed initial world state
|
|
24
|
+
*/
|
|
25
|
+
declare function seed(data: Record<string, Record<string, unknown>>): void;
|
|
26
|
+
/**
|
|
27
|
+
* Check if configured
|
|
28
|
+
*/
|
|
29
|
+
declare function isConfigured(): boolean;
|
|
30
|
+
declare function isSimMode(): boolean;
|
|
31
|
+
interface RegisteredTool {
|
|
32
|
+
name: string;
|
|
33
|
+
description: string;
|
|
34
|
+
fn: (input: unknown) => Promise<unknown>;
|
|
35
|
+
outputSchema?: OutputSchema;
|
|
36
|
+
effects?: EffectsFn<unknown>;
|
|
37
|
+
invariants?: Invariant[];
|
|
38
|
+
}
|
|
39
|
+
declare const registeredTools: Map<string, RegisteredTool>;
|
|
40
|
+
/**
|
|
41
|
+
* Wrap a function for simulation
|
|
42
|
+
*/
|
|
43
|
+
declare function tool<TInput, TOutput>(fn: (input: TInput) => Promise<TOutput>, toolConfig: ToolConfig<TInput, TOutput>): (input: TInput) => Promise<TOutput>;
|
|
44
|
+
/**
|
|
45
|
+
* Get registered tools
|
|
46
|
+
*/
|
|
47
|
+
declare function getTools(): Map<string, RegisteredTool>;
|
|
48
|
+
/**
|
|
49
|
+
* Clear registered tools
|
|
50
|
+
*/
|
|
51
|
+
declare function clearTools(): void;
|
|
52
|
+
interface AgentContext {
|
|
53
|
+
userMessage: string;
|
|
54
|
+
history: Array<{
|
|
55
|
+
role: 'user' | 'assistant';
|
|
56
|
+
content: string;
|
|
57
|
+
}>;
|
|
58
|
+
}
|
|
59
|
+
interface TestOptions {
|
|
60
|
+
/** Agent function to test */
|
|
61
|
+
agent: (context?: AgentContext) => Promise<unknown>;
|
|
62
|
+
/** Task description */
|
|
63
|
+
task: string;
|
|
64
|
+
/** Success function - returns true if task completed */
|
|
65
|
+
success: SuccessFn;
|
|
66
|
+
/** Number of scenarios to generate */
|
|
67
|
+
scenarioCount?: number;
|
|
68
|
+
/** Runs per scenario */
|
|
69
|
+
runsPerScenario?: number;
|
|
70
|
+
/** Enable multi-turn user simulation */
|
|
71
|
+
simulateUser?: boolean;
|
|
72
|
+
/** Max conversation turns */
|
|
73
|
+
maxTurns?: number;
|
|
74
|
+
/** Quality threshold (0-100) */
|
|
75
|
+
qualityThreshold?: number;
|
|
76
|
+
/** Progress callback */
|
|
77
|
+
onProgress?: (progress: {
|
|
78
|
+
completed: number;
|
|
79
|
+
total: number;
|
|
80
|
+
scenario: string;
|
|
81
|
+
}) => void;
|
|
82
|
+
/** CI Integration options */
|
|
83
|
+
ci?: {
|
|
84
|
+
/** Auto-post report to PR (requires GITHUB_TOKEN) */
|
|
85
|
+
postToPR?: boolean;
|
|
86
|
+
/** Print report card to stdout */
|
|
87
|
+
printReport?: boolean;
|
|
88
|
+
};
|
|
89
|
+
}
|
|
90
|
+
/**
|
|
91
|
+
* Run tests against an agent
|
|
92
|
+
*/
|
|
93
|
+
declare function test(options: TestOptions): Promise<TestResults>;
|
|
94
|
+
export { tool, registeredTools };
|
|
95
|
+
export declare const multiverse: {
|
|
96
|
+
configure: typeof configure;
|
|
97
|
+
seed: typeof seed;
|
|
98
|
+
tool: typeof tool;
|
|
99
|
+
test: typeof test;
|
|
100
|
+
getTools: typeof getTools;
|
|
101
|
+
clearTools: typeof clearTools;
|
|
102
|
+
isConfigured: typeof isConfigured;
|
|
103
|
+
isSimMode: typeof isSimMode;
|
|
104
|
+
};
|
|
105
|
+
//# sourceMappingURL=multiverse.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"multiverse.d.ts","sourceRoot":"","sources":["../src/multiverse.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAIH,OAAO,KAAK,EAEV,UAAU,EACV,WAAW,EAEX,SAAS,EACT,SAAS,EAGT,SAAS,EACT,SAAS,EACT,YAAY,EACb,MAAM,oCAAoC,CAAC;AA2F5C,UAAU,gBAAgB;IACxB,uDAAuD;IACvD,OAAO,EAAE,MAAM,CAAC;IAChB,0CAA0C;IAC1C,GAAG,EAAE,SAAS,CAAC;IACf,0CAA0C;IAC1C,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAMD;;GAEG;AACH,iBAAS,SAAS,CAAC,OAAO,EAAE,gBAAgB,GAAG,IAAI,CASlD;AAED;;GAEG;AACH,iBAAS,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,GAAG,IAAI,CAEjE;AAED;;GAEG;AACH,iBAAS,YAAY,IAAI,OAAO,CAE/B;AAoBD,iBAAS,SAAS,IAAI,OAAO,CAE5B;AAMD,UAAU,cAAc;IACtB,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,EAAE,EAAE,CAAC,KAAK,EAAE,OAAO,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;IACzC,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,OAAO,CAAC,EAAE,SAAS,CAAC,OAAO,CAAC,CAAC;IAC7B,UAAU,CAAC,EAAE,SAAS,EAAE,CAAC;CAC1B;AAED,QAAA,MAAM,eAAe,6BAAoC,CAAC;AAE1D;;GAEG;AACH,iBAAS,IAAI,CAAC,MAAM,EAAE,OAAO,EAC3B,EAAE,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC,EACvC,UAAU,EAAE,UAAU,CAAC,MAAM,EAAE,OAAO,CAAC,GACtC,CAAC,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC,CA0ErC;AAED;;GAEG;AACH,iBAAS,QAAQ,IAAI,GAAG,CAAC,MAAM,EAAE,cAAc,CAAC,CAE/C;AAED;;GAEG;AACH,iBAAS,UAAU,IAAI,IAAI,CAE1B;AAMD,UAAU,YAAY;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,GAAG,WAAW,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CACjE;AAED,UAAU,WAAW;IACnB,6BAA6B;IAC7B,KAAK,EAAE,CAAC,OAAO,CAAC,EAAE,YAAY,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;IACpD,uBAAuB;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,wDAAwD;IACxD,OAAO,EAAE,SAAS,CAAC;IACnB,sCAAsC;IACtC,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,wBAAwB;IACxB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,wCAAwC;IACxC,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,6BAA6B;IAC7B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,gCAAgC;IAChC,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,wBAAwB;IACxB,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,KAAK,IAAI,CAAC;IACxF,6BAA6B;IAC7B,EAAE,CAAC,EAAE;QACH,qDAAqD;QACrD,QAAQ,CAAC,EAAE,OAAO,CAAC;QACnB,kCAAkC;QAClC,WAAW,CAAC,EAAE,OAAO,CAAC;KACvB,CAAC;CACH;AAED;;GAEG;AACH,iBAAe,IAAI,CAAC,OAAO,EAAE,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC,CAqR9D;AAOD,OAAO,EAAE,IAAI,EAAE,eAAe,EAAE,CAAC;AAEjC,eAAO,MAAM,UAAU;;;;;;;;;CAStB,CAAC"}
|
|
@@ -0,0 +1,449 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Multiverse SDK - Thin Client
|
|
3
|
+
*
|
|
4
|
+
* All simulation logic runs on the server. This SDK:
|
|
5
|
+
* 1. Wraps tools and forwards calls to the API
|
|
6
|
+
* 2. Orchestrates test runs via API
|
|
7
|
+
* 3. Runs user's agent code and success function locally
|
|
8
|
+
*/
|
|
9
|
+
import { AsyncLocalStorage } from 'node:async_hooks';
|
|
10
|
+
import { nanoid } from 'nanoid';
|
|
11
|
+
import { MultiverseClient } from './client.js';
|
|
12
|
+
import { toolConfigRegistry } from './adapters/langchain.js';
|
|
13
|
+
import { detectCI, hasGitHubToken } from './ci.js';
|
|
14
|
+
import { generateReportCard, generateSummary, calculateTier } from './report.js';
|
|
15
|
+
import { postReportToPR } from './github.js';
|
|
16
|
+
// ============================================================================
|
|
17
|
+
// Schema Serialization (Zod -> JSON)
|
|
18
|
+
// ============================================================================
|
|
19
|
+
/**
|
|
20
|
+
* Convert a Zod schema to a plain JSON object for API transmission.
|
|
21
|
+
* The server uses this to understand the expected output shape.
|
|
22
|
+
*/
|
|
23
|
+
function serializeSchema(schema) {
|
|
24
|
+
if (!schema)
|
|
25
|
+
return undefined;
|
|
26
|
+
const s = schema;
|
|
27
|
+
// Check for zod's internal structure
|
|
28
|
+
if (s._def && typeof s._def === 'object') {
|
|
29
|
+
const def = s._def;
|
|
30
|
+
// Handle ZodObject
|
|
31
|
+
if (def.shape && typeof def.shape === 'function') {
|
|
32
|
+
try {
|
|
33
|
+
const shape = def.shape();
|
|
34
|
+
return {
|
|
35
|
+
type: 'object',
|
|
36
|
+
properties: Object.fromEntries(Object.entries(shape).map(([k, v]) => [k, serializeZodType(v)])),
|
|
37
|
+
};
|
|
38
|
+
}
|
|
39
|
+
catch {
|
|
40
|
+
return { type: 'object' };
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
// Handle ZodArray
|
|
44
|
+
if (def.typeName === 'ZodArray' && def.type) {
|
|
45
|
+
return {
|
|
46
|
+
type: 'array',
|
|
47
|
+
items: serializeSchema(def.type),
|
|
48
|
+
};
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
return { type: 'unknown' };
|
|
52
|
+
}
|
|
53
|
+
function serializeZodType(zodType) {
|
|
54
|
+
const t = zodType;
|
|
55
|
+
if (!t._def)
|
|
56
|
+
return { type: 'unknown' };
|
|
57
|
+
const def = t._def;
|
|
58
|
+
const typeName = def.typeName;
|
|
59
|
+
switch (typeName) {
|
|
60
|
+
case 'ZodString':
|
|
61
|
+
return { type: 'string' };
|
|
62
|
+
case 'ZodNumber':
|
|
63
|
+
return { type: 'number' };
|
|
64
|
+
case 'ZodBoolean':
|
|
65
|
+
return { type: 'boolean' };
|
|
66
|
+
case 'ZodArray':
|
|
67
|
+
return { type: 'array', items: serializeZodType(def.type) };
|
|
68
|
+
case 'ZodObject':
|
|
69
|
+
if (def.shape && typeof def.shape === 'function') {
|
|
70
|
+
const shape = def.shape();
|
|
71
|
+
return {
|
|
72
|
+
type: 'object',
|
|
73
|
+
properties: Object.fromEntries(Object.entries(shape).map(([k, v]) => [k, serializeZodType(v)])),
|
|
74
|
+
};
|
|
75
|
+
}
|
|
76
|
+
return { type: 'object' };
|
|
77
|
+
case 'ZodEnum':
|
|
78
|
+
return { type: 'string', enum: def.values };
|
|
79
|
+
case 'ZodOptional':
|
|
80
|
+
return { ...serializeZodType(def.innerType), optional: true };
|
|
81
|
+
default:
|
|
82
|
+
return { type: typeName.replace('Zod', '').toLowerCase() };
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
let config = null;
|
|
86
|
+
let client = null;
|
|
87
|
+
let seedData = null;
|
|
88
|
+
/**
|
|
89
|
+
* Configure Multiverse
|
|
90
|
+
*/
|
|
91
|
+
function configure(options) {
|
|
92
|
+
if (!options.baseUrl) {
|
|
93
|
+
throw new Error('baseUrl is required - Multiverse requires a server');
|
|
94
|
+
}
|
|
95
|
+
config = options;
|
|
96
|
+
client = new MultiverseClient({
|
|
97
|
+
baseUrl: options.baseUrl,
|
|
98
|
+
apiKey: options.apiKey,
|
|
99
|
+
});
|
|
100
|
+
}
|
|
101
|
+
/**
|
|
102
|
+
* Seed initial world state
|
|
103
|
+
*/
|
|
104
|
+
function seed(data) {
|
|
105
|
+
seedData = data;
|
|
106
|
+
}
|
|
107
|
+
/**
|
|
108
|
+
* Check if configured
|
|
109
|
+
*/
|
|
110
|
+
function isConfigured() {
|
|
111
|
+
return config !== null && client !== null;
|
|
112
|
+
}
|
|
113
|
+
const sessionStorage = new AsyncLocalStorage();
|
|
114
|
+
function getCurrentSession() {
|
|
115
|
+
return sessionStorage.getStore() || null;
|
|
116
|
+
}
|
|
117
|
+
function isSimMode() {
|
|
118
|
+
return getCurrentSession() !== null;
|
|
119
|
+
}
|
|
120
|
+
const registeredTools = new Map();
|
|
121
|
+
/**
|
|
122
|
+
* Wrap a function for simulation
|
|
123
|
+
*/
|
|
124
|
+
function tool(fn, toolConfig) {
|
|
125
|
+
// Register the tool
|
|
126
|
+
registeredTools.set(toolConfig.name, {
|
|
127
|
+
name: toolConfig.name,
|
|
128
|
+
description: toolConfig.description,
|
|
129
|
+
fn: fn,
|
|
130
|
+
outputSchema: toolConfig.outputSchema,
|
|
131
|
+
effects: toolConfig.effects,
|
|
132
|
+
invariants: toolConfig.invariants,
|
|
133
|
+
});
|
|
134
|
+
return async (input) => {
|
|
135
|
+
const session = getCurrentSession();
|
|
136
|
+
// If not in test mode, call the real function
|
|
137
|
+
if (!session || !client) {
|
|
138
|
+
return fn(input);
|
|
139
|
+
}
|
|
140
|
+
// Call the API to simulate
|
|
141
|
+
const result = await client.simulate(session.apiSessionId, session.apiRunId, {
|
|
142
|
+
tool: toolConfig.name,
|
|
143
|
+
input,
|
|
144
|
+
description: toolConfig.description,
|
|
145
|
+
outputSchema: serializeSchema(toolConfig.outputSchema),
|
|
146
|
+
hasClientEffects: !!toolConfig.effects,
|
|
147
|
+
});
|
|
148
|
+
// Compute mutations from effects function (if defined) and send to server
|
|
149
|
+
if (toolConfig.effects) {
|
|
150
|
+
// Create world state accessor from the response
|
|
151
|
+
const worldState = result.worldState;
|
|
152
|
+
const worldAccessor = {
|
|
153
|
+
getCollection: (name) => {
|
|
154
|
+
const coll = worldState[name];
|
|
155
|
+
if (!coll)
|
|
156
|
+
return new Map();
|
|
157
|
+
return new Map(Object.entries(coll).map(([id, data]) => [id, {
|
|
158
|
+
id, type: name, data: data,
|
|
159
|
+
createdAt: Date.now(), updatedAt: Date.now(),
|
|
160
|
+
}]));
|
|
161
|
+
},
|
|
162
|
+
getEntity: (collection, id) => {
|
|
163
|
+
const coll = worldState[collection];
|
|
164
|
+
if (!coll || !coll[id])
|
|
165
|
+
return undefined;
|
|
166
|
+
return { id, type: collection, data: coll[id],
|
|
167
|
+
createdAt: Date.now(), updatedAt: Date.now() };
|
|
168
|
+
},
|
|
169
|
+
hasEntity: (collection, id) => {
|
|
170
|
+
const coll = worldState[collection];
|
|
171
|
+
return coll ? id in coll : false;
|
|
172
|
+
},
|
|
173
|
+
getCollectionNames: () => Object.keys(worldState),
|
|
174
|
+
};
|
|
175
|
+
const effects = toolConfig.effects(result.output, worldAccessor);
|
|
176
|
+
const mutations = effects.map(e => ({
|
|
177
|
+
operation: e.operation,
|
|
178
|
+
collection: e.collection,
|
|
179
|
+
id: e.id,
|
|
180
|
+
data: e.data,
|
|
181
|
+
}));
|
|
182
|
+
if (mutations.length > 0) {
|
|
183
|
+
await client.applyMutations(session.apiSessionId, session.apiRunId, mutations, toolConfig.name);
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
return result.output;
|
|
187
|
+
};
|
|
188
|
+
}
|
|
189
|
+
/**
|
|
190
|
+
* Get registered tools
|
|
191
|
+
*/
|
|
192
|
+
function getTools() {
|
|
193
|
+
return registeredTools;
|
|
194
|
+
}
|
|
195
|
+
/**
|
|
196
|
+
* Clear registered tools
|
|
197
|
+
*/
|
|
198
|
+
function clearTools() {
|
|
199
|
+
registeredTools.clear();
|
|
200
|
+
}
|
|
201
|
+
/**
|
|
202
|
+
* Run tests against an agent
|
|
203
|
+
*/
|
|
204
|
+
async function test(options) {
|
|
205
|
+
if (!config || !client) {
|
|
206
|
+
throw new Error('Multiverse not configured. Call multiverse.configure() first.');
|
|
207
|
+
}
|
|
208
|
+
const { agent, task, success: successFn, scenarioCount = 5, runsPerScenario = 1, simulateUser = false, maxTurns = 10, qualityThreshold = 70, onProgress, } = options;
|
|
209
|
+
const testId = nanoid(10);
|
|
210
|
+
const startTime = Date.now();
|
|
211
|
+
// Get tool definitions for API
|
|
212
|
+
const toolDefs = Array.from(registeredTools.values()).map(t => ({
|
|
213
|
+
name: t.name,
|
|
214
|
+
description: t.description,
|
|
215
|
+
}));
|
|
216
|
+
// Also include tools from langchain adapter
|
|
217
|
+
for (const [name, toolConfig] of toolConfigRegistry) {
|
|
218
|
+
if (!toolDefs.find(t => t.name === name)) {
|
|
219
|
+
toolDefs.push({ name, description: name });
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
// Create session via API
|
|
223
|
+
const { sessionId, scenarios } = await client.createSession({
|
|
224
|
+
task,
|
|
225
|
+
tools: toolDefs,
|
|
226
|
+
scenarioConfig: { count: scenarioCount },
|
|
227
|
+
seed: seedData || undefined,
|
|
228
|
+
llmConfig: config.llm,
|
|
229
|
+
});
|
|
230
|
+
// Build run configs
|
|
231
|
+
const runConfigs = [];
|
|
232
|
+
for (const scenario of scenarios) {
|
|
233
|
+
for (let i = 0; i < runsPerScenario; i++) {
|
|
234
|
+
runConfigs.push({ scenario, runIndex: i });
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
const totalRuns = runConfigs.length;
|
|
238
|
+
let completed = 0;
|
|
239
|
+
// Execute a single run
|
|
240
|
+
async function executeRun(runConfig) {
|
|
241
|
+
const { scenario, runIndex } = runConfig;
|
|
242
|
+
const runStartTime = Date.now();
|
|
243
|
+
// Create run via API
|
|
244
|
+
const { runId, initialUserMessage } = await client.createRun(sessionId, scenario.name, runIndex);
|
|
245
|
+
// Create session context for this run
|
|
246
|
+
const session = {
|
|
247
|
+
id: nanoid(10),
|
|
248
|
+
apiSessionId: sessionId,
|
|
249
|
+
apiRunId: runId,
|
|
250
|
+
scenario,
|
|
251
|
+
toolDescriptions: new Map(toolDefs.map(t => [t.name, t.description])),
|
|
252
|
+
};
|
|
253
|
+
// Track conversation history locally
|
|
254
|
+
const history = [];
|
|
255
|
+
// Run agent within session context
|
|
256
|
+
let agentError = null;
|
|
257
|
+
await sessionStorage.run(session, async () => {
|
|
258
|
+
try {
|
|
259
|
+
if (simulateUser) {
|
|
260
|
+
// Multi-turn conversation
|
|
261
|
+
let userMessage = initialUserMessage;
|
|
262
|
+
history.push({ role: 'user', content: userMessage });
|
|
263
|
+
for (let turn = 0; turn < maxTurns; turn++) {
|
|
264
|
+
// Run agent
|
|
265
|
+
const response = await agent({ userMessage, history: [...history] });
|
|
266
|
+
const agentMessage = typeof response === 'string' ? response : JSON.stringify(response);
|
|
267
|
+
history.push({ role: 'assistant', content: agentMessage });
|
|
268
|
+
// Record agent message to trace
|
|
269
|
+
await client.recordTrace(sessionId, runId, 'agent', agentMessage);
|
|
270
|
+
// Get user response from API
|
|
271
|
+
const userResponse = await client.getUserResponse(sessionId, runId, agentMessage);
|
|
272
|
+
if (userResponse.conversationComplete) {
|
|
273
|
+
break;
|
|
274
|
+
}
|
|
275
|
+
userMessage = userResponse.userMessage;
|
|
276
|
+
history.push({ role: 'user', content: userMessage });
|
|
277
|
+
}
|
|
278
|
+
}
|
|
279
|
+
else {
|
|
280
|
+
// Single turn
|
|
281
|
+
history.push({ role: 'user', content: initialUserMessage });
|
|
282
|
+
const response = await agent({ userMessage: initialUserMessage, history: [] });
|
|
283
|
+
const agentMessage = typeof response === 'string' ? response : JSON.stringify(response);
|
|
284
|
+
await client.recordTrace(sessionId, runId, 'agent', agentMessage);
|
|
285
|
+
}
|
|
286
|
+
}
|
|
287
|
+
catch (error) {
|
|
288
|
+
agentError = error;
|
|
289
|
+
}
|
|
290
|
+
});
|
|
291
|
+
// Get final state from API
|
|
292
|
+
const { worldState, trace } = await client.getState(sessionId, runId);
|
|
293
|
+
// Create world state accessor for success function
|
|
294
|
+
const now = Date.now();
|
|
295
|
+
const worldAccessor = {
|
|
296
|
+
getCollection: (name) => {
|
|
297
|
+
const collection = worldState[name];
|
|
298
|
+
if (!collection)
|
|
299
|
+
return new Map();
|
|
300
|
+
return new Map(Object.entries(collection).map(([id, data]) => [
|
|
301
|
+
id,
|
|
302
|
+
{
|
|
303
|
+
id,
|
|
304
|
+
type: name,
|
|
305
|
+
data: data,
|
|
306
|
+
createdAt: now,
|
|
307
|
+
updatedAt: now,
|
|
308
|
+
},
|
|
309
|
+
]));
|
|
310
|
+
},
|
|
311
|
+
getEntity: (collection, id) => {
|
|
312
|
+
const coll = worldState[collection];
|
|
313
|
+
if (!coll || !coll[id])
|
|
314
|
+
return undefined;
|
|
315
|
+
return {
|
|
316
|
+
id,
|
|
317
|
+
type: collection,
|
|
318
|
+
data: coll[id],
|
|
319
|
+
createdAt: now,
|
|
320
|
+
updatedAt: now,
|
|
321
|
+
};
|
|
322
|
+
},
|
|
323
|
+
hasEntity: (collection, id) => {
|
|
324
|
+
const coll = worldState[collection];
|
|
325
|
+
return coll ? id in coll : false;
|
|
326
|
+
},
|
|
327
|
+
getCollectionNames: () => Object.keys(worldState),
|
|
328
|
+
};
|
|
329
|
+
// Run success function locally
|
|
330
|
+
const taskCompleted = agentError === null && successFn(worldAccessor, trace);
|
|
331
|
+
// Get quality judgment from API
|
|
332
|
+
const quality = await client.judge(sessionId, runId, taskCompleted);
|
|
333
|
+
// Determine pass/fail
|
|
334
|
+
const passed = taskCompleted && quality.score >= qualityThreshold && agentError === null;
|
|
335
|
+
completed++;
|
|
336
|
+
onProgress?.({ completed, total: totalRuns, scenario: scenario.name });
|
|
337
|
+
return {
|
|
338
|
+
id: nanoid(10),
|
|
339
|
+
scenario,
|
|
340
|
+
trace,
|
|
341
|
+
worldState: worldAccessor,
|
|
342
|
+
taskCompleted,
|
|
343
|
+
quality,
|
|
344
|
+
passed,
|
|
345
|
+
duration: Date.now() - runStartTime,
|
|
346
|
+
};
|
|
347
|
+
}
|
|
348
|
+
// Execute all runs in parallel
|
|
349
|
+
const results = await Promise.all(runConfigs.map(executeRun));
|
|
350
|
+
// Aggregate results
|
|
351
|
+
const passedCount = results.filter(r => r.passed).length;
|
|
352
|
+
// Find weak spots
|
|
353
|
+
const scenarioResults = new Map();
|
|
354
|
+
for (const result of results) {
|
|
355
|
+
const name = result.scenario.name;
|
|
356
|
+
const current = scenarioResults.get(name) || { passed: 0, total: 0 };
|
|
357
|
+
current.total++;
|
|
358
|
+
if (result.passed)
|
|
359
|
+
current.passed++;
|
|
360
|
+
scenarioResults.set(name, current);
|
|
361
|
+
}
|
|
362
|
+
const weakSpots = Array.from(scenarioResults.entries())
|
|
363
|
+
.map(([scenario, data]) => ({
|
|
364
|
+
scenario,
|
|
365
|
+
passRate: Math.round((data.passed / data.total) * 100),
|
|
366
|
+
}))
|
|
367
|
+
.sort((a, b) => a.passRate - b.passRate)
|
|
368
|
+
.slice(0, 5);
|
|
369
|
+
const testResults = {
|
|
370
|
+
id: testId,
|
|
371
|
+
runs: results,
|
|
372
|
+
passRate: Math.round((passedCount / results.length) * 100),
|
|
373
|
+
weakSpots,
|
|
374
|
+
duration: Date.now() - startTime,
|
|
375
|
+
url: `${config.baseUrl}/runs/${testId}`,
|
|
376
|
+
};
|
|
377
|
+
// Sync results to server for dashboard
|
|
378
|
+
// Serialize runs (remove non-serializable worldState accessor)
|
|
379
|
+
const serializedRuns = results.map(r => ({
|
|
380
|
+
id: r.id,
|
|
381
|
+
runId: `${testId}-${r.scenario.name}-${r.id}`,
|
|
382
|
+
testId,
|
|
383
|
+
scenario: r.scenario,
|
|
384
|
+
trace: r.trace,
|
|
385
|
+
taskCompleted: r.taskCompleted,
|
|
386
|
+
quality: r.quality,
|
|
387
|
+
passed: r.passed,
|
|
388
|
+
duration: r.duration,
|
|
389
|
+
}));
|
|
390
|
+
try {
|
|
391
|
+
await client.sync({
|
|
392
|
+
id: testId,
|
|
393
|
+
runs: serializedRuns,
|
|
394
|
+
passRate: testResults.passRate,
|
|
395
|
+
weakSpots,
|
|
396
|
+
duration: testResults.duration,
|
|
397
|
+
});
|
|
398
|
+
}
|
|
399
|
+
catch (err) {
|
|
400
|
+
console.warn('Failed to sync results to dashboard:', err);
|
|
401
|
+
}
|
|
402
|
+
// CI Integration
|
|
403
|
+
const ciContext = detectCI();
|
|
404
|
+
const ciOptions = options.ci || {};
|
|
405
|
+
// Generate report card
|
|
406
|
+
const markdown = generateReportCard(testResults, {
|
|
407
|
+
dashboardUrl: testResults.url,
|
|
408
|
+
detailed: true,
|
|
409
|
+
});
|
|
410
|
+
// Add markdown and CI info to results
|
|
411
|
+
const enhancedResults = {
|
|
412
|
+
...testResults,
|
|
413
|
+
markdown,
|
|
414
|
+
summary: generateSummary(testResults),
|
|
415
|
+
tier: calculateTier(testResults.passRate),
|
|
416
|
+
ci: ciContext,
|
|
417
|
+
};
|
|
418
|
+
// Print report if requested or in CI
|
|
419
|
+
if (ciOptions.printReport || ciContext.isCI) {
|
|
420
|
+
console.log('\n' + markdown + '\n');
|
|
421
|
+
}
|
|
422
|
+
// Post to PR if in CI with PR context
|
|
423
|
+
if (ciOptions.postToPR !== false && ciContext.isCI && ciContext.isPR && hasGitHubToken()) {
|
|
424
|
+
const result = await postReportToPR(markdown, ciContext);
|
|
425
|
+
if (result.posted) {
|
|
426
|
+
console.log(`Posted report to PR: ${result.url}`);
|
|
427
|
+
}
|
|
428
|
+
else if (result.error) {
|
|
429
|
+
console.warn(`Failed to post to PR: ${result.error}`);
|
|
430
|
+
}
|
|
431
|
+
}
|
|
432
|
+
return enhancedResults;
|
|
433
|
+
}
|
|
434
|
+
// ============================================================================
|
|
435
|
+
// Exports
|
|
436
|
+
// ============================================================================
|
|
437
|
+
// Export for adapters
|
|
438
|
+
export { tool, registeredTools };
|
|
439
|
+
export const multiverse = {
|
|
440
|
+
configure,
|
|
441
|
+
seed,
|
|
442
|
+
tool,
|
|
443
|
+
test,
|
|
444
|
+
getTools,
|
|
445
|
+
clearTools,
|
|
446
|
+
isConfigured,
|
|
447
|
+
isSimMode,
|
|
448
|
+
};
|
|
449
|
+
//# sourceMappingURL=multiverse.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"multiverse.js","sourceRoot":"","sources":["../src/multiverse.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAchC,OAAO,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAC/C,OAAO,EAAE,kBAAkB,EAAE,MAAM,yBAAyB,CAAC;AAC7D,OAAO,EAAE,QAAQ,EAAE,cAAc,EAAkB,MAAM,SAAS,CAAC;AACnE,OAAO,EAAE,kBAAkB,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AACjF,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAE7C,+EAA+E;AAC/E,qCAAqC;AACrC,+EAA+E;AAE/E;;;GAGG;AACH,SAAS,eAAe,CAAC,MAAgC;IACvD,IAAI,CAAC,MAAM;QAAE,OAAO,SAAS,CAAC;IAE9B,MAAM,CAAC,GAAG,MAA4C,CAAC;IAEvD,qCAAqC;IACrC,IAAI,CAAC,CAAC,IAAI,IAAI,OAAO,CAAC,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;QACzC,MAAM,GAAG,GAAG,CAAC,CAAC,IAA+B,CAAC;QAE9C,mBAAmB;QACnB,IAAI,GAAG,CAAC,KAAK,IAAI,OAAO,GAAG,CAAC,KAAK,KAAK,UAAU,EAAE,CAAC;YACjD,IAAI,CAAC;gBACH,MAAM,KAAK,GAAI,GAAG,CAAC,KAAuC,EAAE,CAAC;gBAC7D,OAAO;oBACL,IAAI,EAAE,QAAQ;oBACd,UAAU,EAAE,MAAM,CAAC,WAAW,CAC5B,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,gBAAgB,CAAC,CAAC,CAAC,CAAC,CAAC,CAChE;iBACF,CAAC;YACJ,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;YAC5B,CAAC;QACH,CAAC;QAED,kBAAkB;QAClB,IAAI,GAAG,CAAC,QAAQ,KAAK,UAAU,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC;YAC5C,OAAO;gBACL,IAAI,EAAE,OAAO;gBACb,KAAK,EAAE,eAAe,CAAC,GAAG,CAAC,IAAoB,CAAC;aACjD,CAAC;QACJ,CAAC;IACH,CAAC;IAED,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC;AAC7B,CAAC;AAED,SAAS,gBAAgB,CAAC,OAAgB;IACxC,MAAM,CAAC,GAAG,OAAkC,CAAC;IAC7C,IAAI,CAAC,CAAC,CAAC,IAAI;QAAE,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC;IAExC,MAAM,GAAG,GAAG,CAAC,CAAC,IAA+B,CAAC;IAC9C,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAkB,CAAC;IAExC,QAAQ,QAAQ,EAAE,CAAC;QACjB,KAAK,WAAW;YACd,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;QAC5B,KAAK,WAAW;YACd,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;QAC5B,KAAK,YAAY;YACf,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC;QAC7B,KAAK,UAAU;YACb,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,gBAAgB,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;QAC9D,KAAK,WAAW;YACd,IAAI,GAAG,CAAC,KAAK,IAAI,OAAO,GAAG,CAAC,KAAK,KAAK,UAAU,EAAE,CAAC;gBACjD,MAAM,KAAK,GAAI,GAAG,CAAC,KAAuC,EAAE,CAAC;gBAC7D,OAAO;oBACL,IAAI,EAAE,QAAQ;oBACd,UAAU,EAAE,MAAM,CAAC,WAAW,CAC5B,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,gBAAgB,CAAC,CAAC,CAAC,CAAC,CAAC,CAChE;iBACF,CAAC;YACJ,CAAC;YACD,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;QAC5B,KAAK,SAAS;YACZ,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC;QAC9C,KAAK,aAAa;YAChB,OAAO,EAAE,GAAG,gBAAgB,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;QAChE;YACE,OAAO,EAAE,IAAI,EAAE,QAAQ,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,WAAW,EAAE,EAAE,CAAC;IAC/D,CAAC;AACH,CAAC;AAeD,IAAI,MAAM,GAA4B,IAAI,CAAC;AAC3C,IAAI,MAAM,GAA4B,IAAI,CAAC;AAC3C,IAAI,QAAQ,GAAmD,IAAI,CAAC;AAEpE;;GAEG;AACH,SAAS,SAAS,CAAC,OAAyB;IAC1C,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;QACrB,MAAM,IAAI,KAAK,CAAC,oDAAoD,CAAC,CAAC;IACxE,CAAC;IACD,MAAM,GAAG,OAAO,CAAC;IACjB,MAAM,GAAG,IAAI,gBAAgB,CAAC;QAC5B,OAAO,EAAE,OAAO,CAAC,OAAO;QACxB,MAAM,EAAE,OAAO,CAAC,MAAM;KACvB,CAAC,CAAC;AACL,CAAC;AAED;;GAEG;AACH,SAAS,IAAI,CAAC,IAA6C;IACzD,QAAQ,GAAG,IAAI,CAAC;AAClB,CAAC;AAED;;GAEG;AACH,SAAS,YAAY;IACnB,OAAO,MAAM,KAAK,IAAI,IAAI,MAAM,KAAK,IAAI,CAAC;AAC5C,CAAC;AAcD,MAAM,cAAc,GAAG,IAAI,iBAAiB,EAAW,CAAC;AAExD,SAAS,iBAAiB;IACxB,OAAO,cAAc,CAAC,QAAQ,EAAE,IAAI,IAAI,CAAC;AAC3C,CAAC;AAED,SAAS,SAAS;IAChB,OAAO,iBAAiB,EAAE,KAAK,IAAI,CAAC;AACtC,CAAC;AAeD,MAAM,eAAe,GAAG,IAAI,GAAG,EAA0B,CAAC;AAE1D;;GAEG;AACH,SAAS,IAAI,CACX,EAAuC,EACvC,UAAuC;IAEvC,oBAAoB;IACpB,eAAe,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,EAAE;QACnC,IAAI,EAAE,UAAU,CAAC,IAAI;QACrB,WAAW,EAAE,UAAU,CAAC,WAAW;QACnC,EAAE,EAAE,EAA0C;QAC9C,YAAY,EAAE,UAAU,CAAC,YAAY;QACrC,OAAO,EAAE,UAAU,CAAC,OAAyC;QAC7D,UAAU,EAAE,UAAU,CAAC,UAAU;KAClC,CAAC,CAAC;IAEH,OAAO,KAAK,EAAE,KAAa,EAAoB,EAAE;QAC/C,MAAM,OAAO,GAAG,iBAAiB,EAAE,CAAC;QAEpC,8CAA8C;QAC9C,IAAI,CAAC,OAAO,IAAI,CAAC,MAAM,EAAE,CAAC;YACxB,OAAO,EAAE,CAAC,KAAK,CAAC,CAAC;QACnB,CAAC;QAED,2BAA2B;QAC3B,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,QAAQ,CAClC,OAAO,CAAC,YAAY,EACpB,OAAO,CAAC,QAAQ,EAChB;YACE,IAAI,EAAE,UAAU,CAAC,IAAI;YACrB,KAAK;YACL,WAAW,EAAE,UAAU,CAAC,WAAW;YACnC,YAAY,EAAE,eAAe,CAAC,UAAU,CAAC,YAAY,CAAC;YACtD,gBAAgB,EAAE,CAAC,CAAC,UAAU,CAAC,OAAO;SACvC,CACF,CAAC;QAEF,0EAA0E;QAC1E,IAAI,UAAU,CAAC,OAAO,EAAE,CAAC;YACvB,gDAAgD;YAChD,MAAM,UAAU,GAAG,MAAM,CAAC,UAAqD,CAAC;YAChF,MAAM,aAAa,GAAuB;gBACxC,aAAa,EAAE,CAAC,IAAY,EAAE,EAAE;oBAC9B,MAAM,IAAI,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC;oBAC9B,IAAI,CAAC,IAAI;wBAAE,OAAO,IAAI,GAAG,EAAE,CAAC;oBAC5B,OAAO,IAAI,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE;4BAC3D,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAA+B;4BACrD,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;yBACnC,CAAC,CAAC,CAAC,CAAC;gBACjB,CAAC;gBACD,SAAS,EAAE,CAAC,UAAkB,EAAE,EAAU,EAAE,EAAE;oBAC5C,MAAM,IAAI,GAAG,UAAU,CAAC,UAAU,CAAC,CAAC;oBACpC,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;wBAAE,OAAO,SAAS,CAAC;oBACzC,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,IAAI,CAAC,EAAE,CAA4B;wBACtE,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;gBACnD,CAAC;gBACD,SAAS,EAAE,CAAC,UAAkB,EAAE,EAAU,EAAE,EAAE;oBAC5C,MAAM,IAAI,GAAG,UAAU,CAAC,UAAU,CAAC,CAAC;oBACpC,OAAO,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC;gBACnC,CAAC;gBACD,kBAAkB,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC;aAClD,CAAC;YAEF,MAAM,OAAO,GAAG,UAAU,CAAC,OAAO,CAAC,MAAM,CAAC,MAAiB,EAAE,aAAa,CAAC,CAAC;YAE5E,MAAM,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;gBAClC,SAAS,EAAE,CAAC,CAAC,SAAS;gBACtB,UAAU,EAAE,CAAC,CAAC,UAAU;gBACxB,EAAE,EAAE,CAAC,CAAC,EAAE;gBACR,IAAI,EAAE,CAAC,CAAC,IAA2C;aACpD,CAAC,CAAC,CAAC;YAEJ,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACzB,MAAM,MAAM,CAAC,cAAc,CAAC,OAAO,CAAC,YAAY,EAAE,OAAO,CAAC,QAAQ,EAAE,SAAS,EAAE,UAAU,CAAC,IAAI,CAAC,CAAC;YAClG,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAC,MAAiB,CAAC;IAClC,CAAC,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,SAAS,QAAQ;IACf,OAAO,eAAe,CAAC;AACzB,CAAC;AAED;;GAEG;AACH,SAAS,UAAU;IACjB,eAAe,CAAC,KAAK,EAAE,CAAC;AAC1B,CAAC;AAuCD;;GAEG;AACH,KAAK,UAAU,IAAI,CAAC,OAAoB;IACtC,IAAI,CAAC,MAAM,IAAI,CAAC,MAAM,EAAE,CAAC;QACvB,MAAM,IAAI,KAAK,CAAC,+DAA+D,CAAC,CAAC;IACnF,CAAC;IAED,MAAM,EACJ,KAAK,EACL,IAAI,EACJ,OAAO,EAAE,SAAS,EAClB,aAAa,GAAG,CAAC,EACjB,eAAe,GAAG,CAAC,EACnB,YAAY,GAAG,KAAK,EACpB,QAAQ,GAAG,EAAE,EACb,gBAAgB,GAAG,EAAE,EACrB,UAAU,GACX,GAAG,OAAO,CAAC;IAEZ,MAAM,MAAM,GAAG,MAAM,CAAC,EAAE,CAAC,CAAC;IAC1B,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAE7B,+BAA+B;IAC/B,MAAM,QAAQ,GAAG,KAAK,CAAC,IAAI,CAAC,eAAe,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAC9D,IAAI,EAAE,CAAC,CAAC,IAAI;QACZ,WAAW,EAAE,CAAC,CAAC,WAAW;KAC3B,CAAC,CAAC,CAAC;IAEJ,4CAA4C;IAC5C,KAAK,MAAM,CAAC,IAAI,EAAE,UAAU,CAAC,IAAI,kBAAkB,EAAE,CAAC;QACpD,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,EAAE,CAAC;YACzC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC,CAAC;QAC7C,CAAC;IACH,CAAC;IAED,yBAAyB;IACzB,MAAM,EAAE,SAAS,EAAE,SAAS,EAAE,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC;QAC1D,IAAI;QACJ,KAAK,EAAE,QAAQ;QACf,cAAc,EAAE,EAAE,KAAK,EAAE,aAAa,EAAE;QACxC,IAAI,EAAE,QAAQ,IAAI,SAAS;QAC3B,SAAS,EAAE,MAAM,CAAC,GAAG;KACtB,CAAC,CAAC;IAEH,oBAAoB;IACpB,MAAM,UAAU,GAAoD,EAAE,CAAC;IACvE,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QACjC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,eAAe,EAAE,CAAC,EAAE,EAAE,CAAC;YACzC,UAAU,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC;QAC7C,CAAC;IACH,CAAC;IAED,MAAM,SAAS,GAAG,UAAU,CAAC,MAAM,CAAC;IACpC,IAAI,SAAS,GAAG,CAAC,CAAC;IAElB,uBAAuB;IACvB,KAAK,UAAU,UAAU,CAAC,SAAmD;QAC3E,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,GAAG,SAAS,CAAC;QACzC,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAEhC,qBAAqB;QACrB,MAAM,EAAE,KAAK,EAAE,kBAAkB,EAAE,GAAG,MAAM,MAAO,CAAC,SAAS,CAC3D,SAAS,EACT,QAAQ,CAAC,IAAI,EACb,QAAQ,CACT,CAAC;QAEF,sCAAsC;QACtC,MAAM,OAAO,GAAY;YACvB,EAAE,EAAE,MAAM,CAAC,EAAE,CAAC;YACd,YAAY,EAAE,SAAS;YACvB,QAAQ,EAAE,KAAK;YACf,QAAQ;YACR,gBAAgB,EAAE,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC;SACtE,CAAC;QAEF,qCAAqC;QACrC,MAAM,OAAO,GAA2D,EAAE,CAAC;QAE3E,mCAAmC;QACnC,IAAI,UAAU,GAAiB,IAAI,CAAC;QAEpC,MAAM,cAAc,CAAC,GAAG,CAAC,OAAO,EAAE,KAAK,IAAI,EAAE;YAC3C,IAAI,CAAC;gBACH,IAAI,YAAY,EAAE,CAAC;oBACjB,0BAA0B;oBAC1B,IAAI,WAAW,GAAG,kBAAkB,CAAC;oBACrC,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE,CAAC,CAAC;oBAErD,KAAK,IAAI,IAAI,GAAG,CAAC,EAAE,IAAI,GAAG,QAAQ,EAAE,IAAI,EAAE,EAAE,CAAC;wBAC3C,YAAY;wBACZ,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,EAAE,WAAW,EAAE,OAAO,EAAE,CAAC,GAAG,OAAO,CAAC,EAAE,CAAC,CAAC;wBACrE,MAAM,YAAY,GAAG,OAAO,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;wBAExF,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,YAAY,EAAE,CAAC,CAAC;wBAE3D,gCAAgC;wBAChC,MAAM,MAAO,CAAC,WAAW,CAAC,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,YAAY,CAAC,CAAC;wBAEnE,6BAA6B;wBAC7B,MAAM,YAAY,GAAG,MAAM,MAAO,CAAC,eAAe,CAAC,SAAS,EAAE,KAAK,EAAE,YAAY,CAAC,CAAC;wBAEnF,IAAI,YAAY,CAAC,oBAAoB,EAAE,CAAC;4BACtC,MAAM;wBACR,CAAC;wBAED,WAAW,GAAG,YAAY,CAAC,WAAW,CAAC;wBACvC,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE,CAAC,CAAC;oBACvD,CAAC;gBACH,CAAC;qBAAM,CAAC;oBACN,cAAc;oBACd,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,kBAAkB,EAAE,CAAC,CAAC;oBAC5D,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,EAAE,WAAW,EAAE,kBAAkB,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC,CAAC;oBAC/E,MAAM,YAAY,GAAG,OAAO,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;oBACxF,MAAM,MAAO,CAAC,WAAW,CAAC,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,YAAY,CAAC,CAAC;gBACrE,CAAC;YACH,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,UAAU,GAAG,KAAc,CAAC;YAC9B,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,2BAA2B;QAC3B,MAAM,EAAE,UAAU,EAAE,KAAK,EAAE,GAAG,MAAM,MAAO,CAAC,QAAQ,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;QAEvE,mDAAmD;QACnD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,MAAM,aAAa,GAAuB;YACxC,aAAa,EAAE,CAAC,IAAY,EAAE,EAAE;gBAC9B,MAAM,UAAU,GAAG,UAAU,CAAC,IAAI,CAAwC,CAAC;gBAC3E,IAAI,CAAC,UAAU;oBAAE,OAAO,IAAI,GAAG,EAAE,CAAC;gBAClC,OAAO,IAAI,GAAG,CACZ,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC;oBAC7C,EAAE;oBACF;wBACE,EAAE;wBACF,IAAI,EAAE,IAAI;wBACV,IAAI,EAAE,IAA+B;wBACrC,SAAS,EAAE,GAAG;wBACd,SAAS,EAAE,GAAG;qBACL;iBACZ,CAAC,CACH,CAAC;YACJ,CAAC;YACD,SAAS,EAAE,CAAC,UAAkB,EAAE,EAAU,EAAsB,EAAE;gBAChE,MAAM,IAAI,GAAG,UAAU,CAAC,UAAU,CAAwC,CAAC;gBAC3E,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;oBAAE,OAAO,SAAS,CAAC;gBACzC,OAAO;oBACL,EAAE;oBACF,IAAI,EAAE,UAAU;oBAChB,IAAI,EAAE,IAAI,CAAC,EAAE,CAA4B;oBACzC,SAAS,EAAE,GAAG;oBACd,SAAS,EAAE,GAAG;iBACf,CAAC;YACJ,CAAC;YACD,SAAS,EAAE,CAAC,UAAkB,EAAE,EAAU,EAAE,EAAE;gBAC5C,MAAM,IAAI,GAAG,UAAU,CAAC,UAAU,CAAwC,CAAC;gBAC3E,OAAO,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC;YACnC,CAAC;YACD,kBAAkB,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC;SAClD,CAAC;QAEF,+BAA+B;QAC/B,MAAM,aAAa,GAAG,UAAU,KAAK,IAAI,IAAI,SAAS,CAAC,aAAa,EAAE,KAAK,CAAC,CAAC;QAE7E,gCAAgC;QAChC,MAAM,OAAO,GAAG,MAAM,MAAO,CAAC,KAAK,CAAC,SAAS,EAAE,KAAK,EAAE,aAAa,CAAC,CAAC;QAErE,sBAAsB;QACtB,MAAM,MAAM,GAAG,aAAa,IAAI,OAAO,CAAC,KAAK,IAAI,gBAAgB,IAAI,UAAU,KAAK,IAAI,CAAC;QAEzF,SAAS,EAAE,CAAC;QACZ,UAAU,EAAE,CAAC,EAAE,SAAS,EAAE,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;QAEvE,OAAO;YACL,EAAE,EAAE,MAAM,CAAC,EAAE,CAAC;YACd,QAAQ;YACR,KAAK;YACL,UAAU,EAAE,aAAa;YACzB,aAAa;YACb,OAAO;YACP,MAAM;YACN,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,YAAY;SACpC,CAAC;IACJ,CAAC;IAED,+BAA+B;IAC/B,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC;IAE9D,oBAAoB;IACpB,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IAEzD,kBAAkB;IAClB,MAAM,eAAe,GAAG,IAAI,GAAG,EAA6C,CAAC;IAC7E,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,MAAM,IAAI,GAAG,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC;QAClC,MAAM,OAAO,GAAG,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;QACrE,OAAO,CAAC,KAAK,EAAE,CAAC;QAChB,IAAI,MAAM,CAAC,MAAM;YAAE,OAAO,CAAC,MAAM,EAAE,CAAC;QACpC,eAAe,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IACrC,CAAC;IAED,MAAM,SAAS,GAAG,KAAK,CAAC,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC;SACpD,GAAG,CAAC,CAAC,CAAC,QAAQ,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC;QAC1B,QAAQ;QACR,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC;KACvD,CAAC,CAAC;SACF,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,QAAQ,CAAC;SACvC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAEf,MAAM,WAAW,GAAgB;QAC/B,EAAE,EAAE,MAAM;QACV,IAAI,EAAE,OAAO;QACb,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC;QAC1D,SAAS;QACT,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;QAChC,GAAG,EAAE,GAAG,MAAM,CAAC,OAAO,SAAS,MAAM,EAAE;KACxC,CAAC;IAEF,uCAAuC;IACvC,+DAA+D;IAC/D,MAAM,cAAc,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QACvC,EAAE,EAAE,CAAC,CAAC,EAAE;QACR,KAAK,EAAE,GAAG,MAAM,IAAI,CAAC,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,CAAC,EAAE,EAAE;QAC7C,MAAM;QACN,QAAQ,EAAE,CAAC,CAAC,QAAQ;QACpB,KAAK,EAAE,CAAC,CAAC,KAAK;QACd,aAAa,EAAE,CAAC,CAAC,aAAa;QAC9B,OAAO,EAAE,CAAC,CAAC,OAAO;QAClB,MAAM,EAAE,CAAC,CAAC,MAAM;QAChB,QAAQ,EAAE,CAAC,CAAC,QAAQ;KACrB,CAAC,CAAC,CAAC;IAEJ,IAAI,CAAC;QACH,MAAM,MAAM,CAAC,IAAI,CAAC;YAChB,EAAE,EAAE,MAAM;YACV,IAAI,EAAE,cAAc;YACpB,QAAQ,EAAE,WAAW,CAAC,QAAQ;YAC9B,SAAS;YACT,QAAQ,EAAE,WAAW,CAAC,QAAQ;SAC/B,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO,CAAC,IAAI,CAAC,sCAAsC,EAAE,GAAG,CAAC,CAAC;IAC5D,CAAC;IAED,iBAAiB;IACjB,MAAM,SAAS,GAAG,QAAQ,EAAE,CAAC;IAC7B,MAAM,SAAS,GAAG,OAAO,CAAC,EAAE,IAAI,EAAE,CAAC;IAEnC,uBAAuB;IACvB,MAAM,QAAQ,GAAG,kBAAkB,CAAC,WAAW,EAAE;QAC/C,YAAY,EAAE,WAAW,CAAC,GAAG;QAC7B,QAAQ,EAAE,IAAI;KACf,CAAC,CAAC;IAEH,sCAAsC;IACtC,MAAM,eAAe,GAAG;QACtB,GAAG,WAAW;QACd,QAAQ;QACR,OAAO,EAAE,eAAe,CAAC,WAAW,CAAC;QACrC,IAAI,EAAE,aAAa,CAAC,WAAW,CAAC,QAAQ,CAAC;QACzC,EAAE,EAAE,SAAS;KACd,CAAC;IAEF,qCAAqC;IACrC,IAAI,SAAS,CAAC,WAAW,IAAI,SAAS,CAAC,IAAI,EAAE,CAAC;QAC5C,OAAO,CAAC,GAAG,CAAC,IAAI,GAAG,QAAQ,GAAG,IAAI,CAAC,CAAC;IACtC,CAAC;IAED,sCAAsC;IACtC,IAAI,SAAS,CAAC,QAAQ,KAAK,KAAK,IAAI,SAAS,CAAC,IAAI,IAAI,SAAS,CAAC,IAAI,IAAI,cAAc,EAAE,EAAE,CAAC;QACzF,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;QACzD,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;YAClB,OAAO,CAAC,GAAG,CAAC,wBAAwB,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC;QACpD,CAAC;aAAM,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;YACxB,OAAO,CAAC,IAAI,CAAC,yBAAyB,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC;QACxD,CAAC;IACH,CAAC;IAED,OAAO,eAAe,CAAC;AACzB,CAAC;AAED,+EAA+E;AAC/E,UAAU;AACV,+EAA+E;AAE/E,sBAAsB;AACtB,OAAO,EAAE,IAAI,EAAE,eAAe,EAAE,CAAC;AAEjC,MAAM,CAAC,MAAM,UAAU,GAAG;IACxB,SAAS;IACT,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,QAAQ;IACR,UAAU;IACV,YAAY;IACZ,SAAS;CACV,CAAC"}
|
package/dist/report.d.ts
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Report Card Generation
|
|
3
|
+
*
|
|
4
|
+
* Generates markdown report cards from test results
|
|
5
|
+
* for posting to PRs and CI output.
|
|
6
|
+
*/
|
|
7
|
+
import type { TestResults } from '@virtualkitchenco/multiverse-types';
|
|
8
|
+
export interface ReportOptions {
|
|
9
|
+
/** Base URL for dashboard links */
|
|
10
|
+
dashboardUrl?: string;
|
|
11
|
+
/** Include detailed scenario breakdown */
|
|
12
|
+
detailed?: boolean;
|
|
13
|
+
}
|
|
14
|
+
/**
|
|
15
|
+
* Calculate tier from pass rate
|
|
16
|
+
*/
|
|
17
|
+
export declare function calculateTier(passRate: number): 'S' | 'A' | 'B' | 'C' | 'D';
|
|
18
|
+
/**
|
|
19
|
+
* Generate markdown report card from test results
|
|
20
|
+
*/
|
|
21
|
+
export declare function generateReportCard(results: TestResults, options?: ReportOptions): string;
|
|
22
|
+
/**
|
|
23
|
+
* Generate short summary for CI output
|
|
24
|
+
*/
|
|
25
|
+
export declare function generateSummary(results: TestResults): string;
|
|
26
|
+
/**
|
|
27
|
+
* Generate badge URL (shields.io style)
|
|
28
|
+
*/
|
|
29
|
+
export declare function generateBadgeUrl(results: TestResults): string;
|
|
30
|
+
//# sourceMappingURL=report.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"report.d.ts","sourceRoot":"","sources":["../src/report.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,WAAW,EAAa,MAAM,oCAAoC,CAAC;AAEjF,MAAM,WAAW,aAAa;IAC5B,mCAAmC;IACnC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,0CAA0C;IAC1C,QAAQ,CAAC,EAAE,OAAO,CAAC;CACpB;AAED;;GAEG;AACH,wBAAgB,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,GAAG,GAAG,GAAG,GAAG,GAAG,GAAG,GAAG,GAAG,GAAG,CAM3E;AAgBD;;GAEG;AACH,wBAAgB,kBAAkB,CAChC,OAAO,EAAE,WAAW,EACpB,OAAO,GAAE,aAAkB,GAC1B,MAAM,CAiER;AAED;;GAEG;AACH,wBAAgB,eAAe,CAAC,OAAO,EAAE,WAAW,GAAG,MAAM,CAM5D;AAmBD;;GAEG;AACH,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,WAAW,GAAG,MAAM,CAU7D"}
|