@jmoak/vibrissa 0.1.0 → 2.0.0-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -21
- package/README.md +268 -254
- package/bin/vib-test.js +1 -1
- package/dist/chunk-7GFB72TI.js +10706 -0
- package/dist/chunk-7GFB72TI.js.map +1 -0
- package/dist/chunk-TFXZJODE.js +2329 -0
- package/dist/chunk-TFXZJODE.js.map +1 -0
- package/dist/cli.d.ts +57 -0
- package/dist/cli.js +5896 -613
- package/dist/cli.js.map +1 -1
- package/dist/index.d.ts +837 -58
- package/dist/index.js +211 -10
- package/dist/index.js.map +1 -1
- package/dist/runner-J_V7pgax.d.ts +1194 -0
- package/dist/vitest.d.ts +44 -0
- package/dist/vitest.js +240 -0
- package/dist/vitest.js.map +1 -0
- package/package.json +27 -9
- package/schema/case.schema.json +405 -14
- package/schema/vibrissa.schema.json +528 -12
- package/dist/chunk-FTVETYUK.js +0 -656
- package/dist/chunk-FTVETYUK.js.map +0 -1
package/dist/vitest.d.ts
ADDED
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import { ae as RunnerOptions, i as RunOptions } from './runner-J_V7pgax.js';
|
|
2
|
+
import '@modelcontextprotocol/client';
|
|
3
|
+
|
|
4
|
+
type TestFn = () => void | Promise<void>;
|
|
5
|
+
/** The slice of the Vitest API the bridge uses. */
|
|
6
|
+
interface VitestApi {
|
|
7
|
+
describe: (name: string, fn: () => void) => void;
|
|
8
|
+
it: ((name: string, fn: TestFn, timeout?: number) => void) & {
|
|
9
|
+
skip?: (name: string, fn: TestFn) => void;
|
|
10
|
+
};
|
|
11
|
+
beforeAll: (fn: TestFn, timeout?: number) => void;
|
|
12
|
+
afterAll: (fn: TestFn, timeout?: number) => void;
|
|
13
|
+
}
|
|
14
|
+
/**
|
|
15
|
+
* Options for `describeCases`. Every RunOptions key may be overridden here (servers,
|
|
16
|
+
* normalize, timeoutMs, tags, grep, manifest, manifestIgnore, requireManifest, ...).
|
|
17
|
+
*/
|
|
18
|
+
interface DescribeCasesOptions extends Partial<Omit<RunnerOptions, 'globs' | 'checks'>> {
|
|
19
|
+
/** Case globs (relative to the config's directory). Default: config `globs`. */
|
|
20
|
+
globs?: string | string[];
|
|
21
|
+
/** Config file to load; `false` skips config loading. Default: vibrissa.json from `cwd`. */
|
|
22
|
+
config?: string | false;
|
|
23
|
+
/** Profile to merge over the config. */
|
|
24
|
+
profile?: string;
|
|
25
|
+
/** Where to look for the config. Default process.cwd(). */
|
|
26
|
+
cwd?: string;
|
|
27
|
+
/** `describe` title. Default "vibrissa". */
|
|
28
|
+
title?: string;
|
|
29
|
+
/**
|
|
30
|
+
* `true` adds a `checks (<server>)` test (spec sanity + description lint, like
|
|
31
|
+
* `vib run --check`); an object also sets `allow` / `severity` (and turns checks on).
|
|
32
|
+
*/
|
|
33
|
+
checks?: boolean | NonNullable<RunOptions['checks']>;
|
|
34
|
+
/** Vitest functions; default: globals, else `import('vitest')`. */
|
|
35
|
+
api?: VitestApi;
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* Register Vitest tests for a Vibrissa suite: manifest lock, optional checks, one test per
|
|
39
|
+
* case. Accepts globs or DescribeCasesOptions. Resolves once the tests are registered;
|
|
40
|
+
* `await` it at the top of the test file unless you pass `api` or use Vitest globals.
|
|
41
|
+
*/
|
|
42
|
+
declare function describeCases(target?: string | string[] | DescribeCasesOptions, extra?: DescribeCasesOptions): Promise<void>;
|
|
43
|
+
|
|
44
|
+
export { type DescribeCasesOptions, type VitestApi, describeCases };
|
package/dist/vitest.js
ADDED
|
@@ -0,0 +1,240 @@
|
|
|
1
|
+
import {
|
|
2
|
+
DEFAULT_MANIFEST_IGNORE,
|
|
3
|
+
DEFAULT_NORMALIZE,
|
|
4
|
+
McpSession,
|
|
5
|
+
captureManifest,
|
|
6
|
+
defaultRunOptions,
|
|
7
|
+
diffManifest,
|
|
8
|
+
executeCase,
|
|
9
|
+
formatDiff,
|
|
10
|
+
isHttpServerConfig,
|
|
11
|
+
loadCasesSync,
|
|
12
|
+
loadConfig,
|
|
13
|
+
manifestPathFor,
|
|
14
|
+
readManifest,
|
|
15
|
+
resolveRootDir,
|
|
16
|
+
resolveServerConfig,
|
|
17
|
+
runChecks
|
|
18
|
+
} from "./chunk-7GFB72TI.js";
|
|
19
|
+
|
|
20
|
+
// src/vitest.ts
|
|
21
|
+
import fs from "fs";
|
|
22
|
+
import path from "path";
|
|
23
|
+
function globalApi() {
|
|
24
|
+
const g = globalThis;
|
|
25
|
+
if (typeof g.describe === "function" && typeof g.it === "function" && typeof g.beforeAll === "function" && typeof g.afterAll === "function") {
|
|
26
|
+
return g;
|
|
27
|
+
}
|
|
28
|
+
return void 0;
|
|
29
|
+
}
|
|
30
|
+
async function importVitest() {
|
|
31
|
+
try {
|
|
32
|
+
return await import("vitest");
|
|
33
|
+
} catch (err) {
|
|
34
|
+
throw new Error(
|
|
35
|
+
`describeCases needs Vitest: could not import "vitest" (${err instanceof Error ? err.message : String(err)}). Install it (npm i -D vitest), enable globals, or pass { api: { describe, it, beforeAll, afterAll } }.`
|
|
36
|
+
);
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
function resolveOptions(input) {
|
|
40
|
+
const globs = input.globs === void 0 ? void 0 : Array.isArray(input.globs) ? input.globs : [input.globs];
|
|
41
|
+
const { config, profile, cwd, title: _title, api: _api, globs: _globs, checks, ...rest } = input;
|
|
42
|
+
const checkConfig = typeof checks === "object" ? { checks } : {};
|
|
43
|
+
let options;
|
|
44
|
+
if (config === false) {
|
|
45
|
+
options = {
|
|
46
|
+
...defaultRunOptions,
|
|
47
|
+
...rest,
|
|
48
|
+
...checkConfig,
|
|
49
|
+
normalize: { ...DEFAULT_NORMALIZE, ...rest.normalize ?? {} },
|
|
50
|
+
globs: globs ?? defaultRunOptions.globs,
|
|
51
|
+
rootDir: rest.rootDir ?? cwd ?? process.cwd()
|
|
52
|
+
};
|
|
53
|
+
} else {
|
|
54
|
+
const loaded = loadConfig(cwd ?? process.cwd(), {
|
|
55
|
+
...config !== void 0 ? { configPath: config } : {},
|
|
56
|
+
...profile !== void 0 ? { profile } : {},
|
|
57
|
+
overrides: { ...globs ? { globs } : {} }
|
|
58
|
+
});
|
|
59
|
+
options = {
|
|
60
|
+
...loaded.options,
|
|
61
|
+
...rest,
|
|
62
|
+
...checkConfig,
|
|
63
|
+
normalize: { ...loaded.options.normalize, ...rest.normalize ?? {} }
|
|
64
|
+
};
|
|
65
|
+
}
|
|
66
|
+
return {
|
|
67
|
+
options,
|
|
68
|
+
checks: checks === true || typeof checks === "object" || options.check === true
|
|
69
|
+
};
|
|
70
|
+
}
|
|
71
|
+
function failureMessage(loaded, outcome) {
|
|
72
|
+
const lines = [outcome.reason ?? "Case failed."];
|
|
73
|
+
for (const m of (outcome.mismatches ?? []).slice(0, 10)) lines.push(` - ${m.path}: ${m.message}`);
|
|
74
|
+
lines.push(` (${loaded.ref.id})`);
|
|
75
|
+
return lines.join("\n");
|
|
76
|
+
}
|
|
77
|
+
function register(api, resolved, title) {
|
|
78
|
+
const { options } = resolved;
|
|
79
|
+
const rootDir = resolveRootDir(options);
|
|
80
|
+
const { cases, warnings } = loadCasesSync({
|
|
81
|
+
rootDir,
|
|
82
|
+
globs: options.globs,
|
|
83
|
+
defaultServer: options.defaultServer,
|
|
84
|
+
servers: Object.keys(options.servers),
|
|
85
|
+
hooks: options.hooks,
|
|
86
|
+
manifest: options.manifest,
|
|
87
|
+
tags: options.tags,
|
|
88
|
+
excludeTags: options.excludeTags,
|
|
89
|
+
grep: options.grep,
|
|
90
|
+
shard: options.shard
|
|
91
|
+
});
|
|
92
|
+
const sessions = /* @__PURE__ */ new Map();
|
|
93
|
+
const servers = [...new Set(cases.filter((c) => !c.error && !c.skip).map((c) => c.ref.server))];
|
|
94
|
+
const timeout = Math.max(3e4, options.timeoutMs * 4);
|
|
95
|
+
const lockOn = options.lockManifest !== false;
|
|
96
|
+
const lockFor = (server) => manifestPathFor({ ...options, rootDir }, server);
|
|
97
|
+
const entryFor = (server) => {
|
|
98
|
+
const entry = sessions.get(server);
|
|
99
|
+
if (!entry)
|
|
100
|
+
throw new Error(`No session for server "${server}" (it failed to start in beforeAll).`);
|
|
101
|
+
return entry;
|
|
102
|
+
};
|
|
103
|
+
api.describe(title, () => {
|
|
104
|
+
api.beforeAll(async () => {
|
|
105
|
+
for (const warning of warnings) console.warn(`vibrissa: ${warning}`);
|
|
106
|
+
for (const server of servers) {
|
|
107
|
+
const session = await McpSession.start(resolveServerConfig(options, server), {
|
|
108
|
+
timeoutMs: options.timeoutMs,
|
|
109
|
+
stderr: "capture"
|
|
110
|
+
});
|
|
111
|
+
const needsManifest = resolved.checks || lockOn && fs.existsSync(lockFor(server));
|
|
112
|
+
const captured = needsManifest ? await captureManifest(session, { toolsListSamples: resolved.checks ? 2 : 1 }) : void 0;
|
|
113
|
+
const tools = new Map(
|
|
114
|
+
(captured?.toolsListSamples[0] ?? await session.listTools()).map((tool) => [
|
|
115
|
+
tool.name,
|
|
116
|
+
tool
|
|
117
|
+
])
|
|
118
|
+
);
|
|
119
|
+
sessions.set(server, { session, tools, captured });
|
|
120
|
+
}
|
|
121
|
+
}, timeout);
|
|
122
|
+
api.afterAll(async () => {
|
|
123
|
+
for (const { session } of sessions.values()) await session.close();
|
|
124
|
+
sessions.clear();
|
|
125
|
+
});
|
|
126
|
+
if (lockOn) {
|
|
127
|
+
for (const server of servers) {
|
|
128
|
+
const file = lockFor(server);
|
|
129
|
+
const shown = path.relative(rootDir, file).replace(/\\/g, "/") || file;
|
|
130
|
+
const record = `vib manifest record${server === options.defaultServer ? "" : ` --server ${server}`}`;
|
|
131
|
+
if (fs.existsSync(file)) {
|
|
132
|
+
api.it(`manifest lock (${shown})`, () => {
|
|
133
|
+
const captured = entryFor(server).captured;
|
|
134
|
+
if (!captured) throw new Error(`The live manifest of "${server}" was not captured.`);
|
|
135
|
+
const diff = diffManifest(readManifest(file), captured.manifest, {
|
|
136
|
+
ignore: options.manifestIgnore ?? DEFAULT_MANIFEST_IGNORE
|
|
137
|
+
});
|
|
138
|
+
if (!diff.identical) {
|
|
139
|
+
throw new Error(
|
|
140
|
+
`${formatDiff(diff)}
|
|
141
|
+
Review the change, then accept it with "${record}" (or vib run --update-manifest).`
|
|
142
|
+
);
|
|
143
|
+
}
|
|
144
|
+
});
|
|
145
|
+
} else if (options.requireManifest) {
|
|
146
|
+
api.it(`manifest lock (${shown})`, () => {
|
|
147
|
+
throw new Error(
|
|
148
|
+
`No manifest lock at ${shown}, and requireManifest is on. Record it with "${record}", review it, and commit it.`
|
|
149
|
+
);
|
|
150
|
+
});
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
if (resolved.checks) {
|
|
155
|
+
for (const server of servers) {
|
|
156
|
+
api.it(`checks (${server})`, () => {
|
|
157
|
+
const entry = entryFor(server);
|
|
158
|
+
if (!entry.captured) throw new Error(`The live manifest of "${server}" was not captured.`);
|
|
159
|
+
const config = resolveServerConfig(options, server);
|
|
160
|
+
const failed = runChecks(
|
|
161
|
+
{
|
|
162
|
+
...entry.captured.manifest,
|
|
163
|
+
transport: isHttpServerConfig(config) ? "http" : "stdio",
|
|
164
|
+
stdoutNoise: entry.session.stdoutNoise
|
|
165
|
+
},
|
|
166
|
+
{
|
|
167
|
+
toolsListSamples: entry.captured.toolsListSamples,
|
|
168
|
+
allow: options.checks?.allow,
|
|
169
|
+
severity: options.checks?.severity
|
|
170
|
+
}
|
|
171
|
+
).filter((outcome) => outcome.status === "failed");
|
|
172
|
+
if (failed.length > 0) {
|
|
173
|
+
throw new Error(
|
|
174
|
+
[
|
|
175
|
+
`${failed.length} check(s) failed on "${server}":`,
|
|
176
|
+
...failed.map(
|
|
177
|
+
(o) => ` - ${o.id}: ${o.title}${o.detail ? `
|
|
178
|
+
${o.detail}` : ""}`
|
|
179
|
+
),
|
|
180
|
+
'Skip a reviewed finding with "checks": { "allow": ["<id>:<subject>"] } in vibrissa.json.'
|
|
181
|
+
].join("\n")
|
|
182
|
+
);
|
|
183
|
+
}
|
|
184
|
+
});
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
if (cases.length === 0 && !options.allowEmpty) {
|
|
188
|
+
api.it("has cases", () => {
|
|
189
|
+
throw new Error(
|
|
190
|
+
`No test cases matched globs: ${options.globs.join(", ")} (from ${rootDir}). Check the globs passed to describeCases.`
|
|
191
|
+
);
|
|
192
|
+
});
|
|
193
|
+
}
|
|
194
|
+
for (const loaded of cases) {
|
|
195
|
+
const name = loaded.ref.name;
|
|
196
|
+
if (loaded.skip) {
|
|
197
|
+
if (api.it.skip) api.it.skip(name, () => {
|
|
198
|
+
});
|
|
199
|
+
continue;
|
|
200
|
+
}
|
|
201
|
+
api.it(
|
|
202
|
+
name,
|
|
203
|
+
async () => {
|
|
204
|
+
if (loaded.error || !loaded.data) throw new Error(loaded.error ?? "Case failed to load.");
|
|
205
|
+
const entry = entryFor(loaded.ref.server);
|
|
206
|
+
const execution = await executeCase(loaded.data, {
|
|
207
|
+
session: entry.session,
|
|
208
|
+
normalize: options.normalize,
|
|
209
|
+
timeoutMs: options.timeoutMs,
|
|
210
|
+
budgetMs: options.budgetMs,
|
|
211
|
+
tools: entry.tools,
|
|
212
|
+
update: "off",
|
|
213
|
+
// `vib run --update` is not reachable from here: keep it out of failure messages.
|
|
214
|
+
updateHint: false,
|
|
215
|
+
caseId: loaded.ref.id
|
|
216
|
+
});
|
|
217
|
+
if (execution.outcome.status === "failed") {
|
|
218
|
+
throw new Error(failureMessage(loaded, execution.outcome));
|
|
219
|
+
}
|
|
220
|
+
},
|
|
221
|
+
timeout
|
|
222
|
+
);
|
|
223
|
+
}
|
|
224
|
+
});
|
|
225
|
+
}
|
|
226
|
+
function describeCases(target = {}, extra = {}) {
|
|
227
|
+
const input = typeof target === "string" || Array.isArray(target) ? { ...extra, globs: target } : { ...target, ...extra };
|
|
228
|
+
const resolved = resolveOptions(input);
|
|
229
|
+
const title = input.title ?? "vibrissa";
|
|
230
|
+
const api = input.api ?? globalApi();
|
|
231
|
+
if (api) {
|
|
232
|
+
register(api, resolved, title);
|
|
233
|
+
return Promise.resolve();
|
|
234
|
+
}
|
|
235
|
+
return importVitest().then((imported) => register(imported, resolved, title));
|
|
236
|
+
}
|
|
237
|
+
export {
|
|
238
|
+
describeCases
|
|
239
|
+
};
|
|
240
|
+
//# sourceMappingURL=vitest.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../src/vitest.ts"],"sourcesContent":["/**\n * Vitest bridge: expand Vibrissa case files into Vitest tests so teams with an existing\n * Vitest suite get one report, gated on the same things as `vib run`.\n *\n * ```ts\n * // tests/mcp.test.ts\n * import { describeCases } from '@jmoak/vibrissa/vitest'\n *\n * await describeCases('tests/integration/**\\/*.json')\n * // or, with options (all optional):\n * await describeCases({\n * config: 'vibrissa.json', // false = no config file (pass servers/globs yourself)\n * profile: 'ci',\n * globs: ['tests/integration/**\\/*.json'],\n * checks: true, // also run `vib check` (spec sanity + description lint)\n * lockManifest: true, // default: verify the manifest lock when one exists\n * title: 'mcp contract',\n * })\n * ```\n *\n * Registers one `describe` per call with, in order:\n * - `manifest lock (<path>)` per server whose lock file exists (or a failing test when the\n * lock is missing and `requireManifest` is on), failing on drift with the rendered diff;\n * - `checks (<server>)` when `checks: true` (or config `check: true`), failing on failed checks;\n * - one `it` per case (`it.skip` for `skip: true`), failing with the reason and mismatches.\n *\n * One session per server for the whole block (beforeAll / afterAll). Vitest is an optional\n * peer dependency: pass `{ api: { describe, it, beforeAll, afterAll } }`, enable Vitest\n * `globals`, or let the bridge import `vitest` lazily (hence the `await`).\n */\nimport fs from 'node:fs'\nimport path from 'node:path'\nimport { type LoadedCase, loadCasesSync } from './cases.js'\nimport { loadConfig } from './config.js'\nimport { DEFAULT_NORMALIZE, defaultRunOptions } from './defaults.js'\nimport { runChecks } from './lint/index.js'\nimport {\n type CapturedManifest,\n DEFAULT_MANIFEST_IGNORE,\n captureManifest,\n diffManifest,\n formatDiff,\n manifestPathFor,\n readManifest,\n} from './manifest.js'\nimport { resolveRootDir, resolveServerConfig } from './paths.js'\nimport { type RunnerOptions, executeCase } from './runner.js'\nimport { McpSession } from './session/index.js'\nimport type { RunOptions, ToolSummary } from './types.js'\nimport { isHttpServerConfig } from './types.js'\n\ntype TestFn = () => void | Promise<void>\n\n/** The slice of the Vitest API the bridge uses. */\nexport interface VitestApi {\n describe: (name: string, fn: () => void) => void\n it: ((name: string, fn: TestFn, timeout?: number) => void) & {\n skip?: (name: string, fn: TestFn) => void\n }\n beforeAll: (fn: TestFn, timeout?: number) => void\n afterAll: (fn: TestFn, timeout?: number) => void\n}\n\n/**\n * Options for `describeCases`. Every RunOptions key may be overridden here (servers,\n * normalize, timeoutMs, tags, grep, manifest, manifestIgnore, requireManifest, ...).\n */\nexport interface DescribeCasesOptions extends Partial<Omit<RunnerOptions, 'globs' | 'checks'>> {\n /** Case globs (relative to the config's directory). Default: config `globs`. */\n globs?: string | string[]\n /** Config file to load; `false` skips config loading. Default: vibrissa.json from `cwd`. */\n config?: string | false\n /** Profile to merge over the config. */\n profile?: string\n /** Where to look for the config. Default process.cwd(). */\n cwd?: string\n /** `describe` title. Default \"vibrissa\". */\n title?: string\n /**\n * `true` adds a `checks (<server>)` test (spec sanity + description lint, like\n * `vib run --check`); an object also sets `allow` / `severity` (and turns checks on).\n */\n checks?: boolean | NonNullable<RunOptions['checks']>\n /** Vitest functions; default: globals, else `import('vitest')`. */\n api?: VitestApi\n}\n\ninterface Resolved {\n options: RunOptions & Pick<RunnerOptions, 'lockManifest'>\n checks: boolean\n}\n\nfunction globalApi(): VitestApi | undefined {\n const g = globalThis as Record<string, unknown>\n if (\n typeof g.describe === 'function' &&\n typeof g.it === 'function' &&\n typeof g.beforeAll === 'function' &&\n typeof g.afterAll === 'function'\n ) {\n return g as unknown as VitestApi\n }\n return undefined\n}\n\nasync function importVitest(): Promise<VitestApi> {\n try {\n // A literal specifier so bundlers and Vitest's module runner resolve it; vitest stays\n // external (optional peer) in the published build.\n return (await import('vitest')) as unknown as VitestApi\n } catch (err) {\n throw new Error(\n `describeCases needs Vitest: could not import \"vitest\" (${err instanceof Error ? err.message : String(err)}). Install it (npm i -D vitest), enable globals, or pass { api: { describe, it, beforeAll, afterAll } }.`,\n )\n }\n}\n\nfunction resolveOptions(input: DescribeCasesOptions): Resolved {\n const globs =\n input.globs === undefined ? undefined : Array.isArray(input.globs) ? input.globs : [input.globs]\n const { config, profile, cwd, title: _title, api: _api, globs: _globs, checks, ...rest } = input\n const checkConfig = typeof checks === 'object' ? { checks } : {}\n let options: Resolved['options']\n if (config === false) {\n options = {\n ...defaultRunOptions,\n ...rest,\n ...checkConfig,\n normalize: { ...DEFAULT_NORMALIZE, ...(rest.normalize ?? {}) },\n globs: globs ?? defaultRunOptions.globs,\n rootDir: rest.rootDir ?? cwd ?? process.cwd(),\n } as Resolved['options']\n } else {\n const loaded = loadConfig(cwd ?? process.cwd(), {\n ...(config !== undefined ? { configPath: config } : {}),\n ...(profile !== undefined ? { profile } : {}),\n overrides: { ...(globs ? { globs } : {}) },\n })\n options = {\n ...loaded.options,\n ...rest,\n ...checkConfig,\n normalize: { ...loaded.options.normalize, ...(rest.normalize ?? {}) },\n } as Resolved['options']\n }\n return {\n options,\n checks: checks === true || typeof checks === 'object' || options.check === true,\n }\n}\n\nfunction failureMessage(\n loaded: LoadedCase,\n outcome: { reason?: string; mismatches?: Array<{ path: string; message: string }> },\n): string {\n const lines = [outcome.reason ?? 'Case failed.']\n for (const m of (outcome.mismatches ?? []).slice(0, 10)) lines.push(` - ${m.path}: ${m.message}`)\n lines.push(` (${loaded.ref.id})`)\n return lines.join('\\n')\n}\n\ninterface ServerEntry {\n session: McpSession\n tools: Map<string, ToolSummary>\n captured?: CapturedManifest\n}\n\nfunction register(api: VitestApi, resolved: Resolved, title: string): void {\n const { options } = resolved\n const rootDir = resolveRootDir(options)\n const { cases, warnings } = loadCasesSync({\n rootDir,\n globs: options.globs,\n defaultServer: options.defaultServer,\n servers: Object.keys(options.servers),\n hooks: options.hooks,\n manifest: options.manifest,\n tags: options.tags,\n excludeTags: options.excludeTags,\n grep: options.grep,\n shard: options.shard,\n })\n const sessions = new Map<string, ServerEntry>()\n const servers = [...new Set(cases.filter((c) => !c.error && !c.skip).map((c) => c.ref.server))]\n const timeout = Math.max(30000, options.timeoutMs * 4)\n const lockOn = options.lockManifest !== false\n const lockFor = (server: string) => manifestPathFor({ ...options, rootDir }, server)\n\n const entryFor = (server: string): ServerEntry => {\n const entry = sessions.get(server)\n if (!entry)\n throw new Error(`No session for server \"${server}\" (it failed to start in beforeAll).`)\n return entry\n }\n\n api.describe(title, () => {\n api.beforeAll(async () => {\n for (const warning of warnings) console.warn(`vibrissa: ${warning}`)\n for (const server of servers) {\n const session = await McpSession.start(resolveServerConfig(options, server), {\n timeoutMs: options.timeoutMs,\n stderr: 'capture',\n })\n const needsManifest = resolved.checks || (lockOn && fs.existsSync(lockFor(server)))\n const captured = needsManifest\n ? await captureManifest(session, { toolsListSamples: resolved.checks ? 2 : 1 })\n : undefined\n const tools = new Map(\n (captured?.toolsListSamples[0] ?? (await session.listTools())).map((tool) => [\n tool.name,\n tool,\n ]),\n )\n sessions.set(server, { session, tools, captured })\n }\n }, timeout)\n\n api.afterAll(async () => {\n for (const { session } of sessions.values()) await session.close()\n sessions.clear()\n })\n\n if (lockOn) {\n for (const server of servers) {\n const file = lockFor(server)\n const shown = path.relative(rootDir, file).replace(/\\\\/g, '/') || file\n const record = `vib manifest record${server === options.defaultServer ? '' : ` --server ${server}`}`\n if (fs.existsSync(file)) {\n api.it(`manifest lock (${shown})`, () => {\n const captured = entryFor(server).captured\n if (!captured) throw new Error(`The live manifest of \"${server}\" was not captured.`)\n const diff = diffManifest(readManifest(file), captured.manifest, {\n ignore: options.manifestIgnore ?? DEFAULT_MANIFEST_IGNORE,\n })\n if (!diff.identical) {\n throw new Error(\n `${formatDiff(diff)}\\nReview the change, then accept it with \"${record}\" (or vib run --update-manifest).`,\n )\n }\n })\n } else if (options.requireManifest) {\n api.it(`manifest lock (${shown})`, () => {\n throw new Error(\n `No manifest lock at ${shown}, and requireManifest is on. Record it with \"${record}\", review it, and commit it.`,\n )\n })\n }\n }\n }\n\n if (resolved.checks) {\n for (const server of servers) {\n api.it(`checks (${server})`, () => {\n const entry = entryFor(server)\n if (!entry.captured) throw new Error(`The live manifest of \"${server}\" was not captured.`)\n const config = resolveServerConfig(options, server)\n const failed = runChecks(\n {\n ...entry.captured.manifest,\n transport: isHttpServerConfig(config) ? 'http' : 'stdio',\n stdoutNoise: entry.session.stdoutNoise,\n },\n {\n toolsListSamples: entry.captured.toolsListSamples,\n allow: options.checks?.allow,\n severity: options.checks?.severity,\n },\n ).filter((outcome) => outcome.status === 'failed')\n if (failed.length > 0) {\n throw new Error(\n [\n `${failed.length} check(s) failed on \"${server}\":`,\n ...failed.map(\n (o) => ` - ${o.id}: ${o.title}${o.detail ? `\\n ${o.detail}` : ''}`,\n ),\n 'Skip a reviewed finding with \"checks\": { \"allow\": [\"<id>:<subject>\"] } in vibrissa.json.',\n ].join('\\n'),\n )\n }\n })\n }\n }\n\n if (cases.length === 0 && !options.allowEmpty) {\n api.it('has cases', () => {\n throw new Error(\n `No test cases matched globs: ${options.globs.join(', ')} (from ${rootDir}). Check the globs passed to describeCases.`,\n )\n })\n }\n\n for (const loaded of cases) {\n const name = loaded.ref.name\n if (loaded.skip) {\n if (api.it.skip) api.it.skip(name, () => {})\n continue\n }\n api.it(\n name,\n async () => {\n if (loaded.error || !loaded.data) throw new Error(loaded.error ?? 'Case failed to load.')\n const entry = entryFor(loaded.ref.server)\n const execution = await executeCase(loaded.data, {\n session: entry.session,\n normalize: options.normalize,\n timeoutMs: options.timeoutMs,\n budgetMs: options.budgetMs,\n tools: entry.tools,\n update: 'off',\n // `vib run --update` is not reachable from here: keep it out of failure messages.\n updateHint: false,\n caseId: loaded.ref.id,\n })\n if (execution.outcome.status === 'failed') {\n throw new Error(failureMessage(loaded, execution.outcome))\n }\n },\n timeout,\n )\n }\n })\n}\n\n/**\n * Register Vitest tests for a Vibrissa suite: manifest lock, optional checks, one test per\n * case. Accepts globs or DescribeCasesOptions. Resolves once the tests are registered;\n * `await` it at the top of the test file unless you pass `api` or use Vitest globals.\n */\nexport function describeCases(\n target: string | string[] | DescribeCasesOptions = {},\n extra: DescribeCasesOptions = {},\n): Promise<void> {\n const input: DescribeCasesOptions =\n typeof target === 'string' || Array.isArray(target)\n ? { ...extra, globs: target }\n : { ...target, ...extra }\n const resolved = resolveOptions(input)\n const title = input.title ?? 'vibrissa'\n const api = input.api ?? globalApi()\n if (api) {\n register(api, resolved, title)\n return Promise.resolve()\n }\n return importVitest().then((imported) => register(imported, resolved, title))\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AA8BA,OAAO,QAAQ;AACf,OAAO,UAAU;AA6DjB,SAAS,YAAmC;AAC1C,QAAM,IAAI;AACV,MACE,OAAO,EAAE,aAAa,cACtB,OAAO,EAAE,OAAO,cAChB,OAAO,EAAE,cAAc,cACvB,OAAO,EAAE,aAAa,YACtB;AACA,WAAO;AAAA,EACT;AACA,SAAO;AACT;AAEA,eAAe,eAAmC;AAChD,MAAI;AAGF,WAAQ,MAAM,OAAO,QAAQ;AAAA,EAC/B,SAAS,KAAK;AACZ,UAAM,IAAI;AAAA,MACR,0DAA0D,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;AAAA,IAC5G;AAAA,EACF;AACF;AAEA,SAAS,eAAe,OAAuC;AAC7D,QAAM,QACJ,MAAM,UAAU,SAAY,SAAY,MAAM,QAAQ,MAAM,KAAK,IAAI,MAAM,QAAQ,CAAC,MAAM,KAAK;AACjG,QAAM,EAAE,QAAQ,SAAS,KAAK,OAAO,QAAQ,KAAK,MAAM,OAAO,QAAQ,QAAQ,GAAG,KAAK,IAAI;AAC3F,QAAM,cAAc,OAAO,WAAW,WAAW,EAAE,OAAO,IAAI,CAAC;AAC/D,MAAI;AACJ,MAAI,WAAW,OAAO;AACpB,cAAU;AAAA,MACR,GAAG;AAAA,MACH,GAAG;AAAA,MACH,GAAG;AAAA,MACH,WAAW,EAAE,GAAG,mBAAmB,GAAI,KAAK,aAAa,CAAC,EAAG;AAAA,MAC7D,OAAO,SAAS,kBAAkB;AAAA,MAClC,SAAS,KAAK,WAAW,OAAO,QAAQ,IAAI;AAAA,IAC9C;AAAA,EACF,OAAO;AACL,UAAM,SAAS,WAAW,OAAO,QAAQ,IAAI,GAAG;AAAA,MAC9C,GAAI,WAAW,SAAY,EAAE,YAAY,OAAO,IAAI,CAAC;AAAA,MACrD,GAAI,YAAY,SAAY,EAAE,QAAQ,IAAI,CAAC;AAAA,MAC3C,WAAW,EAAE,GAAI,QAAQ,EAAE,MAAM,IAAI,CAAC,EAAG;AAAA,IAC3C,CAAC;AACD,cAAU;AAAA,MACR,GAAG,OAAO;AAAA,MACV,GAAG;AAAA,MACH,GAAG;AAAA,MACH,WAAW,EAAE,GAAG,OAAO,QAAQ,WAAW,GAAI,KAAK,aAAa,CAAC,EAAG;AAAA,IACtE;AAAA,EACF;AACA,SAAO;AAAA,IACL;AAAA,IACA,QAAQ,WAAW,QAAQ,OAAO,WAAW,YAAY,QAAQ,UAAU;AAAA,EAC7E;AACF;AAEA,SAAS,eACP,QACA,SACQ;AACR,QAAM,QAAQ,CAAC,QAAQ,UAAU,cAAc;AAC/C,aAAW,MAAM,QAAQ,cAAc,CAAC,GAAG,MAAM,GAAG,EAAE,EAAG,OAAM,KAAK,OAAO,EAAE,IAAI,KAAK,EAAE,OAAO,EAAE;AACjG,QAAM,KAAK,MAAM,OAAO,IAAI,EAAE,GAAG;AACjC,SAAO,MAAM,KAAK,IAAI;AACxB;AAQA,SAAS,SAAS,KAAgB,UAAoB,OAAqB;AACzE,QAAM,EAAE,QAAQ,IAAI;AACpB,QAAM,UAAU,eAAe,OAAO;AACtC,QAAM,EAAE,OAAO,SAAS,IAAI,cAAc;AAAA,IACxC;AAAA,IACA,OAAO,QAAQ;AAAA,IACf,eAAe,QAAQ;AAAA,IACvB,SAAS,OAAO,KAAK,QAAQ,OAAO;AAAA,IACpC,OAAO,QAAQ;AAAA,IACf,UAAU,QAAQ;AAAA,IAClB,MAAM,QAAQ;AAAA,IACd,aAAa,QAAQ;AAAA,IACrB,MAAM,QAAQ;AAAA,IACd,OAAO,QAAQ;AAAA,EACjB,CAAC;AACD,QAAM,WAAW,oBAAI,IAAyB;AAC9C,QAAM,UAAU,CAAC,GAAG,IAAI,IAAI,MAAM,OAAO,CAAC,MAAM,CAAC,EAAE,SAAS,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,MAAM,CAAC,CAAC;AAC9F,QAAM,UAAU,KAAK,IAAI,KAAO,QAAQ,YAAY,CAAC;AACrD,QAAM,SAAS,QAAQ,iBAAiB;AACxC,QAAM,UAAU,CAAC,WAAmB,gBAAgB,EAAE,GAAG,SAAS,QAAQ,GAAG,MAAM;AAEnF,QAAM,WAAW,CAAC,WAAgC;AAChD,UAAM,QAAQ,SAAS,IAAI,MAAM;AACjC,QAAI,CAAC;AACH,YAAM,IAAI,MAAM,0BAA0B,MAAM,sCAAsC;AACxF,WAAO;AAAA,EACT;AAEA,MAAI,SAAS,OAAO,MAAM;AACxB,QAAI,UAAU,YAAY;AACxB,iBAAW,WAAW,SAAU,SAAQ,KAAK,aAAa,OAAO,EAAE;AACnE,iBAAW,UAAU,SAAS;AAC5B,cAAM,UAAU,MAAM,WAAW,MAAM,oBAAoB,SAAS,MAAM,GAAG;AAAA,UAC3E,WAAW,QAAQ;AAAA,UACnB,QAAQ;AAAA,QACV,CAAC;AACD,cAAM,gBAAgB,SAAS,UAAW,UAAU,GAAG,WAAW,QAAQ,MAAM,CAAC;AACjF,cAAM,WAAW,gBACb,MAAM,gBAAgB,SAAS,EAAE,kBAAkB,SAAS,SAAS,IAAI,EAAE,CAAC,IAC5E;AACJ,cAAM,QAAQ,IAAI;AAAA,WACf,UAAU,iBAAiB,CAAC,KAAM,MAAM,QAAQ,UAAU,GAAI,IAAI,CAAC,SAAS;AAAA,YAC3E,KAAK;AAAA,YACL;AAAA,UACF,CAAC;AAAA,QACH;AACA,iBAAS,IAAI,QAAQ,EAAE,SAAS,OAAO,SAAS,CAAC;AAAA,MACnD;AAAA,IACF,GAAG,OAAO;AAEV,QAAI,SAAS,YAAY;AACvB,iBAAW,EAAE,QAAQ,KAAK,SAAS,OAAO,EAAG,OAAM,QAAQ,MAAM;AACjE,eAAS,MAAM;AAAA,IACjB,CAAC;AAED,QAAI,QAAQ;AACV,iBAAW,UAAU,SAAS;AAC5B,cAAM,OAAO,QAAQ,MAAM;AAC3B,cAAM,QAAQ,KAAK,SAAS,SAAS,IAAI,EAAE,QAAQ,OAAO,GAAG,KAAK;AAClE,cAAM,SAAS,sBAAsB,WAAW,QAAQ,gBAAgB,KAAK,aAAa,MAAM,EAAE;AAClG,YAAI,GAAG,WAAW,IAAI,GAAG;AACvB,cAAI,GAAG,kBAAkB,KAAK,KAAK,MAAM;AACvC,kBAAM,WAAW,SAAS,MAAM,EAAE;AAClC,gBAAI,CAAC,SAAU,OAAM,IAAI,MAAM,yBAAyB,MAAM,qBAAqB;AACnF,kBAAM,OAAO,aAAa,aAAa,IAAI,GAAG,SAAS,UAAU;AAAA,cAC/D,QAAQ,QAAQ,kBAAkB;AAAA,YACpC,CAAC;AACD,gBAAI,CAAC,KAAK,WAAW;AACnB,oBAAM,IAAI;AAAA,gBACR,GAAG,WAAW,IAAI,CAAC;AAAA,0CAA6C,MAAM;AAAA,cACxE;AAAA,YACF;AAAA,UACF,CAAC;AAAA,QACH,WAAW,QAAQ,iBAAiB;AAClC,cAAI,GAAG,kBAAkB,KAAK,KAAK,MAAM;AACvC,kBAAM,IAAI;AAAA,cACR,uBAAuB,KAAK,gDAAgD,MAAM;AAAA,YACpF;AAAA,UACF,CAAC;AAAA,QACH;AAAA,MACF;AAAA,IACF;AAEA,QAAI,SAAS,QAAQ;AACnB,iBAAW,UAAU,SAAS;AAC5B,YAAI,GAAG,WAAW,MAAM,KAAK,MAAM;AACjC,gBAAM,QAAQ,SAAS,MAAM;AAC7B,cAAI,CAAC,MAAM,SAAU,OAAM,IAAI,MAAM,yBAAyB,MAAM,qBAAqB;AACzF,gBAAM,SAAS,oBAAoB,SAAS,MAAM;AAClD,gBAAM,SAAS;AAAA,YACb;AAAA,cACE,GAAG,MAAM,SAAS;AAAA,cAClB,WAAW,mBAAmB,MAAM,IAAI,SAAS;AAAA,cACjD,aAAa,MAAM,QAAQ;AAAA,YAC7B;AAAA,YACA;AAAA,cACE,kBAAkB,MAAM,SAAS;AAAA,cACjC,OAAO,QAAQ,QAAQ;AAAA,cACvB,UAAU,QAAQ,QAAQ;AAAA,YAC5B;AAAA,UACF,EAAE,OAAO,CAAC,YAAY,QAAQ,WAAW,QAAQ;AACjD,cAAI,OAAO,SAAS,GAAG;AACrB,kBAAM,IAAI;AAAA,cACR;AAAA,gBACE,GAAG,OAAO,MAAM,wBAAwB,MAAM;AAAA,gBAC9C,GAAG,OAAO;AAAA,kBACR,CAAC,MAAM,OAAO,EAAE,EAAE,KAAK,EAAE,KAAK,GAAG,EAAE,SAAS;AAAA,QAAW,EAAE,MAAM,KAAK,EAAE;AAAA,gBACxE;AAAA,gBACA;AAAA,cACF,EAAE,KAAK,IAAI;AAAA,YACb;AAAA,UACF;AAAA,QACF,CAAC;AAAA,MACH;AAAA,IACF;AAEA,QAAI,MAAM,WAAW,KAAK,CAAC,QAAQ,YAAY;AAC7C,UAAI,GAAG,aAAa,MAAM;AACxB,cAAM,IAAI;AAAA,UACR,gCAAgC,QAAQ,MAAM,KAAK,IAAI,CAAC,UAAU,OAAO;AAAA,QAC3E;AAAA,MACF,CAAC;AAAA,IACH;AAEA,eAAW,UAAU,OAAO;AAC1B,YAAM,OAAO,OAAO,IAAI;AACxB,UAAI,OAAO,MAAM;AACf,YAAI,IAAI,GAAG,KAAM,KAAI,GAAG,KAAK,MAAM,MAAM;AAAA,QAAC,CAAC;AAC3C;AAAA,MACF;AACA,UAAI;AAAA,QACF;AAAA,QACA,YAAY;AACV,cAAI,OAAO,SAAS,CAAC,OAAO,KAAM,OAAM,IAAI,MAAM,OAAO,SAAS,sBAAsB;AACxF,gBAAM,QAAQ,SAAS,OAAO,IAAI,MAAM;AACxC,gBAAM,YAAY,MAAM,YAAY,OAAO,MAAM;AAAA,YAC/C,SAAS,MAAM;AAAA,YACf,WAAW,QAAQ;AAAA,YACnB,WAAW,QAAQ;AAAA,YACnB,UAAU,QAAQ;AAAA,YAClB,OAAO,MAAM;AAAA,YACb,QAAQ;AAAA;AAAA,YAER,YAAY;AAAA,YACZ,QAAQ,OAAO,IAAI;AAAA,UACrB,CAAC;AACD,cAAI,UAAU,QAAQ,WAAW,UAAU;AACzC,kBAAM,IAAI,MAAM,eAAe,QAAQ,UAAU,OAAO,CAAC;AAAA,UAC3D;AAAA,QACF;AAAA,QACA;AAAA,MACF;AAAA,IACF;AAAA,EACF,CAAC;AACH;AAOO,SAAS,cACd,SAAmD,CAAC,GACpD,QAA8B,CAAC,GAChB;AACf,QAAM,QACJ,OAAO,WAAW,YAAY,MAAM,QAAQ,MAAM,IAC9C,EAAE,GAAG,OAAO,OAAO,OAAO,IAC1B,EAAE,GAAG,QAAQ,GAAG,MAAM;AAC5B,QAAM,WAAW,eAAe,KAAK;AACrC,QAAM,QAAQ,MAAM,SAAS;AAC7B,QAAM,MAAM,MAAM,OAAO,UAAU;AACnC,MAAI,KAAK;AACP,aAAS,KAAK,UAAU,KAAK;AAC7B,WAAO,QAAQ,QAAQ;AAAA,EACzB;AACA,SAAO,aAAa,EAAE,KAAK,CAAC,aAAa,SAAS,UAAU,UAAU,KAAK,CAAC;AAC9E;","names":[]}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@jmoak/vibrissa",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "2.0.0-alpha.0",
|
|
4
4
|
"description": "Ultra-light JSON-driven MCP integration test runner",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"engines": {
|
|
@@ -17,21 +17,26 @@
|
|
|
17
17
|
},
|
|
18
18
|
"./cli": {
|
|
19
19
|
"import": "./dist/cli.js"
|
|
20
|
+
},
|
|
21
|
+
"./vitest": {
|
|
22
|
+
"import": "./dist/vitest.js",
|
|
23
|
+
"types": "./dist/vitest.d.ts"
|
|
20
24
|
}
|
|
21
25
|
},
|
|
22
26
|
"files": ["dist", "bin", "schema", "README.md", "LICENSE"],
|
|
23
27
|
"scripts": {
|
|
24
|
-
"build": "tsup src/cli.ts src/index.ts --format esm --target node20 --dts --clean --sourcemap --external fast-glob",
|
|
25
|
-
"dev": "tsup src/cli.ts src/index.ts --format esm --target node20 --dts --watch --sourcemap --external fast-glob",
|
|
28
|
+
"build": "tsup src/cli.ts src/index.ts src/vitest.ts --format esm --target node20 --dts --clean --sourcemap --external fast-glob --external vitest",
|
|
29
|
+
"dev": "tsup src/cli.ts src/index.ts src/vitest.ts --format esm --target node20 --dts --watch --sourcemap --external fast-glob --external vitest",
|
|
26
30
|
"typecheck": "tsc --noEmit",
|
|
27
31
|
"test": "vitest run",
|
|
28
32
|
"test:ci": "vitest run --reporter=default --reporter=junit --outputFile.junit=reports/junit/junit.xml",
|
|
29
|
-
"prepublishOnly": "npm run build && npm run typecheck && npm test",
|
|
30
|
-
"changeset": "changeset",
|
|
31
33
|
"e2e": "npm run build && node bin/vib-test.js --config tests/fixtures/echo-server/vibrissa.json",
|
|
32
34
|
"e2e:demo": "npm run build && node bin/vib-test.js --config tests/fixtures/echo-server/vibrissa.json --cases \"tests/fixtures/echo-server/cases/**/*.json\"",
|
|
33
35
|
"lint": "biome check . --write",
|
|
34
|
-
"lint:fix": "biome check . --write --unsafe"
|
|
36
|
+
"lint:fix": "biome check . --write --unsafe",
|
|
37
|
+
"prepack": "npm run build",
|
|
38
|
+
"release": "semantic-release",
|
|
39
|
+
"test:release": "node --test scripts/release.test.mjs"
|
|
35
40
|
},
|
|
36
41
|
"repository": {
|
|
37
42
|
"type": "git",
|
|
@@ -51,7 +56,8 @@
|
|
|
51
56
|
"license": "MIT",
|
|
52
57
|
"publishConfig": {
|
|
53
58
|
"access": "public",
|
|
54
|
-
"provenance": true
|
|
59
|
+
"provenance": true,
|
|
60
|
+
"registry": "https://registry.npmjs.org/"
|
|
55
61
|
},
|
|
56
62
|
"bugs": {
|
|
57
63
|
"url": "https://github.com/JMoak/vibrissa/issues"
|
|
@@ -59,16 +65,28 @@
|
|
|
59
65
|
"homepage": "https://github.com/JMoak/vibrissa#readme",
|
|
60
66
|
"devDependencies": {
|
|
61
67
|
"@biomejs/biome": "^1.8.3",
|
|
62
|
-
"@
|
|
68
|
+
"@modelcontextprotocol/server": "^2.4.0",
|
|
63
69
|
"@types/node": "^20.19.13",
|
|
64
70
|
"@vitest/coverage-v8": "^4.1.10",
|
|
71
|
+
"conventional-changelog-conventionalcommits": "8.0.0",
|
|
72
|
+
"semantic-release": "25.0.9",
|
|
65
73
|
"tsup": "^8.5.0",
|
|
66
74
|
"typescript": "^5.9.2",
|
|
67
75
|
"vitest": "^4.1.10"
|
|
68
76
|
},
|
|
69
77
|
"dependencies": {
|
|
70
|
-
"@modelcontextprotocol/
|
|
78
|
+
"@modelcontextprotocol/client": "^2.4.0",
|
|
79
|
+
"ajv": "^8.20.0",
|
|
80
|
+
"ajv-formats": "^3.0.1",
|
|
71
81
|
"fast-glob": "^3.3.2",
|
|
72
82
|
"jest-diff": "^30.1.3"
|
|
83
|
+
},
|
|
84
|
+
"peerDependencies": {
|
|
85
|
+
"vitest": ">=1.0.0"
|
|
86
|
+
},
|
|
87
|
+
"peerDependenciesMeta": {
|
|
88
|
+
"vitest": {
|
|
89
|
+
"optional": true
|
|
90
|
+
}
|
|
73
91
|
}
|
|
74
92
|
}
|