@memberjunction/unit-testing 6.1.0-edge.6 → 6.1.0-edge.7

package/dist/ai/llm-tool-conformance.d.ts

@@ -0,0 +1,101 @@
+/**
+ * @fileoverview Shared native-tool-calling conformance suite for `BaseLLM` drivers.
+ *
+ * WHY THIS IS SEPARATE FROM `llm-conformance.ts`: that suite drives the streaming / cancellation /
+ * failure template method, and adopting it requires scripting a vendor mock for all of those paths.
+ * The tool-calling contract is independent of them and much cheaper to adopt, so it ships as its own
+ * entry point — a driver can conform on tools without first conforming on streaming. Run both when
+ * a provider has configs for both; they register separate `describe` blocks and do not interact.
+ *
+ * WHAT THIS ASSERTS — the parts of the tool contract that are the SAME for every provider, because
+ * they live in `BaseLLM` and in the normalized `ChatResult` shape rather than in any SDK mapping:
+ *
+ *  - `SupportsTools` reports the driver's real capability (a CODE-level fact: "has the mapping been
+ *    written?", distinct from the metadata capability flag `LLM.SupportsNativeToolCalling`).
+ *  - A tool-call turn normalizes to `ChatCompletionMessage.toolCalls` with **parsed** arguments —
+ *    never a JSON string, whatever the wire format was.
+ *  - Such a turn reports `finish_reason === 'tool_calls'`.
+ *  - Text and tool calls coexist: a turn carrying both surfaces both, and a turn carrying only
+ *    calls is a SUCCESS with empty content — not a "no output from model" failure.
+ *  - An ordinary turn is untouched: no `toolCalls`, and the driver's own finish reason preserved.
+ *  - Streaming plus tools takes the non-streaming path and flags `streamingSuppressedForTools`
+ *    (native tool calling is non-streaming only).
+ *  - An orphaned tool result — one whose call was never declared by an earlier assistant turn —
+ *    is rejected at the MJ boundary rather than at the provider.
+ *
+ * Provider-SPECIFIC request mapping (Anthropic `input_schema` vs OpenAI `parameters` vs Gemini
+ * `parametersJsonSchema`; `tool_choice` vocabularies; how results are shaped into the history) stays
+ * in each driver's own test file — there is no shared truth to assert about it.
+ *
+ * A driver that does NOT implement tools passes `SupportsTools: false` and gets the negative
+ * contract asserted instead: it declares false, and declaring tools to it is harmless.
+ *
+ * HOW A PROVIDER PACKAGE USES IT:
+ * ```ts
+ * import { RunLLMToolCallingConformanceSuite } from '@memberjunction/unit-testing';
+ *
+ * RunLLMToolCallingConformanceSuite({
+ *     ProviderName: 'MyProvider',
+ *     CreateLLM: () => buildLLMWithMockedVendorClient(),
+ *     SupportsTools: true,
+ *     ScriptToolCallResponse: (calls, text) => { ... },
+ *     ScriptTextResponse: (text) => { ... },
+ *     OrdinaryFinishReason: 'stop',
+ * });
+ * ```
+ *
+ * Same rules as the streaming suite: do NOT `vi.mock('@memberjunction/ai')` in a file that runs
+ * this — the point is to drive the REAL `BaseLLM` and the REAL normalized result types. Mock only
+ * the vendor SDK seam.
+ */
+import type { BaseLLM } from '@memberjunction/ai';
+/** One tool call a scripted vendor response should report. */
+export interface ScriptedToolCall {
+    /** Provider call id. Drivers that synthesize ids (Gemini) may ignore this. */
+    Id: string;
+    /** Tool name — matches a declared {@link ChatTool.name}. */
+    Name: string;
+    /** Arguments as a structured object; the mock renders them in its own wire format. */
+    Arguments: Record<string, unknown>;
+}
+export interface LLMToolConformanceSuiteConfig {
+    /** Human-readable provider name, used in describe() labels. */
+    ProviderName: string;
+    /**
+     * Construct a fresh driver instance wired to the scriptable vendor mock. Called in beforeEach,
+     * so every test gets a clean instance.
+     */
+    CreateLLM: () => BaseLLM;
+    /** Whether the driver implements the tool mapping (asserted against `SupportsTools`). */
+    SupportsTools: boolean;
+    /**
+     * Program the vendor mock: the next call returns these tool calls, plus `text` when the
+     * provider is also emitting prose on the same turn. Required when `SupportsTools` is true.
+     *
+     * The mock must render the calls the way its SDK really does — OpenAI's JSON-STRING arguments,
+     * Anthropic's parsed `input`, Gemini's `functionCall` parts — because turning that back into
+     * parsed arguments is exactly what the suite is checking.
+     */
+    ScriptToolCallResponse?: (calls: ScriptedToolCall[], text?: string) => void;
+    /** Program the vendor mock: the next call returns an ordinary text answer, no tool calls. */
+    ScriptTextResponse?: (text: string) => void;
+    /**
+     * The `finish_reason` this driver reports on an ordinary (non-tool) turn — `'stop'` for the
+     * OpenAI family, `'completed'` for Anthropic, `'STOP'` for Gemini.
+     *
+     * That these differ at all is a known gap: `finish_reason` is typed as a bare `string` and has
+     * never been normalized across drivers. Native tool calling adds the FIRST normalized value
+     * (`'tool_calls'`); the rest is captured here rather than papered over, so the inconsistency is
+     * visible in one place.
+     *
+     * Tracked in MemberJunction/MJ#4335. When that lands, every driver reports the same value here
+     * and this knob should be deleted rather than updated.
+     */
+    OrdinaryFinishReason?: string;
+}
+/**
+ * Run the shared native-tool-calling conformance suite against one provider driver. Registers a
+ * full `describe` block — call it at the top level of a provider test file.
+ */
+export declare function RunLLMToolCallingConformanceSuite(config: LLMToolConformanceSuiteConfig): void;
+//# sourceMappingURL=llm-tool-conformance.d.ts.map

package/dist/ai/llm-tool-conformance.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"llm-tool-conformance.d.ts","sourceRoot":"","sources":["../../src/ai/llm-tool-conformance.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiDG;AAGH,OAAO,KAAK,EAAE,OAAO,EAAmD,MAAM,oBAAoB,CAAC;AAGnG,8DAA8D;AAC9D,MAAM,WAAW,gBAAgB;IAC7B,8EAA8E;IAC9E,EAAE,EAAE,MAAM,CAAC;IACX,4DAA4D;IAC5D,IAAI,EAAE,MAAM,CAAC;IACb,sFAAsF;IACtF,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACtC;AAED,MAAM,WAAW,6BAA6B;IAC1C,+DAA+D;IAC/D,YAAY,EAAE,MAAM,CAAC;IAErB;;;OAGG;IACH,SAAS,EAAE,MAAM,OAAO,CAAC;IAEzB,yFAAyF;IACzF,aAAa,EAAE,OAAO,CAAC;IAEvB;;;;;;;OAOG;IACH,sBAAsB,CAAC,EAAE,CAAC,KAAK,EAAE,gBAAgB,EAAE,EAAE,IAAI,CAAC,EAAE,MAAM,KAAK,IAAI,CAAC;IAE5E,6FAA6F;IAC7F,kBAAkB,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;IAE5C;;;;;;;;;;;OAWG;IACH,oBAAoB,CAAC,EAAE,MAAM,CAAC;CACjC;AA8BD;;;GAGG;AACH,wBAAgB,iCAAiC,CAAC,MAAM,EAAE,6BAA6B,GAAG,IAAI,CAwK7F"}

package/dist/ai/llm-tool-conformance.js

@@ -0,0 +1,212 @@
+/**
+ * @fileoverview Shared native-tool-calling conformance suite for `BaseLLM` drivers.
+ *
+ * WHY THIS IS SEPARATE FROM `llm-conformance.ts`: that suite drives the streaming / cancellation /
+ * failure template method, and adopting it requires scripting a vendor mock for all of those paths.
+ * The tool-calling contract is independent of them and much cheaper to adopt, so it ships as its own
+ * entry point — a driver can conform on tools without first conforming on streaming. Run both when
+ * a provider has configs for both; they register separate `describe` blocks and do not interact.
+ *
+ * WHAT THIS ASSERTS — the parts of the tool contract that are the SAME for every provider, because
+ * they live in `BaseLLM` and in the normalized `ChatResult` shape rather than in any SDK mapping:
+ *
+ *  - `SupportsTools` reports the driver's real capability (a CODE-level fact: "has the mapping been
+ *    written?", distinct from the metadata capability flag `LLM.SupportsNativeToolCalling`).
+ *  - A tool-call turn normalizes to `ChatCompletionMessage.toolCalls` with **parsed** arguments —
+ *    never a JSON string, whatever the wire format was.
+ *  - Such a turn reports `finish_reason === 'tool_calls'`.
+ *  - Text and tool calls coexist: a turn carrying both surfaces both, and a turn carrying only
+ *    calls is a SUCCESS with empty content — not a "no output from model" failure.
+ *  - An ordinary turn is untouched: no `toolCalls`, and the driver's own finish reason preserved.
+ *  - Streaming plus tools takes the non-streaming path and flags `streamingSuppressedForTools`
+ *    (native tool calling is non-streaming only).
+ *  - An orphaned tool result — one whose call was never declared by an earlier assistant turn —
+ *    is rejected at the MJ boundary rather than at the provider.
+ *
+ * Provider-SPECIFIC request mapping (Anthropic `input_schema` vs OpenAI `parameters` vs Gemini
+ * `parametersJsonSchema`; `tool_choice` vocabularies; how results are shaped into the history) stays
+ * in each driver's own test file — there is no shared truth to assert about it.
+ *
+ * A driver that does NOT implement tools passes `SupportsTools: false` and gets the negative
+ * contract asserted instead: it declares false, and declaring tools to it is harmless.
+ *
+ * HOW A PROVIDER PACKAGE USES IT:
+ * ```ts
+ * import { RunLLMToolCallingConformanceSuite } from '@memberjunction/unit-testing';
+ *
+ * RunLLMToolCallingConformanceSuite({
+ *     ProviderName: 'MyProvider',
+ *     CreateLLM: () => buildLLMWithMockedVendorClient(),
+ *     SupportsTools: true,
+ *     ScriptToolCallResponse: (calls, text) => { ... },
+ *     ScriptTextResponse: (text) => { ... },
+ *     OrdinaryFinishReason: 'stop',
+ * });
+ * ```
+ *
+ * Same rules as the streaming suite: do NOT `vi.mock('@memberjunction/ai')` in a file that runs
+ * this — the point is to drive the REAL `BaseLLM` and the REAL normalized result types. Mock only
+ * the vendor SDK seam.
+ */
+import { describe, it, expect, beforeEach } from 'vitest';
+import { ChatParams, ChatMessageRole } from '@memberjunction/ai';
+/** The tool every scripted case declares — shape is irrelevant to the assertions, so keep it small. */
+const WEATHER_TOOL = {
+    name: 'get_weather',
+    description: 'Call this when the user asks about the weather.',
+    inputSchema: {
+        type: 'object',
+        properties: { city: { type: 'string', description: 'City name' } },
+        required: ['city']
+    }
+};
+const TOOL_CALL = { Id: 'call_conformance_1', Name: 'get_weather', Arguments: { city: 'NYC' } };
+function buildToolParams(overrides = {}) {
+    const params = new ChatParams();
+    params.model = 'conformance-test-model';
+    params.messages = [{ role: ChatMessageRole.user, content: 'What is the weather in NYC?' }];
+    params.tools = [WEATHER_TOOL];
+    Object.assign(params, overrides);
+    return params;
+}
+/** The single choice of a result, asserting the shape on the way. */
+function firstChoice(result) {
+    expect(result.data.choices.length).toBeGreaterThanOrEqual(1);
+    return result.data.choices[0];
+}
+/**
+ * Run the shared native-tool-calling conformance suite against one provider driver. Registers a
+ * full `describe` block — call it at the top level of a provider test file.
+ */
+export function RunLLMToolCallingConformanceSuite(config) {
+    if (config.SupportsTools && (!config.ScriptToolCallResponse || !config.ScriptTextResponse)) {
+        throw new Error(`${config.ProviderName}: ScriptToolCallResponse and ScriptTextResponse are required when SupportsTools is true.`);
+    }
+    describe(`${config.ProviderName} — BaseLLM native tool-calling conformance`, () => {
+        let llm;
+        beforeEach(() => {
+            llm = config.CreateLLM();
+        });
+        describe('capability declaration', () => {
+            it(`declares SupportsTools = ${config.SupportsTools}`, () => {
+                expect(llm.SupportsTools).toBe(config.SupportsTools);
+            });
+        });
+        if (!config.SupportsTools) {
+            describe('driver without the tool mapping', () => {
+                it('ignores declared tools rather than failing the request', async () => {
+                    config.ScriptTextResponse?.('plain answer');
+                    const result = await llm.ChatCompletion(buildToolParams());
+                    expect(result.success).toBe(true);
+                    expect(firstChoice(result).message.toolCalls).toBeUndefined();
+                });
+            });
+            return;
+        }
+        describe('response normalization', () => {
+            it('normalizes a tool call, with arguments PARSED rather than left as a JSON string', async () => {
+                config.ScriptToolCallResponse([TOOL_CALL]);
+                const result = await llm.ChatCompletion(buildToolParams());
+                expect(result.success).toBe(true);
+                const calls = firstChoice(result).message.toolCalls;
+                expect(calls).toHaveLength(1);
+                expect(calls[0].name).toBe('get_weather');
+                expect(calls[0].arguments).toEqual({ city: 'NYC' });
+                expect(typeof calls[0].id).toBe('string');
+                expect(calls[0].id.length).toBeGreaterThan(0);
+            });
+            it("reports finish_reason 'tool_calls' on a tool-call turn", async () => {
+                config.ScriptToolCallResponse([TOOL_CALL]);
+                const result = await llm.ChatCompletion(buildToolParams());
+                expect(firstChoice(result).finish_reason).toBe('tool_calls');
+            });
+            it('treats a text-free tool call as SUCCESS — a call is output, not a missing answer', async () => {
+                config.ScriptToolCallResponse([TOOL_CALL]);
+                const result = await llm.ChatCompletion(buildToolParams());
+                expect(result.success).toBe(true);
+                expect(result.errorMessage ?? '').not.toContain('No output received');
+            });
+            it('surfaces text AND tool calls together when the model emits both', async () => {
+                config.ScriptToolCallResponse([TOOL_CALL], 'Let me check that for you.');
+                const result = await llm.ChatCompletion(buildToolParams());
+                const choice = firstChoice(result);
+                expect(choice.message.content).toContain('Let me check that for you.');
+                expect(choice.message.toolCalls).toHaveLength(1);
+            });
+            it('normalizes several calls made in one turn, preserving order', async () => {
+                config.ScriptToolCallResponse([
+                    TOOL_CALL,
+                    { Id: 'call_conformance_2', Name: 'get_weather', Arguments: { city: 'Boston' } }
+                ]);
+                const result = await llm.ChatCompletion(buildToolParams());
+                const calls = firstChoice(result).message.toolCalls;
+                expect(calls).toHaveLength(2);
+                expect(calls.map(c => c.arguments.city)).toEqual(['NYC', 'Boston']);
+            });
+            it('leaves an ordinary turn untouched — no toolCalls, own finish reason preserved', async () => {
+                config.ScriptTextResponse('The weather is fine.');
+                const params = buildToolParams();
+                params.tools = undefined;
+                const result = await llm.ChatCompletion(params);
+                const choice = firstChoice(result);
+                expect(choice.message.toolCalls).toBeUndefined();
+                if (config.OrdinaryFinishReason !== undefined) {
+                    expect(choice.finish_reason).toBe(config.OrdinaryFinishReason);
+                }
+                expect(choice.finish_reason).not.toBe('tool_calls');
+            });
+        });
+        describe('streaming is suppressed while tools are declared (non-streaming only)', () => {
+            it('takes the non-streaming path and flags the downgrade', async () => {
+                config.ScriptToolCallResponse([TOOL_CALL]);
+                const onContent = [];
+                const result = await llm.ChatCompletion(buildToolParams({
+                    streaming: true,
+                    streamingCallbacks: {
+                        OnContent: (chunk) => { onContent.push(chunk); },
+                        OnComplete: () => undefined,
+                        OnError: () => undefined
+                    }
+                }));
+                expect(result.modelSpecificResponseDetails?.streamingSuppressedForTools).toBe(true);
+                expect(onContent).toHaveLength(0);
+                expect(firstChoice(result).message.toolCalls).toHaveLength(1);
+            });
+            it('does not flag the downgrade when no tools are declared', async () => {
+                config.ScriptTextResponse('plain answer');
+                const params = buildToolParams();
+                params.tools = undefined;
+                const result = await llm.ChatCompletion(params);
+                expect(result.modelSpecificResponseDetails?.streamingSuppressedForTools).toBeUndefined();
+            });
+        });
+        describe('conversation validity guard', () => {
+            /** A history whose tool result answers a call no assistant turn ever declared. */
+            const orphanedHistory = () => ([
+                { role: ChatMessageRole.user, content: 'What is the weather in NYC?' },
+                // The assistant turn is missing its `toolCalls` — the mistake this guard exists for.
+                { role: ChatMessageRole.assistant, content: '' },
+                {
+                    role: ChatMessageRole.tool,
+                    content: [{ type: 'tool_result', content: '72F', toolCallId: 'call_conformance_1' }]
+                }
+            ]);
+            it('rejects a tool result whose call was never declared, naming the id', async () => {
+                config.ScriptToolCallResponse([TOOL_CALL]);
+                const params = buildToolParams();
+                params.messages = orphanedHistory();
+                await expect(llm.ChatCompletion(params)).rejects.toThrow(/call_conformance_1/);
+            });
+            it('accepts the same history once the assistant turn carries its toolCalls', async () => {
+                config.ScriptTextResponse('It is 72F in NYC.');
+                const params = buildToolParams();
+                const history = orphanedHistory();
+                history[1].toolCalls = [{ id: 'call_conformance_1', name: 'get_weather', arguments: { city: 'NYC' } }];
+                params.messages = history;
+                const result = await llm.ChatCompletion(params);
+                expect(result.success).toBe(true);
+            });
+        });
+    });
+}
+//# sourceMappingURL=llm-tool-conformance.js.map

package/dist/ai/llm-tool-conformance.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"llm-tool-conformance.js","sourceRoot":"","sources":["../../src/ai/llm-tool-conformance.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiDG;AAEH,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAE1D,OAAO,EAAE,UAAU,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAqDjE,uGAAuG;AACvG,MAAM,YAAY,GAAa;IAC3B,IAAI,EAAE,aAAa;IACnB,WAAW,EAAE,iDAAiD;IAC9D,WAAW,EAAE;QACT,IAAI,EAAE,QAAQ;QACd,UAAU,EAAE,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,WAAW,EAAE,EAAE;QAClE,QAAQ,EAAE,CAAC,MAAM,CAAC;KACrB;CACJ,CAAC;AAEF,MAAM,SAAS,GAAqB,EAAE,EAAE,EAAE,oBAAoB,EAAE,IAAI,EAAE,aAAa,EAAE,SAAS,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;AAElH,SAAS,eAAe,CAAC,YAAiC,EAAE;IACxD,MAAM,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;IAChC,MAAM,CAAC,KAAK,GAAG,wBAAwB,CAAC;IACxC,MAAM,CAAC,QAAQ,GAAG,CAAC,EAAE,IAAI,EAAE,eAAe,CAAC,IAAI,EAAE,OAAO,EAAE,6BAA6B,EAAE,CAAC,CAAC;IAC3F,MAAM,CAAC,KAAK,GAAG,CAAC,YAAY,CAAC,CAAC;IAC9B,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;IACjC,OAAO,MAAM,CAAC;AAClB,CAAC;AAED,qEAAqE;AACrE,SAAS,WAAW,CAAC,MAAkB;IACnC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC;IAC7D,OAAO,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;AAClC,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,iCAAiC,CAAC,MAAqC;IACnF,IAAI,MAAM,CAAC,aAAa,IAAI,CAAC,CAAC,MAAM,CAAC,sBAAsB,IAAI,CAAC,MAAM,CAAC,kBAAkB,CAAC,EAAE,CAAC;QACzF,MAAM,IAAI,KAAK,CACX,GAAG,MAAM,CAAC,YAAY,0FAA0F,CACnH,CAAC;IACN,CAAC;IAED,QAAQ,CAAC,GAAG,MAAM,CAAC,YAAY,4CAA4C,EAAE,GAAG,EAAE;QAC9E,IAAI,GAAY,CAAC;QAEjB,UAAU,CAAC,GAAG,EAAE;YACZ,GAAG,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;QAC7B,CAAC,CAAC,CAAC;QAEH,QAAQ,CAAC,wBAAwB,EAAE,GAAG,EAAE;YACpC,EAAE,CAAC,4BAA4B,MAAM,CAAC,aAAa,EAAE,EAAE,GAAG,EAAE;gBACxD,MAAM,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;YACzD,CAAC,CAAC,CAAC;QACP,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,MAAM,CAAC,aAAa,EAAE,CAAC;YACxB,QAAQ,CAAC,iCAAiC,EAAE,GAAG,EAAE;gBAC7C,EAAE,CAAC,wDAAwD,EAAE,KAAK,IAAI,EAAE;oBACpE,MAAM,CAAC,kBAAkB,EAAE,CAAC,cAAc,CAAC,CAAC;oBAE5C,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,cAAc,CAAC,eAAe,EAAE,CAAC,CAAC;oBAE3D,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;oBAClC,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,aAAa,EAAE,CAAC;gBAClE,CAAC,CAAC,CAAC;YACP,CAAC,CAAC,CAAC;YACH,OAAO;QACX,CAAC;QAED,QAAQ,CAAC,wBAAwB,EAAE,GAAG,EAAE;YACpC,EAAE,CAAC,iFAAiF,EAAE,KAAK,IAAI,EAAE;gBAC7F,MAAM,CAAC,sBAAuB,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC;gBAE5C,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,cAAc,CAAC,eAAe,EAAE,CAAC,CAAC;gBAE3D,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBAClC,MAAM,KAAK,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,SAA2B,CAAC;gBACtE,MAAM,CAAC,KAAK,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;gBAC9B,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;gBAC1C,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;gBACpD,MAAM,CAAC,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBAC1C,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;YAClD,CAAC,CAAC,CAAC;YAEH,EAAE,CAAC,wDAAwD,EAAE,KAAK,IAAI,EAAE;gBACpE,MAAM,CAAC,sBAAuB,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC;gBAE5C,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,cAAc,CAAC,eAAe,EAAE,CAAC,CAAC;gBAE3D,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;YACjE,CAAC,CAAC,CAAC;YAEH,EAAE,CAAC,kFAAkF,EAAE,KAAK,IAAI,EAAE;gBAC9F,MAAM,CAAC,sBAAuB,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC;gBAE5C,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,cAAc,CAAC,eAAe,EAAE,CAAC,CAAC;gBAE3D,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBAClC,MAAM,CAAC,MAAM,CAAC,YAAY,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,oBAAoB,CAAC,CAAC;YAC1E,CAAC,CAAC,CAAC;YAEH,EAAE,CAAC,iEAAiE,EAAE,KAAK,IAAI,EAAE;gBAC7E,MAAM,CAAC,sBAAuB,CAAC,CAAC,SAAS,CAAC,EAAE,4BAA4B,CAAC,CAAC;gBAE1E,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,cAAc,CAAC,eAAe,EAAE,CAAC,CAAC;gBAE3D,MAAM,MAAM,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC;gBACnC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,4BAA4B,CAAC,CAAC;gBACvE,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;YACrD,CAAC,CAAC,CAAC;YAEH,EAAE,CAAC,6DAA6D,EAAE,KAAK,IAAI,EAAE;gBACzE,MAAM,CAAC,sBAAuB,CAAC;oBAC3B,SAAS;oBACT,EAAE,EAAE,EAAE,oBAAoB,EAAE,IAAI,EAAE,aAAa,EAAE,SAAS,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE;iBACnF,CAAC,CAAC;gBAEH,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,cAAc,CAAC,eAAe,EAAE,CAAC,CAAC;gBAE3D,MAAM,KAAK,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,SAA2B,CAAC;gBACtE,MAAM,CAAC,KAAK,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;gBAC9B,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC,CAAC;YACxE,CAAC,CAAC,CAAC;YAEH,EAAE,CAAC,+EAA+E,EAAE,KAAK,IAAI,EAAE;gBAC3F,MAAM,CAAC,kBAAmB,CAAC,sBAAsB,CAAC,CAAC;gBAEnD,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;gBACjC,MAAM,CAAC,KAAK,GAAG,SAAS,CAAC;gBACzB,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;gBAEhD,MAAM,MAAM,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC;gBACnC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,aAAa,EAAE,CAAC;gBACjD,IAAI,MAAM,CAAC,oBAAoB,KAAK,SAAS,EAAE,CAAC;oBAC5C,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,oBAAoB,CAAC,CAAC;gBACnE,CAAC;gBACD,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;YACxD,CAAC,CAAC,CAAC;QACP,CAAC,CAAC,CAAC;QAEH,QAAQ,CAAC,uEAAuE,EAAE,GAAG,EAAE;YACnF,EAAE,CAAC,sDAAsD,EAAE,KAAK,IAAI,EAAE;gBAClE,MAAM,CAAC,sBAAuB,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC;gBAC5C,MAAM,SAAS,GAAG,EAAc,CAAC;gBAEjC,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,cAAc,CAAC,eAAe,CAAC;oBACpD,SAAS,EAAE,IAAI;oBACf,kBAAkB,EAAE;wBAChB,SAAS,EAAE,CAAC,KAAa,EAAE,EAAE,GAAG,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;wBACxD,UAAU,EAAE,GAAG,EAAE,CAAC,SAAS;wBAC3B,OAAO,EAAE,GAAG,EAAE,CAAC,SAAS;qBAC3B;iBACJ,CAAC,CAAC,CAAC;gBAEJ,MAAM,CAAC,MAAM,CAAC,4BAA4B,EAAE,2BAA2B,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACpF,MAAM,CAAC,SAAS,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;gBAClC,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;YAClE,CAAC,CAAC,CAAC;YAEH,EAAE,CAAC,wDAAwD,EAAE,KAAK,IAAI,EAAE;gBACpE,MAAM,CAAC,kBAAmB,CAAC,cAAc,CAAC,CAAC;gBAE3C,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;gBACjC,MAAM,CAAC,KAAK,GAAG,SAAS,CAAC;gBACzB,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;gBAEhD,MAAM,CAAC,MAAM,CAAC,4BAA4B,EAAE,2BAA2B,CAAC,CAAC,aAAa,EAAE,CAAC;YAC7F,CAAC,CAAC,CAAC;QACP,CAAC,CAAC,CAAC;QAEH,QAAQ,CAAC,6BAA6B,EAAE,GAAG,EAAE;YACzC,kFAAkF;YAClF,MAAM,eAAe,GAAG,GAAkB,EAAE,CAAC,CAAC;gBAC1C,EAAE,IAAI,EAAE,eAAe,CAAC,IAAI,EAAE,OAAO,EAAE,6BAA6B,EAAE;gBACtE,qFAAqF;gBACrF,EAAE,IAAI,EAAE,eAAe,CAAC,SAAS,EAAE,OAAO,EAAE,EAAE,EAAE;gBAChD;oBACI,IAAI,EAAE,eAAe,CAAC,IAAI;oBAC1B,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,aAAa,EAAE,OAAO,EAAE,KAAK,EAAE,UAAU,EAAE,oBAAoB,EAAE,CAAC;iBACvF;aACJ,CAAC,CAAC;YAEH,EAAE,CAAC,oEAAoE,EAAE,KAAK,IAAI,EAAE;gBAChF,MAAM,CAAC,sBAAuB,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC;gBAC5C,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;gBACjC,MAAM,CAAC,QAAQ,GAAG,eAAe,EAAE,CAAC;gBAEpC,MAAM,MAAM,CAAC,GAAG,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,oBAAoB,CAAC,CAAC;YACnF,CAAC,CAAC,CAAC;YAEH,EAAE,CAAC,wEAAwE,EAAE,KAAK,IAAI,EAAE;gBACpF,MAAM,CAAC,kBAAmB,CAAC,mBAAmB,CAAC,CAAC;gBAChD,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;gBACjC,MAAM,OAAO,GAAG,eAAe,EAAE,CAAC;gBAClC,OAAO,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,EAAE,EAAE,EAAE,oBAAoB,EAAE,IAAI,EAAE,aAAa,EAAE,SAAS,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC;gBACvG,MAAM,CAAC,QAAQ,GAAG,OAAO,CAAC;gBAE1B,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;gBAEhD,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACtC,CAAC,CAAC,CAAC;QACP,CAAC,CAAC,CAAC;IACP,CAAC,CAAC,CAAC;AACP,CAAC"}

package/dist/index.d.ts

@@ -7,5 +7,6 @@ export { TestLLM, registerTestLLM, type TestLLMOutcome } from './ai/test-llm.js'
 export { makeModelUsage, makeSuccessChatResult, makeFailedChatResult, makeDriverFailureChatResult, makeErrorInfo, makeChatParams, type ModelUsageOverrides, type SuccessChatResultOptions, type FailedChatResultOptions, } from './ai/chat-result-factories.js';
 export { VENDOR_TYPE, VENDOR, MODEL_TYPE, CONFIG, MODEL, makeModelVendor, makeModel, makePromptModel, buildRealisticCatalog, DEFAULT_CONFIGURED_DRIVERS, type FxVendorType, type FxVendor, type FxModelType, type FxConfiguration, type FxModelVendor, type FxModel, type FxPromptModel, type AICatalog, } from './ai/catalog-fixtures.js';
 export { RunLLMConformanceSuite, type ExpectedUsageCounts, type FailureSite, type LLMConformanceDeviationKind, type LLMConformanceDeviation, type LLMConformanceSuiteConfig, } from './ai/llm-conformance.js';
+export { RunLLMToolCallingConformanceSuite, type ScriptedToolCall, type LLMToolConformanceSuiteConfig, } from './ai/llm-tool-conformance.js';
 export { CreateOpenAICompatibleSeamMock, type OpenAICompatibleSeam, type OpenAICompatibleChatClient, type OpenAICompatibleUsagePayload, type OpenAICompatibleResponsePayload, type OpenAICompatibleChunkPayload, type OpenAICompatibleRequestBody, type OpenAICompatibleRequestOptions, } from './ai/openai-compatible-seam.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AAC3F,OAAO,EAAE,gBAAgB,EAAE,KAAK,iBAAiB,EAAE,MAAM,eAAe,CAAC;AACzE,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AAC/E,OAAO,EAAE,qBAAqB,EAAE,MAAM,mBAAmB,CAAC;AAC1D,YAAY,EAAE,MAAM,YAAY,CAAC;AAGjC,OAAO,EAAE,OAAO,EAAE,eAAe,EAAE,KAAK,cAAc,EAAE,MAAM,eAAe,CAAC;AAC9E,OAAO,EACL,cAAc,EACd,qBAAqB,EACrB,oBAAoB,EACpB,2BAA2B,EAC3B,aAAa,EACb,cAAc,EACd,KAAK,mBAAmB,EACxB,KAAK,wBAAwB,EAC7B,KAAK,uBAAuB,GAC7B,MAAM,4BAA4B,CAAC;AACpC,OAAO,EACL,WAAW,EACX,MAAM,EACN,UAAU,EACV,MAAM,EACN,KAAK,EACL,eAAe,EACf,SAAS,EACT,eAAe,EACf,qBAAqB,EACrB,0BAA0B,EAC1B,KAAK,YAAY,EACjB,KAAK,QAAQ,EACb,KAAK,WAAW,EAChB,KAAK,eAAe,EACpB,KAAK,aAAa,EAClB,KAAK,OAAO,EACZ,KAAK,aAAa,EAClB,KAAK,SAAS,GACf,MAAM,uBAAuB,CAAC;AAG/B,OAAO,EACL,sBAAsB,EACtB,KAAK,mBAAmB,EACxB,KAAK,WAAW,EAChB,KAAK,2BAA2B,EAChC,KAAK,uBAAuB,EAC5B,KAAK,yBAAyB,GAC/B,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EACL,8BAA8B,EAC9B,KAAK,oBAAoB,EACzB,KAAK,0BAA0B,EAC/B,KAAK,4BAA4B,EACjC,KAAK,+BAA+B,EACpC,KAAK,4BAA4B,EACjC,KAAK,2BAA2B,EAChC,KAAK,8BAA8B,GACpC,MAAM,6BAA6B,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AAC3F,OAAO,EAAE,gBAAgB,EAAE,KAAK,iBAAiB,EAAE,MAAM,eAAe,CAAC;AACzE,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AAC/E,OAAO,EAAE,qBAAqB,EAAE,MAAM,mBAAmB,CAAC;AAC1D,YAAY,EAAE,MAAM,YAAY,CAAC;AAGjC,OAAO,EAAE,OAAO,EAAE,eAAe,EAAE,KAAK,cAAc,EAAE,MAAM,eAAe,CAAC;AAC9E,OAAO,EACL,cAAc,EACd,qBAAqB,EACrB,oBAAoB,EACpB,2BAA2B,EAC3B,aAAa,EACb,cAAc,EACd,KAAK,mBAAmB,EACxB,KAAK,wBAAwB,EAC7B,KAAK,uBAAuB,GAC7B,MAAM,4BAA4B,CAAC;AACpC,OAAO,EACL,WAAW,EACX,MAAM,EACN,UAAU,EACV,MAAM,EACN,KAAK,EACL,eAAe,EACf,SAAS,EACT,eAAe,EACf,qBAAqB,EACrB,0BAA0B,EAC1B,KAAK,YAAY,EACjB,KAAK,QAAQ,EACb,KAAK,WAAW,EAChB,KAAK,eAAe,EACpB,KAAK,aAAa,EAClB,KAAK,OAAO,EACZ,KAAK,aAAa,EAClB,KAAK,SAAS,GACf,MAAM,uBAAuB,CAAC;AAG/B,OAAO,EACL,sBAAsB,EACtB,KAAK,mBAAmB,EACxB,KAAK,WAAW,EAChB,KAAK,2BAA2B,EAChC,KAAK,uBAAuB,EAC5B,KAAK,yBAAyB,GAC/B,MAAM,sBAAsB,CAAC;AAG9B,OAAO,EACL,iCAAiC,EACjC,KAAK,gBAAgB,EACrB,KAAK,6BAA6B,GACnC,MAAM,2BAA2B,CAAC;AACnC,OAAO,EACL,8BAA8B,EAC9B,KAAK,oBAAoB,EACzB,KAAK,0BAA0B,EAC/B,KAAK,4BAA4B,EACjC,KAAK,+BAA+B,EACpC,KAAK,4BAA4B,EACjC,KAAK,2BAA2B,EAChC,KAAK,8BAA8B,GACpC,MAAM,6BAA6B,CAAC"}

package/dist/index.js

@@ -8,5 +8,7 @@ export { makeModelUsage, makeSuccessChatResult, makeFailedChatResult, makeDriver
 export { VENDOR_TYPE, VENDOR, MODEL_TYPE, CONFIG, MODEL, makeModelVendor, makeModel, makePromptModel, buildRealisticCatalog, DEFAULT_CONFIGURED_DRIVERS, } from './ai/catalog-fixtures.js';
 // ---- Shared BaseLLM streaming/ChatResult conformance suite (applied by provider packages) ----
 export { RunLLMConformanceSuite, } from './ai/llm-conformance.js';
+// ---- Shared native-tool-calling conformance suite (applied by provider packages) ----
+export { RunLLMToolCallingConformanceSuite, } from './ai/llm-tool-conformance.js';
 export { CreateOpenAICompatibleSeamMock, } from './ai/openai-compatible-seam.js';
 //# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AAC3F,OAAO,EAAE,gBAAgB,EAA0B,MAAM,eAAe,CAAC;AACzE,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AAC/E,OAAO,EAAE,qBAAqB,EAAE,MAAM,mBAAmB,CAAC;AAG1D,4FAA4F;AAC5F,OAAO,EAAE,OAAO,EAAE,eAAe,EAAuB,MAAM,eAAe,CAAC;AAC9E,OAAO,EACL,cAAc,EACd,qBAAqB,EACrB,oBAAoB,EACpB,2BAA2B,EAC3B,aAAa,EACb,cAAc,GAIf,MAAM,4BAA4B,CAAC;AACpC,OAAO,EACL,WAAW,EACX,MAAM,EACN,UAAU,EACV,MAAM,EACN,KAAK,EACL,eAAe,EACf,SAAS,EACT,eAAe,EACf,qBAAqB,EACrB,0BAA0B,GAS3B,MAAM,uBAAuB,CAAC;AAE/B,iGAAiG;AACjG,OAAO,EACL,sBAAsB,GAMvB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EACL,8BAA8B,GAQ/B,MAAM,6BAA6B,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AAC3F,OAAO,EAAE,gBAAgB,EAA0B,MAAM,eAAe,CAAC;AACzE,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AAC/E,OAAO,EAAE,qBAAqB,EAAE,MAAM,mBAAmB,CAAC;AAG1D,4FAA4F;AAC5F,OAAO,EAAE,OAAO,EAAE,eAAe,EAAuB,MAAM,eAAe,CAAC;AAC9E,OAAO,EACL,cAAc,EACd,qBAAqB,EACrB,oBAAoB,EACpB,2BAA2B,EAC3B,aAAa,EACb,cAAc,GAIf,MAAM,4BAA4B,CAAC;AACpC,OAAO,EACL,WAAW,EACX,MAAM,EACN,UAAU,EACV,MAAM,EACN,KAAK,EACL,eAAe,EACf,SAAS,EACT,eAAe,EACf,qBAAqB,EACrB,0BAA0B,GAS3B,MAAM,uBAAuB,CAAC;AAE/B,iGAAiG;AACjG,OAAO,EACL,sBAAsB,GAMvB,MAAM,sBAAsB,CAAC;AAE9B,wFAAwF;AACxF,OAAO,EACL,iCAAiC,GAGlC,MAAM,2BAA2B,CAAC;AACnC,OAAO,EACL,8BAA8B,GAQ/B,MAAM,6BAA6B,CAAC"}

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@memberjunction/unit-testing",
   "type": "module",
-  "version": "6.1.0-edge.6",
+  "version": "6.1.0-edge.7",
   "description": "MemberJunction: Unit testing utilities and mock infrastructure for Vitest",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -16,8 +16,8 @@
     "vitest": "^4.0.18"
   },
   "dependencies": {
-    "@memberjunction/ai": "6.1.0-edge.6",
-    "@memberjunction/global": "6.1.0-edge.6"
+    "@memberjunction/ai": "6.1.0-edge.7",
+    "@memberjunction/global": "6.1.0-edge.7"
   },
   "repository": {
     "type": "git",