@memberjunction/unit-testing 6.1.0-edge.1 → 6.1.0-edge.3

package/dist/ai/chat-result-factories.js

@@ -0,0 +1,115 @@
+/**
+ * Factories producing REAL-typed `@memberjunction/ai` chat values for tests.
+ *
+ * Every value built here uses the real classes and interfaces exported by
+ * `@memberjunction/ai` (`ChatResult`, `ChatParams`, `ModelUsage`, `AIErrorInfo`) —
+ * never a structural look-alike. Hand-faked shapes silently drift as the real
+ * contracts evolve (the exact bug class behind the historical failover bug);
+ * building from the real types makes the compiler enforce the contract in test
+ * code too.
+ */
+import { ChatParams, ChatResult, ErrorAnalyzer, ModelUsage, } from '@memberjunction/ai';
+/**
+ * Builds a real {@link ModelUsage} instance. Defaults: 10 prompt tokens,
+ * 5 completion tokens, cost 0.001 — so `totalTokens` is 15 out of the box
+ * (the shape the AI-stack tests have standardized on).
+ */
+export function makeModelUsage(overrides = {}) {
+    const cost = 'cost' in overrides ? overrides.cost : 0.001;
+    const usage = new ModelUsage(overrides.promptTokens ?? 10, overrides.completionTokens ?? 5, cost, overrides.costCurrency);
+    if (overrides.cacheReadTokens !== undefined)
+        usage.cacheReadTokens = overrides.cacheReadTokens;
+    if (overrides.cacheWriteTokens !== undefined)
+        usage.cacheWriteTokens = overrides.cacheWriteTokens;
+    if (overrides.queueTime !== undefined)
+        usage.queueTime = overrides.queueTime;
+    if (overrides.promptTime !== undefined)
+        usage.promptTime = overrides.promptTime;
+    if (overrides.completionTime !== undefined)
+        usage.completionTime = overrides.completionTime;
+    return usage;
+}
+/**
+ * Builds a successful {@link ChatResult} the way real drivers do: one assistant
+ * choice carrying `content`, a real {@link ModelUsage}, `statusText: 'success'`.
+ */
+export function makeSuccessChatResult(content = 'test response', options = {}) {
+    const startTime = options.startTime ?? new Date();
+    const endTime = options.endTime ?? startTime;
+    const result = new ChatResult(true, startTime, endTime);
+    result.statusText = 'success';
+    const choice = {
+        message: { role: 'assistant', content, thinking: options.thinking ?? null },
+        finish_reason: options.finishReason ?? 'stop',
+        index: 0,
+    };
+    result.data = {
+        choices: [choice],
+        model: options.model,
+        usage: options.usage ?? makeModelUsage(),
+    };
+    return result;
+}
+/**
+ * Builds a failed {@link ChatResult} (`success: false`). By default the shape
+ * mirrors BaseLLM's own error construction: empty choices, zero usage,
+ * `statusText: 'error'`. `errorInfo` is only attached when supplied, so tests
+ * can model both diagnosed and undiagnosed failures.
+ */
+export function makeFailedChatResult(options = {}) {
+    const startTime = options.startTime ?? new Date();
+    const endTime = options.endTime ?? startTime;
+    const result = new ChatResult(false, startTime, endTime);
+    result.statusText = options.statusText ?? 'error';
+    result.errorMessage = options.errorMessage ?? 'Test failure';
+    if (!options.omitData) {
+        result.data = { choices: [], usage: new ModelUsage(0, 0) };
+    }
+    if (options.exception !== undefined) {
+        result.exception = options.exception;
+    }
+    if (options.errorInfo !== undefined) {
+        result.errorInfo = options.errorInfo;
+    }
+    return result;
+}
+/**
+ * Builds a failed {@link ChatResult} exactly the way real provider drivers do:
+ * the `errorInfo` comes from the REAL `ErrorAnalyzer`, so `errorType`,
+ * `severity`, and `canFailover` match what AnthropicLLM / OpenAILLM / GeminiLLM
+ * would report for this error. This is the value to use when testing failover
+ * logic — drivers catch provider errors internally and RETURN this shape rather
+ * than throwing (the historical failover bug class).
+ */
+export function makeDriverFailureChatResult(error, providerName = 'TestLLM') {
+    return makeFailedChatResult({
+        errorMessage: error.message,
+        exception: error,
+        errorInfo: ErrorAnalyzer.analyzeError(error, providerName),
+    });
+}
+/**
+ * Builds a real {@link AIErrorInfo} with sensible defaults
+ * (`Unknown` / `Retriable` / failover-eligible), overridable per test.
+ */
+export function makeErrorInfo(overrides = {}) {
+    return {
+        errorType: 'Unknown',
+        severity: 'Retriable',
+        canFailover: true,
+        ...overrides,
+    };
+}
+/**
+ * Builds a real {@link ChatParams} instance (so class-level defaults like
+ * `enableCaching` behave exactly as in production) with a default model and a
+ * single user message, overridable per test.
+ */
+export function makeChatParams(overrides = {}) {
+    const params = new ChatParams();
+    params.model = 'test-model';
+    params.messages = [{ role: 'user', content: 'Hello from makeChatParams' }];
+    Object.assign(params, overrides);
+    return params;
+}
+//# sourceMappingURL=chat-result-factories.js.map

package/dist/ai/chat-result-factories.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"chat-result-factories.js","sourceRoot":"","sources":["../../src/ai/chat-result-factories.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AACH,OAAO,EACL,UAAU,EACV,UAAU,EACV,aAAa,EACb,UAAU,GAGX,MAAM,oBAAoB,CAAC;AAgB5B;;;;GAIG;AACH,MAAM,UAAU,cAAc,CAAC,YAAiC,EAAE;IAChE,MAAM,IAAI,GAAG,MAAM,IAAI,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC;IAC1D,MAAM,KAAK,GAAG,IAAI,UAAU,CAC1B,SAAS,CAAC,YAAY,IAAI,EAAE,EAC5B,SAAS,CAAC,gBAAgB,IAAI,CAAC,EAC/B,IAAI,EACJ,SAAS,CAAC,YAAY,CACvB,CAAC;IACF,IAAI,SAAS,CAAC,eAAe,KAAK,SAAS;QAAE,KAAK,CAAC,eAAe,GAAG,SAAS,CAAC,eAAe,CAAC;IAC/F,IAAI,SAAS,CAAC,gBAAgB,KAAK,SAAS;QAAE,KAAK,CAAC,gBAAgB,GAAG,SAAS,CAAC,gBAAgB,CAAC;IAClG,IAAI,SAAS,CAAC,SAAS,KAAK,SAAS;QAAE,KAAK,CAAC,SAAS,GAAG,SAAS,CAAC,SAAS,CAAC;IAC7E,IAAI,SAAS,CAAC,UAAU,KAAK,SAAS;QAAE,KAAK,CAAC,UAAU,GAAG,SAAS,CAAC,UAAU,CAAC;IAChF,IAAI,SAAS,CAAC,cAAc,KAAK,SAAS;QAAE,KAAK,CAAC,cAAc,GAAG,SAAS,CAAC,cAAc,CAAC;IAC5F,OAAO,KAAK,CAAC;AACf,CAAC;AAcD;;;GAGG;AACH,MAAM,UAAU,qBAAqB,CAAC,OAAO,GAAG,eAAe,EAAE,UAAoC,EAAE;IACrG,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,IAAI,IAAI,EAAE,CAAC;IAClD,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,SAAS,CAAC;IAC7C,MAAM,MAAM,GAAG,IAAI,UAAU,CAAC,IAAI,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;IACxD,MAAM,CAAC,UAAU,GAAG,SAAS,CAAC;IAC9B,MAAM,MAAM,GAAqB;QAC/B,OAAO,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,IAAI,IAAI,EAAE;QAC3E,aAAa,EAAE,OAAO,CAAC,YAAY,IAAI,MAAM;QAC7C,KAAK,EAAE,CAAC;KACT,CAAC;IACF,MAAM,CAAC,IAAI,GAAG;QACZ,OAAO,EAAE,CAAC,MAAM,CAAC;QACjB,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,KAAK,EAAE,OAAO,CAAC,KAAK,IAAI,cAAc,EAAE;KACzC,CAAC;IACF,OAAO,MAAM,CAAC;AAChB,CAAC;AAmBD;;;;;GAKG;AACH,MAAM,UAAU,oBAAoB,CAAC,UAAmC,EAAE;IACxE,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,IAAI,IAAI,EAAE,CAAC;IAClD,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,SAAS,CAAC;IAC7C,MAAM,MAAM,GAAG,IAAI,UAAU,CAAC,KAAK,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;IACzD,MAAM,CAAC,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,OAAO,CAAC;IAClD,MAAM,CAAC,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,cAAc,CAAC;IAC7D,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC;QACtB,MAAM,CAAC,IAAI,GAAG,EAAE,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,IAAI,UAAU,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;IAC7D,CAAC;IACD,IAAI,OAAO,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;QACpC,MAAM,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;IACvC,CAAC;IACD,IAAI,OAAO,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;QACpC,MAAM,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;IACvC,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,2BAA2B,CAAC,KAAY,EAAE,YAAY,GAAG,SAAS;IAChF,OAAO,oBAAoB,CAAC;QAC1B,YAAY,EAAE,KAAK,CAAC,OAAO;QAC3B,SAAS,EAAE,KAAK;QAChB,SAAS,EAAE,aAAa,CAAC,YAAY,CAAC,KAAK,EAAE,YAAY,CAAC;KAC3D,CAAC,CAAC;AACL,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,aAAa,CAAC,YAAkC,EAAE;IAChE,OAAO;QACL,SAAS,EAAE,SAAS;QACpB,QAAQ,EAAE,WAAW;QACrB,WAAW,EAAE,IAAI;QACjB,GAAG,SAAS;KACb,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,cAAc,CAAC,YAAiC,EAAE;IAChE,MAAM,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;IAChC,MAAM,CAAC,KAAK,GAAG,YAAY,CAAC;IAC5B,MAAM,CAAC,QAAQ,GAAG,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,2BAA2B,EAAE,CAAC,CAAC;IAC3E,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;IACjC,OAAO,MAAM,CAAC;AAChB,CAAC"}

package/dist/ai/llm-conformance.d.ts

@@ -0,0 +1,104 @@
+import { BaseLLM } from '@memberjunction/ai';
+/**
+ * Token counts a scripted vendor response reports, expressed provider-agnostically. Adapters map
+ * these into the vendor's native usage shape (e.g. `prompt_tokens` / `promptTokens` /
+ * `prompt_eval_count`); the suite asserts they round-trip into `ModelUsage` unchanged.
+ */
+export interface ExpectedUsageCounts {
+    PromptTokens: number;
+    CompletionTokens: number;
+}
+/** Where a scripted vendor failure strikes. */
+export type FailureSite = 
+/** The non-streaming completion call fails. */
+'nonStreaming'
+/** Creating the stream fails (connection refused, auth error, ...). */
+ | 'streamStart'
+/** The stream fails part-way through, after emitting `ChunkBeforeError`. */
+ | 'midStream';
+/**
+ * The catalog of contract deviations this suite knows how to pin. Each kind flips a specific
+ * assertion from the ideal contract to the driver's actual behavior.
+ */
+export type LLMConformanceDeviationKind = 
+/**
+ * The driver's streaming path reports ModelUsage(0, 0) even though the vendor stream carries
+ * token counts (e.g. the driver's processStreamingChunk never reads chunk usage).
+ */
+'StreamingDropsUsage'
+/**
+ * The driver's failed non-streaming ChatResult ('failedResult' mode) does not populate
+ * `errorInfo`, so retry/failover layers get no structured error classification.
+ */
+ | 'FailedResultLacksErrorInfo';
+/** A documented, pinned deviation from the shared contract. */
+export interface LLMConformanceDeviation {
+    Kind: LLMConformanceDeviationKind;
+    /** WHY the driver deviates — required so the ledger is self-explanatory. */
+    Reason: string;
+}
+/**
+ * Configuration for {@link RunLLMConformanceSuite}. The `Script*` functions program the provider
+ * test file's vendor-SDK mock; the suite tells the adapter WHAT the vendor should do, the adapter
+ * translates that into the vendor's native wire shapes.
+ */
+export interface LLMConformanceSuiteConfig {
+    /** Human-readable provider name, used in describe() labels. */
+    ProviderName: string;
+    /**
+     * Construct a fresh driver instance wired to the scriptable vendor mock. Called in
+     * beforeEach, so every test gets a clean instance (and clean streaming state).
+     */
+    CreateLLM: () => BaseLLM;
+    /** Whether the driver declares streaming support (asserted against `SupportsStreaming`). */
+    SupportsStreaming: boolean;
+    /**
+     * How the driver surfaces NON-cancellation vendor failures on the NON-streaming path.
+     * BaseLLM imposes no rule here (ChatCompletions, plural, is the normalizing layer):
+     *  - 'throws': the vendor error propagates out of ChatCompletion (OpenAI family, Groq,
+     *    Cerebras, Mistral).
+     *  - 'failedResult': the driver catches and resolves ChatResult{success:false} (Ollama, Azure).
+     */
+    NonStreamingFailureMode: 'throws' | 'failedResult';
+    /**
+     * What happens when a STREAMING request is made with an already-aborted token. Required when
+     * `SupportsStreaming` is true.
+     *  - 'resolvesCancelled': the driver detects the pre-abort itself (empty stream + cancelled
+     *    finalize) and the promise resolves with the documented cancelled shape (Groq, Cerebras).
+     *  - 'rejectsErrorResult': createStreamingRequest throws (driver pre-check or SDK abort
+     *    error), and BaseLLM's outer catch — which does NOT special-case cancellation — rejects
+     *    with a generic statusText:'error' ChatResult instead of the cancelled shape (OpenAI
+     *    family, Mistral, Ollama, Azure). Pinned under a `[KNOWN DEVIATION]` test name.
+     */
+    PreAbortedStreamingBehavior?: 'resolvesCancelled' | 'rejectsErrorResult';
+    /** Program the vendor mock: the next non-streaming call succeeds with this content + usage. */
+    ScriptNonStreamingSuccess: (content: string, usage: ExpectedUsageCounts) => void;
+    /**
+     * Program the vendor mock: the next streaming call emits these text chunks (one vendor chunk
+     * per entry, in order) and then completes, reporting `usage` in the vendor's native place for
+     * stream usage (final chunk / usage event) even if the driver is known to drop it.
+     */
+    ScriptStreamingSuccess: (chunks: string[], usage: ExpectedUsageCounts) => void;
+    /**
+     * Program the vendor mock: the next call fails with `error` at the given site. For
+     * 'midStream', the stream must first emit `options.ChunkBeforeError` (as a normal text chunk)
+     * and then throw, so the suite can prove partial content is discarded.
+     */
+    ScriptFailure: (error: Error, at: FailureSite, options?: {
+        ChunkBeforeError?: string;
+    }) => void;
+    /**
+     * Program the vendor mock for a mid-stream cancellation: emit `chunksBeforeAbort` as normal
+     * text chunks, then abort `controller`, then end the stream the way the real SDK ends an
+     * aborted stream (throw its abort error, or end iteration silently — whichever is faithful).
+     */
+    ScriptStreamingCancellation: (chunksBeforeAbort: string[], controller: AbortController) => void;
+    /** Documented deviations from the ideal contract — see {@link LLMConformanceDeviation}. */
+    KnownDeviations?: LLMConformanceDeviation[];
+}
+/**
+ * Run the shared BaseLLM streaming/ChatResult conformance suite against one provider driver.
+ * Registers a full `describe` block — call it at the top level of a provider test file.
+ */
+export declare function RunLLMConformanceSuite(config: LLMConformanceSuiteConfig): void;
+//# sourceMappingURL=llm-conformance.d.ts.map

package/dist/ai/llm-conformance.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"llm-conformance.d.ts","sourceRoot":"","sources":["../../src/ai/llm-conformance.ts"],"names":[],"mappings":"AA+DA,OAAO,EAAE,OAAO,EAA+E,MAAM,oBAAoB,CAAC;AAE1H;;;;GAIG;AACH,MAAM,WAAW,mBAAmB;IAChC,YAAY,EAAE,MAAM,CAAC;IACrB,gBAAgB,EAAE,MAAM,CAAC;CAC5B;AAED,+CAA+C;AAC/C,MAAM,MAAM,WAAW;AACnB,+CAA+C;AAC7C,cAAc;AAChB,uEAAuE;GACrE,aAAa;AACf,4EAA4E;GAC1E,WAAW,CAAC;AAElB;;;GAGG;AACH,MAAM,MAAM,2BAA2B;AACnC;;;GAGG;AACD,qBAAqB;AACvB;;;GAGG;GACD,4BAA4B,CAAC;AAEnC,+DAA+D;AAC/D,MAAM,WAAW,uBAAuB;IACpC,IAAI,EAAE,2BAA2B,CAAC;IAClC,4EAA4E;IAC5E,MAAM,EAAE,MAAM,CAAC;CAClB;AAED;;;;GAIG;AACH,MAAM,WAAW,yBAAyB;IACtC,+DAA+D;IAC/D,YAAY,EAAE,MAAM,CAAC;IAErB;;;OAGG;IACH,SAAS,EAAE,MAAM,OAAO,CAAC;IAEzB,4FAA4F;IAC5F,iBAAiB,EAAE,OAAO,CAAC;IAE3B;;;;;;OAMG;IACH,uBAAuB,EAAE,QAAQ,GAAG,cAAc,CAAC;IAEnD;;;;;;;;;OASG;IACH,2BAA2B,CAAC,EAAE,mBAAmB,GAAG,oBAAoB,CAAC;IAEzE,+FAA+F;IAC/F,yBAAyB,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,mBAAmB,KAAK,IAAI,CAAC;IAEjF;;;;OAIG;IACH,sBAAsB,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,mBAAmB,KAAK,IAAI,CAAC;IAE/E;;;;OAIG;IACH,aAAa,EAAE,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,EAAE,WAAW,EAAE,OAAO,CAAC,EAAE;QAAE,gBAAgB,CAAC,EAAE,MAAM,CAAA;KAAE,KAAK,IAAI,CAAC;IAEhG;;;;OAIG;IACH,2BAA2B,EAAE,CAAC,iBAAiB,EAAE,MAAM,EAAE,EAAE,UAAU,EAAE,eAAe,KAAK,IAAI,CAAC;IAEhG,2FAA2F;IAC3F,eAAe,CAAC,EAAE,uBAAuB,EAAE,CAAC;CAC/C;AA8ED;;;GAGG;AACH,wBAAgB,sBAAsB,CAAC,MAAM,EAAE,yBAAyB,GAAG,IAAI,CAsX9E"}