@memberjunction/unit-testing 6.1.0-edge.1 → 6.1.0-edge.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +180 -4
- package/dist/ai/catalog-fixtures.d.ts +144 -0
- package/dist/ai/catalog-fixtures.d.ts.map +1 -0
- package/dist/ai/catalog-fixtures.js +205 -0
- package/dist/ai/catalog-fixtures.js.map +1 -0
- package/dist/ai/chat-result-factories.d.ts +90 -0
- package/dist/ai/chat-result-factories.d.ts.map +1 -0
- package/dist/ai/chat-result-factories.js +115 -0
- package/dist/ai/chat-result-factories.js.map +1 -0
- package/dist/ai/llm-conformance.d.ts +104 -0
- package/dist/ai/llm-conformance.d.ts.map +1 -0
- package/dist/ai/llm-conformance.js +437 -0
- package/dist/ai/llm-conformance.js.map +1 -0
- package/dist/ai/openai-compatible-seam.d.ts +110 -0
- package/dist/ai/openai-compatible-seam.d.ts.map +1 -0
- package/dist/ai/openai-compatible-seam.js +131 -0
- package/dist/ai/openai-compatible-seam.js.map +1 -0
- package/dist/ai/test-llm.d.ts +151 -0
- package/dist/ai/test-llm.d.ts.map +1 -0
- package/dist/ai/test-llm.js +261 -0
- package/dist/ai/test-llm.js.map +1 -0
- package/dist/index.d.ts +5 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +7 -0
- package/dist/index.js.map +1 -1
- package/package.json +4 -3
|
@@ -0,0 +1,437 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview Shared BaseLLM streaming / ChatResult conformance suite.
|
|
3
|
+
*
|
|
4
|
+
* WHY THIS LIVES IN `@memberjunction/unit-testing`: every LLM provider package implements the
|
|
5
|
+
* same `BaseLLM` template-method contract (chunk accumulation, finalize, cancellation, failure
|
|
6
|
+
* shapes), but each package tests it in isolation. This module packages the SHARED contract as a
|
|
7
|
+
* parameterized Vitest suite that provider packages import in THEIR OWN tests, so the contract is
|
|
8
|
+
* asserted once and applied uniformly. It belongs in the shared test-utilities package — which is
|
|
9
|
+
* where MJ's cross-package test harness (TestLLM, the ChatResult factories) already lives — rather
|
|
10
|
+
* than shipping inside the runtime `@memberjunction/ai` package, so vendor packages never carry
|
|
11
|
+
* test code or an optional `vitest` peer dependency in their published output.
|
|
12
|
+
*
|
|
13
|
+
* HOW A PROVIDER PACKAGE USES IT:
|
|
14
|
+
* ```ts
|
|
15
|
+
* import { RunLLMConformanceSuite } from '@memberjunction/unit-testing';
|
|
16
|
+
* import { MyProviderLLM } from '../models/myProvider';
|
|
17
|
+
*
|
|
18
|
+
* RunLLMConformanceSuite({
|
|
19
|
+
* ProviderName: 'MyProvider',
|
|
20
|
+
* CreateLLM: () => buildLLMWithMockedVendorClient(),
|
|
21
|
+
* ...
|
|
22
|
+
* });
|
|
23
|
+
* ```
|
|
24
|
+
*
|
|
25
|
+
* This module imports `vitest` and is exported from the package index like the rest of
|
|
26
|
+
* `@memberjunction/unit-testing` — a test-only package that consumers depend on as a
|
|
27
|
+
* devDependency, so importing it never pulls test code into a runtime bundle. Only ever import it
|
|
28
|
+
* from test code.
|
|
29
|
+
*
|
|
30
|
+
* RULES FOR CONFORMANCE TEST FILES:
|
|
31
|
+
* - Do NOT `vi.mock('@memberjunction/ai')` in a test file that runs this suite — the whole point
|
|
32
|
+
* is to drive the REAL BaseLLM template method and the REAL ChatResult/ModelUsage classes.
|
|
33
|
+
* - Mock ONLY the vendor SDK boundary (the same seam the Anthropic/OpenAI driver tests use):
|
|
34
|
+
* either module-mock the vendor SDK or inject a scriptable fake client into the driver.
|
|
35
|
+
* - Vendor mocks must mimic the real SDK's abort contract: a scripted call that receives an
|
|
36
|
+
* already-aborted `AbortSignal` must fail with the SDK's abort error (or the DOM-standard
|
|
37
|
+
* `AbortError`) instead of responding. Drivers that pre-check the token themselves never reach
|
|
38
|
+
* the mock, so this requirement is inert for them — but it must hold for drivers that delegate
|
|
39
|
+
* abort detection to their SDK (e.g. the OpenAI family).
|
|
40
|
+
*
|
|
41
|
+
* THE CONTRACT THIS SUITE ASSERTS (verified against BaseLLM, not aspirational):
|
|
42
|
+
* - Non-streaming success resolves ChatResult{success:true} with content + ModelUsage populated.
|
|
43
|
+
* - Streaming accumulates chunks in order, fires OnContent per chunk then `('', true)`, fires
|
|
44
|
+
* OnComplete with the same ChatResult the promise resolves with, and finalizes a result
|
|
45
|
+
* consistent with the non-streaming shape.
|
|
46
|
+
* - Streaming failures REJECT with a failed ChatResult (statusText 'error', errorInfo populated)
|
|
47
|
+
* — never an unwrapped vendor error, and never a truncated success
|
|
48
|
+
* (BaseLLM.handleStreamingChatCompletion builds and rejects that ChatResult itself).
|
|
49
|
+
* - Non-streaming failures are PROVIDER-DEFINED: BaseLLM.ChatCompletion imposes no normalization
|
|
50
|
+
* on that path — some drivers throw the vendor error, others resolve a failed ChatResult. The
|
|
51
|
+
* guaranteed normalizing layer is ChatCompletions (plural), which converts a rejection into a
|
|
52
|
+
* resolved failed ChatResult with errorInfo. `NonStreamingFailureMode` captures which behavior
|
|
53
|
+
* the driver under test has, and the plural guarantee is asserted for every driver.
|
|
54
|
+
* - Cancellation mid-stream RESOLVES (does not reject) with the documented cancelled shape:
|
|
55
|
+
* ChatResult{success:false, statusText:'cancelled' (case-insensitive), errorInfo populated,
|
|
56
|
+
* empty choices} — partial content is never passed off as a success.
|
|
57
|
+
*
|
|
58
|
+
* Real deviations from the ideal contract are not patched over: declare them in
|
|
59
|
+
* `KnownDeviations` (with a reason) and the suite pins the ACTUAL behavior under a test name
|
|
60
|
+
* prefixed `[KNOWN DEVIATION]`, so the deviation is loud in test output and starts failing the
|
|
61
|
+
* moment the driver is fixed (forcing the ledger entry to be removed).
|
|
62
|
+
*/
|
|
63
|
+
import { describe, it, expect, beforeEach } from 'vitest';
|
|
64
|
+
import { ChatMessageRole, ChatParams, ChatResult, ModelUsage } from '@memberjunction/ai';
|
|
65
|
+
// ---------------------------------------------------------------------------------------------
|
|
66
|
+
// Canonical scripted fixtures (shared by every provider so results are comparable across suites)
|
|
67
|
+
// ---------------------------------------------------------------------------------------------
|
|
68
|
+
const NON_STREAMING_CONTENT = 'The quick brown fox jumps over the lazy dog.';
|
|
69
|
+
const NON_STREAMING_USAGE = { PromptTokens: 11, CompletionTokens: 7 };
|
|
70
|
+
const STREAMING_CHUNKS = ['Hello', ', ', 'streaming ', 'world.'];
|
|
71
|
+
const STREAMING_USAGE = { PromptTokens: 5, CompletionTokens: 9 };
|
|
72
|
+
const FAILURE_MESSAGE = 'vendor exploded (conformance)';
|
|
73
|
+
const MID_STREAM_CHUNK = 'partial ';
|
|
74
|
+
const CANCEL_CHUNK = 'Hello ';
|
|
75
|
+
function createStreamingRecorder() {
|
|
76
|
+
const contentCalls = [];
|
|
77
|
+
const completeCalls = [];
|
|
78
|
+
const errorCalls = [];
|
|
79
|
+
return {
|
|
80
|
+
ContentCalls: contentCalls,
|
|
81
|
+
CompleteCalls: completeCalls,
|
|
82
|
+
ErrorCalls: errorCalls,
|
|
83
|
+
Callbacks: {
|
|
84
|
+
OnContent: (chunk, isComplete) => {
|
|
85
|
+
contentCalls.push({ Chunk: chunk, IsComplete: isComplete });
|
|
86
|
+
},
|
|
87
|
+
OnComplete: (finalResponse) => {
|
|
88
|
+
completeCalls.push(finalResponse);
|
|
89
|
+
},
|
|
90
|
+
OnError: (error) => {
|
|
91
|
+
errorCalls.push(error);
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
};
|
|
95
|
+
}
|
|
96
|
+
function buildChatParams(cancellationToken) {
|
|
97
|
+
const params = new ChatParams();
|
|
98
|
+
params.model = 'conformance-test-model';
|
|
99
|
+
params.messages = [{ role: ChatMessageRole.user, content: 'Say hello for the conformance suite' }];
|
|
100
|
+
if (cancellationToken) {
|
|
101
|
+
params.cancellationToken = cancellationToken;
|
|
102
|
+
}
|
|
103
|
+
return params;
|
|
104
|
+
}
|
|
105
|
+
async function settle(promise) {
|
|
106
|
+
try {
|
|
107
|
+
return { Status: 'resolved', Value: await promise };
|
|
108
|
+
}
|
|
109
|
+
catch (reason) {
|
|
110
|
+
return { Status: 'rejected', Reason: reason };
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
/** Narrow a rejection reason to ChatResult, asserting the shape on the way. */
|
|
114
|
+
function expectChatResultRejection(settled) {
|
|
115
|
+
expect(settled.Status).toBe('rejected');
|
|
116
|
+
const reason = settled.Reason;
|
|
117
|
+
expect(reason).toBeInstanceOf(ChatResult);
|
|
118
|
+
return reason;
|
|
119
|
+
}
|
|
120
|
+
/**
|
|
121
|
+
* Run the shared BaseLLM streaming/ChatResult conformance suite against one provider driver.
|
|
122
|
+
* Registers a full `describe` block — call it at the top level of a provider test file.
|
|
123
|
+
*/
|
|
124
|
+
export function RunLLMConformanceSuite(config) {
|
|
125
|
+
if (config.SupportsStreaming && !config.PreAbortedStreamingBehavior) {
|
|
126
|
+
throw new Error(`LLM conformance suite for ${config.ProviderName}: PreAbortedStreamingBehavior is required when SupportsStreaming is true`);
|
|
127
|
+
}
|
|
128
|
+
const declaredDeviations = config.KnownDeviations ?? [];
|
|
129
|
+
const queriedKinds = new Set();
|
|
130
|
+
const hasDeviation = (kind) => {
|
|
131
|
+
queriedKinds.add(kind);
|
|
132
|
+
return declaredDeviations.some((d) => d.Kind === kind);
|
|
133
|
+
};
|
|
134
|
+
// Deviation flags are resolved once, at suite-definition time, so the ledger check below can
|
|
135
|
+
// verify every declared deviation was actually exercised.
|
|
136
|
+
const streamingDropsUsage = config.SupportsStreaming ? hasDeviation('StreamingDropsUsage') : false;
|
|
137
|
+
const failedResultLacksErrorInfo = config.NonStreamingFailureMode === 'failedResult' ? hasDeviation('FailedResultLacksErrorInfo') : false;
|
|
138
|
+
describe(`${config.ProviderName} — BaseLLM streaming/ChatResult conformance`, () => {
|
|
139
|
+
let llm;
|
|
140
|
+
beforeEach(() => {
|
|
141
|
+
llm = config.CreateLLM();
|
|
142
|
+
});
|
|
143
|
+
describe('capability declaration', () => {
|
|
144
|
+
it(`declares SupportsStreaming = ${config.SupportsStreaming}`, () => {
|
|
145
|
+
expect(llm.SupportsStreaming).toBe(config.SupportsStreaming);
|
|
146
|
+
});
|
|
147
|
+
});
|
|
148
|
+
describe('non-streaming ChatCompletion — success contract', () => {
|
|
149
|
+
it('resolves a successful ChatResult carrying the vendor content', async () => {
|
|
150
|
+
config.ScriptNonStreamingSuccess(NON_STREAMING_CONTENT, NON_STREAMING_USAGE);
|
|
151
|
+
const result = await llm.ChatCompletion(buildChatParams());
|
|
152
|
+
expect(result.success).toBe(true);
|
|
153
|
+
expect(result.statusText).toBeTruthy();
|
|
154
|
+
expect(result.data.choices.length).toBeGreaterThanOrEqual(1);
|
|
155
|
+
const choice = result.data.choices[0];
|
|
156
|
+
expect(choice.message.role).toBe('assistant');
|
|
157
|
+
expect(choice.message.content).toBe(NON_STREAMING_CONTENT);
|
|
158
|
+
expect(choice.finish_reason).toBe('stop');
|
|
159
|
+
});
|
|
160
|
+
it('normalizes the vendor token accounting into a populated ModelUsage', async () => {
|
|
161
|
+
config.ScriptNonStreamingSuccess(NON_STREAMING_CONTENT, NON_STREAMING_USAGE);
|
|
162
|
+
const result = await llm.ChatCompletion(buildChatParams());
|
|
163
|
+
expect(result.data.usage).toBeInstanceOf(ModelUsage);
|
|
164
|
+
const usage = result.data.usage;
|
|
165
|
+
expect(usage.promptTokens).toBe(NON_STREAMING_USAGE.PromptTokens);
|
|
166
|
+
expect(usage.completionTokens).toBe(NON_STREAMING_USAGE.CompletionTokens);
|
|
167
|
+
});
|
|
168
|
+
it('reports coherent timing (endTime >= startTime)', async () => {
|
|
169
|
+
config.ScriptNonStreamingSuccess(NON_STREAMING_CONTENT, NON_STREAMING_USAGE);
|
|
170
|
+
const result = await llm.ChatCompletion(buildChatParams());
|
|
171
|
+
expect(result.startTime).toBeInstanceOf(Date);
|
|
172
|
+
expect(result.endTime).toBeInstanceOf(Date);
|
|
173
|
+
expect(result.endTime.getTime()).toBeGreaterThanOrEqual(result.startTime.getTime());
|
|
174
|
+
});
|
|
175
|
+
});
|
|
176
|
+
describe('non-streaming ChatCompletion — failure contract', () => {
|
|
177
|
+
if (config.NonStreamingFailureMode === 'throws') {
|
|
178
|
+
it('propagates the vendor error out of ChatCompletion (this driver does not normalize; ChatCompletions is the normalizing layer)', async () => {
|
|
179
|
+
config.ScriptFailure(new Error(FAILURE_MESSAGE), 'nonStreaming');
|
|
180
|
+
const settled = await settle(llm.ChatCompletion(buildChatParams()));
|
|
181
|
+
expect(settled.Status).toBe('rejected');
|
|
182
|
+
const reason = settled.Reason;
|
|
183
|
+
expect(reason).toBeInstanceOf(Error);
|
|
184
|
+
expect(reason.message).toContain(FAILURE_MESSAGE);
|
|
185
|
+
});
|
|
186
|
+
}
|
|
187
|
+
else {
|
|
188
|
+
it('resolves a failed ChatResult (success=false) with errorMessage and exception populated', async () => {
|
|
189
|
+
config.ScriptFailure(new Error(FAILURE_MESSAGE), 'nonStreaming');
|
|
190
|
+
const result = await llm.ChatCompletion(buildChatParams());
|
|
191
|
+
expect(result.success).toBe(false);
|
|
192
|
+
expect(result.errorMessage).toContain(FAILURE_MESSAGE);
|
|
193
|
+
expect(result.exception).toBeTruthy();
|
|
194
|
+
expect(result.data.choices).toHaveLength(0);
|
|
195
|
+
});
|
|
196
|
+
if (failedResultLacksErrorInfo) {
|
|
197
|
+
it('[KNOWN DEVIATION] failed non-streaming ChatResult lacks errorInfo', async () => {
|
|
198
|
+
config.ScriptFailure(new Error(FAILURE_MESSAGE), 'nonStreaming');
|
|
199
|
+
const result = await llm.ChatCompletion(buildChatParams());
|
|
200
|
+
expect(result.success).toBe(false);
|
|
201
|
+
// Pinned actual behavior — remove the KnownDeviations entry when fixed.
|
|
202
|
+
expect(result.errorInfo).toBeUndefined();
|
|
203
|
+
});
|
|
204
|
+
}
|
|
205
|
+
else {
|
|
206
|
+
it('populates errorInfo on the failed ChatResult', async () => {
|
|
207
|
+
config.ScriptFailure(new Error(FAILURE_MESSAGE), 'nonStreaming');
|
|
208
|
+
const result = await llm.ChatCompletion(buildChatParams());
|
|
209
|
+
expect(result.success).toBe(false);
|
|
210
|
+
expect(result.errorInfo).toBeDefined();
|
|
211
|
+
});
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
it('ChatCompletions (plural) normalizes the failure into a resolved failed ChatResult', async () => {
|
|
215
|
+
config.ScriptFailure(new Error(FAILURE_MESSAGE), 'nonStreaming');
|
|
216
|
+
const results = await llm.ChatCompletions([buildChatParams()]);
|
|
217
|
+
expect(results).toHaveLength(1);
|
|
218
|
+
expect(results[0].success).toBe(false);
|
|
219
|
+
expect(results[0].errorMessage).toContain(FAILURE_MESSAGE);
|
|
220
|
+
if (config.NonStreamingFailureMode === 'throws') {
|
|
221
|
+
// The rejection was normalized by ChatCompletions itself, which always
|
|
222
|
+
// populates errorInfo via ErrorAnalyzer.
|
|
223
|
+
expect(results[0].errorInfo).toBeDefined();
|
|
224
|
+
}
|
|
225
|
+
else if (!failedResultLacksErrorInfo) {
|
|
226
|
+
expect(results[0].errorInfo).toBeDefined();
|
|
227
|
+
}
|
|
228
|
+
});
|
|
229
|
+
});
|
|
230
|
+
if (config.SupportsStreaming) {
|
|
231
|
+
describe('streaming ChatCompletion — success contract', () => {
|
|
232
|
+
let recorder;
|
|
233
|
+
let result;
|
|
234
|
+
beforeEach(async () => {
|
|
235
|
+
config.ScriptStreamingSuccess(STREAMING_CHUNKS, STREAMING_USAGE);
|
|
236
|
+
recorder = createStreamingRecorder();
|
|
237
|
+
const params = buildChatParams();
|
|
238
|
+
params.streaming = true;
|
|
239
|
+
params.streamingCallbacks = recorder.Callbacks;
|
|
240
|
+
result = await llm.ChatCompletion(params);
|
|
241
|
+
});
|
|
242
|
+
it('accumulates the chunks, in order, into the final content', () => {
|
|
243
|
+
expect(result.success).toBe(true);
|
|
244
|
+
expect(result.data.choices[0].message.content).toBe(STREAMING_CHUNKS.join(''));
|
|
245
|
+
});
|
|
246
|
+
it('fires OnContent once per chunk in order, then signals completion with an empty final call', () => {
|
|
247
|
+
const nonFinal = recorder.ContentCalls.filter((c) => !c.IsComplete);
|
|
248
|
+
expect(nonFinal.map((c) => c.Chunk)).toEqual(STREAMING_CHUNKS);
|
|
249
|
+
const last = recorder.ContentCalls[recorder.ContentCalls.length - 1];
|
|
250
|
+
expect(last).toEqual({ Chunk: '', IsComplete: true });
|
|
251
|
+
expect(recorder.ContentCalls).toHaveLength(STREAMING_CHUNKS.length + 1);
|
|
252
|
+
});
|
|
253
|
+
it('invokes OnComplete exactly once, with the same ChatResult the promise resolves with', () => {
|
|
254
|
+
expect(recorder.CompleteCalls).toHaveLength(1);
|
|
255
|
+
expect(recorder.CompleteCalls[0]).toBe(result);
|
|
256
|
+
});
|
|
257
|
+
it('does not invoke OnError on a successful stream', () => {
|
|
258
|
+
expect(recorder.ErrorCalls).toHaveLength(0);
|
|
259
|
+
});
|
|
260
|
+
it('finalizes a ChatResult consistent with the non-streaming shape', () => {
|
|
261
|
+
expect(result).toBeInstanceOf(ChatResult);
|
|
262
|
+
expect(result.statusText).toBeTruthy();
|
|
263
|
+
const choice = result.data.choices[0];
|
|
264
|
+
expect(choice.message.role).toBe('assistant');
|
|
265
|
+
expect(choice.finish_reason).toBe('stop');
|
|
266
|
+
expect(result.data.usage).toBeInstanceOf(ModelUsage);
|
|
267
|
+
});
|
|
268
|
+
if (streamingDropsUsage) {
|
|
269
|
+
it('[KNOWN DEVIATION] drops the vendor-reported usage on the streaming path (reports 0/0)', () => {
|
|
270
|
+
const usage = result.data.usage;
|
|
271
|
+
// Pinned actual behavior — the vendor stream carried
|
|
272
|
+
// STREAMING_USAGE, the driver dropped it. Remove the
|
|
273
|
+
// KnownDeviations entry when fixed.
|
|
274
|
+
expect(usage.promptTokens).toBe(0);
|
|
275
|
+
expect(usage.completionTokens).toBe(0);
|
|
276
|
+
});
|
|
277
|
+
}
|
|
278
|
+
else {
|
|
279
|
+
it('carries the vendor-reported usage into the finalized ChatResult', () => {
|
|
280
|
+
const usage = result.data.usage;
|
|
281
|
+
expect(usage.promptTokens).toBe(STREAMING_USAGE.PromptTokens);
|
|
282
|
+
expect(usage.completionTokens).toBe(STREAMING_USAGE.CompletionTokens);
|
|
283
|
+
});
|
|
284
|
+
}
|
|
285
|
+
it('stamps template-method timing onto the finalized result', () => {
|
|
286
|
+
expect(result.startTime).toBeInstanceOf(Date);
|
|
287
|
+
expect(result.endTime).toBeInstanceOf(Date);
|
|
288
|
+
expect(result.endTime.getTime()).toBeGreaterThanOrEqual(result.startTime.getTime());
|
|
289
|
+
});
|
|
290
|
+
});
|
|
291
|
+
describe('streaming ChatCompletion — failure contract', () => {
|
|
292
|
+
it('rejects with a failed ChatResult when the stream cannot be created — never an unwrapped vendor error', async () => {
|
|
293
|
+
const vendorError = new Error(FAILURE_MESSAGE);
|
|
294
|
+
config.ScriptFailure(vendorError, 'streamStart');
|
|
295
|
+
const recorder = createStreamingRecorder();
|
|
296
|
+
const params = buildChatParams();
|
|
297
|
+
params.streaming = true;
|
|
298
|
+
params.streamingCallbacks = recorder.Callbacks;
|
|
299
|
+
const settled = await settle(llm.ChatCompletion(params));
|
|
300
|
+
const rejection = expectChatResultRejection(settled);
|
|
301
|
+
expect(rejection.success).toBe(false);
|
|
302
|
+
expect(rejection.statusText).toBe('error');
|
|
303
|
+
expect(rejection.errorMessage).toContain(FAILURE_MESSAGE);
|
|
304
|
+
expect(rejection.errorInfo).toBeDefined();
|
|
305
|
+
expect(rejection.data.choices).toHaveLength(0);
|
|
306
|
+
expect(recorder.ErrorCalls).toHaveLength(1);
|
|
307
|
+
expect(recorder.ErrorCalls[0]).toBe(vendorError);
|
|
308
|
+
expect(recorder.CompleteCalls).toHaveLength(0);
|
|
309
|
+
});
|
|
310
|
+
it('rejects with a failed ChatResult when the stream dies mid-flight, discarding the partial content', async () => {
|
|
311
|
+
const vendorError = new Error(FAILURE_MESSAGE);
|
|
312
|
+
config.ScriptFailure(vendorError, 'midStream', { ChunkBeforeError: MID_STREAM_CHUNK });
|
|
313
|
+
const recorder = createStreamingRecorder();
|
|
314
|
+
const params = buildChatParams();
|
|
315
|
+
params.streaming = true;
|
|
316
|
+
params.streamingCallbacks = recorder.Callbacks;
|
|
317
|
+
const settled = await settle(llm.ChatCompletion(params));
|
|
318
|
+
// The stream really was mid-flight: the pre-error chunk reached OnContent...
|
|
319
|
+
expect(recorder.ContentCalls.some((c) => c.Chunk === MID_STREAM_CHUNK && !c.IsComplete)).toBe(true);
|
|
320
|
+
// ...but the truncated content is NOT finalized as a success.
|
|
321
|
+
const rejection = expectChatResultRejection(settled);
|
|
322
|
+
expect(rejection.success).toBe(false);
|
|
323
|
+
expect(rejection.statusText).toBe('error');
|
|
324
|
+
expect(rejection.errorMessage).toContain(FAILURE_MESSAGE);
|
|
325
|
+
expect(rejection.errorInfo).toBeDefined();
|
|
326
|
+
expect(rejection.data.choices).toHaveLength(0);
|
|
327
|
+
expect(recorder.ErrorCalls).toHaveLength(1);
|
|
328
|
+
expect(recorder.CompleteCalls).toHaveLength(0);
|
|
329
|
+
});
|
|
330
|
+
});
|
|
331
|
+
}
|
|
332
|
+
describe('cancellation contract', () => {
|
|
333
|
+
it('resolves a cancelled failed ChatResult when the token is already aborted before a non-streaming call', async () => {
|
|
334
|
+
const controller = new AbortController();
|
|
335
|
+
controller.abort();
|
|
336
|
+
// Script a success so the ONLY way to a failed result is honoring the token.
|
|
337
|
+
config.ScriptNonStreamingSuccess(NON_STREAMING_CONTENT, NON_STREAMING_USAGE);
|
|
338
|
+
const result = await llm.ChatCompletion(buildChatParams(controller.signal));
|
|
339
|
+
expect(result.success).toBe(false);
|
|
340
|
+
expect(result.statusText.toLowerCase()).toBe('cancelled');
|
|
341
|
+
expect(result.errorInfo).toBeDefined();
|
|
342
|
+
expect(result.data.choices).toHaveLength(0);
|
|
343
|
+
});
|
|
344
|
+
if (config.SupportsStreaming) {
|
|
345
|
+
it('resolves the documented cancelled ChatResult when the stream is aborted mid-flight — partial content is not passed off as success', async () => {
|
|
346
|
+
const controller = new AbortController();
|
|
347
|
+
config.ScriptStreamingCancellation([CANCEL_CHUNK], controller);
|
|
348
|
+
const recorder = createStreamingRecorder();
|
|
349
|
+
const params = buildChatParams(controller.signal);
|
|
350
|
+
params.streaming = true;
|
|
351
|
+
params.streamingCallbacks = recorder.Callbacks;
|
|
352
|
+
const result = await llm.ChatCompletion(params);
|
|
353
|
+
expect(result).toBeInstanceOf(ChatResult);
|
|
354
|
+
expect(result.success).toBe(false);
|
|
355
|
+
expect(result.statusText.toLowerCase()).toBe('cancelled');
|
|
356
|
+
expect(result.errorInfo).toBeDefined();
|
|
357
|
+
expect(result.data.choices).toHaveLength(0);
|
|
358
|
+
// Prove the abort really happened mid-stream: the pre-abort chunk flowed.
|
|
359
|
+
expect(recorder.ContentCalls.some((c) => c.Chunk === CANCEL_CHUNK && !c.IsComplete)).toBe(true);
|
|
360
|
+
// The cancelled result still flows through OnComplete (BaseLLM resolves it).
|
|
361
|
+
expect(recorder.CompleteCalls).toHaveLength(1);
|
|
362
|
+
expect(recorder.CompleteCalls[0]).toBe(result);
|
|
363
|
+
expect(recorder.ErrorCalls).toHaveLength(0);
|
|
364
|
+
});
|
|
365
|
+
if (config.PreAbortedStreamingBehavior === 'resolvesCancelled') {
|
|
366
|
+
it('resolves a cancelled ChatResult when the token is aborted before the stream starts', async () => {
|
|
367
|
+
const controller = new AbortController();
|
|
368
|
+
controller.abort();
|
|
369
|
+
config.ScriptStreamingSuccess(STREAMING_CHUNKS, STREAMING_USAGE);
|
|
370
|
+
const recorder = createStreamingRecorder();
|
|
371
|
+
const params = buildChatParams(controller.signal);
|
|
372
|
+
params.streaming = true;
|
|
373
|
+
params.streamingCallbacks = recorder.Callbacks;
|
|
374
|
+
const result = await llm.ChatCompletion(params);
|
|
375
|
+
expect(result.success).toBe(false);
|
|
376
|
+
expect(result.statusText.toLowerCase()).toBe('cancelled');
|
|
377
|
+
expect(result.data.choices).toHaveLength(0);
|
|
378
|
+
});
|
|
379
|
+
}
|
|
380
|
+
else {
|
|
381
|
+
it("[KNOWN DEVIATION] a pre-aborted streaming request rejects with a generic statusText='error' ChatResult instead of resolving the cancelled shape", async () => {
|
|
382
|
+
// BaseLLM's outer streaming catch does not special-case cancellation, so a
|
|
383
|
+
// createStreamingRequest that throws on a pre-aborted token surfaces as a
|
|
384
|
+
// generic 'error' rejection rather than the documented cancelled result.
|
|
385
|
+
const controller = new AbortController();
|
|
386
|
+
controller.abort();
|
|
387
|
+
config.ScriptStreamingSuccess(STREAMING_CHUNKS, STREAMING_USAGE);
|
|
388
|
+
const recorder = createStreamingRecorder();
|
|
389
|
+
const params = buildChatParams(controller.signal);
|
|
390
|
+
params.streaming = true;
|
|
391
|
+
params.streamingCallbacks = recorder.Callbacks;
|
|
392
|
+
const settled = await settle(llm.ChatCompletion(params));
|
|
393
|
+
const rejection = expectChatResultRejection(settled);
|
|
394
|
+
expect(rejection.success).toBe(false);
|
|
395
|
+
expect(rejection.statusText).toBe('error');
|
|
396
|
+
expect(recorder.CompleteCalls).toHaveLength(0);
|
|
397
|
+
});
|
|
398
|
+
}
|
|
399
|
+
}
|
|
400
|
+
});
|
|
401
|
+
describe('streaming fallback', () => {
|
|
402
|
+
if (config.SupportsStreaming) {
|
|
403
|
+
it('falls back to the non-streaming path when streaming is requested without callbacks', async () => {
|
|
404
|
+
config.ScriptNonStreamingSuccess(NON_STREAMING_CONTENT, NON_STREAMING_USAGE);
|
|
405
|
+
const params = buildChatParams();
|
|
406
|
+
params.streaming = true; // no streamingCallbacks -> BaseLLM routes non-streaming
|
|
407
|
+
const result = await llm.ChatCompletion(params);
|
|
408
|
+
expect(result.success).toBe(true);
|
|
409
|
+
expect(result.data.choices[0].message.content).toBe(NON_STREAMING_CONTENT);
|
|
410
|
+
});
|
|
411
|
+
}
|
|
412
|
+
else {
|
|
413
|
+
it('falls back to the non-streaming path when streaming is requested (provider cannot stream) — callbacks are not invoked', async () => {
|
|
414
|
+
config.ScriptNonStreamingSuccess(NON_STREAMING_CONTENT, NON_STREAMING_USAGE);
|
|
415
|
+
const recorder = createStreamingRecorder();
|
|
416
|
+
const params = buildChatParams();
|
|
417
|
+
params.streaming = true;
|
|
418
|
+
params.streamingCallbacks = recorder.Callbacks;
|
|
419
|
+
const result = await llm.ChatCompletion(params);
|
|
420
|
+
expect(result.success).toBe(true);
|
|
421
|
+
expect(result.data.choices[0].message.content).toBe(NON_STREAMING_CONTENT);
|
|
422
|
+
expect(recorder.ContentCalls).toHaveLength(0);
|
|
423
|
+
expect(recorder.CompleteCalls).toHaveLength(0);
|
|
424
|
+
});
|
|
425
|
+
}
|
|
426
|
+
});
|
|
427
|
+
describe('known deviations ledger', () => {
|
|
428
|
+
it('every declared deviation is exercised by this suite and documents a reason', () => {
|
|
429
|
+
for (const deviation of declaredDeviations) {
|
|
430
|
+
expect(deviation.Reason.trim().length, `KnownDeviations['${deviation.Kind}'] must document WHY the driver deviates`).toBeGreaterThan(0);
|
|
431
|
+
expect(queriedKinds.has(deviation.Kind), `KnownDeviations['${deviation.Kind}'] declared but never exercised by this suite configuration — stale entry?`).toBe(true);
|
|
432
|
+
}
|
|
433
|
+
});
|
|
434
|
+
});
|
|
435
|
+
});
|
|
436
|
+
}
|
|
437
|
+
//# sourceMappingURL=llm-conformance.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"llm-conformance.js","sourceRoot":"","sources":["../../src/ai/llm-conformance.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6DG;AACH,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAC1D,OAAO,EAAW,eAAe,EAAE,UAAU,EAAE,UAAU,EAA0B,UAAU,EAAE,MAAM,oBAAoB,CAAC;AA+G1H,gGAAgG;AAChG,iGAAiG;AACjG,gGAAgG;AAEhG,MAAM,qBAAqB,GAAG,8CAA8C,CAAC;AAC7E,MAAM,mBAAmB,GAAwB,EAAE,YAAY,EAAE,EAAE,EAAE,gBAAgB,EAAE,CAAC,EAAE,CAAC;AAC3F,MAAM,gBAAgB,GAAG,CAAC,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,QAAQ,CAAC,CAAC;AACjE,MAAM,eAAe,GAAwB,EAAE,YAAY,EAAE,CAAC,EAAE,gBAAgB,EAAE,CAAC,EAAE,CAAC;AACtF,MAAM,eAAe,GAAG,+BAA+B,CAAC;AACxD,MAAM,gBAAgB,GAAG,UAAU,CAAC;AACpC,MAAM,YAAY,GAAG,QAAQ,CAAC;AAkB9B,SAAS,uBAAuB;IAC5B,MAAM,YAAY,GAA0B,EAAE,CAAC;IAC/C,MAAM,aAAa,GAAiB,EAAE,CAAC;IACvC,MAAM,UAAU,GAAc,EAAE,CAAC;IACjC,OAAO;QACH,YAAY,EAAE,YAAY;QAC1B,aAAa,EAAE,aAAa;QAC5B,UAAU,EAAE,UAAU;QACtB,SAAS,EAAE;YACP,SAAS,EAAE,CAAC,KAAa,EAAE,UAAmB,EAAE,EAAE;gBAC9C,YAAY,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,UAAU,EAAE,UAAU,EAAE,CAAC,CAAC;YAChE,CAAC;YACD,UAAU,EAAE,CAAC,aAAyB,EAAE,EAAE;gBACtC,aAAa,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;YACtC,CAAC;YACD,OAAO,EAAE,CAAC,KAAc,EAAE,EAAE;gBACxB,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC3B,CAAC;SACJ;KACJ,CAAC;AACN,CAAC;AAED,SAAS,eAAe,CAAC,iBAA+B;IACpD,MAAM,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;IAChC,MAAM,CAAC,KAAK,GAAG,wBAAwB,CAAC;IACxC,MAAM,CAAC,QAAQ,GAAG,CAAC,EAAE,IAAI,EAAE,eAAe,CAAC,IAAI,EAAE,OAAO,EAAE,qCAAqC,EAAE,CAAC,CAAC;IACnG,IAAI,iBAAiB,EAAE,CAAC;QACpB,MAAM,CAAC,iBAAiB,GAAG,iBAAiB,CAAC;IACjD,CAAC;IACD,OAAO,MAAM,CAAC;AAClB,CAAC;AAED,KAAK,UAAU,MAAM,CAAC,OAA4B;IAC9C,IAAI,CAAC;QACD,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,OAAO,EAAE,CAAC;IACxD,CAAC;IAAC,OAAO,MAAM,EAAE,CAAC;QACd,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC;IAClD,CAAC;AACL,CAAC;AAED,+EAA+E;AAC/E,SAAS,yBAAyB,CAAC,OAAoB;IACnD,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IACxC,MAAM,MAAM,GAAI,OAAmD,CAAC,MAAM,CAAC;IAC3E,MAAM,CAAC,MAAM,CAAC,CAAC,cAAc,CAAC,UAAU,CAAC,CAAC;IAC1C,OAAO,MAAoB,CAAC;AAChC,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,sBAAsB,CAAC,MAAiC;IACpE,IAAI,MAAM,CAAC,iBAAiB,IAAI,CAAC,MAAM,CAAC,2BAA2B,EAAE,CAAC;QAClE,MAAM,IAAI,KAAK,CACX,6BAA6B,MAAM,CAAC,YAAY,0EAA0E,CAC7H,CAAC;IACN,CAAC;IAED,MAAM,kBAAkB,GAAG,MAAM,CAAC,eAAe,IAAI,EAAE,CAAC;IACxD,MAAM,YAAY,GAAG,IAAI,GAAG,EAA+B,CAAC;IAC5D,MAAM,YAAY,GAAG,CAAC,IAAiC,EAAW,EAAE;QAChE,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACvB,OAAO,kBAAkB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC;IAC3D,CAAC,CAAC;IAEF,6FAA6F;IAC7F,0DAA0D;IAC1D,MAAM,mBAAmB,GAAG,MAAM,CAAC,iBAAiB,CAAC,CAAC,CAAC,YAAY,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;IACnG,MAAM,0BAA0B,GAC5B,MAAM,CAAC,uBAAuB,KAAK,cAAc,CAAC,CAAC,CAAC,YAAY,CAAC,4BAA4B,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;IAE3G,QAAQ,CAAC,GAAG,MAAM,CAAC,YAAY,6CAA6C,EAAE,GAAG,EAAE;QAC/E,IAAI,GAAY,CAAC;QAEjB,UAAU,CAAC,GAAG,EAAE;YACZ,GAAG,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;QAC7B,CAAC,CAAC,CAAC;QAEH,QAAQ,CAAC,wBAAwB,EAAE,GAAG,EAAE;YACpC,EAAE,CAAC,gCAAgC,MAAM,CAAC,iBAAiB,EAAE,EAAE,GAAG,EAAE;gBAChE,MAAM,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC;YACjE,CAAC,CAAC,CAAC;QACP,CAAC,CAAC,CAAC;QAEH,QAAQ,CAAC,iDAAiD,EAAE,GAAG,EAAE;YAC7D,EAAE,CAAC,8DAA8D,EAAE,KAAK,IAAI,EAAE;gBAC1E,MAAM,CAAC,yBAAyB,CAAC,qBAAqB,EAAE,mBAAmB,CAAC,CAAC;gBAE7E,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,cAAc,CAAC,eAAe,EAAE,CAAC,CAAC;gBAE3D,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBAClC,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,UAAU,EAAE,CAAC;gBACvC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC;gBAC7D,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;gBACtC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;gBAC9C,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;gBAC3D,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAC9C,CAAC,CAAC,CAAC;YAEH,EAAE,CAAC,oEAAoE,EAAE,KAAK,IAAI,EAAE;gBAChF,MAAM,CAAC,yBAAyB,CAAC,qBAAqB,EAAE,mBAAmB,CAAC,CAAC;gBAE7E,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,cAAc,CAAC,eAAe,EAAE,CAAC,CAAC;gBAE3D,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,cAAc,CAAC,UAAU,CAAC,CAAC;gBACrD,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,KAAmB,CAAC;gBAC9C,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,mBAAmB,CAAC,YAAY,CAAC,CAAC;gBAClE,MAAM,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC,IAAI,CAAC,mBAAmB,CAAC,gBAAgB,CAAC,CAAC;YAC9E,CAAC,CAAC,CAAC;YAEH,EAAE,CAAC,gDAAgD,EAAE,KAAK,IAAI,EAAE;gBAC5D,MAAM,CAAC,yBAAyB,CAAC,qBAAqB,EAAE,mBAAmB,CAAC,CAAC;gBAE7E,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,cAAc,CAAC,eAAe,EAAE,CAAC,CAAC;gBAE3D,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;gBAC9C,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;gBAC5C,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,sBAAsB,CAAC,MAAM,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC,CAAC;YACxF,CAAC,CAAC,CAAC;QACP,CAAC,CAAC,CAAC;QAEH,QAAQ,CAAC,iDAAiD,EAAE,GAAG,EAAE;YAC7D,IAAI,MAAM,CAAC,uBAAuB,KAAK,QAAQ,EAAE,CAAC;gBAC9C,EAAE,CAAC,8HAA8H,EAAE,KAAK,IAAI,EAAE;oBAC1I,MAAM,CAAC,aAAa,CAAC,IAAI,KAAK,CAAC,eAAe,CAAC,EAAE,cAAc,CAAC,CAAC;oBAEjE,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,GAAG,CAAC,cAAc,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC;oBAEpE,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;oBACxC,MAAM,MAAM,GAAI,OAAmD,CAAC,MAAM,CAAC;oBAC3E,MAAM,CAAC,MAAM,CAAC,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC;oBACrC,MAAM,CAAE,MAAgB,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,eAAe,CAAC,CAAC;gBACjE,CAAC,CAAC,CAAC;YACP,CAAC;iBAAM,CAAC;gBACJ,EAAE,CAAC,wFAAwF,EAAE,KAAK,IAAI,EAAE;oBACpG,MAAM,CAAC,aAAa,CAAC,IAAI,KAAK,CAAC,eAAe,CAAC,EAAE,cAAc,CAAC,CAAC;oBAEjE,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,cAAc,CAAC,eAAe,EAAE,CAAC,CAAC;oBAE3D,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;oBACnC,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,SAAS,CAAC,eAAe,CAAC,CAAC;oBACvD,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,UAAU,EAAE,CAAC;oBACtC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;gBAChD,CAAC,CAAC,CAAC;gBAEH,IAAI,0BAA0B,EAAE,CAAC;oBAC7B,EAAE,CAAC,mEAAmE,EAAE,KAAK,IAAI,EAAE;wBAC/E,MAAM,CAAC,aAAa,CAAC,IAAI,KAAK,CAAC,eAAe,CAAC,EAAE,cAAc,CAAC,CAAC;wBAEjE,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,cAAc,CAAC,eAAe,EAAE,CAAC,CAAC;wBAE3D,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;wBACnC,wEAAwE;wBACxE,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,aAAa,EAAE,CAAC;oBAC7C,CAAC,CAAC,CAAC;gBACP,CAAC;qBAAM,CAAC;oBACJ,EAAE,CAAC,8CAA8C,EAAE,KAAK,IAAI,EAAE;wBAC1D,MAAM,CAAC,aAAa,CAAC,IAAI,KAAK,CAAC,eAAe,CAAC,EAAE,cAAc,CAAC,CAAC;wBAEjE,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,cAAc,CAAC,eAAe,EAAE,CAAC,CAAC;wBAE3D,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;wBACnC,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,WAAW,EAAE,CAAC;oBAC3C,CAAC,CAAC,CAAC;gBACP,CAAC;YACL,CAAC;YAED,EAAE,CAAC,mFAAmF,EAAE,KAAK,IAAI,EAAE;gBAC/F,MAAM,CAAC,aAAa,CAAC,IAAI,KAAK,CAAC,eAAe,CAAC,EAAE,cAAc,CAAC,CAAC;gBAEjE,MAAM,OAAO,GAAG,MAAM,GAAG,CAAC,eAAe,CAAC,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC;gBAE/D,MAAM,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;gBAChC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBACvC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,SAAS,CAAC,eAAe,CAAC,CAAC;gBAC3D,IAAI,MAAM,CAAC,uBAAuB,KAAK,QAAQ,EAAE,CAAC;oBAC9C,uEAAuE;oBACvE,yCAAyC;oBACzC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,WAAW,EAAE,CAAC;gBAC/C,CAAC;qBAAM,IAAI,CAAC,0BAA0B,EAAE,CAAC;oBACrC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,WAAW,EAAE,CAAC;gBAC/C,CAAC;YACL,CAAC,CAAC,CAAC;QACP,CAAC,CAAC,CAAC;QAEH,IAAI,MAAM,CAAC,iBAAiB,EAAE,CAAC;YAC3B,QAAQ,CAAC,6CAA6C,EAAE,GAAG,EAAE;gBACzD,IAAI,QAA2B,CAAC;gBAChC,IAAI,MAAkB,CAAC;gBAEvB,UAAU,CAAC,KAAK,IAAI,EAAE;oBAClB,MAAM,CAAC,sBAAsB,CAAC,gBAAgB,EAAE,eAAe,CAAC,CAAC;oBACjE,QAAQ,GAAG,uBAAuB,EAAE,CAAC;oBACrC,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;oBACjC,MAAM,CAAC,SAAS,GAAG,IAAI,CAAC;oBACxB,MAAM,CAAC,kBAAkB,GAAG,QAAQ,CAAC,SAAS,CAAC;oBAC/C,MAAM,GAAG,MAAM,GAAG,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;gBAC9C,CAAC,CAAC,CAAC;gBAEH,EAAE,CAAC,0DAA0D,EAAE,GAAG,EAAE;oBAChE,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;oBAClC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;gBACnF,CAAC,CAAC,CAAC;gBAEH,EAAE,CAAC,2FAA2F,EAAE,GAAG,EAAE;oBACjG,MAAM,QAAQ,GAAG,QAAQ,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;oBACpE,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC;oBAC/D,MAAM,IAAI,GAAG,QAAQ,CAAC,YAAY,CAAC,QAAQ,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;oBACrE,MAAM,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC;oBACtD,MAAM,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC,YAAY,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;gBAC5E,CAAC,CAAC,CAAC;gBAEH,EAAE,CAAC,qFAAqF,EAAE,GAAG,EAAE;oBAC3F,MAAM,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;oBAC/C,MAAM,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBACnD,CAAC,CAAC,CAAC;gBAEH,EAAE,CAAC,gDAAgD,EAAE,GAAG,EAAE;oBACtD,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;gBAChD,CAAC,CAAC,CAAC;gBAEH,EAAE,CAAC,gEAAgE,EAAE,GAAG,EAAE;oBACtE,MAAM,CAAC,MAAM,CAAC,CAAC,cAAc,CAAC,UAAU,CAAC,CAAC;oBAC1C,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,UAAU,EAAE,CAAC;oBACvC,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;oBACtC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;oBAC9C,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;oBAC1C,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,cAAc,CAAC,UAAU,CAAC,CAAC;gBACzD,CAAC,CAAC,CAAC;gBAEH,IAAI,mBAAmB,EAAE,CAAC;oBACtB,EAAE,CAAC,uFAAuF,EAAE,GAAG,EAAE;wBAC7F,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,KAAmB,CAAC;wBAC9C,qDAAqD;wBACrD,qDAAqD;wBACrD,oCAAoC;wBACpC,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;wBACnC,MAAM,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;oBAC3C,CAAC,CAAC,CAAC;gBACP,CAAC;qBAAM,CAAC;oBACJ,EAAE,CAAC,iEAAiE,EAAE,GAAG,EAAE;wBACvE,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,KAAmB,CAAC;wBAC9C,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,YAAY,CAAC,CAAC;wBAC9D,MAAM,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,gBAAgB,CAAC,CAAC;oBAC1E,CAAC,CAAC,CAAC;gBACP,CAAC;gBAED,EAAE,CAAC,yDAAyD,EAAE,GAAG,EAAE;oBAC/D,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;oBAC9C,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;oBAC5C,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,sBAAsB,CAAC,MAAM,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC,CAAC;gBACxF,CAAC,CAAC,CAAC;YACP,CAAC,CAAC,CAAC;YAEH,QAAQ,CAAC,6CAA6C,EAAE,GAAG,EAAE;gBACzD,EAAE,CAAC,sGAAsG,EAAE,KAAK,IAAI,EAAE;oBAClH,MAAM,WAAW,GAAG,IAAI,KAAK,CAAC,eAAe,CAAC,CAAC;oBAC/C,MAAM,CAAC,aAAa,CAAC,WAAW,EAAE,aAAa,CAAC,CAAC;oBACjD,MAAM,QAAQ,GAAG,uBAAuB,EAAE,CAAC;oBAC3C,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;oBACjC,MAAM,CAAC,SAAS,GAAG,IAAI,CAAC;oBACxB,MAAM,CAAC,kBAAkB,GAAG,QAAQ,CAAC,SAAS,CAAC;oBAE/C,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,GAAG,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC;oBAEzD,MAAM,SAAS,GAAG,yBAAyB,CAAC,OAAO,CAAC,CAAC;oBACrD,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;oBACtC,MAAM,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;oBAC3C,MAAM,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC,SAAS,CAAC,eAAe,CAAC,CAAC;oBAC1D,MAAM,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,WAAW,EAAE,CAAC;oBAC1C,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;oBAC/C,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;oBAC5C,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;oBACjD,MAAM,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;gBACnD,CAAC,CAAC,CAAC;gBAEH,EAAE,CAAC,kGAAkG,EAAE,KAAK,IAAI,EAAE;oBAC9G,MAAM,WAAW,GAAG,IAAI,KAAK,CAAC,eAAe,CAAC,CAAC;oBAC/C,MAAM,CAAC,aAAa,CAAC,WAAW,EAAE,WAAW,EAAE,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,CAAC,CAAC;oBACvF,MAAM,QAAQ,GAAG,uBAAuB,EAAE,CAAC;oBAC3C,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;oBACjC,MAAM,CAAC,SAAS,GAAG,IAAI,CAAC;oBACxB,MAAM,CAAC,kBAAkB,GAAG,QAAQ,CAAC,SAAS,CAAC;oBAE/C,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,GAAG,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC;oBAEzD,6EAA6E;oBAC7E,MAAM,CAAC,QAAQ,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,gBAAgB,IAAI,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;oBACpG,8DAA8D;oBAC9D,MAAM,SAAS,GAAG,yBAAyB,CAAC,OAAO,CAAC,CAAC;oBACrD,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;oBACtC,MAAM,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;oBAC3C,MAAM,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC,SAAS,CAAC,eAAe,CAAC,CAAC;oBAC1D,MAAM,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,WAAW,EAAE,CAAC;oBAC1C,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;oBAC/C,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;oBAC5C,MAAM,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;gBACnD,CAAC,CAAC,CAAC;YACP,CAAC,CAAC,CAAC;QACP,CAAC;QAED,QAAQ,CAAC,uBAAuB,EAAE,GAAG,EAAE;YACnC,EAAE,CAAC,sGAAsG,EAAE,KAAK,IAAI,EAAE;gBAClH,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;gBACzC,UAAU,CAAC,KAAK,EAAE,CAAC;gBACnB,6EAA6E;gBAC7E,MAAM,CAAC,yBAAyB,CAAC,qBAAqB,EAAE,mBAAmB,CAAC,CAAC;gBAE7E,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,cAAc,CAAC,eAAe,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC;gBAE5E,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBACnC,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,WAAW,EAAE,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;gBAC1D,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,WAAW,EAAE,CAAC;gBACvC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;YAChD,CAAC,CAAC,CAAC;YAEH,IAAI,MAAM,CAAC,iBAAiB,EAAE,CAAC;gBAC3B,EAAE,CAAC,mIAAmI,EAAE,KAAK,IAAI,EAAE;oBAC/I,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;oBACzC,MAAM,CAAC,2BAA2B,CAAC,CAAC,YAAY,CAAC,EAAE,UAAU,CAAC,CAAC;oBAC/D,MAAM,QAAQ,GAAG,uBAAuB,EAAE,CAAC;oBAC3C,MAAM,MAAM,GAAG,eAAe,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;oBAClD,MAAM,CAAC,SAAS,GAAG,IAAI,CAAC;oBACxB,MAAM,CAAC,kBAAkB,GAAG,QAAQ,CAAC,SAAS,CAAC;oBAE/C,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;oBAEhD,MAAM,CAAC,MAAM,CAAC,CAAC,cAAc,CAAC,UAAU,CAAC,CAAC;oBAC1C,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;oBACnC,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,WAAW,EAAE,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;oBAC1D,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,WAAW,EAAE,CAAC;oBACvC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;oBAC5C,0EAA0E;oBAC1E,MAAM,CAAC,QAAQ,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,YAAY,IAAI,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;oBAChG,6EAA6E;oBAC7E,MAAM,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;oBAC/C,MAAM,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;oBAC/C,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;gBAChD,CAAC,CAAC,CAAC;gBAEH,IAAI,MAAM,CAAC,2BAA2B,KAAK,mBAAmB,EAAE,CAAC;oBAC7D,EAAE,CAAC,oFAAoF,EAAE,KAAK,IAAI,EAAE;wBAChG,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;wBACzC,UAAU,CAAC,KAAK,EAAE,CAAC;wBACnB,MAAM,CAAC,sBAAsB,CAAC,gBAAgB,EAAE,eAAe,CAAC,CAAC;wBACjE,MAAM,QAAQ,GAAG,uBAAuB,EAAE,CAAC;wBAC3C,MAAM,MAAM,GAAG,eAAe,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;wBAClD,MAAM,CAAC,SAAS,GAAG,IAAI,CAAC;wBACxB,MAAM,CAAC,kBAAkB,GAAG,QAAQ,CAAC,SAAS,CAAC;wBAE/C,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;wBAEhD,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;wBACnC,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,WAAW,EAAE,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;wBAC1D,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;oBAChD,CAAC,CAAC,CAAC;gBACP,CAAC;qBAAM,CAAC;oBACJ,EAAE,CAAC,iJAAiJ,EAAE,KAAK,IAAI,EAAE;wBAC7J,2EAA2E;wBAC3E,0EAA0E;wBAC1E,yEAAyE;wBACzE,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;wBACzC,UAAU,CAAC,KAAK,EAAE,CAAC;wBACnB,MAAM,CAAC,sBAAsB,CAAC,gBAAgB,EAAE,eAAe,CAAC,CAAC;wBACjE,MAAM,QAAQ,GAAG,uBAAuB,EAAE,CAAC;wBAC3C,MAAM,MAAM,GAAG,eAAe,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;wBAClD,MAAM,CAAC,SAAS,GAAG,IAAI,CAAC;wBACxB,MAAM,CAAC,kBAAkB,GAAG,QAAQ,CAAC,SAAS,CAAC;wBAE/C,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,GAAG,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC;wBAEzD,MAAM,SAAS,GAAG,yBAAyB,CAAC,OAAO,CAAC,CAAC;wBACrD,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;wBACtC,MAAM,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;wBAC3C,MAAM,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;oBACnD,CAAC,CAAC,CAAC;gBACP,CAAC;YACL,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,QAAQ,CAAC,oBAAoB,EAAE,GAAG,EAAE;YAChC,IAAI,MAAM,CAAC,iBAAiB,EAAE,CAAC;gBAC3B,EAAE,CAAC,oFAAoF,EAAE,KAAK,IAAI,EAAE;oBAChG,MAAM,CAAC,yBAAyB,CAAC,qBAAqB,EAAE,mBAAmB,CAAC,CAAC;oBAC7E,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;oBACjC,MAAM,CAAC,SAAS,GAAG,IAAI,CAAC,CAAC,wDAAwD;oBAEjF,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;oBAEhD,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;oBAClC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;gBAC/E,CAAC,CAAC,CAAC;YACP,CAAC;iBAAM,CAAC;gBACJ,EAAE,CAAC,uHAAuH,EAAE,KAAK,IAAI,EAAE;oBACnI,MAAM,CAAC,yBAAyB,CAAC,qBAAqB,EAAE,mBAAmB,CAAC,CAAC;oBAC7E,MAAM,QAAQ,GAAG,uBAAuB,EAAE,CAAC;oBAC3C,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;oBACjC,MAAM,CAAC,SAAS,GAAG,IAAI,CAAC;oBACxB,MAAM,CAAC,kBAAkB,GAAG,QAAQ,CAAC,SAAS,CAAC;oBAE/C,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;oBAEhD,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;oBAClC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;oBAC3E,MAAM,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;oBAC9C,MAAM,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;gBACnD,CAAC,CAAC,CAAC;YACP,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,QAAQ,CAAC,yBAAyB,EAAE,GAAG,EAAE;YACrC,EAAE,CAAC,4EAA4E,EAAE,GAAG,EAAE;gBAClF,KAAK,MAAM,SAAS,IAAI,kBAAkB,EAAE,CAAC;oBACzC,MAAM,CACF,SAAS,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,MAAM,EAC9B,oBAAoB,SAAS,CAAC,IAAI,0CAA0C,CAC/E,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;oBACrB,MAAM,CACF,YAAY,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,EAChC,oBAAoB,SAAS,CAAC,IAAI,4EAA4E,CACjH,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACjB,CAAC;YACL,CAAC,CAAC,CAAC;QACP,CAAC,CAAC,CAAC;IACP,CAAC,CAAC,CAAC;AACP,CAAC"}
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview Scriptable OpenAI-compatible chat-completions seam mock for the LLM conformance
|
|
3
|
+
* suite ({@link RunLLMConformanceSuite} in ./llm-conformance).
|
|
4
|
+
*
|
|
5
|
+
* Several MJ providers speak the OpenAI wire shape at the exact same seam —
|
|
6
|
+
* `client.chat.completions.create(body, { signal })` — either through the `openai` SDK itself
|
|
7
|
+
* (OpenAILLM and its subclasses MiniMax, Zhipu, OpenRouter, ...) or through OpenAI-compatible
|
|
8
|
+
* SDKs with the same surface (`groq-sdk`, `@cerebras/cerebras_cloud_sdk`). This factory builds a
|
|
9
|
+
* purely structural fake of that seam (no dependency on any vendor SDK) whose `Script*` functions
|
|
10
|
+
* plug straight into the conformance-suite config.
|
|
11
|
+
*
|
|
12
|
+
* A provider test file injects `Client` in place of the driver's private SDK client:
|
|
13
|
+
* ```ts
|
|
14
|
+
* const seam = CreateOpenAICompatibleSeamMock();
|
|
15
|
+
* const llm = new MiniMaxLLM('test-key');
|
|
16
|
+
* (llm as unknown as { _openAI: OpenAICompatibleChatClient })._openAI = seam.Client;
|
|
17
|
+
* ```
|
|
18
|
+
*
|
|
19
|
+
* SDK-faithful behaviors this fake reproduces:
|
|
20
|
+
* - A call whose request options carry an ALREADY-aborted `AbortSignal` fails with an
|
|
21
|
+
* `AbortError`-named error instead of responding (the stainless SDKs check the signal before
|
|
22
|
+
* dialing; the DOM-standard name keeps every driver's `error.name === 'AbortError'` fallback
|
|
23
|
+
* working without importing any vendor error class).
|
|
24
|
+
* - Streaming responses are async iterables of `chat.completion.chunk`-shaped objects; the final
|
|
25
|
+
* chunk carries `finish_reason: 'stop'` plus usage (both at `chunk.usage`, OpenAI-style, and at
|
|
26
|
+
* `chunk.x_groq.usage`, Groq's native placement — so a driver that drops stream usage is
|
|
27
|
+
* provably dropping data the vendor sent).
|
|
28
|
+
*
|
|
29
|
+
* Exported from `@memberjunction/unit-testing` alongside ./llm-conformance — see that file's
|
|
30
|
+
* header. Test-code only; never import from runtime code.
|
|
31
|
+
*/
|
|
32
|
+
import { ExpectedUsageCounts, FailureSite } from './llm-conformance.js';
|
|
33
|
+
/** OpenAI-wire usage payload. */
|
|
34
|
+
export interface OpenAICompatibleUsagePayload {
|
|
35
|
+
prompt_tokens: number;
|
|
36
|
+
completion_tokens: number;
|
|
37
|
+
total_tokens: number;
|
|
38
|
+
}
|
|
39
|
+
/** OpenAI-wire non-streaming completion payload (the subset MJ drivers consume). */
|
|
40
|
+
export interface OpenAICompatibleResponsePayload {
|
|
41
|
+
id: string;
|
|
42
|
+
object: 'chat.completion';
|
|
43
|
+
created: number;
|
|
44
|
+
model: string;
|
|
45
|
+
choices: Array<{
|
|
46
|
+
message: {
|
|
47
|
+
role: 'assistant';
|
|
48
|
+
content: string;
|
|
49
|
+
};
|
|
50
|
+
finish_reason: string;
|
|
51
|
+
index: number;
|
|
52
|
+
}>;
|
|
53
|
+
usage: OpenAICompatibleUsagePayload;
|
|
54
|
+
}
|
|
55
|
+
/** OpenAI-wire streaming chunk payload (the subset MJ drivers consume). */
|
|
56
|
+
export interface OpenAICompatibleChunkPayload {
|
|
57
|
+
id: string;
|
|
58
|
+
object: 'chat.completion.chunk';
|
|
59
|
+
model: string;
|
|
60
|
+
choices: Array<{
|
|
61
|
+
delta: {
|
|
62
|
+
content?: string;
|
|
63
|
+
};
|
|
64
|
+
finish_reason: string | null;
|
|
65
|
+
index: number;
|
|
66
|
+
}>;
|
|
67
|
+
usage?: OpenAICompatibleUsagePayload;
|
|
68
|
+
x_groq?: {
|
|
69
|
+
usage: OpenAICompatibleUsagePayload;
|
|
70
|
+
};
|
|
71
|
+
}
|
|
72
|
+
/** The request body subset the seam dispatches on. */
|
|
73
|
+
export interface OpenAICompatibleRequestBody {
|
|
74
|
+
model?: string;
|
|
75
|
+
stream?: boolean;
|
|
76
|
+
}
|
|
77
|
+
/** Per-call request options (the stainless SDKs' second argument). */
|
|
78
|
+
export interface OpenAICompatibleRequestOptions {
|
|
79
|
+
signal?: AbortSignal;
|
|
80
|
+
}
|
|
81
|
+
/** The client surface MJ drivers call: `client.chat.completions.create(body, options)`. */
|
|
82
|
+
export interface OpenAICompatibleChatClient {
|
|
83
|
+
chat: {
|
|
84
|
+
completions: {
|
|
85
|
+
create: (body: OpenAICompatibleRequestBody, options?: OpenAICompatibleRequestOptions) => Promise<OpenAICompatibleResponsePayload | AsyncGenerator<OpenAICompatibleChunkPayload>>;
|
|
86
|
+
};
|
|
87
|
+
};
|
|
88
|
+
}
|
|
89
|
+
/** The scriptable seam handed to a provider conformance test file. */
|
|
90
|
+
export interface OpenAICompatibleSeam {
|
|
91
|
+
/** Inject this in place of the driver's private SDK client. */
|
|
92
|
+
Client: OpenAICompatibleChatClient;
|
|
93
|
+
/** Conformance-config hook: next non-streaming call succeeds. */
|
|
94
|
+
ScriptNonStreamingSuccess: (content: string, usage: ExpectedUsageCounts) => void;
|
|
95
|
+
/** Conformance-config hook: next streaming call emits these chunks then completes. */
|
|
96
|
+
ScriptStreamingSuccess: (chunks: string[], usage: ExpectedUsageCounts) => void;
|
|
97
|
+
/** Conformance-config hook: next call fails at the given site. */
|
|
98
|
+
ScriptFailure: (error: Error, at: FailureSite, options?: {
|
|
99
|
+
ChunkBeforeError?: string;
|
|
100
|
+
}) => void;
|
|
101
|
+
/** Conformance-config hook: next streaming call emits chunks, aborts the controller, then dies like an aborted SDK stream. */
|
|
102
|
+
ScriptStreamingCancellation: (chunksBeforeAbort: string[], controller: AbortController) => void;
|
|
103
|
+
}
|
|
104
|
+
/**
|
|
105
|
+
* Build a fresh scriptable OpenAI-compatible seam. One seam instance can be shared across the
|
|
106
|
+
* whole conformance run — each `Script*` call replaces the pending behavior, and the conformance
|
|
107
|
+
* suite scripts before every request.
|
|
108
|
+
*/
|
|
109
|
+
export declare function CreateOpenAICompatibleSeamMock(): OpenAICompatibleSeam;
|
|
110
|
+
//# sourceMappingURL=openai-compatible-seam.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"openai-compatible-seam.d.ts","sourceRoot":"","sources":["../../src/ai/openai-compatible-seam.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AACH,OAAO,EAAE,mBAAmB,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAErE,iCAAiC;AACjC,MAAM,WAAW,4BAA4B;IACzC,aAAa,EAAE,MAAM,CAAC;IACtB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,YAAY,EAAE,MAAM,CAAC;CACxB;AAED,oFAAoF;AACpF,MAAM,WAAW,+BAA+B;IAC5C,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,iBAAiB,CAAC;IAC1B,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,KAAK,CAAC;QACX,OAAO,EAAE;YAAE,IAAI,EAAE,WAAW,CAAC;YAAC,OAAO,EAAE,MAAM,CAAA;SAAE,CAAC;QAChD,aAAa,EAAE,MAAM,CAAC;QACtB,KAAK,EAAE,MAAM,CAAC;KACjB,CAAC,CAAC;IACH,KAAK,EAAE,4BAA4B,CAAC;CACvC;AAED,2EAA2E;AAC3E,MAAM,WAAW,4BAA4B;IACzC,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,uBAAuB,CAAC;IAChC,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,KAAK,CAAC;QACX,KAAK,EAAE;YAAE,OAAO,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC;QAC5B,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;QAC7B,KAAK,EAAE,MAAM,CAAC;KACjB,CAAC,CAAC;IACH,KAAK,CAAC,EAAE,4BAA4B,CAAC;IACrC,MAAM,CAAC,EAAE;QAAE,KAAK,EAAE,4BAA4B,CAAA;KAAE,CAAC;CACpD;AAED,sDAAsD;AACtD,MAAM,WAAW,2BAA2B;IACxC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,OAAO,CAAC;CACpB;AAED,sEAAsE;AACtE,MAAM,WAAW,8BAA8B;IAC3C,MAAM,CAAC,EAAE,WAAW,CAAC;CACxB;AAED,2FAA2F;AAC3F,MAAM,WAAW,0BAA0B;IACvC,IAAI,EAAE;QACF,WAAW,EAAE;YACT,MAAM,EAAE,CACJ,IAAI,EAAE,2BAA2B,EACjC,OAAO,CAAC,EAAE,8BAA8B,KACvC,OAAO,CAAC,+BAA+B,GAAG,cAAc,CAAC,4BAA4B,CAAC,CAAC,CAAC;SAChG,CAAC;KACL,CAAC;CACL;AAED,sEAAsE;AACtE,MAAM,WAAW,oBAAoB;IACjC,+DAA+D;IAC/D,MAAM,EAAE,0BAA0B,CAAC;IACnC,iEAAiE;IACjE,yBAAyB,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,mBAAmB,KAAK,IAAI,CAAC;IACjF,sFAAsF;IACtF,sBAAsB,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,mBAAmB,KAAK,IAAI,CAAC;IAC/E,kEAAkE;IAClE,aAAa,EAAE,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,EAAE,WAAW,EAAE,OAAO,CAAC,EAAE;QAAE,gBAAgB,CAAC,EAAE,MAAM,CAAA;KAAE,KAAK,IAAI,CAAC;IAChG,8HAA8H;IAC9H,2BAA2B,EAAE,CAAC,iBAAiB,EAAE,MAAM,EAAE,EAAE,UAAU,EAAE,eAAe,KAAK,IAAI,CAAC;CACnG;AA0CD;;;;GAIG;AACH,wBAAgB,8BAA8B,IAAI,oBAAoB,CAwGrE"}
|