ai.matey.testing 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/dist/cjs/fixture-capture.js +169 -0
- package/dist/cjs/fixture-capture.js.map +1 -0
- package/dist/cjs/fixture-helpers.js +240 -0
- package/dist/cjs/fixture-helpers.js.map +1 -0
- package/dist/cjs/fixture-loader.js +196 -0
- package/dist/cjs/fixture-loader.js.map +1 -0
- package/dist/cjs/fixture-types.js +20 -0
- package/dist/cjs/fixture-types.js.map +1 -0
- package/dist/cjs/index.js +66 -0
- package/dist/cjs/index.js.map +1 -0
- package/dist/cjs/property-testing.js +288 -0
- package/dist/cjs/property-testing.js.map +1 -0
- package/dist/cjs/test-helpers.js +289 -0
- package/dist/cjs/test-helpers.js.map +1 -0
- package/dist/esm/fixture-capture.js +163 -0
- package/dist/esm/fixture-capture.js.map +1 -0
- package/dist/esm/fixture-helpers.js +228 -0
- package/dist/esm/fixture-helpers.js.map +1 -0
- package/dist/esm/fixture-loader.js +154 -0
- package/dist/esm/fixture-loader.js.map +1 -0
- package/dist/esm/fixture-types.js +16 -0
- package/dist/esm/fixture-types.js.map +1 -0
- package/dist/esm/index.js +15 -0
- package/dist/esm/index.js.map +1 -0
- package/dist/esm/property-testing.js +274 -0
- package/dist/esm/property-testing.js.map +1 -0
- package/dist/esm/test-helpers.js +272 -0
- package/dist/esm/test-helpers.js.map +1 -0
- package/dist/types/fixture-capture.d.ts +55 -0
- package/dist/types/fixture-capture.d.ts.map +1 -0
- package/dist/types/fixture-helpers.d.ts +69 -0
- package/dist/types/fixture-helpers.d.ts.map +1 -0
- package/dist/types/fixture-loader.d.ts +40 -0
- package/dist/types/fixture-loader.d.ts.map +1 -0
- package/dist/types/fixture-types.d.ts +76 -0
- package/dist/types/fixture-types.d.ts.map +1 -0
- package/dist/types/index.d.ts +12 -0
- package/dist/types/index.d.ts.map +1 -0
- package/dist/types/property-testing.d.ts +79 -0
- package/dist/types/property-testing.d.ts.map +1 -0
- package/dist/types/test-helpers.d.ts +81 -0
- package/dist/types/test-helpers.d.ts.map +1 -0
- package/package.json +68 -0
- package/readme.md +31 -0
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Fixture helpers - utilities for using fixtures in tests
|
|
3
|
+
*/
|
|
4
|
+
import type { IRChatRequest, IRChatResponse, IRStreamChunk } from 'ai.matey.types';
|
|
5
|
+
import type { Fixture, ChatFixture, StreamingFixture } from './fixture-types.js';
|
|
6
|
+
/**
|
|
7
|
+
* Simple mock backend from a fixture (for testing)
|
|
8
|
+
*/
|
|
9
|
+
export interface SimpleMockBackend {
|
|
10
|
+
readonly name: string;
|
|
11
|
+
chat(_request: IRChatRequest): Promise<IRChatResponse>;
|
|
12
|
+
chatStream?(_request: IRChatRequest): AsyncIterable<IRStreamChunk>;
|
|
13
|
+
}
|
|
14
|
+
/**
|
|
15
|
+
* Create a mock backend adapter from a fixture
|
|
16
|
+
*/
|
|
17
|
+
export declare function createMockFromFixture(fixture: Fixture): SimpleMockBackend;
|
|
18
|
+
/**
|
|
19
|
+
* Replay streaming fixture with timing
|
|
20
|
+
*/
|
|
21
|
+
export declare function replayStreamWithTiming(fixture: StreamingFixture, options?: {
|
|
22
|
+
/** Delay between chunks in ms */
|
|
23
|
+
chunkDelay?: number;
|
|
24
|
+
/** Speed multiplier (1.0 = real-time, 2.0 = 2x speed, 0.5 = half speed) */
|
|
25
|
+
speedMultiplier?: number;
|
|
26
|
+
}): AsyncIterable<IRStreamChunk>;
|
|
27
|
+
/**
|
|
28
|
+
* Create multiple mocks from multiple fixtures
|
|
29
|
+
*/
|
|
30
|
+
export declare function createMocksFromFixtures(fixtures: Fixture[]): Map<string, SimpleMockBackend>;
|
|
31
|
+
/**
|
|
32
|
+
* Check if a fixture is for streaming
|
|
33
|
+
*/
|
|
34
|
+
export declare function fixtureIsStreaming(fixture: Fixture): boolean;
|
|
35
|
+
/**
|
|
36
|
+
* Validate that a response matches a fixture
|
|
37
|
+
*/
|
|
38
|
+
export declare function validateAgainstFixture(response: IRChatResponse, fixture: ChatFixture, options?: {
|
|
39
|
+
/** Check exact match (default: false) */
|
|
40
|
+
exact?: boolean;
|
|
41
|
+
/** Fields to ignore when comparing */
|
|
42
|
+
ignoreFields?: string[];
|
|
43
|
+
}): boolean;
|
|
44
|
+
/**
|
|
45
|
+
* Extract request from fixture (useful for replaying)
|
|
46
|
+
*/
|
|
47
|
+
export declare function extractRequest(fixture: Fixture): IRChatRequest;
|
|
48
|
+
/**
|
|
49
|
+
* Extract response from fixture
|
|
50
|
+
*/
|
|
51
|
+
export declare function extractResponse(fixture: ChatFixture): IRChatResponse;
|
|
52
|
+
/**
|
|
53
|
+
* Extract streaming chunks from fixture
|
|
54
|
+
*/
|
|
55
|
+
export declare function extractChunks(fixture: StreamingFixture): readonly IRStreamChunk[];
|
|
56
|
+
/**
|
|
57
|
+
* Collect streaming chunks into a single response
|
|
58
|
+
*/
|
|
59
|
+
export declare function collectChunksToResponse(chunks: readonly IRStreamChunk[]): IRChatResponse;
|
|
60
|
+
/**
|
|
61
|
+
* Create a test-friendly mock with custom responses
|
|
62
|
+
*/
|
|
63
|
+
export declare function createConfigurableMock(name: string, defaultResponse?: IRChatResponse): SimpleMockBackend & {
|
|
64
|
+
setResponse: (response: IRChatResponse) => void;
|
|
65
|
+
setStreamChunks: (chunks: IRStreamChunk[]) => void;
|
|
66
|
+
setError: (error: Error) => void;
|
|
67
|
+
reset: () => void;
|
|
68
|
+
};
|
|
69
|
+
//# sourceMappingURL=fixture-helpers.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"fixture-helpers.d.ts","sourceRoot":"","sources":["../../src/fixture-helpers.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AACnF,OAAO,KAAK,EAAE,OAAO,EAAE,WAAW,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAGjF;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,IAAI,CAAC,QAAQ,EAAE,aAAa,GAAG,OAAO,CAAC,cAAc,CAAC,CAAC;IACvD,UAAU,CAAC,CAAC,QAAQ,EAAE,aAAa,GAAG,aAAa,CAAC,aAAa,CAAC,CAAC;CACpE;AAED;;GAEG;AACH,wBAAgB,qBAAqB,CAAC,OAAO,EAAE,OAAO,GAAG,iBAAiB,CAMzE;AAqCD;;GAEG;AACH,wBAAuB,sBAAsB,CAC3C,OAAO,EAAE,gBAAgB,EACzB,OAAO,CAAC,EAAE;IACR,iCAAiC;IACjC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,2EAA2E;IAC3E,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B,GACA,aAAa,CAAC,aAAa,CAAC,CAa9B;AAED;;GAEG;AACH,wBAAgB,uBAAuB,CAAC,QAAQ,EAAE,OAAO,EAAE,GAAG,GAAG,CAAC,MAAM,EAAE,iBAAiB,CAAC,CAS3F;AAED;;GAEG;AACH,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAE5D;AAED;;GAEG;AACH,wBAAgB,sBAAsB,CACpC,QAAQ,EAAE,cAAc,EACxB,OAAO,EAAE,WAAW,EACpB,OAAO,CAAC,EAAE;IACR,yCAAyC;IACzC,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,sCAAsC;IACtC,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;CACzB,GACA,OAAO,CA0BT;AA2CD;;GAEG;AACH,wBAAgB,cAAc,CAAC,OAAO,EAAE,OAAO,GAAG,aAAa,CAE9D;AAED;;GAEG;AACH,wBAAgB,eAAe,CAAC,OAAO,EAAE,WAAW,GAAG,cAAc,CAEpE;AAED;;GAEG;AACH,wBAAgB,aAAa,CAAC,OAAO,EAAE,gBAAgB,GAAG,SAAS,aAAa,EAAE,CAEjF;AAED;;GAEG;AACH,wBAAgB,uBAAuB,CAAC,MAAM,EAAE,SAAS,aAAa,EAAE,GAAG,cAAc,CA0BxF;AAED;;GAEG;AACH,wBAAgB,sBAAsB,CACpC,IAAI,EAAE,MAAM,EACZ,eAAe,CAAC,EAAE,cAAc,GAC/B,iBAAiB,GAAG;IACrB,WAAW,EAAE,CAAC,QAAQ,EAAE,cAAc,KAAK,IAAI,CAAC;IAChD,eAAe,EAAE,CAAC,MAAM,EAAE,aAAa,EAAE,KAAK,IAAI,CAAC;IACnD,QAAQ,EAAE,CAAC,KAAK,EAAE,KAAK,KAAK,IAAI,CAAC;IACjC,KAAK,EAAE,MAAM,IAAI,CAAC;CACnB,CAsDA"}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Fixture loader - loads fixtures from files
|
|
3
|
+
*/
|
|
4
|
+
import type { Fixture, FixtureQuery, FixtureCollection } from './fixture-types.js';
|
|
5
|
+
/**
|
|
6
|
+
* Default fixtures directory (relative to project root)
|
|
7
|
+
*/
|
|
8
|
+
export declare const FIXTURES_DIR: string;
|
|
9
|
+
/**
|
|
10
|
+
* Load a single fixture from file
|
|
11
|
+
*/
|
|
12
|
+
export declare function loadFixture(provider: string, scenario: string, options?: {
|
|
13
|
+
useCache?: boolean;
|
|
14
|
+
}): Promise<Fixture>;
|
|
15
|
+
/**
|
|
16
|
+
* Load all fixtures for a provider
|
|
17
|
+
*/
|
|
18
|
+
export declare function loadProviderFixtures(provider: string, options?: {
|
|
19
|
+
useCache?: boolean;
|
|
20
|
+
}): Promise<Fixture[]>;
|
|
21
|
+
/**
|
|
22
|
+
* Search fixtures by query
|
|
23
|
+
*/
|
|
24
|
+
export declare function findFixtures(query: FixtureQuery): Promise<Fixture[]>;
|
|
25
|
+
/**
|
|
26
|
+
* Clear fixture cache
|
|
27
|
+
*/
|
|
28
|
+
export declare function clearFixtureCache(): void;
|
|
29
|
+
/**
|
|
30
|
+
* Get cache statistics
|
|
31
|
+
*/
|
|
32
|
+
export declare function getFixtureCacheStats(): {
|
|
33
|
+
size: number;
|
|
34
|
+
keys: string[];
|
|
35
|
+
};
|
|
36
|
+
/**
|
|
37
|
+
* Load a fixture collection (multiple fixtures in one file)
|
|
38
|
+
*/
|
|
39
|
+
export declare function loadFixtureCollection(provider: string, collectionName: string): Promise<FixtureCollection>;
|
|
40
|
+
//# sourceMappingURL=fixture-loader.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"fixture-loader.d.ts","sourceRoot":"","sources":["../../src/fixture-loader.ts"],"names":[],"mappings":"AAAA;;GAEG;AAIH,OAAO,KAAK,EAAE,OAAO,EAAE,YAAY,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAGnF;;GAEG;AACH,eAAO,MAAM,YAAY,QAAkC,CAAC;AAO5D;;GAEG;AACH,wBAAsB,WAAW,CAC/B,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,MAAM,EAChB,OAAO,CAAC,EAAE;IAAE,QAAQ,CAAC,EAAE,OAAO,CAAA;CAAE,GAC/B,OAAO,CAAC,OAAO,CAAC,CAgClB;AAED;;GAEG;AACH,wBAAsB,oBAAoB,CACxC,QAAQ,EAAE,MAAM,EAChB,OAAO,CAAC,EAAE;IAAE,QAAQ,CAAC,EAAE,OAAO,CAAA;CAAE,GAC/B,OAAO,CAAC,OAAO,EAAE,CAAC,CAsBpB;AAED;;GAEG;AACH,wBAAsB,YAAY,CAAC,KAAK,EAAE,YAAY,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC,CA8D1E;AAED;;GAEG;AACH,wBAAgB,iBAAiB,IAAI,IAAI,CAExC;AAED;;GAEG;AACH,wBAAgB,oBAAoB,IAAI;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,EAAE,CAAA;CAAE,CAKvE;AAED;;GAEG;AACH,wBAAsB,qBAAqB,CACzC,QAAQ,EAAE,MAAM,EAChB,cAAc,EAAE,MAAM,GACrB,OAAO,CAAC,iBAAiB,CAAC,CAW5B"}
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Type definitions for test fixtures
|
|
3
|
+
*/
|
|
4
|
+
import type { IRChatRequest, IRChatResponse, IRStreamChunk } from 'ai.matey.types';
|
|
5
|
+
/**
|
|
6
|
+
* Fixture metadata
|
|
7
|
+
*/
|
|
8
|
+
export interface FixtureMetadata {
|
|
9
|
+
/** Provider name (e.g., "openai", "anthropic") */
|
|
10
|
+
readonly provider: string;
|
|
11
|
+
/** Scenario description (e.g., "basic-chat", "streaming-with-tools") */
|
|
12
|
+
readonly scenario: string;
|
|
13
|
+
/** API version used to capture this fixture */
|
|
14
|
+
readonly apiVersion?: string;
|
|
15
|
+
/** Model used in this fixture */
|
|
16
|
+
readonly model: string;
|
|
17
|
+
/** When this fixture was captured */
|
|
18
|
+
readonly capturedAt: string;
|
|
19
|
+
/** Description of what this fixture tests */
|
|
20
|
+
readonly description?: string;
|
|
21
|
+
/** Tags for categorization */
|
|
22
|
+
readonly tags?: readonly string[];
|
|
23
|
+
}
|
|
24
|
+
/**
|
|
25
|
+
* Non-streaming fixture
|
|
26
|
+
*/
|
|
27
|
+
export interface ChatFixture {
|
|
28
|
+
readonly metadata: FixtureMetadata;
|
|
29
|
+
readonly request: IRChatRequest;
|
|
30
|
+
readonly response: IRChatResponse;
|
|
31
|
+
readonly providerRequest?: unknown;
|
|
32
|
+
readonly providerResponse?: unknown;
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* Streaming fixture
|
|
36
|
+
*/
|
|
37
|
+
export interface StreamingFixture {
|
|
38
|
+
readonly metadata: FixtureMetadata;
|
|
39
|
+
readonly request: IRChatRequest;
|
|
40
|
+
readonly chunks: readonly IRStreamChunk[];
|
|
41
|
+
readonly finalResponse?: IRChatResponse;
|
|
42
|
+
readonly providerRequest?: unknown;
|
|
43
|
+
readonly providerStreamEvents?: readonly unknown[];
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* Combined fixture type
|
|
47
|
+
*/
|
|
48
|
+
export type Fixture = ChatFixture | StreamingFixture;
|
|
49
|
+
/**
|
|
50
|
+
* Type guard for streaming fixtures
|
|
51
|
+
*/
|
|
52
|
+
export declare function isStreamingFixture(fixture: Fixture): fixture is StreamingFixture;
|
|
53
|
+
/**
|
|
54
|
+
* Type guard for chat fixtures
|
|
55
|
+
*/
|
|
56
|
+
export declare function isChatFixture(fixture: Fixture): fixture is ChatFixture;
|
|
57
|
+
/**
|
|
58
|
+
* Fixture collection (multiple fixtures grouped together)
|
|
59
|
+
*/
|
|
60
|
+
export interface FixtureCollection {
|
|
61
|
+
readonly provider: string;
|
|
62
|
+
readonly fixtures: readonly Fixture[];
|
|
63
|
+
readonly version: string;
|
|
64
|
+
readonly createdAt: string;
|
|
65
|
+
}
|
|
66
|
+
/**
|
|
67
|
+
* Fixture search criteria
|
|
68
|
+
*/
|
|
69
|
+
export interface FixtureQuery {
|
|
70
|
+
readonly provider?: string;
|
|
71
|
+
readonly scenario?: string;
|
|
72
|
+
readonly model?: string;
|
|
73
|
+
readonly tags?: readonly string[];
|
|
74
|
+
readonly streaming?: boolean;
|
|
75
|
+
}
|
|
76
|
+
//# sourceMappingURL=fixture-types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"fixture-types.d.ts","sourceRoot":"","sources":["../../src/fixture-types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAEnF;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,kDAAkD;IAClD,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAE1B,wEAAwE;IACxE,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAE1B,+CAA+C;IAC/C,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;IAE7B,iCAAiC;IACjC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IAEvB,qCAAqC;IACrC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAE5B,6CAA6C;IAC7C,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAE9B,8BAA8B;IAC9B,QAAQ,CAAC,IAAI,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;CACnC;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,QAAQ,EAAE,eAAe,CAAC;IACnC,QAAQ,CAAC,OAAO,EAAE,aAAa,CAAC;IAChC,QAAQ,CAAC,QAAQ,EAAE,cAAc,CAAC;IAClC,QAAQ,CAAC,eAAe,CAAC,EAAE,OAAO,CAAC;IACnC,QAAQ,CAAC,gBAAgB,CAAC,EAAE,OAAO,CAAC;CACrC;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,QAAQ,EAAE,eAAe,CAAC;IACnC,QAAQ,CAAC,OAAO,EAAE,aAAa,CAAC;IAChC,QAAQ,CAAC,MAAM,EAAE,SAAS,aAAa,EAAE,CAAC;IAC1C,QAAQ,CAAC,aAAa,CAAC,EAAE,cAAc,CAAC;IACxC,QAAQ,CAAC,eAAe,CAAC,EAAE,OAAO,CAAC;IACnC,QAAQ,CAAC,oBAAoB,CAAC,EAAE,SAAS,OAAO,EAAE,CAAC;CACpD;AAED;;GAEG;AACH,MAAM,MAAM,OAAO,GAAG,WAAW,GAAG,gBAAgB,CAAC;AAErD;;GAEG;AACH,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,IAAI,gBAAgB,CAEhF;AAED;;GAEG;AACH,wBAAgB,aAAa,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,IAAI,WAAW,CAEtE;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,QAAQ,EAAE,SAAS,OAAO,EAAE,CAAC;IACtC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;CAC5B;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,IAAI,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAClC,QAAQ,CAAC,SAAS,CAAC,EAAE,OAAO,CAAC;CAC9B"}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Testing utilities and fixtures
|
|
3
|
+
*/
|
|
4
|
+
export type { FixtureMetadata, ChatFixture, StreamingFixture, Fixture, FixtureCollection, FixtureQuery, } from './fixture-types.js';
|
|
5
|
+
export { isChatFixture, isStreamingFixture } from './fixture-types.js';
|
|
6
|
+
export { FIXTURES_DIR, loadFixture, loadProviderFixtures, findFixtures, clearFixtureCache, getFixtureCacheStats, loadFixtureCollection, } from './fixture-loader.js';
|
|
7
|
+
export type { CaptureConfig } from './fixture-capture.js';
|
|
8
|
+
export { captureChat, captureStream, createCaptureMiddleware, bulkCapture, } from './fixture-capture.js';
|
|
9
|
+
export { createMockFromFixture, replayStreamWithTiming, createMocksFromFixtures, validateAgainstFixture, extractRequest, extractResponse, extractChunks, collectChunksToResponse, createConfigurableMock, } from './fixture-helpers.js';
|
|
10
|
+
export { assertValidChatResponse, assertValidStreamChunk, assertValidChatRequest, assertValidMessage, assertValidMessageContent, assertResponseHasText, assertResponseHasToolUse, assertValidStreamSequence, buildChatRequest, buildMultiTurnRequest, extractTextFromResponse, extractToolUsesFromResponse, accumulateStreamText, estimateTokens, assertReasonableUsage, } from './test-helpers.js';
|
|
11
|
+
export { SeededRandom, generateTextContent, generateUserMessage, generateAssistantMessage, generateSystemMessage, generateParameters, generateChatRequest, forAll, shrinkChatRequest, propertyValidRequest, propertyMultiTurnAlternates, } from './property-testing.js';
|
|
12
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAGH,YAAY,EACV,eAAe,EACf,WAAW,EACX,gBAAgB,EAChB,OAAO,EACP,iBAAiB,EACjB,YAAY,GACb,MAAM,oBAAoB,CAAC;AAG5B,OAAO,EAAE,aAAa,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAGvE,OAAO,EACL,YAAY,EACZ,WAAW,EACX,oBAAoB,EACpB,YAAY,EACZ,iBAAiB,EACjB,oBAAoB,EACpB,qBAAqB,GACtB,MAAM,qBAAqB,CAAC;AAG7B,YAAY,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAC1D,OAAO,EACL,WAAW,EACX,aAAa,EACb,uBAAuB,EACvB,WAAW,GACZ,MAAM,sBAAsB,CAAC;AAG9B,OAAO,EACL,qBAAqB,EACrB,sBAAsB,EACtB,uBAAuB,EACvB,sBAAsB,EACtB,cAAc,EACd,eAAe,EACf,aAAa,EACb,uBAAuB,EACvB,sBAAsB,GACvB,MAAM,sBAAsB,CAAC;AAG9B,OAAO,EACL,uBAAuB,EACvB,sBAAsB,EACtB,sBAAsB,EACtB,kBAAkB,EAClB,yBAAyB,EACzB,qBAAqB,EACrB,wBAAwB,EACxB,yBAAyB,EACzB,gBAAgB,EAChB,qBAAqB,EACrB,uBAAuB,EACvB,2BAA2B,EAC3B,oBAAoB,EACpB,cAAc,EACd,qBAAqB,GACtB,MAAM,mBAAmB,CAAC;AAG3B,OAAO,EACL,YAAY,EACZ,mBAAmB,EACnB,mBAAmB,EACnB,wBAAwB,EACxB,qBAAqB,EACrB,kBAAkB,EAClB,mBAAmB,EACnB,MAAM,EACN,iBAAiB,EACjB,oBAAoB,EACpB,2BAA2B,GAC5B,MAAM,uBAAuB,CAAC"}
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Property-based testing utilities - generate random valid inputs
|
|
3
|
+
*/
|
|
4
|
+
import type { IRChatRequest, IRMessage, MessageContent, IRParameters } from 'ai.matey.types';
|
|
5
|
+
/**
|
|
6
|
+
* Random seed generator for deterministic randomness
|
|
7
|
+
*/
|
|
8
|
+
export declare class SeededRandom {
|
|
9
|
+
private seed;
|
|
10
|
+
constructor(seed?: number);
|
|
11
|
+
/**
|
|
12
|
+
* Generate next random number between 0 and 1
|
|
13
|
+
*/
|
|
14
|
+
next(): number;
|
|
15
|
+
/**
|
|
16
|
+
* Generate random integer between min and max (inclusive)
|
|
17
|
+
*/
|
|
18
|
+
nextInt(min: number, max: number): number;
|
|
19
|
+
/**
|
|
20
|
+
* Pick random element from array
|
|
21
|
+
*/
|
|
22
|
+
pick<T>(array: readonly T[]): T;
|
|
23
|
+
/**
|
|
24
|
+
* Generate random boolean
|
|
25
|
+
*/
|
|
26
|
+
nextBool(probability?: number): boolean;
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* Generate random text content
|
|
30
|
+
*/
|
|
31
|
+
export declare function generateTextContent(random: SeededRandom): MessageContent;
|
|
32
|
+
/**
|
|
33
|
+
* Generate random user message
|
|
34
|
+
*/
|
|
35
|
+
export declare function generateUserMessage(random: SeededRandom): IRMessage;
|
|
36
|
+
/**
|
|
37
|
+
* Generate random assistant message
|
|
38
|
+
*/
|
|
39
|
+
export declare function generateAssistantMessage(random: SeededRandom): IRMessage;
|
|
40
|
+
/**
|
|
41
|
+
* Generate random system message
|
|
42
|
+
*/
|
|
43
|
+
export declare function generateSystemMessage(random: SeededRandom): IRMessage;
|
|
44
|
+
/**
|
|
45
|
+
* Generate random parameters
|
|
46
|
+
*/
|
|
47
|
+
export declare function generateParameters(random: SeededRandom): IRParameters;
|
|
48
|
+
/**
|
|
49
|
+
* Generate random chat request
|
|
50
|
+
*/
|
|
51
|
+
export declare function generateChatRequest(random: SeededRandom, options?: {
|
|
52
|
+
includeSystem?: boolean;
|
|
53
|
+
multiTurn?: boolean;
|
|
54
|
+
minMessages?: number;
|
|
55
|
+
maxMessages?: number;
|
|
56
|
+
}): IRChatRequest;
|
|
57
|
+
/**
|
|
58
|
+
* Property test: run test with multiple generated inputs
|
|
59
|
+
*/
|
|
60
|
+
export declare function forAll<T>(generator: (random: SeededRandom) => T, test: (value: T) => void | Promise<void>, options?: {
|
|
61
|
+
runs?: number;
|
|
62
|
+
seed?: number;
|
|
63
|
+
}): Promise<void>;
|
|
64
|
+
/**
|
|
65
|
+
* Shrink a failing value to find minimal failing case
|
|
66
|
+
*/
|
|
67
|
+
export declare function shrinkChatRequest(request: IRChatRequest): IRChatRequest[];
|
|
68
|
+
/**
|
|
69
|
+
* Example property tests
|
|
70
|
+
*/
|
|
71
|
+
/**
|
|
72
|
+
* Property: All generated requests should be valid
|
|
73
|
+
*/
|
|
74
|
+
export declare function propertyValidRequest(): Promise<void>;
|
|
75
|
+
/**
|
|
76
|
+
* Property: Multi-turn conversations should alternate user/assistant
|
|
77
|
+
*/
|
|
78
|
+
export declare function propertyMultiTurnAlternates(): Promise<void>;
|
|
79
|
+
//# sourceMappingURL=property-testing.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"property-testing.d.ts","sourceRoot":"","sources":["../../src/property-testing.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAE,SAAS,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAE7F;;GAEG;AACH,qBAAa,YAAY;IACvB,OAAO,CAAC,IAAI,CAAS;gBAET,IAAI,GAAE,MAAmB;IAIrC;;OAEG;IACH,IAAI,IAAI,MAAM;IAKd;;OAEG;IACH,OAAO,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,MAAM;IAIzC;;OAEG;IACH,IAAI,CAAC,CAAC,EAAE,KAAK,EAAE,SAAS,CAAC,EAAE,GAAG,CAAC;IAO/B;;OAEG;IACH,QAAQ,CAAC,WAAW,GAAE,MAAY,GAAG,OAAO;CAG7C;AAED;;GAEG;AACH,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,YAAY,GAAG,cAAc,CAkBxE;AAED;;GAEG;AACH,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,YAAY,GAAG,SAAS,CAYnE;AAED;;GAEG;AACH,wBAAgB,wBAAwB,CAAC,MAAM,EAAE,YAAY,GAAG,SAAS,CAKxE;AAED;;GAEG;AACH,wBAAgB,qBAAqB,CAAC,MAAM,EAAE,YAAY,GAAG,SAAS,CAkBrE;AAED;;GAEG;AACH,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,YAAY,GAAG,YAAY,CAgBrE;AAED;;GAEG;AACH,wBAAgB,mBAAmB,CACjC,MAAM,EAAE,YAAY,EACpB,OAAO,CAAC,EAAE;IACR,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB,GACA,aAAa,CAiCf;AAED;;GAEG;AACH,wBAAsB,MAAM,CAAC,CAAC,EAC5B,SAAS,EAAE,CAAC,MAAM,EAAE,YAAY,KAAK,CAAC,EACtC,IAAI,EAAE,CAAC,KAAK,EAAE,CAAC,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EACxC,OAAO,CAAC,EAAE;IACR,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;CACf,GACA,OAAO,CAAC,IAAI,CAAC,CA+Bf;AAED;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,OAAO,EAAE,aAAa,GAAG,aAAa,EAAE,CAuCzE;AAED;;GAEG;AAEH;;GAEG;AACH,wBAAsB,oBAAoB,IAAI,OAAO,CAAC,IAAI,CAAC,CA8B1D;AAED;;GAEG;AACH,wBAAsB,2BAA2B,IAAI,OAAO,CAAC,IAAI,CAAC,CAoBjE"}
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Test helper utilities - assertions and test builders
|
|
3
|
+
*/
|
|
4
|
+
import type { IRChatRequest, IRChatResponse, IRStreamChunk, IRMessage, MessageContent } from 'ai.matey.types';
|
|
5
|
+
/**
|
|
6
|
+
* Assert that a response has the correct structure
|
|
7
|
+
*/
|
|
8
|
+
export declare function assertValidChatResponse(response: IRChatResponse): void;
|
|
9
|
+
/**
|
|
10
|
+
* Assert that a stream chunk has the correct structure
|
|
11
|
+
*/
|
|
12
|
+
export declare function assertValidStreamChunk(chunk: IRStreamChunk): void;
|
|
13
|
+
/**
|
|
14
|
+
* Assert that a request has the correct structure
|
|
15
|
+
*/
|
|
16
|
+
export declare function assertValidChatRequest(request: IRChatRequest): void;
|
|
17
|
+
/**
|
|
18
|
+
* Assert that a message has the correct structure
|
|
19
|
+
*/
|
|
20
|
+
export declare function assertValidMessage(message: IRMessage): void;
|
|
21
|
+
/**
|
|
22
|
+
* Assert that message content has the correct structure
|
|
23
|
+
*/
|
|
24
|
+
export declare function assertValidMessageContent(content: MessageContent): void;
|
|
25
|
+
/**
|
|
26
|
+
* Assert that response contains text
|
|
27
|
+
*/
|
|
28
|
+
export declare function assertResponseHasText(response: IRChatResponse): void;
|
|
29
|
+
/**
|
|
30
|
+
* Assert that response contains tool use
|
|
31
|
+
*/
|
|
32
|
+
export declare function assertResponseHasToolUse(response: IRChatResponse): void;
|
|
33
|
+
/**
|
|
34
|
+
* Assert that streaming produces expected sequence
|
|
35
|
+
*/
|
|
36
|
+
export declare function assertValidStreamSequence(stream: AsyncIterable<IRStreamChunk>): Promise<IRStreamChunk[]>;
|
|
37
|
+
/**
|
|
38
|
+
* Build a simple chat request for testing
|
|
39
|
+
*/
|
|
40
|
+
export declare function buildChatRequest(userMessage: string, options?: {
|
|
41
|
+
model?: string;
|
|
42
|
+
temperature?: number;
|
|
43
|
+
maxTokens?: number;
|
|
44
|
+
systemMessage?: string;
|
|
45
|
+
}): IRChatRequest;
|
|
46
|
+
/**
|
|
47
|
+
* Build a multi-turn chat request for testing
|
|
48
|
+
*/
|
|
49
|
+
export declare function buildMultiTurnRequest(exchanges: Array<{
|
|
50
|
+
user: string;
|
|
51
|
+
assistant: string;
|
|
52
|
+
}>, finalUserMessage: string, options?: {
|
|
53
|
+
model?: string;
|
|
54
|
+
temperature?: number;
|
|
55
|
+
maxTokens?: number;
|
|
56
|
+
}): IRChatRequest;
|
|
57
|
+
/**
|
|
58
|
+
* Extract text from response
|
|
59
|
+
*/
|
|
60
|
+
export declare function extractTextFromResponse(response: IRChatResponse): string;
|
|
61
|
+
/**
|
|
62
|
+
* Extract tool uses from response
|
|
63
|
+
*/
|
|
64
|
+
export declare function extractToolUsesFromResponse(response: IRChatResponse): Array<{
|
|
65
|
+
id: string;
|
|
66
|
+
name: string;
|
|
67
|
+
input: unknown;
|
|
68
|
+
}>;
|
|
69
|
+
/**
|
|
70
|
+
* Accumulate text from stream chunks
|
|
71
|
+
*/
|
|
72
|
+
export declare function accumulateStreamText(stream: AsyncIterable<IRStreamChunk>): Promise<string>;
|
|
73
|
+
/**
|
|
74
|
+
* Count tokens in text (rough estimate)
|
|
75
|
+
*/
|
|
76
|
+
export declare function estimateTokens(text: string): number;
|
|
77
|
+
/**
|
|
78
|
+
* Assert that usage statistics are reasonable
|
|
79
|
+
*/
|
|
80
|
+
export declare function assertReasonableUsage(response: IRChatResponse): void;
|
|
81
|
+
//# sourceMappingURL=test-helpers.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"test-helpers.d.ts","sourceRoot":"","sources":["../../src/test-helpers.ts"],"names":[],"mappings":"AAAA;;GAEG;AAGH,OAAO,KAAK,EACV,aAAa,EACb,cAAc,EACd,aAAa,EACb,SAAS,EACT,cAAc,EACf,MAAM,gBAAgB,CAAC;AAExB;;GAEG;AACH,wBAAgB,uBAAuB,CAAC,QAAQ,EAAE,cAAc,GAAG,IAAI,CAkBtE;AAED;;GAEG;AACH,wBAAgB,sBAAsB,CAAC,KAAK,EAAE,aAAa,GAAG,IAAI,CAIjE;AAED;;GAEG;AACH,wBAAgB,sBAAsB,CAAC,OAAO,EAAE,aAAa,GAAG,IAAI,CAenE;AAED;;GAEG;AACH,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,SAAS,GAAG,IAAI,CAY3D;AAED;;GAEG;AACH,wBAAgB,yBAAyB,CAAC,OAAO,EAAE,cAAc,GAAG,IAAI,CA0BvE;AAED;;GAEG;AACH,wBAAgB,qBAAqB,CAAC,QAAQ,EAAE,cAAc,GAAG,IAAI,CAUpE;AAED;;GAEG;AACH,wBAAgB,wBAAwB,CAAC,QAAQ,EAAE,cAAc,GAAG,IAAI,CAYvE;AAED;;GAEG;AACH,wBAAsB,yBAAyB,CAC7C,MAAM,EAAE,aAAa,CAAC,aAAa,CAAC,GACnC,OAAO,CAAC,aAAa,EAAE,CAAC,CAuB1B;AAED;;GAEG;AACH,wBAAgB,gBAAgB,CAC9B,WAAW,EAAE,MAAM,EACnB,OAAO,CAAC,EAAE;IACR,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB,GACA,aAAa,CA2Bf;AAED;;GAEG;AACH,wBAAgB,qBAAqB,CACnC,SAAS,EAAE,KAAK,CAAC;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,CAAC,EACrD,gBAAgB,EAAE,MAAM,EACxB,OAAO,CAAC,EAAE;IACR,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,GACA,aAAa,CA+Bf;AAED;;GAEG;AACH,wBAAgB,uBAAuB,CAAC,QAAQ,EAAE,cAAc,GAAG,MAAM,CAMxE;AAED;;GAEG;AACH,wBAAgB,2BAA2B,CAAC,QAAQ,EAAE,cAAc,GAAG,KAAK,CAAC;IAC3E,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,OAAO,CAAC;CAChB,CAAC,CAiBD;AAED;;GAEG;AACH,wBAAsB,oBAAoB,CAAC,MAAM,EAAE,aAAa,CAAC,aAAa,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,CAUhG;AAED;;GAEG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAGnD;AAED;;GAEG;AACH,wBAAgB,qBAAqB,CAAC,QAAQ,EAAE,cAAc,GAAG,IAAI,CAiBpE"}
|
package/package.json
ADDED
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "ai.matey.testing",
|
|
3
|
+
"version": "0.2.0",
|
|
4
|
+
"description": "Testing utilities for AI Matey - Universal AI Adapter System",
|
|
5
|
+
"type": "module",
|
|
6
|
+
"main": "./dist/cjs/index.js",
|
|
7
|
+
"module": "./dist/esm/index.js",
|
|
8
|
+
"types": "./dist/types/index.d.ts",
|
|
9
|
+
"exports": {
|
|
10
|
+
".": {
|
|
11
|
+
"import": {
|
|
12
|
+
"types": "./dist/types/index.d.ts",
|
|
13
|
+
"default": "./dist/esm/index.js"
|
|
14
|
+
},
|
|
15
|
+
"require": {
|
|
16
|
+
"types": "./dist/types/index.d.ts",
|
|
17
|
+
"default": "./dist/cjs/index.js"
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
},
|
|
21
|
+
"files": [
|
|
22
|
+
"dist",
|
|
23
|
+
"readme.md",
|
|
24
|
+
"CHANGELOG.md",
|
|
25
|
+
"LICENSE"
|
|
26
|
+
],
|
|
27
|
+
"scripts": {
|
|
28
|
+
"build": "npm run build:esm && npm run build:cjs && npm run build:types",
|
|
29
|
+
"build:esm": "tsc -p tsconfig.esm.json",
|
|
30
|
+
"build:cjs": "tsc -p tsconfig.cjs.json",
|
|
31
|
+
"build:types": "tsc -p tsconfig.types.json",
|
|
32
|
+
"clean": "rm -rf dist",
|
|
33
|
+
"typecheck": "tsc --noEmit",
|
|
34
|
+
"lint": "eslint src --ext .ts",
|
|
35
|
+
"lint:fix": "eslint src --ext .ts --fix",
|
|
36
|
+
"test": "vitest run",
|
|
37
|
+
"test:watch": "vitest"
|
|
38
|
+
},
|
|
39
|
+
"dependencies": {
|
|
40
|
+
"ai.matey.types": "*",
|
|
41
|
+
"ai.matey.utils": "*"
|
|
42
|
+
},
|
|
43
|
+
"devDependencies": {
|
|
44
|
+
"typescript": "^5.9.3",
|
|
45
|
+
"vitest": "^3.2.4"
|
|
46
|
+
},
|
|
47
|
+
"keywords": [
|
|
48
|
+
"ai",
|
|
49
|
+
"llm",
|
|
50
|
+
"testing",
|
|
51
|
+
"fixtures",
|
|
52
|
+
"ai-matey"
|
|
53
|
+
],
|
|
54
|
+
"author": "AI Matey",
|
|
55
|
+
"license": "MIT",
|
|
56
|
+
"homepage": "https://github.com/johnhenry/ai.matey#readme",
|
|
57
|
+
"bugs": {
|
|
58
|
+
"url": "https://github.com/johnhenry/ai.matey/issues"
|
|
59
|
+
},
|
|
60
|
+
"repository": {
|
|
61
|
+
"type": "git",
|
|
62
|
+
"url": "git+https://github.com/johnhenry/ai.matey.git",
|
|
63
|
+
"directory": "packages/ai.matey.testing"
|
|
64
|
+
},
|
|
65
|
+
"engines": {
|
|
66
|
+
"node": ">=18.0.0"
|
|
67
|
+
}
|
|
68
|
+
}
|
package/readme.md
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
# ai.matey.testing
|
|
2
|
+
|
|
3
|
+
Testing utilities, mocks, and fixtures for ai.matey
|
|
4
|
+
|
|
5
|
+
Part of the [ai.matey](https://github.com/johnhenry/ai.matey) monorepo.
|
|
6
|
+
|
|
7
|
+
## Installation
|
|
8
|
+
|
|
9
|
+
```bash
|
|
10
|
+
npm install ai.matey.testing
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
## Exports
|
|
14
|
+
|
|
15
|
+
- `MockBackendAdapter`
|
|
16
|
+
- `createMockResponse`
|
|
17
|
+
- `assertChatRequest`
|
|
18
|
+
|
|
19
|
+
## Usage
|
|
20
|
+
|
|
21
|
+
```typescript
|
|
22
|
+
import { MockBackendAdapter, createMockResponse, assertChatRequest } from 'ai.matey.testing';
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
## API Reference
|
|
26
|
+
|
|
27
|
+
See the TypeScript definitions for detailed API documentation.
|
|
28
|
+
|
|
29
|
+
## License
|
|
30
|
+
|
|
31
|
+
MIT - see [LICENSE](./LICENSE) for details.
|