jevkit-vitest 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +84 -0
- package/dist/assertions.d.ts +23 -0
- package/dist/assertions.d.ts.map +1 -0
- package/dist/assertions.js +65 -0
- package/dist/assertions.js.map +1 -0
- package/dist/cassette.d.ts +82 -0
- package/dist/cassette.d.ts.map +1 -0
- package/dist/cassette.js +170 -0
- package/dist/cassette.js.map +1 -0
- package/dist/fixture.d.ts +32 -0
- package/dist/fixture.d.ts.map +1 -0
- package/dist/fixture.js +44 -0
- package/dist/fixture.js.map +1 -0
- package/dist/index.d.ts +11 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +11 -0
- package/dist/index.js.map +1 -0
- package/package.json +41 -0
package/README.md
ADDED
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
# jevkit-vitest
|
|
2
|
+
|
|
3
|
+
Record and replay TypeSafe Jev requests in vitest.
|
|
4
|
+
|
|
5
|
+
A model call in a test is slow, costs money, needs a key in CI, and can change
|
|
6
|
+
its answer under you when the alias moves. The fix is the one VCR established for
|
|
7
|
+
HTTP: record real responses once, replay them forever, re-record on purpose.
|
|
8
|
+
|
|
9
|
+
> Unofficial and unaffiliated with TypeSafe.
|
|
10
|
+
|
|
11
|
+
```bash
|
|
12
|
+
npm install -D jevkit-vitest
|
|
13
|
+
```
|
|
14
|
+
|
|
15
|
+
## Use
|
|
16
|
+
|
|
17
|
+
```ts
|
|
18
|
+
import { expect, it } from "vitest";
|
|
19
|
+
import { cassetteFor } from "jevkit-vitest";
|
|
20
|
+
|
|
21
|
+
it("routes billing questions", async () => {
|
|
22
|
+
const cassette = cassetteFor("routes billing questions", { systemOne });
|
|
23
|
+
const response = await cassette.systemOne(
|
|
24
|
+
"I was charged twice for the same order",
|
|
25
|
+
{ team: { type: "choice", instructions: "Which team should handle this",
|
|
26
|
+
criteria: { billing: "Payment issues", technical: "Bugs",
|
|
27
|
+
unknown: "None apply" } } },
|
|
28
|
+
);
|
|
29
|
+
expect(response.answers.team.choice).toBe("billing");
|
|
30
|
+
});
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
Record the first time, then never again:
|
|
34
|
+
|
|
35
|
+
```bash
|
|
36
|
+
JEV_RECORD=1 vitest # record anything missing
|
|
37
|
+
vitest # replay only; a miss is a failure
|
|
38
|
+
JEV_RERECORD=1 vitest # replace every recording
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
To record you need a live client. Pass its `systemOne` when building the
|
|
42
|
+
cassette:
|
|
43
|
+
|
|
44
|
+
```ts
|
|
45
|
+
import { TypeSafeClient } from "@typesafe-ai/sdk";
|
|
46
|
+
|
|
47
|
+
const client = new TypeSafeClient();
|
|
48
|
+
const systemOne = (state, questions, options) =>
|
|
49
|
+
client.systemOne({ state, questions, ...options });
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
Replay-only runs need no client and no API key, which is the point: CI stays
|
|
53
|
+
green without a secret.
|
|
54
|
+
|
|
55
|
+
## Assertions that explain themselves
|
|
56
|
+
|
|
57
|
+
```ts
|
|
58
|
+
import { assertAnswer, assertConfident } from "jevkit-vitest";
|
|
59
|
+
|
|
60
|
+
assertAnswer(response.answers.team, "billing", { minProbability: 0.6 });
|
|
61
|
+
assertConfident(response.answers.urgency, 0.8);
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
On failure these print the whole distribution, because a 0.51/0.49 split and a
|
|
65
|
+
0.99/0.01 split are different bugs and `toBe` cannot tell them apart.
|
|
66
|
+
|
|
67
|
+
`assert_confident` uses the API's confidence for Choice and Score. A Noul carries
|
|
68
|
+
none, so its distance from 0.5 is used and the message says which it used.
|
|
69
|
+
|
|
70
|
+
## Cassettes are golden sets
|
|
71
|
+
|
|
72
|
+
A cassette is a `.jevl` file, the same format `jevkit-drift`, `jevkit-bench` and
|
|
73
|
+
`jevkit-calibrate` read. So the recordings your tests already make are a golden
|
|
74
|
+
set you can replay against the next model version:
|
|
75
|
+
|
|
76
|
+
```bash
|
|
77
|
+
npx jevkit-drift test/cassettes/routes_billing_questions.jevl candidate.jevl
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
That is the whole reason the format was defined before any of these packages.
|
|
81
|
+
|
|
82
|
+
## License
|
|
83
|
+
|
|
84
|
+
MIT
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Assertions over jev answers, with failure messages that say enough.
|
|
3
|
+
*
|
|
4
|
+
* A bare `expect(answer.choice).toBe("billing")` tells you nothing about how
|
|
5
|
+
* close the call was. These assertions print the distribution on failure,
|
|
6
|
+
* because a 0.51/0.49 split and a 0.99/0.01 split are different bugs.
|
|
7
|
+
*/
|
|
8
|
+
export declare function describeAnswer(qid: string, raw: unknown): string;
|
|
9
|
+
export interface AssertAnswerOptions {
|
|
10
|
+
questionId?: string;
|
|
11
|
+
minConfidence?: number;
|
|
12
|
+
minProbability?: number;
|
|
13
|
+
}
|
|
14
|
+
/** Assert an answer selected `expected`, optionally with enough certainty. */
|
|
15
|
+
export declare function assertAnswer(raw: unknown, expected: unknown, options?: AssertAnswerOptions): void;
|
|
16
|
+
/**
|
|
17
|
+
* Assert an answer is decisive, without caring which way it went.
|
|
18
|
+
*
|
|
19
|
+
* Uses the API's confidence for Choice and Score. A Noul has none, so its
|
|
20
|
+
* distance from 0.5 is used and the message says so.
|
|
21
|
+
*/
|
|
22
|
+
export declare function assertConfident(raw: unknown, minimum: number, questionId?: string): void;
|
|
23
|
+
//# sourceMappingURL=assertions.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"assertions.d.ts","sourceRoot":"","sources":["../src/assertions.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAIH,wBAAgB,cAAc,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,OAAO,GAAG,MAAM,CAUhE;AAED,MAAM,WAAW,mBAAmB;IAClC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,8EAA8E;AAC9E,wBAAgB,YAAY,CAC1B,GAAG,EAAE,OAAO,EACZ,QAAQ,EAAE,OAAO,EACjB,OAAO,GAAE,mBAAwB,GAChC,IAAI,CAqCN;AAED;;;;;GAKG;AACH,wBAAgB,eAAe,CAAC,GAAG,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,UAAU,SAAW,GAAG,IAAI,CAW1F"}
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Assertions over jev answers, with failure messages that say enough.
|
|
3
|
+
*
|
|
4
|
+
* A bare `expect(answer.choice).toBe("billing")` tells you nothing about how
|
|
5
|
+
* close the call was. These assertions print the distribution on failure,
|
|
6
|
+
* because a 0.51/0.49 split and a 0.99/0.01 split are different bugs.
|
|
7
|
+
*/
|
|
8
|
+
import { parseAnswer } from "jevkit-core";
|
|
9
|
+
export function describeAnswer(qid, raw) {
|
|
10
|
+
const answer = parseAnswer(qid, raw);
|
|
11
|
+
const probs = Object.entries(answer.probabilities)
|
|
12
|
+
.sort((a, b) => b[1] - a[1])
|
|
13
|
+
.map(([k, v]) => `${k}=${v.toFixed(3)}`)
|
|
14
|
+
.join(", ");
|
|
15
|
+
const parts = [`predicted=${JSON.stringify(answer.predicted())}`];
|
|
16
|
+
if (answer.confidence !== null)
|
|
17
|
+
parts.push(`confidence=${answer.confidence.toFixed(3)}`);
|
|
18
|
+
if (answer.score !== null)
|
|
19
|
+
parts.push(`score=${answer.score.toFixed(3)}`);
|
|
20
|
+
return `${qid}: ${parts.join(", ")}\n probabilities: ${probs}`;
|
|
21
|
+
}
|
|
22
|
+
/** Assert an answer selected `expected`, optionally with enough certainty. */
|
|
23
|
+
export function assertAnswer(raw, expected, options = {}) {
|
|
24
|
+
const qid = options.questionId ?? "answer";
|
|
25
|
+
const answer = parseAnswer(qid, raw);
|
|
26
|
+
if (!answer.isCorrect(expected)) {
|
|
27
|
+
throw new Error(`expected ${JSON.stringify(expected)} but got ${JSON.stringify(answer.predicted())}\n ` +
|
|
28
|
+
describeAnswer(qid, raw));
|
|
29
|
+
}
|
|
30
|
+
if (options.minProbability !== undefined) {
|
|
31
|
+
const actual = answer.probabilityOf(expected);
|
|
32
|
+
if (actual < options.minProbability) {
|
|
33
|
+
throw new Error(`${JSON.stringify(expected)} was selected but carried only ${actual.toFixed(3)} ` +
|
|
34
|
+
`probability, below the required ${options.minProbability.toFixed(3)}\n ` +
|
|
35
|
+
describeAnswer(qid, raw));
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
if (options.minConfidence !== undefined) {
|
|
39
|
+
if (answer.confidence === null) {
|
|
40
|
+
throw new Error(`minConfidence was given but a ${answer.type} answer carries no confidence. ` +
|
|
41
|
+
`Use minProbability instead.`);
|
|
42
|
+
}
|
|
43
|
+
if (answer.confidence < options.minConfidence) {
|
|
44
|
+
throw new Error(`${JSON.stringify(expected)} was selected but confidence was ` +
|
|
45
|
+
`${answer.confidence.toFixed(3)}, below the required ` +
|
|
46
|
+
`${options.minConfidence.toFixed(3)}\n ` + describeAnswer(qid, raw));
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
/**
|
|
51
|
+
* Assert an answer is decisive, without caring which way it went.
|
|
52
|
+
*
|
|
53
|
+
* Uses the API's confidence for Choice and Score. A Noul has none, so its
|
|
54
|
+
* distance from 0.5 is used and the message says so.
|
|
55
|
+
*/
|
|
56
|
+
export function assertConfident(raw, minimum, questionId = "answer") {
|
|
57
|
+
const answer = parseAnswer(questionId, raw);
|
|
58
|
+
const value = answer.decisiveness;
|
|
59
|
+
if (value < minimum) {
|
|
60
|
+
const quantity = answer.type === "noul" ? "decisiveness (|noul - 0.5| * 2)" : "confidence";
|
|
61
|
+
throw new Error(`${quantity} was ${value.toFixed(3)}, below the required ${minimum.toFixed(3)}\n ` +
|
|
62
|
+
describeAnswer(questionId, raw));
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
//# sourceMappingURL=assertions.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"assertions.js","sourceRoot":"","sources":["../src/assertions.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAE1C,MAAM,UAAU,cAAc,CAAC,GAAW,EAAE,GAAY;IACtD,MAAM,MAAM,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;IACrC,MAAM,KAAK,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,aAAa,CAAC;SAC/C,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;SAC3B,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;SACvC,IAAI,CAAC,IAAI,CAAC,CAAC;IACd,MAAM,KAAK,GAAG,CAAC,aAAa,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC,EAAE,CAAC,CAAC;IAClE,IAAI,MAAM,CAAC,UAAU,KAAK,IAAI;QAAE,KAAK,CAAC,IAAI,CAAC,cAAc,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IACzF,IAAI,MAAM,CAAC,KAAK,KAAK,IAAI;QAAE,KAAK,CAAC,IAAI,CAAC,SAAS,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IAC1E,OAAO,GAAG,GAAG,KAAK,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,wBAAwB,KAAK,EAAE,CAAC;AACpE,CAAC;AAQD,8EAA8E;AAC9E,MAAM,UAAU,YAAY,CAC1B,GAAY,EACZ,QAAiB,EACjB,UAA+B,EAAE;IAEjC,MAAM,GAAG,GAAG,OAAO,CAAC,UAAU,IAAI,QAAQ,CAAC;IAC3C,MAAM,MAAM,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;IAErC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,EAAE,CAAC;QAChC,MAAM,IAAI,KAAK,CACb,YAAY,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,YAAY,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC,MAAM;YACtF,cAAc,CAAC,GAAG,EAAE,GAAG,CAAC,CAC3B,CAAC;IACJ,CAAC;IAED,IAAI,OAAO,CAAC,cAAc,KAAK,SAAS,EAAE,CAAC;QACzC,MAAM,MAAM,GAAG,MAAM,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC;QAC9C,IAAI,MAAM,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC;YACpC,MAAM,IAAI,KAAK,CACb,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,kCAAkC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG;gBAC/E,mCAAmC,OAAO,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM;gBAC1E,cAAc,CAAC,GAAG,EAAE,GAAG,CAAC,CAC3B,CAAC;QACJ,CAAC;IACH,CAAC;IAED,IAAI,OAAO,CAAC,aAAa,KAAK,SAAS,EAAE,CAAC;QACxC,IAAI,MAAM,CAAC,UAAU,KAAK,IAAI,EAAE,CAAC;YAC/B,MAAM,IAAI,KAAK,CACb,iCAAiC,MAAM,CAAC,IAAI,iCAAiC;gBAC3E,6BAA6B,CAChC,CAAC;QACJ,CAAC;QACD,IAAI,MAAM,CAAC,UAAU,GAAG,OAAO,CAAC,aAAa,EAAE,CAAC;YAC9C,MAAM,IAAI,KAAK,CACb,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,mCAAmC;gBAC5D,GAAG,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,uBAAuB;gBACtD,GAAG,OAAO,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,GAAG,cAAc,CAAC,GAAG,EAAE,GAAG,CAAC,CACvE,CAAC;QACJ,CAAC;IACH,CAAC;AACH,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,eAAe,CAAC,GAAY,EAAE,OAAe,EAAE,UAAU,GAAG,QAAQ;IAClF,MAAM,MAAM,GAAG,WAAW,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC;IAC5C,MAAM,KAAK,GAAG,MAAM,CAAC,YAAY,CAAC;IAClC,IAAI,KAAK,GAAG,OAAO,EAAE,CAAC;QACpB,MAAM,QAAQ,GACZ,MAAM,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,iCAAiC,CAAC,CAAC,CAAC,YAAY,CAAC;QAC5E,MAAM,IAAI,KAAK,CACb,GAAG,QAAQ,QAAQ,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,wBAAwB,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM;YACjF,cAAc,CAAC,UAAU,EAAE,GAAG,CAAC,CAClC,CAAC;IACJ,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Record and replay jev requests so tests do not hit the API.
|
|
3
|
+
*
|
|
4
|
+
* A model call in a test is slow, costs money, needs a key in CI, and can
|
|
5
|
+
* change its answer under you when the alias moves. The fix is the one VCR
|
|
6
|
+
* established for HTTP: record real responses once, replay them forever,
|
|
7
|
+
* re-record on purpose.
|
|
8
|
+
*
|
|
9
|
+
* A cassette is a `.jevl` file, the same format `drift`, `bench` and
|
|
10
|
+
* `calibrate` read, so a recording made by your test suite is also a golden set
|
|
11
|
+
* you can replay against the next model version.
|
|
12
|
+
*/
|
|
13
|
+
import { Record as JevlRecord } from "jevkit-core";
|
|
14
|
+
export type Mode = "replay" | "record" | "auto" | "passthrough";
|
|
15
|
+
/** A request was not on the cassette and the mode forbids recording. */
|
|
16
|
+
export declare class CassetteMiss extends Error {
|
|
17
|
+
constructor(message: string);
|
|
18
|
+
}
|
|
19
|
+
export type SystemOneCallable = (state: unknown, questions: Record<string, unknown>, options?: Record<string, unknown>) => Promise<unknown> | unknown;
|
|
20
|
+
/** What a cassette hands back in place of a live API response. */
|
|
21
|
+
export declare class ReplayedResponse {
|
|
22
|
+
readonly model: string;
|
|
23
|
+
readonly answers: Record<string, Record<string, unknown>>;
|
|
24
|
+
readonly usage: Record<string, unknown> | null;
|
|
25
|
+
readonly id: string;
|
|
26
|
+
constructor(record: JevlRecord);
|
|
27
|
+
}
|
|
28
|
+
export interface CassetteOptions {
|
|
29
|
+
systemOne?: SystemOneCallable;
|
|
30
|
+
mode?: Mode;
|
|
31
|
+
model?: string;
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* A recorded set of jev requests, replayed by request digest.
|
|
35
|
+
*
|
|
36
|
+
* `mode` controls what happens on a miss:
|
|
37
|
+
*
|
|
38
|
+
* - `replay` throw `CassetteMiss`. The right default for CI.
|
|
39
|
+
* - `auto` replay a hit, call through and append on a miss.
|
|
40
|
+
* - `record` always call through and append, ignoring existing entries.
|
|
41
|
+
* - `passthrough` never touch the cassette.
|
|
42
|
+
*/
|
|
43
|
+
export declare class Cassette {
|
|
44
|
+
readonly path: string;
|
|
45
|
+
readonly mode: Mode;
|
|
46
|
+
readonly model: string;
|
|
47
|
+
readonly played: string[];
|
|
48
|
+
readonly recorded: string[];
|
|
49
|
+
private readonly systemOneImpl;
|
|
50
|
+
private entries;
|
|
51
|
+
constructor(path: string, options?: CassetteOptions);
|
|
52
|
+
/**
|
|
53
|
+
* Index the file by *requested*-model digest.
|
|
54
|
+
*
|
|
55
|
+
* A record stores the model that actually answered, which is what the format
|
|
56
|
+
* requires: `jev-latest` is an alias and the response says `jev-1.13.0`. But
|
|
57
|
+
* a test asks for the alias, so indexing on the answering model would miss
|
|
58
|
+
* every lookup. The requested model is kept in `meta` at record time and used
|
|
59
|
+
* to rebuild the index here.
|
|
60
|
+
*
|
|
61
|
+
* Later entries win, so re-recording appends rather than requiring a rewrite.
|
|
62
|
+
*/
|
|
63
|
+
private load;
|
|
64
|
+
get size(): number;
|
|
65
|
+
/**
|
|
66
|
+
* Entries on the cassette that no test asked for.
|
|
67
|
+
*
|
|
68
|
+
* Usually means a test was deleted or a question reworded, leaving a stale
|
|
69
|
+
* recording that will quietly rot.
|
|
70
|
+
*/
|
|
71
|
+
get unplayed(): string[];
|
|
72
|
+
contains(state: unknown, questions: Record<string, unknown>, model?: string): boolean;
|
|
73
|
+
/**
|
|
74
|
+
* Drop-in for a client's `systemOne`.
|
|
75
|
+
*
|
|
76
|
+
* Always returns a promise, so a cassette can stand in for an async client
|
|
77
|
+
* without the caller knowing whether the answer came from disk or the wire.
|
|
78
|
+
*/
|
|
79
|
+
systemOne(state: unknown, questions: Record<string, unknown>, options?: Record<string, unknown>): Promise<unknown>;
|
|
80
|
+
private callThrough;
|
|
81
|
+
}
|
|
82
|
+
//# sourceMappingURL=cassette.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cassette.d.ts","sourceRoot":"","sources":["../src/cassette.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAIH,OAAO,EAAE,MAAM,IAAI,UAAU,EAAuC,MAAM,aAAa,CAAC;AAExF,MAAM,MAAM,IAAI,GAAG,QAAQ,GAAG,QAAQ,GAAG,MAAM,GAAG,aAAa,CAAC;AAIhE,wEAAwE;AACxE,qBAAa,YAAa,SAAQ,KAAK;gBACzB,OAAO,EAAE,MAAM;CAI5B;AAED,MAAM,MAAM,iBAAiB,GAAG,CAC9B,KAAK,EAAE,OAAO,EACd,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAClC,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAC9B,OAAO,CAAC,OAAO,CAAC,GAAG,OAAO,CAAC;AAyBhC,kEAAkE;AAClE,qBAAa,gBAAgB;IAC3B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;IAC1D,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;IAC/C,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;gBAER,MAAM,EAAE,UAAU;CAM/B;AAED,MAAM,WAAW,eAAe;IAC9B,SAAS,CAAC,EAAE,iBAAiB,CAAC;IAC9B,IAAI,CAAC,EAAE,IAAI,CAAC;IACZ,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;;;;;;;;GASG;AACH,qBAAa,QAAQ;IASP,QAAQ,CAAC,IAAI,EAAE,MAAM;IARjC,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC;IACpB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,MAAM,EAAE,MAAM,EAAE,CAAM;IAC/B,QAAQ,CAAC,QAAQ,EAAE,MAAM,EAAE,CAAM;IAEjC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAgC;IAC9D,OAAO,CAAC,OAAO,CAA0B;gBAEpB,IAAI,EAAE,MAAM,EAAE,OAAO,GAAE,eAAoB;IAWhE;;;;;;;;;;OAUG;IACH,OAAO,CAAC,IAAI;IASZ,IAAI,IAAI,IAAI,MAAM,CAEjB;IAED;;;;;OAKG;IACH,IAAI,QAAQ,IAAI,MAAM,EAAE,CAEvB;IAED,QAAQ,CAAC,KAAK,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,OAAO;IAIrF;;;;;OAKG;IACG,SAAS,CACb,KAAK,EAAE,OAAO,EACd,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAClC,OAAO,GAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAM,GACpC,OAAO,CAAC,OAAO,CAAC;YA6CL,WAAW;CAa1B"}
|
package/dist/cassette.js
ADDED
|
@@ -0,0 +1,170 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Record and replay jev requests so tests do not hit the API.
|
|
3
|
+
*
|
|
4
|
+
* A model call in a test is slow, costs money, needs a key in CI, and can
|
|
5
|
+
* change its answer under you when the alias moves. The fix is the one VCR
|
|
6
|
+
* established for HTTP: record real responses once, replay them forever,
|
|
7
|
+
* re-record on purpose.
|
|
8
|
+
*
|
|
9
|
+
* A cassette is a `.jevl` file, the same format `drift`, `bench` and
|
|
10
|
+
* `calibrate` read, so a recording made by your test suite is also a golden set
|
|
11
|
+
* you can replay against the next model version.
|
|
12
|
+
*/
|
|
13
|
+
import { existsSync } from "node:fs";
|
|
14
|
+
import { Record as JevlRecord, appendRecord, readRecords, recordId } from "jevkit-core";
|
|
15
|
+
const MODES = ["replay", "record", "auto", "passthrough"];
|
|
16
|
+
/** A request was not on the cassette and the mode forbids recording. */
|
|
17
|
+
export class CassetteMiss extends Error {
|
|
18
|
+
constructor(message) {
|
|
19
|
+
super(message);
|
|
20
|
+
this.name = "CassetteMiss";
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
function plain(value) {
|
|
24
|
+
if (value === null || value === undefined || typeof value !== "object")
|
|
25
|
+
return value;
|
|
26
|
+
if (Array.isArray(value))
|
|
27
|
+
return value;
|
|
28
|
+
return { ...value };
|
|
29
|
+
}
|
|
30
|
+
function responseToParts(response) {
|
|
31
|
+
const obj = (response ?? {});
|
|
32
|
+
const answers = (obj["answers"] ?? {});
|
|
33
|
+
const out = {};
|
|
34
|
+
for (const [qid, a] of Object.entries(answers))
|
|
35
|
+
out[qid] = plain(a);
|
|
36
|
+
const usage = obj["usage"];
|
|
37
|
+
return {
|
|
38
|
+
model: String(obj["model"] ?? ""),
|
|
39
|
+
answers: out,
|
|
40
|
+
usage: usage === undefined || usage === null ? null : plain(usage),
|
|
41
|
+
};
|
|
42
|
+
}
|
|
43
|
+
/** What a cassette hands back in place of a live API response. */
|
|
44
|
+
export class ReplayedResponse {
|
|
45
|
+
model;
|
|
46
|
+
answers;
|
|
47
|
+
usage;
|
|
48
|
+
id;
|
|
49
|
+
constructor(record) {
|
|
50
|
+
this.model = record.model;
|
|
51
|
+
this.answers = record.answers;
|
|
52
|
+
this.usage = record.usage;
|
|
53
|
+
this.id = record.id;
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
/**
|
|
57
|
+
* A recorded set of jev requests, replayed by request digest.
|
|
58
|
+
*
|
|
59
|
+
* `mode` controls what happens on a miss:
|
|
60
|
+
*
|
|
61
|
+
* - `replay` throw `CassetteMiss`. The right default for CI.
|
|
62
|
+
* - `auto` replay a hit, call through and append on a miss.
|
|
63
|
+
* - `record` always call through and append, ignoring existing entries.
|
|
64
|
+
* - `passthrough` never touch the cassette.
|
|
65
|
+
*/
|
|
66
|
+
export class Cassette {
|
|
67
|
+
path;
|
|
68
|
+
mode;
|
|
69
|
+
model;
|
|
70
|
+
played = [];
|
|
71
|
+
recorded = [];
|
|
72
|
+
systemOneImpl;
|
|
73
|
+
entries;
|
|
74
|
+
constructor(path, options = {}) {
|
|
75
|
+
this.path = path;
|
|
76
|
+
this.mode = options.mode ?? "replay";
|
|
77
|
+
if (!MODES.includes(this.mode)) {
|
|
78
|
+
throw new RangeError(`mode must be one of ${MODES.join(", ")}, got "${this.mode}"`);
|
|
79
|
+
}
|
|
80
|
+
this.model = options.model ?? "jev-latest";
|
|
81
|
+
this.systemOneImpl = options.systemOne;
|
|
82
|
+
this.entries =
|
|
83
|
+
existsSync(this.path) && this.mode !== "record" ? this.load() : new Map();
|
|
84
|
+
}
|
|
85
|
+
/**
|
|
86
|
+
* Index the file by *requested*-model digest.
|
|
87
|
+
*
|
|
88
|
+
* A record stores the model that actually answered, which is what the format
|
|
89
|
+
* requires: `jev-latest` is an alias and the response says `jev-1.13.0`. But
|
|
90
|
+
* a test asks for the alias, so indexing on the answering model would miss
|
|
91
|
+
* every lookup. The requested model is kept in `meta` at record time and used
|
|
92
|
+
* to rebuild the index here.
|
|
93
|
+
*
|
|
94
|
+
* Later entries win, so re-recording appends rather than requiring a rewrite.
|
|
95
|
+
*/
|
|
96
|
+
load() {
|
|
97
|
+
const entries = new Map();
|
|
98
|
+
for (const record of readRecords(this.path)) {
|
|
99
|
+
const requested = String(record.meta["requested_model"] ?? record.model);
|
|
100
|
+
entries.set(recordId(requested, record.state, record.questions), record);
|
|
101
|
+
}
|
|
102
|
+
return entries;
|
|
103
|
+
}
|
|
104
|
+
get size() {
|
|
105
|
+
return this.entries.size;
|
|
106
|
+
}
|
|
107
|
+
/**
|
|
108
|
+
* Entries on the cassette that no test asked for.
|
|
109
|
+
*
|
|
110
|
+
* Usually means a test was deleted or a question reworded, leaving a stale
|
|
111
|
+
* recording that will quietly rot.
|
|
112
|
+
*/
|
|
113
|
+
get unplayed() {
|
|
114
|
+
return [...this.entries.keys()].filter((k) => !this.played.includes(k)).sort();
|
|
115
|
+
}
|
|
116
|
+
contains(state, questions, model) {
|
|
117
|
+
return this.entries.has(recordId(model ?? this.model, state, questions));
|
|
118
|
+
}
|
|
119
|
+
/**
|
|
120
|
+
* Drop-in for a client's `systemOne`.
|
|
121
|
+
*
|
|
122
|
+
* Always returns a promise, so a cassette can stand in for an async client
|
|
123
|
+
* without the caller knowing whether the answer came from disk or the wire.
|
|
124
|
+
*/
|
|
125
|
+
async systemOne(state, questions, options = {}) {
|
|
126
|
+
const { model: requested, ...rest } = options;
|
|
127
|
+
const model = String(requested ?? this.model);
|
|
128
|
+
if (this.mode === "passthrough") {
|
|
129
|
+
return this.callThrough(state, questions, { ...rest, model });
|
|
130
|
+
}
|
|
131
|
+
const key = recordId(model, state, questions);
|
|
132
|
+
if (this.mode !== "record") {
|
|
133
|
+
const hit = this.entries.get(key);
|
|
134
|
+
if (hit) {
|
|
135
|
+
this.played.push(key);
|
|
136
|
+
return new ReplayedResponse(hit);
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
if (this.mode === "replay") {
|
|
140
|
+
throw new CassetteMiss(`no recording for this request on ${this.path}.\n` +
|
|
141
|
+
` digest: ${key}\n` +
|
|
142
|
+
` Re-run with mode "auto" (or JEV_RECORD=1) to record it, and commit the ` +
|
|
143
|
+
`updated cassette.`);
|
|
144
|
+
}
|
|
145
|
+
const response = await this.callThrough(state, questions, { ...rest, model });
|
|
146
|
+
const { model: actualModel, answers, usage } = responseToParts(response);
|
|
147
|
+
const record = new JevlRecord({
|
|
148
|
+
model: actualModel || model,
|
|
149
|
+
state,
|
|
150
|
+
questions,
|
|
151
|
+
answers,
|
|
152
|
+
usage,
|
|
153
|
+
meta: { requested_model: model },
|
|
154
|
+
});
|
|
155
|
+
appendRecord(this.path, record);
|
|
156
|
+
this.entries.set(key, record);
|
|
157
|
+
this.recorded.push(key);
|
|
158
|
+
// The live response is returned rather than the record: recording must not
|
|
159
|
+
// change what the code under test sees.
|
|
160
|
+
return response;
|
|
161
|
+
}
|
|
162
|
+
async callThrough(state, questions, options) {
|
|
163
|
+
if (!this.systemOneImpl) {
|
|
164
|
+
throw new CassetteMiss(`cassette ${this.path} is in mode "${this.mode}" and needs to call the API, but ` +
|
|
165
|
+
`no client was supplied. Pass systemOne when constructing it.`);
|
|
166
|
+
}
|
|
167
|
+
return this.systemOneImpl(state, questions, options);
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
//# sourceMappingURL=cassette.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cassette.js","sourceRoot":"","sources":["../src/cassette.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAErC,OAAO,EAAE,MAAM,IAAI,UAAU,EAAE,YAAY,EAAE,WAAW,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAIxF,MAAM,KAAK,GAAoB,CAAC,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,aAAa,CAAC,CAAC;AAE3E,wEAAwE;AACxE,MAAM,OAAO,YAAa,SAAQ,KAAK;IACrC,YAAY,OAAe;QACzB,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,cAAc,CAAC;IAC7B,CAAC;CACF;AAQD,SAAS,KAAK,CAAC,KAAc;IAC3B,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,SAAS,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC;IACrF,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IACvC,OAAO,EAAE,GAAI,KAAiC,EAAE,CAAC;AACnD,CAAC;AAED,SAAS,eAAe,CAAC,QAAiB;IAKxC,MAAM,GAAG,GAAG,CAAC,QAAQ,IAAI,EAAE,CAA4B,CAAC;IACxD,MAAM,OAAO,GAAG,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,EAAE,CAA4B,CAAC;IAClE,MAAM,GAAG,GAA4B,EAAE,CAAC;IACxC,KAAK,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC;QAAE,GAAG,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;IACpE,MAAM,KAAK,GAAG,GAAG,CAAC,OAAO,CAAC,CAAC;IAC3B,OAAO;QACL,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;QACjC,OAAO,EAAE,GAAG;QACZ,KAAK,EAAE,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAE,KAAK,CAAC,KAAK,CAA6B;KAChG,CAAC;AACJ,CAAC;AAED,kEAAkE;AAClE,MAAM,OAAO,gBAAgB;IAClB,KAAK,CAAS;IACd,OAAO,CAA0C;IACjD,KAAK,CAAiC;IACtC,EAAE,CAAS;IAEpB,YAAY,MAAkB;QAC5B,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC;QAC1B,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAkD,CAAC;QACzE,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC;QAC1B,IAAI,CAAC,EAAE,GAAG,MAAM,CAAC,EAAE,CAAC;IACtB,CAAC;CACF;AAQD;;;;;;;;;GASG;AACH,MAAM,OAAO,QAAQ;IASE;IARZ,IAAI,CAAO;IACX,KAAK,CAAS;IACd,MAAM,GAAa,EAAE,CAAC;IACtB,QAAQ,GAAa,EAAE,CAAC;IAEhB,aAAa,CAAgC;IACtD,OAAO,CAA0B;IAEzC,YAAqB,IAAY,EAAE,UAA2B,EAAE;QAA3C,SAAI,GAAJ,IAAI,CAAQ;QAC/B,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,QAAQ,CAAC;QACrC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YAC/B,MAAM,IAAI,UAAU,CAAC,uBAAuB,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC;QACtF,CAAC;QACD,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,YAAY,CAAC;QAC3C,IAAI,CAAC,aAAa,GAAG,OAAO,CAAC,SAAS,CAAC;QACvC,IAAI,CAAC,OAAO;YACV,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,GAAG,EAAE,CAAC;IAC9E,CAAC;IAED;;;;;;;;;;OAUG;IACK,IAAI;QACV,MAAM,OAAO,GAAG,IAAI,GAAG,EAAsB,CAAC;QAC9C,KAAK,MAAM,MAAM,IAAI,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YAC5C,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,iBAAiB,CAAC,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC;YACzE,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC,CAAC;QAC3E,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,IAAI,IAAI;QACN,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;IAC3B,CAAC;IAED;;;;;OAKG;IACH,IAAI,QAAQ;QACV,OAAO,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IACjF,CAAC;IAED,QAAQ,CAAC,KAAc,EAAE,SAAkC,EAAE,KAAc;QACzE,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,EAAE,KAAK,EAAE,SAAS,CAAC,CAAC,CAAC;IAC3E,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,SAAS,CACb,KAAc,EACd,SAAkC,EAClC,UAAmC,EAAE;QAErC,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,IAAI,EAAE,GAAG,OAAO,CAAC;QAC9C,MAAM,KAAK,GAAG,MAAM,CAAC,SAAS,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC;QAE9C,IAAI,IAAI,CAAC,IAAI,KAAK,aAAa,EAAE,CAAC;YAChC,OAAO,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,SAAS,EAAE,EAAE,GAAG,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;QAChE,CAAC;QAED,MAAM,GAAG,GAAG,QAAQ,CAAC,KAAK,EAAE,KAAK,EAAE,SAAS,CAAC,CAAC;QAE9C,IAAI,IAAI,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC3B,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAClC,IAAI,GAAG,EAAE,CAAC;gBACR,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBACtB,OAAO,IAAI,gBAAgB,CAAC,GAAG,CAAC,CAAC;YACnC,CAAC;QACH,CAAC;QAED,IAAI,IAAI,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC3B,MAAM,IAAI,YAAY,CACpB,oCAAoC,IAAI,CAAC,IAAI,KAAK;gBAChD,aAAa,GAAG,IAAI;gBACpB,2EAA2E;gBAC3E,mBAAmB,CACtB,CAAC;QACJ,CAAC;QAED,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,SAAS,EAAE,EAAE,GAAG,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;QAC9E,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,eAAe,CAAC,QAAQ,CAAC,CAAC;QACzE,MAAM,MAAM,GAAG,IAAI,UAAU,CAAC;YAC5B,KAAK,EAAE,WAAW,IAAI,KAAK;YAC3B,KAAK;YACL,SAAS;YACT,OAAO;YACP,KAAK;YACL,IAAI,EAAE,EAAE,eAAe,EAAE,KAAK,EAAE;SACjC,CAAC,CAAC;QACH,YAAY,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QAChC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;QAC9B,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACxB,2EAA2E;QAC3E,wCAAwC;QACxC,OAAO,QAAQ,CAAC;IAClB,CAAC;IAEO,KAAK,CAAC,WAAW,CACvB,KAAc,EACd,SAAkC,EAClC,OAAgC;QAEhC,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC;YACxB,MAAM,IAAI,YAAY,CACpB,YAAY,IAAI,CAAC,IAAI,gBAAgB,IAAI,CAAC,IAAI,mCAAmC;gBAC/E,8DAA8D,CACjE,CAAC;QACJ,CAAC;QACD,OAAO,IAAI,CAAC,aAAa,CAAC,KAAK,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;IACvD,CAAC;CACF"}
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Test-runner glue.
|
|
3
|
+
*
|
|
4
|
+
* vitest has no fixture-injection system like pytest's, so this is a helper
|
|
5
|
+
* rather than a plugin: call `cassetteFor` in a test and you get the same
|
|
6
|
+
* per-test file naming and the same environment-driven mode switching.
|
|
7
|
+
*/
|
|
8
|
+
import { Cassette, type Mode, type SystemOneCallable } from "./cassette.js";
|
|
9
|
+
/**
|
|
10
|
+
* Mode from the environment, so CI and local runs differ without code changes.
|
|
11
|
+
*
|
|
12
|
+
* `JEV_RERECORD=1` replaces every recording, `JEV_RECORD=1` records only what
|
|
13
|
+
* is missing, and the default replays and fails on a miss.
|
|
14
|
+
*/
|
|
15
|
+
export declare function modeFromEnv(env?: NodeJS.ProcessEnv): Mode;
|
|
16
|
+
export interface CassetteForOptions {
|
|
17
|
+
dir?: string;
|
|
18
|
+
systemOne?: SystemOneCallable;
|
|
19
|
+
model?: string;
|
|
20
|
+
mode?: Mode;
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* A cassette named after the test.
|
|
24
|
+
*
|
|
25
|
+
* ```ts
|
|
26
|
+
* const cassette = cassetteFor("routes billing questions", { systemOne });
|
|
27
|
+
* ```
|
|
28
|
+
*
|
|
29
|
+
* The name is slugified so one test's recordings never collide with another's.
|
|
30
|
+
*/
|
|
31
|
+
export declare function cassetteFor(name: string, options?: CassetteForOptions): Cassette;
|
|
32
|
+
//# sourceMappingURL=fixture.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"fixture.d.ts","sourceRoot":"","sources":["../src/fixture.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAKH,OAAO,EAAE,QAAQ,EAAE,KAAK,IAAI,EAAE,KAAK,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAE5E;;;;;GAKG;AACH,wBAAgB,WAAW,CAAC,GAAG,GAAE,MAAM,CAAC,UAAwB,GAAG,IAAI,CAItE;AAED,MAAM,WAAW,kBAAkB;IACjC,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,SAAS,CAAC,EAAE,iBAAiB,CAAC;IAC9B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,IAAI,CAAC;CACb;AAED;;;;;;;;GAQG;AACH,wBAAgB,WAAW,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,GAAE,kBAAuB,GAAG,QAAQ,CAUpF"}
|
package/dist/fixture.js
ADDED
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Test-runner glue.
|
|
3
|
+
*
|
|
4
|
+
* vitest has no fixture-injection system like pytest's, so this is a helper
|
|
5
|
+
* rather than a plugin: call `cassetteFor` in a test and you get the same
|
|
6
|
+
* per-test file naming and the same environment-driven mode switching.
|
|
7
|
+
*/
|
|
8
|
+
import { mkdirSync } from "node:fs";
|
|
9
|
+
import { dirname, join } from "node:path";
|
|
10
|
+
import { Cassette } from "./cassette.js";
|
|
11
|
+
/**
|
|
12
|
+
* Mode from the environment, so CI and local runs differ without code changes.
|
|
13
|
+
*
|
|
14
|
+
* `JEV_RERECORD=1` replaces every recording, `JEV_RECORD=1` records only what
|
|
15
|
+
* is missing, and the default replays and fails on a miss.
|
|
16
|
+
*/
|
|
17
|
+
export function modeFromEnv(env = process.env) {
|
|
18
|
+
if (env["JEV_RERECORD"])
|
|
19
|
+
return "record";
|
|
20
|
+
if (env["JEV_RECORD"])
|
|
21
|
+
return "auto";
|
|
22
|
+
return "replay";
|
|
23
|
+
}
|
|
24
|
+
/**
|
|
25
|
+
* A cassette named after the test.
|
|
26
|
+
*
|
|
27
|
+
* ```ts
|
|
28
|
+
* const cassette = cassetteFor("routes billing questions", { systemOne });
|
|
29
|
+
* ```
|
|
30
|
+
*
|
|
31
|
+
* The name is slugified so one test's recordings never collide with another's.
|
|
32
|
+
*/
|
|
33
|
+
export function cassetteFor(name, options = {}) {
|
|
34
|
+
const dir = options.dir ?? join(process.cwd(), "test", "cassettes");
|
|
35
|
+
const safe = [...name].map((c) => (/[A-Za-z0-9\-_.]/.test(c) ? c : "_")).join("");
|
|
36
|
+
const path = join(dir, `${safe}.jevl`);
|
|
37
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
38
|
+
return new Cassette(path, {
|
|
39
|
+
systemOne: options.systemOne,
|
|
40
|
+
model: options.model,
|
|
41
|
+
mode: options.mode ?? modeFromEnv(),
|
|
42
|
+
});
|
|
43
|
+
}
|
|
44
|
+
//# sourceMappingURL=fixture.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"fixture.js","sourceRoot":"","sources":["../src/fixture.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AACpC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAE1C,OAAO,EAAE,QAAQ,EAAqC,MAAM,eAAe,CAAC;AAE5E;;;;;GAKG;AACH,MAAM,UAAU,WAAW,CAAC,MAAyB,OAAO,CAAC,GAAG;IAC9D,IAAI,GAAG,CAAC,cAAc,CAAC;QAAE,OAAO,QAAQ,CAAC;IACzC,IAAI,GAAG,CAAC,YAAY,CAAC;QAAE,OAAO,MAAM,CAAC;IACrC,OAAO,QAAQ,CAAC;AAClB,CAAC;AASD;;;;;;;;GAQG;AACH,MAAM,UAAU,WAAW,CAAC,IAAY,EAAE,UAA8B,EAAE;IACxE,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,IAAI,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,WAAW,CAAC,CAAC;IACpE,MAAM,IAAI,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAClF,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,OAAO,CAAC,CAAC;IACvC,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC9C,OAAO,IAAI,QAAQ,CAAC,IAAI,EAAE;QACxB,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,IAAI,EAAE,OAAO,CAAC,IAAI,IAAI,WAAW,EAAE;KACpC,CAAC,CAAC;AACL,CAAC"}
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Record and replay TypeSafe Jev requests in vitest.
|
|
3
|
+
*
|
|
4
|
+
* Cassettes are `.jevl` files, the same format the drift, bench and calibrate
|
|
5
|
+
* packages read, so a recording made by your tests doubles as a golden set.
|
|
6
|
+
*/
|
|
7
|
+
export { Cassette, CassetteMiss, ReplayedResponse, type CassetteOptions, type Mode, type SystemOneCallable, } from "./cassette.js";
|
|
8
|
+
export { assertAnswer, assertConfident, describeAnswer, type AssertAnswerOptions, } from "./assertions.js";
|
|
9
|
+
export { cassetteFor, modeFromEnv, type CassetteForOptions } from "./fixture.js";
|
|
10
|
+
export declare const VERSION = "0.1.0";
|
|
11
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EACL,QAAQ,EAAE,YAAY,EAAE,gBAAgB,EAAE,KAAK,eAAe,EAAE,KAAK,IAAI,EACzE,KAAK,iBAAiB,GACvB,MAAM,eAAe,CAAC;AACvB,OAAO,EACL,YAAY,EAAE,eAAe,EAAE,cAAc,EAAE,KAAK,mBAAmB,GACxE,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,KAAK,kBAAkB,EAAE,MAAM,cAAc,CAAC;AAEjF,eAAO,MAAM,OAAO,UAAU,CAAC"}
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Record and replay TypeSafe Jev requests in vitest.
|
|
3
|
+
*
|
|
4
|
+
* Cassettes are `.jevl` files, the same format the drift, bench and calibrate
|
|
5
|
+
* packages read, so a recording made by your tests doubles as a golden set.
|
|
6
|
+
*/
|
|
7
|
+
export { Cassette, CassetteMiss, ReplayedResponse, } from "./cassette.js";
|
|
8
|
+
export { assertAnswer, assertConfident, describeAnswer, } from "./assertions.js";
|
|
9
|
+
export { cassetteFor, modeFromEnv } from "./fixture.js";
|
|
10
|
+
export const VERSION = "0.1.0";
|
|
11
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EACL,QAAQ,EAAE,YAAY,EAAE,gBAAgB,GAEzC,MAAM,eAAe,CAAC;AACvB,OAAO,EACL,YAAY,EAAE,eAAe,EAAE,cAAc,GAC9C,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,WAAW,EAAE,WAAW,EAA2B,MAAM,cAAc,CAAC;AAEjF,MAAM,CAAC,MAAM,OAAO,GAAG,OAAO,CAAC"}
|
package/package.json
ADDED
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "jevkit-vitest",
|
|
3
|
+
"version": "0.1.0",
|
|
4
|
+
"description": "Record and replay TypeSafe Jev requests in vitest. Cassettes are .jevl files, so a test recording doubles as a drift golden set.",
|
|
5
|
+
"keywords": [
|
|
6
|
+
"jev",
|
|
7
|
+
"typesafe",
|
|
8
|
+
"system-one",
|
|
9
|
+
"vitest",
|
|
10
|
+
"testing",
|
|
11
|
+
"cassette",
|
|
12
|
+
"vcr"
|
|
13
|
+
],
|
|
14
|
+
"license": "MIT",
|
|
15
|
+
"type": "module",
|
|
16
|
+
"main": "./dist/index.js",
|
|
17
|
+
"types": "./dist/index.d.ts",
|
|
18
|
+
"exports": {
|
|
19
|
+
".": {
|
|
20
|
+
"types": "./dist/index.d.ts",
|
|
21
|
+
"default": "./dist/index.js"
|
|
22
|
+
}
|
|
23
|
+
},
|
|
24
|
+
"files": [
|
|
25
|
+
"dist",
|
|
26
|
+
"README.md"
|
|
27
|
+
],
|
|
28
|
+
"engines": {
|
|
29
|
+
"node": ">=18"
|
|
30
|
+
},
|
|
31
|
+
"repository": {
|
|
32
|
+
"type": "git",
|
|
33
|
+
"url": "git+https://github.com/pjdurden/jevkit-js.git"
|
|
34
|
+
},
|
|
35
|
+
"dependencies": {
|
|
36
|
+
"jevkit-core": "^0.2.0"
|
|
37
|
+
},
|
|
38
|
+
"scripts": {
|
|
39
|
+
"build": "tsc -b"
|
|
40
|
+
}
|
|
41
|
+
}
|