@mercury-fw/cli 0.37.0 → 0.38.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +32 -0
- package/README.md +18 -1
- package/dist/src/app/local-packages.d.ts +11 -5
- package/dist/src/e2e/define.d.ts +26 -7
- package/package.json +4 -4
- package/src/app/local-packages.ts +23 -9
- package/src/e2e/define.ts +25 -5
- package/src/e2e/load.ts +9 -1
- package/src/e2e/runner.ts +84 -27
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,37 @@
|
|
|
1
1
|
# @mercury-fw/cli
|
|
2
2
|
|
|
3
|
+
## 0.38.1
|
|
4
|
+
|
|
5
|
+
### Patch Changes
|
|
6
|
+
|
|
7
|
+
- 972a17f: - `mfw local-packages` and `mfw create --local-packages` name each tarball in `.packs/` after its content, so a package repacked at the same version is installed anew instead of kept from Bun's cache or refused by the lockfile's integrity check
|
|
8
|
+
- @mercury-fw/core@0.38.1
|
|
9
|
+
- @mercury-fw/utils@0.38.1
|
|
10
|
+
|
|
11
|
+
## 0.38.0
|
|
12
|
+
|
|
13
|
+
### Minor Changes
|
|
14
|
+
|
|
15
|
+
- de1ee52: Several people talking at once.
|
|
16
|
+
|
|
17
|
+
- `@mercury-fw/core`: turns of one conversation run one after the other, on every channel. Before, two HTTP `/turn` on the same conversation id ran together and mixed up its history.
|
|
18
|
+
- `@mercury-fw/core`: a turn waiting behind another one on the same conversation is dropped as soon as its client goes away.
|
|
19
|
+
- `@mercury-fw/core`: the idle capture waits for a running turn before it closes a conversation, and a sweep never starts while the previous one is still running.
|
|
20
|
+
- `@mercury-fw/core`: the capture a history compression triggers no longer leaves the capture marker wrong, which could make a later capture take the same messages twice or skip new ones.
|
|
21
|
+
- `@mercury-fw/core`: the tool log is kept per person, so a busy person no longer pushes everyone else's calls out of it.
|
|
22
|
+
- `@mercury-fw/core`: consolidation, `index.md` updates and confirmed promotions compare against the current file in the same queue as the write, so a concurrent update is never lost.
|
|
23
|
+
- `@mercury-fw/core`: vault files are written through a temporary file and a rename, so a reader never sees one half written.
|
|
24
|
+
- `@mercury-fw/core`: a vault file that exists but can't be read fails a write that depends on it, instead of counting as missing.
|
|
25
|
+
- `@mercury-fw/core`: the nightly review rewrites or deletes only the version of a document it read, and rereads it when it changed meanwhile.
|
|
26
|
+
- `@mercury-fw/core`: each pass of the nightly review gets up to 100 steps, up from 15.
|
|
27
|
+
- `@mercury-fw/cli`: an e2e case on HTTP can send its turns in `lanes` that run at the same time, each as its own user, on a conversation of its own or one they share.
|
|
28
|
+
|
|
29
|
+
### Patch Changes
|
|
30
|
+
|
|
31
|
+
- Updated dependencies [de1ee52]
|
|
32
|
+
- @mercury-fw/core@0.38.0
|
|
33
|
+
- @mercury-fw/utils@0.38.0
|
|
34
|
+
|
|
3
35
|
## 0.37.0
|
|
4
36
|
|
|
5
37
|
### Minor Changes
|
package/README.md
CHANGED
|
@@ -222,7 +222,7 @@ Makes the app install `@mercury-fw/*` packages from local tarballs instead of th
|
|
|
222
222
|
|
|
223
223
|
Overrides, and not the dependencies themselves, because a packed package names the packages it depends on by version, and the registry has those versions too: only an override sends them to the tarballs as well (a packed `@mercury-fw/core` depends on `@mercury-fw/plugin-types`, for example). The Dockerfile `mfw create` writes copies `.packs/` before installing, so the image gets the same packages (an app created before 0.31.0 needs `COPY --chown=mercury:mercury .pack[s] ./.packs/` added after the line copying `package.json`); `.gitignore` leaves it out of the repository.
|
|
224
224
|
|
|
225
|
-
Run it again after packing anew: it replaces the tarballs and the overrides of the previous run, and leaves alone the overrides you wrote yourself. `--off` takes the app back to the registry: the overrides it wrote and `.packs/` go, then `bun install`. After either, `mfw start` rebuilds the image with the packages now installed.
|
|
225
|
+
Run it again after packing anew: it replaces the tarballs and the overrides of the previous run, and leaves alone the overrides you wrote yourself. In `.packs/` each tarball is named after its content (`mercury-fw-core-0.38.0-1a2b3c4d.tgz`): Bun doesn't look again at a tarball whose override didn't change, it keeps what its lockfile and its cache hold, so a repack at the same version needs a new name to be installed. `--off` takes the app back to the registry: the overrides it wrote and `.packs/` go, then `bun install`. After either, `mfw start` rebuilds the image with the packages now installed.
|
|
226
226
|
|
|
227
227
|
```bash
|
|
228
228
|
mfw local-packages ../mercury-fw/apps/testbed/.packs
|
|
@@ -316,6 +316,23 @@ export default e2e({
|
|
|
316
316
|
|
|
317
317
|
Each run of an HTTP case is a conversation of its own. A turn also has the response's `status` (200, or 401 for a refused token). The event stream carries no tool results, so on HTTP a call's `input` is the line the surface shows for it (the command, for a CLI tool) and its `output` is there only for a call staged for confirmation, with the token, as on the REPL. A turn may span several lines.
|
|
318
318
|
|
|
319
|
+
To see what happens when people talk at the same time, an HTTP case can have `lanes` instead of `turns`: every lane sends its own turns in order, as its own user (`as`, or the case's), while the other lanes send theirs. Each lane gets a conversation of its own, unless lanes name the same `conversation`, which sends their turns at once on one conversation id, like two tabs on one chat. In the check, `run.lanes` holds each lane's run, and `run.turns` every turn, lane by lane:
|
|
320
|
+
|
|
321
|
+
```ts
|
|
322
|
+
{
|
|
323
|
+
name: "Alice and Bob at once",
|
|
324
|
+
channel: "http",
|
|
325
|
+
lanes: [
|
|
326
|
+
{ as: "alice", turns: ["Remember: tamarind. Reply OK.", "What word did I give you?"] },
|
|
327
|
+
{ as: "bob", turns: ["Remember: persimmon. Reply OK.", "What word did I give you?"] },
|
|
328
|
+
],
|
|
329
|
+
check: (run, expect) => {
|
|
330
|
+
expect.that("Alice gets hers", run.lanes![0]!.last.answer.includes("tamarind"));
|
|
331
|
+
expect.that("Bob gets his", run.lanes![1]!.last.answer.includes("persimmon"));
|
|
332
|
+
},
|
|
333
|
+
}
|
|
334
|
+
```
|
|
335
|
+
|
|
319
336
|
`before`, `after` and `check` also get `cli(command)`, which runs `sh -c command` in the app's container, outside the model, and resolves to its exit code and output: to prepare data (an issue to work on, a note in the wiki with the vault CLI), to check what a turn changed (`jira issue get KEY --select fields.assignee.displayName` after an assignment), to clean up. `after` runs even when the run or the check failed. A test that changes an external system has to clean up after itself, so start from read-only ones.
|
|
320
337
|
|
|
321
338
|
#### What it can and can't test
|
|
@@ -4,7 +4,10 @@
|
|
|
4
4
|
* which the image copies before `bun install`. Overrides, and not the
|
|
5
5
|
* dependencies themselves, because a packed package names its siblings by
|
|
6
6
|
* version, which the registry has too: only an override sends those
|
|
7
|
-
* transitive dependencies to the tarballs as well.
|
|
7
|
+
* transitive dependencies to the tarballs as well. Each tarball is named
|
|
8
|
+
* after its content there: Bun takes a `file:` tarball whose spec didn't
|
|
9
|
+
* change for the one in its lockfile, so a repack at the same version needs
|
|
10
|
+
* another name to be installed.
|
|
8
11
|
*/
|
|
9
12
|
/** The tarballs' folder inside the app, as the Dockerfile copies it. */
|
|
10
13
|
export declare const LOCAL_PACKS_DIR = ".packs";
|
|
@@ -20,8 +23,10 @@ export declare function withLocalOverrides(pkg: Manifest, packs: Array<{
|
|
|
20
23
|
/** `pkg` without the overrides this command writes; without `overrides` at
|
|
21
24
|
* all when none of the user's own is left. */
|
|
22
25
|
export declare function withoutLocalOverrides(pkg: Manifest): Manifest;
|
|
23
|
-
/** A local tarball: its file name in the folder,
|
|
26
|
+
/** A local tarball: its file name in the folder (`source`), its name in the
|
|
27
|
+
* app's `.packs/` (`file`), and the package it holds. */
|
|
24
28
|
export type Pack = {
|
|
29
|
+
source: string;
|
|
25
30
|
file: string;
|
|
26
31
|
name: string;
|
|
27
32
|
version: string;
|
|
@@ -33,9 +38,10 @@ export declare function packageOf(tarball: string): Promise<{
|
|
|
33
38
|
version: string;
|
|
34
39
|
}>;
|
|
35
40
|
/** The tarballs in `from`, in file-name order, each with the package it
|
|
36
|
-
* holds, for an app in `appDir`. Throws when
|
|
37
|
-
* tarball, or is the app's own `.packs/`
|
|
41
|
+
* holds and its name after its content, for an app in `appDir`. Throws when
|
|
42
|
+
* `from` doesn't exist, holds no tarball, or is the app's own `.packs/`
|
|
43
|
+
* (which the copy replaces). */
|
|
38
44
|
export declare function readPacks(from: string, appDir: string): Promise<Pack[]>;
|
|
39
|
-
/** Copies `packs` from `from` into the app's `.packs
|
|
45
|
+
/** Copies `packs` from `from` into the app's `.packs/` under their content names, in place of whatever was there. */
|
|
40
46
|
export declare function copyPacks(from: string, appDir: string, packs: Pack[]): void;
|
|
41
47
|
export {};
|
package/dist/src/e2e/define.d.ts
CHANGED
|
@@ -4,7 +4,8 @@
|
|
|
4
4
|
* app must have, and cases of turns sent to the app's real model, each with
|
|
5
5
|
* checks on the calls it made and the answer it gave. A case talks to the
|
|
6
6
|
* app's REPL, or to its HTTP surface as one of the test's users (each a token
|
|
7
|
-
* the app's auth provider knows)
|
|
7
|
+
* the app's auth provider knows), possibly in lanes that run at the same time.
|
|
8
|
+
* `mfw e2e` runs them (`runner.ts`).
|
|
8
9
|
*/
|
|
9
10
|
import type { Call, TurnData } from "./dump.ts";
|
|
10
11
|
export type { Call } from "./dump.ts";
|
|
@@ -18,10 +19,13 @@ export type Turn = TurnData & {
|
|
|
18
19
|
seconds: number;
|
|
19
20
|
status?: number;
|
|
20
21
|
};
|
|
21
|
-
/** A case's run: every turn in order, and the last one.
|
|
22
|
+
/** A case's run: every turn in order, and the last one. A case in lanes
|
|
23
|
+
* also has each lane's own run; its `turns` are the lanes' turns, lane by
|
|
24
|
+
* lane, and `last` the last lane's last turn. */
|
|
22
25
|
export type Run = {
|
|
23
26
|
turns: Turn[];
|
|
24
27
|
last: Turn;
|
|
28
|
+
lanes?: Run[];
|
|
25
29
|
};
|
|
26
30
|
/** Runs `command` in the app's container (`sh -c`), outside the model:
|
|
27
31
|
* preparing data, reading what a turn changed, cleaning up. */
|
|
@@ -57,8 +61,23 @@ export type Expect = {
|
|
|
57
61
|
};
|
|
58
62
|
/** A turn's text: a message, or a function of the turn before (a follow-up, a confirmation token). */
|
|
59
63
|
export type TurnText = string | ((previous: Turn) => string);
|
|
64
|
+
/** A case's turns, each as text or as text with its own user. */
|
|
65
|
+
export type Turns = Array<TurnText | {
|
|
66
|
+
text: TurnText;
|
|
67
|
+
as: string;
|
|
68
|
+
}>;
|
|
69
|
+
/** One of a case's lanes: turns sent in order, while the other lanes send
|
|
70
|
+
* theirs at the same time. */
|
|
71
|
+
export type Lane = {
|
|
72
|
+
/** The user the lane's turns are sent as; the case's when absent. */
|
|
73
|
+
as?: string;
|
|
74
|
+
/** Lanes naming the same conversation send their turns on one HTTP
|
|
75
|
+
* conversation (two tabs on one chat); a lane without one gets its own. */
|
|
76
|
+
conversation?: string;
|
|
77
|
+
turns: Turns;
|
|
78
|
+
};
|
|
60
79
|
/** One case: the turns sent, in one session (one REPL, or one HTTP
|
|
61
|
-
* conversation), and the checks on them. */
|
|
80
|
+
* conversation), or lanes of turns at the same time, and the checks on them. */
|
|
62
81
|
export type E2eCase = {
|
|
63
82
|
name: string;
|
|
64
83
|
/** Where the turns go: the REPL (the default) or the HTTP surface, which
|
|
@@ -69,10 +88,10 @@ export type E2eCase = {
|
|
|
69
88
|
/** The turns; on HTTP one can name its own user, so a case can stage an
|
|
70
89
|
* action as one person and try to confirm it as another, in the same
|
|
71
90
|
* conversation id. */
|
|
72
|
-
turns
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
91
|
+
turns?: Turns;
|
|
92
|
+
/** On HTTP, instead of `turns`: lanes that run at the same time, to see
|
|
93
|
+
* what several users, or one user twice, get when they talk at once. */
|
|
94
|
+
lanes?: Lane[];
|
|
76
95
|
/** How many times to run it (the model isn't deterministic); default 1. */
|
|
77
96
|
repeat?: number;
|
|
78
97
|
/** How many runs must pass; default every one. */
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@mercury-fw/cli",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.38.1",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
@@ -41,8 +41,8 @@
|
|
|
41
41
|
"comment:shape": "The Mercury CLI (`mfw`). `mfw create <dir>` writes a new Mercury app from the template. The framework packages move in lockstep with it, so a new app gets them at the CLI's own version; plugins and channels at the registry's latest. The plugin and channel packages are devDependencies only for the catalog test. `main(argv)` is exported for create-mercury-agent.",
|
|
42
42
|
"dependencies": {
|
|
43
43
|
"@clack/prompts": "^1.8.1",
|
|
44
|
-
"@mercury-fw/core": "0.
|
|
45
|
-
"@mercury-fw/utils": "0.
|
|
44
|
+
"@mercury-fw/core": "0.38.1",
|
|
45
|
+
"@mercury-fw/utils": "0.38.1",
|
|
46
46
|
"commander": "^15.0.0"
|
|
47
47
|
},
|
|
48
48
|
"devDependencies": {
|
|
@@ -50,7 +50,7 @@
|
|
|
50
50
|
"@mercury-fw/channel-http": "0.4.0",
|
|
51
51
|
"@mercury-fw/auth-oidc": "0.1.0",
|
|
52
52
|
"@mercury-fw/auth-static": "0.1.0",
|
|
53
|
-
"@mercury-fw/formatter": "0.
|
|
53
|
+
"@mercury-fw/formatter": "0.38.1",
|
|
54
54
|
"@mercury-fw/plugin-atlassian-admin": "0.1.2",
|
|
55
55
|
"@mercury-fw/plugin-bitbucket": "0.1.2",
|
|
56
56
|
"@mercury-fw/plugin-jira": "0.4.3",
|
|
@@ -4,10 +4,14 @@
|
|
|
4
4
|
* which the image copies before `bun install`. Overrides, and not the
|
|
5
5
|
* dependencies themselves, because a packed package names its siblings by
|
|
6
6
|
* version, which the registry has too: only an override sends those
|
|
7
|
-
* transitive dependencies to the tarballs as well.
|
|
7
|
+
* transitive dependencies to the tarballs as well. Each tarball is named
|
|
8
|
+
* after its content there: Bun takes a `file:` tarball whose spec didn't
|
|
9
|
+
* change for the one in its lockfile, so a repack at the same version needs
|
|
10
|
+
* another name to be installed.
|
|
8
11
|
*/
|
|
9
12
|
|
|
10
|
-
import {
|
|
13
|
+
import { createHash } from "node:crypto";
|
|
14
|
+
import { cpSync, existsSync, mkdirSync, readdirSync, readFileSync, rmSync } from "node:fs";
|
|
11
15
|
import { join, resolve } from "node:path";
|
|
12
16
|
|
|
13
17
|
/** The tarballs' folder inside the app, as the Dockerfile copies it. */
|
|
@@ -38,8 +42,15 @@ export function withoutLocalOverrides(pkg: Manifest): Manifest {
|
|
|
38
42
|
return Object.keys(own).length > 0 ? { ...rest, overrides: own } : rest;
|
|
39
43
|
}
|
|
40
44
|
|
|
41
|
-
/** A local tarball: its file name in the folder,
|
|
42
|
-
|
|
45
|
+
/** A local tarball: its file name in the folder (`source`), its name in the
|
|
46
|
+
* app's `.packs/` (`file`), and the package it holds. */
|
|
47
|
+
export type Pack = { source: string; file: string; name: string; version: string };
|
|
48
|
+
|
|
49
|
+
/** `file`'s name with the first 8 hex digits of `tarball`'s sha256 before `.tgz`. */
|
|
50
|
+
function contentName(file: string, tarball: string): string {
|
|
51
|
+
const hash = createHash("sha256").update(readFileSync(tarball)).digest("hex").slice(0, 8);
|
|
52
|
+
return `${file.slice(0, -".tgz".length)}-${hash}.tgz`;
|
|
53
|
+
}
|
|
43
54
|
|
|
44
55
|
/** The package name and version in a `bun pm pack` tarball, read from its
|
|
45
56
|
* `package/package.json`. */
|
|
@@ -58,8 +69,9 @@ export async function packageOf(tarball: string): Promise<{ name: string; versio
|
|
|
58
69
|
}
|
|
59
70
|
|
|
60
71
|
/** The tarballs in `from`, in file-name order, each with the package it
|
|
61
|
-
* holds, for an app in `appDir`. Throws when
|
|
62
|
-
* tarball, or is the app's own `.packs/`
|
|
72
|
+
* holds and its name after its content, for an app in `appDir`. Throws when
|
|
73
|
+
* `from` doesn't exist, holds no tarball, or is the app's own `.packs/`
|
|
74
|
+
* (which the copy replaces). */
|
|
63
75
|
export async function readPacks(from: string, appDir: string): Promise<Pack[]> {
|
|
64
76
|
const source = resolve(from);
|
|
65
77
|
if (source === join(resolve(appDir), LOCAL_PACKS_DIR)) {
|
|
@@ -68,13 +80,15 @@ export async function readPacks(from: string, appDir: string): Promise<Pack[]> {
|
|
|
68
80
|
if (!existsSync(source)) throw new Error(`${source} doesn't exist.`);
|
|
69
81
|
const files = readdirSync(source).filter((f) => f.endsWith(".tgz")).sort();
|
|
70
82
|
if (files.length === 0) throw new Error(`No .tgz in ${source}: pack the packages there first (bun pm pack).`);
|
|
71
|
-
return Promise.all(
|
|
83
|
+
return Promise.all(
|
|
84
|
+
files.map(async (file) => ({ source: file, file: contentName(file, join(source, file)), ...(await packageOf(join(source, file))) })),
|
|
85
|
+
);
|
|
72
86
|
}
|
|
73
87
|
|
|
74
|
-
/** Copies `packs` from `from` into the app's `.packs
|
|
88
|
+
/** Copies `packs` from `from` into the app's `.packs/` under their content names, in place of whatever was there. */
|
|
75
89
|
export function copyPacks(from: string, appDir: string, packs: Pack[]): void {
|
|
76
90
|
const target = join(appDir, LOCAL_PACKS_DIR);
|
|
77
91
|
rmSync(target, { recursive: true, force: true });
|
|
78
92
|
mkdirSync(target);
|
|
79
|
-
for (const { file } of packs) cpSync(join(resolve(from),
|
|
93
|
+
for (const { source, file } of packs) cpSync(join(resolve(from), source), join(target, file));
|
|
80
94
|
}
|
package/src/e2e/define.ts
CHANGED
|
@@ -4,7 +4,8 @@
|
|
|
4
4
|
* app must have, and cases of turns sent to the app's real model, each with
|
|
5
5
|
* checks on the calls it made and the answer it gave. A case talks to the
|
|
6
6
|
* app's REPL, or to its HTTP surface as one of the test's users (each a token
|
|
7
|
-
* the app's auth provider knows)
|
|
7
|
+
* the app's auth provider knows), possibly in lanes that run at the same time.
|
|
8
|
+
* `mfw e2e` runs them (`runner.ts`).
|
|
8
9
|
*/
|
|
9
10
|
import type { Call, TurnData } from "./dump.ts";
|
|
10
11
|
|
|
@@ -18,8 +19,10 @@ export type { Call } from "./dump.ts";
|
|
|
18
19
|
* no tool results. */
|
|
19
20
|
export type Turn = TurnData & { seconds: number; status?: number };
|
|
20
21
|
|
|
21
|
-
/** A case's run: every turn in order, and the last one.
|
|
22
|
-
|
|
22
|
+
/** A case's run: every turn in order, and the last one. A case in lanes
|
|
23
|
+
* also has each lane's own run; its `turns` are the lanes' turns, lane by
|
|
24
|
+
* lane, and `last` the last lane's last turn. */
|
|
25
|
+
export type Run = { turns: Turn[]; last: Turn; lanes?: Run[] };
|
|
23
26
|
|
|
24
27
|
/** Runs `command` in the app's container (`sh -c`), outside the model:
|
|
25
28
|
* preparing data, reading what a turn changed, cleaning up. */
|
|
@@ -51,8 +54,22 @@ export type Expect = {
|
|
|
51
54
|
/** A turn's text: a message, or a function of the turn before (a follow-up, a confirmation token). */
|
|
52
55
|
export type TurnText = string | ((previous: Turn) => string);
|
|
53
56
|
|
|
57
|
+
/** A case's turns, each as text or as text with its own user. */
|
|
58
|
+
export type Turns = Array<TurnText | { text: TurnText; as: string }>;
|
|
59
|
+
|
|
60
|
+
/** One of a case's lanes: turns sent in order, while the other lanes send
|
|
61
|
+
* theirs at the same time. */
|
|
62
|
+
export type Lane = {
|
|
63
|
+
/** The user the lane's turns are sent as; the case's when absent. */
|
|
64
|
+
as?: string;
|
|
65
|
+
/** Lanes naming the same conversation send their turns on one HTTP
|
|
66
|
+
* conversation (two tabs on one chat); a lane without one gets its own. */
|
|
67
|
+
conversation?: string;
|
|
68
|
+
turns: Turns;
|
|
69
|
+
};
|
|
70
|
+
|
|
54
71
|
/** One case: the turns sent, in one session (one REPL, or one HTTP
|
|
55
|
-
* conversation), and the checks on them. */
|
|
72
|
+
* conversation), or lanes of turns at the same time, and the checks on them. */
|
|
56
73
|
export type E2eCase = {
|
|
57
74
|
name: string;
|
|
58
75
|
/** Where the turns go: the REPL (the default) or the HTTP surface, which
|
|
@@ -63,7 +80,10 @@ export type E2eCase = {
|
|
|
63
80
|
/** The turns; on HTTP one can name its own user, so a case can stage an
|
|
64
81
|
* action as one person and try to confirm it as another, in the same
|
|
65
82
|
* conversation id. */
|
|
66
|
-
turns
|
|
83
|
+
turns?: Turns;
|
|
84
|
+
/** On HTTP, instead of `turns`: lanes that run at the same time, to see
|
|
85
|
+
* what several users, or one user twice, get when they talk at once. */
|
|
86
|
+
lanes?: Lane[];
|
|
67
87
|
/** How many times to run it (the model isn't deterministic); default 1. */
|
|
68
88
|
repeat?: number;
|
|
69
89
|
/** How many runs must pass; default every one. */
|
package/src/e2e/load.ts
CHANGED
|
@@ -27,7 +27,15 @@ export async function loadTest(file: string): Promise<E2eTest> {
|
|
|
27
27
|
for (const [i, c] of loaded.cases.entries()) {
|
|
28
28
|
const which = `${file}: case ${i + 1} ("${c?.name ?? "?"}")`;
|
|
29
29
|
if (typeof c?.name !== "string") throw new Error(`${which} has no name`);
|
|
30
|
-
|
|
30
|
+
const nonEmpty = (list: unknown) => Array.isArray(list) && list.length > 0;
|
|
31
|
+
if (c.turns !== undefined && c.lanes !== undefined) throw new Error(`${which} has both turns and lanes`);
|
|
32
|
+
if (c.lanes !== undefined) {
|
|
33
|
+
if (c.channel !== "http") throw new Error(`${which} has lanes, which go with channel "http"`);
|
|
34
|
+
if (!nonEmpty(c.lanes)) throw new Error(`${which} has no turns`);
|
|
35
|
+
for (const [j, lane] of c.lanes.entries()) {
|
|
36
|
+
if (!nonEmpty(lane?.turns)) throw new Error(`${which} lane ${j + 1} has no turns`);
|
|
37
|
+
}
|
|
38
|
+
} else if (!nonEmpty(c.turns)) throw new Error(`${which} has no turns`);
|
|
31
39
|
if (typeof c.check !== "function") throw new Error(`${which} has no check`);
|
|
32
40
|
const whole = (n: unknown) => typeof n === "number" && Number.isInteger(n) && n >= 1;
|
|
33
41
|
if (c.repeat !== undefined && !whole(c.repeat)) throw new Error(`${which} repeat takes a positive whole number`);
|
package/src/e2e/runner.ts
CHANGED
|
@@ -1,14 +1,15 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Runs e2e tests against an app: for each case (as many times as it
|
|
3
3
|
* repeats), a fresh session (the REPL, or one conversation on the HTTP
|
|
4
|
-
* surface) gets the case's turns,
|
|
4
|
+
* surface) gets the case's turns, or each of its lanes gets a session and
|
|
5
|
+
* they all send their turns at once; the checks run on what each turn did, and
|
|
5
6
|
* every check is printed; the exit code says whether every case passed enough
|
|
6
7
|
* runs. The sessions, the container commands and the report's destination are
|
|
7
8
|
* injected (`session.ts`, `http-session.ts` and `commands.ts` hold the real
|
|
8
9
|
* ones), so the tests drive the runner with scripted turns.
|
|
9
10
|
*/
|
|
10
11
|
import { CATALOG } from "../catalog.ts";
|
|
11
|
-
import type { Context, E2eCase, E2eTest, Run, Turn } from "./define.ts";
|
|
12
|
+
import type { Context, E2eCase, E2eTest, Run, Turn, Turns, TurnText } from "./define.ts";
|
|
12
13
|
import { turnFromDump, type TurnData } from "./dump.ts";
|
|
13
14
|
import { createExpect, type Check } from "./expect.ts";
|
|
14
15
|
|
|
@@ -137,40 +138,96 @@ function tokenFor(channel: Channel, user: string | undefined, users: Record<stri
|
|
|
137
138
|
return token;
|
|
138
139
|
}
|
|
139
140
|
|
|
140
|
-
/**
|
|
141
|
+
/** A turn ready to send: its text, and the token its user goes with. */
|
|
142
|
+
type PlannedTurn = { text: TurnText; token: string | undefined };
|
|
143
|
+
|
|
144
|
+
/** Settles every turn's user (the turn's own, or `user`) before anything is
|
|
145
|
+
* sent, so a case that doesn't add up costs nothing. */
|
|
146
|
+
function plan(turns: Turns, channel: Channel, user: string | undefined, users: Record<string, string>): PlannedTurn[] {
|
|
147
|
+
return turns.map((next) => {
|
|
148
|
+
const { text, as } = typeof next === "object" ? next : { text: next, as: undefined };
|
|
149
|
+
return { text, token: tokenFor(channel, as ?? user, users) };
|
|
150
|
+
});
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
/** Sends `planned` on `session` one after the other, pushing each turn onto
|
|
154
|
+
* `turns` as it comes back (a function turn sees the one before it there). */
|
|
155
|
+
async function sendTurns(session: Session, planned: PlannedTurn[], channel: Channel, turns: Turn[], deps: RunnerDeps): Promise<void> {
|
|
156
|
+
for (const { text, token } of planned) {
|
|
157
|
+
const line = typeof text === "string" ? text : text(turns.at(-1)!);
|
|
158
|
+
if (channel === "repl" && line.includes("\n")) throw new Error("a turn must be one line (the REPL reads one line per turn)");
|
|
159
|
+
const start = deps.now();
|
|
160
|
+
const reply = await session.turn(line, token);
|
|
161
|
+
const seconds = (deps.now() - start) / 1000;
|
|
162
|
+
if ("status" in reply) {
|
|
163
|
+
turns.push({ ...reply.turn, seconds, status: reply.status });
|
|
164
|
+
continue;
|
|
165
|
+
}
|
|
166
|
+
const data = turnFromDump(reply.dump);
|
|
167
|
+
const answer = data.calls.length === 0 && data.answer === "" ? answerFromOutput(reply.output) : data.answer;
|
|
168
|
+
turns.push({ calls: data.calls, answer, seconds });
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
/** Runs a case's lanes at the same time, each lane's turns in order, lanes
|
|
173
|
+
* naming the same conversation on one session; returns each lane's turns,
|
|
174
|
+
* filled in as they come back. */
|
|
175
|
+
async function runLanes(
|
|
176
|
+
c: E2eCase,
|
|
177
|
+
channel: Channel,
|
|
178
|
+
users: Record<string, string>,
|
|
179
|
+
laneTurns: Turn[][],
|
|
180
|
+
deps: RunnerDeps,
|
|
181
|
+
): Promise<void> {
|
|
182
|
+
if (channel !== "http") throw new Error('"lanes" go with channel "http"');
|
|
183
|
+
const lanes = c.lanes!.map((lane) => ({ conversation: lane.conversation, planned: plan(lane.turns, channel, lane.as ?? c.as, users) }));
|
|
184
|
+
const opened: Session[] = [];
|
|
185
|
+
const byConversation = new Map<string, Session>();
|
|
186
|
+
try {
|
|
187
|
+
const sessions: Session[] = [];
|
|
188
|
+
for (const { conversation } of lanes) {
|
|
189
|
+
let session = conversation === undefined ? undefined : byConversation.get(conversation);
|
|
190
|
+
if (session === undefined) {
|
|
191
|
+
session = await deps.openSession(channel);
|
|
192
|
+
opened.push(session);
|
|
193
|
+
if (conversation !== undefined) byConversation.set(conversation, session);
|
|
194
|
+
}
|
|
195
|
+
sessions.push(session);
|
|
196
|
+
}
|
|
197
|
+
await Promise.all(lanes.map((lane, i) => sendTurns(sessions[i]!, lane.planned, channel, laneTurns[i]!, deps)));
|
|
198
|
+
} finally {
|
|
199
|
+
await Promise.all(opened.map((session) => session.close()));
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
/** One run of a case in a fresh session (or one per lane): `before`, the turns, the checks, `after`. */
|
|
141
204
|
async function runOnce(c: E2eCase, users: Record<string, string>, deps: RunnerDeps): Promise<RunReport> {
|
|
142
205
|
const channel: Channel = c.channel ?? "repl";
|
|
143
206
|
const ctx: Context = { cli: deps.cli };
|
|
144
207
|
const turns: Turn[] = [];
|
|
208
|
+
const laneTurns: Turn[][] = (c.lanes ?? []).map(() => []);
|
|
145
209
|
let checks: Check[] = [];
|
|
146
210
|
try {
|
|
147
211
|
await c.before?.(ctx);
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
continue;
|
|
165
|
-
}
|
|
166
|
-
const data = turnFromDump(reply.dump);
|
|
167
|
-
const answer = data.calls.length === 0 && data.answer === "" ? answerFromOutput(reply.output) : data.answer;
|
|
168
|
-
turns.push({ calls: data.calls, answer, seconds });
|
|
212
|
+
if ((c.turns === undefined) === (c.lanes === undefined)) throw new Error('a case has either "turns" or "lanes"');
|
|
213
|
+
let lanes: Run[] | undefined;
|
|
214
|
+
if (c.lanes !== undefined) {
|
|
215
|
+
try {
|
|
216
|
+
await runLanes(c, channel, users, laneTurns, deps);
|
|
217
|
+
} finally {
|
|
218
|
+
turns.push(...laneTurns.flat());
|
|
219
|
+
}
|
|
220
|
+
lanes = laneTurns.map((laneTurn) => ({ turns: laneTurn, last: laneTurn.at(-1)! }));
|
|
221
|
+
} else {
|
|
222
|
+
const planned = plan(c.turns!, channel, c.as, users);
|
|
223
|
+
const session = await deps.openSession(channel);
|
|
224
|
+
try {
|
|
225
|
+
await sendTurns(session, planned, channel, turns, deps);
|
|
226
|
+
} finally {
|
|
227
|
+
await session.close();
|
|
169
228
|
}
|
|
170
|
-
} finally {
|
|
171
|
-
await session.close();
|
|
172
229
|
}
|
|
173
|
-
const run: Run = { turns, last: turns.at(-1)
|
|
230
|
+
const run: Run = { turns, last: turns.at(-1)!, ...(lanes === undefined ? {} : { lanes }) };
|
|
174
231
|
const recorder = createExpect(run);
|
|
175
232
|
try {
|
|
176
233
|
await c.check(run, recorder.expect, ctx);
|