@kontextmind/kxm 0.7.146 → 0.7.148
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/CHANGELOG.md +38 -3
- package/docs/README.md +1 -1
- package/docs/adr/ADR-0002-browser-automation-steel-doks.md +8 -5
- package/docs/adr/ADR-0005-obscura-default-playwright.md +4 -3
- package/docs/adr/ADR-0006-machine-account-names.md +90 -0
- package/docs/adr/ADR-0007-steel-caddy-authentik.md +97 -0
- package/docs/adr/README.md +3 -1
- package/docs/contributing/ci-and-release.md +80 -43
- package/docs/contributing/operating-rules.md +13 -1
- package/docs/guides/agent-skills.md +1 -1
- package/docs/guides/browser-automation.md +44 -28
- package/docs/kb/how-credentials-retrieved-safely.md +22 -23
- package/docs/kb/how-to-connect-playwright-to-steel.md +8 -5
- package/docs/kb/how-to-recover-expired-session-or-orphan.md +11 -10
- package/docs/kb/why-automation-opened-different-browser.md +4 -3
- package/docs/operations/deploy.md +31 -0
- package/docs/operations/troubleshooting.md +23 -1
- package/docs/prompts/browser-diagnose-recover.md +2 -2
- package/docs/prompts/browser-start.md +1 -1
- package/docs/reference/configuration.md +9 -7
- package/package.json +2 -1
- package/plugins/kxm/.claude-plugin/plugin.json +1 -1
- package/plugins/kxm/dist/claude-hook.js +1 -1
- package/plugins/kxm/dist/cli.js +2 -2
- package/plugins/kxm/dist/extension.js +1 -1
- package/plugins/kxm/dist/mcp-server.js +1 -1
- package/plugins/kxm/dist/runtime.js +3 -3
- package/plugins/kxm/package.json +1 -1
- package/plugins/kxm/skills/kxm-browser-auth/SKILL.md +11 -12
- package/plugins/kxm/skills/kxm-browser-diagnostics/SKILL.md +5 -4
- package/plugins/kxm/skills/kxm-browser-explore/SKILL.md +4 -3
- package/plugins/kxm/skills/kxm-browser-session/SKILL.md +5 -5
- package/plugins/kxm/skills/kxm-browser-verify/SKILL.md +1 -1
- package/plugins/kxm/src/browser.ts +12 -9
- package/plugins/kxm/src/mcp-server.ts +1 -1
- package/plugins/kxm/src/modes.ts +1 -1
- package/plugins/kxm/src/session-work.ts +3 -1
- package/scripts/ci-classify.mjs +69 -0
- package/scripts/ci-unit-shard.mjs +230 -0
|
@@ -15,16 +15,17 @@ Use this skill to investigate and resolve connectivity failures, CDP attachment
|
|
|
15
15
|
- **Diagnosis**:
|
|
16
16
|
- KontextMind Steel is behind Authentik forward auth. Unauthenticated requests redirect to `id.kxmd.dev`. Authentik accepts an app password only as `Authorization: Basic`. A Bearer token is refused.
|
|
17
17
|
- Check that `STEEL_AUTH_BASIC` is set, or that both `STEEL_AUTH_USER` and `STEEL_AUTH_TOKEN` are set (`test -n "$STEEL_AUTH_TOKEN" && echo set`); never print the value.
|
|
18
|
-
-
|
|
19
|
-
-
|
|
18
|
+
- `STEEL_API_KEY` is deprecated. Steel and Caddy do not enforce it. Do not put a credential in the URL. Direct LAN, tailnet, and host-forward connections are blocked.
|
|
19
|
+
- `websocketUrl` `ws://steel-browser/` means the client is older than `kxm` 0.7.135. The server returns `wss://steel.kontextmind.com/`.
|
|
20
|
+
- **Remedy**: Re-read the `svc-steel` field `basic_auth` with `op read 'op://kontextmind/Steel (svc-steel)/basic_auth'` into `STEEL_AUTH_BASIC`. Do not log it or write it to disk. Precedence is `STEEL_AUTH_HEADER`, then `STEEL_AUTH_BASIC`, then `STEEL_AUTH_USER` and `STEEL_AUTH_TOKEN`.
|
|
20
21
|
|
|
21
22
|
### 2. CDP WebSocket Attachment Failure
|
|
22
23
|
|
|
23
24
|
- **Symptom**: `WebSocket connection to wss://... failed: 404/500`.
|
|
24
25
|
- **Diagnosis**:
|
|
25
26
|
- Check if the target session ID has already been released or timed out.
|
|
26
|
-
-
|
|
27
|
-
- **Remedy**: Query `GET /v1/sessions/<id
|
|
27
|
+
- The CDP path is `/v1/devtools` on `wss://steel.kontextmind.com/` with an `Authorization` header. Caddy must pass the WebSocket upgrade. A credential in the URL is not accepted.
|
|
28
|
+
- **Remedy**: Query `GET /v1/sessions/<id>` with the same `Authorization` header. If status is `released`, launch a fresh session.
|
|
28
29
|
|
|
29
30
|
### 3. Session Timeout & Expiration
|
|
30
31
|
|
|
@@ -21,11 +21,12 @@ Use this skill for exploratory navigation, DOM inspection, scraping, and interac
|
|
|
21
21
|
Ensure an active Steel session exists and obtain its CDP endpoint:
|
|
22
22
|
|
|
23
23
|
```bash
|
|
24
|
-
# CDP
|
|
25
|
-
|
|
24
|
+
# CDP path only. The session websocketUrl is wss://steel.kontextmind.com/.
|
|
25
|
+
# The Authentik credential is an Authorization header, not a query parameter.
|
|
26
|
+
CDP_URL="wss://steel.kontextmind.com/v1/devtools?sessionId=<sessionId>"
|
|
26
27
|
```
|
|
27
28
|
|
|
28
|
-
Build that URL with `formatCDPConnect()` so the `Authorization
|
|
29
|
+
Build that URL with `formatCDPConnect()` so the `Authorization` header is available for the handshake. Precedence is `STEEL_AUTH_HEADER`, then `STEEL_AUTH_BASIC`, then `STEEL_AUTH_USER` and `STEEL_AUTH_TOKEN`. Those override `STEEL_API_KEY`, which Steel and Caddy do not enforce. A Bearer token is not accepted. `kxm` 0.7.135 or newer is required. Read `basic_auth` with `op read` and do not write it to disk.
|
|
29
30
|
|
|
30
31
|
### 2. Connect a client that can send the header
|
|
31
32
|
|
|
@@ -5,7 +5,7 @@ description: Start, attach to, inspect, and release self-hosted Steel browser se
|
|
|
5
5
|
|
|
6
6
|
# KXM Browser Session Management
|
|
7
7
|
|
|
8
|
-
Use this skill to create, inspect, attach automation tools to, and release isolated browser sessions
|
|
8
|
+
Use this skill to create, inspect, attach automation tools to, and release isolated browser sessions on the Steel server. `STEEL_API_URL` defaults to `https://steel.kontextmind.com`. `steel.theneuro.me` is an alias of that server. `kxm` 0.7.135 or newer is required.
|
|
9
9
|
|
|
10
10
|
Playwright testing and verification use Obscura by default (`resolveBrowserCdpEndpoint()`, or `npm run e2e`). Use this skill's Steel session for human takeover, MFA, and the live session viewer. Attach Playwright to that session only when `KXM_BROWSER=steel`.
|
|
11
11
|
|
|
@@ -18,9 +18,9 @@ Playwright testing and verification use Obscura by default (`resolveBrowserCdpEn
|
|
|
18
18
|
|
|
19
19
|
## Prerequisites
|
|
20
20
|
|
|
21
|
-
1.
|
|
22
|
-
2.
|
|
23
|
-
3.
|
|
21
|
+
1. `kxm` 0.7.135 or newer. The server is `https://steel.kontextmind.com`. Caddy and Authentik forward auth are the only path. Direct LAN, tailnet, and host-forward access is blocked. Sessions return `websocketUrl` `wss://steel.kontextmind.com/` (previously `ws://steel-browser/`).
|
|
22
|
+
2. The `svc-steel` Authentik credential in the environment. Allowed groups are `steel-users`, `kxmd-users`, `kxmd-admins`, and `kxmd-owners`. Read it at runtime and do not write it to disk: `export STEEL_AUTH_BASIC="$(op read 'op://kontextmind/Steel (svc-steel)/basic_auth')"`. Precedence is `STEEL_AUTH_HEADER`, then `STEEL_AUTH_BASIC`, then `STEEL_AUTH_USER` and `STEEL_AUTH_TOKEN`. Those override `STEEL_API_KEY`. `STEEL_API_KEY` is deprecated. Steel and Caddy do not enforce it. A Bearer token is refused. Never put the credential in a URL.
|
|
23
|
+
3. HTTPS to `steel.kontextmind.com` for the CDP path `/v1/devtools`. Send `Authorization` on the handshake.
|
|
24
24
|
|
|
25
25
|
## Session Lifecycle States
|
|
26
26
|
|
|
@@ -48,7 +48,7 @@ Playwright testing and verification use Obscura by default (`resolveBrowserCdpEn
|
|
|
48
48
|
- **Inputs**: Task ID, target URL, session timeout (default 300s, max 1800s), optional proxy or viewport dimensions.
|
|
49
49
|
- **Outputs**:
|
|
50
50
|
- `sessionId`: Unique session UUID.
|
|
51
|
-
- `cdpUrl`:
|
|
51
|
+
- `cdpUrl`: `wss://steel.kontextmind.com/v1/devtools?sessionId=<id>`. Send `Authorization` on the handshake. The URL has no credential. The session `websocketUrl` is `wss://steel.kontextmind.com/`.
|
|
52
52
|
- `sessionViewerUrl`: Interactive web session viewer URL (`$STEEL_UI_URL?sessionId=<id>`).
|
|
53
53
|
- `status`: `live` | `idle` | `released`.
|
|
54
54
|
|
|
@@ -77,7 +77,7 @@ Local pages require the launcher flag `--allow-private-network` (the launcher al
|
|
|
77
77
|
|
|
78
78
|
## Steel, only for takeover
|
|
79
79
|
|
|
80
|
-
When the case is human takeover, MFA, or the live session viewer, set `KXM_BROWSER=steel` and a session id. `connectBrowserOverCdp()` passes `formatCDPConnect()` headers into `chromium.connectOverCDP
|
|
80
|
+
When the case is human takeover, MFA, or the live session viewer, set `KXM_BROWSER=steel` and a session id. That path needs `kxm` 0.7.135 or newer. `connectBrowserOverCdp()` passes `formatCDPConnect()` headers into `chromium.connectOverCDP` for `/v1/devtools` on `wss://steel.kontextmind.com/`. The `Authorization` header is the `svc-steel` credential (`STEEL_AUTH_HEADER`, then `STEEL_AUTH_BASIC`, then `STEEL_AUTH_USER` and `STEEL_AUTH_TOKEN`). `STEEL_API_KEY` is deprecated and is not enforced. Closing the Playwright browser disconnects the client and does not release the Steel session.
|
|
81
81
|
|
|
82
82
|
```typescript
|
|
83
83
|
import { chromium } from "playwright";
|
|
@@ -5,11 +5,12 @@
|
|
|
5
5
|
* (`resolveBrowserCdpEndpoint()`). Steel remains the client for human
|
|
6
6
|
* takeover, MFA, and the live session viewer (`KXM_BROWSER=steel`).
|
|
7
7
|
* Manages remote Steel sessions, CDP endpoints, human takeover handoffs,
|
|
8
|
-
*
|
|
8
|
+
* and automated cleanup without leaking secrets.
|
|
9
9
|
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
12
|
-
*
|
|
10
|
+
* steel.kontextmind.com is reached only through Caddy and Authentik forward
|
|
11
|
+
* auth. Clients send `Authorization: Basic` for the svc-steel credential.
|
|
12
|
+
* A Bearer token is refused. `STEEL_API_KEY` is deprecated and is not
|
|
13
|
+
* enforced by Steel or Caddy. Do not put a credential in the URL.
|
|
13
14
|
*/
|
|
14
15
|
|
|
15
16
|
import { execSync } from "node:child_process";
|
|
@@ -188,9 +189,9 @@ export class SteelAuthRedirectError extends Error {
|
|
|
188
189
|
}
|
|
189
190
|
|
|
190
191
|
const LEGACY_STEEL_AUTH_WARNING =
|
|
191
|
-
"kxm: STEEL_API_KEY is deprecated for Steel.
|
|
192
|
-
"Set STEEL_AUTH_BASIC, or STEEL_AUTH_USER and STEEL_AUTH_TOKEN. " +
|
|
193
|
-
"
|
|
192
|
+
"kxm: STEEL_API_KEY is deprecated for Steel. Steel and Caddy do not enforce it. " +
|
|
193
|
+
"Set STEEL_AUTH_HEADER, or STEEL_AUTH_BASIC, or STEEL_AUTH_USER and STEEL_AUTH_TOKEN. " +
|
|
194
|
+
"Those override STEEL_API_KEY. Send Authorization on the request, not in the URL.\n";
|
|
194
195
|
|
|
195
196
|
let legacySteelAuthWarned = false;
|
|
196
197
|
|
|
@@ -310,6 +311,7 @@ export function resolvePassCliApiKey(
|
|
|
310
311
|
}
|
|
311
312
|
}
|
|
312
313
|
try {
|
|
314
|
+
// Deprecated lookup. The hosted server does not enforce STEEL_API_KEY.
|
|
313
315
|
const output = execFn(
|
|
314
316
|
'pass-cli item view --vault-name "AI Provider Keys" --item-title "Steel Browser (KontextMind DOKS)" --output json',
|
|
315
317
|
);
|
|
@@ -325,8 +327,9 @@ export function resolvePassCliApiKey(
|
|
|
325
327
|
}
|
|
326
328
|
|
|
327
329
|
/**
|
|
328
|
-
* Resolve Steel configuration from environment
|
|
330
|
+
* Resolve Steel configuration from the environment.
|
|
329
331
|
* Does not write secrets to disk or logs.
|
|
332
|
+
* Authentik Basic auth overrides `STEEL_API_KEY`. Steel and Caddy do not enforce that key.
|
|
330
333
|
*
|
|
331
334
|
* Authentik Basic auth (`STEEL_AUTH_HEADER`, `STEEL_AUTH_BASIC`, or
|
|
332
335
|
* `STEEL_AUTH_USER` + `STEEL_AUTH_TOKEN`) wins over `STEEL_API_KEY`.
|
|
@@ -610,7 +613,7 @@ export class SteelClient {
|
|
|
610
613
|
}
|
|
611
614
|
|
|
612
615
|
/**
|
|
613
|
-
* Launch a new Steel browser session
|
|
616
|
+
* Launch a new Steel browser session.
|
|
614
617
|
*/
|
|
615
618
|
async createSession(options?: CreateSessionOptions): Promise<SteelSession> {
|
|
616
619
|
const timeoutMs = options?.timeoutMs ?? this.config.timeoutMs ?? 300000;
|
|
@@ -11,7 +11,7 @@ import { deliverInboxNotification } from "./inbox.ts";
|
|
|
11
11
|
import type { HubEvent, MessageRecord } from "./protocol.ts";
|
|
12
12
|
import { sessionTokenFixHint } from "./session-token-hint.ts";
|
|
13
13
|
|
|
14
|
-
const VERSION = "0.7.
|
|
14
|
+
const VERSION = "0.7.148";
|
|
15
15
|
const CONFIGURE_PLUGIN = "/plugin configure kxm@kxm";
|
|
16
16
|
const inbox = new Map<string, MessageRecord>();
|
|
17
17
|
const notifiedInbox = new Set<string>();
|
package/plugins/kxm/src/modes.ts
CHANGED
|
@@ -123,7 +123,7 @@ export const DEFAULT_MODES_CONFIG: ModesConfig = Object.freeze({
|
|
|
123
123
|
browser: {
|
|
124
124
|
description: "Remote Steel browser sessions and visual testing",
|
|
125
125
|
tools: ["steel_session", "steel_scrape", "steel_screenshot"],
|
|
126
|
-
promptSnippet: "
|
|
126
|
+
promptSnippet: "Playwright tests use Obscura. Steel is remote browsing and takeover through Caddy and Authentik; send Authorization, never a credential in the URL.",
|
|
127
127
|
},
|
|
128
128
|
},
|
|
129
129
|
});
|
|
@@ -127,7 +127,9 @@ export function formatShipLine(ship?: SessionShipStatus): string {
|
|
|
127
127
|
|
|
128
128
|
export function readGitShip(cwd: string): SessionShipStatus | undefined {
|
|
129
129
|
try {
|
|
130
|
-
|
|
130
|
+
// --no-optional-locks keeps a ship-status read from rewriting .git/index.
|
|
131
|
+
// git status refreshes the index under an optional lock; a dry run must not.
|
|
132
|
+
const dirty = spawnSync("git", ["--no-optional-locks", "-C", cwd, "status", "--porcelain"], { encoding: "utf8", windowsHide: true });
|
|
131
133
|
if (dirty.status !== 0) return undefined;
|
|
132
134
|
const isDirty = dirty.stdout.trim().length > 0;
|
|
133
135
|
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
// Classify a pull request or push into the CI lanes.
|
|
4
|
+
//
|
|
5
|
+
// Docs and plan markdown skip the code jobs. The aggregate `required` job
|
|
6
|
+
// still runs, so a skipped matrix does not leave the status check blank.
|
|
7
|
+
// Platform-sensitive paths add the Windows unit shards on pull requests.
|
|
8
|
+
// Pushes to main and workflow_dispatch keep the full OS x Node validate matrix.
|
|
9
|
+
|
|
10
|
+
import { readFileSync } from "node:fs";
|
|
11
|
+
|
|
12
|
+
const DOCS_RULES = [
|
|
13
|
+
(file) => file.endsWith(".md"),
|
|
14
|
+
(file) => file.startsWith("docs/"),
|
|
15
|
+
(file) => file.startsWith(".kxm/assets/"),
|
|
16
|
+
(file) => file === "LICENSE",
|
|
17
|
+
(file) => file.startsWith(".github/ISSUE_TEMPLATE/"),
|
|
18
|
+
(file) => file === ".github/pull_request_template.md",
|
|
19
|
+
(file) => file === ".github/dependabot.yml",
|
|
20
|
+
];
|
|
21
|
+
|
|
22
|
+
// Filename tokens for path, process, shell, and spawn behavior, plus the
|
|
23
|
+
// dependency and workflow files that change how those paths run.
|
|
24
|
+
const PLATFORM_NAME = /(path|paths|process|shell|spawn|worker|supervisor|repo-root|ssh-remote)/i;
|
|
25
|
+
|
|
26
|
+
export function isDocsPath(file) {
|
|
27
|
+
return DOCS_RULES.some((rule) => rule(file));
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
export function isPlatformPath(file) {
|
|
31
|
+
if (/(^|\/)package\.json$/.test(file)) return true;
|
|
32
|
+
if (/(^|\/)package-lock\.json$/.test(file)) return true;
|
|
33
|
+
if (file.startsWith(".github/workflows/")) return true;
|
|
34
|
+
if (file.startsWith("scripts/")) return true;
|
|
35
|
+
const base = file.split("/").pop() ?? file;
|
|
36
|
+
return PLATFORM_NAME.test(base);
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
export function unboundedClassification() {
|
|
40
|
+
return { code: true, platform: false, runValidate: true };
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
export function classifyPaths(paths, event) {
|
|
44
|
+
const files = paths.map((file) => file.trim()).filter(Boolean);
|
|
45
|
+
const code = files.some((file) => !isDocsPath(file));
|
|
46
|
+
const platform = files.some((file) => isPlatformPath(file));
|
|
47
|
+
const runValidate = event !== "pull_request" && code;
|
|
48
|
+
return { code, platform, runValidate };
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
function emit(result) {
|
|
52
|
+
process.stdout.write(`code=${result.code}\n`);
|
|
53
|
+
process.stdout.write(`platform=${result.platform}\n`);
|
|
54
|
+
process.stdout.write(`run_validate=${result.runValidate}\n`);
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
if (process.argv[1] && process.argv[1].endsWith("ci-classify.mjs")) {
|
|
58
|
+
const event = process.argv[2] ?? "";
|
|
59
|
+
if (event !== "pull_request" && event !== "push" && event !== "workflow_dispatch" && event !== "unbounded") {
|
|
60
|
+
process.stderr.write("usage: ci-classify.mjs <pull_request|push|workflow_dispatch|unbounded>\n");
|
|
61
|
+
process.exit(2);
|
|
62
|
+
}
|
|
63
|
+
if (event === "unbounded") {
|
|
64
|
+
emit(unboundedClassification());
|
|
65
|
+
} else {
|
|
66
|
+
const stdin = readFileSync(0, "utf8");
|
|
67
|
+
emit(classifyPaths(stdin.split(/\r?\n/), event));
|
|
68
|
+
}
|
|
69
|
+
}
|
|
@@ -0,0 +1,230 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
// Unit lanes for CI.
|
|
4
|
+
//
|
|
5
|
+
// Measured on Node 24 (4 cores), one file at a time: engine.test.ts 174s,
|
|
6
|
+
// permission.test.ts 74s, runtime.test.ts 53s. Those files run their tests
|
|
7
|
+
// sequentially, and running them in one process pool stretched the rest of
|
|
8
|
+
// the suite to 268s. engine.test.ts is split by test name across two jobs.
|
|
9
|
+
// permission and runtime run one file at a time in a serial job so they do
|
|
10
|
+
// not steal cores from each other. Every other unit file runs in a light
|
|
11
|
+
// job at concurrency 4.
|
|
12
|
+
//
|
|
13
|
+
// Dynamic `test(\`...\${...}\`)` names stay together as one pattern so a
|
|
14
|
+
// loop is not dropped. Run via `npm run test:ci-shard` so npm_execpath is
|
|
15
|
+
// set for the packed-install test: `engine <index> <total>`, `serial`, or
|
|
16
|
+
// `light`.
|
|
17
|
+
|
|
18
|
+
import { spawn, spawnSync } from "node:child_process";
|
|
19
|
+
import { globSync, readFileSync } from "node:fs";
|
|
20
|
+
import { join } from "node:path";
|
|
21
|
+
|
|
22
|
+
export const SHARD_TOTAL = 2;
|
|
23
|
+
export const ENGINE_FILE = "test/core/engine.test.ts";
|
|
24
|
+
// Solo Node 24 timings: permission 74s, runtime 53s. One file at a time.
|
|
25
|
+
export const SERIAL_FILES = [
|
|
26
|
+
"test/core/permission.test.ts",
|
|
27
|
+
"test/core/runtime.test.ts",
|
|
28
|
+
];
|
|
29
|
+
const LIGHT_CONCURRENCY = 4;
|
|
30
|
+
|
|
31
|
+
function escapeRegExp(value) {
|
|
32
|
+
return value.replace(/[|\\{}()[\]^$+*?.]/g, "\\$&");
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
export function listUnitFiles(root) {
|
|
36
|
+
return [
|
|
37
|
+
...globSync("test/core/*.test.ts", { cwd: root }),
|
|
38
|
+
...globSync("packages/core/*/tests/unit/*.test.ts", { cwd: root }),
|
|
39
|
+
].sort();
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
export function extractTestPatterns(source) {
|
|
43
|
+
const patterns = [];
|
|
44
|
+
const seen = new Set();
|
|
45
|
+
const re = /^[ \t]*test\(\s*(["'`])([\s\S]*?)\1/gm;
|
|
46
|
+
for (const match of source.matchAll(re)) {
|
|
47
|
+
const raw = match[2];
|
|
48
|
+
let sourcePattern;
|
|
49
|
+
let dynamic = false;
|
|
50
|
+
let name;
|
|
51
|
+
if (raw.includes("${")) {
|
|
52
|
+
dynamic = true;
|
|
53
|
+
const marked = raw.replace(/\$\{[^}]*\}/g, "\0");
|
|
54
|
+
sourcePattern = `^${escapeRegExp(marked).replaceAll("\0", ".*?")}$`;
|
|
55
|
+
} else {
|
|
56
|
+
name = raw.replace(/\\(["'`\\])/g, "$1");
|
|
57
|
+
sourcePattern = `^${escapeRegExp(name)}$`;
|
|
58
|
+
}
|
|
59
|
+
if (seen.has(sourcePattern)) continue;
|
|
60
|
+
seen.add(sourcePattern);
|
|
61
|
+
patterns.push({ dynamic, name, source: sourcePattern });
|
|
62
|
+
}
|
|
63
|
+
return patterns;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
function conflicts(patterns) {
|
|
67
|
+
const exact = patterns.filter((pattern) => !pattern.dynamic);
|
|
68
|
+
const dynamic = patterns.filter((pattern) => pattern.dynamic);
|
|
69
|
+
const hits = [];
|
|
70
|
+
for (const pattern of dynamic) {
|
|
71
|
+
const regex = new RegExp(pattern.source);
|
|
72
|
+
for (const item of exact) {
|
|
73
|
+
if (regex.test(item.name)) hits.push(`${item.name} matches ${pattern.source}`);
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
return hits;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
function enginePatterns(root) {
|
|
80
|
+
const patterns = extractTestPatterns(readFileSync(join(root, ENGINE_FILE), "utf8"));
|
|
81
|
+
if (patterns.length === 0) throw new Error(`${ENGINE_FILE} has no recognizable test() names`);
|
|
82
|
+
const overlapped = conflicts(patterns);
|
|
83
|
+
if (overlapped.length > 0) {
|
|
84
|
+
throw new Error(`${ENGINE_FILE} dynamic test names also match exact tests:\n${overlapped.join("\n")}`);
|
|
85
|
+
}
|
|
86
|
+
return patterns;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
export function planEngineShard(root, index, total = SHARD_TOTAL) {
|
|
90
|
+
if (!Number.isInteger(index) || !Number.isInteger(total) || index < 1 || index > total) {
|
|
91
|
+
throw new Error(`shard ${index}/${total} is outside 1..${total}`);
|
|
92
|
+
}
|
|
93
|
+
const patterns = enginePatterns(root).filter((_, patternIndex) => patternIndex % total === index - 1);
|
|
94
|
+
if (patterns.length === 0) throw new Error(`engine shard ${index}/${total} assigned no tests`);
|
|
95
|
+
return { file: ENGINE_FILE, patterns };
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
export function planSerial(root) {
|
|
99
|
+
const files = listUnitFiles(root);
|
|
100
|
+
if (!files.includes(ENGINE_FILE)) throw new Error(`${ENGINE_FILE} is missing`);
|
|
101
|
+
for (const file of SERIAL_FILES) {
|
|
102
|
+
if (!files.includes(file)) throw new Error(`${file} is missing`);
|
|
103
|
+
}
|
|
104
|
+
return [...SERIAL_FILES];
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
export function planLight(root) {
|
|
108
|
+
const serial = new Set(SERIAL_FILES);
|
|
109
|
+
const files = listUnitFiles(root).filter((file) => file !== ENGINE_FILE && !serial.has(file));
|
|
110
|
+
if (files.length === 0) throw new Error("light lane has no unit files");
|
|
111
|
+
return files;
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
export function coverageOfShards(root, total = SHARD_TOTAL) {
|
|
115
|
+
const heavy = new Map();
|
|
116
|
+
for (let index = 1; index <= total; index += 1) {
|
|
117
|
+
const plan = planEngineShard(root, index, total);
|
|
118
|
+
for (const pattern of plan.patterns) heavy.set(pattern.source, (heavy.get(pattern.source) ?? 0) + 1);
|
|
119
|
+
}
|
|
120
|
+
return { files: listUnitFiles(root), serial: planSerial(root), light: planLight(root), heavy };
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
function runNode(args, label, children) {
|
|
124
|
+
return new Promise((resolve) => {
|
|
125
|
+
const child = spawn(process.execPath, args, { env: process.env });
|
|
126
|
+
children.add(child);
|
|
127
|
+
let stdout = "";
|
|
128
|
+
let stderr = "";
|
|
129
|
+
const flush = (buffer, stream, chunk) => {
|
|
130
|
+
buffer += chunk.toString();
|
|
131
|
+
const lines = buffer.split("\n");
|
|
132
|
+
const rest = lines.pop() ?? "";
|
|
133
|
+
for (const line of lines) stream.write(`[${label}] ${line}\n`);
|
|
134
|
+
return rest;
|
|
135
|
+
};
|
|
136
|
+
child.stdout.on("data", (chunk) => { stdout = flush(stdout, process.stdout, chunk); });
|
|
137
|
+
child.stderr.on("data", (chunk) => { stderr = flush(stderr, process.stderr, chunk); });
|
|
138
|
+
child.on("close", (code) => {
|
|
139
|
+
children.delete(child);
|
|
140
|
+
if (stdout) process.stdout.write(`[${label}] ${stdout}\n`);
|
|
141
|
+
if (stderr) process.stderr.write(`[${label}] ${stderr}\n`);
|
|
142
|
+
resolve(code ?? 1);
|
|
143
|
+
});
|
|
144
|
+
child.on("error", (error) => {
|
|
145
|
+
children.delete(child);
|
|
146
|
+
process.stderr.write(`[${label}] ${error.message}\n`);
|
|
147
|
+
resolve(1);
|
|
148
|
+
});
|
|
149
|
+
});
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
function testArgs(concurrency, files, patterns) {
|
|
153
|
+
const args = [
|
|
154
|
+
"--disable-warning=ExperimentalWarning",
|
|
155
|
+
"--experimental-strip-types",
|
|
156
|
+
"--test",
|
|
157
|
+
"--test-force-exit",
|
|
158
|
+
`--test-concurrency=${concurrency}`,
|
|
159
|
+
];
|
|
160
|
+
for (const pattern of patterns ?? []) args.push("--test-name-pattern", pattern.source);
|
|
161
|
+
args.push(...files);
|
|
162
|
+
return args;
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
async function runPool(tasks, limit) {
|
|
166
|
+
const pending = [...tasks];
|
|
167
|
+
const children = new Set();
|
|
168
|
+
let failed = false;
|
|
169
|
+
const workers = Array.from({ length: Math.min(limit, pending.length) }, async () => {
|
|
170
|
+
while (pending.length > 0 && !failed) {
|
|
171
|
+
const task = pending.shift();
|
|
172
|
+
const code = await task(children);
|
|
173
|
+
if (code !== 0) {
|
|
174
|
+
failed = true;
|
|
175
|
+
for (const child of children) child.kill("SIGTERM");
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
});
|
|
179
|
+
await Promise.all(workers);
|
|
180
|
+
return failed ? 1 : 0;
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
function npmBuild() {
|
|
184
|
+
const npmCli = process.env.npm_execpath;
|
|
185
|
+
if (!npmCli) {
|
|
186
|
+
process.stderr.write("npm_execpath is required; run via npm run test:ci-shard\n");
|
|
187
|
+
return 1;
|
|
188
|
+
}
|
|
189
|
+
const result = spawnSync(process.execPath, [npmCli, "run", "build"], {
|
|
190
|
+
stdio: "inherit",
|
|
191
|
+
env: process.env,
|
|
192
|
+
});
|
|
193
|
+
return result.status ?? 1;
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
async function main() {
|
|
197
|
+
const mode = process.argv[2];
|
|
198
|
+
const root = process.cwd();
|
|
199
|
+
const built = npmBuild();
|
|
200
|
+
if (built !== 0) process.exit(built);
|
|
201
|
+
if (mode === "serial") {
|
|
202
|
+
const files = planSerial(root);
|
|
203
|
+
process.exit(await runPool([
|
|
204
|
+
(children) => runNode(testArgs(1, files), "serial", children),
|
|
205
|
+
], 1));
|
|
206
|
+
}
|
|
207
|
+
if (mode === "light") {
|
|
208
|
+
const files = planLight(root);
|
|
209
|
+
process.exit(await runPool([
|
|
210
|
+
(children) => runNode(testArgs(LIGHT_CONCURRENCY, files), "light", children),
|
|
211
|
+
], 1));
|
|
212
|
+
}
|
|
213
|
+
if (mode === "engine") {
|
|
214
|
+
const index = Number(process.argv[3]);
|
|
215
|
+
const total = Number(process.argv[4] ?? SHARD_TOTAL);
|
|
216
|
+
const plan = planEngineShard(root, index, total);
|
|
217
|
+
process.exit(await runPool([
|
|
218
|
+
(children) => runNode(testArgs(1, [plan.file], plan.patterns), "engine", children),
|
|
219
|
+
], 1));
|
|
220
|
+
}
|
|
221
|
+
process.stderr.write("usage: ci-unit-shard.mjs <serial|light|engine> [index total]\n");
|
|
222
|
+
process.exit(2);
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
if (process.argv[1] && process.argv[1].endsWith("ci-unit-shard.mjs")) {
|
|
226
|
+
main().catch((error) => {
|
|
227
|
+
process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`);
|
|
228
|
+
process.exit(1);
|
|
229
|
+
});
|
|
230
|
+
}
|