dsh-test-drive 0.1.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +16 -0
- package/README.es.md +18 -3
- package/README.hi.md +18 -3
- package/README.md +19 -4
- package/README.pt.md +18 -3
- package/README.zh.md +18 -3
- package/cordis.patch.yml +14 -0
- package/lib/index.js +431 -10
- package/lib/types/batch.d.ts +2 -2
- package/lib/types/capability.d.ts +74 -0
- package/lib/types/capability.d.ts.map +1 -0
- package/lib/types/command.d.ts +1 -1
- package/lib/types/config.d.ts +33 -0
- package/lib/types/config.d.ts.map +1 -1
- package/lib/types/domain.d.ts +19 -7
- package/lib/types/domain.d.ts.map +1 -1
- package/lib/types/drive.d.ts +10 -5
- package/lib/types/drive.d.ts.map +1 -1
- package/lib/types/driver.d.ts +13 -2
- package/lib/types/driver.d.ts.map +1 -1
- package/lib/types/index.d.ts +16 -16
- package/lib/types/report.d.ts +2 -2
- package/lib/types/report.d.ts.map +1 -1
- package/lib/types/result.d.ts +31 -1
- package/lib/types/result.d.ts.map +1 -1
- package/lib/types/tools.d.ts +5 -5
- package/lib/types/tools.d.ts.map +1 -1
- package/lib/types/version.d.ts +1 -1
- package/package.json +15 -9
package/lib/index.js
CHANGED
|
@@ -1,11 +1,161 @@
|
|
|
1
1
|
import { randomBytes } from "node:crypto";
|
|
2
|
-
import { access, mkdir, mkdtemp, readFile, rename, rm, writeFile } from "node:fs/promises";
|
|
3
|
-
import { basename, dirname, isAbsolute, join, resolve } from "node:path";
|
|
4
2
|
import { z } from "zod";
|
|
3
|
+
import { access, mkdir, mkdtemp, readFile, readdir, rename, rm, stat, writeFile } from "node:fs/promises";
|
|
4
|
+
import { basename, dirname, isAbsolute, join, resolve } from "node:path";
|
|
5
5
|
import z$1 from "@deepseek-ai/schemastery";
|
|
6
6
|
import { defineDomain, domainTable } from "@deepseek-ai/dsh-storage-domain";
|
|
7
7
|
import { defineTool } from "@deepseek-ai/dsh-tools";
|
|
8
8
|
import { tmpdir } from "node:os";
|
|
9
|
+
//#region src/capability.ts
|
|
10
|
+
/**
|
|
11
|
+
* The capability-assertion stage: beyond "booted and exited", prove one NAMED
|
|
12
|
+
* capability of the installed plugin is registered, invoked, and its effect
|
|
13
|
+
* OBSERVED — the omdsh-INTAKE principle that a clean boot is only a smoke
|
|
14
|
+
* test. The stage runs one headless task that asks the agent to call a tool
|
|
15
|
+
* (or run a command), then analyzes the durable session log for the paired
|
|
16
|
+
* `tool/call` → `tool/result` events (or the command invocation and reply) and
|
|
17
|
+
* matches a caller-supplied expectation against the observed output.
|
|
18
|
+
*
|
|
19
|
+
* Everything here is pure and tolerant: the analyzer never throws on hostile
|
|
20
|
+
* or truncated log lines (it skips them), so the capability stage itself can
|
|
21
|
+
* only fail through its own process outcome, never through log parsing.
|
|
22
|
+
*
|
|
23
|
+
* @module dsh-test-drive/capability
|
|
24
|
+
*/
|
|
25
|
+
/** The capability kinds the stage can assert. */
|
|
26
|
+
const CapabilityKindSchema = z.union([z.literal("tool"), z.literal("command")]);
|
|
27
|
+
/** Outcome of the capability stage (registered + invoked + observed, in order). */
|
|
28
|
+
const CapabilityStatusSchema = z.union([
|
|
29
|
+
z.literal("observed"),
|
|
30
|
+
z.literal("invoked"),
|
|
31
|
+
z.literal("not-registered"),
|
|
32
|
+
z.literal("skipped"),
|
|
33
|
+
z.literal("failed")
|
|
34
|
+
]);
|
|
35
|
+
/** The exact reply the task text demands; also the run-completion marker. */
|
|
36
|
+
const CAPABILITY_DONE_MARKER = "capability-check-done";
|
|
37
|
+
/** Maximum length of the `args` payload handed to the capability task. */
|
|
38
|
+
const MAX_CAPABILITY_ARGS_LENGTH = 4e3;
|
|
39
|
+
/** Maximum length of the `expect` literal matched against the observed output. */
|
|
40
|
+
const MAX_CAPABILITY_EXPECT_LENGTH = 2e3;
|
|
41
|
+
/** Allowed characters for an asserted tool or command name. */
|
|
42
|
+
const CAPABILITY_NAME_PATTERN = /^[a-zA-Z0-9][a-zA-Z0-9._/-]*$/;
|
|
43
|
+
/** Regex-escape a literal for embedding in a pattern. */
|
|
44
|
+
function escapeRegex$1(source) {
|
|
45
|
+
return source.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&");
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* Compile the headless task text that makes the agent exercise the capability.
|
|
49
|
+
*
|
|
50
|
+
* @param spec - the capability to exercise.
|
|
51
|
+
* @returns the task text; never empty.
|
|
52
|
+
*/
|
|
53
|
+
function buildCapabilityTask(spec) {
|
|
54
|
+
if (spec.kind === "tool") return [
|
|
55
|
+
`Call the tool "${spec.name}" with exactly these arguments:`,
|
|
56
|
+
spec.args === "" ? "(no arguments)" : spec.args,
|
|
57
|
+
`After the tool returns, reply with exactly: ${CAPABILITY_DONE_MARKER}`
|
|
58
|
+
].join("\n");
|
|
59
|
+
return [`Run the command /${spec.name}${spec.args === "" ? "" : ` ${spec.args}`}.`, `Then reply with exactly: ${CAPABILITY_DONE_MARKER}`].join("\n");
|
|
60
|
+
}
|
|
61
|
+
/** Parse one JSONL line into `{ type, data }` or undefined for unparseable lines. */
|
|
62
|
+
function parseLine(line) {
|
|
63
|
+
try {
|
|
64
|
+
const value = JSON.parse(line);
|
|
65
|
+
if (typeof value.type !== "string" || value.data === void 0) return void 0;
|
|
66
|
+
return {
|
|
67
|
+
type: value.type,
|
|
68
|
+
data: value.data
|
|
69
|
+
};
|
|
70
|
+
} catch {
|
|
71
|
+
return;
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
/** Lowercased serialization of one event payload for expectation matching. */
|
|
75
|
+
function haystack(data) {
|
|
76
|
+
return JSON.stringify(data).toLowerCase();
|
|
77
|
+
}
|
|
78
|
+
/** Whether the serialized payload contains the expectation (case-insensitive). */
|
|
79
|
+
function matchesExpect(data, expect) {
|
|
80
|
+
return expect !== "" && haystack(data).includes(expect.toLowerCase());
|
|
81
|
+
}
|
|
82
|
+
/**
|
|
83
|
+
* Analyze a durable session log against one capability spec. Tolerant: every
|
|
84
|
+
* unparseable or unrelated line is skipped, and missing evidence degrades to
|
|
85
|
+
* the highest status actually reached (`not-registered` when the invocation
|
|
86
|
+
* never appears, `invoked` when it ran but the expectation never appeared,
|
|
87
|
+
* `observed` only when both hold).
|
|
88
|
+
*
|
|
89
|
+
* @param log - the raw JSONL session text (as persisted by the harness).
|
|
90
|
+
* @param spec - the asserted capability.
|
|
91
|
+
* @returns the settled analysis; never throws.
|
|
92
|
+
*/
|
|
93
|
+
function analyzeSessionLog(log, spec) {
|
|
94
|
+
const lines = [];
|
|
95
|
+
for (const line of log.split("\n")) {
|
|
96
|
+
const parsed = parseLine(line);
|
|
97
|
+
if (parsed !== void 0) lines.push(parsed);
|
|
98
|
+
}
|
|
99
|
+
if (spec.kind === "tool") {
|
|
100
|
+
const callIds = [];
|
|
101
|
+
for (const line of lines) {
|
|
102
|
+
if (line.type !== "tool/call") continue;
|
|
103
|
+
const data = line.data;
|
|
104
|
+
if (data.name === spec.name && typeof data.callId === "string") callIds.push(data.callId);
|
|
105
|
+
}
|
|
106
|
+
if (callIds.length === 0) return {
|
|
107
|
+
status: "not-registered",
|
|
108
|
+
detail: `no tool/call event names "${spec.name}" — the tool was never registered or the agent never invoked it`,
|
|
109
|
+
expectMatched: false
|
|
110
|
+
};
|
|
111
|
+
for (const line of lines) {
|
|
112
|
+
if (line.type !== "tool/result") continue;
|
|
113
|
+
const serialized = JSON.stringify(line.data);
|
|
114
|
+
if (!callIds.some((callId) => serialized.includes(callId))) continue;
|
|
115
|
+
if (matchesExpect(line.data, spec.expect)) return {
|
|
116
|
+
status: "observed",
|
|
117
|
+
detail: `tool "${spec.name}" called and its result contains the expectation`,
|
|
118
|
+
expectMatched: true
|
|
119
|
+
};
|
|
120
|
+
}
|
|
121
|
+
return {
|
|
122
|
+
status: "invoked",
|
|
123
|
+
detail: `tool "${spec.name}" was called (${callIds.length} call(s)) but no paired result contains the expectation`,
|
|
124
|
+
expectMatched: false
|
|
125
|
+
};
|
|
126
|
+
}
|
|
127
|
+
const commandPattern = new RegExp(`/${escapeRegex$1(spec.name)}(?:\\s|$)`, "u");
|
|
128
|
+
let invokedAt = -1;
|
|
129
|
+
for (let i = 0; i < lines.length; i += 1) {
|
|
130
|
+
const line = lines[i];
|
|
131
|
+
if (line === void 0) continue;
|
|
132
|
+
if (line.type !== "user/message") continue;
|
|
133
|
+
if (commandPattern.test(JSON.stringify(line.data))) {
|
|
134
|
+
invokedAt = i;
|
|
135
|
+
break;
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
if (invokedAt === -1) return {
|
|
139
|
+
status: "not-registered",
|
|
140
|
+
detail: `no user message invoking /${spec.name} — the agent never ran the command`,
|
|
141
|
+
expectMatched: false
|
|
142
|
+
};
|
|
143
|
+
for (let i = invokedAt + 1; i < lines.length; i += 1) {
|
|
144
|
+
const line = lines[i];
|
|
145
|
+
if (line === void 0) continue;
|
|
146
|
+
if (line.type === "assistant/message" && matchesExpect(line.data, spec.expect)) return {
|
|
147
|
+
status: "observed",
|
|
148
|
+
detail: `command /${spec.name} ran and a reply contains the expectation`,
|
|
149
|
+
expectMatched: true
|
|
150
|
+
};
|
|
151
|
+
}
|
|
152
|
+
return {
|
|
153
|
+
status: "invoked",
|
|
154
|
+
detail: `command /${spec.name} was invoked but no reply contains the expectation (unknown command or empty output)`,
|
|
155
|
+
expectMatched: false
|
|
156
|
+
};
|
|
157
|
+
}
|
|
158
|
+
//#endregion
|
|
9
159
|
//#region src/driver.ts
|
|
10
160
|
/**
|
|
11
161
|
* The `dsh` CLI driver: locates a spawnable dsh executable (config override,
|
|
@@ -269,6 +419,43 @@ var DshDriver = class {
|
|
|
269
419
|
task
|
|
270
420
|
], options);
|
|
271
421
|
}
|
|
422
|
+
/**
|
|
423
|
+
* Read the newest durable session log from a throwaway home's session store.
|
|
424
|
+
* Used by the capability stage to observe what the headless task actually
|
|
425
|
+
* did (tool calls, command replies). Tolerant by contract: a missing or
|
|
426
|
+
* empty store yields '' so the stage degrades instead of throwing.
|
|
427
|
+
*
|
|
428
|
+
* @param home - the throwaway DSH_HOME.
|
|
429
|
+
* @param maxBytes - cap on the returned text (tail end kept).
|
|
430
|
+
* @returns the newest session file's text, or '' when none exists.
|
|
431
|
+
*/
|
|
432
|
+
async readNewestSession(home, maxBytes) {
|
|
433
|
+
let entries;
|
|
434
|
+
try {
|
|
435
|
+
entries = await readdir(join(home, "sessions"));
|
|
436
|
+
} catch {
|
|
437
|
+
return "";
|
|
438
|
+
}
|
|
439
|
+
let newest;
|
|
440
|
+
for (const entry of entries) {
|
|
441
|
+
if (!/\.jsonl$/u.test(entry)) continue;
|
|
442
|
+
const path = join(home, "sessions", entry);
|
|
443
|
+
try {
|
|
444
|
+
const info = await stat(path);
|
|
445
|
+
if (info.isFile() && (newest === void 0 || info.mtimeMs > newest.mtimeMs)) newest = {
|
|
446
|
+
path,
|
|
447
|
+
mtimeMs: info.mtimeMs
|
|
448
|
+
};
|
|
449
|
+
} catch {}
|
|
450
|
+
}
|
|
451
|
+
if (newest === void 0) return "";
|
|
452
|
+
try {
|
|
453
|
+
const text = await readFile(newest.path, "utf8");
|
|
454
|
+
return text.length <= maxBytes ? text : text.slice(text.length - maxBytes);
|
|
455
|
+
} catch {
|
|
456
|
+
return "";
|
|
457
|
+
}
|
|
458
|
+
}
|
|
272
459
|
/** Run the uninstall stage: `dsh plugin --profile <name> remove <pkg>`. */
|
|
273
460
|
async remove(packageName, options) {
|
|
274
461
|
return this.run([
|
|
@@ -407,6 +594,19 @@ const SmokeStageResultSchema = StageResultSchema.extend({
|
|
|
407
594
|
});
|
|
408
595
|
/** Uninstall stage result. */
|
|
409
596
|
const UninstallStageResultSchema = StageResultSchema;
|
|
597
|
+
/** Capability-assertion stage: one named tool or command, registered → invoked → observed. */
|
|
598
|
+
const CapabilityStageResultSchema = StageResultSchema.extend({
|
|
599
|
+
/** `observed`/`invoked`/`not-registered`/`skipped`/`failed`. */
|
|
600
|
+
status: CapabilityStatusSchema,
|
|
601
|
+
/** What was asserted (tool or command); null when the stage never ran. */
|
|
602
|
+
capabilityKind: CapabilityKindSchema.nullable(),
|
|
603
|
+
/** The asserted tool or command name; '' when the stage never ran. */
|
|
604
|
+
name: z.string(),
|
|
605
|
+
/** Whether the expectation literal was found in the observed output. */
|
|
606
|
+
expectMatched: z.boolean(),
|
|
607
|
+
/** One-line sanitized explanation of the reached status. */
|
|
608
|
+
detail: z.string()
|
|
609
|
+
});
|
|
410
610
|
/** Cleanup stage: quarantine/removal facts for the temp directory this run owned. */
|
|
411
611
|
const CleanupStageResultSchema = z.object({
|
|
412
612
|
/** `pass` when the owned temp root was removed; `skipped` when keepTempDirs was set. */
|
|
@@ -483,6 +683,12 @@ const DriveResultSchema = z.object({
|
|
|
483
683
|
install: InstallStageResultSchema,
|
|
484
684
|
config: ConfigStageResultSchema,
|
|
485
685
|
smoke: SmokeStageResultSchema,
|
|
686
|
+
/**
|
|
687
|
+
* Optional capability-assertion stage (added after v1 records existed;
|
|
688
|
+
* optional so pre-capability records still validate at the durable
|
|
689
|
+
* boundary — no domain-version bump for a backward-compatible addition).
|
|
690
|
+
*/
|
|
691
|
+
capability: CapabilityStageResultSchema.optional(),
|
|
486
692
|
uninstall: UninstallStageResultSchema,
|
|
487
693
|
cleanup: CleanupStageResultSchema
|
|
488
694
|
}),
|
|
@@ -556,7 +762,8 @@ function totalsOf(rows) {
|
|
|
556
762
|
* Derive the overall verdict of one drive run from its stage outcomes.
|
|
557
763
|
*
|
|
558
764
|
* Rules, in order:
|
|
559
|
-
* 1. A failed install
|
|
765
|
+
* 1. A failed install, a failed boot (`smoke.fail`), or a capability stage
|
|
766
|
+
* that reached `not-registered`/`failed` is a hard `fail`.
|
|
560
767
|
* 2. `pass` needs install pass + patch effective + a clean boot (`pass` or
|
|
561
768
|
* `boot-ok`) + a successful uninstall.
|
|
562
769
|
* 3. `partial` covers everything that installed but missed a later assurance
|
|
@@ -567,12 +774,14 @@ function totalsOf(rows) {
|
|
|
567
774
|
* @returns `[verdict, reason]`.
|
|
568
775
|
*/
|
|
569
776
|
function verdictOf(result) {
|
|
570
|
-
const { install, config, smoke, uninstall } = result.stages;
|
|
777
|
+
const { install, config, smoke, uninstall, capability } = result.stages;
|
|
571
778
|
if (install.status === "fail") return ["fail", `install failed: ${install.summary}`];
|
|
572
779
|
if (smoke.status === "fail") return ["fail", `boot smoke failed: ${smoke.summary}`];
|
|
780
|
+
if (capability !== void 0 && (capability.status === "not-registered" || capability.status === "failed")) return ["fail", `capability assertion failed (${capability.name}): ${capability.detail}`];
|
|
573
781
|
if (install.status === "pass" && config.status === "pass" && config.patchEffective && (smoke.status === "pass" || smoke.status === "boot-ok") && uninstall.status === "pass") {
|
|
574
|
-
|
|
575
|
-
return ["pass",
|
|
782
|
+
const capabilityNote = capability !== void 0 && capability.status === "observed" ? `; capability "${capability.name}" registered, invoked, and observed` : "";
|
|
783
|
+
if (smoke.status === "boot-ok") return ["pass", `install, patch, boot, and uninstall verified${capabilityNote}; headless task inconclusive (see smoke.summary)`];
|
|
784
|
+
return ["pass", `install, patch, boot, task, and uninstall all verified${capabilityNote}`];
|
|
576
785
|
}
|
|
577
786
|
if (install.status !== "pass") return ["unknown", "install never ran to completion"];
|
|
578
787
|
if (config.status !== "pass" || !config.patchEffective) return ["partial", `install passed but the bundle patch was not verified (${config.summary})`];
|
|
@@ -674,7 +883,7 @@ function sanitizeOutput(text, tempRoot, maxChars) {
|
|
|
674
883
|
* The structured report records it as `run.pluginVersion`.
|
|
675
884
|
* @module dsh-test-drive/version
|
|
676
885
|
*/
|
|
677
|
-
const VERSION = "0.1
|
|
886
|
+
const VERSION = "0.2.1";
|
|
678
887
|
//#endregion
|
|
679
888
|
//#region src/drive.ts
|
|
680
889
|
/**
|
|
@@ -687,6 +896,8 @@ const VERSION = "0.1.0";
|
|
|
687
896
|
*
|
|
688
897
|
* @module dsh-test-drive/drive
|
|
689
898
|
*/
|
|
899
|
+
/** Cap on the session-log text the capability stage reads back. */
|
|
900
|
+
const SESSION_READ_MAX_BYTES = 1048576;
|
|
690
901
|
/** id prefix of a single drive run. */
|
|
691
902
|
const RUN_ID_PREFIX = "tdr_";
|
|
692
903
|
/** id prefix of a batch matrix. */
|
|
@@ -815,6 +1026,18 @@ var DriveRunner = class {
|
|
|
815
1026
|
bootFailed: false,
|
|
816
1027
|
taskCompleted: false
|
|
817
1028
|
};
|
|
1029
|
+
let capability = {
|
|
1030
|
+
status: "skipped",
|
|
1031
|
+
exitCode: null,
|
|
1032
|
+
durationMs: 0,
|
|
1033
|
+
attempts: 0,
|
|
1034
|
+
outputTail: "",
|
|
1035
|
+
summary: "not reached: earlier stage errored",
|
|
1036
|
+
capabilityKind: null,
|
|
1037
|
+
name: "",
|
|
1038
|
+
expectMatched: false,
|
|
1039
|
+
detail: "not reached: earlier stage errored"
|
|
1040
|
+
};
|
|
818
1041
|
let uninstall = { ...UNREACHED };
|
|
819
1042
|
let resolved;
|
|
820
1043
|
let cleanup = {
|
|
@@ -915,6 +1138,89 @@ var DriveRunner = class {
|
|
|
915
1138
|
taskCompleted: false
|
|
916
1139
|
};
|
|
917
1140
|
}
|
|
1141
|
+
const capSpec = options.capability ?? (config.capability.enabled ? {
|
|
1142
|
+
kind: config.capability.kind,
|
|
1143
|
+
name: config.capability.name,
|
|
1144
|
+
args: config.capability.args,
|
|
1145
|
+
expect: config.capability.expect
|
|
1146
|
+
} : void 0);
|
|
1147
|
+
if (capSpec === void 0) capability = {
|
|
1148
|
+
...capability,
|
|
1149
|
+
status: "skipped",
|
|
1150
|
+
summary: "capability assertion disabled (config.capability.enabled is false and no per-drive capability given)",
|
|
1151
|
+
detail: "capability assertion disabled by config"
|
|
1152
|
+
};
|
|
1153
|
+
else if (install.status !== "pass") capability = {
|
|
1154
|
+
...capability,
|
|
1155
|
+
summary: "install did not pass",
|
|
1156
|
+
detail: "install did not pass"
|
|
1157
|
+
};
|
|
1158
|
+
else if (smoke.status === "fail") capability = {
|
|
1159
|
+
...capability,
|
|
1160
|
+
summary: "boot smoke failed",
|
|
1161
|
+
detail: "boot smoke failed before the capability stage"
|
|
1162
|
+
};
|
|
1163
|
+
else if (options.signal?.aborted ?? false) capability = {
|
|
1164
|
+
...capability,
|
|
1165
|
+
summary: "run aborted before this stage",
|
|
1166
|
+
detail: "run aborted before this stage"
|
|
1167
|
+
};
|
|
1168
|
+
else if (process.env.DEEPSEEK_API_KEY === void 0 && !config.forwardEnv.includes("DEEPSEEK_API_KEY")) capability = {
|
|
1169
|
+
...capability,
|
|
1170
|
+
status: "skipped",
|
|
1171
|
+
summary: "DEEPSEEK_API_KEY not available — the capability task needs a model (set it on the host or list it in forwardEnv)",
|
|
1172
|
+
capabilityKind: capSpec.kind,
|
|
1173
|
+
name: capSpec.name,
|
|
1174
|
+
detail: "no model credential available for the capability task"
|
|
1175
|
+
};
|
|
1176
|
+
else {
|
|
1177
|
+
const run = await driver.smokeTask(buildCapabilityTask(capSpec), {
|
|
1178
|
+
...base,
|
|
1179
|
+
timeoutMs: config.capabilityTimeoutMs
|
|
1180
|
+
});
|
|
1181
|
+
const outputTail = sanitizeOutput(`${run.stdout}\n${run.stderr}`, workspace.root, config.outputTailBytes);
|
|
1182
|
+
if (run.timedOut) capability = {
|
|
1183
|
+
status: "failed",
|
|
1184
|
+
exitCode: run.exitCode,
|
|
1185
|
+
durationMs: run.durationMs,
|
|
1186
|
+
attempts: 1,
|
|
1187
|
+
summary: `timed out after ${run.durationMs} ms`,
|
|
1188
|
+
outputTail,
|
|
1189
|
+
capabilityKind: capSpec.kind,
|
|
1190
|
+
name: capSpec.name,
|
|
1191
|
+
expectMatched: false,
|
|
1192
|
+
detail: "capability task timed out"
|
|
1193
|
+
};
|
|
1194
|
+
else if (run.exitCode !== 0) {
|
|
1195
|
+
const firstLine = tailText(run.stderr || run.stdout, 160).split("\n")[0] ?? "";
|
|
1196
|
+
capability = {
|
|
1197
|
+
status: "failed",
|
|
1198
|
+
exitCode: run.exitCode,
|
|
1199
|
+
durationMs: run.durationMs,
|
|
1200
|
+
attempts: 1,
|
|
1201
|
+
summary: `exit ${String(run.exitCode)}: ${firstLine || "no output"}`,
|
|
1202
|
+
outputTail,
|
|
1203
|
+
capabilityKind: capSpec.kind,
|
|
1204
|
+
name: capSpec.name,
|
|
1205
|
+
expectMatched: false,
|
|
1206
|
+
detail: "capability task did not complete (model/credential failure)"
|
|
1207
|
+
};
|
|
1208
|
+
} else {
|
|
1209
|
+
const analysis = analyzeSessionLog(await driver.readNewestSession(workspace.home, SESSION_READ_MAX_BYTES), capSpec);
|
|
1210
|
+
capability = {
|
|
1211
|
+
status: analysis.status,
|
|
1212
|
+
exitCode: 0,
|
|
1213
|
+
durationMs: run.durationMs,
|
|
1214
|
+
attempts: 1,
|
|
1215
|
+
summary: analysis.detail,
|
|
1216
|
+
outputTail,
|
|
1217
|
+
capabilityKind: capSpec.kind,
|
|
1218
|
+
name: capSpec.name,
|
|
1219
|
+
expectMatched: analysis.expectMatched,
|
|
1220
|
+
detail: analysis.detail
|
|
1221
|
+
};
|
|
1222
|
+
}
|
|
1223
|
+
}
|
|
918
1224
|
if (resolved?.packageName === void 0) uninstall = {
|
|
919
1225
|
...uninstall,
|
|
920
1226
|
summary: "no installed package to remove"
|
|
@@ -940,6 +1246,14 @@ var DriveRunner = class {
|
|
|
940
1246
|
bootFailed: false,
|
|
941
1247
|
taskCompleted: false
|
|
942
1248
|
};
|
|
1249
|
+
else if (capability.status === "skipped") capability = {
|
|
1250
|
+
...erroredStage(summary),
|
|
1251
|
+
status: "failed",
|
|
1252
|
+
capabilityKind: null,
|
|
1253
|
+
name: "",
|
|
1254
|
+
expectMatched: false,
|
|
1255
|
+
detail: summary
|
|
1256
|
+
};
|
|
943
1257
|
else uninstall = { ...erroredStage(summary) };
|
|
944
1258
|
log(`test-drive: pipeline error for ${spec}: ${String(error)}`);
|
|
945
1259
|
} finally {
|
|
@@ -983,6 +1297,7 @@ var DriveRunner = class {
|
|
|
983
1297
|
install,
|
|
984
1298
|
config: configStage,
|
|
985
1299
|
smoke,
|
|
1300
|
+
capability,
|
|
986
1301
|
uninstall,
|
|
987
1302
|
cleanup
|
|
988
1303
|
},
|
|
@@ -1176,6 +1491,8 @@ const DEFAULT_INSTALL_TIMEOUT_MS = 6e5;
|
|
|
1176
1491
|
const DEFAULT_CONFIG_TIMEOUT_MS = 6e4;
|
|
1177
1492
|
/** Default per-stage timeout for the headless boot smoke. */
|
|
1178
1493
|
const DEFAULT_SMOKE_TIMEOUT_MS = 3e5;
|
|
1494
|
+
/** Default per-stage timeout for the capability-assertion task. */
|
|
1495
|
+
const DEFAULT_CAPABILITY_TIMEOUT_MS = 3e5;
|
|
1179
1496
|
/** Default per-stage timeout for `dsh plugin remove`. */
|
|
1180
1497
|
const DEFAULT_UNINSTALL_TIMEOUT_MS = 12e4;
|
|
1181
1498
|
/** Bounds shared by every stage timeout. */
|
|
@@ -1196,6 +1513,20 @@ const Config = z$1.object({
|
|
|
1196
1513
|
installTimeoutMs: z$1.number().min(MIN_STAGE_TIMEOUT_MS).max(MAX_STAGE_TIMEOUT_MS).default(DEFAULT_INSTALL_TIMEOUT_MS),
|
|
1197
1514
|
configTimeoutMs: z$1.number().min(MIN_STAGE_TIMEOUT_MS).max(MAX_STAGE_TIMEOUT_MS).default(DEFAULT_CONFIG_TIMEOUT_MS),
|
|
1198
1515
|
smokeTimeoutMs: z$1.number().min(MIN_STAGE_TIMEOUT_MS).max(MAX_STAGE_TIMEOUT_MS).default(DEFAULT_SMOKE_TIMEOUT_MS),
|
|
1516
|
+
capabilityTimeoutMs: z$1.number().min(MIN_STAGE_TIMEOUT_MS).max(MAX_STAGE_TIMEOUT_MS).default(DEFAULT_CAPABILITY_TIMEOUT_MS),
|
|
1517
|
+
capability: z$1.object({
|
|
1518
|
+
enabled: z$1.boolean().default(false),
|
|
1519
|
+
kind: z$1.union([z$1.const("tool"), z$1.const("command")]).default("tool"),
|
|
1520
|
+
name: z$1.string().default(""),
|
|
1521
|
+
args: z$1.string().default(""),
|
|
1522
|
+
expect: z$1.string().default("")
|
|
1523
|
+
}).default({
|
|
1524
|
+
enabled: false,
|
|
1525
|
+
kind: "tool",
|
|
1526
|
+
name: "",
|
|
1527
|
+
args: "",
|
|
1528
|
+
expect: ""
|
|
1529
|
+
}),
|
|
1199
1530
|
uninstallTimeoutMs: z$1.number().min(MIN_STAGE_TIMEOUT_MS).max(MAX_STAGE_TIMEOUT_MS).default(DEFAULT_UNINSTALL_TIMEOUT_MS),
|
|
1200
1531
|
outputTailBytes: z$1.number().min(256).max(MAX_OUTPUT_TAIL_BYTES).default(DEFAULT_OUTPUT_TAIL_BYTES),
|
|
1201
1532
|
keepTempDirs: z$1.boolean().default(false),
|
|
@@ -1230,6 +1561,21 @@ function resolveConfig(config) {
|
|
|
1230
1561
|
if (!Number.isFinite(configTimeoutMs) || configTimeoutMs < 1e3 || configTimeoutMs > 36e5) invalid("configTimeoutMs", `must be a finite number between ${MIN_STAGE_TIMEOUT_MS} and ${MAX_STAGE_TIMEOUT_MS}`);
|
|
1231
1562
|
const smokeTimeoutMs = config?.smokeTimeoutMs ?? 3e5;
|
|
1232
1563
|
if (!Number.isFinite(smokeTimeoutMs) || smokeTimeoutMs < 1e3 || smokeTimeoutMs > 36e5) invalid("smokeTimeoutMs", `must be a finite number between ${MIN_STAGE_TIMEOUT_MS} and ${MAX_STAGE_TIMEOUT_MS}`);
|
|
1564
|
+
const capabilityTimeoutMs = config?.capabilityTimeoutMs ?? 3e5;
|
|
1565
|
+
if (!Number.isFinite(capabilityTimeoutMs) || capabilityTimeoutMs < 1e3 || capabilityTimeoutMs > 36e5) invalid("capabilityTimeoutMs", `must be a finite number between ${MIN_STAGE_TIMEOUT_MS} and ${MAX_STAGE_TIMEOUT_MS}`);
|
|
1566
|
+
const capabilityEnabled = config?.capability?.enabled ?? false;
|
|
1567
|
+
if (typeof capabilityEnabled !== "boolean") invalid("capability.enabled", "must be a boolean");
|
|
1568
|
+
const capabilityKind = config?.capability?.kind ?? "tool";
|
|
1569
|
+
if (capabilityKind !== "tool" && capabilityKind !== "command") invalid("capability.kind", `must be 'tool' or 'command' (got ${JSON.stringify(capabilityKind)})`);
|
|
1570
|
+
const capabilityName = config?.capability?.name ?? "";
|
|
1571
|
+
const capabilityArgs = config?.capability?.args ?? "";
|
|
1572
|
+
const capabilityExpect = config?.capability?.expect ?? "";
|
|
1573
|
+
if (capabilityEnabled) {
|
|
1574
|
+
if (capabilityName.length === 0) invalid("capability.name", "is required when capability.enabled is true");
|
|
1575
|
+
if (capabilityName.length > 200 || !CAPABILITY_NAME_PATTERN.test(capabilityName)) invalid("capability.name", `must be at most 200 characters of [a-zA-Z0-9._/-] starting alphanumerically`);
|
|
1576
|
+
if (capabilityArgs.length > 4e3) invalid("capability.args", `must be at most ${MAX_CAPABILITY_ARGS_LENGTH} characters`);
|
|
1577
|
+
if (capabilityExpect.length > 2e3) invalid("capability.expect", `must be at most ${MAX_CAPABILITY_EXPECT_LENGTH} characters`);
|
|
1578
|
+
}
|
|
1233
1579
|
const uninstallTimeoutMs = config?.uninstallTimeoutMs ?? 12e4;
|
|
1234
1580
|
if (!Number.isFinite(uninstallTimeoutMs) || uninstallTimeoutMs < 1e3 || uninstallTimeoutMs > 36e5) invalid("uninstallTimeoutMs", `must be a finite number between ${MIN_STAGE_TIMEOUT_MS} and ${MAX_STAGE_TIMEOUT_MS}`);
|
|
1235
1581
|
const outputTailBytes = config?.outputTailBytes ?? 8e3;
|
|
@@ -1249,6 +1595,14 @@ function resolveConfig(config) {
|
|
|
1249
1595
|
installTimeoutMs,
|
|
1250
1596
|
configTimeoutMs,
|
|
1251
1597
|
smokeTimeoutMs,
|
|
1598
|
+
capabilityTimeoutMs,
|
|
1599
|
+
capability: Object.freeze({
|
|
1600
|
+
enabled: capabilityEnabled,
|
|
1601
|
+
kind: capabilityKind,
|
|
1602
|
+
name: capabilityName,
|
|
1603
|
+
args: capabilityArgs,
|
|
1604
|
+
expect: capabilityExpect
|
|
1605
|
+
}),
|
|
1252
1606
|
uninstallTimeoutMs,
|
|
1253
1607
|
outputTailBytes,
|
|
1254
1608
|
keepTempDirs,
|
|
@@ -1301,6 +1655,9 @@ function statusMark(status) {
|
|
|
1301
1655
|
case "boot-ok": return "🟡";
|
|
1302
1656
|
case "partial": return "🟠";
|
|
1303
1657
|
case "skipped": return "⏭️";
|
|
1658
|
+
case "observed": return "✅";
|
|
1659
|
+
case "invoked": return "🟡";
|
|
1660
|
+
case "not-registered": return "❌";
|
|
1304
1661
|
default: return "❓";
|
|
1305
1662
|
}
|
|
1306
1663
|
}
|
|
@@ -1326,6 +1683,7 @@ function renderDriveResult(result) {
|
|
|
1326
1683
|
`| install | ${statusMark(stages.install.status)} ${stages.install.status} | ${String(stages.install.exitCode)} | ${formatDuration(stages.install.durationMs)} | ${stages.install.summary} |`,
|
|
1327
1684
|
`| config (--dump-config) | ${statusMark(stages.config.status)} ${stages.config.status} | ${String(stages.config.exitCode)} | ${formatDuration(stages.config.durationMs)} | ${stages.config.patchEffective ? `patch effective: ${stages.config.layers.join(", ")}` : stages.config.summary} |`,
|
|
1328
1685
|
`| smoke (headless boot) | ${statusMark(stages.smoke.status)} ${stages.smoke.status} | ${String(stages.smoke.exitCode)} | ${formatDuration(stages.smoke.durationMs)} | ${stages.smoke.summary} |`,
|
|
1686
|
+
`| capability | ${stages.capability === void 0 ? "—" : `${statusMark(stages.capability.status)} ${stages.capability.status}`} | ${stages.capability === void 0 ? "—" : String(stages.capability.exitCode)} | ${stages.capability === void 0 ? "—" : formatDuration(stages.capability.durationMs)} | ${stages.capability?.summary ?? "not asserted"} |`,
|
|
1329
1687
|
`| uninstall | ${statusMark(stages.uninstall.status)} ${stages.uninstall.status} | ${String(stages.uninstall.exitCode)} | ${formatDuration(stages.uninstall.durationMs)} | ${stages.uninstall.summary} |`,
|
|
1330
1688
|
`| cleanup | ${statusMark(stages.cleanup.status)} ${stages.cleanup.status} | — | — | ${stages.cleanup.summary} |`,
|
|
1331
1689
|
""
|
|
@@ -1459,6 +1817,26 @@ const driveResultSchema = {
|
|
|
1459
1817
|
taskCompleted: { type: "boolean" }
|
|
1460
1818
|
}
|
|
1461
1819
|
},
|
|
1820
|
+
capability: { oneOf: [{
|
|
1821
|
+
...stageRecordSchema,
|
|
1822
|
+
properties: {
|
|
1823
|
+
...stageRecordSchema.properties,
|
|
1824
|
+
capabilityKind: { oneOf: [
|
|
1825
|
+
{
|
|
1826
|
+
type: "string",
|
|
1827
|
+
const: "tool"
|
|
1828
|
+
},
|
|
1829
|
+
{
|
|
1830
|
+
type: "string",
|
|
1831
|
+
const: "command"
|
|
1832
|
+
},
|
|
1833
|
+
{ type: "null" }
|
|
1834
|
+
] },
|
|
1835
|
+
name: { type: "string" },
|
|
1836
|
+
expectMatched: { type: "boolean" },
|
|
1837
|
+
detail: { type: "string" }
|
|
1838
|
+
}
|
|
1839
|
+
}, { type: "null" }] },
|
|
1462
1840
|
uninstall: stageRecordSchema,
|
|
1463
1841
|
cleanup: {
|
|
1464
1842
|
type: "object",
|
|
@@ -1556,7 +1934,7 @@ function renderReport(value) {
|
|
|
1556
1934
|
}
|
|
1557
1935
|
/** The deadline for one foreground drive: the sum of every stage deadline plus a buffer. */
|
|
1558
1936
|
function driveDeadlineMs(config) {
|
|
1559
|
-
return config.installTimeoutMs + config.configTimeoutMs + config.smokeTimeoutMs + config.uninstallTimeoutMs + 6e4;
|
|
1937
|
+
return config.installTimeoutMs + config.configTimeoutMs + config.smokeTimeoutMs + config.capabilityTimeoutMs + config.uninstallTimeoutMs + 6e4;
|
|
1560
1938
|
}
|
|
1561
1939
|
/** Owner for a background branch; background work needs a live agent to collect it. */
|
|
1562
1940
|
function requireOwner(exec) {
|
|
@@ -1578,6 +1956,30 @@ function allTools(services) {
|
|
|
1578
1956
|
type: "string",
|
|
1579
1957
|
description: "One-shot task text for the boot-smoke stage; empty string skips the smoke stage. Defaults to the plugin config."
|
|
1580
1958
|
},
|
|
1959
|
+
capability: {
|
|
1960
|
+
type: "object",
|
|
1961
|
+
additionalProperties: false,
|
|
1962
|
+
description: "Capability assertion: after the boot smoke, drive one headless task that calls the named tool (or runs the /command), then verify the durable session log records the invocation and that the observed output contains `expect`. Requires DEEPSEEK_API_KEY (host env or forwardEnv); without it the stage is skipped, never failed.",
|
|
1963
|
+
properties: {
|
|
1964
|
+
kind: {
|
|
1965
|
+
type: "string",
|
|
1966
|
+
enum: ["tool", "command"],
|
|
1967
|
+
description: "What to assert: a registered model tool, or a /name command."
|
|
1968
|
+
},
|
|
1969
|
+
name: {
|
|
1970
|
+
type: "string",
|
|
1971
|
+
description: "The tool or command name (without the leading /)."
|
|
1972
|
+
},
|
|
1973
|
+
args: {
|
|
1974
|
+
type: "string",
|
|
1975
|
+
description: "Invocation text: tool arguments (JSON-ish) or command words."
|
|
1976
|
+
},
|
|
1977
|
+
expect: {
|
|
1978
|
+
type: "string",
|
|
1979
|
+
description: "Literal expected in the observed output (case-insensitive substring of the logged event)."
|
|
1980
|
+
}
|
|
1981
|
+
}
|
|
1982
|
+
},
|
|
1581
1983
|
background: {
|
|
1582
1984
|
type: "boolean",
|
|
1583
1985
|
description: "Run the drive as a background job and return its job id instead of waiting."
|
|
@@ -1602,7 +2004,25 @@ function allTools(services) {
|
|
|
1602
2004
|
const target = sanitizeTarget(args.target);
|
|
1603
2005
|
if (target.length === 0) throw new Error("dsh-test-drive: target must be a non-empty string");
|
|
1604
2006
|
const headlessTask = args.headlessTask;
|
|
1605
|
-
if (headlessTask !== void 0 && headlessTask.length > 2e3) throw new Error(`dsh-test-drive: headlessTask must be at most ${MAX_HEADLESS_TASK_LENGTH} characters`);
|
|
2007
|
+
if (headlessTask !== void 0 && headlessTask !== null && headlessTask.length > 2e3) throw new Error(`dsh-test-drive: headlessTask must be at most ${MAX_HEADLESS_TASK_LENGTH} characters`);
|
|
2008
|
+
const capabilityArg = args.capability;
|
|
2009
|
+
let capability;
|
|
2010
|
+
if (capabilityArg !== void 0 && capabilityArg !== null) {
|
|
2011
|
+
const kind = capabilityArg.kind;
|
|
2012
|
+
if (kind !== "tool" && kind !== "command") throw new Error("dsh-test-drive: capability.kind must be 'tool' or 'command'");
|
|
2013
|
+
const name = capabilityArg.name;
|
|
2014
|
+
if (typeof name !== "string" || name.length === 0 || name.length > 200 || !CAPABILITY_NAME_PATTERN.test(name)) throw new Error(`dsh-test-drive: capability.name must be at most 200 characters of [a-zA-Z0-9._/-] starting alphanumerically`);
|
|
2015
|
+
const capArgs = capabilityArg.args;
|
|
2016
|
+
if (typeof capArgs !== "string" || capArgs.length > 4e3) throw new Error(`dsh-test-drive: capability.args must be a string of at most ${MAX_CAPABILITY_ARGS_LENGTH} characters`);
|
|
2017
|
+
const expect = capabilityArg.expect;
|
|
2018
|
+
if (typeof expect !== "string" || expect.length > 2e3) throw new Error(`dsh-test-drive: capability.expect must be a string of at most ${MAX_CAPABILITY_EXPECT_LENGTH} characters`);
|
|
2019
|
+
capability = {
|
|
2020
|
+
kind,
|
|
2021
|
+
name,
|
|
2022
|
+
args: capArgs,
|
|
2023
|
+
expect
|
|
2024
|
+
};
|
|
2025
|
+
}
|
|
1606
2026
|
if (args.background === true) {
|
|
1607
2027
|
const owner = requireOwner(exec);
|
|
1608
2028
|
const label = `test_drive ${target}`;
|
|
@@ -1614,7 +2034,8 @@ function allTools(services) {
|
|
|
1614
2034
|
}
|
|
1615
2035
|
const result = await services.runner.drive(target, {
|
|
1616
2036
|
signal: exec.signal,
|
|
1617
|
-
...headlessTask === void 0 ? {} : { headlessTask }
|
|
2037
|
+
...headlessTask === void 0 || headlessTask === null ? {} : { headlessTask },
|
|
2038
|
+
...capability === void 0 ? {} : { capability }
|
|
1618
2039
|
});
|
|
1619
2040
|
return {
|
|
1620
2041
|
...result,
|
package/lib/types/batch.d.ts
CHANGED
|
@@ -10,8 +10,8 @@
|
|
|
10
10
|
import type { Agent } from '@deepseek-ai/dsh-agent';
|
|
11
11
|
import type { Context } from '@deepseek-ai/cordis';
|
|
12
12
|
import type { JobId } from '@deepseek-ai/dsh-jobs';
|
|
13
|
-
import type { DriveDeps } from './drive.
|
|
14
|
-
import type { MatrixRecord } from './result.
|
|
13
|
+
import type { DriveDeps } from './drive.js';
|
|
14
|
+
import type { MatrixRecord } from './result.js';
|
|
15
15
|
/** The job kind this producer registers (declaration-merged into JobKindMap). */
|
|
16
16
|
export declare const DRIVE_BATCH_KIND: "drive-batch";
|
|
17
17
|
declare module '@deepseek-ai/dsh-jobs' {
|