dsh-test-drive 0.1.0 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/lib/index.js CHANGED
@@ -1,11 +1,161 @@
1
1
  import { randomBytes } from "node:crypto";
2
- import { access, mkdir, mkdtemp, readFile, rename, rm, writeFile } from "node:fs/promises";
3
- import { basename, dirname, isAbsolute, join, resolve } from "node:path";
4
2
  import { z } from "zod";
3
+ import { access, mkdir, mkdtemp, readFile, readdir, rename, rm, stat, writeFile } from "node:fs/promises";
4
+ import { basename, dirname, isAbsolute, join, resolve } from "node:path";
5
5
  import z$1 from "@deepseek-ai/schemastery";
6
6
  import { defineDomain, domainTable } from "@deepseek-ai/dsh-storage-domain";
7
7
  import { defineTool } from "@deepseek-ai/dsh-tools";
8
8
  import { tmpdir } from "node:os";
9
+ //#region src/capability.ts
10
+ /**
11
+ * The capability-assertion stage: beyond "booted and exited", prove one NAMED
12
+ * capability of the installed plugin is registered, invoked, and its effect
13
+ * OBSERVED — the omdsh-INTAKE principle that a clean boot is only a smoke
14
+ * test. The stage runs one headless task that asks the agent to call a tool
15
+ * (or run a command), then analyzes the durable session log for the paired
16
+ * `tool/call` → `tool/result` events (or the command invocation and reply) and
17
+ * matches a caller-supplied expectation against the observed output.
18
+ *
19
+ * Everything here is pure and tolerant: the analyzer never throws on hostile
20
+ * or truncated log lines (it skips them), so the capability stage itself can
21
+ * only fail through its own process outcome, never through log parsing.
22
+ *
23
+ * @module dsh-test-drive/capability
24
+ */
25
+ /** The capability kinds the stage can assert. */
26
+ const CapabilityKindSchema = z.union([z.literal("tool"), z.literal("command")]);
27
+ /** Outcome of the capability stage (registered + invoked + observed, in order). */
28
+ const CapabilityStatusSchema = z.union([
29
+ z.literal("observed"),
30
+ z.literal("invoked"),
31
+ z.literal("not-registered"),
32
+ z.literal("skipped"),
33
+ z.literal("failed")
34
+ ]);
35
+ /** The exact reply the task text demands; also the run-completion marker. */
36
+ const CAPABILITY_DONE_MARKER = "capability-check-done";
37
+ /** Maximum length of the `args` payload handed to the capability task. */
38
+ const MAX_CAPABILITY_ARGS_LENGTH = 4e3;
39
+ /** Maximum length of the `expect` literal matched against the observed output. */
40
+ const MAX_CAPABILITY_EXPECT_LENGTH = 2e3;
41
+ /** Allowed characters for an asserted tool or command name. */
42
+ const CAPABILITY_NAME_PATTERN = /^[a-zA-Z0-9][a-zA-Z0-9._/-]*$/;
43
+ /** Regex-escape a literal for embedding in a pattern. */
44
+ function escapeRegex$1(source) {
45
+ return source.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&");
46
+ }
47
+ /**
48
+ * Compile the headless task text that makes the agent exercise the capability.
49
+ *
50
+ * @param spec - the capability to exercise.
51
+ * @returns the task text; never empty.
52
+ */
53
+ function buildCapabilityTask(spec) {
54
+ if (spec.kind === "tool") return [
55
+ `Call the tool "${spec.name}" with exactly these arguments:`,
56
+ spec.args === "" ? "(no arguments)" : spec.args,
57
+ `After the tool returns, reply with exactly: ${CAPABILITY_DONE_MARKER}`
58
+ ].join("\n");
59
+ return [`Run the command /${spec.name}${spec.args === "" ? "" : ` ${spec.args}`}.`, `Then reply with exactly: ${CAPABILITY_DONE_MARKER}`].join("\n");
60
+ }
61
+ /** Parse one JSONL line into `{ type, data }` or undefined for unparseable lines. */
62
+ function parseLine(line) {
63
+ try {
64
+ const value = JSON.parse(line);
65
+ if (typeof value.type !== "string" || value.data === void 0) return void 0;
66
+ return {
67
+ type: value.type,
68
+ data: value.data
69
+ };
70
+ } catch {
71
+ return;
72
+ }
73
+ }
74
+ /** Lowercased serialization of one event payload for expectation matching. */
75
+ function haystack(data) {
76
+ return JSON.stringify(data).toLowerCase();
77
+ }
78
+ /** Whether the serialized payload contains the expectation (case-insensitive). */
79
+ function matchesExpect(data, expect) {
80
+ return expect !== "" && haystack(data).includes(expect.toLowerCase());
81
+ }
82
+ /**
83
+ * Analyze a durable session log against one capability spec. Tolerant: every
84
+ * unparseable or unrelated line is skipped, and missing evidence degrades to
85
+ * the highest status actually reached (`not-registered` when the invocation
86
+ * never appears, `invoked` when it ran but the expectation never appeared,
87
+ * `observed` only when both hold).
88
+ *
89
+ * @param log - the raw JSONL session text (as persisted by the harness).
90
+ * @param spec - the asserted capability.
91
+ * @returns the settled analysis; never throws.
92
+ */
93
+ function analyzeSessionLog(log, spec) {
94
+ const lines = [];
95
+ for (const line of log.split("\n")) {
96
+ const parsed = parseLine(line);
97
+ if (parsed !== void 0) lines.push(parsed);
98
+ }
99
+ if (spec.kind === "tool") {
100
+ const callIds = [];
101
+ for (const line of lines) {
102
+ if (line.type !== "tool/call") continue;
103
+ const data = line.data;
104
+ if (data.name === spec.name && typeof data.callId === "string") callIds.push(data.callId);
105
+ }
106
+ if (callIds.length === 0) return {
107
+ status: "not-registered",
108
+ detail: `no tool/call event names "${spec.name}" — the tool was never registered or the agent never invoked it`,
109
+ expectMatched: false
110
+ };
111
+ for (const line of lines) {
112
+ if (line.type !== "tool/result") continue;
113
+ const serialized = JSON.stringify(line.data);
114
+ if (!callIds.some((callId) => serialized.includes(callId))) continue;
115
+ if (matchesExpect(line.data, spec.expect)) return {
116
+ status: "observed",
117
+ detail: `tool "${spec.name}" called and its result contains the expectation`,
118
+ expectMatched: true
119
+ };
120
+ }
121
+ return {
122
+ status: "invoked",
123
+ detail: `tool "${spec.name}" was called (${callIds.length} call(s)) but no paired result contains the expectation`,
124
+ expectMatched: false
125
+ };
126
+ }
127
+ const commandPattern = new RegExp(`/${escapeRegex$1(spec.name)}(?:\\s|$)`, "u");
128
+ let invokedAt = -1;
129
+ for (let i = 0; i < lines.length; i += 1) {
130
+ const line = lines[i];
131
+ if (line === void 0) continue;
132
+ if (line.type !== "user/message") continue;
133
+ if (commandPattern.test(JSON.stringify(line.data))) {
134
+ invokedAt = i;
135
+ break;
136
+ }
137
+ }
138
+ if (invokedAt === -1) return {
139
+ status: "not-registered",
140
+ detail: `no user message invoking /${spec.name} — the agent never ran the command`,
141
+ expectMatched: false
142
+ };
143
+ for (let i = invokedAt + 1; i < lines.length; i += 1) {
144
+ const line = lines[i];
145
+ if (line === void 0) continue;
146
+ if (line.type === "assistant/message" && matchesExpect(line.data, spec.expect)) return {
147
+ status: "observed",
148
+ detail: `command /${spec.name} ran and a reply contains the expectation`,
149
+ expectMatched: true
150
+ };
151
+ }
152
+ return {
153
+ status: "invoked",
154
+ detail: `command /${spec.name} was invoked but no reply contains the expectation (unknown command or empty output)`,
155
+ expectMatched: false
156
+ };
157
+ }
158
+ //#endregion
9
159
  //#region src/driver.ts
10
160
  /**
11
161
  * The `dsh` CLI driver: locates a spawnable dsh executable (config override,
@@ -269,6 +419,43 @@ var DshDriver = class {
269
419
  task
270
420
  ], options);
271
421
  }
422
+ /**
423
+ * Read the newest durable session log from a throwaway home's session store.
424
+ * Used by the capability stage to observe what the headless task actually
425
+ * did (tool calls, command replies). Tolerant by contract: a missing or
426
+ * empty store yields '' so the stage degrades instead of throwing.
427
+ *
428
+ * @param home - the throwaway DSH_HOME.
429
+ * @param maxBytes - cap on the returned text (tail end kept).
430
+ * @returns the newest session file's text, or '' when none exists.
431
+ */
432
+ async readNewestSession(home, maxBytes) {
433
+ let entries;
434
+ try {
435
+ entries = await readdir(join(home, "sessions"));
436
+ } catch {
437
+ return "";
438
+ }
439
+ let newest;
440
+ for (const entry of entries) {
441
+ if (!/\.jsonl$/u.test(entry)) continue;
442
+ const path = join(home, "sessions", entry);
443
+ try {
444
+ const info = await stat(path);
445
+ if (info.isFile() && (newest === void 0 || info.mtimeMs > newest.mtimeMs)) newest = {
446
+ path,
447
+ mtimeMs: info.mtimeMs
448
+ };
449
+ } catch {}
450
+ }
451
+ if (newest === void 0) return "";
452
+ try {
453
+ const text = await readFile(newest.path, "utf8");
454
+ return text.length <= maxBytes ? text : text.slice(text.length - maxBytes);
455
+ } catch {
456
+ return "";
457
+ }
458
+ }
272
459
  /** Run the uninstall stage: `dsh plugin --profile <name> remove <pkg>`. */
273
460
  async remove(packageName, options) {
274
461
  return this.run([
@@ -407,6 +594,19 @@ const SmokeStageResultSchema = StageResultSchema.extend({
407
594
  });
408
595
  /** Uninstall stage result. */
409
596
  const UninstallStageResultSchema = StageResultSchema;
597
+ /** Capability-assertion stage: one named tool or command, registered → invoked → observed. */
598
+ const CapabilityStageResultSchema = StageResultSchema.extend({
599
+ /** `observed`/`invoked`/`not-registered`/`skipped`/`failed`. */
600
+ status: CapabilityStatusSchema,
601
+ /** What was asserted (tool or command); null when the stage never ran. */
602
+ capabilityKind: CapabilityKindSchema.nullable(),
603
+ /** The asserted tool or command name; '' when the stage never ran. */
604
+ name: z.string(),
605
+ /** Whether the expectation literal was found in the observed output. */
606
+ expectMatched: z.boolean(),
607
+ /** One-line sanitized explanation of the reached status. */
608
+ detail: z.string()
609
+ });
410
610
  /** Cleanup stage: quarantine/removal facts for the temp directory this run owned. */
411
611
  const CleanupStageResultSchema = z.object({
412
612
  /** `pass` when the owned temp root was removed; `skipped` when keepTempDirs was set. */
@@ -483,6 +683,12 @@ const DriveResultSchema = z.object({
483
683
  install: InstallStageResultSchema,
484
684
  config: ConfigStageResultSchema,
485
685
  smoke: SmokeStageResultSchema,
686
+ /**
687
+ * Optional capability-assertion stage (added after v1 records existed;
688
+ * optional so pre-capability records still validate at the durable
689
+ * boundary — no domain-version bump for a backward-compatible addition).
690
+ */
691
+ capability: CapabilityStageResultSchema.optional(),
486
692
  uninstall: UninstallStageResultSchema,
487
693
  cleanup: CleanupStageResultSchema
488
694
  }),
@@ -556,7 +762,8 @@ function totalsOf(rows) {
556
762
  * Derive the overall verdict of one drive run from its stage outcomes.
557
763
  *
558
764
  * Rules, in order:
559
- * 1. A failed install or a failed boot (`smoke.fail`) is a hard `fail`.
765
+ * 1. A failed install, a failed boot (`smoke.fail`), or a capability stage
766
+ * that reached `not-registered`/`failed` is a hard `fail`.
560
767
  * 2. `pass` needs install pass + patch effective + a clean boot (`pass` or
561
768
  * `boot-ok`) + a successful uninstall.
562
769
  * 3. `partial` covers everything that installed but missed a later assurance
@@ -567,12 +774,14 @@ function totalsOf(rows) {
567
774
  * @returns `[verdict, reason]`.
568
775
  */
569
776
  function verdictOf(result) {
570
- const { install, config, smoke, uninstall } = result.stages;
777
+ const { install, config, smoke, uninstall, capability } = result.stages;
571
778
  if (install.status === "fail") return ["fail", `install failed: ${install.summary}`];
572
779
  if (smoke.status === "fail") return ["fail", `boot smoke failed: ${smoke.summary}`];
780
+ if (capability !== void 0 && (capability.status === "not-registered" || capability.status === "failed")) return ["fail", `capability assertion failed (${capability.name}): ${capability.detail}`];
573
781
  if (install.status === "pass" && config.status === "pass" && config.patchEffective && (smoke.status === "pass" || smoke.status === "boot-ok") && uninstall.status === "pass") {
574
- if (smoke.status === "boot-ok") return ["pass", "install, patch, boot, and uninstall verified; headless task inconclusive (see smoke.summary)"];
575
- return ["pass", "install, patch, boot, task, and uninstall all verified"];
782
+ const capabilityNote = capability !== void 0 && capability.status === "observed" ? `; capability "${capability.name}" registered, invoked, and observed` : "";
783
+ if (smoke.status === "boot-ok") return ["pass", `install, patch, boot, and uninstall verified${capabilityNote}; headless task inconclusive (see smoke.summary)`];
784
+ return ["pass", `install, patch, boot, task, and uninstall all verified${capabilityNote}`];
576
785
  }
577
786
  if (install.status !== "pass") return ["unknown", "install never ran to completion"];
578
787
  if (config.status !== "pass" || !config.patchEffective) return ["partial", `install passed but the bundle patch was not verified (${config.summary})`];
@@ -674,7 +883,7 @@ function sanitizeOutput(text, tempRoot, maxChars) {
674
883
  * The structured report records it as `run.pluginVersion`.
675
884
  * @module dsh-test-drive/version
676
885
  */
677
- const VERSION = "0.1.0";
886
+ const VERSION = "0.2.0";
678
887
  //#endregion
679
888
  //#region src/drive.ts
680
889
  /**
@@ -687,6 +896,8 @@ const VERSION = "0.1.0";
687
896
  *
688
897
  * @module dsh-test-drive/drive
689
898
  */
899
+ /** Cap on the session-log text the capability stage reads back. */
900
+ const SESSION_READ_MAX_BYTES = 1048576;
690
901
  /** id prefix of a single drive run. */
691
902
  const RUN_ID_PREFIX = "tdr_";
692
903
  /** id prefix of a batch matrix. */
@@ -815,6 +1026,18 @@ var DriveRunner = class {
815
1026
  bootFailed: false,
816
1027
  taskCompleted: false
817
1028
  };
1029
+ let capability = {
1030
+ status: "skipped",
1031
+ exitCode: null,
1032
+ durationMs: 0,
1033
+ attempts: 0,
1034
+ outputTail: "",
1035
+ summary: "not reached: earlier stage errored",
1036
+ capabilityKind: null,
1037
+ name: "",
1038
+ expectMatched: false,
1039
+ detail: "not reached: earlier stage errored"
1040
+ };
818
1041
  let uninstall = { ...UNREACHED };
819
1042
  let resolved;
820
1043
  let cleanup = {
@@ -915,6 +1138,89 @@ var DriveRunner = class {
915
1138
  taskCompleted: false
916
1139
  };
917
1140
  }
1141
+ const capSpec = options.capability ?? (config.capability.enabled ? {
1142
+ kind: config.capability.kind,
1143
+ name: config.capability.name,
1144
+ args: config.capability.args,
1145
+ expect: config.capability.expect
1146
+ } : void 0);
1147
+ if (capSpec === void 0) capability = {
1148
+ ...capability,
1149
+ status: "skipped",
1150
+ summary: "capability assertion disabled (config.capability.enabled is false and no per-drive capability given)",
1151
+ detail: "capability assertion disabled by config"
1152
+ };
1153
+ else if (install.status !== "pass") capability = {
1154
+ ...capability,
1155
+ summary: "install did not pass",
1156
+ detail: "install did not pass"
1157
+ };
1158
+ else if (smoke.status === "fail") capability = {
1159
+ ...capability,
1160
+ summary: "boot smoke failed",
1161
+ detail: "boot smoke failed before the capability stage"
1162
+ };
1163
+ else if (options.signal?.aborted ?? false) capability = {
1164
+ ...capability,
1165
+ summary: "run aborted before this stage",
1166
+ detail: "run aborted before this stage"
1167
+ };
1168
+ else if (process.env.DEEPSEEK_API_KEY === void 0 && !config.forwardEnv.includes("DEEPSEEK_API_KEY")) capability = {
1169
+ ...capability,
1170
+ status: "skipped",
1171
+ summary: "DEEPSEEK_API_KEY not available — the capability task needs a model (set it on the host or list it in forwardEnv)",
1172
+ capabilityKind: capSpec.kind,
1173
+ name: capSpec.name,
1174
+ detail: "no model credential available for the capability task"
1175
+ };
1176
+ else {
1177
+ const run = await driver.smokeTask(buildCapabilityTask(capSpec), {
1178
+ ...base,
1179
+ timeoutMs: config.capabilityTimeoutMs
1180
+ });
1181
+ const outputTail = sanitizeOutput(`${run.stdout}\n${run.stderr}`, workspace.root, config.outputTailBytes);
1182
+ if (run.timedOut) capability = {
1183
+ status: "failed",
1184
+ exitCode: run.exitCode,
1185
+ durationMs: run.durationMs,
1186
+ attempts: 1,
1187
+ summary: `timed out after ${run.durationMs} ms`,
1188
+ outputTail,
1189
+ capabilityKind: capSpec.kind,
1190
+ name: capSpec.name,
1191
+ expectMatched: false,
1192
+ detail: "capability task timed out"
1193
+ };
1194
+ else if (run.exitCode !== 0) {
1195
+ const firstLine = tailText(run.stderr || run.stdout, 160).split("\n")[0] ?? "";
1196
+ capability = {
1197
+ status: "failed",
1198
+ exitCode: run.exitCode,
1199
+ durationMs: run.durationMs,
1200
+ attempts: 1,
1201
+ summary: `exit ${String(run.exitCode)}: ${firstLine || "no output"}`,
1202
+ outputTail,
1203
+ capabilityKind: capSpec.kind,
1204
+ name: capSpec.name,
1205
+ expectMatched: false,
1206
+ detail: "capability task did not complete (model/credential failure)"
1207
+ };
1208
+ } else {
1209
+ const analysis = analyzeSessionLog(await driver.readNewestSession(workspace.home, SESSION_READ_MAX_BYTES), capSpec);
1210
+ capability = {
1211
+ status: analysis.status,
1212
+ exitCode: 0,
1213
+ durationMs: run.durationMs,
1214
+ attempts: 1,
1215
+ summary: analysis.detail,
1216
+ outputTail,
1217
+ capabilityKind: capSpec.kind,
1218
+ name: capSpec.name,
1219
+ expectMatched: analysis.expectMatched,
1220
+ detail: analysis.detail
1221
+ };
1222
+ }
1223
+ }
918
1224
  if (resolved?.packageName === void 0) uninstall = {
919
1225
  ...uninstall,
920
1226
  summary: "no installed package to remove"
@@ -940,6 +1246,14 @@ var DriveRunner = class {
940
1246
  bootFailed: false,
941
1247
  taskCompleted: false
942
1248
  };
1249
+ else if (capability.status === "skipped") capability = {
1250
+ ...erroredStage(summary),
1251
+ status: "failed",
1252
+ capabilityKind: null,
1253
+ name: "",
1254
+ expectMatched: false,
1255
+ detail: summary
1256
+ };
943
1257
  else uninstall = { ...erroredStage(summary) };
944
1258
  log(`test-drive: pipeline error for ${spec}: ${String(error)}`);
945
1259
  } finally {
@@ -983,6 +1297,7 @@ var DriveRunner = class {
983
1297
  install,
984
1298
  config: configStage,
985
1299
  smoke,
1300
+ capability,
986
1301
  uninstall,
987
1302
  cleanup
988
1303
  },
@@ -1176,6 +1491,8 @@ const DEFAULT_INSTALL_TIMEOUT_MS = 6e5;
1176
1491
  const DEFAULT_CONFIG_TIMEOUT_MS = 6e4;
1177
1492
  /** Default per-stage timeout for the headless boot smoke. */
1178
1493
  const DEFAULT_SMOKE_TIMEOUT_MS = 3e5;
1494
+ /** Default per-stage timeout for the capability-assertion task. */
1495
+ const DEFAULT_CAPABILITY_TIMEOUT_MS = 3e5;
1179
1496
  /** Default per-stage timeout for `dsh plugin remove`. */
1180
1497
  const DEFAULT_UNINSTALL_TIMEOUT_MS = 12e4;
1181
1498
  /** Bounds shared by every stage timeout. */
@@ -1196,6 +1513,20 @@ const Config = z$1.object({
1196
1513
  installTimeoutMs: z$1.number().min(MIN_STAGE_TIMEOUT_MS).max(MAX_STAGE_TIMEOUT_MS).default(DEFAULT_INSTALL_TIMEOUT_MS),
1197
1514
  configTimeoutMs: z$1.number().min(MIN_STAGE_TIMEOUT_MS).max(MAX_STAGE_TIMEOUT_MS).default(DEFAULT_CONFIG_TIMEOUT_MS),
1198
1515
  smokeTimeoutMs: z$1.number().min(MIN_STAGE_TIMEOUT_MS).max(MAX_STAGE_TIMEOUT_MS).default(DEFAULT_SMOKE_TIMEOUT_MS),
1516
+ capabilityTimeoutMs: z$1.number().min(MIN_STAGE_TIMEOUT_MS).max(MAX_STAGE_TIMEOUT_MS).default(DEFAULT_CAPABILITY_TIMEOUT_MS),
1517
+ capability: z$1.object({
1518
+ enabled: z$1.boolean().default(false),
1519
+ kind: z$1.union([z$1.const("tool"), z$1.const("command")]).default("tool"),
1520
+ name: z$1.string().default(""),
1521
+ args: z$1.string().default(""),
1522
+ expect: z$1.string().default("")
1523
+ }).default({
1524
+ enabled: false,
1525
+ kind: "tool",
1526
+ name: "",
1527
+ args: "",
1528
+ expect: ""
1529
+ }),
1199
1530
  uninstallTimeoutMs: z$1.number().min(MIN_STAGE_TIMEOUT_MS).max(MAX_STAGE_TIMEOUT_MS).default(DEFAULT_UNINSTALL_TIMEOUT_MS),
1200
1531
  outputTailBytes: z$1.number().min(256).max(MAX_OUTPUT_TAIL_BYTES).default(DEFAULT_OUTPUT_TAIL_BYTES),
1201
1532
  keepTempDirs: z$1.boolean().default(false),
@@ -1230,6 +1561,21 @@ function resolveConfig(config) {
1230
1561
  if (!Number.isFinite(configTimeoutMs) || configTimeoutMs < 1e3 || configTimeoutMs > 36e5) invalid("configTimeoutMs", `must be a finite number between ${MIN_STAGE_TIMEOUT_MS} and ${MAX_STAGE_TIMEOUT_MS}`);
1231
1562
  const smokeTimeoutMs = config?.smokeTimeoutMs ?? 3e5;
1232
1563
  if (!Number.isFinite(smokeTimeoutMs) || smokeTimeoutMs < 1e3 || smokeTimeoutMs > 36e5) invalid("smokeTimeoutMs", `must be a finite number between ${MIN_STAGE_TIMEOUT_MS} and ${MAX_STAGE_TIMEOUT_MS}`);
1564
+ const capabilityTimeoutMs = config?.capabilityTimeoutMs ?? 3e5;
1565
+ if (!Number.isFinite(capabilityTimeoutMs) || capabilityTimeoutMs < 1e3 || capabilityTimeoutMs > 36e5) invalid("capabilityTimeoutMs", `must be a finite number between ${MIN_STAGE_TIMEOUT_MS} and ${MAX_STAGE_TIMEOUT_MS}`);
1566
+ const capabilityEnabled = config?.capability?.enabled ?? false;
1567
+ if (typeof capabilityEnabled !== "boolean") invalid("capability.enabled", "must be a boolean");
1568
+ const capabilityKind = config?.capability?.kind ?? "tool";
1569
+ if (capabilityKind !== "tool" && capabilityKind !== "command") invalid("capability.kind", `must be 'tool' or 'command' (got ${JSON.stringify(capabilityKind)})`);
1570
+ const capabilityName = config?.capability?.name ?? "";
1571
+ const capabilityArgs = config?.capability?.args ?? "";
1572
+ const capabilityExpect = config?.capability?.expect ?? "";
1573
+ if (capabilityEnabled) {
1574
+ if (capabilityName.length === 0) invalid("capability.name", "is required when capability.enabled is true");
1575
+ if (capabilityName.length > 200 || !CAPABILITY_NAME_PATTERN.test(capabilityName)) invalid("capability.name", `must be at most 200 characters of [a-zA-Z0-9._/-] starting alphanumerically`);
1576
+ if (capabilityArgs.length > 4e3) invalid("capability.args", `must be at most ${MAX_CAPABILITY_ARGS_LENGTH} characters`);
1577
+ if (capabilityExpect.length > 2e3) invalid("capability.expect", `must be at most ${MAX_CAPABILITY_EXPECT_LENGTH} characters`);
1578
+ }
1233
1579
  const uninstallTimeoutMs = config?.uninstallTimeoutMs ?? 12e4;
1234
1580
  if (!Number.isFinite(uninstallTimeoutMs) || uninstallTimeoutMs < 1e3 || uninstallTimeoutMs > 36e5) invalid("uninstallTimeoutMs", `must be a finite number between ${MIN_STAGE_TIMEOUT_MS} and ${MAX_STAGE_TIMEOUT_MS}`);
1235
1581
  const outputTailBytes = config?.outputTailBytes ?? 8e3;
@@ -1249,6 +1595,14 @@ function resolveConfig(config) {
1249
1595
  installTimeoutMs,
1250
1596
  configTimeoutMs,
1251
1597
  smokeTimeoutMs,
1598
+ capabilityTimeoutMs,
1599
+ capability: Object.freeze({
1600
+ enabled: capabilityEnabled,
1601
+ kind: capabilityKind,
1602
+ name: capabilityName,
1603
+ args: capabilityArgs,
1604
+ expect: capabilityExpect
1605
+ }),
1252
1606
  uninstallTimeoutMs,
1253
1607
  outputTailBytes,
1254
1608
  keepTempDirs,
@@ -1301,6 +1655,9 @@ function statusMark(status) {
1301
1655
  case "boot-ok": return "🟡";
1302
1656
  case "partial": return "🟠";
1303
1657
  case "skipped": return "⏭️";
1658
+ case "observed": return "✅";
1659
+ case "invoked": return "🟡";
1660
+ case "not-registered": return "❌";
1304
1661
  default: return "❓";
1305
1662
  }
1306
1663
  }
@@ -1326,6 +1683,7 @@ function renderDriveResult(result) {
1326
1683
  `| install | ${statusMark(stages.install.status)} ${stages.install.status} | ${String(stages.install.exitCode)} | ${formatDuration(stages.install.durationMs)} | ${stages.install.summary} |`,
1327
1684
  `| config (--dump-config) | ${statusMark(stages.config.status)} ${stages.config.status} | ${String(stages.config.exitCode)} | ${formatDuration(stages.config.durationMs)} | ${stages.config.patchEffective ? `patch effective: ${stages.config.layers.join(", ")}` : stages.config.summary} |`,
1328
1685
  `| smoke (headless boot) | ${statusMark(stages.smoke.status)} ${stages.smoke.status} | ${String(stages.smoke.exitCode)} | ${formatDuration(stages.smoke.durationMs)} | ${stages.smoke.summary} |`,
1686
+ `| capability | ${stages.capability === void 0 ? "—" : `${statusMark(stages.capability.status)} ${stages.capability.status}`} | ${stages.capability === void 0 ? "—" : String(stages.capability.exitCode)} | ${stages.capability === void 0 ? "—" : formatDuration(stages.capability.durationMs)} | ${stages.capability?.summary ?? "not asserted"} |`,
1329
1687
  `| uninstall | ${statusMark(stages.uninstall.status)} ${stages.uninstall.status} | ${String(stages.uninstall.exitCode)} | ${formatDuration(stages.uninstall.durationMs)} | ${stages.uninstall.summary} |`,
1330
1688
  `| cleanup | ${statusMark(stages.cleanup.status)} ${stages.cleanup.status} | — | — | ${stages.cleanup.summary} |`,
1331
1689
  ""
@@ -1459,6 +1817,26 @@ const driveResultSchema = {
1459
1817
  taskCompleted: { type: "boolean" }
1460
1818
  }
1461
1819
  },
1820
+ capability: { oneOf: [{
1821
+ ...stageRecordSchema,
1822
+ properties: {
1823
+ ...stageRecordSchema.properties,
1824
+ capabilityKind: { oneOf: [
1825
+ {
1826
+ type: "string",
1827
+ const: "tool"
1828
+ },
1829
+ {
1830
+ type: "string",
1831
+ const: "command"
1832
+ },
1833
+ { type: "null" }
1834
+ ] },
1835
+ name: { type: "string" },
1836
+ expectMatched: { type: "boolean" },
1837
+ detail: { type: "string" }
1838
+ }
1839
+ }, { type: "null" }] },
1462
1840
  uninstall: stageRecordSchema,
1463
1841
  cleanup: {
1464
1842
  type: "object",
@@ -1556,7 +1934,7 @@ function renderReport(value) {
1556
1934
  }
1557
1935
  /** The deadline for one foreground drive: the sum of every stage deadline plus a buffer. */
1558
1936
  function driveDeadlineMs(config) {
1559
- return config.installTimeoutMs + config.configTimeoutMs + config.smokeTimeoutMs + config.uninstallTimeoutMs + 6e4;
1937
+ return config.installTimeoutMs + config.configTimeoutMs + config.smokeTimeoutMs + config.capabilityTimeoutMs + config.uninstallTimeoutMs + 6e4;
1560
1938
  }
1561
1939
  /** Owner for a background branch; background work needs a live agent to collect it. */
1562
1940
  function requireOwner(exec) {
@@ -1578,6 +1956,30 @@ function allTools(services) {
1578
1956
  type: "string",
1579
1957
  description: "One-shot task text for the boot-smoke stage; empty string skips the smoke stage. Defaults to the plugin config."
1580
1958
  },
1959
+ capability: {
1960
+ type: "object",
1961
+ additionalProperties: false,
1962
+ description: "Capability assertion: after the boot smoke, drive one headless task that calls the named tool (or runs the /command), then verify the durable session log records the invocation and that the observed output contains `expect`. Requires DEEPSEEK_API_KEY (host env or forwardEnv); without it the stage is skipped, never failed.",
1963
+ properties: {
1964
+ kind: {
1965
+ type: "string",
1966
+ enum: ["tool", "command"],
1967
+ description: "What to assert: a registered model tool, or a /name command."
1968
+ },
1969
+ name: {
1970
+ type: "string",
1971
+ description: "The tool or command name (without the leading /)."
1972
+ },
1973
+ args: {
1974
+ type: "string",
1975
+ description: "Invocation text: tool arguments (JSON-ish) or command words."
1976
+ },
1977
+ expect: {
1978
+ type: "string",
1979
+ description: "Literal expected in the observed output (case-insensitive substring of the logged event)."
1980
+ }
1981
+ }
1982
+ },
1581
1983
  background: {
1582
1984
  type: "boolean",
1583
1985
  description: "Run the drive as a background job and return its job id instead of waiting."
@@ -1602,7 +2004,25 @@ function allTools(services) {
1602
2004
  const target = sanitizeTarget(args.target);
1603
2005
  if (target.length === 0) throw new Error("dsh-test-drive: target must be a non-empty string");
1604
2006
  const headlessTask = args.headlessTask;
1605
- if (headlessTask !== void 0 && headlessTask.length > 2e3) throw new Error(`dsh-test-drive: headlessTask must be at most ${MAX_HEADLESS_TASK_LENGTH} characters`);
2007
+ if (headlessTask !== void 0 && headlessTask !== null && headlessTask.length > 2e3) throw new Error(`dsh-test-drive: headlessTask must be at most ${MAX_HEADLESS_TASK_LENGTH} characters`);
2008
+ const capabilityArg = args.capability;
2009
+ let capability;
2010
+ if (capabilityArg !== void 0 && capabilityArg !== null) {
2011
+ const kind = capabilityArg.kind;
2012
+ if (kind !== "tool" && kind !== "command") throw new Error("dsh-test-drive: capability.kind must be 'tool' or 'command'");
2013
+ const name = capabilityArg.name;
2014
+ if (typeof name !== "string" || name.length === 0 || name.length > 200 || !CAPABILITY_NAME_PATTERN.test(name)) throw new Error(`dsh-test-drive: capability.name must be at most 200 characters of [a-zA-Z0-9._/-] starting alphanumerically`);
2015
+ const capArgs = capabilityArg.args;
2016
+ if (typeof capArgs !== "string" || capArgs.length > 4e3) throw new Error(`dsh-test-drive: capability.args must be a string of at most ${MAX_CAPABILITY_ARGS_LENGTH} characters`);
2017
+ const expect = capabilityArg.expect;
2018
+ if (typeof expect !== "string" || expect.length > 2e3) throw new Error(`dsh-test-drive: capability.expect must be a string of at most ${MAX_CAPABILITY_EXPECT_LENGTH} characters`);
2019
+ capability = {
2020
+ kind,
2021
+ name,
2022
+ args: capArgs,
2023
+ expect
2024
+ };
2025
+ }
1606
2026
  if (args.background === true) {
1607
2027
  const owner = requireOwner(exec);
1608
2028
  const label = `test_drive ${target}`;
@@ -1614,7 +2034,8 @@ function allTools(services) {
1614
2034
  }
1615
2035
  const result = await services.runner.drive(target, {
1616
2036
  signal: exec.signal,
1617
- ...headlessTask === void 0 ? {} : { headlessTask }
2037
+ ...headlessTask === void 0 || headlessTask === null ? {} : { headlessTask },
2038
+ ...capability === void 0 ? {} : { capability }
1618
2039
  });
1619
2040
  return {
1620
2041
  ...result,
@@ -0,0 +1,74 @@
1
+ /**
2
+ * The capability-assertion stage: beyond "booted and exited", prove one NAMED
3
+ * capability of the installed plugin is registered, invoked, and its effect
4
+ * OBSERVED — the omdsh-INTAKE principle that a clean boot is only a smoke
5
+ * test. The stage runs one headless task that asks the agent to call a tool
6
+ * (or run a command), then analyzes the durable session log for the paired
7
+ * `tool/call` → `tool/result` events (or the command invocation and reply) and
8
+ * matches a caller-supplied expectation against the observed output.
9
+ *
10
+ * Everything here is pure and tolerant: the analyzer never throws on hostile
11
+ * or truncated log lines (it skips them), so the capability stage itself can
12
+ * only fail through its own process outcome, never through log parsing.
13
+ *
14
+ * @module dsh-test-drive/capability
15
+ */
16
+ import { z } from 'zod';
17
+ /** The capability kinds the stage can assert. */
18
+ export declare const CapabilityKindSchema: z.ZodUnion<readonly [z.ZodLiteral<"tool">, z.ZodLiteral<"command">]>;
19
+ export type CapabilityKind = z.infer<typeof CapabilityKindSchema>;
20
+ /** Outcome of the capability stage (registered + invoked + observed, in order). */
21
+ export declare const CapabilityStatusSchema: z.ZodUnion<readonly [z.ZodLiteral<"observed">, z.ZodLiteral<"invoked">, z.ZodLiteral<"not-registered">, z.ZodLiteral<"skipped">, z.ZodLiteral<"failed">]>;
22
+ export type CapabilityStatus = z.infer<typeof CapabilityStatusSchema>;
23
+ /** The exact reply the task text demands; also the run-completion marker. */
24
+ export declare const CAPABILITY_DONE_MARKER = "capability-check-done";
25
+ /** Maximum length of the `args` payload handed to the capability task. */
26
+ export declare const MAX_CAPABILITY_ARGS_LENGTH = 4000;
27
+ /** Maximum length of the `expect` literal matched against the observed output. */
28
+ export declare const MAX_CAPABILITY_EXPECT_LENGTH = 2000;
29
+ /** Maximum length of the asserted tool or command name. */
30
+ export declare const MAX_CAPABILITY_NAME_LENGTH = 200;
31
+ /** Allowed characters for an asserted tool or command name. */
32
+ export declare const CAPABILITY_NAME_PATTERN: RegExp;
33
+ /** The capability spec the stage works from (config or per-drive override). */
34
+ export interface CapabilitySpec {
35
+ /** What to invoke: a registered model tool, or a `/name` command. */
36
+ kind: CapabilityKind;
37
+ /** The tool or command name (without the leading `/`). */
38
+ name: string;
39
+ /** Raw arguments text for the invocation (tool: JSON-ish argument text; command: argument words). */
40
+ args: string;
41
+ /** Literal expected in the observed output (case-insensitive substring of the serialized event). */
42
+ expect: string;
43
+ }
44
+ /** Regex-escape a literal for embedding in a pattern. */
45
+ export declare function escapeRegex(source: string): string;
46
+ /**
47
+ * Compile the headless task text that makes the agent exercise the capability.
48
+ *
49
+ * @param spec - the capability to exercise.
50
+ * @returns the task text; never empty.
51
+ */
52
+ export declare function buildCapabilityTask(spec: Pick<CapabilitySpec, 'kind' | 'name' | 'args'>): string;
53
+ /** The settled analysis of one session log against one capability spec. */
54
+ export interface CapabilityAnalysis {
55
+ /** Highest reached status. */
56
+ status: Extract<CapabilityStatus, 'observed' | 'invoked' | 'not-registered'>;
57
+ /** One-line sanitized explanation. */
58
+ detail: string;
59
+ /** Whether the expectation literal was found in the observed output. */
60
+ expectMatched: boolean;
61
+ }
62
+ /**
63
+ * Analyze a durable session log against one capability spec. Tolerant: every
64
+ * unparseable or unrelated line is skipped, and missing evidence degrades to
65
+ * the highest status actually reached (`not-registered` when the invocation
66
+ * never appears, `invoked` when it ran but the expectation never appeared,
67
+ * `observed` only when both hold).
68
+ *
69
+ * @param log - the raw JSONL session text (as persisted by the harness).
70
+ * @param spec - the asserted capability.
71
+ * @returns the settled analysis; never throws.
72
+ */
73
+ export declare function analyzeSessionLog(log: string, spec: Pick<CapabilitySpec, 'kind' | 'name' | 'expect'>): CapabilityAnalysis;
74
+ //# sourceMappingURL=capability.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"capability.d.ts","sourceRoot":"","sources":["../../src/capability.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAA;AAEvB,iDAAiD;AACjD,eAAO,MAAM,oBAAoB,sEAAqD,CAAA;AACtF,MAAM,MAAM,cAAc,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,oBAAoB,CAAC,CAAA;AAEjE,mFAAmF;AACnF,eAAO,MAAM,sBAAsB,2JAMjC,CAAA;AACF,MAAM,MAAM,gBAAgB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,sBAAsB,CAAC,CAAA;AAErE,6EAA6E;AAC7E,eAAO,MAAM,sBAAsB,0BAA0B,CAAA;AAE7D,0EAA0E;AAC1E,eAAO,MAAM,0BAA0B,OAAQ,CAAA;AAE/C,kFAAkF;AAClF,eAAO,MAAM,4BAA4B,OAAQ,CAAA;AAEjD,2DAA2D;AAC3D,eAAO,MAAM,0BAA0B,MAAM,CAAA;AAE7C,+DAA+D;AAC/D,eAAO,MAAM,uBAAuB,QAAkC,CAAA;AAEtE,+EAA+E;AAC/E,MAAM,WAAW,cAAc;IAC7B,qEAAqE;IACrE,IAAI,EAAE,cAAc,CAAA;IACpB,0DAA0D;IAC1D,IAAI,EAAE,MAAM,CAAA;IACZ,qGAAqG;IACrG,IAAI,EAAE,MAAM,CAAA;IACZ,oGAAoG;IACpG,MAAM,EAAE,MAAM,CAAA;CACf;AAED,yDAAyD;AACzD,wBAAgB,WAAW,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAElD;AAED;;;;;GAKG;AACH,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,IAAI,CAAC,cAAc,EAAE,MAAM,GAAG,MAAM,GAAG,MAAM,CAAC,GAAG,MAAM,CAYhG;AAED,2EAA2E;AAC3E,MAAM,WAAW,kBAAkB;IACjC,8BAA8B;IAC9B,MAAM,EAAE,OAAO,CAAC,gBAAgB,EAAE,UAAU,GAAG,SAAS,GAAG,gBAAgB,CAAC,CAAA;IAC5E,sCAAsC;IACtC,MAAM,EAAE,MAAM,CAAA;IACd,wEAAwE;IACxE,aAAa,EAAE,OAAO,CAAA;CACvB;AAuBD;;;;;;;;;;GAUG;AACH,wBAAgB,iBAAiB,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,cAAc,EAAE,MAAM,GAAG,MAAM,GAAG,QAAQ,CAAC,GAAG,kBAAkB,CAqEzH"}