dsh-scout 0.9.0 → 0.9.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +91 -0
- package/README.en.md +237 -39
- package/README.md +131 -26
- package/assets/icon.svg +7 -0
- package/dist/i18n.d.ts +115 -0
- package/dist/i18n.js +299 -0
- package/dist/index.d.ts +14 -9
- package/dist/index.js +682 -206
- package/dist/model.d.ts +58 -8
- package/dist/model.js +260 -107
- package/docs/dsh-scout-product-contract.md +51 -5
- package/locale/en.json +6 -0
- package/locale/zh.json +6 -0
- package/package.json +15 -7
- package/scripts/check-release.mjs +219 -4
- package/scripts/profile-smoke.mjs +1118 -41
- package/tests/acceptance.test.mjs +1684 -0
- package/tests/model.test.mjs +496 -778
- package/tests/package.test.mjs +246 -0
- package/tests/tools.test.mjs +974 -0
|
@@ -1,10 +1,213 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* dsh-scout profile smoke gate — real-invocation edition (0.9.2).
|
|
4
|
+
*
|
|
5
|
+
* WHAT THIS GATE PROVES (every item below is an executable assertion):
|
|
6
|
+
* 1. the packed tarball installs into throw-away DSH profiles and the real
|
|
7
|
+
* plugin loader mounts it (`--dump-config` shows the `# == dsh-scout` layer);
|
|
8
|
+
* the profile patch layer's `config:` block is honored too (an absolute
|
|
9
|
+
* `scoutDir` and the deployment `language`), which is how every step below
|
|
10
|
+
* gets a deterministic base directory without touching the plugin;
|
|
11
|
+
* 2. all twelve registered tools are mounted in the registry, `ctx.appExit`
|
|
12
|
+
* (launcher service) and the `fs` seam are available, and the plugin
|
|
13
|
+
* disposer runs on unload (`DSH_SCOUT_UNLOAD`);
|
|
14
|
+
* 3. REAL TOOL CALLS through the public runtime API
|
|
15
|
+
* `ctx.tools.execute({ callId, name, arguments, signal })`, in a SIX-STEP
|
|
16
|
+
* scenario chain run across up to six real host processes:
|
|
17
|
+
* STEP 1 create: `scout_start` + `scout_add_source` (E3 registry origin)
|
|
18
|
+
* + `scout_add_claim` → each `isError === false`; `scout_start`
|
|
19
|
+
* returns an object `value` echoing the requested case id
|
|
20
|
+
* (0.9.1 structured-output contract) whose rendered projection
|
|
21
|
+
* stays under the model-visible token budget;
|
|
22
|
+
* STEP 2 export: `scout_export { targetDir: <ABSOLUTE dir> }` →
|
|
23
|
+
* `persisted === true`, `complete === true`, the resolved
|
|
24
|
+
* `value.targetDir` is the absolute directory that was passed,
|
|
25
|
+
* all SIX files exist on disk and `export.complete.json` carries
|
|
26
|
+
* `complete: true`;
|
|
27
|
+
* STEP 3 cold restart: a second host process with the same DSH_HOME and
|
|
28
|
+
* profile boots successfully and reports a DIFFERENT pid, so no
|
|
29
|
+
* in-memory case state can survive into it;
|
|
30
|
+
* STEP 4 lazy rehydration: in that second process `scout_report` runs
|
|
31
|
+
* with no prior `scout_import` and still returns the exported
|
|
32
|
+
* case — same decision as the export, sources/claims intact, and
|
|
33
|
+
* a follow-up export's `events.jsonl` carries the
|
|
34
|
+
* `case_rehydrated` event pointing at `<scoutDir>/<caseId>`;
|
|
35
|
+
* STEP 5 cross-language: a second profile with `language: en` sharing
|
|
36
|
+
* the same `scoutDir` runs `scout_import` on the zh export →
|
|
37
|
+
* `complete === true`, the rendered report AND the exported
|
|
38
|
+
* `report.md` contain ZERO Han characters, `case.json.language`
|
|
39
|
+
* is `en`, and the ASCII user-authored subject survives verbatim;
|
|
40
|
+
* STEP 6 uninstall: after `dsh plugin remove dsh-scout` the composed
|
|
41
|
+
* config has no dsh-scout layer AND a real boot reports all
|
|
42
|
+
* twelve tools missing — no registry residue;
|
|
43
|
+
* 4. NEGATIVE CONTROLS (each one is what keeps a green run from being vacuous):
|
|
44
|
+
* - OUTPUT-SCHEMA SENSITIVITY: the verifier registers a throw-away tool
|
|
45
|
+
* whose declared output schema is an object while its body returns a
|
|
46
|
+
* string, calls it, and requires the registry to reject the value
|
|
47
|
+
* (`ToolOutputError` / `INVALID_TOOL_OUTPUT`);
|
|
48
|
+
* - COMPLETION MARKER, MISSING: with `export.complete.json` deleted, a
|
|
49
|
+
* cold process must NOT rehydrate — `scout_report` fails with
|
|
50
|
+
* `Unknown case: …`, so STEP 4 really reads the marker;
|
|
51
|
+
* - COMPLETION MARKER, FALSE: the marker rewritten as `complete: false`
|
|
52
|
+
* (payload files still on disk) must be refused the same way, so
|
|
53
|
+
* existence alone is not being trusted.
|
|
54
|
+
* The two marker controls only count when their run mounted all twelve
|
|
55
|
+
* tools (asserted), otherwise they could pass because the plugin failed;
|
|
56
|
+
* 5. DECISION STABILITY ACROSS THE BOUNDARY: the decision captured from the
|
|
57
|
+
* exported `case.json` equals the decision reported before the restart and
|
|
58
|
+
* after rehydration and after the cross-language import.
|
|
59
|
+
*
|
|
60
|
+
* RUNTIME NOTE — failures arrive as RESULTS, not thrown errors (measured on
|
|
61
|
+
* dsh 0.2.0-rc.2): a tool body that violates its declared `output.schema` does
|
|
62
|
+
* not reject `execute()`. The registry converts it to
|
|
63
|
+
* `{ isError: true, error: { message, info: { name: 'ToolOutputError',
|
|
64
|
+
* code: 'INVALID_TOOL_OUTPUT' } } }` (dsh-tools `createSuccessResult` →
|
|
65
|
+
* `toolErrorResult`, `lib/index.js:3541-3544` / `:3616-3622`). An unknown tool
|
|
66
|
+
* name behaves the same way with `ToolNotFoundError` / `UNKNOWN_TOOL`. Assertions
|
|
67
|
+
* must therefore inspect `result.isError` + `result.error.info`, and must not
|
|
68
|
+
* rely on `try`/`catch` around `execute()` to detect a rejected call.
|
|
69
|
+
*
|
|
70
|
+
* WHAT THIS GATE DOES NOT COVER — read before trusting a green run:
|
|
71
|
+
* - no LLM and no agent loop: `execute()` runs with `agent` unset, so
|
|
72
|
+
* agent-scoped tool visibility, approval/authorization policy, model-side
|
|
73
|
+
* argument generation and streaming are NOT exercised. A tool that is
|
|
74
|
+
* invisible or denied for a real agent would still pass here;
|
|
75
|
+
* - no relative-path behavior: with `agent` unset the plugin's `cwd` is
|
|
76
|
+
* `undefined` (src/index.ts:427), so this gate deliberately passes an
|
|
77
|
+
* ABSOLUTE `scoutDir` AND an absolute `targetDir`. That is the documented
|
|
78
|
+
* contract for this host shape; a regression that only shows up with a
|
|
79
|
+
* relative directory is out of scope by construction;
|
|
80
|
+
* - the fixture is ASCII-only on purpose: user-authored copy is never
|
|
81
|
+
* translated (only untouched factory defaults are aligned when the
|
|
82
|
+
* deployment language differs), so "CJK = 0" can only hold when the whole
|
|
83
|
+
* artifact is factory prose. The zh `report.md` CJK > 0 control is what
|
|
84
|
+
* makes STEP 5 non-vacuous — without it an English-only plugin would pass;
|
|
85
|
+
* - six of the twelve tools are executed (`scout_start`, `scout_add_source`,
|
|
86
|
+
* `scout_add_claim`, `scout_export`, `scout_report`, `scout_import`); the
|
|
87
|
+
* other six (`scout_ingest`, `scout_search`, `scout_compare`,
|
|
88
|
+
* `scout_questions`, `scout_verify_identity`, `scout_verify_claim`) are
|
|
89
|
+
* registration-only assertions;
|
|
90
|
+
* - `report.md`/`case.json` are asserted structurally (decision echo, subject,
|
|
91
|
+
* CJK count, language label), never against a golden artifact, so wording
|
|
92
|
+
* and layout regressions can pass;
|
|
93
|
+
* - the token budget covers the *rendered content* projection of
|
|
94
|
+
* `scout_start`, not the bytes of `value`/`meta`;
|
|
95
|
+
* - `autoPersist` stays at its default (`false`): `case_rehydrated` and the
|
|
96
|
+
* completion marker are observed via explicit exports, so "persist after
|
|
97
|
+
* every mutation" is NOT exercised;
|
|
98
|
+
* - torn-export behavior (a marker left `complete: false` by an interrupted
|
|
99
|
+
* write) is only reproduced by rewriting the marker after a clean export,
|
|
100
|
+
* not by killing a process mid-write;
|
|
101
|
+
* - `--dump-config` proves the layer is present, not its order relative to the
|
|
102
|
+
* other profile layers;
|
|
103
|
+
* - DSH's own exit code is not evidence: every assertion lives in this script,
|
|
104
|
+
* which exits non-zero on any gate failure.
|
|
105
|
+
*
|
|
106
|
+
* COST AND HOW TO SKIP. This gate is intentionally heavier than its 0.9.1 form:
|
|
107
|
+
* it boots the real host up to six times (create/export, rehydrate, en import,
|
|
108
|
+
* two marker controls, post-uninstall) and installs the tarball into up to two
|
|
109
|
+
* profiles. Measured wall clock: ~16s for a full run on an M-series Mac with a
|
|
110
|
+
* warm pnpm store (the six host boots and the two profile installs dominate; the
|
|
111
|
+
* tool-call chain itself is a few seconds), and more on a cold store or a slow
|
|
112
|
+
* CI box. Escapes:
|
|
113
|
+
* - DSH_SCOUT_SMOKE_SKIP_CROSS_LANGUAGE=1 → skip STEP 5 (and the second
|
|
114
|
+
* profile's install); U11's cross-language path loses its gate;
|
|
115
|
+
* - DSH_SCOUT_SMOKE_SKIP_REHYDRATION_CONTROL=1 → skip both completion-marker
|
|
116
|
+
* controls; STEP 4 then has no control proving it reads the marker;
|
|
117
|
+
* - DSH_SCOUT_SMOKE_SKIP_NEGATIVE_CONTROL=1 → downgrade the output-schema
|
|
118
|
+
* probe to a warning (pre-existing switch).
|
|
119
|
+
* Every skip is reported in the PASS line as degraded coverage — never silent.
|
|
120
|
+
* DSH_SCOUT_SMOKE_VERBOSE=1 echoes each host process's raw stdout (the
|
|
121
|
+
* `DSH_SCOUT_GATE` evidence lines included) without changing any assertion.
|
|
122
|
+
*
|
|
123
|
+
* Isolation: the DSH profile commands run with `cwd` set to this gate's own
|
|
124
|
+
* temporary directory, every case directory lives under that temporary
|
|
125
|
+
* directory, and the temporary DSH home is removed in `finally`.
|
|
126
|
+
*/
|
|
1
127
|
import assert from 'node:assert/strict'
|
|
2
|
-
import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'
|
|
3
|
-
import { tmpdir } from 'node:os'
|
|
4
|
-
import { join } from 'node:path'
|
|
5
128
|
import { spawnSync } from 'node:child_process'
|
|
129
|
+
import { mkdir, mkdtemp, readdir, readFile, realpath, rm, writeFile } from 'node:fs/promises'
|
|
130
|
+
import { tmpdir } from 'node:os'
|
|
131
|
+
import { isAbsolute, join } from 'node:path'
|
|
6
132
|
|
|
7
133
|
const DSH_VERSION = process.env.DSH_VERSION ?? '0.2.0-rc.2'
|
|
134
|
+
/** Model-visible byte budget for a mutation-tool result (0.9.1 output contract). */
|
|
135
|
+
const TOKEN_BUDGET_BYTES = 1024
|
|
136
|
+
/** Set to 1 to downgrade an unavailable negative control to a loud warning. */
|
|
137
|
+
const SKIP_NEGATIVE_CONTROL = process.env.DSH_SCOUT_SMOKE_SKIP_NEGATIVE_CONTROL === '1'
|
|
138
|
+
/** Set to 1 to skip both completion-marker controls (STEP 4 loses its control). */
|
|
139
|
+
const SKIP_REHYDRATION_CONTROL = process.env.DSH_SCOUT_SMOKE_SKIP_REHYDRATION_CONTROL === '1'
|
|
140
|
+
/** Set to 1 to skip the cross-language profile (STEP 5). */
|
|
141
|
+
const SKIP_CROSS_LANGUAGE = process.env.DSH_SCOUT_SMOKE_SKIP_CROSS_LANGUAGE === '1'
|
|
142
|
+
/** Set to 1 to echo each host process's raw stdout (evidence for a red run). */
|
|
143
|
+
const VERBOSE = process.env.DSH_SCOUT_SMOKE_VERBOSE === '1'
|
|
144
|
+
const GATE_PREFIX = 'DSH_SCOUT_GATE '
|
|
145
|
+
const REQUIRED_TOOLS = [
|
|
146
|
+
'scout_start',
|
|
147
|
+
'scout_add_source',
|
|
148
|
+
'scout_add_claim',
|
|
149
|
+
'scout_verify_identity',
|
|
150
|
+
'scout_verify_claim',
|
|
151
|
+
'scout_ingest',
|
|
152
|
+
'scout_search',
|
|
153
|
+
'scout_compare',
|
|
154
|
+
'scout_report',
|
|
155
|
+
'scout_export',
|
|
156
|
+
'scout_import',
|
|
157
|
+
'scout_questions',
|
|
158
|
+
]
|
|
159
|
+
/** The six-file export: five payload files plus the completion marker. */
|
|
160
|
+
const EXPORT_MARKER_FILE = 'export.complete.json'
|
|
161
|
+
const EXPORT_FILES = [
|
|
162
|
+
'case.json',
|
|
163
|
+
'sources.json',
|
|
164
|
+
'claims.json',
|
|
165
|
+
'events.jsonl',
|
|
166
|
+
'report.md',
|
|
167
|
+
EXPORT_MARKER_FILE,
|
|
168
|
+
]
|
|
169
|
+
/**
|
|
170
|
+
* Deliberately ASCII user-authored copy: user-authored strings are never
|
|
171
|
+
* translated (only untouched factory prose is aligned when the deployment
|
|
172
|
+
* language differs), so an ASCII fixture keeps "a Han character leaked into the
|
|
173
|
+
* en artifact" distinguishable from "the user wrote Chinese". Without that, the
|
|
174
|
+
* STEP 5 CJK count could only be asserted against a moving expectation.
|
|
175
|
+
*/
|
|
176
|
+
const CASE_ARGS = {
|
|
177
|
+
caseId: 'smoke-case-001',
|
|
178
|
+
companyName: 'Sicha Verification Technology Co., Ltd.',
|
|
179
|
+
roleTitle: 'HRBP',
|
|
180
|
+
location: 'Shanghai',
|
|
181
|
+
}
|
|
182
|
+
const SOURCE_ARGS = [
|
|
183
|
+
{
|
|
184
|
+
sourceId: 'src-registry',
|
|
185
|
+
sourceType: 'company_registry',
|
|
186
|
+
title: 'National Enterprise Credit Information Publicity System record',
|
|
187
|
+
url: 'https://www.gsxt.gov.cn/corp-query-search-1.html',
|
|
188
|
+
evidenceLevel: 'E3',
|
|
189
|
+
},
|
|
190
|
+
{
|
|
191
|
+
sourceId: 'src-posting',
|
|
192
|
+
sourceType: 'job_posting',
|
|
193
|
+
title: 'HRBP job posting',
|
|
194
|
+
url: 'https://www.linkedin.com/jobs/view/4012345678',
|
|
195
|
+
evidenceLevel: 'E2',
|
|
196
|
+
},
|
|
197
|
+
]
|
|
198
|
+
const CLAIM_ARGS = {
|
|
199
|
+
claimId: 'clm-1',
|
|
200
|
+
text: 'The HRBP role reports to the founder and owns the hiring plan.',
|
|
201
|
+
status: 'reported',
|
|
202
|
+
evidenceLevel: 'E2',
|
|
203
|
+
dimension: 'reporting_line',
|
|
204
|
+
impact: 'material',
|
|
205
|
+
sourceIds: 'src-posting',
|
|
206
|
+
confidenceNote: 'Stated in the posting; not confirmed by the company yet.',
|
|
207
|
+
nextAction: 'Confirm the reporting line in the first interview.',
|
|
208
|
+
}
|
|
209
|
+
/** Han characters only, matching tests/acceptance.test.mjs. */
|
|
210
|
+
const HAN = /[\u4e00-\u9fff]/g
|
|
8
211
|
const command = name => process.platform === 'win32' ? `${name}.cmd` : name
|
|
9
212
|
const dshCommand = process.env.DSH_BIN
|
|
10
213
|
? { program: process.env.DSH_BIN, prefix: [] }
|
|
@@ -12,7 +215,7 @@ const dshCommand = process.env.DSH_BIN
|
|
|
12
215
|
|
|
13
216
|
function run(program, args, options = {}) {
|
|
14
217
|
const result = spawnSync(command(program), args, {
|
|
15
|
-
cwd: process.cwd(),
|
|
218
|
+
cwd: options.cwd ?? process.cwd(),
|
|
16
219
|
env: options.env ?? process.env,
|
|
17
220
|
encoding: 'utf8',
|
|
18
221
|
stdio: options.capture ? 'pipe' : 'inherit',
|
|
@@ -28,70 +231,944 @@ function run(program, args, options = {}) {
|
|
|
28
231
|
return result.stdout ?? ''
|
|
29
232
|
}
|
|
30
233
|
|
|
234
|
+
/** First 200 chars of a value, for "expected vs actual" diagnostics. */
|
|
235
|
+
function head(value, max = 200) {
|
|
236
|
+
const text = typeof value === 'string' ? value : JSON.stringify(value)
|
|
237
|
+
if (text === undefined) return 'undefined'
|
|
238
|
+
return text.length > max ? `${text.slice(0, max)}…` : text
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
function gateFailure(lines) {
|
|
242
|
+
const error = new Error(['dsh-scout profile smoke gate FAILED', ...lines.map(line => ` ${line}`)].join('\n'))
|
|
243
|
+
error.gateFailure = true
|
|
244
|
+
return error
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
function requireGate(condition, diagnostic) {
|
|
248
|
+
if (condition) return
|
|
249
|
+
const lines = typeof diagnostic === 'function' ? diagnostic() : diagnostic
|
|
250
|
+
throw gateFailure(Array.isArray(lines) ? lines : [lines])
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
function parseGateLines(stdout) {
|
|
254
|
+
const entries = []
|
|
255
|
+
for (const line of stdout.split('\n')) {
|
|
256
|
+
if (!line.startsWith(GATE_PREFIX)) continue
|
|
257
|
+
try {
|
|
258
|
+
entries.push(JSON.parse(line.slice(GATE_PREFIX.length)))
|
|
259
|
+
} catch (error) {
|
|
260
|
+
entries.push({ kind: 'unparsable', raw: head(line, 400), parseError: error.message })
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
return entries
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
/** Rendered text carried by a `ContentBlock[]` projection. */
|
|
267
|
+
function renderedText(content) {
|
|
268
|
+
if (!Array.isArray(content)) return ''
|
|
269
|
+
return content
|
|
270
|
+
.map(block => block && typeof block.text === 'string' ? block.text : '')
|
|
271
|
+
.filter(Boolean)
|
|
272
|
+
.join('\n')
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
/** Han-character count of a string (0 for non-strings). */
|
|
276
|
+
function cjkCount(text) {
|
|
277
|
+
return typeof text === 'string' ? (text.match(HAN) ?? []).length : 0
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
function describeCall(entry, extra = []) {
|
|
281
|
+
const result = entry?.result
|
|
282
|
+
const lines = [
|
|
283
|
+
`tool: ${entry?.tool ?? '(no gate line emitted)'}`,
|
|
284
|
+
`outcome: ${result?.outcome ?? 'no result (call skipped or plugin failed)'}`,
|
|
285
|
+
`callId: ${result?.callId ?? 'n/a'}`,
|
|
286
|
+
]
|
|
287
|
+
if (entry?.skipped) lines.push(`skipped: ${entry.skipped}`)
|
|
288
|
+
if (result?.outcome === 'threw') {
|
|
289
|
+
lines.push(`threw: ${result.error?.name ?? 'Error'}: ${result.error?.message ?? '(no message)'}`)
|
|
290
|
+
if (result.error?.violations) lines.push(`violations: ${JSON.stringify(result.error.violations)}`)
|
|
291
|
+
if (result.error?.code) lines.push(`code: ${result.error.code}`)
|
|
292
|
+
} else if (result) {
|
|
293
|
+
lines.push(`isError: ${result.isError}`)
|
|
294
|
+
lines.push(`value type: ${result.valueType ?? typeof result.value}`)
|
|
295
|
+
lines.push(`contentBytes: ${result.contentBytes}`)
|
|
296
|
+
if (result.isError) lines.push(`error: ${head(result.error, 300)}`)
|
|
297
|
+
}
|
|
298
|
+
lines.push(`content[0:200]: ${head(result?.content)}`)
|
|
299
|
+
return [...lines, ...extra]
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
/** One-line read of a gate entry's result for diagnostics. */
|
|
303
|
+
function describeResult(result) {
|
|
304
|
+
if (!result) return 'no result'
|
|
305
|
+
const bits = [`outcome=${result.outcome}`]
|
|
306
|
+
if (result.outcome === 'returned') {
|
|
307
|
+
bits.push(`isError=${result.isError}`)
|
|
308
|
+
if (result.isError) bits.push(`error=${head(result.error, 200)}`)
|
|
309
|
+
} else {
|
|
310
|
+
bits.push(`error=${result.error?.name}: ${result.error?.message}`)
|
|
311
|
+
}
|
|
312
|
+
return bits.join(' ')
|
|
313
|
+
}
|
|
314
|
+
|
|
31
315
|
const packageJson = JSON.parse(await readFile(new URL('../package.json', import.meta.url), 'utf8'))
|
|
32
316
|
const gateRoot = await mkdtemp(join(tmpdir(), 'dsh-scout-profile-gate-'))
|
|
33
317
|
const dshHome = join(gateRoot, 'home')
|
|
34
318
|
const tarball = join(gateRoot, `${packageJson.name}-${packageJson.version}.tgz`)
|
|
35
319
|
const verifierPath = join(gateRoot, 'verify-lifecycle.mjs')
|
|
36
|
-
|
|
320
|
+
/**
|
|
321
|
+
* One shot of the shared setup: every case lives under this absolute directory
|
|
322
|
+
* and BOTH profiles point their `scoutDir` at it, which is what lets STEP 5
|
|
323
|
+
* import what STEP 2 exported (and lets STEP 4/6 find the case on disk).
|
|
324
|
+
*/
|
|
325
|
+
const scoutDataDir = join(gateRoot, 'scout-data')
|
|
326
|
+
const zhExportDir = join(scoutDataDir, CASE_ARGS.caseId)
|
|
327
|
+
const rehydratedExportDir = join(gateRoot, 'rehydrated-export')
|
|
328
|
+
const enExportDir = join(gateRoot, 'en-export')
|
|
329
|
+
const patchZhPath = join(gateRoot, 'profile-zh.patch.yml')
|
|
330
|
+
const patchEnPath = join(gateRoot, 'profile-en.patch.yml')
|
|
331
|
+
const patchVerifierOnlyPath = join(gateRoot, 'verifier-only.patch.yml')
|
|
332
|
+
const ZH_PROFILE = 'scout-smoke'
|
|
333
|
+
const EN_PROFILE = 'scout-smoke-en'
|
|
37
334
|
const gateEnv = {
|
|
38
335
|
...process.env,
|
|
39
336
|
DSH_HOME: dshHome,
|
|
40
337
|
DSH_TELEMETRY_MODE: 'DISABLED',
|
|
41
338
|
}
|
|
42
339
|
|
|
43
|
-
|
|
44
|
-
|
|
340
|
+
/**
|
|
341
|
+
* The injected verifier plugin. It runs inside the real DSH process with the
|
|
342
|
+
* `tools` service injected, emits one `DSH_SCOUT_GATE <json>` line per observed
|
|
343
|
+
* fact, and always leaves the assertions to this script (single place for
|
|
344
|
+
* expected-vs-actual diagnostics). The scenario (which calls to make) comes from
|
|
345
|
+
* the environment; the plugin never decides whether a fact is correct.
|
|
346
|
+
*/
|
|
347
|
+
const verifierSource = `
|
|
45
348
|
export const inject = ['tools']
|
|
349
|
+
|
|
350
|
+
const REQUIRED = ${JSON.stringify(REQUIRED_TOOLS)}
|
|
351
|
+
const CASE_ARGS = ${JSON.stringify(CASE_ARGS)}
|
|
352
|
+
const SOURCES = ${JSON.stringify(SOURCE_ARGS)}
|
|
353
|
+
const CLAIM = ${JSON.stringify(CLAIM_ARGS)}
|
|
354
|
+
const TOKEN_BUDGET_BYTES = ${TOKEN_BUDGET_BYTES}
|
|
355
|
+
const PROBE_NAME = 'scout_smoke_output_probe'
|
|
356
|
+
const SCENARIO = process.env.DSH_SCOUT_GATE_SCENARIO ?? 'registry-only'
|
|
357
|
+
const CASE_ID = process.env.DSH_SCOUT_GATE_CASE_ID ?? CASE_ARGS.caseId
|
|
358
|
+
const ZH_EXPORT_DIR = process.env.DSH_SCOUT_GATE_ZH_EXPORT_DIR
|
|
359
|
+
const REHYDRATED_EXPORT_DIR = process.env.DSH_SCOUT_GATE_REHYDRATED_EXPORT_DIR
|
|
360
|
+
const EN_EXPORT_DIR = process.env.DSH_SCOUT_GATE_EN_EXPORT_DIR
|
|
361
|
+
const WAIT_MS = Number(process.env.DSH_SCOUT_GATE_WAIT_MS ?? '5000')
|
|
362
|
+
|
|
363
|
+
let callSequence = 0
|
|
364
|
+
|
|
365
|
+
const emit = (kind, payload) => {
|
|
366
|
+
process.stdout.write('${GATE_PREFIX}' + JSON.stringify(Object.assign({ kind: kind }, payload)) + '\\n')
|
|
367
|
+
}
|
|
368
|
+
|
|
369
|
+
const describeError = error => ({
|
|
370
|
+
name: error && error.name ? error.name : typeof error,
|
|
371
|
+
message: error && error.message ? error.message : String(error),
|
|
372
|
+
code: error && error.code ? error.code : null,
|
|
373
|
+
violations: error && Array.isArray(error.violations) ? error.violations : null,
|
|
374
|
+
})
|
|
375
|
+
|
|
376
|
+
const waitForTool = async (ctx, name, deadline) => {
|
|
377
|
+
for (;;) {
|
|
378
|
+
if (ctx.tools.get(name)) return true
|
|
379
|
+
if (Date.now() >= deadline) return false
|
|
380
|
+
await new Promise(resolve => setTimeout(resolve, 25))
|
|
381
|
+
}
|
|
382
|
+
}
|
|
383
|
+
|
|
384
|
+
const callTool = async (ctx, name, args) => {
|
|
385
|
+
callSequence += 1
|
|
386
|
+
const callId = 'smoke-' + name + '-' + callSequence
|
|
387
|
+
try {
|
|
388
|
+
const result = await ctx.tools.execute({
|
|
389
|
+
callId: callId,
|
|
390
|
+
name: name,
|
|
391
|
+
arguments: args,
|
|
392
|
+
signal: new AbortController().signal,
|
|
393
|
+
})
|
|
394
|
+
const content = result.content === undefined ? null : result.content
|
|
395
|
+
return {
|
|
396
|
+
outcome: 'returned',
|
|
397
|
+
callId: callId,
|
|
398
|
+
isError: result.isError === true,
|
|
399
|
+
hasValue: Object.prototype.hasOwnProperty.call(result, 'value'),
|
|
400
|
+
value: result.value === undefined ? null : result.value,
|
|
401
|
+
valueType: Array.isArray(result.value) ? 'array' : typeof result.value,
|
|
402
|
+
content: content,
|
|
403
|
+
contentBytes: Buffer.byteLength(JSON.stringify(content)),
|
|
404
|
+
meta: result.meta === undefined ? null : result.meta,
|
|
405
|
+
error: result.isError === true ? result.error : null,
|
|
406
|
+
}
|
|
407
|
+
} catch (error) {
|
|
408
|
+
return { outcome: 'threw', callId: callId, error: describeError(error) }
|
|
409
|
+
}
|
|
410
|
+
}
|
|
411
|
+
|
|
46
412
|
export async function apply(ctx) {
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
'
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
413
|
+
const startedAt = Date.now()
|
|
414
|
+
try {
|
|
415
|
+
emit('boot', { pid: process.pid, scenario: SCENARIO, caseId: CASE_ID })
|
|
416
|
+
const deadline = startedAt + WAIT_MS
|
|
417
|
+
const missing = []
|
|
418
|
+
for (const name of REQUIRED) {
|
|
419
|
+
if (!(await waitForTool(ctx, name, deadline))) missing.push(name)
|
|
420
|
+
}
|
|
421
|
+
emit('registry', {
|
|
422
|
+
required: REQUIRED,
|
|
423
|
+
missing: missing,
|
|
424
|
+
appExit: typeof ctx.appExit,
|
|
425
|
+
execute: typeof ctx.tools.execute,
|
|
426
|
+
fs: typeof ctx.get('fs'),
|
|
427
|
+
})
|
|
428
|
+
|
|
429
|
+
const runScenario = async () => {
|
|
430
|
+
if (SCENARIO === 'create-export') {
|
|
431
|
+
// STEP 1 — create the case and register a source plus a claim.
|
|
432
|
+
emit('call', { tool: 'scout_start', args: CASE_ARGS, result: await callTool(ctx, 'scout_start', CASE_ARGS) })
|
|
433
|
+
for (const source of SOURCES) {
|
|
434
|
+
emit('call', { tool: 'scout_add_source', args: source, result: await callTool(ctx, 'scout_add_source', Object.assign({ caseId: CASE_ID }, source)) })
|
|
435
|
+
}
|
|
436
|
+
emit('call', { tool: 'scout_add_claim', args: CLAIM, result: await callTool(ctx, 'scout_add_claim', Object.assign({ caseId: CASE_ID }, CLAIM)) })
|
|
437
|
+
// STEP 2 — export to an EXPLICIT absolute directory (never process cwd).
|
|
438
|
+
emit('call', {
|
|
439
|
+
tool: 'scout_export',
|
|
440
|
+
args: { caseId: CASE_ID, targetDir: ZH_EXPORT_DIR },
|
|
441
|
+
result: await callTool(ctx, 'scout_export', { caseId: CASE_ID, targetDir: ZH_EXPORT_DIR }),
|
|
442
|
+
})
|
|
443
|
+
emit('call', { tool: 'scout_report', args: { caseId: CASE_ID }, result: await callTool(ctx, 'scout_report', { caseId: CASE_ID }) })
|
|
444
|
+
} else if (SCENARIO === 'rehydrate') {
|
|
445
|
+
// STEP 4 — no import: the case must come back from <scoutDir>/<caseId>.
|
|
446
|
+
emit('call', { tool: 'scout_report', args: { caseId: CASE_ID }, result: await callTool(ctx, 'scout_report', { caseId: CASE_ID }) })
|
|
447
|
+
// A follow-up export is what puts the in-memory case_rehydrated event on
|
|
448
|
+
// disk; without it the event exists only in the process that read it.
|
|
449
|
+
emit('call', {
|
|
450
|
+
tool: 'scout_export',
|
|
451
|
+
args: { caseId: CASE_ID, targetDir: REHYDRATED_EXPORT_DIR },
|
|
452
|
+
result: await callTool(ctx, 'scout_export', { caseId: CASE_ID, targetDir: REHYDRATED_EXPORT_DIR }),
|
|
453
|
+
})
|
|
454
|
+
} else if (SCENARIO === 'import-en') {
|
|
455
|
+
// STEP 5 — the en deployment imports the zh export from the shared dir.
|
|
456
|
+
emit('call', {
|
|
457
|
+
tool: 'scout_import',
|
|
458
|
+
args: { caseId: CASE_ID, sourceDir: ZH_EXPORT_DIR },
|
|
459
|
+
result: await callTool(ctx, 'scout_import', { caseId: CASE_ID, sourceDir: ZH_EXPORT_DIR }),
|
|
460
|
+
})
|
|
461
|
+
emit('call', { tool: 'scout_report', args: { caseId: CASE_ID }, result: await callTool(ctx, 'scout_report', { caseId: CASE_ID }) })
|
|
462
|
+
emit('call', {
|
|
463
|
+
tool: 'scout_export',
|
|
464
|
+
args: { caseId: CASE_ID, targetDir: EN_EXPORT_DIR },
|
|
465
|
+
result: await callTool(ctx, 'scout_export', { caseId: CASE_ID, targetDir: EN_EXPORT_DIR }),
|
|
466
|
+
})
|
|
467
|
+
} else if (SCENARIO === 'report-only') {
|
|
468
|
+
// Negative controls: the marker is missing or says complete:false, so
|
|
469
|
+
// this call is expected to FAIL — the assertion lives in the host script.
|
|
470
|
+
emit('call', { tool: 'scout_report', args: { caseId: CASE_ID }, result: await callTool(ctx, 'scout_report', { caseId: CASE_ID }) })
|
|
471
|
+
}
|
|
472
|
+
}
|
|
473
|
+
|
|
474
|
+
if (missing.length === 0) await runScenario()
|
|
475
|
+
else emit('skipped', { reason: 'required tools are not mounted', missing: missing })
|
|
476
|
+
|
|
477
|
+
// Negative control: the declared output schema is an object, the body returns
|
|
478
|
+
// a string, so a registry that enforces output schemas must reject the value.
|
|
479
|
+
try {
|
|
480
|
+
const dispose = ctx.tools.register({
|
|
481
|
+
name: PROBE_NAME,
|
|
482
|
+
description: 'Gate self-test: declares an object output schema and deliberately returns a string.',
|
|
483
|
+
parameters: { type: 'object', properties: {}, additionalProperties: false },
|
|
484
|
+
output: {
|
|
485
|
+
schema: { type: 'object', properties: { ok: { type: 'string' } }, additionalProperties: false },
|
|
486
|
+
render: () => [{ type: 'text', text: 'output-schema probe' }],
|
|
487
|
+
},
|
|
488
|
+
execute: async () => 'deliberately-not-an-object',
|
|
489
|
+
})
|
|
490
|
+
try {
|
|
491
|
+
emit('negativeControl', { status: 'ran', tool: PROBE_NAME, result: await callTool(ctx, PROBE_NAME, {}) })
|
|
492
|
+
} finally {
|
|
493
|
+
dispose()
|
|
494
|
+
}
|
|
495
|
+
} catch (error) {
|
|
496
|
+
emit('negativeControl', { status: 'unavailable', tool: PROBE_NAME, error: describeError(error) })
|
|
497
|
+
}
|
|
498
|
+
|
|
499
|
+
process.stdout.write('DSH_SCOUT_LOAD\\n')
|
|
500
|
+
ctx.effect(() => () => process.stdout.write('DSH_SCOUT_UNLOAD\\n'))
|
|
501
|
+
emit('done', { elapsedMs: Date.now() - startedAt })
|
|
502
|
+
queueMicrotask(() => ctx.appExit(0))
|
|
503
|
+
} catch (error) {
|
|
504
|
+
emit('fatal', { error: describeError(error) })
|
|
505
|
+
queueMicrotask(() => ctx.appExit(1))
|
|
506
|
+
}
|
|
507
|
+
}
|
|
508
|
+
`
|
|
509
|
+
|
|
510
|
+
/** The per-profile overlay: absolute scoutDir + language + the verifier insert. */
|
|
511
|
+
function profilePatch(language) {
|
|
512
|
+
return [
|
|
513
|
+
'- id: dsh-scout',
|
|
514
|
+
' config:',
|
|
515
|
+
` scoutDir: ${JSON.stringify(scoutDataDir)}`,
|
|
516
|
+
` language: ${JSON.stringify(language)}`,
|
|
517
|
+
'- insert:',
|
|
518
|
+
' - id: dsh-scout-smoke-verifier',
|
|
519
|
+
` name: ${JSON.stringify(verifierPath)}`,
|
|
520
|
+
'',
|
|
521
|
+
].join('\n')
|
|
522
|
+
}
|
|
523
|
+
|
|
524
|
+
/** Insert only the verifier, for a boot where dsh-scout has been uninstalled. */
|
|
525
|
+
const verifierOnlyPatch = [
|
|
526
|
+
'- insert:',
|
|
527
|
+
' - id: dsh-scout-smoke-verifier',
|
|
528
|
+
` name: ${JSON.stringify(verifierPath)}`,
|
|
529
|
+
'',
|
|
530
|
+
].join('\n')
|
|
531
|
+
|
|
532
|
+
/**
|
|
533
|
+
* Boot the real host once and capture its gate lines. `run()` throws on a
|
|
534
|
+
* non-zero exit, so a boot that cannot even start is a hard failure here.
|
|
535
|
+
*/
|
|
536
|
+
function boot(label, { profile, patch, scenario, env = {} }) {
|
|
537
|
+
const stdout = run(dshCommand.program, [
|
|
68
538
|
...dshCommand.prefix,
|
|
69
|
-
'plugin',
|
|
70
539
|
'--profile',
|
|
71
|
-
|
|
72
|
-
'
|
|
73
|
-
|
|
74
|
-
], {
|
|
540
|
+
profile,
|
|
541
|
+
'--patch',
|
|
542
|
+
patch,
|
|
543
|
+
], {
|
|
544
|
+
capture: true,
|
|
545
|
+
env: {
|
|
546
|
+
...gateEnv,
|
|
547
|
+
DSH_SCOUT_GATE_SCENARIO: scenario,
|
|
548
|
+
DSH_SCOUT_GATE_CASE_ID: CASE_ARGS.caseId,
|
|
549
|
+
DSH_SCOUT_GATE_ZH_EXPORT_DIR: zhExportDir,
|
|
550
|
+
DSH_SCOUT_GATE_REHYDRATED_EXPORT_DIR: rehydratedExportDir,
|
|
551
|
+
DSH_SCOUT_GATE_EN_EXPORT_DIR: enExportDir,
|
|
552
|
+
...env,
|
|
553
|
+
},
|
|
554
|
+
cwd: gateRoot,
|
|
555
|
+
})
|
|
556
|
+
const entry = { label, scenario, profile, stdout, gates: parseGateLines(stdout) }
|
|
557
|
+
entry.find = (kind, tool) => entry.gates.find(gate => gate.kind === kind && (tool === undefined || gate.tool === tool))
|
|
558
|
+
entry.raw = () => [`gate lines captured: ${entry.gates.length}`, `dsh output[0:600]: ${head(entry.stdout, 600)}`]
|
|
559
|
+
// Raw evidence on demand: the assertions only read the parsed gate lines, but
|
|
560
|
+
// a human debugging a red run wants the host's own stdout.
|
|
561
|
+
if (VERBOSE) {
|
|
562
|
+
console.log(`--- ${label} (${scenario}, profile ${profile}) ---`)
|
|
563
|
+
console.log(stdout.trimEnd())
|
|
564
|
+
}
|
|
565
|
+
return entry
|
|
566
|
+
}
|
|
567
|
+
|
|
568
|
+
/** Every boot must load the plugin and dispose it cleanly. */
|
|
569
|
+
function assertLifecycle(entry) {
|
|
570
|
+
requireGate(entry.stdout.includes('DSH_SCOUT_LOAD'), [
|
|
571
|
+
`lifecycle assertion (${entry.label}): the verifier plugin never reached its load line`,
|
|
572
|
+
...entry.raw(),
|
|
573
|
+
])
|
|
574
|
+
requireGate(entry.stdout.includes('DSH_SCOUT_UNLOAD'), [
|
|
575
|
+
`lifecycle assertion (${entry.label}): the plugin disposer did not run on unload (no DSH_SCOUT_UNLOAD)`,
|
|
576
|
+
...entry.raw(),
|
|
577
|
+
])
|
|
578
|
+
requireGate(entry.find('done') !== undefined, [
|
|
579
|
+
`lifecycle assertion (${entry.label}): the verifier plugin never reached its \`done\` line`,
|
|
580
|
+
...entry.raw(),
|
|
581
|
+
])
|
|
582
|
+
}
|
|
583
|
+
|
|
584
|
+
/**
|
|
585
|
+
* Registry assertions shared by every boot. `expectMissing` is the exact set of
|
|
586
|
+
* tools that must NOT be mounted (empty for a healthy profile).
|
|
587
|
+
*/
|
|
588
|
+
function assertRegistry(entry, expectMissing = []) {
|
|
589
|
+
const registry = entry.find('registry')
|
|
590
|
+
requireGate(registry !== undefined, () => [
|
|
591
|
+
`registry assertion (${entry.label}): the verifier plugin emitted no \`registry\` gate line`,
|
|
592
|
+
...entry.raw(),
|
|
593
|
+
])
|
|
594
|
+
requireGate(registry.missing.length === expectMissing.length && expectMissing.every(name => registry.missing.includes(name)), [
|
|
595
|
+
`registry assertion (${entry.label}): unexpected mounted-tool set`,
|
|
596
|
+
`expected missing: ${JSON.stringify(expectMissing)}`,
|
|
597
|
+
`actually missing: ${JSON.stringify(registry.missing)}`,
|
|
598
|
+
])
|
|
599
|
+
requireGate(registry.execute === 'function', [
|
|
600
|
+
`runtime API assertion (${entry.label}): ctx.tools.execute is not a function in this DSH version`,
|
|
601
|
+
`expected: function actual: ${registry.execute}`,
|
|
602
|
+
])
|
|
603
|
+
requireGate(registry.appExit === 'function', [
|
|
604
|
+
`launcher assertion (${entry.label}): ctx.appExit is not a function`,
|
|
605
|
+
`expected: function actual: ${registry.appExit}`,
|
|
606
|
+
])
|
|
607
|
+
return registry
|
|
608
|
+
}
|
|
609
|
+
|
|
610
|
+
/** The result object of a `call` gate, with a fail-fast assertion when absent. */
|
|
611
|
+
function callResult(entry, tool, extra = []) {
|
|
612
|
+
const gate = entry.find('call', tool)
|
|
613
|
+
requireGate(gate?.result !== undefined, () => [
|
|
614
|
+
`tool call assertion (${entry.label}): no result for ${tool}`,
|
|
615
|
+
...describeCall(gate, entry.raw()),
|
|
616
|
+
])
|
|
617
|
+
requireGate(gate.result.outcome === 'returned', () => describeCall(gate, [
|
|
618
|
+
'expected: execute() resolves with a ToolExecutionResult',
|
|
619
|
+
'actual: execute() threw',
|
|
620
|
+
...extra,
|
|
621
|
+
]))
|
|
622
|
+
return gate
|
|
623
|
+
}
|
|
624
|
+
|
|
625
|
+
/** A real call that must succeed. */
|
|
626
|
+
function requireOkCall(entry, tool, extra = []) {
|
|
627
|
+
const gate = callResult(entry, tool, extra)
|
|
628
|
+
requireGate(gate.result.isError === false, () => describeCall(gate, [
|
|
629
|
+
`expected (${entry.label}): isError === false for ${tool}`,
|
|
630
|
+
`actual: isError === ${gate.result.isError}`,
|
|
631
|
+
...extra,
|
|
632
|
+
]))
|
|
633
|
+
return gate.result
|
|
634
|
+
}
|
|
635
|
+
|
|
636
|
+
/** Read a JSON file into an object (used for the exported artifacts). */
|
|
637
|
+
async function readJson(path) {
|
|
638
|
+
return JSON.parse(await readFile(path, 'utf8'))
|
|
639
|
+
}
|
|
640
|
+
|
|
641
|
+
/** Canonical path of an existing file/directory, or null when it does not exist. */
|
|
642
|
+
async function realpathOrNull(path) {
|
|
643
|
+
if (typeof path !== 'string' || path === '') return null
|
|
644
|
+
try {
|
|
645
|
+
return await realpath(path)
|
|
646
|
+
} catch {
|
|
647
|
+
return null
|
|
648
|
+
}
|
|
649
|
+
}
|
|
650
|
+
|
|
651
|
+
const stepLines = []
|
|
652
|
+
const step = (name, detail) => {
|
|
653
|
+
stepLines.push(` ${name}: ${detail}`)
|
|
654
|
+
}
|
|
655
|
+
const warnings = []
|
|
656
|
+
|
|
657
|
+
try {
|
|
658
|
+
await mkdir(scoutDataDir, { recursive: true })
|
|
659
|
+
await writeFile(verifierPath, verifierSource)
|
|
660
|
+
await writeFile(patchZhPath, profilePatch('zh-CN'))
|
|
661
|
+
await writeFile(patchEnPath, profilePatch('en'))
|
|
662
|
+
await writeFile(patchVerifierOnlyPath, verifierOnlyPatch)
|
|
663
|
+
|
|
664
|
+
run('pnpm', ['pack', '--pack-destination', gateRoot])
|
|
665
|
+
const profiles = [ZH_PROFILE]
|
|
666
|
+
if (!SKIP_CROSS_LANGUAGE) profiles.push(EN_PROFILE)
|
|
667
|
+
for (const profile of profiles) {
|
|
668
|
+
run(dshCommand.program, [
|
|
669
|
+
...dshCommand.prefix,
|
|
670
|
+
'plugin',
|
|
671
|
+
'--profile',
|
|
672
|
+
profile,
|
|
673
|
+
'add',
|
|
674
|
+
`file:${tarball}`,
|
|
675
|
+
], { env: gateEnv, cwd: gateRoot })
|
|
676
|
+
}
|
|
677
|
+
|
|
75
678
|
const config = run(dshCommand.program, [
|
|
76
679
|
...dshCommand.prefix,
|
|
77
680
|
'--profile',
|
|
78
|
-
|
|
681
|
+
ZH_PROFILE,
|
|
79
682
|
'--dump-config',
|
|
80
683
|
], {
|
|
81
684
|
capture: true,
|
|
82
685
|
env: gateEnv,
|
|
686
|
+
cwd: gateRoot,
|
|
83
687
|
})
|
|
84
688
|
assert.match(config, /# == dsh-scout\n- id: dsh-scout\n name: dsh-scout/)
|
|
85
|
-
|
|
689
|
+
// The overlay's `config:` block must actually reach the plugin: this is what
|
|
690
|
+
// makes the absolute base directory in every step below a verified fact rather
|
|
691
|
+
// than an assumption about the profile layer.
|
|
692
|
+
const patchedConfig = run(dshCommand.program, [
|
|
86
693
|
...dshCommand.prefix,
|
|
87
694
|
'--profile',
|
|
88
|
-
|
|
695
|
+
ZH_PROFILE,
|
|
89
696
|
'--patch',
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
697
|
+
patchZhPath,
|
|
698
|
+
'--dump-config',
|
|
699
|
+
], {
|
|
700
|
+
capture: true,
|
|
701
|
+
env: gateEnv,
|
|
702
|
+
cwd: gateRoot,
|
|
703
|
+
})
|
|
704
|
+
// The dump folds long scalars into YAML block scalars (`scoutDir: >-` + an
|
|
705
|
+
// indented line), so assert on the patched layer block rather than one line.
|
|
706
|
+
const patchedLayer = patchedConfig.split('# == dsh-scout, patched by')[1]?.split('# ==')[0] ?? ''
|
|
707
|
+
requireGate(
|
|
708
|
+
patchedLayer.includes('scoutDir:') && patchedLayer.includes(scoutDataDir),
|
|
709
|
+
[
|
|
710
|
+
'config-injection assertion: the profile patch `config.scoutDir` did not reach the plugin layer',
|
|
711
|
+
`expected to find: scoutDir: ${scoutDataDir}`,
|
|
712
|
+
`patched layer: ${head(patchedLayer, 400)}`,
|
|
713
|
+
],
|
|
714
|
+
)
|
|
715
|
+
requireGate(
|
|
716
|
+
/language:\s*'?"?zh-CN"?/.test(patchedLayer),
|
|
717
|
+
[
|
|
718
|
+
'config-injection assertion: the profile patch `config.language` did not reach the plugin layer',
|
|
719
|
+
`patched layer: ${head(patchedLayer, 400)}`,
|
|
720
|
+
],
|
|
721
|
+
)
|
|
722
|
+
|
|
723
|
+
// --- STEP 1 + 2: create the case, register, export to an absolute dir -------
|
|
724
|
+
const create = boot('step1-2 zh create+export', { profile: ZH_PROFILE, patch: patchZhPath, scenario: 'create-export' })
|
|
725
|
+
const registry = assertRegistry(create)
|
|
726
|
+
assertLifecycle(create)
|
|
727
|
+
requireGate(registry.fs === 'object', [
|
|
728
|
+
'fs seam assertion: the real host exposed no `fs` service, so the export/import',
|
|
729
|
+
'paths below cannot be exercised at all (do not treat this as a pass).',
|
|
730
|
+
`typeof ctx.get('fs') === ${registry.fs}`,
|
|
731
|
+
])
|
|
732
|
+
|
|
733
|
+
const startGate = callResult(create, 'scout_start')
|
|
734
|
+
const start = requireOkCall(create, 'scout_start')
|
|
735
|
+
requireGate(start.hasValue === true && start.value !== null && typeof start.value === 'object', () => describeCall(startGate, [
|
|
736
|
+
"expected: typeof result.value === 'object' (0.9.1 structured-output contract)",
|
|
737
|
+
`actual: hasValue=${start.hasValue} valueType=${start.valueType}`,
|
|
738
|
+
]))
|
|
739
|
+
// Contract-level shape check from the frozen 0.9.1 envelope: the mutation value
|
|
740
|
+
// must echo the requested case id (a stray `{}` or an unrelated object fails).
|
|
741
|
+
requireGate(start.value.caseId === CASE_ARGS.caseId, () => describeCall(startGate, [
|
|
742
|
+
`expected: result.value.caseId === ${JSON.stringify(CASE_ARGS.caseId)} (mutation envelope echoes the case)`,
|
|
743
|
+
`actual: caseId=${JSON.stringify(start.value.caseId)}`,
|
|
744
|
+
`value keys: ${JSON.stringify(Object.keys(start.value ?? {}))}`,
|
|
745
|
+
]))
|
|
746
|
+
requireGate(start.contentBytes === Buffer.byteLength(JSON.stringify(start.content)), () => describeCall(startGate, [
|
|
747
|
+
'emission fidelity: the JSON-encoded content does not round-trip to the same byte size',
|
|
748
|
+
`plugin-side bytes: ${start.contentBytes}`,
|
|
749
|
+
`script-side bytes: ${Buffer.byteLength(JSON.stringify(start.content))}`,
|
|
750
|
+
]))
|
|
751
|
+
requireGate(start.contentBytes < TOKEN_BUDGET_BYTES, () => describeCall(startGate, [
|
|
752
|
+
`expected: Buffer.byteLength(JSON.stringify(result.content)) < ${TOKEN_BUDGET_BYTES}`,
|
|
753
|
+
`actual: ${start.contentBytes} bytes (budget exceeded by ${start.contentBytes - TOKEN_BUDGET_BYTES})`,
|
|
754
|
+
]))
|
|
755
|
+
|
|
756
|
+
const sourceGates = create.gates.filter(gate => gate.kind === 'call' && gate.tool === 'scout_add_source')
|
|
757
|
+
requireGate(sourceGates.length === SOURCE_ARGS.length, [
|
|
758
|
+
'tool call assertion: expected one scout_add_source gate line per registered source',
|
|
759
|
+
`expected: ${SOURCE_ARGS.length} actual: ${sourceGates.length}`,
|
|
760
|
+
])
|
|
761
|
+
sourceGates.forEach((gate, index) => {
|
|
762
|
+
const sourceId = SOURCE_ARGS[index].sourceId
|
|
763
|
+
requireGate(gate.result?.outcome === 'returned' && gate.result.isError === false, () => describeCall(gate, [
|
|
764
|
+
`expected: scout_add_source(${sourceId}) returns isError === false`,
|
|
765
|
+
`actual: ${describeResult(gate.result)}`,
|
|
766
|
+
]))
|
|
767
|
+
// The envelope is the model-visible status summary; its counts must move.
|
|
768
|
+
requireGate(gate.result.value?.counts?.sources === index + 1, () => describeCall(gate, [
|
|
769
|
+
`expected: the envelope count reflects ${index + 1} registered source(s) after ${sourceId}`,
|
|
770
|
+
`actual: counts=${JSON.stringify(gate.result.value?.counts)}`,
|
|
771
|
+
]))
|
|
772
|
+
})
|
|
773
|
+
const claim = requireOkCall(create, 'scout_add_claim', ['the exported case must carry a claim'])
|
|
774
|
+
requireGate(claim.value?.counts?.claims === 1, () => describeCall(create.find('call', 'scout_add_claim'), [
|
|
775
|
+
'expected: the envelope count reflects 1 registered claim',
|
|
776
|
+
`actual: counts=${JSON.stringify(claim.value?.counts)}`,
|
|
777
|
+
]))
|
|
778
|
+
|
|
779
|
+
const firstReport = requireOkCall(create, 'scout_report')
|
|
780
|
+
const firstReportText = renderedText(firstReport.content)
|
|
781
|
+
requireGate(firstReportText.length >= 200, () => describeCall(create.find('call', 'scout_report'), [
|
|
782
|
+
'expected: rendered content carries the report body (>= 200 chars)',
|
|
783
|
+
`actual: ${firstReportText.length} rendered chars`,
|
|
784
|
+
]))
|
|
785
|
+
requireGate(firstReportText.includes(CASE_ARGS.companyName), () => describeCall(create.find('call', 'scout_report'), [
|
|
786
|
+
`expected: report body contains the case subject ${JSON.stringify(CASE_ARGS.companyName)}`,
|
|
787
|
+
`actual: subject not found in ${firstReportText.length} rendered chars`,
|
|
788
|
+
]))
|
|
789
|
+
const reportHasHeading = /^#\s/m.test(firstReportText)
|
|
790
|
+
if (!reportHasHeading) {
|
|
791
|
+
warnings.push('report body has no Markdown heading; the "report body" assertion only checked length and case subject')
|
|
792
|
+
}
|
|
793
|
+
// Positive control for STEP 5: the zh deployment must really render zh prose,
|
|
794
|
+
// otherwise "CJK = 0 in en" would be satisfied by an English-only plugin.
|
|
795
|
+
requireGate(cjkCount(firstReportText) > 0, () => describeCall(create.find('call', 'scout_report'), [
|
|
796
|
+
'expected: the zh-CN report contains Han characters (positive control for STEP 5)',
|
|
797
|
+
`actual: ${cjkCount(firstReportText)} Han characters`,
|
|
798
|
+
]))
|
|
799
|
+
|
|
800
|
+
// STEP 2 — the export itself.
|
|
801
|
+
const exportGate = callResult(create, 'scout_export')
|
|
802
|
+
const exported = requireOkCall(create, 'scout_export')
|
|
803
|
+
requireGate(exported.value?.persisted === true, () => describeCall(exportGate, [
|
|
804
|
+
'expected: scout_export value.persisted === true',
|
|
805
|
+
`actual: ${JSON.stringify(exported.value)}`,
|
|
806
|
+
]))
|
|
807
|
+
requireGate(exported.value?.complete === true, () => describeCall(exportGate, [
|
|
808
|
+
'expected: scout_export value.complete === true',
|
|
809
|
+
`actual: ${JSON.stringify(exported.value)}`,
|
|
810
|
+
]))
|
|
811
|
+
requireGate(
|
|
812
|
+
Array.isArray(exported.value?.files)
|
|
813
|
+
&& exported.value.files.length === EXPORT_FILES.length
|
|
814
|
+
&& EXPORT_FILES.every(name => exported.value.files.includes(name)),
|
|
815
|
+
() => describeCall(exportGate, [
|
|
816
|
+
`expected: value.files lists the six export files ${JSON.stringify(EXPORT_FILES)}`,
|
|
817
|
+
`actual: ${JSON.stringify(exported.value?.files)}`,
|
|
818
|
+
]),
|
|
819
|
+
)
|
|
820
|
+
// Path-basis assertion: the resolved directory must be the absolute directory
|
|
821
|
+
// that was passed. If the plugin had resolved against the DSH process cwd (or
|
|
822
|
+
// any relative base), this equality fails instead of the gate silently writing
|
|
823
|
+
// somewhere the CI job happens to own. Both sides go through `realpath` so a
|
|
824
|
+
// platform that reports the canonical form (`/private/var` for `/var`) is not
|
|
825
|
+
// mistaken for a different directory.
|
|
826
|
+
const reportedTargetDir = exported.value?.targetDir
|
|
827
|
+
const reportedReal = await realpathOrNull(reportedTargetDir)
|
|
828
|
+
const requestedReal = await realpathOrNull(zhExportDir)
|
|
829
|
+
requireGate(
|
|
830
|
+
isAbsolute(reportedTargetDir ?? '') && reportedReal !== null && reportedReal === requestedReal,
|
|
831
|
+
() => describeCall(exportGate, [
|
|
832
|
+
`expected: value.targetDir resolves to ${JSON.stringify(zhExportDir)} (absolute, as passed)`,
|
|
833
|
+
`actual: ${JSON.stringify(reportedTargetDir)} → ${JSON.stringify(reportedReal)}`,
|
|
834
|
+
`requested: ${JSON.stringify(zhExportDir)} → ${JSON.stringify(requestedReal)}`,
|
|
835
|
+
]),
|
|
836
|
+
)
|
|
837
|
+
|
|
838
|
+
const writtenFiles = (await readdir(zhExportDir)).sort()
|
|
839
|
+
requireGate(
|
|
840
|
+
writtenFiles.length === EXPORT_FILES.length && [...EXPORT_FILES].sort().every(name => writtenFiles.includes(name)),
|
|
841
|
+
[
|
|
842
|
+
'disk assertion: the export directory does not hold exactly the six expected files',
|
|
843
|
+
`expected: ${JSON.stringify([...EXPORT_FILES].sort())}`,
|
|
844
|
+
`actual: ${JSON.stringify(writtenFiles)}`,
|
|
845
|
+
`directory: ${zhExportDir}`,
|
|
846
|
+
],
|
|
847
|
+
)
|
|
848
|
+
const zhMarker = await readJson(join(zhExportDir, EXPORT_MARKER_FILE))
|
|
849
|
+
requireGate(zhMarker.complete === true, [
|
|
850
|
+
'marker assertion: export.complete.json does not read complete: true',
|
|
851
|
+
`actual: ${JSON.stringify(zhMarker)}`,
|
|
852
|
+
])
|
|
853
|
+
requireGate(zhMarker.caseId === CASE_ARGS.caseId, [
|
|
854
|
+
'marker assertion: export.complete.json names a different case',
|
|
855
|
+
`expected: ${CASE_ARGS.caseId} actual: ${JSON.stringify(zhMarker.caseId)}`,
|
|
856
|
+
])
|
|
857
|
+
const zhCase = await readJson(join(zhExportDir, 'case.json'))
|
|
858
|
+
const zhReportFile = await readFile(join(zhExportDir, 'report.md'), 'utf8')
|
|
859
|
+
const zhClaims = await readJson(join(zhExportDir, 'claims.json'))
|
|
860
|
+
requireGate(zhCase.language === 'zh-CN', [
|
|
861
|
+
'artifact assertion: the zh deployment exported a case.json that does not declare zh-CN',
|
|
862
|
+
`actual: ${JSON.stringify(zhCase.language)}`,
|
|
863
|
+
])
|
|
864
|
+
requireGate(zhClaims.length === 1, [
|
|
865
|
+
'artifact assertion: the exported claims.json does not carry the registered claim',
|
|
866
|
+
`actual: ${JSON.stringify(zhClaims.map(entry => entry.claimId))}`,
|
|
867
|
+
])
|
|
868
|
+
requireGate(cjkCount(zhReportFile) > 0, [
|
|
869
|
+
'positive control: the exported zh report.md carries no Han characters, so the',
|
|
870
|
+
'STEP 5 "CJK = 0" assertion below would be vacuous.',
|
|
871
|
+
`actual: ${cjkCount(zhReportFile)} Han characters`,
|
|
872
|
+
])
|
|
873
|
+
const exportedDecision = zhCase.decision
|
|
874
|
+
requireGate(exportedDecision === firstReport.value?.decision, [
|
|
875
|
+
'decision stability: the decision inside the exported case.json differs from the one reported before the export',
|
|
876
|
+
`case.json: ${JSON.stringify(exportedDecision)} scout_report: ${JSON.stringify(firstReport.value?.decision)}`,
|
|
877
|
+
])
|
|
878
|
+
step('step 1 registry+create', `PASS (${REQUIRED_TOOLS.length} tools mounted, fs seam present, scout_start/add_source×2/add_claim isError=false, value=object, scout_start content=${start.contentBytes}B < ${TOKEN_BUDGET_BYTES}, zh report ${cjkCount(firstReportText)} Han chars)`)
|
|
879
|
+
step('step 2 export', `PASS (persisted=true complete=true, targetDir absolute & equal to the requested path, ${writtenFiles.length}/6 files on disk, marker complete=true, decision=${exportedDecision})`)
|
|
880
|
+
|
|
881
|
+
// --- STEP 3 + 4: cold restart, then lazy rehydration ------------------------
|
|
882
|
+
const rehydrate = boot('step3-4 zh rehydrate', { profile: ZH_PROFILE, patch: patchZhPath, scenario: 'rehydrate' })
|
|
883
|
+
assertRegistry(rehydrate)
|
|
884
|
+
assertLifecycle(rehydrate)
|
|
885
|
+
const createPid = create.find('boot')?.pid
|
|
886
|
+
const rehydratePid = rehydrate.find('boot')?.pid
|
|
887
|
+
requireGate(typeof createPid === 'number' && typeof rehydratePid === 'number' && createPid !== rehydratePid, [
|
|
888
|
+
'cold-restart assertion: the rehydration step did not run in a different OS process',
|
|
889
|
+
`step1-2 pid: ${createPid} step3-4 pid: ${rehydratePid}`,
|
|
890
|
+
'a shared pid would mean the in-memory case cache survived, so nothing was rehydrated from disk.',
|
|
891
|
+
])
|
|
892
|
+
step('step 3 cold restart', `PASS (second boot on the same DSH_HOME/profile, pid ${createPid} → ${rehydratePid})`)
|
|
893
|
+
|
|
894
|
+
const rehydratedReport = requireOkCall(rehydrate, 'scout_report')
|
|
895
|
+
const rehydratedText = renderedText(rehydratedReport.content)
|
|
896
|
+
requireGate(rehydratedReport.value?.decision === exportedDecision, () => describeCall(rehydrate.find('call', 'scout_report'), [
|
|
897
|
+
'lazy-rehydration assertion: the decision after a cold start differs from the exported one',
|
|
898
|
+
`exported: ${JSON.stringify(exportedDecision)} rehydrated: ${JSON.stringify(rehydratedReport.value?.decision)}`,
|
|
899
|
+
`rendered[0:300]: ${head(rehydratedText, 300)}`,
|
|
900
|
+
]))
|
|
901
|
+
requireGate(rehydratedText.includes(CASE_ARGS.companyName), () => describeCall(rehydrate.find('call', 'scout_report'), [
|
|
902
|
+
`lazy-rehydration assertion: the rehydrated report lost the case subject ${JSON.stringify(CASE_ARGS.companyName)}`,
|
|
903
|
+
`rendered[0:300]: ${head(rehydratedText, 300)}`,
|
|
904
|
+
]))
|
|
905
|
+
const rehydratedExport = requireOkCall(rehydrate, 'scout_export')
|
|
906
|
+
requireGate(rehydratedExport.value?.persisted === true, () => describeCall(rehydrate.find('call', 'scout_export'), [
|
|
907
|
+
'lazy-rehydration assertion: the follow-up export of the rehydrated case did not persist',
|
|
908
|
+
`actual: ${JSON.stringify(rehydratedExport.value)}`,
|
|
909
|
+
]))
|
|
910
|
+
const rehydratedEvents = await readFile(join(rehydratedExportDir, 'events.jsonl'), 'utf8')
|
|
911
|
+
const rehydratedCase = await readJson(join(rehydratedExportDir, 'case.json'))
|
|
912
|
+
const rehydratedSources = await readJson(join(rehydratedExportDir, 'sources.json'))
|
|
913
|
+
const rehydratedClaims = await readJson(join(rehydratedExportDir, 'claims.json'))
|
|
914
|
+
requireGate(rehydratedEvents.includes('"case_rehydrated"'), [
|
|
915
|
+
'lazy-rehydration assertion: events.jsonl carries no case_rehydrated event',
|
|
916
|
+
`expected: a case_rehydrated line in ${join(rehydratedExportDir, 'events.jsonl')}`,
|
|
917
|
+
// Never let the diagnostic itself throw on a malformed file.
|
|
918
|
+
`event types seen: ${JSON.stringify(rehydratedEvents.trim().split('\n').filter(Boolean).map(line => {
|
|
919
|
+
try {
|
|
920
|
+
return JSON.parse(line).type
|
|
921
|
+
} catch {
|
|
922
|
+
return '(unparsable line)'
|
|
923
|
+
}
|
|
924
|
+
}))}`,
|
|
925
|
+
])
|
|
926
|
+
requireGate(rehydratedCase.decision === exportedDecision, [
|
|
927
|
+
'lazy-rehydration assertion: the re-exported case.json carries a different decision than the original export',
|
|
928
|
+
`original: ${JSON.stringify(exportedDecision)} re-exported: ${JSON.stringify(rehydratedCase.decision)}`,
|
|
929
|
+
])
|
|
930
|
+
requireGate(rehydratedEvents.includes(zhExportDir), [
|
|
931
|
+
'lazy-rehydration assertion: the case_rehydrated event does not name the directory it restored from',
|
|
932
|
+
`expected to find: ${zhExportDir}`,
|
|
933
|
+
`events[0:400]: ${head(rehydratedEvents, 400)}`,
|
|
934
|
+
])
|
|
935
|
+
requireGate(rehydratedSources.length === SOURCE_ARGS.length && rehydratedClaims.length === 1, [
|
|
936
|
+
'lazy-rehydration assertion: the restored case lost sources or claims across the process boundary',
|
|
937
|
+
`expected: ${SOURCE_ARGS.length} sources, 1 claim`,
|
|
938
|
+
`actual: ${rehydratedSources.length} sources, ${rehydratedClaims.length} claims`,
|
|
939
|
+
])
|
|
940
|
+
step('step 4 lazy rehydrate', `PASS (scout_report with no import returned decision=${rehydratedReport.value?.decision}, ${rehydratedSources.length} sources + ${rehydratedClaims.length} claim restored, case_rehydrated persisted to events.jsonl)`)
|
|
941
|
+
|
|
942
|
+
// --- STEP 5: cross-language import into a second profile --------------------
|
|
943
|
+
let enCase = null
|
|
944
|
+
let enReportFile = ''
|
|
945
|
+
|
|
946
|
+
if (SKIP_CROSS_LANGUAGE) {
|
|
947
|
+
step('step 5 cross-language', 'SKIPPED (DSH_SCOUT_SMOKE_SKIP_CROSS_LANGUAGE=1 — coverage degraded: the en import path has no real-host gate)')
|
|
948
|
+
} else {
|
|
949
|
+
const importEn = boot('step5 en import', { profile: EN_PROFILE, patch: patchEnPath, scenario: 'import-en' })
|
|
950
|
+
assertRegistry(importEn)
|
|
951
|
+
assertLifecycle(importEn)
|
|
952
|
+
const imported = requireOkCall(importEn, 'scout_import')
|
|
953
|
+
requireGate(imported.value?.complete === true, () => describeCall(importEn.find('call', 'scout_import'), [
|
|
954
|
+
'cross-language assertion: scout_import did not read the completion marker (complete !== true)',
|
|
955
|
+
`actual: ${JSON.stringify(imported.value)}`,
|
|
956
|
+
]))
|
|
957
|
+
requireGate(imported.value?.caseId === CASE_ARGS.caseId, () => describeCall(importEn.find('call', 'scout_import'), [
|
|
958
|
+
'cross-language assertion: the imported case id does not echo the requested one',
|
|
959
|
+
`actual: ${JSON.stringify(imported.value?.caseId)}`,
|
|
960
|
+
]))
|
|
961
|
+
const enReport = requireOkCall(importEn, 'scout_report')
|
|
962
|
+
const enText = renderedText(enReport.content)
|
|
963
|
+
requireGate(cjkCount(enText) === 0, () => describeCall(importEn.find('call', 'scout_report'), [
|
|
964
|
+
'cross-language assertion: the en deployment rendered Han characters after importing the zh export',
|
|
965
|
+
`actual: ${cjkCount(enText)} Han characters`,
|
|
966
|
+
`rendered[0:400]: ${head(enText, 400)}`,
|
|
967
|
+
]))
|
|
968
|
+
requireGate(enText.includes(CASE_ARGS.companyName), () => describeCall(importEn.find('call', 'scout_report'), [
|
|
969
|
+
`cross-language assertion: the ASCII subject ${JSON.stringify(CASE_ARGS.companyName)} did not survive alignment`,
|
|
970
|
+
`rendered[0:300]: ${head(enText, 300)}`,
|
|
971
|
+
]))
|
|
972
|
+
requireGate(enReport.value?.decision === exportedDecision, () => describeCall(importEn.find('call', 'scout_report'), [
|
|
973
|
+
'cross-language assertion: importing the zh export into the en deployment changed the decision',
|
|
974
|
+
`exported: ${JSON.stringify(exportedDecision)} reported en: ${JSON.stringify(enReport.value?.decision)}`,
|
|
975
|
+
]))
|
|
976
|
+
const enExport = requireOkCall(importEn, 'scout_export')
|
|
977
|
+
requireGate(enExport.value?.persisted === true, () => describeCall(importEn.find('call', 'scout_export'), [
|
|
978
|
+
'cross-language assertion: the en deployment could not export the imported case',
|
|
979
|
+
`actual: ${JSON.stringify(enExport.value)}`,
|
|
980
|
+
]))
|
|
981
|
+
enCase = await readJson(join(enExportDir, 'case.json'))
|
|
982
|
+
enReportFile = await readFile(join(enExportDir, 'report.md'), 'utf8')
|
|
983
|
+
|
|
984
|
+
requireGate(enCase.language === 'en', [
|
|
985
|
+
'cross-language assertion: the exported case.json does not declare language en',
|
|
986
|
+
`actual: ${JSON.stringify(enCase.language)}`,
|
|
987
|
+
])
|
|
988
|
+
requireGate(cjkCount(enReportFile) === 0, [
|
|
989
|
+
'cross-language assertion: the exported en report.md contains Han characters',
|
|
990
|
+
`actual: ${cjkCount(enReportFile)} Han characters`,
|
|
991
|
+
`report.md[0:400]: ${head(enReportFile, 400)}`,
|
|
992
|
+
])
|
|
993
|
+
requireGate(enCase.subject?.name === CASE_ARGS.companyName, [
|
|
994
|
+
'cross-language assertion: user-authored subject data was altered by factory-default alignment',
|
|
995
|
+
`expected: ${JSON.stringify(CASE_ARGS.companyName)} actual: ${JSON.stringify(enCase.subject?.name)}`,
|
|
996
|
+
])
|
|
997
|
+
// Evidence, not an assertion: does the en deployment touch the zh export in
|
|
998
|
+
// the shared scoutDir? Today it does not (no autoPersist), and knowing if
|
|
999
|
+
// that ever changes is more useful than a brittle red gate here.
|
|
1000
|
+
const zhCaseAfter = await readJson(join(zhExportDir, 'case.json'))
|
|
1001
|
+
if (zhCaseAfter.language !== 'zh-CN') {
|
|
1002
|
+
warnings.push(`the en deployment rewrote the zh case.json in the shared scoutDir (language=${JSON.stringify(zhCaseAfter.language)})`)
|
|
1003
|
+
}
|
|
1004
|
+
step('step 5 cross-language', `PASS (en profile imported the zh export: complete=true, render CJK=0, report.md CJK=0 vs zh report.md ${cjkCount(zhReportFile)} Han chars, case.json.language=en, ASCII subject preserved)`)
|
|
1005
|
+
}
|
|
1006
|
+
|
|
1007
|
+
// --- STEP 6 + negative controls: marker semantics, then uninstall -----------
|
|
1008
|
+
let markerControlSummary = 'completion-marker negative controls SKIPPED (coverage degraded: STEP 4 has no control proving it reads the marker)'
|
|
1009
|
+
if (SKIP_REHYDRATION_CONTROL) {
|
|
1010
|
+
step('negative control marker', markerControlSummary)
|
|
1011
|
+
} else {
|
|
1012
|
+
// Control A — the marker file is gone; payload files and the case directory
|
|
1013
|
+
// are still there, so only "read the marker" can explain the refusal.
|
|
1014
|
+
const markerPath = join(zhExportDir, EXPORT_MARKER_FILE)
|
|
1015
|
+
const markerRaw = await readFile(markerPath, 'utf8')
|
|
1016
|
+
await rm(markerPath)
|
|
1017
|
+
const missingMarker = boot('control marker missing', { profile: ZH_PROFILE, patch: patchZhPath, scenario: 'report-only' })
|
|
1018
|
+
assertRegistry(missingMarker)
|
|
1019
|
+
assertLifecycle(missingMarker)
|
|
1020
|
+
const missingResult = callResult(missingMarker, 'scout_report').result
|
|
1021
|
+
requireGate(missingResult.isError === true, () => describeCall(missingMarker.find('call', 'scout_report'), [
|
|
1022
|
+
'marker control A: with export.complete.json deleted, scout_report still returned a case',
|
|
1023
|
+
'expected: isError === true (no restorable export)',
|
|
1024
|
+
`actual: isError === ${missingResult.isError}`,
|
|
1025
|
+
'without this control STEP 4 could pass on a path that never reads the marker.',
|
|
1026
|
+
]))
|
|
1027
|
+
requireGate(
|
|
1028
|
+
typeof missingResult.error?.message === 'string'
|
|
1029
|
+
&& missingResult.error.message.includes('Unknown case')
|
|
1030
|
+
&& missingResult.error.message.includes(CASE_ARGS.caseId),
|
|
1031
|
+
() => describeCall(missingMarker.find('call', 'scout_report'), [
|
|
1032
|
+
'marker control A: scout_report failed for a reason other than the missing export',
|
|
1033
|
+
`expected: an \`Unknown case: ${CASE_ARGS.caseId}\` refusal naming the no-restorable-export directory`,
|
|
1034
|
+
`actual: ${head(missingResult.error?.message ?? '(no message)', 300)}`,
|
|
1035
|
+
]),
|
|
1036
|
+
)
|
|
1037
|
+
|
|
1038
|
+
// Control B — the marker exists but says complete:false. Existence must not
|
|
1039
|
+
// be enough; the content has to be read.
|
|
1040
|
+
const brokenMarker = { ...JSON.parse(markerRaw), complete: false }
|
|
1041
|
+
await writeFile(markerPath, `${JSON.stringify(brokenMarker, null, 2)}\n`)
|
|
1042
|
+
const falseMarker = boot('control marker complete:false', { profile: ZH_PROFILE, patch: patchZhPath, scenario: 'report-only' })
|
|
1043
|
+
assertRegistry(falseMarker)
|
|
1044
|
+
assertLifecycle(falseMarker)
|
|
1045
|
+
const falseResult = callResult(falseMarker, 'scout_report').result
|
|
1046
|
+
requireGate(falseResult.isError === true, () => describeCall(falseMarker.find('call', 'scout_report'), [
|
|
1047
|
+
'marker control B: a marker with complete:false was accepted as a finished export',
|
|
1048
|
+
'expected: isError === true',
|
|
1049
|
+
`actual: isError === ${falseResult.isError}`,
|
|
1050
|
+
`marker on disk: ${JSON.stringify(brokenMarker)}`,
|
|
1051
|
+
]))
|
|
1052
|
+
requireGate(
|
|
1053
|
+
typeof falseResult.error?.message === 'string' && falseResult.error.message.includes('Unknown case'),
|
|
1054
|
+
() => describeCall(falseMarker.find('call', 'scout_report'), [
|
|
1055
|
+
'marker control B: scout_report failed for an unexpected reason',
|
|
1056
|
+
`expected: an \`Unknown case\` refusal`,
|
|
1057
|
+
`actual: ${head(falseResult.error?.message ?? '(no message)', 300)}`,
|
|
1058
|
+
]),
|
|
1059
|
+
)
|
|
1060
|
+
const falseMarkerRendered = renderedText(falseResult.content ?? [])
|
|
1061
|
+
requireGate(falseResult.value === null && cjkCount(falseMarkerRendered) === 0, [
|
|
1062
|
+
'marker control B: the refused call still produced a case body',
|
|
1063
|
+
`actual value: ${head(falseResult.value)}`,
|
|
1064
|
+
])
|
|
1065
|
+
// Both controls only isolate the marker if the payload files were still
|
|
1066
|
+
// there while it was refused — otherwise the refusal would be explained by
|
|
1067
|
+
// a missing export directory rather than by the marker.
|
|
1068
|
+
const survivingPayload = (await readdir(zhExportDir)).sort()
|
|
1069
|
+
requireGate(
|
|
1070
|
+
['case.json', 'claims.json', 'events.jsonl', 'report.md', 'sources.json'].every(name => survivingPayload.includes(name)),
|
|
1071
|
+
[
|
|
1072
|
+
'marker control: the payload files did not survive the marker controls, so this run does',
|
|
1073
|
+
'not isolate the completion marker (a missing export directory would explain the refusal).',
|
|
1074
|
+
`directory: ${zhExportDir}`,
|
|
1075
|
+
`entries: ${JSON.stringify(survivingPayload)}`,
|
|
1076
|
+
],
|
|
1077
|
+
)
|
|
1078
|
+
// Restore the marker so the uninstall step still runs against a complete case.
|
|
1079
|
+
await writeFile(markerPath, markerRaw)
|
|
1080
|
+
const restoredMarker = await readJson(markerPath)
|
|
1081
|
+
requireGate(restoredMarker.complete === true, [
|
|
1082
|
+
'marker control cleanup: the completion marker was not restored before the uninstall step',
|
|
1083
|
+
`actual: ${JSON.stringify(restoredMarker)}`,
|
|
1084
|
+
])
|
|
1085
|
+
markerControlSummary = 'completion-marker negative controls (missing marker AND complete:false both refused rehydration)'
|
|
1086
|
+
step('negative control marker', `PASS (marker deleted → isError=true "Unknown case"; marker complete:false → same refusal; payload files untouched both times)`)
|
|
1087
|
+
}
|
|
1088
|
+
|
|
1089
|
+
// STEP 6 — uninstall: no layer, no tools, and the post-uninstall boot still
|
|
1090
|
+
// proves the verifier itself runs (so "no tools" is not a dead process).
|
|
1091
|
+
run(dshCommand.program, [
|
|
1092
|
+
...dshCommand.prefix,
|
|
1093
|
+
'plugin',
|
|
1094
|
+
'--profile',
|
|
1095
|
+
ZH_PROFILE,
|
|
1096
|
+
'remove',
|
|
1097
|
+
'dsh-scout',
|
|
1098
|
+
], { env: gateEnv, cwd: gateRoot })
|
|
1099
|
+
const uninstalledConfig = run(dshCommand.program, [
|
|
1100
|
+
...dshCommand.prefix,
|
|
1101
|
+
'--profile',
|
|
1102
|
+
ZH_PROFILE,
|
|
1103
|
+
'--dump-config',
|
|
1104
|
+
], {
|
|
1105
|
+
capture: true,
|
|
1106
|
+
env: gateEnv,
|
|
1107
|
+
cwd: gateRoot,
|
|
1108
|
+
})
|
|
1109
|
+
requireGate(!uninstalledConfig.includes('# == dsh-scout'), [
|
|
1110
|
+
'uninstall assertion: the dsh-scout layer is still composed into the profile after `dsh plugin remove`',
|
|
1111
|
+
`dump-config[0:600]: ${head(uninstalledConfig, 600)}`,
|
|
1112
|
+
])
|
|
1113
|
+
const uninstalled = boot('step6 post-uninstall', {
|
|
1114
|
+
profile: ZH_PROFILE,
|
|
1115
|
+
patch: patchVerifierOnlyPath,
|
|
1116
|
+
scenario: 'registry-only',
|
|
1117
|
+
// All twelve tools are absent: one shared deadline is enough.
|
|
1118
|
+
env: { DSH_SCOUT_GATE_WAIT_MS: '1500' },
|
|
1119
|
+
})
|
|
1120
|
+
assertRegistry(uninstalled, REQUIRED_TOOLS)
|
|
1121
|
+
assertLifecycle(uninstalled)
|
|
1122
|
+
step('step 6 uninstall', `PASS (dsh-scout layer gone from --dump-config; post-removal boot mounted 0/${REQUIRED_TOOLS.length} tools and the verifier still ran)`)
|
|
1123
|
+
|
|
1124
|
+
// --- the output-schema negative control (pre-existing; asserted on the boot
|
|
1125
|
+
// --- that made the real calls, so it guards STEP 1/2's evidence) -------------
|
|
1126
|
+
const negative = create.find('negativeControl')
|
|
1127
|
+
requireGate(negative !== undefined, () => [
|
|
1128
|
+
'output-schema self-test: no `negativeControl` gate line',
|
|
1129
|
+
...create.raw(),
|
|
1130
|
+
])
|
|
1131
|
+
if (negative.status === 'unavailable' && !SKIP_NEGATIVE_CONTROL) {
|
|
1132
|
+
throw gateFailure([
|
|
1133
|
+
'output-schema self-test: the probe tool could not be registered, so this run does NOT prove',
|
|
1134
|
+
'that the real calls above were validated against their declared output schemas.',
|
|
1135
|
+
`probe registration error: ${negative.error?.name}: ${negative.error?.message}`,
|
|
1136
|
+
're-run with DSH_SCOUT_SMOKE_SKIP_NEGATIVE_CONTROL=1 only if you accept that reduced coverage.',
|
|
1137
|
+
])
|
|
1138
|
+
}
|
|
1139
|
+
if (negative.status === 'ran') {
|
|
1140
|
+
const probe = negative.result
|
|
1141
|
+
const detected = probe.outcome === 'threw'
|
|
1142
|
+
? probe.error?.name === 'ToolOutputError' || probe.error?.code === 'INVALID_TOOL_OUTPUT'
|
|
1143
|
+
: probe.isError === true && (probe.error?.info?.name === 'ToolOutputError' || probe.error?.info?.code === 'INVALID_TOOL_OUTPUT')
|
|
1144
|
+
requireGate(detected, () => describeCall({ tool: negative.tool, result: probe }, [
|
|
1145
|
+
'expected: the registry rejects a string value against a declared object output schema',
|
|
1146
|
+
' (ToolOutputError / INVALID_TOOL_OUTPUT)',
|
|
1147
|
+
'actual: the invalid value was accepted, so the real-call assertions above are NOT schema gates',
|
|
1148
|
+
]))
|
|
1149
|
+
}
|
|
1150
|
+
|
|
1151
|
+
const negativeSummary = negative.status === 'ran'
|
|
1152
|
+
? 'output-schema negative control (ToolOutputError observed)'
|
|
1153
|
+
: 'output-schema negative control SKIPPED (coverage degraded)'
|
|
1154
|
+
console.log(`dsh-scout profile smoke: PASS (dsh-scout ${packageJson.version}, dsh ${DSH_VERSION})`)
|
|
1155
|
+
console.log(` scenario chain: ${stepLines.length} gate lines (6 steps + the completion-marker control)`)
|
|
1156
|
+
for (const line of stepLines) console.log(line)
|
|
1157
|
+
console.log(` verified: dump-config layer + patch config injection | ${REQUIRED_TOOLS.length} tools mounted | scout_start(isError=false, value=object with caseId echo, content=${start.contentBytes}B < ${TOKEN_BUDGET_BYTES}) | scout_export(absolute targetDir, 6/6 files, marker complete=true) | cold restart (pid change) | lazy rehydration (decision ${exportedDecision}, case_rehydrated persisted) | ${SKIP_CROSS_LANGUAGE ? 'cross-language SKIPPED' : `cross-language import (case.json.language=en, CJK=0, zh control ${cjkCount(zhReportFile)} Han chars)`} | uninstall (0 tools left) | ${negativeSummary} | ${markerControlSummary}`)
|
|
1158
|
+
// Evidence, deliberately NOT an assertion: the frozen 0.9.1 read-tool envelope
|
|
1159
|
+
// (`scout_report.value = {caseId, decision, report, coverage}`) is printed so a
|
|
1160
|
+
// human can compare it against the contract; only content/report-body presence
|
|
1161
|
+
// is enforced above, to avoid over-fitting this gate to one envelope revision.
|
|
1162
|
+
console.log(` observed value keys: scout_start=${JSON.stringify(Object.keys(start.value ?? {}))} scout_report=${JSON.stringify(Object.keys(firstReport.value ?? {}))}${enCase ? ` en case.json keys=${JSON.stringify(Object.keys(enCase))}` : ''}`)
|
|
1163
|
+
console.log(' not covered: no agent/LLM/approval path (agent unset ⇒ absolute dirs only), 6/12 tools executed, report asserted structurally, autoPersist stays false, token budget covers rendered content only')
|
|
1164
|
+
for (const warning of warnings) console.warn(` WARNING: ${warning}`)
|
|
1165
|
+
} catch (error) {
|
|
1166
|
+
if (error?.gateFailure) {
|
|
1167
|
+
console.error(error.message)
|
|
1168
|
+
process.exitCode = 1
|
|
1169
|
+
} else {
|
|
1170
|
+
throw error
|
|
1171
|
+
}
|
|
95
1172
|
} finally {
|
|
96
1173
|
await rm(gateRoot, { recursive: true, force: true })
|
|
97
1174
|
}
|