@substrate-ai/sdlc 0.20.3 → 0.20.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/run-model/index.d.ts +2 -2
- package/dist/run-model/index.d.ts.map +1 -1
- package/dist/run-model/index.js +2 -1
- package/dist/run-model/index.js.map +1 -1
- package/dist/run-model/per-story-state.d.ts +16 -2
- package/dist/run-model/per-story-state.d.ts.map +1 -1
- package/dist/run-model/schemas.d.ts +16 -2
- package/dist/run-model/schemas.d.ts.map +1 -1
- package/dist/run-model/verification-result.d.ts +53 -3
- package/dist/run-model/verification-result.d.ts.map +1 -1
- package/dist/run-model/verification-result.js +28 -0
- package/dist/run-model/verification-result.js.map +1 -1
- package/dist/verification/checks/acceptance-criteria-evidence-check.d.ts.map +1 -1
- package/dist/verification/checks/acceptance-criteria-evidence-check.js +65 -8
- package/dist/verification/checks/acceptance-criteria-evidence-check.js.map +1 -1
- package/dist/verification/checks/build-check.d.ts.map +1 -1
- package/dist/verification/checks/build-check.js +64 -8
- package/dist/verification/checks/build-check.js.map +1 -1
- package/dist/verification/checks/index.d.ts +4 -1
- package/dist/verification/checks/index.d.ts.map +1 -1
- package/dist/verification/checks/index.js +3 -1
- package/dist/verification/checks/index.js.map +1 -1
- package/dist/verification/checks/phantom-review-check.d.ts.map +1 -1
- package/dist/verification/checks/phantom-review-check.js +21 -2
- package/dist/verification/checks/phantom-review-check.js.map +1 -1
- package/dist/verification/checks/runtime-probe-check.d.ts +43 -0
- package/dist/verification/checks/runtime-probe-check.d.ts.map +1 -0
- package/dist/verification/checks/runtime-probe-check.js +152 -0
- package/dist/verification/checks/runtime-probe-check.js.map +1 -0
- package/dist/verification/checks/trivial-output-check.d.ts.map +1 -1
- package/dist/verification/checks/trivial-output-check.js +21 -3
- package/dist/verification/checks/trivial-output-check.js.map +1 -1
- package/dist/verification/findings.d.ts +61 -0
- package/dist/verification/findings.d.ts.map +1 -0
- package/dist/verification/findings.js +39 -0
- package/dist/verification/findings.js.map +1 -0
- package/dist/verification/index.d.ts +6 -1
- package/dist/verification/index.d.ts.map +1 -1
- package/dist/verification/index.js +4 -0
- package/dist/verification/index.js.map +1 -1
- package/dist/verification/probes/executor.d.ts +39 -0
- package/dist/verification/probes/executor.d.ts.map +1 -0
- package/dist/verification/probes/executor.js +109 -0
- package/dist/verification/probes/executor.js.map +1 -0
- package/dist/verification/probes/index.d.ts +14 -0
- package/dist/verification/probes/index.d.ts.map +1 -0
- package/dist/verification/probes/index.js +12 -0
- package/dist/verification/probes/index.js.map +1 -0
- package/dist/verification/probes/parser.d.ts +26 -0
- package/dist/verification/probes/parser.d.ts.map +1 -0
- package/dist/verification/probes/parser.js +134 -0
- package/dist/verification/probes/parser.js.map +1 -0
- package/dist/verification/probes/types.d.ts +122 -0
- package/dist/verification/probes/types.d.ts.map +1 -0
- package/dist/verification/probes/types.js +68 -0
- package/dist/verification/probes/types.js.map +1 -0
- package/dist/verification/types.d.ts +10 -1
- package/dist/verification/types.d.ts.map +1 -1
- package/dist/verification/verification-pipeline.d.ts +3 -1
- package/dist/verification/verification-pipeline.d.ts.map +1 -1
- package/dist/verification/verification-pipeline.js +20 -2
- package/dist/verification/verification-pipeline.js.map +1 -1
- package/package.json +6 -3
|
@@ -2,14 +2,16 @@
|
|
|
2
2
|
* Barrel export for VerificationCheck implementations.
|
|
3
3
|
*
|
|
4
4
|
* Check registration order (Tier A before Tier B, and within Tier A in
|
|
5
|
-
* pipeline sequence order
|
|
5
|
+
* pipeline sequence order):
|
|
6
6
|
* 1. PhantomReviewCheck — story 51-2
|
|
7
7
|
* 2. TrivialOutputCheck — story 51-3
|
|
8
8
|
* 3. AcceptanceCriteriaEvidenceCheck
|
|
9
9
|
* 4. BuildCheck — story 51-4
|
|
10
|
+
* 5. RuntimeProbeCheck — Epic 55 Phase 2
|
|
10
11
|
*/
|
|
11
12
|
export { PhantomReviewCheck } from './phantom-review-check.js';
|
|
12
13
|
export { TrivialOutputCheck, DEFAULT_TRIVIAL_OUTPUT_THRESHOLD } from './trivial-output-check.js';
|
|
13
14
|
export { AcceptanceCriteriaEvidenceCheck, extractAcceptanceCriteriaIds } from './acceptance-criteria-evidence-check.js';
|
|
14
15
|
export { BuildCheck, BUILD_CHECK_TIMEOUT_MS, detectBuildCommand } from './build-check.js';
|
|
16
|
+
export { RuntimeProbeCheck } from './runtime-probe-check.js';
|
|
15
17
|
//# sourceMappingURL=index.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/verification/checks/index.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/verification/checks/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,EAAE,kBAAkB,EAAE,MAAM,2BAA2B,CAAA;AAC9D,OAAO,EAAE,kBAAkB,EAAE,gCAAgC,EAAE,MAAM,2BAA2B,CAAA;AAChG,OAAO,EAAE,+BAA+B,EAAE,4BAA4B,EAAE,MAAM,yCAAyC,CAAA;AACvH,OAAO,EAAE,UAAU,EAAE,sBAAsB,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAA;AACzF,OAAO,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAA"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"phantom-review-check.d.ts","sourceRoot":"","sources":["../../../src/verification/checks/phantom-review-check.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,KAAK,EACV,iBAAiB,EACjB,mBAAmB,
|
|
1
|
+
{"version":3,"file":"phantom-review-check.d.ts","sourceRoot":"","sources":["../../../src/verification/checks/phantom-review-check.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,KAAK,EACV,iBAAiB,EACjB,mBAAmB,EAEnB,kBAAkB,EACnB,MAAM,aAAa,CAAA;AAOpB;;;;;;;;;GASG;AACH,qBAAa,kBAAmB,YAAW,iBAAiB;IAC1D,QAAQ,CAAC,IAAI,oBAAmB;IAChC,QAAQ,CAAC,IAAI,EAAG,GAAG,CAAS;IAEtB,GAAG,CAAC,OAAO,EAAE,mBAAmB,GAAG,OAAO,CAAC,kBAAkB,CAAC;CA+DrE"}
|
|
@@ -10,6 +10,7 @@
|
|
|
10
10
|
* - No shell invocations — pure static signal inspection over VerificationContext fields.
|
|
11
11
|
* - Runs first in Tier A (before TrivialOutputCheck, before BuildCheck).
|
|
12
12
|
*/
|
|
13
|
+
import { renderFindings } from '../findings.js';
|
|
13
14
|
// ---------------------------------------------------------------------------
|
|
14
15
|
// PhantomReviewCheck
|
|
15
16
|
// ---------------------------------------------------------------------------
|
|
@@ -35,6 +36,7 @@ export class PhantomReviewCheck {
|
|
|
35
36
|
status: 'pass',
|
|
36
37
|
details: 'phantom-review: no review result in context — skipping check',
|
|
37
38
|
duration_ms: Date.now() - start,
|
|
39
|
+
findings: [],
|
|
38
40
|
};
|
|
39
41
|
}
|
|
40
42
|
// AC1 + AC3: dispatch itself failed (non-zero exit, timeout, crash, or schema validation failure)
|
|
@@ -42,10 +44,18 @@ export class PhantomReviewCheck {
|
|
|
42
44
|
const reason = review.error === 'schema_validation_failed'
|
|
43
45
|
? 'schema validation failed'
|
|
44
46
|
: `dispatch failed${review.error ? ` — ${review.error}` : ''}`;
|
|
47
|
+
const findings = [
|
|
48
|
+
{
|
|
49
|
+
category: 'phantom-review',
|
|
50
|
+
severity: 'error',
|
|
51
|
+
message: reason,
|
|
52
|
+
},
|
|
53
|
+
];
|
|
45
54
|
return {
|
|
46
55
|
status: 'fail',
|
|
47
|
-
details:
|
|
56
|
+
details: renderFindings(findings),
|
|
48
57
|
duration_ms: Date.now() - start,
|
|
58
|
+
findings,
|
|
49
59
|
};
|
|
50
60
|
}
|
|
51
61
|
// AC2: agent produced no output — only flag when rawOutput is explicitly empty
|
|
@@ -53,10 +63,18 @@ export class PhantomReviewCheck {
|
|
|
53
63
|
// dispatch result may not have captured it (e.g., parsed YAML result without
|
|
54
64
|
// raw output preservation) — treat as pass since dispatchFailed was not set.
|
|
55
65
|
if (review.rawOutput !== undefined && review.rawOutput.trim().length === 0) {
|
|
66
|
+
const findings = [
|
|
67
|
+
{
|
|
68
|
+
category: 'phantom-review',
|
|
69
|
+
severity: 'error',
|
|
70
|
+
message: 'empty review output',
|
|
71
|
+
},
|
|
72
|
+
];
|
|
56
73
|
return {
|
|
57
74
|
status: 'fail',
|
|
58
|
-
details:
|
|
75
|
+
details: renderFindings(findings),
|
|
59
76
|
duration_ms: Date.now() - start,
|
|
77
|
+
findings,
|
|
60
78
|
};
|
|
61
79
|
}
|
|
62
80
|
// AC5: valid review — non-empty output, no dispatch failure
|
|
@@ -64,6 +82,7 @@ export class PhantomReviewCheck {
|
|
|
64
82
|
status: 'pass',
|
|
65
83
|
details: 'phantom-review: review output is valid',
|
|
66
84
|
duration_ms: Date.now() - start,
|
|
85
|
+
findings: [],
|
|
67
86
|
};
|
|
68
87
|
}
|
|
69
88
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"phantom-review-check.js","sourceRoot":"","sources":["../../../src/verification/checks/phantom-review-check.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAQH,8EAA8E;AAC9E,qBAAqB;AACrB,8EAA8E;AAE9E;;;;;;;;;GASG;AACH,MAAM,OAAO,kBAAkB;IACpB,IAAI,GAAG,gBAAgB,CAAA;IACvB,IAAI,GAAG,GAAY,CAAA;IAE5B,KAAK,CAAC,GAAG,CAAC,OAA4B;QACpC,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;QACxB,MAAM,MAAM,GAAG,OAAO,CAAC,YAAY,CAAA;QAEnC,+EAA+E;QAC/E,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,OAAO;gBACL,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,8DAA8D;gBACvE,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;
|
|
1
|
+
{"version":3,"file":"phantom-review-check.js","sourceRoot":"","sources":["../../../src/verification/checks/phantom-review-check.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAQH,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAA;AAE/C,8EAA8E;AAC9E,qBAAqB;AACrB,8EAA8E;AAE9E;;;;;;;;;GASG;AACH,MAAM,OAAO,kBAAkB;IACpB,IAAI,GAAG,gBAAgB,CAAA;IACvB,IAAI,GAAG,GAAY,CAAA;IAE5B,KAAK,CAAC,GAAG,CAAC,OAA4B;QACpC,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;QACxB,MAAM,MAAM,GAAG,OAAO,CAAC,YAAY,CAAA;QAEnC,+EAA+E;QAC/E,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,OAAO;gBACL,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,8DAA8D;gBACvE,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;gBAC/B,QAAQ,EAAE,EAAE;aACb,CAAA;QACH,CAAC;QAED,kGAAkG;QAClG,IAAI,MAAM,CAAC,cAAc,KAAK,IAAI,EAAE,CAAC;YACnC,MAAM,MAAM,GACV,MAAM,CAAC,KAAK,KAAK,0BAA0B;gBACzC,CAAC,CAAC,0BAA0B;gBAC5B,CAAC,CAAC,kBAAkB,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAA;YAClE,MAAM,QAAQ,GAA0B;gBACtC;oBACE,QAAQ,EAAE,gBAAgB;oBAC1B,QAAQ,EAAE,OAAO;oBACjB,OAAO,EAAE,MAAM;iBAChB;aACF,CAAA;YACD,OAAO;gBACL,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,cAAc,CAAC,QAAQ,CAAC;gBACjC,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;gBAC/B,QAAQ;aACT,CAAA;QACH,CAAC;QAED,+EAA+E;QAC/E,+EAA+E;QAC/E,6EAA6E;QAC7E,6EAA6E;QAC7E,IAAI,MAAM,CAAC,SAAS,KAAK,SAAS,IAAI,MAAM,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3E,MAAM,QAAQ,GAA0B;gBACtC;oBACE,QAAQ,EAAE,gBAAgB;oBAC1B,QAAQ,EAAE,OAAO;oBACjB,OAAO,EAAE,qBAAqB;iBAC/B;aACF,CAAA;YACD,OAAO;gBACL,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,cAAc,CAAC,QAAQ,CAAC;gBACjC,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;gBAC/B,QAAQ;aACT,CAAA;QACH,CAAC;QAED,4DAA4D;QAC5D,OAAO;YACL,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,wCAAwC;YACjD,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;YAC/B,QAAQ,EAAE,EAAE;SACb,CAAA;IACH,CAAC;CACF"}
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* RuntimeProbeCheck — Epic 55 / Phase 2.
|
|
3
|
+
*
|
|
4
|
+
* Tier A verification check that parses and executes the author-declared
|
|
5
|
+
* runtime probes in a story's `## Runtime Probes` section. Each probe's
|
|
6
|
+
* outcome becomes a structured VerificationFinding, leveraging the
|
|
7
|
+
* Phase 1 persistence surface to flow through retry prompts, the run
|
|
8
|
+
* manifest, and post-run analysis without string-parsing.
|
|
9
|
+
*
|
|
10
|
+
* Architecture / scoring contract:
|
|
11
|
+
*
|
|
12
|
+
* - No probes declared → pass (skip note finding array is empty)
|
|
13
|
+
* - Probes declared, YAML invalid → fail (one finding per parse error)
|
|
14
|
+
* - Probe `sandbox: twin` → warn (deferred until Phase 3)
|
|
15
|
+
* - Probe `sandbox: host`, exit 0 → pass; no finding emitted for that probe
|
|
16
|
+
* - Probe `sandbox: host`, exit ≠ 0 → fail (exit code + stdout/stderr tails)
|
|
17
|
+
* - Probe `sandbox: host`, timed out → fail (runtime-probe-timeout category)
|
|
18
|
+
* - No storyContent on context → warn (cannot verify without story text)
|
|
19
|
+
*
|
|
20
|
+
* Registered last in the canonical Tier A pipeline, after BuildCheck,
|
|
21
|
+
* because probes may depend on a successful build's output.
|
|
22
|
+
*/
|
|
23
|
+
import type { VerificationCheck, VerificationContext, VerificationResult } from '../types.js';
|
|
24
|
+
import { type ProbeResult, type RuntimeProbe } from '../probes/index.js';
|
|
25
|
+
/**
|
|
26
|
+
* Per-sandbox executors, injected via constructor so tests (and future
|
|
27
|
+
* twin integration in Phase 3) can swap implementations without touching
|
|
28
|
+
* the check's dispatch logic.
|
|
29
|
+
*/
|
|
30
|
+
export interface RuntimeProbeExecutors {
|
|
31
|
+
host: (probe: RuntimeProbe) => Promise<ProbeResult>;
|
|
32
|
+
/** Twin execution is deferred; the default returns undefined so the check
|
|
33
|
+
* emits a `probe-deferred` warn finding. Phase 3 replaces this. */
|
|
34
|
+
twin?: (probe: RuntimeProbe) => Promise<ProbeResult> | undefined;
|
|
35
|
+
}
|
|
36
|
+
export declare class RuntimeProbeCheck implements VerificationCheck {
|
|
37
|
+
readonly name = "runtime-probes";
|
|
38
|
+
readonly tier: "A";
|
|
39
|
+
private readonly _executors;
|
|
40
|
+
constructor(executors?: Partial<RuntimeProbeExecutors>);
|
|
41
|
+
run(context: VerificationContext): Promise<VerificationResult>;
|
|
42
|
+
}
|
|
43
|
+
//# sourceMappingURL=runtime-probe-check.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"runtime-probe-check.d.ts","sourceRoot":"","sources":["../../../src/verification/checks/runtime-probe-check.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAEH,OAAO,KAAK,EACV,iBAAiB,EACjB,mBAAmB,EAEnB,kBAAkB,EACnB,MAAM,aAAa,CAAA;AAEpB,OAAO,EAGL,KAAK,WAAW,EAChB,KAAK,YAAY,EAClB,MAAM,oBAAoB,CAAA;AAgB3B;;;;GAIG;AACH,MAAM,WAAW,qBAAqB;IACpC,IAAI,EAAE,CAAC,KAAK,EAAE,YAAY,KAAK,OAAO,CAAC,WAAW,CAAC,CAAA;IACnD;wEACoE;IACpE,IAAI,CAAC,EAAE,CAAC,KAAK,EAAE,YAAY,KAAK,OAAO,CAAC,WAAW,CAAC,GAAG,SAAS,CAAA;CACjE;AAWD,qBAAa,iBAAkB,YAAW,iBAAiB;IACzD,QAAQ,CAAC,IAAI,oBAAmB;IAChC,QAAQ,CAAC,IAAI,EAAG,GAAG,CAAS;IAE5B,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAuB;gBAEtC,SAAS,CAAC,EAAE,OAAO,CAAC,qBAAqB,CAAC;IAOhD,GAAG,CAAC,OAAO,EAAE,mBAAmB,GAAG,OAAO,CAAC,kBAAkB,CAAC;CAmHrE"}
|
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* RuntimeProbeCheck — Epic 55 / Phase 2.
|
|
3
|
+
*
|
|
4
|
+
* Tier A verification check that parses and executes the author-declared
|
|
5
|
+
* runtime probes in a story's `## Runtime Probes` section. Each probe's
|
|
6
|
+
* outcome becomes a structured VerificationFinding, leveraging the
|
|
7
|
+
* Phase 1 persistence surface to flow through retry prompts, the run
|
|
8
|
+
* manifest, and post-run analysis without string-parsing.
|
|
9
|
+
*
|
|
10
|
+
* Architecture / scoring contract:
|
|
11
|
+
*
|
|
12
|
+
* - No probes declared → pass (skip note finding array is empty)
|
|
13
|
+
* - Probes declared, YAML invalid → fail (one finding per parse error)
|
|
14
|
+
* - Probe `sandbox: twin` → warn (deferred until Phase 3)
|
|
15
|
+
* - Probe `sandbox: host`, exit 0 → pass; no finding emitted for that probe
|
|
16
|
+
* - Probe `sandbox: host`, exit ≠ 0 → fail (exit code + stdout/stderr tails)
|
|
17
|
+
* - Probe `sandbox: host`, timed out → fail (runtime-probe-timeout category)
|
|
18
|
+
* - No storyContent on context → warn (cannot verify without story text)
|
|
19
|
+
*
|
|
20
|
+
* Registered last in the canonical Tier A pipeline, after BuildCheck,
|
|
21
|
+
* because probes may depend on a successful build's output.
|
|
22
|
+
*/
|
|
23
|
+
import { renderFindings } from '../findings.js';
|
|
24
|
+
import { parseRuntimeProbes, executeProbeOnHost, } from '../probes/index.js';
|
|
25
|
+
// ---------------------------------------------------------------------------
|
|
26
|
+
// Finding categories
|
|
27
|
+
// ---------------------------------------------------------------------------
|
|
28
|
+
const CATEGORY_PARSE = 'runtime-probe-parse-error';
|
|
29
|
+
const CATEGORY_SKIP = 'runtime-probe-skip';
|
|
30
|
+
const CATEGORY_DEFERRED = 'runtime-probe-deferred';
|
|
31
|
+
const CATEGORY_FAIL = 'runtime-probe-fail';
|
|
32
|
+
const CATEGORY_TIMEOUT = 'runtime-probe-timeout';
|
|
33
|
+
const defaultExecutors = {
|
|
34
|
+
host: (probe) => executeProbeOnHost(probe, { cwd: process.cwd() }),
|
|
35
|
+
// twin intentionally omitted → RuntimeProbeCheck emits a warn finding
|
|
36
|
+
};
|
|
37
|
+
// ---------------------------------------------------------------------------
|
|
38
|
+
// RuntimeProbeCheck
|
|
39
|
+
// ---------------------------------------------------------------------------
|
|
40
|
+
export class RuntimeProbeCheck {
|
|
41
|
+
name = 'runtime-probes';
|
|
42
|
+
tier = 'A';
|
|
43
|
+
_executors;
|
|
44
|
+
constructor(executors) {
|
|
45
|
+
this._executors = {
|
|
46
|
+
...defaultExecutors,
|
|
47
|
+
...(executors ?? {}),
|
|
48
|
+
};
|
|
49
|
+
}
|
|
50
|
+
async run(context) {
|
|
51
|
+
const start = Date.now();
|
|
52
|
+
// Story content is required to locate the `## Runtime Probes` section.
|
|
53
|
+
// Absence is treated as a warn so reviewers know the check could not
|
|
54
|
+
// make a decision, distinct from "present but no probes declared".
|
|
55
|
+
if (context.storyContent === undefined) {
|
|
56
|
+
const findings = [
|
|
57
|
+
{
|
|
58
|
+
category: CATEGORY_SKIP,
|
|
59
|
+
severity: 'warn',
|
|
60
|
+
message: 'story content unavailable — skipping runtime probe check',
|
|
61
|
+
},
|
|
62
|
+
];
|
|
63
|
+
return {
|
|
64
|
+
status: 'warn',
|
|
65
|
+
details: renderFindings(findings),
|
|
66
|
+
duration_ms: Date.now() - start,
|
|
67
|
+
findings,
|
|
68
|
+
};
|
|
69
|
+
}
|
|
70
|
+
const parsed = parseRuntimeProbes(context.storyContent);
|
|
71
|
+
if (parsed.kind === 'absent') {
|
|
72
|
+
return {
|
|
73
|
+
status: 'pass',
|
|
74
|
+
details: 'runtime-probes: no ## Runtime Probes section declared — skipping',
|
|
75
|
+
duration_ms: Date.now() - start,
|
|
76
|
+
findings: [],
|
|
77
|
+
};
|
|
78
|
+
}
|
|
79
|
+
if (parsed.kind === 'invalid') {
|
|
80
|
+
const findings = [
|
|
81
|
+
{
|
|
82
|
+
category: CATEGORY_PARSE,
|
|
83
|
+
severity: 'error',
|
|
84
|
+
message: parsed.error,
|
|
85
|
+
},
|
|
86
|
+
];
|
|
87
|
+
return {
|
|
88
|
+
status: 'fail',
|
|
89
|
+
details: renderFindings(findings),
|
|
90
|
+
duration_ms: Date.now() - start,
|
|
91
|
+
findings,
|
|
92
|
+
};
|
|
93
|
+
}
|
|
94
|
+
// kind === 'parsed'. Empty list is a valid author-declared state
|
|
95
|
+
// (e.g. they declared the section but intentionally zero probes yet);
|
|
96
|
+
// treat identically to `absent` for the verdict.
|
|
97
|
+
if (parsed.probes.length === 0) {
|
|
98
|
+
return {
|
|
99
|
+
status: 'pass',
|
|
100
|
+
details: 'runtime-probes: 0 probes declared — skipping',
|
|
101
|
+
duration_ms: Date.now() - start,
|
|
102
|
+
findings: [],
|
|
103
|
+
};
|
|
104
|
+
}
|
|
105
|
+
const findings = [];
|
|
106
|
+
for (const probe of parsed.probes) {
|
|
107
|
+
if (probe.sandbox === 'twin') {
|
|
108
|
+
findings.push({
|
|
109
|
+
category: CATEGORY_DEFERRED,
|
|
110
|
+
severity: 'warn',
|
|
111
|
+
message: `probe "${probe.name}" uses sandbox=twin which is deferred until ` +
|
|
112
|
+
`Phase 3 (Digital Twin integration); skipping`,
|
|
113
|
+
});
|
|
114
|
+
continue;
|
|
115
|
+
}
|
|
116
|
+
// sandbox === 'host'
|
|
117
|
+
const result = await this._executors.host(probe);
|
|
118
|
+
if (result.outcome === 'pass') {
|
|
119
|
+
continue;
|
|
120
|
+
}
|
|
121
|
+
const category = result.outcome === 'timeout' ? CATEGORY_TIMEOUT : CATEGORY_FAIL;
|
|
122
|
+
const descriptor = probe.description ? ` (${probe.description})` : '';
|
|
123
|
+
const message = result.outcome === 'timeout'
|
|
124
|
+
? `probe "${probe.name}"${descriptor} timed out after ${result.durationMs}ms`
|
|
125
|
+
: `probe "${probe.name}"${descriptor} failed with exit ${result.exitCode ?? 'unknown'}`;
|
|
126
|
+
findings.push({
|
|
127
|
+
category,
|
|
128
|
+
severity: 'error',
|
|
129
|
+
message,
|
|
130
|
+
command: result.command,
|
|
131
|
+
...(result.exitCode !== undefined ? { exitCode: result.exitCode } : {}),
|
|
132
|
+
stdoutTail: result.stdoutTail,
|
|
133
|
+
stderrTail: result.stderrTail,
|
|
134
|
+
durationMs: result.durationMs,
|
|
135
|
+
});
|
|
136
|
+
}
|
|
137
|
+
const status = findings.some((f) => f.severity === 'error')
|
|
138
|
+
? 'fail'
|
|
139
|
+
: findings.some((f) => f.severity === 'warn')
|
|
140
|
+
? 'warn'
|
|
141
|
+
: 'pass';
|
|
142
|
+
return {
|
|
143
|
+
status,
|
|
144
|
+
details: findings.length > 0
|
|
145
|
+
? renderFindings(findings)
|
|
146
|
+
: `runtime-probes: ${parsed.probes.length} probe(s) passed`,
|
|
147
|
+
duration_ms: Date.now() - start,
|
|
148
|
+
findings,
|
|
149
|
+
};
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
//# sourceMappingURL=runtime-probe-check.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"runtime-probe-check.js","sourceRoot":"","sources":["../../../src/verification/checks/runtime-probe-check.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAQH,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAA;AAC/C,OAAO,EACL,kBAAkB,EAClB,kBAAkB,GAGnB,MAAM,oBAAoB,CAAA;AAE3B,8EAA8E;AAC9E,qBAAqB;AACrB,8EAA8E;AAE9E,MAAM,cAAc,GAAG,2BAA2B,CAAA;AAClD,MAAM,aAAa,GAAG,oBAAoB,CAAA;AAC1C,MAAM,iBAAiB,GAAG,wBAAwB,CAAA;AAClD,MAAM,aAAa,GAAG,oBAAoB,CAAA;AAC1C,MAAM,gBAAgB,GAAG,uBAAuB,CAAA;AAkBhD,MAAM,gBAAgB,GAA0B;IAC9C,IAAI,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,kBAAkB,CAAC,KAAK,EAAE,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,EAAE,CAAC;IAClE,sEAAsE;CACvE,CAAA;AAED,8EAA8E;AAC9E,oBAAoB;AACpB,8EAA8E;AAE9E,MAAM,OAAO,iBAAiB;IACnB,IAAI,GAAG,gBAAgB,CAAA;IACvB,IAAI,GAAG,GAAY,CAAA;IAEX,UAAU,CAAuB;IAElD,YAAY,SAA0C;QACpD,IAAI,CAAC,UAAU,GAAG;YAChB,GAAG,gBAAgB;YACnB,GAAG,CAAC,SAAS,IAAI,EAAE,CAAC;SACrB,CAAA;IACH,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,OAA4B;QACpC,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;QAExB,uEAAuE;QACvE,qEAAqE;QACrE,mEAAmE;QACnE,IAAI,OAAO,CAAC,YAAY,KAAK,SAAS,EAAE,CAAC;YACvC,MAAM,QAAQ,GAA0B;gBACtC;oBACE,QAAQ,EAAE,aAAa;oBACvB,QAAQ,EAAE,MAAM;oBAChB,OAAO,EAAE,0DAA0D;iBACpE;aACF,CAAA;YACD,OAAO;gBACL,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,cAAc,CAAC,QAAQ,CAAC;gBACjC,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;gBAC/B,QAAQ;aACT,CAAA;QACH,CAAC;QAED,MAAM,MAAM,GAAG,kBAAkB,CAAC,OAAO,CAAC,YAAY,CAAC,CAAA;QAEvD,IAAI,MAAM,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC7B,OAAO;gBACL,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,kEAAkE;gBAC3E,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;gBAC/B,QAAQ,EAAE,EAAE;aACb,CAAA;QACH,CAAC;QAED,IAAI,MAAM,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;YAC9B,MAAM,QAAQ,GAA0B;gBACtC;oBACE,QAAQ,EAAE,cAAc;oBACxB,QAAQ,EAAE,OAAO;oBACjB,OAAO,EAAE,MAAM,CAAC,KAAK;iBACtB;aACF,CAAA;YACD,OAAO;gBACL,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,cAAc,CAAC,QAAQ,CAAC;gBACjC,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;gBAC/B,QAAQ;aACT,CAAA;QACH,CAAC;QAED,iEAAiE;QACjE,sEAAsE;QACtE,iDAAiD;QACjD,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC/B,OAAO;gBACL,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,8CAA8C;gBACvD,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;gBAC/B,QAAQ,EAAE,EAAE;aACb,CAAA;QACH,CAAC;QAED,MAAM,QAAQ,GAA0B,EAAE,CAAA;QAC1C,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;YAClC,IAAI,KAAK,CAAC,OAAO,KAAK,MAAM,EAAE,CAAC;gBAC7B,QAAQ,CAAC,IAAI,CAAC;oBACZ,QAAQ,EAAE,iBAAiB;oBAC3B,QAAQ,EAAE,MAAM;oBAChB,OAAO,EACL,UAAU,KAAK,CAAC,IAAI,8CAA8C;wBAClE,8CAA8C;iBACjD,CAAC,CAAA;gBACF,SAAQ;YACV,CAAC;YAED,qBAAqB;YACrB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YAChD,IAAI,MAAM,CAAC,OAAO,KAAK,MAAM,EAAE,CAAC;gBAC9B,SAAQ;YACV,CAAC;YAED,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,aAAa,CAAA;YAChF,MAAM,UAAU,GAAG,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,WAAW,GAAG,CAAC,CAAC,CAAC,EAAE,CAAA;YACrE,MAAM,OAAO,GACX,MAAM,CAAC,OAAO,KAAK,SAAS;gBAC1B,CAAC,CAAC,UAAU,KAAK,CAAC,IAAI,IAAI,UAAU,oBAAoB,MAAM,CAAC,UAAU,IAAI;gBAC7E,CAAC,CAAC,UAAU,KAAK,CAAC,IAAI,IAAI,UAAU,qBAAqB,MAAM,CAAC,QAAQ,IAAI,SAAS,EAAE,CAAA;YAE3F,QAAQ,CAAC,IAAI,CAAC;gBACZ,QAAQ;gBACR,QAAQ,EAAE,OAAO;gBACjB,OAAO;gBACP,OAAO,EAAE,MAAM,CAAC,OAAO;gBACvB,GAAG,CAAC,MAAM,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACvE,UAAU,EAAE,MAAM,CAAC,UAAU;gBAC7B,UAAU,EAAE,MAAM,CAAC,UAAU;gBAC7B,UAAU,EAAE,MAAM,CAAC,UAAU;aAC9B,CAAC,CAAA;QACJ,CAAC;QAED,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,OAAO,CAAC;YACzD,CAAC,CAAC,MAAM;YACR,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,MAAM,CAAC;gBAC3C,CAAC,CAAC,MAAM;gBACR,CAAC,CAAC,MAAM,CAAA;QAEZ,OAAO;YACL,MAAM;YACN,OAAO,EACL,QAAQ,CAAC,MAAM,GAAG,CAAC;gBACjB,CAAC,CAAC,cAAc,CAAC,QAAQ,CAAC;gBAC1B,CAAC,CAAC,mBAAmB,MAAM,CAAC,MAAM,CAAC,MAAM,kBAAkB;YAC/D,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;YAC/B,QAAQ;SACT,CAAA;IACH,CAAC;CACF"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"trivial-output-check.d.ts","sourceRoot":"","sources":["../../../src/verification/checks/trivial-output-check.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,KAAK,EACV,iBAAiB,EACjB,mBAAmB,
|
|
1
|
+
{"version":3,"file":"trivial-output-check.d.ts","sourceRoot":"","sources":["../../../src/verification/checks/trivial-output-check.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,KAAK,EACV,iBAAiB,EACjB,mBAAmB,EAEnB,kBAAkB,EACnB,MAAM,aAAa,CAAA;AAOpB;;;GAGG;AACH,eAAO,MAAM,gCAAgC,MAAM,CAAA;AAMnD;;;;GAIG;AACH,MAAM,WAAW,wBAAwB;IACvC,sBAAsB,CAAC,EAAE,MAAM,CAAA;CAChC;AAMD;;;;;;;;;;;GAWG;AACH,qBAAa,kBAAmB,YAAW,iBAAiB;IAC1D,QAAQ,CAAC,IAAI,oBAAmB;IAChC,QAAQ,CAAC,IAAI,EAAG,GAAG,CAAS;IAE5B,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAQ;gBAEtB,MAAM,CAAC,EAAE,wBAAwB;IAIvC,GAAG,CAAC,OAAO,EAAE,mBAAmB,GAAG,OAAO,CAAC,kBAAkB,CAAC;CAiDrE"}
|
|
@@ -11,6 +11,7 @@
|
|
|
11
11
|
* - No shell invocations — pure in-process computation.
|
|
12
12
|
* - Runs in Tier A: before BuildCheck, after PhantomReviewCheck.
|
|
13
13
|
*/
|
|
14
|
+
import { renderFindings } from '../findings.js';
|
|
14
15
|
// ---------------------------------------------------------------------------
|
|
15
16
|
// Default threshold
|
|
16
17
|
// ---------------------------------------------------------------------------
|
|
@@ -45,20 +46,36 @@ export class TrivialOutputCheck {
|
|
|
45
46
|
const start = Date.now();
|
|
46
47
|
// AC5: missing token data → warn, not fail
|
|
47
48
|
if (context.outputTokenCount === undefined) {
|
|
49
|
+
const findings = [
|
|
50
|
+
{
|
|
51
|
+
category: 'trivial-output',
|
|
52
|
+
severity: 'warn',
|
|
53
|
+
message: 'output token count unavailable — skipping check',
|
|
54
|
+
},
|
|
55
|
+
];
|
|
48
56
|
return {
|
|
49
57
|
status: 'warn',
|
|
50
|
-
details:
|
|
58
|
+
details: renderFindings(findings),
|
|
51
59
|
duration_ms: Date.now() - start,
|
|
60
|
+
findings,
|
|
52
61
|
};
|
|
53
62
|
}
|
|
54
63
|
const count = context.outputTokenCount;
|
|
55
64
|
// AC1 + AC2: below threshold → fail with actionable message
|
|
56
65
|
if (count < this.threshold) {
|
|
66
|
+
const findings = [
|
|
67
|
+
{
|
|
68
|
+
category: 'trivial-output',
|
|
69
|
+
severity: 'error',
|
|
70
|
+
message: `output token count ${count} is below threshold ${this.threshold}` +
|
|
71
|
+
` — Re-run with increased maxTurns`,
|
|
72
|
+
},
|
|
73
|
+
];
|
|
57
74
|
return {
|
|
58
75
|
status: 'fail',
|
|
59
|
-
details:
|
|
60
|
-
` — Re-run with increased maxTurns`,
|
|
76
|
+
details: renderFindings(findings),
|
|
61
77
|
duration_ms: Date.now() - start,
|
|
78
|
+
findings,
|
|
62
79
|
};
|
|
63
80
|
}
|
|
64
81
|
// AC3: at or above threshold → pass
|
|
@@ -66,6 +83,7 @@ export class TrivialOutputCheck {
|
|
|
66
83
|
status: 'pass',
|
|
67
84
|
details: `output token count ${count} meets threshold ${this.threshold}`,
|
|
68
85
|
duration_ms: Date.now() - start,
|
|
86
|
+
findings: [],
|
|
69
87
|
};
|
|
70
88
|
}
|
|
71
89
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"trivial-output-check.js","sourceRoot":"","sources":["../../../src/verification/checks/trivial-output-check.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAQH,8EAA8E;AAC9E,oBAAoB;AACpB,8EAA8E;AAE9E;;;GAGG;AACH,MAAM,CAAC,MAAM,gCAAgC,GAAG,GAAG,CAAA;AAenD,8EAA8E;AAC9E,qBAAqB;AACrB,8EAA8E;AAE9E;;;;;;;;;;;GAWG;AACH,MAAM,OAAO,kBAAkB;IACpB,IAAI,GAAG,gBAAgB,CAAA;IACvB,IAAI,GAAG,GAAY,CAAA;IAEX,SAAS,CAAQ;IAElC,YAAY,MAAiC;QAC3C,IAAI,CAAC,SAAS,GAAG,MAAM,EAAE,sBAAsB,IAAI,gCAAgC,CAAA;IACrF,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,OAA4B;QACpC,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;QAExB,2CAA2C;QAC3C,IAAI,OAAO,CAAC,gBAAgB,KAAK,SAAS,EAAE,CAAC;YAC3C,OAAO;gBACL,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"trivial-output-check.js","sourceRoot":"","sources":["../../../src/verification/checks/trivial-output-check.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAQH,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAA;AAE/C,8EAA8E;AAC9E,oBAAoB;AACpB,8EAA8E;AAE9E;;;GAGG;AACH,MAAM,CAAC,MAAM,gCAAgC,GAAG,GAAG,CAAA;AAenD,8EAA8E;AAC9E,qBAAqB;AACrB,8EAA8E;AAE9E;;;;;;;;;;;GAWG;AACH,MAAM,OAAO,kBAAkB;IACpB,IAAI,GAAG,gBAAgB,CAAA;IACvB,IAAI,GAAG,GAAY,CAAA;IAEX,SAAS,CAAQ;IAElC,YAAY,MAAiC;QAC3C,IAAI,CAAC,SAAS,GAAG,MAAM,EAAE,sBAAsB,IAAI,gCAAgC,CAAA;IACrF,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,OAA4B;QACpC,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;QAExB,2CAA2C;QAC3C,IAAI,OAAO,CAAC,gBAAgB,KAAK,SAAS,EAAE,CAAC;YAC3C,MAAM,QAAQ,GAA0B;gBACtC;oBACE,QAAQ,EAAE,gBAAgB;oBAC1B,QAAQ,EAAE,MAAM;oBAChB,OAAO,EAAE,iDAAiD;iBAC3D;aACF,CAAA;YACD,OAAO;gBACL,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,cAAc,CAAC,QAAQ,CAAC;gBACjC,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;gBAC/B,QAAQ;aACT,CAAA;QACH,CAAC;QAED,MAAM,KAAK,GAAG,OAAO,CAAC,gBAAgB,CAAA;QAEtC,4DAA4D;QAC5D,IAAI,KAAK,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;YAC3B,MAAM,QAAQ,GAA0B;gBACtC;oBACE,QAAQ,EAAE,gBAAgB;oBAC1B,QAAQ,EAAE,OAAO;oBACjB,OAAO,EACL,sBAAsB,KAAK,uBAAuB,IAAI,CAAC,SAAS,EAAE;wBAClE,mCAAmC;iBACtC;aACF,CAAA;YACD,OAAO;gBACL,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,cAAc,CAAC,QAAQ,CAAC;gBACjC,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;gBAC/B,QAAQ;aACT,CAAA;QACH,CAAC;QAED,oCAAoC;QACpC,OAAO;YACL,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,sBAAsB,KAAK,oBAAoB,IAAI,CAAC,SAAS,EAAE;YACxE,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;YAC/B,QAAQ,EAAE,EAAE;SACb,CAAA;IACH,CAAC;CACF"}
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* VerificationFinding — structured per-issue payload emitted by verification checks.
|
|
3
|
+
*
|
|
4
|
+
* Replaces the ad-hoc "stuff everything into VerificationResult.details" pattern
|
|
5
|
+
* that preceded it: every downstream consumer (retry prompts, run manifest,
|
|
6
|
+
* post-run analysis) used to string-parse a free-form blob that the emitting
|
|
7
|
+
* check never promised a schema for. With findings, each issue is an
|
|
8
|
+
* addressable record the pipeline can act on individually.
|
|
9
|
+
*
|
|
10
|
+
* The {command, exitCode, stdoutTail, stderrTail} optional fields are reserved
|
|
11
|
+
* primarily for Phase 2 runtime probes — they cost nothing on the current four
|
|
12
|
+
* Tier A checks (which leave them undefined) but let probe output flow through
|
|
13
|
+
* the same shape without a second refactor.
|
|
14
|
+
*/
|
|
15
|
+
/**
|
|
16
|
+
* Severity of a single verification finding.
|
|
17
|
+
*
|
|
18
|
+
* - `error` → verification check returns `fail` (the aggregate status of the
|
|
19
|
+
* containing VerificationResult is `fail`).
|
|
20
|
+
* - `warn` → verification check returns `warn`.
|
|
21
|
+
* - `info` → a finding captured for diagnostic/telemetry purposes that does
|
|
22
|
+
* not itself raise the containing VerificationResult's status.
|
|
23
|
+
*/
|
|
24
|
+
export type VerificationFindingSeverity = 'error' | 'warn' | 'info';
|
|
25
|
+
/**
|
|
26
|
+
* One structured issue emitted by a verification check.
|
|
27
|
+
*
|
|
28
|
+
* `category` is the stable machine-readable identifier consumers filter/group
|
|
29
|
+
* on (e.g. `'build-error'`, `'ac-missing-evidence'`, `'phantom-review'`,
|
|
30
|
+
* `'trivial-output'`). New categories should be introduced alongside the
|
|
31
|
+
* check that emits them.
|
|
32
|
+
*/
|
|
33
|
+
export interface VerificationFinding {
|
|
34
|
+
/** Stable machine-readable category (e.g. `'build-error'`). */
|
|
35
|
+
category: string;
|
|
36
|
+
/** Severity classification for aggregate-status computation. */
|
|
37
|
+
severity: VerificationFindingSeverity;
|
|
38
|
+
/** Single-line human-readable summary. */
|
|
39
|
+
message: string;
|
|
40
|
+
/** The command that produced this finding, if any. Reserved primarily for Phase 2 runtime probes. */
|
|
41
|
+
command?: string;
|
|
42
|
+
/** Exit status of `command`, if applicable. */
|
|
43
|
+
exitCode?: number;
|
|
44
|
+
/** Last ≤ 4 KiB of stdout from `command`, if captured. */
|
|
45
|
+
stdoutTail?: string;
|
|
46
|
+
/** Last ≤ 4 KiB of stderr from `command`, if captured. */
|
|
47
|
+
stderrTail?: string;
|
|
48
|
+
/** Wall-clock milliseconds the producing action took. */
|
|
49
|
+
durationMs?: number;
|
|
50
|
+
}
|
|
51
|
+
/**
|
|
52
|
+
* Render a list of findings into the multi-line human-readable string that
|
|
53
|
+
* populates VerificationResult.details. One line per finding:
|
|
54
|
+
*
|
|
55
|
+
* `${PREFIX} [${category}] ${message}`
|
|
56
|
+
*
|
|
57
|
+
* Checks that migrate to the findings-first pattern call this helper to derive
|
|
58
|
+
* `details` from the findings they emit, guaranteeing the two stay in sync.
|
|
59
|
+
*/
|
|
60
|
+
export declare function renderFindings(findings: VerificationFinding[]): string;
|
|
61
|
+
//# sourceMappingURL=findings.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"findings.d.ts","sourceRoot":"","sources":["../../src/verification/findings.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAMH;;;;;;;;GAQG;AACH,MAAM,MAAM,2BAA2B,GAAG,OAAO,GAAG,MAAM,GAAG,MAAM,CAAA;AAEnE;;;;;;;GAOG;AACH,MAAM,WAAW,mBAAmB;IAClC,+DAA+D;IAC/D,QAAQ,EAAE,MAAM,CAAA;IAChB,gEAAgE;IAChE,QAAQ,EAAE,2BAA2B,CAAA;IACrC,0CAA0C;IAC1C,OAAO,EAAE,MAAM,CAAA;IACf,qGAAqG;IACrG,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,+CAA+C;IAC/C,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,0DAA0D;IAC1D,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,0DAA0D;IAC1D,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,yDAAyD;IACzD,UAAU,CAAC,EAAE,MAAM,CAAA;CACpB;AAYD;;;;;;;;GAQG;AACH,wBAAgB,cAAc,CAAC,QAAQ,EAAE,mBAAmB,EAAE,GAAG,MAAM,CAKtE"}
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* VerificationFinding — structured per-issue payload emitted by verification checks.
|
|
3
|
+
*
|
|
4
|
+
* Replaces the ad-hoc "stuff everything into VerificationResult.details" pattern
|
|
5
|
+
* that preceded it: every downstream consumer (retry prompts, run manifest,
|
|
6
|
+
* post-run analysis) used to string-parse a free-form blob that the emitting
|
|
7
|
+
* check never promised a schema for. With findings, each issue is an
|
|
8
|
+
* addressable record the pipeline can act on individually.
|
|
9
|
+
*
|
|
10
|
+
* The {command, exitCode, stdoutTail, stderrTail} optional fields are reserved
|
|
11
|
+
* primarily for Phase 2 runtime probes — they cost nothing on the current four
|
|
12
|
+
* Tier A checks (which leave them undefined) but let probe output flow through
|
|
13
|
+
* the same shape without a second refactor.
|
|
14
|
+
*/
|
|
15
|
+
// ---------------------------------------------------------------------------
|
|
16
|
+
// Rendering
|
|
17
|
+
// ---------------------------------------------------------------------------
|
|
18
|
+
const SEVERITY_PREFIX = {
|
|
19
|
+
error: 'ERROR',
|
|
20
|
+
warn: 'WARN',
|
|
21
|
+
info: 'INFO',
|
|
22
|
+
};
|
|
23
|
+
/**
|
|
24
|
+
* Render a list of findings into the multi-line human-readable string that
|
|
25
|
+
* populates VerificationResult.details. One line per finding:
|
|
26
|
+
*
|
|
27
|
+
* `${PREFIX} [${category}] ${message}`
|
|
28
|
+
*
|
|
29
|
+
* Checks that migrate to the findings-first pattern call this helper to derive
|
|
30
|
+
* `details` from the findings they emit, guaranteeing the two stay in sync.
|
|
31
|
+
*/
|
|
32
|
+
export function renderFindings(findings) {
|
|
33
|
+
if (findings.length === 0)
|
|
34
|
+
return '';
|
|
35
|
+
return findings
|
|
36
|
+
.map((f) => `${SEVERITY_PREFIX[f.severity]} [${f.category}] ${f.message}`)
|
|
37
|
+
.join('\n');
|
|
38
|
+
}
|
|
39
|
+
//# sourceMappingURL=findings.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"findings.js","sourceRoot":"","sources":["../../src/verification/findings.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AA4CH,8EAA8E;AAC9E,YAAY;AACZ,8EAA8E;AAE9E,MAAM,eAAe,GAAgD;IACnE,KAAK,EAAE,OAAO;IACd,IAAI,EAAE,MAAM;IACZ,IAAI,EAAE,MAAM;CACb,CAAA;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,cAAc,CAAC,QAA+B;IAC5D,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAA;IACpC,OAAO,QAAQ;SACZ,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,eAAe,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,QAAQ,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC;SACzE,IAAI,CAAC,IAAI,CAAC,CAAA;AACf,CAAC"}
|
|
@@ -5,7 +5,12 @@
|
|
|
5
5
|
* and concrete check implementations.
|
|
6
6
|
*/
|
|
7
7
|
export { VerificationPipeline, createDefaultVerificationPipeline } from './verification-pipeline.js';
|
|
8
|
-
export type { VerificationCheck, VerificationContext, VerificationResult, VerificationCheckResult, VerificationSummary, ReviewSignals, DevStorySignals, } from './types.js';
|
|
8
|
+
export type { VerificationCheck, VerificationContext, VerificationResult, VerificationCheckResult, VerificationSummary, ReviewSignals, DevStorySignals, VerificationFinding, VerificationFindingSeverity, } from './types.js';
|
|
9
|
+
export { renderFindings } from './findings.js';
|
|
10
|
+
export { RuntimeProbeCheck } from './checks/runtime-probe-check.js';
|
|
11
|
+
export type { RuntimeProbeExecutors } from './checks/runtime-probe-check.js';
|
|
12
|
+
export { DEFAULT_PROBE_TIMEOUT_MS, PROBE_TAIL_BYTES, RuntimeProbeListSchema, RuntimeProbeSandboxSchema, RuntimeProbeSchema, parseRuntimeProbes, executeProbeOnHost, } from './probes/index.js';
|
|
13
|
+
export type { HostExecuteOptions, ProbeResult, RuntimeProbe, RuntimeProbeParseResult, RuntimeProbeSandbox, } from './probes/index.js';
|
|
9
14
|
export { PhantomReviewCheck } from './checks/index.js';
|
|
10
15
|
export { TrivialOutputCheck, DEFAULT_TRIVIAL_OUTPUT_THRESHOLD } from './checks/index.js';
|
|
11
16
|
export { AcceptanceCriteriaEvidenceCheck, extractAcceptanceCriteriaIds } from './checks/index.js';
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/verification/index.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,oBAAoB,EAAE,iCAAiC,EAAE,MAAM,4BAA4B,CAAA;AACpG,YAAY,EACV,iBAAiB,EACjB,mBAAmB,EACnB,kBAAkB,EAClB,uBAAuB,EACvB,mBAAmB,EACnB,aAAa,EACb,eAAe,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/verification/index.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,oBAAoB,EAAE,iCAAiC,EAAE,MAAM,4BAA4B,CAAA;AACpG,YAAY,EACV,iBAAiB,EACjB,mBAAmB,EACnB,kBAAkB,EAClB,uBAAuB,EACvB,mBAAmB,EACnB,aAAa,EACb,eAAe,EACf,mBAAmB,EACnB,2BAA2B,GAC5B,MAAM,YAAY,CAAA;AACnB,OAAO,EAAE,cAAc,EAAE,MAAM,eAAe,CAAA;AAG9C,OAAO,EAAE,iBAAiB,EAAE,MAAM,iCAAiC,CAAA;AACnE,YAAY,EAAE,qBAAqB,EAAE,MAAM,iCAAiC,CAAA;AAC5E,OAAO,EACL,wBAAwB,EACxB,gBAAgB,EAChB,sBAAsB,EACtB,yBAAyB,EACzB,kBAAkB,EAClB,kBAAkB,EAClB,kBAAkB,GACnB,MAAM,mBAAmB,CAAA;AAC1B,YAAY,EACV,kBAAkB,EAClB,WAAW,EACX,YAAY,EACZ,uBAAuB,EACvB,mBAAmB,GACpB,MAAM,mBAAmB,CAAA;AAE1B,OAAO,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAA;AACtD,OAAO,EAAE,kBAAkB,EAAE,gCAAgC,EAAE,MAAM,mBAAmB,CAAA;AACxF,OAAO,EAAE,+BAA+B,EAAE,4BAA4B,EAAE,MAAM,mBAAmB,CAAA;AACjG,OAAO,EAAE,UAAU,EAAE,sBAAsB,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAA"}
|
|
@@ -5,6 +5,10 @@
|
|
|
5
5
|
* and concrete check implementations.
|
|
6
6
|
*/
|
|
7
7
|
export { VerificationPipeline, createDefaultVerificationPipeline } from './verification-pipeline.js';
|
|
8
|
+
export { renderFindings } from './findings.js';
|
|
9
|
+
// Epic 55 Phase 2 — Runtime probe support.
|
|
10
|
+
export { RuntimeProbeCheck } from './checks/runtime-probe-check.js';
|
|
11
|
+
export { DEFAULT_PROBE_TIMEOUT_MS, PROBE_TAIL_BYTES, RuntimeProbeListSchema, RuntimeProbeSandboxSchema, RuntimeProbeSchema, parseRuntimeProbes, executeProbeOnHost, } from './probes/index.js';
|
|
8
12
|
// Concrete check implementations (story 51-2+)
|
|
9
13
|
export { PhantomReviewCheck } from './checks/index.js';
|
|
10
14
|
export { TrivialOutputCheck, DEFAULT_TRIVIAL_OUTPUT_THRESHOLD } from './checks/index.js';
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/verification/index.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,oBAAoB,EAAE,iCAAiC,EAAE,MAAM,4BAA4B,CAAA;
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/verification/index.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,oBAAoB,EAAE,iCAAiC,EAAE,MAAM,4BAA4B,CAAA;AAYpG,OAAO,EAAE,cAAc,EAAE,MAAM,eAAe,CAAA;AAE9C,2CAA2C;AAC3C,OAAO,EAAE,iBAAiB,EAAE,MAAM,iCAAiC,CAAA;AAEnE,OAAO,EACL,wBAAwB,EACxB,gBAAgB,EAChB,sBAAsB,EACtB,yBAAyB,EACzB,kBAAkB,EAClB,kBAAkB,EAClB,kBAAkB,GACnB,MAAM,mBAAmB,CAAA;AAQ1B,+CAA+C;AAC/C,OAAO,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAA;AACtD,OAAO,EAAE,kBAAkB,EAAE,gCAAgC,EAAE,MAAM,mBAAmB,CAAA;AACxF,OAAO,EAAE,+BAA+B,EAAE,4BAA4B,EAAE,MAAM,mBAAmB,CAAA;AACjG,OAAO,EAAE,UAAU,EAAE,sBAAsB,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAA"}
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Host-sandbox probe executor — Epic 55 / Phase 2.
|
|
3
|
+
*
|
|
4
|
+
* Runs one runtime probe directly on the operator machine via a detached
|
|
5
|
+
* shell process, with stream-separated capture and a hard timeout that
|
|
6
|
+
* terminates the entire process group. Same safety posture as BuildCheck —
|
|
7
|
+
* specifically the detached-process-group + SIGKILL-on-timeout pattern —
|
|
8
|
+
* so probe execution cannot orphan long-running descendants.
|
|
9
|
+
*
|
|
10
|
+
* Twin-sandbox execution is **deferred to Phase 3** (Digital Twin
|
|
11
|
+
* integration). RuntimeProbeCheck handles the `sandbox: twin` case itself
|
|
12
|
+
* without routing through this module.
|
|
13
|
+
*/
|
|
14
|
+
import { type ProbeResult, type RuntimeProbe } from './types.js';
|
|
15
|
+
/** Options for host execution. Injected primarily to make tests deterministic. */
|
|
16
|
+
export interface HostExecuteOptions {
|
|
17
|
+
/** Working directory for the spawned shell. Defaults to process.cwd(). */
|
|
18
|
+
cwd?: string;
|
|
19
|
+
/** Environment override. Defaults to the parent process's env. */
|
|
20
|
+
env?: NodeJS.ProcessEnv;
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* Execute one probe on the host and return a structured ProbeResult.
|
|
24
|
+
*
|
|
25
|
+
* Behavior notes:
|
|
26
|
+
* - The shell used is `/bin/sh -c '<probe.command>'` inside a detached
|
|
27
|
+
* process group (so the entire tree is killed on timeout).
|
|
28
|
+
* - stdout and stderr are captured independently; each is returned
|
|
29
|
+
* tailed to PROBE_TAIL_BYTES (≤ 4 KiB) so published tarballs of the
|
|
30
|
+
* run manifest stay small.
|
|
31
|
+
* - Timeout defaults to `probe.timeout_ms ?? DEFAULT_PROBE_TIMEOUT_MS`
|
|
32
|
+
* (60 s). When the timeout fires, the process group is SIGKILL'd and
|
|
33
|
+
* the returned result has `outcome: 'timeout'`, `exitCode` undefined.
|
|
34
|
+
* - Never throws. Spawn errors (e.g. exec format error) are returned as
|
|
35
|
+
* `outcome: 'fail'` with exitCode -1 and the error message captured on
|
|
36
|
+
* stderrTail, so the caller can emit a deterministic finding.
|
|
37
|
+
*/
|
|
38
|
+
export declare function executeProbeOnHost(probe: RuntimeProbe, options?: HostExecuteOptions): Promise<ProbeResult>;
|
|
39
|
+
//# sourceMappingURL=executor.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"executor.d.ts","sourceRoot":"","sources":["../../../src/verification/probes/executor.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAGH,OAAO,EAGL,KAAK,WAAW,EAChB,KAAK,YAAY,EAClB,MAAM,YAAY,CAAA;AAenB,kFAAkF;AAClF,MAAM,WAAW,kBAAkB;IACjC,0EAA0E;IAC1E,GAAG,CAAC,EAAE,MAAM,CAAA;IACZ,kEAAkE;IAClE,GAAG,CAAC,EAAE,MAAM,CAAC,UAAU,CAAA;CACxB;AAED;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,kBAAkB,CAChC,KAAK,EAAE,YAAY,EACnB,OAAO,GAAE,kBAAuB,GAC/B,OAAO,CAAC,WAAW,CAAC,CA0EtB"}
|