@phamvuhoang/otto-core 0.30.0 → 0.32.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/context-compressor.d.ts +4 -2
- package/dist/context-compressor.d.ts.map +1 -1
- package/dist/context-compressor.js +13 -1
- package/dist/context-compressor.js.map +1 -1
- package/dist/extension-profiles.js +2 -2
- package/dist/extension-profiles.js.map +1 -1
- package/dist/git.d.ts +7 -0
- package/dist/git.d.ts.map +1 -1
- package/dist/git.js +20 -0
- package/dist/git.js.map +1 -1
- package/dist/headroom-adapter.d.ts +138 -27
- package/dist/headroom-adapter.d.ts.map +1 -1
- package/dist/headroom-adapter.js +335 -49
- package/dist/headroom-adapter.js.map +1 -1
- package/dist/index.d.ts +3 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -1
- package/dist/index.js.map +1 -1
- package/dist/inspect.d.ts.map +1 -1
- package/dist/inspect.js +20 -0
- package/dist/inspect.js.map +1 -1
- package/dist/loop.d.ts.map +1 -1
- package/dist/loop.js +81 -7
- package/dist/loop.js.map +1 -1
- package/dist/report-finalize.d.ts.map +1 -1
- package/dist/report-finalize.js +49 -1
- package/dist/report-finalize.js.map +1 -1
- package/dist/run-report.d.ts +7 -0
- package/dist/run-report.d.ts.map +1 -1
- package/dist/run-report.js.map +1 -1
- package/dist/verification-evidence.d.ts +41 -0
- package/dist/verification-evidence.d.ts.map +1 -0
- package/dist/verification-evidence.js +184 -0
- package/dist/verification-evidence.js.map +1 -0
- package/dist/verification-matrix.d.ts +159 -0
- package/dist/verification-matrix.d.ts.map +1 -0
- package/dist/verification-matrix.js +337 -0
- package/dist/verification-matrix.js.map +1 -0
- package/package.json +1 -1
- package/templates/verify.md +41 -1
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Verification evidence validation + relocation (issue #181 P24, re-review).
|
|
3
|
+
*
|
|
4
|
+
* The pure {@link isValidArtifactReference} checks an artifact reference's *shape*
|
|
5
|
+
* only. This impure layer, run at finalize against the real workspace + git,
|
|
6
|
+
* answers whether the cited proof actually **exists** — a source `file:line` whose
|
|
7
|
+
* file is present and whose line is in bounds, or a commit SHA present in git —
|
|
8
|
+
* and **relocates** scratch file artifacts (screenshots, transcripts the verify
|
|
9
|
+
* stage wrote under `.otto-tmp/`) into the durable run bundle with bundle-relative
|
|
10
|
+
* paths. Both close re-review gaps: agent-cited proof that doesn't exist no longer
|
|
11
|
+
* earns coverage, and a malicious/absolute/`..` path can neither be counted nor
|
|
12
|
+
* copied out of the workspace.
|
|
13
|
+
*
|
|
14
|
+
* INERT outside a `--verify` finalize; pure-ish (fs reads + copies into the
|
|
15
|
+
* bundle, git injected for testability).
|
|
16
|
+
*/
|
|
17
|
+
import { copyFileSync, lstatSync, mkdirSync, readFileSync, realpathSync, statSync, } from "node:fs";
|
|
18
|
+
import { basename, isAbsolute, join, sep } from "node:path";
|
|
19
|
+
import { runReportDir } from "./run-report.js";
|
|
20
|
+
import { BUNDLE_ARTIFACT_PREFIX, isValidArtifactReference, } from "./verification-matrix.js";
|
|
21
|
+
const SHA_RE = /^[0-9a-f]{7,40}$/i;
|
|
22
|
+
const FILE_LINE_RE = /^(.+?):(\d+)(?:-(\d+))?$/;
|
|
23
|
+
/**
|
|
24
|
+
* Real on-disk path of `file` iff it resolves to a regular file **inside** the
|
|
25
|
+
* workspace — rejecting `..` traversal, absolute escapes, and symlinks that point
|
|
26
|
+
* outside (realpath is compared against the workspace's realpath). Null otherwise.
|
|
27
|
+
*/
|
|
28
|
+
function safeWorkspaceFile(file, workspaceDir) {
|
|
29
|
+
if (/(?:^|[\\/])\.\.(?:[\\/]|$)/.test(file))
|
|
30
|
+
return null; // traversal
|
|
31
|
+
const abs = isAbsolute(file) ? file : join(workspaceDir, file);
|
|
32
|
+
let realWs;
|
|
33
|
+
let real;
|
|
34
|
+
try {
|
|
35
|
+
realWs = realpathSync(workspaceDir);
|
|
36
|
+
}
|
|
37
|
+
catch {
|
|
38
|
+
return null;
|
|
39
|
+
}
|
|
40
|
+
try {
|
|
41
|
+
real = realpathSync(abs);
|
|
42
|
+
}
|
|
43
|
+
catch {
|
|
44
|
+
return null; // does not exist
|
|
45
|
+
}
|
|
46
|
+
if (real !== realWs && !real.startsWith(realWs + sep))
|
|
47
|
+
return null; // escaped
|
|
48
|
+
try {
|
|
49
|
+
if (!statSync(real).isFile())
|
|
50
|
+
return null;
|
|
51
|
+
}
|
|
52
|
+
catch {
|
|
53
|
+
return null;
|
|
54
|
+
}
|
|
55
|
+
return real;
|
|
56
|
+
}
|
|
57
|
+
/**
|
|
58
|
+
* Whether an artifact reference points to evidence that actually exists: a commit
|
|
59
|
+
* SHA present in git, or a file inside the workspace whose cited line range is in
|
|
60
|
+
* bounds. Syntactically-invalid refs / URLs / traversal are rejected first.
|
|
61
|
+
*/
|
|
62
|
+
export function artifactReferenceExists(ref, workspaceDir, deps) {
|
|
63
|
+
const r = ref.trim();
|
|
64
|
+
if (!isValidArtifactReference(r))
|
|
65
|
+
return false;
|
|
66
|
+
if (SHA_RE.test(r))
|
|
67
|
+
return deps.commitExists(r);
|
|
68
|
+
const m = FILE_LINE_RE.exec(r);
|
|
69
|
+
const file = m ? m[1] : r;
|
|
70
|
+
const real = safeWorkspaceFile(file, workspaceDir);
|
|
71
|
+
if (!real)
|
|
72
|
+
return false;
|
|
73
|
+
if (m) {
|
|
74
|
+
const start = Number(m[2]);
|
|
75
|
+
const end = m[3] ? Number(m[3]) : start;
|
|
76
|
+
if (start < 1 || end < start)
|
|
77
|
+
return false;
|
|
78
|
+
let lineCount;
|
|
79
|
+
try {
|
|
80
|
+
lineCount = readFileSync(real, "utf8").split("\n").length;
|
|
81
|
+
}
|
|
82
|
+
catch {
|
|
83
|
+
return false;
|
|
84
|
+
}
|
|
85
|
+
if (end > lineCount)
|
|
86
|
+
return false;
|
|
87
|
+
}
|
|
88
|
+
return true;
|
|
89
|
+
}
|
|
90
|
+
/**
|
|
91
|
+
* Validate every entry's artifact against the filesystem/git (setting
|
|
92
|
+
* `artifactExists`), and relocate any **scratch file** artifact (a plain path,
|
|
93
|
+
* inside the workspace, that is not a `file:line`/SHA reference) into the run
|
|
94
|
+
* bundle at `.otto/runs/<run-id>/verification/` with a bundle-relative path, so
|
|
95
|
+
* the persisted report's links resolve and the files are durable. Source
|
|
96
|
+
* `file:line` references and commit SHAs are durable already and left untouched;
|
|
97
|
+
* a path that escapes the workspace is neither counted nor copied.
|
|
98
|
+
*/
|
|
99
|
+
export function validateVerificationEvidence(entries, opts) {
|
|
100
|
+
const { workspaceDir, runId, commitExists } = opts;
|
|
101
|
+
let counter = 0;
|
|
102
|
+
let scratchRoot = null;
|
|
103
|
+
try {
|
|
104
|
+
scratchRoot = realpathSync(join(workspaceDir, ".otto-tmp"));
|
|
105
|
+
}
|
|
106
|
+
catch {
|
|
107
|
+
scratchRoot = null; // no scratch dir ⇒ nothing to relocate
|
|
108
|
+
}
|
|
109
|
+
// The physical run-bundle verification dir. Ensure the bundle dir exists, then
|
|
110
|
+
// realpath it (stripping any symlinks ABOVE it) and require it to stay inside
|
|
111
|
+
// the workspace; the copy destination is then this exact physical dir, and the
|
|
112
|
+
// `verification/` component itself must not be a symlink (checked per copy).
|
|
113
|
+
let destDir = null;
|
|
114
|
+
try {
|
|
115
|
+
const bundlePath = runReportDir(workspaceDir, runId);
|
|
116
|
+
mkdirSync(bundlePath, { recursive: true });
|
|
117
|
+
const bundleReal = realpathSync(bundlePath);
|
|
118
|
+
const wsReal = realpathSync(workspaceDir);
|
|
119
|
+
if (bundleReal === wsReal || bundleReal.startsWith(wsReal + sep)) {
|
|
120
|
+
destDir = join(bundleReal, "verification");
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
catch {
|
|
124
|
+
destDir = null;
|
|
125
|
+
}
|
|
126
|
+
const relocateScratch = (p) => {
|
|
127
|
+
const keep = { path: p, bundled: false };
|
|
128
|
+
if (FILE_LINE_RE.test(p) || SHA_RE.test(p))
|
|
129
|
+
return keep; // a reference, not a file
|
|
130
|
+
const real = safeWorkspaceFile(p, workspaceDir);
|
|
131
|
+
// Only relocate scratch artifacts the verify stage wrote under `.otto-tmp/`
|
|
132
|
+
// — never an arbitrary in-repo or host file (#181 re-review).
|
|
133
|
+
if (!real ||
|
|
134
|
+
!destDir ||
|
|
135
|
+
!scratchRoot ||
|
|
136
|
+
(real !== scratchRoot && !real.startsWith(scratchRoot + sep))) {
|
|
137
|
+
return keep;
|
|
138
|
+
}
|
|
139
|
+
try {
|
|
140
|
+
mkdirSync(destDir, { recursive: true });
|
|
141
|
+
// The destination must be the *physical* bundle verification dir: reject a
|
|
142
|
+
// symlinked `verification/` (which could redirect the copy into a source
|
|
143
|
+
// dir even while staying inside the workspace) by requiring its realpath to
|
|
144
|
+
// equal the path itself (#181 boundary review).
|
|
145
|
+
if (lstatSync(destDir).isSymbolicLink())
|
|
146
|
+
return keep;
|
|
147
|
+
if (realpathSync(destDir) !== destDir)
|
|
148
|
+
return keep;
|
|
149
|
+
const safe = `${counter++}-${basename(p)}`
|
|
150
|
+
.replace(/[^\w.-]+/g, "-")
|
|
151
|
+
.slice(0, 100);
|
|
152
|
+
const target = join(destDir, safe);
|
|
153
|
+
// Reject a pre-existing symlink at the target — copyFileSync would follow it.
|
|
154
|
+
try {
|
|
155
|
+
if (lstatSync(target).isSymbolicLink())
|
|
156
|
+
return keep;
|
|
157
|
+
}
|
|
158
|
+
catch {
|
|
159
|
+
// absent target is the expected case
|
|
160
|
+
}
|
|
161
|
+
copyFileSync(real, target);
|
|
162
|
+
return { path: `${BUNDLE_ARTIFACT_PREFIX}${safe}`, bundled: true };
|
|
163
|
+
}
|
|
164
|
+
catch {
|
|
165
|
+
return keep; // best-effort: never fail finalize over a copy
|
|
166
|
+
}
|
|
167
|
+
};
|
|
168
|
+
return entries.map((e) => {
|
|
169
|
+
const next = { ...e };
|
|
170
|
+
if (e.artifactPath !== undefined) {
|
|
171
|
+
next.artifactExists = artifactReferenceExists(e.artifactPath, workspaceDir, { commitExists });
|
|
172
|
+
const r = relocateScratch(e.artifactPath);
|
|
173
|
+
next.artifactPath = r.path;
|
|
174
|
+
next.artifactBundled = r.bundled;
|
|
175
|
+
}
|
|
176
|
+
if (e.beforePath !== undefined) {
|
|
177
|
+
const r = relocateScratch(e.beforePath);
|
|
178
|
+
next.beforePath = r.path;
|
|
179
|
+
next.beforeBundled = r.bundled;
|
|
180
|
+
}
|
|
181
|
+
return next;
|
|
182
|
+
});
|
|
183
|
+
}
|
|
184
|
+
//# sourceMappingURL=verification-evidence.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"verification-evidence.js","sourceRoot":"","sources":["../src/verification-evidence.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,EACL,YAAY,EACZ,SAAS,EACT,SAAS,EACT,YAAY,EACZ,YAAY,EACZ,QAAQ,GACT,MAAM,SAAS,CAAC;AACjB,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,IAAI,EAAE,GAAG,EAAE,MAAM,WAAW,CAAC;AAE5D,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC/C,OAAO,EACL,sBAAsB,EACtB,wBAAwB,GAEzB,MAAM,0BAA0B,CAAC;AAElC,MAAM,MAAM,GAAG,mBAAmB,CAAC;AACnC,MAAM,YAAY,GAAG,0BAA0B,CAAC;AAOhD;;;;GAIG;AACH,SAAS,iBAAiB,CAAC,IAAY,EAAE,YAAoB;IAC3D,IAAI,4BAA4B,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,IAAI,CAAC,CAAC,YAAY;IACtE,MAAM,GAAG,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,YAAY,EAAE,IAAI,CAAC,CAAC;IAC/D,IAAI,MAAc,CAAC;IACnB,IAAI,IAAY,CAAC;IACjB,IAAI,CAAC;QACH,MAAM,GAAG,YAAY,CAAC,YAAY,CAAC,CAAC;IACtC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;IACD,IAAI,CAAC;QACH,IAAI,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC;IAC3B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC,CAAC,iBAAiB;IAChC,CAAC;IACD,IAAI,IAAI,KAAK,MAAM,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,GAAG,GAAG,CAAC;QAAE,OAAO,IAAI,CAAC,CAAC,UAAU;IAC9E,IAAI,CAAC;QACH,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE;YAAE,OAAO,IAAI,CAAC;IAC5C,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,uBAAuB,CACrC,GAAW,EACX,YAAoB,EACpB,IAAkB;IAElB,MAAM,CAAC,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC;IACrB,IAAI,CAAC,wBAAwB,CAAC,CAAC,CAAC;QAAE,OAAO,KAAK,CAAC;IAC/C,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;IAChD,MAAM,CAAC,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC/B,MAAM,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC1B,MAAM,IAAI,GAAG,iBAAiB,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC;IACnD,IAAI,CAAC,IAAI;QAAE,OAAO,KAAK,CAAC;IACxB,IAAI,CAAC,EAAE,CAAC;QACN,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC3B,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;QACxC,IAAI,KAAK,GAAG,CAAC,IAAI,GAAG,GAAG,KAAK;YAAE,OAAO,KAAK,CAAC;QAC3C,IAAI,SAAiB,CAAC;QACtB,IAAI,CAAC;YACH,SAAS,GAAG,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;QAC5D,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,KAAK,CAAC;QACf,CAAC;QACD,IAAI,GAAG,GAAG,SAAS;YAAE,OAAO,KAAK,CAAC;IACpC,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,4BAA4B,CAC1C,OAA4B,EAC5B,IAA4D;IAE5D,MAAM,EAAE,YAAY,EAAE,KAAK,EAAE,YAAY,EAAE,GAAG,IAAI,CAAC;IACnD,IAAI,OAAO,GAAG,CAAC,CAAC;IAEhB,IAAI,WAAW,GAAkB,IAAI,CAAC;IACtC,IAAI,CAAC;QACH,WAAW,GAAG,YAAY,CAAC,IAAI,CAAC,YAAY,EAAE,WAAW,CAAC,CAAC,CAAC;IAC9D,CAAC;IAAC,MAAM,CAAC;QACP,WAAW,GAAG,IAAI,CAAC,CAAC,uCAAuC;IAC7D,CAAC;IAED,+EAA+E;IAC/E,8EAA8E;IAC9E,+EAA+E;IAC/E,6EAA6E;IAC7E,IAAI,OAAO,GAAkB,IAAI,CAAC;IAClC,IAAI,CAAC;QACH,MAAM,UAAU,GAAG,YAAY,CAAC,YAAY,EAAE,KAAK,CAAC,CAAC;QACrD,SAAS,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC3C,MAAM,UAAU,GAAG,YAAY,CAAC,UAAU,CAAC,CAAC;QAC5C,MAAM,MAAM,GAAG,YAAY,CAAC,YAAY,CAAC,CAAC;QAC1C,IAAI,UAAU,KAAK,MAAM,IAAI,UAAU,CAAC,UAAU,CAAC,MAAM,GAAG,GAAG,CAAC,EAAE,CAAC;YACjE,OAAO,GAAG,IAAI,CAAC,UAAU,EAAE,cAAc,CAAC,CAAC;QAC7C,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,GAAG,IAAI,CAAC;IACjB,CAAC;IAED,MAAM,eAAe,GAAG,CAAC,CAAS,EAAsC,EAAE;QACxE,MAAM,IAAI,GAAG,EAAE,IAAI,EAAE,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;QACzC,IAAI,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;YAAE,OAAO,IAAI,CAAC,CAAC,0BAA0B;QACnF,MAAM,IAAI,GAAG,iBAAiB,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC;QAChD,4EAA4E;QAC5E,8DAA8D;QAC9D,IACE,CAAC,IAAI;YACL,CAAC,OAAO;YACR,CAAC,WAAW;YACZ,CAAC,IAAI,KAAK,WAAW,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,WAAW,GAAG,GAAG,CAAC,CAAC,EAC7D,CAAC;YACD,OAAO,IAAI,CAAC;QACd,CAAC;QACD,IAAI,CAAC;YACH,SAAS,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YACxC,2EAA2E;YAC3E,yEAAyE;YACzE,4EAA4E;YAC5E,gDAAgD;YAChD,IAAI,SAAS,CAAC,OAAO,CAAC,CAAC,cAAc,EAAE;gBAAE,OAAO,IAAI,CAAC;YACrD,IAAI,YAAY,CAAC,OAAO,CAAC,KAAK,OAAO;gBAAE,OAAO,IAAI,CAAC;YACnD,MAAM,IAAI,GAAG,GAAG,OAAO,EAAE,IAAI,QAAQ,CAAC,CAAC,CAAC,EAAE;iBACvC,OAAO,CAAC,WAAW,EAAE,GAAG,CAAC;iBACzB,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;YACjB,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;YACnC,8EAA8E;YAC9E,IAAI,CAAC;gBACH,IAAI,SAAS,CAAC,MAAM,CAAC,CAAC,cAAc,EAAE;oBAAE,OAAO,IAAI,CAAC;YACtD,CAAC;YAAC,MAAM,CAAC;gBACP,qCAAqC;YACvC,CAAC;YACD,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;YAC3B,OAAO,EAAE,IAAI,EAAE,GAAG,sBAAsB,GAAG,IAAI,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;QACrE,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,IAAI,CAAC,CAAC,+CAA+C;QAC9D,CAAC;IACH,CAAC,CAAC;IAEF,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QACvB,MAAM,IAAI,GAAsB,EAAE,GAAG,CAAC,EAAE,CAAC;QACzC,IAAI,CAAC,CAAC,YAAY,KAAK,SAAS,EAAE,CAAC;YACjC,IAAI,CAAC,cAAc,GAAG,uBAAuB,CAC3C,CAAC,CAAC,YAAY,EACd,YAAY,EACZ,EAAE,YAAY,EAAE,CACjB,CAAC;YACF,MAAM,CAAC,GAAG,eAAe,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC;YAC1C,IAAI,CAAC,YAAY,GAAG,CAAC,CAAC,IAAI,CAAC;YAC3B,IAAI,CAAC,eAAe,GAAG,CAAC,CAAC,OAAO,CAAC;QACnC,CAAC;QACD,IAAI,CAAC,CAAC,UAAU,KAAK,SAAS,EAAE,CAAC;YAC/B,MAAM,CAAC,GAAG,eAAe,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;YACxC,IAAI,CAAC,UAAU,GAAG,CAAC,CAAC,IAAI,CAAC;YACzB,IAAI,CAAC,aAAa,GAAG,CAAC,CAAC,OAAO,CAAC;QACjC,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Verification matrix (issue #181 P24, slice 1 — "make verification provable").
|
|
3
|
+
*
|
|
4
|
+
* A pure data model + scorer for the roadmap's verification-artifact matrix:
|
|
5
|
+
* each requirement (a plan task / acceptance criterion) paired with HOW it was
|
|
6
|
+
* checked, the concrete artifact that proves it (a `file:line`, a commit SHA, a
|
|
7
|
+
* command transcript, a screenshot path), the result, and a confidence. Today
|
|
8
|
+
* the `verify` stage emits this as free-form DONE/GAP/DEFERRED prose; P24 turns
|
|
9
|
+
* it into a structured matrix a maintainer can scan, and a "verification
|
|
10
|
+
* gallery" a non-engineer can accept/reject from.
|
|
11
|
+
*
|
|
12
|
+
* This slice is the substrate — the type, a summary scorer (counts, artifact
|
|
13
|
+
* coverage, an overall verdict), and a formatter that renders the matrix and
|
|
14
|
+
* surfaces failures / unproven requirements as explicit risks rather than
|
|
15
|
+
* burying them in logs. INERT on the loop: nothing here runs a stage or changes
|
|
16
|
+
* behavior; later P24 slices populate it from the verify stage and fold it into
|
|
17
|
+
* run reports + `otto-inspect`. Mirrors how the plan/context/report rubrics
|
|
18
|
+
* shipped pure-then-wired.
|
|
19
|
+
*/
|
|
20
|
+
/** How a requirement was checked. */
|
|
21
|
+
export type VerificationMethod = "test" | "command" | "visual" | "inspection" | "manual";
|
|
22
|
+
/** The outcome of checking a requirement. */
|
|
23
|
+
export type VerificationResult = "pass" | "fail" | "partial" | "deferred";
|
|
24
|
+
/** How much the evidence supports the result. */
|
|
25
|
+
export type VerificationConfidence = "high" | "medium" | "low";
|
|
26
|
+
/** One row of the verification matrix: a requirement and how it was proven. */
|
|
27
|
+
export type VerificationEntry = {
|
|
28
|
+
/** The requirement / plan task / acceptance criterion being verified. */
|
|
29
|
+
requirement: string;
|
|
30
|
+
method: VerificationMethod;
|
|
31
|
+
/** The concrete check: the command run, the assertion, or the visual checked. */
|
|
32
|
+
check: string;
|
|
33
|
+
/** Pointer to the proving artifact: `file:line`, a commit SHA, a transcript or
|
|
34
|
+
* screenshot path. Absent ⇒ the requirement is asserted but not artifact-backed.
|
|
35
|
+
* For a `visual` before/after entry this is the **after** screenshot. */
|
|
36
|
+
artifactPath?: string;
|
|
37
|
+
/** For a `visual` entry, the optional **before** screenshot path, paired with
|
|
38
|
+
* `artifactPath` (the after) for a before/after comparison. */
|
|
39
|
+
beforePath?: string;
|
|
40
|
+
/** Set by the loop's filesystem/git validation (issue #181 re-review): whether
|
|
41
|
+
* `artifactPath` actually resolves to an existing file (with in-bounds line)
|
|
42
|
+
* or a commit present in git. `false` ⇒ the cited proof does not exist and the
|
|
43
|
+
* requirement is not counted as covered. Never read from agent JSON. */
|
|
44
|
+
artifactExists?: boolean;
|
|
45
|
+
/** Set true ONLY by the impure layer when `artifactPath` was actually copied
|
|
46
|
+
* into the physical run bundle (issue #181 boundary review). Embedding trusts
|
|
47
|
+
* this flag, not a string prefix the agent could spoof. Never read from agent JSON. */
|
|
48
|
+
artifactBundled?: boolean;
|
|
49
|
+
/** Set true ONLY by the impure layer when `beforePath` was copied into the
|
|
50
|
+
* physical run bundle. Never read from agent JSON. */
|
|
51
|
+
beforeBundled?: boolean;
|
|
52
|
+
result: VerificationResult;
|
|
53
|
+
confidence: VerificationConfidence;
|
|
54
|
+
note?: string;
|
|
55
|
+
};
|
|
56
|
+
/**
|
|
57
|
+
* Whether a string is a *typed*, durable artifact reference that actually proves
|
|
58
|
+
* something — a `file:line` (or `file:line-range`), a path with a separator or a
|
|
59
|
+
* file extension (a transcript / screenshot / source file), or a 7–40 char commit
|
|
60
|
+
* SHA. A bare command (`node --test`), prose (`read the code`), or a placeholder
|
|
61
|
+
* (`TODO`) is NOT proof and returns false, so the coverage signal can't be earned
|
|
62
|
+
* by an unverifiable string (#181 review). Pure.
|
|
63
|
+
*/
|
|
64
|
+
export declare function isValidArtifactReference(ref: string): boolean;
|
|
65
|
+
export type VerificationParseResult = {
|
|
66
|
+
/** Valid entries kept. */
|
|
67
|
+
entries: VerificationEntry[];
|
|
68
|
+
/** Rows present in the JSON array that were rejected as malformed. */
|
|
69
|
+
dropped: number;
|
|
70
|
+
/** False when the input was not parseable JSON or not an array. */
|
|
71
|
+
parsed: boolean;
|
|
72
|
+
};
|
|
73
|
+
/**
|
|
74
|
+
* Parse an agent-emitted verification matrix (a JSON `VerificationEntry[]`,
|
|
75
|
+
* e.g. the `.otto-tmp/verify-matrix.json` the verify stage writes), keeping the
|
|
76
|
+
* parse diagnostics so a malformed/partial matrix is reported, not silently
|
|
77
|
+
* dropped (#181 review). Tolerant: never throws, drops any entry missing a
|
|
78
|
+
* non-empty `requirement` or carrying an unknown `method`/`result`, defaults an
|
|
79
|
+
* absent/invalid `confidence` to `medium`, and counts each dropped row. Non-array
|
|
80
|
+
* / malformed JSON ⇒ `{ entries: [], dropped: 0, parsed: false }`. Pure.
|
|
81
|
+
*/
|
|
82
|
+
export declare function parseVerificationMatrixWithDiagnostics(raw: string): VerificationParseResult;
|
|
83
|
+
/** Parse a verification matrix into valid entries (diagnostics discarded). Pure. */
|
|
84
|
+
export declare function parseVerificationMatrix(raw: string): VerificationEntry[];
|
|
85
|
+
export type VerificationSummary = {
|
|
86
|
+
total: number;
|
|
87
|
+
pass: number;
|
|
88
|
+
fail: number;
|
|
89
|
+
partial: number;
|
|
90
|
+
deferred: number;
|
|
91
|
+
/** Entries carrying a concrete artifact path. */
|
|
92
|
+
withArtifact: number;
|
|
93
|
+
/** Artifact-backed share of the *verifiable* (non-deferred) requirements, 0..1. */
|
|
94
|
+
coverage: number;
|
|
95
|
+
/**
|
|
96
|
+
* Overall verdict:
|
|
97
|
+
* - `empty` — nothing to verify.
|
|
98
|
+
* - `gaps` — at least one requirement failed.
|
|
99
|
+
* - `unproven` — no failures, but no verifiable requirement is artifact-backed.
|
|
100
|
+
* - `partial` — no failures, some but not all verifiable requirements proven.
|
|
101
|
+
* - `verified` — no failures and every verifiable requirement is artifact-backed.
|
|
102
|
+
*/
|
|
103
|
+
verdict: "empty" | "gaps" | "unproven" | "partial" | "verified";
|
|
104
|
+
};
|
|
105
|
+
/** Tally a verification matrix into counts, artifact coverage, and a verdict. Pure. */
|
|
106
|
+
export declare function summarizeVerification(entries: VerificationEntry[]): VerificationSummary;
|
|
107
|
+
export type VerificationCoverageGate = {
|
|
108
|
+
/** True iff no requirement failed, every verifiable one is artifact-backed, and
|
|
109
|
+
* no matrix rows were dropped during parsing. */
|
|
110
|
+
passed: boolean;
|
|
111
|
+
/** Artifact-backed share of the verifiable requirements, 0..1. */
|
|
112
|
+
coverage: number;
|
|
113
|
+
/** Verifiable requirements asserted without a valid proving artifact. */
|
|
114
|
+
unproven: string[];
|
|
115
|
+
/** Requirements that failed verification. */
|
|
116
|
+
failed: string[];
|
|
117
|
+
/** Requirements left `partial` — checked but not fully passing. */
|
|
118
|
+
incomplete: string[];
|
|
119
|
+
/** Malformed matrix rows dropped during parsing — unknown, unverified requirements. */
|
|
120
|
+
dropped: number;
|
|
121
|
+
};
|
|
122
|
+
/**
|
|
123
|
+
* Score a verification matrix against the roadmap's coverage bar (P24): every
|
|
124
|
+
* verifiable requirement must carry a valid artifact, none may fail, none may be
|
|
125
|
+
* left partial, and no rows may have been dropped during parsing. `dropped` rows
|
|
126
|
+
* are unknown requirements that cannot be assumed proven, so any drop FAILs the
|
|
127
|
+
* gate (#181 re-review). The `unproven`/`failed`/`incomplete` lists + `dropped`
|
|
128
|
+
* are exactly why the gate did not pass. Pure.
|
|
129
|
+
*/
|
|
130
|
+
export declare function scoreVerificationCoverage(entries: VerificationEntry[], dropped?: number): VerificationCoverageGate;
|
|
131
|
+
/**
|
|
132
|
+
* Render the verification-coverage gate as a report block (mirrors the emit-time
|
|
133
|
+
* legibility gate): a PASS/FAIL verdict, the artifact-backed coverage, and — on
|
|
134
|
+
* FAIL — the unproven/failed requirements plus how to clear them. Empty string
|
|
135
|
+
* for an empty matrix, so a run with no verification adds no gate. Pure.
|
|
136
|
+
*/
|
|
137
|
+
export declare function formatVerificationCoverageGate(entries: VerificationEntry[], dropped?: number): string;
|
|
138
|
+
/** Path prefix of artifacts relocated into the run bundle (see
|
|
139
|
+
* `verification-evidence.ts`); only these resolve relative to report.md. */
|
|
140
|
+
export declare const BUNDLE_ARTIFACT_PREFIX = "verification/";
|
|
141
|
+
/**
|
|
142
|
+
* Render the visual evidence (P24 visual half) as a markdown "Screenshot
|
|
143
|
+
* Evidence" section that *embeds* each `visual` entry's captured screenshot —
|
|
144
|
+
* a single image, or a before → after pair — so a non-engineer reading the run
|
|
145
|
+
* report sees the proof, not just a path. Only artifacts the impure layer actually
|
|
146
|
+
* **copied into the bundle** (`artifactBundled`/`beforeBundled`) and that are
|
|
147
|
+
* images are embedded; a visual check the environment could not render, or any
|
|
148
|
+
* unbundled/spoofed path, is left to the coverage gate to flag, never emitted as
|
|
149
|
+
* an image. Empty string when no embeddable visual evidence exists. Pure.
|
|
150
|
+
*/
|
|
151
|
+
export declare function formatVisualEvidence(entries: VerificationEntry[]): string;
|
|
152
|
+
/**
|
|
153
|
+
* Render a verification matrix as a scannable report: a header verdict line, one
|
|
154
|
+
* row per requirement (mark · method · requirement · artifact), and an explicit
|
|
155
|
+
* **Risks** section listing every failed or unproven requirement so a reader
|
|
156
|
+
* cannot miss them. Pure.
|
|
157
|
+
*/
|
|
158
|
+
export declare function formatVerificationMatrix(entries: VerificationEntry[]): string;
|
|
159
|
+
//# sourceMappingURL=verification-matrix.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"verification-matrix.d.ts","sourceRoot":"","sources":["../src/verification-matrix.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAEH,qCAAqC;AACrC,MAAM,MAAM,kBAAkB,GAC1B,MAAM,GACN,SAAS,GACT,QAAQ,GACR,YAAY,GACZ,QAAQ,CAAC;AAEb,6CAA6C;AAC7C,MAAM,MAAM,kBAAkB,GAAG,MAAM,GAAG,MAAM,GAAG,SAAS,GAAG,UAAU,CAAC;AAE1E,iDAAiD;AACjD,MAAM,MAAM,sBAAsB,GAAG,MAAM,GAAG,QAAQ,GAAG,KAAK,CAAC;AAE/D,+EAA+E;AAC/E,MAAM,MAAM,iBAAiB,GAAG;IAC9B,yEAAyE;IACzE,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,kBAAkB,CAAC;IAC3B,iFAAiF;IACjF,KAAK,EAAE,MAAM,CAAC;IACd;;8EAE0E;IAC1E,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB;oEACgE;IAChE,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;;6EAGyE;IACzE,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB;;4FAEwF;IACxF,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B;2DACuD;IACvD,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,MAAM,EAAE,kBAAkB,CAAC;IAC3B,UAAU,EAAE,sBAAsB,CAAC;IACnC,IAAI,CAAC,EAAE,MAAM,CAAC;CACf,CAAC;AAiBF;;;;;;;GAOG;AACH,wBAAgB,wBAAwB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAS7D;AAiBD,MAAM,MAAM,uBAAuB,GAAG;IACpC,0BAA0B;IAC1B,OAAO,EAAE,iBAAiB,EAAE,CAAC;IAC7B,sEAAsE;IACtE,OAAO,EAAE,MAAM,CAAC;IAChB,mEAAmE;IACnE,MAAM,EAAE,OAAO,CAAC;CACjB,CAAC;AAEF;;;;;;;;GAQG;AACH,wBAAgB,sCAAsC,CACpD,GAAG,EAAE,MAAM,GACV,uBAAuB,CAgBzB;AA8BD,oFAAoF;AACpF,wBAAgB,uBAAuB,CAAC,GAAG,EAAE,MAAM,GAAG,iBAAiB,EAAE,CAExE;AAOD,MAAM,MAAM,mBAAmB,GAAG;IAChC,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,iDAAiD;IACjD,YAAY,EAAE,MAAM,CAAC;IACrB,mFAAmF;IACnF,QAAQ,EAAE,MAAM,CAAC;IACjB;;;;;;;OAOG;IACH,OAAO,EAAE,OAAO,GAAG,MAAM,GAAG,UAAU,GAAG,SAAS,GAAG,UAAU,CAAC;CACjE,CAAC;AAEF,uFAAuF;AACvF,wBAAgB,qBAAqB,CACnC,OAAO,EAAE,iBAAiB,EAAE,GAC3B,mBAAmB,CAqCrB;AAED,MAAM,MAAM,wBAAwB,GAAG;IACrC;sDACkD;IAClD,MAAM,EAAE,OAAO,CAAC;IAChB,kEAAkE;IAClE,QAAQ,EAAE,MAAM,CAAC;IACjB,yEAAyE;IACzE,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,6CAA6C;IAC7C,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,mEAAmE;IACnE,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,uFAAuF;IACvF,OAAO,EAAE,MAAM,CAAC;CACjB,CAAC;AAEF;;;;;;;GAOG;AACH,wBAAgB,yBAAyB,CACvC,OAAO,EAAE,iBAAiB,EAAE,EAC5B,OAAO,SAAI,GACV,wBAAwB,CAgB1B;AAED;;;;;GAKG;AACH,wBAAgB,8BAA8B,CAC5C,OAAO,EAAE,iBAAiB,EAAE,EAC5B,OAAO,SAAI,GACV,MAAM,CA6CR;AAED;6EAC6E;AAC7E,eAAO,MAAM,sBAAsB,kBAAkB,CAAC;AAsBtD;;;;;;;;;GASG;AACH,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,iBAAiB,EAAE,GAAG,MAAM,CAmBzE;AAiBD;;;;;GAKG;AACH,wBAAgB,wBAAwB,CAAC,OAAO,EAAE,iBAAiB,EAAE,GAAG,MAAM,CAoC7E"}
|