imprint-mcp 0.6.1 → 0.6.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -0
- package/package.json +3 -2
- package/prompts/audit-agent.md +10 -7
- package/prompts/build-planning.md +2 -2
- package/prompts/compile-agent.md +49 -24
- package/prompts/live-verifier-agent.md +114 -0
- package/prompts/playbook-compilation.md +8 -1
- package/src/cli.ts +121 -4
- package/src/imprint/agent.ts +7 -2
- package/src/imprint/audit.ts +430 -39
- package/src/imprint/auth-compile-tools.ts +64 -13
- package/src/imprint/backend-cache.ts +18 -11
- package/src/imprint/backend-ladder.ts +48 -34
- package/src/imprint/cdp-browser-fetch.ts +135 -42
- package/src/imprint/claude-cli-compile.ts +38 -16
- package/src/imprint/codex-cli-compile.ts +120 -21
- package/src/imprint/compile-agent-types.ts +25 -2
- package/src/imprint/compile-agent.ts +238 -36
- package/src/imprint/compile-tools.ts +331 -58
- package/src/imprint/compile-verification.ts +187 -0
- package/src/imprint/compile.ts +71 -2
- package/src/imprint/freeform-redact.ts +27 -6
- package/src/imprint/live-verifier.ts +1826 -0
- package/src/imprint/llm.ts +20 -0
- package/src/imprint/mcp-compile-server.ts +488 -84
- package/src/imprint/mcp-live-verifier-server.ts +359 -0
- package/src/imprint/mcp-server.ts +28 -3
- package/src/imprint/probe-backends.ts +147 -31
- package/src/imprint/runtime.ts +61 -11
- package/src/imprint/teach-state.ts +70 -3
- package/src/imprint/teach.ts +171 -88
- package/src/imprint/types.ts +39 -2
|
@@ -11,7 +11,9 @@ import {
|
|
|
11
11
|
existsSync,
|
|
12
12
|
mkdirSync,
|
|
13
13
|
readFileSync,
|
|
14
|
+
readdirSync,
|
|
14
15
|
realpathSync,
|
|
16
|
+
statSync,
|
|
15
17
|
unlinkSync,
|
|
16
18
|
writeFileSync,
|
|
17
19
|
} from 'node:fs';
|
|
@@ -27,6 +29,11 @@ import {
|
|
|
27
29
|
readBuildPlanFile,
|
|
28
30
|
resolveAssignedModules,
|
|
29
31
|
} from './build-plan.ts';
|
|
32
|
+
import {
|
|
33
|
+
LIVE_EVIDENCE_PATH_ENV,
|
|
34
|
+
type LiveIntegrationEvidence,
|
|
35
|
+
readLiveIntegrationEvidence,
|
|
36
|
+
} from './compile-verification.ts';
|
|
30
37
|
import { splitSetCookieHeader } from './cookie-jar.ts';
|
|
31
38
|
import { isSameRegistrableDomain, registrableDomain } from './etld.ts';
|
|
32
39
|
import {
|
|
@@ -50,6 +57,7 @@ import {
|
|
|
50
57
|
} from './types.ts';
|
|
51
58
|
|
|
52
59
|
const REPO_ROOT = pathJoin(import.meta.dir, '..', '..');
|
|
60
|
+
const MAX_SHARED_MODULE_SOURCE_CHARS = 50_000;
|
|
53
61
|
|
|
54
62
|
// Env var read by the agent-written parser.test.ts to locate the redacted
|
|
55
63
|
// session. The test loads it, finds the load-bearing request seq, and feeds
|
|
@@ -80,7 +88,7 @@ export function buildCompileTools(
|
|
|
80
88
|
? { IMPRINT_TEACH_CREDENTIALS: JSON.stringify(context.teachCredentials) }
|
|
81
89
|
: undefined;
|
|
82
90
|
const tools = [
|
|
83
|
-
buildReadSessionSummaryTool(session, context),
|
|
91
|
+
buildReadSessionSummaryTool(session, context, toolDir),
|
|
84
92
|
buildReadRequestTool(session),
|
|
85
93
|
buildSearchRequestsTool(session),
|
|
86
94
|
buildRevealRequestTool(sessionPath),
|
|
@@ -115,6 +123,10 @@ export interface CompileToolContext {
|
|
|
115
123
|
buildPlanPath?: string;
|
|
116
124
|
/** Shared-module build manifest (verified flags) for this site. */
|
|
117
125
|
sharedModules?: SharedModuleManifestEntry[];
|
|
126
|
+
/** A bounded teach resume is revising an already-generated tool. Surface the
|
|
127
|
+
* existing artifact and durable verifier/audit feedback to the agent so it
|
|
128
|
+
* preserves proven behavior instead of re-deriving the tool from raw capture. */
|
|
129
|
+
revisionMode?: boolean;
|
|
118
130
|
}
|
|
119
131
|
|
|
120
132
|
// ─── Tool: read_build_plan ───────────────────────────────────────────────────
|
|
@@ -149,11 +161,11 @@ function buildReadBuildPlanTool(
|
|
|
149
161
|
];
|
|
150
162
|
if (emitsTokens.length > 0) {
|
|
151
163
|
tokenNotes.push(
|
|
152
|
-
`PRODUCER
|
|
164
|
+
`PRODUCER COMPATIBILITY TARGET: emit ${emitsTokens
|
|
153
165
|
.map((e) => `\`${e.field}\``)
|
|
154
166
|
.join(
|
|
155
167
|
', ',
|
|
156
|
-
)} in
|
|
168
|
+
)} in the exact shape described whenever a live result supplies or proves the full value a sibling consumer needs. Never fabricate a missing token. Useful core records may retain a null/absent token when workflow.limitations names the missing output and affected consumers; preserve their supported input classes. A missing downstream token alone is not a reason to discard valid producer records.`,
|
|
157
169
|
);
|
|
158
170
|
}
|
|
159
171
|
for (const tp of tokenParams) {
|
|
@@ -164,7 +176,7 @@ function buildReadBuildPlanTool(
|
|
|
164
176
|
? ` The producer declares shape: ${producerShape}. If that shape contains requiredKeys=[...], request-transform.ts must decode/unpack the fresh value and reference each required key in the outgoing request.`
|
|
165
177
|
: '';
|
|
166
178
|
tokenNotes.push(
|
|
167
|
-
`CONSUMER CONTRACT: param \`${tp.param}\` is an opaque token minted by the \`${tp.sourceTool}\` tool's \`${tp.sourceField}\` output.${shapeNote} Write a CHAINED \`param:${tp.param}\` integration test that calls \`
|
|
179
|
+
`CONSUMER CONTRACT: param \`${tp.param}\` is an opaque token minted by the \`${tp.sourceTool}\` tool's \`${tp.sourceField}\` output.${shapeNote} Write a CHAINED \`param:${tp.param}\` integration test that calls \`runCapturedIntegrationCase\` on \`../${tp.sourceTool}/workflow.json\` directly inside the \`param:${tp.param}\` test block, reads \`${tp.sourceField}\` from the producer's ACTUAL output shape (top-level field, or an item in any returned collection such as flights/items/results), and passes THAT fresh value (not the recorded constant) into this tool — then asserts the response is non-empty. When several params come from the same producer, mint once and choose one producer item containing all sibling fields so the values stay from the same result. On producer bot/infra error, rethrow so the suite waives. If useful producer records legitimately lack the field, do not fabricate or mutate output: keep this consumer verified:false, waived-chain, or inconclusive with diagnostics, and require an honest producer limitation.`,
|
|
168
180
|
);
|
|
169
181
|
}
|
|
170
182
|
// Per-source wiring guidance for the general dependency contract. These are
|
|
@@ -201,6 +213,7 @@ function buildReadBuildPlanTool(
|
|
|
201
213
|
kind: m.kind,
|
|
202
214
|
purpose: m.purpose,
|
|
203
215
|
exportSignatures: m.exportSignatures,
|
|
216
|
+
...readAssignedSharedModuleSource(buildPlanPath, m.path),
|
|
204
217
|
})),
|
|
205
218
|
parserGuidance: slice.tool.parserGuidance,
|
|
206
219
|
paramChecklist: slice.tool.paramChecklist,
|
|
@@ -227,11 +240,36 @@ function buildReadBuildPlanTool(
|
|
|
227
240
|
};
|
|
228
241
|
}
|
|
229
242
|
|
|
243
|
+
function readAssignedSharedModuleSource(
|
|
244
|
+
buildPlanPath: string,
|
|
245
|
+
modulePath: string,
|
|
246
|
+
): { source?: string; sourceTruncated?: boolean; sourceUnavailable?: string } {
|
|
247
|
+
const siteDir = realpathSync(dirname(buildPlanPath));
|
|
248
|
+
const candidate = pathJoin(siteDir, modulePath);
|
|
249
|
+
if (!existsSync(candidate)) {
|
|
250
|
+
return { sourceUnavailable: 'The verified shared module file is missing.' };
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
const resolved = realpathSync(candidate);
|
|
254
|
+
const relative = pathRelative(siteDir, resolved);
|
|
255
|
+
if (relative.startsWith('..') || relative === '') {
|
|
256
|
+
return { sourceUnavailable: 'The shared module resolved outside the site directory.' };
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
const source = readFileSync(resolved, 'utf8');
|
|
260
|
+
if (source.length <= MAX_SHARED_MODULE_SOURCE_CHARS) return { source };
|
|
261
|
+
return {
|
|
262
|
+
source: source.slice(0, MAX_SHARED_MODULE_SOURCE_CHARS),
|
|
263
|
+
sourceTruncated: true,
|
|
264
|
+
};
|
|
265
|
+
}
|
|
266
|
+
|
|
230
267
|
// ─── Tool: read_session_summary ──────────────────────────────────────────────
|
|
231
268
|
|
|
232
269
|
export function buildReadSessionSummaryTool(
|
|
233
270
|
session: Session,
|
|
234
271
|
context: CompileToolContext,
|
|
272
|
+
toolDir?: string,
|
|
235
273
|
): AgentTool {
|
|
236
274
|
return {
|
|
237
275
|
name: 'read_session_summary',
|
|
@@ -343,6 +381,10 @@ export function buildReadSessionSummaryTool(
|
|
|
343
381
|
captureHints: captureHints.length > 0 ? captureHints : undefined,
|
|
344
382
|
paramGroundingHints: paramGroundingHints.length > 0 ? paramGroundingHints : undefined,
|
|
345
383
|
inputProvenanceHints: inputProvenanceHints.length > 0 ? inputProvenanceHints : undefined,
|
|
384
|
+
revisionContext:
|
|
385
|
+
context.revisionMode && toolDir
|
|
386
|
+
? buildExistingArtifactRevisionContext(toolDir)
|
|
387
|
+
: undefined,
|
|
346
388
|
loadBearingRequests,
|
|
347
389
|
};
|
|
348
390
|
|
|
@@ -361,6 +403,65 @@ export function buildReadSessionSummaryTool(
|
|
|
361
403
|
};
|
|
362
404
|
}
|
|
363
405
|
|
|
406
|
+
const REVISION_ARTIFACT_NAMES = [
|
|
407
|
+
'workflow.json',
|
|
408
|
+
'parser.ts',
|
|
409
|
+
'request-transform.ts',
|
|
410
|
+
'integration.test.ts',
|
|
411
|
+
'playbook.yaml',
|
|
412
|
+
] as const;
|
|
413
|
+
|
|
414
|
+
const REVISION_DIAGNOSTIC_NAMES = [
|
|
415
|
+
'.live-verification.json',
|
|
416
|
+
'.live-verification-evidence.json',
|
|
417
|
+
'.live-verifier-log.jsonl',
|
|
418
|
+
] as const;
|
|
419
|
+
|
|
420
|
+
/** Describe, but do not interpret or gate on, prior compile evidence. The
|
|
421
|
+
* compile agent owns the judgment: it can read any listed file through the
|
|
422
|
+
* existing read_file tool and decide what to preserve, repair, narrow, or
|
|
423
|
+
* document. This is intentionally agent context rather than another runtime
|
|
424
|
+
* validator. */
|
|
425
|
+
function buildExistingArtifactRevisionContext(toolDir: string): Record<string, unknown> {
|
|
426
|
+
const describe = (name: string): { path: string; bytes: number } | null => {
|
|
427
|
+
const absolute = pathJoin(toolDir, name);
|
|
428
|
+
if (!existsSync(absolute)) return null;
|
|
429
|
+
try {
|
|
430
|
+
return { path: name, bytes: statSync(absolute).size };
|
|
431
|
+
} catch {
|
|
432
|
+
return null;
|
|
433
|
+
}
|
|
434
|
+
};
|
|
435
|
+
|
|
436
|
+
const artifacts = REVISION_ARTIFACT_NAMES.map(describe).filter(
|
|
437
|
+
(entry): entry is { path: string; bytes: number } => entry !== null,
|
|
438
|
+
);
|
|
439
|
+
const diagnostics = REVISION_DIAGNOSTIC_NAMES.map(describe).filter(
|
|
440
|
+
(entry): entry is { path: string; bytes: number } => entry !== null,
|
|
441
|
+
);
|
|
442
|
+
const notesDir = pathJoin(toolDir, 'notes');
|
|
443
|
+
const feedbackNotes = existsSync(notesDir)
|
|
444
|
+
? readdirSync(notesDir)
|
|
445
|
+
.filter((name) => name.endsWith('.md'))
|
|
446
|
+
.sort()
|
|
447
|
+
.map((name) => describe(pathJoin('notes', name)))
|
|
448
|
+
.filter((entry): entry is { path: string; bytes: number } => entry !== null)
|
|
449
|
+
: [];
|
|
450
|
+
|
|
451
|
+
return {
|
|
452
|
+
mode: 'revise_existing_artifact',
|
|
453
|
+
instruction: [
|
|
454
|
+
'This bounded resume is a revision, not a from-scratch compile.',
|
|
455
|
+
'Read the existing workflow, parser, request transform, integration test, durable live-verification diagnostics, and feedback notes before re-reading raw recording bodies.',
|
|
456
|
+
'Preserve behavior that live evidence proves works. Repair or narrow only the behavior contradicted by evidence.',
|
|
457
|
+
'If a secondary branch cannot be grounded, remove its dependent parameters and claims together and record the reasoning in workflow limitations; do not guess.',
|
|
458
|
+
].join(' '),
|
|
459
|
+
existingArtifacts: artifacts,
|
|
460
|
+
durableDiagnostics: diagnostics,
|
|
461
|
+
feedbackNotes,
|
|
462
|
+
};
|
|
463
|
+
}
|
|
464
|
+
|
|
364
465
|
// ─── Inline request/response data for candidate-scoped requests ─────────────
|
|
365
466
|
|
|
366
467
|
// claude-cli truncates tool results > ~40K chars. Keep the total summary
|
|
@@ -1345,6 +1446,7 @@ export async function runCommand(
|
|
|
1345
1446
|
cwd: string,
|
|
1346
1447
|
timeoutMs: number,
|
|
1347
1448
|
extraEnv?: Record<string, string>,
|
|
1449
|
+
onOutput?: (stream: 'stdout' | 'stderr', chunk: string) => void,
|
|
1348
1450
|
): Promise<{ result: string; isError?: boolean }> {
|
|
1349
1451
|
return new Promise((resolve) => {
|
|
1350
1452
|
// `detached: true` makes the child its own process-group leader so a timeout
|
|
@@ -1362,11 +1464,15 @@ export async function runCommand(
|
|
|
1362
1464
|
const TRUNCATE_LIMIT = 16 * 1024; // 16KB
|
|
1363
1465
|
|
|
1364
1466
|
proc.stdout.on('data', (chunk) => {
|
|
1365
|
-
|
|
1467
|
+
const text = chunk.toString();
|
|
1468
|
+
stdout += text;
|
|
1469
|
+
onOutput?.('stdout', text);
|
|
1366
1470
|
});
|
|
1367
1471
|
|
|
1368
1472
|
proc.stderr.on('data', (chunk) => {
|
|
1369
|
-
|
|
1473
|
+
const text = chunk.toString();
|
|
1474
|
+
stderr += text;
|
|
1475
|
+
onOutput?.('stderr', text);
|
|
1370
1476
|
});
|
|
1371
1477
|
|
|
1372
1478
|
const timeout = setTimeout(() => {
|
|
@@ -1716,7 +1822,7 @@ interface BunTestRun {
|
|
|
1716
1822
|
|
|
1717
1823
|
/** Per-exposed-parameter verification outcome. `verified` is true only when a
|
|
1718
1824
|
* `param:<name>` integration test actually ran green against live data. */
|
|
1719
|
-
interface ParamVerification {
|
|
1825
|
+
export interface ParamVerification {
|
|
1720
1826
|
name: string;
|
|
1721
1827
|
verified: boolean;
|
|
1722
1828
|
/** Why an exposed param is unverified. Undefined when `verified` is true.
|
|
@@ -1727,7 +1833,7 @@ interface ParamVerification {
|
|
|
1727
1833
|
* - `waived-chain`: the param is a producer-sourced token but the producer
|
|
1728
1834
|
* tool could not be run at compile time (anti-bot / not compiled), so the
|
|
1729
1835
|
* chain could not be verified. */
|
|
1730
|
-
reason?: 'waived-bot' | 'waived-infra' | 'annotated' | 'waived-chain';
|
|
1836
|
+
reason?: 'waived-bot' | 'waived-infra' | 'annotated' | 'waived-chain' | 'semantic-gap';
|
|
1731
1837
|
/** For a producer-sourced token param, the sibling tool + output field its
|
|
1732
1838
|
* value comes from. Stamped into workflow.json (`param.sourcedFrom`) so the
|
|
1733
1839
|
* MCP description tells the orchestrating LLM where to mint it and the audit
|
|
@@ -1751,11 +1857,15 @@ interface TokenSource {
|
|
|
1751
1857
|
* bot-defense / infra detection and error surfacing) and the per-test pass/fail
|
|
1752
1858
|
* names via a JUnit report written to a transient file in the tool dir.
|
|
1753
1859
|
*/
|
|
1754
|
-
async function runBunTestWithResults(
|
|
1860
|
+
export async function runBunTestWithResults(
|
|
1755
1861
|
testPath: string,
|
|
1756
1862
|
toolDir: string,
|
|
1757
1863
|
timeoutMs: number,
|
|
1758
1864
|
env: Record<string, string> = {},
|
|
1865
|
+
opts: {
|
|
1866
|
+
bail?: boolean;
|
|
1867
|
+
onOutput?: (stream: 'stdout' | 'stderr', chunk: string) => void;
|
|
1868
|
+
} = {},
|
|
1759
1869
|
): Promise<BunTestRun> {
|
|
1760
1870
|
const junitPath = pathJoin(toolDir, `.imprint-junit-${basename(testPath)}.xml`);
|
|
1761
1871
|
try {
|
|
@@ -1764,10 +1874,11 @@ async function runBunTestWithResults(
|
|
|
1764
1874
|
// best-effort
|
|
1765
1875
|
}
|
|
1766
1876
|
const result = await runCommand(
|
|
1767
|
-
`bun test ${testPath} --reporter=junit --reporter-outfile=${junitPath}`,
|
|
1877
|
+
`bun test ${testPath} --reporter=junit --reporter-outfile=${junitPath}${opts.bail ? ' --bail=1' : ''}`,
|
|
1768
1878
|
toolDir,
|
|
1769
1879
|
timeoutMs,
|
|
1770
1880
|
env,
|
|
1881
|
+
opts.onOutput,
|
|
1771
1882
|
);
|
|
1772
1883
|
const output = JSON.parse(result.result) as {
|
|
1773
1884
|
stdout: string;
|
|
@@ -1802,20 +1913,121 @@ interface TestBlock {
|
|
|
1802
1913
|
body: string;
|
|
1803
1914
|
}
|
|
1804
1915
|
|
|
1805
|
-
|
|
1806
|
-
|
|
1807
|
-
|
|
1916
|
+
function matchingClosingBrace(source: string, openIndex: number): number | null {
|
|
1917
|
+
let depth = 0;
|
|
1918
|
+
let quote: "'" | '"' | '`' | null = null;
|
|
1919
|
+
let escaped = false;
|
|
1920
|
+
let lineComment = false;
|
|
1921
|
+
let blockComment = false;
|
|
1922
|
+
for (let index = openIndex; index < source.length; index++) {
|
|
1923
|
+
const char = source[index] ?? '';
|
|
1924
|
+
const next = source[index + 1] ?? '';
|
|
1925
|
+
if (lineComment) {
|
|
1926
|
+
if (char === '\n') lineComment = false;
|
|
1927
|
+
continue;
|
|
1928
|
+
}
|
|
1929
|
+
if (blockComment) {
|
|
1930
|
+
if (char === '*' && next === '/') {
|
|
1931
|
+
blockComment = false;
|
|
1932
|
+
index++;
|
|
1933
|
+
}
|
|
1934
|
+
continue;
|
|
1935
|
+
}
|
|
1936
|
+
if (quote) {
|
|
1937
|
+
if (escaped) {
|
|
1938
|
+
escaped = false;
|
|
1939
|
+
} else if (char === '\\') {
|
|
1940
|
+
escaped = true;
|
|
1941
|
+
} else if (char === quote) {
|
|
1942
|
+
quote = null;
|
|
1943
|
+
}
|
|
1944
|
+
continue;
|
|
1945
|
+
}
|
|
1946
|
+
if (char === '/' && next === '/') {
|
|
1947
|
+
lineComment = true;
|
|
1948
|
+
index++;
|
|
1949
|
+
continue;
|
|
1950
|
+
}
|
|
1951
|
+
if (char === '/' && next === '*') {
|
|
1952
|
+
blockComment = true;
|
|
1953
|
+
index++;
|
|
1954
|
+
continue;
|
|
1955
|
+
}
|
|
1956
|
+
if (char === "'" || char === '"' || char === '`') {
|
|
1957
|
+
quote = char;
|
|
1958
|
+
continue;
|
|
1959
|
+
}
|
|
1960
|
+
if (char === '{') depth++;
|
|
1961
|
+
if (char === '}' && --depth === 0) return index;
|
|
1962
|
+
}
|
|
1963
|
+
return null;
|
|
1964
|
+
}
|
|
1965
|
+
|
|
1966
|
+
function hasCapturedLiveWorkflowCall(source: string): boolean {
|
|
1967
|
+
return /\b(?:runWorkflowWithLadder|runCapturedIntegrationCase)\s*\(/.test(source);
|
|
1968
|
+
}
|
|
1969
|
+
|
|
1970
|
+
/** Resolve small local function wrappers around a captured workflow call. The
|
|
1971
|
+
* generated suites commonly factor one invocation helper above their tests;
|
|
1972
|
+
* keeping the resolution local prevents a call in an unrelated param test from
|
|
1973
|
+
* laundering a synthetic baseline elsewhere in the file. */
|
|
1974
|
+
function capturedWorkflowHelperNames(source: string): Set<string> {
|
|
1975
|
+
const functionRe = /\b(?:async\s+)?function\s+([A-Za-z_$][\w$]*)\s*\([^)]*\)\s*\{/g;
|
|
1976
|
+
const functions = [...source.matchAll(functionRe)].map((match) => ({
|
|
1977
|
+
name: match[1] ?? '',
|
|
1978
|
+
index: match.index ?? 0,
|
|
1979
|
+
}));
|
|
1980
|
+
const testStarts = [...source.matchAll(/\b(?:test|it)\s*\(/g)].map((match) => match.index ?? 0);
|
|
1981
|
+
const boundaries = [...functions.map((entry) => entry.index), ...testStarts, source.length].sort(
|
|
1982
|
+
(a, b) => a - b,
|
|
1983
|
+
);
|
|
1984
|
+
const bodies = new Map<string, string>();
|
|
1985
|
+
for (const entry of functions) {
|
|
1986
|
+
const end = boundaries.find((boundary) => boundary > entry.index) ?? source.length;
|
|
1987
|
+
bodies.set(entry.name, source.slice(entry.index, end));
|
|
1988
|
+
}
|
|
1989
|
+
const captured = new Set<string>();
|
|
1990
|
+
let changed = true;
|
|
1991
|
+
while (changed) {
|
|
1992
|
+
changed = false;
|
|
1993
|
+
for (const [name, body] of bodies) {
|
|
1994
|
+
if (captured.has(name)) continue;
|
|
1995
|
+
const callsCapturedHelper = [...captured].some((helper) =>
|
|
1996
|
+
new RegExp(`\\b${helper.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\s*\\(`).test(body),
|
|
1997
|
+
);
|
|
1998
|
+
if (hasCapturedLiveWorkflowCall(body) || callsCapturedHelper) {
|
|
1999
|
+
captured.add(name);
|
|
2000
|
+
changed = true;
|
|
2001
|
+
}
|
|
2002
|
+
}
|
|
2003
|
+
}
|
|
2004
|
+
return captured;
|
|
2005
|
+
}
|
|
2006
|
+
|
|
2007
|
+
/** Split a test file into `test(...)` / `it(...)` blocks (title + callback
|
|
2008
|
+
* source). Good enough to check whether a named per-parameter test's body
|
|
2009
|
+
* actually calls the workflow. */
|
|
1808
2010
|
export function extractTestBlocks(src: string): TestBlock[] {
|
|
1809
2011
|
const re = /\b(?:test|it)\s*\(\s*(['"`])((?:\\.|(?!\1).)*)\1/g;
|
|
1810
|
-
const starts: Array<{ index: number; title: string }> = [];
|
|
2012
|
+
const starts: Array<{ index: number; title: string; afterTitle: number }> = [];
|
|
1811
2013
|
for (const m of src.matchAll(re)) {
|
|
1812
|
-
starts.push({
|
|
2014
|
+
starts.push({
|
|
2015
|
+
index: m.index ?? 0,
|
|
2016
|
+
title: m[2] ?? '',
|
|
2017
|
+
afterTitle: (m.index ?? 0) + m[0].length,
|
|
2018
|
+
});
|
|
1813
2019
|
}
|
|
1814
2020
|
const blocks: TestBlock[] = [];
|
|
1815
2021
|
for (let i = 0; i < starts.length; i++) {
|
|
1816
2022
|
const start = starts[i];
|
|
1817
2023
|
if (!start) continue;
|
|
1818
|
-
const
|
|
2024
|
+
const fallbackEnd = i + 1 < starts.length ? (starts[i + 1]?.index ?? src.length) : src.length;
|
|
2025
|
+
const callbackOpen = src.indexOf('{', start.afterTitle);
|
|
2026
|
+
const callbackClose =
|
|
2027
|
+
callbackOpen >= 0 && callbackOpen < fallbackEnd
|
|
2028
|
+
? matchingClosingBrace(src, callbackOpen)
|
|
2029
|
+
: null;
|
|
2030
|
+
const end = callbackClose === null ? fallbackEnd : callbackClose + 1;
|
|
1819
2031
|
blocks.push({ title: start.title, body: src.slice(start.index, end) });
|
|
1820
2032
|
}
|
|
1821
2033
|
return blocks;
|
|
@@ -2198,6 +2410,7 @@ export function classifyIntegrationOutcome(input: {
|
|
|
2198
2410
|
* - chained pass → `{ verified: true, sourcedFrom }`
|
|
2199
2411
|
* - passed but not chained (the recorded-value tautology) → `unchained` (blocking)
|
|
2200
2412
|
* - suite waived (producer anti-bot) → `{ verified: false, reason: 'waived-chain' }`
|
|
2413
|
+
* - explicitly annotated unavailable producer output → `{ verified: false, reason: 'waived-chain' }`
|
|
2201
2414
|
* - else → `unchained` (blocking)
|
|
2202
2415
|
*/
|
|
2203
2416
|
export function classifyParamCoverage(opts: {
|
|
@@ -2232,6 +2445,10 @@ export function classifyParamCoverage(opts: {
|
|
|
2232
2445
|
const token = `param:${lp.name}`;
|
|
2233
2446
|
const passedLive = [...opts.passedTests].some((n) => n.includes(token));
|
|
2234
2447
|
const block = blocks.find((b) => b.title.includes(token));
|
|
2448
|
+
const annotationRe = new RegExp(
|
|
2449
|
+
`//\\s*exposed-but-not-verified[^\\n]*\\b${lp.name.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\b`,
|
|
2450
|
+
);
|
|
2451
|
+
const isAnnotated = annotationRe.test(opts.integrationSrc);
|
|
2235
2452
|
|
|
2236
2453
|
// Producer-sourced token param: requires a chained test that mints a fresh
|
|
2237
2454
|
// value from the producer's sibling workflow.
|
|
@@ -2241,7 +2458,7 @@ export function classifyParamCoverage(opts: {
|
|
|
2241
2458
|
if (passedLive) {
|
|
2242
2459
|
const chained =
|
|
2243
2460
|
!!block &&
|
|
2244
|
-
|
|
2461
|
+
hasCapturedLiveWorkflowCall(block.body) &&
|
|
2245
2462
|
SIBLING_WORKFLOW_RE.test(block.body);
|
|
2246
2463
|
if (chained) {
|
|
2247
2464
|
paramVerification.push({ name: lp.name, verified: true, sourcedFrom });
|
|
@@ -2255,21 +2472,28 @@ export function classifyParamCoverage(opts: {
|
|
|
2255
2472
|
reason: 'waived-chain',
|
|
2256
2473
|
sourcedFrom,
|
|
2257
2474
|
});
|
|
2475
|
+
} else if (isAnnotated) {
|
|
2476
|
+
// The compile agent may explicitly preserve a limited consumer when
|
|
2477
|
+
// its producer returns useful core records but cannot mint this token.
|
|
2478
|
+
// The durable verified:false+sourcedFrom stamp keeps orchestration
|
|
2479
|
+
// honest; opaque defaults and passing recorded-token tests remain
|
|
2480
|
+
// blocked by the independent guards above and below.
|
|
2481
|
+
paramVerification.push({
|
|
2482
|
+
name: lp.name,
|
|
2483
|
+
verified: false,
|
|
2484
|
+
reason: 'waived-chain',
|
|
2485
|
+
sourcedFrom,
|
|
2486
|
+
});
|
|
2258
2487
|
} else {
|
|
2259
2488
|
unchained.push(lp.name);
|
|
2260
2489
|
}
|
|
2261
2490
|
continue;
|
|
2262
2491
|
}
|
|
2263
2492
|
|
|
2264
|
-
const annotationRe = new RegExp(
|
|
2265
|
-
`//\\s*exposed-but-not-verified[^\\n]*\\b${lp.name.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\b`,
|
|
2266
|
-
);
|
|
2267
|
-
const isAnnotated = annotationRe.test(opts.integrationSrc);
|
|
2268
|
-
|
|
2269
2493
|
if (passedLive) {
|
|
2270
2494
|
// Anti-tautology: a passing per-param test must actually exercise the live
|
|
2271
2495
|
// workflow, not assert a constant.
|
|
2272
|
-
if (block &&
|
|
2496
|
+
if (block && !hasCapturedLiveWorkflowCall(block.body)) {
|
|
2273
2497
|
tautological.push(lp.name);
|
|
2274
2498
|
} else {
|
|
2275
2499
|
paramVerification.push({ name: lp.name, verified: true });
|
|
@@ -2299,13 +2523,20 @@ export function hasLiveBaselineWorkflowTest(integrationSrc: string): boolean {
|
|
|
2299
2523
|
const throwsOnFailure = /if\s*\(\s*!result\.ok\s*\)\s*\{[\s\S]{0,500}\bthrow\b/;
|
|
2300
2524
|
const returnsOnFailedResult =
|
|
2301
2525
|
/if\s*\([^)]*![^)]*\.result\.ok[\s\S]{0,200}\)\s*(?:\{[\s\S]{0,300}\breturn\b|return\b)/;
|
|
2302
|
-
|
|
2303
|
-
|
|
2526
|
+
const capturedHelpers = capturedWorkflowHelperNames(integrationSrc);
|
|
2527
|
+
return extractTestBlocks(integrationSrc).some((block) => {
|
|
2528
|
+
const invokesCapturedWorkflow =
|
|
2529
|
+
hasCapturedLiveWorkflowCall(block.body) ||
|
|
2530
|
+
[...capturedHelpers].some((helper) =>
|
|
2531
|
+
new RegExp(`\\b${helper.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\s*\\(`).test(block.body),
|
|
2532
|
+
);
|
|
2533
|
+
return (
|
|
2304
2534
|
!block.title.includes('param:') &&
|
|
2305
|
-
|
|
2535
|
+
invokesCapturedWorkflow &&
|
|
2306
2536
|
!returnsOnFailedResult.test(block.body) &&
|
|
2307
|
-
(requiresOk.test(block.body) || throwsOnFailure.test(block.body))
|
|
2308
|
-
|
|
2537
|
+
(requiresOk.test(block.body) || throwsOnFailure.test(block.body))
|
|
2538
|
+
);
|
|
2539
|
+
});
|
|
2309
2540
|
}
|
|
2310
2541
|
|
|
2311
2542
|
/**
|
|
@@ -3427,6 +3658,10 @@ export async function externalVerification(
|
|
|
3427
3658
|
* verifier blocks invented ${credential.X} placeholders that are not in this
|
|
3428
3659
|
* set or in requiredInputs. */
|
|
3429
3660
|
credentialNames?: string[];
|
|
3661
|
+
/** Agentic compile path only: validate the integration suite's source here,
|
|
3662
|
+
* but let the independent semantic verifier invoke the final live suite as
|
|
3663
|
+
* a tool. Parser/type/schema checks remain deterministic. */
|
|
3664
|
+
deferLiveIntegrationToSemanticAgent?: boolean;
|
|
3430
3665
|
} = {},
|
|
3431
3666
|
): Promise<{
|
|
3432
3667
|
failures: string[];
|
|
@@ -3440,9 +3675,13 @@ export async function externalVerification(
|
|
|
3440
3675
|
firstError: string;
|
|
3441
3676
|
exhaustedBackends: string[];
|
|
3442
3677
|
};
|
|
3678
|
+
/** Actual tool-level inputs and parsed results captured by the single live
|
|
3679
|
+
* integration run. The caller hands these to the independent semantic agent. */
|
|
3680
|
+
integrationEvidence: LiveIntegrationEvidence[];
|
|
3443
3681
|
}> {
|
|
3444
3682
|
const failures: string[] = [];
|
|
3445
3683
|
const warnings: string[] = [];
|
|
3684
|
+
let integrationEvidence: LiveIntegrationEvidence[] = [];
|
|
3446
3685
|
const paramVerification: ParamVerification[] = [];
|
|
3447
3686
|
let liveVerification:
|
|
3448
3687
|
| { kind: 'waived-bot' | 'waived-infra'; firstError: string; exhaustedBackends: string[] }
|
|
@@ -3647,6 +3886,7 @@ export async function externalVerification(
|
|
|
3647
3886
|
|
|
3648
3887
|
const notTemplated: string[] = [];
|
|
3649
3888
|
const inventedOnly: string[] = [];
|
|
3889
|
+
const intentionallyOmitted: Array<{ name: string; feature: string; reason: string }> = [];
|
|
3650
3890
|
|
|
3651
3891
|
for (const lp of opts.likelyParams) {
|
|
3652
3892
|
const placeholder = `\${param.${lp.name}}`;
|
|
@@ -3684,7 +3924,19 @@ export async function externalVerification(
|
|
|
3684
3924
|
}
|
|
3685
3925
|
|
|
3686
3926
|
if (!inBody && !inHeader && !inOriginalQuery && !inInventedQuery) {
|
|
3687
|
-
|
|
3927
|
+
const limitation = workflow.limitations?.find((entry) =>
|
|
3928
|
+
entry.omittedParameters?.includes(lp.name),
|
|
3929
|
+
);
|
|
3930
|
+
const stillDeclared = workflow.parameters.some((param) => param.name === lp.name);
|
|
3931
|
+
if (limitation && !stillDeclared) {
|
|
3932
|
+
intentionallyOmitted.push({
|
|
3933
|
+
name: lp.name,
|
|
3934
|
+
feature: limitation.feature,
|
|
3935
|
+
reason: limitation.reason,
|
|
3936
|
+
});
|
|
3937
|
+
} else {
|
|
3938
|
+
notTemplated.push(lp.name);
|
|
3939
|
+
}
|
|
3688
3940
|
} else if (!inBody && !inHeader && !inOriginalQuery && inInventedQuery) {
|
|
3689
3941
|
inventedOnly.push(lp.name);
|
|
3690
3942
|
}
|
|
@@ -3692,7 +3944,7 @@ export async function externalVerification(
|
|
|
3692
3944
|
|
|
3693
3945
|
if (notTemplated.length > 0) {
|
|
3694
3946
|
failures.push(
|
|
3695
|
-
`${notTemplated.length} likelyParam(s)
|
|
3947
|
+
`${notTemplated.length} likelyParam(s) have no valid disposition: ${notTemplated.join(', ')}. Each must either appear as \${param.NAME} in a request URL, body, or header, or be removed from the public parameters list and named in workflow.limitations[].omittedParameters with an evidence-based reason. For parameters recorded as null or [] (filters the user toggled but didn\'t apply), find the correct position in the request body and replace the placeholder value with \${param.NAME}.`,
|
|
3696
3948
|
);
|
|
3697
3949
|
}
|
|
3698
3950
|
if (inventedOnly.length > 0) {
|
|
@@ -3700,6 +3952,13 @@ export async function externalVerification(
|
|
|
3700
3952
|
`${inventedOnly.length} likelyParam(s) are templated only in URL query params that do not exist in any recorded request URL: ${inventedOnly.join(', ')}. The API server likely ignores these invented params — wire them into the request body or an existing query param instead. For complex body formats, use a requestTransformModule to construct the body programmatically.`,
|
|
3701
3953
|
);
|
|
3702
3954
|
}
|
|
3955
|
+
if (intentionallyOmitted.length > 0) {
|
|
3956
|
+
warnings.push(
|
|
3957
|
+
`${intentionallyOmitted.length} detector-suggested parameter(s) were intentionally omitted by the compile agent: ${intentionallyOmitted
|
|
3958
|
+
.map(({ name, feature, reason }) => `${name} (${feature}: ${reason})`)
|
|
3959
|
+
.join('; ')}. The limitations are retained in workflow.json and surfaced to callers.`,
|
|
3960
|
+
);
|
|
3961
|
+
}
|
|
3703
3962
|
}
|
|
3704
3963
|
} catch (err) {
|
|
3705
3964
|
failures.push(
|
|
@@ -3816,8 +4075,13 @@ export async function externalVerification(
|
|
|
3816
4075
|
if (!hasLiveBaselineWorkflowTest(integrationSrc)) {
|
|
3817
4076
|
integrationOutcome = 'failed';
|
|
3818
4077
|
failures.push(
|
|
3819
|
-
'integration.test.ts has no strict baseline live workflow test: add a non-`param:` test that calls `
|
|
4078
|
+
'integration.test.ts has no strict baseline live workflow test: add a non-`param:` test that calls `runCapturedIntegrationCase(...)` on this tool workflow, fails when `result.ok` is false, and asserts the successful response contains real data. Static workflow-shape assertions or green returns on bot/infra errors are not enough to verify a generated tool.',
|
|
3820
4079
|
);
|
|
4080
|
+
} else if (opts.deferLiveIntegrationToSemanticAgent) {
|
|
4081
|
+
// The independent verifier owns the final live execution. Keeping the
|
|
4082
|
+
// suite out of this deterministic gate avoids two disconnected verdicts
|
|
4083
|
+
// and lets the reviewer inspect the exact outputs it requested.
|
|
4084
|
+
integrationOutcome = 'absent';
|
|
3821
4085
|
} else {
|
|
3822
4086
|
// Scale the verifier's live-test timeout to the suite size: the baseline plus
|
|
3823
4087
|
// one live `runWorkflowWithLadder` per param, each gated by the ~25s compile
|
|
@@ -3827,26 +4091,30 @@ export async function externalVerification(
|
|
|
3827
4091
|
const paramCount = opts.likelyParams?.length ?? 0;
|
|
3828
4092
|
const pacingMs = Number(process.env.IMPRINT_COMPILE_ACT_SPACING_MS ?? 25_000) || 0;
|
|
3829
4093
|
const verifierTimeoutMs = Math.min(120_000 + paramCount * (pacingMs + 20_000), 10 * 60_000);
|
|
3830
|
-
|
|
3831
|
-
|
|
3832
|
-
|
|
3833
|
-
|
|
3834
|
-
|
|
3835
|
-
|
|
3836
|
-
|
|
3837
|
-
|
|
3838
|
-
|
|
3839
|
-
|
|
3840
|
-
|
|
3841
|
-
|
|
3842
|
-
|
|
3843
|
-
|
|
3844
|
-
|
|
3845
|
-
|
|
3846
|
-
|
|
3847
|
-
|
|
3848
|
-
|
|
3849
|
-
|
|
4094
|
+
const evidencePath = pathJoin(
|
|
4095
|
+
toolDir,
|
|
4096
|
+
`.imprint-live-evidence-${process.pid}-${Date.now()}.jsonl`,
|
|
4097
|
+
);
|
|
4098
|
+
try {
|
|
4099
|
+
if (existsSync(evidencePath)) unlinkSync(evidencePath);
|
|
4100
|
+
} catch {
|
|
4101
|
+
// best-effort cleanup before the run
|
|
4102
|
+
}
|
|
4103
|
+
const run = await runBunTestWithResults(integrationTestPath, toolDir, verifierTimeoutMs, {
|
|
4104
|
+
[LIVE_EVIDENCE_PATH_ENV]: evidencePath,
|
|
4105
|
+
});
|
|
4106
|
+
try {
|
|
4107
|
+
integrationEvidence = readLiveIntegrationEvidence(evidencePath);
|
|
4108
|
+
} catch (error) {
|
|
4109
|
+
failures.push(
|
|
4110
|
+
`could not read captured live integration evidence: ${error instanceof Error ? error.message : String(error)}`,
|
|
4111
|
+
);
|
|
4112
|
+
} finally {
|
|
4113
|
+
try {
|
|
4114
|
+
if (existsSync(evidencePath)) unlinkSync(evidencePath);
|
|
4115
|
+
} catch {
|
|
4116
|
+
// Evidence is ephemeral and intentionally never kept with artifacts.
|
|
4117
|
+
}
|
|
3850
4118
|
}
|
|
3851
4119
|
integrationPassedTests = run.passed;
|
|
3852
4120
|
|
|
@@ -3862,7 +4130,11 @@ export async function externalVerification(
|
|
|
3862
4130
|
integrationOutcome = verdict.outcome;
|
|
3863
4131
|
|
|
3864
4132
|
if (verdict.outcome === 'passed') {
|
|
3865
|
-
|
|
4133
|
+
if (integrationEvidence.length === 0) {
|
|
4134
|
+
failures.push(
|
|
4135
|
+
'integration.test.ts passed but captured no live evidence. Import `runCapturedIntegrationCase` from `imprint/compile-verification` and use it for every live workflow call so the independent verifier can inspect the actual tool inputs and parsed outputs.',
|
|
4136
|
+
);
|
|
4137
|
+
}
|
|
3866
4138
|
} else if (verdict.captureFailName !== null) {
|
|
3867
4139
|
const capName = verdict.captureFailName;
|
|
3868
4140
|
// If the failing capture is a `response_header` on a REPLAYED workflow
|
|
@@ -3913,7 +4185,7 @@ export async function externalVerification(
|
|
|
3913
4185
|
);
|
|
3914
4186
|
} else {
|
|
3915
4187
|
failures.push(
|
|
3916
|
-
`bun test integration.test.ts exited ${run.exitCode} — the workflow failed to produce live data
|
|
4188
|
+
`bun test integration.test.ts exited ${run.exitCode} — the workflow failed to produce live data. The baseline suite runs exactly once per verification cycle.\nstdout:\n${run.stdout}\nstderr:\n${run.stderr}`,
|
|
3917
4189
|
);
|
|
3918
4190
|
}
|
|
3919
4191
|
}
|
|
@@ -3930,7 +4202,8 @@ export async function externalVerification(
|
|
|
3930
4202
|
!referencedStateBroken &&
|
|
3931
4203
|
existsSync(integrationTestPath) &&
|
|
3932
4204
|
opts.likelyParams &&
|
|
3933
|
-
opts.likelyParams.length > 0
|
|
4205
|
+
opts.likelyParams.length > 0 &&
|
|
4206
|
+
!opts.deferLiveIntegrationToSemanticAgent
|
|
3934
4207
|
) {
|
|
3935
4208
|
const integrationSrc = readFileSync(integrationTestPath, 'utf8');
|
|
3936
4209
|
|
|
@@ -4018,7 +4291,7 @@ export async function externalVerification(
|
|
|
4018
4291
|
})
|
|
4019
4292
|
.join(', ');
|
|
4020
4293
|
failures.push(
|
|
4021
|
-
`${coverage.unchained.length} producer-sourced token param(s) lack a CHAINED \`param:<name>\` test that mints a FRESH value from the producer tool: ${details}. Each test must call runWorkflowWithLadder on the named producer's \`workflow.json\`, read the named field from its result, and pass THAT value (not the recorded constant) into this tool — then assert the response is non-empty. If
|
|
4294
|
+
`${coverage.unchained.length} producer-sourced token param(s) lack a CHAINED \`param:<name>\` test that mints a FRESH value from the producer tool: ${details}. Each test must call runWorkflowWithLadder on the named producer's \`workflow.json\`, read the named field from its result, and pass THAT value (not the recorded constant) into this tool — then assert the response is non-empty. If live producer results legitimately cannot supply the token, keep the consumer explicitly unverified with an \`// exposed-but-not-verified: <param> ...\` annotation and document the producer/consumer limitation instead of fabricating it. See prompts/compile-agent.md "Producer-sourced token parameters".`,
|
|
4022
4295
|
);
|
|
4023
4296
|
}
|
|
4024
4297
|
}
|
|
@@ -4044,9 +4317,9 @@ export async function externalVerification(
|
|
|
4044
4317
|
missing.length === 1 ? 'it' : 'them'
|
|
4045
4318
|
} as an input token, but parser.ts does not emit ${
|
|
4046
4319
|
missing.length === 1 ? 'that field' : 'those fields'
|
|
4047
|
-
}.
|
|
4320
|
+
}. Reference ${
|
|
4048
4321
|
missing.length === 1 ? 'it' : 'each'
|
|
4049
|
-
}
|
|
4322
|
+
} under the EXACT field name and emit the full value whenever live evidence supplies it; null/absent values are allowed only for useful records whose missing output and affected consumers are documented in workflow.limitations — see read_build_plan "emitsTokens".`,
|
|
4050
4323
|
);
|
|
4051
4324
|
}
|
|
4052
4325
|
|
|
@@ -4146,5 +4419,5 @@ export async function externalVerification(
|
|
|
4146
4419
|
}
|
|
4147
4420
|
}
|
|
4148
4421
|
|
|
4149
|
-
return { failures, warnings, paramVerification, liveVerification };
|
|
4422
|
+
return { failures, warnings, paramVerification, liveVerification, integrationEvidence };
|
|
4150
4423
|
}
|