imprint-mcp 0.6.1 → 0.6.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -11,7 +11,9 @@ import {
11
11
  existsSync,
12
12
  mkdirSync,
13
13
  readFileSync,
14
+ readdirSync,
14
15
  realpathSync,
16
+ statSync,
15
17
  unlinkSync,
16
18
  writeFileSync,
17
19
  } from 'node:fs';
@@ -27,6 +29,11 @@ import {
27
29
  readBuildPlanFile,
28
30
  resolveAssignedModules,
29
31
  } from './build-plan.ts';
32
+ import {
33
+ LIVE_EVIDENCE_PATH_ENV,
34
+ type LiveIntegrationEvidence,
35
+ readLiveIntegrationEvidence,
36
+ } from './compile-verification.ts';
30
37
  import { splitSetCookieHeader } from './cookie-jar.ts';
31
38
  import { isSameRegistrableDomain, registrableDomain } from './etld.ts';
32
39
  import {
@@ -50,6 +57,7 @@ import {
50
57
  } from './types.ts';
51
58
 
52
59
  const REPO_ROOT = pathJoin(import.meta.dir, '..', '..');
60
+ const MAX_SHARED_MODULE_SOURCE_CHARS = 50_000;
53
61
 
54
62
  // Env var read by the agent-written parser.test.ts to locate the redacted
55
63
  // session. The test loads it, finds the load-bearing request seq, and feeds
@@ -80,7 +88,7 @@ export function buildCompileTools(
80
88
  ? { IMPRINT_TEACH_CREDENTIALS: JSON.stringify(context.teachCredentials) }
81
89
  : undefined;
82
90
  const tools = [
83
- buildReadSessionSummaryTool(session, context),
91
+ buildReadSessionSummaryTool(session, context, toolDir),
84
92
  buildReadRequestTool(session),
85
93
  buildSearchRequestsTool(session),
86
94
  buildRevealRequestTool(sessionPath),
@@ -115,6 +123,10 @@ export interface CompileToolContext {
115
123
  buildPlanPath?: string;
116
124
  /** Shared-module build manifest (verified flags) for this site. */
117
125
  sharedModules?: SharedModuleManifestEntry[];
126
+ /** A bounded teach resume is revising an already-generated tool. Surface the
127
+ * existing artifact and durable verifier/audit feedback to the agent so it
128
+ * preserves proven behavior instead of re-deriving the tool from raw capture. */
129
+ revisionMode?: boolean;
118
130
  }
119
131
 
120
132
  // ─── Tool: read_build_plan ───────────────────────────────────────────────────
@@ -149,11 +161,11 @@ function buildReadBuildPlanTool(
149
161
  ];
150
162
  if (emitsTokens.length > 0) {
151
163
  tokenNotes.push(
152
- `PRODUCER CONTRACT: your parser MUST emit ${emitsTokens
164
+ `PRODUCER COMPATIBILITY TARGET: emit ${emitsTokens
153
165
  .map((e) => `\`${e.field}\``)
154
166
  .join(
155
167
  ', ',
156
- )} in each result item, in the exact shape described (the FULL value a sibling consumer needs — never a bare fragment). Sibling tools mint their input from these fields; the verifier fails this tool if a declared field is missing from the parser output.`,
168
+ )} in the exact shape described whenever a live result supplies or proves the full value a sibling consumer needs. Never fabricate a missing token. Useful core records may retain a null/absent token when workflow.limitations names the missing output and affected consumers; preserve their supported input classes. A missing downstream token alone is not a reason to discard valid producer records.`,
157
169
  );
158
170
  }
159
171
  for (const tp of tokenParams) {
@@ -164,7 +176,7 @@ function buildReadBuildPlanTool(
164
176
  ? ` The producer declares shape: ${producerShape}. If that shape contains requiredKeys=[...], request-transform.ts must decode/unpack the fresh value and reference each required key in the outgoing request.`
165
177
  : '';
166
178
  tokenNotes.push(
167
- `CONSUMER CONTRACT: param \`${tp.param}\` is an opaque token minted by the \`${tp.sourceTool}\` tool's \`${tp.sourceField}\` output.${shapeNote} Write a CHAINED \`param:${tp.param}\` integration test that calls \`runWorkflowWithLadder\` on \`../${tp.sourceTool}/workflow.json\` directly inside the \`param:${tp.param}\` test block, reads \`${tp.sourceField}\` from the producer's ACTUAL output shape (top-level field, or an item in any returned collection such as flights/items/results), and passes THAT fresh value (not the recorded constant) into this tool — then asserts the response is non-empty. When several params come from the same producer, mint once and choose one producer item containing all sibling fields so the values stay from the same result. On producer bot/infra error, rethrow so the suite waives; if the producer ran but the field is absent, fix the producer parser/output contract instead of waiving. Do not fabricate a fallback value or mutate producer.result.data to make the test pass.`,
179
+ `CONSUMER CONTRACT: param \`${tp.param}\` is an opaque token minted by the \`${tp.sourceTool}\` tool's \`${tp.sourceField}\` output.${shapeNote} Write a CHAINED \`param:${tp.param}\` integration test that calls \`runCapturedIntegrationCase\` on \`../${tp.sourceTool}/workflow.json\` directly inside the \`param:${tp.param}\` test block, reads \`${tp.sourceField}\` from the producer's ACTUAL output shape (top-level field, or an item in any returned collection such as flights/items/results), and passes THAT fresh value (not the recorded constant) into this tool — then asserts the response is non-empty. When several params come from the same producer, mint once and choose one producer item containing all sibling fields so the values stay from the same result. On producer bot/infra error, rethrow so the suite waives. If useful producer records legitimately lack the field, do not fabricate or mutate output: keep this consumer verified:false, waived-chain, or inconclusive with diagnostics, and require an honest producer limitation.`,
168
180
  );
169
181
  }
170
182
  // Per-source wiring guidance for the general dependency contract. These are
@@ -201,6 +213,7 @@ function buildReadBuildPlanTool(
201
213
  kind: m.kind,
202
214
  purpose: m.purpose,
203
215
  exportSignatures: m.exportSignatures,
216
+ ...readAssignedSharedModuleSource(buildPlanPath, m.path),
204
217
  })),
205
218
  parserGuidance: slice.tool.parserGuidance,
206
219
  paramChecklist: slice.tool.paramChecklist,
@@ -227,11 +240,36 @@ function buildReadBuildPlanTool(
227
240
  };
228
241
  }
229
242
 
243
+ function readAssignedSharedModuleSource(
244
+ buildPlanPath: string,
245
+ modulePath: string,
246
+ ): { source?: string; sourceTruncated?: boolean; sourceUnavailable?: string } {
247
+ const siteDir = realpathSync(dirname(buildPlanPath));
248
+ const candidate = pathJoin(siteDir, modulePath);
249
+ if (!existsSync(candidate)) {
250
+ return { sourceUnavailable: 'The verified shared module file is missing.' };
251
+ }
252
+
253
+ const resolved = realpathSync(candidate);
254
+ const relative = pathRelative(siteDir, resolved);
255
+ if (relative.startsWith('..') || relative === '') {
256
+ return { sourceUnavailable: 'The shared module resolved outside the site directory.' };
257
+ }
258
+
259
+ const source = readFileSync(resolved, 'utf8');
260
+ if (source.length <= MAX_SHARED_MODULE_SOURCE_CHARS) return { source };
261
+ return {
262
+ source: source.slice(0, MAX_SHARED_MODULE_SOURCE_CHARS),
263
+ sourceTruncated: true,
264
+ };
265
+ }
266
+
230
267
  // ─── Tool: read_session_summary ──────────────────────────────────────────────
231
268
 
232
269
  export function buildReadSessionSummaryTool(
233
270
  session: Session,
234
271
  context: CompileToolContext,
272
+ toolDir?: string,
235
273
  ): AgentTool {
236
274
  return {
237
275
  name: 'read_session_summary',
@@ -343,6 +381,10 @@ export function buildReadSessionSummaryTool(
343
381
  captureHints: captureHints.length > 0 ? captureHints : undefined,
344
382
  paramGroundingHints: paramGroundingHints.length > 0 ? paramGroundingHints : undefined,
345
383
  inputProvenanceHints: inputProvenanceHints.length > 0 ? inputProvenanceHints : undefined,
384
+ revisionContext:
385
+ context.revisionMode && toolDir
386
+ ? buildExistingArtifactRevisionContext(toolDir)
387
+ : undefined,
346
388
  loadBearingRequests,
347
389
  };
348
390
 
@@ -361,6 +403,65 @@ export function buildReadSessionSummaryTool(
361
403
  };
362
404
  }
363
405
 
406
+ const REVISION_ARTIFACT_NAMES = [
407
+ 'workflow.json',
408
+ 'parser.ts',
409
+ 'request-transform.ts',
410
+ 'integration.test.ts',
411
+ 'playbook.yaml',
412
+ ] as const;
413
+
414
+ const REVISION_DIAGNOSTIC_NAMES = [
415
+ '.live-verification.json',
416
+ '.live-verification-evidence.json',
417
+ '.live-verifier-log.jsonl',
418
+ ] as const;
419
+
420
+ /** Describe, but do not interpret or gate on, prior compile evidence. The
421
+ * compile agent owns the judgment: it can read any listed file through the
422
+ * existing read_file tool and decide what to preserve, repair, narrow, or
423
+ * document. This is intentionally agent context rather than another runtime
424
+ * validator. */
425
+ function buildExistingArtifactRevisionContext(toolDir: string): Record<string, unknown> {
426
+ const describe = (name: string): { path: string; bytes: number } | null => {
427
+ const absolute = pathJoin(toolDir, name);
428
+ if (!existsSync(absolute)) return null;
429
+ try {
430
+ return { path: name, bytes: statSync(absolute).size };
431
+ } catch {
432
+ return null;
433
+ }
434
+ };
435
+
436
+ const artifacts = REVISION_ARTIFACT_NAMES.map(describe).filter(
437
+ (entry): entry is { path: string; bytes: number } => entry !== null,
438
+ );
439
+ const diagnostics = REVISION_DIAGNOSTIC_NAMES.map(describe).filter(
440
+ (entry): entry is { path: string; bytes: number } => entry !== null,
441
+ );
442
+ const notesDir = pathJoin(toolDir, 'notes');
443
+ const feedbackNotes = existsSync(notesDir)
444
+ ? readdirSync(notesDir)
445
+ .filter((name) => name.endsWith('.md'))
446
+ .sort()
447
+ .map((name) => describe(pathJoin('notes', name)))
448
+ .filter((entry): entry is { path: string; bytes: number } => entry !== null)
449
+ : [];
450
+
451
+ return {
452
+ mode: 'revise_existing_artifact',
453
+ instruction: [
454
+ 'This bounded resume is a revision, not a from-scratch compile.',
455
+ 'Read the existing workflow, parser, request transform, integration test, durable live-verification diagnostics, and feedback notes before re-reading raw recording bodies.',
456
+ 'Preserve behavior that live evidence proves works. Repair or narrow only the behavior contradicted by evidence.',
457
+ 'If a secondary branch cannot be grounded, remove its dependent parameters and claims together and record the reasoning in workflow limitations; do not guess.',
458
+ ].join(' '),
459
+ existingArtifacts: artifacts,
460
+ durableDiagnostics: diagnostics,
461
+ feedbackNotes,
462
+ };
463
+ }
464
+
364
465
  // ─── Inline request/response data for candidate-scoped requests ─────────────
365
466
 
366
467
  // claude-cli truncates tool results > ~40K chars. Keep the total summary
@@ -1345,6 +1446,7 @@ export async function runCommand(
1345
1446
  cwd: string,
1346
1447
  timeoutMs: number,
1347
1448
  extraEnv?: Record<string, string>,
1449
+ onOutput?: (stream: 'stdout' | 'stderr', chunk: string) => void,
1348
1450
  ): Promise<{ result: string; isError?: boolean }> {
1349
1451
  return new Promise((resolve) => {
1350
1452
  // `detached: true` makes the child its own process-group leader so a timeout
@@ -1362,11 +1464,15 @@ export async function runCommand(
1362
1464
  const TRUNCATE_LIMIT = 16 * 1024; // 16KB
1363
1465
 
1364
1466
  proc.stdout.on('data', (chunk) => {
1365
- stdout += chunk.toString();
1467
+ const text = chunk.toString();
1468
+ stdout += text;
1469
+ onOutput?.('stdout', text);
1366
1470
  });
1367
1471
 
1368
1472
  proc.stderr.on('data', (chunk) => {
1369
- stderr += chunk.toString();
1473
+ const text = chunk.toString();
1474
+ stderr += text;
1475
+ onOutput?.('stderr', text);
1370
1476
  });
1371
1477
 
1372
1478
  const timeout = setTimeout(() => {
@@ -1716,7 +1822,7 @@ interface BunTestRun {
1716
1822
 
1717
1823
  /** Per-exposed-parameter verification outcome. `verified` is true only when a
1718
1824
  * `param:<name>` integration test actually ran green against live data. */
1719
- interface ParamVerification {
1825
+ export interface ParamVerification {
1720
1826
  name: string;
1721
1827
  verified: boolean;
1722
1828
  /** Why an exposed param is unverified. Undefined when `verified` is true.
@@ -1727,7 +1833,7 @@ interface ParamVerification {
1727
1833
  * - `waived-chain`: the param is a producer-sourced token but the producer
1728
1834
  * tool could not be run at compile time (anti-bot / not compiled), so the
1729
1835
  * chain could not be verified. */
1730
- reason?: 'waived-bot' | 'waived-infra' | 'annotated' | 'waived-chain';
1836
+ reason?: 'waived-bot' | 'waived-infra' | 'annotated' | 'waived-chain' | 'semantic-gap';
1731
1837
  /** For a producer-sourced token param, the sibling tool + output field its
1732
1838
  * value comes from. Stamped into workflow.json (`param.sourcedFrom`) so the
1733
1839
  * MCP description tells the orchestrating LLM where to mint it and the audit
@@ -1751,11 +1857,15 @@ interface TokenSource {
1751
1857
  * bot-defense / infra detection and error surfacing) and the per-test pass/fail
1752
1858
  * names via a JUnit report written to a transient file in the tool dir.
1753
1859
  */
1754
- async function runBunTestWithResults(
1860
+ export async function runBunTestWithResults(
1755
1861
  testPath: string,
1756
1862
  toolDir: string,
1757
1863
  timeoutMs: number,
1758
1864
  env: Record<string, string> = {},
1865
+ opts: {
1866
+ bail?: boolean;
1867
+ onOutput?: (stream: 'stdout' | 'stderr', chunk: string) => void;
1868
+ } = {},
1759
1869
  ): Promise<BunTestRun> {
1760
1870
  const junitPath = pathJoin(toolDir, `.imprint-junit-${basename(testPath)}.xml`);
1761
1871
  try {
@@ -1764,10 +1874,11 @@ async function runBunTestWithResults(
1764
1874
  // best-effort
1765
1875
  }
1766
1876
  const result = await runCommand(
1767
- `bun test ${testPath} --reporter=junit --reporter-outfile=${junitPath}`,
1877
+ `bun test ${testPath} --reporter=junit --reporter-outfile=${junitPath}${opts.bail ? ' --bail=1' : ''}`,
1768
1878
  toolDir,
1769
1879
  timeoutMs,
1770
1880
  env,
1881
+ opts.onOutput,
1771
1882
  );
1772
1883
  const output = JSON.parse(result.result) as {
1773
1884
  stdout: string;
@@ -1802,20 +1913,121 @@ interface TestBlock {
1802
1913
  body: string;
1803
1914
  }
1804
1915
 
1805
- /** Split a test file into `test(...)` / `it(...)` blocks (title + source from
1806
- * that test's start to the next test's start). Good enough to check whether a
1807
- * named per-parameter test's body actually calls the workflow. */
1916
+ function matchingClosingBrace(source: string, openIndex: number): number | null {
1917
+ let depth = 0;
1918
+ let quote: "'" | '"' | '`' | null = null;
1919
+ let escaped = false;
1920
+ let lineComment = false;
1921
+ let blockComment = false;
1922
+ for (let index = openIndex; index < source.length; index++) {
1923
+ const char = source[index] ?? '';
1924
+ const next = source[index + 1] ?? '';
1925
+ if (lineComment) {
1926
+ if (char === '\n') lineComment = false;
1927
+ continue;
1928
+ }
1929
+ if (blockComment) {
1930
+ if (char === '*' && next === '/') {
1931
+ blockComment = false;
1932
+ index++;
1933
+ }
1934
+ continue;
1935
+ }
1936
+ if (quote) {
1937
+ if (escaped) {
1938
+ escaped = false;
1939
+ } else if (char === '\\') {
1940
+ escaped = true;
1941
+ } else if (char === quote) {
1942
+ quote = null;
1943
+ }
1944
+ continue;
1945
+ }
1946
+ if (char === '/' && next === '/') {
1947
+ lineComment = true;
1948
+ index++;
1949
+ continue;
1950
+ }
1951
+ if (char === '/' && next === '*') {
1952
+ blockComment = true;
1953
+ index++;
1954
+ continue;
1955
+ }
1956
+ if (char === "'" || char === '"' || char === '`') {
1957
+ quote = char;
1958
+ continue;
1959
+ }
1960
+ if (char === '{') depth++;
1961
+ if (char === '}' && --depth === 0) return index;
1962
+ }
1963
+ return null;
1964
+ }
1965
+
1966
+ function hasCapturedLiveWorkflowCall(source: string): boolean {
1967
+ return /\b(?:runWorkflowWithLadder|runCapturedIntegrationCase)\s*\(/.test(source);
1968
+ }
1969
+
1970
+ /** Resolve small local function wrappers around a captured workflow call. The
1971
+ * generated suites commonly factor one invocation helper above their tests;
1972
+ * keeping the resolution local prevents a call in an unrelated param test from
1973
+ * laundering a synthetic baseline elsewhere in the file. */
1974
+ function capturedWorkflowHelperNames(source: string): Set<string> {
1975
+ const functionRe = /\b(?:async\s+)?function\s+([A-Za-z_$][\w$]*)\s*\([^)]*\)\s*\{/g;
1976
+ const functions = [...source.matchAll(functionRe)].map((match) => ({
1977
+ name: match[1] ?? '',
1978
+ index: match.index ?? 0,
1979
+ }));
1980
+ const testStarts = [...source.matchAll(/\b(?:test|it)\s*\(/g)].map((match) => match.index ?? 0);
1981
+ const boundaries = [...functions.map((entry) => entry.index), ...testStarts, source.length].sort(
1982
+ (a, b) => a - b,
1983
+ );
1984
+ const bodies = new Map<string, string>();
1985
+ for (const entry of functions) {
1986
+ const end = boundaries.find((boundary) => boundary > entry.index) ?? source.length;
1987
+ bodies.set(entry.name, source.slice(entry.index, end));
1988
+ }
1989
+ const captured = new Set<string>();
1990
+ let changed = true;
1991
+ while (changed) {
1992
+ changed = false;
1993
+ for (const [name, body] of bodies) {
1994
+ if (captured.has(name)) continue;
1995
+ const callsCapturedHelper = [...captured].some((helper) =>
1996
+ new RegExp(`\\b${helper.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\s*\\(`).test(body),
1997
+ );
1998
+ if (hasCapturedLiveWorkflowCall(body) || callsCapturedHelper) {
1999
+ captured.add(name);
2000
+ changed = true;
2001
+ }
2002
+ }
2003
+ }
2004
+ return captured;
2005
+ }
2006
+
2007
+ /** Split a test file into `test(...)` / `it(...)` blocks (title + callback
2008
+ * source). Good enough to check whether a named per-parameter test's body
2009
+ * actually calls the workflow. */
1808
2010
  export function extractTestBlocks(src: string): TestBlock[] {
1809
2011
  const re = /\b(?:test|it)\s*\(\s*(['"`])((?:\\.|(?!\1).)*)\1/g;
1810
- const starts: Array<{ index: number; title: string }> = [];
2012
+ const starts: Array<{ index: number; title: string; afterTitle: number }> = [];
1811
2013
  for (const m of src.matchAll(re)) {
1812
- starts.push({ index: m.index ?? 0, title: m[2] ?? '' });
2014
+ starts.push({
2015
+ index: m.index ?? 0,
2016
+ title: m[2] ?? '',
2017
+ afterTitle: (m.index ?? 0) + m[0].length,
2018
+ });
1813
2019
  }
1814
2020
  const blocks: TestBlock[] = [];
1815
2021
  for (let i = 0; i < starts.length; i++) {
1816
2022
  const start = starts[i];
1817
2023
  if (!start) continue;
1818
- const end = i + 1 < starts.length ? (starts[i + 1]?.index ?? src.length) : src.length;
2024
+ const fallbackEnd = i + 1 < starts.length ? (starts[i + 1]?.index ?? src.length) : src.length;
2025
+ const callbackOpen = src.indexOf('{', start.afterTitle);
2026
+ const callbackClose =
2027
+ callbackOpen >= 0 && callbackOpen < fallbackEnd
2028
+ ? matchingClosingBrace(src, callbackOpen)
2029
+ : null;
2030
+ const end = callbackClose === null ? fallbackEnd : callbackClose + 1;
1819
2031
  blocks.push({ title: start.title, body: src.slice(start.index, end) });
1820
2032
  }
1821
2033
  return blocks;
@@ -2198,6 +2410,7 @@ export function classifyIntegrationOutcome(input: {
2198
2410
  * - chained pass → `{ verified: true, sourcedFrom }`
2199
2411
  * - passed but not chained (the recorded-value tautology) → `unchained` (blocking)
2200
2412
  * - suite waived (producer anti-bot) → `{ verified: false, reason: 'waived-chain' }`
2413
+ * - explicitly annotated unavailable producer output → `{ verified: false, reason: 'waived-chain' }`
2201
2414
  * - else → `unchained` (blocking)
2202
2415
  */
2203
2416
  export function classifyParamCoverage(opts: {
@@ -2232,6 +2445,10 @@ export function classifyParamCoverage(opts: {
2232
2445
  const token = `param:${lp.name}`;
2233
2446
  const passedLive = [...opts.passedTests].some((n) => n.includes(token));
2234
2447
  const block = blocks.find((b) => b.title.includes(token));
2448
+ const annotationRe = new RegExp(
2449
+ `//\\s*exposed-but-not-verified[^\\n]*\\b${lp.name.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\b`,
2450
+ );
2451
+ const isAnnotated = annotationRe.test(opts.integrationSrc);
2235
2452
 
2236
2453
  // Producer-sourced token param: requires a chained test that mints a fresh
2237
2454
  // value from the producer's sibling workflow.
@@ -2241,7 +2458,7 @@ export function classifyParamCoverage(opts: {
2241
2458
  if (passedLive) {
2242
2459
  const chained =
2243
2460
  !!block &&
2244
- /runWorkflowWithLadder\s*\(/.test(block.body) &&
2461
+ hasCapturedLiveWorkflowCall(block.body) &&
2245
2462
  SIBLING_WORKFLOW_RE.test(block.body);
2246
2463
  if (chained) {
2247
2464
  paramVerification.push({ name: lp.name, verified: true, sourcedFrom });
@@ -2255,21 +2472,28 @@ export function classifyParamCoverage(opts: {
2255
2472
  reason: 'waived-chain',
2256
2473
  sourcedFrom,
2257
2474
  });
2475
+ } else if (isAnnotated) {
2476
+ // The compile agent may explicitly preserve a limited consumer when
2477
+ // its producer returns useful core records but cannot mint this token.
2478
+ // The durable verified:false+sourcedFrom stamp keeps orchestration
2479
+ // honest; opaque defaults and passing recorded-token tests remain
2480
+ // blocked by the independent guards above and below.
2481
+ paramVerification.push({
2482
+ name: lp.name,
2483
+ verified: false,
2484
+ reason: 'waived-chain',
2485
+ sourcedFrom,
2486
+ });
2258
2487
  } else {
2259
2488
  unchained.push(lp.name);
2260
2489
  }
2261
2490
  continue;
2262
2491
  }
2263
2492
 
2264
- const annotationRe = new RegExp(
2265
- `//\\s*exposed-but-not-verified[^\\n]*\\b${lp.name.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\b`,
2266
- );
2267
- const isAnnotated = annotationRe.test(opts.integrationSrc);
2268
-
2269
2493
  if (passedLive) {
2270
2494
  // Anti-tautology: a passing per-param test must actually exercise the live
2271
2495
  // workflow, not assert a constant.
2272
- if (block && !/runWorkflowWithLadder\s*\(/.test(block.body)) {
2496
+ if (block && !hasCapturedLiveWorkflowCall(block.body)) {
2273
2497
  tautological.push(lp.name);
2274
2498
  } else {
2275
2499
  paramVerification.push({ name: lp.name, verified: true });
@@ -2299,13 +2523,20 @@ export function hasLiveBaselineWorkflowTest(integrationSrc: string): boolean {
2299
2523
  const throwsOnFailure = /if\s*\(\s*!result\.ok\s*\)\s*\{[\s\S]{0,500}\bthrow\b/;
2300
2524
  const returnsOnFailedResult =
2301
2525
  /if\s*\([^)]*![^)]*\.result\.ok[\s\S]{0,200}\)\s*(?:\{[\s\S]{0,300}\breturn\b|return\b)/;
2302
- return extractTestBlocks(integrationSrc).some(
2303
- (block) =>
2526
+ const capturedHelpers = capturedWorkflowHelperNames(integrationSrc);
2527
+ return extractTestBlocks(integrationSrc).some((block) => {
2528
+ const invokesCapturedWorkflow =
2529
+ hasCapturedLiveWorkflowCall(block.body) ||
2530
+ [...capturedHelpers].some((helper) =>
2531
+ new RegExp(`\\b${helper.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\s*\\(`).test(block.body),
2532
+ );
2533
+ return (
2304
2534
  !block.title.includes('param:') &&
2305
- /\brunWorkflowWithLadder\s*\(/.test(block.body) &&
2535
+ invokesCapturedWorkflow &&
2306
2536
  !returnsOnFailedResult.test(block.body) &&
2307
- (requiresOk.test(block.body) || throwsOnFailure.test(block.body)),
2308
- );
2537
+ (requiresOk.test(block.body) || throwsOnFailure.test(block.body))
2538
+ );
2539
+ });
2309
2540
  }
2310
2541
 
2311
2542
  /**
@@ -3427,6 +3658,10 @@ export async function externalVerification(
3427
3658
  * verifier blocks invented ${credential.X} placeholders that are not in this
3428
3659
  * set or in requiredInputs. */
3429
3660
  credentialNames?: string[];
3661
+ /** Agentic compile path only: validate the integration suite's source here,
3662
+ * but let the independent semantic verifier invoke the final live suite as
3663
+ * a tool. Parser/type/schema checks remain deterministic. */
3664
+ deferLiveIntegrationToSemanticAgent?: boolean;
3430
3665
  } = {},
3431
3666
  ): Promise<{
3432
3667
  failures: string[];
@@ -3440,9 +3675,13 @@ export async function externalVerification(
3440
3675
  firstError: string;
3441
3676
  exhaustedBackends: string[];
3442
3677
  };
3678
+ /** Actual tool-level inputs and parsed results captured by the single live
3679
+ * integration run. The caller hands these to the independent semantic agent. */
3680
+ integrationEvidence: LiveIntegrationEvidence[];
3443
3681
  }> {
3444
3682
  const failures: string[] = [];
3445
3683
  const warnings: string[] = [];
3684
+ let integrationEvidence: LiveIntegrationEvidence[] = [];
3446
3685
  const paramVerification: ParamVerification[] = [];
3447
3686
  let liveVerification:
3448
3687
  | { kind: 'waived-bot' | 'waived-infra'; firstError: string; exhaustedBackends: string[] }
@@ -3647,6 +3886,7 @@ export async function externalVerification(
3647
3886
 
3648
3887
  const notTemplated: string[] = [];
3649
3888
  const inventedOnly: string[] = [];
3889
+ const intentionallyOmitted: Array<{ name: string; feature: string; reason: string }> = [];
3650
3890
 
3651
3891
  for (const lp of opts.likelyParams) {
3652
3892
  const placeholder = `\${param.${lp.name}}`;
@@ -3684,7 +3924,19 @@ export async function externalVerification(
3684
3924
  }
3685
3925
 
3686
3926
  if (!inBody && !inHeader && !inOriginalQuery && !inInventedQuery) {
3687
- notTemplated.push(lp.name);
3927
+ const limitation = workflow.limitations?.find((entry) =>
3928
+ entry.omittedParameters?.includes(lp.name),
3929
+ );
3930
+ const stillDeclared = workflow.parameters.some((param) => param.name === lp.name);
3931
+ if (limitation && !stillDeclared) {
3932
+ intentionallyOmitted.push({
3933
+ name: lp.name,
3934
+ feature: limitation.feature,
3935
+ reason: limitation.reason,
3936
+ });
3937
+ } else {
3938
+ notTemplated.push(lp.name);
3939
+ }
3688
3940
  } else if (!inBody && !inHeader && !inOriginalQuery && inInventedQuery) {
3689
3941
  inventedOnly.push(lp.name);
3690
3942
  }
@@ -3692,7 +3944,7 @@ export async function externalVerification(
3692
3944
 
3693
3945
  if (notTemplated.length > 0) {
3694
3946
  failures.push(
3695
- `${notTemplated.length} likelyParam(s) are not templated in any request: ${notTemplated.join(', ')}. Each must appear as \${param.NAME} in a request URL, body, or header. For parameters recorded as null or [] (filters the user toggled but didn\'t apply), find the correct position in the request body and replace the placeholder value with \${param.NAME}.`,
3947
+ `${notTemplated.length} likelyParam(s) have no valid disposition: ${notTemplated.join(', ')}. Each must either appear as \${param.NAME} in a request URL, body, or header, or be removed from the public parameters list and named in workflow.limitations[].omittedParameters with an evidence-based reason. For parameters recorded as null or [] (filters the user toggled but didn\'t apply), find the correct position in the request body and replace the placeholder value with \${param.NAME}.`,
3696
3948
  );
3697
3949
  }
3698
3950
  if (inventedOnly.length > 0) {
@@ -3700,6 +3952,13 @@ export async function externalVerification(
3700
3952
  `${inventedOnly.length} likelyParam(s) are templated only in URL query params that do not exist in any recorded request URL: ${inventedOnly.join(', ')}. The API server likely ignores these invented params — wire them into the request body or an existing query param instead. For complex body formats, use a requestTransformModule to construct the body programmatically.`,
3701
3953
  );
3702
3954
  }
3955
+ if (intentionallyOmitted.length > 0) {
3956
+ warnings.push(
3957
+ `${intentionallyOmitted.length} detector-suggested parameter(s) were intentionally omitted by the compile agent: ${intentionallyOmitted
3958
+ .map(({ name, feature, reason }) => `${name} (${feature}: ${reason})`)
3959
+ .join('; ')}. The limitations are retained in workflow.json and surfaced to callers.`,
3960
+ );
3961
+ }
3703
3962
  }
3704
3963
  } catch (err) {
3705
3964
  failures.push(
@@ -3816,8 +4075,13 @@ export async function externalVerification(
3816
4075
  if (!hasLiveBaselineWorkflowTest(integrationSrc)) {
3817
4076
  integrationOutcome = 'failed';
3818
4077
  failures.push(
3819
- 'integration.test.ts has no strict baseline live workflow test: add a non-`param:` test that calls `runWorkflowWithLadder(...)` on this tool workflow, fails when `result.ok` is false, and asserts the successful response contains real data. Static workflow-shape assertions or green returns on bot/infra errors are not enough to verify a generated tool.',
4078
+ 'integration.test.ts has no strict baseline live workflow test: add a non-`param:` test that calls `runCapturedIntegrationCase(...)` on this tool workflow, fails when `result.ok` is false, and asserts the successful response contains real data. Static workflow-shape assertions or green returns on bot/infra errors are not enough to verify a generated tool.',
3820
4079
  );
4080
+ } else if (opts.deferLiveIntegrationToSemanticAgent) {
4081
+ // The independent verifier owns the final live execution. Keeping the
4082
+ // suite out of this deterministic gate avoids two disconnected verdicts
4083
+ // and lets the reviewer inspect the exact outputs it requested.
4084
+ integrationOutcome = 'absent';
3821
4085
  } else {
3822
4086
  // Scale the verifier's live-test timeout to the suite size: the baseline plus
3823
4087
  // one live `runWorkflowWithLadder` per param, each gated by the ~25s compile
@@ -3827,26 +4091,30 @@ export async function externalVerification(
3827
4091
  const paramCount = opts.likelyParams?.length ?? 0;
3828
4092
  const pacingMs = Number(process.env.IMPRINT_COMPILE_ACT_SPACING_MS ?? 25_000) || 0;
3829
4093
  const verifierTimeoutMs = Math.min(120_000 + paramCount * (pacingMs + 20_000), 10 * 60_000);
3830
- let run: BunTestRun = {
3831
- stdout: '',
3832
- stderr: '',
3833
- exitCode: 1,
3834
- timedOut: false,
3835
- passed: new Set(),
3836
- failed: new Set(),
3837
- };
3838
- for (let attempt = 0; attempt < 3; attempt++) {
3839
- run = await runBunTestWithResults(integrationTestPath, toolDir, verifierTimeoutMs);
3840
- if (run.exitCode === 0) break;
3841
- // A timeout, bot-defense, or ladder-exhaustion failure will NOT clear on a
3842
- // retry — re-running only fires more state-changing calls and deepens the
3843
- // per-IP rate flag. One attempt is enough to classify it; stop early.
3844
- if (run.timedOut) break;
3845
- const out = `${run.stdout}\n${run.stderr}`;
3846
- const ladderExhausted =
3847
- /\bRATE_LIMITED\b|\bFORBIDDEN\b|\bNETWORK\b/.test(out) &&
3848
- /non-escalatable|giving up|ladder exhausted|all backends failed/.test(out);
3849
- if (isBotDefenseFailure(out) || ladderExhausted) break;
4094
+ const evidencePath = pathJoin(
4095
+ toolDir,
4096
+ `.imprint-live-evidence-${process.pid}-${Date.now()}.jsonl`,
4097
+ );
4098
+ try {
4099
+ if (existsSync(evidencePath)) unlinkSync(evidencePath);
4100
+ } catch {
4101
+ // best-effort cleanup before the run
4102
+ }
4103
+ const run = await runBunTestWithResults(integrationTestPath, toolDir, verifierTimeoutMs, {
4104
+ [LIVE_EVIDENCE_PATH_ENV]: evidencePath,
4105
+ });
4106
+ try {
4107
+ integrationEvidence = readLiveIntegrationEvidence(evidencePath);
4108
+ } catch (error) {
4109
+ failures.push(
4110
+ `could not read captured live integration evidence: ${error instanceof Error ? error.message : String(error)}`,
4111
+ );
4112
+ } finally {
4113
+ try {
4114
+ if (existsSync(evidencePath)) unlinkSync(evidencePath);
4115
+ } catch {
4116
+ // Evidence is ephemeral and intentionally never kept with artifacts.
4117
+ }
3850
4118
  }
3851
4119
  integrationPassedTests = run.passed;
3852
4120
 
@@ -3862,7 +4130,11 @@ export async function externalVerification(
3862
4130
  integrationOutcome = verdict.outcome;
3863
4131
 
3864
4132
  if (verdict.outcome === 'passed') {
3865
- // exitCode 0 — nothing to surface.
4133
+ if (integrationEvidence.length === 0) {
4134
+ failures.push(
4135
+ 'integration.test.ts passed but captured no live evidence. Import `runCapturedIntegrationCase` from `imprint/compile-verification` and use it for every live workflow call so the independent verifier can inspect the actual tool inputs and parsed outputs.',
4136
+ );
4137
+ }
3866
4138
  } else if (verdict.captureFailName !== null) {
3867
4139
  const capName = verdict.captureFailName;
3868
4140
  // If the failing capture is a `response_header` on a REPLAYED workflow
@@ -3913,7 +4185,7 @@ export async function externalVerification(
3913
4185
  );
3914
4186
  } else {
3915
4187
  failures.push(
3916
- `bun test integration.test.ts exited ${run.exitCode} — the workflow failed to produce live data (tried 3 times).\nstdout:\n${run.stdout}\nstderr:\n${run.stderr}`,
4188
+ `bun test integration.test.ts exited ${run.exitCode} — the workflow failed to produce live data. The baseline suite runs exactly once per verification cycle.\nstdout:\n${run.stdout}\nstderr:\n${run.stderr}`,
3917
4189
  );
3918
4190
  }
3919
4191
  }
@@ -3930,7 +4202,8 @@ export async function externalVerification(
3930
4202
  !referencedStateBroken &&
3931
4203
  existsSync(integrationTestPath) &&
3932
4204
  opts.likelyParams &&
3933
- opts.likelyParams.length > 0
4205
+ opts.likelyParams.length > 0 &&
4206
+ !opts.deferLiveIntegrationToSemanticAgent
3934
4207
  ) {
3935
4208
  const integrationSrc = readFileSync(integrationTestPath, 'utf8');
3936
4209
 
@@ -4018,7 +4291,7 @@ export async function externalVerification(
4018
4291
  })
4019
4292
  .join(', ');
4020
4293
  failures.push(
4021
- `${coverage.unchained.length} producer-sourced token param(s) lack a CHAINED \`param:<name>\` test that mints a FRESH value from the producer tool: ${details}. Each test must call runWorkflowWithLadder on the named producer's \`workflow.json\`, read the named field from its result, and pass THAT value (not the recorded constant) into this tool — then assert the response is non-empty. If the producer only emits a bare fragment, fix the PRODUCER to emit the full value this tool consumes. See prompts/compile-agent.md "Producer-sourced token parameters".`,
4294
+ `${coverage.unchained.length} producer-sourced token param(s) lack a CHAINED \`param:<name>\` test that mints a FRESH value from the producer tool: ${details}. Each test must call runWorkflowWithLadder on the named producer's \`workflow.json\`, read the named field from its result, and pass THAT value (not the recorded constant) into this tool — then assert the response is non-empty. If live producer results legitimately cannot supply the token, keep the consumer explicitly unverified with an \`// exposed-but-not-verified: <param> ...\` annotation and document the producer/consumer limitation instead of fabricating it. See prompts/compile-agent.md "Producer-sourced token parameters".`,
4022
4295
  );
4023
4296
  }
4024
4297
  }
@@ -4044,9 +4317,9 @@ export async function externalVerification(
4044
4317
  missing.length === 1 ? 'it' : 'them'
4045
4318
  } as an input token, but parser.ts does not emit ${
4046
4319
  missing.length === 1 ? 'that field' : 'those fields'
4047
- }. Emit ${
4320
+ }. Reference ${
4048
4321
  missing.length === 1 ? 'it' : 'each'
4049
- } in every result item under the EXACT field name (the full value a consumer needs, never a bare fragment) — see read_build_plan "emitsTokens".`,
4322
+ } under the EXACT field name and emit the full value whenever live evidence supplies it; null/absent values are allowed only for useful records whose missing output and affected consumers are documented in workflow.limitations — see read_build_plan "emitsTokens".`,
4050
4323
  );
4051
4324
  }
4052
4325
 
@@ -4146,5 +4419,5 @@ export async function externalVerification(
4146
4419
  }
4147
4420
  }
4148
4421
 
4149
- return { failures, warnings, paramVerification, liveVerification };
4422
+ return { failures, warnings, paramVerification, liveVerification, integrationEvidence };
4150
4423
  }