imprint-mcp 0.6.5 → 0.6.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/prompts/audit-agent.md +3 -3
- package/prompts/auth-compile-agent.md +46 -4
- package/prompts/compile-agent.md +36 -12
- package/prompts/live-verifier-agent.md +15 -5
- package/prompts/request-triage.md +51 -9
- package/prompts/tool-candidate-detection.md +7 -1
- package/prompts/tool-planning.md +1 -1
- package/src/cli.ts +10 -4
- package/src/imprint/audit.ts +23 -5
- package/src/imprint/auth-compile-agent.ts +11 -7
- package/src/imprint/auth-compile-tools.ts +127 -18
- package/src/imprint/auth-verifier.ts +2 -1
- package/src/imprint/backend-ladder.ts +20 -0
- package/src/imprint/build-plan.ts +21 -0
- package/src/imprint/claude-cli-compile.ts +7 -1
- package/src/imprint/codex-cli-compile.ts +6 -0
- package/src/imprint/compile-agent.ts +50 -9
- package/src/imprint/compile-tools.ts +308 -25
- package/src/imprint/compile-verification.ts +11 -2
- package/src/imprint/compile.ts +298 -69
- package/src/imprint/effects.ts +9 -0
- package/src/imprint/live-verifier.ts +455 -34
- package/src/imprint/login.ts +30 -12
- package/src/imprint/mcp-compile-server.ts +56 -5
- package/src/imprint/mcp-live-verifier-server.ts +43 -10
- package/src/imprint/probe-backends.ts +6 -0
- package/src/imprint/recording-request.ts +7 -2
- package/src/imprint/replay-capture.ts +48 -29
- package/src/imprint/request-capture.ts +14 -0
- package/src/imprint/runtime.ts +102 -22
- package/src/imprint/teach-state.ts +6 -7
- package/src/imprint/teach.ts +342 -120
- package/src/imprint/tool-candidates.ts +17 -0
- package/src/imprint/tool-plan.ts +2 -0
- package/src/imprint/triage-selection.ts +44 -0
- package/src/imprint/types.ts +39 -3
package/src/imprint/audit.ts
CHANGED
|
@@ -21,6 +21,7 @@ import {
|
|
|
21
21
|
type AuthVerificationReceiptStatus,
|
|
22
22
|
readAuthVerificationReceiptStatus,
|
|
23
23
|
} from './auth-compile-tools.ts';
|
|
24
|
+
import { workflowHasIrreversibleEffect } from './effects.ts';
|
|
24
25
|
import { type ProviderName, preferredAgentModel } from './llm.ts';
|
|
25
26
|
import { createLog } from './log.ts';
|
|
26
27
|
import { imprintHomeDir } from './paths.ts';
|
|
@@ -129,6 +130,7 @@ interface AuditToolPartition {
|
|
|
129
130
|
eligible: ResolvedTool[];
|
|
130
131
|
nonInteractiveAuthNames: string[];
|
|
131
132
|
skippedInteractiveAuth: SkippedInteractiveAuth[];
|
|
133
|
+
skippedIrreversible: string[];
|
|
132
134
|
}
|
|
133
135
|
|
|
134
136
|
interface DetectedButUnemittedTool {
|
|
@@ -185,8 +187,13 @@ export function partitionAuditTools(tools: readonly ResolvedTool[]): AuditToolPa
|
|
|
185
187
|
const eligible: ResolvedTool[] = [];
|
|
186
188
|
const nonInteractiveAuthNames: string[] = [];
|
|
187
189
|
const skippedInteractiveAuth: SkippedInteractiveAuth[] = [];
|
|
190
|
+
const skippedIrreversible: string[] = [];
|
|
188
191
|
|
|
189
192
|
for (const tool of tools) {
|
|
193
|
+
if (workflowHasIrreversibleEffect(tool.workflow)) {
|
|
194
|
+
skippedIrreversible.push(tool.workflow.toolName);
|
|
195
|
+
continue;
|
|
196
|
+
}
|
|
190
197
|
// authConfig is the authoritative capability marker for legacy workflows
|
|
191
198
|
// where the newer optional toolKind field may be absent.
|
|
192
199
|
const auth = tool.workflow.authConfig;
|
|
@@ -229,7 +236,7 @@ export function partitionAuditTools(tools: readonly ResolvedTool[]): AuditToolPa
|
|
|
229
236
|
});
|
|
230
237
|
}
|
|
231
238
|
|
|
232
|
-
return { eligible, nonInteractiveAuthNames, skippedInteractiveAuth };
|
|
239
|
+
return { eligible, nonInteractiveAuthNames, skippedInteractiveAuth, skippedIrreversible };
|
|
233
240
|
}
|
|
234
241
|
|
|
235
242
|
/**
|
|
@@ -566,6 +573,7 @@ export async function runAudit(opts: RunAuditOptions): Promise<AuditScore> {
|
|
|
566
573
|
eligible: tools,
|
|
567
574
|
nonInteractiveAuthNames,
|
|
568
575
|
skippedInteractiveAuth,
|
|
576
|
+
skippedIrreversible,
|
|
569
577
|
} = partitionAuditTools(discoveredTools);
|
|
570
578
|
const auditedToolCount = tools.length;
|
|
571
579
|
const detectedButUnemitted = detectedButUnemittedTools(
|
|
@@ -603,6 +611,9 @@ export async function runAudit(opts: RunAuditOptions): Promise<AuditScore> {
|
|
|
603
611
|
`skipping interactive auth ${skipped.name}: ${skipped.reason}; receipt=${skipped.verificationStatus}`,
|
|
604
612
|
);
|
|
605
613
|
}
|
|
614
|
+
for (const name of skippedIrreversible) {
|
|
615
|
+
log(`skipping irreversible workflow ${name}: live audit is intentionally disabled`);
|
|
616
|
+
}
|
|
606
617
|
|
|
607
618
|
// Parameters that shipped live-unverified at compile time (Fix D). Tell the
|
|
608
619
|
// auditor to probe them especially — these are the most likely to be broken
|
|
@@ -648,7 +659,7 @@ export async function runAudit(opts: RunAuditOptions): Promise<AuditScore> {
|
|
|
648
659
|
tokenDeps,
|
|
649
660
|
});
|
|
650
661
|
if (auditedToolCount === 0) {
|
|
651
|
-
log('all discovered tools
|
|
662
|
+
log('all discovered tools are excluded from live audit; auditor not spawned');
|
|
652
663
|
}
|
|
653
664
|
recordLlmUsageSpan(
|
|
654
665
|
'audit.llm_usage',
|
|
@@ -747,6 +758,7 @@ export async function runAudit(opts: RunAuditOptions): Promise<AuditScore> {
|
|
|
747
758
|
'imprint.audit.detected_tool_count': detectedToolCount,
|
|
748
759
|
'imprint.audit.detected_but_unemitted_count': detectedButUnemitted.length,
|
|
749
760
|
'imprint.audit.skipped_interactive_auth_count': skippedInteractiveAuth.length,
|
|
761
|
+
'imprint.audit.skipped_irreversible_count': skippedIrreversible.length,
|
|
750
762
|
'imprint.audit.verdict': score.verdict,
|
|
751
763
|
'imprint.audit.unverified_and_ungradeable_count': unverifiedAndUngradeable.length,
|
|
752
764
|
'imprint.audit.missing_tool_count': missingToolNames.length,
|
|
@@ -766,6 +778,7 @@ export async function runAudit(opts: RunAuditOptions): Promise<AuditScore> {
|
|
|
766
778
|
detectedToolCount,
|
|
767
779
|
detectedButUnemittedTools: detectedButUnemitted,
|
|
768
780
|
skippedInteractiveAuth,
|
|
781
|
+
skippedIrreversible,
|
|
769
782
|
ungradeableTools: ungradeableNames,
|
|
770
783
|
/** Connected tools without an invocation record. A partial audit
|
|
771
784
|
* cannot pass merely because the auditor named an uncalled tool. */
|
|
@@ -808,6 +821,7 @@ export async function runAudit(opts: RunAuditOptions): Promise<AuditScore> {
|
|
|
808
821
|
report: drive.report,
|
|
809
822
|
auditedToolCount,
|
|
810
823
|
skippedInteractiveAuth,
|
|
824
|
+
skippedIrreversible,
|
|
811
825
|
detectedToolCount,
|
|
812
826
|
detectedButUnemittedTools: detectedButUnemitted,
|
|
813
827
|
});
|
|
@@ -1124,19 +1138,19 @@ export function buildUnverifiedParamNote(
|
|
|
1124
1138
|
.map((u) => `${u.tool}(${u.params.join(', ')})`)
|
|
1125
1139
|
.join(
|
|
1126
1140
|
'; ',
|
|
1127
|
-
)}. Give each one a \`parameters\` verdict (works / no_op / broken / untestable) like any other — do not let an unverified parameter pass without a differential test. Only skip probing when the tool is
|
|
1141
|
+
)}. Give each one a \`parameters\` verdict (works / no_op / broken / untestable) like any other — do not let an unverified parameter pass without a differential test. Only skip probing when the connected tool is state-changing, or when repeated paced attempts for that parameter stay blocked by infrastructure. Irreversible tools are excluded before this prompt is built. A bot-defended idempotent read is still testable after the warm browser session is established.`;
|
|
1128
1142
|
}
|
|
1129
1143
|
|
|
1130
1144
|
export function buildAuditInitialPrompt(opts: DriveAuditOptions, unverifiedNote: string): string {
|
|
1131
1145
|
const nonInteractiveAuthNames = opts.nonInteractiveAuthNames ?? [];
|
|
1132
1146
|
const authNote = nonInteractiveAuthNames.length
|
|
1133
|
-
? `\n\
|
|
1147
|
+
? `\n\nEligible unattended authentication tool(s): ${nonInteractiveAuthNames.join(', ')}. Apply the unattended authentication rules only to these tools, with at most one recovery authentication call total across this audit.`
|
|
1134
1148
|
: '\n\nNo authentication tool is eligible in this audit. If a DATA tool returns AUTH_EXPIRED, classify that invocation as infra/inconclusive evidence; do not attempt to authenticate.';
|
|
1135
1149
|
return `Audit every MCP tool connected to you for the site "${opts.site}".
|
|
1136
1150
|
|
|
1137
1151
|
There are ${opts.toolNames.length} connected tool(s). For each one: read its description and input schema, invoke it with a realistic parameter set, judge the result, and classify each invocation as correct | tool_broken | infra | bad_params per your system prompt.
|
|
1138
1152
|
|
|
1139
|
-
Parameter coverage is mandatory for idempotent read tools. A cold cdp-replay/browser-backed first call may be slow because it launches or warms a trusted browser session, but later calls usually reuse that warm session and are much cheaper. Do NOT use the first call's cold-start latency as a reason to skip parameter probes. For search/list/calendar/lookup/quote/read tools, keep making paced sequential differential calls until each advertised parameter has a verdict, unless the same parameter remains blocked by infrastructure after repeated paced retries. For
|
|
1153
|
+
Parameter coverage is mandatory for idempotent read tools. A cold cdp-replay/browser-backed first call may be slow because it launches or warms a trusted browser session, but later calls usually reuse that warm session and are much cheaper. Do NOT use the first call's cold-start latency as a reason to skip parameter probes. For search/list/calendar/lookup/quote/read tools, keep making paced sequential differential calls until each advertised parameter has a verdict, unless the same parameter remains blocked by infrastructure after repeated paced retries. For connected state-changing but reversible tools, make only one safe baseline call and mark parameters untestable with that state-changing reason. Irreversible workflows are never connected to this audit.
|
|
1140
1154
|
|
|
1141
1155
|
Promo/coupon/voucher/discount-code parameters require a known valid code to prove a visible effect. If neither the schema nor another tool output gives you a valid code, classify that parameter as untestable; do not invent a random code and call an unchanged response no_op.
|
|
1142
1156
|
|
|
@@ -1592,6 +1606,7 @@ function printSummary(
|
|
|
1592
1606
|
report: AuditReport;
|
|
1593
1607
|
auditedToolCount: number;
|
|
1594
1608
|
skippedInteractiveAuth: SkippedInteractiveAuth[];
|
|
1609
|
+
skippedIrreversible: string[];
|
|
1595
1610
|
detectedToolCount: number;
|
|
1596
1611
|
detectedButUnemittedTools: DetectedButUnemittedTool[];
|
|
1597
1612
|
},
|
|
@@ -1614,6 +1629,9 @@ function printSummary(
|
|
|
1614
1629
|
`[imprint] skipped interactive auth ${skipped.name} — ${skipped.reason}; verification receipt: ${skipped.verificationStatus}`,
|
|
1615
1630
|
);
|
|
1616
1631
|
}
|
|
1632
|
+
for (const name of extra.skippedIrreversible) {
|
|
1633
|
+
console.log(`[imprint] skipped irreversible workflow ${name} — live audit disabled`);
|
|
1634
|
+
}
|
|
1617
1635
|
if (extra.detectedButUnemittedTools.length > 0) {
|
|
1618
1636
|
console.log(
|
|
1619
1637
|
`[imprint] ${extra.detectedButUnemittedTools.length}/${extra.detectedToolCount} detected tool(s) were not emitted and are excluded from scoring: ${extra.detectedButUnemittedTools.map((tool) => tool.name).join(', ')}`,
|
|
@@ -27,6 +27,7 @@ import {
|
|
|
27
27
|
AUTH_COMPILE_TOOL_NAMES,
|
|
28
28
|
AUTH_VERIFICATION_ATTEMPT_SENTINEL,
|
|
29
29
|
authExternalVerification,
|
|
30
|
+
authLivePreflightFailures,
|
|
30
31
|
authWorkflowHash,
|
|
31
32
|
authWorkflowPreflightFailures,
|
|
32
33
|
buildAuthCompileTools,
|
|
@@ -85,20 +86,21 @@ function buildAuthInitialMessage(opts: {
|
|
|
85
86
|
authToolPlan: NonNullable<AuthToolPlan>;
|
|
86
87
|
}): string {
|
|
87
88
|
const { site, toolName, toolDir, authToolPlan } = opts;
|
|
88
|
-
const
|
|
89
|
+
const durableCaptures = (authToolPlan.captures ?? []).filter((c) => {
|
|
89
90
|
const u = (c.usedAs ?? '').toLowerCase();
|
|
90
|
-
// Cookies persist automatically
|
|
91
|
-
|
|
91
|
+
// Cookies persist automatically. Every other downstream transport needs a
|
|
92
|
+
// durable credential contract, including JSON/form body fields.
|
|
93
|
+
return u.length > 0 && u !== 'header:cookie' && u !== 'header:set-cookie';
|
|
92
94
|
});
|
|
93
95
|
const durableCaptureNote =
|
|
94
|
-
|
|
95
|
-
? `\n- durable credential contracts (data tools consume these as \${credential.<name>}): ${
|
|
96
|
+
durableCaptures.length > 0
|
|
97
|
+
? `\n- durable credential contracts (data tools consume these as \${credential.<name>}): ${durableCaptures
|
|
96
98
|
.map(
|
|
97
99
|
(c) => `${c.name} (used as ${c.usedAs}; seed source ${c.source}, locator ${c.locator})`,
|
|
98
100
|
)
|
|
99
101
|
.join(
|
|
100
102
|
'; ',
|
|
101
|
-
)}\n → Capture each value on the recording-grounded producing request and include
|
|
103
|
+
)}\n → These names are downstream credential interface names, not required internal capture names. Capture each value on the recording-grounded producing request and include the interface name in authConfig.persist. If you choose a different capture name, declare authConfig.persistBindings[interfaceName] = captureName. The seed source/locator is only a hint.`
|
|
102
104
|
: '';
|
|
103
105
|
return `A new auth compile task is starting.
|
|
104
106
|
|
|
@@ -456,10 +458,12 @@ async function runAuthSegmentLoop(opts: AuthSegmentLoopOptions): Promise<Compile
|
|
|
456
458
|
const checkpointStartedAt = Date.now();
|
|
457
459
|
try {
|
|
458
460
|
if (cp.kind === 'run_verification') {
|
|
459
|
-
const preflightFailures =
|
|
461
|
+
const preflightFailures = await authLivePreflightFailures(
|
|
460
462
|
opts.toolDir,
|
|
461
463
|
opts.session,
|
|
462
464
|
opts.authToolPlan.credentialNames,
|
|
465
|
+
[...opts.authToolPlan.credentialRequestSeqs, ...opts.authToolPlan.authRequestSeqs],
|
|
466
|
+
opts.sessionPath,
|
|
463
467
|
);
|
|
464
468
|
if (Date.now() >= activeDeadlineMs) {
|
|
465
469
|
resultMsg =
|
|
@@ -10,17 +10,21 @@ import { join as pathJoin } from 'node:path';
|
|
|
10
10
|
import type { AgentTool } from './agent.ts';
|
|
11
11
|
import {
|
|
12
12
|
type CompileToolContext,
|
|
13
|
+
bodyEncodingContractFailures,
|
|
13
14
|
buildReadFileTool,
|
|
14
15
|
buildReadRequestTool,
|
|
15
16
|
buildReadResponseBodyTool,
|
|
16
17
|
buildReadSessionSummaryTool,
|
|
17
|
-
|
|
18
|
+
buildRunTestsTool,
|
|
18
19
|
buildSearchRequestsTool,
|
|
19
20
|
buildWriteFileTool,
|
|
21
|
+
irreversibleProvenanceFailures,
|
|
22
|
+
requestEncodingTestContractFailures,
|
|
20
23
|
} from './compile-tools.ts';
|
|
24
|
+
import { isIrreversibleRequest, workflowHasIrreversibleEffect } from './effects.ts';
|
|
21
25
|
import { recordedRequestMatchesWorkflow } from './recording-request.ts';
|
|
22
|
-
import { captureHeader, captureValueMatches,
|
|
23
|
-
import { type Session, type Workflow, WorkflowSchema } from './types.ts';
|
|
26
|
+
import { captureHeader, captureValueMatches, resolveJsonCapture } from './request-capture.ts';
|
|
27
|
+
import { type Session, type Workflow, WorkflowSchema, persistedCaptureName } from './types.ts';
|
|
24
28
|
|
|
25
29
|
type TeachCredentials = { site: string; values: Record<string, string> };
|
|
26
30
|
|
|
@@ -40,7 +44,7 @@ export const AUTH_COMPILE_TOOL_NAMES = [
|
|
|
40
44
|
'read_response_body',
|
|
41
45
|
'write_file',
|
|
42
46
|
'read_file',
|
|
43
|
-
'
|
|
47
|
+
'run_tests',
|
|
44
48
|
'run_verification',
|
|
45
49
|
'inspect_verification_page',
|
|
46
50
|
'prompt_user',
|
|
@@ -50,7 +54,7 @@ export const AUTH_COMPILE_TOOL_NAMES = [
|
|
|
50
54
|
export function buildAuthCompileTools(
|
|
51
55
|
session: Session,
|
|
52
56
|
toolDir: string,
|
|
53
|
-
|
|
57
|
+
sessionPath: string,
|
|
54
58
|
teachCredentials: TeachCredentials,
|
|
55
59
|
): AgentTool[] {
|
|
56
60
|
const context: CompileToolContext = { teachCredentials };
|
|
@@ -61,7 +65,19 @@ export function buildAuthCompileTools(
|
|
|
61
65
|
buildReadResponseBodyTool(session),
|
|
62
66
|
buildWriteFileTool(toolDir),
|
|
63
67
|
buildReadFileTool(toolDir),
|
|
64
|
-
|
|
68
|
+
buildRunTestsTool(
|
|
69
|
+
toolDir,
|
|
70
|
+
sessionPath,
|
|
71
|
+
{
|
|
72
|
+
// Auth request/capture tests are offline and use synthetic credentials.
|
|
73
|
+
// Explicitly mask any inherited teach payload so agent-authored test
|
|
74
|
+
// output cannot expose real credentials to the compile model.
|
|
75
|
+
IMPRINT_TEACH_CREDENTIALS: '',
|
|
76
|
+
},
|
|
77
|
+
{
|
|
78
|
+
networkDisabled: session.requests.some(isIrreversibleRequest),
|
|
79
|
+
},
|
|
80
|
+
),
|
|
65
81
|
];
|
|
66
82
|
}
|
|
67
83
|
|
|
@@ -75,9 +91,57 @@ export function authWorkflowPreflightFailures(
|
|
|
75
91
|
return parseWorkflow(workflowPath, requiredCredentialNames, session).failures;
|
|
76
92
|
}
|
|
77
93
|
|
|
94
|
+
export async function authLivePreflightFailures(
|
|
95
|
+
toolDir: string,
|
|
96
|
+
session?: Session,
|
|
97
|
+
requiredCredentialNames: readonly string[] = [],
|
|
98
|
+
authRequestSeqs: readonly number[] = [],
|
|
99
|
+
sessionPath = '',
|
|
100
|
+
): Promise<string[]> {
|
|
101
|
+
const failures = authWorkflowPreflightFailures(toolDir, session, requiredCredentialNames);
|
|
102
|
+
if (failures.length > 0) return failures;
|
|
103
|
+
const parsed = parseWorkflow(
|
|
104
|
+
pathJoin(toolDir, 'workflow.json'),
|
|
105
|
+
requiredCredentialNames,
|
|
106
|
+
session,
|
|
107
|
+
);
|
|
108
|
+
if (!parsed.workflow) return parsed.failures;
|
|
109
|
+
const requestTestPath = pathJoin(toolDir, 'request.test.ts');
|
|
110
|
+
failures.push(
|
|
111
|
+
...bodyEncodingContractFailures(parsed.workflow),
|
|
112
|
+
...requestEncodingTestContractFailures(
|
|
113
|
+
parsed.workflow,
|
|
114
|
+
existsSync(requestTestPath) ? readFileSync(requestTestPath, 'utf8') : undefined,
|
|
115
|
+
),
|
|
116
|
+
...persistedCaptureTestContractFailures(
|
|
117
|
+
parsed.workflow,
|
|
118
|
+
existsSync(requestTestPath) ? readFileSync(requestTestPath, 'utf8') : undefined,
|
|
119
|
+
),
|
|
120
|
+
);
|
|
121
|
+
if (session) {
|
|
122
|
+
failures.push(
|
|
123
|
+
...irreversibleProvenanceFailures(session, parsed.workflow, {
|
|
124
|
+
candidateRequestSeqs: [...authRequestSeqs],
|
|
125
|
+
}),
|
|
126
|
+
);
|
|
127
|
+
}
|
|
128
|
+
if (workflowHasIrreversibleEffect(parsed.workflow)) {
|
|
129
|
+
failures.push('auth live verification is disabled for workflows with irreversible requests');
|
|
130
|
+
}
|
|
131
|
+
if (failures.length === 0 && existsSync(requestTestPath)) {
|
|
132
|
+
const testResult = await buildRunTestsTool(toolDir, sessionPath, undefined, {
|
|
133
|
+
networkDisabled: workflowHasIrreversibleEffect(parsed.workflow),
|
|
134
|
+
}).handler({});
|
|
135
|
+
if (testResult.isError) {
|
|
136
|
+
failures.push(`agent-authored request tests failed: ${testResult.result}`);
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
return failures;
|
|
140
|
+
}
|
|
141
|
+
|
|
78
142
|
export function authExternalVerification(
|
|
79
143
|
toolDir: string,
|
|
80
|
-
|
|
144
|
+
requiredDurableCredentials: Array<{ name: string; usedAs?: string }> = [],
|
|
81
145
|
options: { requireLiveAttempt?: boolean; requiredCredentialNames?: readonly string[] } = {},
|
|
82
146
|
): string[] {
|
|
83
147
|
const workflowPath = pathJoin(toolDir, 'workflow.json');
|
|
@@ -87,12 +151,27 @@ export function authExternalVerification(
|
|
|
87
151
|
if (!parsed.workflow) return parsed.failures;
|
|
88
152
|
const failures = [...parsed.failures];
|
|
89
153
|
const workflow = parsed.workflow;
|
|
154
|
+
failures.push(...bodyEncodingContractFailures(workflow));
|
|
155
|
+
failures.push(
|
|
156
|
+
...requestEncodingTestContractFailures(
|
|
157
|
+
workflow,
|
|
158
|
+
existsSync(pathJoin(toolDir, 'request.test.ts'))
|
|
159
|
+
? readFileSync(pathJoin(toolDir, 'request.test.ts'), 'utf8')
|
|
160
|
+
: undefined,
|
|
161
|
+
),
|
|
162
|
+
...persistedCaptureTestContractFailures(
|
|
163
|
+
workflow,
|
|
164
|
+
existsSync(pathJoin(toolDir, 'request.test.ts'))
|
|
165
|
+
? readFileSync(pathJoin(toolDir, 'request.test.ts'), 'utf8')
|
|
166
|
+
: undefined,
|
|
167
|
+
),
|
|
168
|
+
);
|
|
90
169
|
const persisted = new Set(workflow.authConfig?.persist ?? []);
|
|
91
170
|
|
|
92
|
-
const missingContracts =
|
|
171
|
+
const missingContracts = requiredDurableCredentials.filter((capture) => {
|
|
93
172
|
const target = (capture.usedAs ?? '').toLowerCase();
|
|
94
173
|
return (
|
|
95
|
-
target.
|
|
174
|
+
target.length > 0 &&
|
|
96
175
|
target !== 'header:cookie' &&
|
|
97
176
|
target !== 'header:set-cookie' &&
|
|
98
177
|
!persisted.has(capture.name)
|
|
@@ -100,7 +179,7 @@ export function authExternalVerification(
|
|
|
100
179
|
});
|
|
101
180
|
if (missingContracts.length > 0) {
|
|
102
181
|
failures.push(
|
|
103
|
-
`authConfig.persist must include downstream credential
|
|
182
|
+
`authConfig.persist must include downstream credential interface(s): ${missingContracts
|
|
104
183
|
.map((capture) => capture.name)
|
|
105
184
|
.join(', ')}`,
|
|
106
185
|
);
|
|
@@ -112,6 +191,25 @@ export function authExternalVerification(
|
|
|
112
191
|
return failures;
|
|
113
192
|
}
|
|
114
193
|
|
|
194
|
+
function persistedCaptureTestContractFailures(
|
|
195
|
+
workflow: Workflow,
|
|
196
|
+
source: string | undefined,
|
|
197
|
+
): string[] {
|
|
198
|
+
const persisted = workflow.authConfig?.persist ?? [];
|
|
199
|
+
if (persisted.length === 0) return [];
|
|
200
|
+
if (source === undefined) {
|
|
201
|
+
return [
|
|
202
|
+
'request.test.ts is required to prove persisted auth captures against their exact recorded response fields',
|
|
203
|
+
];
|
|
204
|
+
}
|
|
205
|
+
return persisted
|
|
206
|
+
.filter((name) => !source.includes(`persisted-capture:${name}`))
|
|
207
|
+
.map(
|
|
208
|
+
(name) =>
|
|
209
|
+
`request.test.ts must include a passing "persisted-capture:${name}" test that independently decodes the recorded producer response and asserts the compiled capture equals that exact structured value`,
|
|
210
|
+
);
|
|
211
|
+
}
|
|
212
|
+
|
|
115
213
|
function parseWorkflow(
|
|
116
214
|
path: string,
|
|
117
215
|
requiredCredentialNames: readonly string[] = [],
|
|
@@ -274,17 +372,28 @@ function authProgramFailures(
|
|
|
274
372
|
}
|
|
275
373
|
|
|
276
374
|
if (!hasSuccess) failures.push('authConfig.actions has no success outcome');
|
|
375
|
+
const persistedCredentials = new Set(config.persist);
|
|
376
|
+
for (const credentialName of Object.keys(config.persistBindings ?? {})) {
|
|
377
|
+
if (!persistedCredentials.has(credentialName)) {
|
|
378
|
+
failures.push(
|
|
379
|
+
`authConfig.persistBindings declares unused credential ${JSON.stringify(credentialName)}`,
|
|
380
|
+
);
|
|
381
|
+
}
|
|
382
|
+
}
|
|
277
383
|
for (const name of config.persist) {
|
|
278
|
-
|
|
279
|
-
|
|
384
|
+
const captureName = persistedCaptureName(config, name);
|
|
385
|
+
if (!captures.has(captureName)) {
|
|
386
|
+
failures.push(
|
|
387
|
+
`authConfig.persist credential ${JSON.stringify(name)} references unknown capture ${JSON.stringify(captureName)}`,
|
|
388
|
+
);
|
|
280
389
|
continue;
|
|
281
390
|
}
|
|
282
|
-
const owner = captureOwners.get(
|
|
391
|
+
const owner = captureOwners.get(captureName);
|
|
283
392
|
if (!owner) continue;
|
|
284
393
|
const { request, capture } = owner;
|
|
285
394
|
if (request.recordingRequestSeq === undefined) {
|
|
286
395
|
failures.push(
|
|
287
|
-
`persisted capture ${JSON.stringify(
|
|
396
|
+
`persisted capture ${JSON.stringify(captureName)} must declare recordingRequestSeq on its producing request`,
|
|
288
397
|
);
|
|
289
398
|
} else if (session) {
|
|
290
399
|
const recorded = session.requests.find(
|
|
@@ -292,16 +401,16 @@ function authProgramFailures(
|
|
|
292
401
|
);
|
|
293
402
|
if (!recorded) {
|
|
294
403
|
failures.push(
|
|
295
|
-
`persisted capture ${JSON.stringify(
|
|
404
|
+
`persisted capture ${JSON.stringify(captureName)} references missing recorded request seq ${request.recordingRequestSeq}`,
|
|
296
405
|
);
|
|
297
406
|
} else if (!recordedRequestMatchesWorkflow(recorded, request)) {
|
|
298
407
|
failures.push(
|
|
299
|
-
`persisted capture ${JSON.stringify(
|
|
408
|
+
`persisted capture ${JSON.stringify(captureName)} recordingRequestSeq ${request.recordingRequestSeq} does not match its workflow request`,
|
|
300
409
|
);
|
|
301
410
|
} else {
|
|
302
411
|
if (!recordedResponseProducesCapture(recorded, capture)) {
|
|
303
412
|
failures.push(
|
|
304
|
-
`persisted capture ${JSON.stringify(
|
|
413
|
+
`persisted capture ${JSON.stringify(captureName)} is not produced by recorded request seq ${request.recordingRequestSeq}`,
|
|
305
414
|
);
|
|
306
415
|
}
|
|
307
416
|
}
|
|
@@ -323,7 +432,7 @@ function recordedResponseProducesCapture(
|
|
|
323
432
|
let value: unknown;
|
|
324
433
|
if (capture.source === 'json') {
|
|
325
434
|
try {
|
|
326
|
-
value =
|
|
435
|
+
value = resolveJsonCapture(JSON.parse(body), capture.path, capture.decodeJsonPath);
|
|
327
436
|
} catch {
|
|
328
437
|
return false;
|
|
329
438
|
}
|
|
@@ -59,7 +59,7 @@ export class AuthVerifier {
|
|
|
59
59
|
async runAction(
|
|
60
60
|
action: string,
|
|
61
61
|
parameters: Record<string, string | number | boolean> = {},
|
|
62
|
-
options: { freshSession?: boolean; cleanSession?: boolean } = {},
|
|
62
|
+
options: { freshSession?: boolean; cleanSession?: boolean; signal?: AbortSignal } = {},
|
|
63
63
|
): Promise<AuthActionResult> {
|
|
64
64
|
if (options.freshSession || options.cleanSession) await this.reset();
|
|
65
65
|
this.rememberSensitiveValues(parameters);
|
|
@@ -74,6 +74,7 @@ export class AuthVerifier {
|
|
|
74
74
|
cdpPool: this.cdpPool,
|
|
75
75
|
forceBackend: 'cdp-replay',
|
|
76
76
|
initialState: this.continuation,
|
|
77
|
+
signal: options.signal,
|
|
77
78
|
});
|
|
78
79
|
const durationMs = Date.now() - startedAt;
|
|
79
80
|
const result = ladder.result;
|
|
@@ -340,6 +340,8 @@ export async function runWithLadder(
|
|
|
340
340
|
initialState?: Record<string, unknown>;
|
|
341
341
|
/** Optional credential override used by auth verification. */
|
|
342
342
|
credentials?: CredentialStore;
|
|
343
|
+
/** Caller cancellation for bounded verification work. */
|
|
344
|
+
signal?: AbortSignal;
|
|
343
345
|
},
|
|
344
346
|
): Promise<LadderResult> {
|
|
345
347
|
if (ladder.length === 0) {
|
|
@@ -437,6 +439,7 @@ export async function runWithLadder(
|
|
|
437
439
|
options?.cdpPool,
|
|
438
440
|
options?.initialState,
|
|
439
441
|
options?.credentials,
|
|
442
|
+
options?.signal,
|
|
440
443
|
);
|
|
441
444
|
break;
|
|
442
445
|
case 'stealth-fetch': {
|
|
@@ -1036,6 +1039,7 @@ async function runCdpReplay(
|
|
|
1036
1039
|
cdpPool?: Map<string, CdpBrowserFetch>,
|
|
1037
1040
|
callerState?: Record<string, unknown>,
|
|
1038
1041
|
credentialOverride?: CredentialStore,
|
|
1042
|
+
signal?: AbortSignal,
|
|
1039
1043
|
): Promise<ToolResult> {
|
|
1040
1044
|
let baseUrl: string;
|
|
1041
1045
|
try {
|
|
@@ -1105,6 +1109,9 @@ async function runCdpReplay(
|
|
|
1105
1109
|
// declares it — never a blanket default. See AuthConfig.crossOriginCookieReinjection.
|
|
1106
1110
|
reinjectCrossOriginCookies: tool.workflow.authConfig?.crossOriginCookieReinjection ?? false,
|
|
1107
1111
|
});
|
|
1112
|
+
// Register immediately so caller cancellation can close a browser that is
|
|
1113
|
+
// still launching or minting its first jar.
|
|
1114
|
+
if (cdpPool) cdpPool.set(poolKey, cf);
|
|
1108
1115
|
}
|
|
1109
1116
|
|
|
1110
1117
|
try {
|
|
@@ -1148,6 +1155,7 @@ async function runCdpReplay(
|
|
|
1148
1155
|
snapshotCookies: cf.snapshotCookies.bind(cf),
|
|
1149
1156
|
}
|
|
1150
1157
|
: undefined,
|
|
1158
|
+
signal,
|
|
1151
1159
|
});
|
|
1152
1160
|
|
|
1153
1161
|
if (result.ok) {
|
|
@@ -1597,6 +1605,8 @@ export async function runWorkflowWithLadder(opts: {
|
|
|
1597
1605
|
/** Pin execution to a single rung, bypassing the parallel probe AND the winner
|
|
1598
1606
|
* memo. AuthVerifier pins `cdp-replay` so all actions share one browser. */
|
|
1599
1607
|
forceBackend?: ConcreteBackend;
|
|
1608
|
+
/** Caller cancellation for bounded verification work. */
|
|
1609
|
+
signal?: AbortSignal;
|
|
1600
1610
|
}): Promise<LadderResult> {
|
|
1601
1611
|
if (!existsSync(opts.workflowPath)) {
|
|
1602
1612
|
throw new Error(`runWorkflowWithLadder: workflow.json not found at ${opts.workflowPath}`);
|
|
@@ -1674,6 +1684,7 @@ export async function runWorkflowWithLadder(opts: {
|
|
|
1674
1684
|
cdpPool,
|
|
1675
1685
|
initialState: opts.initialState,
|
|
1676
1686
|
credentials: opts.credentials,
|
|
1687
|
+
signal: opts.signal,
|
|
1677
1688
|
});
|
|
1678
1689
|
}
|
|
1679
1690
|
|
|
@@ -1731,6 +1742,7 @@ export async function runWorkflowWithLadder(opts: {
|
|
|
1731
1742
|
cdpPool,
|
|
1732
1743
|
initialState: opts.initialState,
|
|
1733
1744
|
credentials: opts.credentials,
|
|
1745
|
+
signal: opts.signal,
|
|
1734
1746
|
});
|
|
1735
1747
|
const r = await settleProbeWithDeadline(inner, PROBE_TIMEOUT_MS, {
|
|
1736
1748
|
result: { ok: false, error: 'NETWORK', message: 'probe deadline exceeded' },
|
|
@@ -1781,6 +1793,7 @@ export async function runWorkflowWithLadder(opts: {
|
|
|
1781
1793
|
cdpPool,
|
|
1782
1794
|
initialState: opts.initialState,
|
|
1783
1795
|
credentials: opts.credentials,
|
|
1796
|
+
signal: opts.signal,
|
|
1784
1797
|
});
|
|
1785
1798
|
if (probeReachable(seqResult.result)) {
|
|
1786
1799
|
compileWinningBackend.set(memoKey, seqResult.usedBackend);
|
|
@@ -1804,6 +1817,7 @@ export async function runWorkflowWithLadder(opts: {
|
|
|
1804
1817
|
cdpPool,
|
|
1805
1818
|
initialState: opts.initialState,
|
|
1806
1819
|
credentials: opts.credentials,
|
|
1820
|
+
signal: opts.signal,
|
|
1807
1821
|
});
|
|
1808
1822
|
if (isProbeReachable(result.result)) {
|
|
1809
1823
|
compileWinningBackend.set(memoKey, result.usedBackend);
|
|
@@ -1849,6 +1863,7 @@ export function resolveWorkflowTool(
|
|
|
1849
1863
|
browser?: BrowserNavigationTransport;
|
|
1850
1864
|
initialState?: Record<string, unknown>;
|
|
1851
1865
|
credentials?: CredentialStore;
|
|
1866
|
+
signal?: AbortSignal;
|
|
1852
1867
|
}
|
|
1853
1868
|
| undefined;
|
|
1854
1869
|
return executeWorkflow({
|
|
@@ -1859,6 +1874,7 @@ export function resolveWorkflowTool(
|
|
|
1859
1874
|
fetchImpl: o?.fetchImpl,
|
|
1860
1875
|
browser: o?.browser,
|
|
1861
1876
|
initialState: o?.initialState,
|
|
1877
|
+
signal: o?.signal,
|
|
1862
1878
|
});
|
|
1863
1879
|
},
|
|
1864
1880
|
};
|
|
@@ -1896,6 +1912,8 @@ export async function renderWorkflowRequests(opts: {
|
|
|
1896
1912
|
params: Record<string, string | number | boolean>;
|
|
1897
1913
|
workflowPath?: string;
|
|
1898
1914
|
credentials?: CredentialStore;
|
|
1915
|
+
/** Synthetic captured state for offline request tests. */
|
|
1916
|
+
initialState?: Record<string, unknown>;
|
|
1899
1917
|
recordedResponseFor?: (
|
|
1900
1918
|
method: string,
|
|
1901
1919
|
url: string,
|
|
@@ -1930,7 +1948,9 @@ export async function renderWorkflowRequests(opts: {
|
|
|
1930
1948
|
params: opts.params,
|
|
1931
1949
|
credentials: opts.credentials,
|
|
1932
1950
|
workflowPath: opts.workflowPath,
|
|
1951
|
+
initialState: opts.initialState,
|
|
1933
1952
|
fetchImpl,
|
|
1953
|
+
persistAuthState: false,
|
|
1934
1954
|
});
|
|
1935
1955
|
return { requests: captured, result };
|
|
1936
1956
|
}
|
|
@@ -183,6 +183,8 @@ type DynamicValueFinding = z.infer<typeof DynamicValueFindingSchema>;
|
|
|
183
183
|
|
|
184
184
|
const PerToolPlanSchema = z.object({
|
|
185
185
|
toolName: z.string().regex(/^[a-z][a-z0-9_]*$/),
|
|
186
|
+
/** Callable prerequisite tools chosen by candidate detection. */
|
|
187
|
+
dependsOnTools: z.array(z.string().regex(/^[a-z][a-z0-9_]*$/)).default([]),
|
|
186
188
|
usesSharedModules: z.array(z.string()).default([]),
|
|
187
189
|
loadBearingSeqs: z.array(z.number().int().nonnegative()).default([]),
|
|
188
190
|
parserGuidance: z.string().default(''),
|
|
@@ -692,6 +694,8 @@ export function topoLevelsForTools<T extends { toolName: string }>(
|
|
|
692
694
|
(t) => {
|
|
693
695
|
if (!plan) return [];
|
|
694
696
|
const deps = new Set<string>();
|
|
697
|
+
const toolPlan = plan.perTool.find((candidate) => candidate.toolName === t.toolName);
|
|
698
|
+
for (const dependency of toolPlan?.dependsOnTools ?? []) deps.add(dependency);
|
|
695
699
|
for (const tp of resolveTokenParams(plan, t.toolName)) deps.add(tp.sourceTool);
|
|
696
700
|
// General contract edges: a producer_tool input depends on its producer; any
|
|
697
701
|
// auth input depends on the authenticate tool (filtered out by kahnLevels
|
|
@@ -862,6 +866,20 @@ export function validateBuildPlan(
|
|
|
862
866
|
const plan = BuildPlanSchema.parse(normalizeRawPlan(input));
|
|
863
867
|
if (selected && selected.length > 0) {
|
|
864
868
|
const names = new Set(selected.map((t) => (typeof t === 'string' ? t : t.toolName)));
|
|
869
|
+
const dependenciesByName = new Map(
|
|
870
|
+
selected.flatMap((candidate) =>
|
|
871
|
+
typeof candidate === 'string'
|
|
872
|
+
? []
|
|
873
|
+
: [
|
|
874
|
+
[
|
|
875
|
+
candidate.toolName,
|
|
876
|
+
candidate.dependsOnTools.filter(
|
|
877
|
+
(dependency) => names.has(dependency) && dependency !== candidate.toolName,
|
|
878
|
+
),
|
|
879
|
+
] as const,
|
|
880
|
+
],
|
|
881
|
+
),
|
|
882
|
+
);
|
|
865
883
|
plan.perTool = plan.perTool.filter((t) => names.has(t.toolName));
|
|
866
884
|
for (const name of names) {
|
|
867
885
|
if (!plan.perTool.some((t) => t.toolName === name)) {
|
|
@@ -873,6 +891,9 @@ export function validateBuildPlan(
|
|
|
873
891
|
if (plan.perTool.length === 0) {
|
|
874
892
|
throw new Error('Build plan has no perTool entries for the selected tools.');
|
|
875
893
|
}
|
|
894
|
+
for (const tool of plan.perTool) {
|
|
895
|
+
tool.dependsOnTools = dependenciesByName.get(tool.toolName) ?? [];
|
|
896
|
+
}
|
|
876
897
|
}
|
|
877
898
|
// Deterministic requiredInputs ordering for stable sidecar JSON.
|
|
878
899
|
for (const t of plan.perTool) {
|
|
@@ -51,6 +51,7 @@ import {
|
|
|
51
51
|
traceLlmIoEnabled,
|
|
52
52
|
traced,
|
|
53
53
|
} from './tracing.ts';
|
|
54
|
+
import type { SharedTriageSelection } from './triage-selection.ts';
|
|
54
55
|
import type { Session } from './types.ts';
|
|
55
56
|
|
|
56
57
|
const log = createLog('compile-claude-cli');
|
|
@@ -127,7 +128,7 @@ interface CompileViaClaudeCliOptions {
|
|
|
127
128
|
onProgress?: (p: CompileAgentProgress) => void;
|
|
128
129
|
/** Called when wall-clock deadline is reached; return ms to extend or null to time out. */
|
|
129
130
|
onDeadlineReached?: OnDeadlineReached;
|
|
130
|
-
/** Retain
|
|
131
|
+
/** Retain agent-generated tests after successful verification. Mirrors the
|
|
131
132
|
* in-process loop's `keepTest`. */
|
|
132
133
|
keepTest?: boolean;
|
|
133
134
|
candidate?: ToolCandidate;
|
|
@@ -140,6 +141,8 @@ interface CompileViaClaudeCliOptions {
|
|
|
140
141
|
toolPlan?: string;
|
|
141
142
|
/** Revise existing generated artifacts from durable verification feedback. */
|
|
142
143
|
revisionMode?: boolean;
|
|
144
|
+
/** Shared triage result for irreversible-effect propagation in the MCP server. */
|
|
145
|
+
sharedTriageSelection?: SharedTriageSelection;
|
|
143
146
|
/** Present → drive an auth compile rather than a data compile. */
|
|
144
147
|
authMode?: AuthCliCompileMode;
|
|
145
148
|
/** Auth segments only: resume a prior segment's claude session with a new user
|
|
@@ -377,6 +380,9 @@ async function runClaudeCliAttempt(opts: CompileViaClaudeCliOptions): Promise<Co
|
|
|
377
380
|
...(opts.buildPlanPath ? ['--build-plan-path', opts.buildPlanPath] : []),
|
|
378
381
|
...(opts.sharedModules ? ['--shared-modules-json', JSON.stringify(opts.sharedModules)] : []),
|
|
379
382
|
...(opts.revisionMode ? ['--revision-mode'] : []),
|
|
383
|
+
...(opts.sharedTriageSelection
|
|
384
|
+
? ['--shared-triage-json', JSON.stringify(opts.sharedTriageSelection)]
|
|
385
|
+
: []),
|
|
380
386
|
];
|
|
381
387
|
allowedToolNames = [
|
|
382
388
|
'read_session_summary',
|
|
@@ -34,6 +34,7 @@ import {
|
|
|
34
34
|
traceToolIoEnabled,
|
|
35
35
|
traced,
|
|
36
36
|
} from './tracing.ts';
|
|
37
|
+
import type { SharedTriageSelection } from './triage-selection.ts';
|
|
37
38
|
import type { Session } from './types.ts';
|
|
38
39
|
|
|
39
40
|
const log = createLog('compile-codex-cli');
|
|
@@ -143,6 +144,8 @@ interface CompileViaCodexCliOptions {
|
|
|
143
144
|
toolPlan?: string;
|
|
144
145
|
/** Revise existing generated artifacts from durable verification feedback. */
|
|
145
146
|
revisionMode?: boolean;
|
|
147
|
+
/** Shared triage result for irreversible-effect propagation in the MCP server. */
|
|
148
|
+
sharedTriageSelection?: SharedTriageSelection;
|
|
146
149
|
/** Present → drive an auth compile rather than a data compile. */
|
|
147
150
|
authMode?: AuthCliCompileMode;
|
|
148
151
|
/** Auth segments only: resume the same non-interactive Codex session. */
|
|
@@ -302,6 +305,9 @@ async function compileViaCodexCliImpl(
|
|
|
302
305
|
...(opts.buildPlanPath ? ['--build-plan-path', opts.buildPlanPath] : []),
|
|
303
306
|
...(opts.sharedModules ? ['--shared-modules-json', JSON.stringify(opts.sharedModules)] : []),
|
|
304
307
|
...(opts.revisionMode ? ['--revision-mode'] : []),
|
|
308
|
+
...(opts.sharedTriageSelection
|
|
309
|
+
? ['--shared-triage-json', JSON.stringify(opts.sharedTriageSelection)]
|
|
310
|
+
: []),
|
|
305
311
|
];
|
|
306
312
|
const { assignedSharedModules } = resolvePlanSliceFromFile(
|
|
307
313
|
opts.buildPlanPath,
|