imprint-mcp 0.6.0 → 0.6.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -0
- package/package.json +4 -2
- package/prompts/audit-agent.md +4 -2
- package/prompts/auth-compile-agent.md +149 -158
- package/prompts/build-planning.md +14 -9
- package/prompts/compile-agent.md +46 -23
- package/prompts/live-verifier-agent.md +101 -0
- package/prompts/playbook-compilation.md +8 -1
- package/prompts/tool-candidate-detection.md +9 -29
- package/src/cli.ts +83 -1
- package/src/imprint/audit.ts +219 -20
- package/src/imprint/auth-compile-agent.ts +215 -155
- package/src/imprint/auth-compile-tools.ts +300 -118
- package/src/imprint/auth-verifier.ts +167 -180
- package/src/imprint/backend-cache.ts +18 -11
- package/src/imprint/backend-ladder.ts +225 -177
- package/src/imprint/build-plan.ts +3 -8
- package/src/imprint/cdp-browser-fetch.ts +930 -83
- package/src/imprint/claude-cli-compile.ts +46 -20
- package/src/imprint/codex-cli-compile.ts +436 -139
- package/src/imprint/compile-agent-types.ts +15 -10
- package/src/imprint/compile-agent.ts +88 -28
- package/src/imprint/compile-tools.ts +228 -39
- package/src/imprint/compile-verification.ts +187 -0
- package/src/imprint/compile.ts +71 -2
- package/src/imprint/credential-store.ts +57 -44
- package/src/imprint/emit.ts +3 -1
- package/src/imprint/freeform-redact.ts +27 -6
- package/src/imprint/live-verifier.ts +1665 -0
- package/src/imprint/llm.ts +22 -3
- package/src/imprint/login.ts +107 -25
- package/src/imprint/mcp-compile-server.ts +447 -114
- package/src/imprint/mcp-live-verifier-server.ts +359 -0
- package/src/imprint/mcp-server.ts +116 -35
- package/src/imprint/probe-backends.ts +88 -7
- package/src/imprint/recording-request.ts +36 -0
- package/src/imprint/request-capture.ts +40 -17
- package/src/imprint/runtime.ts +490 -337
- package/src/imprint/teach-state.ts +36 -2
- package/src/imprint/teach.ts +302 -146
- package/src/imprint/tool-candidates.ts +8 -10
- package/src/imprint/tool-loader.ts +12 -4
- package/src/imprint/types.ts +137 -68
|
@@ -36,6 +36,7 @@ import { RuntimeCookieJar } from './cookie-jar.ts';
|
|
|
36
36
|
import { createLog } from './log.ts';
|
|
37
37
|
import { runPlaybook } from './playbook-runner.ts';
|
|
38
38
|
import {
|
|
39
|
+
type BrowserNavigationTransport,
|
|
39
40
|
type CredentialStore,
|
|
40
41
|
executeWorkflow,
|
|
41
42
|
loadCredentialStore,
|
|
@@ -86,7 +87,7 @@ export function __setPlaybookRunnerForTest(fn: PlaybookRunner | null): void {
|
|
|
86
87
|
|
|
87
88
|
const DEFAULT_LADDER: ConcreteBackend[] = ['fetch', 'stealth-fetch', 'playbook'];
|
|
88
89
|
|
|
89
|
-
const NON_TRANSPORT_ERRORS = new Set(['
|
|
90
|
+
const NON_TRANSPORT_ERRORS = new Set(['ACTION_REQUIRED', 'AUTH_EXPIRED', 'RATE_LIMITED']);
|
|
90
91
|
|
|
91
92
|
function isProbeReachable(result: ToolResult): boolean {
|
|
92
93
|
if (result.ok) return true;
|
|
@@ -94,9 +95,7 @@ function isProbeReachable(result: ToolResult): boolean {
|
|
|
94
95
|
}
|
|
95
96
|
// Generous enough to clear an anti-bot interstitial (Cloudflare/Akamai
|
|
96
97
|
// "checking your browser", which can hold the navigation 10-30s) before the
|
|
97
|
-
// real page's `load` fires.
|
|
98
|
-
// gate, so a 20s per-step cap was too tight and stranded otherwise-correct
|
|
99
|
-
// login playbooks at the navigate step. Overridable via env for tuning.
|
|
98
|
+
// real page's `load` fires. Overridable via env for tuning.
|
|
100
99
|
const DEFAULT_PLAYBOOK_BACKEND_TIMEOUT_MS = 150_000;
|
|
101
100
|
const DEFAULT_PLAYBOOK_BACKEND_STEP_TIMEOUT_MS = 45_000;
|
|
102
101
|
|
|
@@ -210,6 +209,38 @@ function sleepMs(ms: number): Promise<void> {
|
|
|
210
209
|
return new Promise((r) => setTimeout(r, ms));
|
|
211
210
|
}
|
|
212
211
|
|
|
212
|
+
async function settleProbeWithDeadline<T>(
|
|
213
|
+
inner: Promise<T>,
|
|
214
|
+
timeoutMs: number,
|
|
215
|
+
timeoutValue: T,
|
|
216
|
+
): Promise<T> {
|
|
217
|
+
type Observed = { status: 'fulfilled'; value: T } | { status: 'rejected'; reason: unknown };
|
|
218
|
+
const observed: Promise<Observed> = inner.then(
|
|
219
|
+
(value) => ({ status: 'fulfilled', value }),
|
|
220
|
+
(reason) => ({ status: 'rejected', reason }),
|
|
221
|
+
);
|
|
222
|
+
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
223
|
+
const deadline = new Promise<'deadline'>((resolve) => {
|
|
224
|
+
timer = setTimeout(() => resolve('deadline'), timeoutMs);
|
|
225
|
+
});
|
|
226
|
+
const first = await Promise.race([observed, deadline]);
|
|
227
|
+
if (timer) clearTimeout(timer);
|
|
228
|
+
if (first === 'deadline') {
|
|
229
|
+
const pending = observed.then(() => undefined);
|
|
230
|
+
pendingProbeRuns.add(pending);
|
|
231
|
+
void pending.finally(() => pendingProbeRuns.delete(pending));
|
|
232
|
+
return timeoutValue;
|
|
233
|
+
}
|
|
234
|
+
if (first.status === 'rejected') throw first.reason;
|
|
235
|
+
return first.value;
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
const pendingProbeRuns = new Set<Promise<void>>();
|
|
239
|
+
|
|
240
|
+
export async function __drainPendingProbeRunsForTest(): Promise<void> {
|
|
241
|
+
await Promise.allSettled([...pendingProbeRuns]);
|
|
242
|
+
}
|
|
243
|
+
|
|
213
244
|
function playbookBackendTimeoutMs(): number {
|
|
214
245
|
return positiveEnvMs('IMPRINT_PLAYBOOK_BACKEND_TIMEOUT_MS', DEFAULT_PLAYBOOK_BACKEND_TIMEOUT_MS);
|
|
215
246
|
}
|
|
@@ -259,6 +290,10 @@ export function __resetCompilePacingForTest(): void {
|
|
|
259
290
|
compileLastRequestAt.clear();
|
|
260
291
|
}
|
|
261
292
|
|
|
293
|
+
export function cdpReplayPoolKey(site: string, _baseUrl: string): string {
|
|
294
|
+
return site;
|
|
295
|
+
}
|
|
296
|
+
|
|
262
297
|
/** Expand a replayBackend choice into a concrete ladder. 'auto' prefers
|
|
263
298
|
* the probed order (if any), then appends the default fallback ladder.
|
|
264
299
|
* Explicit choice → single rung. */
|
|
@@ -300,21 +335,35 @@ export async function runWithLadder(
|
|
|
300
335
|
* The mcp-server owns one map and ties its lifetime to `cdpPool` (a memoized
|
|
301
336
|
* cdp-replay is only fast while its Chrome is pooled). */
|
|
302
337
|
winnerCache?: Map<string, ConcreteBackend>;
|
|
303
|
-
/** Seed state for `${state.X}` substitution, merged
|
|
304
|
-
*
|
|
305
|
-
* uses it: the caller echoes the AWAITING_2FA `twoFactorContext` back on
|
|
306
|
-
* submit_otp so a body-returned token (e.g. a reauth mfaId) resolves on the
|
|
307
|
-
* stateless second call. Undefined for every non-auth call → no effect. */
|
|
338
|
+
/** Seed state for `${state.X}` substitution, merged under state minted by a
|
|
339
|
+
* rung. Auth actions use this for their declared continuation projection. */
|
|
308
340
|
initialState?: Record<string, unknown>;
|
|
341
|
+
/** Optional credential override used by auth verification. */
|
|
342
|
+
credentials?: CredentialStore;
|
|
309
343
|
},
|
|
310
344
|
): Promise<LadderResult> {
|
|
311
345
|
if (ladder.length === 0) {
|
|
312
346
|
throw new Error('runWithLadder: empty ladder');
|
|
313
347
|
}
|
|
314
348
|
|
|
349
|
+
const permittedLadder =
|
|
350
|
+
tool.workflow.toolKind === 'authenticate'
|
|
351
|
+
? ladder.filter((backend) => backend !== 'playbook')
|
|
352
|
+
: ladder;
|
|
353
|
+
if (permittedLadder.length === 0) {
|
|
354
|
+
return {
|
|
355
|
+
result: {
|
|
356
|
+
ok: false,
|
|
357
|
+
error: 'UNKNOWN',
|
|
358
|
+
message: 'Authenticate workflows do not execute playbooks.',
|
|
359
|
+
},
|
|
360
|
+
usedBackend: 'playbook',
|
|
361
|
+
attempts: [],
|
|
362
|
+
};
|
|
363
|
+
}
|
|
315
364
|
const baseLadder = options?.skipBootstrapSplice
|
|
316
|
-
?
|
|
317
|
-
: effectiveAutoLadder(
|
|
365
|
+
? permittedLadder
|
|
366
|
+
: effectiveAutoLadder(permittedLadder, tool.workflow);
|
|
318
367
|
|
|
319
368
|
// Runtime winner memo. Once a backend has served this tool in THIS session,
|
|
320
369
|
// start there next time instead of re-walking the doomed early rungs (southwest
|
|
@@ -369,14 +418,26 @@ export async function runWithLadder(
|
|
|
369
418
|
const fetchOpts: Record<string, unknown> = {};
|
|
370
419
|
if (proxyFetch) fetchOpts.fetchImpl = proxyFetch;
|
|
371
420
|
if (options?.initialState) fetchOpts.initialState = options.initialState;
|
|
421
|
+
if (options?.credentials) fetchOpts.credentials = options.credentials;
|
|
372
422
|
result = await tool.toolFn(params, fetchOpts);
|
|
373
423
|
break;
|
|
374
424
|
}
|
|
375
425
|
case 'fetch-bootstrap':
|
|
376
|
-
result = await runFetchBootstrap(
|
|
426
|
+
result = await runFetchBootstrap(
|
|
427
|
+
tool,
|
|
428
|
+
params,
|
|
429
|
+
options?.initialState,
|
|
430
|
+
options?.credentials,
|
|
431
|
+
);
|
|
377
432
|
break;
|
|
378
433
|
case 'cdp-replay':
|
|
379
|
-
result = await runCdpReplay(
|
|
434
|
+
result = await runCdpReplay(
|
|
435
|
+
tool,
|
|
436
|
+
params,
|
|
437
|
+
options?.cdpPool,
|
|
438
|
+
options?.initialState,
|
|
439
|
+
options?.credentials,
|
|
440
|
+
);
|
|
380
441
|
break;
|
|
381
442
|
case 'stealth-fetch': {
|
|
382
443
|
const paramsWithDefaults = withWorkflowDefaults(tool.workflow, params);
|
|
@@ -396,7 +457,11 @@ export async function runWithLadder(
|
|
|
396
457
|
options?.initialState || bootstrapState
|
|
397
458
|
? { ...options?.initialState, ...bootstrapState }
|
|
398
459
|
: undefined;
|
|
399
|
-
result = await tool.toolFn(paramsWithDefaults, {
|
|
460
|
+
result = await tool.toolFn(paramsWithDefaults, {
|
|
461
|
+
fetchImpl: sf.fetchImpl,
|
|
462
|
+
initialState,
|
|
463
|
+
credentials: options?.credentials,
|
|
464
|
+
});
|
|
400
465
|
break;
|
|
401
466
|
}
|
|
402
467
|
case 'playbook': {
|
|
@@ -410,11 +475,7 @@ export async function runWithLadder(
|
|
|
410
475
|
site: tool.site,
|
|
411
476
|
stepTimeoutMs: playbookBackendStepTimeoutMs(),
|
|
412
477
|
maxDurationMs: playbookBackendTimeoutMs(),
|
|
413
|
-
// A login playbook mints a fresh session — persist it so downstream
|
|
414
|
-
// data tools reuse the cookies.
|
|
415
|
-
persistCookies: tool.workflow.toolKind === 'authenticate',
|
|
416
478
|
});
|
|
417
|
-
result = reshapePlaybookAuthResult(result, tool.workflow, paramsWithDefaults);
|
|
418
479
|
break;
|
|
419
480
|
}
|
|
420
481
|
}
|
|
@@ -797,6 +858,30 @@ function jarLikelyStale(result: ToolResult): boolean {
|
|
|
797
858
|
return !result.ok && (result.error === 'FORBIDDEN' || result.error === 'AUTH_EXPIRED');
|
|
798
859
|
}
|
|
799
860
|
|
|
861
|
+
function credentialSeedCookies(credentials: CredentialStore): MintedJar['cookies'] {
|
|
862
|
+
return credentials.cookies
|
|
863
|
+
.filter((c) => c.name && c.value && c.domain)
|
|
864
|
+
.map((c) => ({
|
|
865
|
+
name: c.name,
|
|
866
|
+
value: c.value,
|
|
867
|
+
domain: c.domain,
|
|
868
|
+
path: c.path ?? '/',
|
|
869
|
+
expires: c.expires,
|
|
870
|
+
httpOnly: c.httpOnly,
|
|
871
|
+
secure: c.secure,
|
|
872
|
+
sameSite: c.sameSite,
|
|
873
|
+
}));
|
|
874
|
+
}
|
|
875
|
+
|
|
876
|
+
function uniqueSeedCookies(cookies: MintedJar['cookies']): MintedJar['cookies'] | undefined {
|
|
877
|
+
if (cookies.length === 0) return undefined;
|
|
878
|
+
const byScope = new Map<string, MintedJar['cookies'][number]>();
|
|
879
|
+
for (const c of cookies) {
|
|
880
|
+
byScope.set(`${c.domain}\t${c.path ?? '/'}\t${c.name}`, c);
|
|
881
|
+
}
|
|
882
|
+
return [...byScope.values()];
|
|
883
|
+
}
|
|
884
|
+
|
|
800
885
|
/**
|
|
801
886
|
* fetch-bootstrap rung — the API anti-bot path. Mint a validated session jar via
|
|
802
887
|
* cdp-browser (real Chrome, used ONLY to bootstrap), CLOSE the browser, then
|
|
@@ -812,6 +897,7 @@ async function runFetchBootstrap(
|
|
|
812
897
|
tool: ResolvedTool,
|
|
813
898
|
params: Record<string, string | number | boolean>,
|
|
814
899
|
callerState?: Record<string, unknown>,
|
|
900
|
+
credentialOverride?: CredentialStore,
|
|
815
901
|
): Promise<ToolResult> {
|
|
816
902
|
let baseUrl: string;
|
|
817
903
|
try {
|
|
@@ -825,12 +911,13 @@ async function runFetchBootstrap(
|
|
|
825
911
|
};
|
|
826
912
|
}
|
|
827
913
|
|
|
828
|
-
const credentials =
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
914
|
+
const credentials = credentialOverride ??
|
|
915
|
+
(await loadCredentialStore(tool.site)) ?? {
|
|
916
|
+
site: tool.site,
|
|
917
|
+
cookies: [],
|
|
918
|
+
values: {},
|
|
919
|
+
storage: [],
|
|
920
|
+
};
|
|
834
921
|
const paramsWithDefaults = withWorkflowDefaults(tool.workflow, params);
|
|
835
922
|
const bootstrapUrl = tool.workflow.bootstrap
|
|
836
923
|
? substituteString(tool.workflow.bootstrap.url, paramsWithDefaults, credentials, [])
|
|
@@ -948,6 +1035,7 @@ async function runCdpReplay(
|
|
|
948
1035
|
params: Record<string, string | number | boolean>,
|
|
949
1036
|
cdpPool?: Map<string, CdpBrowserFetch>,
|
|
950
1037
|
callerState?: Record<string, unknown>,
|
|
1038
|
+
credentialOverride?: CredentialStore,
|
|
951
1039
|
): Promise<ToolResult> {
|
|
952
1040
|
let baseUrl: string;
|
|
953
1041
|
try {
|
|
@@ -961,19 +1049,20 @@ async function runCdpReplay(
|
|
|
961
1049
|
};
|
|
962
1050
|
}
|
|
963
1051
|
|
|
964
|
-
const credentials =
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
1052
|
+
const credentials = credentialOverride ??
|
|
1053
|
+
(await loadCredentialStore(tool.site)) ?? {
|
|
1054
|
+
site: tool.site,
|
|
1055
|
+
cookies: [],
|
|
1056
|
+
values: {},
|
|
1057
|
+
storage: [],
|
|
1058
|
+
};
|
|
970
1059
|
const paramsWithDefaults = withWorkflowDefaults(tool.workflow, params);
|
|
971
1060
|
const bootstrapUrl = tool.workflow.bootstrap
|
|
972
1061
|
? substituteString(tool.workflow.bootstrap.url, paramsWithDefaults, credentials, [])
|
|
973
1062
|
: undefined;
|
|
974
1063
|
|
|
975
1064
|
const siteDir = pathResolve(tool.dir, '..');
|
|
976
|
-
const poolKey = tool.site;
|
|
1065
|
+
const poolKey = cdpReplayPoolKey(tool.site, baseUrl);
|
|
977
1066
|
const pooled = cdpPool?.get(poolKey);
|
|
978
1067
|
const ownsSession = !pooled;
|
|
979
1068
|
|
|
@@ -982,30 +1071,29 @@ async function runCdpReplay(
|
|
|
982
1071
|
log('cdp-replay: reusing pooled Chrome session');
|
|
983
1072
|
cf = pooled;
|
|
984
1073
|
} else {
|
|
985
|
-
|
|
986
|
-
//
|
|
987
|
-
//
|
|
988
|
-
//
|
|
989
|
-
//
|
|
990
|
-
// `_abck` to `~0~`, but the cross-origin credential POST is edge-403'd ("Failed
|
|
991
|
-
// to fetch", no ACAO). saveJar persists each cdp-replay session, so seeding would
|
|
992
|
-
// otherwise re-arm itself every run (initiate writes a jar that the next initiate
|
|
993
|
-
// seeds → cascade of 403s). Data tools still seed (they reuse an authed session).
|
|
1074
|
+
const seeds: MintedJar['cookies'] = [];
|
|
1075
|
+
// Cached replay jars are recording-derived and may be stale, so authenticate
|
|
1076
|
+
// workflows do not inherit them implicitly. Current credential-store cookies
|
|
1077
|
+
// are different: they represent durable device/session state and remain
|
|
1078
|
+
// available unless the caller intentionally supplies a clean credential store.
|
|
994
1079
|
if (tool.workflow.toolKind !== 'authenticate') {
|
|
995
1080
|
try {
|
|
996
1081
|
const rec = newestRecording(siteDir);
|
|
997
1082
|
let cached = loadJar(siteDir);
|
|
998
1083
|
if (cached && rec && rec.mtimeMs > cached.bootstrapEpoch) cached = null;
|
|
999
1084
|
if (!cached && seedJarFromRecording(siteDir, rec, bootstrapUrl)) cached = loadJar(siteDir);
|
|
1000
|
-
if (cached?.cookies.length)
|
|
1085
|
+
if (cached?.cookies.length) seeds.push(...cached.cookies);
|
|
1001
1086
|
} catch {
|
|
1002
1087
|
// best-effort
|
|
1003
1088
|
}
|
|
1004
1089
|
}
|
|
1090
|
+
seeds.push(...credentialSeedCookies(credentials));
|
|
1091
|
+
const seedCookies = uniqueSeedCookies(seeds);
|
|
1005
1092
|
cf = (cdpBrowserFetchFactoryForTest ?? createCdpBrowserFetch)({
|
|
1006
1093
|
baseUrl,
|
|
1007
1094
|
bootstrapUrl,
|
|
1008
1095
|
seedCookies,
|
|
1096
|
+
seedStorage: credentials.storage,
|
|
1009
1097
|
// Run HEADED for authentication: a strong anti-bot edge (e.g. Akamai Bot
|
|
1010
1098
|
// Manager) fingerprints headless Chrome beyond the `HeadlessChrome` UA token
|
|
1011
1099
|
// we strip, so a cross-origin credential POST that 403s headless passes
|
|
@@ -1053,6 +1141,13 @@ async function runCdpReplay(
|
|
|
1053
1141
|
credentials: bootstrappedCredentials,
|
|
1054
1142
|
initialState: { ...callerState, ...captureResult.state },
|
|
1055
1143
|
fetchImpl: cf.fetchImpl,
|
|
1144
|
+
browser:
|
|
1145
|
+
cf.navigate && cf.snapshotCookies
|
|
1146
|
+
? {
|
|
1147
|
+
navigate: cf.navigate.bind(cf),
|
|
1148
|
+
snapshotCookies: cf.snapshotCookies.bind(cf),
|
|
1149
|
+
}
|
|
1150
|
+
: undefined,
|
|
1056
1151
|
});
|
|
1057
1152
|
|
|
1058
1153
|
if (result.ok) {
|
|
@@ -1065,15 +1160,9 @@ async function runCdpReplay(
|
|
|
1065
1160
|
} finally {
|
|
1066
1161
|
if (!cdpPool && ownsSession) await cf.close();
|
|
1067
1162
|
}
|
|
1068
|
-
} else if (cdpPool && result.error === '
|
|
1069
|
-
//
|
|
1070
|
-
//
|
|
1071
|
-
// completion phase reuses the EXACT page that minted the challenge: the
|
|
1072
|
-
// server-side challenge state and any single-use in-page tokens are bound to
|
|
1073
|
-
// THIS session, so a fresh browser would reset them and the poll/complete
|
|
1074
|
-
// would 401. The caller (AuthVerifier) owns this pool and drains it in its
|
|
1075
|
-
// `finally`. Without this carve-out the generic `else` below evicts+closes
|
|
1076
|
-
// the session here and phase 2 silently starts cold → "tokens missing" 401.
|
|
1163
|
+
} else if (cdpPool && result.error === 'ACTION_REQUIRED') {
|
|
1164
|
+
// A paused auth action is healthy progress. Retain the browser so the next
|
|
1165
|
+
// declared action sees the same cookies, page, and browser-owned state.
|
|
1077
1166
|
if (ownsSession) cdpPool.set(poolKey, cf);
|
|
1078
1167
|
try {
|
|
1079
1168
|
const postJar = await cf.mintJar();
|
|
@@ -1082,14 +1171,31 @@ async function runCdpReplay(
|
|
|
1082
1171
|
// best-effort
|
|
1083
1172
|
}
|
|
1084
1173
|
// Deliberately do NOT close cf — the pool retains it for the completion phase.
|
|
1085
|
-
} else {
|
|
1086
|
-
|
|
1087
|
-
|
|
1088
|
-
|
|
1174
|
+
} else if (cdpPool) {
|
|
1175
|
+
let sessionAlive = !cdpToolResultImpliesDeadSession(result);
|
|
1176
|
+
if (!sessionAlive && cf.inspectPage) {
|
|
1177
|
+
try {
|
|
1178
|
+
await cf.inspectPage();
|
|
1179
|
+
sessionAlive = true;
|
|
1180
|
+
} catch {
|
|
1181
|
+
// A failed liveness probe confirms that the transport is unusable.
|
|
1182
|
+
}
|
|
1183
|
+
}
|
|
1184
|
+
|
|
1185
|
+
if (sessionAlive) {
|
|
1186
|
+
// A caller-owned pool also owns diagnostic failures. Keep the rendered
|
|
1187
|
+
// page available so the caller can inspect what the browser actually saw
|
|
1188
|
+
// before deciding whether to retry, revise the workflow, or reset state.
|
|
1189
|
+
if (ownsSession) cdpPool.set(poolKey, cf);
|
|
1190
|
+
} else {
|
|
1089
1191
|
cdpPool.delete(poolKey);
|
|
1090
1192
|
log('cdp-replay: evicted degraded session from pool');
|
|
1091
1193
|
await cf.close();
|
|
1092
1194
|
}
|
|
1195
|
+
} else {
|
|
1196
|
+
if (ownsSession) {
|
|
1197
|
+
await cf.close();
|
|
1198
|
+
}
|
|
1093
1199
|
}
|
|
1094
1200
|
|
|
1095
1201
|
return result;
|
|
@@ -1099,7 +1205,7 @@ async function runCdpReplay(
|
|
|
1099
1205
|
cdpPool.delete(poolKey);
|
|
1100
1206
|
log('cdp-replay: evicted dead session from pool');
|
|
1101
1207
|
}
|
|
1102
|
-
|
|
1208
|
+
await cf.close();
|
|
1103
1209
|
const msg = err instanceof Error ? err.message : String(err);
|
|
1104
1210
|
return { ok: false, error: 'NETWORK', message: `cdp-replay failed: ${msg}` };
|
|
1105
1211
|
}
|
|
@@ -1450,50 +1556,6 @@ function playbookPath(assetRoot: string, site: string, toolDir?: string): string
|
|
|
1450
1556
|
return pathResolve(assetRoot, site, 'playbook.yaml');
|
|
1451
1557
|
}
|
|
1452
1558
|
|
|
1453
|
-
/**
|
|
1454
|
-
* For a 2FA authenticate tool whose login runs on the playbook rung, the
|
|
1455
|
-
* playbook's success marker is the **2FA challenge state** (per the
|
|
1456
|
-
* auth-compile contract), not full authentication. Reshape that `ok: true`
|
|
1457
|
-
* into the same `AWAITING_2FA` signal the API rungs emit so every consumer
|
|
1458
|
-
* (teach, mcp-server) handles playbook- and API-reached 2FA uniformly — and
|
|
1459
|
-
* carry any best-effort `twoFactorContext` token the playbook captured (named
|
|
1460
|
-
* per `authConfig.twoFactorContext`) across the stateless initiate→submit_otp
|
|
1461
|
-
* gap. Only fires on the login/initiate action; submit_otp/complete run via
|
|
1462
|
-
* the fetch path, not the playbook.
|
|
1463
|
-
*/
|
|
1464
|
-
export function reshapePlaybookAuthResult(
|
|
1465
|
-
result: ToolResult,
|
|
1466
|
-
workflow: Workflow,
|
|
1467
|
-
params: Record<string, string | number | boolean>,
|
|
1468
|
-
): ToolResult {
|
|
1469
|
-
const authCfg = workflow.authConfig;
|
|
1470
|
-
const action = String(params.action ?? 'initiate');
|
|
1471
|
-
if (
|
|
1472
|
-
!result.ok ||
|
|
1473
|
-
workflow.toolKind !== 'authenticate' ||
|
|
1474
|
-
!authCfg ||
|
|
1475
|
-
authCfg.twoFactorType === 'none' ||
|
|
1476
|
-
action === 'submit_otp' ||
|
|
1477
|
-
action === 'complete'
|
|
1478
|
-
) {
|
|
1479
|
-
return result;
|
|
1480
|
-
}
|
|
1481
|
-
const data = (result.data ?? {}) as Record<string, unknown>;
|
|
1482
|
-
const ctx: Record<string, unknown> = {};
|
|
1483
|
-
for (const name of authCfg.twoFactorContext ?? []) {
|
|
1484
|
-
if (data && typeof data === 'object' && name in data && data[name] != null) {
|
|
1485
|
-
ctx[name] = data[name];
|
|
1486
|
-
}
|
|
1487
|
-
}
|
|
1488
|
-
return {
|
|
1489
|
-
ok: false,
|
|
1490
|
-
error: 'AWAITING_2FA',
|
|
1491
|
-
twoFactorType: authCfg.twoFactorType,
|
|
1492
|
-
twoFactorContext: Object.keys(ctx).length > 0 ? ctx : undefined,
|
|
1493
|
-
message: `2FA required (${authCfg.twoFactorType}) — login reached the 2FA challenge via the playbook rung.`,
|
|
1494
|
-
};
|
|
1495
|
-
}
|
|
1496
|
-
|
|
1497
1559
|
/**
|
|
1498
1560
|
* Compile-time integration-test convenience: dispatch a request through
|
|
1499
1561
|
* `runWithLadder` using only a `workflow.json` path. Avoids requiring an
|
|
@@ -1523,71 +1585,30 @@ export async function runWorkflowWithLadder(opts: {
|
|
|
1523
1585
|
/** Optional credential override; otherwise loaded from the credential
|
|
1524
1586
|
* store by executeWorkflow. */
|
|
1525
1587
|
credentials?: CredentialStore;
|
|
1526
|
-
/** Seed state for `${state.X}
|
|
1527
|
-
*
|
|
1528
|
-
* completion request can resolve a token the initiate response returned. */
|
|
1588
|
+
/** Seed state for `${state.X}`. AuthVerifier uses the action program's
|
|
1589
|
+
* declared continuation projection here. */
|
|
1529
1590
|
initialState?: Record<string, unknown>;
|
|
1530
1591
|
/** Caller-owned CDP pool. When provided, cdp-replay pools its live Chrome here
|
|
1531
1592
|
* (reused across calls that pass the SAME map) and the process-global idle
|
|
1532
1593
|
* close is NOT armed — the caller owns the browser's lifecycle and must drain
|
|
1533
|
-
* it.
|
|
1534
|
-
*
|
|
1594
|
+
* it. AuthVerifier uses this to preserve one browser across action
|
|
1595
|
+
* checkpoints. */
|
|
1535
1596
|
cdpPool?: Map<string, CdpBrowserFetch>;
|
|
1536
1597
|
/** Pin execution to a single rung, bypassing the parallel probe AND the winner
|
|
1537
|
-
* memo.
|
|
1538
|
-
* one live browser in `cdpPool` (which the AWAITING_2FA carve-out retains), so
|
|
1539
|
-
* phase 2 can reuse the exact session that minted the challenge. Left to the
|
|
1540
|
-
* probe, the FASTEST rung returning AWAITING_2FA wins (often fetch /
|
|
1541
|
-
* fetch-bootstrap), which can't persist the session → completion 401s. */
|
|
1598
|
+
* memo. AuthVerifier pins `cdp-replay` so all actions share one browser. */
|
|
1542
1599
|
forceBackend?: ConcreteBackend;
|
|
1543
1600
|
}): Promise<LadderResult> {
|
|
1544
1601
|
if (!existsSync(opts.workflowPath)) {
|
|
1545
1602
|
throw new Error(`runWorkflowWithLadder: workflow.json not found at ${opts.workflowPath}`);
|
|
1546
1603
|
}
|
|
1547
|
-
const
|
|
1548
|
-
const
|
|
1604
|
+
const tool = resolveWorkflowTool(opts.workflowPath, opts.credentials);
|
|
1605
|
+
const workflow = tool.workflow;
|
|
1606
|
+
const toolDir = tool.dir;
|
|
1549
1607
|
// assetRoot only matters for playbook-rung path resolution, which this
|
|
1550
1608
|
// ladder skips. Use a conventional value for completeness.
|
|
1551
1609
|
const assetRoot = pathResolve(toolDir, '..', '..');
|
|
1552
1610
|
|
|
1553
|
-
const
|
|
1554
|
-
site: workflow.site ?? '',
|
|
1555
|
-
dir: toolDir,
|
|
1556
|
-
workflow,
|
|
1557
|
-
toolFn: async (params, fnOpts) => {
|
|
1558
|
-
// Thread ALL execution opts the rungs pass — fetchImpl (stealth), and
|
|
1559
|
-
// crucially initialState + credentials minted by fetch-bootstrap's
|
|
1560
|
-
// Chrome navigation. The production generated tool fn (tool-loader path)
|
|
1561
|
-
// forwards these to executeWorkflow; this test/probe-path toolFn must do
|
|
1562
|
-
// the same, otherwise a bootstrap-block tool's csrf/session state is
|
|
1563
|
-
// silently dropped here and the integration test fails a workflow that
|
|
1564
|
-
// actually works in production — a false waiver.
|
|
1565
|
-
const o = fnOpts as
|
|
1566
|
-
| {
|
|
1567
|
-
fetchImpl?: typeof fetch;
|
|
1568
|
-
initialState?: Record<string, unknown>;
|
|
1569
|
-
credentials?: CredentialStore;
|
|
1570
|
-
}
|
|
1571
|
-
| undefined;
|
|
1572
|
-
return executeWorkflow({
|
|
1573
|
-
workflow,
|
|
1574
|
-
params: params as Record<string, string | number | boolean>,
|
|
1575
|
-
credentials: o?.credentials ?? opts.credentials,
|
|
1576
|
-
workflowPath: opts.workflowPath,
|
|
1577
|
-
fetchImpl: o?.fetchImpl,
|
|
1578
|
-
initialState: o?.initialState,
|
|
1579
|
-
});
|
|
1580
|
-
},
|
|
1581
|
-
};
|
|
1582
|
-
|
|
1583
|
-
// Authenticate tools may need the playbook rung: a login whose POST body is
|
|
1584
|
-
// browser-minted (encrypted credentials) can't be API-replayed, so the login
|
|
1585
|
-
// DOM steps must run in a real browser. runWithLadder skips playbook when no
|
|
1586
|
-
// playbook.yaml exists, so including it is safe for tools without one.
|
|
1587
|
-
const ladder: ConcreteBackend[] =
|
|
1588
|
-
workflow.toolKind === 'authenticate'
|
|
1589
|
-
? ['fetch', 'fetch-bootstrap', 'cdp-replay', 'stealth-fetch', 'playbook']
|
|
1590
|
-
: ['fetch', 'fetch-bootstrap', 'cdp-replay', 'stealth-fetch'];
|
|
1611
|
+
const ladder: ConcreteBackend[] = ['fetch', 'fetch-bootstrap', 'cdp-replay', 'stealth-fetch'];
|
|
1591
1612
|
|
|
1592
1613
|
const memoKey = `${tool.site}::${workflow.toolName}`;
|
|
1593
1614
|
let memoWinner = compileWinningBackend.get(memoKey);
|
|
@@ -1652,6 +1673,7 @@ export async function runWorkflowWithLadder(opts: {
|
|
|
1652
1673
|
skipBootstrapSplice: true,
|
|
1653
1674
|
cdpPool,
|
|
1654
1675
|
initialState: opts.initialState,
|
|
1676
|
+
credentials: opts.credentials,
|
|
1655
1677
|
});
|
|
1656
1678
|
}
|
|
1657
1679
|
|
|
@@ -1704,43 +1726,22 @@ export async function runWorkflowWithLadder(opts: {
|
|
|
1704
1726
|
const settled = await Promise.allSettled(
|
|
1705
1727
|
probeBackends.map(async (b) => {
|
|
1706
1728
|
const t0 = Date.now();
|
|
1707
|
-
// Keep a handle to the real backend run (the race's non-timeout arm) so a
|
|
1708
|
-
// backend that LOSES the deadline race — still launching Chrome in the
|
|
1709
|
-
// background — gets settled and its pooled browser drained, not leaked,
|
|
1710
|
-
// once the probe returns.
|
|
1711
1729
|
const inner = runWithLadder([b], tool, opts.params, assetRoot, stealthCache, {
|
|
1712
1730
|
skipBootstrapSplice: true,
|
|
1713
1731
|
cdpPool,
|
|
1714
1732
|
initialState: opts.initialState,
|
|
1733
|
+
credentials: opts.credentials,
|
|
1734
|
+
});
|
|
1735
|
+
const r = await settleProbeWithDeadline(inner, PROBE_TIMEOUT_MS, {
|
|
1736
|
+
result: { ok: false, error: 'NETWORK', message: 'probe deadline exceeded' },
|
|
1737
|
+
usedBackend: b,
|
|
1738
|
+
attempts: [],
|
|
1715
1739
|
});
|
|
1716
|
-
// A backend that finishes AFTER the probe returned (it lost the race but
|
|
1717
|
-
// is still cold-starting Chrome) pools its browser late — arm the idle
|
|
1718
|
-
// close so it's torn down rather than left lingering. (Caller-owned pool
|
|
1719
|
-
// drains itself, so don't arm the global idle close for it.)
|
|
1720
|
-
if (!usingCallerPool) void inner.finally(() => armCompileCdpIdleClose()).catch(() => {});
|
|
1721
|
-
const r = await Promise.race([
|
|
1722
|
-
inner,
|
|
1723
|
-
sleepMs(PROBE_TIMEOUT_MS).then(
|
|
1724
|
-
() =>
|
|
1725
|
-
({
|
|
1726
|
-
result: { ok: false, error: 'NETWORK', message: 'probe deadline exceeded' },
|
|
1727
|
-
usedBackend: b,
|
|
1728
|
-
attempts: [],
|
|
1729
|
-
}) as LadderResult,
|
|
1730
|
-
),
|
|
1731
|
-
]);
|
|
1732
1740
|
return { backend: b, result: r, durationMs: Date.now() - t0 };
|
|
1733
1741
|
}),
|
|
1734
1742
|
);
|
|
1735
1743
|
|
|
1736
|
-
|
|
1737
|
-
// means the login failed, and the playbook rung (only reached via the
|
|
1738
|
-
// sequential fallback below) is the browser-minted login's actual path.
|
|
1739
|
-
// Treating it as "reachable" would let a cdp-replay 401 win the probe and
|
|
1740
|
-
// the playbook would never run. AWAITING_2FA stays reachable (it IS success).
|
|
1741
|
-
const isAuthTool = tool.workflow.toolKind === 'authenticate';
|
|
1742
|
-
const probeReachable = (r: ToolResult): boolean =>
|
|
1743
|
-
isProbeReachable(r) && !(isAuthTool && !r.ok && r.error === 'AUTH_EXPIRED');
|
|
1744
|
+
const probeReachable = isProbeReachable;
|
|
1744
1745
|
const digest = settled.map((s, i) => {
|
|
1745
1746
|
const b = probeBackends[i];
|
|
1746
1747
|
if (s.status === 'rejected')
|
|
@@ -1779,6 +1780,7 @@ export async function runWorkflowWithLadder(opts: {
|
|
|
1779
1780
|
const seqResult = await runWithLadder(ladder, tool, opts.params, assetRoot, stealthCache, {
|
|
1780
1781
|
cdpPool,
|
|
1781
1782
|
initialState: opts.initialState,
|
|
1783
|
+
credentials: opts.credentials,
|
|
1782
1784
|
});
|
|
1783
1785
|
if (probeReachable(seqResult.result)) {
|
|
1784
1786
|
compileWinningBackend.set(memoKey, seqResult.usedBackend);
|
|
@@ -1801,6 +1803,7 @@ export async function runWorkflowWithLadder(opts: {
|
|
|
1801
1803
|
skipBootstrapSplice: true,
|
|
1802
1804
|
cdpPool,
|
|
1803
1805
|
initialState: opts.initialState,
|
|
1806
|
+
credentials: opts.credentials,
|
|
1804
1807
|
});
|
|
1805
1808
|
if (isProbeReachable(result.result)) {
|
|
1806
1809
|
compileWinningBackend.set(memoKey, result.usedBackend);
|
|
@@ -1816,6 +1819,51 @@ export async function runWorkflowWithLadder(opts: {
|
|
|
1816
1819
|
}
|
|
1817
1820
|
}
|
|
1818
1821
|
|
|
1822
|
+
/** Build the same in-memory tool used by compile-time integration execution,
|
|
1823
|
+
* without requiring the emitted index.ts that production discovery loads.
|
|
1824
|
+
* Backend probing reuses this adapter during final pre-emission verification. */
|
|
1825
|
+
export function resolveWorkflowTool(
|
|
1826
|
+
workflowPath: string,
|
|
1827
|
+
fallbackCredentials?: CredentialStore,
|
|
1828
|
+
): ResolvedTool {
|
|
1829
|
+
if (!existsSync(workflowPath)) {
|
|
1830
|
+
throw new Error(`resolveWorkflowTool: workflow.json not found at ${workflowPath}`);
|
|
1831
|
+
}
|
|
1832
|
+
const workflow = WorkflowSchema.parse(JSON.parse(readFileSync(workflowPath, 'utf8')));
|
|
1833
|
+
const toolDir = dirname(workflowPath);
|
|
1834
|
+
return {
|
|
1835
|
+
site: workflow.site ?? '',
|
|
1836
|
+
dir: toolDir,
|
|
1837
|
+
workflow,
|
|
1838
|
+
toolFn: async (params, fnOpts) => {
|
|
1839
|
+
// Thread ALL execution opts the rungs pass — fetchImpl (stealth), and
|
|
1840
|
+
// crucially initialState + credentials minted by fetch-bootstrap's
|
|
1841
|
+
// Chrome navigation. The production generated tool fn (tool-loader path)
|
|
1842
|
+
// forwards these to executeWorkflow; this test/probe-path toolFn must do
|
|
1843
|
+
// the same, otherwise a bootstrap-block tool's csrf/session state is
|
|
1844
|
+
// silently dropped here and the integration test fails a workflow that
|
|
1845
|
+
// actually works in production — a false waiver.
|
|
1846
|
+
const o = fnOpts as
|
|
1847
|
+
| {
|
|
1848
|
+
fetchImpl?: typeof fetch;
|
|
1849
|
+
browser?: BrowserNavigationTransport;
|
|
1850
|
+
initialState?: Record<string, unknown>;
|
|
1851
|
+
credentials?: CredentialStore;
|
|
1852
|
+
}
|
|
1853
|
+
| undefined;
|
|
1854
|
+
return executeWorkflow({
|
|
1855
|
+
workflow,
|
|
1856
|
+
params: params as Record<string, string | number | boolean>,
|
|
1857
|
+
credentials: o?.credentials ?? fallbackCredentials,
|
|
1858
|
+
workflowPath,
|
|
1859
|
+
fetchImpl: o?.fetchImpl,
|
|
1860
|
+
browser: o?.browser,
|
|
1861
|
+
initialState: o?.initialState,
|
|
1862
|
+
});
|
|
1863
|
+
},
|
|
1864
|
+
};
|
|
1865
|
+
}
|
|
1866
|
+
|
|
1819
1867
|
export interface RenderedRequest {
|
|
1820
1868
|
method: string;
|
|
1821
1869
|
/** Final, fully-substituted + transform-applied request URL. */
|
|
@@ -28,7 +28,6 @@ import { compactRequestContexts, requestContextDigest } from './request-context.
|
|
|
28
28
|
import { type ClassifiedValue, looksLikeOpaqueToken, looksLikeToken } from './session-diff.ts';
|
|
29
29
|
import type { SharedCompileContext, ToolCandidate } from './tool-candidates.ts';
|
|
30
30
|
import { setSpanAttributes, traced } from './tracing.ts';
|
|
31
|
-
import { TwoFactorTypeSchema } from './types.ts';
|
|
32
31
|
import type { Session } from './types.ts';
|
|
33
32
|
|
|
34
33
|
const PROMPTS_DIR = pathJoin(import.meta.dir, '..', '..', 'prompts');
|
|
@@ -206,12 +205,8 @@ type PerToolPlan = z.infer<typeof PerToolPlanSchema>;
|
|
|
206
205
|
export const AuthToolPlanSchema = z
|
|
207
206
|
.object({
|
|
208
207
|
toolName: z.string(),
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
twoFactorType: TwoFactorTypeSchema,
|
|
212
|
-
/** OTP only: initiate-response field names the completion request chains via
|
|
213
|
-
* ${state.X} (structural, from the recording). */
|
|
214
|
-
twoFactorContext: z.array(z.string()).default([]),
|
|
208
|
+
credentialRequestSeqs: z.array(z.number().int().nonnegative()).default([]),
|
|
209
|
+
authRequestSeqs: z.array(z.number().int().nonnegative()).default([]),
|
|
215
210
|
credentialNames: z.array(z.string()).default([]),
|
|
216
211
|
captures: z.array(AuthCaptureSchema).default([]),
|
|
217
212
|
notes: z.string().default(''),
|
|
@@ -1391,7 +1386,7 @@ export function deriveRequiredInputHints(payload: {
|
|
|
1391
1386
|
|
|
1392
1387
|
// auth — minted by the login flow's response (any NON-cookie location). A
|
|
1393
1388
|
// login-minted value sent back as a Cookie persists automatically via the
|
|
1394
|
-
// credential jar, so it needs no ${credential.X}
|
|
1389
|
+
// credential jar, so it needs no ${credential.X} persist contract.
|
|
1395
1390
|
if (ev.producerSeq != null && loginSeqs.has(ev.producerSeq) && !isCookieHeaderName(evHeader)) {
|
|
1396
1391
|
const name = nameFromLocationOrSuggestion(ev.location, ev.suggestedStateName);
|
|
1397
1392
|
push({
|