imprint-mcp 0.6.6 → 0.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +22 -0
- package/README.md +340 -16
- package/package.json +2 -2
- package/prompts/audit-agent.md +25 -4
- package/prompts/auth-compile-agent.md +195 -34
- package/prompts/build-planning.md +3 -3
- package/prompts/compile-agent.md +932 -180
- package/prompts/component-recovery.md +40 -0
- package/prompts/live-verifier-agent.md +10 -3
- package/prompts/master-teach-api-researcher.md +813 -0
- package/prompts/master-teach-baseline-mvp-review.md +217 -0
- package/prompts/master-teach-completion-review.md +188 -0
- package/prompts/master-teach-decision.md +910 -0
- package/prompts/master-teach-focused-planner.md +777 -0
- package/prompts/master-teach-parameter-advisor.md +58 -0
- package/prompts/master-teach-recording-evidence.md +19 -0
- package/prompts/master-teach-refine-plan.md +7 -0
- package/prompts/master-teach-tool-advisor.md +130 -0
- package/prompts/playbook-compilation.md +45 -54
- package/prompts/prereq-builder.md +28 -6
- package/prompts/prereq-planner.md +2 -2
- package/prompts/tool-candidate-detection.md +70 -39
- package/prompts/tool-planning.md +3 -3
- package/src/cli.ts +212 -164
- package/src/imprint/agent-tool.ts +15 -0
- package/src/imprint/agent.ts +49 -44
- package/src/imprint/api-research-agent.ts +933 -0
- package/src/imprint/audit.ts +84 -15
- package/src/imprint/auth-browser-discovery.ts +276 -0
- package/src/imprint/auth-compile-agent.ts +610 -251
- package/src/imprint/auth-compile-tools.ts +233 -73
- package/src/imprint/auth-discovery-evidence.ts +81 -0
- package/src/imprint/auth-plan.ts +28 -0
- package/src/imprint/auth-verifier.ts +239 -36
- package/src/imprint/authentication-interaction.ts +137 -0
- package/src/imprint/backend-ladder.ts +711 -395
- package/src/imprint/body-structure.ts +656 -0
- package/src/imprint/build-plan.ts +35 -345
- package/src/imprint/cdp-browser-fetch.ts +926 -82
- package/src/imprint/cdp-jar-cache.ts +7 -12
- package/src/imprint/chromium.ts +93 -29
- package/src/imprint/claude-cli-compile.ts +278 -320
- package/src/imprint/codex-cli-compile.ts +529 -350
- package/src/imprint/codex-sdk-turn.ts +102 -0
- package/src/imprint/compile-agent-types.ts +58 -15
- package/src/imprint/compile-agent.ts +222 -274
- package/src/imprint/compile-done-sentinel.ts +211 -0
- package/src/imprint/compile-provider-control.ts +401 -0
- package/src/imprint/compile-provider-recovery.ts +142 -0
- package/src/imprint/compile-strategy.ts +5 -0
- package/src/imprint/compile-tool-executor.ts +783 -0
- package/src/imprint/compile-tools.ts +1727 -3096
- package/src/imprint/compile-verification.ts +11 -1
- package/src/imprint/compile.ts +110 -29
- package/src/imprint/compiler-log.ts +28 -0
- package/src/imprint/compiler-process.ts +440 -0
- package/src/imprint/component-recovery.ts +120 -0
- package/src/imprint/concurrency.ts +43 -58
- package/src/imprint/cookie-jar.ts +117 -7
- package/src/imprint/credential-extract.ts +10 -90
- package/src/imprint/credential-lock.ts +146 -0
- package/src/imprint/credential-observation.ts +261 -0
- package/src/imprint/credential-store.ts +247 -44
- package/src/imprint/emit.ts +18 -2
- package/src/imprint/evidence-inspection.ts +55 -0
- package/src/imprint/freeform-redact.ts +34 -234
- package/src/imprint/import-module-fresh.ts +30 -0
- package/src/imprint/live-component-recovery.ts +73 -0
- package/src/imprint/live-finesse-runner.ts +299 -0
- package/src/imprint/live-verifier.ts +299 -358
- package/src/imprint/llm.ts +467 -422
- package/src/imprint/log.ts +15 -8
- package/src/imprint/login.ts +9 -0
- package/src/imprint/master-teach-agent-contracts.ts +912 -0
- package/src/imprint/master-teach-agents.ts +3065 -0
- package/src/imprint/master-teach-checks.ts +348 -0
- package/src/imprint/master-teach-controller.ts +7702 -0
- package/src/imprint/master-teach-plan.ts +1410 -0
- package/src/imprint/master-teach-prompt-projections.ts +349 -0
- package/src/imprint/master-teach-store.ts +961 -0
- package/src/imprint/mcp-cdp-sessions.ts +60 -0
- package/src/imprint/mcp-compile-server.ts +64 -1001
- package/src/imprint/mcp-live-verifier-server.ts +246 -190
- package/src/imprint/mcp-server.ts +46 -57
- package/src/imprint/native-agent-pass.ts +260 -0
- package/src/imprint/native-agent-usage.ts +141 -0
- package/src/imprint/native-teach-agents.ts +619 -0
- package/src/imprint/param-grounding.ts +488 -347
- package/src/imprint/playbook-runner.ts +254 -63
- package/src/imprint/prereq-builder.ts +55 -543
- package/src/imprint/private-file.ts +18 -0
- package/src/imprint/probe-backends.ts +129 -33
- package/src/imprint/provider-retry.ts +501 -0
- package/src/imprint/provider-terminal.ts +214 -0
- package/src/imprint/recording-request.ts +290 -1
- package/src/imprint/recording-verification.ts +444 -0
- package/src/imprint/redact.ts +33 -749
- package/src/imprint/refine-worker.ts +326 -0
- package/src/imprint/refine.ts +241 -0
- package/src/imprint/replay-capture.ts +41 -12
- package/src/imprint/replay-evidence.ts +1007 -0
- package/src/imprint/research-response-evidence.ts +45 -0
- package/src/imprint/runtime.ts +867 -138
- package/src/imprint/sensitive-keys.ts +0 -6
- package/src/imprint/session-merge.ts +103 -7
- package/src/imprint/stealth-chromium.ts +67 -7
- package/src/imprint/stealth-fetch.ts +25 -8
- package/src/imprint/stealth-token-cache.ts +16 -16
- package/src/imprint/teach-browser-resources.ts +88 -0
- package/src/imprint/teach-plan.ts +0 -242
- package/src/imprint/teach-research-memory.ts +229 -0
- package/src/imprint/teach-stage-progress.ts +38 -0
- package/src/imprint/teach-state.ts +47 -625
- package/src/imprint/teach.ts +12 -3699
- package/src/imprint/terminal-user-prompt.ts +42 -0
- package/src/imprint/tool-candidates.ts +58 -210
- package/src/imprint/tool-loader.ts +3 -0
- package/src/imprint/tool-plan.ts +1 -134
- package/src/imprint/tracing.ts +24 -7
- package/src/imprint/types.ts +113 -21
- package/src/imprint/user-interaction.ts +836 -0
- package/src/imprint/endpoint-key.ts +0 -23
|
@@ -10,6 +10,8 @@
|
|
|
10
10
|
* scope for v1.
|
|
11
11
|
*/
|
|
12
12
|
|
|
13
|
+
import { createHash } from 'node:crypto';
|
|
14
|
+
|
|
13
15
|
export interface RuntimeCookie {
|
|
14
16
|
name: string;
|
|
15
17
|
value: string;
|
|
@@ -21,6 +23,8 @@ export interface RuntimeCookie {
|
|
|
21
23
|
sameSite?: string;
|
|
22
24
|
hostOnly?: boolean;
|
|
23
25
|
creationIndex?: number;
|
|
26
|
+
/** Exact browser observation when an independent HTTP response changed this scope. */
|
|
27
|
+
httpResponse?: { browserSnapshotHash: string | null; deleted?: boolean };
|
|
24
28
|
}
|
|
25
29
|
|
|
26
30
|
export interface CookieLookupConstraints {
|
|
@@ -52,14 +56,95 @@ export class RuntimeCookieJar {
|
|
|
52
56
|
return this.cookies.map((c) => ({ ...c }));
|
|
53
57
|
}
|
|
54
58
|
|
|
59
|
+
/** Reconcile a complete browser snapshot, including cookies it removed. */
|
|
60
|
+
replace(cookies: RuntimeCookie[]): void {
|
|
61
|
+
this.cookies = [];
|
|
62
|
+
for (const cookie of cookies) this.setCookie(cookie);
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/** Restore actual browser state while retaining HTTP changes whose browser
|
|
66
|
+
* baseline has not changed since that response, including deletion markers. */
|
|
67
|
+
restoreBrowserSnapshot(cookies: RuntimeCookie[]): void {
|
|
68
|
+
const observed = new RuntimeCookieJar(cookies);
|
|
69
|
+
const independent = this.cookies.filter(
|
|
70
|
+
(cookie) =>
|
|
71
|
+
cookie.httpResponse !== undefined &&
|
|
72
|
+
cookie.httpResponse.browserSnapshotHash === observed.scopeHash(cookie),
|
|
73
|
+
);
|
|
74
|
+
this.replace([...cookies, ...independent]);
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
private scopeHash(cookie: RuntimeCookie): string | null {
|
|
78
|
+
const matching = this.cookies.find((entry) => cookieScopeKey(entry) === cookieScopeKey(cookie));
|
|
79
|
+
if (!matching) return null;
|
|
80
|
+
const { name, value, domain, path, expires, sameSite, hostOnly } = matching;
|
|
81
|
+
return createHash('sha256')
|
|
82
|
+
.update(
|
|
83
|
+
JSON.stringify({
|
|
84
|
+
name,
|
|
85
|
+
value,
|
|
86
|
+
domain,
|
|
87
|
+
path,
|
|
88
|
+
expires,
|
|
89
|
+
sameSite,
|
|
90
|
+
hostOnly,
|
|
91
|
+
secure: Boolean(matching.secure),
|
|
92
|
+
httpOnly: Boolean(matching.httpOnly),
|
|
93
|
+
}),
|
|
94
|
+
)
|
|
95
|
+
.digest('hex');
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
/** Reconcile browser changes while retaining independent HTTP Set-Cookie
|
|
99
|
+
* changes that the browser has not observed. The caller owns the previous
|
|
100
|
+
* browser snapshot separately from this mixed-transport jar. */
|
|
101
|
+
reconcileBrowserSnapshot(cookies: RuntimeCookie[], previous: RuntimeCookieJar): void {
|
|
102
|
+
const next = new RuntimeCookieJar(cookies);
|
|
103
|
+
const scope = cookieScopeKey;
|
|
104
|
+
const before = new Map(previous.cookies.map((cookie) => [scope(cookie), cookie]));
|
|
105
|
+
const after = new Map(next.cookies.map((cookie) => [scope(cookie), cookie]));
|
|
106
|
+
const current = new Map(this.cookies.map((cookie) => [scope(cookie), cookie]));
|
|
107
|
+
const equal = (a: RuntimeCookie | undefined, b: RuntimeCookie | undefined) => {
|
|
108
|
+
const fields: (keyof RuntimeCookie)[] = [
|
|
109
|
+
'name',
|
|
110
|
+
'value',
|
|
111
|
+
'domain',
|
|
112
|
+
'path',
|
|
113
|
+
'expires',
|
|
114
|
+
'httpOnly',
|
|
115
|
+
'secure',
|
|
116
|
+
'sameSite',
|
|
117
|
+
'hostOnly',
|
|
118
|
+
];
|
|
119
|
+
return (
|
|
120
|
+
a === b ||
|
|
121
|
+
Boolean(
|
|
122
|
+
a &&
|
|
123
|
+
b &&
|
|
124
|
+
fields.every((field) =>
|
|
125
|
+
field === 'secure' || field === 'httpOnly'
|
|
126
|
+
? Boolean(a[field]) === Boolean(b[field])
|
|
127
|
+
: a[field] === b[field],
|
|
128
|
+
),
|
|
129
|
+
)
|
|
130
|
+
);
|
|
131
|
+
};
|
|
132
|
+
this.cookies = this.cookies.filter((cookie) => {
|
|
133
|
+
const key = scope(cookie);
|
|
134
|
+
return !before.has(key) || after.has(key) || !equal(cookie, before.get(key));
|
|
135
|
+
});
|
|
136
|
+
for (const [key, cookie] of after) {
|
|
137
|
+
if (!equal(cookie, before.get(key)) || equal(current.get(key), before.get(key)))
|
|
138
|
+
this.setCookie(cookie);
|
|
139
|
+
}
|
|
140
|
+
previous.replace(cookies);
|
|
141
|
+
}
|
|
142
|
+
|
|
55
143
|
setCookie(cookie: RuntimeCookie): void {
|
|
56
144
|
const normalized = normalizeCookie(cookie);
|
|
57
|
-
const idx = this.cookies.findIndex(
|
|
58
|
-
(c) =>
|
|
59
|
-
c.name === normalized.name && c.domain === normalized.domain && c.path === normalized.path,
|
|
60
|
-
);
|
|
145
|
+
const idx = this.cookies.findIndex((c) => cookieScopeKey(c) === cookieScopeKey(normalized));
|
|
61
146
|
|
|
62
|
-
if (isExpired(normalized)) {
|
|
147
|
+
if (isExpired(normalized) && !normalized.httpResponse?.deleted) {
|
|
63
148
|
if (idx >= 0) this.cookies.splice(idx, 1);
|
|
64
149
|
return;
|
|
65
150
|
}
|
|
@@ -75,9 +160,20 @@ export class RuntimeCookieJar {
|
|
|
75
160
|
}
|
|
76
161
|
}
|
|
77
162
|
|
|
78
|
-
setCookieFromHeader(
|
|
163
|
+
setCookieFromHeader(
|
|
164
|
+
setCookie: string,
|
|
165
|
+
requestUrl: string,
|
|
166
|
+
browserSnapshot?: RuntimeCookieJar,
|
|
167
|
+
): void {
|
|
79
168
|
const parsed = parseSetCookie(setCookie, requestUrl);
|
|
80
|
-
if (parsed)
|
|
169
|
+
if (parsed) {
|
|
170
|
+
if (browserSnapshot)
|
|
171
|
+
parsed.httpResponse = {
|
|
172
|
+
browserSnapshotHash: browserSnapshot.scopeHash(parsed),
|
|
173
|
+
...(isExpired(parsed) ? { deleted: true } : {}),
|
|
174
|
+
};
|
|
175
|
+
this.setCookie(parsed);
|
|
176
|
+
}
|
|
81
177
|
}
|
|
82
178
|
|
|
83
179
|
getCookieHeader(url: string): string | null {
|
|
@@ -129,6 +225,7 @@ export class RuntimeCookieJar {
|
|
|
129
225
|
const nowSeconds = Date.now() / 1000;
|
|
130
226
|
|
|
131
227
|
return this.cookies.filter((c) => {
|
|
228
|
+
if (c.httpResponse?.deleted) return false;
|
|
132
229
|
if (c.expires !== undefined && c.expires <= nowSeconds) return false;
|
|
133
230
|
if (c.secure && !secure) return false;
|
|
134
231
|
if (!domainMatches(c, host)) return false;
|
|
@@ -199,6 +296,19 @@ function parseSetCookie(setCookie: string, requestUrl: string): RuntimeCookie |
|
|
|
199
296
|
return normalizeCookie(cookie);
|
|
200
297
|
}
|
|
201
298
|
|
|
299
|
+
/** Chromium stores host-only and domain cookies as distinct scopes even when
|
|
300
|
+
* their names, normalized domains, and paths match. */
|
|
301
|
+
export function cookieScopeKey(
|
|
302
|
+
cookie: Pick<RuntimeCookie, 'name' | 'domain' | 'path' | 'hostOnly'>,
|
|
303
|
+
): string {
|
|
304
|
+
return JSON.stringify([
|
|
305
|
+
normalizeDomain(cookie.domain),
|
|
306
|
+
cookie.hostOnly ?? !cookie.domain.startsWith('.'),
|
|
307
|
+
cookie.path || '/',
|
|
308
|
+
cookie.name,
|
|
309
|
+
]);
|
|
310
|
+
}
|
|
311
|
+
|
|
202
312
|
function normalizeCookie(cookie: RuntimeCookie): RuntimeCookie {
|
|
203
313
|
const expires = cookie.expires === -1 ? undefined : cookie.expires;
|
|
204
314
|
return {
|
|
@@ -36,7 +36,7 @@ export interface Replacement {
|
|
|
36
36
|
placeholder: string;
|
|
37
37
|
}
|
|
38
38
|
|
|
39
|
-
|
|
39
|
+
interface CredentialFinding {
|
|
40
40
|
kind: 'login-pair';
|
|
41
41
|
/** `username` for form-login pairs by default. Re-namable by the user. */
|
|
42
42
|
usernameName: string;
|
|
@@ -57,6 +57,11 @@ interface ExtractionResult {
|
|
|
57
57
|
replacements: Replacement[];
|
|
58
58
|
}
|
|
59
59
|
|
|
60
|
+
/** Existing workflow references are not newly recorded secret values. */
|
|
61
|
+
export function isCredentialReference(value: string): boolean {
|
|
62
|
+
return /^\$\{credential\.[^}]+\}$/.test(value);
|
|
63
|
+
}
|
|
64
|
+
|
|
60
65
|
/** Parsers are tried in this order on every request that has a body. Each
|
|
61
66
|
* one is side-effect-free and returns `null` when its input doesn't fit
|
|
62
67
|
* its expected framing — so trying JSON first on a form body, or form on
|
|
@@ -112,7 +117,7 @@ export function extractCredentials(session: Session): ExtractionResult {
|
|
|
112
117
|
confirmedByDom,
|
|
113
118
|
});
|
|
114
119
|
|
|
115
|
-
|
|
120
|
+
for (const replacement of [
|
|
116
121
|
{
|
|
117
122
|
requestSeq: req.seq,
|
|
118
123
|
location: found.usernameLocation,
|
|
@@ -125,7 +130,9 @@ export function extractCredentials(session: Session): ExtractionResult {
|
|
|
125
130
|
originalValue: found.passwordValue,
|
|
126
131
|
placeholder: '${credential.password}',
|
|
127
132
|
},
|
|
128
|
-
)
|
|
133
|
+
]) {
|
|
134
|
+
if (!isCredentialReference(replacement.originalValue)) replacements.push(replacement);
|
|
135
|
+
}
|
|
129
136
|
}
|
|
130
137
|
|
|
131
138
|
return { findings, replacements };
|
|
@@ -403,93 +410,6 @@ function collectFormSubmitUsernames(events: CapturedEvent[]): Set<string> {
|
|
|
403
410
|
return out;
|
|
404
411
|
}
|
|
405
412
|
|
|
406
|
-
/** Fallback credential extraction for logins the username+password pairer
|
|
407
|
-
* misses — passwordless / OTP-only flows (e.g. email + emailed code, magic
|
|
408
|
-
* link) where the only "credential" the user supplies is an identifier, and
|
|
409
|
-
* the second factor replaces the password. Driven by the build plan's
|
|
410
|
-
* `authTool`: we look ONLY in the declared login request(s) and map each
|
|
411
|
-
* planner-declared `credentialNames` entry to a value, gated by either an
|
|
412
|
-
* exact field-name match or a username-like field whose value the user
|
|
413
|
-
* actually typed into a form submit (DOM confirmation) — so a stray email
|
|
414
|
-
* from analytics is never mistaken for a credential. Returns the values plus
|
|
415
|
-
* redaction replacements so the redacted session can show `${credential.X}`.
|
|
416
|
-
* Form-urlencoded bodies only (the common shape for these legacy forms). */
|
|
417
|
-
export function deriveLoginCredentials(
|
|
418
|
-
session: Session,
|
|
419
|
-
loginRequestSeqs: number[],
|
|
420
|
-
credentialNames: string[],
|
|
421
|
-
): { values: Record<string, string>; replacements: Replacement[] } {
|
|
422
|
-
const values: Record<string, string> = {};
|
|
423
|
-
const replacements: Replacement[] = [];
|
|
424
|
-
if (loginRequestSeqs.length === 0 || credentialNames.length === 0) {
|
|
425
|
-
return { values, replacements };
|
|
426
|
-
}
|
|
427
|
-
const typedInDom = collectFormSubmitUsernames(session.events);
|
|
428
|
-
const seqs = new Set(loginRequestSeqs);
|
|
429
|
-
const norm = (s: string) => s.toLowerCase().replace(/[^a-z0-9]/g, '');
|
|
430
|
-
for (const req of session.requests) {
|
|
431
|
-
if (!seqs.has(req.seq) || !req.body) continue;
|
|
432
|
-
const pairs = parseFormBody(req.body);
|
|
433
|
-
if (pairs.length === 0) continue;
|
|
434
|
-
for (const name of credentialNames) {
|
|
435
|
-
if (values[name]) continue;
|
|
436
|
-
// 1. Exact-ish field-name match (e.g. credentialName "email" → key "email").
|
|
437
|
-
let hit = pairs.find((p) => norm(p.key) === norm(name) && p.value.length > 0);
|
|
438
|
-
// 2. A username-like field whose value the user actually typed (DOM-confirmed).
|
|
439
|
-
if (!hit) {
|
|
440
|
-
hit = pairs.find(
|
|
441
|
-
(p) => isUsernameKey(p.key) && p.value.length > 0 && typedInDom.has(p.value),
|
|
442
|
-
);
|
|
443
|
-
}
|
|
444
|
-
if (hit) {
|
|
445
|
-
values[name] = hit.value;
|
|
446
|
-
replacements.push({
|
|
447
|
-
requestSeq: req.seq,
|
|
448
|
-
location: { kind: 'body-form', key: hit.key },
|
|
449
|
-
originalValue: hit.value,
|
|
450
|
-
placeholder: `\${credential.${name}}`,
|
|
451
|
-
});
|
|
452
|
-
}
|
|
453
|
-
}
|
|
454
|
-
}
|
|
455
|
-
return { values, replacements };
|
|
456
|
-
}
|
|
457
|
-
|
|
458
|
-
/** Swap form-field values in a session's request bodies for their
|
|
459
|
-
* `${credential.X}` placeholders, in place. Used to back-fill credential
|
|
460
|
-
* placeholders into an already-redacted session once the build plan has
|
|
461
|
-
* identified passwordless credential fields (see `deriveLoginCredentials`).
|
|
462
|
-
* Preserves the original key encoding; only the value is replaced. */
|
|
463
|
-
export function applyCredentialPlaceholders(session: Session, replacements: Replacement[]): void {
|
|
464
|
-
const bySeq = new Map<number, Map<string, string>>();
|
|
465
|
-
for (const r of replacements) {
|
|
466
|
-
if (r.location.kind !== 'body-form') continue;
|
|
467
|
-
const m = bySeq.get(r.requestSeq) ?? new Map<string, string>();
|
|
468
|
-
m.set(r.location.key, r.placeholder);
|
|
469
|
-
bySeq.set(r.requestSeq, m);
|
|
470
|
-
}
|
|
471
|
-
for (const req of session.requests) {
|
|
472
|
-
const keys = bySeq.get(req.seq);
|
|
473
|
-
if (!keys || !req.body) continue;
|
|
474
|
-
req.body = req.body
|
|
475
|
-
.split('&')
|
|
476
|
-
.map((pair) => {
|
|
477
|
-
const eq = pair.indexOf('=');
|
|
478
|
-
if (eq === -1) return pair;
|
|
479
|
-
const rawK = pair.slice(0, eq);
|
|
480
|
-
let k: string;
|
|
481
|
-
try {
|
|
482
|
-
k = decodeURIComponent(rawK);
|
|
483
|
-
} catch {
|
|
484
|
-
k = rawK;
|
|
485
|
-
}
|
|
486
|
-
const placeholder = keys.get(k);
|
|
487
|
-
return placeholder ? `${rawK}=${placeholder}` : pair;
|
|
488
|
-
})
|
|
489
|
-
.join('&');
|
|
490
|
-
}
|
|
491
|
-
}
|
|
492
|
-
|
|
493
413
|
/** Parse `a=1&b=2` into pairs, URL-decoding both sides. Best-effort: bad
|
|
494
414
|
* pairs get skipped. */
|
|
495
415
|
export function parseFormBody(body: string): Array<{ key: string; value: string }> {
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
import { execFileSync } from 'node:child_process';
|
|
2
|
+
import { randomUUID } from 'node:crypto';
|
|
3
|
+
import {
|
|
4
|
+
mkdirSync,
|
|
5
|
+
readFileSync,
|
|
6
|
+
readdirSync,
|
|
7
|
+
renameSync,
|
|
8
|
+
unlinkSync,
|
|
9
|
+
writeFileSync,
|
|
10
|
+
} from 'node:fs';
|
|
11
|
+
import { join } from 'node:path';
|
|
12
|
+
import { abortableDelay } from './concurrency.ts';
|
|
13
|
+
|
|
14
|
+
interface Ticket {
|
|
15
|
+
pid: number;
|
|
16
|
+
started: string | null;
|
|
17
|
+
nonce: string;
|
|
18
|
+
/** null means the contender is still choosing its ticket number. */
|
|
19
|
+
ticket: number | null;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
function processStart(pid: number): string | null {
|
|
23
|
+
if (process.platform === 'win32') return null;
|
|
24
|
+
try {
|
|
25
|
+
return (
|
|
26
|
+
execFileSync('ps', ['-p', String(pid), '-o', 'lstart='], {
|
|
27
|
+
encoding: 'utf8',
|
|
28
|
+
stdio: ['ignore', 'pipe', 'ignore'],
|
|
29
|
+
timeout: 2_000,
|
|
30
|
+
}).trim() || null
|
|
31
|
+
);
|
|
32
|
+
} catch {
|
|
33
|
+
return null;
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
const ownStart = processStart(process.pid);
|
|
38
|
+
|
|
39
|
+
function ownerIsGone(owner: Ticket): boolean {
|
|
40
|
+
if (owner.pid === process.pid)
|
|
41
|
+
return owner.started !== null && ownStart !== null && owner.started !== ownStart;
|
|
42
|
+
try {
|
|
43
|
+
process.kill(owner.pid, 0);
|
|
44
|
+
} catch (error) {
|
|
45
|
+
return (error as NodeJS.ErrnoException).code === 'ESRCH';
|
|
46
|
+
}
|
|
47
|
+
if (!owner.started) return false;
|
|
48
|
+
const currentStart = processStart(owner.pid);
|
|
49
|
+
return currentStart !== null && currentStart !== owner.started;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
function removeOwned(path: string, nonce: string): void {
|
|
53
|
+
try {
|
|
54
|
+
const value = JSON.parse(readFileSync(path, 'utf8')) as Ticket;
|
|
55
|
+
if (value.nonce === nonce) unlinkSync(path);
|
|
56
|
+
} catch (error) {
|
|
57
|
+
if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error;
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
function readTickets(directory: string): Ticket[] {
|
|
62
|
+
const tickets: Ticket[] = [];
|
|
63
|
+
for (const name of readdirSync(directory)) {
|
|
64
|
+
if (!name.endsWith('.json')) continue;
|
|
65
|
+
const path = join(directory, name);
|
|
66
|
+
let owner: Ticket;
|
|
67
|
+
try {
|
|
68
|
+
owner = JSON.parse(readFileSync(path, 'utf8')) as Ticket;
|
|
69
|
+
} catch (error) {
|
|
70
|
+
if ((error as NodeJS.ErrnoException).code === 'ENOENT') continue;
|
|
71
|
+
throw error;
|
|
72
|
+
}
|
|
73
|
+
if (
|
|
74
|
+
!Number.isSafeInteger(owner.pid) ||
|
|
75
|
+
owner.pid <= 0 ||
|
|
76
|
+
!(owner.started === null || typeof owner.started === 'string') ||
|
|
77
|
+
name !== `${owner.nonce}.json` ||
|
|
78
|
+
!(owner.ticket === null || (Number.isSafeInteger(owner.ticket) && owner.ticket > 0))
|
|
79
|
+
) {
|
|
80
|
+
throw new Error('Invalid credential lock owner metadata');
|
|
81
|
+
}
|
|
82
|
+
if (ownerIsGone(owner)) {
|
|
83
|
+
// Each owner has a unique path. Recovering this dead owner cannot delete
|
|
84
|
+
// a replacement owner's lock, even when several processes notice it.
|
|
85
|
+
removeOwned(path, owner.nonce);
|
|
86
|
+
} else tickets.push(owner);
|
|
87
|
+
}
|
|
88
|
+
return tickets;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/** A cross-process mutex for one durable credential namespace. Unique ticket
|
|
92
|
+
* paths avoid stale-owner cleanup racing a new lock holder. There is no lease
|
|
93
|
+
* timeout: a live owner keeps the lock until it finishes or is cancelled. */
|
|
94
|
+
export async function withCredentialLock<T>(
|
|
95
|
+
directory: string,
|
|
96
|
+
signal: AbortSignal | undefined,
|
|
97
|
+
work: () => Promise<T>,
|
|
98
|
+
): Promise<T> {
|
|
99
|
+
signal?.throwIfAborted();
|
|
100
|
+
mkdirSync(directory, { recursive: true, mode: 0o700 });
|
|
101
|
+
const nonce = randomUUID();
|
|
102
|
+
const path = join(directory, `${nonce}.json`);
|
|
103
|
+
const temporary = join(directory, `${nonce}.tmp`);
|
|
104
|
+
const owner: Ticket = { pid: process.pid, started: ownStart, nonce, ticket: null };
|
|
105
|
+
const publish = () => {
|
|
106
|
+
writeFileSync(temporary, JSON.stringify(owner), { mode: 0o600 });
|
|
107
|
+
renameSync(temporary, path);
|
|
108
|
+
};
|
|
109
|
+
let result: T | undefined;
|
|
110
|
+
let failure: { error: unknown } | undefined;
|
|
111
|
+
try {
|
|
112
|
+
publish();
|
|
113
|
+
const tickets = readTickets(directory);
|
|
114
|
+
owner.ticket = Math.max(0, ...tickets.map(({ ticket }) => ticket ?? 0)) + 1;
|
|
115
|
+
publish();
|
|
116
|
+
while (true) {
|
|
117
|
+
signal?.throwIfAborted();
|
|
118
|
+
const ahead = readTickets(directory).some(
|
|
119
|
+
(other) =>
|
|
120
|
+
other.nonce !== nonce &&
|
|
121
|
+
(other.ticket === null ||
|
|
122
|
+
other.ticket < (owner.ticket ?? 0) ||
|
|
123
|
+
(other.ticket === owner.ticket && other.nonce < nonce)),
|
|
124
|
+
);
|
|
125
|
+
if (!ahead) break;
|
|
126
|
+
await abortableDelay(20, signal);
|
|
127
|
+
}
|
|
128
|
+
signal?.throwIfAborted();
|
|
129
|
+
result = await work();
|
|
130
|
+
} catch (error) {
|
|
131
|
+
failure = { error };
|
|
132
|
+
} finally {
|
|
133
|
+
try {
|
|
134
|
+
removeOwned(path, nonce);
|
|
135
|
+
} catch (error) {
|
|
136
|
+
failure ??= { error };
|
|
137
|
+
}
|
|
138
|
+
try {
|
|
139
|
+
unlinkSync(temporary);
|
|
140
|
+
} catch (error) {
|
|
141
|
+
if ((error as NodeJS.ErrnoException).code !== 'ENOENT') failure ??= { error };
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
if (failure) throw failure.error;
|
|
145
|
+
return result as T;
|
|
146
|
+
}
|
|
@@ -0,0 +1,261 @@
|
|
|
1
|
+
/** Optional, value-free credential path diagnostics. A private per-run key makes
|
|
2
|
+
* identities comparable across child processes without stable secret hashes. */
|
|
3
|
+
import { createHmac, randomBytes } from 'node:crypto';
|
|
4
|
+
import { appendFileSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
|
|
5
|
+
import { dirname } from 'node:path';
|
|
6
|
+
|
|
7
|
+
interface CredentialMaterial {
|
|
8
|
+
site: string;
|
|
9
|
+
cookies: unknown[];
|
|
10
|
+
values: Record<string, string>;
|
|
11
|
+
storage?: unknown[];
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
export interface CredentialObservation {
|
|
15
|
+
scope: string;
|
|
16
|
+
revision: string;
|
|
17
|
+
source: string;
|
|
18
|
+
cookieCount: number;
|
|
19
|
+
valueCount: number;
|
|
20
|
+
storageCount: number;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
export interface DispatchObservation {
|
|
24
|
+
invocationId: string;
|
|
25
|
+
requestIndex: number;
|
|
26
|
+
credentials: CredentialObservation;
|
|
27
|
+
/** Runtime jar after mechanical normalization and earlier response updates. */
|
|
28
|
+
effectiveCredentials?: CredentialObservation;
|
|
29
|
+
/** Exact runtime-prepared headers/url/body, before transport-owned changes. */
|
|
30
|
+
preparedRequestIdentity: string;
|
|
31
|
+
requestTargetIdentity: string;
|
|
32
|
+
preparedCookieHeaderIdentity?: string;
|
|
33
|
+
cookieSource: 'browser-owned' | 'runtime-prepared';
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
interface BrowserCookieStateObservation extends CredentialObservation {
|
|
37
|
+
cookieStates: Array<{ scopeIdentity: string; valueIdentity: string; stateIdentity: string }>;
|
|
38
|
+
seed?: { attempted: number; accepted: number; rejected: number; failed: number };
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
function canonical(value: unknown): string {
|
|
42
|
+
if (Array.isArray(value)) return `[${value.map(canonical).join(',')}]`;
|
|
43
|
+
if (value && typeof value === 'object') {
|
|
44
|
+
return `{${Object.entries(value)
|
|
45
|
+
.sort(([a], [b]) => a.localeCompare(b))
|
|
46
|
+
.filter(([, item]) => item !== undefined)
|
|
47
|
+
.map(([key, item]) => `${JSON.stringify(key)}:${canonical(item)}`)
|
|
48
|
+
.join(',')}}`;
|
|
49
|
+
}
|
|
50
|
+
return JSON.stringify(value) ?? 'null';
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
function diagnosticKey(): Buffer | undefined {
|
|
54
|
+
const path = process.env.IMPRINT_CREDENTIAL_DIAGNOSTICS_PATH;
|
|
55
|
+
if (!path) return undefined;
|
|
56
|
+
try {
|
|
57
|
+
const keyPath = `${path}.key`;
|
|
58
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
59
|
+
try {
|
|
60
|
+
writeFileSync(keyPath, randomBytes(32), { flag: 'wx', mode: 0o600 });
|
|
61
|
+
} catch (error) {
|
|
62
|
+
if ((error as NodeJS.ErrnoException).code !== 'EEXIST') return undefined;
|
|
63
|
+
}
|
|
64
|
+
const key = readFileSync(keyPath);
|
|
65
|
+
return key.length === 32 ? key : undefined;
|
|
66
|
+
} catch {
|
|
67
|
+
return undefined;
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
function identity(key: Buffer, value: unknown): string {
|
|
72
|
+
return createHmac('sha256', key).update(canonical(value)).digest('hex');
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
export function credentialObservation(
|
|
76
|
+
material: CredentialMaterial,
|
|
77
|
+
source: string,
|
|
78
|
+
): CredentialObservation | undefined {
|
|
79
|
+
const key = diagnosticKey();
|
|
80
|
+
if (!key) return undefined;
|
|
81
|
+
return {
|
|
82
|
+
scope: identity(key, 'credential-observation-scope'),
|
|
83
|
+
revision: identity(key, {
|
|
84
|
+
site: material.site,
|
|
85
|
+
cookies: material.cookies,
|
|
86
|
+
values: material.values,
|
|
87
|
+
storage: material.storage ?? [],
|
|
88
|
+
}),
|
|
89
|
+
source,
|
|
90
|
+
cookieCount: material.cookies.length,
|
|
91
|
+
valueCount: Object.keys(material.values).length,
|
|
92
|
+
storageCount: material.storage?.length ?? 0,
|
|
93
|
+
};
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/** Only accept typed metadata at this boundary; never raw credentials/requests. */
|
|
97
|
+
export function recordCredentialEvent(
|
|
98
|
+
event:
|
|
99
|
+
| 'load'
|
|
100
|
+
| 'commit'
|
|
101
|
+
| 'dispatch'
|
|
102
|
+
| 'response'
|
|
103
|
+
| 'browser-request'
|
|
104
|
+
| 'browser-wire'
|
|
105
|
+
| 'browser-response'
|
|
106
|
+
| 'browser-state',
|
|
107
|
+
facts:
|
|
108
|
+
| CredentialObservation
|
|
109
|
+
| DispatchObservation
|
|
110
|
+
| {
|
|
111
|
+
invocationId: string;
|
|
112
|
+
requestIndex: number;
|
|
113
|
+
status: number;
|
|
114
|
+
}
|
|
115
|
+
| BrowserCredentialObservation
|
|
116
|
+
| BrowserCookieStateObservation,
|
|
117
|
+
): void {
|
|
118
|
+
const path = process.env.IMPRINT_CREDENTIAL_DIAGNOSTICS_PATH;
|
|
119
|
+
if (!path) return;
|
|
120
|
+
try {
|
|
121
|
+
appendFileSync(
|
|
122
|
+
path,
|
|
123
|
+
`${JSON.stringify({
|
|
124
|
+
event,
|
|
125
|
+
at: new Date().toISOString(),
|
|
126
|
+
monotonicMs: performance.now(),
|
|
127
|
+
pid: process.pid,
|
|
128
|
+
...facts,
|
|
129
|
+
})}\n`,
|
|
130
|
+
{ mode: 0o600 },
|
|
131
|
+
);
|
|
132
|
+
} catch {
|
|
133
|
+
// Optional diagnostics cannot alter persistence or runtime outcomes.
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
/** Compare exact cookie scopes across seed/bootstrap/request boundaries without
|
|
138
|
+
* recording their names or values. These are facts, never auth verdicts. */
|
|
139
|
+
export function recordBrowserCookieState(
|
|
140
|
+
material: CredentialMaterial,
|
|
141
|
+
phase: 'cdp-before-seed' | 'cdp-after-bootstrap' | 'cdp-after-request',
|
|
142
|
+
seed?: BrowserCookieStateObservation['seed'],
|
|
143
|
+
): void {
|
|
144
|
+
const key = diagnosticKey();
|
|
145
|
+
const observation = credentialObservation(material, phase);
|
|
146
|
+
if (!key || !observation) return;
|
|
147
|
+
recordCredentialEvent('browser-state', {
|
|
148
|
+
...observation,
|
|
149
|
+
cookieStates: material.cookies.map((cookie) => {
|
|
150
|
+
const entry = cookie as Record<string, unknown>;
|
|
151
|
+
const domain = String(entry.domain ?? '');
|
|
152
|
+
const scope = {
|
|
153
|
+
name: entry.name,
|
|
154
|
+
domain: domain.replace(/^\./, ''),
|
|
155
|
+
path: entry.path ?? '/',
|
|
156
|
+
hostOnly: entry.hostOnly ?? !domain.startsWith('.'),
|
|
157
|
+
};
|
|
158
|
+
return {
|
|
159
|
+
scopeIdentity: identity(key, scope),
|
|
160
|
+
valueIdentity: identity(key, entry.value),
|
|
161
|
+
stateIdentity: identity(key, {
|
|
162
|
+
...scope,
|
|
163
|
+
value: entry.value,
|
|
164
|
+
secure: Boolean(entry.secure),
|
|
165
|
+
httpOnly: Boolean(entry.httpOnly),
|
|
166
|
+
sameSite: typeof entry.sameSite === 'string' ? entry.sameSite.toLowerCase() : undefined,
|
|
167
|
+
expires:
|
|
168
|
+
typeof entry.expires === 'number' && entry.expires > 0 ? entry.expires : undefined,
|
|
169
|
+
}),
|
|
170
|
+
};
|
|
171
|
+
}),
|
|
172
|
+
...(seed ? { seed } : {}),
|
|
173
|
+
});
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
export function dispatchObservation(input: {
|
|
177
|
+
invocationId: string;
|
|
178
|
+
requestIndex: number;
|
|
179
|
+
material: CredentialMaterial;
|
|
180
|
+
effectiveMaterial?: CredentialMaterial;
|
|
181
|
+
source: string;
|
|
182
|
+
request: { method: string; url: string; headers: Record<string, string>; body?: string };
|
|
183
|
+
browserOwned: boolean;
|
|
184
|
+
}): DispatchObservation | undefined {
|
|
185
|
+
const key = diagnosticKey();
|
|
186
|
+
const credentials = credentialObservation(input.material, input.source);
|
|
187
|
+
if (!key || !credentials) return undefined;
|
|
188
|
+
const observation: DispatchObservation = {
|
|
189
|
+
invocationId: input.invocationId,
|
|
190
|
+
requestIndex: input.requestIndex,
|
|
191
|
+
credentials,
|
|
192
|
+
...(input.effectiveMaterial
|
|
193
|
+
? { effectiveCredentials: credentialObservation(input.effectiveMaterial, 'runtime-jar') }
|
|
194
|
+
: {}),
|
|
195
|
+
preparedRequestIdentity: identity(key, input.request),
|
|
196
|
+
requestTargetIdentity: identity(key, {
|
|
197
|
+
method: input.request.method.toUpperCase(),
|
|
198
|
+
url: input.request.url,
|
|
199
|
+
}),
|
|
200
|
+
...(Object.entries(input.request.headers).find(
|
|
201
|
+
([name]) => name.toLowerCase() === 'cookie',
|
|
202
|
+
)?.[1] === undefined
|
|
203
|
+
? {}
|
|
204
|
+
: {
|
|
205
|
+
preparedCookieHeaderIdentity: identity(
|
|
206
|
+
key,
|
|
207
|
+
Object.entries(input.request.headers).find(
|
|
208
|
+
([name]) => name.toLowerCase() === 'cookie',
|
|
209
|
+
)?.[1],
|
|
210
|
+
),
|
|
211
|
+
}),
|
|
212
|
+
cookieSource: input.browserOwned ? 'browser-owned' : 'runtime-prepared',
|
|
213
|
+
};
|
|
214
|
+
recordCredentialEvent('dispatch', observation);
|
|
215
|
+
return observation;
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
interface BrowserCredentialObservation {
|
|
219
|
+
transport: 'cdp' | 'browser-http';
|
|
220
|
+
browserRequestIdentity: string;
|
|
221
|
+
requestTargetIdentity?: string;
|
|
222
|
+
wireHeadersIdentity?: string;
|
|
223
|
+
cookieHeaderIdentity?: string;
|
|
224
|
+
status?: number;
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
/** CDP's actual network header event; never retains header/cookie values or URL.
|
|
228
|
+
* Request and extra-info events may arrive in either order, so join opaque IDs. */
|
|
229
|
+
export function recordBrowserCredentialObservation(input: {
|
|
230
|
+
event: 'browser-request' | 'browser-wire' | 'browser-response';
|
|
231
|
+
browserId: string;
|
|
232
|
+
requestId: string;
|
|
233
|
+
transport?: 'cdp' | 'browser-http';
|
|
234
|
+
request?: { method?: string; url?: string };
|
|
235
|
+
headers?: Record<string, unknown>;
|
|
236
|
+
status?: number;
|
|
237
|
+
}): void {
|
|
238
|
+
const key = diagnosticKey();
|
|
239
|
+
if (!key) return;
|
|
240
|
+
const cookie = Object.entries(input.headers ?? {}).find(
|
|
241
|
+
([name]) => name.toLowerCase() === 'cookie',
|
|
242
|
+
)?.[1];
|
|
243
|
+
recordCredentialEvent(input.event, {
|
|
244
|
+
transport: input.transport ?? 'cdp',
|
|
245
|
+
browserRequestIdentity: identity(key, {
|
|
246
|
+
browserId: input.browserId,
|
|
247
|
+
requestId: input.requestId,
|
|
248
|
+
}),
|
|
249
|
+
...(input.request?.url
|
|
250
|
+
? {
|
|
251
|
+
requestTargetIdentity: identity(key, {
|
|
252
|
+
method: (input.request.method ?? 'GET').toUpperCase(),
|
|
253
|
+
url: input.request.url,
|
|
254
|
+
}),
|
|
255
|
+
}
|
|
256
|
+
: {}),
|
|
257
|
+
...(input.headers ? { wireHeadersIdentity: identity(key, input.headers) } : {}),
|
|
258
|
+
...(cookie === undefined ? {} : { cookieHeaderIdentity: identity(key, cookie) }),
|
|
259
|
+
...(input.status === undefined ? {} : { status: input.status }),
|
|
260
|
+
});
|
|
261
|
+
}
|