@foldspace_npm/harness 0.1.15 → 0.1.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +46 -19
- package/README.md +2 -2
- package/bin/attach.mjs +4 -1
- package/bin/inject.mjs +3 -0
- package/bin/observe.mjs +790 -0
- package/package.json +1 -1
- package/src/cdp-client.mjs +233 -0
- package/src/cli-help.mjs +2 -1
- package/src/cli-registry.mjs +53 -3
- package/src/observe-core.mjs +627 -0
- package/src/session-events.mjs +42 -0
- package/templates/agent-starter/README.md +2 -2
package/bin/observe.mjs
ADDED
|
@@ -0,0 +1,790 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* foldspace observe — read-only eyes on the Chrome that `foldspace inject`
|
|
4
|
+
* launched, so nobody hand-writes a CDP script to find where an action's data
|
|
5
|
+
* lives.
|
|
6
|
+
*
|
|
7
|
+
* foldspace observe pages
|
|
8
|
+
* foldspace observe menu
|
|
9
|
+
* foldspace observe screen [--click "<label>"] [--goto /path] [--match "deal stage"] [--wait 4000]
|
|
10
|
+
* foldspace observe read <path-or-url>
|
|
11
|
+
* foldspace observe auth [--wait 5000]
|
|
12
|
+
* foldspace observe styles
|
|
13
|
+
* foldspace observe screenshot [--out file.png]
|
|
14
|
+
* foldspace observe wait-login [--timeout 600]
|
|
15
|
+
*
|
|
16
|
+
* Every subcommand prints ONE JSON document. It is safe beside a running
|
|
17
|
+
* `foldspace attach`: it never enables the Fetch domain, only navigates for
|
|
18
|
+
* `screen`/`auth`, only clicks the app's own navigation, only sends GET, and
|
|
19
|
+
* prints names and shapes — never a header, cookie or storage value.
|
|
20
|
+
*/
|
|
21
|
+
import fs from "node:fs";
|
|
22
|
+
import path from "node:path";
|
|
23
|
+
|
|
24
|
+
import { CdpPage, parseArgs, printJson, readLaunchState, sleep } from "../src/cdp-client.mjs";
|
|
25
|
+
import {
|
|
26
|
+
NAV_ITEM_SELECTOR,
|
|
27
|
+
NAV_SCOPE_SELECTOR,
|
|
28
|
+
analyzeAuthHeaders,
|
|
29
|
+
buildAuthSummary,
|
|
30
|
+
clickRefusal,
|
|
31
|
+
displayUrl,
|
|
32
|
+
isCandidateRequest,
|
|
33
|
+
isSameSite,
|
|
34
|
+
isStandardHeader,
|
|
35
|
+
looksLoggedIn,
|
|
36
|
+
parseMatchWords,
|
|
37
|
+
pickByLabel,
|
|
38
|
+
planRead,
|
|
39
|
+
printableUrl,
|
|
40
|
+
rankCandidates,
|
|
41
|
+
readHint,
|
|
42
|
+
refuseWriteFlags,
|
|
43
|
+
sameOriginPath,
|
|
44
|
+
summarizeJson,
|
|
45
|
+
toHex,
|
|
46
|
+
} from "../src/observe-core.mjs";
|
|
47
|
+
import { recordEvent } from "../src/session-events.mjs";
|
|
48
|
+
import { resolveProjectDir } from "../src/upgrade.mjs";
|
|
49
|
+
|
|
50
|
+
const SUBCOMMANDS = {
|
|
51
|
+
pages: { values: [], flags: [] },
|
|
52
|
+
menu: { values: [], flags: [] },
|
|
53
|
+
screen: { values: ["click", "goto", "match", "wait"], flags: [] },
|
|
54
|
+
read: { values: [], flags: [] },
|
|
55
|
+
auth: { values: ["wait"], flags: [] },
|
|
56
|
+
styles: { values: [], flags: [] },
|
|
57
|
+
screenshot: { values: ["out"], flags: [] },
|
|
58
|
+
"wait-login": { values: ["timeout"], flags: [] },
|
|
59
|
+
};
|
|
60
|
+
|
|
61
|
+
const projectDir = resolveProjectDir();
|
|
62
|
+
|
|
63
|
+
class ObserveExit extends Error {
|
|
64
|
+
constructor(document, code) {
|
|
65
|
+
super(document.error || "observe");
|
|
66
|
+
this.document = document;
|
|
67
|
+
this.code = code;
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
function numberFlag(value, fallback, name) {
|
|
72
|
+
if (value === undefined) return fallback;
|
|
73
|
+
const parsed = Number(value);
|
|
74
|
+
if (!Number.isFinite(parsed) || parsed < 0) throw new Error(`--${name} takes a number.`);
|
|
75
|
+
return parsed;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
// ------------------------------------------------------------ page helpers
|
|
79
|
+
|
|
80
|
+
const VISIBLE_JS = `const visible = (el) => {
|
|
81
|
+
if (!el.getClientRects().length) return false;
|
|
82
|
+
const style = getComputedStyle(el);
|
|
83
|
+
return style.visibility !== "hidden" && style.display !== "none";
|
|
84
|
+
};`;
|
|
85
|
+
|
|
86
|
+
/**
|
|
87
|
+
* Run an expression in the top frame and every child frame of this target,
|
|
88
|
+
* each in an isolated world: the DOM is shared, the app's own JavaScript is
|
|
89
|
+
* not touched. Apps that render inside an iframe keep their menu there.
|
|
90
|
+
*/
|
|
91
|
+
async function evaluateInFrames(page, expression, { stopWhen } = {}) {
|
|
92
|
+
const tree = await page.send("Page.getFrameTree");
|
|
93
|
+
const frames = [];
|
|
94
|
+
(function walk(node) {
|
|
95
|
+
frames.push(node.frame);
|
|
96
|
+
(node.childFrames || []).forEach(walk);
|
|
97
|
+
})(tree.frameTree);
|
|
98
|
+
const values = [];
|
|
99
|
+
for (const frame of frames) {
|
|
100
|
+
try {
|
|
101
|
+
const world = await page.send("Page.createIsolatedWorld", {
|
|
102
|
+
frameId: frame.id,
|
|
103
|
+
worldName: "foldspace-observe",
|
|
104
|
+
});
|
|
105
|
+
const result = await page.send("Runtime.evaluate", {
|
|
106
|
+
contextId: world.executionContextId,
|
|
107
|
+
expression,
|
|
108
|
+
returnByValue: true,
|
|
109
|
+
});
|
|
110
|
+
if (result.exceptionDetails) continue;
|
|
111
|
+
values.push(result.result?.value);
|
|
112
|
+
if (stopWhen && stopWhen(result.result?.value)) break;
|
|
113
|
+
} catch {
|
|
114
|
+
// a frame that went away mid-read is not an error
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
return values;
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
/** Page code shared by `menu` (click = null) and `screen --click`. */
|
|
121
|
+
function navigationScript(clickLabel) {
|
|
122
|
+
return `(() => {
|
|
123
|
+
${pickByLabel}
|
|
124
|
+
${clickRefusal}
|
|
125
|
+
${VISIBLE_JS}
|
|
126
|
+
const labelOf = (el) =>
|
|
127
|
+
(el.innerText || el.textContent || "").replace(/\\s+/g, " ").trim() ||
|
|
128
|
+
(el.getAttribute("aria-label") || "").trim() ||
|
|
129
|
+
(el.getAttribute("title") || "").trim();
|
|
130
|
+
const kindOf = (el) => {
|
|
131
|
+
const role = el.getAttribute("role");
|
|
132
|
+
if (role === "menuitem" || role === "tab" || role === "link") return role;
|
|
133
|
+
return el.tagName === "A" ? "link" : "button";
|
|
134
|
+
};
|
|
135
|
+
const items = [];
|
|
136
|
+
const elements = [];
|
|
137
|
+
const seenElements = new Set();
|
|
138
|
+
const seenKeys = new Set();
|
|
139
|
+
for (const scope of document.querySelectorAll(${JSON.stringify(NAV_SCOPE_SELECTOR)})) {
|
|
140
|
+
for (const el of scope.querySelectorAll(${JSON.stringify(NAV_ITEM_SELECTOR)})) {
|
|
141
|
+
if (seenElements.has(el)) continue;
|
|
142
|
+
seenElements.add(el);
|
|
143
|
+
if (!visible(el)) continue;
|
|
144
|
+
// A <header> also heads dialogs, cards and table rows; controls inside
|
|
145
|
+
// those, or inside a form, are not the app's navigation.
|
|
146
|
+
if (el.closest("dialog, [role=dialog], [role=alertdialog], form, table, [role=row], article")) continue;
|
|
147
|
+
const label = labelOf(el);
|
|
148
|
+
if (!label || label.length > 60) continue;
|
|
149
|
+
let path = null;
|
|
150
|
+
const href = el.getAttribute("href");
|
|
151
|
+
if (href && href !== "#" && !/^javascript:/i.test(href)) {
|
|
152
|
+
try {
|
|
153
|
+
const url = new URL(href, location.href);
|
|
154
|
+
if (url.origin !== location.origin) continue; // leaves the app
|
|
155
|
+
path = url.pathname + url.search + url.hash;
|
|
156
|
+
} catch {
|
|
157
|
+
continue;
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
const key = label + " " + path;
|
|
161
|
+
if (seenKeys.has(key)) continue;
|
|
162
|
+
seenKeys.add(key);
|
|
163
|
+
items.push({
|
|
164
|
+
label,
|
|
165
|
+
kind: kindOf(el),
|
|
166
|
+
path,
|
|
167
|
+
submitsForm: el.tagName === "BUTTON" && Boolean(el.form) && (el.type || "submit") === "submit",
|
|
168
|
+
download: el.hasAttribute("download"),
|
|
169
|
+
expands:
|
|
170
|
+
el.hasAttribute("aria-expanded") || el.hasAttribute("aria-haspopup") || el.hasAttribute("aria-controls"),
|
|
171
|
+
});
|
|
172
|
+
elements.push(el);
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
const want = ${JSON.stringify(clickLabel ?? null)};
|
|
176
|
+
if (want === null) return { items };
|
|
177
|
+
const index = pickByLabel(items, want);
|
|
178
|
+
if (index < 0) {
|
|
179
|
+
const lower = want.toLowerCase();
|
|
180
|
+
const elsewhere = [...document.querySelectorAll("a, button, input[type=submit], [role=button], [role=menuitem], [role=tab], [role=link]")]
|
|
181
|
+
.some((el) => visible(el) && (labelOf(el) || el.value || "").toLowerCase().includes(lower));
|
|
182
|
+
return { clicked: false, outsideNavigation: elsewhere, available: items.map((item) => item.label) };
|
|
183
|
+
}
|
|
184
|
+
const refused = clickRefusal(items[index]);
|
|
185
|
+
if (refused) return { clicked: false, refused };
|
|
186
|
+
elements[index].click();
|
|
187
|
+
return { clicked: true, item: items[index] };
|
|
188
|
+
})()`;
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
// Origins this page was seen calling for its own data. `read` may go to these
|
|
192
|
+
// and nowhere else off-origin. Kept beside the launch state, never in docs/.
|
|
193
|
+
const ORIGINS_FILE = path.join(projectDir, ".foldspace-dev", "observe", "origins.json");
|
|
194
|
+
|
|
195
|
+
function loadObservedOrigins() {
|
|
196
|
+
try {
|
|
197
|
+
const list = JSON.parse(fs.readFileSync(ORIGINS_FILE, "utf8"));
|
|
198
|
+
return Array.isArray(list) ? list.filter((entry) => typeof entry === "string") : [];
|
|
199
|
+
} catch {
|
|
200
|
+
return [];
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
function rememberObservedOrigins(urls, pageUrl, hosts) {
|
|
205
|
+
const known = new Set(loadObservedOrigins());
|
|
206
|
+
const before = known.size;
|
|
207
|
+
for (const url of urls) {
|
|
208
|
+
// The page really called it, AND it looks like the same site. Neither
|
|
209
|
+
// alone is enough: analytics hosts are called too.
|
|
210
|
+
if (!isSameSite(url, pageUrl, hosts)) continue;
|
|
211
|
+
try {
|
|
212
|
+
known.add(new URL(url).origin);
|
|
213
|
+
} catch {}
|
|
214
|
+
}
|
|
215
|
+
if (known.size === before) return;
|
|
216
|
+
fs.mkdirSync(path.dirname(ORIGINS_FILE), { recursive: true });
|
|
217
|
+
fs.writeFileSync(ORIGINS_FILE, `${JSON.stringify([...known].slice(-20), null, 2)}\n`, "utf8");
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
const publicItem = ({ label, kind, path: itemPath }) => ({ label, kind, path: itemPath });
|
|
221
|
+
|
|
222
|
+
function recordNetwork(page) {
|
|
223
|
+
const requests = new Map();
|
|
224
|
+
const extraHeaders = new Map();
|
|
225
|
+
const inflight = new Set();
|
|
226
|
+
let lastActivity = Date.now();
|
|
227
|
+
page.on("Network.requestWillBeSent", (event) => {
|
|
228
|
+
lastActivity = Date.now();
|
|
229
|
+
inflight.add(event.requestId);
|
|
230
|
+
requests.set(event.requestId, {
|
|
231
|
+
id: event.requestId,
|
|
232
|
+
method: event.request.method,
|
|
233
|
+
url: event.request.url,
|
|
234
|
+
type: event.type,
|
|
235
|
+
headers: event.request.headers || {},
|
|
236
|
+
});
|
|
237
|
+
});
|
|
238
|
+
page.on("Network.requestWillBeSentExtraInfo", (event) => {
|
|
239
|
+
extraHeaders.set(event.requestId, event.headers || {});
|
|
240
|
+
});
|
|
241
|
+
page.on("Network.responseReceived", (event) => {
|
|
242
|
+
lastActivity = Date.now();
|
|
243
|
+
const request = requests.get(event.requestId);
|
|
244
|
+
if (!request) return;
|
|
245
|
+
request.status = event.response.status;
|
|
246
|
+
request.mimeType = event.response.mimeType;
|
|
247
|
+
request.type = event.type || request.type;
|
|
248
|
+
});
|
|
249
|
+
page.on("Network.loadingFinished", (event) => {
|
|
250
|
+
lastActivity = Date.now();
|
|
251
|
+
inflight.delete(event.requestId);
|
|
252
|
+
const request = requests.get(event.requestId);
|
|
253
|
+
if (request) request.finished = true;
|
|
254
|
+
});
|
|
255
|
+
page.on("Network.loadingFailed", (event) => {
|
|
256
|
+
inflight.delete(event.requestId);
|
|
257
|
+
});
|
|
258
|
+
return {
|
|
259
|
+
requests,
|
|
260
|
+
extraHeaders,
|
|
261
|
+
/** Wait until the network has been quiet for a moment, at most maxMs. */
|
|
262
|
+
async settle(maxMs) {
|
|
263
|
+
const started = Date.now();
|
|
264
|
+
while (Date.now() - started < maxMs) {
|
|
265
|
+
await sleep(150);
|
|
266
|
+
const quiet = inflight.size === 0 && Date.now() - lastActivity >= 1200;
|
|
267
|
+
if (quiet && Date.now() - started >= 1500) break;
|
|
268
|
+
}
|
|
269
|
+
},
|
|
270
|
+
async jsonBody(requestId) {
|
|
271
|
+
try {
|
|
272
|
+
const body = await page.send("Network.getResponseBody", { requestId }, { timeoutMs: 8000 });
|
|
273
|
+
const text = body.base64Encoded ? Buffer.from(body.body, "base64").toString("utf8") : body.body;
|
|
274
|
+
return JSON.parse(text);
|
|
275
|
+
} catch {
|
|
276
|
+
return undefined; // evicted, navigated away, or not JSON after all
|
|
277
|
+
}
|
|
278
|
+
},
|
|
279
|
+
};
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
const isApiCall = (request) => /^(xhr|fetch)$/i.test(request.type || "");
|
|
283
|
+
|
|
284
|
+
// ------------------------------------------------------------- subcommands
|
|
285
|
+
|
|
286
|
+
async function runPages() {
|
|
287
|
+
const state = readLaunchState(projectDir);
|
|
288
|
+
let targets;
|
|
289
|
+
try {
|
|
290
|
+
targets = await (await fetch(`http://127.0.0.1:${state.port}/json/list`)).json();
|
|
291
|
+
} catch {
|
|
292
|
+
throw new Error(
|
|
293
|
+
`The inject Chrome is not answering on :${state.port}. It was closed; run \`npm run inject\` again.`,
|
|
294
|
+
);
|
|
295
|
+
}
|
|
296
|
+
const pages = targets
|
|
297
|
+
.filter((target) => target.type === "page" && !String(target.url).startsWith("devtools://"))
|
|
298
|
+
.map((target) => ({ url: printableUrl(target.url), title: target.title }));
|
|
299
|
+
return { ok: true, pages };
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
async function runMenu(page) {
|
|
303
|
+
const perFrame = await evaluateInFrames(page, navigationScript(null));
|
|
304
|
+
const seen = new Set();
|
|
305
|
+
const items = [];
|
|
306
|
+
for (const entry of perFrame.flatMap((value) => value?.items || [])) {
|
|
307
|
+
const key = `${entry.label} ${entry.path}`;
|
|
308
|
+
if (seen.has(key) || items.length >= 60) continue;
|
|
309
|
+
seen.add(key);
|
|
310
|
+
items.push(publicItem(entry));
|
|
311
|
+
}
|
|
312
|
+
const url = await page.evaluate("return location.href;");
|
|
313
|
+
return {
|
|
314
|
+
ok: true,
|
|
315
|
+
url: printableUrl(url),
|
|
316
|
+
count: items.length,
|
|
317
|
+
items,
|
|
318
|
+
...(items.length ? {} : { note: "No visible navigation found. The app may still be loading, or showing a login page." }),
|
|
319
|
+
};
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
async function runScreen(page, flags) {
|
|
323
|
+
const waitMs = numberFlag(flags.wait, 4000, "wait");
|
|
324
|
+
const words = parseMatchWords(flags.match);
|
|
325
|
+
const before = await page.evaluate("return location.href;");
|
|
326
|
+
const destination = flags.goto !== undefined ? sameOriginPath(flags.goto, before) : null;
|
|
327
|
+
|
|
328
|
+
await page.send("Network.enable");
|
|
329
|
+
const network = recordNetwork(page);
|
|
330
|
+
let reached;
|
|
331
|
+
|
|
332
|
+
if (destination) {
|
|
333
|
+
await page.send("Page.navigate", { url: destination.url });
|
|
334
|
+
reached = { how: "goto", path: destination.path };
|
|
335
|
+
if (flags.click !== undefined) await network.settle(Math.min(waitMs, 3000));
|
|
336
|
+
}
|
|
337
|
+
if (flags.click !== undefined) {
|
|
338
|
+
const results = await evaluateInFrames(page, navigationScript(String(flags.click)), {
|
|
339
|
+
stopWhen: (value) => value?.clicked === true,
|
|
340
|
+
});
|
|
341
|
+
const hit = results.find((value) => value?.clicked);
|
|
342
|
+
if (!hit) {
|
|
343
|
+
const refused = results.find((value) => value?.refused);
|
|
344
|
+
const outside = results.some((value) => value?.outsideNavigation);
|
|
345
|
+
const error = refused
|
|
346
|
+
? refused.refused
|
|
347
|
+
: outside
|
|
348
|
+
? `"${flags.click}" is on the page but not in the app's navigation. observe only clicks navigation, never a control that could create, send, save or delete.`
|
|
349
|
+
: `No navigation item is labelled "${flags.click}".`;
|
|
350
|
+
throw new ObserveExit(
|
|
351
|
+
{
|
|
352
|
+
ok: false,
|
|
353
|
+
error,
|
|
354
|
+
available: [...new Set(results.flatMap((value) => value?.available || []))].slice(0, 40),
|
|
355
|
+
},
|
|
356
|
+
1,
|
|
357
|
+
);
|
|
358
|
+
}
|
|
359
|
+
reached = { how: "click", ...publicItem(hit.item) };
|
|
360
|
+
}
|
|
361
|
+
if (!reached) {
|
|
362
|
+
// Nothing to reach: reload so this screen's own requests are seen.
|
|
363
|
+
await page.send("Page.reload");
|
|
364
|
+
reached = { how: "reload" };
|
|
365
|
+
}
|
|
366
|
+
await network.settle(waitMs);
|
|
367
|
+
|
|
368
|
+
const url = await page.evaluate("return location.href;").catch(() => before);
|
|
369
|
+
const origin = new URL(url).origin;
|
|
370
|
+
const all = [...network.requests.values()];
|
|
371
|
+
const candidates = [];
|
|
372
|
+
const seenUrls = new Set();
|
|
373
|
+
for (const request of all) {
|
|
374
|
+
if (!request.finished || !isCandidateRequest(request) || seenUrls.has(request.url)) continue;
|
|
375
|
+
if (candidates.length >= 40) break;
|
|
376
|
+
seenUrls.add(request.url);
|
|
377
|
+
const json = await network.jsonBody(request.id);
|
|
378
|
+
if (json === undefined) continue;
|
|
379
|
+
const shape = summarizeJson(json);
|
|
380
|
+
candidates.push({
|
|
381
|
+
method: request.method,
|
|
382
|
+
path: displayUrl(request.url, origin),
|
|
383
|
+
status: request.status,
|
|
384
|
+
contentType: request.mimeType,
|
|
385
|
+
topLevelKeys: shape.topLevelKeys,
|
|
386
|
+
rows: shape.rows,
|
|
387
|
+
fieldNames: shape.fieldNames,
|
|
388
|
+
});
|
|
389
|
+
}
|
|
390
|
+
rememberObservedOrigins([...seenUrls], url, page.info.hosts);
|
|
391
|
+
const writes = new Map();
|
|
392
|
+
for (const request of all) {
|
|
393
|
+
if (!isApiCall(request) || /^(GET|HEAD|OPTIONS)$/i.test(request.method)) continue;
|
|
394
|
+
// No query either: a write's query string is as much payload as its body.
|
|
395
|
+
const entry = { method: request.method, path: displayUrl(request.url, origin, { query: false }) };
|
|
396
|
+
writes.set(`${entry.method} ${entry.path}`, entry);
|
|
397
|
+
}
|
|
398
|
+
const ranked = rankCandidates(candidates, words, 8);
|
|
399
|
+
return {
|
|
400
|
+
ok: true,
|
|
401
|
+
reached,
|
|
402
|
+
url: printableUrl(url),
|
|
403
|
+
matchedOn: words,
|
|
404
|
+
requestsSeen: all.filter(isApiCall).length,
|
|
405
|
+
candidates: ranked,
|
|
406
|
+
writesSeen: [...writes.values()].slice(0, 20),
|
|
407
|
+
...(ranked.length
|
|
408
|
+
? {}
|
|
409
|
+
: { note: "No successful JSON GET was seen. The data may load after another click, or arrive inside the HTML." }),
|
|
410
|
+
};
|
|
411
|
+
}
|
|
412
|
+
|
|
413
|
+
async function runRead(page, flags, rest) {
|
|
414
|
+
const pageUrl = await page.evaluate("return location.href;");
|
|
415
|
+
const plan = planRead({
|
|
416
|
+
target: rest[0],
|
|
417
|
+
flags,
|
|
418
|
+
extra: rest.slice(1),
|
|
419
|
+
pageUrl,
|
|
420
|
+
hosts: page.info.hosts,
|
|
421
|
+
observedOrigins: loadObservedOrigins(),
|
|
422
|
+
});
|
|
423
|
+
const result = await page.evaluate(
|
|
424
|
+
`${summarizeJson}
|
|
425
|
+
const response = await fetch(${JSON.stringify(plan.url)}, {
|
|
426
|
+
method: "GET",
|
|
427
|
+
credentials: "include",
|
|
428
|
+
headers: { Accept: "application/json" },
|
|
429
|
+
});
|
|
430
|
+
const text = await response.text();
|
|
431
|
+
const out = {
|
|
432
|
+
status: response.status,
|
|
433
|
+
ok: response.ok,
|
|
434
|
+
contentType: response.headers.get("content-type"),
|
|
435
|
+
json: false,
|
|
436
|
+
bytes: text.length,
|
|
437
|
+
};
|
|
438
|
+
try {
|
|
439
|
+
Object.assign(out, summarizeJson(JSON.parse(text)), { json: true });
|
|
440
|
+
} catch {}
|
|
441
|
+
return out;`,
|
|
442
|
+
);
|
|
443
|
+
const hint = readHint(result.status);
|
|
444
|
+
const { status, ok, ...shape } = result;
|
|
445
|
+
return {
|
|
446
|
+
ok,
|
|
447
|
+
status,
|
|
448
|
+
method: "GET",
|
|
449
|
+
path: displayUrl(plan.url, new URL(pageUrl).origin),
|
|
450
|
+
...shape,
|
|
451
|
+
...(hint ? { hint } : {}),
|
|
452
|
+
};
|
|
453
|
+
}
|
|
454
|
+
|
|
455
|
+
async function runAuth(page, flags) {
|
|
456
|
+
const waitMs = numberFlag(flags.wait, 5000, "wait");
|
|
457
|
+
await page.send("Network.enable");
|
|
458
|
+
const network = recordNetwork(page);
|
|
459
|
+
await page.send("Page.reload");
|
|
460
|
+
await network.settle(waitMs);
|
|
461
|
+
|
|
462
|
+
const url = await page.evaluate("return location.href;");
|
|
463
|
+
const origin = new URL(url).origin;
|
|
464
|
+
const calls = [...network.requests.values()].filter(
|
|
465
|
+
(request) => isApiCall(request) && isSameSite(request.url, url, page.info.hosts),
|
|
466
|
+
);
|
|
467
|
+
|
|
468
|
+
// Real values are gathered here to be COMPARED; none of them is returned.
|
|
469
|
+
const pageSources = await page.evaluate(`
|
|
470
|
+
const dump = (storage) => {
|
|
471
|
+
const out = {};
|
|
472
|
+
try {
|
|
473
|
+
for (let index = 0; index < storage.length && index < 200; index++) {
|
|
474
|
+
const key = storage.key(index);
|
|
475
|
+
out[key] = storage.getItem(key);
|
|
476
|
+
}
|
|
477
|
+
} catch {}
|
|
478
|
+
return out;
|
|
479
|
+
};
|
|
480
|
+
return {
|
|
481
|
+
localStorage: dump(window.localStorage),
|
|
482
|
+
sessionStorage: dump(window.sessionStorage),
|
|
483
|
+
metas: [...document.querySelectorAll("meta[name][content], meta[property][content]")].map((meta) => ({
|
|
484
|
+
name: meta.getAttribute("name") || meta.getAttribute("property"),
|
|
485
|
+
content: meta.getAttribute("content"),
|
|
486
|
+
})),
|
|
487
|
+
};`);
|
|
488
|
+
const cookies = await page
|
|
489
|
+
.send("Network.getCookies", { urls: [url] })
|
|
490
|
+
.then((result) => result.cookies || [])
|
|
491
|
+
.catch(() => []);
|
|
492
|
+
const responses = [];
|
|
493
|
+
for (const request of calls) {
|
|
494
|
+
if (!request.finished || !/json/i.test(request.mimeType || "") || responses.length >= 30) continue;
|
|
495
|
+
const json = await network.jsonBody(request.id);
|
|
496
|
+
if (json === undefined) continue;
|
|
497
|
+
responses.push({
|
|
498
|
+
request: `${request.method} ${displayUrl(request.url, origin, { query: false })}`,
|
|
499
|
+
json,
|
|
500
|
+
});
|
|
501
|
+
}
|
|
502
|
+
|
|
503
|
+
const observed = calls.map((request) => ({
|
|
504
|
+
method: request.method,
|
|
505
|
+
path: displayUrl(request.url, origin, { query: false }),
|
|
506
|
+
// ExtraInfo is what really went on the wire; the page-set names keep their case.
|
|
507
|
+
headers: { ...lowerKeys(network.extraHeaders.get(request.id), request.headers), ...request.headers },
|
|
508
|
+
}));
|
|
509
|
+
const headers = analyzeAuthHeaders(observed, { ...pageSources, cookies, responses });
|
|
510
|
+
|
|
511
|
+
// Does a plain cookie-only GET get through? Prefer a request the app dressed
|
|
512
|
+
// up with its own headers: that is the one whose answer tells us something.
|
|
513
|
+
const replayable = calls.filter((request) => request.finished && isCandidateRequest(request));
|
|
514
|
+
const custom = (request) => Object.keys(request.headers).filter((name) => !isStandardHeader(name));
|
|
515
|
+
const probe =
|
|
516
|
+
replayable.find((request) => custom(request).length && new URL(request.url).origin === origin) ||
|
|
517
|
+
replayable.find((request) => new URL(request.url).origin === origin) ||
|
|
518
|
+
replayable[0];
|
|
519
|
+
let cookieOnlyGet = null;
|
|
520
|
+
if (probe) {
|
|
521
|
+
const answer = await page
|
|
522
|
+
.evaluate(
|
|
523
|
+
`try {
|
|
524
|
+
const response = await fetch(${JSON.stringify(probe.url)}, {
|
|
525
|
+
method: "GET",
|
|
526
|
+
credentials: "include",
|
|
527
|
+
headers: { Accept: "application/json" },
|
|
528
|
+
});
|
|
529
|
+
return { status: response.status, ok: response.ok };
|
|
530
|
+
} catch (error) {
|
|
531
|
+
return { status: 0, ok: false, error: "blocked before it was sent (CORS)" };
|
|
532
|
+
}`,
|
|
533
|
+
)
|
|
534
|
+
.catch(() => ({ status: 0, ok: false }));
|
|
535
|
+
cookieOnlyGet = {
|
|
536
|
+
path: displayUrl(probe.url, origin, { query: false }),
|
|
537
|
+
...answer,
|
|
538
|
+
appSentHeaders: custom(probe),
|
|
539
|
+
};
|
|
540
|
+
}
|
|
541
|
+
|
|
542
|
+
return {
|
|
543
|
+
ok: true,
|
|
544
|
+
url: printableUrl(url),
|
|
545
|
+
requestsSeen: calls.length,
|
|
546
|
+
headers,
|
|
547
|
+
cookieNames: cookies.map((cookie) => ({ name: cookie.name, httpOnly: cookie.httpOnly })).slice(0, 30),
|
|
548
|
+
cookieOnlyGet,
|
|
549
|
+
summary: buildAuthSummary({ headers, cookieOnlyGet }),
|
|
550
|
+
};
|
|
551
|
+
}
|
|
552
|
+
|
|
553
|
+
/** Wire headers whose name the page did not already set (HTTP/2 lowercases them). */
|
|
554
|
+
function lowerKeys(wire = {}, pageSet = {}) {
|
|
555
|
+
const known = new Set(Object.keys(pageSet).map((name) => name.toLowerCase()));
|
|
556
|
+
return Object.fromEntries(Object.entries(wire).filter(([name]) => !known.has(name.toLowerCase())));
|
|
557
|
+
}
|
|
558
|
+
|
|
559
|
+
async function runStyles(page) {
|
|
560
|
+
const raw = await page.evaluate(`
|
|
561
|
+
${VISIBLE_JS}
|
|
562
|
+
const clear = (color) =>
|
|
563
|
+
!color || color === "transparent" || /,\\s*0\\)$/.test(color) || /\\/\\s*0\\)$/.test(color);
|
|
564
|
+
const top = (tally) => [...tally.entries()].sort((a, b) => b[1] - a[1]).map(([value]) => value);
|
|
565
|
+
const bump = (tally, value, weight = 1) => tally.set(value, (tally.get(value) || 0) + weight);
|
|
566
|
+
const body = getComputedStyle(document.body);
|
|
567
|
+
|
|
568
|
+
// What is actually painted behind the content, not what <body> declares.
|
|
569
|
+
const viewport = { width: innerWidth, height: innerHeight };
|
|
570
|
+
const behind = document
|
|
571
|
+
.elementsFromPoint(viewport.width / 2, viewport.height / 2)
|
|
572
|
+
.find((el) => {
|
|
573
|
+
const rect = el.getBoundingClientRect();
|
|
574
|
+
return (
|
|
575
|
+
rect.width >= viewport.width * 0.6 &&
|
|
576
|
+
rect.height >= viewport.height * 0.6 &&
|
|
577
|
+
!clear(getComputedStyle(el).backgroundColor)
|
|
578
|
+
);
|
|
579
|
+
});
|
|
580
|
+
// A short <body> still paints the whole canvas, so it is the next best answer.
|
|
581
|
+
const background =
|
|
582
|
+
[behind, document.body, document.documentElement]
|
|
583
|
+
.filter(Boolean)
|
|
584
|
+
.map((el) => getComputedStyle(el).backgroundColor)
|
|
585
|
+
.find((color) => !clear(color)) || "rgb(255, 255, 255)";
|
|
586
|
+
|
|
587
|
+
const fonts = new Map(), sizes = new Map(), colors = new Map();
|
|
588
|
+
for (const el of [...document.querySelectorAll("p, span, div, li, td, th, label, h1, h2, h3, h4, a")].slice(0, 3000)) {
|
|
589
|
+
const own = [...el.childNodes]
|
|
590
|
+
.filter((node) => node.nodeType === 3)
|
|
591
|
+
.map((node) => node.textContent.trim())
|
|
592
|
+
.join("");
|
|
593
|
+
if (!own || !visible(el)) continue;
|
|
594
|
+
const style = getComputedStyle(el);
|
|
595
|
+
bump(fonts, style.fontFamily, own.length);
|
|
596
|
+
bump(sizes, style.fontSize, own.length);
|
|
597
|
+
if (el.tagName !== "A") bump(colors, style.color, own.length);
|
|
598
|
+
}
|
|
599
|
+
|
|
600
|
+
const links = new Map();
|
|
601
|
+
for (const el of [...document.querySelectorAll("a[href]")].slice(0, 500)) {
|
|
602
|
+
if (visible(el) && clear(getComputedStyle(el).backgroundColor)) bump(links, getComputedStyle(el).color);
|
|
603
|
+
}
|
|
604
|
+
|
|
605
|
+
// Primary button: filled, button-sized, and the most saturated large one.
|
|
606
|
+
let button = null;
|
|
607
|
+
for (const el of document.querySelectorAll("button, [role=button], a, input[type=submit]")) {
|
|
608
|
+
if (!visible(el)) continue;
|
|
609
|
+
const style = getComputedStyle(el);
|
|
610
|
+
if (clear(style.backgroundColor) || style.backgroundColor === background) continue;
|
|
611
|
+
const rect = el.getBoundingClientRect();
|
|
612
|
+
const label = (el.innerText || el.value || "").trim();
|
|
613
|
+
if (!label || label.length > 40 || rect.height > 80 || rect.width > 420 || rect.height < 16) continue;
|
|
614
|
+
const channels = (style.backgroundColor.match(/[\\d.]+/g) || []).slice(0, 3).map(Number);
|
|
615
|
+
const chroma = (Math.max(...channels) - Math.min(...channels)) / 255;
|
|
616
|
+
const score = (chroma > 0.15 ? 1e7 : 0) + rect.width * rect.height;
|
|
617
|
+
if (!button || score > button.score) {
|
|
618
|
+
button = {
|
|
619
|
+
score,
|
|
620
|
+
background: style.backgroundColor,
|
|
621
|
+
color: style.color,
|
|
622
|
+
borderRadius: style.borderTopLeftRadius,
|
|
623
|
+
fontWeight: style.fontWeight,
|
|
624
|
+
};
|
|
625
|
+
}
|
|
626
|
+
}
|
|
627
|
+
|
|
628
|
+
const borders = new Map(), radii = new Map();
|
|
629
|
+
const boxes = "input:not([type=hidden]), select, textarea, table, th, td, section, article, [class*=card i], [class*=panel i], [class*=tile i]";
|
|
630
|
+
for (const el of [...document.querySelectorAll(boxes)].slice(0, 1500)) {
|
|
631
|
+
if (!visible(el)) continue;
|
|
632
|
+
const style = getComputedStyle(el);
|
|
633
|
+
for (const side of ["Top", "Bottom"]) {
|
|
634
|
+
if (parseFloat(style["border" + side + "Width"]) > 0 && style["border" + side + "Style"] !== "none" && !clear(style["border" + side + "Color"])) {
|
|
635
|
+
bump(borders, style["border" + side + "Color"]);
|
|
636
|
+
}
|
|
637
|
+
}
|
|
638
|
+
if (parseFloat(style.borderTopLeftRadius) > 0) bump(radii, style.borderTopLeftRadius);
|
|
639
|
+
}
|
|
640
|
+
|
|
641
|
+
return {
|
|
642
|
+
url: location.href,
|
|
643
|
+
body: { fontFamily: body.fontFamily, fontSize: body.fontSize, color: body.color },
|
|
644
|
+
fontFamily: top(fonts)[0] || body.fontFamily,
|
|
645
|
+
fontSize: top(sizes)[0] || body.fontSize,
|
|
646
|
+
text: top(colors)[0] || body.color,
|
|
647
|
+
textMuted: top(colors)[1] || null,
|
|
648
|
+
background,
|
|
649
|
+
link: top(links)[0] || null,
|
|
650
|
+
button,
|
|
651
|
+
border: top(borders)[0] || null,
|
|
652
|
+
radius: top(radii)[0] || null,
|
|
653
|
+
};`);
|
|
654
|
+
const { button } = raw;
|
|
655
|
+
return {
|
|
656
|
+
ok: true,
|
|
657
|
+
url: printableUrl(raw.url),
|
|
658
|
+
sampledFrom: "computed styles of visible elements",
|
|
659
|
+
fontFamily: raw.fontFamily,
|
|
660
|
+
fontSize: raw.fontSize,
|
|
661
|
+
text: toHex(raw.text),
|
|
662
|
+
textMuted: raw.textMuted ? toHex(raw.textMuted) : null,
|
|
663
|
+
background: toHex(raw.background),
|
|
664
|
+
link: raw.link ? toHex(raw.link) : null,
|
|
665
|
+
primaryButton: button
|
|
666
|
+
? {
|
|
667
|
+
background: toHex(button.background),
|
|
668
|
+
color: toHex(button.color),
|
|
669
|
+
borderRadius: button.borderRadius,
|
|
670
|
+
fontWeight: button.fontWeight,
|
|
671
|
+
}
|
|
672
|
+
: null,
|
|
673
|
+
border: raw.border ? toHex(raw.border) : null,
|
|
674
|
+
boxRadius: raw.radius,
|
|
675
|
+
body: { ...raw.body, color: toHex(raw.body.color) },
|
|
676
|
+
};
|
|
677
|
+
}
|
|
678
|
+
|
|
679
|
+
async function runScreenshot(page, flags) {
|
|
680
|
+
const stamp = new Date().toISOString().replace(/[:.]/g, "-");
|
|
681
|
+
const file =
|
|
682
|
+
typeof flags.out === "string"
|
|
683
|
+
? path.resolve(flags.out)
|
|
684
|
+
: path.join(projectDir, ".foldspace-dev", "observe", `${stamp}.png`);
|
|
685
|
+
await page.screenshot(file);
|
|
686
|
+
return { ok: true, path: file, url: printableUrl(await page.evaluate("return location.href;")) };
|
|
687
|
+
}
|
|
688
|
+
|
|
689
|
+
async function runWaitLogin(flags) {
|
|
690
|
+
const timeoutMs = numberFlag(flags.timeout, 600, "timeout") * 1000;
|
|
691
|
+
const { hosts } = readLaunchState(projectDir);
|
|
692
|
+
const started = Date.now();
|
|
693
|
+
let streak = 0;
|
|
694
|
+
let url = null;
|
|
695
|
+
let lastError = null;
|
|
696
|
+
while (Date.now() - started < timeoutMs) {
|
|
697
|
+
let page;
|
|
698
|
+
try {
|
|
699
|
+
page = await CdpPage.open({ projectDir });
|
|
700
|
+
const seen = await page.evaluate(
|
|
701
|
+
`${VISIBLE_JS}
|
|
702
|
+
return {
|
|
703
|
+
url: location.href,
|
|
704
|
+
hasPasswordField: [...document.querySelectorAll("input[type=password]")].some(visible),
|
|
705
|
+
};`,
|
|
706
|
+
{ timeoutMs: 5000 },
|
|
707
|
+
);
|
|
708
|
+
// Post-login URLs carry OAuth codes and the like.
|
|
709
|
+
url = printableUrl(seen.url);
|
|
710
|
+
lastError = null;
|
|
711
|
+
streak = looksLoggedIn({ ...seen, hosts }) ? streak + 1 : 0;
|
|
712
|
+
} catch (error) {
|
|
713
|
+
// Mid-navigation, or Chrome not up yet: simply not logged in this poll.
|
|
714
|
+
streak = 0;
|
|
715
|
+
lastError = error instanceof Error ? error.message : String(error);
|
|
716
|
+
} finally {
|
|
717
|
+
page?.close();
|
|
718
|
+
}
|
|
719
|
+
if (streak >= 2) {
|
|
720
|
+
recordEvent(projectDir, "login_seen", { url });
|
|
721
|
+
return { ok: true, loggedIn: true, url, waitedMs: Date.now() - started };
|
|
722
|
+
}
|
|
723
|
+
await sleep(1500);
|
|
724
|
+
}
|
|
725
|
+
throw new ObserveExit({ ok: false, loggedIn: false, url, ...(lastError ? { lastError } : {}) }, 2);
|
|
726
|
+
}
|
|
727
|
+
|
|
728
|
+
// -------------------------------------------------------------------- main
|
|
729
|
+
|
|
730
|
+
function splitEquals(argv) {
|
|
731
|
+
// `--wait=500` and `--wait 500` mean the same thing, as in bin/cli.mjs.
|
|
732
|
+
return argv.flatMap((token) => {
|
|
733
|
+
const at = token.indexOf("=");
|
|
734
|
+
return token.startsWith("--") && at > -1 ? [token.slice(0, at), token.slice(at + 1)] : [token];
|
|
735
|
+
});
|
|
736
|
+
}
|
|
737
|
+
|
|
738
|
+
async function main() {
|
|
739
|
+
const [sub, ...argv] = splitEquals(process.argv.slice(2));
|
|
740
|
+
const spec = SUBCOMMANDS[sub];
|
|
741
|
+
if (!spec) {
|
|
742
|
+
throw new Error(
|
|
743
|
+
`usage: foldspace observe <${Object.keys(SUBCOMMANDS).join("|")}> [options]` +
|
|
744
|
+
(sub ? ` (unknown subcommand '${sub}')` : ""),
|
|
745
|
+
);
|
|
746
|
+
}
|
|
747
|
+
const { flags, rest } = parseArgs(argv, spec.values);
|
|
748
|
+
if (sub === "read") refuseWriteFlags(flags); // before "unknown option": the reason matters
|
|
749
|
+
const unknown = Object.keys(flags).filter((name) => ![...spec.values, ...spec.flags].includes(name));
|
|
750
|
+
if (unknown.length) throw new Error(`unknown option '--${unknown[0]}' for observe ${sub}`);
|
|
751
|
+
for (const name of spec.values) {
|
|
752
|
+
if (name in flags && (flags[name] === undefined || String(flags[name]).startsWith("--"))) {
|
|
753
|
+
throw new Error(`option '--${name}' requires a value`);
|
|
754
|
+
}
|
|
755
|
+
}
|
|
756
|
+
if (sub !== "read" && rest.length) throw new Error(`unexpected argument '${rest[0]}'`);
|
|
757
|
+
|
|
758
|
+
recordEvent(projectDir, "observe", { sub });
|
|
759
|
+
if (sub === "pages") return runPages();
|
|
760
|
+
if (sub === "wait-login") return runWaitLogin(flags);
|
|
761
|
+
|
|
762
|
+
const page = await CdpPage.open({ projectDir });
|
|
763
|
+
try {
|
|
764
|
+
if (sub === "menu") return await runMenu(page);
|
|
765
|
+
if (sub === "screen") return await runScreen(page, flags);
|
|
766
|
+
if (sub === "read") return await runRead(page, flags, rest);
|
|
767
|
+
if (sub === "auth") return await runAuth(page, flags);
|
|
768
|
+
if (sub === "styles") return await runStyles(page);
|
|
769
|
+
return await runScreenshot(page, flags);
|
|
770
|
+
} finally {
|
|
771
|
+
page.close();
|
|
772
|
+
}
|
|
773
|
+
}
|
|
774
|
+
|
|
775
|
+
try {
|
|
776
|
+
const document = await main();
|
|
777
|
+
printJson(document);
|
|
778
|
+
// `read` follows the HTTP answer so a shell `&&` chain stops on a 401.
|
|
779
|
+
process.exitCode = document.ok === false ? 1 : 0;
|
|
780
|
+
} catch (error) {
|
|
781
|
+
if (error instanceof ObserveExit) {
|
|
782
|
+
printJson(error.document);
|
|
783
|
+
process.exitCode = error.code;
|
|
784
|
+
} else {
|
|
785
|
+
printJson({ ok: false, error: error instanceof Error ? error.message : String(error) });
|
|
786
|
+
process.exitCode = 1;
|
|
787
|
+
}
|
|
788
|
+
}
|
|
789
|
+
// A CDP socket that is still closing must not hold the process open.
|
|
790
|
+
setTimeout(() => process.exit(), 50).unref();
|