@rubytech/create-maxy-code 0.1.109 → 0.1.111

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (32) hide show
  1. package/dist/index.js +50 -150
  2. package/dist/snap-chromium.js +1 -2
  3. package/dist/uninstall.js +10 -9
  4. package/package.json +1 -1
  5. package/payload/platform/neo4j/schema.cypher +6 -3
  6. package/payload/platform/plugins/admin/PLUGIN.md +1 -0
  7. package/payload/platform/plugins/admin/mcp/dist/index.js +1 -1
  8. package/payload/platform/plugins/admin/mcp/dist/index.js.map +1 -1
  9. package/payload/platform/plugins/admin/skills/upgrade/SKILL.md +34 -0
  10. package/payload/platform/plugins/cloudflare/.claude-plugin/plugin.json +1 -1
  11. package/payload/platform/plugins/cloudflare/PLUGIN.md +9 -16
  12. package/payload/platform/plugins/cloudflare/mcp/dist/index.js +7 -12
  13. package/payload/platform/plugins/cloudflare/mcp/dist/index.js.map +1 -1
  14. package/payload/platform/plugins/cloudflare/references/dashboard-guide.md +3 -3
  15. package/payload/platform/plugins/cloudflare/references/manual-setup.md +19 -54
  16. package/payload/platform/plugins/cloudflare/references/reset-guide.md +29 -28
  17. package/payload/platform/plugins/cloudflare/skills/setup-tunnel/SKILL.md +29 -149
  18. package/payload/platform/plugins/docs/references/admin-session.md +1 -1
  19. package/payload/platform/plugins/docs/references/admin-ui.md +1 -1
  20. package/payload/platform/plugins/docs/references/cloudflare.md +20 -29
  21. package/payload/platform/plugins/docs/references/platform.md +1 -1
  22. package/payload/platform/plugins/docs/references/plugins-guide.md +1 -1
  23. package/payload/platform/plugins/docs/references/troubleshooting.md +1 -1
  24. package/payload/platform/scripts/check-no-task-id-leaks.mjs +1 -1
  25. package/payload/platform/templates/agents/admin/IDENTITY.md +1 -1
  26. package/payload/platform/templates/specialists/agents/personal-assistant.md +1 -1
  27. package/payload/platform/plugins/cloudflare/scripts/__tests__/tunnel-ingress.test.ts +0 -241
  28. package/payload/platform/plugins/cloudflare/scripts/list-cf-domains.sh +0 -98
  29. package/payload/platform/plugins/cloudflare/scripts/list-cf-domains.ts +0 -715
  30. package/payload/platform/plugins/cloudflare/scripts/reset-tunnel.sh +0 -107
  31. package/payload/platform/plugins/cloudflare/scripts/setup-tunnel.sh +0 -852
  32. package/payload/platform/plugins/cloudflare/scripts/tunnel-ingress.ts +0 -291
@@ -1,715 +0,0 @@
1
- // Raw CDP scrape of the operator's Cloudflare dashboard to discover the
2
- // domains attached to the logged-in account. Output on stdout is a JSON
3
- // `string[]`, sorted and deduped. Every failure mode exits non-zero with
4
- // a `reason=<enum>` token on stderr tagged [list-cf-domains].
5
- //
6
- // Why raw CDP, not playwright:
7
- // - `/json/new?about:blank` + WebSocket to the returned `webSocketDebuggerUrl`
8
- // is ~120 LOC including retries and state machine.
9
- // - Adds zero npm deps (Node 22 ships global `WebSocket`; no `ws`/`playwright`).
10
- // - CDP is already bound to 127.0.0.1 by vnc.sh; the runtime surface is
11
- // the same one setup-tunnel.sh uses for browser drive.
12
- //
13
- // Why URL-pattern extraction, not CSS selectors:
14
- // - Cloudflare's SPA uses hashed, version-churning class names — a selector
15
- // written today drifts within a dashboard redesign window.
16
- // - The `/<accountId>/<domain>` URL shape is load-bearing for CF's own
17
- // routing and cannot drift without breaking their entire dashboard; it
18
- // is the most stable surface to key off.
19
- //
20
- // Why href-only (no page-text fallback): an earlier
21
- // <main>-wide FQDN text walker ("Source B") was removed after a reproduction returned
22
- // `count=1` with a bogus string (`leg.interest` — a static footer FQDN-shape
23
- // that hit the regex). The walker's non-empty result short-circuited the
24
- // poll-to-success branch before zones hydrated 500 ms later, silently
25
- // returning wrong data with `reason=ok`. The walker's intended role —
26
- // "defend against CF redesign removing href links" — is already served by
27
- // the existing `empty-or-drift` body-dump branch when href scraping returns
28
- // nothing; a silent fallback that produces wrong answers is strictly worse
29
- // than a loud drift signal with a snapshotted HTML dump for diagnosis.
30
- //
31
- // Why poll-to-stable: the CF dashboard lazy-loads its zone list after the
32
- // initial document-ready signal. A naive "return on first non-empty poll"
33
- // can race into a partial result (e.g. first zone rendered at t=500 ms, all
34
- // zones at t=1000 ms). The `scrapeDomains` poll waits for the observed count
35
- // to stabilise across two consecutive iterations before returning.
36
- //
37
- // Contract with the caller (the agent invokes list-cf-domains.sh via Bash):
38
- // stdout on exit 0: JSON `string[]` (empty array means signed-in-but-empty)
39
- // stderr on any path: `[list-cf-domains] …` phase lines
40
- // exit 1: `reason=<enum>` on stderr; scrape-empty-but-no-error is exit 0
41
- // with body dump (the enum exists so the operator/agent can
42
- // distinguish "selector drift" from "empty account" — both
43
- // shapes arrive via stdout).
44
-
45
- import { readFileSync } from "node:fs";
46
- import { writeFile } from "node:fs/promises";
47
- import { resolve } from "node:path";
48
- import { homedir } from "node:os";
49
- import { fileURLToPath } from "node:url";
50
-
51
- // CDP endpoint is resolved lazily inside `main()` so importing this module
52
- // for in-process tests (`scrapeCurrentPage` / `scrapeDomains` under JSDOM)
53
- // does not require an installed brand on disk.
54
- //
55
- // source-of-truth contract:
56
- // Runtime: brand.json `cdpPort` at `${MAXY_PLATFORM_ROOT}/config/brand.json`
57
- // is authoritative. Wrapper exports MAXY_PLATFORM_ROOT and BRAND. Missing
58
- // env / file / field → loud-fail with one of three named reasons. A
59
- // silent CDP-port default would make every non-default brand fail
60
- // `cdp-unreachable`; runtime config never falls back silently
61
- // (NEO4J_URI sets the same precedent).
62
- //
63
- // Test overrides: when BOTH `LIST_CF_DOMAINS_CDP_HOST` and
64
- // `LIST_CF_DOMAINS_CDP_PORT` are set, they win over brand.json. The
65
- // existing ECONNREFUSED vitest forces `127.0.0.1:1` (RFC 6335-reserved)
66
- // this way without an installed brand. Production never sets these.
67
-
68
- // Phase budgets total 30s (enforced by the shell wrapper's `timeout`).
69
- // Individual steps get sub-budgets so an early stall does not eat the
70
- // whole envelope silently.
71
- const CONNECT_TIMEOUT_MS = 2_000;
72
- const NAVIGATE_TIMEOUT_MS = 15_000;
73
- const SIGNED_IN_POLL_MS = 10_000;
74
- const SCRAPE_POLL_MS = 10_000;
75
- const POLL_INTERVAL_MS = 500;
76
-
77
- type Reason =
78
- | "cdp-unreachable"
79
- | "target-create-failed"
80
- | "ws-connect-failed"
81
- | "not-signed-in"
82
- | "navigate-failed"
83
- | "table-wait-timeout"
84
- | "runtime-error"
85
- // config-class failures. The first three short-circuit before
86
- // any CDP work begins; the route maps them to `field='config'` so the
87
- // form renders a config-card naming the brand + path instead of a Retry
88
- // button (retrying re-fires the same wrongly-resolved spawn
89
- // pattern).
90
- | "brand-config-missing"
91
- | "cdp-port-unresolved";
92
-
93
- function logPhase(line: string): void {
94
- // Stderr: the agent-via-Bash invocation streams the script's stderr into
95
- // the PTY chat verbatim, so phase lines land in the operator's view
96
- // alongside any other script output.
97
- process.stderr.write(`[list-cf-domains] ${line}\n`);
98
- }
99
-
100
- function die(reason: Reason, detail = ""): never {
101
- logPhase(`phase=error reason=${reason}${detail ? ` detail="${detail.replace(/"/g, "'").slice(0, 300)}"` : ""}`);
102
- process.exit(1);
103
- }
104
-
105
- interface CdpEndpoint {
106
- host: string;
107
- port: number;
108
- source: "env-override" | "brand.json";
109
- }
110
-
111
- // single source of truth for the CDP endpoint. Called once from
112
- // `main()` so the JSDOM scrape test (which imports `scrapeCurrentPage`
113
- // directly without invoking `main`) does not need an installed brand.
114
- function resolveCdpEndpoint(): CdpEndpoint {
115
- const envHost = process.env.LIST_CF_DOMAINS_CDP_HOST;
116
- const envPortRaw = process.env.LIST_CF_DOMAINS_CDP_PORT;
117
- if (envHost !== undefined && envPortRaw !== undefined) {
118
- const envPort = Number(envPortRaw);
119
- if (!Number.isInteger(envPort) || envPort <= 0 || envPort > 65535) {
120
- die(
121
- "cdp-port-unresolved",
122
- `LIST_CF_DOMAINS_CDP_PORT="${envPortRaw}" is not a valid integer port`,
123
- );
124
- }
125
- return { host: envHost, port: envPort, source: "env-override" };
126
- }
127
-
128
- const brand = process.env.BRAND;
129
- if (!brand) {
130
- die(
131
- "brand-config-missing",
132
- "BRAND env var not set by wrapper — refusing to guess brand identity",
133
- );
134
- }
135
- const platformRoot = process.env.MAXY_PLATFORM_ROOT;
136
- if (!platformRoot) {
137
- die(
138
- "brand-config-missing",
139
- "MAXY_PLATFORM_ROOT env var not set — wrapper must export it before spawn",
140
- );
141
- }
142
- const brandPath = resolve(platformRoot, "config", "brand.json");
143
- let raw: string;
144
- try {
145
- raw = readFileSync(brandPath, "utf-8");
146
- } catch (err) {
147
- // Distinct emission (not via `die`) so the path is named on the same line
148
- // — config-card rendering on the route side keys off it.
149
- logPhase(
150
- `phase=error reason=brand-config-missing path=${brandPath} detail="${(err instanceof Error ? err.message : String(err)).replace(/"/g, "'").slice(0, 200)}"`,
151
- );
152
- process.exit(1);
153
- }
154
- let parsed: Record<string, unknown>;
155
- try {
156
- parsed = JSON.parse(raw) as Record<string, unknown>;
157
- } catch (err) {
158
- logPhase(
159
- `phase=error reason=brand-config-missing path=${brandPath} detail="parse failed: ${(err instanceof Error ? err.message : String(err)).replace(/"/g, "'").slice(0, 160)}"`,
160
- );
161
- process.exit(1);
162
- }
163
- const cdpPort = parsed.cdpPort;
164
- if (typeof cdpPort !== "number" || !Number.isInteger(cdpPort) || cdpPort <= 0 || cdpPort > 65535) {
165
- // Names the keys actually present so a schema-drift incident surfaces
166
- // immediately ("oh, brand.json renamed `cdpPort` to `cdp_port` in r123").
167
- const keys = Object.keys(parsed).join(",");
168
- logPhase(
169
- `phase=error reason=cdp-port-unresolved brand=${brand} json_keys=${keys}`,
170
- );
171
- process.exit(1);
172
- }
173
- // Host stays 127.0.0.1 on the runtime path — vnc.sh binds Chromium's CDP
174
- // server to localhost only, by construction. brand.json carries the port,
175
- // not the host, because the host has never varied across brands.
176
- return { host: "127.0.0.1", port: cdpPort, source: "brand.json" };
177
- }
178
-
179
- async function cdpVersion(host: string, port: number): Promise<void> {
180
- const controller = new AbortController();
181
- const timer = setTimeout(() => controller.abort(), CONNECT_TIMEOUT_MS);
182
- try {
183
- const res = await fetch(`http://${host}:${port}/json/version`, { signal: controller.signal });
184
- if (!res.ok) die("cdp-unreachable", `HTTP ${res.status}`);
185
- } catch (err) {
186
- die("cdp-unreachable", err instanceof Error ? err.message : String(err));
187
- } finally {
188
- clearTimeout(timer);
189
- }
190
- }
191
-
192
- interface CdpTarget {
193
- id: string;
194
- webSocketDebuggerUrl: string;
195
- }
196
-
197
- async function createTarget(host: string, port: number): Promise<CdpTarget> {
198
- // `PUT /json/new?<url>` returns JSON describing the target. The URL goes
199
- // as a query-string argument (not a ?url= key), matching Chromium's CDP
200
- // server. about:blank avoids a wasted navigation — the real navigate
201
- // happens over WS after we attach.
202
- const endpoint = `http://${host}:${port}/json/new?about:blank`;
203
- let res: Response;
204
- try {
205
- res = await fetch(endpoint, { method: "PUT", signal: AbortSignal.timeout(CONNECT_TIMEOUT_MS) });
206
- } catch (err) {
207
- die("target-create-failed", err instanceof Error ? err.message : String(err));
208
- }
209
- if (!res.ok) die("target-create-failed", `HTTP ${res.status}`);
210
- const target = (await res.json()) as Partial<CdpTarget>;
211
- if (!target.id || !target.webSocketDebuggerUrl) {
212
- die("target-create-failed", "response missing id or webSocketDebuggerUrl");
213
- }
214
- return target as CdpTarget;
215
- }
216
-
217
- async function closeTarget(host: string, port: number, id: string): Promise<void> {
218
- try {
219
- await fetch(`http://${host}:${port}/json/close/${id}`, {
220
- signal: AbortSignal.timeout(CONNECT_TIMEOUT_MS),
221
- });
222
- } catch {
223
- // Best-effort on cleanup — a leaked target is recovered by the next vnc.sh
224
- // restart; surfacing it as a failure would mask the real scrape error.
225
- }
226
- }
227
-
228
- // Minimal CDP WebSocket client. We only need three commands: Page.enable,
229
- // Runtime.enable, Page.navigate, Runtime.evaluate. A full client would
230
- // manage sessions / targets / events — we just need request/response with
231
- // per-id correlation and an optional event stream for Page.frameNavigated
232
- // (which we do not actually need because Runtime.evaluate on location.href
233
- // is the authoritative signal).
234
- class CdpClient {
235
- private ws: WebSocket;
236
- private nextId = 1;
237
- private pending = new Map<number, { resolve: (v: unknown) => void; reject: (e: Error) => void }>();
238
-
239
- private constructor(ws: WebSocket) {
240
- this.ws = ws;
241
- this.ws.addEventListener("message", (ev) => this.onMessage(ev.data as string));
242
- this.ws.addEventListener("error", () => {
243
- for (const { reject } of this.pending.values()) reject(new Error("WebSocket error"));
244
- this.pending.clear();
245
- });
246
- this.ws.addEventListener("close", () => {
247
- for (const { reject } of this.pending.values()) reject(new Error("WebSocket closed"));
248
- this.pending.clear();
249
- });
250
- }
251
-
252
- static async connect(url: string): Promise<CdpClient> {
253
- const ws = new WebSocket(url);
254
- return await new Promise((resolvePromise, rejectPromise) => {
255
- const timer = setTimeout(() => {
256
- ws.close();
257
- rejectPromise(new Error(`WebSocket connect timeout (${CONNECT_TIMEOUT_MS}ms)`));
258
- }, CONNECT_TIMEOUT_MS);
259
- ws.addEventListener("open", () => {
260
- clearTimeout(timer);
261
- resolvePromise(new CdpClient(ws));
262
- }, { once: true });
263
- ws.addEventListener("error", (ev) => {
264
- clearTimeout(timer);
265
- rejectPromise(new Error(`WebSocket error: ${(ev as ErrorEvent).message ?? "unknown"}`));
266
- }, { once: true });
267
- });
268
- }
269
-
270
- private onMessage(raw: string): void {
271
- let msg: { id?: number; result?: unknown; error?: { message?: string } };
272
- try {
273
- msg = JSON.parse(raw);
274
- } catch {
275
- return;
276
- }
277
- if (typeof msg.id !== "number") return;
278
- const waiter = this.pending.get(msg.id);
279
- if (!waiter) return;
280
- this.pending.delete(msg.id);
281
- if (msg.error) {
282
- waiter.reject(new Error(msg.error.message ?? "CDP error"));
283
- } else {
284
- waiter.resolve(msg.result);
285
- }
286
- }
287
-
288
- send<T = unknown>(method: string, params: Record<string, unknown> = {}): Promise<T> {
289
- const id = this.nextId++;
290
- return new Promise<T>((resolvePromise, rejectPromise) => {
291
- this.pending.set(id, {
292
- resolve: resolvePromise as (v: unknown) => void,
293
- reject: rejectPromise,
294
- });
295
- this.ws.send(JSON.stringify({ id, method, params }));
296
- });
297
- }
298
-
299
- close(): void {
300
- try { this.ws.close(); } catch { /* ws may already be closing */ }
301
- }
302
- }
303
-
304
- interface EvaluateResult {
305
- result: { type: string; value?: unknown; description?: string };
306
- exceptionDetails?: { text?: string; exception?: { description?: string } };
307
- }
308
-
309
- async function evaluate<T = unknown>(cdp: CdpClient, expression: string): Promise<T> {
310
- const res = await cdp.send<EvaluateResult>("Runtime.evaluate", {
311
- expression,
312
- returnByValue: true,
313
- awaitPromise: false,
314
- });
315
- if (res.exceptionDetails) {
316
- throw new Error(res.exceptionDetails.exception?.description ?? res.exceptionDetails.text ?? "evaluate threw");
317
- }
318
- return res.result.value as T;
319
- }
320
-
321
- // Detect whether the dashboard has redirected to a signed-in account path
322
- // (URL like `/<32-hex-accountId>/…`) or the unsigned-in `/login` flow.
323
- // Returns the accountId if signed in, null if still loading, false if
324
- // definitively signed-out.
325
- const ACCOUNT_ID_REGEX = /^\/([a-f0-9]{32})(?:\/|$)/;
326
-
327
- async function readAccountId(cdp: CdpClient): Promise<string | null | false> {
328
- const pathname = await evaluate<string>(cdp, "location.pathname");
329
- if (typeof pathname !== "string") return null;
330
- if (pathname.startsWith("/login") || pathname.startsWith("/sign-in")) return false;
331
- const match = pathname.match(ACCOUNT_ID_REGEX);
332
- return match ? match[1] : null;
333
- }
334
-
335
- async function waitForSignedIn(cdp: CdpClient): Promise<string> {
336
- const deadline = Date.now() + SIGNED_IN_POLL_MS;
337
- while (Date.now() < deadline) {
338
- const state = await readAccountId(cdp);
339
- if (state === false) die("not-signed-in", "dashboard redirected to login");
340
- if (typeof state === "string") {
341
- logPhase(`phase=dashboard-nav-complete accountId=${state.slice(0, 8)}…`);
342
- return state;
343
- }
344
- await new Promise((r) => setTimeout(r, POLL_INTERVAL_MS));
345
- }
346
- // Timed out without seeing either pattern — treat as not-signed-in to
347
- // prompt `tunnel-login`; a genuinely-slow dashboard that later resolves
348
- // would be a false-negative we accept in exchange for deterministic UX.
349
- die("not-signed-in", "no /<accountId>/… path reached within budget");
350
- }
351
-
352
- export interface ScrapeOutcome {
353
- reason: "ok" | "no-account-id";
354
- domains: string[];
355
- }
356
-
357
- // Pure scrape logic — extracts the zones on the current Cloudflare dashboard
358
- // page by matching `<a href="/<accountId>/<zone>…">` links. Exported for
359
- // direct in-process testing under JSDOM (see `list-cf-domains-scrape.test.ts`),
360
- // and serialised into `SCRAPE_EXPRESSION` below for execution inside the
361
- // operator's VNC Chromium via CDP `Runtime.evaluate`. The function takes its
362
- // document and location as arguments (rather than reading globals) so the
363
- // JSDOM test can pass in its window's document+location directly — test and
364
- // prod share the same implementation, with no duplication and no eval.
365
- //
366
- // Note: this function body is serialised via `Function.prototype.toString()`,
367
- // so it must be fully self-contained — no imports, no captures from outer
368
- // scope. Node-side helpers belong outside.
369
- export function scrapeCurrentPage(
370
- document: Document,
371
- location: Location,
372
- ): ScrapeOutcome {
373
- const accountIdMatch = location.pathname.match(/^\/([a-f0-9]{32})/);
374
- if (!accountIdMatch) return { reason: "no-account-id", domains: [] };
375
- const accountId = accountIdMatch[1];
376
-
377
- const out = new Set<string>();
378
- const pushIfDomain = (s: unknown): void => {
379
- if (typeof s !== "string") return;
380
- const t = s.trim().toLowerCase();
381
- if (
382
- !/^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?(?:\.[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?)+$/.test(
383
- t,
384
- )
385
- )
386
- return;
387
- if (t.endsWith(".cloudflare.com") || t === "cloudflare.com") return;
388
- out.add(t);
389
- };
390
-
391
- // The only scrape source: `/<accountId>/<host>` hrefs — the canonical CF
392
- // routing pattern the overview page uses to link each zone to its
393
- // management screens. The full-<main> FQDN text walker was removed
394
- // after it returned a bogus FQDN-shaped string from static page copy
395
- // (see module header). When this source returns empty, the caller's
396
- // `empty-or-drift` branch snapshots the page HTML for diagnosis — loud
397
- // drift signal, not silent wrong data.
398
- const hrefPrefix = "/" + accountId + "/";
399
- for (const a of document.querySelectorAll("a[href]")) {
400
- const href = a.getAttribute("href") || "";
401
- if (!href.startsWith(hrefPrefix)) continue;
402
- const tail = href.slice(hrefPrefix.length).split("?")[0].split("#")[0];
403
- const firstSegment = tail.split("/")[0];
404
- pushIfDomain(firstSegment);
405
- }
406
-
407
- return { reason: "ok", domains: Array.from(out).sort() };
408
- }
409
-
410
- // The serialised form the CDP Runtime.evaluate executes inside the operator's
411
- // VNC Chromium. Deriving it from `scrapeCurrentPage.toString()` means the
412
- // browser-side and the test-side share one implementation — drift between
413
- // them is impossible by construction.
414
- export const SCRAPE_EXPRESSION = `(${scrapeCurrentPage.toString()})(document, location)`;
415
-
416
- // Minimum surface `scrapeDomains` needs from the CDP client: execute a JS
417
- // expression in the target page and return the serialised value. Typing the
418
- // dependency at this width lets the vitest regression inject a mock without
419
- // constructing a full CdpClient + WebSocket.
420
- export type CdpEvaluator = (expression: string) => Promise<unknown>;
421
-
422
- // Trailing-run length (consecutive polls at `max_count`) the complete-line
423
- // asserts before it declares `unstable=false`. Two iterations at 500 ms each
424
- // = 1 s of continuous observation at the high-water mark. removed
425
- // the early-exit on this threshold (it shortcuts multi-zone accounts where
426
- // the first zone hydrates one poll ahead of the second — the plateau at the
427
- // intermediate count trips the threshold before the final zone arrives);
428
- // the threshold now only labels the exit, it does not gate it.
429
- const STABLE_POLL_THRESHOLD = 2;
430
-
431
- // Cap on `document.documentElement.outerHTML` captured into the dump file.
432
- // The observed Cloudflare dashboard outerHTML is ~1 MB; 5 MB is 5× headroom
433
- // and still fits comfortably in one `writeFile` syscall on the Pi. An earlier
434
- // 100 KB cap truncated the dump mid-`<head>` CSS on every scrape,
435
- // rendering the forensic artifact silently broken: the
436
- // zone table lived past the cutoff and `grep <zone-host> <dump>` always
437
- // returned nothing, forcing re-scrapes to answer post-hoc questions.
438
- const OUTER_HTML_CAPTURE_MAX_CHARS = 5_000_000;
439
-
440
- // Cap the `domains=[…]` payload on per-poll and complete phase lines. The
441
- // extractor already implicitly bounds the list via the FQDN regex + Set
442
- // dedupe, but a pathological CF redesign could emit thousands of href
443
- // matches; the cap keeps the stream log readable without masking the real
444
- // capture size (the `count=` field carries the exact n).
445
- const DOMAINS_PAYLOAD_MAX_CHARS = 1000;
446
-
447
- function formatDomains(domains: string[]): string {
448
- const joined = domains.join(",");
449
- return joined.length > DOMAINS_PAYLOAD_MAX_CHARS
450
- ? joined.slice(0, DOMAINS_PAYLOAD_MAX_CHARS - 3) + "..."
451
- : joined;
452
- }
453
-
454
- type DumpMode = "stable" | "unstable" | "empty-or-drift";
455
- type DumpResult = { path: string; truncated: boolean } | { err: string };
456
-
457
- // Snapshot the operator's current dashboard HTML so post-hoc diagnosis of a
458
- // partial, unstable, or empty scrape has the exact DOM the scrape observed
459
- // at exit. Called on all three `scrapeDomains` exit paths — the filename's
460
- // `mode` field names which path fired, and the `pid<pid>` suffix prevents
461
- // collision if two invocations land in the same millisecond. Loud-fail
462
- // preserved: on any throw (evaluator rejected, CONFIG_DIR unset, writeFile
463
- // ENOENT), return `{err}` so the caller emits a separate
464
- // `phase=dump-write-failed` line and the complete line carries
465
- // `dump=failed`. The scrape's real signal must never be masked.
466
- async function dumpHtml(
467
- evaluator: CdpEvaluator,
468
- count: number,
469
- mode: DumpMode,
470
- ): Promise<DumpResult> {
471
- try {
472
- const html = (await evaluator(
473
- `document.documentElement.outerHTML.slice(0, ${OUTER_HTML_CAPTURE_MAX_CHARS})`,
474
- )) as string;
475
- // CONFIG_DIR is set by list-cf-domains.sh before the spawn. A silent
476
- // fallback would dump logs into the wrong brand's directory on a Real
477
- // Agent install — a silent-miswrite masking the wrapper-side break it
478
- // came from. doctrine: loud-fail on absent runtime-derived
479
- // values.
480
- const configDir = process.env.CONFIG_DIR;
481
- if (!configDir) {
482
- throw new Error(
483
- "CONFIG_DIR env var not set by wrapper — refusing to guess brand log directory",
484
- );
485
- }
486
- const logDir = resolve(homedir(), configDir, "logs");
487
- const ts = new Date().toISOString().replace(/[:.]/g, "-");
488
- const dumpPath = resolve(
489
- logDir,
490
- `list-cf-domains-${ts}-count${count}-${mode}-pid${process.pid}.html`,
491
- );
492
- const htmlStr = typeof html === "string" ? html : String(html);
493
- await writeFile(dumpPath, htmlStr, "utf-8");
494
- // Heuristic: if the slice returned exactly OUTER_HTML_CAPTURE_MAX_CHARS,
495
- // the source outerHTML almost certainly exceeded the cap and we
496
- // truncated mid-document. The false positive (outerHTML exactly at the
497
- // ceiling) is a rounding coincidence on the order of 1-in-5M and is
498
- // acceptable — a loud truncation signal with a ~10^-7 false-positive
499
- // rate is strictly better than silent truncation. A downstream
500
- // investigator seeing `truncated=true` can re-scrape with a larger
501
- // cap or `outerHTML` unsliced.
502
- const truncated = htmlStr.length >= OUTER_HTML_CAPTURE_MAX_CHARS;
503
- return { path: dumpPath, truncated };
504
- } catch (err) {
505
- return {
506
- err: (err instanceof Error ? err.message : String(err)).slice(0, 120),
507
- };
508
- }
509
- }
510
-
511
- // Render the `dump=<path>` field for a complete-line, and emit a separate
512
- // loud-fail `phase=dump-write-failed` line on failure. Keeps the on-success
513
- // complete line terse and routes every failure through the same observation
514
- // primitive so investigators have one grep pattern for all dump write errors.
515
- function dumpField(result: DumpResult, mode: DumpMode): string {
516
- if ("path" in result) {
517
- if (result.truncated) {
518
- // Loud separate phase line — an investigator grepping for truncation
519
- // incidents across the stream log has one exact pattern to match.
520
- logPhase(
521
- `phase=dump-truncated mode=${mode} path=${result.path} captured_bytes=${OUTER_HTML_CAPTURE_MAX_CHARS}`,
522
- );
523
- return `dump=${result.path} dump_truncated=true`;
524
- }
525
- return `dump=${result.path}`;
526
- }
527
- logPhase(
528
- `phase=dump-write-failed mode=${mode} detail="${result.err.replace(/"/g, "'")}"`,
529
- );
530
- return "dump=failed";
531
- }
532
-
533
- export async function scrapeDomains(evaluator: CdpEvaluator): Promise<string[]> {
534
- const deadline = Date.now() + SCRAPE_POLL_MS;
535
- // pure poll-to-deadline. The earlier "first N stable polls wins"
536
- // heuristic short-circuited on the first plateau a multi-zone dashboard
537
- // rendered — a 2-zone account whose second zone hydrates one poll later
538
- // than the first would lock `count=1` before the second zone arrived.
539
- // Removing early-exit entirely is the only formulation that eliminates the
540
- // class: any "N-consec-at-max wins" variant still fires on the intermediate
541
- // plateau (`[] [] [a] [a] [a,b] …` → threshold met at poll 4 before zone B
542
- // observed). The cost is ~9 s of happy-path latency under the 10 s budget
543
- // (still within onboarding tolerance). `max_count` tracks the high-water
544
- // count; `consecutiveAtMax` tracks the trailing run at that water-mark;
545
- // `seenDecrease` tracks any mid-trajectory count drop. `unstable=false`
546
- // requires reaching max, ending at max, at least STABLE_POLL_THRESHOLD
547
- // consecutive trailing polls at max, and no mid-trajectory decrease — so a
548
- // trajectory like `[a][a][a][a,b]{never reaches 2 consecutive}` correctly
549
- // reports `unstable=true` even though `final_count === max_count`.
550
- let lastOutcome: ScrapeOutcome | null = null;
551
- let maxCount = 0;
552
- // Snapshot of the domains array at the most-recent observation of
553
- // `maxCount`. Updated on every `>=` poll so it always reflects the latest
554
- // capture at the high-water mark; returned at deadline when `maxCount > 0`.
555
- let maxDomains: string[] = [];
556
- let finalCount = 0;
557
- let consecutiveAtMax = 0;
558
- let seenDecrease = false;
559
- let polls = 0;
560
-
561
- while (Date.now() < deadline) {
562
- polls += 1;
563
- try {
564
- const outcome = (await evaluator(SCRAPE_EXPRESSION)) as ScrapeOutcome;
565
- lastOutcome = outcome;
566
- const count = outcome.domains.length;
567
-
568
- logPhase(
569
- `phase=dom-scrape-poll n=${polls} count=${count} domains=[${formatDomains(outcome.domains)}]`,
570
- );
571
-
572
- if (outcome.reason === "ok" && count > 0) {
573
- if (count > maxCount) {
574
- // New high-water mark. Post-stable increases reset the run counter —
575
- // the new stable window must also satisfy the threshold to qualify
576
- // as `unstable=false`.
577
- maxCount = count;
578
- maxDomains = outcome.domains;
579
- consecutiveAtMax = 1;
580
- } else if (count === maxCount) {
581
- maxDomains = outcome.domains;
582
- consecutiveAtMax += 1;
583
- } else {
584
- // count > 0 && count < maxCount — the page shrunk mid-poll
585
- // (virtual-scroll pagination, zone deletion concurrent with scrape,
586
- // SPA re-render collapsing rows). Preserve the high-water snapshot;
587
- // flag the trajectory as unstable.
588
- seenDecrease = true;
589
- consecutiveAtMax = 0;
590
- }
591
- finalCount = count;
592
- } else {
593
- // Empty poll or non-ok reason (no-account-id — SPA mid-navigation):
594
- // the page momentarily showed no zones, breaking any trailing at-max
595
- // run. Don't flip `seenDecrease` — treating a hydration gap as a
596
- // "shrink" would be a false positive on the leading empty polls of
597
- // every scrape. `finalCount` reflects the last poll's observed count
598
- // (always 0 in this branch, since `scrapeCurrentPage` returns empty
599
- // domains on any non-ok reason), so the stream log's `final_count`
600
- // field accurately names what the last poll saw, not a stale value
601
- // from an earlier successful observation.
602
- consecutiveAtMax = 0;
603
- finalCount = count;
604
- }
605
- } catch (err) {
606
- logPhase(
607
- `phase=scrape-retry n=${polls} err="${(err instanceof Error ? err.message : String(err)).slice(0, 120)}"`,
608
- );
609
- }
610
- await new Promise((r) => setTimeout(r, POLL_INTERVAL_MS));
611
- }
612
-
613
- // Deadline reached. Two branches:
614
- //
615
- // (i) We observed at least one non-empty poll. Return the max-count
616
- // snapshot (which, when `finalCount === maxCount`, is also the last
617
- // non-empty observation). `unstable=true` whenever any signal suggests
618
- // the page wasn't settled: shrinkage, end-below-max, or fewer than
619
- // STABLE_POLL_THRESHOLD consecutive trailing polls at max.
620
- //
621
- // (ii) Every poll was empty — either a genuinely empty account OR CF
622
- // drifted the href shape. The `empty-or-drift` dump is the operator's
623
- // single-artifact signal for distinguishing the two.
624
- if (maxCount > 0) {
625
- const unstable =
626
- seenDecrease ||
627
- finalCount !== maxCount ||
628
- consecutiveAtMax < STABLE_POLL_THRESHOLD;
629
- const mode: DumpMode = unstable ? "unstable" : "stable";
630
- const dump = await dumpHtml(evaluator, maxCount, mode);
631
- logPhase(
632
- `phase=dom-scrape-complete result=ok count=${maxCount} total_polls=${polls} max_count=${maxCount} final_count=${finalCount} stable_polls=${consecutiveAtMax} unstable=${unstable} domains=[${formatDomains(maxDomains)}] ${dumpField(dump, mode)}`,
633
- );
634
- return maxDomains;
635
- }
636
-
637
- const dump = await dumpHtml(evaluator, 0, "empty-or-drift");
638
- logPhase(
639
- `phase=dom-scrape-complete result=empty-or-drift count=0 total_polls=${polls} max_count=0 final_count=0 stable_polls=0 lastReason=${lastOutcome?.reason ?? "unknown"} ${dumpField(dump, "empty-or-drift")}`,
640
- );
641
- return [];
642
- }
643
-
644
- async function navigate(cdp: CdpClient, url: string): Promise<void> {
645
- const result = await cdp.send<{ errorText?: string }>("Page.navigate", { url });
646
- if (result?.errorText) die("navigate-failed", `Page.navigate errorText="${result.errorText}"`);
647
- }
648
-
649
- async function waitForDocumentReady(cdp: CdpClient): Promise<void> {
650
- const deadline = Date.now() + NAVIGATE_TIMEOUT_MS;
651
- while (Date.now() < deadline) {
652
- const state = await evaluate<string>(cdp, "document.readyState");
653
- if (state === "complete" || state === "interactive") return;
654
- await new Promise((r) => setTimeout(r, POLL_INTERVAL_MS));
655
- }
656
- die("navigate-failed", "document.readyState did not reach complete/interactive");
657
- }
658
-
659
- async function main(): Promise<void> {
660
- const endpoint = resolveCdpEndpoint();
661
- logPhase(`phase=script-start cdp=${endpoint.host}:${endpoint.port} source=${endpoint.source}`);
662
-
663
- await cdpVersion(endpoint.host, endpoint.port);
664
- logPhase(`phase=cdp-connect result=ok`);
665
-
666
- const target = await createTarget(endpoint.host, endpoint.port);
667
- logPhase(`phase=target-created targetId=${target.id.slice(0, 8)}…`);
668
-
669
- let cdp: CdpClient;
670
- try {
671
- cdp = await CdpClient.connect(target.webSocketDebuggerUrl);
672
- } catch (err) {
673
- await closeTarget(endpoint.host, endpoint.port, target.id);
674
- die("ws-connect-failed", err instanceof Error ? err.message : String(err));
675
- }
676
-
677
- try {
678
- await cdp.send("Page.enable");
679
- await cdp.send("Runtime.enable");
680
-
681
- logPhase(`phase=dashboard-nav-start url=https://dash.cloudflare.com/`);
682
- await navigate(cdp, "https://dash.cloudflare.com/");
683
- await waitForDocumentReady(cdp);
684
-
685
- const accountId = await waitForSignedIn(cdp);
686
-
687
- const overviewUrl = `https://dash.cloudflare.com/${accountId}/domains/overview`;
688
- logPhase(`phase=table-wait url=${overviewUrl}`);
689
- await navigate(cdp, overviewUrl);
690
- await waitForDocumentReady(cdp);
691
-
692
- const domains = await scrapeDomains((expr) => evaluate(cdp, expr));
693
-
694
- logPhase(`phase=target-closed`);
695
- process.stdout.write(JSON.stringify(domains) + "\n");
696
- logPhase(`phase=script-exit code=0 count=${domains.length}`);
697
- } catch (err) {
698
- // Only runtime errors that escaped the typed-reason paths land here.
699
- die("runtime-error", err instanceof Error ? err.message : String(err));
700
- } finally {
701
- cdp.close();
702
- await closeTarget(endpoint.host, endpoint.port, target.id);
703
- }
704
- }
705
-
706
- // Entry-point gate: only run `main()` when this module is the script Node was
707
- // invoked with, not when imported by the vitest regressions that exercise
708
- // `scrapeCurrentPage` / `scrapeDomains` directly. Without the gate, importing
709
- // this module would trigger a live CDP connection attempt and process.exit(1)
710
- // inside the test process.
711
- if (process.argv[1] && process.argv[1] === fileURLToPath(import.meta.url)) {
712
- main().catch((err: unknown) => {
713
- die("runtime-error", err instanceof Error ? err.message : String(err));
714
- });
715
- }