@ia-qa/self-healing 1.3.2 β 1.3.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -1
- package/TUTORIAL.md +3 -3
- package/dist/browser/discover.d.ts +28 -0
- package/dist/browser/discover.js +107 -0
- package/dist/browser/discover.js.map +1 -0
- package/dist/cli/baseline.js +13 -9
- package/dist/cli/baseline.js.map +1 -1
- package/dist/cli/diff.d.ts +2 -1
- package/dist/cli/diff.js +16 -2
- package/dist/cli/diff.js.map +1 -1
- package/dist/cli/discover.d.ts +17 -0
- package/dist/cli/discover.js +121 -0
- package/dist/cli/discover.js.map +1 -0
- package/dist/cli/fix.d.ts +2 -1
- package/dist/cli/fix.js +10 -9
- package/dist/cli/fix.js.map +1 -1
- package/dist/cli/index.js +26 -5
- package/dist/cli/index.js.map +1 -1
- package/dist/cli/map.d.ts +1 -0
- package/dist/cli/map.js +12 -0
- package/dist/cli/map.js.map +1 -1
- package/dist/cli/run.d.ts +9 -5
- package/dist/cli/run.js +86 -12
- package/dist/cli/run.js.map +1 -1
- package/dist/crawl.d.ts +42 -0
- package/dist/crawl.js +124 -0
- package/dist/crawl.js.map +1 -0
- package/dist/discovery/safety.d.ts +69 -0
- package/dist/discovery/safety.js +129 -0
- package/dist/discovery/safety.js.map +1 -0
- package/dist/discovery/sitemap.d.ts +48 -0
- package/dist/discovery/sitemap.js +123 -0
- package/dist/discovery/sitemap.js.map +1 -0
- package/dist/htmlReport.d.ts +28 -0
- package/dist/htmlReport.js +53 -1
- package/dist/htmlReport.js.map +1 -1
- package/dist/index.d.ts +6 -0
- package/dist/index.js +15 -2
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/crawl.js
ADDED
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.crawlApp = crawlApp;
|
|
4
|
+
const launcher_1 = require("./launcher");
|
|
5
|
+
const map_1 = require("./cli/map");
|
|
6
|
+
const aom_1 = require("./aom");
|
|
7
|
+
const discover_1 = require("./browser/discover");
|
|
8
|
+
const safety_1 = require("./discovery/safety");
|
|
9
|
+
/** How long to let a revealed menu/dialog settle before reading it. */
|
|
10
|
+
const REVEAL_SETTLE_MS = 250;
|
|
11
|
+
/**
|
|
12
|
+
* Crawl an app behind its configured login and return the pages found that are not
|
|
13
|
+
* already in `config.pages`. Launches its own browser, logs in if `config.auth` is
|
|
14
|
+
* set, and always closes the browser β a thrown error must not strand a process.
|
|
15
|
+
*/
|
|
16
|
+
async function crawlApp(config, options = {}) {
|
|
17
|
+
const maxDepth = options.maxDepth ?? 2;
|
|
18
|
+
const maxPages = options.maxPages ?? 60;
|
|
19
|
+
const reveal = options.reveal !== false;
|
|
20
|
+
const base = new URL(config.baseUrl);
|
|
21
|
+
const start = new URL(config.baseUrl).toString();
|
|
22
|
+
const found = new Map();
|
|
23
|
+
const visited = new Set();
|
|
24
|
+
const queue = [{ url: start, depth: 0 }];
|
|
25
|
+
const enqueue = (href, depth) => {
|
|
26
|
+
if (!(0, safety_1.isSafeCandidateUrl)(href, config.baseUrl))
|
|
27
|
+
return;
|
|
28
|
+
const abs = new URL(href, config.baseUrl).toString();
|
|
29
|
+
const key = (0, safety_1.normalizedPath)(abs, config.baseUrl);
|
|
30
|
+
if (!found.has(key)) {
|
|
31
|
+
const candidate = (0, safety_1.toCandidate)(abs, config.baseUrl, 'crawl');
|
|
32
|
+
if (candidate)
|
|
33
|
+
found.set(key, candidate);
|
|
34
|
+
}
|
|
35
|
+
if (depth <= maxDepth && !visited.has(key)) {
|
|
36
|
+
queue.push({ url: `${base.origin}${key}`, depth });
|
|
37
|
+
}
|
|
38
|
+
};
|
|
39
|
+
for (const seed of options.seeds ?? [])
|
|
40
|
+
enqueue(seed, 1);
|
|
41
|
+
const { browser, using } = await (0, launcher_1.launchBrowser)();
|
|
42
|
+
console.log(`π Using ${using}`);
|
|
43
|
+
try {
|
|
44
|
+
const context = config.locale
|
|
45
|
+
? await browser.newContext({ locale: config.locale })
|
|
46
|
+
: await browser.newContext();
|
|
47
|
+
const page = await context.newPage();
|
|
48
|
+
if (config.auth) {
|
|
49
|
+
console.log('π Logging inβ¦');
|
|
50
|
+
await (0, map_1.login)(page, config);
|
|
51
|
+
}
|
|
52
|
+
while (queue.length > 0 && visited.size < maxPages) {
|
|
53
|
+
const { url, depth } = queue.shift();
|
|
54
|
+
const key = (0, safety_1.normalizedPath)(url, config.baseUrl);
|
|
55
|
+
if (visited.has(key))
|
|
56
|
+
continue;
|
|
57
|
+
visited.add(key);
|
|
58
|
+
let hrefs;
|
|
59
|
+
try {
|
|
60
|
+
hrefs = await visit(page, url, reveal && depth < maxDepth);
|
|
61
|
+
}
|
|
62
|
+
catch {
|
|
63
|
+
// A single page that will not load (timeout, transient error) is not fatal
|
|
64
|
+
// to a discovery run β skip it and keep mapping the rest of the app.
|
|
65
|
+
continue;
|
|
66
|
+
}
|
|
67
|
+
options.onVisit?.(key, hrefs.length);
|
|
68
|
+
for (const href of hrefs)
|
|
69
|
+
enqueue(href, depth + 1);
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
finally {
|
|
73
|
+
await browser.close();
|
|
74
|
+
}
|
|
75
|
+
return Array.from(found.values());
|
|
76
|
+
}
|
|
77
|
+
/**
|
|
78
|
+
* Load a URL, harvest its links, and β when `reveal` is on β open the safe
|
|
79
|
+
* disclosure toggles to harvest the links hidden behind them too.
|
|
80
|
+
*
|
|
81
|
+
* Returns raw hrefs (paths or absolute URLs); the caller filters them through
|
|
82
|
+
* `isSafeCandidateUrl`, so nothing here has to know the denylist.
|
|
83
|
+
*/
|
|
84
|
+
async function visit(page, url, reveal) {
|
|
85
|
+
await page.goto(url, { waitUntil: 'networkidle', timeout: 30000 });
|
|
86
|
+
const hrefs = new Set();
|
|
87
|
+
for (const el of await (0, aom_1.extractInteractiveElements)(page)) {
|
|
88
|
+
if (el.href)
|
|
89
|
+
hrefs.add(el.href);
|
|
90
|
+
}
|
|
91
|
+
if (!reveal)
|
|
92
|
+
return Array.from(hrefs);
|
|
93
|
+
// The toggles present on the settled page. Selectors are stamped attributes, so
|
|
94
|
+
// they survive re-query even as the DOM changes around them.
|
|
95
|
+
const targets = await page.evaluate(discover_1.findRevealTargets);
|
|
96
|
+
for (const selector of targets) {
|
|
97
|
+
const urlBefore = page.url();
|
|
98
|
+
try {
|
|
99
|
+
await page.locator(selector).first().click({ timeout: 2000 });
|
|
100
|
+
}
|
|
101
|
+
catch {
|
|
102
|
+
continue; // not clickable / detached β the safe outcome is to skip it
|
|
103
|
+
}
|
|
104
|
+
// GUARD: a disclosure toggle does not navigate. If the URL changed, this was a
|
|
105
|
+
// link in disguise β record where it went and step back, rather than crawling
|
|
106
|
+
// on from an unexpected place.
|
|
107
|
+
if (page.url() !== urlBefore) {
|
|
108
|
+
hrefs.add(page.url());
|
|
109
|
+
await page.goBack({ waitUntil: 'domcontentloaded', timeout: 10000 }).catch(() => { });
|
|
110
|
+
continue;
|
|
111
|
+
}
|
|
112
|
+
await page.waitForTimeout(REVEAL_SETTLE_MS);
|
|
113
|
+
for (const el of await (0, aom_1.extractInteractiveElements)(page)) {
|
|
114
|
+
if (el.href)
|
|
115
|
+
hrefs.add(el.href);
|
|
116
|
+
}
|
|
117
|
+
// Close whatever opened. Escape dismisses the overwhelming majority of menus and
|
|
118
|
+
// dialogs; if it does not, the next goto resets the page anyway. We never click
|
|
119
|
+
// the revealed items β only read their hrefs.
|
|
120
|
+
await page.keyboard.press('Escape').catch(() => { });
|
|
121
|
+
}
|
|
122
|
+
return Array.from(hrefs);
|
|
123
|
+
}
|
|
124
|
+
//# sourceMappingURL=crawl.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"crawl.js","sourceRoot":"","sources":["../src/crawl.ts"],"names":[],"mappings":";;AAoDA,4BA8DC;AAjHD,yCAA2C;AAC3C,mCAAkC;AAClC,+BAAmD;AACnD,iDAAuD;AAEvD,+CAAgG;AAsChG,uEAAuE;AACvE,MAAM,gBAAgB,GAAG,GAAG,CAAC;AAE7B;;;;GAIG;AACI,KAAK,UAAU,QAAQ,CAAC,MAAkB,EAAE,UAAwB,EAAE;IAC3E,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,CAAC,CAAC;IACvC,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAC;IACxC,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,KAAK,KAAK,CAAC;IAExC,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IACrC,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,CAAC;IAEjD,MAAM,KAAK,GAAG,IAAI,GAAG,EAAqB,CAAC;IAC3C,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;IAClC,MAAM,KAAK,GAA0C,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;IAEhF,MAAM,OAAO,GAAG,CAAC,IAAY,EAAE,KAAa,EAAE,EAAE;QAC9C,IAAI,CAAC,IAAA,2BAAkB,EAAC,IAAI,EAAE,MAAM,CAAC,OAAO,CAAC;YAAE,OAAO;QACtD,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,IAAI,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,CAAC;QACrD,MAAM,GAAG,GAAG,IAAA,uBAAc,EAAC,GAAG,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC;QAChD,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YACpB,MAAM,SAAS,GAAG,IAAA,oBAAW,EAAC,GAAG,EAAE,MAAM,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;YAC5D,IAAI,SAAS;gBAAE,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;QAC3C,CAAC;QACD,IAAI,KAAK,IAAI,QAAQ,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YAC3C,KAAK,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,MAAM,GAAG,GAAG,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;QACrD,CAAC;IACH,CAAC,CAAC;IAEF,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,KAAK,IAAI,EAAE;QAAE,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IAEzD,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,MAAM,IAAA,wBAAa,GAAE,CAAC;IACjD,OAAO,CAAC,GAAG,CAAC,YAAY,KAAK,EAAE,CAAC,CAAC;IACjC,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM;YAC3B,CAAC,CAAC,MAAM,OAAO,CAAC,UAAU,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC;YACrD,CAAC,CAAC,MAAM,OAAO,CAAC,UAAU,EAAE,CAAC;QAC/B,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;QAErC,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;YAChB,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;YAC9B,MAAM,IAAA,WAAK,EAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QAC5B,CAAC;QAED,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,OAAO,CAAC,IAAI,GAAG,QAAQ,EAAE,CAAC;YACnD,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,KAAK,CAAC,KAAK,EAAG,CAAC;YACtC,MAAM,GAAG,GAAG,IAAA,uBAAc,EAAC,GAAG,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC;YAChD,IAAI,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC;gBAAE,SAAS;YAC/B,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAEjB,IAAI,KAAe,CAAC;YACpB,IAAI,CAAC;gBACH,KAAK,GAAG,MAAM,KAAK,CAAC,IAAI,EAAE,GAAG,EAAE,MAAM,IAAI,KAAK,GAAG,QAAQ,CAAC,CAAC;YAC7D,CAAC;YAAC,MAAM,CAAC;gBACP,2EAA2E;gBAC3E,qEAAqE;gBACrE,SAAS;YACX,CAAC;YACD,OAAO,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;YACrC,KAAK,MAAM,IAAI,IAAI,KAAK;gBAAE,OAAO,CAAC,IAAI,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;QACrD,CAAC;IACH,CAAC;YAAS,CAAC;QACT,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;IACxB,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;AACpC,CAAC;AAED;;;;;;GAMG;AACH,KAAK,UAAU,KAAK,CAAC,IAAU,EAAE,GAAW,EAAE,MAAe;IAC3D,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,aAAa,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;IACnE,MAAM,KAAK,GAAG,IAAI,GAAG,EAAU,CAAC;IAChC,KAAK,MAAM,EAAE,IAAI,MAAM,IAAA,gCAA0B,EAAC,IAAI,CAAC,EAAE,CAAC;QACxD,IAAI,EAAE,CAAC,IAAI;YAAE,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC;IAClC,CAAC;IACD,IAAI,CAAC,MAAM;QAAE,OAAO,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAEtC,gFAAgF;IAChF,6DAA6D;IAC7D,MAAM,OAAO,GAAa,MAAM,IAAI,CAAC,QAAQ,CAAC,4BAAiB,CAAC,CAAC;IACjE,KAAK,MAAM,QAAQ,IAAI,OAAO,EAAE,CAAC;QAC/B,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7B,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;QAChE,CAAC;QAAC,MAAM,CAAC;YACP,SAAS,CAAC,4DAA4D;QACxE,CAAC;QAED,+EAA+E;QAC/E,8EAA8E;QAC9E,+BAA+B;QAC/B,IAAI,IAAI,CAAC,GAAG,EAAE,KAAK,SAAS,EAAE,CAAC;YAC7B,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;YACtB,MAAM,IAAI,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,kBAAkB,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;YACrF,SAAS;QACX,CAAC;QAED,MAAM,IAAI,CAAC,cAAc,CAAC,gBAAgB,CAAC,CAAC;QAC5C,KAAK,MAAM,EAAE,IAAI,MAAM,IAAA,gCAA0B,EAAC,IAAI,CAAC,EAAE,CAAC;YACxD,IAAI,EAAE,CAAC,IAAI;gBAAE,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC;QAClC,CAAC;QACD,iFAAiF;QACjF,gFAAgF;QAChF,8CAA8C;QAC9C,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IACtD,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;AAC3B,CAAC"}
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
import type { PageTarget } from '../config';
|
|
2
|
+
/**
|
|
3
|
+
* The safety floor shared by both discovery sources (sitemap + crawl).
|
|
4
|
+
*
|
|
5
|
+
* Discovery only ever *proposes* pages to add to `config.pages`; it never maps,
|
|
6
|
+
* heals, or edits a test. But the crawler navigates real URLs on a real (often
|
|
7
|
+
* authenticated) app, so the one thing that must be airtight is *which URLs it is
|
|
8
|
+
* allowed to touch at all*. That decision lives here, as pure functions, so it is
|
|
9
|
+
* unit-tested in isolation rather than trusted inside a browser loop.
|
|
10
|
+
*
|
|
11
|
+
* Two rules, both erring toward doing nothing:
|
|
12
|
+
* 1. Never leave the app's own origin.
|
|
13
|
+
* 2. Never touch a URL whose path or query reads like an action β logout, delete,
|
|
14
|
+
* pay, unsubscribe⦠A crawler that visits `/logout` kills the session for every
|
|
15
|
+
* page after it; one that visits `/orders/42/delete` (a GET that mutates β bad
|
|
16
|
+
* REST, but real) destroys data. Both are same-origin GETs a naive BFS would
|
|
17
|
+
* happily follow, so the guard is a denylist on the URL itself, not on a verb.
|
|
18
|
+
*/
|
|
19
|
+
/**
|
|
20
|
+
* A path or query segment that names an action rather than a view. Matched against
|
|
21
|
+
* the full href (pathname + search), case-insensitive, on word-ish boundaries so
|
|
22
|
+
* `/deleted-items` (a view) is not caught by `delete` but `/item/delete` is.
|
|
23
|
+
*
|
|
24
|
+
* Deliberately broad: a false positive costs one un-discovered page (the user can
|
|
25
|
+
* add it by hand); a false negative can log the crawler out or mutate data. On an
|
|
26
|
+
* "ultra-safe" tool that trade is not close.
|
|
27
|
+
*/
|
|
28
|
+
export declare const UNSAFE_URL_RE: RegExp;
|
|
29
|
+
/**
|
|
30
|
+
* May the crawler touch this href, resolved against `baseUrl`?
|
|
31
|
+
*
|
|
32
|
+
* Returns false for anything off-origin, non-http, a download, or matching
|
|
33
|
+
* `UNSAFE_URL_RE`. Unparseable input is unsafe by default β the whole point is to
|
|
34
|
+
* be conservative about what gets a navigation.
|
|
35
|
+
*/
|
|
36
|
+
export declare function isSafeCandidateUrl(href: string, baseUrl: string): boolean;
|
|
37
|
+
/**
|
|
38
|
+
* Compare two URLs by pathname alone β query and hash do not make another page,
|
|
39
|
+
* and a trailing slash is not a distinction. This is the dedup key for candidates,
|
|
40
|
+
* and mirrors `normalizePath` in overview.ts and `samePath` in map.ts so discovery,
|
|
41
|
+
* the nav graph, and `assertLanded` all agree on when two URLs are one page.
|
|
42
|
+
*/
|
|
43
|
+
export declare function normalizedPath(href: string, baseUrl: string): string;
|
|
44
|
+
export interface Candidate {
|
|
45
|
+
name: string;
|
|
46
|
+
url: string;
|
|
47
|
+
/** Where this candidate came from β shown in the summary, never written to config. */
|
|
48
|
+
source: 'sitemap' | 'crawl';
|
|
49
|
+
}
|
|
50
|
+
/**
|
|
51
|
+
* Turn a raw URL into a candidate page, or null if it is not safe to propose.
|
|
52
|
+
*
|
|
53
|
+
* The stored `url` is the same shape the rest of the config uses: a path for
|
|
54
|
+
* same-origin (`/dashboard`), so a contract stays comparable across environments.
|
|
55
|
+
*/
|
|
56
|
+
export declare function toCandidate(href: string, baseUrl: string, source: Candidate['source']): Candidate | null;
|
|
57
|
+
/**
|
|
58
|
+
* Fold candidates into the pages already in config, dropping duplicates and
|
|
59
|
+
* anything already configured, and making every new page name unique.
|
|
60
|
+
*
|
|
61
|
+
* A page name is a contract's filename ([[config.mappingPath]]), so a collision is
|
|
62
|
+
* not cosmetic β two pages would overwrite each other's mapping. Existing pages win
|
|
63
|
+
* their name; a new candidate that slugs to a taken name gets a numeric suffix.
|
|
64
|
+
* Dedup is by normalized path so `/x`, `/x/`, and `/x?ref=nav` are one page.
|
|
65
|
+
*
|
|
66
|
+
* Returns only the *new* pages to add, in discovery order, so the caller can show
|
|
67
|
+
* "found N, M new" and append them without disturbing the author's ordering.
|
|
68
|
+
*/
|
|
69
|
+
export declare function mergePages(existing: PageTarget[], candidates: Candidate[], baseUrl: string): Candidate[];
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.UNSAFE_URL_RE = void 0;
|
|
4
|
+
exports.isSafeCandidateUrl = isSafeCandidateUrl;
|
|
5
|
+
exports.normalizedPath = normalizedPath;
|
|
6
|
+
exports.toCandidate = toCandidate;
|
|
7
|
+
exports.mergePages = mergePages;
|
|
8
|
+
const pageName_1 = require("../pageName");
|
|
9
|
+
/**
|
|
10
|
+
* The safety floor shared by both discovery sources (sitemap + crawl).
|
|
11
|
+
*
|
|
12
|
+
* Discovery only ever *proposes* pages to add to `config.pages`; it never maps,
|
|
13
|
+
* heals, or edits a test. But the crawler navigates real URLs on a real (often
|
|
14
|
+
* authenticated) app, so the one thing that must be airtight is *which URLs it is
|
|
15
|
+
* allowed to touch at all*. That decision lives here, as pure functions, so it is
|
|
16
|
+
* unit-tested in isolation rather than trusted inside a browser loop.
|
|
17
|
+
*
|
|
18
|
+
* Two rules, both erring toward doing nothing:
|
|
19
|
+
* 1. Never leave the app's own origin.
|
|
20
|
+
* 2. Never touch a URL whose path or query reads like an action β logout, delete,
|
|
21
|
+
* pay, unsubscribe⦠A crawler that visits `/logout` kills the session for every
|
|
22
|
+
* page after it; one that visits `/orders/42/delete` (a GET that mutates β bad
|
|
23
|
+
* REST, but real) destroys data. Both are same-origin GETs a naive BFS would
|
|
24
|
+
* happily follow, so the guard is a denylist on the URL itself, not on a verb.
|
|
25
|
+
*/
|
|
26
|
+
/**
|
|
27
|
+
* A path or query segment that names an action rather than a view. Matched against
|
|
28
|
+
* the full href (pathname + search), case-insensitive, on word-ish boundaries so
|
|
29
|
+
* `/deleted-items` (a view) is not caught by `delete` but `/item/delete` is.
|
|
30
|
+
*
|
|
31
|
+
* Deliberately broad: a false positive costs one un-discovered page (the user can
|
|
32
|
+
* add it by hand); a false negative can log the crawler out or mutate data. On an
|
|
33
|
+
* "ultra-safe" tool that trade is not close.
|
|
34
|
+
*/
|
|
35
|
+
exports.UNSAFE_URL_RE = /(?:^|[/_.?=&-])(?:log[\s_-]?out|sign[\s_-]?out|logout|signout|delete|destroy|remove|deactivate|disable|unsubscribe|revoke|cancel|purchase|checkout|payment|pay|buy|order|confirm|approve|reject|reset|logoff)(?:[/_.?=&-]|$)/i;
|
|
36
|
+
/** File extensions that are downloads, not pages β never worth mapping. */
|
|
37
|
+
const DOWNLOAD_EXT_RE = /\.(?:pdf|zip|gz|tar|rar|7z|csv|xlsx?|docx?|pptx?|png|jpe?g|gif|svg|webp|ico|mp[34]|mov|avi|woff2?|ttf|eot|dmg|exe|apk|pkg)(?:$|[?#])/i;
|
|
38
|
+
/**
|
|
39
|
+
* May the crawler touch this href, resolved against `baseUrl`?
|
|
40
|
+
*
|
|
41
|
+
* Returns false for anything off-origin, non-http, a download, or matching
|
|
42
|
+
* `UNSAFE_URL_RE`. Unparseable input is unsafe by default β the whole point is to
|
|
43
|
+
* be conservative about what gets a navigation.
|
|
44
|
+
*/
|
|
45
|
+
function isSafeCandidateUrl(href, baseUrl) {
|
|
46
|
+
let url;
|
|
47
|
+
let base;
|
|
48
|
+
try {
|
|
49
|
+
base = new URL(baseUrl);
|
|
50
|
+
url = new URL(href, baseUrl);
|
|
51
|
+
}
|
|
52
|
+
catch {
|
|
53
|
+
return false;
|
|
54
|
+
}
|
|
55
|
+
if (url.protocol !== 'http:' && url.protocol !== 'https:')
|
|
56
|
+
return false;
|
|
57
|
+
if (url.origin !== base.origin)
|
|
58
|
+
return false;
|
|
59
|
+
if (DOWNLOAD_EXT_RE.test(url.pathname + url.search))
|
|
60
|
+
return false;
|
|
61
|
+
if (exports.UNSAFE_URL_RE.test(url.pathname + url.search))
|
|
62
|
+
return false;
|
|
63
|
+
return true;
|
|
64
|
+
}
|
|
65
|
+
/**
|
|
66
|
+
* Compare two URLs by pathname alone β query and hash do not make another page,
|
|
67
|
+
* and a trailing slash is not a distinction. This is the dedup key for candidates,
|
|
68
|
+
* and mirrors `normalizePath` in overview.ts and `samePath` in map.ts so discovery,
|
|
69
|
+
* the nav graph, and `assertLanded` all agree on when two URLs are one page.
|
|
70
|
+
*/
|
|
71
|
+
function normalizedPath(href, baseUrl) {
|
|
72
|
+
let url;
|
|
73
|
+
try {
|
|
74
|
+
url = new URL(href, baseUrl);
|
|
75
|
+
}
|
|
76
|
+
catch {
|
|
77
|
+
return href;
|
|
78
|
+
}
|
|
79
|
+
const p = url.pathname;
|
|
80
|
+
return p.length > 1 && p.endsWith('/') ? p.slice(0, -1) : p;
|
|
81
|
+
}
|
|
82
|
+
/**
|
|
83
|
+
* Turn a raw URL into a candidate page, or null if it is not safe to propose.
|
|
84
|
+
*
|
|
85
|
+
* The stored `url` is the same shape the rest of the config uses: a path for
|
|
86
|
+
* same-origin (`/dashboard`), so a contract stays comparable across environments.
|
|
87
|
+
*/
|
|
88
|
+
function toCandidate(href, baseUrl, source) {
|
|
89
|
+
if (!isSafeCandidateUrl(href, baseUrl))
|
|
90
|
+
return null;
|
|
91
|
+
const path = normalizedPath(href, baseUrl);
|
|
92
|
+
return { name: (0, pageName_1.pageNameFromUrl)(new URL(href, baseUrl).toString()), url: path, source };
|
|
93
|
+
}
|
|
94
|
+
/**
|
|
95
|
+
* Fold candidates into the pages already in config, dropping duplicates and
|
|
96
|
+
* anything already configured, and making every new page name unique.
|
|
97
|
+
*
|
|
98
|
+
* A page name is a contract's filename ([[config.mappingPath]]), so a collision is
|
|
99
|
+
* not cosmetic β two pages would overwrite each other's mapping. Existing pages win
|
|
100
|
+
* their name; a new candidate that slugs to a taken name gets a numeric suffix.
|
|
101
|
+
* Dedup is by normalized path so `/x`, `/x/`, and `/x?ref=nav` are one page.
|
|
102
|
+
*
|
|
103
|
+
* Returns only the *new* pages to add, in discovery order, so the caller can show
|
|
104
|
+
* "found N, M new" and append them without disturbing the author's ordering.
|
|
105
|
+
*/
|
|
106
|
+
function mergePages(existing, candidates, baseUrl) {
|
|
107
|
+
const takenPaths = new Set(existing
|
|
108
|
+
.filter((p) => !p.steps?.length)
|
|
109
|
+
.map((p) => normalizedPath(p.url, baseUrl)));
|
|
110
|
+
const takenNames = new Set(existing.map((p) => p.name));
|
|
111
|
+
const added = [];
|
|
112
|
+
for (const c of candidates) {
|
|
113
|
+
const key = normalizedPath(c.url, baseUrl);
|
|
114
|
+
if (takenPaths.has(key))
|
|
115
|
+
continue;
|
|
116
|
+
takenPaths.add(key);
|
|
117
|
+
let name = c.name;
|
|
118
|
+
if (takenNames.has(name)) {
|
|
119
|
+
let n = 2;
|
|
120
|
+
while (takenNames.has(`${name}-${n}`))
|
|
121
|
+
n++;
|
|
122
|
+
name = `${name}-${n}`;
|
|
123
|
+
}
|
|
124
|
+
takenNames.add(name);
|
|
125
|
+
added.push({ ...c, name });
|
|
126
|
+
}
|
|
127
|
+
return added;
|
|
128
|
+
}
|
|
129
|
+
//# sourceMappingURL=safety.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"safety.js","sourceRoot":"","sources":["../../src/discovery/safety.ts"],"names":[],"mappings":";;;AA4CA,gDAcC;AAQD,wCASC;AAeD,kCAQC;AAcD,gCA4BC;AA5ID,0CAA8C;AAG9C;;;;;;;;;;;;;;;;GAgBG;AAEH;;;;;;;;GAQG;AACU,QAAA,aAAa,GACxB,+NAA+N,CAAC;AAElO,2EAA2E;AAC3E,MAAM,eAAe,GACnB,uIAAuI,CAAC;AAE1I;;;;;;GAMG;AACH,SAAgB,kBAAkB,CAAC,IAAY,EAAE,OAAe;IAC9D,IAAI,GAAQ,CAAC;IACb,IAAI,IAAS,CAAC;IACd,IAAI,CAAC;QACH,IAAI,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC;QACxB,GAAG,GAAG,IAAI,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC/B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;IACD,IAAI,GAAG,CAAC,QAAQ,KAAK,OAAO,IAAI,GAAG,CAAC,QAAQ,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC;IACxE,IAAI,GAAG,CAAC,MAAM,KAAK,IAAI,CAAC,MAAM;QAAE,OAAO,KAAK,CAAC;IAC7C,IAAI,eAAe,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,GAAG,GAAG,CAAC,MAAM,CAAC;QAAE,OAAO,KAAK,CAAC;IAClE,IAAI,qBAAa,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,GAAG,GAAG,CAAC,MAAM,CAAC;QAAE,OAAO,KAAK,CAAC;IAChE,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;;;GAKG;AACH,SAAgB,cAAc,CAAC,IAAY,EAAE,OAAe;IAC1D,IAAI,GAAQ,CAAC;IACb,IAAI,CAAC;QACH,GAAG,GAAG,IAAI,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC/B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;IACD,MAAM,CAAC,GAAG,GAAG,CAAC,QAAQ,CAAC;IACvB,OAAO,CAAC,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAC9D,CAAC;AASD;;;;;GAKG;AACH,SAAgB,WAAW,CACzB,IAAY,EACZ,OAAe,EACf,MAA2B;IAE3B,IAAI,CAAC,kBAAkB,CAAC,IAAI,EAAE,OAAO,CAAC;QAAE,OAAO,IAAI,CAAC;IACpD,MAAM,IAAI,GAAG,cAAc,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC3C,OAAO,EAAE,IAAI,EAAE,IAAA,0BAAe,EAAC,IAAI,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,QAAQ,EAAE,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;AACzF,CAAC;AAED;;;;;;;;;;;GAWG;AACH,SAAgB,UAAU,CACxB,QAAsB,EACtB,UAAuB,EACvB,OAAe;IAEf,MAAM,UAAU,GAAG,IAAI,GAAG,CACxB,QAAQ;SACL,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,EAAE,MAAM,CAAC;SAC/B,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,cAAc,CAAC,CAAC,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,CAC9C,CAAC;IACF,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IACxD,MAAM,KAAK,GAAgB,EAAE,CAAC;IAE9B,KAAK,MAAM,CAAC,IAAI,UAAU,EAAE,CAAC;QAC3B,MAAM,GAAG,GAAG,cAAc,CAAC,CAAC,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QAC3C,IAAI,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,SAAS;QAClC,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAEpB,IAAI,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC;QAClB,IAAI,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YACzB,IAAI,CAAC,GAAG,CAAC,CAAC;YACV,OAAO,UAAU,CAAC,GAAG,CAAC,GAAG,IAAI,IAAI,CAAC,EAAE,CAAC;gBAAE,CAAC,EAAE,CAAC;YAC3C,IAAI,GAAG,GAAG,IAAI,IAAI,CAAC,EAAE,CAAC;QACxB,CAAC;QACD,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACrB,KAAK,CAAC,IAAI,CAAC,EAAE,GAAG,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC;IAC7B,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC"}
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
import { Candidate } from './safety';
|
|
2
|
+
/**
|
|
3
|
+
* Sitemap discovery β the zero-risk source.
|
|
4
|
+
*
|
|
5
|
+
* A sitemap is a manifest the site *publishes about itself*, so reading it is a
|
|
6
|
+
* single GET and never touches the app's state: no browser, no login, no clicks.
|
|
7
|
+
* Its blind spot is the mirror image of the crawler's: it sees the public, SEO
|
|
8
|
+
* surface and stops exactly where the authenticated app begins. That is why the
|
|
9
|
+
* two exist side by side rather than one replacing the other.
|
|
10
|
+
*
|
|
11
|
+
* Dependency-free by the same rule as `browser/extract.js`: `<loc>` extraction is a
|
|
12
|
+
* regex, not an XML library, because namespaces vary between generators and the
|
|
13
|
+
* data we need is trivial. `fetch` is injectable so the parser is tested without a
|
|
14
|
+
* network.
|
|
15
|
+
*/
|
|
16
|
+
export type FetchLike = (url: string) => Promise<{
|
|
17
|
+
ok: boolean;
|
|
18
|
+
status: number;
|
|
19
|
+
text: () => Promise<string>;
|
|
20
|
+
}>;
|
|
21
|
+
export interface SitemapOptions {
|
|
22
|
+
/** Explicit sitemap URL. Defaults to `<baseUrl>/sitemap.xml`, then robots.txt. */
|
|
23
|
+
sitemapUrl?: string;
|
|
24
|
+
/** Injected for tests; defaults to the global `fetch` (Node 18+). */
|
|
25
|
+
fetchImpl?: FetchLike;
|
|
26
|
+
/** Hard cap on URLs returned, to keep a 10k-URL catalogue from flooding config. */
|
|
27
|
+
maxUrls?: number;
|
|
28
|
+
/** Bound on nested sitemap files followed from an index. */
|
|
29
|
+
maxSitemaps?: number;
|
|
30
|
+
}
|
|
31
|
+
/** Pull every `<loc>` out of a sitemap or sitemap-index document. Pure. */
|
|
32
|
+
export declare function extractLocs(xml: string): string[];
|
|
33
|
+
/** A sitemap index points at other sitemaps rather than at pages. */
|
|
34
|
+
export declare function isSitemapIndex(xml: string): boolean;
|
|
35
|
+
/** `Sitemap:` directives in a robots.txt β the standard way to advertise one. */
|
|
36
|
+
export declare function sitemapsFromRobots(robots: string): string[];
|
|
37
|
+
/**
|
|
38
|
+
* Discover candidate pages from a site's sitemap(s).
|
|
39
|
+
*
|
|
40
|
+
* Resolution: the explicit `sitemapUrl` if given, else `/sitemap.xml`; if that
|
|
41
|
+
* 404s, the `Sitemap:` lines in `/robots.txt`. A sitemap index is followed one
|
|
42
|
+
* level into its child sitemaps (bounded by `maxSitemaps`). Everything is filtered
|
|
43
|
+
* through `toCandidate`, so off-origin, unsafe, and download URLs never survive.
|
|
44
|
+
*
|
|
45
|
+
* Never throws on a missing or malformed sitemap: absence is the common case
|
|
46
|
+
* (SPAs, staging), and it is a "found nothing", not an error.
|
|
47
|
+
*/
|
|
48
|
+
export declare function discoverFromSitemap(baseUrl: string, options?: SitemapOptions): Promise<Candidate[]>;
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.extractLocs = extractLocs;
|
|
4
|
+
exports.isSitemapIndex = isSitemapIndex;
|
|
5
|
+
exports.sitemapsFromRobots = sitemapsFromRobots;
|
|
6
|
+
exports.discoverFromSitemap = discoverFromSitemap;
|
|
7
|
+
const safety_1 = require("./safety");
|
|
8
|
+
const LOC_RE = /<loc>\s*([^<\s][^<]*?)\s*<\/loc>/gi;
|
|
9
|
+
/** Pull every `<loc>` out of a sitemap or sitemap-index document. Pure. */
|
|
10
|
+
function extractLocs(xml) {
|
|
11
|
+
const out = [];
|
|
12
|
+
let m;
|
|
13
|
+
LOC_RE.lastIndex = 0;
|
|
14
|
+
while ((m = LOC_RE.exec(xml)) !== null) {
|
|
15
|
+
const loc = decodeEntities(m[1].trim());
|
|
16
|
+
if (loc)
|
|
17
|
+
out.push(loc);
|
|
18
|
+
}
|
|
19
|
+
return out;
|
|
20
|
+
}
|
|
21
|
+
/** A sitemap index points at other sitemaps rather than at pages. */
|
|
22
|
+
function isSitemapIndex(xml) {
|
|
23
|
+
return /<sitemapindex[\s>]/i.test(xml);
|
|
24
|
+
}
|
|
25
|
+
function decodeEntities(s) {
|
|
26
|
+
return s
|
|
27
|
+
.replace(/&/g, '&')
|
|
28
|
+
.replace(/</g, '<')
|
|
29
|
+
.replace(/>/g, '>')
|
|
30
|
+
.replace(/"/g, '"')
|
|
31
|
+
.replace(/'/g, "'");
|
|
32
|
+
}
|
|
33
|
+
/** `Sitemap:` directives in a robots.txt β the standard way to advertise one. */
|
|
34
|
+
function sitemapsFromRobots(robots) {
|
|
35
|
+
const out = [];
|
|
36
|
+
for (const line of robots.split(/\r?\n/)) {
|
|
37
|
+
const m = /^\s*sitemap:\s*(\S+)/i.exec(line);
|
|
38
|
+
if (m)
|
|
39
|
+
out.push(m[1].trim());
|
|
40
|
+
}
|
|
41
|
+
return out;
|
|
42
|
+
}
|
|
43
|
+
/**
|
|
44
|
+
* Discover candidate pages from a site's sitemap(s).
|
|
45
|
+
*
|
|
46
|
+
* Resolution: the explicit `sitemapUrl` if given, else `/sitemap.xml`; if that
|
|
47
|
+
* 404s, the `Sitemap:` lines in `/robots.txt`. A sitemap index is followed one
|
|
48
|
+
* level into its child sitemaps (bounded by `maxSitemaps`). Everything is filtered
|
|
49
|
+
* through `toCandidate`, so off-origin, unsafe, and download URLs never survive.
|
|
50
|
+
*
|
|
51
|
+
* Never throws on a missing or malformed sitemap: absence is the common case
|
|
52
|
+
* (SPAs, staging), and it is a "found nothing", not an error.
|
|
53
|
+
*/
|
|
54
|
+
async function discoverFromSitemap(baseUrl, options = {}) {
|
|
55
|
+
const fetchImpl = options.fetchImpl || globalThis.fetch;
|
|
56
|
+
if (!fetchImpl)
|
|
57
|
+
throw new Error('No fetch implementation available (Node 18+ or pass fetchImpl).');
|
|
58
|
+
const maxUrls = options.maxUrls ?? 200;
|
|
59
|
+
const maxSitemaps = options.maxSitemaps ?? 20;
|
|
60
|
+
const roots = options.sitemapUrl
|
|
61
|
+
? [options.sitemapUrl]
|
|
62
|
+
: await resolveSitemapUrls(baseUrl, fetchImpl);
|
|
63
|
+
const seenSitemaps = new Set();
|
|
64
|
+
const pageUrls = [];
|
|
65
|
+
const queue = [...roots];
|
|
66
|
+
while (queue.length > 0 && seenSitemaps.size < maxSitemaps && pageUrls.length < maxUrls) {
|
|
67
|
+
const url = queue.shift();
|
|
68
|
+
if (seenSitemaps.has(url))
|
|
69
|
+
continue;
|
|
70
|
+
seenSitemaps.add(url);
|
|
71
|
+
const xml = await safeText(fetchImpl, url);
|
|
72
|
+
if (!xml)
|
|
73
|
+
continue;
|
|
74
|
+
if (isSitemapIndex(xml)) {
|
|
75
|
+
for (const child of extractLocs(xml)) {
|
|
76
|
+
if (!seenSitemaps.has(child))
|
|
77
|
+
queue.push(child);
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
else {
|
|
81
|
+
pageUrls.push(...extractLocs(xml));
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
const candidates = [];
|
|
85
|
+
const seenPaths = new Set();
|
|
86
|
+
for (const raw of pageUrls) {
|
|
87
|
+
const candidate = (0, safety_1.toCandidate)(raw, baseUrl, 'sitemap');
|
|
88
|
+
if (!candidate)
|
|
89
|
+
continue;
|
|
90
|
+
if (seenPaths.has(candidate.url))
|
|
91
|
+
continue;
|
|
92
|
+
seenPaths.add(candidate.url);
|
|
93
|
+
candidates.push(candidate);
|
|
94
|
+
if (candidates.length >= maxUrls)
|
|
95
|
+
break;
|
|
96
|
+
}
|
|
97
|
+
return candidates;
|
|
98
|
+
}
|
|
99
|
+
async function resolveSitemapUrls(baseUrl, fetchImpl) {
|
|
100
|
+
const primary = new URL('/sitemap.xml', baseUrl).toString();
|
|
101
|
+
const head = await safeText(fetchImpl, primary);
|
|
102
|
+
if (head)
|
|
103
|
+
return [primary];
|
|
104
|
+
const robots = await safeText(fetchImpl, new URL('/robots.txt', baseUrl).toString());
|
|
105
|
+
if (robots) {
|
|
106
|
+
const advertised = sitemapsFromRobots(robots);
|
|
107
|
+
if (advertised.length > 0)
|
|
108
|
+
return advertised;
|
|
109
|
+
}
|
|
110
|
+
return [primary];
|
|
111
|
+
}
|
|
112
|
+
async function safeText(fetchImpl, url) {
|
|
113
|
+
try {
|
|
114
|
+
const res = await fetchImpl(url);
|
|
115
|
+
if (!res.ok)
|
|
116
|
+
return null;
|
|
117
|
+
return await res.text();
|
|
118
|
+
}
|
|
119
|
+
catch {
|
|
120
|
+
return null;
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
//# sourceMappingURL=sitemap.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"sitemap.js","sourceRoot":"","sources":["../../src/discovery/sitemap.ts"],"names":[],"mappings":";;AAiCA,kCASC;AAGD,wCAEC;AAYD,gDAOC;AAaD,kDA6CC;AA5HD,qCAAkD;AA8BlD,MAAM,MAAM,GAAG,oCAAoC,CAAC;AAEpD,2EAA2E;AAC3E,SAAgB,WAAW,CAAC,GAAW;IACrC,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,IAAI,CAAyB,CAAC;IAC9B,MAAM,CAAC,SAAS,GAAG,CAAC,CAAC;IACrB,OAAO,CAAC,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;QACvC,MAAM,GAAG,GAAG,cAAc,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QACxC,IAAI,GAAG;YAAE,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACzB,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,qEAAqE;AACrE,SAAgB,cAAc,CAAC,GAAW;IACxC,OAAO,qBAAqB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AACzC,CAAC;AAED,SAAS,cAAc,CAAC,CAAS;IAC/B,OAAO,CAAC;SACL,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC;SACtB,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC;SACrB,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC;SACrB,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;SACvB,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;AAC5B,CAAC;AAED,iFAAiF;AACjF,SAAgB,kBAAkB,CAAC,MAAc;IAC/C,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;QACzC,MAAM,CAAC,GAAG,uBAAuB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7C,IAAI,CAAC;YAAE,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IAC/B,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;;;;;;;GAUG;AACI,KAAK,UAAU,mBAAmB,CACvC,OAAe,EACf,UAA0B,EAAE;IAE5B,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAK,UAAU,CAAC,KAA8B,CAAC;IAClF,IAAI,CAAC,SAAS;QAAE,MAAM,IAAI,KAAK,CAAC,iEAAiE,CAAC,CAAC;IACnG,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,GAAG,CAAC;IACvC,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,EAAE,CAAC;IAE9C,MAAM,KAAK,GAAG,OAAO,CAAC,UAAU;QAC9B,CAAC,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC;QACtB,CAAC,CAAC,MAAM,kBAAkB,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;IAEjD,MAAM,YAAY,GAAG,IAAI,GAAG,EAAU,CAAC;IACvC,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,MAAM,KAAK,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC;IAEzB,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,YAAY,CAAC,IAAI,GAAG,WAAW,IAAI,QAAQ,CAAC,MAAM,GAAG,OAAO,EAAE,CAAC;QACxF,MAAM,GAAG,GAAG,KAAK,CAAC,KAAK,EAAG,CAAC;QAC3B,IAAI,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,SAAS;QACpC,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAEtB,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;QAC3C,IAAI,CAAC,GAAG;YAAE,SAAS;QAEnB,IAAI,cAAc,CAAC,GAAG,CAAC,EAAE,CAAC;YACxB,KAAK,MAAM,KAAK,IAAI,WAAW,CAAC,GAAG,CAAC,EAAE,CAAC;gBACrC,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC;oBAAE,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAClD,CAAC;QACH,CAAC;aAAM,CAAC;YACN,QAAQ,CAAC,IAAI,CAAC,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC;QACrC,CAAC;IACH,CAAC;IAED,MAAM,UAAU,GAAgB,EAAE,CAAC;IACnC,MAAM,SAAS,GAAG,IAAI,GAAG,EAAU,CAAC;IACpC,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;QAC3B,MAAM,SAAS,GAAG,IAAA,oBAAW,EAAC,GAAG,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;QACvD,IAAI,CAAC,SAAS;YAAE,SAAS;QACzB,IAAI,SAAS,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC;YAAE,SAAS;QAC3C,SAAS,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;QAC7B,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAC3B,IAAI,UAAU,CAAC,MAAM,IAAI,OAAO;YAAE,MAAM;IAC1C,CAAC;IACD,OAAO,UAAU,CAAC;AACpB,CAAC;AAED,KAAK,UAAU,kBAAkB,CAAC,OAAe,EAAE,SAAoB;IACrE,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,cAAc,EAAE,OAAO,CAAC,CAAC,QAAQ,EAAE,CAAC;IAC5D,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;IAChD,IAAI,IAAI;QAAE,OAAO,CAAC,OAAO,CAAC,CAAC;IAE3B,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,SAAS,EAAE,IAAI,GAAG,CAAC,aAAa,EAAE,OAAO,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC;IACrF,IAAI,MAAM,EAAE,CAAC;QACX,MAAM,UAAU,GAAG,kBAAkB,CAAC,MAAM,CAAC,CAAC;QAC9C,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC;YAAE,OAAO,UAAU,CAAC;IAC/C,CAAC;IACD,OAAO,CAAC,OAAO,CAAC,CAAC;AACnB,CAAC;AAED,KAAK,UAAU,QAAQ,CAAC,SAAoB,EAAE,GAAW;IACvD,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,SAAS,CAAC,GAAG,CAAC,CAAC;QACjC,IAAI,CAAC,GAAG,CAAC,EAAE;YAAE,OAAO,IAAI,CAAC;QACzB,OAAO,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;IAC1B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC"}
|
package/dist/htmlReport.d.ts
CHANGED
|
@@ -1,11 +1,39 @@
|
|
|
1
1
|
import type { DirReport, Report } from './cli/diff';
|
|
2
2
|
import type { Usage } from './ingest';
|
|
3
|
+
/** One test file the deterministic `fix` actually rewrote, with the exact oldβnew per selector. */
|
|
4
|
+
export interface AppliedEdit {
|
|
5
|
+
/** Relativized by the caller β htmlReport stays free of fs/path. */
|
|
6
|
+
file: string;
|
|
7
|
+
replacements: Array<{
|
|
8
|
+
from: string;
|
|
9
|
+
to: string;
|
|
10
|
+
count: number;
|
|
11
|
+
lines?: number[];
|
|
12
|
+
}>;
|
|
13
|
+
}
|
|
14
|
+
/** The verdict of re-running the suite AFTER the rewrite β the answer to "does it pass now?". */
|
|
15
|
+
export interface VerifyResult {
|
|
16
|
+
verdict: 'PASS' | 'FAIL' | 'SKIPPED';
|
|
17
|
+
command?: string;
|
|
18
|
+
exitCode?: number;
|
|
19
|
+
/** Why the re-run did not happen, when verdict is SKIPPED. */
|
|
20
|
+
reason?: string;
|
|
21
|
+
}
|
|
22
|
+
/** What `ia-qa-heal run` actually did after the diff: the applied rewrites and the post-fix re-run. */
|
|
23
|
+
export interface FixOutcome {
|
|
24
|
+
totalReplacements: number;
|
|
25
|
+
filesEdited: number;
|
|
26
|
+
edits: AppliedEdit[];
|
|
27
|
+
verify?: VerifyResult;
|
|
28
|
+
}
|
|
3
29
|
export interface DriftReportMeta {
|
|
4
30
|
baselineName: string;
|
|
5
31
|
currentName: string;
|
|
6
32
|
filesDiffed: number;
|
|
7
33
|
usage?: Usage | null;
|
|
8
34
|
baseUrl?: string;
|
|
35
|
+
/** Present only for a post-fix report (`run`): the rewrites applied and whether the re-run passed. */
|
|
36
|
+
outcome?: FixOutcome;
|
|
9
37
|
}
|
|
10
38
|
export declare function renderDriftReportHtml(agg: DirReport, reports: Array<{
|
|
11
39
|
name: string;
|
package/dist/htmlReport.js
CHANGED
|
@@ -120,6 +120,51 @@ function nextActionDrift(agg) {
|
|
|
120
120
|
}
|
|
121
121
|
return `<div class="next-action"><span class="na-label">What now?</span>${body}</div>`;
|
|
122
122
|
}
|
|
123
|
+
/**
|
|
124
|
+
* The post-fix banner: did the suite pass once the rewrite landed? This is the
|
|
125
|
+
* half a detection-only report could never show β the fix is only worth anything
|
|
126
|
+
* if the tests are green after it, so the re-run verdict leads.
|
|
127
|
+
*/
|
|
128
|
+
function verifyBanner(v) {
|
|
129
|
+
if (v.verdict === 'SKIPPED') {
|
|
130
|
+
return `<div class="next-action"><span class="na-label">Re-run skipped</span>${esc(v.reason || 'No test command configured.')} Run your suite to confirm the rewrite is green.</div>`;
|
|
131
|
+
}
|
|
132
|
+
const tone = v.verdict === 'PASS' ? 'pass' : 'fail';
|
|
133
|
+
const label = v.verdict === 'PASS' ? 'β
TESTS PASS' : 'β STILL FAILING';
|
|
134
|
+
const msg = v.verdict === 'PASS'
|
|
135
|
+
? 'The suite was re-run after the rewrite and passed β the fix is verified green.'
|
|
136
|
+
: `The suite was re-run after the rewrite and still failed (exit ${esc(v.exitCode)}). The selector rewrite landed, but something else is red β a human is needed.`;
|
|
137
|
+
return `<div class="verify verify-${tone}"><div class="verify-head"><span class="verdict-inline verdict-${tone}">${label}</span><span>${esc(msg)}</span></div>${v.command ? `<div class="verify-cmd">re-ran <code>${esc(v.command)}</code></div>` : ''}</div>`;
|
|
138
|
+
}
|
|
139
|
+
/**
|
|
140
|
+
* The applied-rewrite section of a post-fix report: exactly which selector became
|
|
141
|
+
* which, in which file and on which lines β the file-level before/after the diff's
|
|
142
|
+
* "healable" transitions only promised. Led by the re-run verdict.
|
|
143
|
+
*/
|
|
144
|
+
function outcomeSection(outcome) {
|
|
145
|
+
const verify = outcome.verify ? verifyBanner(outcome.verify) : '';
|
|
146
|
+
const editItems = outcome.edits
|
|
147
|
+
.map((e) => {
|
|
148
|
+
const reps = e.replacements
|
|
149
|
+
.map((r) => {
|
|
150
|
+
const lines = r.lines && r.lines.length > 0
|
|
151
|
+
? ` <span class="how">(line${r.lines.length === 1 ? '' : 's'} ${esc(r.lines.join(', '))})</span>`
|
|
152
|
+
: '';
|
|
153
|
+
return `<div class="drift-body"><span class="how">${esc(r.count)}Γ</span> <code class="old">${esc(r.from)}</code> <span class="arr">β</span> <code class="new">${esc(r.to)}</code>${lines}</div>`;
|
|
154
|
+
})
|
|
155
|
+
.join('');
|
|
156
|
+
return `<div class="drift-row"><div class="drift-head"><span class="badge badge-pass">β edited</span><span class="drift-label">${esc(e.file)}</span></div>${reps}</div>`;
|
|
157
|
+
})
|
|
158
|
+
.join('');
|
|
159
|
+
const summary = outcome.totalReplacements > 0
|
|
160
|
+
? `Rewrote ${plural(outcome.totalReplacements, 'selector reference')} across ${plural(outcome.filesEdited, 'test file')}, directly in your working tree. Nothing was committed β review with <code>git diff</code>.`
|
|
161
|
+
: `No test file referenced a healable selector β nothing was rewritten.`;
|
|
162
|
+
return `<section><h2>π§ Fix applied</h2>
|
|
163
|
+
<p class="run-summary">${summary}</p>
|
|
164
|
+
${verify}
|
|
165
|
+
${editItems}
|
|
166
|
+
</section>`;
|
|
167
|
+
}
|
|
123
168
|
/** A plain-English gloss of what a drift status means for the reader's tests. */
|
|
124
169
|
function statusPlain(status) {
|
|
125
170
|
switch (status) {
|
|
@@ -267,6 +312,12 @@ const REPORT_CSS = `
|
|
|
267
312
|
.drift-body .old{color:#fca5a5}.drift-body .new{color:#6ee7b7}.drift-body .arr{color:var(--orange2);font-weight:700;margin:0 4px}
|
|
268
313
|
.drift-body .how{color:var(--muted)}
|
|
269
314
|
.rebound{margin-top:7px;font-size:12px;color:#fca5a5;background:#ef444410;border:1px solid #ef444430;border-radius:7px;padding:6px 9px}
|
|
315
|
+
.verify{border-radius:10px;padding:12px 14px;margin-bottom:14px;font-size:13.5px;line-height:1.55}
|
|
316
|
+
.verify-pass{background:#10b98114;border:1px solid #10b98140;color:var(--text)}
|
|
317
|
+
.verify-fail{background:#ef444414;border:1px solid #ef444440;color:var(--text)}
|
|
318
|
+
.verify-head{display:flex;align-items:center;gap:11px;flex-wrap:wrap}
|
|
319
|
+
.verify-cmd{margin-top:7px;font-size:12px;color:var(--muted)}
|
|
320
|
+
.verify-cmd code{background:var(--bg);border:1px solid var(--border);border-radius:6px;padding:1px 6px;color:var(--orange2)}
|
|
270
321
|
.use{margin-left:auto;font-size:11px;padding:1px 8px;border-radius:99px;white-space:nowrap}
|
|
271
322
|
.use-hit{background:#8b5cf618;color:#c4b5fd;border:1px solid #8b5cf633}
|
|
272
323
|
.use-none{background:#3f3f4633;color:var(--muted);border:1px solid var(--border)}
|
|
@@ -380,9 +431,10 @@ ${reportHeader('Selector drift report', `<code>${esc(meta.baselineName)}</code>
|
|
|
380
431
|
<section class="hero verdict-${tone}">
|
|
381
432
|
<div class="verdict verdict-${tone}"><span class="big">${esc(agg.verdict)}</span><span class="headline">${esc(execHeadline(agg))}</span></div>
|
|
382
433
|
${runSummary}
|
|
383
|
-
${nextActionDrift(agg)}
|
|
434
|
+
${meta.outcome ? '' : nextActionDrift(agg)}
|
|
384
435
|
<div class="stats">${statsHtml}</div>
|
|
385
436
|
</section>
|
|
437
|
+
${meta.outcome ? outcomeSection(meta.outcome) : ''}
|
|
386
438
|
${layoutHtml}
|
|
387
439
|
<section>
|
|
388
440
|
<h2>π Pages (${agg.pageReports.length})</h2>
|