@qa-ai-stlc/explorer 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -0
- package/dist/allowlist.d.ts +5 -0
- package/dist/allowlist.d.ts.map +1 -0
- package/dist/allowlist.js +22 -0
- package/dist/allowlist.js.map +1 -0
- package/dist/crawl.d.ts +34 -0
- package/dist/crawl.d.ts.map +1 -0
- package/dist/crawl.js +98 -0
- package/dist/crawl.js.map +1 -0
- package/dist/extract-links.d.ts +9 -0
- package/dist/extract-links.d.ts.map +1 -0
- package/dist/extract-links.js +17 -0
- package/dist/extract-links.js.map +1 -0
- package/dist/index.d.ts +5 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +7 -0
- package/dist/index.js.map +1 -0
- package/dist/request-log.d.ts +9 -0
- package/dist/request-log.d.ts.map +1 -0
- package/dist/request-log.js +21 -0
- package/dist/request-log.js.map +1 -0
- package/dist/safe-mode.d.ts +10 -0
- package/dist/safe-mode.d.ts.map +1 -0
- package/dist/safe-mode.js +19 -0
- package/dist/safe-mode.js.map +1 -0
- package/package.json +37 -0
package/README.md
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
# @qa-ai-stlc/explorer
|
|
2
|
+
|
|
3
|
+
The crawler: route discovery from a start URL, following only in-allowlist links, in safe mode.
|
|
4
|
+
Every non-GET request is intercepted and cancelled before it reaches the network, so a crawl can
|
|
5
|
+
never mutate the application under test. Produces a `RouteMap` and a redactable HAR-shaped
|
|
6
|
+
request log for the engine to register as evidence.
|
|
7
|
+
|
|
8
|
+
Part of [QA-AI-STLC](https://github.com/Klim-101/QA-AI-STLC). See the repository root for
|
|
9
|
+
license, contributing and security information.
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
/** Checks `url`'s hostname against the configured domain allowlist (development plan section 6.4). */
|
|
2
|
+
export declare function isAllowedUrl(url: string, allowlist: readonly string[]): boolean;
|
|
3
|
+
/** Drops the fragment so `#section` links do not look like distinct routes. */
|
|
4
|
+
export declare function normalizeUrl(url: string): string;
|
|
5
|
+
//# sourceMappingURL=allowlist.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"allowlist.d.ts","sourceRoot":"","sources":["../src/allowlist.ts"],"names":[],"mappings":"AAGA,sGAAsG;AACtG,wBAAgB,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,SAAS,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAG/E;AAUD,+EAA+E;AAC/E,wBAAgB,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAIhD"}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
// Copyright The QA-AI-STLC Authors
|
|
2
|
+
// SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
/** Checks `url`'s hostname against the configured domain allowlist (development plan section 6.4). */
|
|
4
|
+
export function isAllowedUrl(url, allowlist) {
|
|
5
|
+
const hostname = tryHostname(url);
|
|
6
|
+
return hostname !== undefined && allowlist.includes(hostname);
|
|
7
|
+
}
|
|
8
|
+
function tryHostname(url) {
|
|
9
|
+
try {
|
|
10
|
+
return new URL(url).hostname;
|
|
11
|
+
}
|
|
12
|
+
catch {
|
|
13
|
+
return undefined;
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
/** Drops the fragment so `#section` links do not look like distinct routes. */
|
|
17
|
+
export function normalizeUrl(url) {
|
|
18
|
+
const parsed = new URL(url);
|
|
19
|
+
parsed.hash = '';
|
|
20
|
+
return parsed.toString();
|
|
21
|
+
}
|
|
22
|
+
//# sourceMappingURL=allowlist.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"allowlist.js","sourceRoot":"","sources":["../src/allowlist.ts"],"names":[],"mappings":"AAAA,mCAAmC;AACnC,sCAAsC;AAEtC,sGAAsG;AACtG,MAAM,UAAU,YAAY,CAAC,GAAW,EAAE,SAA4B;IACpE,MAAM,QAAQ,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC;IAClC,OAAO,QAAQ,KAAK,SAAS,IAAI,SAAS,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;AAChE,CAAC;AAED,SAAS,WAAW,CAAC,GAAW;IAC9B,IAAI,CAAC;QACH,OAAO,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;IAC/B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC;AAED,+EAA+E;AAC/E,MAAM,UAAU,YAAY,CAAC,GAAW;IACtC,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;IAC5B,MAAM,CAAC,IAAI,GAAG,EAAE,CAAC;IACjB,OAAO,MAAM,CAAC,QAAQ,EAAE,CAAC;AAC3B,CAAC"}
|
package/dist/crawl.d.ts
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import { type BrowserLauncher, type Clock } from '@qa-ai-stlc/core';
|
|
2
|
+
import { type IdentityConfig, type RouteMap } from '@qa-ai-stlc/schemas';
|
|
3
|
+
export interface CrawlIdentity {
|
|
4
|
+
readonly config: IdentityConfig;
|
|
5
|
+
readonly env: Readonly<Record<string, string | undefined>>;
|
|
6
|
+
/** Required, and only used, when `identity.config.auth` is `"cdp-attach"`. */
|
|
7
|
+
readonly cdpEndpointUrl?: string;
|
|
8
|
+
}
|
|
9
|
+
export interface CrawlOptions {
|
|
10
|
+
readonly startUrl: string;
|
|
11
|
+
/** Hostnames the crawl may navigate to (development plan section 6.4); a link outside it is dropped. */
|
|
12
|
+
readonly allowlist: readonly string[];
|
|
13
|
+
readonly browserLauncher: BrowserLauncher;
|
|
14
|
+
/** Signs in before crawling, reusing `authenticate()` (P1-05). Omit to crawl anonymously. */
|
|
15
|
+
readonly identity?: CrawlIdentity;
|
|
16
|
+
/** A safety cap on pages visited, independent of how many links are actually reachable. */
|
|
17
|
+
readonly maxPages?: number;
|
|
18
|
+
readonly clock?: Clock;
|
|
19
|
+
}
|
|
20
|
+
export interface CrawlResult {
|
|
21
|
+
readonly routeMap: RouteMap;
|
|
22
|
+
/** HAR-shaped, redact before registering as `network-har` evidence (development plan section 6.4). */
|
|
23
|
+
readonly requestLogHar: string;
|
|
24
|
+
/** Every non-GET request safe mode intercepted and cancelled; always 0 unless something is broken. */
|
|
25
|
+
readonly blockedRequestCount: number;
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* Crawls from `startUrl` by following in-allowlist links only, in safe mode: every non-GET
|
|
29
|
+
* request is intercepted and aborted before it reaches the network (development plan section 6.4
|
|
30
|
+
* — mutation budget zero). Produces a `RouteMap` and a redactable request log, never a live
|
|
31
|
+
* browser or page the caller has to manage.
|
|
32
|
+
*/
|
|
33
|
+
export declare function crawl(options: CrawlOptions): Promise<CrawlResult>;
|
|
34
|
+
//# sourceMappingURL=crawl.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"crawl.d.ts","sourceRoot":"","sources":["../src/crawl.ts"],"names":[],"mappings":"AAGA,OAAO,EAIL,KAAK,eAAe,EACpB,KAAK,KAAK,EAEX,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAGL,KAAK,cAAc,EACnB,KAAK,QAAQ,EACd,MAAM,qBAAqB,CAAC;AAQ7B,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,MAAM,EAAE,cAAc,CAAC;IAChC,QAAQ,CAAC,GAAG,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC;IAC3D,8EAA8E;IAC9E,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM,CAAC;CAClC;AAED,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,wGAAwG;IACxG,QAAQ,CAAC,SAAS,EAAE,SAAS,MAAM,EAAE,CAAC;IACtC,QAAQ,CAAC,eAAe,EAAE,eAAe,CAAC;IAC1C,6FAA6F;IAC7F,QAAQ,CAAC,QAAQ,CAAC,EAAE,aAAa,CAAC;IAClC,2FAA2F;IAC3F,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,KAAK,CAAC,EAAE,KAAK,CAAC;CACxB;AAED,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC;IAC5B,sGAAsG;IACtG,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,sGAAsG;IACtG,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;CACtC;AAQD;;;;;GAKG;AACH,wBAAsB,KAAK,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,WAAW,CAAC,CAoCvE"}
|
package/dist/crawl.js
ADDED
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
// Copyright The QA-AI-STLC Authors
|
|
2
|
+
// SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
import { authenticate, systemClock, } from '@qa-ai-stlc/core';
|
|
4
|
+
import { SCHEMA_VERSION, } from '@qa-ai-stlc/schemas';
|
|
5
|
+
import { isAllowedUrl, normalizeUrl } from './allowlist.js';
|
|
6
|
+
import { extractLinks } from './extract-links.js';
|
|
7
|
+
import { buildRequestLogHar } from './request-log.js';
|
|
8
|
+
import { createSafeModeRouteHandler } from './safe-mode.js';
|
|
9
|
+
const DEFAULT_MAX_PAGES = 50;
|
|
10
|
+
/**
|
|
11
|
+
* Crawls from `startUrl` by following in-allowlist links only, in safe mode: every non-GET
|
|
12
|
+
* request is intercepted and aborted before it reaches the network (development plan section 6.4
|
|
13
|
+
* — mutation budget zero). Produces a `RouteMap` and a redactable request log, never a live
|
|
14
|
+
* browser or page the caller has to manage.
|
|
15
|
+
*/
|
|
16
|
+
export async function crawl(options) {
|
|
17
|
+
const clock = options.clock ?? systemClock;
|
|
18
|
+
const maxPages = options.maxPages ?? DEFAULT_MAX_PAGES;
|
|
19
|
+
const storageState = await resolveStorageState(options);
|
|
20
|
+
const browser = await options.browserLauncher.launch();
|
|
21
|
+
try {
|
|
22
|
+
const context = await browser.newContext(storageState === undefined ? {} : { storageState });
|
|
23
|
+
const page = await context.newPage();
|
|
24
|
+
const logEntries = [];
|
|
25
|
+
let blockedRequestCount = 0;
|
|
26
|
+
await page.route('**/*', createSafeModeRouteHandler((entry) => {
|
|
27
|
+
blockedRequestCount += 1;
|
|
28
|
+
logEntries.push(entry);
|
|
29
|
+
}));
|
|
30
|
+
const routes = await visitAllowedRoutes(page, options, maxPages, logEntries);
|
|
31
|
+
const routeMap = {
|
|
32
|
+
schemaVersion: SCHEMA_VERSION,
|
|
33
|
+
generatedAt: clock.now().toISOString(),
|
|
34
|
+
startUrl: options.startUrl,
|
|
35
|
+
routes,
|
|
36
|
+
};
|
|
37
|
+
return {
|
|
38
|
+
routeMap,
|
|
39
|
+
requestLogHar: buildRequestLogHar(logEntries, routeMap.generatedAt),
|
|
40
|
+
blockedRequestCount,
|
|
41
|
+
};
|
|
42
|
+
}
|
|
43
|
+
finally {
|
|
44
|
+
await browser.close();
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
async function resolveStorageState(options) {
|
|
48
|
+
if (options.identity === undefined) {
|
|
49
|
+
return undefined;
|
|
50
|
+
}
|
|
51
|
+
return authenticate(options.browserLauncher, options.identity.config, options.identity.env, {
|
|
52
|
+
...(options.identity.cdpEndpointUrl !== undefined
|
|
53
|
+
? { cdpEndpointUrl: options.identity.cdpEndpointUrl }
|
|
54
|
+
: {}),
|
|
55
|
+
});
|
|
56
|
+
}
|
|
57
|
+
async function visitAllowedRoutes(page, options, maxPages, logEntries) {
|
|
58
|
+
const visited = new Set();
|
|
59
|
+
const routes = [];
|
|
60
|
+
const queue = [{ url: options.startUrl, discoveredVia: 'link' }];
|
|
61
|
+
while (queue.length > 0 && visited.size < maxPages) {
|
|
62
|
+
const next = queue.shift();
|
|
63
|
+
// The while condition already guarantees an element; noUncheckedIndexedAccess still types
|
|
64
|
+
// shift()'s result as possibly undefined, so this satisfies the type without being reachable.
|
|
65
|
+
/* v8 ignore next 3 */
|
|
66
|
+
if (next === undefined) {
|
|
67
|
+
break;
|
|
68
|
+
}
|
|
69
|
+
const url = normalizeUrl(next.url);
|
|
70
|
+
if (visited.has(url) || !isAllowedUrl(url, options.allowlist)) {
|
|
71
|
+
continue;
|
|
72
|
+
}
|
|
73
|
+
visited.add(url);
|
|
74
|
+
const response = await page.goto(url);
|
|
75
|
+
const httpStatus = response?.status();
|
|
76
|
+
logEntries.push({
|
|
77
|
+
method: 'GET',
|
|
78
|
+
url,
|
|
79
|
+
blocked: false,
|
|
80
|
+
...(httpStatus !== undefined ? { status: httpStatus } : {}),
|
|
81
|
+
});
|
|
82
|
+
routes.push({
|
|
83
|
+
url,
|
|
84
|
+
discoveredVia: next.discoveredVia,
|
|
85
|
+
...(next.discoveredFrom !== undefined ? { discoveredFrom: next.discoveredFrom } : {}),
|
|
86
|
+
...(httpStatus !== undefined ? { httpStatus } : {}),
|
|
87
|
+
});
|
|
88
|
+
if (httpStatus !== undefined && httpStatus < 400) {
|
|
89
|
+
for (const link of await extractLinks(page)) {
|
|
90
|
+
if (isAllowedUrl(link, options.allowlist)) {
|
|
91
|
+
queue.push({ url: link, discoveredVia: 'link', discoveredFrom: url });
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
return routes;
|
|
97
|
+
}
|
|
98
|
+
//# sourceMappingURL=crawl.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"crawl.js","sourceRoot":"","sources":["../src/crawl.ts"],"names":[],"mappings":"AAAA,mCAAmC;AACnC,sCAAsC;AAEtC,OAAO,EACL,YAAY,EACZ,WAAW,GAKZ,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EACL,cAAc,GAIf,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC5D,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,kBAAkB,EAAwB,MAAM,kBAAkB,CAAC;AAC5E,OAAO,EAAE,0BAA0B,EAAE,MAAM,gBAAgB,CAAC;AAE5D,MAAM,iBAAiB,GAAG,EAAE,CAAC;AAmC7B;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,KAAK,CAAC,OAAqB;IAC/C,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,WAAW,CAAC;IAC3C,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,iBAAiB,CAAC;IACvD,MAAM,YAAY,GAAG,MAAM,mBAAmB,CAAC,OAAO,CAAC,CAAC;IAExD,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,eAAe,CAAC,MAAM,EAAE,CAAC;IACvD,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC,YAAY,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,CAAC,CAAC;QAC7F,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;QAErC,MAAM,UAAU,GAAsB,EAAE,CAAC;QACzC,IAAI,mBAAmB,GAAG,CAAC,CAAC;QAC5B,MAAM,IAAI,CAAC,KAAK,CACd,MAAM,EACN,0BAA0B,CAAC,CAAC,KAAK,EAAE,EAAE;YACnC,mBAAmB,IAAI,CAAC,CAAC;YACzB,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACzB,CAAC,CAAC,CACH,CAAC;QAEF,MAAM,MAAM,GAAG,MAAM,kBAAkB,CAAC,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,UAAU,CAAC,CAAC;QAE7E,MAAM,QAAQ,GAAa;YACzB,aAAa,EAAE,cAAc;YAC7B,WAAW,EAAE,KAAK,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE;YACtC,QAAQ,EAAE,OAAO,CAAC,QAAQ;YAC1B,MAAM;SACP,CAAC;QACF,OAAO;YACL,QAAQ;YACR,aAAa,EAAE,kBAAkB,CAAC,UAAU,EAAE,QAAQ,CAAC,WAAW,CAAC;YACnE,mBAAmB;SACpB,CAAC;IACJ,CAAC;YAAS,CAAC;QACT,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;IACxB,CAAC;AACH,CAAC;AAED,KAAK,UAAU,mBAAmB,CAAC,OAAqB;IACtD,IAAI,OAAO,CAAC,QAAQ,KAAK,SAAS,EAAE,CAAC;QACnC,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,OAAO,YAAY,CAAC,OAAO,CAAC,eAAe,EAAE,OAAO,CAAC,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC,QAAQ,CAAC,GAAG,EAAE;QAC1F,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,cAAc,KAAK,SAAS;YAC/C,CAAC,CAAC,EAAE,cAAc,EAAE,OAAO,CAAC,QAAQ,CAAC,cAAc,EAAE;YACrD,CAAC,CAAC,EAAE,CAAC;KACR,CAAC,CAAC;AACL,CAAC;AAED,KAAK,UAAU,kBAAkB,CAC/B,IAAc,EACd,OAAqB,EACrB,QAAgB,EAChB,UAA6B;IAE7B,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;IAClC,MAAM,MAAM,GAAsB,EAAE,CAAC;IACrC,MAAM,KAAK,GAAgB,CAAC,EAAE,GAAG,EAAE,OAAO,CAAC,QAAQ,EAAE,aAAa,EAAE,MAAM,EAAE,CAAC,CAAC;IAE9E,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,OAAO,CAAC,IAAI,GAAG,QAAQ,EAAE,CAAC;QACnD,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,EAAE,CAAC;QAC3B,0FAA0F;QAC1F,8FAA8F;QAC9F,sBAAsB;QACtB,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;YACvB,MAAM;QACR,CAAC;QACD,MAAM,GAAG,GAAG,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACnC,IAAI,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,CAAC,GAAG,EAAE,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC;YAC9D,SAAS;QACX,CAAC;QACD,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAEjB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACtC,MAAM,UAAU,GAAG,QAAQ,EAAE,MAAM,EAAE,CAAC;QACtC,UAAU,CAAC,IAAI,CAAC;YACd,MAAM,EAAE,KAAK;YACb,GAAG;YACH,OAAO,EAAE,KAAK;YACd,GAAG,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAC5D,CAAC,CAAC;QACH,MAAM,CAAC,IAAI,CAAC;YACV,GAAG;YACH,aAAa,EAAE,IAAI,CAAC,aAAa;YACjC,GAAG,CAAC,IAAI,CAAC,cAAc,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,IAAI,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACrF,GAAG,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACpD,CAAC,CAAC;QAEH,IAAI,UAAU,KAAK,SAAS,IAAI,UAAU,GAAG,GAAG,EAAE,CAAC;YACjD,KAAK,MAAM,IAAI,IAAI,MAAM,YAAY,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC5C,IAAI,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC;oBAC1C,KAAK,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,aAAa,EAAE,MAAM,EAAE,cAAc,EAAE,GAAG,EAAE,CAAC,CAAC;gBACxE,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import type { AuthPage } from '@qa-ai-stlc/core';
|
|
2
|
+
/**
|
|
3
|
+
* Reads every `<a href>` on the current page. The callback runs inside the browser (Playwright
|
|
4
|
+
* serializes it across the CDP boundary), so it can reference DOM globals unavailable to the rest
|
|
5
|
+
* of this package; `AuthPage.evaluate` is untyped for exactly that reason, so the result is
|
|
6
|
+
* narrowed here instead of trusted.
|
|
7
|
+
*/
|
|
8
|
+
export declare function extractLinks(page: AuthPage): Promise<readonly string[]>;
|
|
9
|
+
//# sourceMappingURL=extract-links.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"extract-links.d.ts","sourceRoot":"","sources":["../src/extract-links.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAEjD;;;;;GAKG;AACH,wBAAsB,YAAY,CAAC,IAAI,EAAE,QAAQ,GAAG,OAAO,CAAC,SAAS,MAAM,EAAE,CAAC,CAS7E"}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
// Copyright The QA-AI-STLC Authors
|
|
2
|
+
// SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
/**
|
|
4
|
+
* Reads every `<a href>` on the current page. The callback runs inside the browser (Playwright
|
|
5
|
+
* serializes it across the CDP boundary), so it can reference DOM globals unavailable to the rest
|
|
6
|
+
* of this package; `AuthPage.evaluate` is untyped for exactly that reason, so the result is
|
|
7
|
+
* narrowed here instead of trusted.
|
|
8
|
+
*/
|
|
9
|
+
export async function extractLinks(page) {
|
|
10
|
+
/* v8 ignore next 3 -- runs in the browser's own V8 instance, invisible to Node coverage */
|
|
11
|
+
const result = await page.evaluate(() => Array.from(document.querySelectorAll('a[href]')).map((anchor) => anchor.href));
|
|
12
|
+
if (!Array.isArray(result)) {
|
|
13
|
+
return [];
|
|
14
|
+
}
|
|
15
|
+
return result.filter((item) => typeof item === 'string');
|
|
16
|
+
}
|
|
17
|
+
//# sourceMappingURL=extract-links.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"extract-links.js","sourceRoot":"","sources":["../src/extract-links.ts"],"names":[],"mappings":"AAAA,mCAAmC;AACnC,sCAAsC;AAItC;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,IAAc;IAC/C,2FAA2F;IAC3F,MAAM,MAAM,GAAY,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,CAC/C,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAE,MAA4B,CAAC,IAAI,CAAC,CACrG,CAAC;IACF,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QAC3B,OAAO,EAAE,CAAC;IACZ,CAAC;IACD,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,EAAkB,EAAE,CAAC,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC;AAC3E,CAAC"}
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAGA,cAAc,gBAAgB,CAAC;AAC/B,cAAc,YAAY,CAAC;AAC3B,cAAc,oBAAoB,CAAC;AACnC,cAAc,kBAAkB,CAAC"}
|
package/dist/index.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,mCAAmC;AACnC,sCAAsC;AAEtC,cAAc,gBAAgB,CAAC;AAC/B,cAAc,YAAY,CAAC;AAC3B,cAAc,oBAAoB,CAAC;AACnC,cAAc,kBAAkB,CAAC"}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
export interface RequestLogEntry {
|
|
2
|
+
readonly method: string;
|
|
3
|
+
readonly url: string;
|
|
4
|
+
/** Absent for a request safe mode blocked before it ever reached the network. */
|
|
5
|
+
readonly status?: number;
|
|
6
|
+
readonly blocked: boolean;
|
|
7
|
+
}
|
|
8
|
+
export declare function buildRequestLogHar(entries: readonly RequestLogEntry[], generatedAt: string): string;
|
|
9
|
+
//# sourceMappingURL=request-log.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"request-log.d.ts","sourceRoot":"","sources":["../src/request-log.ts"],"names":[],"mappings":"AAGA,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,iFAAiF;IACjF,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;CAC3B;AAMD,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,SAAS,eAAe,EAAE,EAAE,WAAW,EAAE,MAAM,GAAG,MAAM,CAanG"}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
// Copyright The QA-AI-STLC Authors
|
|
2
|
+
// SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
// HAR 1.2 shaped so `@qa-ai-stlc/core`'s `redactHar` (development plan section 6.4) can strip
|
|
4
|
+
// authorization headers, cookies and set-cookie headers before this is ever registered as
|
|
5
|
+
// evidence; headers and cookies are always present, even empty, so redaction has something to
|
|
6
|
+
// walk.
|
|
7
|
+
export function buildRequestLogHar(entries, generatedAt) {
|
|
8
|
+
return JSON.stringify({
|
|
9
|
+
log: {
|
|
10
|
+
version: '1.2',
|
|
11
|
+
creator: { name: 'qa-ai-stlc-explorer', version: '0' },
|
|
12
|
+
entries: entries.map((entry) => ({
|
|
13
|
+
startedDateTime: generatedAt,
|
|
14
|
+
blocked: entry.blocked,
|
|
15
|
+
request: { method: entry.method, url: entry.url, headers: [], cookies: [] },
|
|
16
|
+
response: entry.status === undefined ? undefined : { status: entry.status, headers: [], cookies: [] },
|
|
17
|
+
})),
|
|
18
|
+
},
|
|
19
|
+
});
|
|
20
|
+
}
|
|
21
|
+
//# sourceMappingURL=request-log.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"request-log.js","sourceRoot":"","sources":["../src/request-log.ts"],"names":[],"mappings":"AAAA,mCAAmC;AACnC,sCAAsC;AAUtC,8FAA8F;AAC9F,0FAA0F;AAC1F,8FAA8F;AAC9F,QAAQ;AACR,MAAM,UAAU,kBAAkB,CAAC,OAAmC,EAAE,WAAmB;IACzF,OAAO,IAAI,CAAC,SAAS,CAAC;QACpB,GAAG,EAAE;YACH,OAAO,EAAE,KAAK;YACd,OAAO,EAAE,EAAE,IAAI,EAAE,qBAAqB,EAAE,OAAO,EAAE,GAAG,EAAE;YACtD,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;gBAC/B,eAAe,EAAE,WAAW;gBAC5B,OAAO,EAAE,KAAK,CAAC,OAAO;gBACtB,OAAO,EAAE,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,GAAG,EAAE,KAAK,CAAC,GAAG,EAAE,OAAO,EAAE,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE;gBAC3E,QAAQ,EAAE,KAAK,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,OAAO,EAAE,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE;aACtG,CAAC,CAAC;SACJ;KACF,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { RouteHandler } from '@qa-ai-stlc/core';
|
|
2
|
+
import type { RequestLogEntry } from './request-log.js';
|
|
3
|
+
/**
|
|
4
|
+
* Safe mode's enforcement point (development plan section 6.4): every non-GET request is
|
|
5
|
+
* intercepted and aborted before it reaches the network, so a crawl can never mutate the
|
|
6
|
+
* application under test. `onBlocked` is called once per aborted request, for callers that need
|
|
7
|
+
* to count or log it.
|
|
8
|
+
*/
|
|
9
|
+
export declare function createSafeModeRouteHandler(onBlocked: (entry: RequestLogEntry) => void): RouteHandler;
|
|
10
|
+
//# sourceMappingURL=safe-mode.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"safe-mode.d.ts","sourceRoot":"","sources":["../src/safe-mode.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAExD;;;;;GAKG;AACH,wBAAgB,0BAA0B,CAAC,SAAS,EAAE,CAAC,KAAK,EAAE,eAAe,KAAK,IAAI,GAAG,YAAY,CASpG"}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
// Copyright The QA-AI-STLC Authors
|
|
2
|
+
// SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
/**
|
|
4
|
+
* Safe mode's enforcement point (development plan section 6.4): every non-GET request is
|
|
5
|
+
* intercepted and aborted before it reaches the network, so a crawl can never mutate the
|
|
6
|
+
* application under test. `onBlocked` is called once per aborted request, for callers that need
|
|
7
|
+
* to count or log it.
|
|
8
|
+
*/
|
|
9
|
+
export function createSafeModeRouteHandler(onBlocked) {
|
|
10
|
+
return (route) => {
|
|
11
|
+
const request = route.request();
|
|
12
|
+
if (request.method() === 'GET') {
|
|
13
|
+
return route.continue();
|
|
14
|
+
}
|
|
15
|
+
onBlocked({ method: request.method(), url: request.url(), blocked: true });
|
|
16
|
+
return route.abort();
|
|
17
|
+
};
|
|
18
|
+
}
|
|
19
|
+
//# sourceMappingURL=safe-mode.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"safe-mode.js","sourceRoot":"","sources":["../src/safe-mode.ts"],"names":[],"mappings":"AAAA,mCAAmC;AACnC,sCAAsC;AAKtC;;;;;GAKG;AACH,MAAM,UAAU,0BAA0B,CAAC,SAA2C;IACpF,OAAO,CAAC,KAAK,EAAE,EAAE;QACf,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,EAAE,CAAC;QAChC,IAAI,OAAO,CAAC,MAAM,EAAE,KAAK,KAAK,EAAE,CAAC;YAC/B,OAAO,KAAK,CAAC,QAAQ,EAAE,CAAC;QAC1B,CAAC;QACD,SAAS,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;QAC3E,OAAO,KAAK,CAAC,KAAK,EAAE,CAAC;IACvB,CAAC,CAAC;AACJ,CAAC"}
|
package/package.json
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@qa-ai-stlc/explorer",
|
|
3
|
+
"version": "0.1.0",
|
|
4
|
+
"description": "The crawler: route discovery in safe mode, redacted request logging, and the route map every later explorer phase builds on.",
|
|
5
|
+
"type": "module",
|
|
6
|
+
"license": "Apache-2.0",
|
|
7
|
+
"homepage": "https://github.com/Klim-101/QA-AI-STLC/tree/main/packages/explorer#readme",
|
|
8
|
+
"repository": {
|
|
9
|
+
"type": "git",
|
|
10
|
+
"url": "git+https://github.com/Klim-101/QA-AI-STLC.git",
|
|
11
|
+
"directory": "packages/explorer"
|
|
12
|
+
},
|
|
13
|
+
"bugs": "https://github.com/Klim-101/QA-AI-STLC/issues",
|
|
14
|
+
"files": [
|
|
15
|
+
"dist"
|
|
16
|
+
],
|
|
17
|
+
"exports": {
|
|
18
|
+
".": "./dist/index.js"
|
|
19
|
+
},
|
|
20
|
+
"scripts": {
|
|
21
|
+
"pretypecheck": "npm run build --workspace @qa-ai-stlc/schemas --workspace @qa-ai-stlc/core --workspace @qa-ai-stlc/test-utils",
|
|
22
|
+
"typecheck": "tsc --noEmit -p .",
|
|
23
|
+
"pretest": "npm run build --workspace @qa-ai-stlc/schemas --workspace @qa-ai-stlc/core --workspace @qa-ai-stlc/test-utils",
|
|
24
|
+
"test": "vitest run",
|
|
25
|
+
"pretest:coverage": "npm run build --workspace @qa-ai-stlc/schemas --workspace @qa-ai-stlc/core --workspace @qa-ai-stlc/test-utils",
|
|
26
|
+
"test:coverage": "vitest run --coverage",
|
|
27
|
+
"build": "tsc -b tsconfig.build.json",
|
|
28
|
+
"clean": "node ../../scripts/clean-dir.mjs dist"
|
|
29
|
+
},
|
|
30
|
+
"dependencies": {
|
|
31
|
+
"@qa-ai-stlc/core": "0.1.0",
|
|
32
|
+
"@qa-ai-stlc/schemas": "0.1.0"
|
|
33
|
+
},
|
|
34
|
+
"devDependencies": {
|
|
35
|
+
"@qa-ai-stlc/test-utils": "0.0.0"
|
|
36
|
+
}
|
|
37
|
+
}
|