@detiq/crawler 1.1.13 → 1.1.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Dockerfile +1 -1
- package/dist/accessibility-tree-crawler.js +156 -0
- package/dist/accessibility-tree-crawler.js.map +7 -0
- package/dist/advanced-ui-probes.js +310 -0
- package/dist/advanced-ui-probes.js.map +7 -0
- package/dist/content-chunker.js +2 -2
- package/dist/content-chunker.js.map +2 -2
- package/dist/crawl-session-manager.js +145 -0
- package/dist/crawl-session-manager.js.map +7 -0
- package/dist/crawler.js +1142 -263
- package/dist/crawler.js.map +2 -2
- package/dist/crud-interaction.js +264 -0
- package/dist/crud-interaction.js.map +7 -0
- package/dist/dropdown-enumerator.js +230 -0
- package/dist/dropdown-enumerator.js.map +7 -0
- package/dist/filter-pagination-crawler.js +299 -0
- package/dist/filter-pagination-crawler.js.map +7 -0
- package/dist/frontier.js +5 -3
- package/dist/frontier.js.map +2 -2
- package/dist/gap-pass.js +47 -14
- package/dist/gap-pass.js.map +2 -2
- package/dist/http-engine.js +28 -15
- package/dist/http-engine.js.map +2 -2
- package/dist/index.js +15 -1
- package/dist/index.js.map +2 -2
- package/dist/job-queue.js +12 -5
- package/dist/job-queue.js.map +2 -2
- package/dist/locator-extractor.js +23 -0
- package/dist/locator-extractor.js.map +2 -2
- package/dist/multi-context-handler.js +182 -0
- package/dist/multi-context-handler.js.map +7 -0
- package/dist/nested-window-crawler.js +172 -0
- package/dist/nested-window-crawler.js.map +7 -0
- package/dist/network-action-correlator.js +209 -0
- package/dist/network-action-correlator.js.map +7 -0
- package/dist/performance-metrics-collector.js +155 -0
- package/dist/performance-metrics-collector.js.map +7 -0
- package/dist/performance-metrics.js.map +2 -2
- package/dist/policies/crawl-safety.js +5 -1
- package/dist/policies/crawl-safety.js.map +2 -2
- package/dist/profiles/domain-profile.js +21 -1
- package/dist/profiles/domain-profile.js.map +2 -2
- package/dist/rate-limiter.js +2 -2
- package/dist/rate-limiter.js.map +2 -2
- package/dist/screen-reader.js +1 -0
- package/dist/screen-reader.js.map +2 -2
- package/dist/service-worker-interceptor.js +136 -0
- package/dist/service-worker-interceptor.js.map +7 -0
- package/dist/session-capture.js +4 -2
- package/dist/session-capture.js.map +2 -2
- package/dist/shadow-dom-crawler.js +132 -0
- package/dist/shadow-dom-crawler.js.map +7 -0
- package/dist/sitemap.js +16 -10
- package/dist/sitemap.js.map +2 -2
- package/dist/state-graph.js +193 -0
- package/dist/state-graph.js.map +7 -0
- package/dist/stealth/human-behavior.js +36 -3
- package/dist/stealth/human-behavior.js.map +2 -2
- package/dist/url-pattern-completer.js +48 -14
- package/dist/url-pattern-completer.js.map +2 -2
- package/dist/visual-diff-verdict.js +3 -1
- package/dist/visual-diff-verdict.js.map +2 -2
- package/package.json +2 -1
- package/src/accessibility-tree-crawler.ts +246 -0
- package/src/advanced-ui-probes.ts +470 -0
- package/src/crawl-session-manager.ts +231 -0
- package/src/crawler.ts +1071 -142
- package/src/crud-interaction.ts +386 -0
- package/src/dropdown-enumerator.ts +312 -0
- package/src/filter-pagination-crawler.ts +500 -0
- package/src/gap-pass.ts +58 -16
- package/src/http-engine.ts +29 -15
- package/src/index.ts +14 -1
- package/src/job-queue.ts +3 -1
- package/src/locator-extractor.ts +45 -0
- package/src/multi-context-handler.ts +296 -0
- package/src/nested-window-crawler.ts +310 -0
- package/src/network-action-correlator.ts +320 -0
- package/src/performance-metrics-collector.ts +243 -0
- package/src/performance-metrics.ts +1 -1
- package/src/screen-reader.ts +4 -1
- package/src/service-worker-interceptor.ts +185 -0
- package/src/session-capture.ts +3 -0
- package/src/shadow-dom-crawler.ts +198 -0
- package/src/sitemap.ts +17 -10
- package/src/state-graph.ts +280 -0
- package/src/stealth/human-behavior.ts +38 -4
- package/src/url-pattern-completer.ts +61 -15
- package/tsconfig.json +1 -1
- package/types/accessibility-tree-crawler.d.ts +44 -0
- package/types/advanced-ui-probes.d.ts +47 -0
- package/types/crawl-session-manager.d.ts +44 -0
- package/types/crawler.d.ts +3 -1
- package/types/crud-interaction.d.ts +31 -0
- package/types/dropdown-enumerator.d.ts +20 -0
- package/types/filter-pagination-crawler.d.ts +52 -0
- package/types/gap-pass.d.ts +1 -0
- package/types/index.d.ts +14 -1
- package/types/job-queue.d.ts +18 -9
- package/types/locator-extractor.d.ts +7 -0
- package/types/multi-context-handler.d.ts +55 -0
- package/types/nested-window-crawler.d.ts +41 -0
- package/types/network-action-correlator.d.ts +47 -0
- package/types/performance-metrics-collector.d.ts +50 -0
- package/types/performance-metrics.d.ts +1 -2
- package/types/screen-reader.d.ts +1 -2
- package/types/service-worker-interceptor.d.ts +36 -0
- package/types/session-capture.d.ts +1 -1
- package/types/shadow-dom-crawler.d.ts +35 -0
- package/types/sitemap.d.ts +2 -2
- package/types/state-graph.d.ts +59 -0
- package/types/stealth/human-behavior.d.ts +1 -1
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Session management for authenticated crawl sessions.
|
|
3
|
+
* Detects auth expiry via HTTP status codes and login-page redirects,
|
|
4
|
+
* then attempts recovery via a token refresh endpoint or re-login flow.
|
|
5
|
+
* Designed to keep long-running crawls alive without manual intervention.
|
|
6
|
+
*/
|
|
7
|
+
import type { Page } from 'playwright';
|
|
8
|
+
export interface SessionConfig {
|
|
9
|
+
loginUrl?: string;
|
|
10
|
+
email?: string;
|
|
11
|
+
password?: string;
|
|
12
|
+
healthCheckUrl?: string;
|
|
13
|
+
refreshEndpoint?: string;
|
|
14
|
+
tokenLocalStorageKey?: string;
|
|
15
|
+
maxRefreshAttempts?: number;
|
|
16
|
+
}
|
|
17
|
+
export type SessionEventType = 'HEALTHY' | 'EXPIRY_DETECTED' | 'REFRESH_ATTEMPT' | 'REFRESH_SUCCESS' | 'REFRESH_FAILED' | 'AUTH_FAILURE';
|
|
18
|
+
export interface SessionEvent {
|
|
19
|
+
type: SessionEventType;
|
|
20
|
+
isoTimestamp: string;
|
|
21
|
+
url?: string;
|
|
22
|
+
strategy?: string;
|
|
23
|
+
attemptCount?: number;
|
|
24
|
+
}
|
|
25
|
+
export interface SessionManager {
|
|
26
|
+
onResponse(url: string, status: number): void;
|
|
27
|
+
onNavigation(url: string): void;
|
|
28
|
+
isHealthy(): boolean;
|
|
29
|
+
needsRefresh(): boolean;
|
|
30
|
+
markHealthy(isoTimestamp: string): void;
|
|
31
|
+
refresh(page: Page, isoTimestamp: string): Promise<boolean>;
|
|
32
|
+
getEvents(): SessionEvent[];
|
|
33
|
+
getStats(): {
|
|
34
|
+
healthy: boolean;
|
|
35
|
+
refreshAttempts: number;
|
|
36
|
+
authFailure: boolean;
|
|
37
|
+
};
|
|
38
|
+
}
|
|
39
|
+
export declare function createSessionManager(config: SessionConfig): SessionManager;
|
|
40
|
+
/**
|
|
41
|
+
* Attaches a Playwright response listener to feed HTTP status codes into the
|
|
42
|
+
* session manager. Returns a cleanup function to remove the listener.
|
|
43
|
+
*/
|
|
44
|
+
export declare function attachSessionWatcher(page: Page, manager: SessionManager): () => void;
|
package/types/crawler.d.ts
CHANGED
|
@@ -164,6 +164,7 @@ export declare function crawlProject(opts: {
|
|
|
164
164
|
segmentIndex: number;
|
|
165
165
|
hasMoreSegments: boolean;
|
|
166
166
|
domHashes: string[];
|
|
167
|
+
crawlWarnings?: undefined;
|
|
167
168
|
} | {
|
|
168
169
|
screensFound: number;
|
|
169
170
|
screensErrored: number;
|
|
@@ -188,9 +189,10 @@ export declare function crawlProject(opts: {
|
|
|
188
189
|
segmentIndex: number;
|
|
189
190
|
hasMoreSegments: boolean;
|
|
190
191
|
domHashes: string[];
|
|
192
|
+
crawlWarnings: string[] | undefined;
|
|
191
193
|
discoveredUrls?: undefined;
|
|
192
194
|
}>;
|
|
193
|
-
export declare function domFallbackElements(page: any): Promise<Array<{
|
|
195
|
+
export declare function domFallbackElements(page: any, rawPage?: any): Promise<Array<{
|
|
194
196
|
meaning: string;
|
|
195
197
|
role: string;
|
|
196
198
|
expectedData?: string;
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import type { Page } from 'playwright';
|
|
2
|
+
export type EditFlowType = 'inline' | 'modal' | 'navigation' | 'unknown';
|
|
3
|
+
export interface CrudFlowCapture {
|
|
4
|
+
entityName: string;
|
|
5
|
+
createTrigger?: string;
|
|
6
|
+
editFlowType: EditFlowType;
|
|
7
|
+
editFields: Array<{
|
|
8
|
+
name: string;
|
|
9
|
+
type: string;
|
|
10
|
+
prefilledValue?: string;
|
|
11
|
+
}>;
|
|
12
|
+
deleteConfirmation?: {
|
|
13
|
+
message: string;
|
|
14
|
+
confirmLabel: string;
|
|
15
|
+
cancelLabel: string;
|
|
16
|
+
};
|
|
17
|
+
createSuccessToast?: string;
|
|
18
|
+
editSuccessToast?: string;
|
|
19
|
+
rowCount: number;
|
|
20
|
+
hasDeleteButton: boolean;
|
|
21
|
+
hasEditButton: boolean;
|
|
22
|
+
}
|
|
23
|
+
export interface CrudProbeResult {
|
|
24
|
+
flows: CrudFlowCapture[];
|
|
25
|
+
toastsObserved: Array<{
|
|
26
|
+
action: string;
|
|
27
|
+
text: string;
|
|
28
|
+
role: string;
|
|
29
|
+
}>;
|
|
30
|
+
}
|
|
31
|
+
export declare function probeCrudFlows(page: Page, entityName?: string): Promise<CrudProbeResult>;
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import type { Page } from 'playwright';
|
|
2
|
+
export interface DropdownOption {
|
|
3
|
+
value: string;
|
|
4
|
+
label: string;
|
|
5
|
+
disabled?: boolean;
|
|
6
|
+
}
|
|
7
|
+
export interface DropdownDef {
|
|
8
|
+
kind: 'native-select' | 'combobox' | 'listbox' | 'multiselect';
|
|
9
|
+
label: string;
|
|
10
|
+
locatorHint: string;
|
|
11
|
+
options: DropdownOption[];
|
|
12
|
+
defaultValue?: string;
|
|
13
|
+
cascadesFrom?: string;
|
|
14
|
+
}
|
|
15
|
+
export interface DropdownEnumerationResult {
|
|
16
|
+
dropdowns: DropdownDef[];
|
|
17
|
+
conditionalRevealMap: Record<string, string[]>;
|
|
18
|
+
totalOptionsFound: number;
|
|
19
|
+
}
|
|
20
|
+
export declare function enumerateDropdowns(page: Page): Promise<DropdownEnumerationResult>;
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Filter state crawling and pagination capture.
|
|
3
|
+
*
|
|
4
|
+
* Discovers filter controls (select, checkbox-group, radio-group, search),
|
|
5
|
+
* applies each meaningful option and records result counts and first-row
|
|
6
|
+
* values, then detects and walks pagination (numbered, load-more, infinite
|
|
7
|
+
* scroll). Also captures sort column behaviour.
|
|
8
|
+
*
|
|
9
|
+
* All per-filter / per-page errors are caught and skipped so one bad control
|
|
10
|
+
* never aborts the full crawl.
|
|
11
|
+
*/
|
|
12
|
+
import type { Page } from 'playwright';
|
|
13
|
+
export interface FilterOption {
|
|
14
|
+
value: string;
|
|
15
|
+
label: string;
|
|
16
|
+
}
|
|
17
|
+
export interface FilterDef {
|
|
18
|
+
kind: 'select' | 'checkbox-group' | 'radio-group' | 'search';
|
|
19
|
+
label: string;
|
|
20
|
+
locatorHint: string;
|
|
21
|
+
options: FilterOption[];
|
|
22
|
+
}
|
|
23
|
+
export interface FilterState {
|
|
24
|
+
filterLabel: string;
|
|
25
|
+
filterValue: string;
|
|
26
|
+
resultCount: number | null;
|
|
27
|
+
firstRowValues: string[];
|
|
28
|
+
}
|
|
29
|
+
export interface PaginationCapture {
|
|
30
|
+
kind: 'numbered' | 'load-more' | 'infinite-scroll' | 'none';
|
|
31
|
+
totalPages?: number;
|
|
32
|
+
totalItems?: number;
|
|
33
|
+
pageSizes?: number[];
|
|
34
|
+
pages: Array<{
|
|
35
|
+
pageNum: number;
|
|
36
|
+
rowValues: string[];
|
|
37
|
+
}>;
|
|
38
|
+
}
|
|
39
|
+
export interface FilterPaginationResult {
|
|
40
|
+
filters: FilterDef[];
|
|
41
|
+
filterStates: FilterState[];
|
|
42
|
+
pagination: PaginationCapture;
|
|
43
|
+
sortColumns: Array<{
|
|
44
|
+
header: string;
|
|
45
|
+
ariaSortAfterClick: string | null;
|
|
46
|
+
}>;
|
|
47
|
+
searchCapture?: {
|
|
48
|
+
noResultsText: string | null;
|
|
49
|
+
sampleSuggestions: string[];
|
|
50
|
+
};
|
|
51
|
+
}
|
|
52
|
+
export declare function crawlFiltersAndPagination(page: Page): Promise<FilterPaginationResult>;
|
package/types/gap-pass.d.ts
CHANGED
package/types/index.d.ts
CHANGED
|
@@ -35,5 +35,18 @@ export { startSpan, endSpan, withSpan, increment, flushMetrics } from './telemet
|
|
|
35
35
|
export type { Span, FinishedSpan } from './telemetry.js';
|
|
36
36
|
export { analyzeFailure } from './failure-analysis-agent.js';
|
|
37
37
|
export type { FailureContext } from './failure-analysis-agent.js';
|
|
38
|
-
export { elementToPlaywrightLocator, elementsToPlaywrightLocators, formatLocatorsForPrompt } from './locator-extractor.js';
|
|
38
|
+
export { elementToPlaywrightLocator, elementsToPlaywrightLocators, formatLocatorsForPrompt, detectAuthWallLocators } from './locator-extractor.js';
|
|
39
39
|
export type { PlaywrightLocator } from './locator-extractor.js';
|
|
40
|
+
export * from './dropdown-enumerator.js';
|
|
41
|
+
export * from './crud-interaction.js';
|
|
42
|
+
export * from './filter-pagination-crawler.js';
|
|
43
|
+
export * from './multi-context-handler.js';
|
|
44
|
+
export * from './advanced-ui-probes.js';
|
|
45
|
+
export * from './state-graph.js';
|
|
46
|
+
export * from './nested-window-crawler.js';
|
|
47
|
+
export * from './shadow-dom-crawler.js';
|
|
48
|
+
export * from './network-action-correlator.js';
|
|
49
|
+
export * from './crawl-session-manager.js';
|
|
50
|
+
export * from './accessibility-tree-crawler.js';
|
|
51
|
+
export * from './performance-metrics.js';
|
|
52
|
+
export * from './service-worker-interceptor.js';
|
package/types/job-queue.d.ts
CHANGED
|
@@ -111,22 +111,24 @@ export declare function createTrackedJob(type: string, config: {
|
|
|
111
111
|
projectId: string;
|
|
112
112
|
[k: string]: any;
|
|
113
113
|
}): Promise<string>;
|
|
114
|
-
/** Mark a tracked job DONE or FAILED. Safe to call in finally — swallows errors.
|
|
114
|
+
/** Mark a tracked job DONE or FAILED. Safe to call in finally — swallows errors.
|
|
115
|
+
* Gap 30: When marking FAILED, sets nextRetryAt (1 min) so the retry worker picks it up promptly. */
|
|
115
116
|
export declare function finishTrackedJob(tenantId: string, jobId: string, error?: string): Promise<void>;
|
|
116
117
|
/** Enqueue a generic platform job — returns the job ID immediately. */
|
|
117
118
|
export declare function enqueueJob(type: string, config: any): Promise<string>;
|
|
118
119
|
/** Get the current state of a platform job. */
|
|
119
120
|
export declare function getJob(tenantId: string, jobId: string): Promise<{
|
|
120
121
|
progress: number;
|
|
121
|
-
|
|
122
|
+
id: string;
|
|
123
|
+
name: string | null;
|
|
122
124
|
type: string;
|
|
125
|
+
error: string | null;
|
|
126
|
+
result: import("@prisma/client/runtime/library").JsonValue | null;
|
|
123
127
|
status: string;
|
|
124
128
|
tenantId: string;
|
|
125
129
|
projectId: string;
|
|
126
|
-
name: string | null;
|
|
127
|
-
result: import("@prisma/client/runtime/library").JsonValue | null;
|
|
128
|
-
id: string;
|
|
129
130
|
config: import("@prisma/client/runtime/library").JsonValue;
|
|
131
|
+
retryCount: number;
|
|
130
132
|
queuedAt: Date;
|
|
131
133
|
startedAt: Date | null;
|
|
132
134
|
finishedAt: Date | null;
|
|
@@ -137,19 +139,23 @@ export declare function getJob(tenantId: string, jobId: string): Promise<{
|
|
|
137
139
|
totalCacheReadTokens: number | null;
|
|
138
140
|
totalEstimatedCostUsd: number | null;
|
|
139
141
|
llmCallCount: number;
|
|
142
|
+
nextRetryAt: Date | null;
|
|
143
|
+
dlq: boolean;
|
|
144
|
+
dlqReason: string | null;
|
|
140
145
|
} | null>;
|
|
141
146
|
/** List recent jobs for a project. */
|
|
142
147
|
export declare function listJobs(tenantId: string, projectId: string): Promise<{
|
|
143
148
|
progress: number;
|
|
144
|
-
|
|
149
|
+
id: string;
|
|
150
|
+
name: string | null;
|
|
145
151
|
type: string;
|
|
152
|
+
error: string | null;
|
|
153
|
+
result: import("@prisma/client/runtime/library").JsonValue | null;
|
|
146
154
|
status: string;
|
|
147
155
|
tenantId: string;
|
|
148
156
|
projectId: string;
|
|
149
|
-
name: string | null;
|
|
150
|
-
result: import("@prisma/client/runtime/library").JsonValue | null;
|
|
151
|
-
id: string;
|
|
152
157
|
config: import("@prisma/client/runtime/library").JsonValue;
|
|
158
|
+
retryCount: number;
|
|
153
159
|
queuedAt: Date;
|
|
154
160
|
startedAt: Date | null;
|
|
155
161
|
finishedAt: Date | null;
|
|
@@ -160,6 +166,9 @@ export declare function listJobs(tenantId: string, projectId: string): Promise<{
|
|
|
160
166
|
totalCacheReadTokens: number | null;
|
|
161
167
|
totalEstimatedCostUsd: number | null;
|
|
162
168
|
llmCallCount: number;
|
|
169
|
+
nextRetryAt: Date | null;
|
|
170
|
+
dlq: boolean;
|
|
171
|
+
dlqReason: string | null;
|
|
163
172
|
}[]>;
|
|
164
173
|
/** Cancel a RUNNING or QUEUED job. Aborts the in-process crawl and marks DB CANCELLED. */
|
|
165
174
|
export declare function cancelJob(tenantId: string, jobId: string): Promise<void>;
|
|
@@ -22,6 +22,7 @@ export declare function elementToPlaywrightLocator(element: {
|
|
|
22
22
|
/**
|
|
23
23
|
* Convert an array of DB Element records to deduplicated Playwright locators.
|
|
24
24
|
* Deduplicates by key — higher confidence wins.
|
|
25
|
+
* Filters out crawl-analysis meta-annotations and produces an auth-wall warning when applicable.
|
|
25
26
|
*/
|
|
26
27
|
export declare function elementsToPlaywrightLocators(elements: Array<{
|
|
27
28
|
role?: string | null;
|
|
@@ -32,6 +33,12 @@ export declare function elementsToPlaywrightLocators(elements: Array<{
|
|
|
32
33
|
isInteractive?: boolean | null;
|
|
33
34
|
ariaState?: Record<string, unknown> | null;
|
|
34
35
|
}>): PlaywrightLocator[];
|
|
36
|
+
/**
|
|
37
|
+
* Detect whether a set of locators was extracted from an auth-wall page.
|
|
38
|
+
* Returns a warning message if auth-wall detected, null otherwise.
|
|
39
|
+
* Heuristic: >50% of locators are low-confidence text matches AND at least one matches auth-wall pattern.
|
|
40
|
+
*/
|
|
41
|
+
export declare function detectAuthWallLocators(locators: PlaywrightLocator[]): string | null;
|
|
35
42
|
/**
|
|
36
43
|
* Format locators as a block for injection into LLM script generation prompts.
|
|
37
44
|
* Returns empty string if no locators.
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Multi-context browser handling — detects and records new tabs, iframes,
|
|
3
|
+
* popups, and file downloads triggered during a page crawl.
|
|
4
|
+
* This lets ZeTa discover cross-origin flows, OAuth redirects, embedded
|
|
5
|
+
* third-party content, and downloadable resources that single-page HTTP
|
|
6
|
+
* crawlers would miss entirely.
|
|
7
|
+
*/
|
|
8
|
+
import type { Page, BrowserContext } from 'playwright';
|
|
9
|
+
export interface NewTabRecord {
|
|
10
|
+
url: string;
|
|
11
|
+
linkText: string;
|
|
12
|
+
isInternal: boolean;
|
|
13
|
+
httpStatus?: number;
|
|
14
|
+
}
|
|
15
|
+
export interface IframeRecord {
|
|
16
|
+
src: string;
|
|
17
|
+
crossOrigin: boolean;
|
|
18
|
+
sandboxed: boolean;
|
|
19
|
+
sandboxValue: string | null;
|
|
20
|
+
accessible: boolean;
|
|
21
|
+
contentSample: string;
|
|
22
|
+
hasForm: boolean;
|
|
23
|
+
}
|
|
24
|
+
export interface PopupRecord {
|
|
25
|
+
url: string;
|
|
26
|
+
triggerLabel: string;
|
|
27
|
+
isOAuthLike: boolean;
|
|
28
|
+
}
|
|
29
|
+
export interface DownloadRecord {
|
|
30
|
+
filename: string;
|
|
31
|
+
downloadUrl: string;
|
|
32
|
+
triggerLabel: string;
|
|
33
|
+
}
|
|
34
|
+
export interface MultiContextResult {
|
|
35
|
+
newTabs: NewTabRecord[];
|
|
36
|
+
iframes: IframeRecord[];
|
|
37
|
+
popups: PopupRecord[];
|
|
38
|
+
downloads: DownloadRecord[];
|
|
39
|
+
}
|
|
40
|
+
/**
|
|
41
|
+
* Attaches a listener to the BrowserContext that records every new tab
|
|
42
|
+
* opened while the listener is active. Returns an unsubscribe function.
|
|
43
|
+
*
|
|
44
|
+
* Call this before navigating / clicking so that tabs opened during the
|
|
45
|
+
* crawl are captured, then call the returned function to stop listening.
|
|
46
|
+
*/
|
|
47
|
+
export declare function attachNewTabListener(context: BrowserContext, seedUrl: string, result: MultiContextResult): () => void;
|
|
48
|
+
/**
|
|
49
|
+
* Probes the current page for multi-context browser events: iframes,
|
|
50
|
+
* popup windows, and file downloads.
|
|
51
|
+
*
|
|
52
|
+
* New-tab tracking is intentionally excluded here — it must be set up at
|
|
53
|
+
* the crawler level (before navigation) using `attachNewTabListener`.
|
|
54
|
+
*/
|
|
55
|
+
export declare function probeMultiContext(page: Page, context: BrowserContext, seedUrl: string): Promise<MultiContextResult>;
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Nested window / browsing-context crawler — discovers and records frames,
|
|
3
|
+
* embedded apps, and popup windows that are already open in the browser
|
|
4
|
+
* context at the time of a crawl step.
|
|
5
|
+
*
|
|
6
|
+
* Complements multi-context-handler.ts: that module handles proactive popup
|
|
7
|
+
* interception via context.on('page'); this module snapshots what is already
|
|
8
|
+
* present in the frame tree and context page list.
|
|
9
|
+
*/
|
|
10
|
+
import type { BrowserContext } from 'playwright';
|
|
11
|
+
export type BrowsingContextType = 'page' | 'frame' | 'popup' | 'embedded-app';
|
|
12
|
+
export interface BrowsingContextNode {
|
|
13
|
+
id: string;
|
|
14
|
+
type: BrowsingContextType;
|
|
15
|
+
url: string;
|
|
16
|
+
parentId: string | null;
|
|
17
|
+
depth: number;
|
|
18
|
+
title: string;
|
|
19
|
+
interactiveCount: number;
|
|
20
|
+
accessible: boolean;
|
|
21
|
+
isEmbeddedApp: boolean;
|
|
22
|
+
}
|
|
23
|
+
export interface NestedCrawlResult {
|
|
24
|
+
contextTree: BrowsingContextNode[];
|
|
25
|
+
totalContextsFound: number;
|
|
26
|
+
maxDepthReached: number;
|
|
27
|
+
inaccessibleCount: number;
|
|
28
|
+
frameUrls: string[];
|
|
29
|
+
embeddedAppUrls: string[];
|
|
30
|
+
}
|
|
31
|
+
/**
|
|
32
|
+
* Discovers all nested browsing contexts reachable from the current page:
|
|
33
|
+
* sub-frames, embedded applications, and already-open popup pages.
|
|
34
|
+
*
|
|
35
|
+
* All failures are non-fatal — errors are caught and an empty result is
|
|
36
|
+
* returned so the caller's crawl step can continue uninterrupted.
|
|
37
|
+
*/
|
|
38
|
+
export declare function crawlNestedContexts(page: any, context: BrowserContext | null, seedUrl: string, opts?: {
|
|
39
|
+
maxFrameDepth?: number;
|
|
40
|
+
maxPopupDepth?: number;
|
|
41
|
+
}): Promise<NestedCrawlResult>;
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Network-action correlator — captures XHR/fetch requests that occur during
|
|
3
|
+
* a named UI action (click, form submit, navigation) and maps them to API
|
|
4
|
+
* endpoint contracts.
|
|
5
|
+
*
|
|
6
|
+
* Usage:
|
|
7
|
+
* const correlator = createCorrelator();
|
|
8
|
+
* const detach = attachNetworkInterceptor(page, correlator);
|
|
9
|
+
* correlator.startWindow('Submit login form');
|
|
10
|
+
* await page.click('#login-btn');
|
|
11
|
+
* const correlation = correlator.endWindow();
|
|
12
|
+
* detach();
|
|
13
|
+
* const result = correlator.getResult();
|
|
14
|
+
*/
|
|
15
|
+
import type { Page } from 'playwright';
|
|
16
|
+
export interface NetworkRequestRecord {
|
|
17
|
+
url: string;
|
|
18
|
+
urlPattern: string;
|
|
19
|
+
method: string;
|
|
20
|
+
bodySchema: Record<string, string> | null;
|
|
21
|
+
status: number | null;
|
|
22
|
+
responseSchema: Record<string, string> | null;
|
|
23
|
+
isGraphQL: boolean;
|
|
24
|
+
graphQLOperation?: string;
|
|
25
|
+
resourceType: string;
|
|
26
|
+
correlatedActionLabel?: string;
|
|
27
|
+
}
|
|
28
|
+
export interface ActionCorrelation {
|
|
29
|
+
actionLabel: string;
|
|
30
|
+
requests: NetworkRequestRecord[];
|
|
31
|
+
}
|
|
32
|
+
export interface NetworkCorrelationResult {
|
|
33
|
+
correlations: ActionCorrelation[];
|
|
34
|
+
uniqueEndpoints: string[];
|
|
35
|
+
graphqlOperations: string[];
|
|
36
|
+
totalRequestsCaptured: number;
|
|
37
|
+
coverage: number;
|
|
38
|
+
}
|
|
39
|
+
interface NetworkCorrelator {
|
|
40
|
+
startWindow(actionLabel: string): void;
|
|
41
|
+
recordRequest(rec: NetworkRequestRecord): void;
|
|
42
|
+
endWindow(): ActionCorrelation;
|
|
43
|
+
getResult(): NetworkCorrelationResult;
|
|
44
|
+
}
|
|
45
|
+
export declare function createCorrelator(): NetworkCorrelator;
|
|
46
|
+
export declare function attachNetworkInterceptor(page: Page, correlator: ReturnType<typeof createCorrelator>): () => void;
|
|
47
|
+
export {};
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Performance metrics collector — captures Core Web Vitals and navigation timing
|
|
3
|
+
* for pages visited during crawl runs.
|
|
4
|
+
* Collects LCP, CLS, INP, TTFB, long tasks, and slow resources via PerformanceObserver
|
|
5
|
+
* and the Navigation/Resource Timing APIs.
|
|
6
|
+
*/
|
|
7
|
+
import type { Page } from 'playwright';
|
|
8
|
+
export type CwvScore = 'good' | 'needs-improvement' | 'poor';
|
|
9
|
+
export interface LcpMetric {
|
|
10
|
+
value: number;
|
|
11
|
+
score: CwvScore;
|
|
12
|
+
elementTag: string | null;
|
|
13
|
+
url: string | null;
|
|
14
|
+
}
|
|
15
|
+
export interface ClsMetric {
|
|
16
|
+
value: number;
|
|
17
|
+
score: CwvScore;
|
|
18
|
+
shiftCount: number;
|
|
19
|
+
}
|
|
20
|
+
export interface InpMetric {
|
|
21
|
+
p75: number;
|
|
22
|
+
score: CwvScore;
|
|
23
|
+
sampleCount: number;
|
|
24
|
+
}
|
|
25
|
+
export interface NavTimingMetric {
|
|
26
|
+
ttfb: number;
|
|
27
|
+
ttfbScore: CwvScore;
|
|
28
|
+
domContentLoaded: number;
|
|
29
|
+
domComplete: number;
|
|
30
|
+
loadComplete: number;
|
|
31
|
+
transferSizeBytes: number;
|
|
32
|
+
protocol: string;
|
|
33
|
+
}
|
|
34
|
+
export interface SlowResource {
|
|
35
|
+
path: string;
|
|
36
|
+
durationMs: number;
|
|
37
|
+
transferSizeBytes: number;
|
|
38
|
+
initiatorType: string;
|
|
39
|
+
}
|
|
40
|
+
export interface PerformanceMetricsResult {
|
|
41
|
+
lcp: LcpMetric | null;
|
|
42
|
+
cls: ClsMetric | null;
|
|
43
|
+
inp: InpMetric | null;
|
|
44
|
+
navTiming: NavTimingMetric | null;
|
|
45
|
+
slowResources: SlowResource[];
|
|
46
|
+
longTaskCount: number;
|
|
47
|
+
captureSuccess: boolean;
|
|
48
|
+
}
|
|
49
|
+
export declare function injectPerformanceObservers(page: Page): Promise<void>;
|
|
50
|
+
export declare function collectPagePerformance(page: Page): Promise<PerformanceMetricsResult>;
|
|
@@ -11,7 +11,6 @@
|
|
|
11
11
|
* - DOMContentLoaded — from navigation timing
|
|
12
12
|
* - Resource count and size by type
|
|
13
13
|
*/
|
|
14
|
-
import type { Page } from 'playwright';
|
|
15
14
|
export interface CoreWebVitals {
|
|
16
15
|
lcp?: number;
|
|
17
16
|
cls?: number;
|
|
@@ -50,4 +49,4 @@ export interface PagePerfMetrics {
|
|
|
50
49
|
requestCount: number;
|
|
51
50
|
measuredAt: string;
|
|
52
51
|
}
|
|
53
|
-
export declare function collectPerfMetrics(page:
|
|
52
|
+
export declare function collectPerfMetrics(page: any, url: string): Promise<PagePerfMetrics>;
|
package/types/screen-reader.d.ts
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import type { Page } from 'playwright';
|
|
2
1
|
/**
|
|
3
2
|
* Reads a page as a structured outline (cheap + reliable), the way
|
|
4
3
|
* Playwright MCP / the accessibility tree works — NOT by sending pixels to a
|
|
@@ -13,4 +12,4 @@ export interface RawElement {
|
|
|
13
12
|
selectorHint: string;
|
|
14
13
|
expectedData?: string;
|
|
15
14
|
}
|
|
16
|
-
export declare function readScreen(page:
|
|
15
|
+
export declare function readScreen(page: any): Promise<RawElement[]>;
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Service Worker detection — inspects SW registrations, cache storage, background
|
|
3
|
+
* sync queues, and PWA installability for a crawled page.
|
|
4
|
+
* Surfaces offline-first contracts and push-capable surfaces that HTTP-only
|
|
5
|
+
* crawlers cannot observe.
|
|
6
|
+
*/
|
|
7
|
+
export interface SwRegistration {
|
|
8
|
+
scope: string;
|
|
9
|
+
scriptURL: string | null;
|
|
10
|
+
state: 'active' | 'installing' | 'waiting' | 'none';
|
|
11
|
+
updateViaCache: string;
|
|
12
|
+
}
|
|
13
|
+
export interface SwCacheEntry {
|
|
14
|
+
cacheName: string;
|
|
15
|
+
entryCount: number;
|
|
16
|
+
sampleUrls: string[];
|
|
17
|
+
}
|
|
18
|
+
export interface BackgroundSyncInfo {
|
|
19
|
+
scope: string;
|
|
20
|
+
pendingTags: string[];
|
|
21
|
+
}
|
|
22
|
+
export interface PwaInfo {
|
|
23
|
+
isInstallable: boolean;
|
|
24
|
+
manifestUrl: string | null;
|
|
25
|
+
hasPushPermission: boolean;
|
|
26
|
+
}
|
|
27
|
+
export interface ServiceWorkerResult {
|
|
28
|
+
supported: boolean;
|
|
29
|
+
active: boolean;
|
|
30
|
+
registrations: SwRegistration[];
|
|
31
|
+
cacheStorages: SwCacheEntry[];
|
|
32
|
+
backgroundSync: BackgroundSyncInfo[];
|
|
33
|
+
pwa: PwaInfo;
|
|
34
|
+
totalCachedEntries: number;
|
|
35
|
+
}
|
|
36
|
+
export declare function inspectServiceWorker(page: any): Promise<ServiceWorkerResult>;
|
|
@@ -31,7 +31,7 @@ export declare function validateAndCaptureSession(appUrl: string, creds: {
|
|
|
31
31
|
}, captchaOpts?: {
|
|
32
32
|
apiKey?: string;
|
|
33
33
|
provider?: '2captcha' | 'capsolver';
|
|
34
|
-
}): Promise<{
|
|
34
|
+
}, loginInstructions?: string): Promise<{
|
|
35
35
|
storageState: string;
|
|
36
36
|
cookies: number;
|
|
37
37
|
localStorage: number;
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shadow DOM crawler — enumerate shadow hosts and detect web component libraries.
|
|
3
|
+
*
|
|
4
|
+
* Walks the live DOM recursively (up to depth 8) to find all open shadow roots
|
|
5
|
+
* and custom elements with closed shadow roots, then classifies the component
|
|
6
|
+
* framework in use. Results feed ZeTa's test generator so it can account for
|
|
7
|
+
* elements that are invisible to standard Playwright locators.
|
|
8
|
+
*/
|
|
9
|
+
export interface ShadowInteractiveEl {
|
|
10
|
+
role: string;
|
|
11
|
+
label: string;
|
|
12
|
+
type: string;
|
|
13
|
+
tagName: string;
|
|
14
|
+
}
|
|
15
|
+
export interface ShadowHostInfo {
|
|
16
|
+
tagName: string;
|
|
17
|
+
hostId: string;
|
|
18
|
+
shadowMode: 'open' | 'closed';
|
|
19
|
+
depth: number;
|
|
20
|
+
interactiveCount: number;
|
|
21
|
+
interactiveElements: ShadowInteractiveEl[];
|
|
22
|
+
slotNames: string[];
|
|
23
|
+
childShadowHosts: number;
|
|
24
|
+
}
|
|
25
|
+
export interface ShadowDomCrawlResult {
|
|
26
|
+
shadowHostCount: number;
|
|
27
|
+
openShadowCount: number;
|
|
28
|
+
closedShadowCount: number;
|
|
29
|
+
maxNestingDepth: number;
|
|
30
|
+
hosts: ShadowHostInfo[];
|
|
31
|
+
totalInteractiveInShadow: number;
|
|
32
|
+
detectedLibraries: string[];
|
|
33
|
+
blindSpots: string[];
|
|
34
|
+
}
|
|
35
|
+
export declare function crawlShadowDom(page: any): Promise<ShadowDomCrawlResult>;
|
package/types/sitemap.d.ts
CHANGED
|
@@ -10,7 +10,7 @@ export declare function sameSite(candidateUrl: string, siteOrigin: string): bool
|
|
|
10
10
|
* and deduped. Pure function — no I/O — for direct unit testing without a
|
|
11
11
|
* real HTTP fetch.
|
|
12
12
|
*/
|
|
13
|
-
export declare function extractUrlsFromParsedSitemap(parsed: any, origin: string): {
|
|
13
|
+
export declare function extractUrlsFromParsedSitemap(parsed: any, origin: string, maxUrls?: number): {
|
|
14
14
|
pageUrls: string[];
|
|
15
15
|
subSitemapUrls: string[];
|
|
16
16
|
};
|
|
@@ -18,7 +18,7 @@ export declare function extractUrlsFromParsedSitemap(parsed: any, origin: string
|
|
|
18
18
|
* Fetches and parses a sitemap (or sitemap index, recursed up to a small
|
|
19
19
|
* depth) into a flat, deduped list of same-origin URLs.
|
|
20
20
|
*/
|
|
21
|
-
export declare function fetchSitemapUrls(sitemapUrl: string, origin: string, depth?: number, dispatcher?: any): Promise<string[]>;
|
|
21
|
+
export declare function fetchSitemapUrls(sitemapUrl: string, origin: string, depth?: number, dispatcher?: any, maxUrls?: number): Promise<string[]>;
|
|
22
22
|
/**
|
|
23
23
|
* Discovers a project's sitemap (default `${origin}/sitemap.xml`, or an
|
|
24
24
|
* explicit override) and returns the URLs it lists. Never throws.
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
import type { Page } from 'playwright';
|
|
2
|
+
export declare function simHash32(text: string): number;
|
|
3
|
+
export declare function hammingDistance32(a: number, b: number): number;
|
|
4
|
+
export declare function normalizeUrl(raw: string): string;
|
|
5
|
+
export declare function computeStateId(normalizedUrl: string, hash: number): string;
|
|
6
|
+
export interface UIStateNode {
|
|
7
|
+
id: string;
|
|
8
|
+
url: string;
|
|
9
|
+
normalizedUrl: string;
|
|
10
|
+
title: string;
|
|
11
|
+
interactiveElementCount: number;
|
|
12
|
+
domFingerprint: string;
|
|
13
|
+
simHashValue: number;
|
|
14
|
+
visitedAt: string;
|
|
15
|
+
}
|
|
16
|
+
export interface StateTransition {
|
|
17
|
+
fromId: string;
|
|
18
|
+
toId: string;
|
|
19
|
+
action: string;
|
|
20
|
+
label: string;
|
|
21
|
+
triggeredAt: string;
|
|
22
|
+
}
|
|
23
|
+
export interface CoverageReport {
|
|
24
|
+
totalStates: number;
|
|
25
|
+
totalTransitions: number;
|
|
26
|
+
uniqueUrls: number;
|
|
27
|
+
interactiveElementsTotal: number;
|
|
28
|
+
estimatedCoveragePercent: number;
|
|
29
|
+
}
|
|
30
|
+
export declare function fingerprintPage(page: Page, visitedAt: string): Promise<{
|
|
31
|
+
fingerprint: string;
|
|
32
|
+
visibleText: string;
|
|
33
|
+
interactiveCount: number;
|
|
34
|
+
simHashValue: number;
|
|
35
|
+
}>;
|
|
36
|
+
export declare class ScreenTransitionGraph {
|
|
37
|
+
private states;
|
|
38
|
+
private transitions;
|
|
39
|
+
/** Maps simHash value → array of state ids with that hash (for near-dup lookup) */
|
|
40
|
+
private hashIndex;
|
|
41
|
+
addState(node: UIStateNode): {
|
|
42
|
+
added: boolean;
|
|
43
|
+
duplicateOf?: string;
|
|
44
|
+
};
|
|
45
|
+
addTransition(fromId: string, toId: string, action: string, label: string, triggeredAt: string): void;
|
|
46
|
+
isDuplicate(url: string, visibleText: string, threshold?: number): boolean;
|
|
47
|
+
getCoverageReport(): CoverageReport;
|
|
48
|
+
toJSON(): {
|
|
49
|
+
states: UIStateNode[];
|
|
50
|
+
transitions: StateTransition[];
|
|
51
|
+
};
|
|
52
|
+
static fromJSON(data: {
|
|
53
|
+
states: UIStateNode[];
|
|
54
|
+
transitions: StateTransition[];
|
|
55
|
+
}): ScreenTransitionGraph;
|
|
56
|
+
getState(id: string): UIStateNode | undefined;
|
|
57
|
+
getAllStates(): UIStateNode[];
|
|
58
|
+
getTransitionsFrom(id: string): StateTransition[];
|
|
59
|
+
}
|