@detiq/crawler 1.1.13 → 1.1.15

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (112) hide show
  1. package/Dockerfile +1 -1
  2. package/dist/accessibility-tree-crawler.js +156 -0
  3. package/dist/accessibility-tree-crawler.js.map +7 -0
  4. package/dist/advanced-ui-probes.js +310 -0
  5. package/dist/advanced-ui-probes.js.map +7 -0
  6. package/dist/content-chunker.js +2 -2
  7. package/dist/content-chunker.js.map +2 -2
  8. package/dist/crawl-session-manager.js +145 -0
  9. package/dist/crawl-session-manager.js.map +7 -0
  10. package/dist/crawler.js +1142 -263
  11. package/dist/crawler.js.map +2 -2
  12. package/dist/crud-interaction.js +264 -0
  13. package/dist/crud-interaction.js.map +7 -0
  14. package/dist/dropdown-enumerator.js +230 -0
  15. package/dist/dropdown-enumerator.js.map +7 -0
  16. package/dist/filter-pagination-crawler.js +299 -0
  17. package/dist/filter-pagination-crawler.js.map +7 -0
  18. package/dist/frontier.js +5 -3
  19. package/dist/frontier.js.map +2 -2
  20. package/dist/gap-pass.js +47 -14
  21. package/dist/gap-pass.js.map +2 -2
  22. package/dist/http-engine.js +28 -15
  23. package/dist/http-engine.js.map +2 -2
  24. package/dist/index.js +15 -1
  25. package/dist/index.js.map +2 -2
  26. package/dist/job-queue.js +12 -5
  27. package/dist/job-queue.js.map +2 -2
  28. package/dist/locator-extractor.js +23 -0
  29. package/dist/locator-extractor.js.map +2 -2
  30. package/dist/multi-context-handler.js +182 -0
  31. package/dist/multi-context-handler.js.map +7 -0
  32. package/dist/nested-window-crawler.js +172 -0
  33. package/dist/nested-window-crawler.js.map +7 -0
  34. package/dist/network-action-correlator.js +209 -0
  35. package/dist/network-action-correlator.js.map +7 -0
  36. package/dist/performance-metrics-collector.js +155 -0
  37. package/dist/performance-metrics-collector.js.map +7 -0
  38. package/dist/performance-metrics.js.map +2 -2
  39. package/dist/policies/crawl-safety.js +5 -1
  40. package/dist/policies/crawl-safety.js.map +2 -2
  41. package/dist/profiles/domain-profile.js +21 -1
  42. package/dist/profiles/domain-profile.js.map +2 -2
  43. package/dist/rate-limiter.js +2 -2
  44. package/dist/rate-limiter.js.map +2 -2
  45. package/dist/screen-reader.js +1 -0
  46. package/dist/screen-reader.js.map +2 -2
  47. package/dist/service-worker-interceptor.js +136 -0
  48. package/dist/service-worker-interceptor.js.map +7 -0
  49. package/dist/session-capture.js +4 -2
  50. package/dist/session-capture.js.map +2 -2
  51. package/dist/shadow-dom-crawler.js +132 -0
  52. package/dist/shadow-dom-crawler.js.map +7 -0
  53. package/dist/sitemap.js +16 -10
  54. package/dist/sitemap.js.map +2 -2
  55. package/dist/state-graph.js +193 -0
  56. package/dist/state-graph.js.map +7 -0
  57. package/dist/stealth/human-behavior.js +36 -3
  58. package/dist/stealth/human-behavior.js.map +2 -2
  59. package/dist/url-pattern-completer.js +48 -14
  60. package/dist/url-pattern-completer.js.map +2 -2
  61. package/dist/visual-diff-verdict.js +3 -1
  62. package/dist/visual-diff-verdict.js.map +2 -2
  63. package/package.json +2 -1
  64. package/src/accessibility-tree-crawler.ts +246 -0
  65. package/src/advanced-ui-probes.ts +470 -0
  66. package/src/crawl-session-manager.ts +231 -0
  67. package/src/crawler.ts +1071 -142
  68. package/src/crud-interaction.ts +386 -0
  69. package/src/dropdown-enumerator.ts +312 -0
  70. package/src/filter-pagination-crawler.ts +500 -0
  71. package/src/gap-pass.ts +58 -16
  72. package/src/http-engine.ts +29 -15
  73. package/src/index.ts +14 -1
  74. package/src/job-queue.ts +3 -1
  75. package/src/locator-extractor.ts +45 -0
  76. package/src/multi-context-handler.ts +296 -0
  77. package/src/nested-window-crawler.ts +310 -0
  78. package/src/network-action-correlator.ts +320 -0
  79. package/src/performance-metrics-collector.ts +243 -0
  80. package/src/performance-metrics.ts +1 -1
  81. package/src/screen-reader.ts +4 -1
  82. package/src/service-worker-interceptor.ts +185 -0
  83. package/src/session-capture.ts +3 -0
  84. package/src/shadow-dom-crawler.ts +198 -0
  85. package/src/sitemap.ts +17 -10
  86. package/src/state-graph.ts +280 -0
  87. package/src/stealth/human-behavior.ts +38 -4
  88. package/src/url-pattern-completer.ts +61 -15
  89. package/tsconfig.json +1 -1
  90. package/types/accessibility-tree-crawler.d.ts +44 -0
  91. package/types/advanced-ui-probes.d.ts +47 -0
  92. package/types/crawl-session-manager.d.ts +44 -0
  93. package/types/crawler.d.ts +3 -1
  94. package/types/crud-interaction.d.ts +31 -0
  95. package/types/dropdown-enumerator.d.ts +20 -0
  96. package/types/filter-pagination-crawler.d.ts +52 -0
  97. package/types/gap-pass.d.ts +1 -0
  98. package/types/index.d.ts +14 -1
  99. package/types/job-queue.d.ts +18 -9
  100. package/types/locator-extractor.d.ts +7 -0
  101. package/types/multi-context-handler.d.ts +55 -0
  102. package/types/nested-window-crawler.d.ts +41 -0
  103. package/types/network-action-correlator.d.ts +47 -0
  104. package/types/performance-metrics-collector.d.ts +50 -0
  105. package/types/performance-metrics.d.ts +1 -2
  106. package/types/screen-reader.d.ts +1 -2
  107. package/types/service-worker-interceptor.d.ts +36 -0
  108. package/types/session-capture.d.ts +1 -1
  109. package/types/shadow-dom-crawler.d.ts +35 -0
  110. package/types/sitemap.d.ts +2 -2
  111. package/types/state-graph.d.ts +59 -0
  112. package/types/stealth/human-behavior.d.ts +1 -1
@@ -0,0 +1,44 @@
1
+ /**
2
+ * Session management for authenticated crawl sessions.
3
+ * Detects auth expiry via HTTP status codes and login-page redirects,
4
+ * then attempts recovery via a token refresh endpoint or re-login flow.
5
+ * Designed to keep long-running crawls alive without manual intervention.
6
+ */
7
+ import type { Page } from 'playwright';
8
+ export interface SessionConfig {
9
+ loginUrl?: string;
10
+ email?: string;
11
+ password?: string;
12
+ healthCheckUrl?: string;
13
+ refreshEndpoint?: string;
14
+ tokenLocalStorageKey?: string;
15
+ maxRefreshAttempts?: number;
16
+ }
17
+ export type SessionEventType = 'HEALTHY' | 'EXPIRY_DETECTED' | 'REFRESH_ATTEMPT' | 'REFRESH_SUCCESS' | 'REFRESH_FAILED' | 'AUTH_FAILURE';
18
+ export interface SessionEvent {
19
+ type: SessionEventType;
20
+ isoTimestamp: string;
21
+ url?: string;
22
+ strategy?: string;
23
+ attemptCount?: number;
24
+ }
25
+ export interface SessionManager {
26
+ onResponse(url: string, status: number): void;
27
+ onNavigation(url: string): void;
28
+ isHealthy(): boolean;
29
+ needsRefresh(): boolean;
30
+ markHealthy(isoTimestamp: string): void;
31
+ refresh(page: Page, isoTimestamp: string): Promise<boolean>;
32
+ getEvents(): SessionEvent[];
33
+ getStats(): {
34
+ healthy: boolean;
35
+ refreshAttempts: number;
36
+ authFailure: boolean;
37
+ };
38
+ }
39
+ export declare function createSessionManager(config: SessionConfig): SessionManager;
40
+ /**
41
+ * Attaches a Playwright response listener to feed HTTP status codes into the
42
+ * session manager. Returns a cleanup function to remove the listener.
43
+ */
44
+ export declare function attachSessionWatcher(page: Page, manager: SessionManager): () => void;
@@ -164,6 +164,7 @@ export declare function crawlProject(opts: {
164
164
  segmentIndex: number;
165
165
  hasMoreSegments: boolean;
166
166
  domHashes: string[];
167
+ crawlWarnings?: undefined;
167
168
  } | {
168
169
  screensFound: number;
169
170
  screensErrored: number;
@@ -188,9 +189,10 @@ export declare function crawlProject(opts: {
188
189
  segmentIndex: number;
189
190
  hasMoreSegments: boolean;
190
191
  domHashes: string[];
192
+ crawlWarnings: string[] | undefined;
191
193
  discoveredUrls?: undefined;
192
194
  }>;
193
- export declare function domFallbackElements(page: any): Promise<Array<{
195
+ export declare function domFallbackElements(page: any, rawPage?: any): Promise<Array<{
194
196
  meaning: string;
195
197
  role: string;
196
198
  expectedData?: string;
@@ -0,0 +1,31 @@
1
+ import type { Page } from 'playwright';
2
+ export type EditFlowType = 'inline' | 'modal' | 'navigation' | 'unknown';
3
+ export interface CrudFlowCapture {
4
+ entityName: string;
5
+ createTrigger?: string;
6
+ editFlowType: EditFlowType;
7
+ editFields: Array<{
8
+ name: string;
9
+ type: string;
10
+ prefilledValue?: string;
11
+ }>;
12
+ deleteConfirmation?: {
13
+ message: string;
14
+ confirmLabel: string;
15
+ cancelLabel: string;
16
+ };
17
+ createSuccessToast?: string;
18
+ editSuccessToast?: string;
19
+ rowCount: number;
20
+ hasDeleteButton: boolean;
21
+ hasEditButton: boolean;
22
+ }
23
+ export interface CrudProbeResult {
24
+ flows: CrudFlowCapture[];
25
+ toastsObserved: Array<{
26
+ action: string;
27
+ text: string;
28
+ role: string;
29
+ }>;
30
+ }
31
+ export declare function probeCrudFlows(page: Page, entityName?: string): Promise<CrudProbeResult>;
@@ -0,0 +1,20 @@
1
+ import type { Page } from 'playwright';
2
+ export interface DropdownOption {
3
+ value: string;
4
+ label: string;
5
+ disabled?: boolean;
6
+ }
7
+ export interface DropdownDef {
8
+ kind: 'native-select' | 'combobox' | 'listbox' | 'multiselect';
9
+ label: string;
10
+ locatorHint: string;
11
+ options: DropdownOption[];
12
+ defaultValue?: string;
13
+ cascadesFrom?: string;
14
+ }
15
+ export interface DropdownEnumerationResult {
16
+ dropdowns: DropdownDef[];
17
+ conditionalRevealMap: Record<string, string[]>;
18
+ totalOptionsFound: number;
19
+ }
20
+ export declare function enumerateDropdowns(page: Page): Promise<DropdownEnumerationResult>;
@@ -0,0 +1,52 @@
1
+ /**
2
+ * Filter state crawling and pagination capture.
3
+ *
4
+ * Discovers filter controls (select, checkbox-group, radio-group, search),
5
+ * applies each meaningful option and records result counts and first-row
6
+ * values, then detects and walks pagination (numbered, load-more, infinite
7
+ * scroll). Also captures sort column behaviour.
8
+ *
9
+ * All per-filter / per-page errors are caught and skipped so one bad control
10
+ * never aborts the full crawl.
11
+ */
12
+ import type { Page } from 'playwright';
13
+ export interface FilterOption {
14
+ value: string;
15
+ label: string;
16
+ }
17
+ export interface FilterDef {
18
+ kind: 'select' | 'checkbox-group' | 'radio-group' | 'search';
19
+ label: string;
20
+ locatorHint: string;
21
+ options: FilterOption[];
22
+ }
23
+ export interface FilterState {
24
+ filterLabel: string;
25
+ filterValue: string;
26
+ resultCount: number | null;
27
+ firstRowValues: string[];
28
+ }
29
+ export interface PaginationCapture {
30
+ kind: 'numbered' | 'load-more' | 'infinite-scroll' | 'none';
31
+ totalPages?: number;
32
+ totalItems?: number;
33
+ pageSizes?: number[];
34
+ pages: Array<{
35
+ pageNum: number;
36
+ rowValues: string[];
37
+ }>;
38
+ }
39
+ export interface FilterPaginationResult {
40
+ filters: FilterDef[];
41
+ filterStates: FilterState[];
42
+ pagination: PaginationCapture;
43
+ sortColumns: Array<{
44
+ header: string;
45
+ ariaSortAfterClick: string | null;
46
+ }>;
47
+ searchCapture?: {
48
+ noResultsText: string | null;
49
+ sampleSuggestions: string[];
50
+ };
51
+ }
52
+ export declare function crawlFiltersAndPagination(page: Page): Promise<FilterPaginationResult>;
@@ -26,6 +26,7 @@ interface LlmCfg {
26
26
  provider: string;
27
27
  model: string;
28
28
  apiKey?: string;
29
+ baseUrl?: string;
29
30
  }
30
31
  /**
31
32
  * Run LLM gap analysis + HEAD validation.
package/types/index.d.ts CHANGED
@@ -35,5 +35,18 @@ export { startSpan, endSpan, withSpan, increment, flushMetrics } from './telemet
35
35
  export type { Span, FinishedSpan } from './telemetry.js';
36
36
  export { analyzeFailure } from './failure-analysis-agent.js';
37
37
  export type { FailureContext } from './failure-analysis-agent.js';
38
- export { elementToPlaywrightLocator, elementsToPlaywrightLocators, formatLocatorsForPrompt } from './locator-extractor.js';
38
+ export { elementToPlaywrightLocator, elementsToPlaywrightLocators, formatLocatorsForPrompt, detectAuthWallLocators } from './locator-extractor.js';
39
39
  export type { PlaywrightLocator } from './locator-extractor.js';
40
+ export * from './dropdown-enumerator.js';
41
+ export * from './crud-interaction.js';
42
+ export * from './filter-pagination-crawler.js';
43
+ export * from './multi-context-handler.js';
44
+ export * from './advanced-ui-probes.js';
45
+ export * from './state-graph.js';
46
+ export * from './nested-window-crawler.js';
47
+ export * from './shadow-dom-crawler.js';
48
+ export * from './network-action-correlator.js';
49
+ export * from './crawl-session-manager.js';
50
+ export * from './accessibility-tree-crawler.js';
51
+ export * from './performance-metrics.js';
52
+ export * from './service-worker-interceptor.js';
@@ -111,22 +111,24 @@ export declare function createTrackedJob(type: string, config: {
111
111
  projectId: string;
112
112
  [k: string]: any;
113
113
  }): Promise<string>;
114
- /** Mark a tracked job DONE or FAILED. Safe to call in finally — swallows errors. */
114
+ /** Mark a tracked job DONE or FAILED. Safe to call in finally — swallows errors.
115
+ * Gap 30: When marking FAILED, sets nextRetryAt (1 min) so the retry worker picks it up promptly. */
115
116
  export declare function finishTrackedJob(tenantId: string, jobId: string, error?: string): Promise<void>;
116
117
  /** Enqueue a generic platform job — returns the job ID immediately. */
117
118
  export declare function enqueueJob(type: string, config: any): Promise<string>;
118
119
  /** Get the current state of a platform job. */
119
120
  export declare function getJob(tenantId: string, jobId: string): Promise<{
120
121
  progress: number;
121
- error: string | null;
122
+ id: string;
123
+ name: string | null;
122
124
  type: string;
125
+ error: string | null;
126
+ result: import("@prisma/client/runtime/library").JsonValue | null;
123
127
  status: string;
124
128
  tenantId: string;
125
129
  projectId: string;
126
- name: string | null;
127
- result: import("@prisma/client/runtime/library").JsonValue | null;
128
- id: string;
129
130
  config: import("@prisma/client/runtime/library").JsonValue;
131
+ retryCount: number;
130
132
  queuedAt: Date;
131
133
  startedAt: Date | null;
132
134
  finishedAt: Date | null;
@@ -137,19 +139,23 @@ export declare function getJob(tenantId: string, jobId: string): Promise<{
137
139
  totalCacheReadTokens: number | null;
138
140
  totalEstimatedCostUsd: number | null;
139
141
  llmCallCount: number;
142
+ nextRetryAt: Date | null;
143
+ dlq: boolean;
144
+ dlqReason: string | null;
140
145
  } | null>;
141
146
  /** List recent jobs for a project. */
142
147
  export declare function listJobs(tenantId: string, projectId: string): Promise<{
143
148
  progress: number;
144
- error: string | null;
149
+ id: string;
150
+ name: string | null;
145
151
  type: string;
152
+ error: string | null;
153
+ result: import("@prisma/client/runtime/library").JsonValue | null;
146
154
  status: string;
147
155
  tenantId: string;
148
156
  projectId: string;
149
- name: string | null;
150
- result: import("@prisma/client/runtime/library").JsonValue | null;
151
- id: string;
152
157
  config: import("@prisma/client/runtime/library").JsonValue;
158
+ retryCount: number;
153
159
  queuedAt: Date;
154
160
  startedAt: Date | null;
155
161
  finishedAt: Date | null;
@@ -160,6 +166,9 @@ export declare function listJobs(tenantId: string, projectId: string): Promise<{
160
166
  totalCacheReadTokens: number | null;
161
167
  totalEstimatedCostUsd: number | null;
162
168
  llmCallCount: number;
169
+ nextRetryAt: Date | null;
170
+ dlq: boolean;
171
+ dlqReason: string | null;
163
172
  }[]>;
164
173
  /** Cancel a RUNNING or QUEUED job. Aborts the in-process crawl and marks DB CANCELLED. */
165
174
  export declare function cancelJob(tenantId: string, jobId: string): Promise<void>;
@@ -22,6 +22,7 @@ export declare function elementToPlaywrightLocator(element: {
22
22
  /**
23
23
  * Convert an array of DB Element records to deduplicated Playwright locators.
24
24
  * Deduplicates by key — higher confidence wins.
25
+ * Filters out crawl-analysis meta-annotations and produces an auth-wall warning when applicable.
25
26
  */
26
27
  export declare function elementsToPlaywrightLocators(elements: Array<{
27
28
  role?: string | null;
@@ -32,6 +33,12 @@ export declare function elementsToPlaywrightLocators(elements: Array<{
32
33
  isInteractive?: boolean | null;
33
34
  ariaState?: Record<string, unknown> | null;
34
35
  }>): PlaywrightLocator[];
36
+ /**
37
+ * Detect whether a set of locators was extracted from an auth-wall page.
38
+ * Returns a warning message if auth-wall detected, null otherwise.
39
+ * Heuristic: >50% of locators are low-confidence text matches AND at least one matches auth-wall pattern.
40
+ */
41
+ export declare function detectAuthWallLocators(locators: PlaywrightLocator[]): string | null;
35
42
  /**
36
43
  * Format locators as a block for injection into LLM script generation prompts.
37
44
  * Returns empty string if no locators.
@@ -0,0 +1,55 @@
1
+ /**
2
+ * Multi-context browser handling — detects and records new tabs, iframes,
3
+ * popups, and file downloads triggered during a page crawl.
4
+ * This lets ZeTa discover cross-origin flows, OAuth redirects, embedded
5
+ * third-party content, and downloadable resources that single-page HTTP
6
+ * crawlers would miss entirely.
7
+ */
8
+ import type { Page, BrowserContext } from 'playwright';
9
+ export interface NewTabRecord {
10
+ url: string;
11
+ linkText: string;
12
+ isInternal: boolean;
13
+ httpStatus?: number;
14
+ }
15
+ export interface IframeRecord {
16
+ src: string;
17
+ crossOrigin: boolean;
18
+ sandboxed: boolean;
19
+ sandboxValue: string | null;
20
+ accessible: boolean;
21
+ contentSample: string;
22
+ hasForm: boolean;
23
+ }
24
+ export interface PopupRecord {
25
+ url: string;
26
+ triggerLabel: string;
27
+ isOAuthLike: boolean;
28
+ }
29
+ export interface DownloadRecord {
30
+ filename: string;
31
+ downloadUrl: string;
32
+ triggerLabel: string;
33
+ }
34
+ export interface MultiContextResult {
35
+ newTabs: NewTabRecord[];
36
+ iframes: IframeRecord[];
37
+ popups: PopupRecord[];
38
+ downloads: DownloadRecord[];
39
+ }
40
+ /**
41
+ * Attaches a listener to the BrowserContext that records every new tab
42
+ * opened while the listener is active. Returns an unsubscribe function.
43
+ *
44
+ * Call this before navigating / clicking so that tabs opened during the
45
+ * crawl are captured, then call the returned function to stop listening.
46
+ */
47
+ export declare function attachNewTabListener(context: BrowserContext, seedUrl: string, result: MultiContextResult): () => void;
48
+ /**
49
+ * Probes the current page for multi-context browser events: iframes,
50
+ * popup windows, and file downloads.
51
+ *
52
+ * New-tab tracking is intentionally excluded here — it must be set up at
53
+ * the crawler level (before navigation) using `attachNewTabListener`.
54
+ */
55
+ export declare function probeMultiContext(page: Page, context: BrowserContext, seedUrl: string): Promise<MultiContextResult>;
@@ -0,0 +1,41 @@
1
+ /**
2
+ * Nested window / browsing-context crawler — discovers and records frames,
3
+ * embedded apps, and popup windows that are already open in the browser
4
+ * context at the time of a crawl step.
5
+ *
6
+ * Complements multi-context-handler.ts: that module handles proactive popup
7
+ * interception via context.on('page'); this module snapshots what is already
8
+ * present in the frame tree and context page list.
9
+ */
10
+ import type { BrowserContext } from 'playwright';
11
+ export type BrowsingContextType = 'page' | 'frame' | 'popup' | 'embedded-app';
12
+ export interface BrowsingContextNode {
13
+ id: string;
14
+ type: BrowsingContextType;
15
+ url: string;
16
+ parentId: string | null;
17
+ depth: number;
18
+ title: string;
19
+ interactiveCount: number;
20
+ accessible: boolean;
21
+ isEmbeddedApp: boolean;
22
+ }
23
+ export interface NestedCrawlResult {
24
+ contextTree: BrowsingContextNode[];
25
+ totalContextsFound: number;
26
+ maxDepthReached: number;
27
+ inaccessibleCount: number;
28
+ frameUrls: string[];
29
+ embeddedAppUrls: string[];
30
+ }
31
+ /**
32
+ * Discovers all nested browsing contexts reachable from the current page:
33
+ * sub-frames, embedded applications, and already-open popup pages.
34
+ *
35
+ * All failures are non-fatal — errors are caught and an empty result is
36
+ * returned so the caller's crawl step can continue uninterrupted.
37
+ */
38
+ export declare function crawlNestedContexts(page: any, context: BrowserContext | null, seedUrl: string, opts?: {
39
+ maxFrameDepth?: number;
40
+ maxPopupDepth?: number;
41
+ }): Promise<NestedCrawlResult>;
@@ -0,0 +1,47 @@
1
+ /**
2
+ * Network-action correlator — captures XHR/fetch requests that occur during
3
+ * a named UI action (click, form submit, navigation) and maps them to API
4
+ * endpoint contracts.
5
+ *
6
+ * Usage:
7
+ * const correlator = createCorrelator();
8
+ * const detach = attachNetworkInterceptor(page, correlator);
9
+ * correlator.startWindow('Submit login form');
10
+ * await page.click('#login-btn');
11
+ * const correlation = correlator.endWindow();
12
+ * detach();
13
+ * const result = correlator.getResult();
14
+ */
15
+ import type { Page } from 'playwright';
16
+ export interface NetworkRequestRecord {
17
+ url: string;
18
+ urlPattern: string;
19
+ method: string;
20
+ bodySchema: Record<string, string> | null;
21
+ status: number | null;
22
+ responseSchema: Record<string, string> | null;
23
+ isGraphQL: boolean;
24
+ graphQLOperation?: string;
25
+ resourceType: string;
26
+ correlatedActionLabel?: string;
27
+ }
28
+ export interface ActionCorrelation {
29
+ actionLabel: string;
30
+ requests: NetworkRequestRecord[];
31
+ }
32
+ export interface NetworkCorrelationResult {
33
+ correlations: ActionCorrelation[];
34
+ uniqueEndpoints: string[];
35
+ graphqlOperations: string[];
36
+ totalRequestsCaptured: number;
37
+ coverage: number;
38
+ }
39
+ interface NetworkCorrelator {
40
+ startWindow(actionLabel: string): void;
41
+ recordRequest(rec: NetworkRequestRecord): void;
42
+ endWindow(): ActionCorrelation;
43
+ getResult(): NetworkCorrelationResult;
44
+ }
45
+ export declare function createCorrelator(): NetworkCorrelator;
46
+ export declare function attachNetworkInterceptor(page: Page, correlator: ReturnType<typeof createCorrelator>): () => void;
47
+ export {};
@@ -0,0 +1,50 @@
1
+ /**
2
+ * Performance metrics collector — captures Core Web Vitals and navigation timing
3
+ * for pages visited during crawl runs.
4
+ * Collects LCP, CLS, INP, TTFB, long tasks, and slow resources via PerformanceObserver
5
+ * and the Navigation/Resource Timing APIs.
6
+ */
7
+ import type { Page } from 'playwright';
8
+ export type CwvScore = 'good' | 'needs-improvement' | 'poor';
9
+ export interface LcpMetric {
10
+ value: number;
11
+ score: CwvScore;
12
+ elementTag: string | null;
13
+ url: string | null;
14
+ }
15
+ export interface ClsMetric {
16
+ value: number;
17
+ score: CwvScore;
18
+ shiftCount: number;
19
+ }
20
+ export interface InpMetric {
21
+ p75: number;
22
+ score: CwvScore;
23
+ sampleCount: number;
24
+ }
25
+ export interface NavTimingMetric {
26
+ ttfb: number;
27
+ ttfbScore: CwvScore;
28
+ domContentLoaded: number;
29
+ domComplete: number;
30
+ loadComplete: number;
31
+ transferSizeBytes: number;
32
+ protocol: string;
33
+ }
34
+ export interface SlowResource {
35
+ path: string;
36
+ durationMs: number;
37
+ transferSizeBytes: number;
38
+ initiatorType: string;
39
+ }
40
+ export interface PerformanceMetricsResult {
41
+ lcp: LcpMetric | null;
42
+ cls: ClsMetric | null;
43
+ inp: InpMetric | null;
44
+ navTiming: NavTimingMetric | null;
45
+ slowResources: SlowResource[];
46
+ longTaskCount: number;
47
+ captureSuccess: boolean;
48
+ }
49
+ export declare function injectPerformanceObservers(page: Page): Promise<void>;
50
+ export declare function collectPagePerformance(page: Page): Promise<PerformanceMetricsResult>;
@@ -11,7 +11,6 @@
11
11
  * - DOMContentLoaded — from navigation timing
12
12
  * - Resource count and size by type
13
13
  */
14
- import type { Page } from 'playwright';
15
14
  export interface CoreWebVitals {
16
15
  lcp?: number;
17
16
  cls?: number;
@@ -50,4 +49,4 @@ export interface PagePerfMetrics {
50
49
  requestCount: number;
51
50
  measuredAt: string;
52
51
  }
53
- export declare function collectPerfMetrics(page: Page, url: string): Promise<PagePerfMetrics>;
52
+ export declare function collectPerfMetrics(page: any, url: string): Promise<PagePerfMetrics>;
@@ -1,4 +1,3 @@
1
- import type { Page } from 'playwright';
2
1
  /**
3
2
  * Reads a page as a structured outline (cheap + reliable), the way
4
3
  * Playwright MCP / the accessibility tree works — NOT by sending pixels to a
@@ -13,4 +12,4 @@ export interface RawElement {
13
12
  selectorHint: string;
14
13
  expectedData?: string;
15
14
  }
16
- export declare function readScreen(page: Page): Promise<RawElement[]>;
15
+ export declare function readScreen(page: any): Promise<RawElement[]>;
@@ -0,0 +1,36 @@
1
+ /**
2
+ * Service Worker detection — inspects SW registrations, cache storage, background
3
+ * sync queues, and PWA installability for a crawled page.
4
+ * Surfaces offline-first contracts and push-capable surfaces that HTTP-only
5
+ * crawlers cannot observe.
6
+ */
7
+ export interface SwRegistration {
8
+ scope: string;
9
+ scriptURL: string | null;
10
+ state: 'active' | 'installing' | 'waiting' | 'none';
11
+ updateViaCache: string;
12
+ }
13
+ export interface SwCacheEntry {
14
+ cacheName: string;
15
+ entryCount: number;
16
+ sampleUrls: string[];
17
+ }
18
+ export interface BackgroundSyncInfo {
19
+ scope: string;
20
+ pendingTags: string[];
21
+ }
22
+ export interface PwaInfo {
23
+ isInstallable: boolean;
24
+ manifestUrl: string | null;
25
+ hasPushPermission: boolean;
26
+ }
27
+ export interface ServiceWorkerResult {
28
+ supported: boolean;
29
+ active: boolean;
30
+ registrations: SwRegistration[];
31
+ cacheStorages: SwCacheEntry[];
32
+ backgroundSync: BackgroundSyncInfo[];
33
+ pwa: PwaInfo;
34
+ totalCachedEntries: number;
35
+ }
36
+ export declare function inspectServiceWorker(page: any): Promise<ServiceWorkerResult>;
@@ -31,7 +31,7 @@ export declare function validateAndCaptureSession(appUrl: string, creds: {
31
31
  }, captchaOpts?: {
32
32
  apiKey?: string;
33
33
  provider?: '2captcha' | 'capsolver';
34
- }): Promise<{
34
+ }, loginInstructions?: string): Promise<{
35
35
  storageState: string;
36
36
  cookies: number;
37
37
  localStorage: number;
@@ -0,0 +1,35 @@
1
+ /**
2
+ * Shadow DOM crawler — enumerate shadow hosts and detect web component libraries.
3
+ *
4
+ * Walks the live DOM recursively (up to depth 8) to find all open shadow roots
5
+ * and custom elements with closed shadow roots, then classifies the component
6
+ * framework in use. Results feed ZeTa's test generator so it can account for
7
+ * elements that are invisible to standard Playwright locators.
8
+ */
9
+ export interface ShadowInteractiveEl {
10
+ role: string;
11
+ label: string;
12
+ type: string;
13
+ tagName: string;
14
+ }
15
+ export interface ShadowHostInfo {
16
+ tagName: string;
17
+ hostId: string;
18
+ shadowMode: 'open' | 'closed';
19
+ depth: number;
20
+ interactiveCount: number;
21
+ interactiveElements: ShadowInteractiveEl[];
22
+ slotNames: string[];
23
+ childShadowHosts: number;
24
+ }
25
+ export interface ShadowDomCrawlResult {
26
+ shadowHostCount: number;
27
+ openShadowCount: number;
28
+ closedShadowCount: number;
29
+ maxNestingDepth: number;
30
+ hosts: ShadowHostInfo[];
31
+ totalInteractiveInShadow: number;
32
+ detectedLibraries: string[];
33
+ blindSpots: string[];
34
+ }
35
+ export declare function crawlShadowDom(page: any): Promise<ShadowDomCrawlResult>;
@@ -10,7 +10,7 @@ export declare function sameSite(candidateUrl: string, siteOrigin: string): bool
10
10
  * and deduped. Pure function — no I/O — for direct unit testing without a
11
11
  * real HTTP fetch.
12
12
  */
13
- export declare function extractUrlsFromParsedSitemap(parsed: any, origin: string): {
13
+ export declare function extractUrlsFromParsedSitemap(parsed: any, origin: string, maxUrls?: number): {
14
14
  pageUrls: string[];
15
15
  subSitemapUrls: string[];
16
16
  };
@@ -18,7 +18,7 @@ export declare function extractUrlsFromParsedSitemap(parsed: any, origin: string
18
18
  * Fetches and parses a sitemap (or sitemap index, recursed up to a small
19
19
  * depth) into a flat, deduped list of same-origin URLs.
20
20
  */
21
- export declare function fetchSitemapUrls(sitemapUrl: string, origin: string, depth?: number, dispatcher?: any): Promise<string[]>;
21
+ export declare function fetchSitemapUrls(sitemapUrl: string, origin: string, depth?: number, dispatcher?: any, maxUrls?: number): Promise<string[]>;
22
22
  /**
23
23
  * Discovers a project's sitemap (default `${origin}/sitemap.xml`, or an
24
24
  * explicit override) and returns the URLs it lists. Never throws.
@@ -0,0 +1,59 @@
1
+ import type { Page } from 'playwright';
2
+ export declare function simHash32(text: string): number;
3
+ export declare function hammingDistance32(a: number, b: number): number;
4
+ export declare function normalizeUrl(raw: string): string;
5
+ export declare function computeStateId(normalizedUrl: string, hash: number): string;
6
+ export interface UIStateNode {
7
+ id: string;
8
+ url: string;
9
+ normalizedUrl: string;
10
+ title: string;
11
+ interactiveElementCount: number;
12
+ domFingerprint: string;
13
+ simHashValue: number;
14
+ visitedAt: string;
15
+ }
16
+ export interface StateTransition {
17
+ fromId: string;
18
+ toId: string;
19
+ action: string;
20
+ label: string;
21
+ triggeredAt: string;
22
+ }
23
+ export interface CoverageReport {
24
+ totalStates: number;
25
+ totalTransitions: number;
26
+ uniqueUrls: number;
27
+ interactiveElementsTotal: number;
28
+ estimatedCoveragePercent: number;
29
+ }
30
+ export declare function fingerprintPage(page: Page, visitedAt: string): Promise<{
31
+ fingerprint: string;
32
+ visibleText: string;
33
+ interactiveCount: number;
34
+ simHashValue: number;
35
+ }>;
36
+ export declare class ScreenTransitionGraph {
37
+ private states;
38
+ private transitions;
39
+ /** Maps simHash value → array of state ids with that hash (for near-dup lookup) */
40
+ private hashIndex;
41
+ addState(node: UIStateNode): {
42
+ added: boolean;
43
+ duplicateOf?: string;
44
+ };
45
+ addTransition(fromId: string, toId: string, action: string, label: string, triggeredAt: string): void;
46
+ isDuplicate(url: string, visibleText: string, threshold?: number): boolean;
47
+ getCoverageReport(): CoverageReport;
48
+ toJSON(): {
49
+ states: UIStateNode[];
50
+ transitions: StateTransition[];
51
+ };
52
+ static fromJSON(data: {
53
+ states: UIStateNode[];
54
+ transitions: StateTransition[];
55
+ }): ScreenTransitionGraph;
56
+ getState(id: string): UIStateNode | undefined;
57
+ getAllStates(): UIStateNode[];
58
+ getTransitionsFrom(id: string): StateTransition[];
59
+ }