@detiq/crawler 1.1.14 → 1.1.16

package/types/nested-window-crawler.d.ts

@@ -0,0 +1,41 @@
+/**
+ * Nested window / browsing-context crawler — discovers and records frames,
+ * embedded apps, and popup windows that are already open in the browser
+ * context at the time of a crawl step.
+ *
+ * Complements multi-context-handler.ts: that module handles proactive popup
+ * interception via context.on('page'); this module snapshots what is already
+ * present in the frame tree and context page list.
+ */
+import type { BrowserContext } from 'playwright';
+export type BrowsingContextType = 'page' | 'frame' | 'popup' | 'embedded-app';
+export interface BrowsingContextNode {
+    id: string;
+    type: BrowsingContextType;
+    url: string;
+    parentId: string | null;
+    depth: number;
+    title: string;
+    interactiveCount: number;
+    accessible: boolean;
+    isEmbeddedApp: boolean;
+}
+export interface NestedCrawlResult {
+    contextTree: BrowsingContextNode[];
+    totalContextsFound: number;
+    maxDepthReached: number;
+    inaccessibleCount: number;
+    frameUrls: string[];
+    embeddedAppUrls: string[];
+}
+/**
+ * Discovers all nested browsing contexts reachable from the current page:
+ * sub-frames, embedded applications, and already-open popup pages.
+ *
+ * All failures are non-fatal — errors are caught and an empty result is
+ * returned so the caller's crawl step can continue uninterrupted.
+ */
+export declare function crawlNestedContexts(page: any, context: BrowserContext | null, seedUrl: string, opts?: {
+    maxFrameDepth?: number;
+    maxPopupDepth?: number;
+}): Promise<NestedCrawlResult>;

package/types/network-action-correlator.d.ts

@@ -0,0 +1,47 @@
+/**
+ * Network-action correlator — captures XHR/fetch requests that occur during
+ * a named UI action (click, form submit, navigation) and maps them to API
+ * endpoint contracts.
+ *
+ * Usage:
+ *   const correlator = createCorrelator();
+ *   const detach = attachNetworkInterceptor(page, correlator);
+ *   correlator.startWindow('Submit login form');
+ *   await page.click('#login-btn');
+ *   const correlation = correlator.endWindow();
+ *   detach();
+ *   const result = correlator.getResult();
+ */
+import type { Page } from 'playwright';
+export interface NetworkRequestRecord {
+    url: string;
+    urlPattern: string;
+    method: string;
+    bodySchema: Record<string, string> | null;
+    status: number | null;
+    responseSchema: Record<string, string> | null;
+    isGraphQL: boolean;
+    graphQLOperation?: string;
+    resourceType: string;
+    correlatedActionLabel?: string;
+}
+export interface ActionCorrelation {
+    actionLabel: string;
+    requests: NetworkRequestRecord[];
+}
+export interface NetworkCorrelationResult {
+    correlations: ActionCorrelation[];
+    uniqueEndpoints: string[];
+    graphqlOperations: string[];
+    totalRequestsCaptured: number;
+    coverage: number;
+}
+interface NetworkCorrelator {
+    startWindow(actionLabel: string): void;
+    recordRequest(rec: NetworkRequestRecord): void;
+    endWindow(): ActionCorrelation;
+    getResult(): NetworkCorrelationResult;
+}
+export declare function createCorrelator(): NetworkCorrelator;
+export declare function attachNetworkInterceptor(page: Page, correlator: ReturnType<typeof createCorrelator>): () => void;
+export {};

package/types/performance-metrics-collector.d.ts

@@ -0,0 +1,50 @@
+/**
+ * Performance metrics collector — captures Core Web Vitals and navigation timing
+ * for pages visited during crawl runs.
+ * Collects LCP, CLS, INP, TTFB, long tasks, and slow resources via PerformanceObserver
+ * and the Navigation/Resource Timing APIs.
+ */
+import type { Page } from 'playwright';
+export type CwvScore = 'good' | 'needs-improvement' | 'poor';
+export interface LcpMetric {
+    value: number;
+    score: CwvScore;
+    elementTag: string | null;
+    url: string | null;
+}
+export interface ClsMetric {
+    value: number;
+    score: CwvScore;
+    shiftCount: number;
+}
+export interface InpMetric {
+    p75: number;
+    score: CwvScore;
+    sampleCount: number;
+}
+export interface NavTimingMetric {
+    ttfb: number;
+    ttfbScore: CwvScore;
+    domContentLoaded: number;
+    domComplete: number;
+    loadComplete: number;
+    transferSizeBytes: number;
+    protocol: string;
+}
+export interface SlowResource {
+    path: string;
+    durationMs: number;
+    transferSizeBytes: number;
+    initiatorType: string;
+}
+export interface PerformanceMetricsResult {
+    lcp: LcpMetric | null;
+    cls: ClsMetric | null;
+    inp: InpMetric | null;
+    navTiming: NavTimingMetric | null;
+    slowResources: SlowResource[];
+    longTaskCount: number;
+    captureSuccess: boolean;
+}
+export declare function injectPerformanceObservers(page: Page): Promise<void>;
+export declare function collectPagePerformance(page: Page): Promise<PerformanceMetricsResult>;

package/types/performance-metrics.d.ts

@@ -11,7 +11,6 @@
  * - DOMContentLoaded — from navigation timing
  * - Resource count and size by type
  */
-import type { Page } from 'playwright';
 export interface CoreWebVitals {
     lcp?: number;
     cls?: number;
@@ -50,4 +49,4 @@ export interface PagePerfMetrics {
     requestCount: number;
     measuredAt: string;
 }
-export declare function collectPerfMetrics(page: Page, url: string): Promise<PagePerfMetrics>;
+export declare function collectPerfMetrics(page: any, url: string): Promise<PagePerfMetrics>;

package/types/screen-reader.d.ts

@@ -1,4 +1,3 @@
-import type { Page } from 'playwright';
 /**
  * Reads a page as a structured outline (cheap + reliable), the way
  * Playwright MCP / the accessibility tree works — NOT by sending pixels to a
@@ -13,4 +12,4 @@ export interface RawElement {
     selectorHint: string;
     expectedData?: string;
 }
-export declare function readScreen(page: Page): Promise<RawElement[]>;
+export declare function readScreen(page: any): Promise<RawElement[]>;

package/types/service-worker-interceptor.d.ts

@@ -0,0 +1,36 @@
+/**
+ * Service Worker detection — inspects SW registrations, cache storage, background
+ * sync queues, and PWA installability for a crawled page.
+ * Surfaces offline-first contracts and push-capable surfaces that HTTP-only
+ * crawlers cannot observe.
+ */
+export interface SwRegistration {
+    scope: string;
+    scriptURL: string | null;
+    state: 'active' | 'installing' | 'waiting' | 'none';
+    updateViaCache: string;
+}
+export interface SwCacheEntry {
+    cacheName: string;
+    entryCount: number;
+    sampleUrls: string[];
+}
+export interface BackgroundSyncInfo {
+    scope: string;
+    pendingTags: string[];
+}
+export interface PwaInfo {
+    isInstallable: boolean;
+    manifestUrl: string | null;
+    hasPushPermission: boolean;
+}
+export interface ServiceWorkerResult {
+    supported: boolean;
+    active: boolean;
+    registrations: SwRegistration[];
+    cacheStorages: SwCacheEntry[];
+    backgroundSync: BackgroundSyncInfo[];
+    pwa: PwaInfo;
+    totalCachedEntries: number;
+}
+export declare function inspectServiceWorker(page: any): Promise<ServiceWorkerResult>;

package/types/session-capture.d.ts

@@ -31,7 +31,7 @@ export declare function validateAndCaptureSession(appUrl: string, creds: {
 }, captchaOpts?: {
     apiKey?: string;
     provider?: '2captcha' | 'capsolver';
-}): Promise<{
+}, loginInstructions?: string): Promise<{
     storageState: string;
     cookies: number;
     localStorage: number;

package/types/shadow-dom-crawler.d.ts

@@ -0,0 +1,35 @@
+/**
+ * Shadow DOM crawler — enumerate shadow hosts and detect web component libraries.
+ *
+ * Walks the live DOM recursively (up to depth 8) to find all open shadow roots
+ * and custom elements with closed shadow roots, then classifies the component
+ * framework in use. Results feed ZeTa's test generator so it can account for
+ * elements that are invisible to standard Playwright locators.
+ */
+export interface ShadowInteractiveEl {
+    role: string;
+    label: string;
+    type: string;
+    tagName: string;
+}
+export interface ShadowHostInfo {
+    tagName: string;
+    hostId: string;
+    shadowMode: 'open' | 'closed';
+    depth: number;
+    interactiveCount: number;
+    interactiveElements: ShadowInteractiveEl[];
+    slotNames: string[];
+    childShadowHosts: number;
+}
+export interface ShadowDomCrawlResult {
+    shadowHostCount: number;
+    openShadowCount: number;
+    closedShadowCount: number;
+    maxNestingDepth: number;
+    hosts: ShadowHostInfo[];
+    totalInteractiveInShadow: number;
+    detectedLibraries: string[];
+    blindSpots: string[];
+}
+export declare function crawlShadowDom(page: any): Promise<ShadowDomCrawlResult>;

package/types/sitemap.d.ts

@@ -10,7 +10,7 @@ export declare function sameSite(candidateUrl: string, siteOrigin: string): bool
  * and deduped. Pure function — no I/O — for direct unit testing without a
  * real HTTP fetch.
  */
-export declare function extractUrlsFromParsedSitemap(parsed: any, origin: string): {
+export declare function extractUrlsFromParsedSitemap(parsed: any, origin: string, maxUrls?: number): {
     pageUrls: string[];
     subSitemapUrls: string[];
 };
@@ -18,7 +18,7 @@ export declare function extractUrlsFromParsedSitemap(parsed: any, origin: string
  * Fetches and parses a sitemap (or sitemap index, recursed up to a small
  * depth) into a flat, deduped list of same-origin URLs.
  */
-export declare function fetchSitemapUrls(sitemapUrl: string, origin: string, depth?: number, dispatcher?: any): Promise<string[]>;
+export declare function fetchSitemapUrls(sitemapUrl: string, origin: string, depth?: number, dispatcher?: any, maxUrls?: number): Promise<string[]>;
 /**
  * Discovers a project's sitemap (default `${origin}/sitemap.xml`, or an
  * explicit override) and returns the URLs it lists. Never throws.

package/types/state-graph.d.ts

@@ -0,0 +1,59 @@
+import type { Page } from 'playwright';
+export declare function simHash32(text: string): number;
+export declare function hammingDistance32(a: number, b: number): number;
+export declare function normalizeUrl(raw: string): string;
+export declare function computeStateId(normalizedUrl: string, hash: number): string;
+export interface UIStateNode {
+    id: string;
+    url: string;
+    normalizedUrl: string;
+    title: string;
+    interactiveElementCount: number;
+    domFingerprint: string;
+    simHashValue: number;
+    visitedAt: string;
+}
+export interface StateTransition {
+    fromId: string;
+    toId: string;
+    action: string;
+    label: string;
+    triggeredAt: string;
+}
+export interface CoverageReport {
+    totalStates: number;
+    totalTransitions: number;
+    uniqueUrls: number;
+    interactiveElementsTotal: number;
+    estimatedCoveragePercent: number;
+}
+export declare function fingerprintPage(page: Page, visitedAt: string): Promise<{
+    fingerprint: string;
+    visibleText: string;
+    interactiveCount: number;
+    simHashValue: number;
+}>;
+export declare class ScreenTransitionGraph {
+    private states;
+    private transitions;
+    /** Maps simHash value → array of state ids with that hash (for near-dup lookup) */
+    private hashIndex;
+    addState(node: UIStateNode): {
+        added: boolean;
+        duplicateOf?: string;
+    };
+    addTransition(fromId: string, toId: string, action: string, label: string, triggeredAt: string): void;
+    isDuplicate(url: string, visibleText: string, threshold?: number): boolean;
+    getCoverageReport(): CoverageReport;
+    toJSON(): {
+        states: UIStateNode[];
+        transitions: StateTransition[];
+    };
+    static fromJSON(data: {
+        states: UIStateNode[];
+        transitions: StateTransition[];
+    }): ScreenTransitionGraph;
+    getState(id: string): UIStateNode | undefined;
+    getAllStates(): UIStateNode[];
+    getTransitionsFrom(id: string): StateTransition[];
+}

package/types/stealth/human-behavior.d.ts

@@ -8,7 +8,7 @@ export declare function infiniteScrollToBottom(page: Page, opts?: {
     maxScrolls?: number;
     scrollDelay?: number;
     stabilizeMs?: number;
-}): Promise<{
+}, rawPage?: any): Promise<{
     scrollCount: number;
     heightReached: number;
 }>;