@detiq/crawler 1.1.7 → 1.1.9

package/dist/websocket-detector.js

@@ -1,60 +1,72 @@
+/**
+ * WebSocket detection — intercepts WS connections opened during page crawl.
+ * Records connection URLs, message schemas, and message counts.
+ * This lets ZeTa discover real-time API contracts (chat, notifications, presence)
+ * that would be invisible to HTTP-only crawlers.
+ */
 function inferProtocol(sample) {
-  try {
-    const obj = JSON.parse(sample);
-    if (obj?.jsonrpc) return "json-rpc";
-    if (obj?.type === "connection_init" || obj?.type === "subscribe") return "graphql-ws";
-    if (typeof obj?.sid === "string" || sample.startsWith("0{")) return "socket.io";
-    return "json";
-  } catch {
-    return "raw";
-  }
-}
-function attachWebSocketDetector(cdpSession) {
-  const connections = /* @__PURE__ */ new Map();
-  const onCreated = (evt) => {
-    const { requestId, url, timestamp } = evt ?? {};
-    if (!requestId || !url) return;
-    connections.set(requestId, {
-      url,
-      framesSent: 0,
-      framesReceived: 0,
-      samples: [],
-      protocol: "unknown",
-      openedAt: new Date(timestamp * 1e3).toISOString()
-    });
-  };
-  const onFrame = (evt, direction) => {
-    const { requestId, response } = evt ?? {};
-    if (!requestId) return;
-    const conn = connections.get(requestId);
-    if (!conn) return;
-    if (direction === "sent") conn.framesSent++;
-    else conn.framesReceived++;
-    if (conn.samples.length < 10 && response?.payloadData) {
-      const payload = String(response.payloadData).slice(0, 500);
-      conn.samples.push({ direction, payload, ts: (/* @__PURE__ */ new Date()).toISOString() });
-      if (conn.protocol === "unknown") conn.protocol = inferProtocol(payload);
+    try {
+        const obj = JSON.parse(sample);
+        if (obj?.jsonrpc)
+            return 'json-rpc';
+        if (obj?.type === 'connection_init' || obj?.type === 'subscribe')
+            return 'graphql-ws';
+        if (typeof obj?.sid === 'string' || sample.startsWith('0{'))
+            return 'socket.io';
+        return 'json';
+    }
+    catch {
+        return 'raw';
     }
-  };
-  const onSent = (evt) => onFrame(evt, "sent");
-  const onReceived = (evt) => onFrame(evt, "received");
-  if (cdpSession?.on) {
-    cdpSession.on("Network.webSocketCreated", onCreated);
-    cdpSession.on("Network.webSocketFrameSent", onSent);
-    cdpSession.on("Network.webSocketFrameReceived", onReceived);
-  }
-  return {
-    getConnections: () => [...connections.values()],
-    detach() {
-      if (cdpSession?.off) {
-        cdpSession.off("Network.webSocketCreated", onCreated);
-        cdpSession.off("Network.webSocketFrameSent", onSent);
-        cdpSession.off("Network.webSocketFrameReceived", onReceived);
-      }
+}
+export function attachWebSocketDetector(cdpSession) {
+    const connections = new Map();
+    const onCreated = (evt) => {
+        const { requestId, url, timestamp } = evt ?? {};
+        if (!requestId || !url)
+            return;
+        connections.set(requestId, {
+            url,
+            framesSent: 0,
+            framesReceived: 0,
+            samples: [],
+            protocol: 'unknown',
+            openedAt: new Date(timestamp * 1000).toISOString(),
+        });
+    };
+    const onFrame = (evt, direction) => {
+        const { requestId, response } = evt ?? {};
+        if (!requestId)
+            return;
+        const conn = connections.get(requestId);
+        if (!conn)
+            return;
+        if (direction === 'sent')
+            conn.framesSent++;
+        else
+            conn.framesReceived++;
+        if (conn.samples.length < 10 && response?.payloadData) {
+            const payload = String(response.payloadData).slice(0, 500);
+            conn.samples.push({ direction, payload, ts: new Date().toISOString() });
+            if (conn.protocol === 'unknown')
+                conn.protocol = inferProtocol(payload);
+        }
+    };
+    const onSent = (evt) => onFrame(evt, 'sent');
+    const onReceived = (evt) => onFrame(evt, 'received');
+    if (cdpSession?.on) {
+        cdpSession.on('Network.webSocketCreated', onCreated);
+        cdpSession.on('Network.webSocketFrameSent', onSent);
+        cdpSession.on('Network.webSocketFrameReceived', onReceived);
     }
-  };
+    return {
+        getConnections: () => [...connections.values()],
+        detach() {
+            if (cdpSession?.off) {
+                cdpSession.off('Network.webSocketCreated', onCreated);
+                cdpSession.off('Network.webSocketFrameSent', onSent);
+                cdpSession.off('Network.webSocketFrameReceived', onReceived);
+            }
+        },
+    };
 }
-export {
-  attachWebSocketDetector
-};
-//# sourceMappingURL=websocket-detector.js.map

package/dist/worker.js

@@ -1,33 +1,42 @@
-import { startJobWorkerLoop, processDataRetentionPurge } from "./job-queue.js";
-import { logger } from "./logger.js";
+/**
+ * Standalone worker process entrypoint for a dedicated crawler/worker container,
+ * separate from the API server (see infrastructure/ecs-task-definitions/crawler.json).
+ *
+ * index.ts is a side-effect-free library barrel imported by @detiq/api — it must NOT
+ * start any background loop itself, or importing a single helper from @detiq/crawler
+ * would silently spin up a second worker inside the API process too. This file is
+ * the actual entrypoint that keeps a dedicated worker container's process alive and
+ * consuming jobs: `node dist/worker.js`.
+ */
+import { startJobWorkerLoop, processDataRetentionPurge } from './job-queue.js';
+import { logger } from './logger.js';
 async function main() {
-  logger.info("[worker] starting standalone crawler worker process");
-  startJobWorkerLoop();
-  const runPurge = () => {
-    processDataRetentionPurge().catch((err) => {
-      logger.error({ err: String(err) }, "[worker] data-retention purge failed");
-    });
-  };
-  runPurge();
-  setInterval(runPurge, 24 * 60 * 60 * 1e3);
-  if (process.env.JOB_QUEUE_BACKEND === "sqs") {
-    const { startSqsWorker } = await import("./sqs-queue.js");
-    startSqsWorker().catch((err) => {
-      logger.error({ err: String(err) }, "[worker] SQS worker crashed");
-      process.exit(1);
-    });
-  }
+    logger.info('[worker] starting standalone crawler worker process');
+    // Durable DB-backed queue — claims every non-CRAWL job type, and CRAWL too when
+    // JOB_QUEUE_BACKEND isn't 'sqs'. Safe to run in many worker replicas at once.
+    startJobWorkerLoop();
+    // Daily data-retention purge: runs immediately on startup then every 24 h.
+    // Only runs in the dedicated worker container (this file), not in the API
+    // process, so there is exactly one purge sweep regardless of API replica count.
+    const runPurge = () => {
+        processDataRetentionPurge().catch((err) => {
+            logger.error({ err: String(err) }, '[worker] data-retention purge failed');
+        });
+    };
+    runPurge();
+    setInterval(runPurge, 24 * 60 * 60 * 1000);
+    // Optional: also consume the SQS transport for horizontally-scaled CRAWL fleets.
+    if (process.env.JOB_QUEUE_BACKEND === 'sqs') {
+        const { startSqsWorker } = await import('./sqs-queue.js');
+        startSqsWorker().catch((err) => {
+            logger.error({ err: String(err) }, '[worker] SQS worker crashed');
+            process.exit(1);
+        });
+    }
 }
-process.on("SIGTERM", () => {
-  logger.info("[worker] SIGTERM received, exiting");
-  process.exit(0);
-});
-process.on("SIGINT", () => {
-  logger.info("[worker] SIGINT received, exiting");
-  process.exit(0);
-});
+process.on('SIGTERM', () => { logger.info('[worker] SIGTERM received, exiting'); process.exit(0); });
+process.on('SIGINT', () => { logger.info('[worker] SIGINT received, exiting'); process.exit(0); });
 main().catch((err) => {
-  logger.error({ err: String(err) }, "[worker] fatal startup error");
-  process.exit(1);
+    logger.error({ err: String(err) }, '[worker] fatal startup error');
+    process.exit(1);
 });
-//# sourceMappingURL=worker.js.map

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@detiq/crawler",
-  "version": "1.1.7",
+  "version": "1.1.9",
   "private": false,
   "publishConfig": {
     "access": "public",

package/src/index.ts

@@ -35,3 +35,5 @@ export { startSpan, endSpan, withSpan, increment, flushMetrics } from './telemet
 export type { Span, FinishedSpan } from './telemetry.js';
 export { analyzeFailure } from './failure-analysis-agent.js';
 export type { FailureContext } from './failure-analysis-agent.js';
+export { elementToPlaywrightLocator, elementsToPlaywrightLocators, formatLocatorsForPrompt } from './locator-extractor.js';
+export type { PlaywrightLocator } from './locator-extractor.js';

package/src/locator-extractor.ts

@@ -0,0 +1,226 @@
+/**
+ * Converts raw DB Element records (role, meaning, notes JSON) into Playwright-idiomatic locator strings.
+ * Priority: data-testid → aria-label/placeholder (inputs) → role+name (buttons/links) → heading → text → id
+ */
+
+export interface PlaywrightLocator {
+  key: string;           // camelCase property name for page object
+  expression: string;    // e.g. "page.getByRole('button', { name: 'Submit' })"
+  elementType: string;   // button | input | link | heading | text | element
+  confidence: 'high' | 'medium' | 'low';
+  source: 'testid' | 'label' | 'placeholder' | 'role-name' | 'heading' | 'text' | 'id';
+  description: string;   // human-readable label
+}
+
+/** Parse the notes JSON string stored on Element records. */
+function parseNotes(notes: unknown): Record<string, string> {
+  if (!notes) return {};
+  try {
+    const parsed = typeof notes === 'string' ? JSON.parse(notes) : notes;
+    if (parsed && typeof parsed === 'object') return parsed as Record<string, string>;
+  } catch {}
+  return {};
+}
+
+/** Map crawler role strings to Playwright role values. */
+function mapRole(role?: string | null): string | null {
+  if (!role) return null;
+  const r = role.toLowerCase().trim();
+  if (r === 'button' || r === 'submit' || r === 'reset') return 'button';
+  if (r === 'link' || r === 'a' || r === 'anchor') return 'link';
+  if (r === 'textbox' || r === 'input' || r === 'text' || r === 'email' || r === 'password' || r === 'tel' || r === 'number' || r === 'search') return 'textbox';
+  if (r === 'combobox' || r === 'select' || r === 'dropdown') return 'combobox';
+  if (r === 'checkbox') return 'checkbox';
+  if (r === 'radio') return 'radio';
+  if (r === 'heading' || r === 'h1' || r === 'h2' || r === 'h3' || r === 'h4' || r === 'h5' || r === 'h6') return 'heading';
+  if (r === 'img' || r === 'image') return 'img';
+  if (r === 'nav' || r === 'navigation') return 'navigation';
+  return r;
+}
+
+/** Convert "Submit Form Button" → "submitFormButton" */
+function toKey(text: string): string {
+  return text
+    .replace(/[^a-zA-Z0-9\s]/g, ' ')
+    .replace(/\s+(.)/g, (_, c) => c.toUpperCase())
+    .replace(/^(.)/, (_, c) => c.toLowerCase())
+    .replace(/\s/g, '')
+    .slice(0, 40)
+    || 'element';
+}
+
+/** Convert a single DB Element record to a Playwright locator. Returns null if insufficient data. */
+export function elementToPlaywrightLocator(element: {
+  role?: string | null;
+  meaning?: string | null;
+  text?: string | null;
+  notes?: unknown;
+  selector?: string | null;
+}): PlaywrightLocator | null {
+  const attrs = parseNotes(element.notes);
+  const testId = attrs['data-testid'];
+  const ariaLabel = attrs['aria-label'] || element.meaning;
+  const id = attrs['id'];
+  const placeholder = attrs['placeholder'];
+  const name = attrs['name'];
+  const text = (element.text ?? '').trim();
+  const meaning = (element.meaning ?? '').trim();
+  const role = mapRole(element.role);
+
+  const label = meaning || text || testId || ariaLabel || placeholder || id || name || 'element';
+  const key = toKey(label);
+
+  // Priority 1: data-testid
+  if (testId) {
+    return {
+      key,
+      expression: `page.getByTestId(${JSON.stringify(testId)})`,
+      elementType: role ?? 'element',
+      confidence: 'high',
+      source: 'testid',
+      description: meaning || testId,
+    };
+  }
+
+  // Priority 2: input/combobox/textarea with label → getByLabel; with placeholder → getByPlaceholder
+  if (role === 'textbox' || role === 'combobox') {
+    if (ariaLabel) {
+      return {
+        key,
+        expression: `page.getByLabel(${JSON.stringify(ariaLabel)})`,
+        elementType: 'input',
+        confidence: 'high',
+        source: 'label',
+        description: ariaLabel,
+      };
+    }
+    if (placeholder) {
+      return {
+        key,
+        expression: `page.getByPlaceholder(${JSON.stringify(placeholder)})`,
+        elementType: 'input',
+        confidence: 'high',
+        source: 'placeholder',
+        description: placeholder,
+      };
+    }
+  }
+
+  // Priority 3: checkbox/radio → getByLabel
+  if ((role === 'checkbox' || role === 'radio') && ariaLabel) {
+    return {
+      key,
+      expression: `page.getByLabel(${JSON.stringify(ariaLabel)})`,
+      elementType: role,
+      confidence: 'high',
+      source: 'label',
+      description: ariaLabel,
+    };
+  }
+
+  // Priority 4: button/link with meaningful name → getByRole
+  if ((role === 'button' || role === 'link') && meaning) {
+    return {
+      key,
+      expression: `page.getByRole(${JSON.stringify(role)}, { name: ${JSON.stringify(meaning)} })`,
+      elementType: role,
+      confidence: 'high',
+      source: 'role-name',
+      description: meaning,
+    };
+  }
+
+  // Priority 5: heading → getByRole heading
+  if (role === 'heading' && (meaning || text)) {
+    const name2 = meaning || text;
+    return {
+      key,
+      expression: `page.getByRole('heading', { name: ${JSON.stringify(name2)} })`,
+      elementType: 'heading',
+      confidence: 'medium',
+      source: 'heading',
+      description: name2,
+    };
+  }
+
+  // Priority 6: existing CSS selector from crawler
+  if (element.selector && element.selector.length > 0) {
+    return {
+      key,
+      expression: `page.locator(${JSON.stringify(element.selector)})`,
+      elementType: role ?? 'element',
+      confidence: 'medium',
+      source: 'id',
+      description: meaning || text || element.selector,
+    };
+  }
+
+  // Priority 7: id from attrs
+  if (id) {
+    return {
+      key,
+      expression: `page.locator(${JSON.stringify('#' + id)})`,
+      elementType: role ?? 'element',
+      confidence: 'medium',
+      source: 'id',
+      description: meaning || id,
+    };
+  }
+
+  // Priority 8: fallback to text content
+  if (meaning || text) {
+    const t = meaning || text;
+    return {
+      key,
+      expression: `page.getByText(${JSON.stringify(t)}, { exact: false })`,
+      elementType: role ?? 'text',
+      confidence: 'low',
+      source: 'text',
+      description: t,
+    };
+  }
+
+  return null;
+}
+
+/**
+ * Convert an array of DB Element records to deduplicated Playwright locators.
+ * Deduplicates by key — higher confidence wins.
+ */
+export function elementsToPlaywrightLocators(elements: Array<{
+  role?: string | null;
+  meaning?: string | null;
+  text?: string | null;
+  notes?: unknown;
+  selector?: string | null;
+  isInteractive?: boolean | null;
+}>): PlaywrightLocator[] {
+  const CONFIDENCE_RANK = { high: 0, medium: 1, low: 2 };
+  const byKey = new Map<string, PlaywrightLocator>();
+
+  for (const el of elements) {
+    const loc = elementToPlaywrightLocator(el);
+    if (!loc) continue;
+    const existing = byKey.get(loc.key);
+    if (!existing || CONFIDENCE_RANK[loc.confidence] < CONFIDENCE_RANK[existing.confidence]) {
+      byKey.set(loc.key, loc);
+    }
+  }
+
+  return Array.from(byKey.values());
+}
+
+/**
+ * Format locators as a block for injection into LLM script generation prompts.
+ * Returns empty string if no locators.
+ */
+export function formatLocatorsForPrompt(locators: PlaywrightLocator[]): string {
+  if (locators.length === 0) return '';
+  const lines = locators.map(l =>
+    `  ${l.key}: ${l.expression}  // ${l.description} [${l.confidence}]`
+  );
+  return [
+    'EXTRACTED PAGE LOCATORS (use these EXACTLY — do not invent other selectors; these were verified against the live page):',
+    ...lines,
+  ].join('\n');
+}

package/types/index.d.ts

@@ -35,3 +35,5 @@ export { startSpan, endSpan, withSpan, increment, flushMetrics } from './telemet
 export type { Span, FinishedSpan } from './telemetry.js';
 export { analyzeFailure } from './failure-analysis-agent.js';
 export type { FailureContext } from './failure-analysis-agent.js';
+export { elementToPlaywrightLocator, elementsToPlaywrightLocators, formatLocatorsForPrompt } from './locator-extractor.js';
+export type { PlaywrightLocator } from './locator-extractor.js';

package/types/locator-extractor.d.ts

@@ -0,0 +1,37 @@
+/**
+ * Converts raw DB Element records (role, meaning, notes JSON) into Playwright-idiomatic locator strings.
+ * Priority: data-testid → aria-label/placeholder (inputs) → role+name (buttons/links) → heading → text → id
+ */
+export interface PlaywrightLocator {
+    key: string;
+    expression: string;
+    elementType: string;
+    confidence: 'high' | 'medium' | 'low';
+    source: 'testid' | 'label' | 'placeholder' | 'role-name' | 'heading' | 'text' | 'id';
+    description: string;
+}
+/** Convert a single DB Element record to a Playwright locator. Returns null if insufficient data. */
+export declare function elementToPlaywrightLocator(element: {
+    role?: string | null;
+    meaning?: string | null;
+    text?: string | null;
+    notes?: unknown;
+    selector?: string | null;
+}): PlaywrightLocator | null;
+/**
+ * Convert an array of DB Element records to deduplicated Playwright locators.
+ * Deduplicates by key — higher confidence wins.
+ */
+export declare function elementsToPlaywrightLocators(elements: Array<{
+    role?: string | null;
+    meaning?: string | null;
+    text?: string | null;
+    notes?: unknown;
+    selector?: string | null;
+    isInteractive?: boolean | null;
+}>): PlaywrightLocator[];
+/**
+ * Format locators as a block for injection into LLM script generation prompts.
+ * Returns empty string if no locators.
+ */
+export declare function formatLocatorsForPrompt(locators: PlaywrightLocator[]): string;