@detiq/crawler 1.1.7 → 1.1.8

package/dist/extractors/advanced.d.ts

@@ -13,11 +13,21 @@ export interface PageMetadata {
     ogImage: string | null;
     ogUrl: string | null;
     ogType: string | null;
+    ogSiteName: string | null;
+    ogLocale: string | null;
     twitterCard: string | null;
     twitterTitle: string | null;
     twitterDescription: string | null;
+    twitterImage: string | null;
+    twitterSite: string | null;
     canonical: string | null;
     robots: string | null;
+    hreflang: Array<{
+        lang: string;
+        href: string;
+    }>;
+    themeColor: string | null;
+    favicon: string | null;
     jsonLd: object[];
     schemaTypes: string[];
     lang: string | null;

package/dist/index.d.ts

@@ -35,3 +35,5 @@ export { startSpan, endSpan, withSpan, increment, flushMetrics } from './telemet
 export type { Span, FinishedSpan } from './telemetry.js';
 export { analyzeFailure } from './failure-analysis-agent.js';
 export type { FailureContext } from './failure-analysis-agent.js';
+export { elementToPlaywrightLocator, elementsToPlaywrightLocators, formatLocatorsForPrompt } from './locator-extractor.js';
+export type { PlaywrightLocator } from './locator-extractor.js';

package/dist/index.js

@@ -25,6 +25,7 @@ import { TieredProxyManager } from "./proxy.js";
 import { RedisCrawlFrontier, isRedisFrontierEnabled } from "./redis-frontier.js";
 import { startSpan, endSpan, withSpan, increment, flushMetrics } from "./telemetry.js";
 import { analyzeFailure } from "./failure-analysis-agent.js";
+import { elementToPlaywrightLocator, elementsToPlaywrightLocators, formatLocatorsForPrompt } from "./locator-extractor.js";
 export {
   CHROME_TLS_HEADERS,
   RedisCrawlFrontier,
@@ -38,9 +39,12 @@ export {
   createRedisQueue,
   discoverProjectUrls,
   discoverSitemapUrls,
+  elementToPlaywrightLocator,
+  elementsToPlaywrightLocators,
   endSpan,
   extractMarkdown,
   flushMetrics,
+  formatLocatorsForPrompt,
   getFrame,
   getInboxEmail,
   getSsoClientCredentialsToken,

package/dist/index.js.map

@@ -1,7 +1,7 @@
 {
   "version": 3,
   "sources": ["../src/index.ts"],
-  "sourcesContent": ["export * from './crawler.js';\nexport * from './visual-diff-verdict.js';\nexport * from './auth.js';\nexport * from './screen-reader.js';\nexport * from './job-queue.js';\nexport { startSqsWorker } from './sqs-queue.js';\nexport * from './session-capture.js';\nexport * from './session-export.js';\nexport * from './test-executor.js';\nexport { startLiveView, getFrame, liveViewNavigate, completeLiveView, closeLiveView, sendInput } from './live-view.js';\nexport { waitForEmailOTP, getInboxEmail } from './email-otp.js';\nexport { getSsoToken, getSsoClientCredentialsToken } from './sso-token.js';\nexport type { SsoConfig, SsoProvider } from './sso-token.js';\nexport * from './engine-router.js';\nexport * from './profiles/domain-profile.js';\nexport * from './policies/crawl-safety.js';\nexport * from './extractors/artifact-envelope.js';\nexport * from './network-observer.js';\nexport { createChromeTlsAgent, probeTlsFingerprint, chromeTlsAgent, CHROME_TLS_HEADERS } from './tls-fingerprint.js';\nexport type { TlsFingerprintOptions } from './tls-fingerprint.js';\nexport { prepareExtensionProfile, adBlockerProfile } from './browser-extension.js';\nexport type { ExtensionSource, ExtensionProfile } from './browser-extension.js';\nexport { createRedisQueue, maybeCreateRedisQueue, isRedisQueueEnabled } from './redis-queue.js';\nexport type { RedisQueueJob, RedisQueueClient } from './redis-queue.js';\nexport { discoverProjectUrls, discoverSitemapUrls } from './sitemap.js';\nexport type { UrlCandidate } from './sitemap.js';\nexport { httpCrawl } from './http-engine.js';\nexport { extractMarkdown } from './markdown-extract.js';\nexport type { ExtractedMarkdown } from './markdown-extract.js';\nexport type { HttpScreen, HttpCrawlOpts, HttpCrawlResult, HttpOutputFormat } from './http-engine.js';\nexport { TieredProxyManager } from './proxy.js';\nexport type { ProxyTier, ProxyTierConfig } from './proxy.js';\nexport { RedisCrawlFrontier, isRedisFrontierEnabled } from './redis-frontier.js';\nexport { startSpan, endSpan, withSpan, increment, flushMetrics } from './telemetry.js';\nexport type { Span, FinishedSpan } from './telemetry.js';\nexport { analyzeFailure } from './failure-analysis-agent.js';\nexport type { FailureContext } from './failure-analysis-agent.js';\n"],
-  "mappings": "AAAA,cAAc;AACd,cAAc;AACd,cAAc;AACd,cAAc;AACd,cAAc;AACd,SAAS,sBAAsB;AAC/B,cAAc;AACd,cAAc;AACd,cAAc;AACd,SAAS,eAAe,UAAU,kBAAkB,kBAAkB,eAAe,iBAAiB;AACtG,SAAS,iBAAiB,qBAAqB;AAC/C,SAAS,aAAa,oCAAoC;AAE1D,cAAc;AACd,cAAc;AACd,cAAc;AACd,cAAc;AACd,cAAc;AACd,SAAS,sBAAsB,qBAAqB,gBAAgB,0BAA0B;AAE9F,SAAS,yBAAyB,wBAAwB;AAE1D,SAAS,kBAAkB,uBAAuB,2BAA2B;AAE7E,SAAS,qBAAqB,2BAA2B;AAEzD,SAAS,iBAAiB;AAC1B,SAAS,uBAAuB;AAGhC,SAAS,0BAA0B;AAEnC,SAAS,oBAAoB,8BAA8B;AAC3D,SAAS,WAAW,SAAS,UAAU,WAAW,oBAAoB;AAEtE,SAAS,sBAAsB;",
+  "sourcesContent": ["export * from './crawler.js';\nexport * from './visual-diff-verdict.js';\nexport * from './auth.js';\nexport * from './screen-reader.js';\nexport * from './job-queue.js';\nexport { startSqsWorker } from './sqs-queue.js';\nexport * from './session-capture.js';\nexport * from './session-export.js';\nexport * from './test-executor.js';\nexport { startLiveView, getFrame, liveViewNavigate, completeLiveView, closeLiveView, sendInput } from './live-view.js';\nexport { waitForEmailOTP, getInboxEmail } from './email-otp.js';\nexport { getSsoToken, getSsoClientCredentialsToken } from './sso-token.js';\nexport type { SsoConfig, SsoProvider } from './sso-token.js';\nexport * from './engine-router.js';\nexport * from './profiles/domain-profile.js';\nexport * from './policies/crawl-safety.js';\nexport * from './extractors/artifact-envelope.js';\nexport * from './network-observer.js';\nexport { createChromeTlsAgent, probeTlsFingerprint, chromeTlsAgent, CHROME_TLS_HEADERS } from './tls-fingerprint.js';\nexport type { TlsFingerprintOptions } from './tls-fingerprint.js';\nexport { prepareExtensionProfile, adBlockerProfile } from './browser-extension.js';\nexport type { ExtensionSource, ExtensionProfile } from './browser-extension.js';\nexport { createRedisQueue, maybeCreateRedisQueue, isRedisQueueEnabled } from './redis-queue.js';\nexport type { RedisQueueJob, RedisQueueClient } from './redis-queue.js';\nexport { discoverProjectUrls, discoverSitemapUrls } from './sitemap.js';\nexport type { UrlCandidate } from './sitemap.js';\nexport { httpCrawl } from './http-engine.js';\nexport { extractMarkdown } from './markdown-extract.js';\nexport type { ExtractedMarkdown } from './markdown-extract.js';\nexport type { HttpScreen, HttpCrawlOpts, HttpCrawlResult, HttpOutputFormat } from './http-engine.js';\nexport { TieredProxyManager } from './proxy.js';\nexport type { ProxyTier, ProxyTierConfig } from './proxy.js';\nexport { RedisCrawlFrontier, isRedisFrontierEnabled } from './redis-frontier.js';\nexport { startSpan, endSpan, withSpan, increment, flushMetrics } from './telemetry.js';\nexport type { Span, FinishedSpan } from './telemetry.js';\nexport { analyzeFailure } from './failure-analysis-agent.js';\nexport type { FailureContext } from './failure-analysis-agent.js';\nexport { elementToPlaywrightLocator, elementsToPlaywrightLocators, formatLocatorsForPrompt } from './locator-extractor.js';\nexport type { PlaywrightLocator } from './locator-extractor.js';\n"],
+  "mappings": "AAAA,cAAc;AACd,cAAc;AACd,cAAc;AACd,cAAc;AACd,cAAc;AACd,SAAS,sBAAsB;AAC/B,cAAc;AACd,cAAc;AACd,cAAc;AACd,SAAS,eAAe,UAAU,kBAAkB,kBAAkB,eAAe,iBAAiB;AACtG,SAAS,iBAAiB,qBAAqB;AAC/C,SAAS,aAAa,oCAAoC;AAE1D,cAAc;AACd,cAAc;AACd,cAAc;AACd,cAAc;AACd,cAAc;AACd,SAAS,sBAAsB,qBAAqB,gBAAgB,0BAA0B;AAE9F,SAAS,yBAAyB,wBAAwB;AAE1D,SAAS,kBAAkB,uBAAuB,2BAA2B;AAE7E,SAAS,qBAAqB,2BAA2B;AAEzD,SAAS,iBAAiB;AAC1B,SAAS,uBAAuB;AAGhC,SAAS,0BAA0B;AAEnC,SAAS,oBAAoB,8BAA8B;AAC3D,SAAS,WAAW,SAAS,UAAU,WAAW,oBAAoB;AAEtE,SAAS,sBAAsB;AAE/B,SAAS,4BAA4B,8BAA8B,+BAA+B;",
   "names": []
 }

package/dist/interactive-probe.d.ts

@@ -43,6 +43,35 @@ export interface ConditionalReveal {
     triggerValue: string;
     revealedText: string;
 }
+export interface TooltipCapture {
+    triggerLabel: string;
+    tooltipText: string;
+}
+export interface TabPanelCapture {
+    tabLabel: string;
+    panelContent: string;
+    panelHeading?: string;
+}
+export interface CarouselSlide {
+    slideIndex: number;
+    heading?: string;
+    content: string;
+}
+export interface WizardStep {
+    stepIndex: number;
+    stepLabel?: string;
+    content: string;
+    fields: string[];
+}
+export interface FocusTrapResult {
+    modalTrigger: string;
+    focusTrapped: boolean;
+    sampledFocused: string[];
+}
+export interface ContextMenuCapture {
+    trigger: string;
+    items: string[];
+}
 export interface ProbeResult {
     screenId: string;
     url: string;
@@ -51,6 +80,13 @@ export interface ProbeResult {
     modals: ModalObservation[];
     conditionals: ConditionalReveal[];
     loadingPatterns: string[];
+    tooltips?: TooltipCapture[];
+    tabPanels?: TabPanelCapture[];
+    carouselSlides?: CarouselSlide[];
+    wizardSteps?: WizardStep[];
+    focusTraps?: FocusTrapResult[];
+    skipNavIssues?: string[];
+    contextMenus?: ContextMenuCapture[];
 }
 /**
  * Run interactive probing on a single screen page.

package/dist/locator-extractor.d.ts

@@ -0,0 +1,37 @@
+/**
+ * Converts raw DB Element records (role, meaning, notes JSON) into Playwright-idiomatic locator strings.
+ * Priority: data-testid → aria-label/placeholder (inputs) → role+name (buttons/links) → heading → text → id
+ */
+export interface PlaywrightLocator {
+    key: string;
+    expression: string;
+    elementType: string;
+    confidence: 'high' | 'medium' | 'low';
+    source: 'testid' | 'label' | 'placeholder' | 'role-name' | 'heading' | 'text' | 'id';
+    description: string;
+}
+/** Convert a single DB Element record to a Playwright locator. Returns null if insufficient data. */
+export declare function elementToPlaywrightLocator(element: {
+    role?: string | null;
+    meaning?: string | null;
+    text?: string | null;
+    notes?: unknown;
+    selector?: string | null;
+}): PlaywrightLocator | null;
+/**
+ * Convert an array of DB Element records to deduplicated Playwright locators.
+ * Deduplicates by key — higher confidence wins.
+ */
+export declare function elementsToPlaywrightLocators(elements: Array<{
+    role?: string | null;
+    meaning?: string | null;
+    text?: string | null;
+    notes?: unknown;
+    selector?: string | null;
+    isInteractive?: boolean | null;
+}>): PlaywrightLocator[];
+/**
+ * Format locators as a block for injection into LLM script generation prompts.
+ * Returns empty string if no locators.
+ */
+export declare function formatLocatorsForPrompt(locators: PlaywrightLocator[]): string;

package/dist/locator-extractor.js

@@ -0,0 +1,163 @@
+function parseNotes(notes) {
+  if (!notes) return {};
+  try {
+    const parsed = typeof notes === "string" ? JSON.parse(notes) : notes;
+    if (parsed && typeof parsed === "object") return parsed;
+  } catch {
+  }
+  return {};
+}
+function mapRole(role) {
+  if (!role) return null;
+  const r = role.toLowerCase().trim();
+  if (r === "button" || r === "submit" || r === "reset") return "button";
+  if (r === "link" || r === "a" || r === "anchor") return "link";
+  if (r === "textbox" || r === "input" || r === "text" || r === "email" || r === "password" || r === "tel" || r === "number" || r === "search") return "textbox";
+  if (r === "combobox" || r === "select" || r === "dropdown") return "combobox";
+  if (r === "checkbox") return "checkbox";
+  if (r === "radio") return "radio";
+  if (r === "heading" || r === "h1" || r === "h2" || r === "h3" || r === "h4" || r === "h5" || r === "h6") return "heading";
+  if (r === "img" || r === "image") return "img";
+  if (r === "nav" || r === "navigation") return "navigation";
+  return r;
+}
+function toKey(text) {
+  return text.replace(/[^a-zA-Z0-9\s]/g, " ").replace(/\s+(.)/g, (_, c) => c.toUpperCase()).replace(/^(.)/, (_, c) => c.toLowerCase()).replace(/\s/g, "").slice(0, 40) || "element";
+}
+function elementToPlaywrightLocator(element) {
+  const attrs = parseNotes(element.notes);
+  const testId = attrs["data-testid"];
+  const ariaLabel = attrs["aria-label"] || element.meaning;
+  const id = attrs["id"];
+  const placeholder = attrs["placeholder"];
+  const name = attrs["name"];
+  const text = (element.text ?? "").trim();
+  const meaning = (element.meaning ?? "").trim();
+  const role = mapRole(element.role);
+  const label = meaning || text || testId || ariaLabel || placeholder || id || name || "element";
+  const key = toKey(label);
+  if (testId) {
+    return {
+      key,
+      expression: `page.getByTestId(${JSON.stringify(testId)})`,
+      elementType: role ?? "element",
+      confidence: "high",
+      source: "testid",
+      description: meaning || testId
+    };
+  }
+  if (role === "textbox" || role === "combobox") {
+    if (ariaLabel) {
+      return {
+        key,
+        expression: `page.getByLabel(${JSON.stringify(ariaLabel)})`,
+        elementType: "input",
+        confidence: "high",
+        source: "label",
+        description: ariaLabel
+      };
+    }
+    if (placeholder) {
+      return {
+        key,
+        expression: `page.getByPlaceholder(${JSON.stringify(placeholder)})`,
+        elementType: "input",
+        confidence: "high",
+        source: "placeholder",
+        description: placeholder
+      };
+    }
+  }
+  if ((role === "checkbox" || role === "radio") && ariaLabel) {
+    return {
+      key,
+      expression: `page.getByLabel(${JSON.stringify(ariaLabel)})`,
+      elementType: role,
+      confidence: "high",
+      source: "label",
+      description: ariaLabel
+    };
+  }
+  if ((role === "button" || role === "link") && meaning) {
+    return {
+      key,
+      expression: `page.getByRole(${JSON.stringify(role)}, { name: ${JSON.stringify(meaning)} })`,
+      elementType: role,
+      confidence: "high",
+      source: "role-name",
+      description: meaning
+    };
+  }
+  if (role === "heading" && (meaning || text)) {
+    const name2 = meaning || text;
+    return {
+      key,
+      expression: `page.getByRole('heading', { name: ${JSON.stringify(name2)} })`,
+      elementType: "heading",
+      confidence: "medium",
+      source: "heading",
+      description: name2
+    };
+  }
+  if (element.selector && element.selector.length > 0) {
+    return {
+      key,
+      expression: `page.locator(${JSON.stringify(element.selector)})`,
+      elementType: role ?? "element",
+      confidence: "medium",
+      source: "id",
+      description: meaning || text || element.selector
+    };
+  }
+  if (id) {
+    return {
+      key,
+      expression: `page.locator(${JSON.stringify("#" + id)})`,
+      elementType: role ?? "element",
+      confidence: "medium",
+      source: "id",
+      description: meaning || id
+    };
+  }
+  if (meaning || text) {
+    const t = meaning || text;
+    return {
+      key,
+      expression: `page.getByText(${JSON.stringify(t)}, { exact: false })`,
+      elementType: role ?? "text",
+      confidence: "low",
+      source: "text",
+      description: t
+    };
+  }
+  return null;
+}
+function elementsToPlaywrightLocators(elements) {
+  const CONFIDENCE_RANK = { high: 0, medium: 1, low: 2 };
+  const byKey = /* @__PURE__ */ new Map();
+  for (const el of elements) {
+    const loc = elementToPlaywrightLocator(el);
+    if (!loc) continue;
+    const existing = byKey.get(loc.key);
+    if (!existing || CONFIDENCE_RANK[loc.confidence] < CONFIDENCE_RANK[existing.confidence]) {
+      byKey.set(loc.key, loc);
+    }
+  }
+  return Array.from(byKey.values());
+}
+function formatLocatorsForPrompt(locators) {
+  if (locators.length === 0) return "";
+  const lines = locators.map(
+    (l) => `  ${l.key}: ${l.expression}  // ${l.description} [${l.confidence}]`
+  );
+  return [
+    "EXTRACTED PAGE LOCATORS (use these EXACTLY \u2014 do not invent other selectors; these were verified against the live page):",
+    ...lines
+  ].join("\n");
+}
+export {
+  elementToPlaywrightLocator,
+  elementsToPlaywrightLocators,
+  formatLocatorsForPrompt
+};
+//# sourceMappingURL=locator-extractor.js.map

package/dist/locator-extractor.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../src/locator-extractor.ts"],
+  "sourcesContent": ["/**\n * Converts raw DB Element records (role, meaning, notes JSON) into Playwright-idiomatic locator strings.\n * Priority: data-testid \u2192 aria-label/placeholder (inputs) \u2192 role+name (buttons/links) \u2192 heading \u2192 text \u2192 id\n */\n\nexport interface PlaywrightLocator {\n  key: string;           // camelCase property name for page object\n  expression: string;    // e.g. \"page.getByRole('button', { name: 'Submit' })\"\n  elementType: string;   // button | input | link | heading | text | element\n  confidence: 'high' | 'medium' | 'low';\n  source: 'testid' | 'label' | 'placeholder' | 'role-name' | 'heading' | 'text' | 'id';\n  description: string;   // human-readable label\n}\n\n/** Parse the notes JSON string stored on Element records. */\nfunction parseNotes(notes: unknown): Record<string, string> {\n  if (!notes) return {};\n  try {\n    const parsed = typeof notes === 'string' ? JSON.parse(notes) : notes;\n    if (parsed && typeof parsed === 'object') return parsed as Record<string, string>;\n  } catch {}\n  return {};\n}\n\n/** Map crawler role strings to Playwright role values. */\nfunction mapRole(role?: string | null): string | null {\n  if (!role) return null;\n  const r = role.toLowerCase().trim();\n  if (r === 'button' || r === 'submit' || r === 'reset') return 'button';\n  if (r === 'link' || r === 'a' || r === 'anchor') return 'link';\n  if (r === 'textbox' || r === 'input' || r === 'text' || r === 'email' || r === 'password' || r === 'tel' || r === 'number' || r === 'search') return 'textbox';\n  if (r === 'combobox' || r === 'select' || r === 'dropdown') return 'combobox';\n  if (r === 'checkbox') return 'checkbox';\n  if (r === 'radio') return 'radio';\n  if (r === 'heading' || r === 'h1' || r === 'h2' || r === 'h3' || r === 'h4' || r === 'h5' || r === 'h6') return 'heading';\n  if (r === 'img' || r === 'image') return 'img';\n  if (r === 'nav' || r === 'navigation') return 'navigation';\n  return r;\n}\n\n/** Convert \"Submit Form Button\" \u2192 \"submitFormButton\" */\nfunction toKey(text: string): string {\n  return text\n    .replace(/[^a-zA-Z0-9\\s]/g, ' ')\n    .replace(/\\s+(.)/g, (_, c) => c.toUpperCase())\n    .replace(/^(.)/, (_, c) => c.toLowerCase())\n    .replace(/\\s/g, '')\n    .slice(0, 40)\n    || 'element';\n}\n\n/** Convert a single DB Element record to a Playwright locator. Returns null if insufficient data. */\nexport function elementToPlaywrightLocator(element: {\n  role?: string | null;\n  meaning?: string | null;\n  text?: string | null;\n  notes?: unknown;\n  selector?: string | null;\n}): PlaywrightLocator | null {\n  const attrs = parseNotes(element.notes);\n  const testId = attrs['data-testid'];\n  const ariaLabel = attrs['aria-label'] || element.meaning;\n  const id = attrs['id'];\n  const placeholder = attrs['placeholder'];\n  const name = attrs['name'];\n  const text = (element.text ?? '').trim();\n  const meaning = (element.meaning ?? '').trim();\n  const role = mapRole(element.role);\n\n  const label = meaning || text || testId || ariaLabel || placeholder || id || name || 'element';\n  const key = toKey(label);\n\n  // Priority 1: data-testid\n  if (testId) {\n    return {\n      key,\n      expression: `page.getByTestId(${JSON.stringify(testId)})`,\n      elementType: role ?? 'element',\n      confidence: 'high',\n      source: 'testid',\n      description: meaning || testId,\n    };\n  }\n\n  // Priority 2: input/combobox/textarea with label \u2192 getByLabel; with placeholder \u2192 getByPlaceholder\n  if (role === 'textbox' || role === 'combobox') {\n    if (ariaLabel) {\n      return {\n        key,\n        expression: `page.getByLabel(${JSON.stringify(ariaLabel)})`,\n        elementType: 'input',\n        confidence: 'high',\n        source: 'label',\n        description: ariaLabel,\n      };\n    }\n    if (placeholder) {\n      return {\n        key,\n        expression: `page.getByPlaceholder(${JSON.stringify(placeholder)})`,\n        elementType: 'input',\n        confidence: 'high',\n        source: 'placeholder',\n        description: placeholder,\n      };\n    }\n  }\n\n  // Priority 3: checkbox/radio \u2192 getByLabel\n  if ((role === 'checkbox' || role === 'radio') && ariaLabel) {\n    return {\n      key,\n      expression: `page.getByLabel(${JSON.stringify(ariaLabel)})`,\n      elementType: role,\n      confidence: 'high',\n      source: 'label',\n      description: ariaLabel,\n    };\n  }\n\n  // Priority 4: button/link with meaningful name \u2192 getByRole\n  if ((role === 'button' || role === 'link') && meaning) {\n    return {\n      key,\n      expression: `page.getByRole(${JSON.stringify(role)}, { name: ${JSON.stringify(meaning)} })`,\n      elementType: role,\n      confidence: 'high',\n      source: 'role-name',\n      description: meaning,\n    };\n  }\n\n  // Priority 5: heading \u2192 getByRole heading\n  if (role === 'heading' && (meaning || text)) {\n    const name2 = meaning || text;\n    return {\n      key,\n      expression: `page.getByRole('heading', { name: ${JSON.stringify(name2)} })`,\n      elementType: 'heading',\n      confidence: 'medium',\n      source: 'heading',\n      description: name2,\n    };\n  }\n\n  // Priority 6: existing CSS selector from crawler\n  if (element.selector && element.selector.length > 0) {\n    return {\n      key,\n      expression: `page.locator(${JSON.stringify(element.selector)})`,\n      elementType: role ?? 'element',\n      confidence: 'medium',\n      source: 'id',\n      description: meaning || text || element.selector,\n    };\n  }\n\n  // Priority 7: id from attrs\n  if (id) {\n    return {\n      key,\n      expression: `page.locator(${JSON.stringify('#' + id)})`,\n      elementType: role ?? 'element',\n      confidence: 'medium',\n      source: 'id',\n      description: meaning || id,\n    };\n  }\n\n  // Priority 8: fallback to text content\n  if (meaning || text) {\n    const t = meaning || text;\n    return {\n      key,\n      expression: `page.getByText(${JSON.stringify(t)}, { exact: false })`,\n      elementType: role ?? 'text',\n      confidence: 'low',\n      source: 'text',\n      description: t,\n    };\n  }\n\n  return null;\n}\n\n/**\n * Convert an array of DB Element records to deduplicated Playwright locators.\n * Deduplicates by key \u2014 higher confidence wins.\n */\nexport function elementsToPlaywrightLocators(elements: Array<{\n  role?: string | null;\n  meaning?: string | null;\n  text?: string | null;\n  notes?: unknown;\n  selector?: string | null;\n  isInteractive?: boolean | null;\n}>): PlaywrightLocator[] {\n  const CONFIDENCE_RANK = { high: 0, medium: 1, low: 2 };\n  const byKey = new Map<string, PlaywrightLocator>();\n\n  for (const el of elements) {\n    const loc = elementToPlaywrightLocator(el);\n    if (!loc) continue;\n    const existing = byKey.get(loc.key);\n    if (!existing || CONFIDENCE_RANK[loc.confidence] < CONFIDENCE_RANK[existing.confidence]) {\n      byKey.set(loc.key, loc);\n    }\n  }\n\n  return Array.from(byKey.values());\n}\n\n/**\n * Format locators as a block for injection into LLM script generation prompts.\n * Returns empty string if no locators.\n */\nexport function formatLocatorsForPrompt(locators: PlaywrightLocator[]): string {\n  if (locators.length === 0) return '';\n  const lines = locators.map(l =>\n    `  ${l.key}: ${l.expression}  // ${l.description} [${l.confidence}]`\n  );\n  return [\n    'EXTRACTED PAGE LOCATORS (use these EXACTLY \u2014 do not invent other selectors; these were verified against the live page):',\n    ...lines,\n  ].join('\\n');\n}\n"],
+  "mappings": "AAeA,SAAS,WAAW,OAAwC;AAC1D,MAAI,CAAC,MAAO,QAAO,CAAC;AACpB,MAAI;AACF,UAAM,SAAS,OAAO,UAAU,WAAW,KAAK,MAAM,KAAK,IAAI;AAC/D,QAAI,UAAU,OAAO,WAAW,SAAU,QAAO;AAAA,EACnD,QAAQ;AAAA,EAAC;AACT,SAAO,CAAC;AACV;AAGA,SAAS,QAAQ,MAAqC;AACpD,MAAI,CAAC,KAAM,QAAO;AAClB,QAAM,IAAI,KAAK,YAAY,EAAE,KAAK;AAClC,MAAI,MAAM,YAAY,MAAM,YAAY,MAAM,QAAS,QAAO;AAC9D,MAAI,MAAM,UAAU,MAAM,OAAO,MAAM,SAAU,QAAO;AACxD,MAAI,MAAM,aAAa,MAAM,WAAW,MAAM,UAAU,MAAM,WAAW,MAAM,cAAc,MAAM,SAAS,MAAM,YAAY,MAAM,SAAU,QAAO;AACrJ,MAAI,MAAM,cAAc,MAAM,YAAY,MAAM,WAAY,QAAO;AACnE,MAAI,MAAM,WAAY,QAAO;AAC7B,MAAI,MAAM,QAAS,QAAO;AAC1B,MAAI,MAAM,aAAa,MAAM,QAAQ,MAAM,QAAQ,MAAM,QAAQ,MAAM,QAAQ,MAAM,QAAQ,MAAM,KAAM,QAAO;AAChH,MAAI,MAAM,SAAS,MAAM,QAAS,QAAO;AACzC,MAAI,MAAM,SAAS,MAAM,aAAc,QAAO;AAC9C,SAAO;AACT;AAGA,SAAS,MAAM,MAAsB;AACnC,SAAO,KACJ,QAAQ,mBAAmB,GAAG,EAC9B,QAAQ,WAAW,CAAC,GAAG,MAAM,EAAE,YAAY,CAAC,EAC5C,QAAQ,QAAQ,CAAC,GAAG,MAAM,EAAE,YAAY,CAAC,EACzC,QAAQ,OAAO,EAAE,EACjB,MAAM,GAAG,EAAE,KACT;AACP;AAGO,SAAS,2BAA2B,SAMd;AAC3B,QAAM,QAAQ,WAAW,QAAQ,KAAK;AACtC,QAAM,SAAS,MAAM,aAAa;AAClC,QAAM,YAAY,MAAM,YAAY,KAAK,QAAQ;AACjD,QAAM,KAAK,MAAM,IAAI;AACrB,QAAM,cAAc,MAAM,aAAa;AACvC,QAAM,OAAO,MAAM,MAAM;AACzB,QAAM,QAAQ,QAAQ,QAAQ,IAAI,KAAK;AACvC,QAAM,WAAW,QAAQ,WAAW,IAAI,KAAK;AAC7C,QAAM,OAAO,QAAQ,QAAQ,IAAI;AAEjC,QAAM,QAAQ,WAAW,QAAQ,UAAU,aAAa,eAAe,MAAM,QAAQ;AACrF,QAAM,MAAM,MAAM,KAAK;AAGvB,MAAI,QAAQ;AACV,WAAO;AAAA,MACL;AAAA,MACA,YAAY,oBAAoB,KAAK,UAAU,MAAM,CAAC;AAAA,MACtD,aAAa,QAAQ;AAAA,MACrB,YAAY;AAAA,MACZ,QAAQ;AAAA,MACR,aAAa,WAAW;AAAA,IAC1B;AAAA,EACF;AAGA,MAAI,SAAS,aAAa,SAAS,YAAY;AAC7C,QAAI,WAAW;AACb,aAAO;AAAA,QACL;AAAA,QACA,YAAY,mBAAmB,KAAK,UAAU,SAAS,CAAC;AAAA,QACxD,aAAa;AAAA,QACb,YAAY;AAAA,QACZ,QAAQ;AAAA,QACR,aAAa;AAAA,MACf;AAAA,IACF;AACA,QAAI,aAAa;AACf,aAAO;AAAA,QACL;AAAA,QACA,YAAY,yBAAyB,KAAK,UAAU,WAAW,CAAC;AAAA,QAChE,aAAa;AAAA,QACb,YAAY;AAAA,QACZ,QAAQ;AAAA,QACR,aAAa;AAAA,MACf;AAAA,IACF;AAAA,EACF;AAGA,OAAK,SAAS,cAAc,SAAS,YAAY,WAAW;AAC1D,WAAO;AAAA,MACL;AAAA,MACA,YAAY,mBAAmB,KAAK,UAAU,SAAS,CAAC;AAAA,MACxD,aAAa;AAAA,MACb,YAAY;AAAA,MACZ,QAAQ;AAAA,MACR,aAAa;AAAA,IACf;AAAA,EACF;AAGA,OAAK,SAAS,YAAY,SAAS,WAAW,SAAS;AACrD,WAAO;AAAA,MACL;AAAA,MACA,YAAY,kBAAkB,KAAK,UAAU,IAAI,CAAC,aAAa,KAAK,UAAU,OAAO,CAAC;AAAA,MACtF,aAAa;AAAA,MACb,YAAY;AAAA,MACZ,QAAQ;AAAA,MACR,aAAa;AAAA,IACf;AAAA,EACF;AAGA,MAAI,SAAS,cAAc,WAAW,OAAO;AAC3C,UAAM,QAAQ,WAAW;AACzB,WAAO;AAAA,MACL;AAAA,MACA,YAAY,qCAAqC,KAAK,UAAU,KAAK,CAAC;AAAA,MACtE,aAAa;AAAA,MACb,YAAY;AAAA,MACZ,QAAQ;AAAA,MACR,aAAa;AAAA,IACf;AAAA,EACF;AAGA,MAAI,QAAQ,YAAY,QAAQ,SAAS,SAAS,GAAG;AACnD,WAAO;AAAA,MACL;AAAA,MACA,YAAY,gBAAgB,KAAK,UAAU,QAAQ,QAAQ,CAAC;AAAA,MAC5D,aAAa,QAAQ;AAAA,MACrB,YAAY;AAAA,MACZ,QAAQ;AAAA,MACR,aAAa,WAAW,QAAQ,QAAQ;AAAA,IAC1C;AAAA,EACF;AAGA,MAAI,IAAI;AACN,WAAO;AAAA,MACL;AAAA,MACA,YAAY,gBAAgB,KAAK,UAAU,MAAM,EAAE,CAAC;AAAA,MACpD,aAAa,QAAQ;AAAA,MACrB,YAAY;AAAA,MACZ,QAAQ;AAAA,MACR,aAAa,WAAW;AAAA,IAC1B;AAAA,EACF;AAGA,MAAI,WAAW,MAAM;AACnB,UAAM,IAAI,WAAW;AACrB,WAAO;AAAA,MACL;AAAA,MACA,YAAY,kBAAkB,KAAK,UAAU,CAAC,CAAC;AAAA,MAC/C,aAAa,QAAQ;AAAA,MACrB,YAAY;AAAA,MACZ,QAAQ;AAAA,MACR,aAAa;AAAA,IACf;AAAA,EACF;AAEA,SAAO;AACT;AAMO,SAAS,6BAA6B,UAOpB;AACvB,QAAM,kBAAkB,EAAE,MAAM,GAAG,QAAQ,GAAG,KAAK,EAAE;AACrD,QAAM,QAAQ,oBAAI,IAA+B;AAEjD,aAAW,MAAM,UAAU;AACzB,UAAM,MAAM,2BAA2B,EAAE;AACzC,QAAI,CAAC,IAAK;AACV,UAAM,WAAW,MAAM,IAAI,IAAI,GAAG;AAClC,QAAI,CAAC,YAAY,gBAAgB,IAAI,UAAU,IAAI,gBAAgB,SAAS,UAAU,GAAG;AACvF,YAAM,IAAI,IAAI,KAAK,GAAG;AAAA,IACxB;AAAA,EACF;AAEA,SAAO,MAAM,KAAK,MAAM,OAAO,CAAC;AAClC;AAMO,SAAS,wBAAwB,UAAuC;AAC7E,MAAI,SAAS,WAAW,EAAG,QAAO;AAClC,QAAM,QAAQ,SAAS;AAAA,IAAI,OACzB,KAAK,EAAE,GAAG,KAAK,EAAE,UAAU,QAAQ,EAAE,WAAW,KAAK,EAAE,UAAU;AAAA,EACnE;AACA,SAAO;AAAA,IACL;AAAA,IACA,GAAG;AAAA,EACL,EAAE,KAAK,IAAI;AACb;",
+  "names": []
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@detiq/crawler",
-  "version": "1.1.7",
+  "version": "1.1.8",
   "private": false,
   "publishConfig": {
     "access": "public",

package/src/index.ts

@@ -35,3 +35,5 @@ export { startSpan, endSpan, withSpan, increment, flushMetrics } from './telemet
 export type { Span, FinishedSpan } from './telemetry.js';
 export { analyzeFailure } from './failure-analysis-agent.js';
 export type { FailureContext } from './failure-analysis-agent.js';
+export { elementToPlaywrightLocator, elementsToPlaywrightLocators, formatLocatorsForPrompt } from './locator-extractor.js';
+export type { PlaywrightLocator } from './locator-extractor.js';

package/src/locator-extractor.ts

@@ -0,0 +1,226 @@
+/**
+ * Converts raw DB Element records (role, meaning, notes JSON) into Playwright-idiomatic locator strings.
+ * Priority: data-testid → aria-label/placeholder (inputs) → role+name (buttons/links) → heading → text → id
+ */
+
+export interface PlaywrightLocator {
+  key: string;           // camelCase property name for page object
+  expression: string;    // e.g. "page.getByRole('button', { name: 'Submit' })"
+  elementType: string;   // button | input | link | heading | text | element
+  confidence: 'high' | 'medium' | 'low';
+  source: 'testid' | 'label' | 'placeholder' | 'role-name' | 'heading' | 'text' | 'id';
+  description: string;   // human-readable label
+}
+
+/** Parse the notes JSON string stored on Element records. */
+function parseNotes(notes: unknown): Record<string, string> {
+  if (!notes) return {};
+  try {
+    const parsed = typeof notes === 'string' ? JSON.parse(notes) : notes;
+    if (parsed && typeof parsed === 'object') return parsed as Record<string, string>;
+  } catch {}
+  return {};
+}
+
+/** Map crawler role strings to Playwright role values. */
+function mapRole(role?: string | null): string | null {
+  if (!role) return null;
+  const r = role.toLowerCase().trim();
+  if (r === 'button' || r === 'submit' || r === 'reset') return 'button';
+  if (r === 'link' || r === 'a' || r === 'anchor') return 'link';
+  if (r === 'textbox' || r === 'input' || r === 'text' || r === 'email' || r === 'password' || r === 'tel' || r === 'number' || r === 'search') return 'textbox';
+  if (r === 'combobox' || r === 'select' || r === 'dropdown') return 'combobox';
+  if (r === 'checkbox') return 'checkbox';
+  if (r === 'radio') return 'radio';
+  if (r === 'heading' || r === 'h1' || r === 'h2' || r === 'h3' || r === 'h4' || r === 'h5' || r === 'h6') return 'heading';
+  if (r === 'img' || r === 'image') return 'img';
+  if (r === 'nav' || r === 'navigation') return 'navigation';
+  return r;
+}
+
+/** Convert "Submit Form Button" → "submitFormButton" */
+function toKey(text: string): string {
+  return text
+    .replace(/[^a-zA-Z0-9\s]/g, ' ')
+    .replace(/\s+(.)/g, (_, c) => c.toUpperCase())
+    .replace(/^(.)/, (_, c) => c.toLowerCase())
+    .replace(/\s/g, '')
+    .slice(0, 40)
+    || 'element';
+}
+
+/** Convert a single DB Element record to a Playwright locator. Returns null if insufficient data. */
+export function elementToPlaywrightLocator(element: {
+  role?: string | null;
+  meaning?: string | null;
+  text?: string | null;
+  notes?: unknown;
+  selector?: string | null;
+}): PlaywrightLocator | null {
+  const attrs = parseNotes(element.notes);
+  const testId = attrs['data-testid'];
+  const ariaLabel = attrs['aria-label'] || element.meaning;
+  const id = attrs['id'];
+  const placeholder = attrs['placeholder'];
+  const name = attrs['name'];
+  const text = (element.text ?? '').trim();
+  const meaning = (element.meaning ?? '').trim();
+  const role = mapRole(element.role);
+
+  const label = meaning || text || testId || ariaLabel || placeholder || id || name || 'element';
+  const key = toKey(label);
+
+  // Priority 1: data-testid
+  if (testId) {
+    return {
+      key,
+      expression: `page.getByTestId(${JSON.stringify(testId)})`,
+      elementType: role ?? 'element',
+      confidence: 'high',
+      source: 'testid',
+      description: meaning || testId,
+    };
+  }
+
+  // Priority 2: input/combobox/textarea with label → getByLabel; with placeholder → getByPlaceholder
+  if (role === 'textbox' || role === 'combobox') {
+    if (ariaLabel) {
+      return {
+        key,
+        expression: `page.getByLabel(${JSON.stringify(ariaLabel)})`,
+        elementType: 'input',
+        confidence: 'high',
+        source: 'label',
+        description: ariaLabel,
+      };
+    }
+    if (placeholder) {
+      return {
+        key,
+        expression: `page.getByPlaceholder(${JSON.stringify(placeholder)})`,
+        elementType: 'input',
+        confidence: 'high',
+        source: 'placeholder',
+        description: placeholder,
+      };
+    }
+  }
+
+  // Priority 3: checkbox/radio → getByLabel
+  if ((role === 'checkbox' || role === 'radio') && ariaLabel) {
+    return {
+      key,
+      expression: `page.getByLabel(${JSON.stringify(ariaLabel)})`,
+      elementType: role,
+      confidence: 'high',
+      source: 'label',
+      description: ariaLabel,
+    };
+  }
+
+  // Priority 4: button/link with meaningful name → getByRole
+  if ((role === 'button' || role === 'link') && meaning) {
+    return {
+      key,
+      expression: `page.getByRole(${JSON.stringify(role)}, { name: ${JSON.stringify(meaning)} })`,
+      elementType: role,
+      confidence: 'high',
+      source: 'role-name',
+      description: meaning,
+    };
+  }
+
+  // Priority 5: heading → getByRole heading
+  if (role === 'heading' && (meaning || text)) {
+    const name2 = meaning || text;
+    return {
+      key,
+      expression: `page.getByRole('heading', { name: ${JSON.stringify(name2)} })`,
+      elementType: 'heading',
+      confidence: 'medium',
+      source: 'heading',
+      description: name2,
+    };
+  }
+
+  // Priority 6: existing CSS selector from crawler
+  if (element.selector && element.selector.length > 0) {
+    return {
+      key,
+      expression: `page.locator(${JSON.stringify(element.selector)})`,
+      elementType: role ?? 'element',
+      confidence: 'medium',
+      source: 'id',
+      description: meaning || text || element.selector,
+    };
+  }
+
+  // Priority 7: id from attrs
+  if (id) {
+    return {
+      key,
+      expression: `page.locator(${JSON.stringify('#' + id)})`,
+      elementType: role ?? 'element',
+      confidence: 'medium',
+      source: 'id',
+      description: meaning || id,
+    };
+  }
+
+  // Priority 8: fallback to text content
+  if (meaning || text) {
+    const t = meaning || text;
+    return {
+      key,
+      expression: `page.getByText(${JSON.stringify(t)}, { exact: false })`,
+      elementType: role ?? 'text',
+      confidence: 'low',
+      source: 'text',
+      description: t,
+    };
+  }
+
+  return null;
+}
+
+/**
+ * Convert an array of DB Element records to deduplicated Playwright locators.
+ * Deduplicates by key — higher confidence wins.
+ */
+export function elementsToPlaywrightLocators(elements: Array<{
+  role?: string | null;
+  meaning?: string | null;
+  text?: string | null;
+  notes?: unknown;
+  selector?: string | null;
+  isInteractive?: boolean | null;
+}>): PlaywrightLocator[] {
+  const CONFIDENCE_RANK = { high: 0, medium: 1, low: 2 };
+  const byKey = new Map<string, PlaywrightLocator>();
+
+  for (const el of elements) {
+    const loc = elementToPlaywrightLocator(el);
+    if (!loc) continue;
+    const existing = byKey.get(loc.key);
+    if (!existing || CONFIDENCE_RANK[loc.confidence] < CONFIDENCE_RANK[existing.confidence]) {
+      byKey.set(loc.key, loc);
+    }
+  }
+
+  return Array.from(byKey.values());
+}
+
+/**
+ * Format locators as a block for injection into LLM script generation prompts.
+ * Returns empty string if no locators.
+ */
+export function formatLocatorsForPrompt(locators: PlaywrightLocator[]): string {
+  if (locators.length === 0) return '';
+  const lines = locators.map(l =>
+    `  ${l.key}: ${l.expression}  // ${l.description} [${l.confidence}]`
+  );
+  return [
+    'EXTRACTED PAGE LOCATORS (use these EXACTLY — do not invent other selectors; these were verified against the live page):',
+    ...lines,
+  ].join('\n');
+}

package/types/index.d.ts

@@ -35,3 +35,5 @@ export { startSpan, endSpan, withSpan, increment, flushMetrics } from './telemet
 export type { Span, FinishedSpan } from './telemetry.js';
 export { analyzeFailure } from './failure-analysis-agent.js';
 export type { FailureContext } from './failure-analysis-agent.js';
+export { elementToPlaywrightLocator, elementsToPlaywrightLocators, formatLocatorsForPrompt } from './locator-extractor.js';
+export type { PlaywrightLocator } from './locator-extractor.js';

package/types/locator-extractor.d.ts

@@ -0,0 +1,37 @@
+/**
+ * Converts raw DB Element records (role, meaning, notes JSON) into Playwright-idiomatic locator strings.
+ * Priority: data-testid → aria-label/placeholder (inputs) → role+name (buttons/links) → heading → text → id
+ */
+export interface PlaywrightLocator {
+    key: string;
+    expression: string;
+    elementType: string;
+    confidence: 'high' | 'medium' | 'low';
+    source: 'testid' | 'label' | 'placeholder' | 'role-name' | 'heading' | 'text' | 'id';
+    description: string;
+}
+/** Convert a single DB Element record to a Playwright locator. Returns null if insufficient data. */
+export declare function elementToPlaywrightLocator(element: {
+    role?: string | null;
+    meaning?: string | null;
+    text?: string | null;
+    notes?: unknown;
+    selector?: string | null;
+}): PlaywrightLocator | null;
+/**
+ * Convert an array of DB Element records to deduplicated Playwright locators.
+ * Deduplicates by key — higher confidence wins.
+ */
+export declare function elementsToPlaywrightLocators(elements: Array<{
+    role?: string | null;
+    meaning?: string | null;
+    text?: string | null;
+    notes?: unknown;
+    selector?: string | null;
+    isInteractive?: boolean | null;
+}>): PlaywrightLocator[];
+/**
+ * Format locators as a block for injection into LLM script generation prompts.
+ * Returns empty string if no locators.
+ */
+export declare function formatLocatorsForPrompt(locators: PlaywrightLocator[]): string;