@detiq/crawler 0.1.3

package/dist/engine-router.d.ts

@@ -0,0 +1,47 @@
+import type { DomainProfile } from './profiles/domain-profile.js';
+export type CrawlEngineName = 'http' | 'browser' | 'browser-stealth' | 'stagehand' | 'vision' | 'recorded-session' | 'native';
+export interface CrawlRouteDecisionInput {
+    appUrl: string;
+    appType?: string | null;
+    hasCredentials: boolean;
+    hasStorageState: boolean;
+    hasAuthToken: boolean;
+    hasSkyvern: boolean;
+    hasProxy: boolean;
+    profile: DomainProfile;
+    previousFailureReason?: string;
+}
+export interface CrawlRouteDecision {
+    engines: CrawlEngineName[];
+    reasons: string[];
+    confidence: number;
+}
+/**
+ * Per-domain rendering predictor — learns whether a domain requires a full browser
+ * or can be served by the fast HTTP engine. Implements adaptive crawlee-python pattern.
+ *
+ * Strategy:
+ * 1. On first encounter of a domain, classify the HTTP response body.
+ * 2. Store the decision in memory (Map). All future URLs on same domain skip re-testing.
+ * 3. "Requires browser" signals: React/Vue/Angular root with no content, tiny body (<500 chars meaningful text).
+ */
+export declare class RenderingTypePredictor {
+    private cache;
+    /** Return cached decision if available, undefined if unknown. */
+    getCached(url: string): 'http' | 'browser' | undefined;
+    /**
+     * Analyze an HTTP response body to predict if JS rendering is required.
+     * Returns 'browser' if JS-heavy signals detected, 'http' otherwise.
+     */
+    predict(url: string, htmlBody: string, statusCode: number): 'http' | 'browser';
+    /** Force a domain to use a specific engine (e.g. after observing crawl errors). */
+    set(url: string, engine: 'http' | 'browser'): void;
+    stats(): {
+        cached: number;
+        browserDomains: string[];
+        httpDomains: string[];
+    };
+}
+/** Singleton predictor — shared across crawl jobs in the same process. */
+export declare const renderingPredictor: RenderingTypePredictor;
+export declare function chooseCrawlEngines(input: CrawlRouteDecisionInput): CrawlRouteDecision;

package/dist/engine-router.js

@@ -0,0 +1,94 @@
+class RenderingTypePredictor {
+  cache = /* @__PURE__ */ new Map();
+  /** Return cached decision if available, undefined if unknown. */
+  getCached(url) {
+    try {
+      return this.cache.get(new URL(url).hostname);
+    } catch {
+      return void 0;
+    }
+  }
+  /**
+   * Analyze an HTTP response body to predict if JS rendering is required.
+   * Returns 'browser' if JS-heavy signals detected, 'http' otherwise.
+   */
+  predict(url, htmlBody, statusCode) {
+    let decision = "http";
+    try {
+      if (statusCode >= 400) {
+        decision = "browser";
+      } else {
+        const bodyLen = htmlBody.length;
+        const textOnly = htmlBody.replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").trim();
+        const wordCount = textOnly.split(/\s+/).filter((w) => w.length > 2).length;
+        const hasSpaRoot = /<div[^>]*\s+id=["'](?:root|app|__nuxt|__next|app-root)["'][^>]*>\s*<\/div>/i.test(htmlBody) || /<app-root[^>]*>\s*<\/app-root>/i.test(htmlBody);
+        const hasNoscriptFallback = /<noscript[^>]*>[\s\S]{100,}/i.test(htmlBody);
+        const tinyBody = bodyLen < 3e3 && wordCount < 50;
+        if (hasSpaRoot && wordCount < 100 || tinyBody && !hasNoscriptFallback) {
+          decision = "browser";
+        }
+      }
+    } catch {
+    }
+    try {
+      this.cache.set(new URL(url).hostname, decision);
+    } catch {
+    }
+    return decision;
+  }
+  /** Force a domain to use a specific engine (e.g. after observing crawl errors). */
+  set(url, engine) {
+    try {
+      this.cache.set(new URL(url).hostname, engine);
+    } catch {
+    }
+  }
+  stats() {
+    const browserDomains = [];
+    const httpDomains = [];
+    for (const [domain, engine] of this.cache) {
+      if (engine === "browser") browserDomains.push(domain);
+      else httpDomains.push(domain);
+    }
+    return { cached: this.cache.size, browserDomains, httpDomains };
+  }
+}
+const renderingPredictor = new RenderingTypePredictor();
+function chooseCrawlEngines(input) {
+  const engines = [];
+  const reasons = [];
+  const appType = input.appType?.toLowerCase();
+  if (appType === "android" || appType === "ios" || appType === "windows" || appType === "macos") {
+    return { engines: ["native"], reasons: [`appType=${input.appType}`], confidence: 0.95 };
+  }
+  if (input.hasStorageState || input.hasAuthToken) {
+    engines.push("recorded-session");
+    reasons.push("session/token auth available");
+  }
+  engines.push(input.hasProxy ? "browser-stealth" : "browser");
+  reasons.push(input.hasProxy ? "proxy configured; prefer stealth browser path" : "default browser path");
+  if (input.hasCredentials || input.profile.industry !== "unknown") {
+    engines.push("stagehand");
+    reasons.push(input.hasCredentials ? "credentials available for AI login/navigation" : `domain profile=${input.profile.industry}`);
+  }
+  const hardUi = input.profile.industry === "legacy-enterprise" || /captcha|blocked|canvas|iframe|bot/i.test(input.previousFailureReason ?? "");
+  if (input.hasSkyvern && hardUi) {
+    engines.push("vision");
+    reasons.push("hard UI or previous block detected; vision fallback enabled");
+  }
+  if (engines.length === 1) {
+    engines.unshift("http");
+    reasons.push("HTTP/static extraction can be attempted before browser escalation");
+  }
+  return {
+    engines: Array.from(new Set(engines)),
+    reasons,
+    confidence: input.profile.confidence
+  };
+}
+export {
+  RenderingTypePredictor,
+  chooseCrawlEngines,
+  renderingPredictor
+};
+//# sourceMappingURL=engine-router.js.map

package/dist/engine-router.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../src/engine-router.ts"],
+  "sourcesContent": ["import type { DomainProfile } from './profiles/domain-profile.js';\n\nexport type CrawlEngineName =\n  | 'http'\n  | 'browser'\n  | 'browser-stealth'\n  | 'stagehand'\n  | 'vision'\n  | 'recorded-session'\n  | 'native';\n\nexport interface CrawlRouteDecisionInput {\n  appUrl: string;\n  appType?: string | null;\n  hasCredentials: boolean;\n  hasStorageState: boolean;\n  hasAuthToken: boolean;\n  hasSkyvern: boolean;\n  hasProxy: boolean;\n  profile: DomainProfile;\n  previousFailureReason?: string;\n}\n\nexport interface CrawlRouteDecision {\n  engines: CrawlEngineName[];\n  reasons: string[];\n  confidence: number;\n}\n\n/**\n * Per-domain rendering predictor \u2014 learns whether a domain requires a full browser\n * or can be served by the fast HTTP engine. Implements adaptive crawlee-python pattern.\n *\n * Strategy:\n * 1. On first encounter of a domain, classify the HTTP response body.\n * 2. Store the decision in memory (Map). All future URLs on same domain skip re-testing.\n * 3. \"Requires browser\" signals: React/Vue/Angular root with no content, tiny body (<500 chars meaningful text).\n */\nexport class RenderingTypePredictor {\n  private cache = new Map<string, 'http' | 'browser'>();\n\n  /** Return cached decision if available, undefined if unknown. */\n  getCached(url: string): 'http' | 'browser' | undefined {\n    try {\n      return this.cache.get(new URL(url).hostname);\n    } catch {\n      return undefined;\n    }\n  }\n\n  /**\n   * Analyze an HTTP response body to predict if JS rendering is required.\n   * Returns 'browser' if JS-heavy signals detected, 'http' otherwise.\n   */\n  predict(url: string, htmlBody: string, statusCode: number): 'http' | 'browser' {\n    let decision: 'http' | 'browser' = 'http';\n    try {\n      if (statusCode >= 400) {\n        decision = 'browser'; // let browser handle error pages\n      } else {\n        const bodyLen = htmlBody.length;\n        // Strip tags to get raw text\n        const textOnly = htmlBody.replace(/<[^>]+>/g, ' ').replace(/\\s+/g, ' ').trim();\n        const wordCount = textOnly.split(/\\s+/).filter(w => w.length > 2).length;\n\n        // SPA signals: very short text, presence of React/Vue/Angular root markers\n        const hasSpaRoot = /<div[^>]*\\s+id=[\"'](?:root|app|__nuxt|__next|app-root)[\"'][^>]*>\\s*<\\/div>/i.test(htmlBody)\n          || /<app-root[^>]*>\\s*<\\/app-root>/i.test(htmlBody);\n        const hasNoscriptFallback = /<noscript[^>]*>[\\s\\S]{100,}/i.test(htmlBody);\n        const tinyBody = bodyLen < 3000 && wordCount < 50;\n\n        if ((hasSpaRoot && wordCount < 100) || (tinyBody && !hasNoscriptFallback)) {\n          decision = 'browser';\n        }\n      }\n    } catch { /* default http */ }\n\n    try {\n      this.cache.set(new URL(url).hostname, decision);\n    } catch { /* ignore */ }\n    return decision;\n  }\n\n  /** Force a domain to use a specific engine (e.g. after observing crawl errors). */\n  set(url: string, engine: 'http' | 'browser'): void {\n    try { this.cache.set(new URL(url).hostname, engine); } catch { /* ignore */ }\n  }\n\n  stats(): { cached: number; browserDomains: string[]; httpDomains: string[] } {\n    const browserDomains: string[] = [];\n    const httpDomains: string[] = [];\n    for (const [domain, engine] of this.cache) {\n      if (engine === 'browser') browserDomains.push(domain);\n      else httpDomains.push(domain);\n    }\n    return { cached: this.cache.size, browserDomains, httpDomains };\n  }\n}\n\n/** Singleton predictor \u2014 shared across crawl jobs in the same process. */\nexport const renderingPredictor = new RenderingTypePredictor();\n\nexport function chooseCrawlEngines(input: CrawlRouteDecisionInput): CrawlRouteDecision {\n  const engines: CrawlEngineName[] = [];\n  const reasons: string[] = [];\n\n  const appType = input.appType?.toLowerCase();\n  if (appType === 'android' || appType === 'ios' || appType === 'windows' || appType === 'macos') {\n    return { engines: ['native'], reasons: [`appType=${input.appType}`], confidence: 0.95 };\n  }\n\n  if (input.hasStorageState || input.hasAuthToken) {\n    engines.push('recorded-session');\n    reasons.push('session/token auth available');\n  }\n\n  engines.push(input.hasProxy ? 'browser-stealth' : 'browser');\n  reasons.push(input.hasProxy ? 'proxy configured; prefer stealth browser path' : 'default browser path');\n\n  if (input.hasCredentials || input.profile.industry !== 'unknown') {\n    engines.push('stagehand');\n    reasons.push(input.hasCredentials ? 'credentials available for AI login/navigation' : `domain profile=${input.profile.industry}`);\n  }\n\n  const hardUi = input.profile.industry === 'legacy-enterprise' || /captcha|blocked|canvas|iframe|bot/i.test(input.previousFailureReason ?? '');\n  if (input.hasSkyvern && hardUi) {\n    engines.push('vision');\n    reasons.push('hard UI or previous block detected; vision fallback enabled');\n  }\n\n  if (engines.length === 1) {\n    engines.unshift('http');\n    reasons.push('HTTP/static extraction can be attempted before browser escalation');\n  }\n\n  return {\n    engines: Array.from(new Set(engines)),\n    reasons,\n    confidence: input.profile.confidence,\n  };\n}\n"],
+  "mappings": "AAsCO,MAAM,uBAAuB;AAAA,EAC1B,QAAQ,oBAAI,IAAgC;AAAA;AAAA,EAGpD,UAAU,KAA6C;AACrD,QAAI;AACF,aAAO,KAAK,MAAM,IAAI,IAAI,IAAI,GAAG,EAAE,QAAQ;AAAA,IAC7C,QAAQ;AACN,aAAO;AAAA,IACT;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,QAAQ,KAAa,UAAkB,YAAwC;AAC7E,QAAI,WAA+B;AACnC,QAAI;AACF,UAAI,cAAc,KAAK;AACrB,mBAAW;AAAA,MACb,OAAO;AACL,cAAM,UAAU,SAAS;AAEzB,cAAM,WAAW,SAAS,QAAQ,YAAY,GAAG,EAAE,QAAQ,QAAQ,GAAG,EAAE,KAAK;AAC7E,cAAM,YAAY,SAAS,MAAM,KAAK,EAAE,OAAO,OAAK,EAAE,SAAS,CAAC,EAAE;AAGlE,cAAM,aAAa,8EAA8E,KAAK,QAAQ,KACzG,kCAAkC,KAAK,QAAQ;AACpD,cAAM,sBAAsB,+BAA+B,KAAK,QAAQ;AACxE,cAAM,WAAW,UAAU,OAAQ,YAAY;AAE/C,YAAK,cAAc,YAAY,OAAS,YAAY,CAAC,qBAAsB;AACzE,qBAAW;AAAA,QACb;AAAA,MACF;AAAA,IACF,QAAQ;AAAA,IAAqB;AAE7B,QAAI;AACF,WAAK,MAAM,IAAI,IAAI,IAAI,GAAG,EAAE,UAAU,QAAQ;AAAA,IAChD,QAAQ;AAAA,IAAe;AACvB,WAAO;AAAA,EACT;AAAA;AAAA,EAGA,IAAI,KAAa,QAAkC;AACjD,QAAI;AAAE,WAAK,MAAM,IAAI,IAAI,IAAI,GAAG,EAAE,UAAU,MAAM;AAAA,IAAG,QAAQ;AAAA,IAAe;AAAA,EAC9E;AAAA,EAEA,QAA6E;AAC3E,UAAM,iBAA2B,CAAC;AAClC,UAAM,cAAwB,CAAC;AAC/B,eAAW,CAAC,QAAQ,MAAM,KAAK,KAAK,OAAO;AACzC,UAAI,WAAW,UAAW,gBAAe,KAAK,MAAM;AAAA,UAC/C,aAAY,KAAK,MAAM;AAAA,IAC9B;AACA,WAAO,EAAE,QAAQ,KAAK,MAAM,MAAM,gBAAgB,YAAY;AAAA,EAChE;AACF;AAGO,MAAM,qBAAqB,IAAI,uBAAuB;AAEtD,SAAS,mBAAmB,OAAoD;AACrF,QAAM,UAA6B,CAAC;AACpC,QAAM,UAAoB,CAAC;AAE3B,QAAM,UAAU,MAAM,SAAS,YAAY;AAC3C,MAAI,YAAY,aAAa,YAAY,SAAS,YAAY,aAAa,YAAY,SAAS;AAC9F,WAAO,EAAE,SAAS,CAAC,QAAQ,GAAG,SAAS,CAAC,WAAW,MAAM,OAAO,EAAE,GAAG,YAAY,KAAK;AAAA,EACxF;AAEA,MAAI,MAAM,mBAAmB,MAAM,cAAc;AAC/C,YAAQ,KAAK,kBAAkB;AAC/B,YAAQ,KAAK,8BAA8B;AAAA,EAC7C;AAEA,UAAQ,KAAK,MAAM,WAAW,oBAAoB,SAAS;AAC3D,UAAQ,KAAK,MAAM,WAAW,kDAAkD,sBAAsB;AAEtG,MAAI,MAAM,kBAAkB,MAAM,QAAQ,aAAa,WAAW;AAChE,YAAQ,KAAK,WAAW;AACxB,YAAQ,KAAK,MAAM,iBAAiB,kDAAkD,kBAAkB,MAAM,QAAQ,QAAQ,EAAE;AAAA,EAClI;AAEA,QAAM,SAAS,MAAM,QAAQ,aAAa,uBAAuB,qCAAqC,KAAK,MAAM,yBAAyB,EAAE;AAC5I,MAAI,MAAM,cAAc,QAAQ;AAC9B,YAAQ,KAAK,QAAQ;AACrB,YAAQ,KAAK,6DAA6D;AAAA,EAC5E;AAEA,MAAI,QAAQ,WAAW,GAAG;AACxB,YAAQ,QAAQ,MAAM;AACtB,YAAQ,KAAK,mEAAmE;AAAA,EAClF;AAEA,SAAO;AAAA,IACL,SAAS,MAAM,KAAK,IAAI,IAAI,OAAO,CAAC;AAAA,IACpC;AAAA,IACA,YAAY,MAAM,QAAQ;AAAA,EAC5B;AACF;",
+  "names": []
+}

package/dist/export/bulk-export.d.ts

@@ -0,0 +1,26 @@
+/**
+ * Bulk export — export crawled screens to CSV / JSON / Markdown digest.
+ *
+ * Outputs:
+ * - CSV: url, name, wordCount, screenshotPath, markdownPath, createdAt — for spreadsheet analysis
+ * - JSON: full screen records array — for API integration
+ * - Markdown digest: concatenated markdown of all screens — for LLM context windows
+ */
+export interface ExportScreen {
+    id: string;
+    name: string;
+    url: string;
+    projectId: string;
+    wordCount?: number;
+    screenshotPath?: string | null;
+    markdownPath?: string | null;
+    markdown?: string | null;
+    createdAt?: Date;
+    updatedAt?: Date;
+}
+export declare function exportToCsv(screens: ExportScreen[], outputPath: string): Promise<void>;
+export declare function exportToJson(screens: ExportScreen[], outputPath: string): Promise<void>;
+export declare function exportToMarkdownDigest(screens: ExportScreen[], outputPath: string, opts?: {
+    readMarkdownFromPath?: boolean;
+    maxScreens?: number;
+}): Promise<void>;

package/dist/export/bulk-export.js

@@ -0,0 +1,76 @@
+import * as fs from "fs/promises";
+import * as path from "path";
+function escapeCsv(val) {
+  if (val.includes(",") || val.includes('"') || val.includes("\n")) {
+    return `"${val.replace(/"/g, '""')}"`;
+  }
+  return val;
+}
+async function exportToCsv(screens, outputPath) {
+  const headers = ["id", "name", "url", "projectId", "wordCount", "screenshotPath", "markdownPath", "createdAt"];
+  const rows = screens.map((s) => [
+    s.id,
+    s.name,
+    s.url,
+    s.projectId,
+    String(s.wordCount ?? 0),
+    s.screenshotPath ?? "",
+    s.markdownPath ?? "",
+    s.createdAt?.toISOString() ?? ""
+  ].map(escapeCsv).join(","));
+  await fs.mkdir(path.dirname(outputPath), { recursive: true });
+  await fs.writeFile(outputPath, [headers.join(","), ...rows].join("\n"), "utf8");
+}
+async function exportToJson(screens, outputPath) {
+  await fs.mkdir(path.dirname(outputPath), { recursive: true });
+  const data = screens.map((s) => ({
+    id: s.id,
+    name: s.name,
+    url: s.url,
+    projectId: s.projectId,
+    wordCount: s.wordCount,
+    screenshotPath: s.screenshotPath,
+    markdownPath: s.markdownPath,
+    createdAt: s.createdAt?.toISOString(),
+    updatedAt: s.updatedAt?.toISOString()
+  }));
+  await fs.writeFile(outputPath, JSON.stringify(data, null, 2), "utf8");
+}
+async function exportToMarkdownDigest(screens, outputPath, opts = {}) {
+  const { readMarkdownFromPath = true, maxScreens = 200 } = opts;
+  await fs.mkdir(path.dirname(outputPath), { recursive: true });
+  const handle = await fs.open(outputPath, "w");
+  try {
+    await handle.write(`# ZeTa Crawl Digest
+
+Exported: ${(/* @__PURE__ */ new Date()).toISOString()}
+Screens: ${Math.min(screens.length, maxScreens)}
+
+---
+
+`);
+    for (const screen of screens.slice(0, maxScreens)) {
+      let md = screen.markdown ?? "";
+      if (!md && readMarkdownFromPath && screen.markdownPath) {
+        md = await fs.readFile(screen.markdownPath, "utf8").catch(() => "");
+      }
+      await handle.write(`## ${screen.name}
+
+**URL:** ${screen.url}
+
+${md.trim()}
+
+---
+
+`);
+    }
+  } finally {
+    await handle.close();
+  }
+}
+export {
+  exportToCsv,
+  exportToJson,
+  exportToMarkdownDigest
+};
+//# sourceMappingURL=bulk-export.js.map

package/dist/export/bulk-export.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../../src/export/bulk-export.ts"],
+  "sourcesContent": ["/**\n * Bulk export \u2014 export crawled screens to CSV / JSON / Markdown digest.\n *\n * Outputs:\n * - CSV: url, name, wordCount, screenshotPath, markdownPath, createdAt \u2014 for spreadsheet analysis\n * - JSON: full screen records array \u2014 for API integration\n * - Markdown digest: concatenated markdown of all screens \u2014 for LLM context windows\n */\n\nimport * as fs from 'fs/promises';\nimport * as path from 'path';\n\nexport interface ExportScreen {\n  id: string;\n  name: string;\n  url: string;\n  projectId: string;\n  wordCount?: number;\n  screenshotPath?: string | null;\n  markdownPath?: string | null;\n  markdown?: string | null;\n  createdAt?: Date;\n  updatedAt?: Date;\n}\n\nfunction escapeCsv(val: string): string {\n  if (val.includes(',') || val.includes('\"') || val.includes('\\n')) {\n    return `\"${val.replace(/\"/g, '\"\"')}\"`;\n  }\n  return val;\n}\n\nexport async function exportToCsv(screens: ExportScreen[], outputPath: string): Promise<void> {\n  const headers = ['id', 'name', 'url', 'projectId', 'wordCount', 'screenshotPath', 'markdownPath', 'createdAt'];\n  const rows = screens.map(s => [\n    s.id, s.name, s.url, s.projectId,\n    String(s.wordCount ?? 0),\n    s.screenshotPath ?? '',\n    s.markdownPath ?? '',\n    s.createdAt?.toISOString() ?? '',\n  ].map(escapeCsv).join(','));\n\n  await fs.mkdir(path.dirname(outputPath), { recursive: true });\n  await fs.writeFile(outputPath, [headers.join(','), ...rows].join('\\n'), 'utf8');\n}\n\nexport async function exportToJson(screens: ExportScreen[], outputPath: string): Promise<void> {\n  await fs.mkdir(path.dirname(outputPath), { recursive: true });\n  const data = screens.map(s => ({\n    id: s.id,\n    name: s.name,\n    url: s.url,\n    projectId: s.projectId,\n    wordCount: s.wordCount,\n    screenshotPath: s.screenshotPath,\n    markdownPath: s.markdownPath,\n    createdAt: s.createdAt?.toISOString(),\n    updatedAt: s.updatedAt?.toISOString(),\n  }));\n  await fs.writeFile(outputPath, JSON.stringify(data, null, 2), 'utf8');\n}\n\nexport async function exportToMarkdownDigest(\n  screens: ExportScreen[],\n  outputPath: string,\n  opts: { readMarkdownFromPath?: boolean; maxScreens?: number } = {},\n): Promise<void> {\n  const { readMarkdownFromPath = true, maxScreens = 200 } = opts;\n  await fs.mkdir(path.dirname(outputPath), { recursive: true });\n\n  const handle = await fs.open(outputPath, 'w');\n  try {\n    await handle.write(`# ZeTa Crawl Digest\\n\\nExported: ${new Date().toISOString()}\\nScreens: ${Math.min(screens.length, maxScreens)}\\n\\n---\\n\\n`);\n\n    for (const screen of screens.slice(0, maxScreens)) {\n      let md = screen.markdown ?? '';\n      if (!md && readMarkdownFromPath && screen.markdownPath) {\n        md = await fs.readFile(screen.markdownPath, 'utf8').catch(() => '');\n      }\n      await handle.write(`## ${screen.name}\\n\\n**URL:** ${screen.url}\\n\\n${md.trim()}\\n\\n---\\n\\n`);\n    }\n  } finally {\n    await handle.close();\n  }\n}\n"],
+  "mappings": "AASA,YAAY,QAAQ;AACpB,YAAY,UAAU;AAetB,SAAS,UAAU,KAAqB;AACtC,MAAI,IAAI,SAAS,GAAG,KAAK,IAAI,SAAS,GAAG,KAAK,IAAI,SAAS,IAAI,GAAG;AAChE,WAAO,IAAI,IAAI,QAAQ,MAAM,IAAI,CAAC;AAAA,EACpC;AACA,SAAO;AACT;AAEA,eAAsB,YAAY,SAAyB,YAAmC;AAC5F,QAAM,UAAU,CAAC,MAAM,QAAQ,OAAO,aAAa,aAAa,kBAAkB,gBAAgB,WAAW;AAC7G,QAAM,OAAO,QAAQ,IAAI,OAAK;AAAA,IAC5B,EAAE;AAAA,IAAI,EAAE;AAAA,IAAM,EAAE;AAAA,IAAK,EAAE;AAAA,IACvB,OAAO,EAAE,aAAa,CAAC;AAAA,IACvB,EAAE,kBAAkB;AAAA,IACpB,EAAE,gBAAgB;AAAA,IAClB,EAAE,WAAW,YAAY,KAAK;AAAA,EAChC,EAAE,IAAI,SAAS,EAAE,KAAK,GAAG,CAAC;AAE1B,QAAM,GAAG,MAAM,KAAK,QAAQ,UAAU,GAAG,EAAE,WAAW,KAAK,CAAC;AAC5D,QAAM,GAAG,UAAU,YAAY,CAAC,QAAQ,KAAK,GAAG,GAAG,GAAG,IAAI,EAAE,KAAK,IAAI,GAAG,MAAM;AAChF;AAEA,eAAsB,aAAa,SAAyB,YAAmC;AAC7F,QAAM,GAAG,MAAM,KAAK,QAAQ,UAAU,GAAG,EAAE,WAAW,KAAK,CAAC;AAC5D,QAAM,OAAO,QAAQ,IAAI,QAAM;AAAA,IAC7B,IAAI,EAAE;AAAA,IACN,MAAM,EAAE;AAAA,IACR,KAAK,EAAE;AAAA,IACP,WAAW,EAAE;AAAA,IACb,WAAW,EAAE;AAAA,IACb,gBAAgB,EAAE;AAAA,IAClB,cAAc,EAAE;AAAA,IAChB,WAAW,EAAE,WAAW,YAAY;AAAA,IACpC,WAAW,EAAE,WAAW,YAAY;AAAA,EACtC,EAAE;AACF,QAAM,GAAG,UAAU,YAAY,KAAK,UAAU,MAAM,MAAM,CAAC,GAAG,MAAM;AACtE;AAEA,eAAsB,uBACpB,SACA,YACA,OAAgE,CAAC,GAClD;AACf,QAAM,EAAE,uBAAuB,MAAM,aAAa,IAAI,IAAI;AAC1D,QAAM,GAAG,MAAM,KAAK,QAAQ,UAAU,GAAG,EAAE,WAAW,KAAK,CAAC;AAE5D,QAAM,SAAS,MAAM,GAAG,KAAK,YAAY,GAAG;AAC5C,MAAI;AACF,UAAM,OAAO,MAAM;AAAA;AAAA,aAAoC,oBAAI,KAAK,GAAE,YAAY,CAAC;AAAA,WAAc,KAAK,IAAI,QAAQ,QAAQ,UAAU,CAAC;AAAA;AAAA;AAAA;AAAA,CAAa;AAE9I,eAAW,UAAU,QAAQ,MAAM,GAAG,UAAU,GAAG;AACjD,UAAI,KAAK,OAAO,YAAY;AAC5B,UAAI,CAAC,MAAM,wBAAwB,OAAO,cAAc;AACtD,aAAK,MAAM,GAAG,SAAS,OAAO,cAAc,MAAM,EAAE,MAAM,MAAM,EAAE;AAAA,MACpE;AACA,YAAM,OAAO,MAAM,MAAM,OAAO,IAAI;AAAA;AAAA,WAAgB,OAAO,GAAG;AAAA;AAAA,EAAO,GAAG,KAAK,CAAC;AAAA;AAAA;AAAA;AAAA,CAAa;AAAA,IAC7F;AAAA,EACF,UAAE;AACA,UAAM,OAAO,MAAM;AAAA,EACrB;AACF;",
+  "names": []
+}

package/dist/export/vector-export.d.ts

@@ -0,0 +1,76 @@
+/**
+ * Vector DB export — convert crawled screens to embedding-ready chunks.
+ *
+ * Produces a JSONL file where each line is one chunk ready for upsert into
+ * Pinecone, Qdrant, Weaviate, or any vector store. The embedding itself is
+ * NOT generated here (requires an embedding model at the user's end) — we
+ * produce the text chunks + metadata, and the caller embeds them.
+ *
+ * Format per line (JSONL):
+ * {
+ *   "id": "<screenId>-<chunkIndex>",
+ *   "text": "<chunk text>",
+ *   "metadata": {
+ *     "screenId": "...",
+ *     "projectId": "...",
+ *     "tenantId": "...",
+ *     "url": "...",
+ *     "screenName": "...",
+ *     "chunkIndex": 0,
+ *     "totalChunks": 3,
+ *     "crawledAt": "2026-...",
+ *     "wordCount": 142
+ *   }
+ * }
+ */
+export interface VectorChunk {
+    id: string;
+    text: string;
+    metadata: {
+        screenId: string;
+        projectId: string;
+        tenantId: string;
+        url: string;
+        screenName: string;
+        chunkIndex: number;
+        totalChunks: number;
+        crawledAt: string;
+        wordCount: number;
+    };
+}
+export interface ScreenRecord {
+    id: string;
+    name: string;
+    url: string;
+    projectId: string;
+    tenantId: string;
+    markdown?: string | null;
+    markdownPath?: string | null;
+    updatedAt?: Date;
+}
+export declare function exportScreensToVectorJsonl(screens: ScreenRecord[], outputPath: string, opts?: {
+    maxChunkWords?: number;
+    readMarkdownFromPath?: boolean;
+}): Promise<{
+    totalChunks: number;
+    skipped: number;
+}>;
+/**
+ * Pinecone-compatible upsert batch format.
+ * Call this after generating embeddings to produce the final upsert payload.
+ */
+export declare function toPineconeUpsert(chunks: VectorChunk[], embeddings: number[][], namespace?: string): Array<{
+    id: string;
+    values: number[];
+    metadata: VectorChunk['metadata'];
+}>;
+/**
+ * Qdrant-compatible point format.
+ */
+export declare function toQdrantPoints(chunks: VectorChunk[], embeddings: number[][]): Array<{
+    id: string;
+    vector: number[];
+    payload: VectorChunk['metadata'] & {
+        text: string;
+    };
+}>;

package/dist/export/vector-export.js

@@ -0,0 +1,85 @@
+import * as fs from "fs/promises";
+import * as path from "path";
+function splitIntoChunks(text, maxChunkWords = 300) {
+  const paragraphs = text.split(/\n{2,}/).filter((p) => p.trim().length > 0);
+  const chunks = [];
+  let current = "";
+  let wordCount = 0;
+  for (const para of paragraphs) {
+    const words = para.split(/\s+/).length;
+    if (wordCount + words > maxChunkWords && current.length > 0) {
+      chunks.push(current.trim());
+      current = "";
+      wordCount = 0;
+    }
+    current += (current ? "\n\n" : "") + para;
+    wordCount += words;
+  }
+  if (current.trim()) chunks.push(current.trim());
+  return chunks.length > 0 ? chunks : [text.slice(0, 2e3)];
+}
+async function exportScreensToVectorJsonl(screens, outputPath, opts = {}) {
+  const { maxChunkWords = 300, readMarkdownFromPath = true } = opts;
+  await fs.mkdir(path.dirname(outputPath), { recursive: true });
+  const handle = await fs.open(outputPath, "w");
+  let totalChunks = 0;
+  let skipped = 0;
+  try {
+    for (const screen of screens) {
+      let markdown = screen.markdown ?? "";
+      if (!markdown && readMarkdownFromPath && screen.markdownPath) {
+        try {
+          markdown = await fs.readFile(screen.markdownPath, "utf8");
+        } catch {
+        }
+      }
+      if (!markdown || markdown.trim().length < 20) {
+        skipped++;
+        continue;
+      }
+      const textChunks = splitIntoChunks(markdown, maxChunkWords);
+      for (let i = 0; i < textChunks.length; i++) {
+        const chunk = {
+          id: `${screen.id}-${i}`,
+          text: textChunks[i],
+          metadata: {
+            screenId: screen.id,
+            projectId: screen.projectId,
+            tenantId: screen.tenantId,
+            url: screen.url,
+            screenName: screen.name,
+            chunkIndex: i,
+            totalChunks: textChunks.length,
+            crawledAt: screen.updatedAt?.toISOString() ?? (/* @__PURE__ */ new Date()).toISOString(),
+            wordCount: textChunks[i].split(/\s+/).length
+          }
+        };
+        await handle.write(JSON.stringify(chunk) + "\n");
+        totalChunks++;
+      }
+    }
+  } finally {
+    await handle.close();
+  }
+  return { totalChunks, skipped };
+}
+function toPineconeUpsert(chunks, embeddings, namespace = "zeta-crawl") {
+  return chunks.map((chunk, i) => ({
+    id: chunk.id,
+    values: embeddings[i],
+    metadata: chunk.metadata
+  }));
+}
+function toQdrantPoints(chunks, embeddings) {
+  return chunks.map((chunk, i) => ({
+    id: chunk.id,
+    vector: embeddings[i],
+    payload: { ...chunk.metadata, text: chunk.text }
+  }));
+}
+export {
+  exportScreensToVectorJsonl,
+  toPineconeUpsert,
+  toQdrantPoints
+};
+//# sourceMappingURL=vector-export.js.map

package/dist/export/vector-export.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../../src/export/vector-export.ts"],
+  "sourcesContent": ["/**\n * Vector DB export \u2014 convert crawled screens to embedding-ready chunks.\n *\n * Produces a JSONL file where each line is one chunk ready for upsert into\n * Pinecone, Qdrant, Weaviate, or any vector store. The embedding itself is\n * NOT generated here (requires an embedding model at the user's end) \u2014 we\n * produce the text chunks + metadata, and the caller embeds them.\n *\n * Format per line (JSONL):\n * {\n *   \"id\": \"<screenId>-<chunkIndex>\",\n *   \"text\": \"<chunk text>\",\n *   \"metadata\": {\n *     \"screenId\": \"...\",\n *     \"projectId\": \"...\",\n *     \"tenantId\": \"...\",\n *     \"url\": \"...\",\n *     \"screenName\": \"...\",\n *     \"chunkIndex\": 0,\n *     \"totalChunks\": 3,\n *     \"crawledAt\": \"2026-...\",\n *     \"wordCount\": 142\n *   }\n * }\n */\n\nimport * as fs from 'fs/promises';\nimport * as path from 'path';\n\nexport interface VectorChunk {\n  id: string;\n  text: string;\n  metadata: {\n    screenId: string;\n    projectId: string;\n    tenantId: string;\n    url: string;\n    screenName: string;\n    chunkIndex: number;\n    totalChunks: number;\n    crawledAt: string;\n    wordCount: number;\n  };\n}\n\nexport interface ScreenRecord {\n  id: string;\n  name: string;\n  url: string;\n  projectId: string;\n  tenantId: string;\n  markdown?: string | null;\n  markdownPath?: string | null;\n  updatedAt?: Date;\n}\n\nfunction splitIntoChunks(text: string, maxChunkWords = 300): string[] {\n  const paragraphs = text.split(/\\n{2,}/).filter(p => p.trim().length > 0);\n  const chunks: string[] = [];\n  let current = '';\n  let wordCount = 0;\n\n  for (const para of paragraphs) {\n    const words = para.split(/\\s+/).length;\n    if (wordCount + words > maxChunkWords && current.length > 0) {\n      chunks.push(current.trim());\n      current = '';\n      wordCount = 0;\n    }\n    current += (current ? '\\n\\n' : '') + para;\n    wordCount += words;\n  }\n  if (current.trim()) chunks.push(current.trim());\n  return chunks.length > 0 ? chunks : [text.slice(0, 2000)];\n}\n\nexport async function exportScreensToVectorJsonl(\n  screens: ScreenRecord[],\n  outputPath: string,\n  opts: { maxChunkWords?: number; readMarkdownFromPath?: boolean } = {},\n): Promise<{ totalChunks: number; skipped: number }> {\n  const { maxChunkWords = 300, readMarkdownFromPath = true } = opts;\n  await fs.mkdir(path.dirname(outputPath), { recursive: true });\n\n  const handle = await fs.open(outputPath, 'w');\n  let totalChunks = 0;\n  let skipped = 0;\n\n  try {\n    for (const screen of screens) {\n      let markdown = screen.markdown ?? '';\n\n      if (!markdown && readMarkdownFromPath && screen.markdownPath) {\n        try {\n          markdown = await fs.readFile(screen.markdownPath, 'utf8');\n        } catch { /* file missing \u2014 skip */ }\n      }\n\n      if (!markdown || markdown.trim().length < 20) {\n        skipped++;\n        continue;\n      }\n\n      const textChunks = splitIntoChunks(markdown, maxChunkWords);\n      for (let i = 0; i < textChunks.length; i++) {\n        const chunk: VectorChunk = {\n          id: `${screen.id}-${i}`,\n          text: textChunks[i],\n          metadata: {\n            screenId: screen.id,\n            projectId: screen.projectId,\n            tenantId: screen.tenantId,\n            url: screen.url,\n            screenName: screen.name,\n            chunkIndex: i,\n            totalChunks: textChunks.length,\n            crawledAt: screen.updatedAt?.toISOString() ?? new Date().toISOString(),\n            wordCount: textChunks[i].split(/\\s+/).length,\n          },\n        };\n        await handle.write(JSON.stringify(chunk) + '\\n');\n        totalChunks++;\n      }\n    }\n  } finally {\n    await handle.close();\n  }\n\n  return { totalChunks, skipped };\n}\n\n/**\n * Pinecone-compatible upsert batch format.\n * Call this after generating embeddings to produce the final upsert payload.\n */\nexport function toPineconeUpsert(\n  chunks: VectorChunk[],\n  embeddings: number[][],\n  namespace = 'zeta-crawl',\n): Array<{ id: string; values: number[]; metadata: VectorChunk['metadata'] }> {\n  return chunks.map((chunk, i) => ({\n    id: chunk.id,\n    values: embeddings[i],\n    metadata: chunk.metadata,\n  }));\n}\n\n/**\n * Qdrant-compatible point format.\n */\nexport function toQdrantPoints(\n  chunks: VectorChunk[],\n  embeddings: number[][],\n): Array<{ id: string; vector: number[]; payload: VectorChunk['metadata'] & { text: string } }> {\n  return chunks.map((chunk, i) => ({\n    id: chunk.id,\n    vector: embeddings[i],\n    payload: { ...chunk.metadata, text: chunk.text },\n  }));\n}\n"],
+  "mappings": "AA0BA,YAAY,QAAQ;AACpB,YAAY,UAAU;AA6BtB,SAAS,gBAAgB,MAAc,gBAAgB,KAAe;AACpE,QAAM,aAAa,KAAK,MAAM,QAAQ,EAAE,OAAO,OAAK,EAAE,KAAK,EAAE,SAAS,CAAC;AACvE,QAAM,SAAmB,CAAC;AAC1B,MAAI,UAAU;AACd,MAAI,YAAY;AAEhB,aAAW,QAAQ,YAAY;AAC7B,UAAM,QAAQ,KAAK,MAAM,KAAK,EAAE;AAChC,QAAI,YAAY,QAAQ,iBAAiB,QAAQ,SAAS,GAAG;AAC3D,aAAO,KAAK,QAAQ,KAAK,CAAC;AAC1B,gBAAU;AACV,kBAAY;AAAA,IACd;AACA,gBAAY,UAAU,SAAS,MAAM;AACrC,iBAAa;AAAA,EACf;AACA,MAAI,QAAQ,KAAK,EAAG,QAAO,KAAK,QAAQ,KAAK,CAAC;AAC9C,SAAO,OAAO,SAAS,IAAI,SAAS,CAAC,KAAK,MAAM,GAAG,GAAI,CAAC;AAC1D;AAEA,eAAsB,2BACpB,SACA,YACA,OAAmE,CAAC,GACjB;AACnD,QAAM,EAAE,gBAAgB,KAAK,uBAAuB,KAAK,IAAI;AAC7D,QAAM,GAAG,MAAM,KAAK,QAAQ,UAAU,GAAG,EAAE,WAAW,KAAK,CAAC;AAE5D,QAAM,SAAS,MAAM,GAAG,KAAK,YAAY,GAAG;AAC5C,MAAI,cAAc;AAClB,MAAI,UAAU;AAEd,MAAI;AACF,eAAW,UAAU,SAAS;AAC5B,UAAI,WAAW,OAAO,YAAY;AAElC,UAAI,CAAC,YAAY,wBAAwB,OAAO,cAAc;AAC5D,YAAI;AACF,qBAAW,MAAM,GAAG,SAAS,OAAO,cAAc,MAAM;AAAA,QAC1D,QAAQ;AAAA,QAA4B;AAAA,MACtC;AAEA,UAAI,CAAC,YAAY,SAAS,KAAK,EAAE,SAAS,IAAI;AAC5C;AACA;AAAA,MACF;AAEA,YAAM,aAAa,gBAAgB,UAAU,aAAa;AAC1D,eAAS,IAAI,GAAG,IAAI,WAAW,QAAQ,KAAK;AAC1C,cAAM,QAAqB;AAAA,UACzB,IAAI,GAAG,OAAO,EAAE,IAAI,CAAC;AAAA,UACrB,MAAM,WAAW,CAAC;AAAA,UAClB,UAAU;AAAA,YACR,UAAU,OAAO;AAAA,YACjB,WAAW,OAAO;AAAA,YAClB,UAAU,OAAO;AAAA,YACjB,KAAK,OAAO;AAAA,YACZ,YAAY,OAAO;AAAA,YACnB,YAAY;AAAA,YACZ,aAAa,WAAW;AAAA,YACxB,WAAW,OAAO,WAAW,YAAY,MAAK,oBAAI,KAAK,GAAE,YAAY;AAAA,YACrE,WAAW,WAAW,CAAC,EAAE,MAAM,KAAK,EAAE;AAAA,UACxC;AAAA,QACF;AACA,cAAM,OAAO,MAAM,KAAK,UAAU,KAAK,IAAI,IAAI;AAC/C;AAAA,MACF;AAAA,IACF;AAAA,EACF,UAAE;AACA,UAAM,OAAO,MAAM;AAAA,EACrB;AAEA,SAAO,EAAE,aAAa,QAAQ;AAChC;AAMO,SAAS,iBACd,QACA,YACA,YAAY,cACgE;AAC5E,SAAO,OAAO,IAAI,CAAC,OAAO,OAAO;AAAA,IAC/B,IAAI,MAAM;AAAA,IACV,QAAQ,WAAW,CAAC;AAAA,IACpB,UAAU,MAAM;AAAA,EAClB,EAAE;AACJ;AAKO,SAAS,eACd,QACA,YAC8F;AAC9F,SAAO,OAAO,IAAI,CAAC,OAAO,OAAO;AAAA,IAC/B,IAAI,MAAM;AAAA,IACV,QAAQ,WAAW,CAAC;AAAA,IACpB,SAAS,EAAE,GAAG,MAAM,UAAU,MAAM,MAAM,KAAK;AAAA,EACjD,EAAE;AACJ;",
+  "names": []
+}

package/dist/extractors/advanced.d.ts

@@ -0,0 +1,54 @@
+export interface ExtractionResult {
+    matches: string[];
+    count: number;
+    method: string;
+}
+export type TableRow = Record<string, string>;
+export interface PageMetadata {
+    title: string | null;
+    description: string | null;
+    keywords: string | null;
+    ogTitle: string | null;
+    ogDescription: string | null;
+    ogImage: string | null;
+    ogUrl: string | null;
+    ogType: string | null;
+    twitterCard: string | null;
+    twitterTitle: string | null;
+    twitterDescription: string | null;
+    canonical: string | null;
+    robots: string | null;
+    jsonLd: object[];
+    schemaTypes: string[];
+    lang: string | null;
+}
+export declare function extractByCssSelector(html: string, selector: string, attr?: string): ExtractionResult;
+export declare function extractByXPath(html: string, xpath: string): ExtractionResult;
+export declare function extractByRegex(text: string, pattern: string, flags?: string): ExtractionResult;
+export declare function extractTables(html: string): TableRow[][];
+export declare function extractMetadata(html: string, url: string): PageMetadata;
+export interface ProductData {
+    name: string | null;
+    description: string | null;
+    brand: string | null;
+    sku: string | null;
+    price: number | null;
+    priceCurrency: string | null;
+    availability: string | null;
+    rating: number | null;
+    reviewCount: number | null;
+    image: string | null;
+    url: string | null;
+}
+export declare function extractStructuredProduct(jsonLd: object[]): ProductData | null;
+export declare function extractImages(html: string, baseUrl: string): Array<{
+    src: string;
+    alt: string;
+    width?: number;
+    height?: number;
+}>;
+export declare function extractMediaFiles(html: string, baseUrl: string): Array<{
+    url: string;
+    type: 'image' | 'video' | 'audio' | 'document';
+    tag: string;
+}>;