@detiq/crawler 1.1.7 → 1.1.9

package/dist/session-export.js

@@ -1,117 +1,143 @@
+/**
+ * Converts a Playwright storageState (the format ZeTa already captures via
+ * session-capture.ts/live-view.ts and stores per-project) into snippets for
+ * other automation frameworks. Playwright's storageState is the de-facto
+ * standard for browser session export — see docs/UNIVERSAL-SESSION-CAPTURE-
+ * STRATEGY.md — so every other framework's import mechanism is a small,
+ * mechanical conversion away, not a new capture flow.
+ *
+ * Pure, dependency-free functions — no I/O, no network, fully unit-testable.
+ */
 function jsStringLiteral(s) {
-  return JSON.stringify(s);
+    return JSON.stringify(s);
 }
+/** Python dict-literal repr — JSON.stringify's `true`/`false`/`null` aren't
+ *  valid Python (Python needs `True`/`False`/`None`), so a plain
+ *  JSON.stringify would emit syntactically invalid Python for any cookie
+ *  with a boolean field. */
 function pythonDictLiteral(obj) {
-  const parts = Object.entries(obj).map(([k, v]) => {
-    let repr;
-    if (typeof v === "boolean") repr = v ? "True" : "False";
-    else if (v === null || v === void 0) repr = "None";
-    else if (typeof v === "number") repr = String(v);
-    else repr = JSON.stringify(v);
-    return `${JSON.stringify(k)}: ${repr}`;
-  });
-  return `{${parts.join(", ")}}`;
+    const parts = Object.entries(obj).map(([k, v]) => {
+        let repr;
+        if (typeof v === 'boolean')
+            repr = v ? 'True' : 'False';
+        else if (v === null || v === undefined)
+            repr = 'None';
+        else if (typeof v === 'number')
+            repr = String(v);
+        else
+            repr = JSON.stringify(v);
+        return `${JSON.stringify(k)}: ${repr}`;
+    });
+    return `{${parts.join(', ')}}`;
 }
-function toCypressSnippet(state) {
-  const cookieLines = state.cookies.map((c) => {
-    const opts = [`domain: ${jsStringLiteral(c.domain)}`, `path: ${jsStringLiteral(c.path)}`];
-    if (c.secure) opts.push("secure: true");
-    if (c.httpOnly) opts.push("httpOnly: true");
-    if (c.sameSite) opts.push(`sameSite: ${jsStringLiteral(c.sameSite.toLowerCase())}`);
-    return `  cy.setCookie(${jsStringLiteral(c.name)}, ${jsStringLiteral(c.value)}, { ${opts.join(", ")} });`;
-  });
-  const localStorageBlocks = state.origins.filter((o) => o.localStorage.length > 0).map((o) => {
-    const sets = o.localStorage.map((kv) => `      window.localStorage.setItem(${jsStringLiteral(kv.name)}, ${jsStringLiteral(kv.value)});`).join("\n");
-    return `  cy.visit(${jsStringLiteral(o.origin)});
-  cy.window().then((window) => {
-${sets}
-  });`;
-  });
-  return [
-    "// Generated by ZeTa-AI \u2014 restores a captured browser session in Cypress.",
-    "// Usage: import this file and call restoreZetaSession() inside cy.session(), e.g.:",
-    "//   cy.session('zeta-session', restoreZetaSession)",
-    "Cypress.Commands.add('restoreZetaSession', () => {",
-    ...cookieLines,
-    ...localStorageBlocks,
-    "});"
-  ].join("\n");
+/** cy.setCookie() + localStorage.setItem() inside a reusable custom command,
+ *  following Cypress's documented cy.session() pattern (see references in
+ *  docs/UNIVERSAL-SESSION-CAPTURE-STRATEGY.md). */
+export function toCypressSnippet(state) {
+    const cookieLines = state.cookies.map((c) => {
+        const opts = [`domain: ${jsStringLiteral(c.domain)}`, `path: ${jsStringLiteral(c.path)}`];
+        if (c.secure)
+            opts.push('secure: true');
+        if (c.httpOnly)
+            opts.push('httpOnly: true');
+        if (c.sameSite)
+            opts.push(`sameSite: ${jsStringLiteral(c.sameSite.toLowerCase())}`);
+        return `  cy.setCookie(${jsStringLiteral(c.name)}, ${jsStringLiteral(c.value)}, { ${opts.join(', ')} });`;
+    });
+    const localStorageBlocks = state.origins
+        .filter((o) => o.localStorage.length > 0)
+        .map((o) => {
+        const sets = o.localStorage
+            .map((kv) => `      window.localStorage.setItem(${jsStringLiteral(kv.name)}, ${jsStringLiteral(kv.value)});`)
+            .join('\n');
+        return `  cy.visit(${jsStringLiteral(o.origin)});\n  cy.window().then((window) => {\n${sets}\n  });`;
+    });
+    return [
+        '// Generated by ZeTa-AI — restores a captured browser session in Cypress.',
+        '// Usage: import this file and call restoreZetaSession() inside cy.session(), e.g.:',
+        "//   cy.session('zeta-session', restoreZetaSession)",
+        "Cypress.Commands.add('restoreZetaSession', () => {",
+        ...cookieLines,
+        ...localStorageBlocks,
+        '});',
+    ].join('\n');
 }
-function toSeleniumPythonDict(state) {
-  const byDomain = /* @__PURE__ */ new Map();
-  for (const c of state.cookies) {
-    const list = byDomain.get(c.domain) ?? [];
-    list.push(c);
-    byDomain.set(c.domain, list);
-  }
-  const lines = [
-    "# Generated by ZeTa-AI \u2014 restores a captured browser session in Selenium.",
-    "# Selenium requires the driver to already be on a matching domain before",
-    "# add_cookie() will accept a cookie for it \u2014 navigate first, then restore.",
-    "def restore_zeta_session(driver):"
-  ];
-  for (const [domain, cookies] of byDomain) {
-    lines.push(`    driver.get(${JSON.stringify(`https://${domain.replace(/^\./, "")}`)})`);
-    for (const c of cookies) {
-      const cookieDict = { name: c.name, value: c.value, path: c.path };
-      if (c.expires && c.expires > 0) cookieDict.expiry = Math.floor(c.expires);
-      if (c.secure) cookieDict.secure = true;
-      lines.push(`    driver.add_cookie(${pythonDictLiteral(cookieDict)})`);
+/** driver.add_cookie() (Selenium requires the browser already be on the
+ *  cookie's domain before adding it) + execute_script() for localStorage,
+ *  per Selenium's documented Cookie API. */
+export function toSeleniumPythonDict(state) {
+    const byDomain = new Map();
+    for (const c of state.cookies) {
+        const list = byDomain.get(c.domain) ?? [];
+        list.push(c);
+        byDomain.set(c.domain, list);
     }
-  }
-  for (const o of state.origins) {
-    if (o.localStorage.length === 0) continue;
-    lines.push(`    driver.get(${JSON.stringify(o.origin)})`);
-    for (const kv of o.localStorage) {
-      lines.push(
-        `    driver.execute_script(${JSON.stringify(`window.localStorage.setItem(arguments[0], arguments[1]);`)}, ${JSON.stringify(kv.name)}, ${JSON.stringify(kv.value)})`
-      );
+    const lines = [
+        '# Generated by ZeTa-AI — restores a captured browser session in Selenium.',
+        '# Selenium requires the driver to already be on a matching domain before',
+        '# add_cookie() will accept a cookie for it — navigate first, then restore.',
+        'def restore_zeta_session(driver):',
+    ];
+    for (const [domain, cookies] of byDomain) {
+        lines.push(`    driver.get(${JSON.stringify(`https://${domain.replace(/^\./, '')}`)})`);
+        for (const c of cookies) {
+            const cookieDict = { name: c.name, value: c.value, path: c.path };
+            if (c.expires && c.expires > 0)
+                cookieDict.expiry = Math.floor(c.expires);
+            if (c.secure)
+                cookieDict.secure = true;
+            lines.push(`    driver.add_cookie(${pythonDictLiteral(cookieDict)})`);
+        }
     }
-  }
-  return lines.join("\n");
+    for (const o of state.origins) {
+        if (o.localStorage.length === 0)
+            continue;
+        lines.push(`    driver.get(${JSON.stringify(o.origin)})`);
+        for (const kv of o.localStorage) {
+            lines.push(`    driver.execute_script(${JSON.stringify(`window.localStorage.setItem(arguments[0], arguments[1]);`)}, ${JSON.stringify(kv.name)}, ${JSON.stringify(kv.value)})`);
+        }
+    }
+    return lines.join('\n');
 }
-function toWebdriverIOSnippet(state) {
-  const cookies = state.cookies.map((c) => ({
-    name: c.name,
-    value: c.value,
-    domain: c.domain,
-    path: c.path,
-    secure: !!c.secure,
-    httpOnly: !!c.httpOnly,
-    ...c.expires && c.expires > 0 ? { expiry: Math.floor(c.expires) } : {}
-  }));
-  const localStorageSets = state.origins.filter((o) => o.localStorage.length > 0).map((o) => {
-    const sets = o.localStorage.map((kv) => `    localStorage.setItem(${jsStringLiteral(kv.name)}, ${jsStringLiteral(kv.value)});`).join("\n");
-    return `  await browser.url(${jsStringLiteral(o.origin)});
-  await browser.execute(() => {
-${sets}
-  });`;
-  });
-  return [
-    "// Generated by ZeTa-AI \u2014 restores a captured browser session in WebdriverIO.",
-    "async function restoreZetaSession(browser) {",
-    `  await browser.setCookies(${JSON.stringify(cookies, null, 2).split("\n").join("\n  ")});`,
-    ...localStorageSets,
-    "}",
-    "module.exports = { restoreZetaSession };"
-  ].join("\n");
+/** browser.setCookies() accepts an array shape very close to Playwright's own
+ *  cookie objects (per WebdriverIO's documented API) — closest 1:1 mapping
+ *  of the three target formats. */
+export function toWebdriverIOSnippet(state) {
+    const cookies = state.cookies.map((c) => ({
+        name: c.name,
+        value: c.value,
+        domain: c.domain,
+        path: c.path,
+        secure: !!c.secure,
+        httpOnly: !!c.httpOnly,
+        ...(c.expires && c.expires > 0 ? { expiry: Math.floor(c.expires) } : {}),
+    }));
+    const localStorageSets = state.origins
+        .filter((o) => o.localStorage.length > 0)
+        .map((o) => {
+        const sets = o.localStorage
+            .map((kv) => `    localStorage.setItem(${jsStringLiteral(kv.name)}, ${jsStringLiteral(kv.value)});`)
+            .join('\n');
+        return `  await browser.url(${jsStringLiteral(o.origin)});\n  await browser.execute(() => {\n${sets}\n  });`;
+    });
+    return [
+        '// Generated by ZeTa-AI — restores a captured browser session in WebdriverIO.',
+        'async function restoreZetaSession(browser) {',
+        `  await browser.setCookies(${JSON.stringify(cookies, null, 2).split('\n').join('\n  ')});`,
+        ...localStorageSets,
+        '}',
+        'module.exports = { restoreZetaSession };',
+    ].join('\n');
 }
-function exportSessionAs(format, state) {
-  switch (format) {
-    case "cypress":
-      return { content: toCypressSnippet(state), filename: "zeta-session.cy-commands.js", mimeType: "application/javascript" };
-    case "selenium-python":
-      return { content: toSeleniumPythonDict(state), filename: "zeta_session.py", mimeType: "text/x-python" };
-    case "webdriverio":
-      return { content: toWebdriverIOSnippet(state), filename: "zeta-session.wdio.js", mimeType: "application/javascript" };
-    default:
-      throw new Error(`Unknown session export format: ${format}`);
-  }
+export function exportSessionAs(format, state) {
+    switch (format) {
+        case 'cypress':
+            return { content: toCypressSnippet(state), filename: 'zeta-session.cy-commands.js', mimeType: 'application/javascript' };
+        case 'selenium-python':
+            return { content: toSeleniumPythonDict(state), filename: 'zeta_session.py', mimeType: 'text/x-python' };
+        case 'webdriverio':
+            return { content: toWebdriverIOSnippet(state), filename: 'zeta-session.wdio.js', mimeType: 'application/javascript' };
+        default:
+            throw new Error(`Unknown session export format: ${format}`);
+    }
 }
-export {
-  exportSessionAs,
-  toCypressSnippet,
-  toSeleniumPythonDict,
-  toWebdriverIOSnippet
-};
-//# sourceMappingURL=session-export.js.map

package/dist/sitemap-writer.js

@@ -1,47 +1,50 @@
-import * as fs from "fs/promises";
-import * as path from "path";
+/**
+ * Sitemap writer — generate XML sitemap from crawled screens.
+ *
+ * Compliant with sitemaps.org protocol (sitemap index + individual sitemap files).
+ * Splits into 50k-URL files per the spec when needed.
+ * Used for: SEO audit export, future crawl seed, and crawler/robots.txt comparison.
+ */
+import * as fs from 'fs/promises';
+import * as path from 'path';
 const XML_HEADER = '<?xml version="1.0" encoding="UTF-8"?>\n';
 const SITEMAP_NS = 'xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"';
 function escape(s) {
-  return s.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;").replace(/"/g, "&quot;").replace(/'/g, "&apos;");
+    return s.replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;').replace(/"/g, '&quot;').replace(/'/g, '&apos;');
 }
 function buildSitemapXml(urls) {
-  const entries = urls.map((u) => [
-    "  <url>",
-    `    <loc>${escape(u.loc)}</loc>`,
-    u.lastmod ? `    <lastmod>${u.lastmod}</lastmod>` : "",
-    u.changefreq ? `    <changefreq>${u.changefreq}</changefreq>` : "",
-    u.priority !== void 0 ? `    <priority>${u.priority.toFixed(1)}</priority>` : "",
-    "  </url>"
-  ].filter(Boolean).join("\n")).join("\n");
-  return `${XML_HEADER}<urlset ${SITEMAP_NS}>
-${entries}
-</urlset>`;
+    const entries = urls.map(u => [
+        '  <url>',
+        `    <loc>${escape(u.loc)}</loc>`,
+        u.lastmod ? `    <lastmod>${u.lastmod}</lastmod>` : '',
+        u.changefreq ? `    <changefreq>${u.changefreq}</changefreq>` : '',
+        u.priority !== undefined ? `    <priority>${u.priority.toFixed(1)}</priority>` : '',
+        '  </url>',
+    ].filter(Boolean).join('\n')).join('\n');
+    return `${XML_HEADER}<urlset ${SITEMAP_NS}>\n${entries}\n</urlset>`;
 }
-async function writeSitemap(urls, outputPath) {
-  await fs.mkdir(path.dirname(outputPath), { recursive: true });
-  await fs.writeFile(outputPath, buildSitemapXml(urls), "utf8");
+export async function writeSitemap(urls, outputPath) {
+    await fs.mkdir(path.dirname(outputPath), { recursive: true });
+    await fs.writeFile(outputPath, buildSitemapXml(urls), 'utf8');
 }
-function screensToSitemapUrls(screens) {
-  return screens.filter((s) => {
-    try {
-      return new URL(s.url).protocol.startsWith("http");
-    } catch {
-      return false;
-    }
-  }).map((s) => ({
-    loc: s.url,
-    lastmod: s.updatedAt ? s.updatedAt.toISOString().split("T")[0] : void 0,
-    changefreq: "weekly",
-    priority: s.depth === 0 ? 1 : s.depth === 1 ? 0.8 : 0.6
-  }));
+/** Convert crawled screen records to SitemapUrl objects */
+export function screensToSitemapUrls(screens) {
+    return screens
+        .filter(s => {
+        try {
+            return new URL(s.url).protocol.startsWith('http');
+        }
+        catch {
+            return false;
+        }
+    })
+        .map(s => ({
+        loc: s.url,
+        lastmod: s.updatedAt ? s.updatedAt.toISOString().split('T')[0] : undefined,
+        changefreq: 'weekly',
+        priority: s.depth === 0 ? 1.0 : s.depth === 1 ? 0.8 : 0.6,
+    }));
 }
-function buildSitemapString(urls) {
-  return buildSitemapXml(urls);
+export function buildSitemapString(urls) {
+    return buildSitemapXml(urls);
 }
-export {
-  buildSitemapString,
-  screensToSitemapUrls,
-  writeSitemap
-};
-//# sourceMappingURL=sitemap-writer.js.map