@detiq/crawler 1.1.7 → 1.1.9

package/dist/export/bulk-export.js

@@ -1,76 +1,61 @@
-import * as fs from "fs/promises";
-import * as path from "path";
+/**
+ * Bulk export — export crawled screens to CSV / JSON / Markdown digest.
+ *
+ * Outputs:
+ * - CSV: url, name, wordCount, screenshotPath, markdownPath, createdAt — for spreadsheet analysis
+ * - JSON: full screen records array — for API integration
+ * - Markdown digest: concatenated markdown of all screens — for LLM context windows
+ */
+import * as fs from 'fs/promises';
+import * as path from 'path';
 function escapeCsv(val) {
-  if (val.includes(",") || val.includes('"') || val.includes("\n")) {
-    return `"${val.replace(/"/g, '""')}"`;
-  }
-  return val;
+    if (val.includes(',') || val.includes('"') || val.includes('\n')) {
+        return `"${val.replace(/"/g, '""')}"`;
+    }
+    return val;
 }
-async function exportToCsv(screens, outputPath) {
-  const headers = ["id", "name", "url", "projectId", "wordCount", "screenshotPath", "markdownPath", "createdAt"];
-  const rows = screens.map((s) => [
-    s.id,
-    s.name,
-    s.url,
-    s.projectId,
-    String(s.wordCount ?? 0),
-    s.screenshotPath ?? "",
-    s.markdownPath ?? "",
-    s.createdAt?.toISOString() ?? ""
-  ].map(escapeCsv).join(","));
-  await fs.mkdir(path.dirname(outputPath), { recursive: true });
-  await fs.writeFile(outputPath, [headers.join(","), ...rows].join("\n"), "utf8");
+export async function exportToCsv(screens, outputPath) {
+    const headers = ['id', 'name', 'url', 'projectId', 'wordCount', 'screenshotPath', 'markdownPath', 'createdAt'];
+    const rows = screens.map(s => [
+        s.id, s.name, s.url, s.projectId,
+        String(s.wordCount ?? 0),
+        s.screenshotPath ?? '',
+        s.markdownPath ?? '',
+        s.createdAt?.toISOString() ?? '',
+    ].map(escapeCsv).join(','));
+    await fs.mkdir(path.dirname(outputPath), { recursive: true });
+    await fs.writeFile(outputPath, [headers.join(','), ...rows].join('\n'), 'utf8');
 }
-async function exportToJson(screens, outputPath) {
-  await fs.mkdir(path.dirname(outputPath), { recursive: true });
-  const data = screens.map((s) => ({
-    id: s.id,
-    name: s.name,
-    url: s.url,
-    projectId: s.projectId,
-    wordCount: s.wordCount,
-    screenshotPath: s.screenshotPath,
-    markdownPath: s.markdownPath,
-    createdAt: s.createdAt?.toISOString(),
-    updatedAt: s.updatedAt?.toISOString()
-  }));
-  await fs.writeFile(outputPath, JSON.stringify(data, null, 2), "utf8");
+export async function exportToJson(screens, outputPath) {
+    await fs.mkdir(path.dirname(outputPath), { recursive: true });
+    const data = screens.map(s => ({
+        id: s.id,
+        name: s.name,
+        url: s.url,
+        projectId: s.projectId,
+        wordCount: s.wordCount,
+        screenshotPath: s.screenshotPath,
+        markdownPath: s.markdownPath,
+        createdAt: s.createdAt?.toISOString(),
+        updatedAt: s.updatedAt?.toISOString(),
+    }));
+    await fs.writeFile(outputPath, JSON.stringify(data, null, 2), 'utf8');
 }
-async function exportToMarkdownDigest(screens, outputPath, opts = {}) {
-  const { readMarkdownFromPath = true, maxScreens = 200 } = opts;
-  await fs.mkdir(path.dirname(outputPath), { recursive: true });
-  const handle = await fs.open(outputPath, "w");
-  try {
-    await handle.write(`# ZeTa Crawl Digest
-
-Exported: ${(/* @__PURE__ */ new Date()).toISOString()}
-Screens: ${Math.min(screens.length, maxScreens)}
-
----
-
-`);
-    for (const screen of screens.slice(0, maxScreens)) {
-      let md = screen.markdown ?? "";
-      if (!md && readMarkdownFromPath && screen.markdownPath) {
-        md = await fs.readFile(screen.markdownPath, "utf8").catch(() => "");
-      }
-      await handle.write(`## ${screen.name}
-
-**URL:** ${screen.url}
-
-${md.trim()}
-
----
-
-`);
+export async function exportToMarkdownDigest(screens, outputPath, opts = {}) {
+    const { readMarkdownFromPath = true, maxScreens = 200 } = opts;
+    await fs.mkdir(path.dirname(outputPath), { recursive: true });
+    const handle = await fs.open(outputPath, 'w');
+    try {
+        await handle.write(`# ZeTa Crawl Digest\n\nExported: ${new Date().toISOString()}\nScreens: ${Math.min(screens.length, maxScreens)}\n\n---\n\n`);
+        for (const screen of screens.slice(0, maxScreens)) {
+            let md = screen.markdown ?? '';
+            if (!md && readMarkdownFromPath && screen.markdownPath) {
+                md = await fs.readFile(screen.markdownPath, 'utf8').catch(() => '');
+            }
+            await handle.write(`## ${screen.name}\n\n**URL:** ${screen.url}\n\n${md.trim()}\n\n---\n\n`);
+        }
+    }
+    finally {
+        await handle.close();
     }
-  } finally {
-    await handle.close();
-  }
 }
-export {
-  exportToCsv,
-  exportToJson,
-  exportToMarkdownDigest
-};
-//# sourceMappingURL=bulk-export.js.map

package/dist/export/vector-export.js

@@ -1,85 +1,113 @@
-import * as fs from "fs/promises";
-import * as path from "path";
+/**
+ * Vector DB export — convert crawled screens to embedding-ready chunks.
+ *
+ * Produces a JSONL file where each line is one chunk ready for upsert into
+ * Pinecone, Qdrant, Weaviate, or any vector store. The embedding itself is
+ * NOT generated here (requires an embedding model at the user's end) — we
+ * produce the text chunks + metadata, and the caller embeds them.
+ *
+ * Format per line (JSONL):
+ * {
+ *   "id": "<screenId>-<chunkIndex>",
+ *   "text": "<chunk text>",
+ *   "metadata": {
+ *     "screenId": "...",
+ *     "projectId": "...",
+ *     "tenantId": "...",
+ *     "url": "...",
+ *     "screenName": "...",
+ *     "chunkIndex": 0,
+ *     "totalChunks": 3,
+ *     "crawledAt": "2026-...",
+ *     "wordCount": 142
+ *   }
+ * }
+ */
+import * as fs from 'fs/promises';
+import * as path from 'path';
 function splitIntoChunks(text, maxChunkWords = 300) {
-  const paragraphs = text.split(/\n{2,}/).filter((p) => p.trim().length > 0);
-  const chunks = [];
-  let current = "";
-  let wordCount = 0;
-  for (const para of paragraphs) {
-    const words = para.split(/\s+/).length;
-    if (wordCount + words > maxChunkWords && current.length > 0) {
-      chunks.push(current.trim());
-      current = "";
-      wordCount = 0;
+    const paragraphs = text.split(/\n{2,}/).filter(p => p.trim().length > 0);
+    const chunks = [];
+    let current = '';
+    let wordCount = 0;
+    for (const para of paragraphs) {
+        const words = para.split(/\s+/).length;
+        if (wordCount + words > maxChunkWords && current.length > 0) {
+            chunks.push(current.trim());
+            current = '';
+            wordCount = 0;
+        }
+        current += (current ? '\n\n' : '') + para;
+        wordCount += words;
     }
-    current += (current ? "\n\n" : "") + para;
-    wordCount += words;
-  }
-  if (current.trim()) chunks.push(current.trim());
-  return chunks.length > 0 ? chunks : [text.slice(0, 2e3)];
+    if (current.trim())
+        chunks.push(current.trim());
+    return chunks.length > 0 ? chunks : [text.slice(0, 2000)];
 }
-async function exportScreensToVectorJsonl(screens, outputPath, opts = {}) {
-  const { maxChunkWords = 300, readMarkdownFromPath = true } = opts;
-  await fs.mkdir(path.dirname(outputPath), { recursive: true });
-  const handle = await fs.open(outputPath, "w");
-  let totalChunks = 0;
-  let skipped = 0;
-  try {
-    for (const screen of screens) {
-      let markdown = screen.markdown ?? "";
-      if (!markdown && readMarkdownFromPath && screen.markdownPath) {
-        try {
-          markdown = await fs.readFile(screen.markdownPath, "utf8");
-        } catch {
+export async function exportScreensToVectorJsonl(screens, outputPath, opts = {}) {
+    const { maxChunkWords = 300, readMarkdownFromPath = true } = opts;
+    await fs.mkdir(path.dirname(outputPath), { recursive: true });
+    const handle = await fs.open(outputPath, 'w');
+    let totalChunks = 0;
+    let skipped = 0;
+    try {
+        for (const screen of screens) {
+            let markdown = screen.markdown ?? '';
+            if (!markdown && readMarkdownFromPath && screen.markdownPath) {
+                try {
+                    markdown = await fs.readFile(screen.markdownPath, 'utf8');
+                }
+                catch { /* file missing — skip */ }
+            }
+            if (!markdown || markdown.trim().length < 20) {
+                skipped++;
+                continue;
+            }
+            const textChunks = splitIntoChunks(markdown, maxChunkWords);
+            for (let i = 0; i < textChunks.length; i++) {
+                const chunk = {
+                    id: `${screen.id}-${i}`,
+                    text: textChunks[i],
+                    metadata: {
+                        screenId: screen.id,
+                        projectId: screen.projectId,
+                        tenantId: screen.tenantId,
+                        url: screen.url,
+                        screenName: screen.name,
+                        chunkIndex: i,
+                        totalChunks: textChunks.length,
+                        crawledAt: screen.updatedAt?.toISOString() ?? new Date().toISOString(),
+                        wordCount: textChunks[i].split(/\s+/).length,
+                    },
+                };
+                await handle.write(JSON.stringify(chunk) + '\n');
+                totalChunks++;
+            }
         }
-      }
-      if (!markdown || markdown.trim().length < 20) {
-        skipped++;
-        continue;
-      }
-      const textChunks = splitIntoChunks(markdown, maxChunkWords);
-      for (let i = 0; i < textChunks.length; i++) {
-        const chunk = {
-          id: `${screen.id}-${i}`,
-          text: textChunks[i],
-          metadata: {
-            screenId: screen.id,
-            projectId: screen.projectId,
-            tenantId: screen.tenantId,
-            url: screen.url,
-            screenName: screen.name,
-            chunkIndex: i,
-            totalChunks: textChunks.length,
-            crawledAt: screen.updatedAt?.toISOString() ?? (/* @__PURE__ */ new Date()).toISOString(),
-            wordCount: textChunks[i].split(/\s+/).length
-          }
-        };
-        await handle.write(JSON.stringify(chunk) + "\n");
-        totalChunks++;
-      }
     }
-  } finally {
-    await handle.close();
-  }
-  return { totalChunks, skipped };
+    finally {
+        await handle.close();
+    }
+    return { totalChunks, skipped };
 }
-function toPineconeUpsert(chunks, embeddings, namespace = "zeta-crawl") {
-  return chunks.map((chunk, i) => ({
-    id: chunk.id,
-    values: embeddings[i],
-    metadata: chunk.metadata
-  }));
+/**
+ * Pinecone-compatible upsert batch format.
+ * Call this after generating embeddings to produce the final upsert payload.
+ */
+export function toPineconeUpsert(chunks, embeddings, namespace = 'zeta-crawl') {
+    return chunks.map((chunk, i) => ({
+        id: chunk.id,
+        values: embeddings[i],
+        metadata: chunk.metadata,
+    }));
 }
-function toQdrantPoints(chunks, embeddings) {
-  return chunks.map((chunk, i) => ({
-    id: chunk.id,
-    vector: embeddings[i],
-    payload: { ...chunk.metadata, text: chunk.text }
-  }));
+/**
+ * Qdrant-compatible point format.
+ */
+export function toQdrantPoints(chunks, embeddings) {
+    return chunks.map((chunk, i) => ({
+        id: chunk.id,
+        vector: embeddings[i],
+        payload: { ...chunk.metadata, text: chunk.text },
+    }));
 }
-export {
-  exportScreensToVectorJsonl,
-  toPineconeUpsert,
-  toQdrantPoints
-};
-//# sourceMappingURL=vector-export.js.map

package/dist/extractors/advanced.d.ts

@@ -13,11 +13,21 @@ export interface PageMetadata {
     ogImage: string | null;
     ogUrl: string | null;
     ogType: string | null;
+    ogSiteName: string | null;
+    ogLocale: string | null;
     twitterCard: string | null;
     twitterTitle: string | null;
     twitterDescription: string | null;
+    twitterImage: string | null;
+    twitterSite: string | null;
     canonical: string | null;
     robots: string | null;
+    hreflang: Array<{
+        lang: string;
+        href: string;
+    }>;
+    themeColor: string | null;
+    favicon: string | null;
     jsonLd: object[];
     schemaTypes: string[];
     lang: string | null;