@detiq/crawler 1.1.9 → 1.1.11

package/dist/export/bulk-export.js

@@ -1,61 +1,76 @@
-/**
- * Bulk export — export crawled screens to CSV / JSON / Markdown digest.
- *
- * Outputs:
- * - CSV: url, name, wordCount, screenshotPath, markdownPath, createdAt — for spreadsheet analysis
- * - JSON: full screen records array — for API integration
- * - Markdown digest: concatenated markdown of all screens — for LLM context windows
- */
-import * as fs from 'fs/promises';
-import * as path from 'path';
+import * as fs from "fs/promises";
+import * as path from "path";
 function escapeCsv(val) {
-    if (val.includes(',') || val.includes('"') || val.includes('\n')) {
-        return `"${val.replace(/"/g, '""')}"`;
-    }
-    return val;
+  if (val.includes(",") || val.includes('"') || val.includes("\n")) {
+    return `"${val.replace(/"/g, '""')}"`;
+  }
+  return val;
 }
-export async function exportToCsv(screens, outputPath) {
-    const headers = ['id', 'name', 'url', 'projectId', 'wordCount', 'screenshotPath', 'markdownPath', 'createdAt'];
-    const rows = screens.map(s => [
-        s.id, s.name, s.url, s.projectId,
-        String(s.wordCount ?? 0),
-        s.screenshotPath ?? '',
-        s.markdownPath ?? '',
-        s.createdAt?.toISOString() ?? '',
-    ].map(escapeCsv).join(','));
-    await fs.mkdir(path.dirname(outputPath), { recursive: true });
-    await fs.writeFile(outputPath, [headers.join(','), ...rows].join('\n'), 'utf8');
+async function exportToCsv(screens, outputPath) {
+  const headers = ["id", "name", "url", "projectId", "wordCount", "screenshotPath", "markdownPath", "createdAt"];
+  const rows = screens.map((s) => [
+    s.id,
+    s.name,
+    s.url,
+    s.projectId,
+    String(s.wordCount ?? 0),
+    s.screenshotPath ?? "",
+    s.markdownPath ?? "",
+    s.createdAt?.toISOString() ?? ""
+  ].map(escapeCsv).join(","));
+  await fs.mkdir(path.dirname(outputPath), { recursive: true });
+  await fs.writeFile(outputPath, [headers.join(","), ...rows].join("\n"), "utf8");
 }
-export async function exportToJson(screens, outputPath) {
-    await fs.mkdir(path.dirname(outputPath), { recursive: true });
-    const data = screens.map(s => ({
-        id: s.id,
-        name: s.name,
-        url: s.url,
-        projectId: s.projectId,
-        wordCount: s.wordCount,
-        screenshotPath: s.screenshotPath,
-        markdownPath: s.markdownPath,
-        createdAt: s.createdAt?.toISOString(),
-        updatedAt: s.updatedAt?.toISOString(),
-    }));
-    await fs.writeFile(outputPath, JSON.stringify(data, null, 2), 'utf8');
+async function exportToJson(screens, outputPath) {
+  await fs.mkdir(path.dirname(outputPath), { recursive: true });
+  const data = screens.map((s) => ({
+    id: s.id,
+    name: s.name,
+    url: s.url,
+    projectId: s.projectId,
+    wordCount: s.wordCount,
+    screenshotPath: s.screenshotPath,
+    markdownPath: s.markdownPath,
+    createdAt: s.createdAt?.toISOString(),
+    updatedAt: s.updatedAt?.toISOString()
+  }));
+  await fs.writeFile(outputPath, JSON.stringify(data, null, 2), "utf8");
 }
-export async function exportToMarkdownDigest(screens, outputPath, opts = {}) {
-    const { readMarkdownFromPath = true, maxScreens = 200 } = opts;
-    await fs.mkdir(path.dirname(outputPath), { recursive: true });
-    const handle = await fs.open(outputPath, 'w');
-    try {
-        await handle.write(`# ZeTa Crawl Digest\n\nExported: ${new Date().toISOString()}\nScreens: ${Math.min(screens.length, maxScreens)}\n\n---\n\n`);
-        for (const screen of screens.slice(0, maxScreens)) {
-            let md = screen.markdown ?? '';
-            if (!md && readMarkdownFromPath && screen.markdownPath) {
-                md = await fs.readFile(screen.markdownPath, 'utf8').catch(() => '');
-            }
-            await handle.write(`## ${screen.name}\n\n**URL:** ${screen.url}\n\n${md.trim()}\n\n---\n\n`);
-        }
-    }
-    finally {
-        await handle.close();
+async function exportToMarkdownDigest(screens, outputPath, opts = {}) {
+  const { readMarkdownFromPath = true, maxScreens = 200 } = opts;
+  await fs.mkdir(path.dirname(outputPath), { recursive: true });
+  const handle = await fs.open(outputPath, "w");
+  try {
+    await handle.write(`# ZeTa Crawl Digest
+
+Exported: ${(/* @__PURE__ */ new Date()).toISOString()}
+Screens: ${Math.min(screens.length, maxScreens)}
+
+---
+
+`);
+    for (const screen of screens.slice(0, maxScreens)) {
+      let md = screen.markdown ?? "";
+      if (!md && readMarkdownFromPath && screen.markdownPath) {
+        md = await fs.readFile(screen.markdownPath, "utf8").catch(() => "");
+      }
+      await handle.write(`## ${screen.name}
+
+**URL:** ${screen.url}
+
+${md.trim()}
+
+---
+
+`);
     }
+  } finally {
+    await handle.close();
+  }
 }
+export {
+  exportToCsv,
+  exportToJson,
+  exportToMarkdownDigest
+};
+//# sourceMappingURL=bulk-export.js.map

package/dist/export/vector-export.js

@@ -1,113 +1,85 @@
-/**
- * Vector DB export — convert crawled screens to embedding-ready chunks.
- *
- * Produces a JSONL file where each line is one chunk ready for upsert into
- * Pinecone, Qdrant, Weaviate, or any vector store. The embedding itself is
- * NOT generated here (requires an embedding model at the user's end) — we
- * produce the text chunks + metadata, and the caller embeds them.
- *
- * Format per line (JSONL):
- * {
- *   "id": "<screenId>-<chunkIndex>",
- *   "text": "<chunk text>",
- *   "metadata": {
- *     "screenId": "...",
- *     "projectId": "...",
- *     "tenantId": "...",
- *     "url": "...",
- *     "screenName": "...",
- *     "chunkIndex": 0,
- *     "totalChunks": 3,
- *     "crawledAt": "2026-...",
- *     "wordCount": 142
- *   }
- * }
- */
-import * as fs from 'fs/promises';
-import * as path from 'path';
+import * as fs from "fs/promises";
+import * as path from "path";
 function splitIntoChunks(text, maxChunkWords = 300) {
-    const paragraphs = text.split(/\n{2,}/).filter(p => p.trim().length > 0);
-    const chunks = [];
-    let current = '';
-    let wordCount = 0;
-    for (const para of paragraphs) {
-        const words = para.split(/\s+/).length;
-        if (wordCount + words > maxChunkWords && current.length > 0) {
-            chunks.push(current.trim());
-            current = '';
-            wordCount = 0;
-        }
-        current += (current ? '\n\n' : '') + para;
-        wordCount += words;
+  const paragraphs = text.split(/\n{2,}/).filter((p) => p.trim().length > 0);
+  const chunks = [];
+  let current = "";
+  let wordCount = 0;
+  for (const para of paragraphs) {
+    const words = para.split(/\s+/).length;
+    if (wordCount + words > maxChunkWords && current.length > 0) {
+      chunks.push(current.trim());
+      current = "";
+      wordCount = 0;
     }
-    if (current.trim())
-        chunks.push(current.trim());
-    return chunks.length > 0 ? chunks : [text.slice(0, 2000)];
+    current += (current ? "\n\n" : "") + para;
+    wordCount += words;
+  }
+  if (current.trim()) chunks.push(current.trim());
+  return chunks.length > 0 ? chunks : [text.slice(0, 2e3)];
 }
-export async function exportScreensToVectorJsonl(screens, outputPath, opts = {}) {
-    const { maxChunkWords = 300, readMarkdownFromPath = true } = opts;
-    await fs.mkdir(path.dirname(outputPath), { recursive: true });
-    const handle = await fs.open(outputPath, 'w');
-    let totalChunks = 0;
-    let skipped = 0;
-    try {
-        for (const screen of screens) {
-            let markdown = screen.markdown ?? '';
-            if (!markdown && readMarkdownFromPath && screen.markdownPath) {
-                try {
-                    markdown = await fs.readFile(screen.markdownPath, 'utf8');
-                }
-                catch { /* file missing — skip */ }
-            }
-            if (!markdown || markdown.trim().length < 20) {
-                skipped++;
-                continue;
-            }
-            const textChunks = splitIntoChunks(markdown, maxChunkWords);
-            for (let i = 0; i < textChunks.length; i++) {
-                const chunk = {
-                    id: `${screen.id}-${i}`,
-                    text: textChunks[i],
-                    metadata: {
-                        screenId: screen.id,
-                        projectId: screen.projectId,
-                        tenantId: screen.tenantId,
-                        url: screen.url,
-                        screenName: screen.name,
-                        chunkIndex: i,
-                        totalChunks: textChunks.length,
-                        crawledAt: screen.updatedAt?.toISOString() ?? new Date().toISOString(),
-                        wordCount: textChunks[i].split(/\s+/).length,
-                    },
-                };
-                await handle.write(JSON.stringify(chunk) + '\n');
-                totalChunks++;
-            }
+async function exportScreensToVectorJsonl(screens, outputPath, opts = {}) {
+  const { maxChunkWords = 300, readMarkdownFromPath = true } = opts;
+  await fs.mkdir(path.dirname(outputPath), { recursive: true });
+  const handle = await fs.open(outputPath, "w");
+  let totalChunks = 0;
+  let skipped = 0;
+  try {
+    for (const screen of screens) {
+      let markdown = screen.markdown ?? "";
+      if (!markdown && readMarkdownFromPath && screen.markdownPath) {
+        try {
+          markdown = await fs.readFile(screen.markdownPath, "utf8");
+        } catch {
         }
+      }
+      if (!markdown || markdown.trim().length < 20) {
+        skipped++;
+        continue;
+      }
+      const textChunks = splitIntoChunks(markdown, maxChunkWords);
+      for (let i = 0; i < textChunks.length; i++) {
+        const chunk = {
+          id: `${screen.id}-${i}`,
+          text: textChunks[i],
+          metadata: {
+            screenId: screen.id,
+            projectId: screen.projectId,
+            tenantId: screen.tenantId,
+            url: screen.url,
+            screenName: screen.name,
+            chunkIndex: i,
+            totalChunks: textChunks.length,
+            crawledAt: screen.updatedAt?.toISOString() ?? (/* @__PURE__ */ new Date()).toISOString(),
+            wordCount: textChunks[i].split(/\s+/).length
+          }
+        };
+        await handle.write(JSON.stringify(chunk) + "\n");
+        totalChunks++;
+      }
     }
-    finally {
-        await handle.close();
-    }
-    return { totalChunks, skipped };
+  } finally {
+    await handle.close();
+  }
+  return { totalChunks, skipped };
 }
-/**
- * Pinecone-compatible upsert batch format.
- * Call this after generating embeddings to produce the final upsert payload.
- */
-export function toPineconeUpsert(chunks, embeddings, namespace = 'zeta-crawl') {
-    return chunks.map((chunk, i) => ({
-        id: chunk.id,
-        values: embeddings[i],
-        metadata: chunk.metadata,
-    }));
+function toPineconeUpsert(chunks, embeddings, namespace = "zeta-crawl") {
+  return chunks.map((chunk, i) => ({
+    id: chunk.id,
+    values: embeddings[i],
+    metadata: chunk.metadata
+  }));
 }
-/**
- * Qdrant-compatible point format.
- */
-export function toQdrantPoints(chunks, embeddings) {
-    return chunks.map((chunk, i) => ({
-        id: chunk.id,
-        vector: embeddings[i],
-        payload: { ...chunk.metadata, text: chunk.text },
-    }));
+function toQdrantPoints(chunks, embeddings) {
+  return chunks.map((chunk, i) => ({
+    id: chunk.id,
+    vector: embeddings[i],
+    payload: { ...chunk.metadata, text: chunk.text }
+  }));
 }
+export {
+  exportScreensToVectorJsonl,
+  toPineconeUpsert,
+  toQdrantPoints
+};
+//# sourceMappingURL=vector-export.js.map