context-vault 2.4.2 → 2.6.0

package/node_modules/@context-vault/core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@context-vault/core",
-  "version": "2.4.2",
+  "version": "2.6.0",
   "type": "module",
   "description": "Shared core: capture, index, retrieve, tools, and utilities for context-vault",
   "main": "src/index.js",
@@ -19,7 +19,11 @@
     "./core/config": "./src/core/config.js",
     "./core/files": "./src/core/files.js",
     "./core/frontmatter": "./src/core/frontmatter.js",
-    "./core/status": "./src/core/status.js"
+    "./core/status": "./src/core/status.js",
+    "./capture/importers": "./src/capture/importers.js",
+    "./capture/import-pipeline": "./src/capture/import-pipeline.js",
+    "./capture/ingest-url": "./src/capture/ingest-url.js",
+    "./sync": "./src/sync/sync.js"
   },
   "files": [
     "src/"
@@ -31,8 +35,8 @@
   "author": "Felix Hellstrom",
   "repository": {
     "type": "git",
-    "url": "git+https://github.com/fellanH/context-mcp.git",
+    "url": "git+https://github.com/fellanH/context-vault.git",
     "directory": "packages/core"
   },
-  "homepage": "https://github.com/fellanH/context-mcp"
+  "homepage": "https://github.com/fellanH/context-vault"
 }

package/node_modules/@context-vault/core/src/capture/file-ops.js

@@ -10,7 +10,7 @@ import { formatFrontmatter } from "../core/frontmatter.js";
 import { slugify, kindToPath } from "../core/files.js";
 import { formatBody } from "./formatters.js";
 
-function safeFolderPath(vaultDir, kind, folder) {
+export function safeFolderPath(vaultDir, kind, folder) {
   const base = resolve(vaultDir, kindToPath(kind));
   if (!folder) return base;
   const resolved = resolve(base, folder);

package/node_modules/@context-vault/core/src/capture/import-pipeline.js

@@ -0,0 +1,85 @@
+/**
+ * import-pipeline.js — Batch import orchestrator
+ *
+ * Processes an array of EntryData through captureAndIndex(),
+ * reporting progress and collecting results.
+ */
+
+import { captureAndIndex } from "./index.js";
+import { indexEntry } from "../index/index.js";
+
+/**
+ * @typedef {object} EntryData
+ * @property {string} kind
+ * @property {string} [title]
+ * @property {string} body
+ * @property {string[]} [tags]
+ * @property {object} [meta]
+ * @property {string} [source]
+ * @property {string} [identity_key]
+ * @property {string} [expires_at]
+ */
+
+/**
+ * @typedef {object} ImportResult
+ * @property {number} imported
+ * @property {number} failed
+ * @property {Array<{ index: number, title?: string, error: string }>} errors
+ */
+
+/**
+ * Import an array of entries into the vault.
+ *
+ * @param {object} ctx — Vault context (db, config, stmts, embed, insertVec, deleteVec)
+ * @param {EntryData[]} entries
+ * @param {{ onProgress?: (current: number, total: number) => void, source?: string }} [opts]
+ * @returns {Promise<ImportResult>}
+ */
+export async function importEntries(ctx, entries, opts = {}) {
+  const { onProgress, source } = opts;
+  let imported = 0;
+  let failed = 0;
+  const errors = [];
+
+  for (let i = 0; i < entries.length; i++) {
+    const entry = entries[i];
+
+    if (onProgress) {
+      onProgress(i + 1, entries.length);
+    }
+
+    try {
+      if (!entry.body?.trim()) {
+        failed++;
+        errors.push({ index: i, title: entry.title, error: "Empty body" });
+        continue;
+      }
+
+      await captureAndIndex(
+        ctx,
+        {
+          kind: entry.kind || "insight",
+          title: entry.title || null,
+          body: entry.body,
+          meta: entry.meta,
+          tags: entry.tags,
+          source: entry.source || source || "import",
+          identity_key: entry.identity_key,
+          expires_at: entry.expires_at,
+          userId: ctx.userId || null,
+        },
+        indexEntry
+      );
+      imported++;
+    } catch (err) {
+      failed++;
+      errors.push({
+        index: i,
+        title: entry.title || null,
+        error: err.message,
+      });
+    }
+  }
+
+  return { imported, failed, errors };
+}

package/node_modules/@context-vault/core/src/capture/importers.js

@@ -0,0 +1,360 @@
+/**
+ * importers.js — Format detection + parsers for bulk import
+ *
+ * Detects and parses markdown, CSV/TSV, JSON, and plain text files into
+ * the EntryData shape that captureAndIndex() accepts.
+ *
+ * No external dependencies — CSV parsed with split + quote handling,
+ * markdown uses existing parseFrontmatter().
+ */
+
+import { readdirSync, readFileSync, statSync } from "node:fs";
+import { join, extname, basename } from "node:path";
+import { parseFrontmatter, parseEntryFromMarkdown } from "../core/frontmatter.js";
+import { dirToKind } from "../core/files.js";
+
+// ─── Format Detection ────────────────────────────────────────────────────────
+
+/**
+ * Detect the format of a file by extension and content heuristics.
+ * @param {string} filePath
+ * @param {string} [content]
+ * @returns {"markdown"|"csv"|"tsv"|"json"|"text"}
+ */
+export function detectFormat(filePath, content) {
+  const ext = extname(filePath).toLowerCase();
+
+  if (ext === ".md" || ext === ".markdown") return "markdown";
+  if (ext === ".csv") return "csv";
+  if (ext === ".tsv") return "tsv";
+  if (ext === ".json" || ext === ".jsonl") return "json";
+
+  // Content-based heuristics if extension is ambiguous
+  if (content) {
+    const trimmed = content.trimStart();
+    if (trimmed.startsWith("---\n")) return "markdown";
+    if (trimmed.startsWith("[") || trimmed.startsWith("{")) return "json";
+  }
+
+  return "text";
+}
+
+// ─── CSV Parsing Helpers ─────────────────────────────────────────────────────
+
+/**
+ * Parse a CSV line respecting quoted fields.
+ * @param {string} line
+ * @param {string} delimiter
+ * @returns {string[]}
+ */
+function parseCsvLine(line, delimiter) {
+  const fields = [];
+  let current = "";
+  let inQuotes = false;
+
+  for (let i = 0; i < line.length; i++) {
+    const ch = line[i];
+    if (inQuotes) {
+      if (ch === '"') {
+        if (i + 1 < line.length && line[i + 1] === '"') {
+          current += '"';
+          i++;
+        } else {
+          inQuotes = false;
+        }
+      } else {
+        current += ch;
+      }
+    } else if (ch === '"') {
+      inQuotes = true;
+    } else if (ch === delimiter) {
+      fields.push(current.trim());
+      current = "";
+    } else {
+      current += ch;
+    }
+  }
+  fields.push(current.trim());
+  return fields;
+}
+
+// ─── Recognized CSV columns ─────────────────────────────────────────────────
+
+const KNOWN_COLUMNS = new Set([
+  "kind", "title", "body", "tags", "source",
+  "identity_key", "expires_at",
+]);
+
+// ─── Parsers ─────────────────────────────────────────────────────────────────
+
+/**
+ * Parse a markdown file into EntryData.
+ * Reuses parseFrontmatter + parseEntryFromMarkdown from core.
+ *
+ * @param {string} content
+ * @param {{ kind?: string, source?: string }} [opts]
+ * @returns {import("./import-pipeline.js").EntryData[]}
+ */
+export function parseMarkdown(content, opts = {}) {
+  const { meta: fmMeta, body: rawBody } = parseFrontmatter(content);
+
+  // Derive kind from frontmatter or option
+  const kind = fmMeta.kind || opts.kind || "insight";
+  const parsed = parseEntryFromMarkdown(kind, rawBody, fmMeta);
+
+  return [{
+    kind,
+    title: parsed.title || fmMeta.title || null,
+    body: parsed.body || rawBody,
+    tags: Array.isArray(fmMeta.tags) ? fmMeta.tags : undefined,
+    meta: parsed.meta || undefined,
+    source: fmMeta.source || opts.source || "import",
+    identity_key: fmMeta.identity_key || undefined,
+    expires_at: fmMeta.expires_at || undefined,
+  }];
+}
+
+/**
+ * Parse a CSV or TSV file into EntryData[].
+ * Header row required. Recognized columns map directly; unknown → meta.
+ * Tags column is comma-separated within field.
+ *
+ * @param {string} content
+ * @param {string} delimiter - "," for CSV, "\t" for TSV
+ * @param {{ kind?: string, source?: string }} [opts]
+ * @returns {import("./import-pipeline.js").EntryData[]}
+ */
+export function parseCsv(content, delimiter, opts = {}) {
+  const lines = content.split(/\r?\n/).filter((l) => l.trim());
+  if (lines.length < 2) return [];
+
+  const headers = parseCsvLine(lines[0], delimiter).map((h) => h.toLowerCase().trim());
+  const entries = [];
+
+  for (let i = 1; i < lines.length; i++) {
+    const values = parseCsvLine(lines[i], delimiter);
+    if (values.every((v) => !v)) continue; // skip empty rows
+
+    const entry = {
+      kind: opts.kind || "insight",
+      body: "",
+      source: opts.source || "csv-import",
+    };
+    const meta = {};
+
+    for (let j = 0; j < headers.length; j++) {
+      const col = headers[j];
+      const val = values[j] || "";
+
+      if (col === "kind" && val) {
+        entry.kind = val;
+      } else if (col === "title" && val) {
+        entry.title = val;
+      } else if (col === "body" && val) {
+        entry.body = val;
+      } else if (col === "tags" && val) {
+        entry.tags = val.split(",").map((t) => t.trim()).filter(Boolean);
+      } else if (col === "source" && val) {
+        entry.source = val;
+      } else if (col === "identity_key" && val) {
+        entry.identity_key = val;
+      } else if (col === "expires_at" && val) {
+        entry.expires_at = val;
+      } else if (val && !KNOWN_COLUMNS.has(col)) {
+        meta[col] = val;
+      }
+    }
+
+    if (!entry.body) continue; // skip rows with no body
+    if (Object.keys(meta).length) entry.meta = meta;
+    entries.push(entry);
+  }
+
+  return entries;
+}
+
+/**
+ * Parse a JSON file into EntryData[].
+ * Supports: array-of-entries, {entries:[...]}, or ChatGPT export format.
+ *
+ * @param {string} content
+ * @param {{ kind?: string, source?: string }} [opts]
+ * @returns {import("./import-pipeline.js").EntryData[]}
+ */
+export function parseJson(content, opts = {}) {
+  let data;
+  try {
+    data = JSON.parse(content);
+  } catch {
+    return [];
+  }
+
+  // Detect format
+  let rawEntries;
+
+  if (Array.isArray(data)) {
+    // Array-of-entries OR ChatGPT export format
+    if (data.length > 0 && data[0].mapping && data[0].create_time !== undefined) {
+      return parseChatGptExport(data, opts);
+    }
+    rawEntries = data;
+  } else if (data && Array.isArray(data.entries)) {
+    rawEntries = data.entries;
+  } else {
+    // Single entry object
+    rawEntries = [data];
+  }
+
+  return rawEntries
+    .filter((e) => e && typeof e === "object" && e.body)
+    .map((e) => ({
+      kind: e.kind || opts.kind || "insight",
+      title: e.title || null,
+      body: e.body,
+      tags: Array.isArray(e.tags) ? e.tags : undefined,
+      meta: e.meta && typeof e.meta === "object" ? e.meta : undefined,
+      source: e.source || opts.source || "json-import",
+      identity_key: e.identity_key || undefined,
+      expires_at: e.expires_at || undefined,
+    }));
+}
+
+/**
+ * Parse ChatGPT export format (array of conversations with mapping + create_time).
+ */
+function parseChatGptExport(conversations, opts = {}) {
+  const entries = [];
+
+  for (const conv of conversations) {
+    if (!conv.title || !conv.mapping) continue;
+
+    // Extract all assistant messages from the mapping
+    const messages = Object.values(conv.mapping)
+      .filter((m) => m.message?.author?.role === "assistant" && m.message.content?.parts?.length)
+      .map((m) => m.message.content.parts.join("\n"))
+      .filter(Boolean);
+
+    if (!messages.length) continue;
+
+    const body = messages.join("\n\n---\n\n");
+    const created = conv.create_time
+      ? new Date(conv.create_time * 1000).toISOString()
+      : undefined;
+
+    entries.push({
+      kind: opts.kind || "conversation",
+      title: conv.title,
+      body,
+      tags: ["chatgpt-import"],
+      meta: { conversation_id: conv.id, created_at_original: created },
+      source: opts.source || "chatgpt-export",
+    });
+  }
+
+  return entries;
+}
+
+/**
+ * Parse a plain text file into a single EntryData.
+ *
+ * @param {string} content
+ * @param {string} filePath
+ * @param {{ kind?: string, source?: string }} [opts]
+ * @returns {import("./import-pipeline.js").EntryData[]}
+ */
+export function parseText(content, filePath, opts = {}) {
+  const trimmed = content.trim();
+  if (!trimmed) return [];
+
+  const name = basename(filePath, extname(filePath));
+  const title = name.replace(/[-_]/g, " ").replace(/\b\w/g, (c) => c.toUpperCase());
+
+  return [{
+    kind: opts.kind || "insight",
+    title,
+    body: trimmed,
+    source: opts.source || "text-import",
+  }];
+}
+
+/**
+ * Parse a single file (auto-detect format).
+ *
+ * @param {string} filePath
+ * @param {string} content
+ * @param {{ kind?: string, source?: string }} [opts]
+ * @returns {import("./import-pipeline.js").EntryData[]}
+ */
+export function parseFile(filePath, content, opts = {}) {
+  const format = detectFormat(filePath, content);
+
+  switch (format) {
+    case "markdown":
+      return parseMarkdown(content, opts);
+    case "csv":
+      return parseCsv(content, ",", opts);
+    case "tsv":
+      return parseCsv(content, "\t", opts);
+    case "json":
+      return parseJson(content, opts);
+    case "text":
+      return parseText(content, filePath, opts);
+    default:
+      return [];
+  }
+}
+
+/**
+ * Recursively parse a directory of files.
+ * Walks subdirectories, filters by extension, infers kind from directory name.
+ *
+ * @param {string} dirPath
+ * @param {{ kind?: string, source?: string, extensions?: string[] }} [opts]
+ * @returns {import("./import-pipeline.js").EntryData[]}
+ */
+export function parseDirectory(dirPath, opts = {}) {
+  const extensions = opts.extensions || [".md", ".markdown", ".csv", ".tsv", ".json", ".txt"];
+  const entries = [];
+
+  function walk(dir, inferredKind) {
+    let items;
+    try {
+      items = readdirSync(dir, { withFileTypes: true });
+    } catch {
+      return;
+    }
+
+    for (const item of items) {
+      if (item.name.startsWith(".") || item.name.startsWith("_")) continue;
+
+      const fullPath = join(dir, item.name);
+
+      if (item.isDirectory()) {
+        // Try to infer kind from directory name
+        const kind = dirToKind(item.name) !== item.name
+          ? dirToKind(item.name)
+          : inferredKind;
+        walk(fullPath, kind);
+      } else if (item.isFile()) {
+        const ext = extname(item.name).toLowerCase();
+        if (!extensions.includes(ext)) continue;
+
+        try {
+          const content = readFileSync(fullPath, "utf-8");
+          const fileOpts = { ...opts };
+          if (inferredKind && !fileOpts.kind) fileOpts.kind = inferredKind;
+          const parsed = parseFile(fullPath, content, fileOpts);
+          entries.push(...parsed);
+        } catch {
+          // Skip unreadable files
+        }
+      }
+    }
+  }
+
+  // Infer kind from the top-level directory name
+  const topKind = opts.kind || undefined;
+  walk(dirPath, topKind);
+
+  return entries;
+}

package/node_modules/@context-vault/core/src/capture/ingest-url.js

@@ -0,0 +1,216 @@
+/**
+ * ingest-url.js — URL fetch + HTML → markdown conversion
+ *
+ * Fetches a URL, extracts readable content, converts to markdown,
+ * and returns an EntryData object ready for captureAndIndex().
+ *
+ * Uses Node built-in fetch() (Node 18+). No external dependencies.
+ */
+
+// ─── HTML → Markdown ─────────────────────────────────────────────────────────
+
+/**
+ * Convert HTML to simplified markdown.
+ * Strips scripts/styles, converts headings/links/lists/code.
+ *
+ * @param {string} html
+ * @returns {string}
+ */
+export function htmlToMarkdown(html) {
+  let md = html;
+
+  // Remove scripts, styles, nav, header, footer, aside
+  md = md.replace(/<script[\s\S]*?<\/script>/gi, "");
+  md = md.replace(/<style[\s\S]*?<\/style>/gi, "");
+  md = md.replace(/<nav[\s\S]*?<\/nav>/gi, "");
+  md = md.replace(/<header[\s\S]*?<\/header>/gi, "");
+  md = md.replace(/<footer[\s\S]*?<\/footer>/gi, "");
+  md = md.replace(/<aside[\s\S]*?<\/aside>/gi, "");
+
+  // Convert headings
+  md = md.replace(/<h1[^>]*>([\s\S]*?)<\/h1>/gi, (_, c) => `\n# ${stripTags(c).trim()}\n`);
+  md = md.replace(/<h2[^>]*>([\s\S]*?)<\/h2>/gi, (_, c) => `\n## ${stripTags(c).trim()}\n`);
+  md = md.replace(/<h3[^>]*>([\s\S]*?)<\/h3>/gi, (_, c) => `\n### ${stripTags(c).trim()}\n`);
+  md = md.replace(/<h4[^>]*>([\s\S]*?)<\/h4>/gi, (_, c) => `\n#### ${stripTags(c).trim()}\n`);
+  md = md.replace(/<h5[^>]*>([\s\S]*?)<\/h5>/gi, (_, c) => `\n##### ${stripTags(c).trim()}\n`);
+  md = md.replace(/<h6[^>]*>([\s\S]*?)<\/h6>/gi, (_, c) => `\n###### ${stripTags(c).trim()}\n`);
+
+  // Convert links
+  md = md.replace(/<a[^>]*href="([^"]*)"[^>]*>([\s\S]*?)<\/a>/gi, (_, href, text) => {
+    const cleanText = stripTags(text).trim();
+    return cleanText ? `[${cleanText}](${href})` : "";
+  });
+
+  // Convert code blocks
+  md = md.replace(/<pre[^>]*><code[^>]*>([\s\S]*?)<\/code><\/pre>/gi, (_, c) => `\n\`\`\`\n${decodeEntities(c).trim()}\n\`\`\`\n`);
+  md = md.replace(/<pre[^>]*>([\s\S]*?)<\/pre>/gi, (_, c) => `\n\`\`\`\n${decodeEntities(stripTags(c)).trim()}\n\`\`\`\n`);
+
+  // Convert inline code
+  md = md.replace(/<code[^>]*>([\s\S]*?)<\/code>/gi, (_, c) => `\`${decodeEntities(c).trim()}\``);
+
+  // Convert strong/em
+  md = md.replace(/<(strong|b)[^>]*>([\s\S]*?)<\/\1>/gi, (_, __, c) => `**${stripTags(c).trim()}**`);
+  md = md.replace(/<(em|i)[^>]*>([\s\S]*?)<\/\1>/gi, (_, __, c) => `*${stripTags(c).trim()}*`);
+
+  // Convert list items
+  md = md.replace(/<li[^>]*>([\s\S]*?)<\/li>/gi, (_, c) => `- ${stripTags(c).trim()}\n`);
+
+  // Convert paragraphs and line breaks
+  md = md.replace(/<br\s*\/?>/gi, "\n");
+  md = md.replace(/<p[^>]*>([\s\S]*?)<\/p>/gi, (_, c) => `\n${stripTags(c).trim()}\n`);
+  md = md.replace(/<blockquote[^>]*>([\s\S]*?)<\/blockquote>/gi, (_, c) => {
+    return "\n" + stripTags(c).trim().split("\n").map((l) => `> ${l}`).join("\n") + "\n";
+  });
+
+  // Remove remaining HTML tags
+  md = stripTags(md);
+
+  // Decode HTML entities
+  md = decodeEntities(md);
+
+  // Clean up whitespace
+  md = md.replace(/\n{3,}/g, "\n\n").trim();
+
+  return md;
+}
+
+function stripTags(html) {
+  return html.replace(/<[^>]+>/g, "");
+}
+
+function decodeEntities(text) {
+  return text
+    .replace(/&amp;/g, "&")
+    .replace(/&lt;/g, "<")
+    .replace(/&gt;/g, ">")
+    .replace(/&quot;/g, '"')
+    .replace(/&#39;/g, "'")
+    .replace(/&nbsp;/g, " ")
+    .replace(/&#(\d+);/g, (_, n) => String.fromCharCode(parseInt(n, 10)))
+    .replace(/&#x([0-9a-f]+);/gi, (_, n) => String.fromCharCode(parseInt(n, 16)));
+}
+
+// ─── HTML Content Extraction ─────────────────────────────────────────────────
+
+/**
+ * Extract the main readable content from an HTML page.
+ * Prefers <article> or <main>, falls back to <body>.
+ *
+ * @param {string} html
+ * @param {string} url
+ * @returns {{ title: string, body: string }}
+ */
+export function extractHtmlContent(html, url) {
+  // Extract <title>
+  const titleMatch = html.match(/<title[^>]*>([\s\S]*?)<\/title>/i);
+  const title = titleMatch ? stripTags(decodeEntities(titleMatch[1])).trim() : "";
+
+  // Try to extract main content area
+  let contentHtml = "";
+
+  const articleMatch = html.match(/<article[^>]*>([\s\S]*?)<\/article>/i);
+  const mainMatch = html.match(/<main[^>]*>([\s\S]*?)<\/main>/i);
+
+  if (articleMatch) {
+    contentHtml = articleMatch[1];
+  } else if (mainMatch) {
+    contentHtml = mainMatch[1];
+  } else {
+    // Fall back to <body>
+    const bodyMatch = html.match(/<body[^>]*>([\s\S]*?)<\/body>/i);
+    contentHtml = bodyMatch ? bodyMatch[1] : html;
+  }
+
+  const body = htmlToMarkdown(contentHtml);
+
+  return { title, body };
+}
+
+// ─── URL Ingestion ───────────────────────────────────────────────────────────
+
+/**
+ * Fetch a URL, extract readable content, and return an EntryData object.
+ *
+ * @param {string} url
+ * @param {{ kind?: string, tags?: string[], source?: string, maxBodyLength?: number, timeoutMs?: number }} [opts]
+ * @returns {Promise<{ kind: string, title: string, body: string, tags: string[], meta: object, source: string }>}
+ */
+export async function ingestUrl(url, opts = {}) {
+  const {
+    kind = "reference",
+    tags = [],
+    source,
+    maxBodyLength = 50000,
+    timeoutMs = 15000,
+  } = opts;
+
+  let domain;
+  try {
+    domain = new URL(url).hostname;
+  } catch {
+    throw new Error(`Invalid URL: ${url}`);
+  }
+
+  const controller = new AbortController();
+  const timeout = setTimeout(() => controller.abort(), timeoutMs);
+
+  let response;
+  try {
+    response = await fetch(url, {
+      signal: controller.signal,
+      headers: {
+        "User-Agent": "ContextVault/1.0 (+https://github.com/fellanH/context-vault)",
+        Accept: "text/html,application/xhtml+xml,text/plain,*/*",
+      },
+    });
+  } catch (err) {
+    if (err.name === "AbortError") {
+      throw new Error(`Request timed out after ${timeoutMs}ms`);
+    }
+    throw new Error(`Fetch failed: ${err.message}`);
+  } finally {
+    clearTimeout(timeout);
+  }
+
+  if (!response.ok) {
+    throw new Error(`HTTP ${response.status}: ${response.statusText}`);
+  }
+
+  const contentType = response.headers.get("content-type") || "";
+  const html = await response.text();
+
+  let title, body;
+
+  if (contentType.includes("text/html") || contentType.includes("application/xhtml")) {
+    const extracted = extractHtmlContent(html, url);
+    title = extracted.title;
+    body = extracted.body;
+  } else {
+    // Plain text or other — use as-is
+    title = domain;
+    body = html;
+  }
+
+  // Truncate if too long
+  if (body.length > maxBodyLength) {
+    body = body.slice(0, maxBodyLength) + "\n\n[Content truncated]";
+  }
+
+  if (!body.trim()) {
+    throw new Error("No readable content extracted from URL");
+  }
+
+  return {
+    kind,
+    title: title || domain,
+    body,
+    tags: [...tags, "web-import"],
+    meta: {
+      url,
+      domain,
+      fetched_at: new Date().toISOString(),
+      content_type: contentType.split(";")[0].trim() || "text/html",
+    },
+    source: source || domain,
+  };
+}