@context-vault/core 2.8.3

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Felix Hellstrom
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/package.json

@@ -0,0 +1,52 @@
+{
+  "name": "@context-vault/core",
+  "version": "2.8.3",
+  "type": "module",
+  "description": "Shared core: capture, index, retrieve, tools, and utilities for context-vault",
+  "main": "src/index.js",
+  "exports": {
+    ".": "./src/index.js",
+    "./capture": "./src/capture/index.js",
+    "./capture/formatters": "./src/capture/formatters.js",
+    "./capture/file-ops": "./src/capture/file-ops.js",
+    "./index/db": "./src/index/db.js",
+    "./index/embed": "./src/index/embed.js",
+    "./index": "./src/index/index.js",
+    "./retrieve": "./src/retrieve/index.js",
+    "./server/tools": "./src/server/tools.js",
+    "./server/helpers": "./src/server/helpers.js",
+    "./core/categories": "./src/core/categories.js",
+    "./core/config": "./src/core/config.js",
+    "./core/files": "./src/core/files.js",
+    "./core/frontmatter": "./src/core/frontmatter.js",
+    "./core/status": "./src/core/status.js",
+    "./capture/importers": "./src/capture/importers.js",
+    "./capture/import-pipeline": "./src/capture/import-pipeline.js",
+    "./capture/ingest-url": "./src/capture/ingest-url.js",
+    "./sync": "./src/sync/sync.js",
+    "./constants": "./src/constants.js"
+  },
+  "files": [
+    "src/"
+  ],
+  "license": "MIT",
+  "engines": {
+    "node": ">=20"
+  },
+  "author": "Felix Hellstrom",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/fellanH/context-vault.git",
+    "directory": "packages/core"
+  },
+  "homepage": "https://github.com/fellanH/context-vault",
+  "publishConfig": {
+    "access": "public"
+  },
+  "dependencies": {
+    "@huggingface/transformers": "^3.0.0",
+    "@modelcontextprotocol/sdk": "^1.26.0",
+    "better-sqlite3": "^12.6.2",
+    "sqlite-vec": "^0.1.0"
+  }
+}

package/src/capture/file-ops.js

@@ -0,0 +1,93 @@
+/**
+ * file-ops.js — Capture-specific file operations
+ *
+ * Writes markdown entry files with frontmatter to the vault directory.
+ */
+
+import { mkdirSync, writeFileSync } from "node:fs";
+import { resolve, relative } from "node:path";
+import { formatFrontmatter } from "../core/frontmatter.js";
+import { slugify, kindToPath } from "../core/files.js";
+import { formatBody } from "./formatters.js";
+
+export function safeFolderPath(vaultDir, kind, folder) {
+  const base = resolve(vaultDir, kindToPath(kind));
+  if (!folder) return base;
+  const resolved = resolve(base, folder);
+  const rel = relative(base, resolved);
+  if (rel.startsWith("..") || resolve(base, rel) !== resolved) {
+    throw new Error(`Folder path escapes vault: "${folder}"`);
+  }
+  return resolved;
+}
+
+export function writeEntryFile(
+  vaultDir,
+  kind,
+  {
+    id,
+    title,
+    body,
+    meta,
+    tags,
+    source,
+    createdAt,
+    folder,
+    category,
+    identity_key,
+    expires_at,
+  },
+) {
+  // P5: folder is now a top-level param; also accept from meta for backward compat
+  const resolvedFolder = folder || meta?.folder || "";
+  const dir = safeFolderPath(vaultDir, kind, resolvedFolder);
+
+  try {
+    mkdirSync(dir, { recursive: true });
+  } catch (e) {
+    throw new Error(`Failed to create directory "${dir}": ${e.message}`);
+  }
+
+  const created = createdAt || new Date().toISOString();
+  const fmFields = { id };
+
+  // Add kind-specific meta fields to frontmatter (flattened, not nested)
+  if (meta) {
+    for (const [k, v] of Object.entries(meta)) {
+      if (k === "folder") continue;
+      if (v !== null && v !== undefined) fmFields[k] = v;
+    }
+  }
+
+  if (identity_key) fmFields.identity_key = identity_key;
+  if (expires_at) fmFields.expires_at = expires_at;
+  fmFields.tags = tags || [];
+  fmFields.source = source || "claude-code";
+  fmFields.created = created;
+
+  const mdBody = formatBody(kind, { title, body, meta });
+
+  // Entity kinds: deterministic filename from identity_key (no ULID suffix)
+  let filename;
+  if (category === "entity" && identity_key) {
+    const identitySlug = slugify(identity_key);
+    filename = identitySlug
+      ? `${identitySlug}.md`
+      : `${id.slice(-8).toLowerCase()}.md`;
+  } else {
+    const slug = slugify((title || body).slice(0, 40));
+    const shortId = id.slice(-8).toLowerCase();
+    filename = slug ? `${slug}-${shortId}.md` : `${shortId}.md`;
+  }
+
+  const filePath = resolve(dir, filename);
+  const md = formatFrontmatter(fmFields) + mdBody;
+
+  try {
+    writeFileSync(filePath, md);
+  } catch (e) {
+    throw new Error(`Failed to write entry file "${filePath}": ${e.message}`);
+  }
+
+  return filePath;
+}

package/src/capture/formatters.js

@@ -0,0 +1,29 @@
+/**
+ * formatters.js — Kind-specific markdown body templates
+ *
+ * Maps entry kinds to their markdown body format.
+ * Default formatter used for unknown kinds.
+ */
+
+const FORMATTERS = {
+  insight: ({ body }) => "\n" + body + "\n",
+
+  decision: ({ title, body }) => {
+    const t = title || body.slice(0, 80);
+    return "\n## Decision\n\n" + t + "\n\n## Rationale\n\n" + body + "\n";
+  },
+
+  pattern: ({ title, body, meta }) => {
+    const t = title || body.slice(0, 80);
+    const lang = meta?.language || "";
+    return "\n# " + t + "\n\n```" + lang + "\n" + body + "\n```\n";
+  },
+};
+
+const DEFAULT_FORMATTER = ({ title, body }) =>
+  title ? "\n# " + title + "\n\n" + body + "\n" : "\n" + body + "\n";
+
+export function formatBody(kind, { title, body, meta }) {
+  const fn = FORMATTERS[kind] || DEFAULT_FORMATTER;
+  return fn({ title, body, meta });
+}

package/src/capture/import-pipeline.js

@@ -0,0 +1,46 @@
+import { captureAndIndex } from "./index.js";
+
+export async function importEntries(ctx, entries, opts = {}) {
+  const { onProgress, source } = opts;
+  let imported = 0;
+  let failed = 0;
+  const errors = [];
+
+  for (let i = 0; i < entries.length; i++) {
+    const entry = entries[i];
+
+    if (onProgress) {
+      onProgress(i + 1, entries.length);
+    }
+
+    try {
+      if (!entry.body?.trim()) {
+        failed++;
+        errors.push({ index: i, title: entry.title, error: "Empty body" });
+        continue;
+      }
+
+      await captureAndIndex(ctx, {
+        kind: entry.kind || "insight",
+        title: entry.title || null,
+        body: entry.body,
+        meta: entry.meta,
+        tags: entry.tags,
+        source: entry.source || source || "import",
+        identity_key: entry.identity_key,
+        expires_at: entry.expires_at,
+        userId: ctx.userId || null,
+      });
+      imported++;
+    } catch (err) {
+      failed++;
+      errors.push({
+        index: i,
+        title: entry.title || null,
+        error: err.message,
+      });
+    }
+  }
+
+  return { imported, failed, errors };
+}

package/src/capture/importers.js

@@ -0,0 +1,387 @@
+/**
+ * importers.js — Format detection + parsers for bulk import
+ *
+ * Detects and parses markdown, CSV/TSV, JSON, and plain text files into
+ * the EntryData shape that captureAndIndex() accepts.
+ *
+ * No external dependencies — CSV parsed with split + quote handling,
+ * markdown uses existing parseFrontmatter().
+ */
+
+import { readdirSync, readFileSync, statSync } from "node:fs";
+import { join, extname, basename } from "node:path";
+import {
+  parseFrontmatter,
+  parseEntryFromMarkdown,
+} from "../core/frontmatter.js";
+import { dirToKind } from "../core/files.js";
+
+/**
+ * Detect the format of a file by extension and content heuristics.
+ * @param {string} filePath
+ * @param {string} [content]
+ * @returns {"markdown"|"csv"|"tsv"|"json"|"text"}
+ */
+export function detectFormat(filePath, content) {
+  const ext = extname(filePath).toLowerCase();
+
+  if (ext === ".md" || ext === ".markdown") return "markdown";
+  if (ext === ".csv") return "csv";
+  if (ext === ".tsv") return "tsv";
+  if (ext === ".json" || ext === ".jsonl") return "json";
+
+  // Content-based heuristics if extension is ambiguous
+  if (content) {
+    const trimmed = content.trimStart();
+    if (trimmed.startsWith("---\n")) return "markdown";
+    if (trimmed.startsWith("[") || trimmed.startsWith("{")) return "json";
+  }
+
+  return "text";
+}
+
+/**
+ * Parse a CSV line respecting quoted fields.
+ * @param {string} line
+ * @param {string} delimiter
+ * @returns {string[]}
+ */
+function parseCsvLine(line, delimiter) {
+  const fields = [];
+  let current = "";
+  let inQuotes = false;
+
+  for (let i = 0; i < line.length; i++) {
+    const ch = line[i];
+    if (inQuotes) {
+      if (ch === '"') {
+        if (i + 1 < line.length && line[i + 1] === '"') {
+          current += '"';
+          i++;
+        } else {
+          inQuotes = false;
+        }
+      } else {
+        current += ch;
+      }
+    } else if (ch === '"') {
+      inQuotes = true;
+    } else if (ch === delimiter) {
+      fields.push(current.trim());
+      current = "";
+    } else {
+      current += ch;
+    }
+  }
+  fields.push(current.trim());
+  return fields;
+}
+
+const KNOWN_COLUMNS = new Set([
+  "kind",
+  "title",
+  "body",
+  "tags",
+  "source",
+  "identity_key",
+  "expires_at",
+]);
+
+/**
+ * Parse a markdown file into EntryData.
+ * Reuses parseFrontmatter + parseEntryFromMarkdown from core.
+ *
+ * @param {string} content
+ * @param {{ kind?: string, source?: string }} [opts]
+ * @returns {import("./import-pipeline.js").EntryData[]}
+ */
+export function parseMarkdown(content, opts = {}) {
+  const { meta: fmMeta, body: rawBody } = parseFrontmatter(content);
+
+  // Derive kind from frontmatter or option
+  const kind = fmMeta.kind || opts.kind || "insight";
+  const parsed = parseEntryFromMarkdown(kind, rawBody, fmMeta);
+
+  return [
+    {
+      kind,
+      title: parsed.title || fmMeta.title || null,
+      body: parsed.body || rawBody,
+      tags: Array.isArray(fmMeta.tags) ? fmMeta.tags : undefined,
+      meta: parsed.meta || undefined,
+      source: fmMeta.source || opts.source || "import",
+      identity_key: fmMeta.identity_key || undefined,
+      expires_at: fmMeta.expires_at || undefined,
+    },
+  ];
+}
+
+/**
+ * Parse a CSV or TSV file into EntryData[].
+ * Header row required. Recognized columns map directly; unknown → meta.
+ * Tags column is comma-separated within field.
+ *
+ * @param {string} content
+ * @param {string} delimiter - "," for CSV, "\t" for TSV
+ * @param {{ kind?: string, source?: string }} [opts]
+ * @returns {import("./import-pipeline.js").EntryData[]}
+ */
+export function parseCsv(content, delimiter, opts = {}) {
+  const lines = content.split(/\r?\n/).filter((l) => l.trim());
+  if (lines.length < 2) return [];
+
+  const headers = parseCsvLine(lines[0], delimiter).map((h) =>
+    h.toLowerCase().trim(),
+  );
+  const entries = [];
+
+  for (let i = 1; i < lines.length; i++) {
+    const values = parseCsvLine(lines[i], delimiter);
+    if (values.every((v) => !v)) continue; // skip empty rows
+
+    const entry = {
+      kind: opts.kind || "insight",
+      body: "",
+      source: opts.source || "csv-import",
+    };
+    const meta = {};
+
+    for (let j = 0; j < headers.length; j++) {
+      const col = headers[j];
+      const val = values[j] || "";
+
+      if (col === "kind" && val) {
+        entry.kind = val;
+      } else if (col === "title" && val) {
+        entry.title = val;
+      } else if (col === "body" && val) {
+        entry.body = val;
+      } else if (col === "tags" && val) {
+        entry.tags = val
+          .split(",")
+          .map((t) => t.trim())
+          .filter(Boolean);
+      } else if (col === "source" && val) {
+        entry.source = val;
+      } else if (col === "identity_key" && val) {
+        entry.identity_key = val;
+      } else if (col === "expires_at" && val) {
+        entry.expires_at = val;
+      } else if (val && !KNOWN_COLUMNS.has(col)) {
+        meta[col] = val;
+      }
+    }
+
+    if (!entry.body) continue; // skip rows with no body
+    if (Object.keys(meta).length) entry.meta = meta;
+    entries.push(entry);
+  }
+
+  return entries;
+}
+
+/**
+ * Parse a JSON file into EntryData[].
+ * Supports: array-of-entries, {entries:[...]}, or ChatGPT export format.
+ *
+ * @param {string} content
+ * @param {{ kind?: string, source?: string }} [opts]
+ * @returns {import("./import-pipeline.js").EntryData[]}
+ */
+export function parseJson(content, opts = {}) {
+  let data;
+  try {
+    data = JSON.parse(content);
+  } catch {
+    return [];
+  }
+
+  // Detect format
+  let rawEntries;
+
+  if (Array.isArray(data)) {
+    // Array-of-entries OR ChatGPT export format
+    if (
+      data.length > 0 &&
+      data[0].mapping &&
+      data[0].create_time !== undefined
+    ) {
+      return parseChatGptExport(data, opts);
+    }
+    rawEntries = data;
+  } else if (data && Array.isArray(data.entries)) {
+    rawEntries = data.entries;
+  } else {
+    // Single entry object
+    rawEntries = [data];
+  }
+
+  return rawEntries
+    .filter((e) => e && typeof e === "object" && e.body)
+    .map((e) => ({
+      kind: e.kind || opts.kind || "insight",
+      title: e.title || null,
+      body: e.body,
+      tags: Array.isArray(e.tags) ? e.tags : undefined,
+      meta: e.meta && typeof e.meta === "object" ? e.meta : undefined,
+      source: e.source || opts.source || "json-import",
+      identity_key: e.identity_key || undefined,
+      expires_at: e.expires_at || undefined,
+    }));
+}
+
+/**
+ * Parse ChatGPT export format (array of conversations with mapping + create_time).
+ */
+function parseChatGptExport(conversations, opts = {}) {
+  const entries = [];
+
+  for (const conv of conversations) {
+    if (!conv.title || !conv.mapping) continue;
+
+    // Extract all assistant messages from the mapping
+    const messages = Object.values(conv.mapping)
+      .filter(
+        (m) =>
+          m.message?.author?.role === "assistant" &&
+          m.message.content?.parts?.length,
+      )
+      .map((m) => m.message.content.parts.join("\n"))
+      .filter(Boolean);
+
+    if (!messages.length) continue;
+
+    const body = messages.join("\n\n---\n\n");
+    const created = conv.create_time
+      ? new Date(conv.create_time * 1000).toISOString()
+      : undefined;
+
+    entries.push({
+      kind: opts.kind || "conversation",
+      title: conv.title,
+      body,
+      tags: ["chatgpt-import"],
+      meta: { conversation_id: conv.id, created_at_original: created },
+      source: opts.source || "chatgpt-export",
+    });
+  }
+
+  return entries;
+}
+
+/**
+ * Parse a plain text file into a single EntryData.
+ *
+ * @param {string} content
+ * @param {string} filePath
+ * @param {{ kind?: string, source?: string }} [opts]
+ * @returns {import("./import-pipeline.js").EntryData[]}
+ */
+export function parseText(content, filePath, opts = {}) {
+  const trimmed = content.trim();
+  if (!trimmed) return [];
+
+  const name = basename(filePath, extname(filePath));
+  const title = name
+    .replace(/[-_]/g, " ")
+    .replace(/\b\w/g, (c) => c.toUpperCase());
+
+  return [
+    {
+      kind: opts.kind || "insight",
+      title,
+      body: trimmed,
+      source: opts.source || "text-import",
+    },
+  ];
+}
+
+/**
+ * Parse a single file (auto-detect format).
+ *
+ * @param {string} filePath
+ * @param {string} content
+ * @param {{ kind?: string, source?: string }} [opts]
+ * @returns {import("./import-pipeline.js").EntryData[]}
+ */
+export function parseFile(filePath, content, opts = {}) {
+  const format = detectFormat(filePath, content);
+
+  switch (format) {
+    case "markdown":
+      return parseMarkdown(content, opts);
+    case "csv":
+      return parseCsv(content, ",", opts);
+    case "tsv":
+      return parseCsv(content, "\t", opts);
+    case "json":
+      return parseJson(content, opts);
+    case "text":
+      return parseText(content, filePath, opts);
+    default:
+      return [];
+  }
+}
+
+/**
+ * Recursively parse a directory of files.
+ * Walks subdirectories, filters by extension, infers kind from directory name.
+ *
+ * @param {string} dirPath
+ * @param {{ kind?: string, source?: string, extensions?: string[] }} [opts]
+ * @returns {import("./import-pipeline.js").EntryData[]}
+ */
+export function parseDirectory(dirPath, opts = {}) {
+  const extensions = opts.extensions || [
+    ".md",
+    ".markdown",
+    ".csv",
+    ".tsv",
+    ".json",
+    ".txt",
+  ];
+  const entries = [];
+
+  function walk(dir, inferredKind) {
+    let items;
+    try {
+      items = readdirSync(dir, { withFileTypes: true });
+    } catch {
+      return;
+    }
+
+    for (const item of items) {
+      if (item.name.startsWith(".") || item.name.startsWith("_")) continue;
+
+      const fullPath = join(dir, item.name);
+
+      if (item.isDirectory()) {
+        // Try to infer kind from directory name
+        const kind =
+          dirToKind(item.name) !== item.name
+            ? dirToKind(item.name)
+            : inferredKind;
+        walk(fullPath, kind);
+      } else if (item.isFile()) {
+        const ext = extname(item.name).toLowerCase();
+        if (!extensions.includes(ext)) continue;
+
+        try {
+          const content = readFileSync(fullPath, "utf-8");
+          const fileOpts = { ...opts };
+          if (inferredKind && !fileOpts.kind) fileOpts.kind = inferredKind;
+          const parsed = parseFile(fullPath, content, fileOpts);
+          entries.push(...parsed);
+        } catch {
+          // Skip unreadable files
+        }
+      }
+    }
+  }
+
+  // Infer kind from the top-level directory name
+  const topKind = opts.kind || undefined;
+  walk(dirPath, topKind);
+
+  return entries;
+}