@context-vault/core 2.17.0 → 3.0.0

package/src/capture/importers.js

@@ -1,387 +0,0 @@
-/**
- * importers.js — Format detection + parsers for bulk import
- *
- * Detects and parses markdown, CSV/TSV, JSON, and plain text files into
- * the EntryData shape that captureAndIndex() accepts.
- *
- * No external dependencies — CSV parsed with split + quote handling,
- * markdown uses existing parseFrontmatter().
- */
-
-import { readdirSync, readFileSync, statSync } from "node:fs";
-import { join, extname, basename } from "node:path";
-import {
-  parseFrontmatter,
-  parseEntryFromMarkdown,
-} from "../core/frontmatter.js";
-import { dirToKind } from "../core/files.js";
-
-/**
- * Detect the format of a file by extension and content heuristics.
- * @param {string} filePath
- * @param {string} [content]
- * @returns {"markdown"|"csv"|"tsv"|"json"|"text"}
- */
-export function detectFormat(filePath, content) {
-  const ext = extname(filePath).toLowerCase();
-
-  if (ext === ".md" || ext === ".markdown") return "markdown";
-  if (ext === ".csv") return "csv";
-  if (ext === ".tsv") return "tsv";
-  if (ext === ".json" || ext === ".jsonl") return "json";
-
-  // Content-based heuristics if extension is ambiguous
-  if (content) {
-    const trimmed = content.trimStart();
-    if (trimmed.startsWith("---\n")) return "markdown";
-    if (trimmed.startsWith("[") || trimmed.startsWith("{")) return "json";
-  }
-
-  return "text";
-}
-
-/**
- * Parse a CSV line respecting quoted fields.
- * @param {string} line
- * @param {string} delimiter
- * @returns {string[]}
- */
-function parseCsvLine(line, delimiter) {
-  const fields = [];
-  let current = "";
-  let inQuotes = false;
-
-  for (let i = 0; i < line.length; i++) {
-    const ch = line[i];
-    if (inQuotes) {
-      if (ch === '"') {
-        if (i + 1 < line.length && line[i + 1] === '"') {
-          current += '"';
-          i++;
-        } else {
-          inQuotes = false;
-        }
-      } else {
-        current += ch;
-      }
-    } else if (ch === '"') {
-      inQuotes = true;
-    } else if (ch === delimiter) {
-      fields.push(current.trim());
-      current = "";
-    } else {
-      current += ch;
-    }
-  }
-  fields.push(current.trim());
-  return fields;
-}
-
-const KNOWN_COLUMNS = new Set([
-  "kind",
-  "title",
-  "body",
-  "tags",
-  "source",
-  "identity_key",
-  "expires_at",
-]);
-
-/**
- * Parse a markdown file into EntryData.
- * Reuses parseFrontmatter + parseEntryFromMarkdown from core.
- *
- * @param {string} content
- * @param {{ kind?: string, source?: string }} [opts]
- * @returns {import("./import-pipeline.js").EntryData[]}
- */
-export function parseMarkdown(content, opts = {}) {
-  const { meta: fmMeta, body: rawBody } = parseFrontmatter(content);
-
-  // Derive kind from frontmatter or option
-  const kind = fmMeta.kind || opts.kind || "insight";
-  const parsed = parseEntryFromMarkdown(kind, rawBody, fmMeta);
-
-  return [
-    {
-      kind,
-      title: parsed.title || fmMeta.title || null,
-      body: parsed.body || rawBody,
-      tags: Array.isArray(fmMeta.tags) ? fmMeta.tags : undefined,
-      meta: parsed.meta || undefined,
-      source: fmMeta.source || opts.source || "import",
-      identity_key: fmMeta.identity_key || undefined,
-      expires_at: fmMeta.expires_at || undefined,
-    },
-  ];
-}
-
-/**
- * Parse a CSV or TSV file into EntryData[].
- * Header row required. Recognized columns map directly; unknown → meta.
- * Tags column is comma-separated within field.
- *
- * @param {string} content
- * @param {string} delimiter - "," for CSV, "\t" for TSV
- * @param {{ kind?: string, source?: string }} [opts]
- * @returns {import("./import-pipeline.js").EntryData[]}
- */
-export function parseCsv(content, delimiter, opts = {}) {
-  const lines = content.split(/\r?\n/).filter((l) => l.trim());
-  if (lines.length < 2) return [];
-
-  const headers = parseCsvLine(lines[0], delimiter).map((h) =>
-    h.toLowerCase().trim(),
-  );
-  const entries = [];
-
-  for (let i = 1; i < lines.length; i++) {
-    const values = parseCsvLine(lines[i], delimiter);
-    if (values.every((v) => !v)) continue; // skip empty rows
-
-    const entry = {
-      kind: opts.kind || "insight",
-      body: "",
-      source: opts.source || "csv-import",
-    };
-    const meta = {};
-
-    for (let j = 0; j < headers.length; j++) {
-      const col = headers[j];
-      const val = values[j] || "";
-
-      if (col === "kind" && val) {
-        entry.kind = val;
-      } else if (col === "title" && val) {
-        entry.title = val;
-      } else if (col === "body" && val) {
-        entry.body = val;
-      } else if (col === "tags" && val) {
-        entry.tags = val
-          .split(",")
-          .map((t) => t.trim())
-          .filter(Boolean);
-      } else if (col === "source" && val) {
-        entry.source = val;
-      } else if (col === "identity_key" && val) {
-        entry.identity_key = val;
-      } else if (col === "expires_at" && val) {
-        entry.expires_at = val;
-      } else if (val && !KNOWN_COLUMNS.has(col)) {
-        meta[col] = val;
-      }
-    }
-
-    if (!entry.body) continue; // skip rows with no body
-    if (Object.keys(meta).length) entry.meta = meta;
-    entries.push(entry);
-  }
-
-  return entries;
-}
-
-/**
- * Parse a JSON file into EntryData[].
- * Supports: array-of-entries, {entries:[...]}, or ChatGPT export format.
- *
- * @param {string} content
- * @param {{ kind?: string, source?: string }} [opts]
- * @returns {import("./import-pipeline.js").EntryData[]}
- */
-export function parseJson(content, opts = {}) {
-  let data;
-  try {
-    data = JSON.parse(content);
-  } catch {
-    return [];
-  }
-
-  // Detect format
-  let rawEntries;
-
-  if (Array.isArray(data)) {
-    // Array-of-entries OR ChatGPT export format
-    if (
-      data.length > 0 &&
-      data[0].mapping &&
-      data[0].create_time !== undefined
-    ) {
-      return parseChatGptExport(data, opts);
-    }
-    rawEntries = data;
-  } else if (data && Array.isArray(data.entries)) {
-    rawEntries = data.entries;
-  } else {
-    // Single entry object
-    rawEntries = [data];
-  }
-
-  return rawEntries
-    .filter((e) => e && typeof e === "object" && e.body)
-    .map((e) => ({
-      kind: e.kind || opts.kind || "insight",
-      title: e.title || null,
-      body: e.body,
-      tags: Array.isArray(e.tags) ? e.tags : undefined,
-      meta: e.meta && typeof e.meta === "object" ? e.meta : undefined,
-      source: e.source || opts.source || "json-import",
-      identity_key: e.identity_key || undefined,
-      expires_at: e.expires_at || undefined,
-    }));
-}
-
-/**
- * Parse ChatGPT export format (array of conversations with mapping + create_time).
- */
-function parseChatGptExport(conversations, opts = {}) {
-  const entries = [];
-
-  for (const conv of conversations) {
-    if (!conv.title || !conv.mapping) continue;
-
-    // Extract all assistant messages from the mapping
-    const messages = Object.values(conv.mapping)
-      .filter(
-        (m) =>
-          m.message?.author?.role === "assistant" &&
-          m.message.content?.parts?.length,
-      )
-      .map((m) => m.message.content.parts.join("\n"))
-      .filter(Boolean);
-
-    if (!messages.length) continue;
-
-    const body = messages.join("\n\n---\n\n");
-    const created = conv.create_time
-      ? new Date(conv.create_time * 1000).toISOString()
-      : undefined;
-
-    entries.push({
-      kind: opts.kind || "conversation",
-      title: conv.title,
-      body,
-      tags: ["chatgpt-import"],
-      meta: { conversation_id: conv.id, created_at_original: created },
-      source: opts.source || "chatgpt-export",
-    });
-  }
-
-  return entries;
-}
-
-/**
- * Parse a plain text file into a single EntryData.
- *
- * @param {string} content
- * @param {string} filePath
- * @param {{ kind?: string, source?: string }} [opts]
- * @returns {import("./import-pipeline.js").EntryData[]}
- */
-export function parseText(content, filePath, opts = {}) {
-  const trimmed = content.trim();
-  if (!trimmed) return [];
-
-  const name = basename(filePath, extname(filePath));
-  const title = name
-    .replace(/[-_]/g, " ")
-    .replace(/\b\w/g, (c) => c.toUpperCase());
-
-  return [
-    {
-      kind: opts.kind || "insight",
-      title,
-      body: trimmed,
-      source: opts.source || "text-import",
-    },
-  ];
-}
-
-/**
- * Parse a single file (auto-detect format).
- *
- * @param {string} filePath
- * @param {string} content
- * @param {{ kind?: string, source?: string }} [opts]
- * @returns {import("./import-pipeline.js").EntryData[]}
- */
-export function parseFile(filePath, content, opts = {}) {
-  const format = detectFormat(filePath, content);
-
-  switch (format) {
-    case "markdown":
-      return parseMarkdown(content, opts);
-    case "csv":
-      return parseCsv(content, ",", opts);
-    case "tsv":
-      return parseCsv(content, "\t", opts);
-    case "json":
-      return parseJson(content, opts);
-    case "text":
-      return parseText(content, filePath, opts);
-    default:
-      return [];
-  }
-}
-
-/**
- * Recursively parse a directory of files.
- * Walks subdirectories, filters by extension, infers kind from directory name.
- *
- * @param {string} dirPath
- * @param {{ kind?: string, source?: string, extensions?: string[] }} [opts]
- * @returns {import("./import-pipeline.js").EntryData[]}
- */
-export function parseDirectory(dirPath, opts = {}) {
-  const extensions = opts.extensions || [
-    ".md",
-    ".markdown",
-    ".csv",
-    ".tsv",
-    ".json",
-    ".txt",
-  ];
-  const entries = [];
-
-  function walk(dir, inferredKind) {
-    let items;
-    try {
-      items = readdirSync(dir, { withFileTypes: true });
-    } catch {
-      return;
-    }
-
-    for (const item of items) {
-      if (item.name.startsWith(".") || item.name.startsWith("_")) continue;
-
-      const fullPath = join(dir, item.name);
-
-      if (item.isDirectory()) {
-        // Try to infer kind from directory name
-        const kind =
-          dirToKind(item.name) !== item.name
-            ? dirToKind(item.name)
-            : inferredKind;
-        walk(fullPath, kind);
-      } else if (item.isFile()) {
-        const ext = extname(item.name).toLowerCase();
-        if (!extensions.includes(ext)) continue;
-
-        try {
-          const content = readFileSync(fullPath, "utf-8");
-          const fileOpts = { ...opts };
-          if (inferredKind && !fileOpts.kind) fileOpts.kind = inferredKind;
-          const parsed = parseFile(fullPath, content, fileOpts);
-          entries.push(...parsed);
-        } catch {
-          // Skip unreadable files
-        }
-      }
-    }
-  }
-
-  // Infer kind from the top-level directory name
-  const topKind = opts.kind || undefined;
-  walk(dirPath, topKind);
-
-  return entries;
-}

package/src/capture/index.js

@@ -1,236 +0,0 @@
-/**
- * Capture Layer — Public API
- *
- * Writes knowledge entries to vault as .md files and indexes them.
- * captureAndIndex() is the write-through entry point (capture + index + rollback on failure).
- */
-
-import { existsSync, readFileSync, unlinkSync, writeFileSync } from "node:fs";
-import { resolve } from "node:path";
-import { ulid, slugify, kindToPath } from "../core/files.js";
-import { categoryFor } from "../core/categories.js";
-import { parseFrontmatter, formatFrontmatter } from "../core/frontmatter.js";
-import { formatBody } from "./formatters.js";
-import { writeEntryFile } from "./file-ops.js";
-import { indexEntry } from "../index/index.js";
-
-export function writeEntry(
-  ctx,
-  {
-    kind,
-    title,
-    body,
-    meta,
-    tags,
-    source,
-    folder,
-    identity_key,
-    expires_at,
-    supersedes,
-    source_files,
-    tier,
-    userId,
-  },
-) {
-  if (!kind || typeof kind !== "string") {
-    throw new Error("writeEntry: kind is required (non-empty string)");
-  }
-  if (!body || typeof body !== "string" || !body.trim()) {
-    throw new Error("writeEntry: body is required (non-empty string)");
-  }
-  if (tags != null && !Array.isArray(tags)) {
-    throw new Error("writeEntry: tags must be an array if provided");
-  }
-  if (meta != null && typeof meta !== "object") {
-    throw new Error("writeEntry: meta must be an object if provided");
-  }
-
-  const category = categoryFor(kind);
-
-  // Entity upsert: check for existing file at deterministic path
-  let id;
-  let createdAt;
-  let updatedAt;
-  if (category === "entity" && identity_key) {
-    const identitySlug = slugify(identity_key);
-    const dir = resolve(ctx.config.vaultDir, kindToPath(kind));
-    const existingPath = resolve(dir, `${identitySlug}.md`);
-
-    if (existsSync(existingPath)) {
-      // Preserve original ID and created timestamp from existing file
-      const raw = readFileSync(existingPath, "utf-8");
-      const { meta: fmMeta } = parseFrontmatter(raw);
-      id = fmMeta.id || ulid();
-      createdAt = fmMeta.created || new Date().toISOString();
-      updatedAt = new Date().toISOString();
-    } else {
-      id = ulid();
-      createdAt = new Date().toISOString();
-      updatedAt = createdAt;
-    }
-  } else {
-    id = ulid();
-    createdAt = new Date().toISOString();
-    updatedAt = createdAt;
-  }
-
-  const filePath = writeEntryFile(ctx.config.vaultDir, kind, {
-    id,
-    title,
-    body,
-    meta,
-    tags,
-    source,
-    createdAt,
-    updatedAt,
-    folder,
-    category,
-    identity_key,
-    expires_at,
-    supersedes,
-  });
-
-  return {
-    id,
-    filePath,
-    kind,
-    category,
-    title,
-    body,
-    meta,
-    tags,
-    source,
-    createdAt,
-    updatedAt,
-    identity_key,
-    expires_at,
-    supersedes,
-    source_files: source_files || null,
-    tier: tier || null,
-    userId: userId || null,
-  };
-}
-
-/**
- * Update an existing entry's file on disk (merge provided fields with existing).
- * Does NOT re-index — caller must call indexEntry after.
- *
- * @param {{ config, stmts }} ctx
- * @param {object} existing — Row from vault table (from getEntryById)
- * @param {{ title?, body?, tags?, meta?, source?, expires_at? }} updates
- * @returns {object} Entry object suitable for indexEntry
- */
-export function updateEntryFile(ctx, existing, updates) {
-  const raw = readFileSync(existing.file_path, "utf-8");
-  const { meta: fmMeta } = parseFrontmatter(raw);
-
-  const existingMeta = existing.meta ? JSON.parse(existing.meta) : {};
-  const existingTags = existing.tags ? JSON.parse(existing.tags) : [];
-
-  const title = updates.title !== undefined ? updates.title : existing.title;
-  const body = updates.body !== undefined ? updates.body : existing.body;
-  const tags = updates.tags !== undefined ? updates.tags : existingTags;
-  const source =
-    updates.source !== undefined ? updates.source : existing.source;
-  const expires_at =
-    updates.expires_at !== undefined ? updates.expires_at : existing.expires_at;
-  const supersedes =
-    updates.supersedes !== undefined
-      ? updates.supersedes
-      : fmMeta.supersedes || null;
-  const source_files =
-    updates.source_files !== undefined
-      ? updates.source_files
-      : existing.source_files
-        ? JSON.parse(existing.source_files)
-        : null;
-
-  let mergedMeta;
-  if (updates.meta !== undefined) {
-    mergedMeta = { ...existingMeta, ...(updates.meta || {}) };
-  } else {
-    mergedMeta = { ...existingMeta };
-  }
-
-  // Build frontmatter
-  const now = new Date().toISOString();
-  const fmFields = { id: existing.id };
-  for (const [k, v] of Object.entries(mergedMeta)) {
-    if (k === "folder") continue;
-    if (v !== null && v !== undefined) fmFields[k] = v;
-  }
-  if (existing.identity_key) fmFields.identity_key = existing.identity_key;
-  if (expires_at) fmFields.expires_at = expires_at;
-  if (supersedes?.length) fmFields.supersedes = supersedes;
-  fmFields.tags = tags;
-  fmFields.source = source || "claude-code";
-  fmFields.created = fmMeta.created || existing.created_at;
-  if (now !== fmFields.created) fmFields.updated = now;
-
-  const mdBody = formatBody(existing.kind, { title, body, meta: mergedMeta });
-  const md = formatFrontmatter(fmFields) + mdBody;
-
-  writeFileSync(existing.file_path, md);
-
-  const finalMeta = Object.keys(mergedMeta).length ? mergedMeta : undefined;
-
-  return {
-    id: existing.id,
-    filePath: existing.file_path,
-    kind: existing.kind,
-    category: existing.category,
-    title,
-    body,
-    meta: finalMeta,
-    tags,
-    source,
-    createdAt: fmMeta.created || existing.created_at,
-    updatedAt: now,
-    identity_key: existing.identity_key,
-    expires_at,
-    supersedes,
-    source_files: source_files || null,
-    userId: existing.user_id || null,
-  };
-}
-
-export async function captureAndIndex(ctx, data) {
-  // For entity upserts, preserve previous file content for safe rollback
-  let previousContent = null;
-  if (categoryFor(data.kind) === "entity" && data.identity_key) {
-    const identitySlug = slugify(data.identity_key);
-    const dir = resolve(ctx.config.vaultDir, kindToPath(data.kind));
-    const existingPath = resolve(dir, `${identitySlug}.md`);
-    if (existsSync(existingPath)) {
-      previousContent = readFileSync(existingPath, "utf-8");
-    }
-  }
-
-  const entry = writeEntry(ctx, data);
-  try {
-    await indexEntry(ctx, entry);
-    // Apply supersedes: mark referenced entries as superseded by this entry
-    if (entry.supersedes?.length && ctx.stmts.updateSupersededBy) {
-      for (const supersededId of entry.supersedes) {
-        if (typeof supersededId === "string" && supersededId.trim()) {
-          ctx.stmts.updateSupersededBy.run(entry.id, supersededId.trim());
-        }
-      }
-    }
-    return entry;
-  } catch (err) {
-    // Rollback: restore previous content for entity upserts, delete for new entries
-    if (previousContent) {
-      try {
-        writeFileSync(entry.filePath, previousContent);
-      } catch {}
-    } else {
-      try {
-        unlinkSync(entry.filePath);
-      } catch {}
-    }
-    throw new Error(
-      `Capture succeeded but indexing failed — file rolled back. ${err.message}`,
-    );
-  }
-}