npm - @context-vault/core - Versions diffs - 2.17.0 → 3.0.0 - Mend

@context-vault/core 2.17.0 → 3.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (101) hide show

package/dist/capture.d.ts +21 -0
package/dist/capture.d.ts.map +1 -0
package/dist/capture.js +269 -0
package/dist/capture.js.map +1 -0
package/dist/categories.d.ts +6 -0
package/dist/categories.d.ts.map +1 -0
package/dist/categories.js +50 -0
package/dist/categories.js.map +1 -0
package/dist/config.d.ts +4 -0
package/dist/config.d.ts.map +1 -0
package/dist/config.js +190 -0
package/dist/config.js.map +1 -0
package/dist/constants.d.ts +33 -0
package/dist/constants.d.ts.map +1 -0
package/dist/constants.js +23 -0
package/dist/constants.js.map +1 -0
package/dist/db.d.ts +13 -0
package/dist/db.d.ts.map +1 -0
package/dist/db.js +191 -0
package/dist/db.js.map +1 -0
package/dist/embed.d.ts +5 -0
package/dist/embed.d.ts.map +1 -0
package/dist/embed.js +78 -0
package/dist/embed.js.map +1 -0
package/dist/files.d.ts +13 -0
package/dist/files.d.ts.map +1 -0
package/dist/files.js +66 -0
package/dist/files.js.map +1 -0
package/dist/formatters.d.ts +8 -0
package/dist/formatters.d.ts.map +1 -0
package/dist/formatters.js +18 -0
package/dist/formatters.js.map +1 -0
package/dist/frontmatter.d.ts +12 -0
package/dist/frontmatter.d.ts.map +1 -0
package/dist/frontmatter.js +101 -0
package/dist/frontmatter.js.map +1 -0
package/dist/index.d.ts +10 -0
package/dist/index.d.ts.map +1 -0
package/dist/index.js +297 -0
package/dist/index.js.map +1 -0
package/dist/ingest-url.d.ts +20 -0
package/dist/ingest-url.d.ts.map +1 -0
package/dist/ingest-url.js +113 -0
package/dist/ingest-url.js.map +1 -0
package/dist/main.d.ts +14 -0
package/dist/main.d.ts.map +1 -0
package/dist/main.js +25 -0
package/dist/main.js.map +1 -0
package/dist/search.d.ts +18 -0
package/dist/search.d.ts.map +1 -0
package/dist/search.js +238 -0
package/dist/search.js.map +1 -0
package/dist/types.d.ts +176 -0
package/dist/types.d.ts.map +1 -0
package/dist/types.js +2 -0
package/dist/types.js.map +1 -0
package/package.json +66 -17
package/src/capture.ts +308 -0
package/src/categories.ts +54 -0
package/src/{core/config.js → config.ts} +34 -33
package/src/{constants.js → constants.ts} +6 -3
package/src/db.ts +229 -0
package/src/{index/embed.js → embed.ts} +10 -35
package/src/files.ts +80 -0
package/src/{capture/formatters.js → formatters.ts} +13 -11
package/src/{core/frontmatter.js → frontmatter.ts} +27 -33
package/src/index.ts +351 -0
package/src/ingest-url.ts +99 -0
package/src/main.ts +111 -0
package/src/search.ts +285 -0
package/src/types.ts +166 -0
package/src/capture/file-ops.js +0 -97
package/src/capture/import-pipeline.js +0 -46
package/src/capture/importers.js +0 -387
package/src/capture/index.js +0 -236
package/src/capture/ingest-url.js +0 -252
package/src/consolidation/index.js +0 -112
package/src/core/categories.js +0 -72
package/src/core/error-log.js +0 -54
package/src/core/files.js +0 -108
package/src/core/status.js +0 -350
package/src/core/telemetry.js +0 -90
package/src/index/db.js +0 -416
package/src/index/index.js +0 -522
package/src/index.js +0 -66
package/src/retrieve/index.js +0 -500
package/src/server/helpers.js +0 -44
package/src/server/tools/clear-context.js +0 -47
package/src/server/tools/context-status.js +0 -182
package/src/server/tools/create-snapshot.js +0 -231
package/src/server/tools/delete-context.js +0 -60
package/src/server/tools/get-context.js +0 -678
package/src/server/tools/ingest-project.js +0 -244
package/src/server/tools/ingest-url.js +0 -88
package/src/server/tools/list-buckets.js +0 -116
package/src/server/tools/list-context.js +0 -163
package/src/server/tools/save-context.js +0 -609
package/src/server/tools/session-start.js +0 -285
package/src/server/tools/submit-feedback.js +0 -55
package/src/server/tools.js +0 -174
package/src/sync/sync.js +0 -235

package/src/capture/ingest-url.js DELETED Viewed

@@ -1,252 +0,0 @@
-export function htmlToMarkdown(html) {
-  let md = html;
-  // Remove scripts, styles, nav, header, footer, aside
-  md = md.replace(/<script[\s\S]*?<\/script>/gi, "");
-  md = md.replace(/<style[\s\S]*?<\/style>/gi, "");
-  md = md.replace(/<nav[\s\S]*?<\/nav>/gi, "");
-  md = md.replace(/<header[\s\S]*?<\/header>/gi, "");
-  md = md.replace(/<footer[\s\S]*?<\/footer>/gi, "");
-  md = md.replace(/<aside[\s\S]*?<\/aside>/gi, "");
-  // Convert headings
-  md = md.replace(
-    /<h1[^>]*>([\s\S]*?)<\/h1>/gi,
-    (_, c) => `\n# ${stripTags(c).trim()}\n`,
-  );
-  md = md.replace(
-    /<h2[^>]*>([\s\S]*?)<\/h2>/gi,
-    (_, c) => `\n## ${stripTags(c).trim()}\n`,
-  );
-  md = md.replace(
-    /<h3[^>]*>([\s\S]*?)<\/h3>/gi,
-    (_, c) => `\n### ${stripTags(c).trim()}\n`,
-  );
-  md = md.replace(
-    /<h4[^>]*>([\s\S]*?)<\/h4>/gi,
-    (_, c) => `\n#### ${stripTags(c).trim()}\n`,
-  );
-  md = md.replace(
-    /<h5[^>]*>([\s\S]*?)<\/h5>/gi,
-    (_, c) => `\n##### ${stripTags(c).trim()}\n`,
-  );
-  md = md.replace(
-    /<h6[^>]*>([\s\S]*?)<\/h6>/gi,
-    (_, c) => `\n###### ${stripTags(c).trim()}\n`,
-  );
-  // Convert links
-  md = md.replace(
-    /<a[^>]*href="([^"]*)"[^>]*>([\s\S]*?)<\/a>/gi,
-    (_, href, text) => {
-      const cleanText = stripTags(text).trim();
-      return cleanText ? `[${cleanText}](${href})` : "";
-    },
-  );
-  // Convert code blocks
-  md = md.replace(
-    /<pre[^>]*><code[^>]*>([\s\S]*?)<\/code><\/pre>/gi,
-    (_, c) => `\n\`\`\`\n${decodeEntities(c).trim()}\n\`\`\`\n`,
-  );
-  md = md.replace(
-    /<pre[^>]*>([\s\S]*?)<\/pre>/gi,
-    (_, c) => `\n\`\`\`\n${decodeEntities(stripTags(c)).trim()}\n\`\`\`\n`,
-  );
-  // Convert inline code
-  md = md.replace(
-    /<code[^>]*>([\s\S]*?)<\/code>/gi,
-    (_, c) => `\`${decodeEntities(c).trim()}\``,
-  );
-  // Convert strong/em
-  md = md.replace(
-    /<(strong|b)[^>]*>([\s\S]*?)<\/\1>/gi,
-    (_, __, c) => `**${stripTags(c).trim()}**`,
-  );
-  md = md.replace(
-    /<(em|i)[^>]*>([\s\S]*?)<\/\1>/gi,
-    (_, __, c) => `*${stripTags(c).trim()}*`,
-  );
-  // Convert list items
-  md = md.replace(
-    /<li[^>]*>([\s\S]*?)<\/li>/gi,
-    (_, c) => `- ${stripTags(c).trim()}\n`,
-  );
-  // Convert paragraphs and line breaks
-  md = md.replace(/<br\s*\/?>/gi, "\n");
-  md = md.replace(
-    /<p[^>]*>([\s\S]*?)<\/p>/gi,
-    (_, c) => `\n${stripTags(c).trim()}\n`,
-  );
-  md = md.replace(/<blockquote[^>]*>([\s\S]*?)<\/blockquote>/gi, (_, c) => {
-    return (
-      "\n" +
-      stripTags(c)
-        .trim()
-        .split("\n")
-        .map((l) => `> ${l}`)
-        .join("\n") +
-      "\n"
-    );
-  });
-  // Remove remaining HTML tags
-  md = stripTags(md);
-  // Decode HTML entities
-  md = decodeEntities(md);
-  // Clean up whitespace
-  md = md.replace(/\n{3,}/g, "\n\n").trim();
-  return md;
-}
-function stripTags(html) {
-  return html.replace(/<[^>]+>/g, "");
-}
-function decodeEntities(text) {
-  return text
-    .replace(/&amp;/g, "&")
-    .replace(/&lt;/g, "<")
-    .replace(/&gt;/g, ">")
-    .replace(/&quot;/g, '"')
-    .replace(/&#39;/g, "'")
-    .replace(/&nbsp;/g, " ")
-    .replace(/&#(\d+);/g, (_, n) => String.fromCharCode(parseInt(n, 10)))
-    .replace(/&#x([0-9a-f]+);/gi, (_, n) =>
-      String.fromCharCode(parseInt(n, 16)),
-    );
-}
-/**
- * Extract the main readable content from an HTML page.
- * Prefers <article> or <main>, falls back to <body>.
- *
- * @param {string} html
- * @param {string} url
- * @returns {{ title: string, body: string }}
- */
-export function extractHtmlContent(html, url) {
-  // Extract <title>
-  const titleMatch = html.match(/<title[^>]*>([\s\S]*?)<\/title>/i);
-  const title = titleMatch
-    ? stripTags(decodeEntities(titleMatch[1])).trim()
-    : "";
-  // Try to extract main content area
-  let contentHtml = "";
-  const articleMatch = html.match(/<article[^>]*>([\s\S]*?)<\/article>/i);
-  const mainMatch = html.match(/<main[^>]*>([\s\S]*?)<\/main>/i);
-  if (articleMatch) {
-    contentHtml = articleMatch[1];
-  } else if (mainMatch) {
-    contentHtml = mainMatch[1];
-  } else {
-    // Fall back to <body>
-    const bodyMatch = html.match(/<body[^>]*>([\s\S]*?)<\/body>/i);
-    contentHtml = bodyMatch ? bodyMatch[1] : html;
-  }
-  const body = htmlToMarkdown(contentHtml);
-  return { title, body };
-}
-/**
- * Fetch a URL, extract readable content, and return an EntryData object.
- *
- * @param {string} url
- * @param {{ kind?: string, tags?: string[], source?: string, maxBodyLength?: number, timeoutMs?: number }} [opts]
- * @returns {Promise<{ kind: string, title: string, body: string, tags: string[], meta: object, source: string }>}
- */
-export async function ingestUrl(url, opts = {}) {
-  const {
-    kind = "reference",
-    tags = [],
-    source,
-    maxBodyLength = 50000,
-    timeoutMs = 15000,
-  } = opts;
-  let domain;
-  try {
-    domain = new URL(url).hostname;
-  } catch {
-    throw new Error(`Invalid URL: ${url}`);
-  }
-  const controller = new AbortController();
-  const timeout = setTimeout(() => controller.abort(), timeoutMs);
-  let response;
-  try {
-    response = await fetch(url, {
-      signal: controller.signal,
-      headers: {
-        "User-Agent":
-          "ContextVault/1.0 (+https://github.com/fellanH/context-vault)",
-        Accept: "text/html,application/xhtml+xml,text/plain,*/*",
-      },
-    });
-  } catch (err) {
-    if (err.name === "AbortError") {
-      throw new Error(`Request timed out after ${timeoutMs}ms`);
-    }
-    throw new Error(`Fetch failed: ${err.message}`);
-  } finally {
-    clearTimeout(timeout);
-  }
-  if (!response.ok) {
-    throw new Error(`HTTP ${response.status}: ${response.statusText}`);
-  }
-  const contentType = response.headers.get("content-type") || "";
-  const html = await response.text();
-  let title, body;
-  if (
-    contentType.includes("text/html") ||
-    contentType.includes("application/xhtml")
-  ) {
-    const extracted = extractHtmlContent(html, url);
-    title = extracted.title;
-    body = extracted.body;
-  } else {
-    // Plain text or other — use as-is
-    title = domain;
-    body = html;
-  }
-  // Truncate if too long
-  if (body.length > maxBodyLength) {
-    body = body.slice(0, maxBodyLength) + "\n\n[Content truncated]";
-  }
-  if (!body.trim()) {
-    throw new Error("No readable content extracted from URL");
-  }
-  return {
-    kind,
-    title: title || domain,
-    body,
-    tags: [...tags, "web-import"],
-    meta: {
-      url,
-      domain,
-      fetched_at: new Date().toISOString(),
-      content_type: contentType.split(";")[0].trim() || "text/html",
-    },
-    source: source || domain,
-  };
-}

package/src/consolidation/index.js DELETED Viewed

@@ -1,112 +0,0 @@
-/**
- * Consolidation utilities — identifies tags and entries that warrant maintenance.
- *
- * These are pure DB queries with no LLM calls. The caller decides what to do
- * with the results (e.g. run create_snapshot, archive entries, report to user).
- */
-/**
- * Identifies tags that have accumulated enough entries to warrant consolidation.
- *
- * A tag is "hot" when it has >= tagThreshold non-superseded entries AND no
- * brief/snapshot was saved for it within the last maxSnapshotAgeDays days.
- *
- * @param {import('node:sqlite').DatabaseSync} db
- * @param {{ tagThreshold?: number, maxSnapshotAgeDays?: number }} [opts]
- * @returns {{ tag: string, entryCount: number, lastSnapshotAge: number | null }[]}
- */
-export function findHotTags(
-  db,
-  { tagThreshold = 10, maxSnapshotAgeDays = 7 } = {},
-) {
-  const rows = db
-    .prepare(
-      `SELECT id, tags, kind FROM vault
-       WHERE superseded_by IS NULL
-         AND tags IS NOT NULL
-         AND tags != '[]'`,
-    )
-    .all();
-  const tagCounts = new Map();
-  for (const row of rows) {
-    let tags;
-    try {
-      tags = JSON.parse(row.tags);
-    } catch {
-      continue;
-    }
-    if (!Array.isArray(tags)) continue;
-    for (const tag of tags) {
-      if (typeof tag !== "string" || !tag) continue;
-      tagCounts.set(tag, (tagCounts.get(tag) ?? 0) + 1);
-    }
-  }
-  const hotTags = [];
-  for (const [tag, count] of tagCounts) {
-    if (count < tagThreshold) continue;
-    const snapshotRow = db
-      .prepare(
-        `SELECT created_at FROM vault
-         WHERE kind = 'brief'
-           AND tags LIKE ?
-           AND created_at > datetime('now', '-' || ? || ' days')
-         ORDER BY created_at DESC
-         LIMIT 1`,
-      )
-      .get(`%"${tag}"%`, String(maxSnapshotAgeDays));
-    if (snapshotRow) continue;
-    const lastSnapshotAny = db
-      .prepare(
-        `SELECT created_at FROM vault
-         WHERE kind = 'brief'
-           AND tags LIKE ?
-         ORDER BY created_at DESC
-         LIMIT 1`,
-      )
-      .get(`%"${tag}"%`);
-    let lastSnapshotAge = null;
-    if (lastSnapshotAny) {
-      const ms = Date.now() - new Date(lastSnapshotAny.created_at).getTime();
-      lastSnapshotAge = Math.floor(ms / (1000 * 60 * 60 * 24));
-    }
-    hotTags.push({ tag, entryCount: count, lastSnapshotAge });
-  }
-  hotTags.sort((a, b) => b.entryCount - a.entryCount);
-  return hotTags;
-}
-/**
- * Identifies cold entries (old, never or rarely accessed) that can be archived.
- *
- * Returns IDs of entries that are old enough, have low hit counts, are not
- * superseded, and are not in permanent kinds (decision, architecture, brief).
- *
- * @param {import('node:sqlite').DatabaseSync} db
- * @param {{ maxAgeDays?: number, maxHitCount?: number }} [opts]
- * @returns {string[]} Entry IDs eligible for archiving
- */
-export function findColdEntries(db, { maxAgeDays = 90, maxHitCount = 0 } = {}) {
-  const rows = db
-    .prepare(
-      `SELECT id FROM vault
-       WHERE hit_count <= ?
-         AND created_at < datetime('now', '-' || ? || ' days')
-         AND superseded_by IS NULL
-         AND kind NOT IN ('decision', 'architecture', 'brief')`,
-    )
-    .all(maxHitCount, String(maxAgeDays));
-  return rows.map((r) => r.id);
-}

package/src/core/categories.js DELETED Viewed

@@ -1,72 +0,0 @@
-/**
- * categories.js — Static kind→category mapping
- *
- * Three categories with distinct write semantics:
- *   knowledge — append-only, enduring (default)
- *   entity    — upsert by identity_key, enduring
- *   event     — append-only, decaying relevance
- */
-const KIND_CATEGORY = {
-  // Knowledge — append-only, enduring
-  insight: "knowledge",
-  decision: "knowledge",
-  pattern: "knowledge",
-  prompt: "knowledge",
-  note: "knowledge",
-  document: "knowledge",
-  reference: "knowledge",
-  // Entity — upsert, enduring
-  contact: "entity",
-  project: "entity",
-  tool: "entity",
-  source: "entity",
-  bucket: "entity",
-  // Event — append-only, decaying
-  conversation: "event",
-  message: "event",
-  session: "event",
-  task: "event",
-  log: "event",
-  feedback: "event",
-};
-/** Map category name → directory name on disk */
-const CATEGORY_DIR_NAMES = {
-  knowledge: "knowledge",
-  entity: "entities",
-  event: "events",
-};
-/** Set of valid category directory names (for reindex discovery) */
-export const CATEGORY_DIRS = new Set(Object.values(CATEGORY_DIR_NAMES));
-/**
- * Staleness thresholds (in days) per knowledge kind.
- * Kinds not listed here are considered enduring (no staleness threshold).
- * Based on updated_at; falls back to created_at if updated_at is null.
- */
-export const KIND_STALENESS_DAYS = {
-  pattern: 180,
-  decision: 365,
-  reference: 90,
-};
-const DURABLE_KINDS = new Set(["decision", "architecture", "pattern"]);
-const EPHEMERAL_KINDS = new Set(["session", "observation"]);
-export function categoryFor(kind) {
-  return KIND_CATEGORY[kind] || "knowledge";
-}
-export function defaultTierFor(kind) {
-  if (DURABLE_KINDS.has(kind)) return "durable";
-  if (EPHEMERAL_KINDS.has(kind)) return "ephemeral";
-  return "working";
-}
-/** Returns the category directory name for a given kind (e.g. "insight" → "knowledge") */
-export function categoryDirFor(kind) {
-  const cat = categoryFor(kind);
-  return CATEGORY_DIR_NAMES[cat] || "knowledge";
-}

package/src/core/error-log.js DELETED Viewed

@@ -1,54 +0,0 @@
-import {
-  appendFileSync,
-  existsSync,
-  mkdirSync,
-  readFileSync,
-  statSync,
-  writeFileSync,
-} from "node:fs";
-import { join } from "node:path";
-const MAX_LOG_SIZE = 1024 * 1024; // 1MB
-export function errorLogPath(dataDir) {
-  return join(dataDir, "error.log");
-}
-/**
- * Append a structured JSON entry to the startup error log.
- * Rotates the file if it exceeds MAX_LOG_SIZE.
- * Never throws — logging failures must not mask the original error.
- *
- * @param {string} dataDir
- * @param {object} entry
- */
-export function appendErrorLog(dataDir, entry) {
-  try {
-    mkdirSync(dataDir, { recursive: true });
-    const logPath = errorLogPath(dataDir);
-    if (existsSync(logPath) && statSync(logPath).size >= MAX_LOG_SIZE) {
-      writeFileSync(logPath, "");
-    }
-    appendFileSync(logPath, JSON.stringify(entry) + "\n");
-  } catch {
-    // intentionally swallowed
-  }
-}
-/**
- * Return number of log lines in the error log, or 0 if absent.
- *
- * @param {string} dataDir
- * @returns {number}
- */
-export function errorLogCount(dataDir) {
-  try {
-    const logPath = errorLogPath(dataDir);
-    if (!existsSync(logPath)) return 0;
-    return readFileSync(logPath, "utf-8")
-      .split("\n")
-      .filter((l) => l.trim()).length;
-  } catch {
-    return 0;
-  }
-}

package/src/core/files.js DELETED Viewed

@@ -1,108 +0,0 @@
-/**
- * files.js — Shared file system utilities used across layers
- *
- * ULID generation, slugify, kind/dir mapping, directory walking.
- */
-import { readdirSync } from "node:fs";
-import { join, resolve, sep } from "node:path";
-import { categoryDirFor } from "./categories.js";
-const CROCKFORD = "0123456789ABCDEFGHJKMNPQRSTVWXYZ";
-export function ulid() {
-  const now = Date.now();
-  let ts = "";
-  let t = now;
-  for (let i = 0; i < 10; i++) {
-    ts = CROCKFORD[t & 31] + ts;
-    t = Math.floor(t / 32);
-  }
-  let rand = "";
-  for (let i = 0; i < 16; i++) {
-    rand += CROCKFORD[Math.floor(Math.random() * 32)];
-  }
-  return ts + rand;
-}
-export function slugify(text, maxLen = 60) {
-  let slug = text
-    .toLowerCase()
-    .replace(/[^a-z0-9]+/g, "-")
-    .replace(/^-+|-+$/g, "");
-  if (slug.length > maxLen) {
-    slug =
-      slug.slice(0, maxLen).replace(/-[^-]*$/, "") || slug.slice(0, maxLen);
-  }
-  return slug;
-}
-const PLURAL_MAP = {
-  insight: "insights",
-  decision: "decisions",
-  pattern: "patterns",
-  status: "statuses",
-  analysis: "analyses",
-  contact: "contacts",
-  project: "projects",
-  tool: "tools",
-  source: "sources",
-  conversation: "conversations",
-  message: "messages",
-  session: "sessions",
-  log: "logs",
-  feedback: "feedbacks",
-};
-const SINGULAR_MAP = Object.fromEntries(
-  Object.entries(PLURAL_MAP).map(([k, v]) => [v, k]),
-);
-export function kindToDir(kind) {
-  if (PLURAL_MAP[kind]) return PLURAL_MAP[kind];
-  return kind.endsWith("s") ? kind : kind + "s";
-}
-export function dirToKind(dirName) {
-  if (SINGULAR_MAP[dirName]) return SINGULAR_MAP[dirName];
-  return dirName.replace(/s$/, "");
-}
-/** Normalize a kind input (singular or plural) to its canonical singular form. */
-export function normalizeKind(input) {
-  if (PLURAL_MAP[input]) return input; // Already a known singular kind
-  if (SINGULAR_MAP[input]) return SINGULAR_MAP[input]; // Known plural → singular
-  return input; // Unknown — use as-is (don't strip 's')
-}
-/** Returns relative path from vault root → kind dir: "knowledge/insights", "events/sessions", etc. */
-export function kindToPath(kind) {
-  return `${categoryDirFor(kind)}/${kindToDir(kind)}`;
-}
-export function safeJoin(base, ...parts) {
-  const resolvedBase = resolve(base);
-  const result = resolve(join(base, ...parts));
-  if (!result.startsWith(resolvedBase + sep) && result !== resolvedBase) {
-    throw new Error(
-      `Path traversal blocked: resolved path escapes base directory`,
-    );
-  }
-  return result;
-}
-export function walkDir(dir) {
-  const results = [];
-  function walk(currentDir, relDir) {
-    for (const entry of readdirSync(currentDir, { withFileTypes: true })) {
-      const fullPath = join(currentDir, entry.name);
-      if (entry.isDirectory() && !entry.name.startsWith("_")) {
-        walk(fullPath, relDir ? join(relDir, entry.name) : entry.name);
-      } else if (entry.isFile() && entry.name.endsWith(".md")) {
-        results.push({ filePath: fullPath, relDir });
-      }
-    }
-  }
-  walk(dir, "");
-  return results;
-}