@cerefox/memory 0.7.2 → 0.8.1

package/dist/server-assets/_shared/mcp-tools/ingest.ts

@@ -0,0 +1,315 @@
+/**
+ * `cerefox_ingest` — save a note or document to the knowledge base.
+ *
+ * Three code paths:
+ * - `document_id` set → ID-based update (preferred; deterministic).
+ * - `update_if_exists: true` + title matches existing → update by title.
+ * - Otherwise → create (with hash-dedup short-circuit).
+ *
+ * Project-membership semantics on update (issue #38 / v0.1.20):
+ * - `project_names: [...]` → destructive replace.
+ * - `project_name: "..."` → non-destructive add (preserves other memberships).
+ * - Neither → no change.
+ *
+ * Mirrors `supabase/functions/cerefox-mcp/tools/ingest.ts` for v0.4.0
+ * extraction (no behaviour change).
+ */
+
+import type { MCPSupabaseClient } from "./types.ts";
+
+import { chunkMarkdown, normalizeContent, sha256hex } from "./_chunker.ts";
+import { embedBatch, OPENAI_MODEL } from "../embeddings/index.ts";
+import { ensureDocumentInProject, setDocumentProjectsByName } from "./_projects.ts";
+import { logUsage } from "./_utils.ts";
+import { McpInvalidParams, type ToolContext, type ToolDefinition } from "./types.ts";
+
+async function handler(
+  supabase: MCPSupabaseClient,
+  args: Record<string, unknown>,
+  ctx: ToolContext,
+): Promise<string> {
+  const title = (args.title as string | undefined)?.trim();
+  const content = args.content as string | undefined;
+  const document_id = (args.document_id as string | undefined) ?? null;
+  const project_name = args.project_name as string | undefined;
+  const project_names_raw = args.project_names;
+  const source = (args.source as string | undefined) ?? "agent";
+  const metadata = (args.metadata as Record<string, unknown> | undefined) ?? {};
+  const update_if_exists = (args.update_if_exists as boolean | undefined) ?? false;
+  const author = (args.author as string | undefined) ?? "mcp-agent";
+  const author_type = "agent"; // MCP path is always agent
+
+  if (!title || !content?.trim()) {
+    throw new McpInvalidParams("title and content are required");
+  }
+  if (
+    project_names_raw !== undefined &&
+    project_names_raw !== null &&
+    !Array.isArray(project_names_raw)
+  ) {
+    throw new McpInvalidParams(
+      "project_names must be a JSON array of strings; for a single project use project_name (string)",
+    );
+  }
+  const project_names: string[] | null = Array.isArray(project_names_raw)
+    ? project_names_raw.filter((s): s is string => typeof s === "string" && s.length > 0)
+    : null;
+
+  if (!ctx.openaiApiKey) {
+    throw new Error(
+      "OpenAI API key not configured. Set OPENAI_API_KEY (Edge Function) or CEREFOX_OPENAI_API_KEY (.env, local).",
+    );
+  }
+
+  const contentHash = await sha256hex(normalizeContent(content));
+  const reviewStatus = author_type === "agent" ? "pending_review" : "approved";
+
+  // ── ID-based update path ─────────────────────────────────────────────────
+  if (document_id) {
+    const { data: existing } = await supabase
+      .from("cerefox_documents")
+      .select("id, title, content_hash")
+      .eq("id", document_id)
+      .is("deleted_at", null)
+      .limit(1);
+
+    if (!existing?.length) {
+      throw new Error(`Document not found: ${document_id}`);
+    }
+
+    const existingDoc = existing[0];
+
+    if (existingDoc.content_hash === contentHash) {
+      const note = update_if_exists
+        ? ""
+        : " Note: update_if_exists flag was overridden by document_id.";
+      return `Document already up-to-date: "${existingDoc.title}" (id: ${existingDoc.id}). Content hash unchanged.${note}`;
+    }
+
+    const chunks = chunkMarkdown(content);
+    if (chunks.length === 0) throw new Error("Content produced no chunks");
+
+    const texts = chunks.map((c) => `# ${title}\n${c.content}`);
+    const embeddings = await embedBatch(texts, ctx.openaiApiKey);
+    const totalChars = chunks.reduce((s, c) => s + c.char_count, 0);
+
+    const chunkData = chunks.map((chunk, i) => ({
+      chunk_index: i,
+      heading_path: chunk.heading_path,
+      heading_level: chunk.heading_level,
+      title: chunk.title,
+      content: chunk.content,
+      char_count: chunk.char_count,
+      embedding: embeddings[i],
+      embedder: OPENAI_MODEL,
+    }));
+
+    const { error: ingestErr } = await supabase.rpc("cerefox_ingest_document", {
+      p_document_id: existingDoc.id,
+      p_title: title,
+      p_source: source,
+      p_content_hash: contentHash,
+      p_metadata: metadata,
+      p_review_status: reviewStatus,
+      p_chunks: chunkData,
+      p_author: author,
+      p_author_type: author_type,
+      p_source_label: source,
+    });
+
+    if (ingestErr) throw new Error(`Ingest RPC failed: ${ingestErr.message}`);
+
+    logUsage(supabase, {
+      operation: "ingest",
+      accessPath: ctx.accessPath,
+      requestor: author,
+      document_id: existingDoc.id,
+      result_count: chunks.length,
+    });
+
+    if (project_names !== null) {
+      await setDocumentProjectsByName(supabase, existingDoc.id, project_names);
+    } else if (project_name) {
+      await ensureDocumentInProject(supabase, existingDoc.id, project_name);
+    }
+
+    const note = update_if_exists
+      ? ""
+      : " Note: update_if_exists flag was overridden by document_id.";
+    return `Document updated: "${title}" (id: ${existingDoc.id}), ${chunks.length} chunk(s), ${totalChars} chars.${note}`;
+  }
+
+  // ── Update-existing path ─────────────────────────────────────────────────
+  if (update_if_exists) {
+    const { data: existing } = await supabase
+      .from("cerefox_documents")
+      .select("id, title, content_hash")
+      .eq("title", title)
+      .order("updated_at", { ascending: false })
+      .limit(1);
+
+    if (existing?.length) {
+      const existingDoc = existing[0];
+
+      if (existingDoc.content_hash === contentHash) {
+        return `Document already up-to-date: "${existingDoc.title}" (id: ${existingDoc.id}). Content hash unchanged.`;
+      }
+
+      const chunks = chunkMarkdown(content);
+      if (chunks.length === 0) throw new Error("Content produced no chunks");
+
+      const texts = chunks.map((c) => `# ${title}\n${c.content}`);
+      const embeddings = await embedBatch(texts, ctx.openaiApiKey);
+      const totalChars = chunks.reduce((s, c) => s + c.char_count, 0);
+
+      const chunkData = chunks.map((chunk, i) => ({
+        chunk_index: i,
+        heading_path: chunk.heading_path,
+        heading_level: chunk.heading_level,
+        title: chunk.title,
+        content: chunk.content,
+        char_count: chunk.char_count,
+        embedding: embeddings[i],
+        embedder: OPENAI_MODEL,
+      }));
+
+      const { error: ingestErr } = await supabase.rpc("cerefox_ingest_document", {
+        p_document_id: existingDoc.id,
+        p_title: existingDoc.title,
+        p_source: source,
+        p_content_hash: contentHash,
+        p_metadata: metadata,
+        p_review_status: reviewStatus,
+        p_chunks: chunkData,
+        p_author: author,
+        p_author_type: author_type,
+        p_source_label: source,
+      });
+
+      if (ingestErr) throw new Error(`Ingest RPC failed: ${ingestErr.message}`);
+
+      logUsage(supabase, {
+        operation: "ingest",
+        accessPath: ctx.accessPath,
+        requestor: author,
+        document_id: existingDoc.id,
+        result_count: chunks.length,
+      });
+
+      if (project_names !== null) {
+        await setDocumentProjectsByName(supabase, existingDoc.id, project_names);
+      } else if (project_name) {
+        await ensureDocumentInProject(supabase, existingDoc.id, project_name);
+      }
+
+      return `Document updated: "${existingDoc.title}" (id: ${existingDoc.id}), ${chunks.length} chunk(s), ${totalChars} chars.`;
+    }
+    // Fall through to create path
+  }
+
+  // ── Hash deduplication (create path) ─────────────────────────────────────
+  const { data: hashMatch } = await supabase
+    .from("cerefox_documents")
+    .select("id, title")
+    .eq("content_hash", contentHash)
+    .limit(1);
+
+  if (hashMatch?.length) {
+    return `Document already up-to-date: "${hashMatch[0].title}" (id: ${hashMatch[0].id}). Content hash unchanged.`;
+  }
+
+  const chunks = chunkMarkdown(content);
+  if (chunks.length === 0) throw new Error("Content produced no chunks");
+
+  const texts = chunks.map((c) => `# ${title}\n${c.content}`);
+  const embeddings = await embedBatch(texts, ctx.openaiApiKey);
+  const totalChars = chunks.reduce((s, c) => s + c.char_count, 0);
+
+  const chunkData = chunks.map((chunk, i) => ({
+    chunk_index: i,
+    heading_path: chunk.heading_path,
+    heading_level: chunk.heading_level,
+    title: chunk.title,
+    content: chunk.content,
+    char_count: chunk.char_count,
+    embedding: embeddings[i],
+    embedder: OPENAI_MODEL,
+  }));
+
+  const { data: ingestResult, error: ingestErr } = await supabase.rpc("cerefox_ingest_document", {
+    p_document_id: null,
+    p_title: title,
+    p_source: source,
+    p_content_hash: contentHash,
+    p_metadata: metadata,
+    p_review_status: reviewStatus,
+    p_chunks: chunkData,
+    p_author: author,
+    p_author_type: author_type,
+  });
+
+  if (ingestErr || !ingestResult?.length) {
+    throw new Error(`Ingest RPC failed: ${ingestErr?.message ?? "no data returned"}`);
+  }
+
+  const documentId = ingestResult[0].document_id;
+
+  if (project_names !== null && project_names.length > 0) {
+    await setDocumentProjectsByName(supabase, documentId, project_names);
+  } else if (project_name) {
+    await ensureDocumentInProject(supabase, documentId, project_name);
+  }
+
+  logUsage(supabase, {
+    operation: "ingest",
+    accessPath: ctx.accessPath,
+    requestor: author,
+    document_id: documentId,
+    result_count: chunks.length,
+  });
+
+  const projectInfo = project_name ? `, project: "${project_name}"` : "";
+  return `Document saved: "${title}" (id: ${documentId}), ${chunks.length} chunk(s), ${totalChars} chars${projectInfo}.`;
+}
+
+export const ingestTool: ToolDefinition = {
+  name: "cerefox_ingest",
+  description: "Save a note or document to the Cerefox knowledge base.",
+  inputSchema: {
+    type: "object",
+    required: ["title", "content"],
+    properties: {
+      title: { type: "string", description: "Document title" },
+      content: { type: "string", description: "Markdown content" },
+      document_id: {
+        type: "string",
+        description:
+          "UUID of an existing document to update. When provided, updates that specific document regardless of update_if_exists. Returns an error if the document does not exist. Workflow: cerefox_search → note the [id: ...] → pass here for deterministic update.",
+      },
+      project_name: {
+        type: "string",
+        description:
+          "Optional: single project name (created if absent). On update: non-destructive add — ensures this membership exists; preserves other memberships an operator may have added via the web UI. For explicit set-the-full-list semantics, use project_names (list) instead, or call cerefox_set_document_projects.",
+      },
+      project_names: {
+        type: "array",
+        items: { type: "string" },
+        description:
+          "Optional: explicit list of project names (each created if absent). Full-set semantics — on update this REPLACES the document's project memberships with exactly this set. Use when you want to set multiple projects at once or deliberately change the membership list. Wins over project_name when both are passed.",
+      },
+      source: { type: "string", description: 'Origin label (default: "agent")' },
+      update_if_exists: {
+        type: "boolean",
+        description:
+          "When true, update an existing document with the same title instead of creating a new one (default: false). Ignored when document_id is provided.",
+      },
+      metadata: { type: "object", description: "Arbitrary JSON metadata (optional)" },
+      author: {
+        type: "string",
+        description:
+          'Name of the agent or tool performing the ingestion (e.g., "Claude Code", "archiver"). Recorded in the audit log for attribution. Defaults to "mcp-agent" if not provided. May be enforced via server config.',
+      },
+    },
+  },
+  handler,
+};

package/dist/server-assets/_shared/mcp-tools/list-metadata-keys.ts

@@ -0,0 +1,55 @@
+/**
+ * `cerefox_list_metadata_keys` — discover the metadata vocabulary in use
+ * across all documents. Returns each key with doc count + up to 5 example
+ * values. Agents call this before `cerefox_metadata_search` or before
+ * supplying their own metadata on `cerefox_ingest`.
+ */
+
+import type { MCPSupabaseClient } from "./types.ts";
+
+import { logUsage } from "./_utils.ts";
+import type { ToolContext, ToolDefinition } from "./types.ts";
+
+async function handler(
+  supabase: MCPSupabaseClient,
+  args: Record<string, unknown>,
+  ctx: ToolContext,
+): Promise<string> {
+  const { data, error } = await supabase.rpc("cerefox_list_metadata_keys");
+
+  if (error) throw new Error(`RPC error: ${error.message}`);
+
+  const keys = (data ?? []) as Array<{
+    key: string;
+    doc_count: number;
+    example_values: string[];
+  }>;
+
+  logUsage(supabase, {
+    operation: "list_metadata_keys",
+    accessPath: ctx.accessPath,
+    requestor: args.requestor as string | undefined,
+    result_count: keys.length,
+  });
+
+  if (keys.length === 0) return "No metadata keys found across documents.";
+
+  return JSON.stringify(keys, null, 2);
+}
+
+export const listMetadataKeysTool: ToolDefinition = {
+  name: "cerefox_list_metadata_keys",
+  description:
+    "List all metadata keys currently in use across documents in the Cerefox knowledge base. Returns each key with its document count and up to 5 example values.",
+  inputSchema: {
+    type: "object",
+    properties: {
+      requestor: {
+        type: "string",
+        description:
+          'Name of the agent or user making this request. Recorded in the usage log. Defaults to "mcp-agent" if not provided. May be enforced via server config.',
+      },
+    },
+  },
+  handler,
+};

package/dist/server-assets/_shared/mcp-tools/list-projects.ts

@@ -0,0 +1,59 @@
+/**
+ * `cerefox_list_projects` — list all projects so agents can discover names
+ * before filtering by `project_name` in other tools.
+ *
+ * Calls `cerefox_list_projects` RPC. Trivial; usage logged.
+ */
+
+import type { MCPSupabaseClient } from "./types.ts";
+
+import { logUsage } from "./_utils.ts";
+import type { ToolContext, ToolDefinition } from "./types.ts";
+
+async function handler(
+  supabase: MCPSupabaseClient,
+  args: Record<string, unknown>,
+  ctx: ToolContext,
+): Promise<string> {
+  const { data, error } = await supabase.rpc("cerefox_list_projects");
+
+  if (error) throw new Error(`RPC error: ${error.message}`);
+
+  const projects = (data ?? []) as Array<{
+    id: string;
+    name: string;
+    description: string | null;
+  }>;
+
+  logUsage(supabase, {
+    operation: "list_projects",
+    accessPath: ctx.accessPath,
+    requestor: args.requestor as string | undefined,
+    result_count: projects.length,
+  });
+
+  if (projects.length === 0) return "No projects found.";
+
+  const lines = projects.map((p) => {
+    const desc = p.description ? ` -- ${p.description}` : "";
+    return `- ${p.name} (id: ${p.id})${desc}`;
+  });
+  return `Projects (${projects.length}):\n\n${lines.join("\n")}`;
+}
+
+export const listProjectsTool: ToolDefinition = {
+  name: "cerefox_list_projects",
+  description:
+    "List all projects with their names and IDs. Use this to discover available projects before filtering by project_name in other tools.",
+  inputSchema: {
+    type: "object",
+    properties: {
+      requestor: {
+        type: "string",
+        description:
+          'Name of the agent or user making this request. Recorded in the usage log. Defaults to "mcp-agent" if not provided. May be enforced via server config.',
+      },
+    },
+  },
+  handler,
+};

package/dist/server-assets/_shared/mcp-tools/list-versions.ts

@@ -0,0 +1,72 @@
+/**
+ * `cerefox_list_versions` — list a document's archived version history
+ * (newest first). Returns version_id (use with `cerefox_get_document`),
+ * version_number, source, chunk_count, total_chars, created_at.
+ */
+
+import type { MCPSupabaseClient } from "./types.ts";
+
+import { logUsage } from "./_utils.ts";
+import { McpInvalidParams, type ToolContext, type ToolDefinition } from "./types.ts";
+
+async function handler(
+  supabase: MCPSupabaseClient,
+  args: Record<string, unknown>,
+  ctx: ToolContext,
+): Promise<string> {
+  const document_id = args.document_id as string | undefined;
+  if (!document_id) throw new McpInvalidParams("document_id is required");
+
+  const { data, error } = await supabase.rpc("cerefox_list_document_versions", {
+    p_document_id: document_id,
+  });
+
+  if (error) throw new Error(`RPC error: ${error.message}`);
+
+  const versions = (data ?? []) as Array<{
+    version_id: string;
+    version_number: number;
+    source: string;
+    chunk_count: number;
+    total_chars: number;
+    created_at: string;
+  }>;
+
+  logUsage(supabase, {
+    operation: "list_versions",
+    accessPath: ctx.accessPath,
+    requestor: args.requestor as string | undefined,
+    document_id,
+    result_count: versions.length,
+  });
+
+  if (!versions.length) return "No archived versions found for this document.";
+
+  const lines = versions.map(
+    (v) =>
+      `v${v.version_number} | ${v.created_at.slice(0, 10)} | ${v.source} | ${v.chunk_count} chunks / ${v.total_chars.toLocaleString()} chars | id: ${v.version_id}`,
+  );
+  return `Archived versions (newest first):\n\n${lines.join("\n")}`;
+}
+
+export const listVersionsTool: ToolDefinition = {
+  name: "cerefox_list_versions",
+  description:
+    "List all archived versions of a document, newest first. Returns version_id (use with cerefox_get_document), version_number, source, chunk_count, total_chars, and created_at.",
+  inputSchema: {
+    type: "object",
+    required: ["document_id"],
+    properties: {
+      document_id: {
+        type: "string",
+        description: "UUID of the document whose version history to list",
+      },
+      requestor: {
+        type: "string",
+        description:
+          'Name of the agent or user making this request. Recorded in the usage log. Defaults to "mcp-agent" if not provided. May be enforced via server config.',
+      },
+    },
+  },
+  handler,
+};

package/dist/server-assets/_shared/mcp-tools/metadata-search.ts

@@ -0,0 +1,154 @@
+/**
+ * `cerefox_metadata_search` — find documents by metadata key-value
+ * criteria without a text search term. JSONB containment filter with AND
+ * semantics; optional project + time-range filters; optional content
+ * inclusion with a byte budget.
+ */
+
+import type { MCPSupabaseClient } from "./types.ts";
+
+import { applyByteBudget, logUsage, MAX_RESPONSE_BYTES } from "./_utils.ts";
+import { lookupProjectId } from "./_projects.ts";
+import { McpInvalidParams, type ToolContext, type ToolDefinition } from "./types.ts";
+
+async function handler(
+  supabase: MCPSupabaseClient,
+  args: Record<string, unknown>,
+  ctx: ToolContext,
+): Promise<string> {
+  const metadata_filter = args.metadata_filter as Record<string, string> | undefined;
+  const project_name = args.project_name as string | undefined;
+  const updated_since = args.updated_since as string | undefined;
+  const created_since = args.created_since as string | undefined;
+  const limit = (args.limit as number | undefined) ?? 10;
+  const include_content = (args.include_content as boolean | undefined) ?? false;
+  const requested_max_bytes = args.max_bytes as number | undefined;
+
+  if (!metadata_filter || typeof metadata_filter !== "object" || Array.isArray(metadata_filter)) {
+    throw new McpInvalidParams("metadata_filter is required and must be a JSON object");
+  }
+  if (Object.keys(metadata_filter).length === 0) {
+    throw new McpInvalidParams("metadata_filter must contain at least one key-value pair");
+  }
+
+  // Resolve project name to UUID if provided
+  let projectId: string | null = null;
+  if (project_name) {
+    projectId = await lookupProjectId(supabase, project_name);
+    if (!projectId) throw new Error(`Project not found: ${project_name}`);
+  }
+
+  // Enforce byte ceiling for content mode
+  const max_bytes = include_content
+    ? Math.min(requested_max_bytes ?? MAX_RESPONSE_BYTES, MAX_RESPONSE_BYTES)
+    : null;
+
+  const params: Record<string, unknown> = {
+    p_metadata_filter: metadata_filter,
+    p_project_id: projectId,
+    p_updated_since: updated_since ?? null,
+    p_created_since: created_since ?? null,
+    p_limit: limit,
+    p_include_content: include_content,
+  };
+  if (max_bytes !== null) params.p_max_bytes = max_bytes;
+
+  const { data, error } = await supabase.rpc("cerefox_metadata_search", params);
+
+  if (error) throw new Error(`RPC error: ${error.message}`);
+
+  const rows = (data ?? []) as Array<{
+    document_id: string;
+    title: string;
+    doc_metadata: Record<string, unknown>;
+    review_status: string;
+    source: string | null;
+    created_at: string;
+    updated_at: string;
+    total_chars: number;
+    chunk_count: number;
+    project_ids: string[];
+    project_names: string[];
+    version_count: number;
+    content: string | null;
+  }>;
+
+  logUsage(supabase, {
+    operation: "metadata_search",
+    accessPath: ctx.accessPath,
+    requestor: args.requestor as string | undefined,
+    query_text: JSON.stringify(metadata_filter),
+    project_id: projectId,
+    result_count: rows.length,
+  });
+
+  if (rows.length === 0) return "No documents match the metadata filter.";
+
+  // Note: when include_content is true the RPC already respects p_max_bytes
+  // server-side. The applyByteBudget helper is retained here only for
+  // parity with the EF implementation and as a defensive trim — see the
+  // EF original for the same shape.
+  void applyByteBudget; // referenced for symmetry; kept for v0.5 work
+
+  const parts: string[] = rows.map((row) => {
+    const projects = row.project_names?.length
+      ? ` | projects: ${row.project_names.join(", ")}`
+      : "";
+    const meta = Object.entries(row.doc_metadata ?? {})
+      .map(([k, v]) => `${k}=${v}`)
+      .join(", ");
+    const header =
+      `## ${row.title} [id: ${row.document_id}]\n` +
+      `${meta}${projects} | ${row.total_chars} chars | ${row.review_status} | updated ${row.updated_at?.slice(0, 10) ?? "?"}`;
+
+    if (include_content && row.content) {
+      return `${header}\n\n${row.content}`;
+    }
+    return header;
+  });
+
+  return parts.join("\n\n---\n\n");
+}
+
+export const metadataSearchTool: ToolDefinition = {
+  name: "cerefox_metadata_search",
+  description:
+    "Find documents by metadata key-value criteria without a text search term. Use to discover documents tagged with specific attributes, browse by taxonomy, or retrieve messages/tasks by type and status.",
+  inputSchema: {
+    type: "object",
+    required: ["metadata_filter"],
+    properties: {
+      metadata_filter: {
+        type: "object",
+        description:
+          'Key-value pairs; ALL must match (AND semantics). Example: {"type": "decision", "status": "active"}. Call cerefox_list_metadata_keys first to discover available keys.',
+        additionalProperties: { type: "string" },
+      },
+      project_name: { type: "string", description: "Restrict to a project by name (optional)" },
+      updated_since: {
+        type: "string",
+        description: "ISO-8601 timestamp; only docs updated on/after (optional)",
+      },
+      created_since: {
+        type: "string",
+        description: "ISO-8601 timestamp; only docs created on/after (optional)",
+      },
+      limit: { type: "integer", description: "Max results (default 10)" },
+      include_content: {
+        type: "boolean",
+        description: "Include full document text (default false)",
+      },
+      max_bytes: {
+        type: "integer",
+        description:
+          "Soft cap on total response bytes when include_content is true. Defaults to server maximum (200000).",
+      },
+      requestor: {
+        type: "string",
+        description:
+          'Name of the agent or user making this request. Recorded in the usage log. Defaults to "mcp-agent" if not provided. May be enforced via server config.',
+      },
+    },
+  },
+  handler,
+};