@cerefox/memory 1.17.4 → 1.19.0

package/dist/server-assets/_shared/mcp-tools/ingest.ts

@@ -27,7 +27,14 @@ import {
 } from "./_chunker.ts";
 import { activeEmbedderName, embedBatch, resolveEmbedderKind } from "../embeddings/index.ts";
 import { ensureDocumentInProject, setDocumentProjectsByName } from "./_projects.ts";
-import { extractConflictHashes, logUsage } from "./_utils.ts";
+import {
+  duplicateContentDocumentId,
+  extractConflictHashes,
+  legacyContentUniqueError,
+  logUsage,
+  metadataTokenError,
+  metadataTokenParams,
+} from "./_utils.ts";
 import { McpInvalidParams, type ToolContext, type ToolDefinition } from "./types.ts";
 import { AUTHOR_PARAM_WRITE, DEFAULT_IDENTITY, callerIdentity } from "./identity.ts";
 
@@ -49,6 +56,8 @@ function conflictError(documentId: string, expectedHash: string, currentHash: st
 
 /** Map RPC-side CEREFOX_CONFLICT / CEREFOX_TOKEN_REQUIRED errors to agent-first text. */
 function mapIngestRpcError(message: string, documentId: string): Error {
+  const metaErr = metadataTokenError(message);
+  if (metaErr) return metaErr;
   if (message.includes("CEREFOX_CONFLICT")) {
     const { expected, current } = extractConflictHashes(message);
     return conflictError(documentId, expected, current);
@@ -82,7 +91,7 @@ function mapIngestRpcError(message: string, documentId: string): Error {
         `regardless of concurrent changes, pass last_write_wins=true.`,
     );
   }
-  return new Error(`Ingest RPC failed: ${message}`);
+  return legacyContentUniqueError(message) ?? new Error(`Ingest RPC failed: ${message}`);
 }
 
 async function handler(
@@ -118,6 +127,14 @@ async function handler(
   const author_type = "agent"; // MCP path is always agent
   const expected_content_hash = (args.expected_content_hash as string | undefined)?.trim() || null;
   const last_write_wins = (args.last_write_wins as boolean | undefined) ?? false;
+  const expectedMetaRaw = args.expected_metadata_revision;
+  if (
+    expectedMetaRaw !== undefined && expectedMetaRaw !== null &&
+    !(typeof expectedMetaRaw === "number" && Number.isInteger(expectedMetaRaw))
+  ) {
+    throw new McpInvalidParams("expected_metadata_revision must be an integer (the metadata_revision you read)");
+  }
+  const expected_metadata_revision = (expectedMetaRaw as number | undefined) ?? null;
 
   if (!title || !content?.trim()) {
     throw new McpInvalidParams("title and content are required");
@@ -214,6 +231,7 @@ async function handler(
       p_expected_content_hash: expected_content_hash,
       p_last_write_wins: last_write_wins,
       p_content_format: CONTENT_FORMAT_BLIND_STITCH,
+      ...metadataTokenParams(expected_metadata_revision, false),
     });
 
     if (ingestErr) throw mapIngestRpcError(ingestErr.message, existingDoc.id);
@@ -315,6 +333,7 @@ async function handler(
         p_expected_content_hash: expected_content_hash,
         p_last_write_wins: last_write_wins,
       p_content_format: CONTENT_FORMAT_BLIND_STITCH,
+      ...metadataTokenParams(expected_metadata_revision, false),
       });
 
       if (ingestErr) throw mapIngestRpcError(ingestErr.message, existingDoc.id);
@@ -339,26 +358,19 @@ async function handler(
   }
 
   // ── Hash deduplication (create path) ─────────────────────────────────────
-  // content_hash is unique STORE-WIDE, trashed documents included, so this
-  // check deliberately sees them: an insert would collide either way.
+  // Creates only, against LIVE documents (#332): content is not unique across
+  // the store, and a trashed document no longer blocks a new one. This saves
+  // the embedding spend; the RPC repeats the check authoritatively.
   const { data: hashMatch } = await supabase
     .from("cerefox_documents")
-    .select("id, title, deleted_at")
+    .select("id, title")
     .eq("content_hash", contentHash)
+    .is("deleted_at", null)
+    .order("created_at", { ascending: true })
     .limit(1);
 
   if (hashMatch?.length) {
-    if (hashMatch[0].deleted_at) {
-      // "Already up-to-date" while search shows nothing is the classic
-      // re-upload-after-delete confusion. Say where the content actually is.
-      return (
-        `This exact content is already in the TRASH as "${hashMatch[0].title}" ` +
-        `(soft-deleted, id: ${hashMatch[0].id}). Restore it with ` +
-        `cerefox_restore_document instead of re-ingesting — or have your user ` +
-        `purge it from the web UI Trash to start fresh.`
-      );
-    }
-    return `Document already up-to-date: "${hashMatch[0].title}" (id: ${hashMatch[0].id}). Content hash unchanged (${contentHash}) — pass it as expected_content_hash to edit it.`;
+    return alreadyExistsMessage(hashMatch[0].title, hashMatch[0].id, contentHash);
   }
 
   const chunks = chunkMarkdown(content);
@@ -391,7 +403,20 @@ async function handler(
     p_content_format: CONTENT_FORMAT_BLIND_STITCH,
   });
 
+  // The RPC's duplicate check caught an identical create that landed after
+  // the pre-check above. Nothing was written.
+  const duplicateId = duplicateContentDocumentId(ingestErr?.message ?? "");
+  if (duplicateId) {
+    const { data: existing } = await supabase
+      .from("cerefox_documents")
+      .select("title")
+      .eq("id", duplicateId)
+      .maybeSingle();
+    return alreadyExistsMessage(existing?.title ?? title, duplicateId, contentHash);
+  }
   if (ingestErr || !ingestResult?.length) {
+    const legacy = legacyContentUniqueError(ingestErr?.message ?? "");
+    if (legacy) throw legacy;
     throw new Error(`Ingest RPC failed: ${ingestErr?.message ?? "no data returned"}`);
   }
 
@@ -424,6 +449,10 @@ async function handler(
   );
 }
 
+function alreadyExistsMessage(title: string, id: string, contentHash: string): string {
+  return `Document already up-to-date: "${title}" (id: ${id}). Content hash unchanged (${contentHash}) — pass it as expected_content_hash to edit it.`;
+}
+
 export const ingestTool: ToolDefinition = {
   name: "cerefox_ingest",
   description:
@@ -477,7 +506,16 @@ export const ingestTool: ToolDefinition = {
         description:
           "Explicitly skip the concurrency check and overwrite regardless of concurrent changes (default: false). Use ONLY when an external source of truth makes conflicts meaningless (e.g. re-syncing from files). Recorded in the audit log.",
       },
-      metadata: { type: "object", description: "Arbitrary JSON metadata (optional)" },
+      metadata: {
+        type: "object",
+        description:
+          "JSON metadata (optional). On an UPDATE it is MERGED into the existing metadata: keys you pass are set, a null value removes that key, keys you omit are kept. To change only metadata, use cerefox_set_document_metadata instead.",
+      },
+      expected_metadata_revision: {
+        type: "integer",
+        description:
+          "Optional, with metadata on an update: the metadata_revision you read (cerefox_get_document / cerefox_metadata_search). The write fails if the metadata changed since.",
+      },
       author: AUTHOR_PARAM_WRITE,
     },
   },

package/dist/server-assets/_shared/mcp-tools/metadata-search.ts

@@ -89,6 +89,7 @@ async function handler(
     project_names: string[];
     version_count: number;
     content_hash: string | null;
+    metadata_revision?: number | null;
     content: string | null;
   }>;
 
@@ -203,7 +204,8 @@ async function handler(
     const meta = Object.entries(row.doc_metadata ?? {})
       .map(([k, v]) => `${k}=${v}`)
       .join(", ");
-    const hash = row.content_hash ? `\nhash: ${row.content_hash}` : "";
+    const rev = row.metadata_revision != null ? ` · metadata_revision: ${row.metadata_revision}` : "";
+    const hash = row.content_hash ? `\nhash: ${row.content_hash}${rev}` : "";
     const header =
       `## ${row.title} [id: ${row.document_id}]\n` +
       `${meta}${projects} | ${row.total_chars} chars | ${showReview ? `${row.review_status} | ` : ""}updated ${row.updated_at?.slice(0, 10) ?? "?"}${hash}`;

package/dist/server-assets/_shared/mcp-tools/partial-edits.ts

@@ -36,7 +36,12 @@ import {
   sha256hex,
 } from "./_chunker.ts";
 import { activeEmbedderName, embedBatch, resolveEmbedderKind } from "../embeddings/index.ts";
-import { extractConflictHashes, isMissingFunctionError, logUsage } from "./_utils.ts";
+import {
+  extractConflictHashes,
+  isMissingFunctionError,
+  legacyContentUniqueError,
+  logUsage,
+} from "./_utils.ts";
 import { McpInvalidParams, type MCPSupabaseClient, type ToolContext, type ToolDefinition } from "./types.ts";
 import { AUTHOR_PARAM_WRITE, DEFAULT_IDENTITY, callerIdentity } from "./identity.ts";
 
@@ -329,18 +334,6 @@ async function applyAndWrite(
     if (message.includes("CEREFOX_CONFLICT")) {
       throw conflictError(documentId, expectedHash, extractConflictHashes(message).current);
     }
-    if (message.includes("cerefox_documents_hash_unique")) {
-      // content_hash is UNIQUE store-wide, so an edit whose result matches
-      // ANOTHER document collides. Raw, this reads as a database fault; it is
-      // actually a meaningful statement about the store.
-      throw new Error(
-        `This edit would make "${doc.title}" (id: ${documentId}) byte-identical to another ` +
-          `document in the store, and content must be unique. No write was performed. ` +
-          `Usually this means the two documents have converged and one should be removed or ` +
-          `merged, or that this edit was already applied to the other one — ` +
-          `cerefox_search for the resulting content to find it.`,
-      );
-    }
     if (message.includes("CEREFOX_UNRESOLVED_LINKS")) {
       // Only ids INTRODUCED by this write reject (the RPC tolerates dead
       // links the document already carried), so the diagnosis is specific.
@@ -365,6 +358,8 @@ async function applyAndWrite(
           `Run \`cerefox server deploy\`, then retry. (${message})`,
       );
     }
+    const legacy = legacyContentUniqueError(message);
+    if (legacy) throw legacy;
     throw new Error(`Edit failed: ${message}`);
   }
 

package/dist/server-assets/_shared/mcp-tools/search.ts

@@ -43,6 +43,7 @@ interface SearchRow {
   chunk_count?: number;
   total_chars?: number;
   content_hash?: string;
+  metadata_revision?: number;
   below_confidence?: boolean;
 }
 
@@ -104,6 +105,14 @@ function shortLabel(row: SearchRow): string {
   return `${leaf ? `${doc} › ${leaf}` : doc}${chunk}${id}`;
 }
 
+/** The concurrency tokens: content_hash for cerefox_ingest updates (iter-32),
+ *  metadata_revision for metadata writes (#333). */
+function hashLine(row: SearchRow): string {
+  if (!row.content_hash) return "";
+  const rev = row.metadata_revision != null ? ` · metadata_revision: ${row.metadata_revision}` : "";
+  return `\nhash: ${row.content_hash}${rev}`;
+}
+
 /** One rendered result: heading, score, then the row's text. */
 function renderRow(row: SearchRow): string {
   const raw = row.best_score ?? row.score;
@@ -112,7 +121,7 @@ function renderRow(row: SearchRow): string {
     ? ` -- partial (${row.chunk_count} of ${(row.total_chars ?? 0).toLocaleString()} chars)`
     : "";
   // content_hash = the concurrency token for cerefox_ingest updates (iter-32).
-  const hash = row.content_hash ? `\nhash: ${row.content_hash}` : "";
+  const hash = hashLine(row);
   // Whichever column this mode returns (#259): `cerefox_search_docs` gives
   // `full_content`, the chunk RPCs give `content`, and reading only the first
   // rendered every hybrid/fts result as a title with an empty body.
@@ -124,7 +133,7 @@ function headerLine(row: SearchRow): string {
   const raw = row.best_score ?? row.score;
   const score = raw != null ? ` (score: ${raw.toFixed(3)})` : "";
   const size = row.total_chars != null ? ` -- ${row.total_chars.toLocaleString()} chars` : "";
-  const hash = row.content_hash ? `\nhash: ${row.content_hash}` : "";
+  const hash = hashLine(row);
   return `## ${rowHeading(row)}${score}${size}${hash}`;
 }
 

package/dist/server-assets/_shared/mcp-tools/set-document-metadata.ts

@@ -19,7 +19,7 @@
 
 import type { MCPSupabaseClient } from "./types.ts";
 
-import { logUsage } from "./_utils.ts";
+import { logUsage, metadataTokenError, metadataTokenParams } from "./_utils.ts";
 import { McpInvalidParams, type ToolContext, type ToolDefinition } from "./types.ts";
 import { AUTHOR_PARAM_WRITE, DEFAULT_IDENTITY, callerIdentity } from "./identity.ts";
 
@@ -31,6 +31,14 @@ async function handler(
   const document_id = args.document_id as string | undefined;
   const metadata = args.metadata as Record<string, unknown> | undefined;
   const replace = (args.replace as boolean | undefined) ?? false;
+  const expectedRevision = args.expected_metadata_revision;
+  if (
+    expectedRevision !== undefined && expectedRevision !== null &&
+    !(typeof expectedRevision === "number" && Number.isInteger(expectedRevision))
+  ) {
+    throw new McpInvalidParams("expected_metadata_revision must be an integer (the metadata_revision you read)");
+  }
+  const lastWriteWins = (args.last_write_wins as boolean | undefined) ?? false;
 
   if (!document_id) throw new McpInvalidParams("document_id is required");
   if (metadata === undefined || metadata === null) {
@@ -60,12 +68,19 @@ async function handler(
     p_replace: replace,
     p_author: author ?? "unknown",
     p_author_type: authorType,
+    ...metadataTokenParams(expectedRevision as number | null | undefined, lastWriteWins),
   });
 
-  if (error) throw new Error(`RPC error: ${error.message}`);
+  if (error) throw metadataTokenError(error.message) ?? new Error(`RPC error: ${error.message}`);
 
   const row = data?.[0] as
-    | { document_id?: string; metadata?: Record<string, unknown>; keys_set?: number; keys_removed?: number }
+    | {
+        document_id?: string;
+        metadata?: Record<string, unknown>;
+        keys_set?: number;
+        keys_removed?: number;
+        metadata_revision?: number;
+      }
     | undefined;
   if (!row) throw new Error("cerefox_set_document_metadata returned no data");
 
@@ -89,6 +104,7 @@ async function handler(
   return (
     `Metadata ${replace ? "replaced" : "merged"} on ${document_id}. ${summary}\n` +
     `Now: ${JSON.stringify(row.metadata ?? {})}\n` +
+    (row.metadata_revision != null ? `metadata_revision: ${row.metadata_revision}\n` : "") +
     `Content untouched — no new version, no re-embedding.`
   );
 }
@@ -96,7 +112,7 @@ async function handler(
 export const setDocumentMetadataTool: ToolDefinition = {
   name: "cerefox_set_document_metadata",
   description:
-    "Change a document's metadata WITHOUT resending its content. MERGES by default: the keys you pass are set, every other key is left alone — so you do not need to read the document first, and you cannot accidentally drop tags another agent set. To REMOVE a key, pass it with a null value ({\"stale_key\": null}). Pass replace: true to set the metadata to exactly the object given, discarding everything else (rare; the same destructive contract as cerefox_set_document_projects). Content, chunks and embeddings are untouched and no new version is created. Use this instead of cerefox_ingest whenever only the tags are changing.",
+    "Change a document's metadata WITHOUT resending its content. MERGES by default: the keys you pass are set, every other key is left alone — so you do not need to read the document first, and you cannot accidentally drop tags another agent set. To REMOVE a key, pass it with a null value ({\"stale_key\": null}). Pass replace: true to set the metadata to exactly the object given, discarding everything else (rare; it requires expected_metadata_revision or last_write_wins). To make a merge fail if anyone changed the metadata since you read it, pass expected_metadata_revision. Content, chunks and embeddings are untouched and no new version is created. Use this instead of cerefox_ingest whenever only the tags are changing.",
   annotations: {
     title: "Set document metadata",
     readOnlyHint: false,
@@ -118,7 +134,17 @@ export const setDocumentMetadataTool: ToolDefinition = {
       replace: {
         type: "boolean",
         description:
-          "Set the metadata to EXACTLY this object, discarding any key not listed. Defaults to false (merge). Use only when you mean to reset a document's tags wholesale.",
+          "Set the metadata to EXACTLY this object, discarding any key not listed. Defaults to false (merge). Use only when you mean to reset a document's tags wholesale. Requires expected_metadata_revision (or last_write_wins).",
+      },
+      expected_metadata_revision: {
+        type: "integer",
+        description:
+          "The metadata_revision you read (from cerefox_get_document or cerefox_metadata_search). Optional on a merge: when given, the write fails if the metadata changed since. Required with replace: true.",
+      },
+      last_write_wins: {
+        type: "boolean",
+        description:
+          "Replace without a metadata_revision, regardless of concurrent changes (default false). Recorded in the audit log. Use only when an external source of truth makes conflicts meaningless.",
       },
       author: AUTHOR_PARAM_WRITE,
     },

package/dist/server-assets/db/migrations/0012_content_format.sql

@@ -2,7 +2,7 @@
 --
 -- Records how each chunk's content reconstructs into full document text:
 --   1 = legacy    — chunk contents were trimmed sections; reconstruction re-joins
---                   them with E'\n\n' (the pre-28D behaviour). All existing chunks.
+--                   them with E'\n\n' (the pre-28D behavior). All existing chunks.
 --   2 = blind-stitch — chunk contents are an exact, gapless partition of the
 --                   document; reconstruction is a plain concatenation (no separator
 --                   synthesized on read). Written by the exact-partition chunker.

package/dist/server-assets/db/migrations/0014_document_relations.sql

@@ -12,7 +12,7 @@ CREATE TABLE IF NOT EXISTS cerefox_document_relations (
     source_id    UUID        NOT NULL REFERENCES cerefox_documents(id) ON DELETE CASCADE,
     target_id    UUID        NOT NULL REFERENCES cerefox_documents(id) ON DELETE CASCADE,
     -- Free-text by design: agents define new types without a migration. The
-    -- type dictionary (in the RPCs) gives known types behaviour; unknown types
+    -- type dictionary (in the RPCs) gives known types behavior; unknown types
     -- are stored and returned, just without special handling.
     rel_type     TEXT        NOT NULL,
     metadata     JSONB       NOT NULL DEFAULT '{}'::jsonb,

package/dist/server-assets/db/migrations/0020_ingest_preserves_source.sql

@@ -55,5 +55,5 @@ BEGIN
         'Migration 0020: cerefox_ingest_document now preserves a document''s '
         'source when p_source is omitted. Before this, any content update '
         'without an explicit source silently rewrote provenance to ''agent'', '
-        'and migrate-format relabelled every document it converted (#191).';
+        'and migrate-format relabeled every document it converted (#191).';
 END $$;

package/dist/server-assets/db/migrations/0022_rls_on_document_relations.sql

@@ -26,7 +26,7 @@
 
 ALTER TABLE cerefox_document_relations ENABLE ROW LEVEL SECURITY;
 
--- Defence in depth: revoke the legacy blanket grants so the table is denied by
+-- Defense in depth: revoke the legacy blanket grants so the table is denied by
 -- privilege as well as by RLS. Harmless where the grants were never made.
 REVOKE ALL ON TABLE cerefox_document_relations FROM anon;
 

package/dist/server-assets/db/migrations/0032_derive_grants.sql

@@ -32,7 +32,7 @@ BEGIN
 
     GRANT USAGE ON SCHEMA public TO service_role;
 
-    -- Derived from the catalogue, not from a list someone has to remember.
+    -- Derived from the catalog, not from a list someone has to remember.
     FOR t IN
         SELECT c.relname
         FROM pg_class c

package/dist/server-assets/db/migrations/0034_drop_vector_indexes.sql

@@ -0,0 +1,22 @@
+-- 0034_drop_vector_indexes.sql — drop the unused HNSW vector indexes (schema
+-- 0.19.0, release 1.18.0).
+--
+-- Since 0.18.1 the hybrid search RPCs rank vector candidates by an exact scan
+-- (`ORDER BY vec_score DESC`, see cerefox_hybrid_search in rpcs.sql), because
+-- the HNSW index answered the bare-distance ordering with at most
+-- `hnsw.ef_search` (40) rows, and on a store with heavy version churn those
+-- were mostly not the nearest ones. The two indexes have served no query since,
+-- while every chunk write still paid to maintain them and their quality kept
+-- degrading.
+--
+-- The exact scan is fast at knowledge-base scale: measured on a 1-CPU / 1 GB
+-- Postgres with 768-dim vectors, top-500 candidates took 2 ms at 2k chunks,
+-- 56 ms at 50k and 181 ms at 100k. A default-configured HNSW index was slower
+-- than that from 50k chunks and recalled 27% of the true top-250 at 100k.
+-- `cerefox doctor` reports the current chunk count and warns past 100k; a
+-- tuned index path for stores that large is tracked separately.
+--
+-- Idempotent: safe to re-run.
+
+DROP INDEX IF EXISTS idx_cerefox_chunks_emb_primary;
+DROP INDEX IF EXISTS idx_cerefox_chunks_emb_upgrade;

package/dist/server-assets/db/migrations/0035_drop_content_hash_unique.sql

@@ -0,0 +1,21 @@
+-- 0035_drop_content_hash_unique.sql — content no longer has to be unique
+-- across the store (schema 0.20.0, #332).
+--
+-- `cerefox_documents_hash_unique` made every document's body unique store-wide,
+-- trashed documents included. It was meant to stop the same file being
+-- ingested twice, but as an unconditional constraint it also refused
+-- legitimate writes: an update whose new body matched any OTHER document (two
+-- unrelated documents converging on a short or templated body, a revision
+-- back to an earlier state), and any write matching a document in the trash,
+-- which the caller cannot see.
+--
+-- Duplicate detection now happens only where it is useful, when CREATING a
+-- document, and only against live documents: `cerefox_ingest_document`
+-- returns the existing document with operation 'skipped' instead of inserting
+-- a copy. Updates are never refused for matching another document.
+--
+-- The plain index idx_cerefox_docs_hash keeps hash lookups fast.
+--
+-- Idempotent: safe to re-run.
+
+ALTER TABLE cerefox_documents DROP CONSTRAINT IF EXISTS cerefox_documents_hash_unique;

package/dist/server-assets/db/migrations/0036_metadata_revision.sql

@@ -0,0 +1,43 @@
+-- 0036_metadata_revision.sql — a concurrency token and a history for metadata
+-- (schema 0.21.0, #333).
+--
+-- Optimistic concurrency covered the body only: content_hash does not move on
+-- a metadata change, and no metadata write checked a token, so two writers
+-- setting the same key, a `replace`, or a content update carrying metadata
+-- could silently overwrite another writer's change. Metadata also had no
+-- history: content versions never captured it and audit entries recorded only
+-- key counts.
+--
+-- 1. cerefox_documents.metadata_revision: the metadata's own token, bumped by a
+--    trigger whenever metadata actually changes (every writer is covered).
+-- 2. cerefox_audit_log.metadata_before / metadata_after: written on
+--    'update-metadata' entries, so any change can be reconstructed.
+--
+-- The RPCs that check the token and write the history ship in rpcs.sql.
+-- Idempotent: safe to re-run.
+
+ALTER TABLE cerefox_documents
+    ADD COLUMN IF NOT EXISTS metadata_revision INT NOT NULL DEFAULT 1;
+
+ALTER TABLE cerefox_audit_log
+    ADD COLUMN IF NOT EXISTS metadata_before JSONB,
+    ADD COLUMN IF NOT EXISTS metadata_after  JSONB;
+
+CREATE OR REPLACE FUNCTION cerefox_bump_metadata_revision()
+RETURNS TRIGGER
+LANGUAGE plpgsql
+SET search_path = public, pg_catalog
+AS $$
+BEGIN
+    IF NEW.metadata IS DISTINCT FROM OLD.metadata THEN
+        NEW.metadata_revision := OLD.metadata_revision + 1;
+    ELSE
+        NEW.metadata_revision := OLD.metadata_revision;
+    END IF;
+    RETURN NEW;
+END;
+$$;
+
+CREATE OR REPLACE TRIGGER trig_cerefox_documents_metadata_revision
+    BEFORE UPDATE ON cerefox_documents
+    FOR EACH ROW EXECUTE FUNCTION cerefox_bump_metadata_revision();