@cerefox/memory 0.11.1 → 1.0.0-beta.2

package/dist/server-assets/_shared/mcp-auth/index.ts

@@ -0,0 +1,352 @@
+/**
+ * mcp-auth — in-function authentication for the `cerefox-mcp` Edge Function.
+ *
+ * Context: to be an OAuth 2.1 protected resource server, `cerefox-mcp` must serve
+ * unauthenticated discovery routes and issue its own 401 challenges. That requires
+ * deploying it with `--no-verify-jwt` (the Supabase gateway otherwise rejects the
+ * request before the function runs). With the gateway gate removed, **this module
+ * becomes the only auth gate** — see the invariants in
+ * `docs/specs/oauth-mcp-server-design.md` §5–6.
+ *
+ * Two accepted credentials (design §5):
+ *   Path 1 — an optional static Bearer, constant-time compared against an
+ *            explicitly-set value. Left unset by cerefox-mcp since iter-28E (its
+ *            static path is the Cerefox access token, checked in the EF via
+ *            _shared/ef-auth); kept here as a generic, fail-closed capability.
+ *   Path 2 — an OAuth 2.1 access token: a project-signed JWT validated against the
+ *            project JWKS (asymmetric alg allowlist, iss/aud/exp/nbf, owner `sub`).
+ *
+ * Portability: this file uses ONLY Web Platform globals (`crypto.subtle`, `fetch`,
+ * `atob`, `TextEncoder`) — no `node:`, `jsr:`, or `npm:` imports — so the identical
+ * source runs under Deno (the Edge Function) and Bun (`bun test`). JWT verification
+ * is implemented directly on SubtleCrypto rather than pulling in `jose`; ES256/RS256
+ * JWS signatures are already in the raw formats SubtleCrypto's `verify` expects.
+ */
+
+// ── Types ────────────────────────────────────────────────────────────────────
+
+export type AuthPath = "static" | "oauth";
+
+export interface AuthSuccess {
+  ok: true;
+  path: AuthPath;
+  /** The authenticated subject (owner user id) for OAuth; undefined for static. */
+  sub?: string;
+}
+
+export interface AuthFailure {
+  ok: false;
+  /** Machine-readable reason, also used to shape the WWW-Authenticate challenge. */
+  reason:
+    | "no_token"
+    | "malformed_token"
+    | "bad_signature"
+    | "bad_claims"
+    | "not_owner"
+    | "no_verifier";
+  /** Human detail for logs (never returned to the client). */
+  detail?: string;
+}
+
+export type AuthResult = AuthSuccess | AuthFailure;
+
+interface Jwk {
+  kty: string;
+  kid?: string;
+  alg?: string;
+  crv?: string;
+  n?: string;
+  e?: string;
+  x?: string;
+  y?: string;
+  use?: string;
+}
+
+interface Jwks {
+  keys: Jwk[];
+}
+
+export interface McpAuthConfig {
+  /** Expected token issuer, e.g. `https://<ref>.supabase.co/auth/v1`. */
+  issuer: string;
+  /** JWKS URL, e.g. `<issuer>/.well-known/jwks.json`. */
+  jwksUri: string;
+  /** Expected `aud` claim. Supabase issues `"authenticated"`. */
+  expectedAudience: string;
+  /**
+   * Pinned owner user id. When set, an OAuth token's `sub` MUST equal it. When
+   * null/undefined the OAuth path FAILS CLOSED (rejects) unless `allowAnyUser` is
+   * true — because with Supabase's default email sign-ups on, an unpinned server
+   * would accept any self-registered user's token (design §6 / Finding 3).
+   */
+  ownerUserId?: string | null;
+  /**
+   * Explicit opt-out of the owner pin: accept any validly-signed `authenticated`
+   * token when `ownerUserId` is unset. For deliberate multi-user / sign-ups-disabled
+   * setups only. Default false (fail closed when unpinned).
+   */
+  allowAnyUser?: boolean;
+  /**
+   * Expected value for the optional static-Bearer path. When null/undefined the
+   * static path is disabled and rejects everything (fail-closed — never
+   * accept-all). cerefox-mcp leaves this null since iter-28E (its static path is
+   * the Cerefox access token, validated in the EF via `_shared/ef-auth`).
+   */
+  staticBearer?: string | null;
+  /** Accepted JWS algorithms. Default `["ES256", "RS256"]`. Never HS256/none. */
+  allowedAlgs?: string[];
+  /** Clock skew tolerance in seconds. Default 60. */
+  clockSkewSec?: number;
+  /** JWKS cache TTL in seconds. Default 600. */
+  jwksCacheTtlSec?: number;
+  /** Injectable clock (ms since epoch). Default `Date.now`. */
+  now?: () => number;
+  /** Injectable fetch (for tests). Default global `fetch`. */
+  fetchImpl?: typeof fetch;
+}
+
+export interface McpAuthenticator {
+  authenticate(authorizationHeader: string | null): Promise<AuthResult>;
+}
+
+// ── Base64url / encoding helpers ─────────────────────────────────────────────
+
+function base64UrlToBytes(input: string): Uint8Array {
+  const b64 = input.replace(/-/g, "+").replace(/_/g, "/");
+  const pad = b64.length % 4 === 0 ? "" : "=".repeat(4 - (b64.length % 4));
+  const bin = atob(b64 + pad);
+  const out = new Uint8Array(bin.length);
+  for (let i = 0; i < bin.length; i++) out[i] = bin.charCodeAt(i);
+  return out;
+}
+
+/**
+ * Copy a view's bytes into a fresh, plain `ArrayBuffer`. SubtleCrypto expects a
+ * `BufferSource`; under strict typed-array generics (TS 5.7) a `Uint8Array` may be
+ * inferred as `ArrayBufferLike`-backed, so we normalize to `ArrayBuffer` here.
+ */
+function toArrayBuffer(view: Uint8Array): ArrayBuffer {
+  return view.buffer.slice(view.byteOffset, view.byteOffset + view.byteLength) as ArrayBuffer;
+}
+
+function base64UrlToString(input: string): string {
+  return new TextDecoder().decode(base64UrlToBytes(input));
+}
+
+/**
+ * Length-independent constant-time string comparison. Iterates over the longer
+ * length so the loop count doesn't leak which operand is shorter, and folds a
+ * length mismatch into the result rather than short-circuiting.
+ */
+export function constantTimeEqual(a: string, b: string): boolean {
+  const ab = new TextEncoder().encode(a);
+  const bb = new TextEncoder().encode(b);
+  const len = Math.max(ab.length, bb.length);
+  let diff = ab.length ^ bb.length;
+  for (let i = 0; i < len; i++) {
+    diff |= (ab[i] ?? 0) ^ (bb[i] ?? 0);
+  }
+  return diff === 0;
+}
+
+// ── JWT verification (SubtleCrypto) ──────────────────────────────────────────
+
+interface JwtHeader {
+  alg: string;
+  kid?: string;
+  typ?: string;
+}
+
+interface JwtPayload {
+  iss?: string;
+  sub?: string;
+  aud?: string | string[];
+  exp?: number;
+  nbf?: number;
+  iat?: number;
+  role?: string;
+  client_id?: string;
+  [k: string]: unknown;
+}
+
+function importParamsFor(alg: string, jwk: Jwk): {
+  importAlgo: EcKeyImportParams | RsaHashedImportParams;
+  verifyAlgo: EcdsaParams | AlgorithmIdentifier;
+} | null {
+  if (alg === "ES256" && jwk.kty === "EC") {
+    return {
+      importAlgo: { name: "ECDSA", namedCurve: "P-256" },
+      verifyAlgo: { name: "ECDSA", hash: { name: "SHA-256" } },
+    };
+  }
+  if (alg === "RS256" && jwk.kty === "RSA") {
+    return {
+      importAlgo: { name: "RSASSA-PKCS1-v1_5", hash: "SHA-256" },
+      verifyAlgo: { name: "RSASSA-PKCS1-v1_5" },
+    };
+  }
+  return null;
+}
+
+async function verifyJwtSignature(
+  token: string,
+  header: JwtHeader,
+  jwks: Jwks,
+): Promise<boolean> {
+  const parts = token.split(".");
+  if (parts.length !== 3) return false;
+  const [headerB64, payloadB64, sigB64] = parts;
+
+  // Select the key by kid; if the token has no kid, accept a lone matching key.
+  const candidates = jwks.keys.filter((k) => {
+    if (header.kid) return k.kid === header.kid;
+    return true;
+  });
+  if (candidates.length === 0) return false;
+
+  const signature = toArrayBuffer(base64UrlToBytes(sigB64));
+  const signed = toArrayBuffer(new TextEncoder().encode(`${headerB64}.${payloadB64}`));
+
+  for (const jwk of candidates) {
+    const params = importParamsFor(header.alg, jwk);
+    if (!params) continue;
+    try {
+      const key = await crypto.subtle.importKey(
+        "jwk",
+        jwk as JsonWebKey,
+        params.importAlgo,
+        false,
+        ["verify"],
+      );
+      const ok = await crypto.subtle.verify(params.verifyAlgo, key, signature, signed);
+      if (ok) return true;
+    } catch {
+      // try the next candidate key
+    }
+  }
+  return false;
+}
+
+// ── Authenticator factory ────────────────────────────────────────────────────
+
+const BEARER_PREFIX = "Bearer ";
+
+export function createMcpAuthenticator(config: McpAuthConfig): McpAuthenticator {
+  const allowedAlgs = config.allowedAlgs ?? ["ES256", "RS256"];
+  const clockSkewSec = config.clockSkewSec ?? 60;
+  const jwksCacheTtlSec = config.jwksCacheTtlSec ?? 600;
+  const now = config.now ?? (() => Date.now());
+  const fetchImpl = config.fetchImpl ?? fetch;
+
+  // Isolate-lifetime JWKS cache (closure state). Key rotation is picked up on
+  // TTL expiry or isolate recycle.
+  let cache: { jwks: Jwks; fetchedAt: number } | null = null;
+
+  async function getJwks(): Promise<Jwks | null> {
+    if (cache && now() - cache.fetchedAt < jwksCacheTtlSec * 1000) {
+      return cache.jwks;
+    }
+    try {
+      const resp = await fetchImpl(config.jwksUri);
+      if (!resp.ok) return cache?.jwks ?? null;
+      const jwks = (await resp.json()) as Jwks;
+      if (!jwks || !Array.isArray(jwks.keys)) return cache?.jwks ?? null;
+      cache = { jwks, fetchedAt: now() };
+      return jwks;
+    } catch {
+      // On a transient fetch failure, fall back to a still-cached set if any;
+      // otherwise fail closed (caller rejects).
+      return cache?.jwks ?? null;
+    }
+  }
+
+  function validateClaims(payload: JwtPayload): AuthResult {
+    const nowSec = Math.floor(now() / 1000);
+
+    if (payload.iss !== config.issuer) {
+      return { ok: false, reason: "bad_claims", detail: `iss=${payload.iss} want=${config.issuer}` };
+    }
+    const auds = Array.isArray(payload.aud) ? payload.aud : [payload.aud];
+    if (!auds.includes(config.expectedAudience)) {
+      return {
+        ok: false,
+        reason: "bad_claims",
+        detail: `aud=${JSON.stringify(payload.aud)} want=${config.expectedAudience}`,
+      };
+    }
+    if (typeof payload.exp !== "number" || payload.exp + clockSkewSec < nowSec) {
+      return { ok: false, reason: "bad_claims", detail: `expired (exp=${payload.exp})` };
+    }
+    if (typeof payload.nbf === "number" && payload.nbf - clockSkewSec > nowSec) {
+      return { ok: false, reason: "bad_claims", detail: "not yet valid" };
+    }
+    if (!payload.sub) {
+      return { ok: false, reason: "bad_claims", detail: "missing sub" };
+    }
+    if (config.ownerUserId) {
+      if (payload.sub !== config.ownerUserId) {
+        return { ok: false, reason: "not_owner", detail: `sub=${payload.sub} owner=${config.ownerUserId}` };
+      }
+    } else if (!config.allowAnyUser) {
+      // Fail closed: no owner pinned and no explicit opt-out. Accepting here would
+      // let any self-registered user in (Supabase email sign-ups default on).
+      return {
+        ok: false,
+        reason: "not_owner",
+        detail: "owner not pinned; set CEREFOX_OAUTH_OWNER_ID (or CEREFOX_OAUTH_ALLOW_ANY_USER=true to opt out)",
+      };
+    }
+    return { ok: true, path: "oauth", sub: payload.sub };
+  }
+
+  async function authenticate(authorizationHeader: string | null): Promise<AuthResult> {
+    if (!authorizationHeader || !authorizationHeader.startsWith(BEARER_PREFIX)) {
+      return { ok: false, reason: "no_token" };
+    }
+    const token = authorizationHeader.slice(BEARER_PREFIX.length).trim();
+    if (!token) return { ok: false, reason: "no_token" };
+
+    // Path 1 — legacy static Bearer (constant-time). Fail-closed when unset.
+    if (config.staticBearer && constantTimeEqual(token, config.staticBearer)) {
+      return { ok: true, path: "static" };
+    }
+
+    // Path 2 — OAuth access token (JWT against JWKS).
+    const parts = token.split(".");
+    if (parts.length !== 3) {
+      return { ok: false, reason: "malformed_token", detail: "not a JWT" };
+    }
+    let header: JwtHeader;
+    let payload: JwtPayload;
+    try {
+      header = JSON.parse(base64UrlToString(parts[0])) as JwtHeader;
+      payload = JSON.parse(base64UrlToString(parts[1])) as JwtPayload;
+    } catch {
+      return { ok: false, reason: "malformed_token", detail: "bad JSON" };
+    }
+
+    // Algorithm allowlist BEFORE any crypto — reject none/HS256 outright, which
+    // is what defends against alg-confusion with the HS256 legacy anon JWT.
+    if (!allowedAlgs.includes(header.alg)) {
+      return { ok: false, reason: "bad_signature", detail: `alg ${header.alg} not allowed` };
+    }
+
+    const jwks = await getJwks();
+    if (!jwks) return { ok: false, reason: "no_verifier", detail: "JWKS unavailable" };
+
+    const verified = await verifyJwtSignature(token, header, jwks);
+    if (!verified) {
+      return {
+        ok: false,
+        reason: "bad_signature",
+        detail: `signature invalid (alg=${header.alg} kid=${header.kid} jwks_kids=${
+          jwks.keys.map((k) => k.kid).join(",")
+        })`,
+      };
+    }
+
+    return validateClaims(payload);
+  }
+
+  return { authenticate };
+}

package/dist/server-assets/_shared/mcp-tools/_chunker.ts

@@ -1,179 +1,25 @@
 /**
- * Heading-aware markdown chunker.
+ * Chunker + content-hash utilities for the MCP ingest tool.
  *
- * Mirrors:
- * - `src/cerefox/chunking/markdown.py` (Python pipeline)
- * - `supabase/functions/cerefox-ingest/index.ts` (standalone ingest EF)
- *
- * Greedy section accumulation: H1/H2/H3 sections are joined into a buffer
- * until adding the next would exceed `MAX_CHUNK_CHARS`. Oversized sections
- * are paragraph-split. Short documents collapse to a single chunk.
- *
- * The hash of the chunked output (via `_hash.ts:sha256hex(normalizeContent(...))`)
- * must match the Python pipeline byte-for-byte so dedup works across access
- * paths. Don't change chunk boundaries without updating both.
+ * iter-28D Phase 1: the chunker is now the single exact-partition implementation
+ * in `_shared/ingest/chunker.ts` — this module re-exports it (the previous copy
+ * here was removed) and keeps the content-hash helpers used for dedup.
  */
 
-export const MAX_CHUNK_CHARS = 4000;
-
-interface Section {
-  level: number;
-  headings: string[];
-  heading: string;
-  content: string;
-  body: string;
-}
-
-export interface Chunk {
-  heading_path: string[];
-  heading_level: number;
-  title: string;
-  content: string;
-  char_count: number;
-}
-
-function parseSections(text: string): Section[] {
-  const lines = text.split("\n");
-  const sections: Section[] = [];
-  let currentHeadings: string[] = [];
-  let currentLevel = 0;
-  let bodyLines: string[] = [];
-
-  function collectSection() {
-    const body = bodyLines.join("\n").trim();
-    bodyLines = [];
-    let content: string;
-    if (currentLevel > 0) {
-      const headerLine = "#".repeat(currentLevel) + " " +
-        (currentHeadings[currentHeadings.length - 1] ?? "");
-      content = body ? headerLine + "\n\n" + body : headerLine;
-    } else {
-      content = body;
-    }
-    if (!content.trim()) return;
-    sections.push({
-      level: currentLevel,
-      headings: [...currentHeadings],
-      heading: currentHeadings[currentHeadings.length - 1] ?? "",
-      content,
-      body,
-    });
-  }
-
-  for (const line of lines) {
-    const h1 = line.match(/^# (.+)/);
-    const h2 = line.match(/^## (.+)/);
-    const h3 = line.match(/^### (.+)/);
-
-    if (h1) {
-      collectSection();
-      currentHeadings = [h1[1].trim()];
-      currentLevel = 1;
-    } else if (h2) {
-      collectSection();
-      currentHeadings = [currentHeadings[0] ?? "", h2[1].trim()].filter(Boolean);
-      currentLevel = 2;
-    } else if (h3) {
-      collectSection();
-      currentHeadings = [
-        currentHeadings[0] ?? "",
-        currentHeadings[1] ?? "",
-        h3[1].trim(),
-      ].filter(Boolean);
-      currentLevel = 3;
-    } else {
-      bodyLines.push(line);
-    }
-  }
-  collectSection();
-  return sections;
-}
-
-function makeChunk(headings: string[], level: number, content: string): Chunk {
-  const title = headings[headings.length - 1] ?? "";
-  return {
-    heading_path: [...headings],
-    heading_level: level,
-    title,
-    content,
-    char_count: content.length,
-  };
-}
+export {
+  chunkMarkdown,
+  embeddingInputFor,
+  CONTENT_FORMAT_BLIND_STITCH,
+  type ChunkData,
+  type ChunkData as Chunk,
+} from "../ingest/chunker.ts";
 
-export function chunkMarkdown(text: string): Chunk[] {
-  const trimmed = text.trim();
-  if (!trimmed) return [];
-
-  if (trimmed.length <= MAX_CHUNK_CHARS) {
-    return [makeChunk([], 0, trimmed)];
-  }
-
-  const sections = parseSections(trimmed);
-  const chunks: Chunk[] = [];
-
-  let bufParts: string[] = [];
-  let bufHeadings: string[] = [];
-  let bufLevel = 0;
-  let bufChars = 0;
-
-  function flushBuf() {
-    if (bufParts.length === 0) return;
-    chunks.push(makeChunk(bufHeadings, bufLevel, bufParts.join("\n\n")));
-    bufParts = [];
-    bufHeadings = [];
-    bufLevel = 0;
-    bufChars = 0;
-  }
-
-  for (const section of sections) {
-    const { level, headings, heading, content, body } = section;
-
-    if (content.length > MAX_CHUNK_CHARS) {
-      flushBuf();
-      const headerPrefix = level > 0 ? "#".repeat(level) + " " + heading + "\n\n" : "";
-      const bodyToSplit = body || content;
-      const paragraphs = bodyToSplit.split(/\n\n+/);
-      let sub = "";
-      let isFirst = true;
-      for (const para of paragraphs) {
-        const prefix = isFirst ? headerPrefix : "";
-        if (sub.length + prefix.length + para.length + 2 > MAX_CHUNK_CHARS && sub.length > 0) {
-          chunks.push(makeChunk(headings, level, sub.trim()));
-          sub = para;
-          isFirst = false;
-        } else {
-          sub = sub ? sub + "\n\n" + para : prefix + para;
-          isFirst = false;
-        }
-      }
-      if (sub.trim()) chunks.push(makeChunk(headings, level, sub.trim()));
-      continue;
-    }
-
-    const addition = content.length + (bufParts.length > 0 ? 2 : 0);
-
-    if (bufChars + addition <= MAX_CHUNK_CHARS) {
-      if (bufParts.length === 0) {
-        bufHeadings = headings;
-        bufLevel = level;
-      }
-      bufParts.push(content);
-      bufChars += addition;
-    } else {
-      flushBuf();
-      bufParts = [content];
-      bufHeadings = headings;
-      bufLevel = level;
-      bufChars = content.length;
-    }
-  }
-
-  flushBuf();
-  return chunks;
-}
+export const MAX_CHUNK_CHARS = 4000;
 
-/** Content-hash normalization. Must match `pipeline.py::_normalize`
- *  byte-for-byte so cross-runtime dedup works. */
+/**
+ * Content-hash normalization. Kept stable so the content_hash dedup key is
+ * consistent across access paths and over time.
+ */
 export function normalizeContent(text: string): string {
   return text.trim().replace(/\r\n/g, "\n").replace(/\r/g, "\n").replace(/\n{3,}/g, "\n\n");
 }

package/dist/server-assets/_shared/mcp-tools/get-help-content.ts

@@ -11,7 +11,7 @@
  * docs/specs/polish-and-distribution-design.md §10d.
  */
 
-export const HELP_FULL = "# Cerefox Knowledge Base -- Agent Quick Reference\n\nCerefox is a persistent, shared knowledge base. You have **10 MCP tools** (9 of them have CLI equivalents — `cerefox_get_help` is MCP-only). For the full guide, search Cerefox for \"How AI Agents Use Cerefox\" or call `cerefox_get_help` to retrieve this content over MCP.\n\n## Tools\n\n| Tool | Purpose | Key params |\n|------|---------|------------|\n| `cerefox_search` | Find documents (hybrid FTS + semantic) | `query` (required), `project_name`, `metadata_filter`, `requestor` |\n| `cerefox_ingest` | Save or update a document | `title`, `content` (required), `document_id` (update by ID), `expected_content_hash` (**required on content updates** — see rule 9), `last_write_wins`, `update_if_exists`, `project_name` (single, non-destructive add on update), `project_names` (list, destructive replace on update), `metadata` (omit on update to keep existing tags; `{}` clears), `author` |\n| `cerefox_get_document` | Get full document by ID (header includes `content_hash` — the update token) | `document_id` (required) |\n| `cerefox_list_versions` | Version history of a document | `document_id` (required) |\n| `cerefox_metadata_search` | Find or list docs by metadata, project, or time (no text query) | `metadata_filter`, `project_name` (list a project's docs), `updated_since`, `include_content` — **at least one** of metadata_filter/project_name/updated_since/created_since |\n| `cerefox_list_metadata_keys` | Discover available metadata keys | (none required) |\n| `cerefox_list_projects` | List all projects | (none required) |\n| `cerefox_set_document_projects` | Set doc's project memberships to exactly the given list (destructive replace; metadata-only, no content change) | `document_id`, `project_names` (required) |\n| `cerefox_get_audit_log` | Query write operation history | `document_id`, `author`, `operation`, `since` |\n| `cerefox_get_help` | Retrieve Cerefox conventions (this reference) over MCP. **Call this whenever uncertain.** | `topic` (optional, case-insensitive H2 substring match) |\n\n## Essential Rules\n\n1. **Search before ingesting** -- check if the document exists first.\n2. **Prefer ID-based updates** -- pass `document_id` from search results for deterministic updates. Falls back to title-matching with `update_if_exists: true`.\n3. **Set `author`/`requestor`** to your name on every call (e.g., \"Claude Code\", \"archiver\"). On MCP, pass as parameters. On CLI, pass `--author`/`--author-type`/`--requestor` flags, or rely on `CEREFOX_AUTHOR_NAME`/`CEREFOX_AUTHOR_TYPE`/`CEREFOX_REQUESTOR_NAME` env vars set in the user's `.env`.\n4. **Use `document_id` from search results** `[id: uuid]` for get_document and list_versions.\n5. **Add metadata** -- at minimum `type` (\"decision-log\", \"research\", \"design-doc\") and `status` (\"active\", \"draft\").\n6. **Write structured Markdown** with H1/H2/H3 headings for good chunking and search.\n7. **Deletes are soft (recoverable); purge is web-UI-only.** If you decide to delete, surface it to the user (`I soft-deleted X — recoverable from the Cerefox web UI trash`). You cannot un-do your own delete from agent code by design.\n8. **Cross-doc links inside content**: **always use `[Text](document-uuid)`.** UUIDs are the only fully reliable link form — stable across title changes, never ambiguous, no encoding gotchas. Every `cerefox_search` result shows `[id: <uuid>]` after the title; grab it and use it. Title-based linking (`[Text](<Title With Spaces>)`) is fragile (breaks on colons, parens, ampersands, brackets — silently navigates to wrong page) — **don't write title-based links**; do an extra search to get the UUID instead. Repo-path forms (`[Text](docs/path.md)`) exist for repo-ingested files; don't construct manually. See `AGENT_GUIDE.md → Writing linkable content` for the full rule.\n9. **Concurrency: content updates require `expected_content_hash`.** Pass the `content_hash` you read (shown by `cerefox_get_document`, `cerefox_search`, and `cerefox_metadata_search`) when updating a document. If it's stale you get a **conflict** — re-read the document, merge your changes into the latest content, retry with the new hash. **Never resolve a conflict by overwriting blindly** — the current content includes another writer's work. `last_write_wins: true` skips the check; use it ONLY when an external source of truth makes conflicts meaningless (file re-sync), never to silence a conflict.\n10. **Project memberships — non-destructive by default**: on `cerefox_ingest` updates, **`project_name` (singular) is a non-destructive add** (ensures membership, preserves others). Use **`project_names` (list)** when you want to set the doc's full project set in one call (destructive replace). For metadata-only project changes without writing content, use **`cerefox_set_document_projects(document_id, project_names)`** — that tool is the destructive-replace contract made explicit. Never call `cerefox_set_document_projects` with a single name when you mean \"add\" — that would REMOVE the doc from all other projects. When in doubt, use `cerefox_ingest` with singular `project_name`.\n\n## Update Workflow (ID-based -- preferred)\n\n```\nsearch(\"topic\") -> find doc [id: abc123] -> get_document(abc123) -> note its content_hash -> modify ->\ningest(title=\"Same Title\", content=\"...\", document_id=\"abc123\",\n       expected_content_hash=\"<the hash you read>\", author=\"my-agent\")\n```\n\nOn a **conflict** error: get_document again (fresh content + fresh hash) -> merge your changes -> retry with the new hash.\n\n## Update Workflow (title-based -- fallback)\n\n```\nsearch(\"topic\") -> find doc (note its hash) -> modify ->\ningest(title=\"Same Title\", content=\"...\", update_if_exists=true,\n       expected_content_hash=\"<the hash you read>\", author=\"my-agent\")\n```\n\n## Catch-Up Workflow\n\n```\nmetadata_search(metadata_filter={\"type\": \"decision-log\"}, updated_since=\"2026-03-28T00:00:00Z\")\n```\n\n## CLI fallback (when MCP is unavailable)\n\nIf `cerefox_search` is not in your tool list, your user has likely installed the Cerefox CLI. The canonical invocation is plain **`cerefox <subcommand>`** (the TypeScript CLI, installed via `npm install -g @cerefox/memory`). It uses a resource-verb shape (`cerefox document get`, `cerefox project list`, …). The legacy Python `uv run cerefox` is now a frozen husk as of v0.9 — only `uv run cerefox mcp` still works.\n\nSame operations, same conventions. Full reference: [`docs/guides/cli.md`](docs/guides/cli.md). CLI flag names match MCP parameter names exactly (e.g. `metadata_filter` ↔ `--metadata-filter`); common flags also have single-letter short forms (`-f`, `-p`, `-c`, `-m`, `-u`, `-a`, `-r`). Use the canonical long name (what `--help` shows) or its short form — there are no long-form aliases like `--filter` or `--count`.\n\n| MCP tool | CLI |\n|---|---|\n| `cerefox_search` | `cerefox search \"<q>\" --requestor \"<your-name>\"` |\n| `cerefox_ingest` (paste) | `printf '...' \\| cerefox document ingest --paste --title \"<t>\" --author \"<your-name>\" --author-type agent` |\n| `cerefox_ingest` (update by ID) | `printf '...' \\| cerefox document ingest --paste --title \"<t>\" --document-id \"<uuid>\" --expected-content-hash \"<hash>\" --author \"<your-name>\" --author-type agent` |\n| `cerefox_get_document` | `cerefox document get <id> --version-id <vid> --requestor \"<your-name>\"` |\n| `cerefox_list_versions` | `cerefox document version list <id> --requestor \"<your-name>\"` |\n| `cerefox_list_projects` | `cerefox project list --requestor \"<your-name>\"` |\n| `cerefox_list_metadata_keys` | `cerefox metadata keys` |\n| `cerefox_metadata_search` | `cerefox metadata search --metadata-filter '<json>' --requestor \"<your-name>\"` (list a project: `cerefox document list --project <name>`) |\n| `cerefox_set_document_projects` | `cerefox document set-projects <id> <name...> --author \"<your-name>\" --author-type agent` (or `--clear` to remove all) |\n| `cerefox_get_audit_log` | `cerefox audit list --requestor \"<your-name>\"` (add `--json` for scripted access) |\n| `cerefox_get_help` | `cerefox guides show agent-quick-reference` (or `cerefox guides list` for the full bundled-docs index) |\n\n**Set identity on every call**, exactly as you would on MCP:\n- Writes (`document ingest`, `document ingest-dir`): `--author \"<your-name>\" --author-type agent`\n- Reads: `--requestor \"<your-name>\"`\n\nOr have your user set `CEREFOX_AUTHOR_NAME` / `CEREFOX_AUTHOR_TYPE` / `CEREFOX_REQUESTOR_NAME` in their `.env` to apply defaults once.\n";
+export const HELP_FULL = "# Cerefox Knowledge Base -- Agent Quick Reference\n\nCerefox is a persistent, shared knowledge base. You have **10 MCP tools** (9 of them have CLI equivalents — `cerefox_get_help` is MCP-only). For the full guide, search Cerefox for \"How AI Agents Use Cerefox\" or call `cerefox_get_help` to retrieve this content over MCP.\n\n## Tools\n\n| Tool | Purpose | Key params |\n|------|---------|------------|\n| `cerefox_search` | Find documents (hybrid FTS + semantic) | `query` (required), `project_name`, `metadata_filter`, `requestor` |\n| `cerefox_ingest` | Save or update a document | `title`, `content` (required), `document_id` (update by ID), `expected_content_hash` (**required on content updates** — see rule 9), `last_write_wins`, `update_if_exists`, `project_name` (single, non-destructive add on update), `project_names` (list, destructive replace on update), `metadata` (omit on update to keep existing tags; `{}` clears), `author` |\n| `cerefox_get_document` | Get full document by ID (header includes `content_hash` — the update token) | `document_id` (required) |\n| `cerefox_list_versions` | Version history of a document | `document_id` (required) |\n| `cerefox_metadata_search` | Find or list docs by metadata, project, or time (no text query) | `metadata_filter`, `project_name` (list a project's docs), `updated_since`, `include_content` — **at least one** of metadata_filter/project_name/updated_since/created_since |\n| `cerefox_list_metadata_keys` | Discover available metadata keys | (none required) |\n| `cerefox_list_projects` | List all projects | (none required) |\n| `cerefox_set_document_projects` | Set doc's project memberships to exactly the given list (destructive replace; metadata-only, no content change) | `document_id`, `project_names` (required) |\n| `cerefox_get_audit_log` | Query write operation history | `document_id`, `author`, `operation`, `since` |\n| `cerefox_get_help` | Retrieve Cerefox conventions (this reference) over MCP. **Call this whenever uncertain.** | `topic` (optional, case-insensitive H2 substring match) |\n\n## Essential Rules\n\n1. **Search before ingesting** -- check if the document exists first.\n2. **Prefer ID-based updates** -- pass `document_id` from search results for deterministic updates. Falls back to title-matching with `update_if_exists: true`.\n3. **Set `author`/`requestor`** to your name on every call (e.g., \"Claude Code\", \"archiver\"). On MCP, pass as parameters. On CLI, pass `--author`/`--author-type`/`--requestor` flags, or rely on `CEREFOX_AUTHOR_NAME`/`CEREFOX_AUTHOR_TYPE`/`CEREFOX_REQUESTOR_NAME` env vars set in the user's `.env`.\n4. **Use `document_id` from search results** `[id: uuid]` for get_document and list_versions.\n5. **Add metadata** -- at minimum `type` (\"decision-log\", \"research\", \"design-doc\") and `status` (\"active\", \"draft\").\n6. **Write structured Markdown** with H1/H2/H3 headings for good chunking and search.\n7. **Deletes are soft (recoverable); purge is web-UI-only.** If you decide to delete, surface it to the user (`I soft-deleted X — recoverable from the Cerefox web UI trash`). You cannot un-do your own delete from agent code by design.\n8. **Cross-doc links inside content**: **always use `[Text](document-uuid)`.** UUIDs are the only fully reliable link form — stable across title changes, never ambiguous, no encoding gotchas. Every `cerefox_search` result shows `[id: <uuid>]` after the title; grab it and use it. Title-based linking (`[Text](<Title With Spaces>)`) is fragile (breaks on colons, parens, ampersands, brackets — silently navigates to wrong page) — **don't write title-based links**; do an extra search to get the UUID instead. Repo-path forms (`[Text](docs/path.md)`) exist for repo-ingested files; don't construct manually. See `AGENT_GUIDE.md → Writing linkable content` for the full rule.\n9. **Concurrency: content updates require `expected_content_hash`.** Pass the `content_hash` you read (shown by `cerefox_get_document`, `cerefox_search`, and `cerefox_metadata_search`) when updating a document. If it's stale you get a **conflict** — re-read the document, merge your changes into the latest content, retry with the new hash. **Never resolve a conflict by overwriting blindly** — the current content includes another writer's work. `last_write_wins: true` skips the check; use it ONLY when an external source of truth makes conflicts meaningless (file re-sync), never to silence a conflict.\n10. **Project memberships — non-destructive by default**: on `cerefox_ingest` updates, **`project_name` (singular) is a non-destructive add** (ensures membership, preserves others). Use **`project_names` (list)** when you want to set the doc's full project set in one call (destructive replace). For metadata-only project changes without writing content, use **`cerefox_set_document_projects(document_id, project_names)`** — that tool is the destructive-replace contract made explicit. Never call `cerefox_set_document_projects` with a single name when you mean \"add\" — that would REMOVE the doc from all other projects. When in doubt, use `cerefox_ingest` with singular `project_name`.\n\n## Update Workflow (ID-based -- preferred)\n\n```\nsearch(\"topic\") -> find doc [id: abc123] -> get_document(abc123) -> note its content_hash -> modify ->\ningest(title=\"Same Title\", content=\"...\", document_id=\"abc123\",\n       expected_content_hash=\"<the hash you read>\", author=\"my-agent\")\n```\n\nOn a **conflict** error: get_document again (fresh content + fresh hash) -> merge your changes -> retry with the new hash.\n\n## Update Workflow (title-based -- fallback)\n\n```\nsearch(\"topic\") -> find doc (note its hash) -> modify ->\ningest(title=\"Same Title\", content=\"...\", update_if_exists=true,\n       expected_content_hash=\"<the hash you read>\", author=\"my-agent\")\n```\n\n## Catch-Up Workflow\n\n```\nmetadata_search(metadata_filter={\"type\": \"decision-log\"}, updated_since=\"2026-03-28T00:00:00Z\")\n```\n\n## CLI fallback (when MCP is unavailable)\n\nIf `cerefox_search` is not in your tool list, your user has likely installed the Cerefox CLI. The canonical invocation is plain **`cerefox <subcommand>`** (the TypeScript CLI, installed via `npm install -g @cerefox/memory`). It uses a resource-verb shape (`cerefox document get`, `cerefox project list`, …).\n\nSame operations, same conventions. Full reference: [`docs/guides/cli.md`](docs/guides/cli.md). CLI flag names match MCP parameter names exactly (e.g. `metadata_filter` ↔ `--metadata-filter`); common flags also have single-letter short forms (`-f`, `-p`, `-c`, `-m`, `-u`, `-a`, `-r`). Use the canonical long name (what `--help` shows) or its short form — there are no long-form aliases like `--filter` or `--count`.\n\n| MCP tool | CLI |\n|---|---|\n| `cerefox_search` | `cerefox search \"<q>\" --requestor \"<your-name>\"` |\n| `cerefox_ingest` (paste) | `printf '...' \\| cerefox document ingest --paste --title \"<t>\" --author \"<your-name>\" --author-type agent` |\n| `cerefox_ingest` (update by ID) | `printf '...' \\| cerefox document ingest --paste --title \"<t>\" --document-id \"<uuid>\" --expected-content-hash \"<hash>\" --author \"<your-name>\" --author-type agent` |\n| `cerefox_get_document` | `cerefox document get <id> --version-id <vid> --requestor \"<your-name>\"` |\n| `cerefox_list_versions` | `cerefox document version list <id> --requestor \"<your-name>\"` |\n| `cerefox_list_projects` | `cerefox project list --requestor \"<your-name>\"` |\n| `cerefox_list_metadata_keys` | `cerefox metadata keys` |\n| `cerefox_metadata_search` | `cerefox metadata search --metadata-filter '<json>' --requestor \"<your-name>\"` (list a project: `cerefox document list --project <name>`) |\n| `cerefox_set_document_projects` | `cerefox document set-projects <id> <name...> --author \"<your-name>\" --author-type agent` (or `--clear` to remove all) |\n| `cerefox_get_audit_log` | `cerefox audit list --requestor \"<your-name>\"` (add `--json` for scripted access) |\n| `cerefox_get_help` | `cerefox guides show agent-quick-reference` (or `cerefox guides list` for the full bundled-docs index) |\n\n**Set identity on every call**, exactly as you would on MCP:\n- Writes (`document ingest`, `document ingest-dir`): `--author \"<your-name>\" --author-type agent`\n- Reads: `--requestor \"<your-name>\"`\n\nOr have your user set `CEREFOX_AUTHOR_NAME` / `CEREFOX_AUTHOR_TYPE` / `CEREFOX_REQUESTOR_NAME` in their `.env` to apply defaults once.\n";
 
 /** Sections keyed by their H2 heading text (lower-cased for matching). */
 export const HELP_SECTIONS: Record<string, string> = {
@@ -20,7 +20,7 @@ export const HELP_SECTIONS: Record<string, string> = {
   "Update Workflow (ID-based -- preferred)": "## Update Workflow (ID-based -- preferred)\n\n```\nsearch(\"topic\") -> find doc [id: abc123] -> get_document(abc123) -> note its content_hash -> modify ->\ningest(title=\"Same Title\", content=\"...\", document_id=\"abc123\",\n       expected_content_hash=\"<the hash you read>\", author=\"my-agent\")\n```\n\nOn a **conflict** error: get_document again (fresh content + fresh hash) -> merge your changes -> retry with the new hash.",
   "Update Workflow (title-based -- fallback)": "## Update Workflow (title-based -- fallback)\n\n```\nsearch(\"topic\") -> find doc (note its hash) -> modify ->\ningest(title=\"Same Title\", content=\"...\", update_if_exists=true,\n       expected_content_hash=\"<the hash you read>\", author=\"my-agent\")\n```",
   "Catch-Up Workflow": "## Catch-Up Workflow\n\n```\nmetadata_search(metadata_filter={\"type\": \"decision-log\"}, updated_since=\"2026-03-28T00:00:00Z\")\n```",
-  "CLI fallback (when MCP is unavailable)": "## CLI fallback (when MCP is unavailable)\n\nIf `cerefox_search` is not in your tool list, your user has likely installed the Cerefox CLI. The canonical invocation is plain **`cerefox <subcommand>`** (the TypeScript CLI, installed via `npm install -g @cerefox/memory`). It uses a resource-verb shape (`cerefox document get`, `cerefox project list`, …). The legacy Python `uv run cerefox` is now a frozen husk as of v0.9 — only `uv run cerefox mcp` still works.\n\nSame operations, same conventions. Full reference: [`docs/guides/cli.md`](docs/guides/cli.md). CLI flag names match MCP parameter names exactly (e.g. `metadata_filter` ↔ `--metadata-filter`); common flags also have single-letter short forms (`-f`, `-p`, `-c`, `-m`, `-u`, `-a`, `-r`). Use the canonical long name (what `--help` shows) or its short form — there are no long-form aliases like `--filter` or `--count`.\n\n| MCP tool | CLI |\n|---|---|\n| `cerefox_search` | `cerefox search \"<q>\" --requestor \"<your-name>\"` |\n| `cerefox_ingest` (paste) | `printf '...' \\| cerefox document ingest --paste --title \"<t>\" --author \"<your-name>\" --author-type agent` |\n| `cerefox_ingest` (update by ID) | `printf '...' \\| cerefox document ingest --paste --title \"<t>\" --document-id \"<uuid>\" --expected-content-hash \"<hash>\" --author \"<your-name>\" --author-type agent` |\n| `cerefox_get_document` | `cerefox document get <id> --version-id <vid> --requestor \"<your-name>\"` |\n| `cerefox_list_versions` | `cerefox document version list <id> --requestor \"<your-name>\"` |\n| `cerefox_list_projects` | `cerefox project list --requestor \"<your-name>\"` |\n| `cerefox_list_metadata_keys` | `cerefox metadata keys` |\n| `cerefox_metadata_search` | `cerefox metadata search --metadata-filter '<json>' --requestor \"<your-name>\"` (list a project: `cerefox document list --project <name>`) |\n| `cerefox_set_document_projects` | `cerefox document set-projects <id> <name...> --author \"<your-name>\" --author-type agent` (or `--clear` to remove all) |\n| `cerefox_get_audit_log` | `cerefox audit list --requestor \"<your-name>\"` (add `--json` for scripted access) |\n| `cerefox_get_help` | `cerefox guides show agent-quick-reference` (or `cerefox guides list` for the full bundled-docs index) |\n\n**Set identity on every call**, exactly as you would on MCP:\n- Writes (`document ingest`, `document ingest-dir`): `--author \"<your-name>\" --author-type agent`\n- Reads: `--requestor \"<your-name>\"`\n\nOr have your user set `CEREFOX_AUTHOR_NAME` / `CEREFOX_AUTHOR_TYPE` / `CEREFOX_REQUESTOR_NAME` in their `.env` to apply defaults once.",
+  "CLI fallback (when MCP is unavailable)": "## CLI fallback (when MCP is unavailable)\n\nIf `cerefox_search` is not in your tool list, your user has likely installed the Cerefox CLI. The canonical invocation is plain **`cerefox <subcommand>`** (the TypeScript CLI, installed via `npm install -g @cerefox/memory`). It uses a resource-verb shape (`cerefox document get`, `cerefox project list`, …).\n\nSame operations, same conventions. Full reference: [`docs/guides/cli.md`](docs/guides/cli.md). CLI flag names match MCP parameter names exactly (e.g. `metadata_filter` ↔ `--metadata-filter`); common flags also have single-letter short forms (`-f`, `-p`, `-c`, `-m`, `-u`, `-a`, `-r`). Use the canonical long name (what `--help` shows) or its short form — there are no long-form aliases like `--filter` or `--count`.\n\n| MCP tool | CLI |\n|---|---|\n| `cerefox_search` | `cerefox search \"<q>\" --requestor \"<your-name>\"` |\n| `cerefox_ingest` (paste) | `printf '...' \\| cerefox document ingest --paste --title \"<t>\" --author \"<your-name>\" --author-type agent` |\n| `cerefox_ingest` (update by ID) | `printf '...' \\| cerefox document ingest --paste --title \"<t>\" --document-id \"<uuid>\" --expected-content-hash \"<hash>\" --author \"<your-name>\" --author-type agent` |\n| `cerefox_get_document` | `cerefox document get <id> --version-id <vid> --requestor \"<your-name>\"` |\n| `cerefox_list_versions` | `cerefox document version list <id> --requestor \"<your-name>\"` |\n| `cerefox_list_projects` | `cerefox project list --requestor \"<your-name>\"` |\n| `cerefox_list_metadata_keys` | `cerefox metadata keys` |\n| `cerefox_metadata_search` | `cerefox metadata search --metadata-filter '<json>' --requestor \"<your-name>\"` (list a project: `cerefox document list --project <name>`) |\n| `cerefox_set_document_projects` | `cerefox document set-projects <id> <name...> --author \"<your-name>\" --author-type agent` (or `--clear` to remove all) |\n| `cerefox_get_audit_log` | `cerefox audit list --requestor \"<your-name>\"` (add `--json` for scripted access) |\n| `cerefox_get_help` | `cerefox guides show agent-quick-reference` (or `cerefox guides list` for the full bundled-docs index) |\n\n**Set identity on every call**, exactly as you would on MCP:\n- Writes (`document ingest`, `document ingest-dir`): `--author \"<your-name>\" --author-type agent`\n- Reads: `--requestor \"<your-name>\"`\n\nOr have your user set `CEREFOX_AUTHOR_NAME` / `CEREFOX_AUTHOR_TYPE` / `CEREFOX_REQUESTOR_NAME` in their `.env` to apply defaults once.",
 };
 
 export const HELP_SECTION_HEADINGS: string[] = ["Tools", "Essential Rules", "Update Workflow (ID-based -- preferred)", "Update Workflow (title-based -- fallback)", "Catch-Up Workflow", "CLI fallback (when MCP is unavailable)"];

package/dist/server-assets/_shared/mcp-tools/ingest.ts

@@ -17,7 +17,13 @@
 
 import type { MCPSupabaseClient } from "./types.ts";
 
-import { chunkMarkdown, normalizeContent, sha256hex } from "./_chunker.ts";
+import {
+  chunkMarkdown,
+  embeddingInputFor,
+  CONTENT_FORMAT_BLIND_STITCH,
+  normalizeContent,
+  sha256hex,
+} from "./_chunker.ts";
 import { embedBatch, OPENAI_MODEL } from "../embeddings/index.ts";
 import { ensureDocumentInProject, setDocumentProjectsByName } from "./_projects.ts";
 import { logUsage } from "./_utils.ts";
@@ -137,7 +143,7 @@ async function handler(
     const chunks = chunkMarkdown(content);
     if (chunks.length === 0) throw new Error("Content produced no chunks");
 
-    const texts = chunks.map((c) => `# ${title}\n${c.content}`);
+    const texts = chunks.map((c) => embeddingInputFor(title, c));
     const embeddings = await embedBatch(texts, ctx.openaiApiKey);
     const totalChars = chunks.reduce((s, c) => s + c.char_count, 0);
 
@@ -165,6 +171,7 @@ async function handler(
       p_source_label: source,
       p_expected_content_hash: expected_content_hash,
       p_last_write_wins: last_write_wins,
+      p_content_format: CONTENT_FORMAT_BLIND_STITCH,
     });
 
     if (ingestErr) throw mapIngestRpcError(ingestErr.message, existingDoc.id);
@@ -214,7 +221,7 @@ async function handler(
       const chunks = chunkMarkdown(content);
       if (chunks.length === 0) throw new Error("Content produced no chunks");
 
-      const texts = chunks.map((c) => `# ${title}\n${c.content}`);
+      const texts = chunks.map((c) => embeddingInputFor(title, c));
       const embeddings = await embedBatch(texts, ctx.openaiApiKey);
       const totalChars = chunks.reduce((s, c) => s + c.char_count, 0);
 
@@ -242,6 +249,7 @@ async function handler(
         p_source_label: source,
         p_expected_content_hash: expected_content_hash,
         p_last_write_wins: last_write_wins,
+      p_content_format: CONTENT_FORMAT_BLIND_STITCH,
       });
 
       if (ingestErr) throw mapIngestRpcError(ingestErr.message, existingDoc.id);
@@ -279,7 +287,7 @@ async function handler(
   const chunks = chunkMarkdown(content);
   if (chunks.length === 0) throw new Error("Content produced no chunks");
 
-  const texts = chunks.map((c) => `# ${title}\n${c.content}`);
+  const texts = chunks.map((c) => embeddingInputFor(title, c));
   const embeddings = await embedBatch(texts, ctx.openaiApiKey);
   const totalChars = chunks.reduce((s, c) => s + c.char_count, 0);
 
@@ -304,6 +312,7 @@ async function handler(
     p_chunks: chunkData,
     p_author: author,
     p_author_type: author_type,
+    p_content_format: CONTENT_FORMAT_BLIND_STITCH,
   });
 
   if (ingestErr || !ingestResult?.length) {