@cerefox/memory 0.11.0 → 1.0.0-beta.1

package/dist/server-assets/_shared/mcp-tools/ingest.ts

@@ -17,7 +17,13 @@
 
 import type { MCPSupabaseClient } from "./types.ts";
 
-import { chunkMarkdown, normalizeContent, sha256hex } from "./_chunker.ts";
+import {
+  chunkMarkdown,
+  embeddingInputFor,
+  CONTENT_FORMAT_BLIND_STITCH,
+  normalizeContent,
+  sha256hex,
+} from "./_chunker.ts";
 import { embedBatch, OPENAI_MODEL } from "../embeddings/index.ts";
 import { ensureDocumentInProject, setDocumentProjectsByName } from "./_projects.ts";
 import { logUsage } from "./_utils.ts";
@@ -71,7 +77,10 @@ async function handler(
   const project_name = args.project_name as string | undefined;
   const project_names_raw = args.project_names;
   const source = (args.source as string | undefined) ?? "agent";
-  const metadata = (args.metadata as Record<string, unknown> | undefined) ?? {};
+  // null = "not provided": the RPC keeps existing metadata on update and uses
+  // {} on create (v0.11.1 — defaulting to {} here used to wipe a document's
+  // tags on every content update that didn't re-pass them).
+  const metadata = (args.metadata as Record<string, unknown> | undefined) ?? null;
   const update_if_exists = (args.update_if_exists as boolean | undefined) ?? false;
   const author = (args.author as string | undefined) ?? "mcp-agent";
   const author_type = "agent"; // MCP path is always agent
@@ -134,7 +143,7 @@ async function handler(
     const chunks = chunkMarkdown(content);
     if (chunks.length === 0) throw new Error("Content produced no chunks");
 
-    const texts = chunks.map((c) => `# ${title}\n${c.content}`);
+    const texts = chunks.map((c) => embeddingInputFor(title, c));
     const embeddings = await embedBatch(texts, ctx.openaiApiKey);
     const totalChars = chunks.reduce((s, c) => s + c.char_count, 0);
 
@@ -162,6 +171,7 @@ async function handler(
       p_source_label: source,
       p_expected_content_hash: expected_content_hash,
       p_last_write_wins: last_write_wins,
+      p_content_format: CONTENT_FORMAT_BLIND_STITCH,
     });
 
     if (ingestErr) throw mapIngestRpcError(ingestErr.message, existingDoc.id);
@@ -211,7 +221,7 @@ async function handler(
       const chunks = chunkMarkdown(content);
       if (chunks.length === 0) throw new Error("Content produced no chunks");
 
-      const texts = chunks.map((c) => `# ${title}\n${c.content}`);
+      const texts = chunks.map((c) => embeddingInputFor(title, c));
       const embeddings = await embedBatch(texts, ctx.openaiApiKey);
       const totalChars = chunks.reduce((s, c) => s + c.char_count, 0);
 
@@ -239,6 +249,7 @@ async function handler(
         p_source_label: source,
         p_expected_content_hash: expected_content_hash,
         p_last_write_wins: last_write_wins,
+      p_content_format: CONTENT_FORMAT_BLIND_STITCH,
       });
 
       if (ingestErr) throw mapIngestRpcError(ingestErr.message, existingDoc.id);
@@ -276,7 +287,7 @@ async function handler(
   const chunks = chunkMarkdown(content);
   if (chunks.length === 0) throw new Error("Content produced no chunks");
 
-  const texts = chunks.map((c) => `# ${title}\n${c.content}`);
+  const texts = chunks.map((c) => embeddingInputFor(title, c));
   const embeddings = await embedBatch(texts, ctx.openaiApiKey);
   const totalChars = chunks.reduce((s, c) => s + c.char_count, 0);
 
@@ -301,6 +312,7 @@ async function handler(
     p_chunks: chunkData,
     p_author: author,
     p_author_type: author_type,
+    p_content_format: CONTENT_FORMAT_BLIND_STITCH,
   });
 
   if (ingestErr || !ingestResult?.length) {

package/dist/server-assets/db/migrations/0012_content_format.sql

@@ -0,0 +1,20 @@
+-- Migration 0012: content_format on cerefox_chunks (iter-28D)
+--
+-- Records how each chunk's content reconstructs into full document text:
+--   1 = legacy    — chunk contents were trimmed sections; reconstruction re-joins
+--                   them with E'\n\n' (the pre-28D behaviour). All existing chunks.
+--   2 = blind-stitch — chunk contents are an exact, gapless partition of the
+--                   document; reconstruction is a plain concatenation (no separator
+--                   synthesized on read). Written by the exact-partition chunker.
+--
+-- Placed on the CHUNK (not the document) so an archived version reconstructs with
+-- its OWN format, since Cerefox uses chunks-anchored versioning
+-- (cerefox_chunks.version_id). The reconstruction RPCs branch on
+-- MAX(content_format) >= 2 per aggregated group.
+--
+-- Adding a NOT NULL column with a constant default is a metadata-only change in
+-- PostgreSQL 11+ (no rewrite of the chunks table). Existing rows read back as 1.
+-- Explanation for users: docs/guides/content-format.md.
+
+ALTER TABLE cerefox_chunks
+    ADD COLUMN IF NOT EXISTS content_format SMALLINT NOT NULL DEFAULT 1;

package/dist/server-assets/db/rpcs.sql

@@ -403,7 +403,7 @@ AS $$
         ARRAY(SELECT p.name FROM cerefox_projects p
               JOIN cerefox_document_projects dp ON p.id = dp.project_id
               WHERE dp.document_id = d.id) AS doc_project_names,
-        STRING_AGG(c.content, E'\n\n' ORDER BY c.chunk_index) AS full_content,
+        CASE WHEN MAX(c.content_format) >= 2 THEN STRING_AGG(c.content, '' ORDER BY c.chunk_index) ELSE STRING_AGG(c.content, E'\n\n' ORDER BY c.chunk_index) END AS full_content,
         COUNT(*)::INT   AS chunk_count,
         SUM(c.char_count)::INT AS total_chars,
         (SELECT COUNT(*)::INT FROM cerefox_document_versions dv
@@ -680,17 +680,18 @@ AS $$
     large_doc_content AS (
         SELECT
             e.document_id,
-            STRING_AGG(e.content, E'\n\n' ORDER BY e.chunk_index) AS full_content,
+            CASE WHEN MAX(ch.content_format) >= 2 THEN STRING_AGG(e.content, '' ORDER BY e.chunk_index) ELSE STRING_AGG(e.content, E'\n\n' ORDER BY e.chunk_index) END AS full_content,
             COUNT(*)::INT AS chunk_count,
             TRUE          AS is_partial
         FROM expanded e
+        JOIN cerefox_chunks ch ON ch.id = e.chunk_id
         GROUP BY e.document_id
     ),
     -- Full content for small documents (is_partial = FALSE).
     small_doc_content AS (
         SELECT
             c.document_id,
-            STRING_AGG(c.content, E'\n\n' ORDER BY c.chunk_index) AS full_content,
+            CASE WHEN MAX(c.content_format) >= 2 THEN STRING_AGG(c.content, '' ORDER BY c.chunk_index) ELSE STRING_AGG(c.content, E'\n\n' ORDER BY c.chunk_index) END AS full_content,
             COUNT(*)::INT AS chunk_count,
             FALSE         AS is_partial
         FROM cerefox_chunks c
@@ -866,7 +867,7 @@ AS $$
               JOIN cerefox_document_projects dp ON p.id = dp.project_id
               WHERE dp.document_id = d.id) AS doc_project_names,
         p_version_id    AS version_id,
-        STRING_AGG(c.content, E'\n\n' ORDER BY c.chunk_index) AS full_content,
+        CASE WHEN MAX(c.content_format) >= 2 THEN STRING_AGG(c.content, '' ORDER BY c.chunk_index) ELSE STRING_AGG(c.content, E'\n\n' ORDER BY c.chunk_index) END AS full_content,
         COUNT(*)::INT   AS chunk_count,
         SUM(c.char_count)::INT AS total_chars,
         d.created_at,
@@ -1045,7 +1046,10 @@ $$;
 --
 -- Parameters:
 --   p_document_id     : NULL for create, UUID for update
---   p_title, p_source, p_source_path, p_content_hash, p_metadata : document fields
+--   p_title, p_source, p_source_path, p_content_hash : document fields
+--   p_metadata        : JSONB metadata. NULL = "not provided" → create uses '{}',
+--                       update keeps the existing metadata (v0.11.1). Pass '{}'
+--                       explicitly to clear all metadata.
 --   p_review_status   : 'approved' or 'pending_review' (based on author_type)
 --   p_chunks          : JSONB array of chunk objects, each with:
 --                        chunk_index, heading_path, heading_level, title,
@@ -1069,13 +1073,17 @@ $$;
 
 DROP FUNCTION IF EXISTS cerefox_ingest_document(UUID, TEXT, TEXT, TEXT, TEXT, JSONB, TEXT, JSONB, TEXT, TEXT, TEXT, INT, BOOLEAN);
 DROP FUNCTION IF EXISTS cerefox_ingest_document(UUID, TEXT, TEXT, TEXT, TEXT, JSONB, TEXT, JSONB, TEXT, TEXT, TEXT, INT, BOOLEAN, TEXT, BOOLEAN);
+DROP FUNCTION IF EXISTS cerefox_ingest_document(UUID, TEXT, TEXT, TEXT, TEXT, JSONB, TEXT, JSONB, TEXT, TEXT, TEXT, INT, BOOLEAN, TEXT, BOOLEAN, SMALLINT);
 CREATE FUNCTION cerefox_ingest_document(
     p_document_id       UUID        DEFAULT NULL,
     p_title             TEXT        DEFAULT 'Untitled',
     p_source            TEXT        DEFAULT 'agent',
     p_source_path       TEXT        DEFAULT NULL,
     p_content_hash      TEXT        DEFAULT '',
-    p_metadata          JSONB       DEFAULT '{}',
+    -- NULL = "not provided": create uses '{}', update KEEPS existing metadata
+    -- (v0.11.1 fix — content updates without metadata used to wipe tags).
+    -- Pass '{}' explicitly to deliberately clear all metadata.
+    p_metadata          JSONB       DEFAULT NULL,
     p_review_status     TEXT        DEFAULT 'approved',
     p_chunks            JSONB       DEFAULT '[]',
     p_author            TEXT        DEFAULT 'unknown',
@@ -1084,7 +1092,11 @@ CREATE FUNCTION cerefox_ingest_document(
     p_retention_hours   INT         DEFAULT 48,
     p_cleanup_enabled   BOOLEAN     DEFAULT TRUE,
     p_expected_content_hash TEXT    DEFAULT NULL,
-    p_last_write_wins   BOOLEAN     DEFAULT FALSE
+    p_last_write_wins   BOOLEAN     DEFAULT FALSE,
+    -- content_format for the chunks being written (iter-28D). 2 = exact-partition
+    -- (blind-stitch reconstruction); default 1 = legacy (E'\n\n'-join). Stamped on
+    -- every chunk this call inserts.
+    p_content_format    SMALLINT    DEFAULT 1
 )
 RETURNS TABLE (
     document_id     UUID,
@@ -1181,13 +1193,14 @@ BEGIN
         SELECT sv.version_id INTO v_version_id
         FROM cerefox_snapshot_version(v_doc_id, p_source_label, p_retention_hours, p_cleanup_enabled) sv;
 
-        -- Update document record
+        -- Update document record. metadata: NULL = keep existing (v0.11.1 —
+        -- a content update without metadata must not wipe the document's tags).
         UPDATE cerefox_documents SET
             title = p_title,
             source = p_source,
             source_path = COALESCE(p_source_path, source_path),
             content_hash = p_content_hash,
-            metadata = p_metadata,
+            metadata = COALESCE(p_metadata, metadata),
             chunk_count = v_chunk_count,
             total_chars = v_total_chars,
             review_status = v_status,
@@ -1202,7 +1215,7 @@ BEGIN
             title, source, source_path, content_hash, metadata,
             chunk_count, total_chars, review_status
         ) VALUES (
-            p_title, p_source, p_source_path, p_content_hash, p_metadata,
+            p_title, p_source, p_source_path, p_content_hash, COALESCE(p_metadata, '{}'::JSONB),
             v_chunk_count, v_total_chars, v_status
         )
         RETURNING id INTO v_doc_id;
@@ -1215,7 +1228,7 @@ BEGIN
     -- Formula: doc_title (A) || chunk_heading (A) || body_content (B)
     INSERT INTO cerefox_chunks (
         document_id, chunk_index, heading_path, heading_level,
-        title, content, char_count, embedding_primary, embedder_primary, fts
+        title, content, char_count, content_format, embedding_primary, embedder_primary, fts
     )
     SELECT
         v_doc_id,
@@ -1225,6 +1238,7 @@ BEGIN
         c->>'title',
         c->>'content',
         (c->>'char_count')::INT,
+        p_content_format,
         (SELECT array_agg(e::FLOAT)::VECTOR(768) FROM jsonb_array_elements_text(c->'embedding') AS e),
         c->>'embedder',
         setweight(to_tsvector('english', COALESCE(p_title, '')), 'A') ||
@@ -1502,7 +1516,7 @@ BEGIN
              WHERE dv.document_id = d.id) AS version_count,
             d.content_hash,
             CASE WHEN p_include_content THEN
-                (SELECT STRING_AGG(c.content, E'\n\n' ORDER BY c.chunk_index)
+                (SELECT CASE WHEN MAX(c.content_format) >= 2 THEN STRING_AGG(c.content, '' ORDER BY c.chunk_index) ELSE STRING_AGG(c.content, E'\n\n' ORDER BY c.chunk_index) END
                  FROM cerefox_chunks c
                  WHERE c.document_id = d.id AND c.version_id IS NULL)
             ELSE NULL END AS content
@@ -1760,7 +1774,30 @@ SET search_path = public, pg_catalog
 AS $$
     -- Keep in lockstep with the `@version:` marker in schema.sql (cut_release.ts
     -- enforces it). Bump whenever schema.sql OR rpcs.sql changes.
-    SELECT '0.5.0'::TEXT;
+    SELECT '0.8.0'::TEXT;
+$$;
+
+-- ── cerefox_content_format_stats ─────────────────────────────────────────────
+-- Counts how many (non-deleted) documents still use the legacy chunk
+-- reconstruction format (content_format = 1) vs the total. Powers the
+-- informational `cerefox doctor` line. See docs/guides/content-format.md.
+CREATE OR REPLACE FUNCTION cerefox_content_format_stats()
+RETURNS TABLE (legacy_docs INT, total_docs INT)
+LANGUAGE sql
+STABLE
+SECURITY DEFINER
+SET search_path = public, pg_catalog
+AS $$
+    SELECT
+        COUNT(*) FILTER (WHERE cf.min_format < 2)::INT AS legacy_docs,
+        COUNT(*)::INT                                  AS total_docs
+    FROM cerefox_documents d
+    LEFT JOIN LATERAL (
+        SELECT MIN(c.content_format) AS min_format
+        FROM cerefox_chunks c
+        WHERE c.document_id = d.id AND c.version_id IS NULL
+    ) cf ON TRUE
+    WHERE d.deleted_at IS NULL;
 $$;
 
 
@@ -1840,3 +1877,41 @@ AS $$
     WHERE a.document_id = ANY(p_doc_ids)
     ORDER BY a.document_id, a.created_at DESC;
 $$;
+
+-- ── Function privilege lockdown (schema 0.7.0) ────────────────────────────────
+-- SECURITY (critical): every cerefox_* RPC is SECURITY DEFINER, so it bypasses the
+-- Row Level Security enabled on the tables. PostgreSQL grants EXECUTE to PUBLIC by
+-- default, and Supabase's PostgREST exposes public-schema functions at
+-- /rest/v1/rpc/<name>. WITHOUT this block, the anon key (and the new sb_publishable_
+-- key, which maps to the same anon role) could call every RPC directly via the Data
+-- API — reading and writing the entire KB, bypassing BOTH the Edge Functions and
+-- RLS. Every legitimate caller uses the service_role key instead (Edge Functions via
+-- SUPABASE_SERVICE_ROLE_KEY; the CLI/web via the secret key; local World B via a
+-- container-minted service_role JWT), so we REVOKE EXECUTE from PUBLIC/anon/
+-- authenticated and GRANT only to service_role.
+--
+-- Applied over ALL cerefox_* functions (so new functions are covered on the next
+-- deploy) and idempotent — re-applied each time rpcs.sql is deployed. Guarded on
+-- role existence so it is safe on non-Supabase Postgres (e.g. World B bootstrap).
+DO $$
+DECLARE
+  fn regprocedure;
+  r  text;
+BEGIN
+  FOR fn IN
+    SELECT p.oid::regprocedure
+    FROM pg_proc p
+    JOIN pg_namespace n ON n.oid = p.pronamespace
+    WHERE n.nspname = 'public' AND p.proname LIKE 'cerefox\_%'
+  LOOP
+    EXECUTE format('REVOKE EXECUTE ON FUNCTION %s FROM PUBLIC', fn);
+    FOREACH r IN ARRAY ARRAY['anon', 'authenticated'] LOOP
+      IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = r) THEN
+        EXECUTE format('REVOKE EXECUTE ON FUNCTION %s FROM %I', fn, r);
+      END IF;
+    END LOOP;
+    IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'service_role') THEN
+      EXECUTE format('GRANT EXECUTE ON FUNCTION %s TO service_role', fn);
+    END IF;
+  END LOOP;
+END $$;

package/dist/server-assets/db/schema.sql

@@ -5,7 +5,7 @@
 -- Requires extensions: vector (pgvector), uuid-ossp
 -- These are enabled at the top of db_deploy.py before this file is applied.
 --
--- @version: 0.5.0
+-- @version: 0.8.0
 -- The `@version` marker above is read by the schema-version-mismatch banner
 -- (see /api/v1/schema-version). Bump it whenever schema.sql OR rpcs.sql
 -- changes in a way that requires `cerefox server deploy` to be re-run —
@@ -157,6 +157,12 @@ CREATE TABLE IF NOT EXISTS cerefox_chunks (
     title           TEXT,
     content         TEXT        NOT NULL,
     char_count      INT         NOT NULL,
+    -- content_format: how this chunk's content reconstructs into full document text
+    -- (iter-28D). 1 = legacy (chunk contents re-joined with E'\n\n' on read); 2 =
+    -- blind-stitch (chunk contents are an exact, gapless partition, reconstructed by
+    -- plain concat). It lives on the chunk (not the document) so an archived version
+    -- reconstructs with its OWN format. See docs/guides/content-format.md.
+    content_format  SMALLINT    NOT NULL DEFAULT 1,
 
     -- Primary embedding: always computed, cloud API (default: OpenAI text-embedding-3-small)
     embedding_primary  VECTOR(768) NOT NULL,

package/dist/server-assets/supabase/functions/cerefox-get-audit-log/index.ts

@@ -1,6 +1,7 @@
 import "jsr:@supabase/functions-js/edge-runtime.d.ts";
 import { createClient } from "jsr:@supabase/supabase-js@2";
 import { isVersionRequest, versionResponse } from "../../../_shared/ef-meta/index.ts";
+import { efAuthGate } from "../../../_shared/ef-auth/index.ts";
 
 /**
  * cerefox-get-audit-log -- Supabase Edge Function
@@ -35,6 +36,13 @@ Deno.serve(async (req: Request): Promise<Response> => {
     return new Response(null, { status: 200, headers: CORS_HEADERS });
   }
 
+  const authFail = efAuthGate(
+    req.headers.get("Authorization"),
+    Deno.env.get("CEREFOX_ACCESS_TOKENS"),
+    { ...CORS_HEADERS, "Content-Type": "application/json" },
+  );
+  if (authFail) return authFail;
+
   if (isVersionRequest(req)) {
     return versionResponse("cerefox-get-audit-log", { ...CORS_HEADERS, "Content-Type": "application/json" });
   }

package/dist/server-assets/supabase/functions/cerefox-get-document/index.ts

@@ -1,6 +1,7 @@
 import "jsr:@supabase/functions-js/edge-runtime.d.ts";
 import { createClient } from "jsr:@supabase/supabase-js@2";
 import { isVersionRequest, versionResponse } from "../../../_shared/ef-meta/index.ts";
+import { efAuthGate } from "../../../_shared/ef-auth/index.ts";
 
 /**
  * cerefox-get-document — Supabase Edge Function
@@ -35,6 +36,13 @@ Deno.serve(async (req: Request): Promise<Response> => {
     return new Response(null, { status: 200, headers: CORS_HEADERS });
   }
 
+  const authFail = efAuthGate(
+    req.headers.get("Authorization"),
+    Deno.env.get("CEREFOX_ACCESS_TOKENS"),
+    { ...CORS_HEADERS, "Content-Type": "application/json" },
+  );
+  if (authFail) return authFail;
+
   if (isVersionRequest(req)) {
     return versionResponse("cerefox-get-document", { ...CORS_HEADERS, "Content-Type": "application/json" });
   }

package/dist/server-assets/supabase/functions/cerefox-ingest/index.ts

@@ -1,6 +1,13 @@
 import "jsr:@supabase/functions-js/edge-runtime.d.ts";
 import { createClient } from "jsr:@supabase/supabase-js@2";
 import { isVersionRequest, versionResponse } from "../../../_shared/ef-meta/index.ts";
+import { efAuthGate } from "../../../_shared/ef-auth/index.ts";
+import { capEmbeddingInput } from "../../../_shared/embeddings/index.ts";
+import {
+  chunkMarkdown,
+  embeddingInputFor,
+  CONTENT_FORMAT_BLIND_STITCH,
+} from "../../../_shared/ingest/chunker.ts";
 
 /**
  * cerefox-ingest — Supabase Edge Function
@@ -17,7 +24,9 @@ import { isVersionRequest, versionResponse } from "../../../_shared/ef-meta/inde
  *   content      string   required  Markdown content
  *   project_name string   optional  Project to assign to (looked up by name, created if absent)
  *   source       string   optional  Origin label (default: "agent")
- *   metadata     object   optional  Arbitrary JSONB metadata
+ *   metadata     object   optional  Arbitrary JSONB metadata. Omitted on an
+ *                                   update → existing metadata is KEPT
+ *                                   (v0.11.1); pass {} explicitly to clear.
  *
  * Response: { document_id, title, chunk_count, project_id? }
  */
@@ -78,173 +87,6 @@ function concurrencyErrorResponse(
   return null;
 }
 
-interface Chunk {
-  heading_path: string[];
-  heading_level: number;
-  title: string;
-  content: string;
-  char_count: number;
-}
-
-// ── Heading-aware chunker (mirrors Python logic) ───────────────────────────
-//
-// Design notes:
-//   • Short-circuit for small documents: if the entire document fits within
-//     MAX_CHUNK_CHARS, it is returned as a single chunk with no splitting.
-//   • Greedy accumulation: sections are collected into a buffer until adding
-//     the next would exceed MAX_CHUNK_CHARS. This keeps chunks close to the
-//     target size and avoids many tiny fragments at every heading boundary.
-//     All heading levels (H1/H2/H3) are treated equally — size alone controls
-//     when a chunk is flushed; there are no hard heading-level boundaries.
-//   • Oversized sections (> MAX_CHUNK_CHARS) are paragraph-split with no overlap.
-//   • The first section's heading metadata anchors each chunk's breadcrumb.
-//   • No overlaps between chunks — the heading breadcrumb in the content
-//     provides sufficient context. Overlaps caused duplication on reconstruction.
-
-interface Section {
-  level: number;
-  headings: string[]; // full heading stack at this section
-  heading: string;    // just the current heading text
-  content: string;    // heading line + body
-  body: string;       // body only (no heading line)
-}
-
-function parseSections(text: string): Section[] {
-  const lines = text.split("\n");
-  const sections: Section[] = [];
-  let currentHeadings: string[] = [];
-  let currentLevel = 0;
-  let bodyLines: string[] = [];
-
-  function collectSection() {
-    const body = bodyLines.join("\n").trim();
-    bodyLines = [];
-    let content: string;
-    if (currentLevel > 0) {
-      const headerLine = "#".repeat(currentLevel) + " " + (currentHeadings[currentHeadings.length - 1] ?? "");
-      content = body ? headerLine + "\n\n" + body : headerLine;
-    } else {
-      content = body;
-    }
-    if (!content.trim()) return;
-    sections.push({
-      level: currentLevel,
-      headings: [...currentHeadings],
-      heading: currentHeadings[currentHeadings.length - 1] ?? "",
-      content,
-      body,
-    });
-  }
-
-  for (const line of lines) {
-    const h1 = line.match(/^# (.+)/);
-    const h2 = line.match(/^## (.+)/);
-    const h3 = line.match(/^### (.+)/);
-
-    if (h1) {
-      collectSection();
-      currentHeadings = [h1[1].trim()];
-      currentLevel = 1;
-    } else if (h2) {
-      collectSection();
-      currentHeadings = [currentHeadings[0] ?? "", h2[1].trim()].filter(Boolean);
-      currentLevel = 2;
-    } else if (h3) {
-      collectSection();
-      currentHeadings = [
-        currentHeadings[0] ?? "",
-        currentHeadings[1] ?? "",
-        h3[1].trim(),
-      ].filter(Boolean);
-      currentLevel = 3;
-    } else {
-      bodyLines.push(line);
-    }
-  }
-  collectSection();
-  return sections;
-}
-
-function chunkMarkdown(text: string): Chunk[] {
-  const trimmed = text.trim();
-  if (!trimmed) return [];
-
-  // Short-circuit: entire document fits in one chunk — skip heading splitting.
-  if (trimmed.length <= MAX_CHUNK_CHARS) {
-    return [makeChunk([], 0, trimmed)];
-  }
-
-  const sections = parseSections(trimmed);
-  const chunks: Chunk[] = [];
-
-  // Greedy accumulation buffer
-  let bufParts: string[] = [];
-  let bufHeadings: string[] = [];
-  let bufLevel = 0;
-  let bufChars = 0;
-
-  function flushBuf() {
-    if (bufParts.length === 0) return;
-    chunks.push(makeChunk(bufHeadings, bufLevel, bufParts.join("\n\n")));
-    bufParts = [];
-    bufHeadings = [];
-    bufLevel = 0;
-    bufChars = 0;
-  }
-
-  for (const section of sections) {
-    const { level, headings, heading, content, body } = section;
-
-    // Oversized section: flush buffer, then paragraph-split.
-    if (content.length > MAX_CHUNK_CHARS) {
-      flushBuf();
-      const headerPrefix = level > 0 ? "#".repeat(level) + " " + heading + "\n\n" : "";
-      const bodyToSplit = body || content;
-      const paragraphs = bodyToSplit.split(/\n\n+/);
-      let sub = "";
-      let isFirst = true;
-      for (const para of paragraphs) {
-        const prefix = isFirst ? headerPrefix : "";
-        if (sub.length + prefix.length + para.length + 2 > MAX_CHUNK_CHARS && sub.length > 0) {
-          chunks.push(makeChunk(headings, level, sub.trim()));
-          sub = para;
-          isFirst = false;
-        } else {
-          sub = sub ? sub + "\n\n" + para : prefix + para;
-          isFirst = false;
-        }
-      }
-      if (sub.trim()) chunks.push(makeChunk(headings, level, sub.trim()));
-      continue;
-    }
-
-    // Section fits. Try to accumulate into the buffer.
-    const addition = content.length + (bufParts.length > 0 ? 2 : 0);
-
-    if (bufChars + addition <= MAX_CHUNK_CHARS) {
-      if (bufParts.length === 0) {
-        bufHeadings = headings;
-        bufLevel = level;
-      }
-      bufParts.push(content);
-      bufChars += addition;
-    } else {
-      flushBuf();
-      bufParts = [content];
-      bufHeadings = headings;
-      bufLevel = level;
-      bufChars = content.length;
-    }
-  }
-
-  flushBuf();
-  return chunks;
-}
-
-function makeChunk(headings: string[], level: number, content: string): Chunk {
-  const title = headings[headings.length - 1] ?? "";
-  return { heading_path: [...headings], heading_level: level, title, content, char_count: content.length };
-}
 
 // ── Embedding ──────────────────────────────────────────────────────────────
 
@@ -253,6 +95,7 @@ const EMBEDDING_INITIAL_BACKOFF_MS = 500; // 500ms, 1s, 2s exponential backoff
 
 async function embedBatch(texts: string[], apiKey: string): Promise<number[][]> {
   let lastError: Error | null = null;
+  const inputs = texts.map(capEmbeddingInput); // iter-28D Phase 0: cap oversized inputs
 
   for (let attempt = 0; attempt < EMBEDDING_MAX_RETRIES; attempt++) {
     try {
@@ -264,7 +107,7 @@ async function embedBatch(texts: string[], apiKey: string): Promise<number[][]>
         },
         body: JSON.stringify({
           model: OPENAI_MODEL,
-          input: texts,
+          input: inputs,
           dimensions: EMBEDDING_DIMENSIONS,
         }),
       });
@@ -440,6 +283,13 @@ Deno.serve(async (req: Request) => {
     });
   }
 
+  const authFail = efAuthGate(
+    req.headers.get("Authorization"),
+    Deno.env.get("CEREFOX_ACCESS_TOKENS"),
+    { "Content-Type": "application/json", "Access-Control-Allow-Origin": "*" },
+  );
+  if (authFail) return authFail;
+
   if (isVersionRequest(req)) {
     return versionResponse("cerefox-ingest", {
       "Content-Type": "application/json",
@@ -464,7 +314,10 @@ Deno.serve(async (req: Request) => {
     });
   }
 
-  const { title, content, document_id = null, project_name, source = "agent", metadata = {}, update_if_exists = false, author = "agent", author_type = "agent", expected_content_hash = null, last_write_wins = false } = body;
+  // metadata: null = "not provided" — the RPC keeps existing metadata on
+  // update and uses {} on create (v0.11.1; a `= {}` default here used to wipe
+  // a document's tags on every content update that didn't re-pass them).
+  const { title, content, document_id = null, project_name, source = "agent", metadata = null, update_if_exists = false, author = "agent", author_type = "agent", expected_content_hash = null, last_write_wins = false } = body;
 
   // Validate + normalize project_names if provided (full-set destructive form)
   let project_names: string[] | null = null;
@@ -577,7 +430,7 @@ Deno.serve(async (req: Request) => {
       return new Response(JSON.stringify({ error: "Content produced no chunks" }), { status: 422, headers });
     }
 
-    const texts = chunks.map((c) => `# ${title.trim()}\n${c.content}`);
+    const texts = chunks.map((c) => embeddingInputFor(title.trim(), c));
     let embeddings: number[][];
     try {
       embeddings = await embedBatch(texts, openaiKey);
@@ -610,6 +463,7 @@ Deno.serve(async (req: Request) => {
       p_source_label: source,
       p_expected_content_hash: expected_content_hash,
       p_last_write_wins: last_write_wins,
+      p_content_format: CONTENT_FORMAT_BLIND_STITCH,
     });
 
     if (ingestErr) {
@@ -693,7 +547,7 @@ Deno.serve(async (req: Request) => {
       }
 
       // Prepend document title for contextual enrichment (stored content unchanged)
-      const texts = chunks.map((c) => `# ${title.trim()}\n${c.content}`);
+      const texts = chunks.map((c) => embeddingInputFor(title.trim(), c));
       let embeddings: number[][];
       try {
         embeddings = await embedBatch(texts, openaiKey);
@@ -728,6 +582,7 @@ Deno.serve(async (req: Request) => {
         p_source_label: source,
         p_expected_content_hash: expected_content_hash,
         p_last_write_wins: last_write_wins,
+      p_content_format: CONTENT_FORMAT_BLIND_STITCH,
       });
 
       if (ingestErr) {
@@ -801,7 +656,7 @@ Deno.serve(async (req: Request) => {
   }
 
   // Embed all chunks with title prefix for contextual enrichment (stored content unchanged)
-  const texts = chunks.map((c) => `# ${title.trim()}\n${c.content}`);
+  const texts = chunks.map((c) => embeddingInputFor(title.trim(), c));
   let embeddings: number[][];
   try {
     embeddings = await embedBatch(texts, openaiKey);
@@ -836,6 +691,7 @@ Deno.serve(async (req: Request) => {
     p_chunks: chunkData,
     p_author: author,
     p_author_type: author_type,
+    p_content_format: CONTENT_FORMAT_BLIND_STITCH,
   });
 
   if (ingestErr || !ingestResult?.length) {