@cerefox/memory 0.11.1 → 1.0.0-beta.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. package/AGENT_GUIDE.md +2 -2
  2. package/AGENT_QUICK_REFERENCE.md +1 -1
  3. package/dist/bin/cerefox.js +1079 -836
  4. package/dist/frontend/assets/index-D3FshoP3.js +125 -0
  5. package/dist/frontend/assets/index-D3FshoP3.js.map +1 -0
  6. package/dist/frontend/index.html +1 -1
  7. package/dist/server-assets/_shared/ef-auth/index.ts +134 -0
  8. package/dist/server-assets/_shared/ef-meta/index.ts +1 -1
  9. package/dist/server-assets/_shared/embeddings/index.ts +42 -2
  10. package/dist/server-assets/_shared/ingest/chunker.ts +210 -0
  11. package/dist/server-assets/_shared/ingest/index.ts +32 -0
  12. package/dist/server-assets/_shared/ingest/pipeline-helpers.ts +135 -0
  13. package/dist/server-assets/_shared/mcp-auth/index.ts +352 -0
  14. package/dist/server-assets/_shared/mcp-tools/_chunker.ts +16 -170
  15. package/dist/server-assets/_shared/mcp-tools/get-help-content.ts +2 -2
  16. package/dist/server-assets/_shared/mcp-tools/ingest.ts +13 -4
  17. package/dist/server-assets/db/migrations/0012_content_format.sql +20 -0
  18. package/dist/server-assets/db/rpcs.sql +76 -8
  19. package/dist/server-assets/db/schema.sql +7 -1
  20. package/dist/server-assets/supabase/functions/cerefox-get-audit-log/index.ts +8 -0
  21. package/dist/server-assets/supabase/functions/cerefox-get-document/index.ts +8 -0
  22. package/dist/server-assets/supabase/functions/cerefox-ingest/index.ts +39 -173
  23. package/dist/server-assets/supabase/functions/cerefox-list-projects/index.ts +8 -0
  24. package/dist/server-assets/supabase/functions/cerefox-list-versions/index.ts +8 -0
  25. package/dist/server-assets/supabase/functions/cerefox-mcp/index.ts +54 -0
  26. package/dist/server-assets/supabase/functions/cerefox-mcp/oauth.ts +121 -0
  27. package/dist/server-assets/supabase/functions/cerefox-metadata/index.ts +8 -0
  28. package/dist/server-assets/supabase/functions/cerefox-metadata-search/index.ts +8 -0
  29. package/dist/server-assets/supabase/functions/cerefox-search/index.ts +11 -1
  30. package/docs/guides/access-paths.md +84 -35
  31. package/docs/guides/cli.md +29 -0
  32. package/docs/guides/configuration.md +10 -8
  33. package/docs/guides/connect-agents.md +99 -101
  34. package/docs/guides/content-format.md +55 -0
  35. package/docs/guides/migration-1.0.md +96 -0
  36. package/docs/guides/ops-scripts.md +5 -7
  37. package/docs/guides/quickstart.md +22 -2
  38. package/docs/guides/setup-cloud-run.md +5 -9
  39. package/docs/guides/setup-supabase.md +157 -16
  40. package/docs/guides/upgrading.md +7 -8
  41. package/package.json +1 -1
  42. package/dist/frontend/assets/index-ojNhWSxm.js +0 -125
  43. package/dist/frontend/assets/index-ojNhWSxm.js.map +0 -1
@@ -0,0 +1,20 @@
1
+ -- Migration 0012: content_format on cerefox_chunks (iter-28D)
2
+ --
3
+ -- Records how each chunk's content reconstructs into full document text:
4
+ -- 1 = legacy — chunk contents were trimmed sections; reconstruction re-joins
5
+ -- them with E'\n\n' (the pre-28D behaviour). All existing chunks.
6
+ -- 2 = blind-stitch — chunk contents are an exact, gapless partition of the
7
+ -- document; reconstruction is a plain concatenation (no separator
8
+ -- synthesized on read). Written by the exact-partition chunker.
9
+ --
10
+ -- Placed on the CHUNK (not the document) so an archived version reconstructs with
11
+ -- its OWN format, since Cerefox uses chunks-anchored versioning
12
+ -- (cerefox_chunks.version_id). The reconstruction RPCs branch on
13
+ -- MAX(content_format) >= 2 per aggregated group.
14
+ --
15
+ -- Adding a NOT NULL column with a constant default is a metadata-only change in
16
+ -- PostgreSQL 11+ (no rewrite of the chunks table). Existing rows read back as 1.
17
+ -- Explanation for users: docs/guides/content-format.md.
18
+
19
+ ALTER TABLE cerefox_chunks
20
+ ADD COLUMN IF NOT EXISTS content_format SMALLINT NOT NULL DEFAULT 1;
@@ -403,7 +403,7 @@ AS $$
403
403
  ARRAY(SELECT p.name FROM cerefox_projects p
404
404
  JOIN cerefox_document_projects dp ON p.id = dp.project_id
405
405
  WHERE dp.document_id = d.id) AS doc_project_names,
406
- STRING_AGG(c.content, E'\n\n' ORDER BY c.chunk_index) AS full_content,
406
+ CASE WHEN MAX(c.content_format) >= 2 THEN STRING_AGG(c.content, '' ORDER BY c.chunk_index) ELSE STRING_AGG(c.content, E'\n\n' ORDER BY c.chunk_index) END AS full_content,
407
407
  COUNT(*)::INT AS chunk_count,
408
408
  SUM(c.char_count)::INT AS total_chars,
409
409
  (SELECT COUNT(*)::INT FROM cerefox_document_versions dv
@@ -680,17 +680,18 @@ AS $$
680
680
  large_doc_content AS (
681
681
  SELECT
682
682
  e.document_id,
683
- STRING_AGG(e.content, E'\n\n' ORDER BY e.chunk_index) AS full_content,
683
+ CASE WHEN MAX(ch.content_format) >= 2 THEN STRING_AGG(e.content, '' ORDER BY e.chunk_index) ELSE STRING_AGG(e.content, E'\n\n' ORDER BY e.chunk_index) END AS full_content,
684
684
  COUNT(*)::INT AS chunk_count,
685
685
  TRUE AS is_partial
686
686
  FROM expanded e
687
+ JOIN cerefox_chunks ch ON ch.id = e.chunk_id
687
688
  GROUP BY e.document_id
688
689
  ),
689
690
  -- Full content for small documents (is_partial = FALSE).
690
691
  small_doc_content AS (
691
692
  SELECT
692
693
  c.document_id,
693
- STRING_AGG(c.content, E'\n\n' ORDER BY c.chunk_index) AS full_content,
694
+ CASE WHEN MAX(c.content_format) >= 2 THEN STRING_AGG(c.content, '' ORDER BY c.chunk_index) ELSE STRING_AGG(c.content, E'\n\n' ORDER BY c.chunk_index) END AS full_content,
694
695
  COUNT(*)::INT AS chunk_count,
695
696
  FALSE AS is_partial
696
697
  FROM cerefox_chunks c
@@ -866,7 +867,7 @@ AS $$
866
867
  JOIN cerefox_document_projects dp ON p.id = dp.project_id
867
868
  WHERE dp.document_id = d.id) AS doc_project_names,
868
869
  p_version_id AS version_id,
869
- STRING_AGG(c.content, E'\n\n' ORDER BY c.chunk_index) AS full_content,
870
+ CASE WHEN MAX(c.content_format) >= 2 THEN STRING_AGG(c.content, '' ORDER BY c.chunk_index) ELSE STRING_AGG(c.content, E'\n\n' ORDER BY c.chunk_index) END AS full_content,
870
871
  COUNT(*)::INT AS chunk_count,
871
872
  SUM(c.char_count)::INT AS total_chars,
872
873
  d.created_at,
@@ -1072,6 +1073,7 @@ $$;
1072
1073
 
1073
1074
  DROP FUNCTION IF EXISTS cerefox_ingest_document(UUID, TEXT, TEXT, TEXT, TEXT, JSONB, TEXT, JSONB, TEXT, TEXT, TEXT, INT, BOOLEAN);
1074
1075
  DROP FUNCTION IF EXISTS cerefox_ingest_document(UUID, TEXT, TEXT, TEXT, TEXT, JSONB, TEXT, JSONB, TEXT, TEXT, TEXT, INT, BOOLEAN, TEXT, BOOLEAN);
1076
+ DROP FUNCTION IF EXISTS cerefox_ingest_document(UUID, TEXT, TEXT, TEXT, TEXT, JSONB, TEXT, JSONB, TEXT, TEXT, TEXT, INT, BOOLEAN, TEXT, BOOLEAN, SMALLINT);
1075
1077
  CREATE FUNCTION cerefox_ingest_document(
1076
1078
  p_document_id UUID DEFAULT NULL,
1077
1079
  p_title TEXT DEFAULT 'Untitled',
@@ -1090,7 +1092,11 @@ CREATE FUNCTION cerefox_ingest_document(
1090
1092
  p_retention_hours INT DEFAULT 48,
1091
1093
  p_cleanup_enabled BOOLEAN DEFAULT TRUE,
1092
1094
  p_expected_content_hash TEXT DEFAULT NULL,
1093
- p_last_write_wins BOOLEAN DEFAULT FALSE
1095
+ p_last_write_wins BOOLEAN DEFAULT FALSE,
1096
+ -- content_format for the chunks being written (iter-28D). 2 = exact-partition
1097
+ -- (blind-stitch reconstruction); default 1 = legacy (E'\n\n'-join). Stamped on
1098
+ -- every chunk this call inserts.
1099
+ p_content_format SMALLINT DEFAULT 1
1094
1100
  )
1095
1101
  RETURNS TABLE (
1096
1102
  document_id UUID,
@@ -1222,7 +1228,7 @@ BEGIN
1222
1228
  -- Formula: doc_title (A) || chunk_heading (A) || body_content (B)
1223
1229
  INSERT INTO cerefox_chunks (
1224
1230
  document_id, chunk_index, heading_path, heading_level,
1225
- title, content, char_count, embedding_primary, embedder_primary, fts
1231
+ title, content, char_count, content_format, embedding_primary, embedder_primary, fts
1226
1232
  )
1227
1233
  SELECT
1228
1234
  v_doc_id,
@@ -1232,6 +1238,7 @@ BEGIN
1232
1238
  c->>'title',
1233
1239
  c->>'content',
1234
1240
  (c->>'char_count')::INT,
1241
+ p_content_format,
1235
1242
  (SELECT array_agg(e::FLOAT)::VECTOR(768) FROM jsonb_array_elements_text(c->'embedding') AS e),
1236
1243
  c->>'embedder',
1237
1244
  setweight(to_tsvector('english', COALESCE(p_title, '')), 'A') ||
@@ -1509,7 +1516,7 @@ BEGIN
1509
1516
  WHERE dv.document_id = d.id) AS version_count,
1510
1517
  d.content_hash,
1511
1518
  CASE WHEN p_include_content THEN
1512
- (SELECT STRING_AGG(c.content, E'\n\n' ORDER BY c.chunk_index)
1519
+ (SELECT CASE WHEN MAX(c.content_format) >= 2 THEN STRING_AGG(c.content, '' ORDER BY c.chunk_index) ELSE STRING_AGG(c.content, E'\n\n' ORDER BY c.chunk_index) END
1513
1520
  FROM cerefox_chunks c
1514
1521
  WHERE c.document_id = d.id AND c.version_id IS NULL)
1515
1522
  ELSE NULL END AS content
@@ -1767,7 +1774,30 @@ SET search_path = public, pg_catalog
1767
1774
  AS $$
1768
1775
  -- Keep in lockstep with the `@version:` marker in schema.sql (cut_release.ts
1769
1776
  -- enforces it). Bump whenever schema.sql OR rpcs.sql changes.
1770
- SELECT '0.6.0'::TEXT;
1777
+ SELECT '0.8.0'::TEXT;
1778
+ $$;
1779
+
1780
+ -- ── cerefox_content_format_stats ─────────────────────────────────────────────
1781
+ -- Counts how many (non-deleted) documents still use the legacy chunk
1782
+ -- reconstruction format (content_format = 1) vs the total. Powers the
1783
+ -- informational `cerefox doctor` line. See docs/guides/content-format.md.
1784
+ CREATE OR REPLACE FUNCTION cerefox_content_format_stats()
1785
+ RETURNS TABLE (legacy_docs INT, total_docs INT)
1786
+ LANGUAGE sql
1787
+ STABLE
1788
+ SECURITY DEFINER
1789
+ SET search_path = public, pg_catalog
1790
+ AS $$
1791
+ SELECT
1792
+ COUNT(*) FILTER (WHERE cf.min_format < 2)::INT AS legacy_docs,
1793
+ COUNT(*)::INT AS total_docs
1794
+ FROM cerefox_documents d
1795
+ LEFT JOIN LATERAL (
1796
+ SELECT MIN(c.content_format) AS min_format
1797
+ FROM cerefox_chunks c
1798
+ WHERE c.document_id = d.id AND c.version_id IS NULL
1799
+ ) cf ON TRUE
1800
+ WHERE d.deleted_at IS NULL;
1771
1801
  $$;
1772
1802
 
1773
1803
 
@@ -1847,3 +1877,41 @@ AS $$
1847
1877
  WHERE a.document_id = ANY(p_doc_ids)
1848
1878
  ORDER BY a.document_id, a.created_at DESC;
1849
1879
  $$;
1880
+
1881
+ -- ── Function privilege lockdown (schema 0.7.0) ────────────────────────────────
1882
+ -- SECURITY (critical): every cerefox_* RPC is SECURITY DEFINER, so it bypasses the
1883
+ -- Row Level Security enabled on the tables. PostgreSQL grants EXECUTE to PUBLIC by
1884
+ -- default, and Supabase's PostgREST exposes public-schema functions at
1885
+ -- /rest/v1/rpc/<name>. WITHOUT this block, the anon key (and the new sb_publishable_
1886
+ -- key, which maps to the same anon role) could call every RPC directly via the Data
1887
+ -- API — reading and writing the entire KB, bypassing BOTH the Edge Functions and
1888
+ -- RLS. Every legitimate caller uses the service_role key instead (Edge Functions via
1889
+ -- SUPABASE_SERVICE_ROLE_KEY; the CLI/web via the secret key; local World B via a
1890
+ -- container-minted service_role JWT), so we REVOKE EXECUTE from PUBLIC/anon/
1891
+ -- authenticated and GRANT only to service_role.
1892
+ --
1893
+ -- Applied over ALL cerefox_* functions (so new functions are covered on the next
1894
+ -- deploy) and idempotent — re-applied each time rpcs.sql is deployed. Guarded on
1895
+ -- role existence so it is safe on non-Supabase Postgres (e.g. World B bootstrap).
1896
+ DO $$
1897
+ DECLARE
1898
+ fn regprocedure;
1899
+ r text;
1900
+ BEGIN
1901
+ FOR fn IN
1902
+ SELECT p.oid::regprocedure
1903
+ FROM pg_proc p
1904
+ JOIN pg_namespace n ON n.oid = p.pronamespace
1905
+ WHERE n.nspname = 'public' AND p.proname LIKE 'cerefox\_%'
1906
+ LOOP
1907
+ EXECUTE format('REVOKE EXECUTE ON FUNCTION %s FROM PUBLIC', fn);
1908
+ FOREACH r IN ARRAY ARRAY['anon', 'authenticated'] LOOP
1909
+ IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = r) THEN
1910
+ EXECUTE format('REVOKE EXECUTE ON FUNCTION %s FROM %I', fn, r);
1911
+ END IF;
1912
+ END LOOP;
1913
+ IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'service_role') THEN
1914
+ EXECUTE format('GRANT EXECUTE ON FUNCTION %s TO service_role', fn);
1915
+ END IF;
1916
+ END LOOP;
1917
+ END $$;
@@ -5,7 +5,7 @@
5
5
  -- Requires extensions: vector (pgvector), uuid-ossp
6
6
  -- These are enabled at the top of db_deploy.py before this file is applied.
7
7
  --
8
- -- @version: 0.6.0
8
+ -- @version: 0.8.0
9
9
  -- The `@version` marker above is read by the schema-version-mismatch banner
10
10
  -- (see /api/v1/schema-version). Bump it whenever schema.sql OR rpcs.sql
11
11
  -- changes in a way that requires `cerefox server deploy` to be re-run —
@@ -157,6 +157,12 @@ CREATE TABLE IF NOT EXISTS cerefox_chunks (
157
157
  title TEXT,
158
158
  content TEXT NOT NULL,
159
159
  char_count INT NOT NULL,
160
+ -- content_format: how this chunk's content reconstructs into full document text
161
+ -- (iter-28D). 1 = legacy (chunk contents re-joined with E'\n\n' on read); 2 =
162
+ -- blind-stitch (chunk contents are an exact, gapless partition, reconstructed by
163
+ -- plain concat). It lives on the chunk (not the document) so an archived version
164
+ -- reconstructs with its OWN format. See docs/guides/content-format.md.
165
+ content_format SMALLINT NOT NULL DEFAULT 1,
160
166
 
161
167
  -- Primary embedding: always computed, cloud API (default: OpenAI text-embedding-3-small)
162
168
  embedding_primary VECTOR(768) NOT NULL,
@@ -1,6 +1,7 @@
1
1
  import "jsr:@supabase/functions-js/edge-runtime.d.ts";
2
2
  import { createClient } from "jsr:@supabase/supabase-js@2";
3
3
  import { isVersionRequest, versionResponse } from "../../../_shared/ef-meta/index.ts";
4
+ import { efAuthGate } from "../../../_shared/ef-auth/index.ts";
4
5
 
5
6
  /**
6
7
  * cerefox-get-audit-log -- Supabase Edge Function
@@ -35,6 +36,13 @@ Deno.serve(async (req: Request): Promise<Response> => {
35
36
  return new Response(null, { status: 200, headers: CORS_HEADERS });
36
37
  }
37
38
 
39
+ const authFail = efAuthGate(
40
+ req.headers.get("Authorization"),
41
+ Deno.env.get("CEREFOX_ACCESS_TOKENS"),
42
+ { ...CORS_HEADERS, "Content-Type": "application/json" },
43
+ );
44
+ if (authFail) return authFail;
45
+
38
46
  if (isVersionRequest(req)) {
39
47
  return versionResponse("cerefox-get-audit-log", { ...CORS_HEADERS, "Content-Type": "application/json" });
40
48
  }
@@ -1,6 +1,7 @@
1
1
  import "jsr:@supabase/functions-js/edge-runtime.d.ts";
2
2
  import { createClient } from "jsr:@supabase/supabase-js@2";
3
3
  import { isVersionRequest, versionResponse } from "../../../_shared/ef-meta/index.ts";
4
+ import { efAuthGate } from "../../../_shared/ef-auth/index.ts";
4
5
 
5
6
  /**
6
7
  * cerefox-get-document — Supabase Edge Function
@@ -35,6 +36,13 @@ Deno.serve(async (req: Request): Promise<Response> => {
35
36
  return new Response(null, { status: 200, headers: CORS_HEADERS });
36
37
  }
37
38
 
39
+ const authFail = efAuthGate(
40
+ req.headers.get("Authorization"),
41
+ Deno.env.get("CEREFOX_ACCESS_TOKENS"),
42
+ { ...CORS_HEADERS, "Content-Type": "application/json" },
43
+ );
44
+ if (authFail) return authFail;
45
+
38
46
  if (isVersionRequest(req)) {
39
47
  return versionResponse("cerefox-get-document", { ...CORS_HEADERS, "Content-Type": "application/json" });
40
48
  }
@@ -1,6 +1,13 @@
1
1
  import "jsr:@supabase/functions-js/edge-runtime.d.ts";
2
2
  import { createClient } from "jsr:@supabase/supabase-js@2";
3
3
  import { isVersionRequest, versionResponse } from "../../../_shared/ef-meta/index.ts";
4
+ import { efAuthGate } from "../../../_shared/ef-auth/index.ts";
5
+ import { capEmbeddingInput } from "../../../_shared/embeddings/index.ts";
6
+ import {
7
+ chunkMarkdown,
8
+ embeddingInputFor,
9
+ CONTENT_FORMAT_BLIND_STITCH,
10
+ } from "../../../_shared/ingest/chunker.ts";
4
11
 
5
12
  /**
6
13
  * cerefox-ingest — Supabase Edge Function
@@ -49,12 +56,27 @@ interface IngestRequest {
49
56
  last_write_wins?: boolean;
50
57
  }
51
58
 
52
- // Map the RPC's CEREFOX_CONFLICT / CEREFOX_TOKEN_REQUIRED errors to HTTP
53
- // responses (409 conflict / 400 missing token). Returns null for other errors.
59
+ // Map the RPC's expected/validation errors to proper HTTP responses:
60
+ // CEREFOX_CONFLICT → 409 (stale optimistic-concurrency token)
61
+ // CEREFOX_TOKEN_REQUIRED → 400 (missing expected_content_hash)
62
+ // cerefox_documents_hash_unique → 409 (content de-dup: another doc already
63
+ // holds identical content)
64
+ // Returns null for genuinely unexpected errors (→ 500 at the call site).
54
65
  function concurrencyErrorResponse(
55
66
  message: string,
56
67
  headers: Record<string, string>,
57
68
  ): Response | null {
69
+ if (message.includes("cerefox_documents_hash_unique")) {
70
+ return new Response(
71
+ JSON.stringify({
72
+ error: "duplicate_content",
73
+ message:
74
+ "Another document already has identical content. Cerefox de-duplicates by content hash — update that document instead of creating or editing a second copy to match it.",
75
+ detail: message,
76
+ }),
77
+ { status: 409, headers },
78
+ );
79
+ }
58
80
  if (message.includes("CEREFOX_CONFLICT")) {
59
81
  return new Response(
60
82
  JSON.stringify({
@@ -80,173 +102,6 @@ function concurrencyErrorResponse(
80
102
  return null;
81
103
  }
82
104
 
83
- interface Chunk {
84
- heading_path: string[];
85
- heading_level: number;
86
- title: string;
87
- content: string;
88
- char_count: number;
89
- }
90
-
91
- // ── Heading-aware chunker (mirrors Python logic) ───────────────────────────
92
- //
93
- // Design notes:
94
- // • Short-circuit for small documents: if the entire document fits within
95
- // MAX_CHUNK_CHARS, it is returned as a single chunk with no splitting.
96
- // • Greedy accumulation: sections are collected into a buffer until adding
97
- // the next would exceed MAX_CHUNK_CHARS. This keeps chunks close to the
98
- // target size and avoids many tiny fragments at every heading boundary.
99
- // All heading levels (H1/H2/H3) are treated equally — size alone controls
100
- // when a chunk is flushed; there are no hard heading-level boundaries.
101
- // • Oversized sections (> MAX_CHUNK_CHARS) are paragraph-split with no overlap.
102
- // • The first section's heading metadata anchors each chunk's breadcrumb.
103
- // • No overlaps between chunks — the heading breadcrumb in the content
104
- // provides sufficient context. Overlaps caused duplication on reconstruction.
105
-
106
- interface Section {
107
- level: number;
108
- headings: string[]; // full heading stack at this section
109
- heading: string; // just the current heading text
110
- content: string; // heading line + body
111
- body: string; // body only (no heading line)
112
- }
113
-
114
- function parseSections(text: string): Section[] {
115
- const lines = text.split("\n");
116
- const sections: Section[] = [];
117
- let currentHeadings: string[] = [];
118
- let currentLevel = 0;
119
- let bodyLines: string[] = [];
120
-
121
- function collectSection() {
122
- const body = bodyLines.join("\n").trim();
123
- bodyLines = [];
124
- let content: string;
125
- if (currentLevel > 0) {
126
- const headerLine = "#".repeat(currentLevel) + " " + (currentHeadings[currentHeadings.length - 1] ?? "");
127
- content = body ? headerLine + "\n\n" + body : headerLine;
128
- } else {
129
- content = body;
130
- }
131
- if (!content.trim()) return;
132
- sections.push({
133
- level: currentLevel,
134
- headings: [...currentHeadings],
135
- heading: currentHeadings[currentHeadings.length - 1] ?? "",
136
- content,
137
- body,
138
- });
139
- }
140
-
141
- for (const line of lines) {
142
- const h1 = line.match(/^# (.+)/);
143
- const h2 = line.match(/^## (.+)/);
144
- const h3 = line.match(/^### (.+)/);
145
-
146
- if (h1) {
147
- collectSection();
148
- currentHeadings = [h1[1].trim()];
149
- currentLevel = 1;
150
- } else if (h2) {
151
- collectSection();
152
- currentHeadings = [currentHeadings[0] ?? "", h2[1].trim()].filter(Boolean);
153
- currentLevel = 2;
154
- } else if (h3) {
155
- collectSection();
156
- currentHeadings = [
157
- currentHeadings[0] ?? "",
158
- currentHeadings[1] ?? "",
159
- h3[1].trim(),
160
- ].filter(Boolean);
161
- currentLevel = 3;
162
- } else {
163
- bodyLines.push(line);
164
- }
165
- }
166
- collectSection();
167
- return sections;
168
- }
169
-
170
- function chunkMarkdown(text: string): Chunk[] {
171
- const trimmed = text.trim();
172
- if (!trimmed) return [];
173
-
174
- // Short-circuit: entire document fits in one chunk — skip heading splitting.
175
- if (trimmed.length <= MAX_CHUNK_CHARS) {
176
- return [makeChunk([], 0, trimmed)];
177
- }
178
-
179
- const sections = parseSections(trimmed);
180
- const chunks: Chunk[] = [];
181
-
182
- // Greedy accumulation buffer
183
- let bufParts: string[] = [];
184
- let bufHeadings: string[] = [];
185
- let bufLevel = 0;
186
- let bufChars = 0;
187
-
188
- function flushBuf() {
189
- if (bufParts.length === 0) return;
190
- chunks.push(makeChunk(bufHeadings, bufLevel, bufParts.join("\n\n")));
191
- bufParts = [];
192
- bufHeadings = [];
193
- bufLevel = 0;
194
- bufChars = 0;
195
- }
196
-
197
- for (const section of sections) {
198
- const { level, headings, heading, content, body } = section;
199
-
200
- // Oversized section: flush buffer, then paragraph-split.
201
- if (content.length > MAX_CHUNK_CHARS) {
202
- flushBuf();
203
- const headerPrefix = level > 0 ? "#".repeat(level) + " " + heading + "\n\n" : "";
204
- const bodyToSplit = body || content;
205
- const paragraphs = bodyToSplit.split(/\n\n+/);
206
- let sub = "";
207
- let isFirst = true;
208
- for (const para of paragraphs) {
209
- const prefix = isFirst ? headerPrefix : "";
210
- if (sub.length + prefix.length + para.length + 2 > MAX_CHUNK_CHARS && sub.length > 0) {
211
- chunks.push(makeChunk(headings, level, sub.trim()));
212
- sub = para;
213
- isFirst = false;
214
- } else {
215
- sub = sub ? sub + "\n\n" + para : prefix + para;
216
- isFirst = false;
217
- }
218
- }
219
- if (sub.trim()) chunks.push(makeChunk(headings, level, sub.trim()));
220
- continue;
221
- }
222
-
223
- // Section fits. Try to accumulate into the buffer.
224
- const addition = content.length + (bufParts.length > 0 ? 2 : 0);
225
-
226
- if (bufChars + addition <= MAX_CHUNK_CHARS) {
227
- if (bufParts.length === 0) {
228
- bufHeadings = headings;
229
- bufLevel = level;
230
- }
231
- bufParts.push(content);
232
- bufChars += addition;
233
- } else {
234
- flushBuf();
235
- bufParts = [content];
236
- bufHeadings = headings;
237
- bufLevel = level;
238
- bufChars = content.length;
239
- }
240
- }
241
-
242
- flushBuf();
243
- return chunks;
244
- }
245
-
246
- function makeChunk(headings: string[], level: number, content: string): Chunk {
247
- const title = headings[headings.length - 1] ?? "";
248
- return { heading_path: [...headings], heading_level: level, title, content, char_count: content.length };
249
- }
250
105
 
251
106
  // ── Embedding ──────────────────────────────────────────────────────────────
252
107
 
@@ -255,6 +110,7 @@ const EMBEDDING_INITIAL_BACKOFF_MS = 500; // 500ms, 1s, 2s exponential backoff
255
110
 
256
111
  async function embedBatch(texts: string[], apiKey: string): Promise<number[][]> {
257
112
  let lastError: Error | null = null;
113
+ const inputs = texts.map(capEmbeddingInput); // iter-28D Phase 0: cap oversized inputs
258
114
 
259
115
  for (let attempt = 0; attempt < EMBEDDING_MAX_RETRIES; attempt++) {
260
116
  try {
@@ -266,7 +122,7 @@ async function embedBatch(texts: string[], apiKey: string): Promise<number[][]>
266
122
  },
267
123
  body: JSON.stringify({
268
124
  model: OPENAI_MODEL,
269
- input: texts,
125
+ input: inputs,
270
126
  dimensions: EMBEDDING_DIMENSIONS,
271
127
  }),
272
128
  });
@@ -442,6 +298,13 @@ Deno.serve(async (req: Request) => {
442
298
  });
443
299
  }
444
300
 
301
+ const authFail = efAuthGate(
302
+ req.headers.get("Authorization"),
303
+ Deno.env.get("CEREFOX_ACCESS_TOKENS"),
304
+ { "Content-Type": "application/json", "Access-Control-Allow-Origin": "*" },
305
+ );
306
+ if (authFail) return authFail;
307
+
445
308
  if (isVersionRequest(req)) {
446
309
  return versionResponse("cerefox-ingest", {
447
310
  "Content-Type": "application/json",
@@ -582,7 +445,7 @@ Deno.serve(async (req: Request) => {
582
445
  return new Response(JSON.stringify({ error: "Content produced no chunks" }), { status: 422, headers });
583
446
  }
584
447
 
585
- const texts = chunks.map((c) => `# ${title.trim()}\n${c.content}`);
448
+ const texts = chunks.map((c) => embeddingInputFor(title.trim(), c));
586
449
  let embeddings: number[][];
587
450
  try {
588
451
  embeddings = await embedBatch(texts, openaiKey);
@@ -615,6 +478,7 @@ Deno.serve(async (req: Request) => {
615
478
  p_source_label: source,
616
479
  p_expected_content_hash: expected_content_hash,
617
480
  p_last_write_wins: last_write_wins,
481
+ p_content_format: CONTENT_FORMAT_BLIND_STITCH,
618
482
  });
619
483
 
620
484
  if (ingestErr) {
@@ -698,7 +562,7 @@ Deno.serve(async (req: Request) => {
698
562
  }
699
563
 
700
564
  // Prepend document title for contextual enrichment (stored content unchanged)
701
- const texts = chunks.map((c) => `# ${title.trim()}\n${c.content}`);
565
+ const texts = chunks.map((c) => embeddingInputFor(title.trim(), c));
702
566
  let embeddings: number[][];
703
567
  try {
704
568
  embeddings = await embedBatch(texts, openaiKey);
@@ -733,6 +597,7 @@ Deno.serve(async (req: Request) => {
733
597
  p_source_label: source,
734
598
  p_expected_content_hash: expected_content_hash,
735
599
  p_last_write_wins: last_write_wins,
600
+ p_content_format: CONTENT_FORMAT_BLIND_STITCH,
736
601
  });
737
602
 
738
603
  if (ingestErr) {
@@ -806,7 +671,7 @@ Deno.serve(async (req: Request) => {
806
671
  }
807
672
 
808
673
  // Embed all chunks with title prefix for contextual enrichment (stored content unchanged)
809
- const texts = chunks.map((c) => `# ${title.trim()}\n${c.content}`);
674
+ const texts = chunks.map((c) => embeddingInputFor(title.trim(), c));
810
675
  let embeddings: number[][];
811
676
  try {
812
677
  embeddings = await embedBatch(texts, openaiKey);
@@ -841,6 +706,7 @@ Deno.serve(async (req: Request) => {
841
706
  p_chunks: chunkData,
842
707
  p_author: author,
843
708
  p_author_type: author_type,
709
+ p_content_format: CONTENT_FORMAT_BLIND_STITCH,
844
710
  });
845
711
 
846
712
  if (ingestErr || !ingestResult?.length) {
@@ -1,6 +1,7 @@
1
1
  import "jsr:@supabase/functions-js/edge-runtime.d.ts";
2
2
  import { createClient } from "jsr:@supabase/supabase-js@2";
3
3
  import { isVersionRequest, versionResponse } from "../../../_shared/ef-meta/index.ts";
4
+ import { efAuthGate } from "../../../_shared/ef-auth/index.ts";
4
5
 
5
6
  /**
6
7
  * cerefox-list-projects -- Supabase Edge Function
@@ -29,6 +30,13 @@ Deno.serve(async (req: Request): Promise<Response> => {
29
30
  return new Response(null, { status: 200, headers: CORS_HEADERS });
30
31
  }
31
32
 
33
+ const authFail = efAuthGate(
34
+ req.headers.get("Authorization"),
35
+ Deno.env.get("CEREFOX_ACCESS_TOKENS"),
36
+ { ...CORS_HEADERS, "Content-Type": "application/json" },
37
+ );
38
+ if (authFail) return authFail;
39
+
32
40
  if (isVersionRequest(req)) {
33
41
  return versionResponse("cerefox-list-projects", { ...CORS_HEADERS, "Content-Type": "application/json" });
34
42
  }
@@ -1,6 +1,7 @@
1
1
  import "jsr:@supabase/functions-js/edge-runtime.d.ts";
2
2
  import { createClient } from "jsr:@supabase/supabase-js@2";
3
3
  import { isVersionRequest, versionResponse } from "../../../_shared/ef-meta/index.ts";
4
+ import { efAuthGate } from "../../../_shared/ef-auth/index.ts";
4
5
 
5
6
  /**
6
7
  * cerefox-list-versions — Supabase Edge Function
@@ -34,6 +35,13 @@ Deno.serve(async (req: Request): Promise<Response> => {
34
35
  return new Response(null, { status: 200, headers: CORS_HEADERS });
35
36
  }
36
37
 
38
+ const authFail = efAuthGate(
39
+ req.headers.get("Authorization"),
40
+ Deno.env.get("CEREFOX_ACCESS_TOKENS"),
41
+ { ...CORS_HEADERS, "Content-Type": "application/json" },
42
+ );
43
+ if (authFail) return authFail;
44
+
37
45
  if (isVersionRequest(req)) {
38
46
  return versionResponse("cerefox-list-versions", { ...CORS_HEADERS, "Content-Type": "application/json" });
39
47
  }
@@ -27,6 +27,14 @@ import {
27
27
  makeSupabaseClient,
28
28
  notificationResponse,
29
29
  } from "./shared.ts";
30
+ import {
31
+ buildAuthenticator,
32
+ isProtectedResourceMetadata,
33
+ protectedResourceMetadata,
34
+ unauthorizedChallenge,
35
+ } from "./oauth.ts";
36
+ import type { AuthResult, McpAuthenticator } from "../../../_shared/mcp-auth/index.ts";
37
+ import { checkAccessToken, parseAccessTokens } from "../../../_shared/ef-auth/index.ts";
30
38
  import {
31
39
  ALL_TOOLS,
32
40
  McpInvalidParams,
@@ -233,11 +241,57 @@ async function handleVersion(req: Request): Promise<Response> {
233
241
 
234
242
  // ── Main handler ─────────────────────────────────────────────────────────────
235
243
 
244
+ // Isolate-lifetime authenticator (holds the JWKS cache). Issuer/JWKS come from the
245
+ // injected SUPABASE_URL (not request headers — see oauth.ts projectOrigin).
246
+ let authenticator: McpAuthenticator | null = null;
247
+ function getAuthenticator(): McpAuthenticator {
248
+ if (!authenticator) authenticator = buildAuthenticator();
249
+ return authenticator;
250
+ }
251
+
236
252
  Deno.serve(async (req: Request): Promise<Response> => {
237
253
  if (req.method === "OPTIONS") {
238
254
  return new Response(null, { status: 200, headers: CORS_HEADERS });
239
255
  }
240
256
 
257
+ // Public discovery route (RFC 9728): the ONLY unauthenticated non-OPTIONS
258
+ // response. Served before auth so OAuth clients can bootstrap.
259
+ if (req.method === "GET" && isProtectedResourceMetadata(req)) {
260
+ return protectedResourceMetadata();
261
+ }
262
+
263
+ // ── Auth-first dispatch (design §6) ────────────────────────────────────────
264
+ // The function is deployed with --no-verify-jwt, so this in-function check is
265
+ // the ONLY gate. Accept EITHER a Cerefox access token (the static path — same
266
+ // credential the primitive EFs take, iter-28E) OR a valid OAuth JWT.
267
+ const authHeader = req.headers.get("Authorization");
268
+ let authResult: AuthResult;
269
+ // Static token first (cheap constant-time compare). A real OAuth JWT won't match
270
+ // any token, so it falls through to the JWKS path below — no false rejection.
271
+ const tokenResult = checkAccessToken(authHeader, {
272
+ tokens: parseAccessTokens(Deno.env.get("CEREFOX_ACCESS_TOKENS")),
273
+ });
274
+ if (tokenResult.ok) {
275
+ authResult = { ok: true, path: "static" };
276
+ } else {
277
+ authResult = await getAuthenticator().authenticate(authHeader);
278
+ }
279
+ if (!authResult.ok) {
280
+ // Log the machine reason (never the token) so the dashboard logs are
281
+ // actionable on a real auth failure. `no_token` is the normal OAuth-discovery
282
+ // probe (every cloud client sends one first) — don't log it as noise. The
283
+ // enriched detail (aud/sub/alg values from _shared/mcp-auth) names which claim
284
+ // a rejected token tripped. To debug a "Claude never sends the token" case
285
+ // (claude-ai #482), temporarily also log `Array.from(req.headers.keys())`.
286
+ if (authResult.reason !== "no_token") {
287
+ console.warn(
288
+ `[cerefox-mcp] auth rejected: ${authResult.reason}` +
289
+ (authResult.detail ? ` (${authResult.detail})` : ""),
290
+ );
291
+ }
292
+ return unauthorizedChallenge(authResult);
293
+ }
294
+
241
295
  // GET — the only supported GET is the /version surface (iter-26). Per MCP
242
296
  // spec (2025-03-26) this server otherwise returns 405 on GET to signal it
243
297
  // does not support SSE notifications (prevents MCP clients from holding a