@cerefox/memory 0.11.1 → 1.0.0-beta.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. package/dist/bin/cerefox.js +1077 -834
  2. package/dist/frontend/assets/index-CCkg5PXt.js +125 -0
  3. package/dist/frontend/assets/index-CCkg5PXt.js.map +1 -0
  4. package/dist/frontend/index.html +1 -1
  5. package/dist/server-assets/_shared/ef-auth/index.ts +134 -0
  6. package/dist/server-assets/_shared/ef-meta/index.ts +1 -1
  7. package/dist/server-assets/_shared/embeddings/index.ts +42 -2
  8. package/dist/server-assets/_shared/ingest/chunker.ts +210 -0
  9. package/dist/server-assets/_shared/ingest/index.ts +32 -0
  10. package/dist/server-assets/_shared/ingest/pipeline-helpers.ts +135 -0
  11. package/dist/server-assets/_shared/mcp-auth/index.ts +352 -0
  12. package/dist/server-assets/_shared/mcp-tools/_chunker.ts +16 -170
  13. package/dist/server-assets/_shared/mcp-tools/ingest.ts +13 -4
  14. package/dist/server-assets/db/migrations/0012_content_format.sql +20 -0
  15. package/dist/server-assets/db/rpcs.sql +76 -8
  16. package/dist/server-assets/db/schema.sql +7 -1
  17. package/dist/server-assets/supabase/functions/cerefox-get-audit-log/index.ts +8 -0
  18. package/dist/server-assets/supabase/functions/cerefox-get-document/index.ts +8 -0
  19. package/dist/server-assets/supabase/functions/cerefox-ingest/index.ts +22 -171
  20. package/dist/server-assets/supabase/functions/cerefox-list-projects/index.ts +8 -0
  21. package/dist/server-assets/supabase/functions/cerefox-list-versions/index.ts +8 -0
  22. package/dist/server-assets/supabase/functions/cerefox-mcp/index.ts +54 -0
  23. package/dist/server-assets/supabase/functions/cerefox-mcp/oauth.ts +121 -0
  24. package/dist/server-assets/supabase/functions/cerefox-metadata/index.ts +8 -0
  25. package/dist/server-assets/supabase/functions/cerefox-metadata-search/index.ts +8 -0
  26. package/dist/server-assets/supabase/functions/cerefox-search/index.ts +11 -1
  27. package/docs/guides/access-paths.md +81 -30
  28. package/docs/guides/cli.md +29 -0
  29. package/docs/guides/configuration.md +4 -1
  30. package/docs/guides/connect-agents.md +98 -54
  31. package/docs/guides/content-format.md +55 -0
  32. package/docs/guides/migration-1.0.md +87 -0
  33. package/docs/guides/ops-scripts.md +1 -1
  34. package/docs/guides/quickstart.md +20 -0
  35. package/docs/guides/setup-supabase.md +154 -13
  36. package/docs/guides/upgrading.md +4 -3
  37. package/package.json +1 -1
  38. package/dist/frontend/assets/index-ojNhWSxm.js +0 -125
  39. package/dist/frontend/assets/index-ojNhWSxm.js.map +0 -1
@@ -403,7 +403,7 @@ AS $$
403
403
  ARRAY(SELECT p.name FROM cerefox_projects p
404
404
  JOIN cerefox_document_projects dp ON p.id = dp.project_id
405
405
  WHERE dp.document_id = d.id) AS doc_project_names,
406
- STRING_AGG(c.content, E'\n\n' ORDER BY c.chunk_index) AS full_content,
406
+ CASE WHEN MAX(c.content_format) >= 2 THEN STRING_AGG(c.content, '' ORDER BY c.chunk_index) ELSE STRING_AGG(c.content, E'\n\n' ORDER BY c.chunk_index) END AS full_content,
407
407
  COUNT(*)::INT AS chunk_count,
408
408
  SUM(c.char_count)::INT AS total_chars,
409
409
  (SELECT COUNT(*)::INT FROM cerefox_document_versions dv
@@ -680,17 +680,18 @@ AS $$
680
680
  large_doc_content AS (
681
681
  SELECT
682
682
  e.document_id,
683
- STRING_AGG(e.content, E'\n\n' ORDER BY e.chunk_index) AS full_content,
683
+ CASE WHEN MAX(ch.content_format) >= 2 THEN STRING_AGG(e.content, '' ORDER BY e.chunk_index) ELSE STRING_AGG(e.content, E'\n\n' ORDER BY e.chunk_index) END AS full_content,
684
684
  COUNT(*)::INT AS chunk_count,
685
685
  TRUE AS is_partial
686
686
  FROM expanded e
687
+ JOIN cerefox_chunks ch ON ch.id = e.chunk_id
687
688
  GROUP BY e.document_id
688
689
  ),
689
690
  -- Full content for small documents (is_partial = FALSE).
690
691
  small_doc_content AS (
691
692
  SELECT
692
693
  c.document_id,
693
- STRING_AGG(c.content, E'\n\n' ORDER BY c.chunk_index) AS full_content,
694
+ CASE WHEN MAX(c.content_format) >= 2 THEN STRING_AGG(c.content, '' ORDER BY c.chunk_index) ELSE STRING_AGG(c.content, E'\n\n' ORDER BY c.chunk_index) END AS full_content,
694
695
  COUNT(*)::INT AS chunk_count,
695
696
  FALSE AS is_partial
696
697
  FROM cerefox_chunks c
@@ -866,7 +867,7 @@ AS $$
866
867
  JOIN cerefox_document_projects dp ON p.id = dp.project_id
867
868
  WHERE dp.document_id = d.id) AS doc_project_names,
868
869
  p_version_id AS version_id,
869
- STRING_AGG(c.content, E'\n\n' ORDER BY c.chunk_index) AS full_content,
870
+ CASE WHEN MAX(c.content_format) >= 2 THEN STRING_AGG(c.content, '' ORDER BY c.chunk_index) ELSE STRING_AGG(c.content, E'\n\n' ORDER BY c.chunk_index) END AS full_content,
870
871
  COUNT(*)::INT AS chunk_count,
871
872
  SUM(c.char_count)::INT AS total_chars,
872
873
  d.created_at,
@@ -1072,6 +1073,7 @@ $$;
1072
1073
 
1073
1074
  DROP FUNCTION IF EXISTS cerefox_ingest_document(UUID, TEXT, TEXT, TEXT, TEXT, JSONB, TEXT, JSONB, TEXT, TEXT, TEXT, INT, BOOLEAN);
1074
1075
  DROP FUNCTION IF EXISTS cerefox_ingest_document(UUID, TEXT, TEXT, TEXT, TEXT, JSONB, TEXT, JSONB, TEXT, TEXT, TEXT, INT, BOOLEAN, TEXT, BOOLEAN);
1076
+ DROP FUNCTION IF EXISTS cerefox_ingest_document(UUID, TEXT, TEXT, TEXT, TEXT, JSONB, TEXT, JSONB, TEXT, TEXT, TEXT, INT, BOOLEAN, TEXT, BOOLEAN, SMALLINT);
1075
1077
  CREATE FUNCTION cerefox_ingest_document(
1076
1078
  p_document_id UUID DEFAULT NULL,
1077
1079
  p_title TEXT DEFAULT 'Untitled',
@@ -1090,7 +1092,11 @@ CREATE FUNCTION cerefox_ingest_document(
1090
1092
  p_retention_hours INT DEFAULT 48,
1091
1093
  p_cleanup_enabled BOOLEAN DEFAULT TRUE,
1092
1094
  p_expected_content_hash TEXT DEFAULT NULL,
1093
- p_last_write_wins BOOLEAN DEFAULT FALSE
1095
+ p_last_write_wins BOOLEAN DEFAULT FALSE,
1096
+ -- content_format for the chunks being written (iter-28D). 2 = exact-partition
1097
+ -- (blind-stitch reconstruction); default 1 = legacy (E'\n\n'-join). Stamped on
1098
+ -- every chunk this call inserts.
1099
+ p_content_format SMALLINT DEFAULT 1
1094
1100
  )
1095
1101
  RETURNS TABLE (
1096
1102
  document_id UUID,
@@ -1222,7 +1228,7 @@ BEGIN
1222
1228
  -- Formula: doc_title (A) || chunk_heading (A) || body_content (B)
1223
1229
  INSERT INTO cerefox_chunks (
1224
1230
  document_id, chunk_index, heading_path, heading_level,
1225
- title, content, char_count, embedding_primary, embedder_primary, fts
1231
+ title, content, char_count, content_format, embedding_primary, embedder_primary, fts
1226
1232
  )
1227
1233
  SELECT
1228
1234
  v_doc_id,
@@ -1232,6 +1238,7 @@ BEGIN
1232
1238
  c->>'title',
1233
1239
  c->>'content',
1234
1240
  (c->>'char_count')::INT,
1241
+ p_content_format,
1235
1242
  (SELECT array_agg(e::FLOAT)::VECTOR(768) FROM jsonb_array_elements_text(c->'embedding') AS e),
1236
1243
  c->>'embedder',
1237
1244
  setweight(to_tsvector('english', COALESCE(p_title, '')), 'A') ||
@@ -1509,7 +1516,7 @@ BEGIN
1509
1516
  WHERE dv.document_id = d.id) AS version_count,
1510
1517
  d.content_hash,
1511
1518
  CASE WHEN p_include_content THEN
1512
- (SELECT STRING_AGG(c.content, E'\n\n' ORDER BY c.chunk_index)
1519
+ (SELECT CASE WHEN MAX(c.content_format) >= 2 THEN STRING_AGG(c.content, '' ORDER BY c.chunk_index) ELSE STRING_AGG(c.content, E'\n\n' ORDER BY c.chunk_index) END
1513
1520
  FROM cerefox_chunks c
1514
1521
  WHERE c.document_id = d.id AND c.version_id IS NULL)
1515
1522
  ELSE NULL END AS content
@@ -1767,7 +1774,30 @@ SET search_path = public, pg_catalog
1767
1774
  AS $$
1768
1775
  -- Keep in lockstep with the `@version:` marker in schema.sql (cut_release.ts
1769
1776
  -- enforces it). Bump whenever schema.sql OR rpcs.sql changes.
1770
- SELECT '0.6.0'::TEXT;
1777
+ SELECT '0.8.0'::TEXT;
1778
+ $$;
1779
+
1780
+ -- ── cerefox_content_format_stats ─────────────────────────────────────────────
1781
+ -- Counts how many (non-deleted) documents still use the legacy chunk
1782
+ -- reconstruction format (content_format = 1) vs the total. Powers the
1783
+ -- informational `cerefox doctor` line. See docs/guides/content-format.md.
1784
+ CREATE OR REPLACE FUNCTION cerefox_content_format_stats()
1785
+ RETURNS TABLE (legacy_docs INT, total_docs INT)
1786
+ LANGUAGE sql
1787
+ STABLE
1788
+ SECURITY DEFINER
1789
+ SET search_path = public, pg_catalog
1790
+ AS $$
1791
+ SELECT
1792
+ COUNT(*) FILTER (WHERE cf.min_format < 2)::INT AS legacy_docs,
1793
+ COUNT(*)::INT AS total_docs
1794
+ FROM cerefox_documents d
1795
+ LEFT JOIN LATERAL (
1796
+ SELECT MIN(c.content_format) AS min_format
1797
+ FROM cerefox_chunks c
1798
+ WHERE c.document_id = d.id AND c.version_id IS NULL
1799
+ ) cf ON TRUE
1800
+ WHERE d.deleted_at IS NULL;
1771
1801
  $$;
1772
1802
 
1773
1803
 
@@ -1847,3 +1877,41 @@ AS $$
1847
1877
  WHERE a.document_id = ANY(p_doc_ids)
1848
1878
  ORDER BY a.document_id, a.created_at DESC;
1849
1879
  $$;
1880
+
1881
+ -- ── Function privilege lockdown (schema 0.7.0) ────────────────────────────────
1882
+ -- SECURITY (critical): every cerefox_* RPC is SECURITY DEFINER, so it bypasses the
1883
+ -- Row Level Security enabled on the tables. PostgreSQL grants EXECUTE to PUBLIC by
1884
+ -- default, and Supabase's PostgREST exposes public-schema functions at
1885
+ -- /rest/v1/rpc/<name>. WITHOUT this block, the anon key (and the new sb_publishable_
1886
+ -- key, which maps to the same anon role) could call every RPC directly via the Data
1887
+ -- API — reading and writing the entire KB, bypassing BOTH the Edge Functions and
1888
+ -- RLS. Every legitimate caller uses the service_role key instead (Edge Functions via
1889
+ -- SUPABASE_SERVICE_ROLE_KEY; the CLI/web via the secret key; local World B via a
1890
+ -- container-minted service_role JWT), so we REVOKE EXECUTE from PUBLIC/anon/
1891
+ -- authenticated and GRANT only to service_role.
1892
+ --
1893
+ -- Applied over ALL cerefox_* functions (so new functions are covered on the next
1894
+ -- deploy) and idempotent — re-applied each time rpcs.sql is deployed. Guarded on
1895
+ -- role existence so it is safe on non-Supabase Postgres (e.g. World B bootstrap).
1896
+ DO $$
1897
+ DECLARE
1898
+ fn regprocedure;
1899
+ r text;
1900
+ BEGIN
1901
+ FOR fn IN
1902
+ SELECT p.oid::regprocedure
1903
+ FROM pg_proc p
1904
+ JOIN pg_namespace n ON n.oid = p.pronamespace
1905
+ WHERE n.nspname = 'public' AND p.proname LIKE 'cerefox\_%'
1906
+ LOOP
1907
+ EXECUTE format('REVOKE EXECUTE ON FUNCTION %s FROM PUBLIC', fn);
1908
+ FOREACH r IN ARRAY ARRAY['anon', 'authenticated'] LOOP
1909
+ IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = r) THEN
1910
+ EXECUTE format('REVOKE EXECUTE ON FUNCTION %s FROM %I', fn, r);
1911
+ END IF;
1912
+ END LOOP;
1913
+ IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'service_role') THEN
1914
+ EXECUTE format('GRANT EXECUTE ON FUNCTION %s TO service_role', fn);
1915
+ END IF;
1916
+ END LOOP;
1917
+ END $$;
@@ -5,7 +5,7 @@
5
5
  -- Requires extensions: vector (pgvector), uuid-ossp
6
6
  -- These are enabled at the top of db_deploy.py before this file is applied.
7
7
  --
8
- -- @version: 0.6.0
8
+ -- @version: 0.8.0
9
9
  -- The `@version` marker above is read by the schema-version-mismatch banner
10
10
  -- (see /api/v1/schema-version). Bump it whenever schema.sql OR rpcs.sql
11
11
  -- changes in a way that requires `cerefox server deploy` to be re-run —
@@ -157,6 +157,12 @@ CREATE TABLE IF NOT EXISTS cerefox_chunks (
157
157
  title TEXT,
158
158
  content TEXT NOT NULL,
159
159
  char_count INT NOT NULL,
160
+ -- content_format: how this chunk's content reconstructs into full document text
161
+ -- (iter-28D). 1 = legacy (chunk contents re-joined with E'\n\n' on read); 2 =
162
+ -- blind-stitch (chunk contents are an exact, gapless partition, reconstructed by
163
+ -- plain concat). It lives on the chunk (not the document) so an archived version
164
+ -- reconstructs with its OWN format. See docs/guides/content-format.md.
165
+ content_format SMALLINT NOT NULL DEFAULT 1,
160
166
 
161
167
  -- Primary embedding: always computed, cloud API (default: OpenAI text-embedding-3-small)
162
168
  embedding_primary VECTOR(768) NOT NULL,
@@ -1,6 +1,7 @@
1
1
  import "jsr:@supabase/functions-js/edge-runtime.d.ts";
2
2
  import { createClient } from "jsr:@supabase/supabase-js@2";
3
3
  import { isVersionRequest, versionResponse } from "../../../_shared/ef-meta/index.ts";
4
+ import { efAuthGate } from "../../../_shared/ef-auth/index.ts";
4
5
 
5
6
  /**
6
7
  * cerefox-get-audit-log -- Supabase Edge Function
@@ -35,6 +36,13 @@ Deno.serve(async (req: Request): Promise<Response> => {
35
36
  return new Response(null, { status: 200, headers: CORS_HEADERS });
36
37
  }
37
38
 
39
+ const authFail = efAuthGate(
40
+ req.headers.get("Authorization"),
41
+ Deno.env.get("CEREFOX_ACCESS_TOKENS"),
42
+ { ...CORS_HEADERS, "Content-Type": "application/json" },
43
+ );
44
+ if (authFail) return authFail;
45
+
38
46
  if (isVersionRequest(req)) {
39
47
  return versionResponse("cerefox-get-audit-log", { ...CORS_HEADERS, "Content-Type": "application/json" });
40
48
  }
@@ -1,6 +1,7 @@
1
1
  import "jsr:@supabase/functions-js/edge-runtime.d.ts";
2
2
  import { createClient } from "jsr:@supabase/supabase-js@2";
3
3
  import { isVersionRequest, versionResponse } from "../../../_shared/ef-meta/index.ts";
4
+ import { efAuthGate } from "../../../_shared/ef-auth/index.ts";
4
5
 
5
6
  /**
6
7
  * cerefox-get-document — Supabase Edge Function
@@ -35,6 +36,13 @@ Deno.serve(async (req: Request): Promise<Response> => {
35
36
  return new Response(null, { status: 200, headers: CORS_HEADERS });
36
37
  }
37
38
 
39
+ const authFail = efAuthGate(
40
+ req.headers.get("Authorization"),
41
+ Deno.env.get("CEREFOX_ACCESS_TOKENS"),
42
+ { ...CORS_HEADERS, "Content-Type": "application/json" },
43
+ );
44
+ if (authFail) return authFail;
45
+
38
46
  if (isVersionRequest(req)) {
39
47
  return versionResponse("cerefox-get-document", { ...CORS_HEADERS, "Content-Type": "application/json" });
40
48
  }
@@ -1,6 +1,13 @@
1
1
  import "jsr:@supabase/functions-js/edge-runtime.d.ts";
2
2
  import { createClient } from "jsr:@supabase/supabase-js@2";
3
3
  import { isVersionRequest, versionResponse } from "../../../_shared/ef-meta/index.ts";
4
+ import { efAuthGate } from "../../../_shared/ef-auth/index.ts";
5
+ import { capEmbeddingInput } from "../../../_shared/embeddings/index.ts";
6
+ import {
7
+ chunkMarkdown,
8
+ embeddingInputFor,
9
+ CONTENT_FORMAT_BLIND_STITCH,
10
+ } from "../../../_shared/ingest/chunker.ts";
4
11
 
5
12
  /**
6
13
  * cerefox-ingest — Supabase Edge Function
@@ -80,173 +87,6 @@ function concurrencyErrorResponse(
80
87
  return null;
81
88
  }
82
89
 
83
- interface Chunk {
84
- heading_path: string[];
85
- heading_level: number;
86
- title: string;
87
- content: string;
88
- char_count: number;
89
- }
90
-
91
- // ── Heading-aware chunker (mirrors Python logic) ───────────────────────────
92
- //
93
- // Design notes:
94
- // • Short-circuit for small documents: if the entire document fits within
95
- // MAX_CHUNK_CHARS, it is returned as a single chunk with no splitting.
96
- // • Greedy accumulation: sections are collected into a buffer until adding
97
- // the next would exceed MAX_CHUNK_CHARS. This keeps chunks close to the
98
- // target size and avoids many tiny fragments at every heading boundary.
99
- // All heading levels (H1/H2/H3) are treated equally — size alone controls
100
- // when a chunk is flushed; there are no hard heading-level boundaries.
101
- // • Oversized sections (> MAX_CHUNK_CHARS) are paragraph-split with no overlap.
102
- // • The first section's heading metadata anchors each chunk's breadcrumb.
103
- // • No overlaps between chunks — the heading breadcrumb in the content
104
- // provides sufficient context. Overlaps caused duplication on reconstruction.
105
-
106
- interface Section {
107
- level: number;
108
- headings: string[]; // full heading stack at this section
109
- heading: string; // just the current heading text
110
- content: string; // heading line + body
111
- body: string; // body only (no heading line)
112
- }
113
-
114
- function parseSections(text: string): Section[] {
115
- const lines = text.split("\n");
116
- const sections: Section[] = [];
117
- let currentHeadings: string[] = [];
118
- let currentLevel = 0;
119
- let bodyLines: string[] = [];
120
-
121
- function collectSection() {
122
- const body = bodyLines.join("\n").trim();
123
- bodyLines = [];
124
- let content: string;
125
- if (currentLevel > 0) {
126
- const headerLine = "#".repeat(currentLevel) + " " + (currentHeadings[currentHeadings.length - 1] ?? "");
127
- content = body ? headerLine + "\n\n" + body : headerLine;
128
- } else {
129
- content = body;
130
- }
131
- if (!content.trim()) return;
132
- sections.push({
133
- level: currentLevel,
134
- headings: [...currentHeadings],
135
- heading: currentHeadings[currentHeadings.length - 1] ?? "",
136
- content,
137
- body,
138
- });
139
- }
140
-
141
- for (const line of lines) {
142
- const h1 = line.match(/^# (.+)/);
143
- const h2 = line.match(/^## (.+)/);
144
- const h3 = line.match(/^### (.+)/);
145
-
146
- if (h1) {
147
- collectSection();
148
- currentHeadings = [h1[1].trim()];
149
- currentLevel = 1;
150
- } else if (h2) {
151
- collectSection();
152
- currentHeadings = [currentHeadings[0] ?? "", h2[1].trim()].filter(Boolean);
153
- currentLevel = 2;
154
- } else if (h3) {
155
- collectSection();
156
- currentHeadings = [
157
- currentHeadings[0] ?? "",
158
- currentHeadings[1] ?? "",
159
- h3[1].trim(),
160
- ].filter(Boolean);
161
- currentLevel = 3;
162
- } else {
163
- bodyLines.push(line);
164
- }
165
- }
166
- collectSection();
167
- return sections;
168
- }
169
-
170
- function chunkMarkdown(text: string): Chunk[] {
171
- const trimmed = text.trim();
172
- if (!trimmed) return [];
173
-
174
- // Short-circuit: entire document fits in one chunk — skip heading splitting.
175
- if (trimmed.length <= MAX_CHUNK_CHARS) {
176
- return [makeChunk([], 0, trimmed)];
177
- }
178
-
179
- const sections = parseSections(trimmed);
180
- const chunks: Chunk[] = [];
181
-
182
- // Greedy accumulation buffer
183
- let bufParts: string[] = [];
184
- let bufHeadings: string[] = [];
185
- let bufLevel = 0;
186
- let bufChars = 0;
187
-
188
- function flushBuf() {
189
- if (bufParts.length === 0) return;
190
- chunks.push(makeChunk(bufHeadings, bufLevel, bufParts.join("\n\n")));
191
- bufParts = [];
192
- bufHeadings = [];
193
- bufLevel = 0;
194
- bufChars = 0;
195
- }
196
-
197
- for (const section of sections) {
198
- const { level, headings, heading, content, body } = section;
199
-
200
- // Oversized section: flush buffer, then paragraph-split.
201
- if (content.length > MAX_CHUNK_CHARS) {
202
- flushBuf();
203
- const headerPrefix = level > 0 ? "#".repeat(level) + " " + heading + "\n\n" : "";
204
- const bodyToSplit = body || content;
205
- const paragraphs = bodyToSplit.split(/\n\n+/);
206
- let sub = "";
207
- let isFirst = true;
208
- for (const para of paragraphs) {
209
- const prefix = isFirst ? headerPrefix : "";
210
- if (sub.length + prefix.length + para.length + 2 > MAX_CHUNK_CHARS && sub.length > 0) {
211
- chunks.push(makeChunk(headings, level, sub.trim()));
212
- sub = para;
213
- isFirst = false;
214
- } else {
215
- sub = sub ? sub + "\n\n" + para : prefix + para;
216
- isFirst = false;
217
- }
218
- }
219
- if (sub.trim()) chunks.push(makeChunk(headings, level, sub.trim()));
220
- continue;
221
- }
222
-
223
- // Section fits. Try to accumulate into the buffer.
224
- const addition = content.length + (bufParts.length > 0 ? 2 : 0);
225
-
226
- if (bufChars + addition <= MAX_CHUNK_CHARS) {
227
- if (bufParts.length === 0) {
228
- bufHeadings = headings;
229
- bufLevel = level;
230
- }
231
- bufParts.push(content);
232
- bufChars += addition;
233
- } else {
234
- flushBuf();
235
- bufParts = [content];
236
- bufHeadings = headings;
237
- bufLevel = level;
238
- bufChars = content.length;
239
- }
240
- }
241
-
242
- flushBuf();
243
- return chunks;
244
- }
245
-
246
- function makeChunk(headings: string[], level: number, content: string): Chunk {
247
- const title = headings[headings.length - 1] ?? "";
248
- return { heading_path: [...headings], heading_level: level, title, content, char_count: content.length };
249
- }
250
90
 
251
91
  // ── Embedding ──────────────────────────────────────────────────────────────
252
92
 
@@ -255,6 +95,7 @@ const EMBEDDING_INITIAL_BACKOFF_MS = 500; // 500ms, 1s, 2s exponential backoff
255
95
 
256
96
  async function embedBatch(texts: string[], apiKey: string): Promise<number[][]> {
257
97
  let lastError: Error | null = null;
98
+ const inputs = texts.map(capEmbeddingInput); // iter-28D Phase 0: cap oversized inputs
258
99
 
259
100
  for (let attempt = 0; attempt < EMBEDDING_MAX_RETRIES; attempt++) {
260
101
  try {
@@ -266,7 +107,7 @@ async function embedBatch(texts: string[], apiKey: string): Promise<number[][]>
266
107
  },
267
108
  body: JSON.stringify({
268
109
  model: OPENAI_MODEL,
269
- input: texts,
110
+ input: inputs,
270
111
  dimensions: EMBEDDING_DIMENSIONS,
271
112
  }),
272
113
  });
@@ -442,6 +283,13 @@ Deno.serve(async (req: Request) => {
442
283
  });
443
284
  }
444
285
 
286
+ const authFail = efAuthGate(
287
+ req.headers.get("Authorization"),
288
+ Deno.env.get("CEREFOX_ACCESS_TOKENS"),
289
+ { "Content-Type": "application/json", "Access-Control-Allow-Origin": "*" },
290
+ );
291
+ if (authFail) return authFail;
292
+
445
293
  if (isVersionRequest(req)) {
446
294
  return versionResponse("cerefox-ingest", {
447
295
  "Content-Type": "application/json",
@@ -582,7 +430,7 @@ Deno.serve(async (req: Request) => {
582
430
  return new Response(JSON.stringify({ error: "Content produced no chunks" }), { status: 422, headers });
583
431
  }
584
432
 
585
- const texts = chunks.map((c) => `# ${title.trim()}\n${c.content}`);
433
+ const texts = chunks.map((c) => embeddingInputFor(title.trim(), c));
586
434
  let embeddings: number[][];
587
435
  try {
588
436
  embeddings = await embedBatch(texts, openaiKey);
@@ -615,6 +463,7 @@ Deno.serve(async (req: Request) => {
615
463
  p_source_label: source,
616
464
  p_expected_content_hash: expected_content_hash,
617
465
  p_last_write_wins: last_write_wins,
466
+ p_content_format: CONTENT_FORMAT_BLIND_STITCH,
618
467
  });
619
468
 
620
469
  if (ingestErr) {
@@ -698,7 +547,7 @@ Deno.serve(async (req: Request) => {
698
547
  }
699
548
 
700
549
  // Prepend document title for contextual enrichment (stored content unchanged)
701
- const texts = chunks.map((c) => `# ${title.trim()}\n${c.content}`);
550
+ const texts = chunks.map((c) => embeddingInputFor(title.trim(), c));
702
551
  let embeddings: number[][];
703
552
  try {
704
553
  embeddings = await embedBatch(texts, openaiKey);
@@ -733,6 +582,7 @@ Deno.serve(async (req: Request) => {
733
582
  p_source_label: source,
734
583
  p_expected_content_hash: expected_content_hash,
735
584
  p_last_write_wins: last_write_wins,
585
+ p_content_format: CONTENT_FORMAT_BLIND_STITCH,
736
586
  });
737
587
 
738
588
  if (ingestErr) {
@@ -806,7 +656,7 @@ Deno.serve(async (req: Request) => {
806
656
  }
807
657
 
808
658
  // Embed all chunks with title prefix for contextual enrichment (stored content unchanged)
809
- const texts = chunks.map((c) => `# ${title.trim()}\n${c.content}`);
659
+ const texts = chunks.map((c) => embeddingInputFor(title.trim(), c));
810
660
  let embeddings: number[][];
811
661
  try {
812
662
  embeddings = await embedBatch(texts, openaiKey);
@@ -841,6 +691,7 @@ Deno.serve(async (req: Request) => {
841
691
  p_chunks: chunkData,
842
692
  p_author: author,
843
693
  p_author_type: author_type,
694
+ p_content_format: CONTENT_FORMAT_BLIND_STITCH,
844
695
  });
845
696
 
846
697
  if (ingestErr || !ingestResult?.length) {
@@ -1,6 +1,7 @@
1
1
  import "jsr:@supabase/functions-js/edge-runtime.d.ts";
2
2
  import { createClient } from "jsr:@supabase/supabase-js@2";
3
3
  import { isVersionRequest, versionResponse } from "../../../_shared/ef-meta/index.ts";
4
+ import { efAuthGate } from "../../../_shared/ef-auth/index.ts";
4
5
 
5
6
  /**
6
7
  * cerefox-list-projects -- Supabase Edge Function
@@ -29,6 +30,13 @@ Deno.serve(async (req: Request): Promise<Response> => {
29
30
  return new Response(null, { status: 200, headers: CORS_HEADERS });
30
31
  }
31
32
 
33
+ const authFail = efAuthGate(
34
+ req.headers.get("Authorization"),
35
+ Deno.env.get("CEREFOX_ACCESS_TOKENS"),
36
+ { ...CORS_HEADERS, "Content-Type": "application/json" },
37
+ );
38
+ if (authFail) return authFail;
39
+
32
40
  if (isVersionRequest(req)) {
33
41
  return versionResponse("cerefox-list-projects", { ...CORS_HEADERS, "Content-Type": "application/json" });
34
42
  }
@@ -1,6 +1,7 @@
1
1
  import "jsr:@supabase/functions-js/edge-runtime.d.ts";
2
2
  import { createClient } from "jsr:@supabase/supabase-js@2";
3
3
  import { isVersionRequest, versionResponse } from "../../../_shared/ef-meta/index.ts";
4
+ import { efAuthGate } from "../../../_shared/ef-auth/index.ts";
4
5
 
5
6
  /**
6
7
  * cerefox-list-versions — Supabase Edge Function
@@ -34,6 +35,13 @@ Deno.serve(async (req: Request): Promise<Response> => {
34
35
  return new Response(null, { status: 200, headers: CORS_HEADERS });
35
36
  }
36
37
 
38
+ const authFail = efAuthGate(
39
+ req.headers.get("Authorization"),
40
+ Deno.env.get("CEREFOX_ACCESS_TOKENS"),
41
+ { ...CORS_HEADERS, "Content-Type": "application/json" },
42
+ );
43
+ if (authFail) return authFail;
44
+
37
45
  if (isVersionRequest(req)) {
38
46
  return versionResponse("cerefox-list-versions", { ...CORS_HEADERS, "Content-Type": "application/json" });
39
47
  }
@@ -27,6 +27,14 @@ import {
27
27
  makeSupabaseClient,
28
28
  notificationResponse,
29
29
  } from "./shared.ts";
30
+ import {
31
+ buildAuthenticator,
32
+ isProtectedResourceMetadata,
33
+ protectedResourceMetadata,
34
+ unauthorizedChallenge,
35
+ } from "./oauth.ts";
36
+ import type { AuthResult, McpAuthenticator } from "../../../_shared/mcp-auth/index.ts";
37
+ import { checkAccessToken, parseAccessTokens } from "../../../_shared/ef-auth/index.ts";
30
38
  import {
31
39
  ALL_TOOLS,
32
40
  McpInvalidParams,
@@ -233,11 +241,57 @@ async function handleVersion(req: Request): Promise<Response> {
233
241
 
234
242
  // ── Main handler ─────────────────────────────────────────────────────────────
235
243
 
244
+ // Isolate-lifetime authenticator (holds the JWKS cache). Issuer/JWKS come from the
245
+ // injected SUPABASE_URL (not request headers — see oauth.ts projectOrigin).
246
+ let authenticator: McpAuthenticator | null = null;
247
+ function getAuthenticator(): McpAuthenticator {
248
+ if (!authenticator) authenticator = buildAuthenticator();
249
+ return authenticator;
250
+ }
251
+
236
252
  Deno.serve(async (req: Request): Promise<Response> => {
237
253
  if (req.method === "OPTIONS") {
238
254
  return new Response(null, { status: 200, headers: CORS_HEADERS });
239
255
  }
240
256
 
257
+ // Public discovery route (RFC 9728): the ONLY unauthenticated non-OPTIONS
258
+ // response. Served before auth so OAuth clients can bootstrap.
259
+ if (req.method === "GET" && isProtectedResourceMetadata(req)) {
260
+ return protectedResourceMetadata();
261
+ }
262
+
263
+ // ── Auth-first dispatch (design §6) ────────────────────────────────────────
264
+ // The function is deployed with --no-verify-jwt, so this in-function check is
265
+ // the ONLY gate. Accept EITHER a Cerefox access token (the static path — same
266
+ // credential the primitive EFs take, iter-28E) OR a valid OAuth JWT.
267
+ const authHeader = req.headers.get("Authorization");
268
+ let authResult: AuthResult;
269
+ // Static token first (cheap constant-time compare). A real OAuth JWT won't match
270
+ // any token, so it falls through to the JWKS path below — no false rejection.
271
+ const tokenResult = checkAccessToken(authHeader, {
272
+ tokens: parseAccessTokens(Deno.env.get("CEREFOX_ACCESS_TOKENS")),
273
+ });
274
+ if (tokenResult.ok) {
275
+ authResult = { ok: true, path: "static" };
276
+ } else {
277
+ authResult = await getAuthenticator().authenticate(authHeader);
278
+ }
279
+ if (!authResult.ok) {
280
+ // Log the machine reason (never the token) so the dashboard logs are
281
+ // actionable on a real auth failure. `no_token` is the normal OAuth-discovery
282
+ // probe (every cloud client sends one first) — don't log it as noise. The
283
+ // enriched detail (aud/sub/alg values from _shared/mcp-auth) names which claim
284
+ // a rejected token tripped. To debug a "Claude never sends the token" case
285
+ // (claude-ai #482), temporarily also log `Array.from(req.headers.keys())`.
286
+ if (authResult.reason !== "no_token") {
287
+ console.warn(
288
+ `[cerefox-mcp] auth rejected: ${authResult.reason}` +
289
+ (authResult.detail ? ` (${authResult.detail})` : ""),
290
+ );
291
+ }
292
+ return unauthorizedChallenge(authResult);
293
+ }
294
+
241
295
  // GET — the only supported GET is the /version surface (iter-26). Per MCP
242
296
  // spec (2025-03-26) this server otherwise returns 405 on GET to signal it
243
297
  // does not support SSE notifications (prevents MCP clients from holding a