@cerefox/memory 0.11.1 → 1.0.0-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/cerefox.js +1077 -834
- package/dist/frontend/assets/index-CCkg5PXt.js +125 -0
- package/dist/frontend/assets/index-CCkg5PXt.js.map +1 -0
- package/dist/frontend/index.html +1 -1
- package/dist/server-assets/_shared/ef-auth/index.ts +134 -0
- package/dist/server-assets/_shared/ef-meta/index.ts +1 -1
- package/dist/server-assets/_shared/embeddings/index.ts +42 -2
- package/dist/server-assets/_shared/ingest/chunker.ts +210 -0
- package/dist/server-assets/_shared/ingest/index.ts +32 -0
- package/dist/server-assets/_shared/ingest/pipeline-helpers.ts +135 -0
- package/dist/server-assets/_shared/mcp-auth/index.ts +352 -0
- package/dist/server-assets/_shared/mcp-tools/_chunker.ts +16 -170
- package/dist/server-assets/_shared/mcp-tools/ingest.ts +13 -4
- package/dist/server-assets/db/migrations/0012_content_format.sql +20 -0
- package/dist/server-assets/db/rpcs.sql +76 -8
- package/dist/server-assets/db/schema.sql +7 -1
- package/dist/server-assets/supabase/functions/cerefox-get-audit-log/index.ts +8 -0
- package/dist/server-assets/supabase/functions/cerefox-get-document/index.ts +8 -0
- package/dist/server-assets/supabase/functions/cerefox-ingest/index.ts +22 -171
- package/dist/server-assets/supabase/functions/cerefox-list-projects/index.ts +8 -0
- package/dist/server-assets/supabase/functions/cerefox-list-versions/index.ts +8 -0
- package/dist/server-assets/supabase/functions/cerefox-mcp/index.ts +54 -0
- package/dist/server-assets/supabase/functions/cerefox-mcp/oauth.ts +121 -0
- package/dist/server-assets/supabase/functions/cerefox-metadata/index.ts +8 -0
- package/dist/server-assets/supabase/functions/cerefox-metadata-search/index.ts +8 -0
- package/dist/server-assets/supabase/functions/cerefox-search/index.ts +11 -1
- package/docs/guides/access-paths.md +81 -30
- package/docs/guides/cli.md +29 -0
- package/docs/guides/configuration.md +4 -1
- package/docs/guides/connect-agents.md +98 -54
- package/docs/guides/content-format.md +55 -0
- package/docs/guides/migration-1.0.md +87 -0
- package/docs/guides/ops-scripts.md +1 -1
- package/docs/guides/quickstart.md +20 -0
- package/docs/guides/setup-supabase.md +154 -13
- package/docs/guides/upgrading.md +4 -3
- package/package.json +1 -1
- package/dist/frontend/assets/index-ojNhWSxm.js +0 -125
- package/dist/frontend/assets/index-ojNhWSxm.js.map +0 -1
|
@@ -403,7 +403,7 @@ AS $$
|
|
|
403
403
|
ARRAY(SELECT p.name FROM cerefox_projects p
|
|
404
404
|
JOIN cerefox_document_projects dp ON p.id = dp.project_id
|
|
405
405
|
WHERE dp.document_id = d.id) AS doc_project_names,
|
|
406
|
-
STRING_AGG(c.content, E'\n\n' ORDER BY c.chunk_index) AS full_content,
|
|
406
|
+
CASE WHEN MAX(c.content_format) >= 2 THEN STRING_AGG(c.content, '' ORDER BY c.chunk_index) ELSE STRING_AGG(c.content, E'\n\n' ORDER BY c.chunk_index) END AS full_content,
|
|
407
407
|
COUNT(*)::INT AS chunk_count,
|
|
408
408
|
SUM(c.char_count)::INT AS total_chars,
|
|
409
409
|
(SELECT COUNT(*)::INT FROM cerefox_document_versions dv
|
|
@@ -680,17 +680,18 @@ AS $$
|
|
|
680
680
|
large_doc_content AS (
|
|
681
681
|
SELECT
|
|
682
682
|
e.document_id,
|
|
683
|
-
STRING_AGG(e.content, E'\n\n' ORDER BY e.chunk_index) AS full_content,
|
|
683
|
+
CASE WHEN MAX(ch.content_format) >= 2 THEN STRING_AGG(e.content, '' ORDER BY e.chunk_index) ELSE STRING_AGG(e.content, E'\n\n' ORDER BY e.chunk_index) END AS full_content,
|
|
684
684
|
COUNT(*)::INT AS chunk_count,
|
|
685
685
|
TRUE AS is_partial
|
|
686
686
|
FROM expanded e
|
|
687
|
+
JOIN cerefox_chunks ch ON ch.id = e.chunk_id
|
|
687
688
|
GROUP BY e.document_id
|
|
688
689
|
),
|
|
689
690
|
-- Full content for small documents (is_partial = FALSE).
|
|
690
691
|
small_doc_content AS (
|
|
691
692
|
SELECT
|
|
692
693
|
c.document_id,
|
|
693
|
-
STRING_AGG(c.content, E'\n\n' ORDER BY c.chunk_index) AS full_content,
|
|
694
|
+
CASE WHEN MAX(c.content_format) >= 2 THEN STRING_AGG(c.content, '' ORDER BY c.chunk_index) ELSE STRING_AGG(c.content, E'\n\n' ORDER BY c.chunk_index) END AS full_content,
|
|
694
695
|
COUNT(*)::INT AS chunk_count,
|
|
695
696
|
FALSE AS is_partial
|
|
696
697
|
FROM cerefox_chunks c
|
|
@@ -866,7 +867,7 @@ AS $$
|
|
|
866
867
|
JOIN cerefox_document_projects dp ON p.id = dp.project_id
|
|
867
868
|
WHERE dp.document_id = d.id) AS doc_project_names,
|
|
868
869
|
p_version_id AS version_id,
|
|
869
|
-
STRING_AGG(c.content, E'\n\n' ORDER BY c.chunk_index) AS full_content,
|
|
870
|
+
CASE WHEN MAX(c.content_format) >= 2 THEN STRING_AGG(c.content, '' ORDER BY c.chunk_index) ELSE STRING_AGG(c.content, E'\n\n' ORDER BY c.chunk_index) END AS full_content,
|
|
870
871
|
COUNT(*)::INT AS chunk_count,
|
|
871
872
|
SUM(c.char_count)::INT AS total_chars,
|
|
872
873
|
d.created_at,
|
|
@@ -1072,6 +1073,7 @@ $$;
|
|
|
1072
1073
|
|
|
1073
1074
|
DROP FUNCTION IF EXISTS cerefox_ingest_document(UUID, TEXT, TEXT, TEXT, TEXT, JSONB, TEXT, JSONB, TEXT, TEXT, TEXT, INT, BOOLEAN);
|
|
1074
1075
|
DROP FUNCTION IF EXISTS cerefox_ingest_document(UUID, TEXT, TEXT, TEXT, TEXT, JSONB, TEXT, JSONB, TEXT, TEXT, TEXT, INT, BOOLEAN, TEXT, BOOLEAN);
|
|
1076
|
+
DROP FUNCTION IF EXISTS cerefox_ingest_document(UUID, TEXT, TEXT, TEXT, TEXT, JSONB, TEXT, JSONB, TEXT, TEXT, TEXT, INT, BOOLEAN, TEXT, BOOLEAN, SMALLINT);
|
|
1075
1077
|
CREATE FUNCTION cerefox_ingest_document(
|
|
1076
1078
|
p_document_id UUID DEFAULT NULL,
|
|
1077
1079
|
p_title TEXT DEFAULT 'Untitled',
|
|
@@ -1090,7 +1092,11 @@ CREATE FUNCTION cerefox_ingest_document(
|
|
|
1090
1092
|
p_retention_hours INT DEFAULT 48,
|
|
1091
1093
|
p_cleanup_enabled BOOLEAN DEFAULT TRUE,
|
|
1092
1094
|
p_expected_content_hash TEXT DEFAULT NULL,
|
|
1093
|
-
p_last_write_wins BOOLEAN DEFAULT FALSE
|
|
1095
|
+
p_last_write_wins BOOLEAN DEFAULT FALSE,
|
|
1096
|
+
-- content_format for the chunks being written (iter-28D). 2 = exact-partition
|
|
1097
|
+
-- (blind-stitch reconstruction); default 1 = legacy (E'\n\n'-join). Stamped on
|
|
1098
|
+
-- every chunk this call inserts.
|
|
1099
|
+
p_content_format SMALLINT DEFAULT 1
|
|
1094
1100
|
)
|
|
1095
1101
|
RETURNS TABLE (
|
|
1096
1102
|
document_id UUID,
|
|
@@ -1222,7 +1228,7 @@ BEGIN
|
|
|
1222
1228
|
-- Formula: doc_title (A) || chunk_heading (A) || body_content (B)
|
|
1223
1229
|
INSERT INTO cerefox_chunks (
|
|
1224
1230
|
document_id, chunk_index, heading_path, heading_level,
|
|
1225
|
-
title, content, char_count, embedding_primary, embedder_primary, fts
|
|
1231
|
+
title, content, char_count, content_format, embedding_primary, embedder_primary, fts
|
|
1226
1232
|
)
|
|
1227
1233
|
SELECT
|
|
1228
1234
|
v_doc_id,
|
|
@@ -1232,6 +1238,7 @@ BEGIN
|
|
|
1232
1238
|
c->>'title',
|
|
1233
1239
|
c->>'content',
|
|
1234
1240
|
(c->>'char_count')::INT,
|
|
1241
|
+
p_content_format,
|
|
1235
1242
|
(SELECT array_agg(e::FLOAT)::VECTOR(768) FROM jsonb_array_elements_text(c->'embedding') AS e),
|
|
1236
1243
|
c->>'embedder',
|
|
1237
1244
|
setweight(to_tsvector('english', COALESCE(p_title, '')), 'A') ||
|
|
@@ -1509,7 +1516,7 @@ BEGIN
|
|
|
1509
1516
|
WHERE dv.document_id = d.id) AS version_count,
|
|
1510
1517
|
d.content_hash,
|
|
1511
1518
|
CASE WHEN p_include_content THEN
|
|
1512
|
-
(SELECT STRING_AGG(c.content, E'\n\n' ORDER BY c.chunk_index)
|
|
1519
|
+
(SELECT CASE WHEN MAX(c.content_format) >= 2 THEN STRING_AGG(c.content, '' ORDER BY c.chunk_index) ELSE STRING_AGG(c.content, E'\n\n' ORDER BY c.chunk_index) END
|
|
1513
1520
|
FROM cerefox_chunks c
|
|
1514
1521
|
WHERE c.document_id = d.id AND c.version_id IS NULL)
|
|
1515
1522
|
ELSE NULL END AS content
|
|
@@ -1767,7 +1774,30 @@ SET search_path = public, pg_catalog
|
|
|
1767
1774
|
AS $$
|
|
1768
1775
|
-- Keep in lockstep with the `@version:` marker in schema.sql (cut_release.ts
|
|
1769
1776
|
-- enforces it). Bump whenever schema.sql OR rpcs.sql changes.
|
|
1770
|
-
SELECT '0.
|
|
1777
|
+
SELECT '0.8.0'::TEXT;
|
|
1778
|
+
$$;
|
|
1779
|
+
|
|
1780
|
+
-- ── cerefox_content_format_stats ─────────────────────────────────────────────
|
|
1781
|
+
-- Counts how many (non-deleted) documents still use the legacy chunk
|
|
1782
|
+
-- reconstruction format (content_format = 1) vs the total. Powers the
|
|
1783
|
+
-- informational `cerefox doctor` line. See docs/guides/content-format.md.
|
|
1784
|
+
CREATE OR REPLACE FUNCTION cerefox_content_format_stats()
|
|
1785
|
+
RETURNS TABLE (legacy_docs INT, total_docs INT)
|
|
1786
|
+
LANGUAGE sql
|
|
1787
|
+
STABLE
|
|
1788
|
+
SECURITY DEFINER
|
|
1789
|
+
SET search_path = public, pg_catalog
|
|
1790
|
+
AS $$
|
|
1791
|
+
SELECT
|
|
1792
|
+
COUNT(*) FILTER (WHERE cf.min_format < 2)::INT AS legacy_docs,
|
|
1793
|
+
COUNT(*)::INT AS total_docs
|
|
1794
|
+
FROM cerefox_documents d
|
|
1795
|
+
LEFT JOIN LATERAL (
|
|
1796
|
+
SELECT MIN(c.content_format) AS min_format
|
|
1797
|
+
FROM cerefox_chunks c
|
|
1798
|
+
WHERE c.document_id = d.id AND c.version_id IS NULL
|
|
1799
|
+
) cf ON TRUE
|
|
1800
|
+
WHERE d.deleted_at IS NULL;
|
|
1771
1801
|
$$;
|
|
1772
1802
|
|
|
1773
1803
|
|
|
@@ -1847,3 +1877,41 @@ AS $$
|
|
|
1847
1877
|
WHERE a.document_id = ANY(p_doc_ids)
|
|
1848
1878
|
ORDER BY a.document_id, a.created_at DESC;
|
|
1849
1879
|
$$;
|
|
1880
|
+
|
|
1881
|
+
-- ── Function privilege lockdown (schema 0.7.0) ────────────────────────────────
|
|
1882
|
+
-- SECURITY (critical): every cerefox_* RPC is SECURITY DEFINER, so it bypasses the
|
|
1883
|
+
-- Row Level Security enabled on the tables. PostgreSQL grants EXECUTE to PUBLIC by
|
|
1884
|
+
-- default, and Supabase's PostgREST exposes public-schema functions at
|
|
1885
|
+
-- /rest/v1/rpc/<name>. WITHOUT this block, the anon key (and the new sb_publishable_
|
|
1886
|
+
-- key, which maps to the same anon role) could call every RPC directly via the Data
|
|
1887
|
+
-- API — reading and writing the entire KB, bypassing BOTH the Edge Functions and
|
|
1888
|
+
-- RLS. Every legitimate caller uses the service_role key instead (Edge Functions via
|
|
1889
|
+
-- SUPABASE_SERVICE_ROLE_KEY; the CLI/web via the secret key; local World B via a
|
|
1890
|
+
-- container-minted service_role JWT), so we REVOKE EXECUTE from PUBLIC/anon/
|
|
1891
|
+
-- authenticated and GRANT only to service_role.
|
|
1892
|
+
--
|
|
1893
|
+
-- Applied over ALL cerefox_* functions (so new functions are covered on the next
|
|
1894
|
+
-- deploy) and idempotent — re-applied each time rpcs.sql is deployed. Guarded on
|
|
1895
|
+
-- role existence so it is safe on non-Supabase Postgres (e.g. World B bootstrap).
|
|
1896
|
+
DO $$
|
|
1897
|
+
DECLARE
|
|
1898
|
+
fn regprocedure;
|
|
1899
|
+
r text;
|
|
1900
|
+
BEGIN
|
|
1901
|
+
FOR fn IN
|
|
1902
|
+
SELECT p.oid::regprocedure
|
|
1903
|
+
FROM pg_proc p
|
|
1904
|
+
JOIN pg_namespace n ON n.oid = p.pronamespace
|
|
1905
|
+
WHERE n.nspname = 'public' AND p.proname LIKE 'cerefox\_%'
|
|
1906
|
+
LOOP
|
|
1907
|
+
EXECUTE format('REVOKE EXECUTE ON FUNCTION %s FROM PUBLIC', fn);
|
|
1908
|
+
FOREACH r IN ARRAY ARRAY['anon', 'authenticated'] LOOP
|
|
1909
|
+
IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = r) THEN
|
|
1910
|
+
EXECUTE format('REVOKE EXECUTE ON FUNCTION %s FROM %I', fn, r);
|
|
1911
|
+
END IF;
|
|
1912
|
+
END LOOP;
|
|
1913
|
+
IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'service_role') THEN
|
|
1914
|
+
EXECUTE format('GRANT EXECUTE ON FUNCTION %s TO service_role', fn);
|
|
1915
|
+
END IF;
|
|
1916
|
+
END LOOP;
|
|
1917
|
+
END $$;
|
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
-- Requires extensions: vector (pgvector), uuid-ossp
|
|
6
6
|
-- These are enabled at the top of db_deploy.py before this file is applied.
|
|
7
7
|
--
|
|
8
|
-
-- @version: 0.
|
|
8
|
+
-- @version: 0.8.0
|
|
9
9
|
-- The `@version` marker above is read by the schema-version-mismatch banner
|
|
10
10
|
-- (see /api/v1/schema-version). Bump it whenever schema.sql OR rpcs.sql
|
|
11
11
|
-- changes in a way that requires `cerefox server deploy` to be re-run —
|
|
@@ -157,6 +157,12 @@ CREATE TABLE IF NOT EXISTS cerefox_chunks (
|
|
|
157
157
|
title TEXT,
|
|
158
158
|
content TEXT NOT NULL,
|
|
159
159
|
char_count INT NOT NULL,
|
|
160
|
+
-- content_format: how this chunk's content reconstructs into full document text
|
|
161
|
+
-- (iter-28D). 1 = legacy (chunk contents re-joined with E'\n\n' on read); 2 =
|
|
162
|
+
-- blind-stitch (chunk contents are an exact, gapless partition, reconstructed by
|
|
163
|
+
-- plain concat). It lives on the chunk (not the document) so an archived version
|
|
164
|
+
-- reconstructs with its OWN format. See docs/guides/content-format.md.
|
|
165
|
+
content_format SMALLINT NOT NULL DEFAULT 1,
|
|
160
166
|
|
|
161
167
|
-- Primary embedding: always computed, cloud API (default: OpenAI text-embedding-3-small)
|
|
162
168
|
embedding_primary VECTOR(768) NOT NULL,
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import "jsr:@supabase/functions-js/edge-runtime.d.ts";
|
|
2
2
|
import { createClient } from "jsr:@supabase/supabase-js@2";
|
|
3
3
|
import { isVersionRequest, versionResponse } from "../../../_shared/ef-meta/index.ts";
|
|
4
|
+
import { efAuthGate } from "../../../_shared/ef-auth/index.ts";
|
|
4
5
|
|
|
5
6
|
/**
|
|
6
7
|
* cerefox-get-audit-log -- Supabase Edge Function
|
|
@@ -35,6 +36,13 @@ Deno.serve(async (req: Request): Promise<Response> => {
|
|
|
35
36
|
return new Response(null, { status: 200, headers: CORS_HEADERS });
|
|
36
37
|
}
|
|
37
38
|
|
|
39
|
+
const authFail = efAuthGate(
|
|
40
|
+
req.headers.get("Authorization"),
|
|
41
|
+
Deno.env.get("CEREFOX_ACCESS_TOKENS"),
|
|
42
|
+
{ ...CORS_HEADERS, "Content-Type": "application/json" },
|
|
43
|
+
);
|
|
44
|
+
if (authFail) return authFail;
|
|
45
|
+
|
|
38
46
|
if (isVersionRequest(req)) {
|
|
39
47
|
return versionResponse("cerefox-get-audit-log", { ...CORS_HEADERS, "Content-Type": "application/json" });
|
|
40
48
|
}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import "jsr:@supabase/functions-js/edge-runtime.d.ts";
|
|
2
2
|
import { createClient } from "jsr:@supabase/supabase-js@2";
|
|
3
3
|
import { isVersionRequest, versionResponse } from "../../../_shared/ef-meta/index.ts";
|
|
4
|
+
import { efAuthGate } from "../../../_shared/ef-auth/index.ts";
|
|
4
5
|
|
|
5
6
|
/**
|
|
6
7
|
* cerefox-get-document — Supabase Edge Function
|
|
@@ -35,6 +36,13 @@ Deno.serve(async (req: Request): Promise<Response> => {
|
|
|
35
36
|
return new Response(null, { status: 200, headers: CORS_HEADERS });
|
|
36
37
|
}
|
|
37
38
|
|
|
39
|
+
const authFail = efAuthGate(
|
|
40
|
+
req.headers.get("Authorization"),
|
|
41
|
+
Deno.env.get("CEREFOX_ACCESS_TOKENS"),
|
|
42
|
+
{ ...CORS_HEADERS, "Content-Type": "application/json" },
|
|
43
|
+
);
|
|
44
|
+
if (authFail) return authFail;
|
|
45
|
+
|
|
38
46
|
if (isVersionRequest(req)) {
|
|
39
47
|
return versionResponse("cerefox-get-document", { ...CORS_HEADERS, "Content-Type": "application/json" });
|
|
40
48
|
}
|
|
@@ -1,6 +1,13 @@
|
|
|
1
1
|
import "jsr:@supabase/functions-js/edge-runtime.d.ts";
|
|
2
2
|
import { createClient } from "jsr:@supabase/supabase-js@2";
|
|
3
3
|
import { isVersionRequest, versionResponse } from "../../../_shared/ef-meta/index.ts";
|
|
4
|
+
import { efAuthGate } from "../../../_shared/ef-auth/index.ts";
|
|
5
|
+
import { capEmbeddingInput } from "../../../_shared/embeddings/index.ts";
|
|
6
|
+
import {
|
|
7
|
+
chunkMarkdown,
|
|
8
|
+
embeddingInputFor,
|
|
9
|
+
CONTENT_FORMAT_BLIND_STITCH,
|
|
10
|
+
} from "../../../_shared/ingest/chunker.ts";
|
|
4
11
|
|
|
5
12
|
/**
|
|
6
13
|
* cerefox-ingest — Supabase Edge Function
|
|
@@ -80,173 +87,6 @@ function concurrencyErrorResponse(
|
|
|
80
87
|
return null;
|
|
81
88
|
}
|
|
82
89
|
|
|
83
|
-
interface Chunk {
|
|
84
|
-
heading_path: string[];
|
|
85
|
-
heading_level: number;
|
|
86
|
-
title: string;
|
|
87
|
-
content: string;
|
|
88
|
-
char_count: number;
|
|
89
|
-
}
|
|
90
|
-
|
|
91
|
-
// ── Heading-aware chunker (mirrors Python logic) ───────────────────────────
|
|
92
|
-
//
|
|
93
|
-
// Design notes:
|
|
94
|
-
// • Short-circuit for small documents: if the entire document fits within
|
|
95
|
-
// MAX_CHUNK_CHARS, it is returned as a single chunk with no splitting.
|
|
96
|
-
// • Greedy accumulation: sections are collected into a buffer until adding
|
|
97
|
-
// the next would exceed MAX_CHUNK_CHARS. This keeps chunks close to the
|
|
98
|
-
// target size and avoids many tiny fragments at every heading boundary.
|
|
99
|
-
// All heading levels (H1/H2/H3) are treated equally — size alone controls
|
|
100
|
-
// when a chunk is flushed; there are no hard heading-level boundaries.
|
|
101
|
-
// • Oversized sections (> MAX_CHUNK_CHARS) are paragraph-split with no overlap.
|
|
102
|
-
// • The first section's heading metadata anchors each chunk's breadcrumb.
|
|
103
|
-
// • No overlaps between chunks — the heading breadcrumb in the content
|
|
104
|
-
// provides sufficient context. Overlaps caused duplication on reconstruction.
|
|
105
|
-
|
|
106
|
-
interface Section {
|
|
107
|
-
level: number;
|
|
108
|
-
headings: string[]; // full heading stack at this section
|
|
109
|
-
heading: string; // just the current heading text
|
|
110
|
-
content: string; // heading line + body
|
|
111
|
-
body: string; // body only (no heading line)
|
|
112
|
-
}
|
|
113
|
-
|
|
114
|
-
function parseSections(text: string): Section[] {
|
|
115
|
-
const lines = text.split("\n");
|
|
116
|
-
const sections: Section[] = [];
|
|
117
|
-
let currentHeadings: string[] = [];
|
|
118
|
-
let currentLevel = 0;
|
|
119
|
-
let bodyLines: string[] = [];
|
|
120
|
-
|
|
121
|
-
function collectSection() {
|
|
122
|
-
const body = bodyLines.join("\n").trim();
|
|
123
|
-
bodyLines = [];
|
|
124
|
-
let content: string;
|
|
125
|
-
if (currentLevel > 0) {
|
|
126
|
-
const headerLine = "#".repeat(currentLevel) + " " + (currentHeadings[currentHeadings.length - 1] ?? "");
|
|
127
|
-
content = body ? headerLine + "\n\n" + body : headerLine;
|
|
128
|
-
} else {
|
|
129
|
-
content = body;
|
|
130
|
-
}
|
|
131
|
-
if (!content.trim()) return;
|
|
132
|
-
sections.push({
|
|
133
|
-
level: currentLevel,
|
|
134
|
-
headings: [...currentHeadings],
|
|
135
|
-
heading: currentHeadings[currentHeadings.length - 1] ?? "",
|
|
136
|
-
content,
|
|
137
|
-
body,
|
|
138
|
-
});
|
|
139
|
-
}
|
|
140
|
-
|
|
141
|
-
for (const line of lines) {
|
|
142
|
-
const h1 = line.match(/^# (.+)/);
|
|
143
|
-
const h2 = line.match(/^## (.+)/);
|
|
144
|
-
const h3 = line.match(/^### (.+)/);
|
|
145
|
-
|
|
146
|
-
if (h1) {
|
|
147
|
-
collectSection();
|
|
148
|
-
currentHeadings = [h1[1].trim()];
|
|
149
|
-
currentLevel = 1;
|
|
150
|
-
} else if (h2) {
|
|
151
|
-
collectSection();
|
|
152
|
-
currentHeadings = [currentHeadings[0] ?? "", h2[1].trim()].filter(Boolean);
|
|
153
|
-
currentLevel = 2;
|
|
154
|
-
} else if (h3) {
|
|
155
|
-
collectSection();
|
|
156
|
-
currentHeadings = [
|
|
157
|
-
currentHeadings[0] ?? "",
|
|
158
|
-
currentHeadings[1] ?? "",
|
|
159
|
-
h3[1].trim(),
|
|
160
|
-
].filter(Boolean);
|
|
161
|
-
currentLevel = 3;
|
|
162
|
-
} else {
|
|
163
|
-
bodyLines.push(line);
|
|
164
|
-
}
|
|
165
|
-
}
|
|
166
|
-
collectSection();
|
|
167
|
-
return sections;
|
|
168
|
-
}
|
|
169
|
-
|
|
170
|
-
function chunkMarkdown(text: string): Chunk[] {
|
|
171
|
-
const trimmed = text.trim();
|
|
172
|
-
if (!trimmed) return [];
|
|
173
|
-
|
|
174
|
-
// Short-circuit: entire document fits in one chunk — skip heading splitting.
|
|
175
|
-
if (trimmed.length <= MAX_CHUNK_CHARS) {
|
|
176
|
-
return [makeChunk([], 0, trimmed)];
|
|
177
|
-
}
|
|
178
|
-
|
|
179
|
-
const sections = parseSections(trimmed);
|
|
180
|
-
const chunks: Chunk[] = [];
|
|
181
|
-
|
|
182
|
-
// Greedy accumulation buffer
|
|
183
|
-
let bufParts: string[] = [];
|
|
184
|
-
let bufHeadings: string[] = [];
|
|
185
|
-
let bufLevel = 0;
|
|
186
|
-
let bufChars = 0;
|
|
187
|
-
|
|
188
|
-
function flushBuf() {
|
|
189
|
-
if (bufParts.length === 0) return;
|
|
190
|
-
chunks.push(makeChunk(bufHeadings, bufLevel, bufParts.join("\n\n")));
|
|
191
|
-
bufParts = [];
|
|
192
|
-
bufHeadings = [];
|
|
193
|
-
bufLevel = 0;
|
|
194
|
-
bufChars = 0;
|
|
195
|
-
}
|
|
196
|
-
|
|
197
|
-
for (const section of sections) {
|
|
198
|
-
const { level, headings, heading, content, body } = section;
|
|
199
|
-
|
|
200
|
-
// Oversized section: flush buffer, then paragraph-split.
|
|
201
|
-
if (content.length > MAX_CHUNK_CHARS) {
|
|
202
|
-
flushBuf();
|
|
203
|
-
const headerPrefix = level > 0 ? "#".repeat(level) + " " + heading + "\n\n" : "";
|
|
204
|
-
const bodyToSplit = body || content;
|
|
205
|
-
const paragraphs = bodyToSplit.split(/\n\n+/);
|
|
206
|
-
let sub = "";
|
|
207
|
-
let isFirst = true;
|
|
208
|
-
for (const para of paragraphs) {
|
|
209
|
-
const prefix = isFirst ? headerPrefix : "";
|
|
210
|
-
if (sub.length + prefix.length + para.length + 2 > MAX_CHUNK_CHARS && sub.length > 0) {
|
|
211
|
-
chunks.push(makeChunk(headings, level, sub.trim()));
|
|
212
|
-
sub = para;
|
|
213
|
-
isFirst = false;
|
|
214
|
-
} else {
|
|
215
|
-
sub = sub ? sub + "\n\n" + para : prefix + para;
|
|
216
|
-
isFirst = false;
|
|
217
|
-
}
|
|
218
|
-
}
|
|
219
|
-
if (sub.trim()) chunks.push(makeChunk(headings, level, sub.trim()));
|
|
220
|
-
continue;
|
|
221
|
-
}
|
|
222
|
-
|
|
223
|
-
// Section fits. Try to accumulate into the buffer.
|
|
224
|
-
const addition = content.length + (bufParts.length > 0 ? 2 : 0);
|
|
225
|
-
|
|
226
|
-
if (bufChars + addition <= MAX_CHUNK_CHARS) {
|
|
227
|
-
if (bufParts.length === 0) {
|
|
228
|
-
bufHeadings = headings;
|
|
229
|
-
bufLevel = level;
|
|
230
|
-
}
|
|
231
|
-
bufParts.push(content);
|
|
232
|
-
bufChars += addition;
|
|
233
|
-
} else {
|
|
234
|
-
flushBuf();
|
|
235
|
-
bufParts = [content];
|
|
236
|
-
bufHeadings = headings;
|
|
237
|
-
bufLevel = level;
|
|
238
|
-
bufChars = content.length;
|
|
239
|
-
}
|
|
240
|
-
}
|
|
241
|
-
|
|
242
|
-
flushBuf();
|
|
243
|
-
return chunks;
|
|
244
|
-
}
|
|
245
|
-
|
|
246
|
-
function makeChunk(headings: string[], level: number, content: string): Chunk {
|
|
247
|
-
const title = headings[headings.length - 1] ?? "";
|
|
248
|
-
return { heading_path: [...headings], heading_level: level, title, content, char_count: content.length };
|
|
249
|
-
}
|
|
250
90
|
|
|
251
91
|
// ── Embedding ──────────────────────────────────────────────────────────────
|
|
252
92
|
|
|
@@ -255,6 +95,7 @@ const EMBEDDING_INITIAL_BACKOFF_MS = 500; // 500ms, 1s, 2s exponential backoff
|
|
|
255
95
|
|
|
256
96
|
async function embedBatch(texts: string[], apiKey: string): Promise<number[][]> {
|
|
257
97
|
let lastError: Error | null = null;
|
|
98
|
+
const inputs = texts.map(capEmbeddingInput); // iter-28D Phase 0: cap oversized inputs
|
|
258
99
|
|
|
259
100
|
for (let attempt = 0; attempt < EMBEDDING_MAX_RETRIES; attempt++) {
|
|
260
101
|
try {
|
|
@@ -266,7 +107,7 @@ async function embedBatch(texts: string[], apiKey: string): Promise<number[][]>
|
|
|
266
107
|
},
|
|
267
108
|
body: JSON.stringify({
|
|
268
109
|
model: OPENAI_MODEL,
|
|
269
|
-
input:
|
|
110
|
+
input: inputs,
|
|
270
111
|
dimensions: EMBEDDING_DIMENSIONS,
|
|
271
112
|
}),
|
|
272
113
|
});
|
|
@@ -442,6 +283,13 @@ Deno.serve(async (req: Request) => {
|
|
|
442
283
|
});
|
|
443
284
|
}
|
|
444
285
|
|
|
286
|
+
const authFail = efAuthGate(
|
|
287
|
+
req.headers.get("Authorization"),
|
|
288
|
+
Deno.env.get("CEREFOX_ACCESS_TOKENS"),
|
|
289
|
+
{ "Content-Type": "application/json", "Access-Control-Allow-Origin": "*" },
|
|
290
|
+
);
|
|
291
|
+
if (authFail) return authFail;
|
|
292
|
+
|
|
445
293
|
if (isVersionRequest(req)) {
|
|
446
294
|
return versionResponse("cerefox-ingest", {
|
|
447
295
|
"Content-Type": "application/json",
|
|
@@ -582,7 +430,7 @@ Deno.serve(async (req: Request) => {
|
|
|
582
430
|
return new Response(JSON.stringify({ error: "Content produced no chunks" }), { status: 422, headers });
|
|
583
431
|
}
|
|
584
432
|
|
|
585
|
-
const texts = chunks.map((c) =>
|
|
433
|
+
const texts = chunks.map((c) => embeddingInputFor(title.trim(), c));
|
|
586
434
|
let embeddings: number[][];
|
|
587
435
|
try {
|
|
588
436
|
embeddings = await embedBatch(texts, openaiKey);
|
|
@@ -615,6 +463,7 @@ Deno.serve(async (req: Request) => {
|
|
|
615
463
|
p_source_label: source,
|
|
616
464
|
p_expected_content_hash: expected_content_hash,
|
|
617
465
|
p_last_write_wins: last_write_wins,
|
|
466
|
+
p_content_format: CONTENT_FORMAT_BLIND_STITCH,
|
|
618
467
|
});
|
|
619
468
|
|
|
620
469
|
if (ingestErr) {
|
|
@@ -698,7 +547,7 @@ Deno.serve(async (req: Request) => {
|
|
|
698
547
|
}
|
|
699
548
|
|
|
700
549
|
// Prepend document title for contextual enrichment (stored content unchanged)
|
|
701
|
-
const texts = chunks.map((c) =>
|
|
550
|
+
const texts = chunks.map((c) => embeddingInputFor(title.trim(), c));
|
|
702
551
|
let embeddings: number[][];
|
|
703
552
|
try {
|
|
704
553
|
embeddings = await embedBatch(texts, openaiKey);
|
|
@@ -733,6 +582,7 @@ Deno.serve(async (req: Request) => {
|
|
|
733
582
|
p_source_label: source,
|
|
734
583
|
p_expected_content_hash: expected_content_hash,
|
|
735
584
|
p_last_write_wins: last_write_wins,
|
|
585
|
+
p_content_format: CONTENT_FORMAT_BLIND_STITCH,
|
|
736
586
|
});
|
|
737
587
|
|
|
738
588
|
if (ingestErr) {
|
|
@@ -806,7 +656,7 @@ Deno.serve(async (req: Request) => {
|
|
|
806
656
|
}
|
|
807
657
|
|
|
808
658
|
// Embed all chunks with title prefix for contextual enrichment (stored content unchanged)
|
|
809
|
-
const texts = chunks.map((c) =>
|
|
659
|
+
const texts = chunks.map((c) => embeddingInputFor(title.trim(), c));
|
|
810
660
|
let embeddings: number[][];
|
|
811
661
|
try {
|
|
812
662
|
embeddings = await embedBatch(texts, openaiKey);
|
|
@@ -841,6 +691,7 @@ Deno.serve(async (req: Request) => {
|
|
|
841
691
|
p_chunks: chunkData,
|
|
842
692
|
p_author: author,
|
|
843
693
|
p_author_type: author_type,
|
|
694
|
+
p_content_format: CONTENT_FORMAT_BLIND_STITCH,
|
|
844
695
|
});
|
|
845
696
|
|
|
846
697
|
if (ingestErr || !ingestResult?.length) {
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import "jsr:@supabase/functions-js/edge-runtime.d.ts";
|
|
2
2
|
import { createClient } from "jsr:@supabase/supabase-js@2";
|
|
3
3
|
import { isVersionRequest, versionResponse } from "../../../_shared/ef-meta/index.ts";
|
|
4
|
+
import { efAuthGate } from "../../../_shared/ef-auth/index.ts";
|
|
4
5
|
|
|
5
6
|
/**
|
|
6
7
|
* cerefox-list-projects -- Supabase Edge Function
|
|
@@ -29,6 +30,13 @@ Deno.serve(async (req: Request): Promise<Response> => {
|
|
|
29
30
|
return new Response(null, { status: 200, headers: CORS_HEADERS });
|
|
30
31
|
}
|
|
31
32
|
|
|
33
|
+
const authFail = efAuthGate(
|
|
34
|
+
req.headers.get("Authorization"),
|
|
35
|
+
Deno.env.get("CEREFOX_ACCESS_TOKENS"),
|
|
36
|
+
{ ...CORS_HEADERS, "Content-Type": "application/json" },
|
|
37
|
+
);
|
|
38
|
+
if (authFail) return authFail;
|
|
39
|
+
|
|
32
40
|
if (isVersionRequest(req)) {
|
|
33
41
|
return versionResponse("cerefox-list-projects", { ...CORS_HEADERS, "Content-Type": "application/json" });
|
|
34
42
|
}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import "jsr:@supabase/functions-js/edge-runtime.d.ts";
|
|
2
2
|
import { createClient } from "jsr:@supabase/supabase-js@2";
|
|
3
3
|
import { isVersionRequest, versionResponse } from "../../../_shared/ef-meta/index.ts";
|
|
4
|
+
import { efAuthGate } from "../../../_shared/ef-auth/index.ts";
|
|
4
5
|
|
|
5
6
|
/**
|
|
6
7
|
* cerefox-list-versions — Supabase Edge Function
|
|
@@ -34,6 +35,13 @@ Deno.serve(async (req: Request): Promise<Response> => {
|
|
|
34
35
|
return new Response(null, { status: 200, headers: CORS_HEADERS });
|
|
35
36
|
}
|
|
36
37
|
|
|
38
|
+
const authFail = efAuthGate(
|
|
39
|
+
req.headers.get("Authorization"),
|
|
40
|
+
Deno.env.get("CEREFOX_ACCESS_TOKENS"),
|
|
41
|
+
{ ...CORS_HEADERS, "Content-Type": "application/json" },
|
|
42
|
+
);
|
|
43
|
+
if (authFail) return authFail;
|
|
44
|
+
|
|
37
45
|
if (isVersionRequest(req)) {
|
|
38
46
|
return versionResponse("cerefox-list-versions", { ...CORS_HEADERS, "Content-Type": "application/json" });
|
|
39
47
|
}
|
|
@@ -27,6 +27,14 @@ import {
|
|
|
27
27
|
makeSupabaseClient,
|
|
28
28
|
notificationResponse,
|
|
29
29
|
} from "./shared.ts";
|
|
30
|
+
import {
|
|
31
|
+
buildAuthenticator,
|
|
32
|
+
isProtectedResourceMetadata,
|
|
33
|
+
protectedResourceMetadata,
|
|
34
|
+
unauthorizedChallenge,
|
|
35
|
+
} from "./oauth.ts";
|
|
36
|
+
import type { AuthResult, McpAuthenticator } from "../../../_shared/mcp-auth/index.ts";
|
|
37
|
+
import { checkAccessToken, parseAccessTokens } from "../../../_shared/ef-auth/index.ts";
|
|
30
38
|
import {
|
|
31
39
|
ALL_TOOLS,
|
|
32
40
|
McpInvalidParams,
|
|
@@ -233,11 +241,57 @@ async function handleVersion(req: Request): Promise<Response> {
|
|
|
233
241
|
|
|
234
242
|
// ── Main handler ─────────────────────────────────────────────────────────────
|
|
235
243
|
|
|
244
|
+
// Isolate-lifetime authenticator (holds the JWKS cache). Issuer/JWKS come from the
|
|
245
|
+
// injected SUPABASE_URL (not request headers — see oauth.ts projectOrigin).
|
|
246
|
+
let authenticator: McpAuthenticator | null = null;
|
|
247
|
+
function getAuthenticator(): McpAuthenticator {
|
|
248
|
+
if (!authenticator) authenticator = buildAuthenticator();
|
|
249
|
+
return authenticator;
|
|
250
|
+
}
|
|
251
|
+
|
|
236
252
|
Deno.serve(async (req: Request): Promise<Response> => {
|
|
237
253
|
if (req.method === "OPTIONS") {
|
|
238
254
|
return new Response(null, { status: 200, headers: CORS_HEADERS });
|
|
239
255
|
}
|
|
240
256
|
|
|
257
|
+
// Public discovery route (RFC 9728): the ONLY unauthenticated non-OPTIONS
|
|
258
|
+
// response. Served before auth so OAuth clients can bootstrap.
|
|
259
|
+
if (req.method === "GET" && isProtectedResourceMetadata(req)) {
|
|
260
|
+
return protectedResourceMetadata();
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
// ── Auth-first dispatch (design §6) ────────────────────────────────────────
|
|
264
|
+
// The function is deployed with --no-verify-jwt, so this in-function check is
|
|
265
|
+
// the ONLY gate. Accept EITHER a Cerefox access token (the static path — same
|
|
266
|
+
// credential the primitive EFs take, iter-28E) OR a valid OAuth JWT.
|
|
267
|
+
const authHeader = req.headers.get("Authorization");
|
|
268
|
+
let authResult: AuthResult;
|
|
269
|
+
// Static token first (cheap constant-time compare). A real OAuth JWT won't match
|
|
270
|
+
// any token, so it falls through to the JWKS path below — no false rejection.
|
|
271
|
+
const tokenResult = checkAccessToken(authHeader, {
|
|
272
|
+
tokens: parseAccessTokens(Deno.env.get("CEREFOX_ACCESS_TOKENS")),
|
|
273
|
+
});
|
|
274
|
+
if (tokenResult.ok) {
|
|
275
|
+
authResult = { ok: true, path: "static" };
|
|
276
|
+
} else {
|
|
277
|
+
authResult = await getAuthenticator().authenticate(authHeader);
|
|
278
|
+
}
|
|
279
|
+
if (!authResult.ok) {
|
|
280
|
+
// Log the machine reason (never the token) so the dashboard logs are
|
|
281
|
+
// actionable on a real auth failure. `no_token` is the normal OAuth-discovery
|
|
282
|
+
// probe (every cloud client sends one first) — don't log it as noise. The
|
|
283
|
+
// enriched detail (aud/sub/alg values from _shared/mcp-auth) names which claim
|
|
284
|
+
// a rejected token tripped. To debug a "Claude never sends the token" case
|
|
285
|
+
// (claude-ai #482), temporarily also log `Array.from(req.headers.keys())`.
|
|
286
|
+
if (authResult.reason !== "no_token") {
|
|
287
|
+
console.warn(
|
|
288
|
+
`[cerefox-mcp] auth rejected: ${authResult.reason}` +
|
|
289
|
+
(authResult.detail ? ` (${authResult.detail})` : ""),
|
|
290
|
+
);
|
|
291
|
+
}
|
|
292
|
+
return unauthorizedChallenge(authResult);
|
|
293
|
+
}
|
|
294
|
+
|
|
241
295
|
// GET — the only supported GET is the /version surface (iter-26). Per MCP
|
|
242
296
|
// spec (2025-03-26) this server otherwise returns 405 on GET to signal it
|
|
243
297
|
// does not support SSE notifications (prevents MCP clients from holding a
|