@cerefox/memory 0.11.1 → 1.0.0-beta.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENT_GUIDE.md +2 -2
- package/AGENT_QUICK_REFERENCE.md +1 -1
- package/dist/bin/cerefox.js +1079 -836
- package/dist/frontend/assets/index-D3FshoP3.js +125 -0
- package/dist/frontend/assets/index-D3FshoP3.js.map +1 -0
- package/dist/frontend/index.html +1 -1
- package/dist/server-assets/_shared/ef-auth/index.ts +134 -0
- package/dist/server-assets/_shared/ef-meta/index.ts +1 -1
- package/dist/server-assets/_shared/embeddings/index.ts +42 -2
- package/dist/server-assets/_shared/ingest/chunker.ts +210 -0
- package/dist/server-assets/_shared/ingest/index.ts +32 -0
- package/dist/server-assets/_shared/ingest/pipeline-helpers.ts +135 -0
- package/dist/server-assets/_shared/mcp-auth/index.ts +352 -0
- package/dist/server-assets/_shared/mcp-tools/_chunker.ts +16 -170
- package/dist/server-assets/_shared/mcp-tools/get-help-content.ts +2 -2
- package/dist/server-assets/_shared/mcp-tools/ingest.ts +13 -4
- package/dist/server-assets/db/migrations/0012_content_format.sql +20 -0
- package/dist/server-assets/db/rpcs.sql +76 -8
- package/dist/server-assets/db/schema.sql +7 -1
- package/dist/server-assets/supabase/functions/cerefox-get-audit-log/index.ts +8 -0
- package/dist/server-assets/supabase/functions/cerefox-get-document/index.ts +8 -0
- package/dist/server-assets/supabase/functions/cerefox-ingest/index.ts +39 -173
- package/dist/server-assets/supabase/functions/cerefox-list-projects/index.ts +8 -0
- package/dist/server-assets/supabase/functions/cerefox-list-versions/index.ts +8 -0
- package/dist/server-assets/supabase/functions/cerefox-mcp/index.ts +54 -0
- package/dist/server-assets/supabase/functions/cerefox-mcp/oauth.ts +121 -0
- package/dist/server-assets/supabase/functions/cerefox-metadata/index.ts +8 -0
- package/dist/server-assets/supabase/functions/cerefox-metadata-search/index.ts +8 -0
- package/dist/server-assets/supabase/functions/cerefox-search/index.ts +11 -1
- package/docs/guides/access-paths.md +84 -35
- package/docs/guides/cli.md +29 -0
- package/docs/guides/configuration.md +10 -8
- package/docs/guides/connect-agents.md +99 -101
- package/docs/guides/content-format.md +55 -0
- package/docs/guides/migration-1.0.md +96 -0
- package/docs/guides/ops-scripts.md +5 -7
- package/docs/guides/quickstart.md +22 -2
- package/docs/guides/setup-cloud-run.md +5 -9
- package/docs/guides/setup-supabase.md +157 -16
- package/docs/guides/upgrading.md +7 -8
- package/package.json +1 -1
- package/dist/frontend/assets/index-ojNhWSxm.js +0 -125
- package/dist/frontend/assets/index-ojNhWSxm.js.map +0 -1
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
-- Migration 0012: content_format on cerefox_chunks (iter-28D)
|
|
2
|
+
--
|
|
3
|
+
-- Records how each chunk's content reconstructs into full document text:
|
|
4
|
+
-- 1 = legacy — chunk contents were trimmed sections; reconstruction re-joins
|
|
5
|
+
-- them with E'\n\n' (the pre-28D behaviour). All existing chunks.
|
|
6
|
+
-- 2 = blind-stitch — chunk contents are an exact, gapless partition of the
|
|
7
|
+
-- document; reconstruction is a plain concatenation (no separator
|
|
8
|
+
-- synthesized on read). Written by the exact-partition chunker.
|
|
9
|
+
--
|
|
10
|
+
-- Placed on the CHUNK (not the document) so an archived version reconstructs with
|
|
11
|
+
-- its OWN format, since Cerefox uses chunks-anchored versioning
|
|
12
|
+
-- (cerefox_chunks.version_id). The reconstruction RPCs branch on
|
|
13
|
+
-- MAX(content_format) >= 2 per aggregated group.
|
|
14
|
+
--
|
|
15
|
+
-- Adding a NOT NULL column with a constant default is a metadata-only change in
|
|
16
|
+
-- PostgreSQL 11+ (no rewrite of the chunks table). Existing rows read back as 1.
|
|
17
|
+
-- Explanation for users: docs/guides/content-format.md.
|
|
18
|
+
|
|
19
|
+
ALTER TABLE cerefox_chunks
|
|
20
|
+
ADD COLUMN IF NOT EXISTS content_format SMALLINT NOT NULL DEFAULT 1;
|
|
@@ -403,7 +403,7 @@ AS $$
|
|
|
403
403
|
ARRAY(SELECT p.name FROM cerefox_projects p
|
|
404
404
|
JOIN cerefox_document_projects dp ON p.id = dp.project_id
|
|
405
405
|
WHERE dp.document_id = d.id) AS doc_project_names,
|
|
406
|
-
STRING_AGG(c.content, E'\n\n' ORDER BY c.chunk_index) AS full_content,
|
|
406
|
+
CASE WHEN MAX(c.content_format) >= 2 THEN STRING_AGG(c.content, '' ORDER BY c.chunk_index) ELSE STRING_AGG(c.content, E'\n\n' ORDER BY c.chunk_index) END AS full_content,
|
|
407
407
|
COUNT(*)::INT AS chunk_count,
|
|
408
408
|
SUM(c.char_count)::INT AS total_chars,
|
|
409
409
|
(SELECT COUNT(*)::INT FROM cerefox_document_versions dv
|
|
@@ -680,17 +680,18 @@ AS $$
|
|
|
680
680
|
large_doc_content AS (
|
|
681
681
|
SELECT
|
|
682
682
|
e.document_id,
|
|
683
|
-
STRING_AGG(e.content, E'\n\n' ORDER BY e.chunk_index) AS full_content,
|
|
683
|
+
CASE WHEN MAX(ch.content_format) >= 2 THEN STRING_AGG(e.content, '' ORDER BY e.chunk_index) ELSE STRING_AGG(e.content, E'\n\n' ORDER BY e.chunk_index) END AS full_content,
|
|
684
684
|
COUNT(*)::INT AS chunk_count,
|
|
685
685
|
TRUE AS is_partial
|
|
686
686
|
FROM expanded e
|
|
687
|
+
JOIN cerefox_chunks ch ON ch.id = e.chunk_id
|
|
687
688
|
GROUP BY e.document_id
|
|
688
689
|
),
|
|
689
690
|
-- Full content for small documents (is_partial = FALSE).
|
|
690
691
|
small_doc_content AS (
|
|
691
692
|
SELECT
|
|
692
693
|
c.document_id,
|
|
693
|
-
STRING_AGG(c.content, E'\n\n' ORDER BY c.chunk_index) AS full_content,
|
|
694
|
+
CASE WHEN MAX(c.content_format) >= 2 THEN STRING_AGG(c.content, '' ORDER BY c.chunk_index) ELSE STRING_AGG(c.content, E'\n\n' ORDER BY c.chunk_index) END AS full_content,
|
|
694
695
|
COUNT(*)::INT AS chunk_count,
|
|
695
696
|
FALSE AS is_partial
|
|
696
697
|
FROM cerefox_chunks c
|
|
@@ -866,7 +867,7 @@ AS $$
|
|
|
866
867
|
JOIN cerefox_document_projects dp ON p.id = dp.project_id
|
|
867
868
|
WHERE dp.document_id = d.id) AS doc_project_names,
|
|
868
869
|
p_version_id AS version_id,
|
|
869
|
-
STRING_AGG(c.content, E'\n\n' ORDER BY c.chunk_index) AS full_content,
|
|
870
|
+
CASE WHEN MAX(c.content_format) >= 2 THEN STRING_AGG(c.content, '' ORDER BY c.chunk_index) ELSE STRING_AGG(c.content, E'\n\n' ORDER BY c.chunk_index) END AS full_content,
|
|
870
871
|
COUNT(*)::INT AS chunk_count,
|
|
871
872
|
SUM(c.char_count)::INT AS total_chars,
|
|
872
873
|
d.created_at,
|
|
@@ -1072,6 +1073,7 @@ $$;
|
|
|
1072
1073
|
|
|
1073
1074
|
DROP FUNCTION IF EXISTS cerefox_ingest_document(UUID, TEXT, TEXT, TEXT, TEXT, JSONB, TEXT, JSONB, TEXT, TEXT, TEXT, INT, BOOLEAN);
|
|
1074
1075
|
DROP FUNCTION IF EXISTS cerefox_ingest_document(UUID, TEXT, TEXT, TEXT, TEXT, JSONB, TEXT, JSONB, TEXT, TEXT, TEXT, INT, BOOLEAN, TEXT, BOOLEAN);
|
|
1076
|
+
DROP FUNCTION IF EXISTS cerefox_ingest_document(UUID, TEXT, TEXT, TEXT, TEXT, JSONB, TEXT, JSONB, TEXT, TEXT, TEXT, INT, BOOLEAN, TEXT, BOOLEAN, SMALLINT);
|
|
1075
1077
|
CREATE FUNCTION cerefox_ingest_document(
|
|
1076
1078
|
p_document_id UUID DEFAULT NULL,
|
|
1077
1079
|
p_title TEXT DEFAULT 'Untitled',
|
|
@@ -1090,7 +1092,11 @@ CREATE FUNCTION cerefox_ingest_document(
|
|
|
1090
1092
|
p_retention_hours INT DEFAULT 48,
|
|
1091
1093
|
p_cleanup_enabled BOOLEAN DEFAULT TRUE,
|
|
1092
1094
|
p_expected_content_hash TEXT DEFAULT NULL,
|
|
1093
|
-
p_last_write_wins BOOLEAN DEFAULT FALSE
|
|
1095
|
+
p_last_write_wins BOOLEAN DEFAULT FALSE,
|
|
1096
|
+
-- content_format for the chunks being written (iter-28D). 2 = exact-partition
|
|
1097
|
+
-- (blind-stitch reconstruction); default 1 = legacy (E'\n\n'-join). Stamped on
|
|
1098
|
+
-- every chunk this call inserts.
|
|
1099
|
+
p_content_format SMALLINT DEFAULT 1
|
|
1094
1100
|
)
|
|
1095
1101
|
RETURNS TABLE (
|
|
1096
1102
|
document_id UUID,
|
|
@@ -1222,7 +1228,7 @@ BEGIN
|
|
|
1222
1228
|
-- Formula: doc_title (A) || chunk_heading (A) || body_content (B)
|
|
1223
1229
|
INSERT INTO cerefox_chunks (
|
|
1224
1230
|
document_id, chunk_index, heading_path, heading_level,
|
|
1225
|
-
title, content, char_count, embedding_primary, embedder_primary, fts
|
|
1231
|
+
title, content, char_count, content_format, embedding_primary, embedder_primary, fts
|
|
1226
1232
|
)
|
|
1227
1233
|
SELECT
|
|
1228
1234
|
v_doc_id,
|
|
@@ -1232,6 +1238,7 @@ BEGIN
|
|
|
1232
1238
|
c->>'title',
|
|
1233
1239
|
c->>'content',
|
|
1234
1240
|
(c->>'char_count')::INT,
|
|
1241
|
+
p_content_format,
|
|
1235
1242
|
(SELECT array_agg(e::FLOAT)::VECTOR(768) FROM jsonb_array_elements_text(c->'embedding') AS e),
|
|
1236
1243
|
c->>'embedder',
|
|
1237
1244
|
setweight(to_tsvector('english', COALESCE(p_title, '')), 'A') ||
|
|
@@ -1509,7 +1516,7 @@ BEGIN
|
|
|
1509
1516
|
WHERE dv.document_id = d.id) AS version_count,
|
|
1510
1517
|
d.content_hash,
|
|
1511
1518
|
CASE WHEN p_include_content THEN
|
|
1512
|
-
(SELECT STRING_AGG(c.content, E'\n\n' ORDER BY c.chunk_index)
|
|
1519
|
+
(SELECT CASE WHEN MAX(c.content_format) >= 2 THEN STRING_AGG(c.content, '' ORDER BY c.chunk_index) ELSE STRING_AGG(c.content, E'\n\n' ORDER BY c.chunk_index) END
|
|
1513
1520
|
FROM cerefox_chunks c
|
|
1514
1521
|
WHERE c.document_id = d.id AND c.version_id IS NULL)
|
|
1515
1522
|
ELSE NULL END AS content
|
|
@@ -1767,7 +1774,30 @@ SET search_path = public, pg_catalog
|
|
|
1767
1774
|
AS $$
|
|
1768
1775
|
-- Keep in lockstep with the `@version:` marker in schema.sql (cut_release.ts
|
|
1769
1776
|
-- enforces it). Bump whenever schema.sql OR rpcs.sql changes.
|
|
1770
|
-
SELECT '0.
|
|
1777
|
+
SELECT '0.8.0'::TEXT;
|
|
1778
|
+
$$;
|
|
1779
|
+
|
|
1780
|
+
-- ── cerefox_content_format_stats ─────────────────────────────────────────────
|
|
1781
|
+
-- Counts how many (non-deleted) documents still use the legacy chunk
|
|
1782
|
+
-- reconstruction format (content_format = 1) vs the total. Powers the
|
|
1783
|
+
-- informational `cerefox doctor` line. See docs/guides/content-format.md.
|
|
1784
|
+
CREATE OR REPLACE FUNCTION cerefox_content_format_stats()
|
|
1785
|
+
RETURNS TABLE (legacy_docs INT, total_docs INT)
|
|
1786
|
+
LANGUAGE sql
|
|
1787
|
+
STABLE
|
|
1788
|
+
SECURITY DEFINER
|
|
1789
|
+
SET search_path = public, pg_catalog
|
|
1790
|
+
AS $$
|
|
1791
|
+
SELECT
|
|
1792
|
+
COUNT(*) FILTER (WHERE cf.min_format < 2)::INT AS legacy_docs,
|
|
1793
|
+
COUNT(*)::INT AS total_docs
|
|
1794
|
+
FROM cerefox_documents d
|
|
1795
|
+
LEFT JOIN LATERAL (
|
|
1796
|
+
SELECT MIN(c.content_format) AS min_format
|
|
1797
|
+
FROM cerefox_chunks c
|
|
1798
|
+
WHERE c.document_id = d.id AND c.version_id IS NULL
|
|
1799
|
+
) cf ON TRUE
|
|
1800
|
+
WHERE d.deleted_at IS NULL;
|
|
1771
1801
|
$$;
|
|
1772
1802
|
|
|
1773
1803
|
|
|
@@ -1847,3 +1877,41 @@ AS $$
|
|
|
1847
1877
|
WHERE a.document_id = ANY(p_doc_ids)
|
|
1848
1878
|
ORDER BY a.document_id, a.created_at DESC;
|
|
1849
1879
|
$$;
|
|
1880
|
+
|
|
1881
|
+
-- ── Function privilege lockdown (schema 0.7.0) ────────────────────────────────
|
|
1882
|
+
-- SECURITY (critical): every cerefox_* RPC is SECURITY DEFINER, so it bypasses the
|
|
1883
|
+
-- Row Level Security enabled on the tables. PostgreSQL grants EXECUTE to PUBLIC by
|
|
1884
|
+
-- default, and Supabase's PostgREST exposes public-schema functions at
|
|
1885
|
+
-- /rest/v1/rpc/<name>. WITHOUT this block, the anon key (and the new sb_publishable_
|
|
1886
|
+
-- key, which maps to the same anon role) could call every RPC directly via the Data
|
|
1887
|
+
-- API — reading and writing the entire KB, bypassing BOTH the Edge Functions and
|
|
1888
|
+
-- RLS. Every legitimate caller uses the service_role key instead (Edge Functions via
|
|
1889
|
+
-- SUPABASE_SERVICE_ROLE_KEY; the CLI/web via the secret key; local World B via a
|
|
1890
|
+
-- container-minted service_role JWT), so we REVOKE EXECUTE from PUBLIC/anon/
|
|
1891
|
+
-- authenticated and GRANT only to service_role.
|
|
1892
|
+
--
|
|
1893
|
+
-- Applied over ALL cerefox_* functions (so new functions are covered on the next
|
|
1894
|
+
-- deploy) and idempotent — re-applied each time rpcs.sql is deployed. Guarded on
|
|
1895
|
+
-- role existence so it is safe on non-Supabase Postgres (e.g. World B bootstrap).
|
|
1896
|
+
DO $$
|
|
1897
|
+
DECLARE
|
|
1898
|
+
fn regprocedure;
|
|
1899
|
+
r text;
|
|
1900
|
+
BEGIN
|
|
1901
|
+
FOR fn IN
|
|
1902
|
+
SELECT p.oid::regprocedure
|
|
1903
|
+
FROM pg_proc p
|
|
1904
|
+
JOIN pg_namespace n ON n.oid = p.pronamespace
|
|
1905
|
+
WHERE n.nspname = 'public' AND p.proname LIKE 'cerefox\_%'
|
|
1906
|
+
LOOP
|
|
1907
|
+
EXECUTE format('REVOKE EXECUTE ON FUNCTION %s FROM PUBLIC', fn);
|
|
1908
|
+
FOREACH r IN ARRAY ARRAY['anon', 'authenticated'] LOOP
|
|
1909
|
+
IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = r) THEN
|
|
1910
|
+
EXECUTE format('REVOKE EXECUTE ON FUNCTION %s FROM %I', fn, r);
|
|
1911
|
+
END IF;
|
|
1912
|
+
END LOOP;
|
|
1913
|
+
IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'service_role') THEN
|
|
1914
|
+
EXECUTE format('GRANT EXECUTE ON FUNCTION %s TO service_role', fn);
|
|
1915
|
+
END IF;
|
|
1916
|
+
END LOOP;
|
|
1917
|
+
END $$;
|
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
-- Requires extensions: vector (pgvector), uuid-ossp
|
|
6
6
|
-- These are enabled at the top of db_deploy.py before this file is applied.
|
|
7
7
|
--
|
|
8
|
-
-- @version: 0.
|
|
8
|
+
-- @version: 0.8.0
|
|
9
9
|
-- The `@version` marker above is read by the schema-version-mismatch banner
|
|
10
10
|
-- (see /api/v1/schema-version). Bump it whenever schema.sql OR rpcs.sql
|
|
11
11
|
-- changes in a way that requires `cerefox server deploy` to be re-run —
|
|
@@ -157,6 +157,12 @@ CREATE TABLE IF NOT EXISTS cerefox_chunks (
|
|
|
157
157
|
title TEXT,
|
|
158
158
|
content TEXT NOT NULL,
|
|
159
159
|
char_count INT NOT NULL,
|
|
160
|
+
-- content_format: how this chunk's content reconstructs into full document text
|
|
161
|
+
-- (iter-28D). 1 = legacy (chunk contents re-joined with E'\n\n' on read); 2 =
|
|
162
|
+
-- blind-stitch (chunk contents are an exact, gapless partition, reconstructed by
|
|
163
|
+
-- plain concat). It lives on the chunk (not the document) so an archived version
|
|
164
|
+
-- reconstructs with its OWN format. See docs/guides/content-format.md.
|
|
165
|
+
content_format SMALLINT NOT NULL DEFAULT 1,
|
|
160
166
|
|
|
161
167
|
-- Primary embedding: always computed, cloud API (default: OpenAI text-embedding-3-small)
|
|
162
168
|
embedding_primary VECTOR(768) NOT NULL,
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import "jsr:@supabase/functions-js/edge-runtime.d.ts";
|
|
2
2
|
import { createClient } from "jsr:@supabase/supabase-js@2";
|
|
3
3
|
import { isVersionRequest, versionResponse } from "../../../_shared/ef-meta/index.ts";
|
|
4
|
+
import { efAuthGate } from "../../../_shared/ef-auth/index.ts";
|
|
4
5
|
|
|
5
6
|
/**
|
|
6
7
|
* cerefox-get-audit-log -- Supabase Edge Function
|
|
@@ -35,6 +36,13 @@ Deno.serve(async (req: Request): Promise<Response> => {
|
|
|
35
36
|
return new Response(null, { status: 200, headers: CORS_HEADERS });
|
|
36
37
|
}
|
|
37
38
|
|
|
39
|
+
const authFail = efAuthGate(
|
|
40
|
+
req.headers.get("Authorization"),
|
|
41
|
+
Deno.env.get("CEREFOX_ACCESS_TOKENS"),
|
|
42
|
+
{ ...CORS_HEADERS, "Content-Type": "application/json" },
|
|
43
|
+
);
|
|
44
|
+
if (authFail) return authFail;
|
|
45
|
+
|
|
38
46
|
if (isVersionRequest(req)) {
|
|
39
47
|
return versionResponse("cerefox-get-audit-log", { ...CORS_HEADERS, "Content-Type": "application/json" });
|
|
40
48
|
}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import "jsr:@supabase/functions-js/edge-runtime.d.ts";
|
|
2
2
|
import { createClient } from "jsr:@supabase/supabase-js@2";
|
|
3
3
|
import { isVersionRequest, versionResponse } from "../../../_shared/ef-meta/index.ts";
|
|
4
|
+
import { efAuthGate } from "../../../_shared/ef-auth/index.ts";
|
|
4
5
|
|
|
5
6
|
/**
|
|
6
7
|
* cerefox-get-document — Supabase Edge Function
|
|
@@ -35,6 +36,13 @@ Deno.serve(async (req: Request): Promise<Response> => {
|
|
|
35
36
|
return new Response(null, { status: 200, headers: CORS_HEADERS });
|
|
36
37
|
}
|
|
37
38
|
|
|
39
|
+
const authFail = efAuthGate(
|
|
40
|
+
req.headers.get("Authorization"),
|
|
41
|
+
Deno.env.get("CEREFOX_ACCESS_TOKENS"),
|
|
42
|
+
{ ...CORS_HEADERS, "Content-Type": "application/json" },
|
|
43
|
+
);
|
|
44
|
+
if (authFail) return authFail;
|
|
45
|
+
|
|
38
46
|
if (isVersionRequest(req)) {
|
|
39
47
|
return versionResponse("cerefox-get-document", { ...CORS_HEADERS, "Content-Type": "application/json" });
|
|
40
48
|
}
|
|
@@ -1,6 +1,13 @@
|
|
|
1
1
|
import "jsr:@supabase/functions-js/edge-runtime.d.ts";
|
|
2
2
|
import { createClient } from "jsr:@supabase/supabase-js@2";
|
|
3
3
|
import { isVersionRequest, versionResponse } from "../../../_shared/ef-meta/index.ts";
|
|
4
|
+
import { efAuthGate } from "../../../_shared/ef-auth/index.ts";
|
|
5
|
+
import { capEmbeddingInput } from "../../../_shared/embeddings/index.ts";
|
|
6
|
+
import {
|
|
7
|
+
chunkMarkdown,
|
|
8
|
+
embeddingInputFor,
|
|
9
|
+
CONTENT_FORMAT_BLIND_STITCH,
|
|
10
|
+
} from "../../../_shared/ingest/chunker.ts";
|
|
4
11
|
|
|
5
12
|
/**
|
|
6
13
|
* cerefox-ingest — Supabase Edge Function
|
|
@@ -49,12 +56,27 @@ interface IngestRequest {
|
|
|
49
56
|
last_write_wins?: boolean;
|
|
50
57
|
}
|
|
51
58
|
|
|
52
|
-
// Map the RPC's
|
|
53
|
-
//
|
|
59
|
+
// Map the RPC's expected/validation errors to proper HTTP responses:
|
|
60
|
+
// CEREFOX_CONFLICT → 409 (stale optimistic-concurrency token)
|
|
61
|
+
// CEREFOX_TOKEN_REQUIRED → 400 (missing expected_content_hash)
|
|
62
|
+
// cerefox_documents_hash_unique → 409 (content de-dup: another doc already
|
|
63
|
+
// holds identical content)
|
|
64
|
+
// Returns null for genuinely unexpected errors (→ 500 at the call site).
|
|
54
65
|
function concurrencyErrorResponse(
|
|
55
66
|
message: string,
|
|
56
67
|
headers: Record<string, string>,
|
|
57
68
|
): Response | null {
|
|
69
|
+
if (message.includes("cerefox_documents_hash_unique")) {
|
|
70
|
+
return new Response(
|
|
71
|
+
JSON.stringify({
|
|
72
|
+
error: "duplicate_content",
|
|
73
|
+
message:
|
|
74
|
+
"Another document already has identical content. Cerefox de-duplicates by content hash — update that document instead of creating or editing a second copy to match it.",
|
|
75
|
+
detail: message,
|
|
76
|
+
}),
|
|
77
|
+
{ status: 409, headers },
|
|
78
|
+
);
|
|
79
|
+
}
|
|
58
80
|
if (message.includes("CEREFOX_CONFLICT")) {
|
|
59
81
|
return new Response(
|
|
60
82
|
JSON.stringify({
|
|
@@ -80,173 +102,6 @@ function concurrencyErrorResponse(
|
|
|
80
102
|
return null;
|
|
81
103
|
}
|
|
82
104
|
|
|
83
|
-
interface Chunk {
|
|
84
|
-
heading_path: string[];
|
|
85
|
-
heading_level: number;
|
|
86
|
-
title: string;
|
|
87
|
-
content: string;
|
|
88
|
-
char_count: number;
|
|
89
|
-
}
|
|
90
|
-
|
|
91
|
-
// ── Heading-aware chunker (mirrors Python logic) ───────────────────────────
|
|
92
|
-
//
|
|
93
|
-
// Design notes:
|
|
94
|
-
// • Short-circuit for small documents: if the entire document fits within
|
|
95
|
-
// MAX_CHUNK_CHARS, it is returned as a single chunk with no splitting.
|
|
96
|
-
// • Greedy accumulation: sections are collected into a buffer until adding
|
|
97
|
-
// the next would exceed MAX_CHUNK_CHARS. This keeps chunks close to the
|
|
98
|
-
// target size and avoids many tiny fragments at every heading boundary.
|
|
99
|
-
// All heading levels (H1/H2/H3) are treated equally — size alone controls
|
|
100
|
-
// when a chunk is flushed; there are no hard heading-level boundaries.
|
|
101
|
-
// • Oversized sections (> MAX_CHUNK_CHARS) are paragraph-split with no overlap.
|
|
102
|
-
// • The first section's heading metadata anchors each chunk's breadcrumb.
|
|
103
|
-
// • No overlaps between chunks — the heading breadcrumb in the content
|
|
104
|
-
// provides sufficient context. Overlaps caused duplication on reconstruction.
|
|
105
|
-
|
|
106
|
-
interface Section {
|
|
107
|
-
level: number;
|
|
108
|
-
headings: string[]; // full heading stack at this section
|
|
109
|
-
heading: string; // just the current heading text
|
|
110
|
-
content: string; // heading line + body
|
|
111
|
-
body: string; // body only (no heading line)
|
|
112
|
-
}
|
|
113
|
-
|
|
114
|
-
function parseSections(text: string): Section[] {
|
|
115
|
-
const lines = text.split("\n");
|
|
116
|
-
const sections: Section[] = [];
|
|
117
|
-
let currentHeadings: string[] = [];
|
|
118
|
-
let currentLevel = 0;
|
|
119
|
-
let bodyLines: string[] = [];
|
|
120
|
-
|
|
121
|
-
function collectSection() {
|
|
122
|
-
const body = bodyLines.join("\n").trim();
|
|
123
|
-
bodyLines = [];
|
|
124
|
-
let content: string;
|
|
125
|
-
if (currentLevel > 0) {
|
|
126
|
-
const headerLine = "#".repeat(currentLevel) + " " + (currentHeadings[currentHeadings.length - 1] ?? "");
|
|
127
|
-
content = body ? headerLine + "\n\n" + body : headerLine;
|
|
128
|
-
} else {
|
|
129
|
-
content = body;
|
|
130
|
-
}
|
|
131
|
-
if (!content.trim()) return;
|
|
132
|
-
sections.push({
|
|
133
|
-
level: currentLevel,
|
|
134
|
-
headings: [...currentHeadings],
|
|
135
|
-
heading: currentHeadings[currentHeadings.length - 1] ?? "",
|
|
136
|
-
content,
|
|
137
|
-
body,
|
|
138
|
-
});
|
|
139
|
-
}
|
|
140
|
-
|
|
141
|
-
for (const line of lines) {
|
|
142
|
-
const h1 = line.match(/^# (.+)/);
|
|
143
|
-
const h2 = line.match(/^## (.+)/);
|
|
144
|
-
const h3 = line.match(/^### (.+)/);
|
|
145
|
-
|
|
146
|
-
if (h1) {
|
|
147
|
-
collectSection();
|
|
148
|
-
currentHeadings = [h1[1].trim()];
|
|
149
|
-
currentLevel = 1;
|
|
150
|
-
} else if (h2) {
|
|
151
|
-
collectSection();
|
|
152
|
-
currentHeadings = [currentHeadings[0] ?? "", h2[1].trim()].filter(Boolean);
|
|
153
|
-
currentLevel = 2;
|
|
154
|
-
} else if (h3) {
|
|
155
|
-
collectSection();
|
|
156
|
-
currentHeadings = [
|
|
157
|
-
currentHeadings[0] ?? "",
|
|
158
|
-
currentHeadings[1] ?? "",
|
|
159
|
-
h3[1].trim(),
|
|
160
|
-
].filter(Boolean);
|
|
161
|
-
currentLevel = 3;
|
|
162
|
-
} else {
|
|
163
|
-
bodyLines.push(line);
|
|
164
|
-
}
|
|
165
|
-
}
|
|
166
|
-
collectSection();
|
|
167
|
-
return sections;
|
|
168
|
-
}
|
|
169
|
-
|
|
170
|
-
function chunkMarkdown(text: string): Chunk[] {
|
|
171
|
-
const trimmed = text.trim();
|
|
172
|
-
if (!trimmed) return [];
|
|
173
|
-
|
|
174
|
-
// Short-circuit: entire document fits in one chunk — skip heading splitting.
|
|
175
|
-
if (trimmed.length <= MAX_CHUNK_CHARS) {
|
|
176
|
-
return [makeChunk([], 0, trimmed)];
|
|
177
|
-
}
|
|
178
|
-
|
|
179
|
-
const sections = parseSections(trimmed);
|
|
180
|
-
const chunks: Chunk[] = [];
|
|
181
|
-
|
|
182
|
-
// Greedy accumulation buffer
|
|
183
|
-
let bufParts: string[] = [];
|
|
184
|
-
let bufHeadings: string[] = [];
|
|
185
|
-
let bufLevel = 0;
|
|
186
|
-
let bufChars = 0;
|
|
187
|
-
|
|
188
|
-
function flushBuf() {
|
|
189
|
-
if (bufParts.length === 0) return;
|
|
190
|
-
chunks.push(makeChunk(bufHeadings, bufLevel, bufParts.join("\n\n")));
|
|
191
|
-
bufParts = [];
|
|
192
|
-
bufHeadings = [];
|
|
193
|
-
bufLevel = 0;
|
|
194
|
-
bufChars = 0;
|
|
195
|
-
}
|
|
196
|
-
|
|
197
|
-
for (const section of sections) {
|
|
198
|
-
const { level, headings, heading, content, body } = section;
|
|
199
|
-
|
|
200
|
-
// Oversized section: flush buffer, then paragraph-split.
|
|
201
|
-
if (content.length > MAX_CHUNK_CHARS) {
|
|
202
|
-
flushBuf();
|
|
203
|
-
const headerPrefix = level > 0 ? "#".repeat(level) + " " + heading + "\n\n" : "";
|
|
204
|
-
const bodyToSplit = body || content;
|
|
205
|
-
const paragraphs = bodyToSplit.split(/\n\n+/);
|
|
206
|
-
let sub = "";
|
|
207
|
-
let isFirst = true;
|
|
208
|
-
for (const para of paragraphs) {
|
|
209
|
-
const prefix = isFirst ? headerPrefix : "";
|
|
210
|
-
if (sub.length + prefix.length + para.length + 2 > MAX_CHUNK_CHARS && sub.length > 0) {
|
|
211
|
-
chunks.push(makeChunk(headings, level, sub.trim()));
|
|
212
|
-
sub = para;
|
|
213
|
-
isFirst = false;
|
|
214
|
-
} else {
|
|
215
|
-
sub = sub ? sub + "\n\n" + para : prefix + para;
|
|
216
|
-
isFirst = false;
|
|
217
|
-
}
|
|
218
|
-
}
|
|
219
|
-
if (sub.trim()) chunks.push(makeChunk(headings, level, sub.trim()));
|
|
220
|
-
continue;
|
|
221
|
-
}
|
|
222
|
-
|
|
223
|
-
// Section fits. Try to accumulate into the buffer.
|
|
224
|
-
const addition = content.length + (bufParts.length > 0 ? 2 : 0);
|
|
225
|
-
|
|
226
|
-
if (bufChars + addition <= MAX_CHUNK_CHARS) {
|
|
227
|
-
if (bufParts.length === 0) {
|
|
228
|
-
bufHeadings = headings;
|
|
229
|
-
bufLevel = level;
|
|
230
|
-
}
|
|
231
|
-
bufParts.push(content);
|
|
232
|
-
bufChars += addition;
|
|
233
|
-
} else {
|
|
234
|
-
flushBuf();
|
|
235
|
-
bufParts = [content];
|
|
236
|
-
bufHeadings = headings;
|
|
237
|
-
bufLevel = level;
|
|
238
|
-
bufChars = content.length;
|
|
239
|
-
}
|
|
240
|
-
}
|
|
241
|
-
|
|
242
|
-
flushBuf();
|
|
243
|
-
return chunks;
|
|
244
|
-
}
|
|
245
|
-
|
|
246
|
-
function makeChunk(headings: string[], level: number, content: string): Chunk {
|
|
247
|
-
const title = headings[headings.length - 1] ?? "";
|
|
248
|
-
return { heading_path: [...headings], heading_level: level, title, content, char_count: content.length };
|
|
249
|
-
}
|
|
250
105
|
|
|
251
106
|
// ── Embedding ──────────────────────────────────────────────────────────────
|
|
252
107
|
|
|
@@ -255,6 +110,7 @@ const EMBEDDING_INITIAL_BACKOFF_MS = 500; // 500ms, 1s, 2s exponential backoff
|
|
|
255
110
|
|
|
256
111
|
async function embedBatch(texts: string[], apiKey: string): Promise<number[][]> {
|
|
257
112
|
let lastError: Error | null = null;
|
|
113
|
+
const inputs = texts.map(capEmbeddingInput); // iter-28D Phase 0: cap oversized inputs
|
|
258
114
|
|
|
259
115
|
for (let attempt = 0; attempt < EMBEDDING_MAX_RETRIES; attempt++) {
|
|
260
116
|
try {
|
|
@@ -266,7 +122,7 @@ async function embedBatch(texts: string[], apiKey: string): Promise<number[][]>
|
|
|
266
122
|
},
|
|
267
123
|
body: JSON.stringify({
|
|
268
124
|
model: OPENAI_MODEL,
|
|
269
|
-
input:
|
|
125
|
+
input: inputs,
|
|
270
126
|
dimensions: EMBEDDING_DIMENSIONS,
|
|
271
127
|
}),
|
|
272
128
|
});
|
|
@@ -442,6 +298,13 @@ Deno.serve(async (req: Request) => {
|
|
|
442
298
|
});
|
|
443
299
|
}
|
|
444
300
|
|
|
301
|
+
const authFail = efAuthGate(
|
|
302
|
+
req.headers.get("Authorization"),
|
|
303
|
+
Deno.env.get("CEREFOX_ACCESS_TOKENS"),
|
|
304
|
+
{ "Content-Type": "application/json", "Access-Control-Allow-Origin": "*" },
|
|
305
|
+
);
|
|
306
|
+
if (authFail) return authFail;
|
|
307
|
+
|
|
445
308
|
if (isVersionRequest(req)) {
|
|
446
309
|
return versionResponse("cerefox-ingest", {
|
|
447
310
|
"Content-Type": "application/json",
|
|
@@ -582,7 +445,7 @@ Deno.serve(async (req: Request) => {
|
|
|
582
445
|
return new Response(JSON.stringify({ error: "Content produced no chunks" }), { status: 422, headers });
|
|
583
446
|
}
|
|
584
447
|
|
|
585
|
-
const texts = chunks.map((c) =>
|
|
448
|
+
const texts = chunks.map((c) => embeddingInputFor(title.trim(), c));
|
|
586
449
|
let embeddings: number[][];
|
|
587
450
|
try {
|
|
588
451
|
embeddings = await embedBatch(texts, openaiKey);
|
|
@@ -615,6 +478,7 @@ Deno.serve(async (req: Request) => {
|
|
|
615
478
|
p_source_label: source,
|
|
616
479
|
p_expected_content_hash: expected_content_hash,
|
|
617
480
|
p_last_write_wins: last_write_wins,
|
|
481
|
+
p_content_format: CONTENT_FORMAT_BLIND_STITCH,
|
|
618
482
|
});
|
|
619
483
|
|
|
620
484
|
if (ingestErr) {
|
|
@@ -698,7 +562,7 @@ Deno.serve(async (req: Request) => {
|
|
|
698
562
|
}
|
|
699
563
|
|
|
700
564
|
// Prepend document title for contextual enrichment (stored content unchanged)
|
|
701
|
-
const texts = chunks.map((c) =>
|
|
565
|
+
const texts = chunks.map((c) => embeddingInputFor(title.trim(), c));
|
|
702
566
|
let embeddings: number[][];
|
|
703
567
|
try {
|
|
704
568
|
embeddings = await embedBatch(texts, openaiKey);
|
|
@@ -733,6 +597,7 @@ Deno.serve(async (req: Request) => {
|
|
|
733
597
|
p_source_label: source,
|
|
734
598
|
p_expected_content_hash: expected_content_hash,
|
|
735
599
|
p_last_write_wins: last_write_wins,
|
|
600
|
+
p_content_format: CONTENT_FORMAT_BLIND_STITCH,
|
|
736
601
|
});
|
|
737
602
|
|
|
738
603
|
if (ingestErr) {
|
|
@@ -806,7 +671,7 @@ Deno.serve(async (req: Request) => {
|
|
|
806
671
|
}
|
|
807
672
|
|
|
808
673
|
// Embed all chunks with title prefix for contextual enrichment (stored content unchanged)
|
|
809
|
-
const texts = chunks.map((c) =>
|
|
674
|
+
const texts = chunks.map((c) => embeddingInputFor(title.trim(), c));
|
|
810
675
|
let embeddings: number[][];
|
|
811
676
|
try {
|
|
812
677
|
embeddings = await embedBatch(texts, openaiKey);
|
|
@@ -841,6 +706,7 @@ Deno.serve(async (req: Request) => {
|
|
|
841
706
|
p_chunks: chunkData,
|
|
842
707
|
p_author: author,
|
|
843
708
|
p_author_type: author_type,
|
|
709
|
+
p_content_format: CONTENT_FORMAT_BLIND_STITCH,
|
|
844
710
|
});
|
|
845
711
|
|
|
846
712
|
if (ingestErr || !ingestResult?.length) {
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import "jsr:@supabase/functions-js/edge-runtime.d.ts";
|
|
2
2
|
import { createClient } from "jsr:@supabase/supabase-js@2";
|
|
3
3
|
import { isVersionRequest, versionResponse } from "../../../_shared/ef-meta/index.ts";
|
|
4
|
+
import { efAuthGate } from "../../../_shared/ef-auth/index.ts";
|
|
4
5
|
|
|
5
6
|
/**
|
|
6
7
|
* cerefox-list-projects -- Supabase Edge Function
|
|
@@ -29,6 +30,13 @@ Deno.serve(async (req: Request): Promise<Response> => {
|
|
|
29
30
|
return new Response(null, { status: 200, headers: CORS_HEADERS });
|
|
30
31
|
}
|
|
31
32
|
|
|
33
|
+
const authFail = efAuthGate(
|
|
34
|
+
req.headers.get("Authorization"),
|
|
35
|
+
Deno.env.get("CEREFOX_ACCESS_TOKENS"),
|
|
36
|
+
{ ...CORS_HEADERS, "Content-Type": "application/json" },
|
|
37
|
+
);
|
|
38
|
+
if (authFail) return authFail;
|
|
39
|
+
|
|
32
40
|
if (isVersionRequest(req)) {
|
|
33
41
|
return versionResponse("cerefox-list-projects", { ...CORS_HEADERS, "Content-Type": "application/json" });
|
|
34
42
|
}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import "jsr:@supabase/functions-js/edge-runtime.d.ts";
|
|
2
2
|
import { createClient } from "jsr:@supabase/supabase-js@2";
|
|
3
3
|
import { isVersionRequest, versionResponse } from "../../../_shared/ef-meta/index.ts";
|
|
4
|
+
import { efAuthGate } from "../../../_shared/ef-auth/index.ts";
|
|
4
5
|
|
|
5
6
|
/**
|
|
6
7
|
* cerefox-list-versions — Supabase Edge Function
|
|
@@ -34,6 +35,13 @@ Deno.serve(async (req: Request): Promise<Response> => {
|
|
|
34
35
|
return new Response(null, { status: 200, headers: CORS_HEADERS });
|
|
35
36
|
}
|
|
36
37
|
|
|
38
|
+
const authFail = efAuthGate(
|
|
39
|
+
req.headers.get("Authorization"),
|
|
40
|
+
Deno.env.get("CEREFOX_ACCESS_TOKENS"),
|
|
41
|
+
{ ...CORS_HEADERS, "Content-Type": "application/json" },
|
|
42
|
+
);
|
|
43
|
+
if (authFail) return authFail;
|
|
44
|
+
|
|
37
45
|
if (isVersionRequest(req)) {
|
|
38
46
|
return versionResponse("cerefox-list-versions", { ...CORS_HEADERS, "Content-Type": "application/json" });
|
|
39
47
|
}
|
|
@@ -27,6 +27,14 @@ import {
|
|
|
27
27
|
makeSupabaseClient,
|
|
28
28
|
notificationResponse,
|
|
29
29
|
} from "./shared.ts";
|
|
30
|
+
import {
|
|
31
|
+
buildAuthenticator,
|
|
32
|
+
isProtectedResourceMetadata,
|
|
33
|
+
protectedResourceMetadata,
|
|
34
|
+
unauthorizedChallenge,
|
|
35
|
+
} from "./oauth.ts";
|
|
36
|
+
import type { AuthResult, McpAuthenticator } from "../../../_shared/mcp-auth/index.ts";
|
|
37
|
+
import { checkAccessToken, parseAccessTokens } from "../../../_shared/ef-auth/index.ts";
|
|
30
38
|
import {
|
|
31
39
|
ALL_TOOLS,
|
|
32
40
|
McpInvalidParams,
|
|
@@ -233,11 +241,57 @@ async function handleVersion(req: Request): Promise<Response> {
|
|
|
233
241
|
|
|
234
242
|
// ── Main handler ─────────────────────────────────────────────────────────────
|
|
235
243
|
|
|
244
|
+
// Isolate-lifetime authenticator (holds the JWKS cache). Issuer/JWKS come from the
|
|
245
|
+
// injected SUPABASE_URL (not request headers — see oauth.ts projectOrigin).
|
|
246
|
+
let authenticator: McpAuthenticator | null = null;
|
|
247
|
+
function getAuthenticator(): McpAuthenticator {
|
|
248
|
+
if (!authenticator) authenticator = buildAuthenticator();
|
|
249
|
+
return authenticator;
|
|
250
|
+
}
|
|
251
|
+
|
|
236
252
|
Deno.serve(async (req: Request): Promise<Response> => {
|
|
237
253
|
if (req.method === "OPTIONS") {
|
|
238
254
|
return new Response(null, { status: 200, headers: CORS_HEADERS });
|
|
239
255
|
}
|
|
240
256
|
|
|
257
|
+
// Public discovery route (RFC 9728): the ONLY unauthenticated non-OPTIONS
|
|
258
|
+
// response. Served before auth so OAuth clients can bootstrap.
|
|
259
|
+
if (req.method === "GET" && isProtectedResourceMetadata(req)) {
|
|
260
|
+
return protectedResourceMetadata();
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
// ── Auth-first dispatch (design §6) ────────────────────────────────────────
|
|
264
|
+
// The function is deployed with --no-verify-jwt, so this in-function check is
|
|
265
|
+
// the ONLY gate. Accept EITHER a Cerefox access token (the static path — same
|
|
266
|
+
// credential the primitive EFs take, iter-28E) OR a valid OAuth JWT.
|
|
267
|
+
const authHeader = req.headers.get("Authorization");
|
|
268
|
+
let authResult: AuthResult;
|
|
269
|
+
// Static token first (cheap constant-time compare). A real OAuth JWT won't match
|
|
270
|
+
// any token, so it falls through to the JWKS path below — no false rejection.
|
|
271
|
+
const tokenResult = checkAccessToken(authHeader, {
|
|
272
|
+
tokens: parseAccessTokens(Deno.env.get("CEREFOX_ACCESS_TOKENS")),
|
|
273
|
+
});
|
|
274
|
+
if (tokenResult.ok) {
|
|
275
|
+
authResult = { ok: true, path: "static" };
|
|
276
|
+
} else {
|
|
277
|
+
authResult = await getAuthenticator().authenticate(authHeader);
|
|
278
|
+
}
|
|
279
|
+
if (!authResult.ok) {
|
|
280
|
+
// Log the machine reason (never the token) so the dashboard logs are
|
|
281
|
+
// actionable on a real auth failure. `no_token` is the normal OAuth-discovery
|
|
282
|
+
// probe (every cloud client sends one first) — don't log it as noise. The
|
|
283
|
+
// enriched detail (aud/sub/alg values from _shared/mcp-auth) names which claim
|
|
284
|
+
// a rejected token tripped. To debug a "Claude never sends the token" case
|
|
285
|
+
// (claude-ai #482), temporarily also log `Array.from(req.headers.keys())`.
|
|
286
|
+
if (authResult.reason !== "no_token") {
|
|
287
|
+
console.warn(
|
|
288
|
+
`[cerefox-mcp] auth rejected: ${authResult.reason}` +
|
|
289
|
+
(authResult.detail ? ` (${authResult.detail})` : ""),
|
|
290
|
+
);
|
|
291
|
+
}
|
|
292
|
+
return unauthorizedChallenge(authResult);
|
|
293
|
+
}
|
|
294
|
+
|
|
241
295
|
// GET — the only supported GET is the /version surface (iter-26). Per MCP
|
|
242
296
|
// spec (2025-03-26) this server otherwise returns 405 on GET to signal it
|
|
243
297
|
// does not support SSE notifications (prevents MCP clients from holding a
|