@cerefox/memory 0.11.1 → 1.0.0-beta.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. package/AGENT_GUIDE.md +2 -2
  2. package/AGENT_QUICK_REFERENCE.md +1 -1
  3. package/dist/bin/cerefox.js +1079 -836
  4. package/dist/frontend/assets/index-D3FshoP3.js +125 -0
  5. package/dist/frontend/assets/index-D3FshoP3.js.map +1 -0
  6. package/dist/frontend/index.html +1 -1
  7. package/dist/server-assets/_shared/ef-auth/index.ts +134 -0
  8. package/dist/server-assets/_shared/ef-meta/index.ts +1 -1
  9. package/dist/server-assets/_shared/embeddings/index.ts +42 -2
  10. package/dist/server-assets/_shared/ingest/chunker.ts +210 -0
  11. package/dist/server-assets/_shared/ingest/index.ts +32 -0
  12. package/dist/server-assets/_shared/ingest/pipeline-helpers.ts +135 -0
  13. package/dist/server-assets/_shared/mcp-auth/index.ts +352 -0
  14. package/dist/server-assets/_shared/mcp-tools/_chunker.ts +16 -170
  15. package/dist/server-assets/_shared/mcp-tools/get-help-content.ts +2 -2
  16. package/dist/server-assets/_shared/mcp-tools/ingest.ts +13 -4
  17. package/dist/server-assets/db/migrations/0012_content_format.sql +20 -0
  18. package/dist/server-assets/db/rpcs.sql +76 -8
  19. package/dist/server-assets/db/schema.sql +7 -1
  20. package/dist/server-assets/supabase/functions/cerefox-get-audit-log/index.ts +8 -0
  21. package/dist/server-assets/supabase/functions/cerefox-get-document/index.ts +8 -0
  22. package/dist/server-assets/supabase/functions/cerefox-ingest/index.ts +39 -173
  23. package/dist/server-assets/supabase/functions/cerefox-list-projects/index.ts +8 -0
  24. package/dist/server-assets/supabase/functions/cerefox-list-versions/index.ts +8 -0
  25. package/dist/server-assets/supabase/functions/cerefox-mcp/index.ts +54 -0
  26. package/dist/server-assets/supabase/functions/cerefox-mcp/oauth.ts +121 -0
  27. package/dist/server-assets/supabase/functions/cerefox-metadata/index.ts +8 -0
  28. package/dist/server-assets/supabase/functions/cerefox-metadata-search/index.ts +8 -0
  29. package/dist/server-assets/supabase/functions/cerefox-search/index.ts +11 -1
  30. package/docs/guides/access-paths.md +84 -35
  31. package/docs/guides/cli.md +29 -0
  32. package/docs/guides/configuration.md +10 -8
  33. package/docs/guides/connect-agents.md +99 -101
  34. package/docs/guides/content-format.md +55 -0
  35. package/docs/guides/migration-1.0.md +96 -0
  36. package/docs/guides/ops-scripts.md +5 -7
  37. package/docs/guides/quickstart.md +22 -2
  38. package/docs/guides/setup-cloud-run.md +5 -9
  39. package/docs/guides/setup-supabase.md +157 -16
  40. package/docs/guides/upgrading.md +7 -8
  41. package/package.json +1 -1
  42. package/dist/frontend/assets/index-ojNhWSxm.js +0 -125
  43. package/dist/frontend/assets/index-ojNhWSxm.js.map +0 -1
package/dist/frontend/index.html CHANGED
@@ -15,7 +15,7 @@
15
15
  href="https://fonts.googleapis.com/css2?family=Geist:wght@300;400;500;600;700&display=swap"
16
16
  />
17
17
  <title>Cerefox</title>
18
- <script type="module" crossorigin src="/app/assets/index-ojNhWSxm.js"></script>
18
+ <script type="module" crossorigin src="/app/assets/index-D3FshoP3.js"></script>
19
19
  <link rel="stylesheet" crossorigin href="/app/assets/index-Asx5wD7g.css">
20
20
  </head>
21
21
  <body>
package/dist/server-assets/_shared/ef-auth/index.ts ADDED
@@ -0,0 +1,134 @@
1
+ /**
2
+ * ef-auth — in-function access-token authentication for the primitive Edge
3
+ * Functions (and `cerefox-mcp`'s static-token path).
4
+ *
5
+ * Context (iter-28E, `docs/specs/ef-auth-migration-design.md`): the 8 primitive
6
+ * EFs move to `--no-verify-jwt` and authenticate the caller *in-function* against
7
+ * a **rotatable, Cerefox-managed access token** instead of the unrotatable legacy
8
+ * anon JWT. With the gateway gate removed, **this check is the only auth gate** on
9
+ * those functions — so it fails closed and compares in constant time.
10
+ *
11
+ * The accepted set (`CEREFOX_ACCESS_TOKENS`, comma-separated) holds one or more
12
+ * tokens so rotation is zero-downtime: add the new token, migrate clients, drop
13
+ * the old. A request is accepted iff its Bearer credential equals ANY token in the
14
+ * set (constant-time, no short-circuit on the first match).
15
+ *
16
+ * Portability: Web-Platform globals only (`TextEncoder` via the reused
17
+ * `constantTimeEqual`) — the identical source runs under Deno (the EF) and Bun
18
+ * (`bun test`). The single audited constant-time primitive lives in
19
+ * `../mcp-auth/index.ts` and is reused here rather than re-implemented.
20
+ */
21
+
22
+ import { constantTimeEqual } from "../mcp-auth/index.ts";
23
+
24
+ // ── Types ────────────────────────────────────────────────────────────────────
25
+
26
+ export interface TokenAuthSuccess {
27
+ ok: true;
28
+ }
29
+
30
+ export interface TokenAuthFailure {
31
+ ok: false;
32
+ /** Machine-readable reason. Human detail (never returned to the client) in `detail`. */
33
+ reason: "no_token" | "no_tokens_configured" | "bad_token";
34
+ detail?: string;
35
+ }
36
+
37
+ export type TokenAuthResult = TokenAuthSuccess | TokenAuthFailure;
38
+
39
+ export interface AccessTokenConfig {
40
+ /**
41
+ * The accepted token set (from `CEREFOX_ACCESS_TOKENS`). When empty the check
42
+ * FAILS CLOSED (rejects everything) — an EF must never accept-all because a
43
+ * missing secret left the gate open.
44
+ */
45
+ tokens: string[];
46
+ }
47
+
48
+ // ── Parsing ──────────────────────────────────────────────────────────────────
49
+
50
+ /**
51
+ * Parse the `CEREFOX_ACCESS_TOKENS` secret into a token set: split on commas,
52
+ * trim, drop empties (so trailing commas / whitespace don't create a `""` token
53
+ * that a blank credential could match). Returns `[]` for null/undefined/blank —
54
+ * which makes the check fail closed.
55
+ */
56
+ export function parseAccessTokens(raw: string | null | undefined): string[] {
57
+ if (!raw) return [];
58
+ return raw
59
+ .split(",")
60
+ .map((t) => t.trim())
61
+ .filter((t) => t.length > 0);
62
+ }
63
+
64
+ // ── Check ────────────────────────────────────────────────────────────────────
65
+
66
+ const BEARER_PREFIX = "Bearer ";
67
+
68
+ /**
69
+ * Validate the `Authorization: Bearer <token>` header against the accepted set.
70
+ * Constant-time, fail-closed. Never logs or returns the token value.
71
+ */
72
+ export function checkAccessToken(
73
+ authorizationHeader: string | null,
74
+ config: AccessTokenConfig,
75
+ ): TokenAuthResult {
76
+ if (!authorizationHeader || !authorizationHeader.startsWith(BEARER_PREFIX)) {
77
+ return { ok: false, reason: "no_token" };
78
+ }
79
+ const token = authorizationHeader.slice(BEARER_PREFIX.length).trim();
80
+ if (!token) return { ok: false, reason: "no_token" };
81
+
82
+ // Fail closed: no tokens configured => reject everything (never accept-all).
83
+ if (config.tokens.length === 0) {
84
+ return {
85
+ ok: false,
86
+ reason: "no_tokens_configured",
87
+ detail: "CEREFOX_ACCESS_TOKENS is unset/empty; refusing to accept-all",
88
+ };
89
+ }
90
+
91
+ // Compare against every accepted token WITHOUT short-circuiting, so timing
92
+ // doesn't leak which (or how many) tokens matched.
93
+ let matched = false;
94
+ for (const accepted of config.tokens) {
95
+ if (constantTimeEqual(token, accepted)) matched = true;
96
+ }
97
+
98
+ return matched ? { ok: true } : { ok: false, reason: "bad_token" };
99
+ }
100
+
101
+ // ── Edge Function gate (drop-in for each primitive EF) ───────────────────────
102
+
103
+ /**
104
+ * One-call auth gate for a primitive Edge Function. Returns a 401 `Response` to
105
+ * short-circuit the handler when the token is missing/wrong/unconfigured, or
106
+ * `null` to continue. Placed **before** the `/version` branch so version is gated
107
+ * too (design §3, decision 2026-07-10).
108
+ *
109
+ * Deno-free by design: the caller passes the raw `CEREFOX_ACCESS_TOKENS` env
110
+ * value (read via `Deno.env.get` in the EF) rather than this module touching the
111
+ * `Deno` global — so the identical source still imports cleanly under Bun for
112
+ * unit tests. `Response`/`console` are Web globals available in both runtimes.
113
+ */
114
+ export function efAuthGate(
115
+ authorization: string | null,
116
+ tokensRaw: string | null | undefined,
117
+ headers: Record<string, string>,
118
+ ): Response | null {
119
+ const result = checkAccessToken(authorization, { tokens: parseAccessTokens(tokensRaw) });
120
+ if (result.ok) return null;
121
+
122
+ // Log the machine reason (never the token). `no_token` is the normal
123
+ // unauthenticated probe (noise); `bad_token` / `no_tokens_configured` are
124
+ // worth surfacing in the dashboard logs.
125
+ if (result.reason !== "no_token") {
126
+ console.warn(
127
+ `[ef-auth] rejected: ${result.reason}${result.detail ? ` (${result.detail})` : ""}`,
128
+ );
129
+ }
130
+ return new Response(JSON.stringify({ error: "Unauthorized" }), {
131
+ status: 401,
132
+ headers: { ...headers, "WWW-Authenticate": "Bearer" },
133
+ });
134
+ }
package/dist/server-assets/_shared/ef-meta/index.ts CHANGED
@@ -18,7 +18,7 @@
18
18
  * doesn't touch `supabase/functions/` leaves it alone).
19
19
  */
20
20
 
21
- export const EF_VERSION = "0.11.1";
21
+ export const EF_VERSION = "1.0.0-beta.2";
22
22
 
23
23
  /**
24
24
  * The 8 peer EFs the cerefox-mcp aggregator probes (excludes cerefox-mcp
package/dist/server-assets/_shared/embeddings/index.ts CHANGED
@@ -43,6 +43,44 @@ export function openaiEmbeddingConfig(): { url: string; model: string; dimension
43
43
  };
44
44
  }
45
45
 
46
+ /**
47
+ * Safety cap (iter-28D Phase 0) on the characters sent to the embedding model per
48
+ * input — a conservative proxy for the model's token limit (`text-embedding-3-small`
49
+ * is 8191 tokens). Normal chunks are far below this (`max_chunk_chars` ≈ 2000); the
50
+ * cap only bites on an oversized *keep-whole* chunk (a huge table or blank-line-free
51
+ * paragraph the interim chunker fix keeps intact). Default 20000 chars sits well under
52
+ * 8191 tokens for markdown/English; override with `CEREFOX_EMBED_MAX_INPUT_CHARS`.
53
+ */
54
+ export const DEFAULT_EMBED_MAX_INPUT_CHARS = 20000;
55
+
56
+ export function embeddingMaxInputChars(): number {
57
+ const env =
58
+ (globalThis as { process?: { env?: Record<string, string | undefined> } }).process?.env ?? {};
59
+ const raw = Number.parseInt(env.CEREFOX_EMBED_MAX_INPUT_CHARS ?? "", 10);
60
+ return Number.isNaN(raw) || raw <= 0 ? DEFAULT_EMBED_MAX_INPUT_CHARS : raw;
61
+ }
62
+
63
+ /**
64
+ * Truncate one embedding input to the char cap. The full chunk `content` is stored and
65
+ * reconstructed untouched; only its *embedding* is computed on this prefix — so at worst
66
+ * search quality for one oversized chunk is degraded, and an ingest **never fails** on an
67
+ * over-limit embedding input (the failure mode this prevents). Surrogate-safe (never leaves
68
+ * a dangling high surrogate). Warns when it truncates.
69
+ */
70
+ export function capEmbeddingInput(text: string): string {
71
+ const max = embeddingMaxInputChars();
72
+ if (text.length <= max) return text;
73
+ let cut = text.slice(0, max);
74
+ const last = cut.charCodeAt(cut.length - 1);
75
+ if (last >= 0xd800 && last <= 0xdbff) cut = cut.slice(0, -1); // don't split a surrogate pair
76
+ console.warn(
77
+ `[embeddings] truncated an embedding input: ${text.length} → ${cut.length} chars ` +
78
+ `(cap CEREFOX_EMBED_MAX_INPUT_CHARS=${max}). The full content is stored and reconstructed ` +
79
+ `untouched; only this chunk's embedding uses the prefix (degraded search for this chunk).`,
80
+ );
81
+ return cut;
82
+ }
83
+
46
84
  const EMBEDDING_MAX_RETRIES = 3;
47
85
  const EMBEDDING_INITIAL_BACKOFF_MS = 500; // 500ms → 1s → 2s
48
86
 
@@ -50,6 +88,7 @@ const EMBEDDING_INITIAL_BACKOFF_MS = 500; // 500ms → 1s → 2s
50
88
  export async function getEmbedding(text: string, apiKey: string): Promise<number[]> {
51
89
  let lastError: Error | null = null;
52
90
  const cfg = openaiEmbeddingConfig();
91
+ const input = capEmbeddingInput(text); // cap once, before the retry loop
53
92
 
54
93
  for (let attempt = 0; attempt < EMBEDDING_MAX_RETRIES; attempt++) {
55
94
  try {
@@ -61,7 +100,7 @@ export async function getEmbedding(text: string, apiKey: string): Promise<number
61
100
  },
62
101
  body: JSON.stringify({
63
102
  model: cfg.model,
64
- input: text,
103
+ input,
65
104
  dimensions: cfg.dimensions,
66
105
  }),
67
106
  });
@@ -121,6 +160,7 @@ async function embedBatchSingleCall(
121
160
  ): Promise<number[][]> {
122
161
  let lastError: Error | null = null;
123
162
  const cfg = openaiEmbeddingConfig();
163
+ const inputs = texts.map(capEmbeddingInput); // cap once, before the retry loop
124
164
 
125
165
  for (let attempt = 0; attempt < EMBEDDING_MAX_RETRIES; attempt++) {
126
166
  try {
@@ -132,7 +172,7 @@ async function embedBatchSingleCall(
132
172
  },
133
173
  body: JSON.stringify({
134
174
  model: cfg.model,
135
- input: texts,
175
+ input: inputs,
136
176
  dimensions: cfg.dimensions,
137
177
  }),
138
178
  });
package/dist/server-assets/_shared/ingest/chunker.ts ADDED
@@ -0,0 +1,210 @@
1
+ /**
2
+ * Exact-partition ("blind-stitch") markdown chunker — the single TS chunker
3
+ * (iter-28D Phase 1). Consolidated: `_shared/mcp-tools/_chunker.ts` and the
4
+ * `cerefox-ingest` Edge Function both import from here (the earlier duplicate
5
+ * copies were removed).
6
+ *
7
+ * **The invariant:** `blindStitch(chunkMarkdown(doc)) === doc.trim()`, byte-for-
8
+ * byte. Chunk `content` values are consecutive, gapless, non-overlapping slices
9
+ * of the trimmed document, so reconstruction is a plain concatenation — no `\n\n`
10
+ * separator is synthesized on read (the bug the old chunker+reconstruction had).
11
+ * That is what lets a chunk boundary fall *anywhere* (including mid-paragraph at
12
+ * a size limit) with zero corruption, which in turn lets us bound chunk size.
13
+ *
14
+ * These chunks are stored with `content_format = 2` and reconstructed by blind
15
+ * concat (`STRING_AGG(content, '')`). Documents written before Phase 1 stay
16
+ * `content_format = 1` and reconstruct with the legacy `\n\n`-join (see
17
+ * `rpcs.sql`); they are untouched until re-written. Design:
18
+ * `docs/specs/chunk-reconstruction-design.md`.
19
+ *
20
+ * Heading context is NOT stored in mid-section chunk content (that would break
21
+ * the exact partition); it lives in `heading_path` metadata, and the embedding
22
+ * input adds it back as a breadcrumb (`# {title}\n{breadcrumb}\n{content}`).
23
+ *
24
+ * Length is code-point-based (`cpLen`), matching the historical size semantics.
25
+ */
26
+
27
+ /** Single chunk produced by the markdown chunker. */
28
+ export interface ChunkData {
29
+ chunk_index: number;
30
+ heading_path: string[];
31
+ /** 0 = no heading (preamble); 1-3 = the innermost active H1-H3. */
32
+ heading_level: number;
33
+ /** Last element of heading_path, or "" for preamble. */
34
+ title: string;
35
+ /** Exact slice of the trimmed document (a gapless partition member). */
36
+ content: string;
37
+ char_count: number;
38
+ }
39
+
40
+ // ── Helpers ──────────────────────────────────────────────────────────────────
41
+
42
+ /** Code-point length (`"🎉".length` is 2 in JS but 1 code point). */
43
+ function cpLen(s: string): number {
44
+ let n = 0;
45
+ for (const _ of s) n++;
46
+ return n;
47
+ }
48
+
49
+ /** Strip trailing `#` (matches a markdown closing-hash heading). */
50
+ function rstripHash(s: string): string {
51
+ let i = s.length;
52
+ while (i > 0 && s[i - 1] === "#") i--;
53
+ return s.slice(0, i);
54
+ }
55
+
56
+ interface HeadingMark {
57
+ /** UTF-16 offset of the `#` in the trimmed doc. */
58
+ offset: number;
59
+ level: number;
60
+ text: string;
61
+ }
62
+
63
+ /** All H1/H2/H3 heading lines in `doc`, in order (heading text on one line). */
64
+ function findHeadings(doc: string): HeadingMark[] {
65
+ const out: HeadingMark[] = [];
66
+ const re = /^(#{1,3})[ \t]+(.+)$/gm;
67
+ let m: RegExpExecArray | null;
68
+ while ((m = re.exec(doc)) !== null) {
69
+ out.push({ offset: m.index, level: m[1].length, text: rstripHash(m[2]).trim() });
70
+ }
71
+ return out;
72
+ }
73
+
74
+ /** The active H1/H2/H3 nesting stack at `offset` (headings at or before it). */
75
+ function activeHeadings(headings: HeadingMark[], offset: number): HeadingMark[] {
76
+ const stack: HeadingMark[] = [];
77
+ for (const h of headings) {
78
+ if (h.offset > offset) break;
79
+ while (stack.length && stack[stack.length - 1].level >= h.level) stack.pop();
80
+ stack.push(h);
81
+ }
82
+ return stack;
83
+ }
84
+
85
+ /**
86
+ * Split `s` into consecutive pieces of at most `maxCp` code points each, with
87
+ * `pieces.join("") === s`. Never splits a code point (surrogate pair).
88
+ */
89
+ function hardSplitCp(s: string, maxCp: number): string[] {
90
+ const out: string[] = [];
91
+ let buf = "";
92
+ let n = 0;
93
+ for (const ch of s) {
94
+ if (n >= maxCp) {
95
+ out.push(buf);
96
+ buf = "";
97
+ n = 0;
98
+ }
99
+ buf += ch;
100
+ n++;
101
+ }
102
+ if (buf) out.push(buf);
103
+ return out;
104
+ }
105
+
106
+ // ── Public API ───────────────────────────────────────────────────────────────
107
+
108
+ /**
109
+ * Split markdown into exact-partition chunks. `minChunkChars` is accepted for
110
+ * signature compatibility but unused (tiny trailing atoms merge into the current
111
+ * chunk naturally via greedy accumulation).
112
+ */
113
+ export function chunkMarkdown(
114
+ text: string,
115
+ maxChunkChars = 4000,
116
+ _minChunkChars = 100,
117
+ ): ChunkData[] {
118
+ const doc = text.trim();
119
+ if (!doc) return [];
120
+
121
+ if (cpLen(doc) <= maxChunkChars) {
122
+ return [
123
+ { chunk_index: 0, heading_path: [], heading_level: 0, title: "", content: doc, char_count: cpLen(doc) },
124
+ ];
125
+ }
126
+
127
+ const headings = findHeadings(doc);
128
+
129
+ // Atomize preserving every character: split on blank-line runs, attach each
130
+ // separator to its PRECEDING block (so a separator never starts a chunk), then
131
+ // hard-split any atom that alone exceeds the size limit. atoms.join("") === doc.
132
+ const parts = doc.split(/(\n{2,})/); // [block, sep, block, sep, …, block]
133
+ const atoms: string[] = [];
134
+ for (let i = 0; i < parts.length; i += 2) {
135
+ const unit = (parts[i] ?? "") + (parts[i + 1] ?? ""); // block + trailing separator
136
+ if (unit === "") continue;
137
+ if (cpLen(unit) > maxChunkChars) atoms.push(...hardSplitCp(unit, maxChunkChars));
138
+ else atoms.push(unit);
139
+ }
140
+
141
+ const chunks: ChunkData[] = [];
142
+ let buf = "";
143
+ let bufCp = 0;
144
+ let bufStart = 0;
145
+ let offset = 0;
146
+
147
+ const flush = (): void => {
148
+ if (buf === "") return;
149
+ const stack = activeHeadings(headings, bufStart);
150
+ chunks.push({
151
+ chunk_index: chunks.length,
152
+ heading_path: stack.map((h) => h.text),
153
+ heading_level: stack.length ? stack[stack.length - 1].level : 0,
154
+ title: stack.length ? stack[stack.length - 1].text : "",
155
+ content: buf,
156
+ char_count: bufCp,
157
+ });
158
+ buf = "";
159
+ bufCp = 0;
160
+ };
161
+
162
+ for (const atom of atoms) {
163
+ const cp = cpLen(atom);
164
+ if (buf === "") {
165
+ bufStart = offset;
166
+ buf = atom;
167
+ bufCp = cp;
168
+ } else if (bufCp + cp <= maxChunkChars) {
169
+ buf += atom;
170
+ bufCp += cp;
171
+ } else {
172
+ flush();
173
+ bufStart = offset;
174
+ buf = atom;
175
+ bufCp = cp;
176
+ }
177
+ offset += atom.length; // UTF-16 length — same units as heading offsets
178
+ }
179
+ flush();
180
+ return chunks;
181
+ }
182
+
183
+ /** Format-2 reconstruction: blind concatenation (mirrors the RPC's `STRING_AGG(content,'')`). */
184
+ export function blindStitch(chunks: Pick<ChunkData, "content">[]): string {
185
+ return chunks.map((c) => c.content).join("");
186
+ }
187
+
188
+ /**
189
+ * The content format this chunker produces. Stamped on each chunk (via the ingest
190
+ * RPC's p_content_format) and used by the reconstruction RPCs to pick the join
191
+ * strategy (2 = blind concat). See docs/guides/content-format.md.
192
+ */
193
+ export const CONTENT_FORMAT_BLIND_STITCH = 2;
194
+
195
+ /**
196
+ * Build the title-boosted embedding input for a chunk. Format-2 chunk `content` is
197
+ * an exact slice that no longer carries the heading line for mid-section chunks, so
198
+ * the heading context is re-added here as a breadcrumb from `heading_path` (search
199
+ * quality parity). Stored `content` is unchanged; this string is embedded only.
200
+ * `# {docTitle}` (chunk at the doc's top / preamble)
201
+ * `# {docTitle}\n{A > B > C}` (chunk under headings A > B > C)
202
+ */
203
+ export function embeddingInputFor(
204
+ docTitle: string,
205
+ chunk: Pick<ChunkData, "heading_path" | "content">,
206
+ ): string {
207
+ const breadcrumb = chunk.heading_path.join(" > ");
208
+ const head = breadcrumb ? `# ${docTitle}\n${breadcrumb}` : `# ${docTitle}`;
209
+ return `${head}\n${chunk.content}`;
210
+ }
package/dist/server-assets/_shared/ingest/index.ts ADDED
@@ -0,0 +1,32 @@
1
+ /**
2
+ * `_shared/ingest/` — chunking + embedding orchestration + content-hashing
3
+ * utilities used by the v0.7 TS ingestion pipeline + the CLI ingest path.
4
+ *
5
+ * Consumers:
6
+ * - `packages/memory/src/ingestion/pipeline.ts` (the IngestionPipeline)
7
+ * - `packages/memory/src/cli/commands/ingest.ts` (v0.7 in-process variant)
8
+ * - `packages/memory/src/web/routes/ingest.ts` (the 3 endpoints unblocked
9
+ * in Part 25F)
10
+ * - `packages/memory/src/web/routes/documents-write.ts` (v0.6's /edit
11
+ * content-hash short-circuit, now using the shared helper)
12
+ *
13
+ * Also consumed by `supabase/functions/cerefox-ingest/` and
14
+ * `_shared/mcp-tools/_chunker.ts` (iter-28D Phase 1 consolidation) — the
15
+ * chunker lives here only; the EF bundles `_shared/ingest/` via
16
+ * `bundle_server_assets.ts`, the same way it bundles `ef-auth`/`embeddings`.
17
+ */
18
+
19
+ export {
20
+ chunkMarkdown,
21
+ blindStitch,
22
+ embeddingInputFor,
23
+ CONTENT_FORMAT_BLIND_STITCH,
24
+ type ChunkData,
25
+ } from "./chunker.js";
26
+ export {
27
+ normalizeForHash,
28
+ contentHash,
29
+ deriveSourcePath,
30
+ resolveProjectIds,
31
+ type ProjectResolveInput,
32
+ } from "./pipeline-helpers.js";
package/dist/server-assets/_shared/ingest/pipeline-helpers.ts ADDED
@@ -0,0 +1,135 @@
1
+ /**
2
+ * Cross-consumer helpers for the v0.7 TS ingestion pipeline.
3
+ *
4
+ * Consumers (all TS surfaces that need to dedup by content hash, mint a
5
+ * source_path from a title, or resolve mixed project-id/name caller
6
+ * styles into a final UUID list):
7
+ *
8
+ * - `packages/memory/src/ingestion/pipeline.ts` (IngestionPipeline)
9
+ * - `packages/memory/src/web/routes/documents-write.ts` (v0.6 /edit
10
+ * content-hash short-circuit — promoted out of the inline version)
11
+ * - `packages/memory/src/cli/commands/ingest.ts` (v0.7 in-process)
12
+ *
13
+ * NOT consumed by `supabase/functions/cerefox-ingest/` (Deno Edge
14
+ * Runtime can't reach `_shared/`; the EF keeps its own copies).
15
+ * Cross-runtime parity is enforced by shared fixtures + manual smoke.
16
+ *
17
+ * Python parity: `normalizeForHash` + `contentHash` match `_normalize`
18
+ * + `_hash` in `src/cerefox/ingestion/pipeline.py` exactly. Drift =
19
+ * dedup breaks across the CLI / web / Python paths.
20
+ */
21
+
22
+ import { createHash } from "node:crypto";
23
+
24
+ // ── Content normalization + hash ────────────────────────────────────────────
25
+
26
+ /**
27
+ * Normalize content before hashing (mirrors Python's `_normalize`):
28
+ * 1. CRLF → LF
29
+ * 2. Bare CR → LF
30
+ * 3. Strip leading/trailing whitespace
31
+ * 4. Collapse 3+ consecutive newlines to 2
32
+ *
33
+ * Stable across round-trips through the web edit form (browsers submit
34
+ * textarea content with CRLF per the HTML spec).
35
+ */
36
+ export function normalizeForHash(text: string): string {
37
+ return text
38
+ .replace(/\r\n/g, "\n")
39
+ .replace(/\r/g, "\n")
40
+ .trim()
41
+ .replace(/\n{3,}/g, "\n\n");
42
+ }
43
+
44
+ /**
45
+ * SHA-256 hex digest of the normalized UTF-8-encoded text. Same
46
+ * algorithm Python uses in `_hash`. Output is a 64-character lowercase
47
+ * hex string.
48
+ */
49
+ export function contentHash(text: string): string {
50
+ return createHash("sha256")
51
+ .update(normalizeForHash(text), "utf8")
52
+ .digest("hex");
53
+ }
54
+
55
+ // ── source_path derivation from title ───────────────────────────────────────
56
+
57
+ /**
58
+ * Derive a default `source_path` from a document title when none was
59
+ * provided (e.g. paste ingestion). Matches Python's behaviour exactly:
60
+ *
61
+ * slug = re.sub(r"[^\w\s-]", "", title.lower())
62
+ * slug = re.sub(r"[\s_-]+", "-", slug).strip("-") or "document"
63
+ * source_path = f"{slug}.md"
64
+ *
65
+ * Used for download filenames + Obsidian-style link resolution.
66
+ */
67
+ export function deriveSourcePath(title: string): string {
68
+ let slug = title.toLowerCase();
69
+ // Python's `\w` matches [A-Za-z0-9_] PLUS Unicode word characters.
70
+ // For Cerefox titles we accept ASCII + word + whitespace + hyphen;
71
+ // anything else is stripped. JS doesn't have a one-line equivalent
72
+ // that matches Python's Unicode-aware `\w`, but in practice Cerefox
73
+ // titles are ASCII-dominant. Use `[^\p{L}\p{N}\s_-]` to be Unicode-
74
+ // aware (matches Python more closely than `[^\w\s-]`).
75
+ slug = slug.replace(/[^\p{L}\p{N}\s_-]/gu, "");
76
+ slug = slug.replace(/[\s_-]+/g, "-").replace(/^-+|-+$/g, "");
77
+ if (!slug) slug = "document";
78
+ return `${slug}.md`;
79
+ }
80
+
81
+ // ── Project ID resolution ───────────────────────────────────────────────────
82
+
83
+ /**
84
+ * Caller styles for project assignment. Mirror Python's
85
+ * `_resolve_project_ids` precedence (highest wins):
86
+ *
87
+ * 1. `projectIds` (explicit UUID list) — used as-is, empties stripped.
88
+ * 2. `projectNames` (explicit name list) — each name resolved via
89
+ * the caller's `getOrCreateProject` (so this helper stays pure).
90
+ * 3. `projectId` (single UUID) — wrapped in a list.
91
+ * 4. `projectName` (single name) — resolved and wrapped in a list.
92
+ *
93
+ * Tiers 1 and 2 carry **full-set semantics** (destructive replace when
94
+ * passed to `assignDocumentProjects`) and tiers 3 and 4 carry
95
+ * **single-hint semantics** (non-destructive add via
96
+ * `addDocumentToProjects`). The caller chooses semantics by which
97
+ * argument they pass; this helper just resolves the values.
98
+ */
99
+ export interface ProjectResolveInput {
100
+ projectIds?: string[] | null;
101
+ projectId?: string | null;
102
+ projectName?: string | null;
103
+ projectNames?: string[] | null;
104
+ }
105
+
106
+ /**
107
+ * Async because tiers 2 and 4 need to call `getOrCreateProject` against
108
+ * the DB; the caller passes a thunk to keep this helper pure (no
109
+ * Supabase client dependency).
110
+ */
111
+ export async function resolveProjectIds(
112
+ input: ProjectResolveInput,
113
+ getOrCreateProject: (name: string) => Promise<{ id: string }>,
114
+ ): Promise<string[]> {
115
+ if (input.projectIds !== undefined && input.projectIds !== null) {
116
+ return input.projectIds.filter((p) => p);
117
+ }
118
+ if (input.projectNames !== undefined && input.projectNames !== null) {
119
+ const resolved: string[] = [];
120
+ for (const name of input.projectNames) {
121
+ if (!name) continue;
122
+ const project = await getOrCreateProject(name);
123
+ resolved.push(project.id);
124
+ }
125
+ return resolved;
126
+ }
127
+ if (input.projectId) {
128
+ return [input.projectId];
129
+ }
130
+ if (input.projectName) {
131
+ const project = await getOrCreateProject(input.projectName);
132
+ return [project.id];
133
+ }
134
+ return [];
135
+ }