npm - @cerefox/memory - Versions diffs - 0.11.1 → 1.0.0-beta.1 - Mend

@cerefox/memory 0.11.1 → 1.0.0-beta.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (39) hide show

package/dist/bin/cerefox.js +1077 -834
package/dist/frontend/assets/index-CCkg5PXt.js +125 -0
package/dist/frontend/assets/index-CCkg5PXt.js.map +1 -0
package/dist/frontend/index.html +1 -1
package/dist/server-assets/_shared/ef-auth/index.ts +134 -0
package/dist/server-assets/_shared/ef-meta/index.ts +1 -1
package/dist/server-assets/_shared/embeddings/index.ts +42 -2
package/dist/server-assets/_shared/ingest/chunker.ts +210 -0
package/dist/server-assets/_shared/ingest/index.ts +32 -0
package/dist/server-assets/_shared/ingest/pipeline-helpers.ts +135 -0
package/dist/server-assets/_shared/mcp-auth/index.ts +352 -0
package/dist/server-assets/_shared/mcp-tools/_chunker.ts +16 -170
package/dist/server-assets/_shared/mcp-tools/ingest.ts +13 -4
package/dist/server-assets/db/migrations/0012_content_format.sql +20 -0
package/dist/server-assets/db/rpcs.sql +76 -8
package/dist/server-assets/db/schema.sql +7 -1
package/dist/server-assets/supabase/functions/cerefox-get-audit-log/index.ts +8 -0
package/dist/server-assets/supabase/functions/cerefox-get-document/index.ts +8 -0
package/dist/server-assets/supabase/functions/cerefox-ingest/index.ts +22 -171
package/dist/server-assets/supabase/functions/cerefox-list-projects/index.ts +8 -0
package/dist/server-assets/supabase/functions/cerefox-list-versions/index.ts +8 -0
package/dist/server-assets/supabase/functions/cerefox-mcp/index.ts +54 -0
package/dist/server-assets/supabase/functions/cerefox-mcp/oauth.ts +121 -0
package/dist/server-assets/supabase/functions/cerefox-metadata/index.ts +8 -0
package/dist/server-assets/supabase/functions/cerefox-metadata-search/index.ts +8 -0
package/dist/server-assets/supabase/functions/cerefox-search/index.ts +11 -1
package/docs/guides/access-paths.md +81 -30
package/docs/guides/cli.md +29 -0
package/docs/guides/configuration.md +4 -1
package/docs/guides/connect-agents.md +98 -54
package/docs/guides/content-format.md +55 -0
package/docs/guides/migration-1.0.md +87 -0
package/docs/guides/ops-scripts.md +1 -1
package/docs/guides/quickstart.md +20 -0
package/docs/guides/setup-supabase.md +154 -13
package/docs/guides/upgrading.md +4 -3
package/package.json +1 -1
package/dist/frontend/assets/index-ojNhWSxm.js +0 -125
package/dist/frontend/assets/index-ojNhWSxm.js.map +0 -1

package/dist/frontend/index.html CHANGED Viewed

@@ -15,7 +15,7 @@
       href="https://fonts.googleapis.com/css2?family=Geist:wght@300;400;500;600;700&display=swap"
     />
     <title>Cerefox</title>
-    <script type="module" crossorigin src="/app/assets/index-ojNhWSxm.js"></script>
+    <script type="module" crossorigin src="/app/assets/index-CCkg5PXt.js"></script>
     <link rel="stylesheet" crossorigin href="/app/assets/index-Asx5wD7g.css">
   </head>
   <body>

package/dist/server-assets/_shared/ef-auth/index.ts ADDED Viewed

@@ -0,0 +1,134 @@
+/**
+ * ef-auth — in-function access-token authentication for the primitive Edge
+ * Functions (and `cerefox-mcp`'s static-token path).
+ *
+ * Context (iter-28E, `docs/specs/ef-auth-migration-design.md`): the 8 primitive
+ * EFs move to `--no-verify-jwt` and authenticate the caller *in-function* against
+ * a **rotatable, Cerefox-managed access token** instead of the unrotatable legacy
+ * anon JWT. With the gateway gate removed, **this check is the only auth gate** on
+ * those functions — so it fails closed and compares in constant time.
+ *
+ * The accepted set (`CEREFOX_ACCESS_TOKENS`, comma-separated) holds one or more
+ * tokens so rotation is zero-downtime: add the new token, migrate clients, drop
+ * the old. A request is accepted iff its Bearer credential equals ANY token in the
+ * set (constant-time, no short-circuit on the first match).
+ *
+ * Portability: Web-Platform globals only (`TextEncoder` via the reused
+ * `constantTimeEqual`) — the identical source runs under Deno (the EF) and Bun
+ * (`bun test`). The single audited constant-time primitive lives in
+ * `../mcp-auth/index.ts` and is reused here rather than re-implemented.
+ */
+import { constantTimeEqual } from "../mcp-auth/index.ts";
+// ── Types ────────────────────────────────────────────────────────────────────
+export interface TokenAuthSuccess {
+  ok: true;
+}
+export interface TokenAuthFailure {
+  ok: false;
+  /** Machine-readable reason. Human detail (never returned to the client) in `detail`. */
+  reason: "no_token" | "no_tokens_configured" | "bad_token";
+  detail?: string;
+}
+export type TokenAuthResult = TokenAuthSuccess | TokenAuthFailure;
+export interface AccessTokenConfig {
+  /**
+   * The accepted token set (from `CEREFOX_ACCESS_TOKENS`). When empty the check
+   * FAILS CLOSED (rejects everything) — an EF must never accept-all because a
+   * missing secret left the gate open.
+   */
+  tokens: string[];
+}
+// ── Parsing ──────────────────────────────────────────────────────────────────
+/**
+ * Parse the `CEREFOX_ACCESS_TOKENS` secret into a token set: split on commas,
+ * trim, drop empties (so trailing commas / whitespace don't create a `""` token
+ * that a blank credential could match). Returns `[]` for null/undefined/blank —
+ * which makes the check fail closed.
+ */
+export function parseAccessTokens(raw: string | null | undefined): string[] {
+  if (!raw) return [];
+  return raw
+    .split(",")
+    .map((t) => t.trim())
+    .filter((t) => t.length > 0);
+}
+// ── Check ────────────────────────────────────────────────────────────────────
+const BEARER_PREFIX = "Bearer ";
+/**
+ * Validate the `Authorization: Bearer <token>` header against the accepted set.
+ * Constant-time, fail-closed. Never logs or returns the token value.
+ */
+export function checkAccessToken(
+  authorizationHeader: string | null,
+  config: AccessTokenConfig,
+): TokenAuthResult {
+  if (!authorizationHeader || !authorizationHeader.startsWith(BEARER_PREFIX)) {
+    return { ok: false, reason: "no_token" };
+  }
+  const token = authorizationHeader.slice(BEARER_PREFIX.length).trim();
+  if (!token) return { ok: false, reason: "no_token" };
+  // Fail closed: no tokens configured => reject everything (never accept-all).
+  if (config.tokens.length === 0) {
+    return {
+      ok: false,
+      reason: "no_tokens_configured",
+      detail: "CEREFOX_ACCESS_TOKENS is unset/empty; refusing to accept-all",
+    };
+  }
+  // Compare against every accepted token WITHOUT short-circuiting, so timing
+  // doesn't leak which (or how many) tokens matched.
+  let matched = false;
+  for (const accepted of config.tokens) {
+    if (constantTimeEqual(token, accepted)) matched = true;
+  }
+  return matched ? { ok: true } : { ok: false, reason: "bad_token" };
+}
+// ── Edge Function gate (drop-in for each primitive EF) ───────────────────────
+/**
+ * One-call auth gate for a primitive Edge Function. Returns a 401 `Response` to
+ * short-circuit the handler when the token is missing/wrong/unconfigured, or
+ * `null` to continue. Placed **before** the `/version` branch so version is gated
+ * too (design §3, decision 2026-07-10).
+ *
+ * Deno-free by design: the caller passes the raw `CEREFOX_ACCESS_TOKENS` env
+ * value (read via `Deno.env.get` in the EF) rather than this module touching the
+ * `Deno` global — so the identical source still imports cleanly under Bun for
+ * unit tests. `Response`/`console` are Web globals available in both runtimes.
+ */
+export function efAuthGate(
+  authorization: string | null,
+  tokensRaw: string | null | undefined,
+  headers: Record<string, string>,
+): Response | null {
+  const result = checkAccessToken(authorization, { tokens: parseAccessTokens(tokensRaw) });
+  if (result.ok) return null;
+  // Log the machine reason (never the token). `no_token` is the normal
+  // unauthenticated probe (noise); `bad_token` / `no_tokens_configured` are
+  // worth surfacing in the dashboard logs.
+  if (result.reason !== "no_token") {
+    console.warn(
+      `[ef-auth] rejected: ${result.reason}${result.detail ? ` (${result.detail})` : ""}`,
+    );
+  }
+  return new Response(JSON.stringify({ error: "Unauthorized" }), {
+    status: 401,
+    headers: { ...headers, "WWW-Authenticate": "Bearer" },
+  });
+}

package/dist/server-assets/_shared/ef-meta/index.ts CHANGED Viewed

@@ -18,7 +18,7 @@
  * doesn't touch `supabase/functions/` leaves it alone).
  */
-export const EF_VERSION = "0.11.1";
+export const EF_VERSION = "1.0.0-beta.1";
 /**
  * The 8 peer EFs the cerefox-mcp aggregator probes (excludes cerefox-mcp

package/dist/server-assets/_shared/embeddings/index.ts CHANGED Viewed

@@ -43,6 +43,44 @@ export function openaiEmbeddingConfig(): { url: string; model: string; dimension
   };
 }
+/**
+ * Safety cap (iter-28D Phase 0) on the characters sent to the embedding model per
+ * input — a conservative proxy for the model's token limit (`text-embedding-3-small`
+ * is 8191 tokens). Normal chunks are far below this (`max_chunk_chars` ≈ 2000); the
+ * cap only bites on an oversized *keep-whole* chunk (a huge table or blank-line-free
+ * paragraph the interim chunker fix keeps intact). Default 20000 chars sits well under
+ * 8191 tokens for markdown/English; override with `CEREFOX_EMBED_MAX_INPUT_CHARS`.
+ */
+export const DEFAULT_EMBED_MAX_INPUT_CHARS = 20000;
+export function embeddingMaxInputChars(): number {
+  const env =
+    (globalThis as { process?: { env?: Record<string, string | undefined> } }).process?.env ?? {};
+  const raw = Number.parseInt(env.CEREFOX_EMBED_MAX_INPUT_CHARS ?? "", 10);
+  return Number.isNaN(raw) || raw <= 0 ? DEFAULT_EMBED_MAX_INPUT_CHARS : raw;
+}
+/**
+ * Truncate one embedding input to the char cap. The full chunk `content` is stored and
+ * reconstructed untouched; only its *embedding* is computed on this prefix — so at worst
+ * search quality for one oversized chunk is degraded, and an ingest **never fails** on an
+ * over-limit embedding input (the failure mode this prevents). Surrogate-safe (never leaves
+ * a dangling high surrogate). Warns when it truncates.
+ */
+export function capEmbeddingInput(text: string): string {
+  const max = embeddingMaxInputChars();
+  if (text.length <= max) return text;
+  let cut = text.slice(0, max);
+  const last = cut.charCodeAt(cut.length - 1);
+  if (last >= 0xd800 && last <= 0xdbff) cut = cut.slice(0, -1); // don't split a surrogate pair
+  console.warn(
+    `[embeddings] truncated an embedding input: ${text.length} → ${cut.length} chars ` +
+      `(cap CEREFOX_EMBED_MAX_INPUT_CHARS=${max}). The full content is stored and reconstructed ` +
+      `untouched; only this chunk's embedding uses the prefix (degraded search for this chunk).`,
+  );
+  return cut;
+}
 const EMBEDDING_MAX_RETRIES = 3;
 const EMBEDDING_INITIAL_BACKOFF_MS = 500; // 500ms → 1s → 2s
@@ -50,6 +88,7 @@ const EMBEDDING_INITIAL_BACKOFF_MS = 500; // 500ms → 1s → 2s
 export async function getEmbedding(text: string, apiKey: string): Promise<number[]> {
   let lastError: Error | null = null;
   const cfg = openaiEmbeddingConfig();
+  const input = capEmbeddingInput(text); // cap once, before the retry loop
   for (let attempt = 0; attempt < EMBEDDING_MAX_RETRIES; attempt++) {
     try {
@@ -61,7 +100,7 @@ export async function getEmbedding(text: string, apiKey: string): Promise<number
         },
         body: JSON.stringify({
           model: cfg.model,
-          input: text,
+          input,
           dimensions: cfg.dimensions,
         }),
       });
@@ -121,6 +160,7 @@ async function embedBatchSingleCall(
 ): Promise<number[][]> {
   let lastError: Error | null = null;
   const cfg = openaiEmbeddingConfig();
+  const inputs = texts.map(capEmbeddingInput); // cap once, before the retry loop
   for (let attempt = 0; attempt < EMBEDDING_MAX_RETRIES; attempt++) {
     try {
@@ -132,7 +172,7 @@ async function embedBatchSingleCall(
         },
         body: JSON.stringify({
           model: cfg.model,
-          input: texts,
+          input: inputs,
           dimensions: cfg.dimensions,
         }),
       });

package/dist/server-assets/_shared/ingest/chunker.ts ADDED Viewed

@@ -0,0 +1,210 @@
+/**
+ * Exact-partition ("blind-stitch") markdown chunker — the single TS chunker
+ * (iter-28D Phase 1). Consolidated: `_shared/mcp-tools/_chunker.ts` and the
+ * `cerefox-ingest` Edge Function both import from here (the earlier duplicate
+ * copies were removed).
+ *
+ * **The invariant:** `blindStitch(chunkMarkdown(doc)) === doc.trim()`, byte-for-
+ * byte. Chunk `content` values are consecutive, gapless, non-overlapping slices
+ * of the trimmed document, so reconstruction is a plain concatenation — no `\n\n`
+ * separator is synthesized on read (the bug the old chunker+reconstruction had).
+ * That is what lets a chunk boundary fall *anywhere* (including mid-paragraph at
+ * a size limit) with zero corruption, which in turn lets us bound chunk size.
+ *
+ * These chunks are stored with `content_format = 2` and reconstructed by blind
+ * concat (`STRING_AGG(content, '')`). Documents written before Phase 1 stay
+ * `content_format = 1` and reconstruct with the legacy `\n\n`-join (see
+ * `rpcs.sql`); they are untouched until re-written. Design:
+ * `docs/specs/chunk-reconstruction-design.md`.
+ *
+ * Heading context is NOT stored in mid-section chunk content (that would break
+ * the exact partition); it lives in `heading_path` metadata, and the embedding
+ * input adds it back as a breadcrumb (`# {title}\n{breadcrumb}\n{content}`).
+ *
+ * Length is code-point-based (`cpLen`), matching the historical size semantics.
+ */
+/** Single chunk produced by the markdown chunker. */
+export interface ChunkData {
+  chunk_index: number;
+  heading_path: string[];
+  /** 0 = no heading (preamble); 1-3 = the innermost active H1-H3. */
+  heading_level: number;
+  /** Last element of heading_path, or "" for preamble. */
+  title: string;
+  /** Exact slice of the trimmed document (a gapless partition member). */
+  content: string;
+  char_count: number;
+}
+// ── Helpers ──────────────────────────────────────────────────────────────────
+/** Code-point length (`"🎉".length` is 2 in JS but 1 code point). */
+function cpLen(s: string): number {
+  let n = 0;
+  for (const _ of s) n++;
+  return n;
+}
+/** Strip trailing `#` (matches a markdown closing-hash heading). */
+function rstripHash(s: string): string {
+  let i = s.length;
+  while (i > 0 && s[i - 1] === "#") i--;
+  return s.slice(0, i);
+}
+interface HeadingMark {
+  /** UTF-16 offset of the `#` in the trimmed doc. */
+  offset: number;
+  level: number;
+  text: string;
+}
+/** All H1/H2/H3 heading lines in `doc`, in order (heading text on one line). */
+function findHeadings(doc: string): HeadingMark[] {
+  const out: HeadingMark[] = [];
+  const re = /^(#{1,3})[ \t]+(.+)$/gm;
+  let m: RegExpExecArray | null;
+  while ((m = re.exec(doc)) !== null) {
+    out.push({ offset: m.index, level: m[1].length, text: rstripHash(m[2]).trim() });
+  }
+  return out;
+}
+/** The active H1/H2/H3 nesting stack at `offset` (headings at or before it). */
+function activeHeadings(headings: HeadingMark[], offset: number): HeadingMark[] {
+  const stack: HeadingMark[] = [];
+  for (const h of headings) {
+    if (h.offset > offset) break;
+    while (stack.length && stack[stack.length - 1].level >= h.level) stack.pop();
+    stack.push(h);
+  }
+  return stack;
+}
+/**
+ * Split `s` into consecutive pieces of at most `maxCp` code points each, with
+ * `pieces.join("") === s`. Never splits a code point (surrogate pair).
+ */
+function hardSplitCp(s: string, maxCp: number): string[] {
+  const out: string[] = [];
+  let buf = "";
+  let n = 0;
+  for (const ch of s) {
+    if (n >= maxCp) {
+      out.push(buf);
+      buf = "";
+      n = 0;
+    }
+    buf += ch;
+    n++;
+  }
+  if (buf) out.push(buf);
+  return out;
+}
+// ── Public API ───────────────────────────────────────────────────────────────
+/**
+ * Split markdown into exact-partition chunks. `minChunkChars` is accepted for
+ * signature compatibility but unused (tiny trailing atoms merge into the current
+ * chunk naturally via greedy accumulation).
+ */
+export function chunkMarkdown(
+  text: string,
+  maxChunkChars = 4000,
+  _minChunkChars = 100,
+): ChunkData[] {
+  const doc = text.trim();
+  if (!doc) return [];
+  if (cpLen(doc) <= maxChunkChars) {
+    return [
+      { chunk_index: 0, heading_path: [], heading_level: 0, title: "", content: doc, char_count: cpLen(doc) },
+    ];
+  }
+  const headings = findHeadings(doc);
+  // Atomize preserving every character: split on blank-line runs, attach each
+  // separator to its PRECEDING block (so a separator never starts a chunk), then
+  // hard-split any atom that alone exceeds the size limit. atoms.join("") === doc.
+  const parts = doc.split(/(\n{2,})/); // [block, sep, block, sep, …, block]
+  const atoms: string[] = [];
+  for (let i = 0; i < parts.length; i += 2) {
+    const unit = (parts[i] ?? "") + (parts[i + 1] ?? ""); // block + trailing separator
+    if (unit === "") continue;
+    if (cpLen(unit) > maxChunkChars) atoms.push(...hardSplitCp(unit, maxChunkChars));
+    else atoms.push(unit);
+  }
+  const chunks: ChunkData[] = [];
+  let buf = "";
+  let bufCp = 0;
+  let bufStart = 0;
+  let offset = 0;
+  const flush = (): void => {
+    if (buf === "") return;
+    const stack = activeHeadings(headings, bufStart);
+    chunks.push({
+      chunk_index: chunks.length,
+      heading_path: stack.map((h) => h.text),
+      heading_level: stack.length ? stack[stack.length - 1].level : 0,
+      title: stack.length ? stack[stack.length - 1].text : "",
+      content: buf,
+      char_count: bufCp,
+    });
+    buf = "";
+    bufCp = 0;
+  };
+  for (const atom of atoms) {
+    const cp = cpLen(atom);
+    if (buf === "") {
+      bufStart = offset;
+      buf = atom;
+      bufCp = cp;
+    } else if (bufCp + cp <= maxChunkChars) {
+      buf += atom;
+      bufCp += cp;
+    } else {
+      flush();
+      bufStart = offset;
+      buf = atom;
+      bufCp = cp;
+    }
+    offset += atom.length; // UTF-16 length — same units as heading offsets
+  }
+  flush();
+  return chunks;
+}
+/** Format-2 reconstruction: blind concatenation (mirrors the RPC's `STRING_AGG(content,'')`). */
+export function blindStitch(chunks: Pick<ChunkData, "content">[]): string {
+  return chunks.map((c) => c.content).join("");
+}
+/**
+ * The content format this chunker produces. Stamped on each chunk (via the ingest
+ * RPC's p_content_format) and used by the reconstruction RPCs to pick the join
+ * strategy (2 = blind concat). See docs/guides/content-format.md.
+ */
+export const CONTENT_FORMAT_BLIND_STITCH = 2;
+/**
+ * Build the title-boosted embedding input for a chunk. Format-2 chunk `content` is
+ * an exact slice that no longer carries the heading line for mid-section chunks, so
+ * the heading context is re-added here as a breadcrumb from `heading_path` (search
+ * quality parity). Stored `content` is unchanged; this string is embedded only.
+ *   `# {docTitle}`                       (chunk at the doc's top / preamble)
+ *   `# {docTitle}\n{A > B > C}`          (chunk under headings A > B > C)
+ */
+export function embeddingInputFor(
+  docTitle: string,
+  chunk: Pick<ChunkData, "heading_path" | "content">,
+): string {
+  const breadcrumb = chunk.heading_path.join(" > ");
+  const head = breadcrumb ? `# ${docTitle}\n${breadcrumb}` : `# ${docTitle}`;
+  return `${head}\n${chunk.content}`;
+}

package/dist/server-assets/_shared/ingest/index.ts ADDED Viewed

@@ -0,0 +1,32 @@
+/**
+ * `_shared/ingest/` — chunking + embedding orchestration + content-hashing
+ * utilities used by the v0.7 TS ingestion pipeline + the CLI ingest path.
+ *
+ * Consumers:
+ *   - `packages/memory/src/ingestion/pipeline.ts` (the IngestionPipeline)
+ *   - `packages/memory/src/cli/commands/ingest.ts` (v0.7 in-process variant)
+ *   - `packages/memory/src/web/routes/ingest.ts` (the 3 endpoints unblocked
+ *     in Part 25F)
+ *   - `packages/memory/src/web/routes/documents-write.ts` (v0.6's /edit
+ *     content-hash short-circuit, now using the shared helper)
+ *
+ * Also consumed by `supabase/functions/cerefox-ingest/` and
+ * `_shared/mcp-tools/_chunker.ts` (iter-28D Phase 1 consolidation) — the
+ * chunker lives here only; the EF bundles `_shared/ingest/` via
+ * `bundle_server_assets.ts`, the same way it bundles `ef-auth`/`embeddings`.
+ */
+export {
+  chunkMarkdown,
+  blindStitch,
+  embeddingInputFor,
+  CONTENT_FORMAT_BLIND_STITCH,
+  type ChunkData,
+} from "./chunker.js";
+export {
+  normalizeForHash,
+  contentHash,
+  deriveSourcePath,
+  resolveProjectIds,
+  type ProjectResolveInput,
+} from "./pipeline-helpers.js";

package/dist/server-assets/_shared/ingest/pipeline-helpers.ts ADDED Viewed

@@ -0,0 +1,135 @@
+/**
+ * Cross-consumer helpers for the v0.7 TS ingestion pipeline.
+ *
+ * Consumers (all TS surfaces that need to dedup by content hash, mint a
+ * source_path from a title, or resolve mixed project-id/name caller
+ * styles into a final UUID list):
+ *
+ *   - `packages/memory/src/ingestion/pipeline.ts` (IngestionPipeline)
+ *   - `packages/memory/src/web/routes/documents-write.ts` (v0.6 /edit
+ *     content-hash short-circuit — promoted out of the inline version)
+ *   - `packages/memory/src/cli/commands/ingest.ts` (v0.7 in-process)
+ *
+ * NOT consumed by `supabase/functions/cerefox-ingest/` (Deno Edge
+ * Runtime can't reach `_shared/`; the EF keeps its own copies).
+ * Cross-runtime parity is enforced by shared fixtures + manual smoke.
+ *
+ * Python parity: `normalizeForHash` + `contentHash` match `_normalize`
+ * + `_hash` in `src/cerefox/ingestion/pipeline.py` exactly. Drift =
+ * dedup breaks across the CLI / web / Python paths.
+ */
+import { createHash } from "node:crypto";
+// ── Content normalization + hash ────────────────────────────────────────────
+/**
+ * Normalize content before hashing (mirrors Python's `_normalize`):
+ *   1. CRLF → LF
+ *   2. Bare CR → LF
+ *   3. Strip leading/trailing whitespace
+ *   4. Collapse 3+ consecutive newlines to 2
+ *
+ * Stable across round-trips through the web edit form (browsers submit
+ * textarea content with CRLF per the HTML spec).
+ */
+export function normalizeForHash(text: string): string {
+  return text
+    .replace(/\r\n/g, "\n")
+    .replace(/\r/g, "\n")
+    .trim()
+    .replace(/\n{3,}/g, "\n\n");
+}
+/**
+ * SHA-256 hex digest of the normalized UTF-8-encoded text. Same
+ * algorithm Python uses in `_hash`. Output is a 64-character lowercase
+ * hex string.
+ */
+export function contentHash(text: string): string {
+  return createHash("sha256")
+    .update(normalizeForHash(text), "utf8")
+    .digest("hex");
+}
+// ── source_path derivation from title ───────────────────────────────────────
+/**
+ * Derive a default `source_path` from a document title when none was
+ * provided (e.g. paste ingestion). Matches Python's behaviour exactly:
+ *
+ *     slug = re.sub(r"[^\w\s-]", "", title.lower())
+ *     slug = re.sub(r"[\s_-]+", "-", slug).strip("-") or "document"
+ *     source_path = f"{slug}.md"
+ *
+ * Used for download filenames + Obsidian-style link resolution.
+ */
+export function deriveSourcePath(title: string): string {
+  let slug = title.toLowerCase();
+  // Python's `\w` matches [A-Za-z0-9_] PLUS Unicode word characters.
+  // For Cerefox titles we accept ASCII + word + whitespace + hyphen;
+  // anything else is stripped. JS doesn't have a one-line equivalent
+  // that matches Python's Unicode-aware `\w`, but in practice Cerefox
+  // titles are ASCII-dominant. Use `[^\p{L}\p{N}\s_-]` to be Unicode-
+  // aware (matches Python more closely than `[^\w\s-]`).
+  slug = slug.replace(/[^\p{L}\p{N}\s_-]/gu, "");
+  slug = slug.replace(/[\s_-]+/g, "-").replace(/^-+|-+$/g, "");
+  if (!slug) slug = "document";
+  return `${slug}.md`;
+}
+// ── Project ID resolution ───────────────────────────────────────────────────
+/**
+ * Caller styles for project assignment. Mirror Python's
+ * `_resolve_project_ids` precedence (highest wins):
+ *
+ *   1. `projectIds` (explicit UUID list) — used as-is, empties stripped.
+ *   2. `projectNames` (explicit name list) — each name resolved via
+ *      the caller's `getOrCreateProject` (so this helper stays pure).
+ *   3. `projectId` (single UUID) — wrapped in a list.
+ *   4. `projectName` (single name) — resolved and wrapped in a list.
+ *
+ * Tiers 1 and 2 carry **full-set semantics** (destructive replace when
+ * passed to `assignDocumentProjects`) and tiers 3 and 4 carry
+ * **single-hint semantics** (non-destructive add via
+ * `addDocumentToProjects`). The caller chooses semantics by which
+ * argument they pass; this helper just resolves the values.
+ */
+export interface ProjectResolveInput {
+  projectIds?: string[] | null;
+  projectId?: string | null;
+  projectName?: string | null;
+  projectNames?: string[] | null;
+}
+/**
+ * Async because tiers 2 and 4 need to call `getOrCreateProject` against
+ * the DB; the caller passes a thunk to keep this helper pure (no
+ * Supabase client dependency).
+ */
+export async function resolveProjectIds(
+  input: ProjectResolveInput,
+  getOrCreateProject: (name: string) => Promise<{ id: string }>,
+): Promise<string[]> {
+  if (input.projectIds !== undefined && input.projectIds !== null) {
+    return input.projectIds.filter((p) => p);
+  }
+  if (input.projectNames !== undefined && input.projectNames !== null) {
+    const resolved: string[] = [];
+    for (const name of input.projectNames) {
+      if (!name) continue;
+      const project = await getOrCreateProject(name);
+      resolved.push(project.id);
+    }
+    return resolved;
+  }
+  if (input.projectId) {
+    return [input.projectId];
+  }
+  if (input.projectName) {
+    const project = await getOrCreateProject(input.projectName);
+    return [project.id];
+  }
+  return [];
+}