@remit/search-service 0.0.1

package/src/backends/sqlite-vec.ts

@@ -0,0 +1,250 @@
+import { mkdirSync } from "node:fs";
+import { dirname } from "node:path";
+import type SqliteDatabase from "better-sqlite3";
+import type {
+	ChunkMetadata,
+	VectorMatch,
+	VectorQuery,
+	VectorQueryFilter,
+	VectorRecord,
+} from "../types.js";
+import type { VectorStoreService } from "./memory.js";
+import { runtimeImport } from "./runtime-import.js";
+
+type Database = SqliteDatabase.Database;
+
+type BetterSqlite3Module = {
+	default: new (path: string) => Database;
+};
+
+type SqliteVecModule = {
+	load: (db: Database) => void;
+};
+
+/**
+ * vec0 stores each chunk's vector alongside the scalar fields the query path
+ * filters on, so equality / range filters are pushed into the KNN instead of
+ * post-filtering a fixed top-k (which would silently drop recall). The full
+ * metadata object rides along in an auxiliary (`+`) column for reconstruction.
+ *
+ * `mailboxId` is a single-value scalar here because the indexing pipeline always
+ * writes a one-element `mailboxIds`; membership therefore reduces to equality on
+ * the stored value.
+ */
+const CREATE_TABLE = (dimensions: number): string => `
+	CREATE VIRTUAL TABLE IF NOT EXISTS vec_chunks USING vec0(
+		chunk_id TEXT PRIMARY KEY,
+		message_id TEXT,
+		account_config_id TEXT,
+		mailbox_id TEXT,
+		chunk_type TEXT,
+		category TEXT,
+		sent_date INTEGER,
+		is_read INTEGER,
+		has_attachment INTEGER,
+		has_stars INTEGER,
+		embedding FLOAT[${dimensions}] distance_metric=cosine,
+		+meta TEXT
+	);
+`;
+
+// vec0 INTEGER metadata columns are strict and better-sqlite3 binds a plain JS
+// number as REAL, so every integer value bound against an INTEGER column (the
+// booleans and sent_date) must be a BigInt to avoid an "Expected integer" error.
+const bool = (value: boolean): bigint => (value ? 1n : 0n);
+
+type BindValue = string | number | bigint;
+
+interface WhereClause {
+	sql: string;
+	params: BindValue[];
+}
+
+const buildFilterClause = (
+	filter: VectorQueryFilter | undefined,
+): WhereClause => {
+	const sql: string[] = [];
+	const params: BindValue[] = [];
+	if (!filter) return { sql: "", params };
+
+	if (filter.accountConfigId !== undefined) {
+		sql.push("account_config_id = ?");
+		params.push(filter.accountConfigId);
+	}
+	if (filter.mailboxId !== undefined) {
+		sql.push("mailbox_id = ?");
+		params.push(filter.mailboxId);
+	}
+	if (filter.chunkType !== undefined) {
+		sql.push("chunk_type = ?");
+		params.push(filter.chunkType);
+	}
+	if (filter.category !== undefined) {
+		sql.push("category = ?");
+		params.push(filter.category);
+	}
+	if (filter.hasAttachment !== undefined) {
+		sql.push("has_attachment = ?");
+		params.push(bool(filter.hasAttachment));
+	}
+	if (filter.hasStars !== undefined) {
+		sql.push("has_stars = ?");
+		params.push(bool(filter.hasStars));
+	}
+	if (filter.isRead !== undefined) {
+		sql.push("is_read = ?");
+		params.push(bool(filter.isRead));
+	}
+	if (filter.sentDateRange) {
+		if (filter.sentDateRange.from !== undefined) {
+			sql.push("sent_date >= ?");
+			params.push(BigInt(Math.trunc(filter.sentDateRange.from)));
+		}
+		if (filter.sentDateRange.to !== undefined) {
+			sql.push("sent_date <= ?");
+			params.push(BigInt(Math.trunc(filter.sentDateRange.to)));
+		}
+	}
+
+	return { sql: sql.length > 0 ? ` AND ${sql.join(" AND ")}` : "", params };
+};
+
+export interface SqliteVectorStoreConfig {
+	path: string;
+	dimensions?: number;
+}
+
+const DEFAULT_DIMENSIONS = 384;
+
+export const createSqliteVectorStore = (
+	config: SqliteVectorStoreConfig,
+): VectorStoreService => {
+	const dimensions = config.dimensions ?? DEFAULT_DIMENSIONS;
+	let dbPromise: Promise<Database> | null = null;
+
+	const getDb = async (): Promise<Database> => {
+		if (dbPromise) return dbPromise;
+		dbPromise = (async () => {
+			const { default: Database } =
+				await runtimeImport<BetterSqlite3Module>("better-sqlite3");
+			const sqliteVec = await runtimeImport<SqliteVecModule>("sqlite-vec");
+			if (config.path !== ":memory:") {
+				mkdirSync(dirname(config.path), { recursive: true });
+			}
+			const db = new Database(config.path);
+			db.pragma("journal_mode = WAL");
+			sqliteVec.load(db);
+			db.exec(CREATE_TABLE(dimensions));
+			return db;
+		})();
+		return dbPromise;
+	};
+
+	const upsert = async (vectors: VectorRecord[]): Promise<void> => {
+		if (vectors.length === 0) return;
+		const db = await getDb();
+		const del = db.prepare("DELETE FROM vec_chunks WHERE chunk_id = ?");
+		const ins = db.prepare(`
+			INSERT INTO vec_chunks (
+				chunk_id, message_id, account_config_id, mailbox_id, chunk_type,
+				category, sent_date, is_read, has_attachment, has_stars, embedding, meta
+			) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+		`);
+		const writeAll = db.transaction((records: VectorRecord[]) => {
+			for (const r of records) {
+				const m = r.metadata;
+				del.run(r.chunkId);
+				ins.run(
+					r.chunkId,
+					m.messageId,
+					m.accountConfigId,
+					m.mailboxIds[0] ?? "",
+					m.chunkType,
+					// vec0 filterable metadata columns are strict NOT NULL; a chunk
+					// with no category binds an empty-string sentinel (a `category = ?`
+					// filter never matches it) rather than NULL, which vec0 rejects.
+					// The returned metadata reads from the `meta` JSON column, so
+					// category stays correctly absent there.
+					m.category ?? "",
+					BigInt(Math.trunc(m.sentDate)),
+					bool(m.isRead),
+					bool(m.hasAttachment),
+					bool(m.hasStars),
+					JSON.stringify(r.vector),
+					JSON.stringify(m),
+				);
+			}
+		});
+		writeAll(vectors);
+	};
+
+	const query = async (params: VectorQuery): Promise<VectorMatch[]> => {
+		const db = await getDb();
+		const { sql: filterSql, params: filterParams } = buildFilterClause(
+			params.filter,
+		);
+		const stmt = db.prepare(`
+			SELECT chunk_id AS chunkId, distance, meta
+			FROM vec_chunks
+			WHERE embedding MATCH ? AND k = ?${filterSql}
+			ORDER BY distance
+		`);
+		const rows = stmt.all(
+			JSON.stringify(params.vector),
+			params.topK,
+			...filterParams,
+		) as { chunkId: string; distance: number; meta: string }[];
+		return rows.map((row) => ({
+			chunkId: row.chunkId,
+			score: 1 - row.distance,
+			metadata: JSON.parse(row.meta) as ChunkMetadata,
+		}));
+	};
+
+	const existingContentHashes = async (
+		chunkIds: string[],
+	): Promise<Map<string, string>> => {
+		const out = new Map<string, string>();
+		if (chunkIds.length === 0) return out;
+		const db = await getDb();
+		const placeholders = chunkIds.map(() => "?").join(", ");
+		const stmt = db.prepare(
+			`SELECT chunk_id AS chunkId, meta FROM vec_chunks WHERE chunk_id IN (${placeholders})`,
+		);
+		const rows = stmt.all(...chunkIds) as { chunkId: string; meta: string }[];
+		for (const row of rows) {
+			const meta = JSON.parse(row.meta) as ChunkMetadata;
+			if (typeof meta.contentHash === "string") {
+				out.set(row.chunkId, meta.contentHash);
+			}
+		}
+		return out;
+	};
+
+	const getByMessage = async (messageId: string): Promise<VectorRecord[]> => {
+		const db = await getDb();
+		const stmt = db.prepare(
+			`SELECT chunk_id AS chunkId, vec_to_json(embedding) AS embedding, meta
+			 FROM vec_chunks WHERE message_id = ?`,
+		);
+		const rows = stmt.all(messageId) as {
+			chunkId: string;
+			embedding: string;
+			meta: string;
+		}[];
+		return rows.map((row) => ({
+			chunkId: row.chunkId,
+			vector: JSON.parse(row.embedding) as number[],
+			metadata: JSON.parse(row.meta) as ChunkMetadata,
+		}));
+	};
+
+	const del = async (filter: { messageId: string }): Promise<void> => {
+		const db = await getDb();
+		db.prepare("DELETE FROM vec_chunks WHERE message_id = ?").run(
+			filter.messageId,
+		);
+	};
+
+	return { upsert, query, existingContentHashes, getByMessage, delete: del };
+};

package/src/bedrock.ts

@@ -0,0 +1,4 @@
+export {
+	type BedrockEmbeddingConfig,
+	BedrockEmbeddingService,
+} from "./backends/bedrock.js";

package/src/chunking/chunker.test.ts

@@ -0,0 +1,79 @@
+import assert from "node:assert";
+import { describe, it } from "node:test";
+import type {
+	ChunkType,
+	EnvelopeChunkInput,
+	ParsedBodyForChunking,
+} from "../types.js";
+import { createEmailChunker } from "./chunker.js";
+import { candidateChunkKeys } from "./keys.js";
+
+const MESSAGE_ID = "msg-parity-1";
+
+const envelope: EnvelopeChunkInput = {
+	from: { name: "Alice", email: "alice@example.com" },
+	to: [{ name: "Bob", email: "bob@example.com" }],
+	cc: [{ name: "Carol", email: "carol@example.com" }],
+	bcc: [],
+	subject: "Q1 2026 invoice review and operational metrics",
+	attachments: [
+		{
+			filename: "invoice-q1-2026.pdf",
+			contentType: "application/pdf",
+			size: 245_000,
+		},
+	],
+};
+
+// Long, high-entropy prose with many UNIQUE entities so the chunker is forced to
+// emit multiple body-N (oversized prose split past the per-chunk cap) and
+// multiple entities-N (the entity summary exceeds the embed char budget).
+const richBody = (): string =>
+	Array.from(
+		{ length: 200 },
+		(_, i) =>
+			`Section ${i}: the quarterly review covered revenue growth and detailed ` +
+			`operational metrics across every region with specific commentary. ` +
+			`Contact person${i}@example.com or open ` +
+			`https://reports.example.com/region/${i}/full-quarterly-summary before ` +
+			`2026-03-15 regarding the €${i},500 budget allocation under discussion.`,
+	).join("\n\n");
+
+describe("createEmailChunker producer/deleter parity", () => {
+	const chunker = createEmailChunker();
+	const parsedBody: ParsedBodyForChunking = { text: richBody(), html: null };
+	const chunks = chunker.chunk({ envelope, parsedBody, messageId: MESSAGE_ID });
+
+	it("exercises every chunk type with multiple body and entity chunks", () => {
+		const types = new Set<ChunkType>(chunks.map((c) => c.chunkType));
+		const expectedTypes: ChunkType[] = [
+			"sender",
+			"recipient",
+			"subject",
+			"attachment",
+			"body",
+			"entities",
+		];
+		for (const expected of expectedTypes) {
+			assert.ok(types.has(expected), `missing chunk type: ${expected}`);
+		}
+
+		const bodyChunks = chunks.filter((c) => c.chunkId.includes("::body-"));
+		const entityChunks = chunks.filter((c) =>
+			c.chunkId.includes("::entities-"),
+		);
+		assert.ok(bodyChunks.length > 1, "expected multiple body-N chunks");
+		assert.ok(entityChunks.length > 1, "expected multiple entities-N chunks");
+	});
+
+	it("emits only chunkIds the deleter can reap via candidateChunkKeys", () => {
+		const reapable = new Set(candidateChunkKeys(MESSAGE_ID));
+		for (const chunk of chunks) {
+			assert.ok(
+				reapable.has(chunk.chunkId),
+				`produced chunkId ${chunk.chunkId} is not in candidateChunkKeys — ` +
+					`the deleter can never reap it`,
+			);
+		}
+	});
+});

package/src/chunking/chunker.ts

@@ -0,0 +1,56 @@
+import type {
+	Chunk,
+	EnvelopeChunkInput,
+	ParsedBodyForChunking,
+} from "../types.js";
+import { buildEntityChunks } from "./entities.js";
+import { buildBodyChunks } from "./entropy.js";
+import { chunkKeyFor } from "./keys.js";
+import { buildStructuredChunks } from "./structured.js";
+
+export interface ChunkInput {
+	envelope: EnvelopeChunkInput;
+	parsedBody: ParsedBodyForChunking;
+	messageId: string;
+}
+
+export interface EmailChunker {
+	chunk(input: ChunkInput): Chunk[];
+}
+
+const stripHtml = (html: string): string =>
+	html
+		.replace(/<style[^>]*>[\s\S]*?<\/style>/gi, "")
+		.replace(/<script[^>]*>[\s\S]*?<\/script>/gi, "")
+		.replace(/<[^>]+>/g, " ")
+		.replace(/&nbsp;/g, " ")
+		.replace(/&amp;/g, "&")
+		.replace(/&lt;/g, "<")
+		.replace(/&gt;/g, ">")
+		.replace(/&quot;/g, '"')
+		.replace(/\s+/g, " ")
+		.trim();
+
+const bodyText = (parsed: ParsedBodyForChunking): string => {
+	if (parsed.text && parsed.text.trim().length > 0) return parsed.text;
+	if (parsed.html && parsed.html.trim().length > 0)
+		return stripHtml(parsed.html);
+	return "";
+};
+
+export const createEmailChunker = (): EmailChunker => ({
+	chunk: ({ envelope, parsedBody, messageId }: ChunkInput): Chunk[] => {
+		const chunkIdFor = (suffix: string): string =>
+			chunkKeyFor(messageId, suffix);
+		const chunks: Chunk[] = [];
+		chunks.push(...buildStructuredChunks(envelope, chunkIdFor));
+
+		const text = bodyText(parsedBody);
+		if (text.length > 0) {
+			chunks.push(...buildBodyChunks(text, chunkIdFor));
+			chunks.push(...buildEntityChunks(text, chunkIdFor));
+		}
+
+		return chunks;
+	},
+});

package/src/chunking/entities.test.ts

@@ -0,0 +1,82 @@
+import assert from "node:assert";
+import { describe, it } from "node:test";
+import { buildEntityChunks, extractEntities } from "./entities.js";
+
+const idFor = (suffix: string): string => `msg-1::${suffix}`;
+
+describe("extractEntities", () => {
+	it("extracts email addresses", () => {
+		const text = "Contact alice@example.com or bob@example.com for details.";
+		const ents = extractEntities(text);
+		assert.deepStrictEqual(ents.emails.sort(), [
+			"alice@example.com",
+			"bob@example.com",
+		]);
+	});
+
+	it("dedupes repeated email addresses", () => {
+		const text = "alice@example.com is alice@example.com everywhere";
+		const ents = extractEntities(text);
+		assert.deepStrictEqual(ents.emails, ["alice@example.com"]);
+	});
+
+	it("extracts URLs", () => {
+		const text =
+			"Open https://docs.example.com/report and https://example.com/x";
+		const ents = extractEntities(text);
+		assert.ok(ents.urls.includes("https://docs.example.com/report"));
+		assert.ok(ents.urls.includes("https://example.com/x"));
+	});
+
+	it("extracts ISO and long-form dates", () => {
+		const text =
+			"The deadline is 2026-03-15 or March 15, 2026 — Q1 2026 cutoff.";
+		const ents = extractEntities(text);
+		assert.ok(ents.dates.includes("2026-03-15"));
+		assert.ok(ents.dates.some((d) => /March 15/.test(d)));
+		assert.ok(ents.dates.includes("Q1 2026"));
+	});
+
+	it("extracts currency amounts and percentages", () => {
+		const text = "The deal is worth €14,500 and the team beat target by 14%.";
+		const ents = extractEntities(text);
+		assert.ok(ents.amounts.some((a) => a.includes("14,500")));
+		assert.ok(ents.amounts.includes("14%"));
+	});
+});
+
+describe("buildEntityChunks", () => {
+	it("emits a single entities chunk when entities are found", () => {
+		const text = "Email alice@example.com about Q1 2026 invoice for €14,500.";
+		const chunks = buildEntityChunks(text, idFor);
+		assert.strictEqual(chunks.length, 1);
+		assert.strictEqual(chunks[0].chunkType, "entities");
+		assert.strictEqual(chunks[0].chunkId, "msg-1::entities");
+		assert.match(chunks[0].text, /alice@example\.com/);
+		assert.match(chunks[0].text, /Q1 2026/);
+	});
+
+	it("emits no chunk when no entities are found", () => {
+		const text = "just plain prose with no specific entities at all";
+		const chunks = buildEntityChunks(text, idFor);
+		assert.strictEqual(chunks.length, 0);
+	});
+
+	it("splits a huge Links list into multiple capped chunks", () => {
+		const links = Array.from(
+			{ length: 4000 },
+			(_, i) =>
+				`Visit https://news.example.com/articles/${i}/read-the-full-story`,
+		).join(" ");
+		const chunks = buildEntityChunks(links, idFor);
+
+		assert.ok(chunks.length > 1, "expected the oversized list to split");
+		for (const chunk of chunks) {
+			assert.strictEqual(chunk.chunkType, "entities");
+			assert.ok(
+				chunk.text.length <= 6000,
+				`chunk ${chunk.chunkId} length ${chunk.text.length} exceeds budget`,
+			);
+		}
+	});
+});

package/src/chunking/entities.ts

@@ -0,0 +1,74 @@
+import type { Chunk } from "../types.js";
+import { EMBED_CHAR_BUDGET, splitToCharBudget } from "./entropy.js";
+import { MAX_CHUNKS_PER_TYPE } from "./keys.js";
+
+const EMAIL_RE = /\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b/g;
+const URL_RE = /\bhttps?:\/\/[^\s<>"]+/gi;
+const CURRENCY_RE =
+	/(?:[€$£¥]\s?\d{1,3}(?:[.,]\d{3})*(?:[.,]\d{1,2})?|\b\d{1,3}(?:[.,]\d{3})*(?:[.,]\d{1,2})?\s?(?:EUR|USD|GBP|JPY)\b)/g;
+const PERCENT_RE = /\b\d{1,3}(?:[.,]\d+)?%/g;
+const ISO_DATE_RE = /\b\d{4}-\d{2}-\d{2}\b/g;
+const LONG_DATE_RE =
+	/\b(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)[a-z]*\.?\s+\d{1,2}(?:,\s*\d{4})?\b/gi;
+const QUARTER_RE = /\bQ[1-4]\s?\d{4}\b/g;
+
+const dedupe = (values: string[]): string[] => Array.from(new Set(values));
+
+export interface ExtractedEntities {
+	emails: string[];
+	urls: string[];
+	dates: string[];
+	amounts: string[];
+}
+
+export const extractEntities = (text: string): ExtractedEntities => {
+	const emails = dedupe(text.match(EMAIL_RE) ?? []);
+	const urls = dedupe(text.match(URL_RE) ?? []);
+	const dates = dedupe([
+		...(text.match(ISO_DATE_RE) ?? []),
+		...(text.match(LONG_DATE_RE) ?? []),
+		...(text.match(QUARTER_RE) ?? []),
+	]);
+	const amounts = dedupe([
+		...(text.match(CURRENCY_RE) ?? []),
+		...(text.match(PERCENT_RE) ?? []),
+	]);
+	return { emails, urls, dates, amounts };
+};
+
+const formatEntities = (entities: ExtractedEntities): string => {
+	const parts: string[] = [];
+	if (entities.emails.length > 0) {
+		parts.push(`Emails: ${entities.emails.join(", ")}`);
+	}
+	if (entities.urls.length > 0) {
+		parts.push(`Links: ${entities.urls.join(", ")}`);
+	}
+	if (entities.dates.length > 0) {
+		parts.push(`Dates: ${entities.dates.join(", ")}`);
+	}
+	if (entities.amounts.length > 0) {
+		parts.push(`Amounts: ${entities.amounts.join(", ")}`);
+	}
+	return parts.join("\n");
+};
+
+export const buildEntityChunks = (
+	text: string,
+	chunkIdFor: (suffix: string) => string,
+): Chunk[] => {
+	const entities = extractEntities(text);
+	const summary = formatEntities(entities);
+	if (summary.length === 0) return [];
+	// Cap at MAX_CHUNKS_PER_TYPE so every produced entities-N has a matching key
+	// in candidateChunkKeys (keys.ts) and stays reapable on delete.
+	const parts = splitToCharBudget(summary, EMBED_CHAR_BUDGET).slice(
+		0,
+		MAX_CHUNKS_PER_TYPE,
+	);
+	return parts.map((part, idx) => ({
+		chunkId: chunkIdFor(parts.length === 1 ? "entities" : `entities-${idx}`),
+		chunkType: "entities",
+		text: part,
+	}));
+};

package/src/chunking/entropy.test.ts

@@ -0,0 +1,98 @@
+import assert from "node:assert";
+import { describe, it } from "node:test";
+import {
+	buildBodyChunks,
+	shannonEntropy,
+	stripBoilerplate,
+} from "./entropy.js";
+
+const idFor = (suffix: string): string => `msg-1::${suffix}`;
+
+describe("shannonEntropy", () => {
+	it("returns zero for an empty string", () => {
+		assert.strictEqual(shannonEntropy(""), 0);
+	});
+
+	it("returns zero for a single repeating character", () => {
+		assert.strictEqual(shannonEntropy("aaaa"), 0);
+	});
+
+	it("returns higher entropy for varied text", () => {
+		const lo = shannonEntropy("aaaaaaaaa");
+		const hi = shannonEntropy("the quick brown fox jumps over the lazy dog");
+		assert.ok(hi > lo);
+	});
+});
+
+describe("stripBoilerplate", () => {
+	it("removes content after a > quoted reply", () => {
+		const input = "Hello bob,\nThis is the new content.\n> quoted reply line";
+		const out = stripBoilerplate(input);
+		assert.ok(!out.includes("quoted reply"));
+		assert.ok(out.includes("new content"));
+	});
+
+	it("removes a 'Best regards alice' signature", () => {
+		const input = "Real body text here.\n\nBest regards,\nAlice";
+		const out = stripBoilerplate(input);
+		assert.ok(!out.includes("Alice"));
+		assert.ok(out.includes("Real body text"));
+	});
+
+	it("removes 'Sent from my iPhone' boilerplate", () => {
+		const input = "Body content.\nSent from my iPhone";
+		const out = stripBoilerplate(input);
+		assert.ok(!out.includes("Sent from my iPhone"));
+	});
+});
+
+describe("buildBodyChunks", () => {
+	it("returns no chunks when the body is just a greeting and signature", () => {
+		const text = "Hi alice,\n\nBest regards,\nBob";
+		const chunks = buildBodyChunks(text, idFor);
+		assert.strictEqual(chunks.length, 0);
+	});
+
+	it("produces chunks from substantive prose", () => {
+		const body = `Hi alice,
+
+I have reviewed the Q1 numbers and the team exceeded the target by fourteen percent.
+The next milestone needs more attention from engineering and the renewal cycle is
+approaching faster than the planning team anticipated. Please confirm by Friday so we
+can schedule the follow-up working session for next week.
+
+Best regards,
+Bob`;
+		const chunks = buildBodyChunks(body, idFor);
+		assert.ok(chunks.length >= 1);
+		assert.ok(chunks[0].chunkId.startsWith("msg-1::body-"));
+		assert.strictEqual(chunks[0].chunkType, "body");
+		assert.ok(!chunks[0].text.includes("Hi alice"));
+		assert.ok(!chunks[0].text.includes("Best regards"));
+		assert.ok(chunks[0].text.includes("Q1 numbers"));
+	});
+
+	it("strips quoted content before chunking", () => {
+		const body = `Substantive answer text covering the renewal discussion in
+some depth so it passes the entropy threshold and minimum length check.
+
+> On Mon, alice wrote:
+> Could you please send the report?`;
+		const chunks = buildBodyChunks(body, idFor);
+		assert.ok(chunks.length >= 1);
+		assert.ok(!chunks[0].text.includes("alice wrote"));
+		assert.ok(!chunks[0].text.includes("send the report"));
+	});
+
+	it("caps individual chunks below the size limit", () => {
+		const repeated = `This paragraph contains enough varied language to clear
+the entropy threshold every time it is repeated, with several distinct words
+and clauses appearing in each iteration to keep entropy high.`;
+		const body = Array.from({ length: 20 }, () => repeated).join("\n\n");
+		const chunks = buildBodyChunks(body, idFor);
+		assert.ok(chunks.length >= 2);
+		for (const chunk of chunks) {
+			assert.ok(chunk.text.length <= 2000);
+		}
+	});
+});