emdash 0.32.0 → 0.33.0

package/src/query.ts

@@ -29,6 +29,7 @@ import { getFallbackChain, getI18nConfig, isI18nEnabled } from "./i18n/config.js
 import {
 	CURSOR_RAW_VALUES,
 	FOLDED_BYLINES,
+	FOLDED_BYLINES_EXIST,
 	FOLDED_TERMS,
 	type WhereRange,
 	type WhereValue,
@@ -310,17 +311,6 @@ function dataStr(data: Record<string, unknown>, key: string, fallback = ""): str
 	return typeof val === "string" ? val : fallback;
 }
 
-/** Safely read a date-like field from a Record */
-function dataDate(data: Record<string, unknown>, key: string): Date | undefined {
-	const val = data[key];
-	if (val instanceof Date) {
-		return Number.isNaN(val.getTime()) ? undefined : val;
-	}
-	if (typeof val !== "string" && typeof val !== "number") return undefined;
-	const date = new Date(val);
-	return Number.isNaN(date.getTime()) ? undefined : date;
-}
-
 /** Type guard for Record<string, unknown> */
 function isRecord(value: unknown): value is Record<string, unknown> {
 	return typeof value === "object" && value !== null && !Array.isArray(value);
@@ -828,23 +818,9 @@ export async function getEmDashEntry<T extends string, D = InferCollectionData<T
 		};
 	}
 
-	/** Check if an entry is publicly visible (published or scheduled past its time) */
+	/** Check if an entry has completed publication. */
 	function isVisible(entry: ContentEntry<D>): boolean {
-		const data = entryData(entry);
-		const status = dataStr(data, "status");
-		const scheduledAt = dataDate(data, "scheduledAt");
-		const isPublished = status === "published";
-		const isScheduledAndReady =
-			status === "scheduled" && scheduledAt !== undefined && scheduledAt.getTime() <= Date.now();
-		return isPublished || !!isScheduledAndReady;
-	}
-
-	/** True when an entry is scheduled to become visible at a future time. */
-	function isPendingScheduled(entry: ContentEntry<D>): boolean {
-		const data = entryData(entry);
-		if (dataStr(data, "status") !== "scheduled") return false;
-		const scheduledAt = dataDate(data, "scheduledAt");
-		return scheduledAt !== undefined && scheduledAt.getTime() > Date.now();
+		return dataStr(entryData(entry), "status") === "published";
 	}
 
 	// Build the fallback chain: [requestedLocale, fallback1, ..., defaultLocale]
@@ -954,11 +930,6 @@ export async function getEmDashEntry<T extends string, D = InferCollectionData<T
 	// hydration) is wrapped in the distributed L2 cache, keyed by the requested
 	// locale. Preview/edit requests took the `serveDrafts` branch above and
 	// never reach here; the object cache additionally bypasses them.
-	// A scheduled entry becomes visible on a future clock tick, not on a write,
-	// so an L2 snapshot taken before its time would keep it hidden past go-live
-	// (until the publish sweep bumps the epoch or the TTL lapses). Mark such a
-	// resolution time-sensitive and skip caching it.
-	let timeSensitive = false;
 
 	const resolveNormal = async (): Promise<EntryResult<D>> => {
 		for (let i = 0; i < localeChain.length; i++) {
@@ -977,9 +948,6 @@ export async function getEmDashEntry<T extends string, D = InferCollectionData<T
 					cacheHint: cacheHint ?? {},
 				});
 			}
-			if (entry && isPendingScheduled(entry)) {
-				timeSensitive = true;
-			}
 			// Entry not found or not visible in this locale — try next
 		}
 		return { entry: null, isPreview: false, cacheHint: {} };
@@ -1003,7 +971,7 @@ export async function getEmDashEntry<T extends string, D = InferCollectionData<T
 				},
 			};
 		},
-		cacheable: (snap) => snap.ok && !timeSensitive,
+		cacheable: (snap) => snap.ok,
 	});
 
 	if (!snapshot.ok) {
@@ -1063,18 +1031,29 @@ async function hydrateEntryBylines<D>(type: string, entries: ContentEntry<D>[]):
 			return { data, credits };
 		});
 
+		// An empty `_emdash_bylines` table makes an empty fold authoritative: no
+		// credit can exist in any locale and the author fallback has no byline
+		// to resolve to, so skip the query path and the custom-fields probe
+		// entirely. A missing marker (older cached rows) means "unknown" and
+		// keeps the conservative fallback below.
+		const knownEmpty = entries.every(
+			(e) => Reflect.get(entryData(e), FOLDED_BYLINES_EXIST) === false,
+		);
+
 		// Fall back to the full query path when the fold can't be trusted to be
 		// complete: an entry with a byline reference (explicit primary, or an
 		// author for the author-fallback) but no folded credits — e.g. a credit
 		// in a different locale than the row, which the locale-correlated subquery
 		// skips, or the author-fallback path which the fold doesn't express.
-		let needsQueryPath = parsed.some(
-			(p) =>
-				p.credits.length === 0 &&
-				(dataStr(p.data, "authorId") !== "" || dataStr(p.data, "primaryBylineId") !== ""),
-		);
+		let needsQueryPath =
+			!knownEmpty &&
+			parsed.some(
+				(p) =>
+					p.credits.length === 0 &&
+					(dataStr(p.data, "authorId") !== "" || dataStr(p.data, "primaryBylineId") !== ""),
+			);
 		let hasCustomFields = false;
-		if (!needsQueryPath) {
+		if (!needsQueryPath && !knownEmpty) {
 			try {
 				const { getDb } = await import("./loader.js");
 				const db = await getDb();

package/src/redirects/cache.ts

@@ -1,19 +1,27 @@
 /**
  * Redirect rule cache.
  *
- * Module-level cache for enabled redirect rules. The middleware populates this
- * on first request; route handlers invalidate it on writes.
+ * Worker-isolate cache for enabled redirect rules. The middleware populates
+ * this on first request; route handlers invalidate it on writes. Cached rules
+ * expire so writes handled by another isolate become visible here too.
  *
  * Both exact-match and pattern rules are loaded from one query and cached
  * together: exact rules indexed by source path in a Map, pattern rules
  * pre-compiled into an array. A single warm request issues zero database
- * queries; a cold isolate issues one.
+ * queries; a cold or expired isolate issues one.
  *
  * This module deliberately has NO Astro imports so it can be safely imported
  * from handlers, seed, CLI, and tests without dragging in `astro:middleware`.
  */
 
+import { after } from "../after.js";
 import type { Redirect } from "../database/repositories/redirect.js";
+import {
+	createSingleFlightCache,
+	type SingleFlightCache,
+	invalidateSingleFlightCache,
+	singleFlightCached,
+} from "../utils/single-flight-cache.js";
 import type { CompiledPattern } from "./patterns.js";
 import { compilePattern, interpolateDestination, matchPattern } from "./patterns.js";
 
@@ -29,33 +37,54 @@ export interface CachedRedirects {
 	patterns: CachedRedirectRule[];
 }
 
-/**
- * Cached enabled redirects.
- * null = not yet populated, object = cached.
- */
-let cachedRedirects: CachedRedirects | null = null;
+interface RedirectCacheState {
+	redirects: CachedRedirects | null;
+	expiresAt: number;
+	generation: number;
+	refresh: SingleFlightCache<CachedRedirects>;
+}
+
+const REDIRECT_CACHE_TTL_MS = 30_000;
+const REDIRECT_CACHE_MAX_REFRESH_ATTEMPTS = 3;
+const REDIRECT_CACHE_KEY = Symbol.for("emdash:redirect-cache");
+const g = globalThis as Record<symbol, unknown>;
+const cacheState: RedirectCacheState =
+	// eslint-disable-next-line typescript/no-unsafe-type-assertion -- globalThis singleton pattern (see request-context.ts)
+	(g[REDIRECT_CACHE_KEY] as RedirectCacheState | undefined) ??
+	(() => {
+		const state: RedirectCacheState = {
+			redirects: null,
+			expiresAt: 0,
+			generation: 0,
+			refresh: createSingleFlightCache<CachedRedirects>(),
+		};
+		g[REDIRECT_CACHE_KEY] = state;
+		return state;
+	})();
 
 /**
  * Invalidate the cached redirects (both exact and pattern).
  * Call when redirects are created, updated, or deleted.
  */
 export function invalidateRedirectCache(): void {
-	cachedRedirects = null;
+	cacheState.generation++;
+	cacheState.redirects = null;
+	cacheState.expiresAt = 0;
+	invalidateSingleFlightCache(cacheState.refresh);
 }
 
 /**
  * Get the cached redirects, or null if the cache is cold.
  */
-export function getCachedRedirects(): CachedRedirects | null {
-	return cachedRedirects;
+function getCachedRedirects(): CachedRedirects | null {
+	if (cacheState.redirects && Date.now() >= cacheState.expiresAt) {
+		invalidateRedirectCache();
+	}
+	return cacheState.redirects;
 }
 
-/**
- * Populate the cache from a list of enabled redirects (both exact and
- * pattern). The caller is responsible for passing only enabled rows — the
- * cache stores them as-is.
- */
-export function setCachedRedirects(redirects: Redirect[]): CachedRedirects {
+/** Compile enabled database rows into the in-memory lookup structures. */
+function compileRedirects(redirects: Redirect[]): CachedRedirects {
 	const exact = new Map<string, Redirect>();
 	const patterns: CachedRedirectRule[] = [];
 	for (const r of redirects) {
@@ -65,8 +94,39 @@ export function setCachedRedirects(redirects: Redirect[]): CachedRedirects {
 			exact.set(r.source, r);
 		}
 	}
-	cachedRedirects = { exact, patterns };
-	return cachedRedirects;
+	return { exact, patterns };
+}
+
+function installCachedRedirects(redirects: CachedRedirects): CachedRedirects {
+	cacheState.redirects = redirects;
+	cacheState.expiresAt = Date.now() + REDIRECT_CACHE_TTL_MS;
+	return cacheState.redirects;
+}
+
+export async function loadCachedRedirects(
+	load: () => Promise<Redirect[]>,
+): Promise<CachedRedirects> {
+	for (let attempt = 0; attempt < REDIRECT_CACHE_MAX_REFRESH_ATTEMPTS; attempt++) {
+		const cached = getCachedRedirects();
+		if (cached) return cached;
+
+		const generation = cacheState.generation;
+		const loaded = await singleFlightCached(
+			cacheState.refresh,
+			async () => compileRedirects(await load()),
+			{ anchor: (promise) => after(() => promise), ownerTimeoutMs: 30_000 },
+		);
+
+		if (generation === cacheState.generation) {
+			return installCachedRedirects(loaded);
+		}
+
+		if (attempt === REDIRECT_CACHE_MAX_REFRESH_ATTEMPTS - 1) {
+			return loaded;
+		}
+	}
+
+	throw new Error("Redirect cache refresh exhausted without loading rules");
 }
 
 /**

package/src/request-context.ts

@@ -20,6 +20,7 @@
 import { AsyncLocalStorage } from "node:async_hooks";
 
 import type { QueryRecorder } from "./database/instrumentation.js";
+import type { DeferredTaskTracker } from "./deferred-tasks.js";
 
 /**
  * Lightweight always-on counters surfaced in Server-Timing.
@@ -102,6 +103,8 @@ export interface EmDashRequestContext {
 	 * and `requestCached`.
 	 */
 	metrics?: RequestMetrics;
+	/** Deferred work that owns resources scoped to this request. */
+	deferredTasks?: DeferredTaskTracker;
 }
 
 const ALS_KEY = Symbol.for("emdash:request-context");

package/src/scheduled-publish.ts

@@ -40,7 +40,11 @@ export const SCHEDULED_PUBLISH_BATCH_LIMIT = 100;
 export type ScheduledPublishFn = (
 	collection: string,
 	id: string,
-	options: { publishedAt?: string; requireScheduledDue?: boolean },
+	options: {
+		publishedAt?: string;
+		requireScheduledDue?: boolean;
+		expectedScheduledAt?: string;
+	},
 ) => Promise<{ success: boolean; error?: { code?: string } }>;
 
 export interface PublishDueContentOptions {
@@ -114,6 +118,7 @@ export async function publishDueContent(
 				const result = await doPublish(collection.slug, item.id, {
 					publishedAt,
 					requireScheduledDue: true,
+					expectedScheduledAt: item.scheduledAt ?? undefined,
 				});
 				if (result.success) {
 					batch.push({ collection: collection.slug, id: item.id });

package/src/schema/registry.ts

@@ -73,6 +73,19 @@ const VALID_COLLECTION_SUPPORTS: ReadonlySet<string> = new Set<CollectionSupport
 // multi-row INSERT below D1's 100-parameter statement limit.
 const SEED_FIELD_INSERT_BATCH_SIZE = 6;
 
+/**
+ * Rank given to collections without an explicit `sort_order`. SQLite sorts
+ * NULL first on ASC while Postgres sorts it last, so the fallback is
+ * materialised with COALESCE rather than left to the dialect.
+ */
+const UNORDERED_COLLECTION_RANK = 2147483647;
+
+/**
+ * Collection ordering shared by every list read: explicit `sort_order`
+ * first (ascending), then alphabetically by slug.
+ */
+const collectionOrder = sql<number>`coalesce(sort_order, ${sql.lit(UNORDERED_COLLECTION_RANK)})`;
+
 function isCollectionSupport(value: unknown): value is CollectionSupport {
 	return typeof value === "string" && VALID_COLLECTION_SUPPORTS.has(value);
 }
@@ -126,6 +139,7 @@ export class SchemaRegistry {
 		const rows = await this.db
 			.selectFrom("_emdash_collections")
 			.selectAll()
+			.orderBy(collectionOrder, "asc")
 			.orderBy("slug", "asc")
 			.execute();
 
@@ -177,6 +191,7 @@ export class SchemaRegistry {
 		const collectionRows = await this.db
 			.selectFrom("_emdash_collections")
 			.selectAll()
+			.orderBy(collectionOrder, "asc")
 			.orderBy("slug", "asc")
 			.execute();
 
@@ -256,6 +271,8 @@ export class SchemaRegistry {
 					supports: JSON.stringify(supports),
 					source: input.source ?? "manual",
 					has_seo: hasSeo ? 1 : 0,
+					hidden: input.hidden ? 1 : 0,
+					sort_order: input.sortOrder ?? null,
 					comments_enabled: input.commentsEnabled ? 1 : 0,
 					url_pattern: input.urlPattern ?? null,
 				})
@@ -354,6 +371,8 @@ export class SchemaRegistry {
 						supports: JSON.stringify(supports),
 						source: "seed",
 						has_seo: hasSeo ? 1 : 0,
+						hidden: input.hidden ? 1 : 0,
+						sort_order: input.sortOrder ?? null,
 						comments_enabled: input.commentsEnabled ? 1 : 0,
 						url_pattern: input.urlPattern ?? null,
 					})
@@ -416,6 +435,9 @@ export class SchemaRegistry {
 							? (input.urlPattern ?? null)
 							: (existing.urlPattern ?? null),
 					has_seo: hasSeo ? 1 : 0,
+					hidden: input.hidden !== undefined ? (input.hidden ? 1 : 0) : existing.hidden ? 1 : 0,
+					sort_order:
+						input.sortOrder !== undefined ? input.sortOrder : (existing.sortOrder ?? null),
 					comments_enabled:
 						input.commentsEnabled !== undefined
 							? input.commentsEnabled
@@ -822,7 +844,12 @@ export class SchemaRegistry {
 		const ftsActive = config?.enabled === true;
 
 		if (wantsSearch && searchableFields.length > 0 && ftsActive) {
-			await ftsManager.rebuildIndex(collectionSlug, searchableFields, config?.weights);
+			await ftsManager.rebuildIndex(
+				collectionSlug,
+				searchableFields,
+				config?.weights,
+				config?.tokenize,
+			);
 		} else if (ftsActive && (!wantsSearch || searchableFields.length === 0)) {
 			await ftsManager.disableSearch(collectionSlug);
 		}
@@ -876,6 +903,60 @@ export class SchemaRegistry {
 		}
 	}
 
+	/**
+	 * Reorder collections in the admin sidebar.
+	 *
+	 * `slugs` is the full desired order: every listed collection gets its
+	 * index as `sort_order`, and any collection left out has its explicit
+	 * position cleared, dropping it back to the alphabetical tail. Unknown or
+	 * duplicate slugs throw before anything is written.
+	 */
+	async reorderCollections(slugs: string[]): Promise<void> {
+		const known = new Set((await this.listCollections()).map((collection) => collection.slug));
+
+		const unknown = slugs.filter((slug) => !known.has(slug));
+		if (unknown.length > 0) {
+			throw new SchemaError(
+				`Unknown collection${unknown.length > 1 ? "s" : ""}: ${unknown.join(", ")}`,
+				"COLLECTION_NOT_FOUND",
+				{ slugs: unknown },
+			);
+		}
+
+		const duplicates = slugs.filter((slug, index) => slugs.indexOf(slug) !== index);
+		if (duplicates.length > 0) {
+			throw new SchemaError(
+				`Duplicate collection${duplicates.length > 1 ? "s" : ""}: ${[...new Set(duplicates)].join(", ")}`,
+				"DUPLICATE_SLUG",
+				{ slugs: [...new Set(duplicates)] },
+			);
+		}
+
+		const now = new Date().toISOString();
+		const ordered = new Set(slugs);
+
+		await withTransaction(this.db, async (trx) => {
+			for (const [index, slug] of slugs.entries()) {
+				await trx
+					.updateTable("_emdash_collections")
+					.set({ sort_order: index, updated_at: now })
+					.where("slug", "=", slug)
+					.execute();
+			}
+
+			const cleared = [...known].filter((slug) => !ordered.has(slug));
+			// Chunked to stay under D1's bound-parameter limit; typical sites
+			// clear far fewer than one chunk.
+			for (const slugChunk of chunks(cleared, SQL_BATCH_SIZE)) {
+				await trx
+					.updateTable("_emdash_collections")
+					.set({ sort_order: null, updated_at: now })
+					.where("slug", "in", slugChunk)
+					.execute();
+			}
+		});
+	}
+
 	/**
 	 * Reorder fields
 	 */
@@ -984,9 +1065,18 @@ export class SchemaRegistry {
 			ON ${sql.ref(tableName)} (locale)
 		`.execute(conn);
 
+		// Names must stay identical to migration 055, which creates these indexes
+		// on tables that already exist. Lookups that don't constrain `deleted_at`
+		// (menu and reference resolution) need the first; reads that do need the
+		// second.
+		await sql`
+			CREATE INDEX ${sql.ref(`idx_${tableName}_tg_locale`)}
+			ON ${sql.ref(tableName)} (translation_group, locale)
+		`.execute(conn);
+
 		await sql`
-			CREATE INDEX ${sql.ref(`idx_${tableName}_translation_group`)}
-			ON ${sql.ref(tableName)} (translation_group)
+			CREATE INDEX ${sql.ref(`idx_${tableName}_del_tg_locale`)}
+			ON ${sql.ref(tableName)} (deleted_at, translation_group, locale)
 		`.execute(conn);
 
 		// Composite indexes for optimized query performance (see migration 033)
@@ -1228,6 +1318,8 @@ export class SchemaRegistry {
 			source: row.source && isCollectionSource(row.source) ? row.source : undefined,
 			hasSeo: row.has_seo === 1,
 			urlPattern: row.url_pattern ?? undefined,
+			hidden: row.hidden === 1,
+			sortOrder: row.sort_order ?? undefined,
 			commentsEnabled: row.comments_enabled === 1,
 			commentsModeration:
 				moderation === "all" || moderation === "first_time" || moderation === "none"

package/src/schema/types.ts

@@ -171,6 +171,19 @@ export interface Collection {
 	hasSeo: boolean;
 	/** URL pattern with {slug} placeholder (e.g. "/{slug}", "/blog/{slug}") */
 	urlPattern?: string;
+	/**
+	 * Omit this collection's auto-generated entry from the admin sidebar.
+	 * The collection stays fully functional everywhere else (API, MCP, hooks,
+	 * direct `/content/:collection` URLs) — this only hides the nav link, so a
+	 * plugin that owns the collection can point editors at its own admin UI.
+	 */
+	hidden: boolean;
+	/**
+	 * Explicit position in the admin sidebar. Collections with a `sortOrder`
+	 * come first, in ascending order; the rest keep the alphabetical-by-slug
+	 * order and follow. `undefined` means "no explicit position".
+	 */
+	sortOrder?: number;
 	/** Whether comments are enabled for this collection */
 	commentsEnabled: boolean;
 	/** Moderation strategy: "all" | "first_time" | "none" */
@@ -219,6 +232,10 @@ export interface CreateCollectionInput {
 	source?: CollectionSource;
 	urlPattern?: string;
 	hasSeo?: boolean;
+	/** Omit the auto-generated admin sidebar entry (defaults to false) */
+	hidden?: boolean;
+	/** Explicit admin sidebar position (omit for the alphabetical fallback) */
+	sortOrder?: number | null;
 	commentsEnabled?: boolean;
 }
 
@@ -233,6 +250,10 @@ export interface UpdateCollectionInput {
 	supports?: CollectionSupport[];
 	urlPattern?: string;
 	hasSeo?: boolean;
+	/** Omit the auto-generated admin sidebar entry */
+	hidden?: boolean;
+	/** Explicit admin sidebar position; `null` clears it back to alphabetical */
+	sortOrder?: number | null;
 	commentsEnabled?: boolean;
 	commentsModeration?: "all" | "first_time" | "none";
 	commentsClosedAfterDays?: number;
@@ -331,6 +352,9 @@ export const RESERVED_COLLECTION_SLUGS = [
 	"taxonomies",
 	"options",
 	"audit_logs",
+	// Shadowed by the static POST /schema/collections/reorder route: a
+	// collection with this slug could never be addressed at its own URL.
+	"reorder",
 ];
 
 /**

package/src/search/fts-manager.ts

@@ -10,7 +10,22 @@ import { sql } from "kysely";
 import { isSqlite, tableExists as dialectTableExists } from "../database/dialect-helpers.js";
 import type { Database } from "../database/types.js";
 import { validateIdentifier } from "../database/validate.js";
-import type { SearchConfig } from "./types.js";
+import { SEARCH_TOKENIZERS } from "./types.js";
+import type { SearchConfig, SearchTokenizer } from "./types.js";
+
+const DEFAULT_SEARCH_TOKENIZER: SearchTokenizer = "porter unicode61";
+
+function isSearchTokenizer(value: unknown): value is SearchTokenizer {
+	return SEARCH_TOKENIZERS.some((tokenizer) => tokenizer === value);
+}
+
+function resolveSearchTokenizer(tokenize?: SearchTokenizer): SearchTokenizer {
+	if (tokenize === undefined) return DEFAULT_SEARCH_TOKENIZER;
+	if (!isSearchTokenizer(tokenize)) {
+		throw new Error(`Unsupported FTS5 tokenizer: "${String(tokenize)}"`);
+	}
+	return tokenize;
+}
 
 /**
  * FTS5 Manager
@@ -66,12 +81,15 @@ export class FTSManager {
 	 * @param collectionSlug - The collection slug
 	 * @param searchableFields - Array of field names to index
 	 * @param weights - Optional field weights for ranking
+	 * @param tokenize - Optional FTS5 tokenizer configuration
 	 */
 	async createFtsTable(
 		collectionSlug: string,
 		searchableFields: string[],
 		_weights?: Record<string, number>,
+		tokenize?: SearchTokenizer,
 	): Promise<void> {
+		const tokenizer = resolveSearchTokenizer(tokenize);
 		if (!isSqlite(this.db)) return;
 		this.validateInputs(collectionSlug, searchableFields);
 		const ftsTable = this.getFtsTableName(collectionSlug);
@@ -88,14 +106,13 @@ export class FTSManager {
 		// `createTriggers` keep the index in sync; they MUST use the
 		// external-content-safe `'delete'` command (see notes there) to
 		// avoid `SQLITE_CORRUPT_VTAB` on UPDATE/DELETE.
-		// tokenize='porter unicode61' enables stemming (run matches running, ran, etc.)
 		await sql
 			.raw(`
 			CREATE VIRTUAL TABLE IF NOT EXISTS "${ftsTable}" USING fts5(
 				${columns},
 				content='${contentTable}',
 				content_rowid='rowid',
-				tokenize='porter unicode61'
+				tokenize='${tokenizer}'
 			)
 		`)
 			.execute(this.db);
@@ -247,13 +264,15 @@ export class FTSManager {
 		collectionSlug: string,
 		searchableFields: string[],
 		weights?: Record<string, number>,
+		tokenize?: SearchTokenizer,
 	): Promise<void> {
+		resolveSearchTokenizer(tokenize);
 		if (!isSqlite(this.db)) return;
 		// Drop existing table and triggers
 		await this.dropFtsTable(collectionSlug);
 
 		// Recreate table and triggers
-		await this.createFtsTable(collectionSlug, searchableFields, weights);
+		await this.createFtsTable(collectionSlug, searchableFields, weights, tokenize);
 
 		// Populate from existing content
 		await this.populateFromContent(collectionSlug, searchableFields);
@@ -314,6 +333,12 @@ export class FTSManager {
 				}
 				config.weights = weights;
 			}
+			if ("tokenize" in parsed) {
+				if (!isSearchTokenizer(parsed.tokenize)) {
+					return null;
+				}
+				config.tokenize = parsed.tokenize;
+			}
 			return config;
 		} catch {
 			return null;
@@ -324,6 +349,9 @@ export class FTSManager {
 	 * Update the search configuration for a collection
 	 */
 	async setSearchConfig(collectionSlug: string, config: SearchConfig): Promise<void> {
+		if (config.tokenize !== undefined) {
+			resolveSearchTokenizer(config.tokenize);
+		}
 		await this.db
 			.updateTable("_emdash_collections")
 			.set({ search_config: JSON.stringify(config) })
@@ -399,8 +427,11 @@ export class FTSManager {
 	 */
 	async enableSearch(
 		collectionSlug: string,
-		options?: { weights?: Record<string, number> },
+		options?: { weights?: Record<string, number>; tokenize?: SearchTokenizer },
 	): Promise<void> {
+		if (options?.tokenize !== undefined) {
+			resolveSearchTokenizer(options.tokenize);
+		}
 		if (!isSqlite(this.db)) {
 			throw new Error("Full-text search is only available with SQLite databases");
 		}
@@ -414,13 +445,18 @@ export class FTSManager {
 			);
 		}
 
+		const existing = await this.getSearchConfig(collectionSlug);
+		const weights = options?.weights ?? existing?.weights;
+		const tokenize = options?.tokenize ?? existing?.tokenize;
+
 		// Rebuild from scratch to ensure clean state (no duplicate rows)
-		await this.rebuildIndex(collectionSlug, searchableFields, options?.weights);
+		await this.rebuildIndex(collectionSlug, searchableFields, weights, tokenize);
 
 		// Update search config
 		await this.setSearchConfig(collectionSlug, {
 			enabled: true,
-			weights: options?.weights,
+			weights,
+			tokenize,
 		});
 	}
 
@@ -433,7 +469,11 @@ export class FTSManager {
 		if (!isSqlite(this.db)) return;
 		await this.dropFtsTable(collectionSlug);
 		const existing = await this.getSearchConfig(collectionSlug);
-		await this.setSearchConfig(collectionSlug, { enabled: false, weights: existing?.weights });
+		await this.setSearchConfig(collectionSlug, {
+			enabled: false,
+			weights: existing?.weights,
+			tokenize: existing?.tokenize,
+		});
 	}
 
 	/**
@@ -490,7 +530,7 @@ export class FTSManager {
 			}
 
 			console.warn(`FTS index for "${collectionSlug}" is missing. Rebuilding.`);
-			await this.rebuildIndex(collectionSlug, fields, config.weights);
+			await this.rebuildIndex(collectionSlug, fields, config.weights, config.tokenize);
 			return true;
 		}
 
@@ -515,7 +555,7 @@ export class FTSManager {
 				`FTS index for "${collectionSlug}" has ${ftsRows} rows but content table has ${contentRows}. Rebuilding.`,
 			);
 			if (fields.length > 0) {
-				await this.rebuildIndex(collectionSlug, fields, config?.weights);
+				await this.rebuildIndex(collectionSlug, fields, config?.weights, config?.tokenize);
 			}
 			return true;
 		}

package/src/search/index.ts

@@ -7,6 +7,7 @@
 // Types
 export type {
 	SearchConfig,
+	SearchTokenizer,
 	SearchOptions,
 	CollectionSearchOptions,
 	SearchResult,
@@ -15,6 +16,7 @@ export type {
 	Suggestion,
 	SearchStats,
 } from "./types.js";
+export { SEARCH_TOKENIZERS } from "./types.js";
 
 // FTS Manager
 export { FTSManager } from "./fts-manager.js";