emdash 0.0.0-a → 0.0.1

package/src/search/query.ts

@@ -0,0 +1,396 @@
+/**
+ * Search Query Functions
+ *
+ * Programmatic API for searching content using FTS5.
+ */
+
+import type { Kysely } from "kysely";
+import { sql } from "kysely";
+
+import type { Database } from "../database/types.js";
+import { validateIdentifier } from "../database/validate.js";
+import { getDb } from "../loader.js";
+import { FTSManager } from "./fts-manager.js";
+import type {
+	SearchOptions,
+	CollectionSearchOptions,
+	SearchResult,
+	SearchResponse,
+	SuggestOptions,
+	Suggestion,
+	SearchStats,
+} from "./types.js";
+
+/** Pattern to split on whitespace for query term extraction */
+const WHITESPACE_SPLIT_PATTERN = /\s+/;
+const FTS_OPERATORS_PATTERN = /\b(AND|OR|NOT|NEAR)\b/i;
+const DOUBLE_QUOTE_PATTERN = /"/g;
+
+/**
+ * Search across multiple collections
+ *
+ * Public API that auto-injects the database.
+ *
+ * @param query - Search query (FTS5 syntax supported)
+ * @param options - Search options
+ * @returns Search results with pagination
+ *
+ * @example
+ * ```typescript
+ * import { search } from "emdash";
+ *
+ * const results = await search("hello world", {
+ *   collections: ["posts", "pages"],
+ *   limit: 20
+ * });
+ * ```
+ */
+export async function search(query: string, options: SearchOptions = {}): Promise<SearchResponse> {
+	const db = await getDb();
+	return searchWithDb(db, query, options);
+}
+
+/**
+ * Search across multiple collections (with explicit db)
+ *
+ * @internal Use `search()` in templates. This variant is for admin routes
+ * that already have a database handle.
+ *
+ * @param db - Kysely database instance
+ * @param query - Search query (FTS5 syntax supported)
+ * @param options - Search options
+ * @returns Search results with pagination
+ */
+export async function searchWithDb(
+	db: Kysely<Database>,
+	query: string,
+	options: SearchOptions = {},
+): Promise<SearchResponse> {
+	const ftsManager = new FTSManager(db);
+	const limit = options.limit ?? 20;
+	const status = options.status ?? "published";
+
+	// Get searchable collections
+	let collections = options.collections;
+	if (!collections || collections.length === 0) {
+		collections = await getSearchableCollections(db);
+	}
+
+	if (collections.length === 0) {
+		return { items: [] };
+	}
+
+	// Search each collection and merge results
+	const allResults: SearchResult[] = [];
+
+	for (const collection of collections) {
+		const config = await ftsManager.getSearchConfig(collection);
+		if (!config?.enabled) {
+			continue;
+		}
+
+		const collectionResults = await searchSingleCollection(
+			db,
+			collection,
+			query,
+			{
+				status,
+				locale: options.locale,
+				limit: limit * 2, // Get extra for merging
+			},
+			config.weights,
+		);
+
+		allResults.push(...collectionResults);
+	}
+
+	// Sort by score descending
+	allResults.sort((a, b) => b.score - a.score);
+
+	// Apply limit
+	const items = allResults.slice(0, limit);
+
+	return { items };
+}
+
+/**
+ * Search within a single collection
+ *
+ * @param db - Kysely database instance
+ * @param collection - Collection slug
+ * @param query - Search query (FTS5 syntax supported)
+ * @param options - Search options
+ * @returns Search results with pagination
+ *
+ * @example
+ * ```typescript
+ * const results = await searchCollection(db, "posts", "hello world", {
+ *   limit: 10
+ * });
+ * ```
+ */
+export async function searchCollection(
+	db: Kysely<Database>,
+	collection: string,
+	query: string,
+	options: CollectionSearchOptions = {},
+): Promise<SearchResponse> {
+	const ftsManager = new FTSManager(db);
+	const config = await ftsManager.getSearchConfig(collection);
+
+	if (!config?.enabled) {
+		return { items: [] };
+	}
+
+	const items = await searchSingleCollection(db, collection, query, options, config.weights);
+
+	return { items };
+}
+
+/**
+ * Internal function to search a single collection
+ */
+async function searchSingleCollection(
+	db: Kysely<Database>,
+	collection: string,
+	query: string,
+	options: CollectionSearchOptions,
+	weights?: Record<string, number>,
+): Promise<SearchResult[]> {
+	// Validate before any raw SQL interpolation
+	validateIdentifier(collection, "collection slug");
+
+	const ftsManager = new FTSManager(db);
+	const ftsTable = ftsManager.getFtsTableName(collection);
+	const contentTable = ftsManager.getContentTableName(collection);
+	const limit = options.limit ?? 20;
+	const status = options.status ?? "published";
+	const locale = options.locale;
+
+	// Check if FTS table exists
+	if (!(await ftsManager.ftsTableExists(collection))) {
+		return [];
+	}
+
+	// Escape the query for FTS5
+	const escapedQuery = escapeQuery(query);
+	if (!escapedQuery) {
+		return [];
+	}
+
+	// Get searchable fields for snippet generation
+	const searchableFields = await ftsManager.getSearchableFields(collection);
+
+	// Build weight string for bm25 if weights provided
+	// Format: bm25(table, weight1, weight2, ...)
+	// First two weights are for 'id' and 'locale' columns (UNINDEXED, so 0)
+	let bm25Args = "";
+	if (weights && searchableFields.length > 0) {
+		const weightValues = ["0", "0"]; // id column, locale column
+		for (const field of searchableFields) {
+			weightValues.push(String(weights[field] ?? 1));
+		}
+		bm25Args = weightValues.join(", ");
+	}
+
+	// Build and execute the search query
+	// Using raw SQL because Kysely doesn't have FTS5 support
+	const bm25Expr = bm25Args ? `bm25("${ftsTable}", ${bm25Args})` : `bm25("${ftsTable}")`;
+
+	// Snippet column index is 2 (after id=0, locale=1, first searchable field=2)
+	const results = await sql<{
+		id: string;
+		slug: string | null;
+		locale: string;
+		title: string | null;
+		snippet: string;
+		score: number;
+	}>`
+		SELECT 
+			c.id,
+			c.slug,
+			c.locale,
+			c.title,
+			snippet("${sql.raw(ftsTable)}", 2, '<mark>', '</mark>', '...', 32) as snippet,
+			${sql.raw(bm25Expr)} as score
+		FROM "${sql.raw(ftsTable)}" f
+		JOIN "${sql.raw(contentTable)}" c ON f.id = c.id
+		WHERE "${sql.raw(ftsTable)}" MATCH ${escapedQuery}
+		AND c.status = ${status}
+		AND c.deleted_at IS NULL
+		${locale ? sql`AND c.locale = ${locale}` : sql``}
+		ORDER BY score
+		LIMIT ${limit}
+	`.execute(db);
+
+	return results.rows.map((row) => ({
+		collection,
+		id: row.id,
+		slug: row.slug,
+		locale: row.locale,
+		title: row.title ?? undefined,
+		snippet: row.snippet,
+		score: Math.abs(row.score), // bm25 returns negative scores
+	}));
+}
+
+/**
+ * Get search suggestions for autocomplete
+ *
+ * @param db - Kysely database instance
+ * @param query - Partial search query
+ * @param options - Suggestion options
+ * @returns Array of suggestions
+ */
+export async function getSuggestions(
+	db: Kysely<Database>,
+	query: string,
+	options: SuggestOptions = {},
+): Promise<Suggestion[]> {
+	const limit = options.limit ?? 5;
+	const locale = options.locale;
+
+	// Get searchable collections
+	let collections = options.collections;
+	if (!collections || collections.length === 0) {
+		collections = await getSearchableCollections(db);
+	}
+
+	if (collections.length === 0) {
+		return [];
+	}
+
+	const suggestions: Suggestion[] = [];
+
+	for (const collection of collections) {
+		const ftsManager = new FTSManager(db);
+		const config = await ftsManager.getSearchConfig(collection);
+		if (!config?.enabled) {
+			continue;
+		}
+
+		// Validate before raw SQL interpolation
+		validateIdentifier(collection, "collection slug");
+
+		const ftsTable = ftsManager.getFtsTableName(collection);
+		const contentTable = ftsManager.getContentTableName(collection);
+
+		// Use prefix search for autocomplete
+		const prefixQuery = `${escapeQuery(query)}*`;
+		if (!prefixQuery || prefixQuery === "*") {
+			continue;
+		}
+
+		const results = await sql<{
+			id: string;
+			title: string;
+		}>`
+			SELECT 
+				c.id,
+				c.title
+			FROM "${sql.raw(ftsTable)}" f
+			JOIN "${sql.raw(contentTable)}" c ON f.id = c.id
+			WHERE "${sql.raw(ftsTable)}" MATCH ${prefixQuery}
+			AND c.status = 'published'
+			AND c.deleted_at IS NULL
+			AND c.title IS NOT NULL
+			${locale ? sql`AND c.locale = ${locale}` : sql``}
+			ORDER BY bm25("${sql.raw(ftsTable)}")
+			LIMIT ${limit}
+		`.execute(db);
+
+		for (const row of results.rows) {
+			suggestions.push({
+				collection,
+				id: row.id,
+				title: row.title,
+			});
+		}
+	}
+
+	return suggestions.slice(0, limit);
+}
+
+/**
+ * Get search statistics for all collections
+ */
+export async function getSearchStats(db: Kysely<Database>): Promise<SearchStats> {
+	const ftsManager = new FTSManager(db);
+	const collections = await getSearchableCollections(db);
+	const stats: SearchStats = { collections: {} };
+
+	for (const collection of collections) {
+		const collectionStats = await ftsManager.getIndexStats(collection);
+		if (collectionStats) {
+			stats.collections[collection] = collectionStats;
+		}
+	}
+
+	return stats;
+}
+
+/**
+ * Get list of collections with search enabled
+ */
+async function getSearchableCollections(db: Kysely<Database>): Promise<string[]> {
+	const results = await db
+		.selectFrom("_emdash_collections")
+		.select(["slug", "search_config"])
+		.execute();
+
+	return results
+		.filter((r) => {
+			if (!r.search_config) return false;
+			try {
+				const config = JSON.parse(r.search_config);
+				return config.enabled === true;
+			} catch {
+				return false;
+			}
+		})
+		.map((r) => r.slug);
+}
+
+/**
+ * Escape a query string for FTS5
+ *
+ * Handles special characters and prevents injection.
+ */
+function escapeQuery(query: string): string {
+	if (!query || typeof query !== "string") {
+		return "";
+	}
+
+	// Trim whitespace
+	query = query.trim();
+
+	if (query.length === 0) {
+		return "";
+	}
+
+	// FTS5 special characters that need escaping in terms: " * ^
+	// We'll wrap terms in quotes to handle most cases
+	// But first, escape any existing quotes
+	const escaped = query.replace(DOUBLE_QUOTE_PATTERN, '""');
+
+	// If the query contains FTS5 operators (AND, OR, NOT, NEAR),
+	// pass through as-is (user knows what they're doing)
+	if (FTS_OPERATORS_PATTERN.test(query)) {
+		return escaped;
+	}
+
+	// If already quoted, pass through
+	if (query.startsWith('"') && query.endsWith('"')) {
+		return query;
+	}
+
+	// For simple queries, wrap each word to handle special chars
+	const terms = escaped.split(WHITESPACE_SPLIT_PATTERN).filter((t) => t.length > 0);
+	if (terms.length === 0) {
+		return "";
+	}
+
+	// Join with implicit AND, add prefix matching (*) to all terms
+	// This allows "hel wor" to match "hello world"
+	return terms.map((t) => `"${t}"*`).join(" ");
+}

package/src/search/text-extraction.ts

@@ -0,0 +1,162 @@
+/**
+ * Text Extraction
+ *
+ * Extracts plain text from Portable Text blocks for FTS indexing.
+ * Uses @portabletext/toolkit as base with extensions for custom block types.
+ */
+
+import { toPlainText } from "@portabletext/toolkit";
+
+import type { PortableTextBlock } from "../content/converters/types.js";
+
+/**
+ * Validate that a value looks like a Portable Text block array.
+ * Each element must have at least a `_type` string property.
+ */
+function isPortableTextArray(value: unknown[]): value is PortableTextBlock[] {
+	return value.every(
+		(item) =>
+			typeof item === "object" &&
+			item !== null &&
+			"_type" in item &&
+			typeof item._type === "string",
+	);
+}
+
+/**
+ * Extract additional text from custom block types that toPlainText doesn't handle
+ */
+function extractCustomBlockText(block: PortableTextBlock): string {
+	// Code blocks - include the code content
+	if (block._type === "code" && "code" in block && typeof block.code === "string") {
+		return block.code;
+	}
+
+	// Image blocks - include alt text and caption
+	if (block._type === "image") {
+		const parts: string[] = [];
+		if ("alt" in block && typeof block.alt === "string" && block.alt) {
+			parts.push(block.alt);
+		}
+		if ("caption" in block && typeof block.caption === "string" && block.caption) {
+			parts.push(block.caption);
+		}
+		return parts.join(" ");
+	}
+
+	return "";
+}
+
+/**
+ * Extract plain text from Portable Text blocks
+ *
+ * Uses @portabletext/toolkit's toPlainText for standard blocks,
+ * plus extracts text from custom block types (code, images with alt/caption).
+ *
+ * @param blocks - Array of Portable Text blocks (or a JSON string)
+ * @returns Plain text content
+ *
+ * @example
+ * ```typescript
+ * const text = extractPlainText([
+ *   {
+ *     _type: "block",
+ *     _key: "abc",
+ *     children: [{ _type: "span", _key: "s1", text: "Hello World" }]
+ *   }
+ * ]);
+ * // Returns: "Hello World"
+ * ```
+ */
+export function extractPlainText(blocks: PortableTextBlock[] | string | null | undefined): string {
+	if (!blocks) {
+		return "";
+	}
+
+	// Handle JSON string input
+	let parsedBlocks: PortableTextBlock[];
+	if (typeof blocks === "string") {
+		try {
+			parsedBlocks = JSON.parse(blocks);
+		} catch {
+			// If it's not valid JSON, treat as plain text
+			return blocks;
+		}
+	} else {
+		parsedBlocks = blocks;
+	}
+
+	if (!Array.isArray(parsedBlocks)) {
+		return "";
+	}
+
+	// Use official toPlainText for standard blocks.
+	// toPlainText expects `{ _type: string; [key: string]: any }[]` but our blocks use
+	// `unknown` index sigs. They're structurally compatible at runtime — spread each block
+	// to satisfy the wider index signature without an unsafe cast.
+	const toolkitBlocks = parsedBlocks.map((b) => {
+		const obj: Record<string, unknown> & { _type: string } = { _type: b._type };
+		for (const [key, val] of Object.entries(b)) {
+			obj[key] = val;
+		}
+		return obj;
+	});
+	const standardText = toPlainText(toolkitBlocks);
+
+	// Extract text from custom block types that toPlainText doesn't handle
+	const customTexts = parsedBlocks.map(extractCustomBlockText).filter((text) => text.length > 0);
+
+	// Combine both
+	const allTexts = [standardText, ...customTexts].filter((t) => t.length > 0);
+	return allTexts.join("\n");
+}
+
+/**
+ * Extract searchable text from a content entry
+ *
+ * Extracts text from specified fields, handling both plain text and Portable Text.
+ *
+ * @param entry - Content entry data
+ * @param fields - Field names to extract text from
+ * @returns Object mapping field names to extracted text
+ */
+export function extractSearchableFields(
+	entry: Record<string, unknown>,
+	fields: string[],
+): Record<string, string> {
+	const result: Record<string, string> = {};
+
+	for (const field of fields) {
+		const value = entry[field];
+
+		if (value === null || value === undefined) {
+			result[field] = "";
+			continue;
+		}
+
+		if (typeof value === "string") {
+			// Could be plain text or JSON Portable Text
+			if (value.startsWith("[")) {
+				result[field] = extractPlainText(value);
+			} else {
+				result[field] = value;
+			}
+		} else if (Array.isArray(value)) {
+			// Validate the array looks like Portable Text before treating it as such
+			if (isPortableTextArray(value)) {
+				result[field] = extractPlainText(value);
+			} else {
+				result[field] = JSON.stringify(value);
+			}
+		} else if (typeof value === "object") {
+			// Object — serialize to JSON for searchable text
+			result[field] = JSON.stringify(value);
+		} else if (typeof value === "number" || typeof value === "boolean") {
+			result[field] = `${value}`;
+		} else {
+			result[field] = "";
+		}
+	}
+
+	return result;
+}

package/src/search/types.ts

@@ -0,0 +1,114 @@
+/**
+ * Search Types
+ *
+ * Type definitions for the EmDash search system.
+ */
+
+/**
+ * Search configuration for a collection
+ */
+export interface SearchConfig {
+	/** Whether search is enabled for this collection */
+	enabled: boolean;
+	/** Field weights for ranking (higher = more important) */
+	weights?: Record<string, number>;
+}
+
+/**
+ * Options for search queries
+ */
+export interface SearchOptions {
+	/** Collections to search (defaults to all searchable collections) */
+	collections?: string[];
+	/** Filter by content status (defaults to 'published') */
+	status?: string;
+	/** Filter by locale (omit to search all locales) */
+	locale?: string;
+	/** Maximum results to return (defaults to 20) */
+	limit?: number;
+	/** Pagination cursor */
+	cursor?: string;
+}
+
+/**
+ * Options for collection-specific search
+ */
+export interface CollectionSearchOptions {
+	/** Filter by content status (defaults to 'published') */
+	status?: string;
+	/** Filter by locale (omit to search all locales) */
+	locale?: string;
+	/** Maximum results to return (defaults to 20) */
+	limit?: number;
+	/** Pagination cursor */
+	cursor?: string;
+}
+
+/**
+ * A single search result
+ */
+export interface SearchResult {
+	/** Collection the result belongs to */
+	collection: string;
+	/** Entry ID */
+	id: string;
+	/** Entry slug */
+	slug: string | null;
+	/** Content locale */
+	locale: string;
+	/** Entry title (if available) */
+	title?: string;
+	/** Highlighted snippet showing match context */
+	snippet?: string;
+	/** Relevance score (higher = more relevant) */
+	score: number;
+}
+
+/**
+ * Response from a search query
+ */
+export interface SearchResponse {
+	/** Search results */
+	items: SearchResult[];
+	/** Cursor for next page of results */
+	nextCursor?: string;
+}
+
+/**
+ * Options for suggestion/autocomplete queries
+ */
+export interface SuggestOptions {
+	/** Collections to search (defaults to all searchable collections) */
+	collections?: string[];
+	/** Filter by locale (omit to search all locales) */
+	locale?: string;
+	/** Maximum suggestions to return (defaults to 5) */
+	limit?: number;
+}
+
+/**
+ * A single suggestion result
+ */
+export interface Suggestion {
+	/** Collection the suggestion belongs to */
+	collection: string;
+	/** Entry ID */
+	id: string;
+	/** Entry title */
+	title: string;
+}
+
+/**
+ * Search index statistics
+ */
+export interface SearchStats {
+	collections: Record<
+		string,
+		{
+			/** Number of indexed entries */
+			indexed: number;
+			/** When the index was last rebuilt */
+			lastRebuilt?: string;
+		}
+	>;
+}