emdash 0.0.0-b → 0.0.2

package/src/plugins/storage-indexes.ts

@@ -0,0 +1,326 @@
+/**
+ * Plugin Storage Index Management
+ *
+ * Manages expression indexes on the _plugin_storage table for efficient queries.
+ *
+ * @see PLUGIN-SYSTEM.md § Plugin Storage > Index Management
+ */
+
+import type { Kysely, RawBuilder } from "kysely";
+import { sql } from "kysely";
+
+import { jsonExtractExpr, isPostgres } from "../database/dialect-helpers.js";
+import type { Database } from "../database/types.js";
+import {
+	validateIdentifier,
+	validateJsonFieldName,
+	validatePluginIdentifier,
+} from "../database/validate.js";
+
+/**
+ * Generate a deterministic index name.
+ * Unique indexes use a `uidx_` prefix to avoid collisions with regular indexes on the same fields.
+ */
+export function generateIndexName(
+	pluginId: string,
+	collection: string,
+	fields: string[],
+	options?: { unique?: boolean },
+): string {
+	const prefix = options?.unique ? "uidx" : "idx";
+	const fieldPart = fields.join("_");
+	// SQLite index names have no length limit, but keep it reasonable
+	return `${prefix}_plugin_${pluginId}_${collection}_${fieldPart}`.substring(0, 128);
+}
+
+/**
+ * Generate a Kysely sql expression for creating an expression index.
+ *
+ * Validates all identifiers before interpolation to prevent SQL injection.
+ * Plugin ID and collection values are parameterized in the WHERE clause.
+ */
+// eslint-disable-next-line @typescript-eslint/no-explicit-any -- accepts any Kysely instance
+export function generateCreateIndexSql(
+	db: Kysely<any>,
+	pluginId: string,
+	collection: string,
+	fields: string[],
+	options?: { unique?: boolean },
+): RawBuilder<unknown> {
+	// Validate all identifiers
+	validatePluginIdentifier(pluginId, "plugin ID");
+	validateIdentifier(collection, "collection name");
+	for (const field of fields) {
+		validateJsonFieldName(field, "index field name");
+	}
+
+	const indexName = generateIndexName(pluginId, collection, fields, options);
+
+	// Build the indexed expressions
+	// Fields are validated above, safe to interpolate into json path
+	const expressions = fields
+		.map((field) => {
+			if (isPostgres(db)) {
+				// Postgres expression indexes need parens around the expression
+				return `(${jsonExtractExpr(db, "data", field)})`;
+			}
+			return jsonExtractExpr(db, "data", field);
+		})
+		.join(", ");
+
+	// Partial index filtered to this plugin/collection
+	// SQLite prohibits bound parameters in partial index WHERE clauses,
+	// so we use sql.lit() for literal string values. Both pluginId and
+	// collection are validated above, so this is safe.
+	const createKeyword = options?.unique ? "CREATE UNIQUE INDEX" : "CREATE INDEX";
+	return sql`${sql.raw(createKeyword)} IF NOT EXISTS ${sql.ref(indexName)}
+		ON _plugin_storage(${sql.raw(expressions)})
+		WHERE plugin_id = ${sql.lit(pluginId)} AND collection = ${sql.lit(collection)}
+	`;
+}
+
+/**
+ * Generate a Kysely sql expression for dropping an index.
+ *
+ * Uses sql.ref() for safe identifier quoting.
+ */
+export function generateDropIndexSql(indexName: string): RawBuilder<unknown> {
+	return sql`DROP INDEX IF EXISTS ${sql.ref(indexName)}`;
+}
+
+/**
+ * Normalize index declarations to arrays of field arrays
+ */
+export function normalizeIndexes(indexes: Array<string | string[]>): string[][] {
+	return indexes.map((index) => (Array.isArray(index) ? index : [index]));
+}
+
+/**
+ * Create all declared indexes for a plugin collection
+ */
+export async function createStorageIndexes(
+	db: Kysely<Database>,
+	pluginId: string,
+	collection: string,
+	indexes: Array<string | string[]>,
+	options?: { uniqueIndexes?: Array<string | string[]> },
+): Promise<{
+	created: string[];
+	errors: Array<{ index: string; error: string }>;
+}> {
+	const normalized = normalizeIndexes(indexes);
+	const uniqueNormalized = options?.uniqueIndexes ? normalizeIndexes(options.uniqueIndexes) : [];
+	const uniqueSet = new Set(uniqueNormalized.map((f) => f.join(",")));
+
+	// Deduplicate: if fields appear in both indexes and uniqueIndexes, only create the unique version
+	const deduped = normalized.filter((f) => !uniqueSet.has(f.join(",")));
+	const allEntries: Array<{ fields: string[]; unique: boolean }> = [
+		...deduped.map((fields) => ({ fields, unique: false })),
+		...uniqueNormalized.map((fields) => ({ fields, unique: true })),
+	];
+
+	const created: string[] = [];
+	const errors: Array<{ index: string; error: string }> = [];
+
+	for (const entry of allEntries) {
+		const { fields } = entry;
+		const indexName = generateIndexName(pluginId, collection, fields, { unique: entry.unique });
+
+		try {
+			// Create the index
+			const createSql = generateCreateIndexSql(db, pluginId, collection, fields, {
+				unique: entry.unique,
+			});
+			await createSql.execute(db);
+
+			// Track in _plugin_indexes table
+			await db
+				.insertInto("_plugin_indexes")
+				.values({
+					plugin_id: pluginId,
+					collection,
+					index_name: indexName,
+					fields: JSON.stringify(fields),
+				})
+				.onConflict((oc) =>
+					oc
+						.columns(["plugin_id", "collection", "index_name"])
+						.doUpdateSet({ fields: JSON.stringify(fields) }),
+				)
+				.execute();
+
+			created.push(indexName);
+		} catch (error) {
+			errors.push({
+				index: indexName,
+				error: error instanceof Error ? error.message : String(error),
+			});
+		}
+	}
+
+	return { created, errors };
+}
+
+/**
+ * Remove indexes that are no longer declared
+ */
+export async function removeOrphanedIndexes(
+	db: Kysely<Database>,
+	pluginId: string,
+	collection: string,
+	currentIndexes: Array<string | string[]>,
+	options?: { uniqueIndexes?: Array<string | string[]> },
+): Promise<{
+	removed: string[];
+	errors: Array<{ index: string; error: string }>;
+}> {
+	const normalized = normalizeIndexes(currentIndexes);
+	const uniqueNormalized = options?.uniqueIndexes ? normalizeIndexes(options.uniqueIndexes) : [];
+	const uniqueSet = new Set(uniqueNormalized.map((f) => f.join(",")));
+
+	// Build the set of expected index names (regular + unique with correct prefix)
+	const currentIndexNames = new Set<string>();
+	for (const fields of normalized) {
+		// If this field set is in both, only the unique version exists (deduplication in create)
+		if (!uniqueSet.has(fields.join(","))) {
+			currentIndexNames.add(generateIndexName(pluginId, collection, fields));
+		}
+	}
+	for (const fields of uniqueNormalized) {
+		currentIndexNames.add(generateIndexName(pluginId, collection, fields, { unique: true }));
+	}
+
+	// Get existing indexes from tracking table
+	const existingIndexes = await db
+		.selectFrom("_plugin_indexes")
+		.select(["index_name"])
+		.where("plugin_id", "=", pluginId)
+		.where("collection", "=", collection)
+		.execute();
+
+	const removed: string[] = [];
+	const errors: Array<{ index: string; error: string }> = [];
+
+	for (const { index_name } of existingIndexes) {
+		if (!currentIndexNames.has(index_name)) {
+			try {
+				// Drop the index
+				await generateDropIndexSql(index_name).execute(db);
+
+				// Remove from tracking table
+				await db
+					.deleteFrom("_plugin_indexes")
+					.where("plugin_id", "=", pluginId)
+					.where("collection", "=", collection)
+					.where("index_name", "=", index_name)
+					.execute();
+
+				removed.push(index_name);
+			} catch (error) {
+				errors.push({
+					index: index_name,
+					error: error instanceof Error ? error.message : String(error),
+				});
+			}
+		}
+	}
+
+	return { removed, errors };
+}
+
+/**
+ * Sync indexes for a plugin collection (create new, remove old)
+ */
+export async function syncStorageIndexes(
+	db: Kysely<Database>,
+	pluginId: string,
+	collection: string,
+	indexes: Array<string | string[]>,
+	options?: { uniqueIndexes?: Array<string | string[]> },
+): Promise<{
+	created: string[];
+	removed: string[];
+	errors: Array<{ index: string; error: string }>;
+}> {
+	const [createResult, removeResult] = await Promise.all([
+		createStorageIndexes(db, pluginId, collection, indexes, options),
+		removeOrphanedIndexes(db, pluginId, collection, indexes, options),
+	]);
+
+	return {
+		created: createResult.created,
+		removed: removeResult.removed,
+		errors: [...createResult.errors, ...removeResult.errors],
+	};
+}
+
+/**
+ * Remove all indexes for a plugin
+ */
+export async function removeAllPluginIndexes(
+	db: Kysely<Database>,
+	pluginId: string,
+): Promise<{
+	removed: string[];
+	errors: Array<{ index: string; error: string }>;
+}> {
+	const existingIndexes = await db
+		.selectFrom("_plugin_indexes")
+		.select(["index_name", "collection"])
+		.where("plugin_id", "=", pluginId)
+		.execute();
+
+	const removed: string[] = [];
+	const errors: Array<{ index: string; error: string }> = [];
+
+	for (const { index_name } of existingIndexes) {
+		try {
+			await generateDropIndexSql(index_name).execute(db);
+			removed.push(index_name);
+		} catch (error) {
+			errors.push({
+				index: index_name,
+				error: error instanceof Error ? error.message : String(error),
+			});
+		}
+	}
+
+	// Clean up tracking table
+	await db.deleteFrom("_plugin_indexes").where("plugin_id", "=", pluginId).execute();
+
+	return { removed, errors };
+}
+
+/**
+ * Get current index status for a plugin
+ */
+export async function getPluginIndexStatus(
+	db: Kysely<Database>,
+	pluginId: string,
+): Promise<
+	Array<{
+		collection: string;
+		indexName: string;
+		fields: string[];
+		createdAt: string;
+	}>
+> {
+	const rows = await db
+		.selectFrom("_plugin_indexes")
+		.select(["collection", "index_name", "fields", "created_at"])
+		.where("plugin_id", "=", pluginId)
+		.execute();
+
+	return rows.map((row) => {
+		const parsed: unknown = JSON.parse(row.fields);
+		const fields = Array.isArray(parsed)
+			? parsed.filter((f): f is string => typeof f === "string")
+			: [];
+		return {
+			collection: row.collection,
+			indexName: row.index_name,
+			fields,
+			createdAt: row.created_at,
+		};
+	});
+}

package/src/plugins/storage-query.ts

@@ -0,0 +1,240 @@
+/**
+ * Plugin Storage Query Validation and Building
+ *
+ * Validates that queries only use indexed fields and builds SQL WHERE clauses.
+ *
+ * @see PLUGIN-SYSTEM.md § Plugin Storage > Query Validation
+ */
+
+import type { Kysely } from "kysely";
+
+import { jsonExtractExpr } from "../database/dialect-helpers.js";
+import { validateJsonFieldName } from "../database/validate.js";
+import type { WhereClause, WhereValue, RangeFilter, InFilter, StartsWithFilter } from "./types.js";
+
+/**
+ * Error thrown when querying non-indexed fields
+ */
+export class StorageQueryError extends Error {
+	constructor(
+		message: string,
+		public field?: string,
+		public suggestion?: string,
+	) {
+		super(message);
+		this.name = "StorageQueryError";
+	}
+}
+
+/**
+ * Check if a value is a range filter
+ */
+export function isRangeFilter(value: WhereValue): value is RangeFilter {
+	if (typeof value !== "object" || value === null) return false;
+	return "gt" in value || "gte" in value || "lt" in value || "lte" in value;
+}
+
+/**
+ * Check if a value is an IN filter
+ */
+export function isInFilter(value: WhereValue): value is InFilter {
+	if (typeof value !== "object" || value === null) return false;
+	return "in" in value && Array.isArray(value.in);
+}
+
+/**
+ * Check if a value is a startsWith filter
+ */
+export function isStartsWithFilter(value: WhereValue): value is StartsWithFilter {
+	if (typeof value !== "object" || value === null) return false;
+	return "startsWith" in value && typeof value.startsWith === "string";
+}
+
+/**
+ * Get the set of indexed fields from index declarations
+ */
+export function getIndexedFields(indexes: Array<string | string[]>): Set<string> {
+	const fields = new Set<string>();
+	for (const index of indexes) {
+		if (Array.isArray(index)) {
+			for (const field of index) {
+				fields.add(field);
+			}
+		} else {
+			fields.add(index);
+		}
+	}
+	return fields;
+}
+
+/**
+ * Validate that all fields in a where clause are indexed
+ */
+export function validateWhereClause(
+	where: WhereClause,
+	indexedFields: Set<string>,
+	pluginId: string,
+	collection: string,
+): void {
+	for (const field of Object.keys(where)) {
+		if (!indexedFields.has(field)) {
+			throw new StorageQueryError(
+				`Cannot query on non-indexed field '${field}'.`,
+				field,
+				`Add '${field}' to storage.${collection}.indexes in plugin '${pluginId}' to enable this query.`,
+			);
+		}
+	}
+}
+
+/**
+ * Validate orderBy fields are indexed
+ */
+export function validateOrderByClause(
+	orderBy: Record<string, "asc" | "desc">,
+	indexedFields: Set<string>,
+	pluginId: string,
+	collection: string,
+): void {
+	for (const field of Object.keys(orderBy)) {
+		if (!indexedFields.has(field)) {
+			throw new StorageQueryError(
+				`Cannot order by non-indexed field '${field}'.`,
+				field,
+				`Add '${field}' to storage.${collection}.indexes in plugin '${pluginId}' to enable ordering by this field.`,
+			);
+		}
+	}
+}
+
+/**
+ * SQL expression for extracting JSON field.
+ *
+ * Validates the field name before interpolation to prevent SQL injection
+ * via crafted JSON path expressions.
+ */
+// eslint-disable-next-line @typescript-eslint/no-explicit-any -- accepts any Kysely instance
+export function jsonExtract(db: Kysely<any>, field: string): string {
+	validateJsonFieldName(field, "query field name");
+	return jsonExtractExpr(db, "data", field);
+}
+
+/**
+ * Build a WHERE clause condition for a single field
+ */
+// eslint-disable-next-line @typescript-eslint/no-explicit-any -- accepts any Kysely instance
+export function buildCondition(
+	db: Kysely<any>,
+	field: string,
+	value: WhereValue,
+): { sql: string; params: unknown[] } {
+	const extract = jsonExtract(db, field);
+
+	if (value === null) {
+		return { sql: `${extract} IS NULL`, params: [] };
+	}
+
+	if (typeof value === "string" || typeof value === "number") {
+		return { sql: `${extract} = ?`, params: [value] };
+	}
+
+	if (typeof value === "boolean") {
+		// JSON booleans are stored as true/false strings
+		return { sql: `${extract} = ?`, params: [value] };
+	}
+
+	if (isInFilter(value)) {
+		const placeholders = value.in.map(() => "?").join(", ");
+		return {
+			sql: `${extract} IN (${placeholders})`,
+			params: value.in,
+		};
+	}
+
+	if (isStartsWithFilter(value)) {
+		return {
+			sql: `${extract} LIKE ?`,
+			params: [`${value.startsWith}%`],
+		};
+	}
+
+	if (isRangeFilter(value)) {
+		const conditions: string[] = [];
+		const params: unknown[] = [];
+
+		if (value.gt !== undefined) {
+			conditions.push(`${extract} > ?`);
+			params.push(value.gt);
+		}
+		if (value.gte !== undefined) {
+			conditions.push(`${extract} >= ?`);
+			params.push(value.gte);
+		}
+		if (value.lt !== undefined) {
+			conditions.push(`${extract} < ?`);
+			params.push(value.lt);
+		}
+		if (value.lte !== undefined) {
+			conditions.push(`${extract} <= ?`);
+			params.push(value.lte);
+		}
+
+		return {
+			sql: conditions.join(" AND "),
+			params,
+		};
+	}
+
+	throw new StorageQueryError(`Unknown filter type for field '${field}'`);
+}
+
+/**
+ * Build a complete WHERE clause from a WhereClause object
+ */
+// eslint-disable-next-line @typescript-eslint/no-explicit-any -- accepts any Kysely instance
+export function buildWhereClause(
+	db: Kysely<any>,
+	where: WhereClause,
+): {
+	sql: string;
+	params: unknown[];
+} {
+	const conditions: string[] = [];
+	const params: unknown[] = [];
+
+	for (const [field, value] of Object.entries(where)) {
+		const condition = buildCondition(db, field, value);
+		conditions.push(condition.sql);
+		params.push(...condition.params);
+	}
+
+	if (conditions.length === 0) {
+		return { sql: "", params: [] };
+	}
+
+	return {
+		sql: conditions.join(" AND "),
+		params,
+	};
+}
+
+/**
+ * Build ORDER BY clause
+ */
+// eslint-disable-next-line @typescript-eslint/no-explicit-any -- accepts any Kysely instance
+export function buildOrderByClause(
+	db: Kysely<any>,
+	orderBy: Record<string, "asc" | "desc">,
+): string {
+	const clauses: string[] = [];
+
+	for (const [field, direction] of Object.entries(orderBy)) {
+		clauses.push(`${jsonExtract(db, field)} ${direction.toUpperCase()}`);
+	}
+
+	if (clauses.length === 0) {
+		return "";
+	}
+
+	return `ORDER BY ${clauses.join(", ")}`;
+}