@shivaedev/test-sql 0.0.0 → 0.9.3

package/src/server/internal/databaseCleanup.ts

@@ -0,0 +1,40 @@
+import { Clock, Effect, type Exit, Fiber, Predicate, Schedule } from "effect";
+import type { SqlError } from "effect/sql/SqlError";
+import { abandonedDatabase } from "./databaseOwner.ts";
+import { administer } from "./databases.ts";
+
+export const databaseCleanup = Effect.fn("PostgresTest.databaseCleanup")(function* (url: Parameters<typeof administer>[0]) {
+	const pending: Fiber.Fiber<Exit.Exit<unknown, SqlError>>[] = [];
+	function drop(name: string) {
+		return administer(url, (admin) => admin`drop database if exists ${admin(name)} with (force)`).pipe(
+			Effect.retry({
+				schedule: Schedule.spaced("100 millis"),
+				times: 300,
+				while: (error) => Predicate.hasProperty(error.reason.cause, "code") && error.reason.cause.code === "53300",
+			}),
+			Effect.provideService(Clock.Clock, Clock.Clock.defaultValue()),
+		);
+	}
+	const awaitCleanup = Effect.suspend(() =>
+		Effect.gen(function* () {
+			const exits = yield* Effect.forEach([...pending], Fiber.join);
+			yield* Effect.forEach(exits, (exit) => exit).pipe(Effect.orDie);
+		}),
+	);
+	yield* Effect.addFinalizer(() => awaitCleanup);
+	const leftovers = yield* administer(url, (admin) => admin<{ datname: string }>`select datname from pg_database`);
+	for (const { datname } of leftovers) {
+		if (abandonedDatabase(datname)) {
+			yield* drop(datname);
+		}
+	}
+	function release(name: string) {
+		return Effect.gen(function* () {
+			const previous = pending.at(-1);
+			const ready = previous === undefined ? Effect.void : Fiber.join(previous);
+			const fiber = yield* Effect.andThen(ready, Effect.exit(drop(name))).pipe(Effect.uninterruptible, Effect.forkDetach);
+			pending.push(fiber);
+		});
+	}
+	return { awaitCleanup, release };
+});

package/src/server/internal/databaseOwner.ts

@@ -0,0 +1,25 @@
+import { createHash } from "node:crypto";
+import { hostname } from "node:os";
+import process from "node:process";
+
+const host = createHash("sha256").update(hostname()).digest("hex").slice(0, 8);
+
+export function databaseName() {
+	return `effect_test_${host}_${process.pid}_${crypto.randomUUID().replaceAll("-", "")}`;
+}
+
+export function abandonedDatabase(name: string) {
+	if (/^effect_test_[a-f0-9]{32}$/u.test(name)) {
+		return true;
+	}
+	const owner = /^effect_test_(?<host>[a-f0-9]{8})_(?<pid>[0-9]+)_[a-f0-9]{32}$/u.exec(name);
+	if (owner === null || owner.groups?.host !== host) {
+		return false;
+	}
+	try {
+		process.kill(Number(owner.groups.pid), 0);
+		return false;
+	} catch (error) {
+		return error instanceof Error && "code" in error && error.code === "ESRCH";
+	}
+}

package/src/server/internal/databases.ts

@@ -0,0 +1,47 @@
+import * as PgClient from "@effect/sql-pg/PgClient";
+import { Context, Effect, Layer, Redacted } from "effect";
+import * as Reactivity from "effect/reactivity/Reactivity";
+import { SqlClient } from "effect/sql";
+import type { SqlError } from "effect/sql/SqlError";
+import type { SqlNameTransforms } from "@shivaedev/sql/clientOptions.ts";
+import { connectionLimitHint } from "./connectionLimitHint.ts";
+import { databaseName } from "./databaseOwner.ts";
+
+export function connection(url: Redacted.Redacted<string>, maxConnections: number, transforms: SqlNameTransforms) {
+	return Layer.effectContext(
+		Effect.map(
+			PgClient.make({
+				maxConnections,
+				transformQueryNames: transforms.transformQueryNames,
+				transformResultNames: transforms.transformResultNames,
+				url,
+			}),
+			(client) => Context.make(SqlClient.SqlClient, client).pipe(Context.add(PgClient.PgClient, client)),
+		),
+	).pipe(Layer.provideMerge(Layer.fresh(Reactivity.layer)));
+}
+
+export const administer = Effect.fn("PostgresTest.administer")(function* <A>(
+	url: Redacted.Redacted<string>,
+	operation: (admin: PgClient.PgClient) => Effect.Effect<A, SqlError>,
+) {
+	return yield* Effect.scoped(Effect.flatMap(PgClient.makeClient({ url }), operation).pipe(Effect.mapError(connectionLimitHint))).pipe(
+		Effect.provide(Reactivity.layer),
+	);
+});
+
+export const database = Effect.fn("PostgresTest.database")(function* (
+	url: Redacted.Redacted<string>,
+	release: (name: string) => Effect.Effect<unknown>,
+	template = "template0",
+) {
+	const name = databaseName();
+	const target = new URL(Redacted.value(url));
+	target.pathname = `/${name}`;
+	target.searchParams.delete("database");
+	yield* Effect.acquireRelease(
+		administer(url, (admin) => admin`create database ${admin(name)} template ${admin(template)}`),
+		() => release(name),
+	);
+	return { name, url: Redacted.make(target.toString()) };
+});

package/src/server/internal/modelColumnProblems.ts

@@ -0,0 +1,45 @@
+import type { PostgresColumn } from "#migrationUpgrade.ts";
+
+interface NameTransforms {
+	readonly transformQueryNames?: ((name: string) => string) | undefined;
+	readonly transformResultNames?: ((name: string) => string) | undefined;
+}
+
+export function modelColumnProblems(
+	columns: readonly PostgresColumn[],
+	fields: object,
+	expectedTypes: Readonly<Record<string, string>>,
+	transforms: NameTransforms | undefined,
+): readonly string[] {
+	const queryFields = new Map(Object.keys(fields).map((name) => [transforms?.transformQueryNames?.(name) ?? name, name]));
+	function fieldName(name: string) {
+		return queryFields.get(name) ?? transforms?.transformResultNames?.(name) ?? name;
+	}
+	function columnKey(name: string) {
+		return transforms?.transformQueryNames === undefined ? fieldName(name) : name;
+	}
+	const actual = new Map(columns.map(({ name, type }) => [columnKey(name), type]));
+	const problems: string[] = [];
+	for (const name of Object.keys(fields)) {
+		const type = actual.get(transforms?.transformQueryNames?.(name) ?? name);
+		if (type === undefined) {
+			problems.push(`Missing column: ${name}`);
+		} else if (type !== expectedTypes[name]) {
+			problems.push(`Wrong type for ${name}: expected ${expectedTypes[name]}, received ${type}`);
+		}
+		if (!Object.hasOwn(expectedTypes, name)) {
+			problems.push(`Missing expected PostgreSQL type: ${name}`);
+		}
+	}
+	for (const column of columns) {
+		if (!queryFields.has(columnKey(column.name))) {
+			problems.push(`Extra column: ${fieldName(column.name)}`);
+		}
+	}
+	for (const name of Object.keys(expectedTypes)) {
+		if (!Object.hasOwn(fields, name)) {
+			problems.push(`Extra expected PostgreSQL type: ${name}`);
+		}
+	}
+	return problems;
+}

package/src/server/migrationSteps.ts

@@ -0,0 +1,54 @@
+import { Effect, Option } from "effect";
+import { Migrator, SqlClient } from "effect/sql";
+import { migratePostgres, type PostgresMigrationOptions } from "@shivaedev/sql/migrations.ts";
+
+function invalid(message: string) {
+	return Effect.fail(new Migrator.MigrationError({ kind: "BadState", message }));
+}
+
+const outsideTransaction = Effect.fnUntraced(function* (sql: SqlClient.SqlClient) {
+	if (Option.isSome(yield* Effect.serviceOption(sql.transactionService))) {
+		return yield* invalid("Migration steps must run outside a wrapping transaction.");
+	}
+});
+
+export const migrationSteps = Effect.fn("PostgresTest.migrationSteps")(function* <R>(migrations: PostgresMigrationOptions<R>) {
+	const sql = yield* SqlClient.SqlClient;
+	const context = yield* Effect.context<R | SqlClient.SqlClient>();
+	yield* sql.onDialectOrElse({ orElse: () => invalid("Migration steps require PostgreSQL."), pg: () => Effect.void });
+	yield* outsideTransaction(sql);
+	const resolved = yield* migrations.loader;
+	const ledger = migrations.table ?? "effect_sql_migrations";
+	const identifier = sql`${sql(ledger)}`.compile()[0];
+	function run(loader: Migrator.Loader, target?: number) {
+		return Effect.gen(function* () {
+			yield* outsideTransaction(sql);
+			if (target !== undefined) {
+				const [existing] = yield* sql<{ readonly name: string | null }>`select to_regclass(${identifier})::text as name`.withoutTransform;
+				if (existing !== undefined && existing.name !== null) {
+					const later = yield* sql`select migration_id from ${sql(ledger)} where migration_id > ${target} limit 1`;
+					if (later.length > 0) {
+						return yield* invalid(`Cannot migrate to ${target}: later migrations are already applied.`);
+					}
+				}
+			}
+			return yield* migratePostgres({ ...migrations, loader }).pipe(Effect.provideContext(context));
+		});
+	}
+	return {
+		migrateTo: (name: string | number) => {
+			const match = /^(?<id>\d+)(?:_(?<label>.+))?$/u.exec(String(name))?.groups;
+			const matches =
+				match === undefined ? [] : resolved.filter(([id, label]) => id === Number(match.id) && (match.label === undefined || label === match.label));
+			const [target] = matches;
+			if (target === undefined) {
+				return invalid(`Migration "${name}" is not in the loader.`);
+			}
+			if (matches.length > 1) {
+				return invalid(`Migration "${name}" is ambiguous in the loader.`);
+			}
+			return run(Effect.succeed(resolved.filter(([id]) => id <= target[0])), target[0]);
+		},
+		migrateToLatest: run(Effect.succeed(resolved)),
+	};
+});

package/src/server/migrationUpgrade.ts

@@ -0,0 +1,45 @@
+import { Effect, Option } from "effect";
+import { Migrator, SqlClient } from "effect/sql";
+import { migratePostgres, type PostgresMigrationOptions } from "@shivaedev/sql/migrations.ts";
+
+export interface PostgresColumn {
+	readonly name: string;
+	readonly type: string;
+}
+
+export interface MigrationUpgradeOptions<R = never> {
+	readonly migrations: PostgresMigrationOptions<R>;
+	readonly table: string;
+	readonly upTo: number;
+}
+
+export const migrationUpgrade = Effect.fn("PostgresTest.migrationUpgrade")(function* <R>(options: MigrationUpgradeOptions<R>) {
+	const sql = yield* SqlClient.SqlClient;
+	function invalid(message: string) {
+		return Effect.fail(new Migrator.MigrationError({ kind: "BadState", message }));
+	}
+	yield* sql.onDialectOrElse({ orElse: () => invalid("Migration upgrade checks require PostgreSQL."), pg: () => Effect.void });
+	if (Option.isSome(yield* Effect.serviceOption(sql.transactionService))) {
+		return yield* invalid("Migration upgrade checks must run outside a wrapping transaction.");
+	}
+	const ledger = options.migrations.table ?? "effect_sql_migrations";
+	const identifier = sql`${sql(ledger)}`.compile()[0];
+	const existing = yield* sql<{ readonly name: string | null }>`select to_regclass(${identifier})::text as name`.withoutTransform;
+	if (existing[0]?.name !== null) {
+		const applied = yield* sql`select migration_id from ${sql(ledger)} limit 1`;
+		if (applied.length > 0) {
+			return yield* invalid("Migration upgrade checks require an empty migration ledger.");
+		}
+	}
+	const resolved = yield* options.migrations.loader;
+	if (!resolved.some(([id]) => id === options.upTo)) {
+		return yield* invalid(`Migration ${options.upTo} is not in the loader.`);
+	}
+	yield* migratePostgres({ ...options.migrations, loader: Effect.succeed(resolved.filter(([id]) => id <= options.upTo)) });
+	yield* migratePostgres({ ...options.migrations, loader: Effect.succeed(resolved) });
+	return yield* sql<PostgresColumn>`
+		select attname as name, format_type(atttypid, atttypmod) as type
+		from pg_attribute
+		where attrelid = ${options.table}::regclass and attnum > 0 and not attisdropped
+		order by attname`.withoutTransform;
+});

package/src/server/postgres.ts

@@ -0,0 +1,72 @@
+import { Context, Effect, Layer, type Redacted, type Scope } from "effect";
+import * as Reactivity from "effect/reactivity/Reactivity";
+import type { SqlNameTransforms } from "@shivaedev/sql/clientOptions.ts";
+import { migratePostgres, type PostgresMigrationOptions } from "@shivaedev/sql/migrations.ts";
+import { PostgresTestDatabase, type PostgresTestDatabaseService, type PostgresTestServices } from "#database.ts";
+import { connectionLimitHint } from "#internal/connectionLimitHint.ts";
+import { databaseCleanup } from "#internal/databaseCleanup.ts";
+import { connection, database } from "#internal/databases.ts";
+
+export type PostgresTestLoader<E = never, R = never> = (
+	url: Redacted.Redacted<string>,
+) => Effect.Effect<unknown, E, R | PostgresTestServices | Scope.Scope>;
+
+interface PostgresTestCommonOptions<E, TFixtures> extends SqlNameTransforms {
+	readonly fixtures?: Effect.Effect<unknown, E, TFixtures | PostgresTestServices | Scope.Scope> | PostgresTestLoader<E, TFixtures>;
+	readonly maxConnections?: number;
+	readonly prepare?: boolean;
+	readonly url: Redacted.Redacted<string>;
+}
+
+export type PostgresTestOptions<E = never, R = never, TFixtures = never> = PostgresTestCommonOptions<E, TFixtures> &
+	(
+		| { readonly migrations: PostgresMigrationOptions<R>; readonly initialize?: never }
+		| {
+				readonly initialize: Effect.Effect<unknown, E, R | PostgresTestServices | Scope.Scope> | PostgresTestLoader<E, R>;
+				readonly migrations?: never;
+		  }
+	);
+
+function runLoader<E, R>(
+	loader: Effect.Effect<unknown, E, R> | ((url: Redacted.Redacted<string>) => Effect.Effect<unknown, E, R>),
+	url: Redacted.Redacted<string>,
+) {
+	return typeof loader === "function" ? loader(url) : loader;
+}
+
+export function makePostgresTestLayer<E = never, R = never, TFixtures = never>(options: PostgresTestOptions<E, R, TFixtures>) {
+	return Layer.effectContext(
+		Effect.gen(function* () {
+			const maxConnections = options.maxConnections ?? 2;
+			const { awaitCleanup, release } = yield* databaseCleanup(options.url);
+			const empty = yield* database(options.url, release);
+			const template = options.prepare === false ? empty : yield* database(options.url, release, empty.name);
+			if (options.prepare !== false) {
+				yield* Effect.scoped(
+					Effect.gen(function* () {
+						const services = yield* Layer.build(connection(template.url, maxConnections, options));
+						const initialize = options.initialize;
+						if (initialize === undefined) {
+							yield* migratePostgres(options.migrations).pipe(Effect.provideContext(services));
+						} else {
+							yield* runLoader(initialize, template.url).pipe(Effect.provideContext(services));
+						}
+						if (options.fixtures !== undefined) {
+							yield* runLoader(options.fixtures, template.url).pipe(Effect.provideContext(services));
+						}
+					}),
+				);
+			}
+			const shared = yield* database(options.url, release, template.name);
+			const services = yield* Layer.build(connection(shared.url, maxConnections, options));
+			const clone: PostgresTestDatabaseService["clone"] = (source) =>
+				Effect.gen(function* () {
+					const target = yield* database(options.url, release, source === "empty" ? empty.name : template.name);
+					const context = yield* Layer.build(connection(target.url, maxConnections, options));
+					return Context.add(context, PostgresTestDatabase, { awaitCleanup, clone, isolated, url: target.url });
+				});
+			const isolated = clone("migrated");
+			return Context.add(services, PostgresTestDatabase, { awaitCleanup, clone, isolated, url: shared.url });
+		}).pipe(Effect.mapError(connectionLimitHint)),
+	).pipe(Layer.provide(Reactivity.layer));
+}

package/src/server/withIsolatedDatabase.ts

@@ -0,0 +1,13 @@
+import { Effect } from "effect";
+import { connectionLimitHint } from "#internal/connectionLimitHint.ts";
+import { PostgresTestDatabase, type PostgresTestTemplate } from "./database.ts";
+
+export function withIsolatedDatabase<A, E, R>(effect: Effect.Effect<A, E, R>, options?: { readonly template?: PostgresTestTemplate }) {
+	return Effect.scoped(
+		Effect.gen(function* () {
+			const database = yield* PostgresTestDatabase;
+			const services = yield* options?.template === undefined ? database.isolated : database.clone(options.template);
+			return yield* Effect.scoped(effect.pipe(Effect.provideContext(services)));
+		}),
+	).pipe(Effect.mapError(connectionLimitHint));
+}

package/src/server/withTestTransaction.ts

@@ -0,0 +1,17 @@
+import { Effect, Exit } from "effect";
+import type { SqlError } from "effect/sql/SqlError";
+import { transact } from "@shivaedev/sql/transact.ts";
+import { connectionLimitHint } from "#internal/connectionLimitHint.ts";
+
+function isTestExit<A, E>(value: Exit.Exit<A, E> | SqlError): value is Exit.Exit<A, E> {
+	return Exit.isExit(value);
+}
+
+export function withTestTransaction<A, E, R>(effect: Effect.Effect<A, E, R>) {
+	return Effect.uninterruptibleMask((restore) =>
+		transact(Effect.flatMap(Effect.exit(Effect.scoped(restore(effect))), Effect.fail), { onSqlError: (error) => error }).pipe(
+			Effect.catchIf(isTestExit, (exit) => exit),
+			Effect.mapError(connectionLimitHint),
+		),
+	);
+}