@shivaedev/sql 0.31.0 → 0.33.0

package/src/coordinateWorkerRelease.ts

@@ -1,16 +1,24 @@
-import { Duration, Effect, Exit, Schedule, Scope } from "effect";
+import { Duration, Effect, Exit, Predicate, Schedule, Scope } from "effect";
 import { isSqlError } from "effect/sql/SqlError";
 import { readPostgresRelease } from "./postgresRelease.ts";
 import { releaseWorkers } from "./releaseWorkers.ts";
 import { makeWorkerRelease } from "./WorkerRelease.ts";
 
+function isCloseable(scope: Scope.Scope): scope is Scope.Closeable {
+	return Predicate.hasProperty(scope, "~effect/Scope/Closeable") && scope["~effect/Scope/Closeable"] === "~effect/Scope/Closeable";
+}
+
 export const coordinateWorkerRelease = Effect.fn("EffectSql.coordinateWorkerRelease")(function* (
 	release: string,
 	pollInterval: Duration.Input = "1 second",
 	table = "effect_sql_migrations",
 ) {
 	const gate = yield* makeWorkerRelease();
-	const scope = yield* Scope.fork(yield* Effect.scope);
+	const owningScope = yield* Effect.scope;
+	if (!isCloseable(owningScope)) {
+		return yield* Effect.die(new Error("Worker release coordination requires a closeable owning scope"));
+	}
+	const scope = owningScope;
 	const retries = Schedule.spaced(Duration.toMillis(pollInterval) / 4).pipe(Schedule.upTo({ times: 3 }));
 	const rows = yield* releaseWorkers(release, Duration.toMillis(pollInterval) * 5, table);
 	let claimed = false;
@@ -46,7 +54,9 @@ export const coordinateWorkerRelease = Effect.fn("EffectSql.coordinateWorkerRele
 	}
 	yield* Scope.addFinalizer(
 		scope,
-		rows.remove.pipe(
+		gate.setActive(false).pipe(
+			Effect.orDie,
+			Effect.andThen(rows.remove),
 			Effect.tapCause((cause) => Effect.logError("Worker release cleanup failed", cause)),
 			Effect.orDie,
 		),

package/src/inValues.ts

@@ -0,0 +1,11 @@
+import { Effect } from "effect";
+import { SqlClient } from "effect/sql";
+import type { Fragment } from "effect/sql/Statement";
+
+export const inValues = Effect.fnUntraced(function* (column: string | Fragment, values: readonly unknown[]) {
+	const sql = yield* SqlClient.SqlClient;
+	if (values.length === 0) {
+		return sql`1 = 0`;
+	}
+	return sql`${typeof column === "string" ? sql(column) : column} in ${sql.in(values)}`;
+});

package/src/int4Array.ts

@@ -0,0 +1,14 @@
+import * as PgTypes from "@effect/sql-pg/PgTypes";
+import { Result, Schema, SchemaGetter, SchemaParser } from "effect";
+
+const Integers = Schema.Array(Schema.Int);
+
+export function int4Array<TField extends Schema.Top & { readonly Encoded: readonly number[] }>(field: TField) {
+	return Schema.Unknown.pipe(
+		Schema.decodeTo(Integers, {
+			decode: SchemaGetter.transformEffect((value) => SchemaParser.decodeUnknownEffect(Integers)(PgTypes.isParameter(value) ? value.value : value)),
+			encode: SchemaGetter.transform((values) => Result.getOrThrow(PgTypes.array(values, PgTypes.OID.int4))),
+		}),
+		Schema.decodeTo(field),
+	);
+}

package/src/int8.ts

@@ -0,0 +1,14 @@
+import * as PgTypes from "@effect/sql-pg/PgTypes";
+import { Schema, SchemaGetter, SchemaParser } from "effect";
+
+export function int8<TField extends Schema.Top & { readonly Encoded: bigint }>(field: TField) {
+	return Schema.Unknown.pipe(
+		Schema.decodeTo(Schema.BigInt, {
+			decode: SchemaGetter.transformEffect((value) =>
+				SchemaParser.decodeUnknownEffect(Schema.BigInt)(PgTypes.isParameter(value) ? value.value : value),
+			),
+			encode: SchemaGetter.transform(PgTypes.int8),
+		}),
+		Schema.decodeTo(field),
+	);
+}

package/src/internal/MigrationStartedAt.ts

@@ -0,0 +1,5 @@
+import { Context } from "effect";
+
+export const MigrationStartedAt = Context.Reference<string | undefined>("@shivaedev/sql/internal/MigrationStartedAt", {
+	defaultValue: () => undefined,
+});

package/src/internal/migrationLock.ts

@@ -0,0 +1,20 @@
+import { Effect } from "effect";
+import { SqlClient } from "effect/sql";
+import type { Statement } from "effect/sql/Statement";
+
+export const migrationLock = Effect.fnUntraced(function* (table: string, timeout: string) {
+	const sql = yield* SqlClient.SqlClient;
+	const connection = yield* sql.reserve;
+	function execute(statement: Statement<Readonly<Record<string, unknown>>>) {
+		const [text, parameters] = statement.compile();
+		return connection.execute(text, parameters, undefined);
+	}
+	const [previous] = yield* execute(sql`select current_setting('lock_timeout') as timeout`);
+	yield* Effect.acquireRelease(execute(sql`select set_config('lock_timeout', ${timeout}, false)`), () =>
+		execute(sql`select set_config('lock_timeout', ${previous?.timeout}, false)`).pipe(Effect.orDie),
+	);
+	const key = `effect_sql_migrations:${table}`;
+	// Register cleanup before waiting so interruption also releases a lock granted just before cancellation.
+	yield* Effect.addFinalizer(() => execute(sql`select pg_advisory_unlock(hashtextextended(${key}, 0))`).pipe(Effect.orDie));
+	yield* execute(sql`select pg_advisory_lock(hashtextextended(${key}, 0))`);
+});

package/src/listenForChanges.ts

@@ -89,11 +89,11 @@ export const listenForChanges = Effect.fn("PostgresChanges.listenForChanges")(fu
 						return;
 					}
 					if (lostAt !== undefined) {
-						yield* Effect.logDebug("PostgreSQL change listener reconnect failed; retrying", cause);
+						yield* Effect.logDebug("PostgreSQL change listener reconnect failed; retrying");
 						return;
 					}
 					lostAt = yield* Clock.currentTimeMillis;
-					yield* Effect.logWarning("PostgreSQL change listener lost; resynchronizing live streams", cause);
+					yield* Effect.logWarning("PostgreSQL change listener lost; resynchronizing live streams");
 				}).pipe(Effect.andThen(Effect.sleep(options.retryDelay ?? "1 second"))),
 			),
 		),

package/src/makeConflictClause.ts

@@ -2,6 +2,7 @@ import { Effect, Schema, SchemaAST, SchemaIssue } from "effect";
 import type { Model } from "effect/schema";
 import type { SqlClient } from "effect/sql";
 import { makeFilter } from "#repositoryFilter.ts";
+import type { UpsertAssignment } from "#upsertColumns.ts";
 
 export function makeConflictClause(
 	model: Model.Any,
@@ -33,18 +34,33 @@ export function makeConflictClause(
 			}
 		});
 	}
-	function clause(encoded: Readonly<Record<string, unknown>>, updateColumns?: readonly string[], updateWhere?: Readonly<Record<string, unknown>>) {
+	function assignment(entry: UpsertAssignment) {
+		const column = typeof entry === "string" ? entry : entry.column;
+		const incoming = sql`excluded.${sql(column)}`;
+		const stored = sql`${sql(options.tableName)}.${sql(column)}`;
+		let value = incoming;
+		if (typeof entry !== "string") {
+			value = entry.mode === "add" ? sql`${stored} + ${incoming}` : sql`coalesce(${incoming}, ${stored})`;
+		}
+		return sql`${sql(column)} = ${value}`;
+	}
+	function clause(
+		encoded: Readonly<Record<string, unknown>>,
+		updateColumns?: readonly UpsertAssignment[],
+		updateWhere?: Readonly<Record<string, unknown>>,
+	) {
 		return Effect.gen(function* () {
 			yield* validateColumns(encoded, conflictColumns);
 			if (updateColumns !== undefined) {
-				yield* validateColumns(encoded, updateColumns, [...idColumns, ...conflictColumns, ...immutable]);
+				yield* validateColumns(
+					encoded,
+					updateColumns.map((entry) => (typeof entry === "string" ? entry : entry.column)),
+					[...idColumns, ...conflictColumns, ...immutable],
+				);
 			}
 			const conflict = sql.csv(conflictColumns.map((column) => sql`${sql(column)}`));
 			const where = options.conflictWhere === undefined ? sql`` : sql`where ${yield* filter.where(options.conflictWhere)}`;
-			const action =
-				updateColumns === undefined
-					? sql`do nothing`
-					: sql`do update set ${sql.csv(updateColumns.map((column) => sql`${sql(column)} = excluded.${sql(column)}`))}`;
+			const action = updateColumns === undefined ? sql`do nothing` : sql`do update set ${sql.csv(updateColumns.map(assignment))}`;
 			const condition = updateWhere === undefined ? sql`` : sql`where ${yield* updateFilter.where(updateWhere, true, true)}`;
 			return sql`on conflict (${conflict}) ${where} ${action} ${condition}`;
 		});

package/src/migrations.ts

@@ -2,6 +2,8 @@ import { Duration, Effect, Option, Schema } from "effect";
 import * as Reactivity from "effect/reactivity/Reactivity";
 import { Migrator, SqlClient } from "effect/sql";
 import type { SqlError } from "effect/sql/SqlError";
+import { MigrationStartedAt } from "#internal/MigrationStartedAt.ts";
+import { migrationLock } from "#internal/migrationLock.ts";
 import { advisoryXactLock } from "./advisoryXactLock.ts";
 import { postgresRelease } from "./postgresRelease.ts";
 import { runSchemaComponent, type SchemaComponent } from "./SchemaComponent.ts";
@@ -25,19 +27,31 @@ export const migratePostgres = Effect.fn("EffectSql.migratePostgres")(function*
 	release,
 }: PostgresMigrationOptions<R, E>) {
 	const sql = yield* SqlClient.SqlClient;
-	const available = yield* Effect.serviceOption(Reactivity.Reactivity);
-	const reactivity = Option.isSome(available) ? available.value : yield* Reactivity.make;
+	const [invocation] =
+		release === undefined
+			? []
+			: yield* sql`select clock_timestamp()::text as "startedAt"`.withoutTransform.pipe(
+					Effect.flatMap(Schema.decodeUnknownEffect(Schema.Array(Schema.Struct({ startedAt: Schema.String })))),
+				);
+	const current = [...(yield* loader)].sort(([left], [right]) => left - right);
+	const extendedLock = components.length > 0 || release !== undefined;
 	const millis = Duration.toMillis(lockTimeout);
 	const timeout = millis === Number.POSITIVE_INFINITY ? "0" : `${Math.max(1, Math.ceil(millis))}ms`;
+	if (extendedLock) {
+		yield* migrationLock(table, timeout);
+	}
+	const available = yield* Effect.serviceOption(Reactivity.Reactivity);
+	const reactivity = Option.isSome(available) ? available.value : yield* Reactivity.make;
 	const appliedMigrations = yield* Effect.gen(function* () {
 		yield* sql`select set_config('lock_timeout', ${timeout}, true)`;
-		yield* advisoryXactLock(`effect_sql_migrations:${table}`).pipe(Effect.catchTag("AdvisoryLockOutsideTransaction", Effect.die));
+		if (!extendedLock) {
+			yield* advisoryXactLock(`effect_sql_migrations:${table}`).pipe(Effect.catchTag("AdvisoryLockOutsideTransaction", Effect.die));
+		}
 		yield* sql`create table if not exists ${sql(table)} (
   migration_id integer primary key,
   created_at timestamp with time zone not null default now(),
   name text not null
 )`;
-		const current = [...(yield* loader)].sort(([left], [right]) => left - right);
 		const applied = yield* sql`select migration_id as id from ${sql(table)}`.withoutTransform.pipe(
 			Effect.flatMap(Schema.decodeUnknownEffect(Schema.Array(Schema.Struct({ id: Schema.Int })))),
 		);
@@ -59,7 +73,7 @@ export const migratePostgres = Effect.fn("EffectSql.migratePostgres")(function*
 		...(release === undefined ? [] : [postgresRelease(release, table)]),
 	];
 	for (const component of schemaComponents) {
-		yield* runSchemaComponent(component.name, component.apply);
+		yield* runSchemaComponent(component.name, component.apply).pipe(Effect.provideService(MigrationStartedAt, invocation?.startedAt));
 	}
 	return appliedMigrations;
-});
+}, Effect.scoped);

package/src/postgresRelease.ts

@@ -1,5 +1,6 @@
 import { Effect, Schema } from "effect";
 import { SqlClient } from "effect/sql";
+import { MigrationStartedAt } from "#internal/MigrationStartedAt.ts";
 import type { SchemaComponent } from "./SchemaComponent.ts";
 
 export const readPostgresRelease = Effect.fn("EffectSql.readPostgresRelease")(function* (table = "effect_sql_migrations") {
@@ -22,6 +23,11 @@ export function postgresRelease(release: string, table = "effect_sql_migrations"
 	return {
 		apply: Effect.gen(function* () {
 			const sql = yield* SqlClient.SqlClient;
+			const startedAt = yield* MigrationStartedAt;
+			const timestamp = startedAt === undefined ? sql`clock_timestamp()` : sql`${startedAt}::timestamptz`;
+			const [invocation] = yield* sql`select ${timestamp}::text as at`.withoutTransform.pipe(
+				Effect.flatMap(Schema.decodeUnknownEffect(Schema.Array(Schema.Struct({ at: Schema.String })))),
+			);
 			yield* sql`create table if not exists ${sql(`${table}_release`)} (
 				id boolean primary key default true check (id),
 				release text not null,
@@ -33,8 +39,9 @@ export function postgresRelease(release: string, table = "effect_sql_migrations"
 				heartbeat timestamptz not null
 			)`;
 			yield* sql`insert into ${sql(`${table}_release`)} (id, release, recorded_at)
-				values (true, ${release}, clock_timestamp())
-				on conflict (id) do update set release = excluded.release, recorded_at = excluded.recorded_at`;
+				values (true, ${release}, ${invocation?.at}::timestamptz)
+				on conflict (id) do update set release = excluded.release, recorded_at = excluded.recorded_at
+				where ${sql(`${table}_release`)}.recorded_at < excluded.recorded_at`;
 		}),
 		name: "database release",
 		pending: Effect.gen(function* () {

package/src/repositoryCollection.ts

@@ -18,8 +18,12 @@ export class BucketRegistrationError extends Data.TaggedError("BucketRegistratio
 	readonly collectionName: string | undefined;
 	readonly modelFields: readonly string[];
 	readonly collectionFields: readonly string[];
+	readonly invalidFields: readonly string[];
 }> {
 	override get message() {
+		if (this.invalidFields.length > 0) {
+			return `Bucket registration for ${this.tableName}: fields ${JSON.stringify(this.invalidFields)} must decode to strings, numbers or absent values`;
+		}
 		return `Bucket registration for ${this.tableName}: model fields ${JSON.stringify(this.modelFields)} disagree with collection ${this.collectionName ?? "(missing)"} fields ${JSON.stringify(this.collectionFields)}`;
 	}
 }
@@ -28,19 +32,29 @@ export const validateRepositoryCollection = Effect.fnUntraced(function* (
 	fields: Schema.Struct.Fields,
 	options: { readonly tableName: string; readonly collection?: RepositoryCollection },
 ) {
-	const modelFields = declaredBuckets(fields)
+	const buckets = declaredBuckets(fields);
+	const invalidFields = buckets
+		.filter(({ supported }) => !supported)
 		.map(({ field }) => field)
 		.sort(Order.String);
+	const modelFields = buckets.map(({ field }) => field).sort(Order.String);
 	const collectionFields = Object.keys(options.collection?.bucket ?? {}).sort(Order.String);
 	if (
-		(options.collection === undefined && modelFields.length === 0)
-		|| (options.collection?.name === options.tableName
-			&& modelFields.length === collectionFields.length
-			&& modelFields.every((field, index) => field === collectionFields[index]))
+		invalidFields.length === 0
+		&& ((options.collection === undefined && modelFields.length === 0)
+			|| (options.collection?.name === options.tableName
+				&& modelFields.length === collectionFields.length
+				&& modelFields.every((field, index) => field === collectionFields[index])))
 	) {
 		return;
 	}
 	return yield* Effect.fail(
-		new BucketRegistrationError({ collectionFields, collectionName: options.collection?.name, modelFields, tableName: options.tableName }),
+		new BucketRegistrationError({
+			collectionFields,
+			collectionName: options.collection?.name,
+			invalidFields,
+			modelFields,
+			tableName: options.tableName,
+		}),
 	);
 });

package/src/repositoryFilter.ts

@@ -70,11 +70,20 @@ export function makeFilter<TModel extends Model.Any>(model: TModel, sql: SqlClie
 	}
 	function predicate(key: string, value: unknown, allowIn: boolean, allowComparisons: boolean) {
 		return Effect.flatMap(fieldAt(key), (field) => {
+			const encode = Schema.encodeEffect(field);
+			function equality() {
+				return Effect.map(encode(value), (encoded) =>
+					value === null || encoded === null ? sql`${column(key)} is null` : sql`${column(key)} = ${encoded}`,
+				);
+			}
+			const acceptsValue: (input: unknown) => boolean = Schema.is(field);
+			if (acceptsValue(value)) {
+				return equality();
+			}
 			const incoming = reference(value);
 			if (incoming !== undefined) {
 				return Effect.as(fieldAt(incoming), sql`${column(key)} = excluded.${sql(incoming)}`);
 			}
-			const encode = Schema.encodeEffect(field);
 			if (allowComparisons && typeof value === "object" && value !== null && Object.keys(value).some(isOperator)) {
 				return Effect.map(
 					Effect.forEach(Object.entries(value), ([operator, operand]) => comparison(key, field, operator, operand)),
@@ -86,7 +95,7 @@ export function makeFilter<TModel extends Model.Any>(model: TModel, sql: SqlClie
 						Effect.forEach(value.in, (entry) => encode(entry)),
 						(encoded) => inPredicate(key, encoded),
 					)
-				: Effect.map(encode(value), (encoded) => (encoded === null ? sql`${column(key)} is null` : sql`${column(key)} = ${encoded}`));
+				: equality();
 		});
 	}
 	function where(input: Readonly<Record<string, unknown>> & Partial<TupleFilter<Model.Any>> = {}, allowIn = false, allowComparisons = false) {

package/src/transact.ts

@@ -58,8 +58,13 @@ export const transact: {
 	2,
 	Effect.fn("Transact.transact")(function* <A, E, R, TError>(effect: Effect.Effect<A, E, R>, options: TransactOptions<TError>) {
 		const sql = yield* SqlClient.SqlClient;
+		const native = yield* Effect.serviceOption(sql.transactionService);
 		return yield* invalidations
-			.within(sql.withTransaction)(effect)
+			.within((body, boundary) =>
+				native._tag === "Some"
+					? sql.withTransaction(body)
+					: boundary.restore(sql.withTransaction(Effect.tap(body, () => sql.afterCommit(boundary.committed)))),
+			)(effect)
 			.pipe(Effect.catchIf(isSqlError, (error) => Effect.fail(options.onSqlError(error))));
 	}),
 );

package/src/upsertColumns.ts

@@ -0,0 +1,16 @@
+import type { BigDecimal } from "effect";
+import type { Model } from "effect/schema";
+
+export type UpsertColumn<TModel extends Model.Any, TColumn extends keyof TModel["fields"]> =
+	| TColumn
+	| {
+			readonly column: TColumn;
+			readonly mode: "keepOnNull";
+	  }
+	| {
+			[TKey in TColumn]: NonNullable<TModel["fields"][TKey]["Type"]> extends number | bigint | BigDecimal.BigDecimal
+				? { readonly column: TKey; readonly mode: "add" }
+				: never;
+	  }[TColumn];
+
+export type UpsertAssignment = string | { readonly column: string; readonly mode: "add" | "keepOnNull" };