@shivaedev/sql 0.23.0 → 0.24.0

package/src/conditionalUpsert.ts

@@ -5,6 +5,7 @@ import type { SqlError } from "effect/sql/SqlError";
 import { makeConflictClause } from "#makeConflictClause.ts";
 import type { Comparisons, FindManyFilter } from "#repositoryFilter.ts";
 import type { RepositoryChanges } from "./repositoryChanges.ts";
+import { makeUpsertDecoder, type UpsertRow, upsertReturning } from "./upsertResult.ts";
 
 type Fields<TModel extends Model.Any> = Schema.Struct.Type<TModel["fields"]>;
 interface Incoming<TModel extends Model.Any, TValue> {
@@ -19,8 +20,8 @@ export type UpsertFilter<TModel extends Model.Any> = {
 		| Comparisons<Fields<TModel>[TKey], NonNullable<Fields<TModel>[TKey]> | Incoming<TModel, NonNullable<Fields<TModel>[TKey]>>>;
 };
 export type UpsertResult<TRow> =
-	| { readonly _tag: "Inserted"; readonly row: TRow }
-	| { readonly _tag: "Updated"; readonly row: TRow }
+	| { readonly _tag: "Inserted"; readonly row: TRow; readonly inserted: true }
+	| { readonly _tag: "Updated"; readonly row: TRow; readonly inserted: false }
 	| { readonly _tag: "Unchanged" };
 export interface ConditionalUpsert<TModel extends Model.Any, TOptions, TServices> {
 	(
@@ -34,7 +35,7 @@ export interface ConditionalUpsert<TModel extends Model.Any, TOptions, TServices
 	(
 		input: TModel["insert"]["Type"],
 		options: TOptions & { readonly updateWhere?: never },
-	): Effect.Effect<TModel["Type"], SqlError | Schema.SchemaError, TServices>;
+	): Effect.Effect<UpsertRow<TModel["Type"]>, SqlError | Schema.SchemaError, TServices>;
 }
 export interface ConditionalUpsertMany<TModel extends Model.Any, TOptions, TServices> {
 	(
@@ -48,7 +49,7 @@ export interface ConditionalUpsertMany<TModel extends Model.Any, TOptions, TServ
 	(
 		inputs: readonly TModel["insert"]["Type"][],
 		options: TOptions & { readonly updateWhere?: never },
-	): Effect.Effect<TModel["Type"][], SqlError | Schema.SchemaError, TServices>;
+	): Effect.Effect<UpsertRow<TModel["Type"]>[], SqlError | Schema.SchemaError, TServices>;
 }
 
 export function makeConditionalUpsert(
@@ -65,7 +66,7 @@ export function makeConditionalUpsert(
 ) {
 	const encode = Schema.encodeEffect(model.insert);
 	const record = Schema.decodeUnknownEffect(Schema.Record(Schema.String, Schema.Unknown));
-	const decode = Schema.decodeUnknownEffect(model);
+	const decode = makeUpsertDecoder(model);
 	const clause = makeConflictClause(model, sql, options);
 	return (
 		inputs: readonly unknown[],
@@ -76,7 +77,7 @@ export function makeConditionalUpsert(
 				Effect.gen(function* () {
 					const row = yield* record(yield* encode(input));
 					const suffix = yield* clause(row, settings.updateColumns, settings.updateWhere);
-					return sql`insert into ${sql(options.tableName)} ${sql.insert(row)} ${suffix} returning *, (xmax = 0) as ${sql("effectSqlInserted")}`;
+					return sql`insert into ${sql(options.tableName)} ${sql.insert(row)} ${suffix} returning ${options.changes.returning(upsertReturning(sql))}`;
 				}),
 			);
 			const changedRows: Readonly<Record<string, unknown>>[] = [];
@@ -88,7 +89,10 @@ export function makeConditionalUpsert(
 					if (row === undefined) {
 						return { _tag: "Unchanged" } as const;
 					}
-					return { _tag: row.effectSqlInserted ? "Inserted" : "Updated", row: yield* decode(row) } as const;
+					const result = yield* decode(row);
+					return result.inserted
+						? ({ ...result, _tag: "Inserted", inserted: true } as const)
+						: ({ ...result, _tag: "Updated", inserted: false } as const);
 				}),
 			);
 			yield* options.changes.record(changedRows);

package/src/conflictWrites.ts

@@ -7,6 +7,7 @@ import { type ConditionalUpsert, type ConditionalUpsertMany, makeConditionalUpse
 import { type InsertFromSelect, makeInsertFromSelect } from "#insertFromSelect.ts";
 import { makeConflictClause } from "#makeConflictClause.ts";
 import type { RepositoryChanges } from "./repositoryChanges.ts";
+import { makeUpsertDecoder, upsertReturning } from "./upsertResult.ts";
 
 export type InsertColumn<TModel extends Model.Any> = Extract<keyof TModel["insert"]["Type"] & keyof TModel["fields"], string>;
 export type ConflictColumns<TModel extends Model.Any> = readonly [InsertColumn<TModel>, ...InsertColumn<TModel>[]];
@@ -53,6 +54,7 @@ export function makeConflictWrites(
 	const { tableName, spanPrefix } = options;
 	const encode = Schema.encodeEffect(model.insert);
 	const decode = Schema.decodeUnknownEffect(model);
+	const decodeUpsert = makeUpsertDecoder(model);
 	const record = Schema.decodeUnknownEffect(Schema.Record(Schema.String, Schema.Unknown));
 	const conditional = makeConditionalUpsert(model, sql, options);
 	const clause = makeConflictClause(model, sql, options);
@@ -60,7 +62,7 @@ export function makeConflictWrites(
 		return Effect.gen(function* () {
 			const encoded = yield* record(yield* encode(input));
 			const suffix = yield* clause(encoded, updateColumns);
-			return sql`insert into ${sql(tableName)} ${sql.insert(encoded)} ${suffix} returning *`;
+			return sql`insert into ${sql(tableName)} ${sql.insert(encoded)} ${suffix} returning ${options.changes.returning(updateColumns === undefined ? sql`*` : upsertReturning(sql))}`;
 		});
 	}
 	return {
@@ -79,7 +81,7 @@ export function makeConflictWrites(
 				? Effect.gen(function* () {
 						const rows = yield* yield* statement(input, settings.updateColumns);
 						yield* options.changes.record(rows);
-						return yield* decode(rows[0]);
+						return yield* decodeUpsert(rows[0]);
 					}).pipe(options.changes.write, Effect.withSpan(`${spanPrefix}.upsert`))
 				: conditional([input], { ...settings, updateWhere: settings.updateWhere }).pipe(Effect.map((results) => results[0])),
 		upsertMany: (
@@ -87,7 +89,7 @@ export function makeConflictWrites(
 			settings: { readonly updateColumns: readonly string[]; readonly updateWhere?: Readonly<Record<string, unknown>> },
 		) =>
 			settings.updateWhere === undefined
-				? makeInsertMany(model, sql, options, "upsertMany", (row) => clause(row, settings.updateColumns))(inputs)
+				? makeInsertMany(model, sql, options, "upsertMany", (row) => clause(row, settings.updateColumns), true)(inputs)
 				: conditional(inputs, { ...settings, updateWhere: settings.updateWhere }),
 	};
 }

package/src/insertFromSelect.ts

@@ -40,7 +40,7 @@ export function makeInsertFromSelect<TServices = never>(
 			const suffix = yield* prepare(names);
 			const statement = yield* source.statement;
 			const rows =
-				yield* sql`insert into ${sql(options.tableName)} (${target}) select ${selected} from (${statement}) as ${sql("source")} where true ${suffix} returning *`;
+				yield* sql`insert into ${sql(options.tableName)} (${target}) select ${selected} from (${statement}) as ${sql("source")} where true ${suffix} returning ${options.changes.returning()}`;
 			yield* options.changes.record(rows);
 			return yield* decode(rows);
 		}).pipe(options.changes.write, Effect.withSpan(`${options.spanPrefix}.${operation}`));

package/src/mapSqlError.ts

@@ -0,0 +1,31 @@
+import type { SqlError } from "effect/sql/SqlError";
+
+export function mapSqlError<TConstraints extends Readonly<Record<string, (error: SqlError) => unknown>>, TFallback>(options: {
+	readonly constraints: TConstraints;
+	readonly fallback: (error: SqlError) => TFallback;
+}): (error: SqlError) => ReturnType<TConstraints[keyof TConstraints]> | TFallback;
+export function mapSqlError(options: {
+	readonly constraints: Readonly<Record<string, (error: SqlError) => unknown>>;
+	readonly fallback: (error: SqlError) => unknown;
+}) {
+	return (error: SqlError) => {
+		const reason = error.reason;
+		let constraint: string | undefined;
+		if (reason._tag === "UniqueViolation") {
+			constraint = reason.constraint;
+		} else if (
+			reason._tag === "ConstraintError"
+			&& typeof reason.cause === "object"
+			&& reason.cause !== null
+			&& "code" in reason.cause
+			&& (reason.cause.code === "23503" || reason.cause.code === "23514")
+			&& "constraint" in reason.cause
+			&& typeof reason.cause.constraint === "string"
+		) {
+			constraint = reason.cause.constraint;
+		}
+		return constraint !== undefined && Object.hasOwn(options.constraints, constraint)
+			? (options.constraints[constraint] ?? options.fallback)(error)
+			: options.fallback(error);
+	};
+}

package/src/repository.ts

@@ -119,7 +119,7 @@ export function makeRepository<TModel extends Model.Any, TId extends IdColumn<TM
 		const sql = yield* SqlClient.SqlClient;
 		const composite = typeof options.idColumn === "object";
 		const idColumns = (composite ? options.idColumn : [options.idColumn]).map(String);
-		const changes = yield* repositoryChanges(sql, options.tableName, idColumns);
+		const changes = yield* repositoryChanges(sql, options.tableName, idColumns, model.fields);
 		const crud = makeCompositeCrud(model, sql, { ...options, columns: idColumns }, changes);
 		const queries = makeQueries(model, sql, options);
 		return options.versionColumn === undefined

package/src/repositoryChanges.ts

@@ -1,17 +1,49 @@
-import { Effect, Option } from "effect";
+import { Effect, Hash, Option, type Schema } from "effect";
 import * as Reactivity from "effect/reactivity/Reactivity";
 import { SqlClient } from "effect/sql";
+import { declaredBuckets } from "@shivaedev/changes/buckets.ts";
+import { invalidationKeys, type Key } from "@shivaedev/changes/keys.ts";
+import { ChangeDeclarations } from "#ChangeDeclarations.ts";
 import { invalidateOnCommit, transact } from "#transact.ts";
 
-const ITEM_KEY_LIMIT = 1000;
-
 function encodeKeyValue(_key: string, value: unknown) {
 	return typeof value === "bigint" ? String(value) : value;
 }
 
+function recordId(value: unknown) {
+	return typeof value === "string" || typeof value === "number" || typeof value === "bigint" || typeof value === "boolean"
+		? String(value)
+		: Hash.hash(value);
+}
+
 export type RepositoryChanges = Effect.Success<ReturnType<typeof repositoryChanges>>;
 
-export const repositoryChanges = Effect.fnUntraced(function* (sql: SqlClient.SqlClient, table: string, columns: readonly string[]) {
+export const repositoryChanges = Effect.fnUntraced(function* (
+	sql: SqlClient.SqlClient,
+	table: string,
+	columns: readonly string[],
+	fields: Schema.Struct.Fields = {},
+) {
+	const buckets = declaredBuckets(fields);
+	const bucketKeys = Effect.fnUntraced(function* (row: Readonly<Record<string, unknown>>) {
+		const keys: Key[] = [];
+		for (const [index, { field, decode }] of buckets.entries()) {
+			for (const value of [row[field], row[`shivaeOldBucket${index}`]]) {
+				if (value === null || value === undefined) {
+					continue;
+				}
+				const decoded = yield* decode(value);
+				if (decoded === null || decoded === undefined) {
+					continue;
+				}
+				keys.push({ _tag: "Bucket", collection: table, field, value: decoded });
+			}
+		}
+		return invalidationKeys(keys);
+	});
+	function returning(base = sql`*`) {
+		return sql.csv([base, ...buckets.map(({ field }, index) => sql`old.${sql(field)} as ${sql(`shivaeOldBucket${index}`)}`)]);
+	}
 	const initial = yield* Effect.serviceOption(Reactivity.Reactivity);
 	const fallback = Option.isSome(initial) ? initial.value : yield* Reactivity.make;
 	function provide<A, E, R>(effect: Effect.Effect<A, E, R | Reactivity.Reactivity | SqlClient.SqlClient>) {
@@ -25,25 +57,30 @@ export const repositoryChanges = Effect.fnUntraced(function* (sql: SqlClient.Sql
 			),
 		);
 	}
-	function record(rows: readonly Readonly<Record<string, unknown>>[]) {
+	const record = Effect.fnUntraced(function* (rows: readonly Readonly<Record<string, unknown>>[]) {
 		if (rows.length === 0) {
-			return Effect.void;
+			return;
 		}
-		if (rows.length > ITEM_KEY_LIMIT) {
-			return provide(invalidateOnCommit([table, `${table}:*`]));
+		const keys = new Set<string>([table]);
+		for (const row of rows) {
+			const id =
+				columns.length === 1
+					? row[columns[0] ?? ""]
+					: JSON.stringify(
+							columns.map((column) => row[column]),
+							encodeKeyValue,
+						);
+			keys.add(`${table}:${recordId(id)}`);
+			for (const key of yield* bucketKeys(row)) {
+				keys.add(key);
+			}
 		}
-		const ids = rows.map((row) =>
-			columns.length === 1
-				? row[columns[0] ?? ""]
-				: JSON.stringify(
-						columns.map((column) => row[column]),
-						encodeKeyValue,
-					),
-		);
-		return provide(invalidateOnCommit({ [table]: ids }));
-	}
+		const declarations = yield* ChangeDeclarations;
+		yield* declarations.declare(table, [...keys], rows.length);
+		yield* provide(invalidateOnCommit([...keys]));
+	});
 	function write<A, E, R>(effect: Effect.Effect<A, E, R>) {
 		return provide(transact(effect, { onSqlError: (error) => error }));
 	}
-	return { record, write };
+	return { record, returning, write };
 });

package/src/repositoryWrites.ts

@@ -20,14 +20,16 @@ export function repositoryWrites(
 	function insert(input: unknown) {
 		return Effect.flatMap(record(input), (row) =>
 			(Object.keys(row).length === 0
-				? sql`insert into ${sql(tableName)} default values returning *`
-				: sql`insert into ${sql(tableName)} ${sql.insert(row)} returning *`
+				? sql`insert into ${sql(tableName)} default values returning ${changes.returning()}`
+				: sql`insert into ${sql(tableName)} ${sql.insert(row)} returning ${changes.returning()}`
 			).pipe(Effect.tap(changes.record)),
 		);
 	}
 	function update(input: unknown) {
 		return Effect.flatMap(record(input), (row) =>
-			sql`update ${sql(tableName)} set ${sql.update(row, idColumns)} where ${predicate(row)} returning *`.pipe(Effect.tap(changes.record)),
+			sql`update ${sql(tableName)} set ${sql.update(row, idColumns)} where ${predicate(row)} returning ${changes.returning()}`.pipe(
+				Effect.tap(changes.record),
+			),
 		);
 	}
 	const insertRow = SqlSchema.findOne({ execute: insert, Request: model.insert, Result: model });
@@ -43,7 +45,7 @@ export function repositoryWrites(
 	const removeByKey = SqlSchema.void({
 		execute: (id) =>
 			Effect.flatMap(record({ [idColumn]: id }), (encoded) =>
-				sql`delete from ${sql(tableName)} where ${predicate(encoded)} returning *`.pipe(Effect.tap(changes.record)),
+				sql`delete from ${sql(tableName)} where ${predicate(encoded)} returning ${changes.returning()}`.pipe(Effect.tap(changes.record)),
 			),
 		Request: key,
 	});
@@ -52,7 +54,7 @@ export function repositoryWrites(
 			? removeByKey(input)
 			: Effect.flatMap(filterRecord(input), (filter) =>
 					Effect.flatMap(where(filter, true), (clause) =>
-						sql`delete from ${sql(tableName)} where ${clause} returning *`.pipe(Effect.tap(changes.record), Effect.asVoid),
+						sql`delete from ${sql(tableName)} where ${clause} returning ${changes.returning()}`.pipe(Effect.tap(changes.record), Effect.asVoid),
 					),
 				);
 	}

package/src/testing.ts

@@ -1,11 +1,13 @@
 import { Data, Effect } from "effect";
 import { SqlClient } from "effect/sql";
-import { ChangeObserver, transact } from "./transact.ts";
+import { type BucketKey, type ItemKey, invalidationKeys } from "@shivaedev/changes/keys.ts";
+import { ChangeDeclarations } from "#ChangeDeclarations.ts";
+import { ChangeObserver, transact } from "#transact.ts";
 
 export class UnannouncedWrites extends Data.TaggedError("UnannouncedWrites")<{ readonly tables: readonly string[] }> {}
 
 export interface ChangeCoverage {
-	readonly covers: Readonly<Record<string, readonly unknown[]>>;
+	readonly covers: Readonly<Record<string, readonly (ItemKey | BucketKey)[]>>;
 }
 
 interface TableWrites {
@@ -21,13 +23,22 @@ export const withChangeCoverage = Effect.fn("Changes.withChangeCoverage")(functi
 	const observer = yield* ChangeObserver;
 	const sql = yield* SqlClient.SqlClient;
 	const named = new Set<unknown>();
+	const declarations = new Map<string, { keys: Set<string>; rows: number }>();
+	const outer = yield* ChangeDeclarations;
 	const written = sql<TableWrites>`select relid, relname, (n_tup_ins + n_tup_upd + n_tup_del)::int as writes from pg_stat_xact_user_tables`;
 	const checked = Effect.gen(function* () {
 		const before = new Map((yield* written).map((row) => [row.relid, row.writes]));
 		const result = yield* body;
-		const missing = (yield* written).filter(
-			(row) => row.writes > (before.get(row.relid) ?? 0) && !named.has(row.relname) && !options?.covers[row.relname]?.some((key) => named.has(key)),
-		);
+		const missing = (yield* written).filter((row) => {
+			const count = row.writes - (before.get(row.relid) ?? 0);
+			if (count <= 0) {
+				return false;
+			}
+			const automatic = declarations.get(row.relname);
+			const explicit = options?.covers[row.relname] ?? [];
+			const expected = [...(automatic?.keys ?? []), ...invalidationKeys(explicit)];
+			return (count > (automatic?.rows ?? 0) && explicit.length === 0) || expected.some((key) => !named.has(key));
+		});
 		if (missing.length > 0) {
 			return yield* Effect.fail(new UnannouncedWrites({ tables: missing.map((row) => row.relname).sort() }));
 		}
@@ -35,6 +46,20 @@ export const withChangeCoverage = Effect.fn("Changes.withChangeCoverage")(functi
 	});
 	return yield* checked.pipe(
 		transact({ onSqlError: (error) => error }),
+		Effect.provideService(ChangeDeclarations, {
+			declare: (table, keys, rows) =>
+				Effect.andThen(
+					Effect.sync(() => {
+						const entry = declarations.get(table) ?? { keys: new Set<string>(), rows: 0 };
+						for (const key of keys) {
+							entry.keys.add(key);
+						}
+						entry.rows += rows;
+						declarations.set(table, entry);
+					}),
+					outer.declare(table, keys, rows),
+				),
+		}),
 		Effect.provideService(ChangeObserver, (event) => {
 			if (event._tag === "Recorded") {
 				for (const key of event.changes) {

package/src/upsertResult.ts

@@ -0,0 +1,18 @@
+import { Effect, Schema } from "effect";
+import type { Model } from "effect/schema";
+import type { SqlClient } from "effect/sql";
+
+export interface UpsertRow<TRow> {
+	readonly inserted: boolean;
+	readonly row: TRow;
+}
+
+export function upsertReturning(sql: SqlClient.SqlClient) {
+	return sql`*, (xmax = 0) as ${sql("effectSqlInserted")}`;
+}
+
+export function makeUpsertDecoder(model: Model.Any) {
+	const decode = Schema.decodeUnknownEffect(model);
+	return (row: Readonly<Record<string, unknown>> | undefined) =>
+		Effect.map(decode(row), (decoded) => ({ inserted: row?.effectSqlInserted === true, row: decoded }));
+}

package/src/versionedRepository.ts

@@ -53,7 +53,7 @@ export function makeVersionedWrites(
 	function insert(input: unknown) {
 		return Effect.gen(function* () {
 			const statement = yield* statements.insert(input);
-			const rows = yield* sql`${statement} returning *`;
+			const rows = yield* sql`${statement} returning ${changes.returning()}`;
 			yield* changes.record(rows);
 			return yield* decodeRow(rows[0]);
 		}).pipe(changes.write, Effect.withSpan(`${spanPrefix}.insert`));
@@ -61,14 +61,14 @@ export function makeVersionedWrites(
 	function insertVoid(input: unknown) {
 		return Effect.gen(function* () {
 			const statement = yield* statements.insert(input);
-			const rows = yield* sql`${statement} returning ${statements.returningKeys}`;
+			const rows = yield* sql`${statement} returning ${changes.returning()}`;
 			yield* changes.record(rows);
 		}).pipe(changes.write, Effect.withSpan(`${spanPrefix}.insertVoid`));
 	}
 	function insertOrIgnore(input: unknown) {
 		return Effect.gen(function* () {
 			const statement = yield* statements.insert(input, true);
-			const rows = yield* sql`${statement} returning *`;
+			const rows = yield* sql`${statement} returning ${changes.returning()}`;
 			yield* changes.record(rows);
 			return rows.length === 0 ? Option.none() : Option.some(yield* decodeRow(rows[0]));
 		}).pipe(changes.write, Effect.withSpan(`${spanPrefix}.insertOrIgnore`));
@@ -76,7 +76,7 @@ export function makeVersionedWrites(
 	function update(input: Readonly<Record<string, unknown>>) {
 		return Effect.gen(function* () {
 			const { statement, classify } = yield* statements.update(input);
-			const rows = yield* sql`${statement} returning *`;
+			const rows = yield* sql`${statement} returning ${changes.returning()}`;
 			if (rows.length === 0) {
 				return yield* classify();
 			}
@@ -87,7 +87,7 @@ export function makeVersionedWrites(
 	function updateVoid(input: Readonly<Record<string, unknown>>) {
 		return Effect.gen(function* () {
 			const { statement, classify } = yield* statements.update(input);
-			const rows = yield* sql`${statement} returning ${statements.returningKeys}`;
+			const rows = yield* sql`${statement} returning ${changes.returning()}`;
 			if (rows.length === 0) {
 				return yield* classify();
 			}
@@ -97,7 +97,7 @@ export function makeVersionedWrites(
 	function deleteRow(id: unknown, expected: number) {
 		return Effect.gen(function* () {
 			const { statement, classify } = yield* statements.remove(id, expected);
-			const rows = yield* sql`${statement} returning ${statements.returningKeys}`;
+			const rows = yield* sql`${statement} returning ${changes.returning()}`;
 			if (rows.length === 0) {
 				return yield* classify();
 			}

package/src/versionedStatements.ts

@@ -37,7 +37,6 @@ export function versionedStatements(model: Model.Any, sql: SqlClient.SqlClient,
 	const validateVersion = Schema.decodeUnknownEffect(Version);
 	const clause =
 		options.conflictColumns === undefined ? undefined : makeConflictClause(model, sql, { ...options, conflictColumns: options.conflictColumns });
-	const returningKeys = sql.csv(idColumns.map((column) => sql`${sql(column)}`));
 	function key(input: unknown) {
 		return idColumns.length === 1 ? { [idColumns[0] ?? ""]: input } : input;
 	}
@@ -86,5 +85,5 @@ export function versionedStatements(model: Model.Any, sql: SqlClient.SqlClient,
 			};
 		});
 	}
-	return { insert, remove, returningKeys, update };
+	return { insert, remove, update };
 }