@shivaedev/sql 0.23.0 → 0.25.0

package/src/repositoryChanges.ts

@@ -1,17 +1,49 @@
-import { Effect, Option } from "effect";
+import { Effect, Hash, Option, type Schema } from "effect";
 import * as Reactivity from "effect/reactivity/Reactivity";
 import { SqlClient } from "effect/sql";
+import { declaredBuckets } from "@shivaedev/changes/buckets.ts";
+import { invalidationKeys, type Key } from "@shivaedev/changes/keys.ts";
+import { ChangeDeclarations } from "#ChangeDeclarations.ts";
 import { invalidateOnCommit, transact } from "#transact.ts";
 
-const ITEM_KEY_LIMIT = 1000;
-
 function encodeKeyValue(_key: string, value: unknown) {
 	return typeof value === "bigint" ? String(value) : value;
 }
 
+function recordId(value: unknown) {
+	return typeof value === "string" || typeof value === "number" || typeof value === "bigint" || typeof value === "boolean"
+		? String(value)
+		: Hash.hash(value);
+}
+
 export type RepositoryChanges = Effect.Success<ReturnType<typeof repositoryChanges>>;
 
-export const repositoryChanges = Effect.fnUntraced(function* (sql: SqlClient.SqlClient, table: string, columns: readonly string[]) {
+export const repositoryChanges = Effect.fnUntraced(function* (
+	sql: SqlClient.SqlClient,
+	table: string,
+	columns: readonly string[],
+	fields: Schema.Struct.Fields = {},
+) {
+	const buckets = declaredBuckets(fields);
+	const bucketKeys = Effect.fnUntraced(function* (row: Readonly<Record<string, unknown>>) {
+		const keys: Key[] = [];
+		for (const [index, { field, decode }] of buckets.entries()) {
+			for (const value of [row[field], row[`shivaeOldBucket${index}`]]) {
+				if (value === null || value === undefined) {
+					continue;
+				}
+				const decoded = yield* decode(value);
+				if (decoded === null || decoded === undefined) {
+					continue;
+				}
+				keys.push({ _tag: "Bucket", collection: table, field, value: decoded });
+			}
+		}
+		return invalidationKeys(keys);
+	});
+	function returning(base = sql`*`) {
+		return sql.csv([base, ...buckets.map(({ field }, index) => sql`old.${sql(field)} as ${sql(`shivaeOldBucket${index}`)}`)]);
+	}
 	const initial = yield* Effect.serviceOption(Reactivity.Reactivity);
 	const fallback = Option.isSome(initial) ? initial.value : yield* Reactivity.make;
 	function provide<A, E, R>(effect: Effect.Effect<A, E, R | Reactivity.Reactivity | SqlClient.SqlClient>) {
@@ -25,25 +57,30 @@ export const repositoryChanges = Effect.fnUntraced(function* (sql: SqlClient.Sql
 			),
 		);
 	}
-	function record(rows: readonly Readonly<Record<string, unknown>>[]) {
+	const record = Effect.fnUntraced(function* (rows: readonly Readonly<Record<string, unknown>>[]) {
 		if (rows.length === 0) {
-			return Effect.void;
+			return;
 		}
-		if (rows.length > ITEM_KEY_LIMIT) {
-			return provide(invalidateOnCommit([table, `${table}:*`]));
+		const keys = new Set<string>([table]);
+		for (const row of rows) {
+			const id =
+				columns.length === 1
+					? row[columns[0] ?? ""]
+					: JSON.stringify(
+							columns.map((column) => row[column]),
+							encodeKeyValue,
+						);
+			keys.add(`${table}:${recordId(id)}`);
+			for (const key of yield* bucketKeys(row)) {
+				keys.add(key);
+			}
 		}
-		const ids = rows.map((row) =>
-			columns.length === 1
-				? row[columns[0] ?? ""]
-				: JSON.stringify(
-						columns.map((column) => row[column]),
-						encodeKeyValue,
-					),
-		);
-		return provide(invalidateOnCommit({ [table]: ids }));
-	}
+		const declarations = yield* ChangeDeclarations;
+		yield* declarations.declare(table, [...keys], rows.length);
+		yield* provide(invalidateOnCommit([...keys]));
+	});
 	function write<A, E, R>(effect: Effect.Effect<A, E, R>) {
 		return provide(transact(effect, { onSqlError: (error) => error }));
 	}
-	return { record, write };
+	return { record, returning, write };
 });

package/src/repositoryWrites.ts

@@ -20,14 +20,16 @@ export function repositoryWrites(
 	function insert(input: unknown) {
 		return Effect.flatMap(record(input), (row) =>
 			(Object.keys(row).length === 0
-				? sql`insert into ${sql(tableName)} default values returning *`
-				: sql`insert into ${sql(tableName)} ${sql.insert(row)} returning *`
+				? sql`insert into ${sql(tableName)} default values returning ${changes.returning()}`
+				: sql`insert into ${sql(tableName)} ${sql.insert(row)} returning ${changes.returning()}`
 			).pipe(Effect.tap(changes.record)),
 		);
 	}
 	function update(input: unknown) {
 		return Effect.flatMap(record(input), (row) =>
-			sql`update ${sql(tableName)} set ${sql.update(row, idColumns)} where ${predicate(row)} returning *`.pipe(Effect.tap(changes.record)),
+			sql`update ${sql(tableName)} set ${sql.update(row, idColumns)} where ${predicate(row)} returning ${changes.returning()}`.pipe(
+				Effect.tap(changes.record),
+			),
 		);
 	}
 	const insertRow = SqlSchema.findOne({ execute: insert, Request: model.insert, Result: model });
@@ -43,7 +45,7 @@ export function repositoryWrites(
 	const removeByKey = SqlSchema.void({
 		execute: (id) =>
 			Effect.flatMap(record({ [idColumn]: id }), (encoded) =>
-				sql`delete from ${sql(tableName)} where ${predicate(encoded)} returning *`.pipe(Effect.tap(changes.record)),
+				sql`delete from ${sql(tableName)} where ${predicate(encoded)} returning ${changes.returning()}`.pipe(Effect.tap(changes.record)),
 			),
 		Request: key,
 	});
@@ -52,7 +54,7 @@ export function repositoryWrites(
 			? removeByKey(input)
 			: Effect.flatMap(filterRecord(input), (filter) =>
 					Effect.flatMap(where(filter, true), (clause) =>
-						sql`delete from ${sql(tableName)} where ${clause} returning *`.pipe(Effect.tap(changes.record), Effect.asVoid),
+						sql`delete from ${sql(tableName)} where ${clause} returning ${changes.returning()}`.pipe(Effect.tap(changes.record), Effect.asVoid),
 					),
 				);
 	}

package/src/testing.ts

@@ -1,11 +1,13 @@
 import { Data, Effect } from "effect";
 import { SqlClient } from "effect/sql";
-import { ChangeObserver, transact } from "./transact.ts";
+import { type BucketKey, type ItemKey, invalidationKeys } from "@shivaedev/changes/keys.ts";
+import { ChangeDeclarations } from "#ChangeDeclarations.ts";
+import { ChangeObserver, transact } from "#transact.ts";
 
 export class UnannouncedWrites extends Data.TaggedError("UnannouncedWrites")<{ readonly tables: readonly string[] }> {}
 
 export interface ChangeCoverage {
-	readonly covers: Readonly<Record<string, readonly unknown[]>>;
+	readonly covers: Readonly<Record<string, readonly (ItemKey | BucketKey)[]>>;
 }
 
 interface TableWrites {
@@ -21,13 +23,22 @@ export const withChangeCoverage = Effect.fn("Changes.withChangeCoverage")(functi
 	const observer = yield* ChangeObserver;
 	const sql = yield* SqlClient.SqlClient;
 	const named = new Set<unknown>();
+	const declarations = new Map<string, { keys: Set<string>; rows: number }>();
+	const outer = yield* ChangeDeclarations;
 	const written = sql<TableWrites>`select relid, relname, (n_tup_ins + n_tup_upd + n_tup_del)::int as writes from pg_stat_xact_user_tables`;
 	const checked = Effect.gen(function* () {
 		const before = new Map((yield* written).map((row) => [row.relid, row.writes]));
 		const result = yield* body;
-		const missing = (yield* written).filter(
-			(row) => row.writes > (before.get(row.relid) ?? 0) && !named.has(row.relname) && !options?.covers[row.relname]?.some((key) => named.has(key)),
-		);
+		const missing = (yield* written).filter((row) => {
+			const count = row.writes - (before.get(row.relid) ?? 0);
+			if (count <= 0) {
+				return false;
+			}
+			const automatic = declarations.get(row.relname);
+			const explicit = options?.covers[row.relname] ?? [];
+			const expected = [...(automatic?.keys ?? []), ...invalidationKeys(explicit)];
+			return (count > (automatic?.rows ?? 0) && explicit.length === 0) || expected.some((key) => !named.has(key));
+		});
 		if (missing.length > 0) {
 			return yield* Effect.fail(new UnannouncedWrites({ tables: missing.map((row) => row.relname).sort() }));
 		}
@@ -35,6 +46,20 @@ export const withChangeCoverage = Effect.fn("Changes.withChangeCoverage")(functi
 	});
 	return yield* checked.pipe(
 		transact({ onSqlError: (error) => error }),
+		Effect.provideService(ChangeDeclarations, {
+			declare: (table, keys, rows) =>
+				Effect.andThen(
+					Effect.sync(() => {
+						const entry = declarations.get(table) ?? { keys: new Set<string>(), rows: 0 };
+						for (const key of keys) {
+							entry.keys.add(key);
+						}
+						entry.rows += rows;
+						declarations.set(table, entry);
+					}),
+					outer.declare(table, keys, rows),
+				),
+		}),
 		Effect.provideService(ChangeObserver, (event) => {
 			if (event._tag === "Recorded") {
 				for (const key of event.changes) {

package/src/transact.ts

@@ -35,7 +35,7 @@ const guard = Effect.fn("Transact.guard")(function* () {
 const invalidations = makeChannel<unknown, SqlClient.SqlClient | Reactivity.Reactivity, SqlError>({
 	name: "@shivaedev/sql/transact",
 	owner: Effect.map(SqlClient.SqlClient, (sql) => sql.transactionService),
-	publish: (keys) => Reactivity.invalidate(keys),
+	publish: (keys) => Reactivity.Reactivity.use((reactivity) => Effect.sync(() => reactivity.invalidateUnsafe(keys))),
 	unowned: guard(),
 });
 

package/src/upsertResult.ts

@@ -0,0 +1,18 @@
+import { Effect, Schema } from "effect";
+import type { Model } from "effect/schema";
+import type { SqlClient } from "effect/sql";
+
+export interface UpsertRow<TRow> {
+	readonly inserted: boolean;
+	readonly row: TRow;
+}
+
+export function upsertReturning(sql: SqlClient.SqlClient) {
+	return sql`*, (xmax = 0) as ${sql("effectSqlInserted")}`;
+}
+
+export function makeUpsertDecoder(model: Model.Any) {
+	const decode = Schema.decodeUnknownEffect(model);
+	return (row: Readonly<Record<string, unknown>> | undefined) =>
+		Effect.map(decode(row), (decoded) => ({ inserted: row?.effectSqlInserted === true, row: decoded }));
+}

package/src/versionedRepository.ts

@@ -53,7 +53,7 @@ export function makeVersionedWrites(
 	function insert(input: unknown) {
 		return Effect.gen(function* () {
 			const statement = yield* statements.insert(input);
-			const rows = yield* sql`${statement} returning *`;
+			const rows = yield* sql`${statement} returning ${changes.returning()}`;
 			yield* changes.record(rows);
 			return yield* decodeRow(rows[0]);
 		}).pipe(changes.write, Effect.withSpan(`${spanPrefix}.insert`));
@@ -61,14 +61,14 @@ export function makeVersionedWrites(
 	function insertVoid(input: unknown) {
 		return Effect.gen(function* () {
 			const statement = yield* statements.insert(input);
-			const rows = yield* sql`${statement} returning ${statements.returningKeys}`;
+			const rows = yield* sql`${statement} returning ${changes.returning()}`;
 			yield* changes.record(rows);
 		}).pipe(changes.write, Effect.withSpan(`${spanPrefix}.insertVoid`));
 	}
 	function insertOrIgnore(input: unknown) {
 		return Effect.gen(function* () {
 			const statement = yield* statements.insert(input, true);
-			const rows = yield* sql`${statement} returning *`;
+			const rows = yield* sql`${statement} returning ${changes.returning()}`;
 			yield* changes.record(rows);
 			return rows.length === 0 ? Option.none() : Option.some(yield* decodeRow(rows[0]));
 		}).pipe(changes.write, Effect.withSpan(`${spanPrefix}.insertOrIgnore`));
@@ -76,7 +76,7 @@ export function makeVersionedWrites(
 	function update(input: Readonly<Record<string, unknown>>) {
 		return Effect.gen(function* () {
 			const { statement, classify } = yield* statements.update(input);
-			const rows = yield* sql`${statement} returning *`;
+			const rows = yield* sql`${statement} returning ${changes.returning()}`;
 			if (rows.length === 0) {
 				return yield* classify();
 			}
@@ -87,7 +87,7 @@ export function makeVersionedWrites(
 	function updateVoid(input: Readonly<Record<string, unknown>>) {
 		return Effect.gen(function* () {
 			const { statement, classify } = yield* statements.update(input);
-			const rows = yield* sql`${statement} returning ${statements.returningKeys}`;
+			const rows = yield* sql`${statement} returning ${changes.returning()}`;
 			if (rows.length === 0) {
 				return yield* classify();
 			}
@@ -97,7 +97,7 @@ export function makeVersionedWrites(
 	function deleteRow(id: unknown, expected: number) {
 		return Effect.gen(function* () {
 			const { statement, classify } = yield* statements.remove(id, expected);
-			const rows = yield* sql`${statement} returning ${statements.returningKeys}`;
+			const rows = yield* sql`${statement} returning ${changes.returning()}`;
 			if (rows.length === 0) {
 				return yield* classify();
 			}

package/src/versionedStatements.ts

@@ -37,7 +37,6 @@ export function versionedStatements(model: Model.Any, sql: SqlClient.SqlClient,
 	const validateVersion = Schema.decodeUnknownEffect(Version);
 	const clause =
 		options.conflictColumns === undefined ? undefined : makeConflictClause(model, sql, { ...options, conflictColumns: options.conflictColumns });
-	const returningKeys = sql.csv(idColumns.map((column) => sql`${sql(column)}`));
 	function key(input: unknown) {
 		return idColumns.length === 1 ? { [idColumns[0] ?? ""]: input } : input;
 	}
@@ -86,5 +85,5 @@ export function versionedStatements(model: Model.Any, sql: SqlClient.SqlClient,
 			};
 		});
 	}
-	return { insert, remove, returningKeys, update };
+	return { insert, remove, update };
 }