@firtoz/drizzle-utils 1.0.1 → 1.1.0

package/CHANGELOG.md

@@ -1,5 +1,17 @@
 # @firtoz/drizzle-utils
 
+## 1.1.0
+
+### Minor Changes
+
+- [`b714ebb`](https://github.com/firtoz/fullstack-toolkit/commit/b714ebbb62ec0e3c3aa56c4105e7499fac11d1e5) Thanks [@firtoz](https://github.com/firtoz)! - Extract shared SQLite TanStack sync backend (`createSqliteTableSyncBackend`, IR→Drizzle helpers, `SQLOperation` types) into `@firtoz/drizzle-utils`. Add `@firtoz/drizzle-durable-sqlite` for Durable Object SQLite collections (`durableSqliteCollectionOptions`). Refactor `@firtoz/drizzle-sqlite-wasm` to use the shared backend with `driverMode: "async"`. `durableSqliteCollectionOptions` accepts optional `readyPromise` (defaults to immediate readiness). README documents the class-field Hono pattern, `app.fetch(request, env)` for bindings, optional `on-demand` + `preload` vs eager + `onFirstReady`, and `honoDoFetcherWithName` without a separate exported app type. Restore JSDoc on `DrizzleSqliteCollectionConfig` (`debug`, `checkpoint`, `interceptor`) for editor tooltips. Align `createStandaloneCollection` generics with `InsertToSelectSchema` from `@firtoz/drizzle-utils`.
+
+## 1.0.2
+
+### Patch Changes
+
+- [`bca3758`](https://github.com/firtoz/fullstack-toolkit/commit/bca3758ab5ad2661b950360dc35edda2680c3b4e) Thanks [@firtoz](https://github.com/firtoz)! - Bump minimum `valibot` peer dependency from `>=1.0.0` to `>=1.3.1`.
+
 ## 1.0.1
 
 ### Patch Changes

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@firtoz/drizzle-utils",
-	"version": "1.0.1",
+	"version": "1.1.0",
 	"description": "Shared utilities and types for Drizzle-based packages",
 	"main": "./src/index.ts",
 	"module": "./src/index.ts",
@@ -56,13 +56,13 @@
 		"@tanstack/db": ">=0.5.33",
 		"drizzle-orm": ">=0.45.1",
 		"drizzle-valibot": ">=0.4.0",
-		"valibot": ">=1.0.0"
+		"valibot": ">=1.3.1"
 	},
 	"devDependencies": {
 		"@tanstack/db": "^0.5.33",
 		"drizzle-orm": "^0.45.1",
 		"drizzle-valibot": "^0.4.2",
-		"valibot": "^1.2.0"
+		"valibot": "^1.3.1"
 	},
 	"dependencies": {
 		"@firtoz/db-helpers": "^2.0.0",

package/src/collection-utils.ts

@@ -66,6 +66,15 @@ export type InsertSchema<TTable extends Table> = BuildSchema<
 	undefined
 >;
 
+/**
+ * Schema type with insert input (optionals for defaults) and select output (all fields present).
+ * Represents the standard input/output pair for collection schemas.
+ */
+export type InsertToSelectSchema<TTable extends Table> = v.GenericSchema<
+	v.InferInput<InsertSchema<TTable>>,
+	v.InferOutput<SelectSchema<TTable>>
+>;
+
 /**
  * Helper type to get the table from schema by name
  */
@@ -82,7 +91,7 @@ export type InferCollectionFromTable<TTable extends Table> = Collection<
 	TTable["$inferSelect"],
 	IdOf<TTable>,
 	UtilsRecord,
-	SelectSchema<TTable>,
+	InsertToSelectSchema<TTable>,
 	Omit<
 		TTable["$inferInsert"],
 		"id"
@@ -135,7 +144,7 @@ export function createSyncFunction<TTable extends Table>(
  */
 export function createInsertSchemaWithDefaults<TTable extends Table>(
 	table: TTable,
-): v.GenericSchema<unknown> {
+): InsertToSelectSchema<TTable> {
 	const insertSchema = createInsertSchema(table);
 	const columns = getTableColumns(table);
 
@@ -194,7 +203,7 @@ export function createInsertSchemaWithDefaults<TTable extends Table>(
 
 			return result;
 		}),
-	) as v.GenericSchema<unknown>;
+	) as InsertToSelectSchema<TTable>;
 }
 
 /**
@@ -203,7 +212,7 @@ export function createInsertSchemaWithDefaults<TTable extends Table>(
  */
 export function createInsertSchemaWithIdDefault<TTable extends Table>(
 	table: TTable,
-): v.GenericSchema<unknown> {
+): InsertToSelectSchema<TTable> {
 	const insertSchema = createInsertSchema(table);
 	const columns = getTableColumns(table);
 	const idColumn = columns.id;
@@ -220,7 +229,7 @@ export function createInsertSchemaWithIdDefault<TTable extends Table>(
 
 			return result;
 		}),
-	) as v.GenericSchema<unknown>;
+	) as InsertToSelectSchema<TTable>;
 }
 
 /**
@@ -247,44 +256,28 @@ export function createCollectionConfig<
 	onInsert?: CollectionConfig<
 		InferSchemaOutput<SelectSchema<TTable>>,
 		string,
-		// biome-ignore lint/suspicious/noExplicitAny: Schema type parameter needs to be flexible
-		any
+		TSchema
 	>["onInsert"];
 	onUpdate?: CollectionConfig<
 		InferSchemaOutput<SelectSchema<TTable>>,
 		string,
-		// biome-ignore lint/suspicious/noExplicitAny: Schema type parameter needs to be flexible
-		any
+		TSchema
 	>["onUpdate"];
 	onDelete?: CollectionConfig<
 		InferSchemaOutput<SelectSchema<TTable>>,
 		string,
-		// biome-ignore lint/suspicious/noExplicitAny: Schema type parameter needs to be flexible
-		any
+		TSchema
 	>["onDelete"];
 	syncMode?: SyncMode;
 }): Omit<
-	CollectionConfig<
-		InferSchemaOutput<SelectSchema<TTable>>,
-		string,
-		// biome-ignore lint/suspicious/noExplicitAny: Schema type parameter needs to be flexible
-		any
-	>,
+	CollectionConfig<InferSchemaOutput<SelectSchema<TTable>>, string, TSchema>,
 	"utils"
 > & {
 	schema: TSchema;
 	utils: CollectionUtils<InferSchemaOutput<SelectSchema<TTable>>>;
 } {
 	type TItem = InferSchemaOutput<SelectSchema<TTable>>;
-	type ReturnType = Omit<
-		CollectionConfig<
-			TItem,
-			string,
-			// biome-ignore lint/suspicious/noExplicitAny: Schema type parameter needs to be flexible
-			any
-		>,
-		"utils"
-	> & {
+	type ReturnType = Omit<CollectionConfig<TItem, string, TSchema>, "utils"> & {
 		schema: TSchema;
 		utils: CollectionUtils<TItem>;
 	};

package/src/index.ts

@@ -22,6 +22,7 @@ export type {
 	IdOf,
 	SelectSchema,
 	InsertSchema,
+	InsertToSelectSchema,
 	GetTableFromSchema,
 	InferCollectionFromTable,
 	BaseSyncConfig,
@@ -47,3 +48,15 @@ export {
 } from "./syncableTable";
 
 export type { TableWithRequiredFields } from "./syncableTable";
+
+export type {
+	SQLOperation,
+	SQLInterceptor,
+	SqliteDriverMode,
+	SqliteTableSyncBackendConfig,
+} from "./sqlite-table-sync";
+export {
+	convertBasicExpressionToDrizzle,
+	convertOrderByToDrizzle,
+	createSqliteTableSyncBackend,
+} from "./sqlite-table-sync";

package/src/sqlite-table-sync/convert-ir.ts

@@ -0,0 +1,121 @@
+import type { IR } from "@tanstack/db";
+import {
+	eq,
+	sql,
+	type Table,
+	gt,
+	gte,
+	lt,
+	lte,
+	ne,
+	and,
+	or,
+	not,
+	isNull,
+	isNotNull,
+	like,
+	inArray,
+	asc,
+	desc,
+	type SQL,
+} from "drizzle-orm";
+import { SQLiteColumn } from "drizzle-orm/sqlite-core";
+import { exhaustiveGuard } from "@firtoz/maybe-error";
+
+/**
+ * Converts TanStack DB IR BasicExpression to Drizzle SQL expression.
+ */
+export function convertBasicExpressionToDrizzle<TTable extends Table>(
+	expression: IR.BasicExpression,
+	table: TTable,
+): SQL {
+	switch (expression.type) {
+		case "ref": {
+			const propRef = expression;
+			const columnName = propRef.path[propRef.path.length - 1];
+			const column = table[columnName as keyof typeof table];
+
+			if (!column || !(column instanceof SQLiteColumn)) {
+				console.error("[SQLite sync backend] Column lookup failed:", {
+					columnName,
+					column,
+					tableKeys: Object.keys(table),
+					hasColumn: columnName in table,
+				});
+				throw new Error(`Column ${String(columnName)} not found in table`);
+			}
+
+			return column as unknown as SQL;
+		}
+		case "val": {
+			const value = expression;
+			return sql`${value.value}`;
+		}
+		case "func": {
+			const func = expression;
+			const args = func.args.map((arg) =>
+				convertBasicExpressionToDrizzle(arg, table),
+			);
+
+			switch (func.name) {
+				case "eq":
+					return eq(args[0], args[1]);
+				case "ne":
+					return ne(args[0], args[1]);
+				case "gt":
+					return gt(args[0], args[1]);
+				case "gte":
+					return gte(args[0], args[1]);
+				case "lt":
+					return lt(args[0], args[1]);
+				case "lte":
+					return lte(args[0], args[1]);
+				case "and": {
+					const result = and(...args);
+					if (!result) {
+						throw new Error("Invalid 'and' expression - no arguments provided");
+					}
+					return result;
+				}
+				case "or": {
+					const result = or(...args);
+					if (!result) {
+						throw new Error("Invalid 'or' expression - no arguments provided");
+					}
+					return result;
+				}
+				case "not":
+					return not(args[0]);
+				case "isNull":
+					return isNull(args[0]);
+				case "isNotNull":
+					return isNotNull(args[0]);
+				case "like":
+					return like(args[0], args[1]);
+				case "in":
+					return inArray(args[0], args[1]);
+				case "isUndefined":
+					return isNull(args[0]);
+				default:
+					throw new Error(`Unsupported function: ${func.name}`);
+			}
+		}
+		default:
+			exhaustiveGuard(expression);
+	}
+}
+
+export function convertOrderByToDrizzle<TTable extends Table>(
+	orderBy: IR.OrderBy,
+	table: TTable,
+): SQL[] {
+	return orderBy.map((clause) => {
+		const expression = convertBasicExpressionToDrizzle(
+			clause.expression,
+			table,
+		);
+		const direction = clause.compareOptions.direction || "asc";
+
+		return direction === "asc" ? asc(expression) : desc(expression);
+	});
+}

package/src/sqlite-table-sync/index.ts

@@ -0,0 +1,10 @@
+export type { SQLOperation, SQLInterceptor } from "./types";
+export {
+	convertBasicExpressionToDrizzle,
+	convertOrderByToDrizzle,
+} from "./convert-ir";
+export {
+	createSqliteTableSyncBackend,
+	type SqliteDriverMode,
+	type SqliteTableSyncBackendConfig,
+} from "./sqlite-table-sync-backend";

package/src/sqlite-table-sync/sqlite-table-sync-backend.ts

@@ -0,0 +1,361 @@
+import type { InferSchemaOutput } from "@tanstack/db";
+import { and, eq, type SQL } from "drizzle-orm";
+import type {
+	SQLiteInsertValue,
+	SQLiteUpdateSetSource,
+} from "drizzle-orm/sqlite-core";
+import type { SelectSchema, SyncBackend } from "../collection-utils";
+import type { TableWithRequiredFields } from "../syncableTable";
+import {
+	convertBasicExpressionToDrizzle,
+	convertOrderByToDrizzle,
+} from "./convert-ir";
+import type { SQLInterceptor } from "./types";
+
+export type SqliteDriverMode = "async" | "sync";
+
+export interface SqliteTableSyncBackendConfig<
+	TTable extends TableWithRequiredFields,
+> {
+	/** drizzle-orm SQLite database (async WASM/libsql or sync Durable Object) */
+	// biome-ignore lint/suspicious/noExplicitAny: generic over sync/async Drizzle DB shapes
+	drizzle: any;
+	table: TTable;
+	tableName: string;
+	debug?: boolean;
+	checkpoint?: () => Promise<void>;
+	interceptor?: SQLInterceptor;
+	/**
+	 * `async`: libsql/WASM — use `await db.transaction(async (tx) => …)`.
+	 * `sync`: Cloudflare DO SQLite — `transactionSync` requires a **synchronous** callback; use `.all()` / `.run()` on builders inside `tx`.
+	 */
+	driverMode: SqliteDriverMode;
+}
+
+export function createSqliteTableSyncBackend<
+	TTable extends TableWithRequiredFields,
+>(config: SqliteTableSyncBackendConfig<TTable>): SyncBackend<TTable> {
+	const table = config.table;
+	const driverMode = config.driverMode;
+
+	let transactionQueue = Promise.resolve();
+	const queueTransaction = <T>(fn: () => Promise<T>): Promise<T> => {
+		const result = transactionQueue.then(fn, fn);
+		transactionQueue = result.then(
+			() => {},
+			() => {},
+		);
+		return result;
+	};
+
+	const backend: SyncBackend<TTable> = {
+		initialLoad: async () => {
+			const items = (await config.drizzle
+				.select()
+				.from(table)) as unknown as InferSchemaOutput<SelectSchema<TTable>>[];
+
+			if (config.interceptor?.onOperation) {
+				config.interceptor.onOperation({
+					type: "select-all",
+					tableName: config.tableName,
+					itemsReturned: items,
+					itemCount: items.length,
+					context: "Initial load (eager mode)",
+					timestamp: Date.now(),
+				});
+			}
+			if (config.interceptor?.onOperation) {
+				config.interceptor.onOperation({
+					type: "write",
+					tableName: config.tableName,
+					itemsWritten: items,
+					writeCount: items.length,
+					context: "Initial load (eager mode)",
+					timestamp: Date.now(),
+				});
+			}
+
+			return items as unknown as InferSchemaOutput<SelectSchema<TTable>>[];
+		},
+
+		loadSubset: async (options) => {
+			let query = config.drizzle.select().from(table).$dynamic();
+
+			let hasWhere = false;
+			if (options.where || options.cursor?.whereFrom) {
+				let drizzleWhere: SQL | undefined;
+
+				if (options.where && options.cursor?.whereFrom) {
+					const mainWhere = convertBasicExpressionToDrizzle(
+						options.where,
+						table,
+					);
+					const cursorWhere = convertBasicExpressionToDrizzle(
+						options.cursor.whereFrom,
+						table,
+					);
+					drizzleWhere = and(mainWhere, cursorWhere);
+				} else if (options.where) {
+					drizzleWhere = convertBasicExpressionToDrizzle(options.where, table);
+				} else if (options.cursor?.whereFrom) {
+					drizzleWhere = convertBasicExpressionToDrizzle(
+						options.cursor.whereFrom,
+						table,
+					);
+				}
+
+				if (drizzleWhere) {
+					query = query.where(drizzleWhere);
+					hasWhere = true;
+				}
+			}
+
+			if (options.orderBy) {
+				const drizzleOrderBy = convertOrderByToDrizzle(options.orderBy, table);
+				query = query.orderBy(...drizzleOrderBy);
+			}
+
+			if (options.limit !== undefined) {
+				query = query.limit(options.limit);
+			}
+
+			if (options.offset !== undefined && options.offset > 0) {
+				query = query.offset(options.offset);
+			}
+
+			const items = (await query) as unknown as InferSchemaOutput<
+				SelectSchema<TTable>
+			>[];
+
+			if (config.interceptor?.onOperation) {
+				const contextParts: string[] = ["On-demand load"];
+				if (options.orderBy) contextParts.push("with sorting");
+				if (options.limit !== undefined)
+					contextParts.push(`limit ${options.limit}`);
+				if (options.offset !== undefined && options.offset > 0)
+					contextParts.push(`offset ${options.offset}`);
+				if (options.cursor) contextParts.push("with cursor pagination");
+
+				if (hasWhere) {
+					config.interceptor.onOperation({
+						type: "select-where",
+						tableName: config.tableName,
+						whereClause: "WHERE clause applied",
+						itemsReturned: items,
+						itemCount: items.length,
+						context: contextParts.join(", "),
+						timestamp: Date.now(),
+					});
+				} else {
+					config.interceptor.onOperation({
+						type: "select-all",
+						tableName: config.tableName,
+						itemsReturned: items,
+						itemCount: items.length,
+						context: contextParts.join(", "),
+						timestamp: Date.now(),
+					});
+				}
+			}
+
+			if (config.interceptor?.onOperation) {
+				const contextParts: string[] = ["On-demand load"];
+				if (hasWhere) contextParts.push("with WHERE clause");
+				if (options.orderBy) contextParts.push("with sorting");
+				if (options.limit !== undefined)
+					contextParts.push(`limit ${options.limit}`);
+				if (options.offset !== undefined && options.offset > 0)
+					contextParts.push(`offset ${options.offset}`);
+
+				config.interceptor.onOperation({
+					type: "write",
+					tableName: config.tableName,
+					itemsWritten: items,
+					writeCount: items.length,
+					context: contextParts.join(", "),
+					timestamp: Date.now(),
+				});
+			}
+
+			return items as unknown as InferSchemaOutput<SelectSchema<TTable>>[];
+		},
+
+		handleInsert: async (items) => {
+			const results: Array<InferSchemaOutput<SelectSchema<TTable>>> = [];
+
+			await queueTransaction(async () => {
+				if (driverMode === "sync") {
+					config.drizzle.transaction((tx: typeof config.drizzle) => {
+						for (const itemToInsert of items) {
+							if (config.debug) {
+								console.log(
+									`[${new Date().toISOString()}] insertListener inserting`,
+									itemToInsert,
+								);
+							}
+							const result = tx
+								.insert(table)
+								.values(
+									itemToInsert as unknown as SQLiteInsertValue<typeof table>,
+								)
+								.returning()
+								.all() as Array<InferSchemaOutput<SelectSchema<TTable>>>;
+							if (config.debug) {
+								console.log(
+									`[${new Date().toISOString()}] insertListener result`,
+									result,
+								);
+							}
+							if (result.length > 0) {
+								results.push(result[0]);
+							}
+						}
+					});
+				} else {
+					await config.drizzle.transaction(
+						async (tx: typeof config.drizzle) => {
+							for (const itemToInsert of items) {
+								if (config.debug) {
+									console.log(
+										`[${new Date().toISOString()}] insertListener inserting`,
+										itemToInsert,
+									);
+								}
+								const result = (await tx
+									.insert(table)
+									.values(
+										itemToInsert as unknown as SQLiteInsertValue<typeof table>,
+									)
+									.returning()) as Array<
+									InferSchemaOutput<SelectSchema<TTable>>
+								>;
+								if (config.debug) {
+									console.log(
+										`[${new Date().toISOString()}] insertListener result`,
+										result,
+									);
+								}
+								if (result.length > 0) {
+									results.push(result[0]);
+								}
+							}
+						},
+					);
+				}
+
+				if (config.checkpoint) {
+					await config.checkpoint();
+				}
+			});
+
+			return results;
+		},
+
+		handleUpdate: async (mutations) => {
+			const results: Array<InferSchemaOutput<SelectSchema<TTable>>> = [];
+
+			await queueTransaction(async () => {
+				if (driverMode === "sync") {
+					config.drizzle.transaction((tx: typeof config.drizzle) => {
+						for (const mutation of mutations) {
+							if (config.debug) {
+								console.log(
+									`[${new Date().toISOString()}] updateListener updating`,
+									mutation,
+								);
+							}
+							const updateTime = new Date();
+							const result = tx
+								.update(table)
+								.set({
+									...mutation.changes,
+									updatedAt: updateTime,
+								} as SQLiteUpdateSetSource<typeof table>)
+								// biome-ignore lint/suspicious/noExplicitAny: branded id key
+								.where(eq(table.id, mutation.key as any))
+								.returning()
+								.all() as Array<InferSchemaOutput<SelectSchema<TTable>>>;
+							if (config.debug) {
+								console.log(
+									`[${new Date().toISOString()}] updateListener result`,
+									result,
+								);
+							}
+							results.push(...result);
+						}
+					});
+				} else {
+					await config.drizzle.transaction(
+						async (tx: typeof config.drizzle) => {
+							for (const mutation of mutations) {
+								if (config.debug) {
+									console.log(
+										`[${new Date().toISOString()}] updateListener updating`,
+										mutation,
+									);
+								}
+								const updateTime = new Date();
+								const result = (await tx
+									.update(table)
+									.set({
+										...mutation.changes,
+										updatedAt: updateTime,
+									} as SQLiteUpdateSetSource<typeof table>)
+									// biome-ignore lint/suspicious/noExplicitAny: branded id key
+									.where(eq(table.id, mutation.key as any))
+									.returning()) as Array<
+									InferSchemaOutput<SelectSchema<TTable>>
+								>;
+								if (config.debug) {
+									console.log(
+										`[${new Date().toISOString()}] updateListener result`,
+										result,
+									);
+								}
+								results.push(...result);
+							}
+						},
+					);
+				}
+
+				if (config.checkpoint) {
+					await config.checkpoint();
+				}
+			});
+
+			return results;
+		},
+
+		handleDelete: async (mutations) => {
+			await queueTransaction(async () => {
+				if (driverMode === "sync") {
+					config.drizzle.transaction((tx: typeof config.drizzle) => {
+						for (const mutation of mutations) {
+							tx.delete(table)
+								// biome-ignore lint/suspicious/noExplicitAny: branded id key
+								.where(eq(table.id, mutation.key as any))
+								.run();
+						}
+					});
+				} else {
+					await config.drizzle.transaction(
+						async (tx: typeof config.drizzle) => {
+							for (const mutation of mutations) {
+								await tx
+									.delete(table)
+									// biome-ignore lint/suspicious/noExplicitAny: branded id key
+									.where(eq(table.id, mutation.key as any));
+							}
+						},
+					);
+				}
+
+				if (config.checkpoint) {
+					await config.checkpoint();
+				}
+			});
+		},
+	};
+
+	return backend;
+}

package/src/sqlite-table-sync/types.ts

@@ -0,0 +1,65 @@
+/**
+ * Operation tracking for SQLite-backed TanStack sync backends.
+ * Uses discriminated unions so TypeScript can narrow on `type`.
+ */
+export type SQLOperation =
+	| {
+			type: "select-all";
+			tableName: string;
+			itemsReturned: unknown[];
+			itemCount: number;
+			context: string;
+			sql?: string;
+			timestamp: number;
+	  }
+	| {
+			type: "select-where";
+			tableName: string;
+			whereClause: string;
+			itemsReturned: unknown[];
+			itemCount: number;
+			context: string;
+			sql?: string;
+			timestamp: number;
+	  }
+	| {
+			type: "write";
+			tableName: string;
+			itemsWritten: unknown[];
+			writeCount: number;
+			context: string;
+			timestamp: number;
+	  }
+	| {
+			type: "insert";
+			tableName: string;
+			item: unknown;
+			sql?: string;
+			timestamp: number;
+	  }
+	| {
+			type: "update";
+			tableName: string;
+			updates: unknown;
+			sql?: string;
+			timestamp: number;
+	  }
+	| {
+			type: "delete";
+			tableName: string;
+			sql?: string;
+			timestamp: number;
+	  }
+	| {
+			type: "raw-query";
+			sql: string;
+			params?: unknown[];
+			method: string;
+			rowCount: number;
+			context: string;
+			timestamp: number;
+	  };
+
+export interface SQLInterceptor {
+	onOperation?: (operation: SQLOperation) => void;
+}