@firtoz/drizzle-sqlite-wasm 0.1.0 → 0.2.0

package/CHANGELOG.md

@@ -1,5 +1,22 @@
 # @firtoz/drizzle-sqlite-wasm
 
+## 0.2.0
+
+### Minor Changes
+
+- [`58d2cba`](https://github.com/firtoz/fullstack-toolkit/commit/58d2cbac8ea4e540b5460b7088b6b62e50357558) Thanks [@firtoz](https://github.com/firtoz)! - Add sync mode functionality for IndexedDB and SQLite collections
+
+  - Introduced support for both eager and on-demand sync modes in Drizzle providers
+  - Implemented operation tracking via interceptors to monitor database operations during queries
+  - Enhanced DrizzleIndexedDBProvider and DrizzleSqliteProvider to accept interceptors for debugging and testing purposes
+  - Added createInsertSchemaWithDefaults and createInsertSchemaWithIdDefault utilities for better schema management
+  - Refactored collection utilities to improve data handling and consistency across collections
+
+### Patch Changes
+
+- Updated dependencies [[`58d2cba`](https://github.com/firtoz/fullstack-toolkit/commit/58d2cbac8ea4e540b5460b7088b6b62e50357558)]:
+  - @firtoz/drizzle-utils@0.2.0
+
 ## 0.1.0
 
 ### Minor Changes

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@firtoz/drizzle-sqlite-wasm",
-	"version": "0.1.0",
+	"version": "0.2.0",
 	"description": "Drizzle SQLite WASM bindings",
 	"main": "./src/index.ts",
 	"module": "./src/index.ts",
@@ -71,7 +71,7 @@
 		"access": "public"
 	},
 	"dependencies": {
-		"@firtoz/drizzle-utils": "^0.1.0",
+		"@firtoz/drizzle-utils": "^0.2.0",
 		"@firtoz/maybe-error": "^1.5.1",
 		"@firtoz/worker-helper": "^1.0.0",
 		"@sqlite.org/sqlite-wasm": "^3.51.1-build1",

package/src/collections/sqlite-collection.ts

@@ -1,15 +1,5 @@
-// const selectSchema = createSelectSchema(todoTable);
-
-import type {
-	CollectionConfig,
-	InferSchemaOutput,
-	LoadSubsetOptions,
-	SyncConfig,
-	SyncConfigRes,
-	SyncMode,
-} from "@tanstack/db";
+import type { InferSchemaOutput, SyncMode } from "@tanstack/db";
 import type { IR } from "@tanstack/db";
-import { DeduplicatedLoadSubset } from "@tanstack/db";
 import {
 	eq,
 	sql,
@@ -29,25 +19,25 @@ import {
 	asc,
 	desc,
 	type SQL,
-	getTableColumns,
 } from "drizzle-orm";
-import type {
-	SQLiteUpdateSetSource,
-	BaseSQLiteDatabase,
-	SQLiteInsertValue,
+import {
+	type SQLiteUpdateSetSource,
+	type BaseSQLiteDatabase,
+	type SQLiteInsertValue,
+	SQLiteColumn,
 } from "drizzle-orm/sqlite-core";
-import { createInsertSchema } from "drizzle-valibot";
-import * as v from "valibot";
 import type {
 	SelectSchema,
 	TableWithRequiredFields,
+	BaseSyncConfig,
+	SyncBackend,
+} from "@firtoz/drizzle-utils";
+import {
+	createSyncFunction,
+	createInsertSchemaWithIdDefault,
+	createGetKeyFunction,
+	createCollectionConfig,
 } from "@firtoz/drizzle-utils";
-
-// WORKAROUND: DeduplicatedLoadSubset has a bug where toggling queries (e.g., isNull/isNotNull)
-// creates invalid expressions like not(or(isNull(...), not(isNull(...))))
-// See: https://github.com/TanStack/db/issues/828
-// TODO: Re-enable once the bug is fixed
-const useDedupe = false as boolean;
 
 export type AnyDrizzleDatabase = BaseSQLiteDatabase<
 	"async",
@@ -59,6 +49,70 @@ export type AnyDrizzleDatabase = BaseSQLiteDatabase<
 export type DrizzleSchema<TDrizzle extends AnyDrizzleDatabase> =
 	TDrizzle["_"]["fullSchema"];
 
+/**
+ * Operation tracking for SQLite queries
+ * Useful for testing and debugging to verify what operations are actually performed
+ *
+ * Uses discriminated unions for type safety - TypeScript can narrow the type based on the 'type' field
+ */
+export type SQLOperation =
+	| {
+			type: "select-all";
+			tableName: string;
+			itemsReturned: unknown[];
+			itemCount: number;
+			context: string;
+			sql?: string;
+			timestamp: number;
+	  }
+	| {
+			type: "select-where";
+			tableName: string;
+			whereClause: string;
+			itemsReturned: unknown[];
+			itemCount: number;
+			context: string;
+			sql?: string;
+			timestamp: number;
+	  }
+	| {
+			type: "write";
+			tableName: string;
+			itemsWritten: unknown[];
+			writeCount: number;
+			context: string;
+			timestamp: number;
+	  }
+	| {
+			type: "insert";
+			tableName: string;
+			item: unknown;
+			sql?: string;
+			timestamp: number;
+	  }
+	| {
+			type: "update";
+			tableName: string;
+			updates: unknown;
+			sql?: string;
+			timestamp: number;
+	  }
+	| {
+			type: "delete";
+			tableName: string;
+			sql?: string;
+			timestamp: number;
+	  };
+
+/**
+ * Interceptor interface for tracking SQLite operations
+ * Allows tests and debugging tools to observe what operations are performed
+ */
+export interface SQLInterceptor {
+	/** Called when any SQLite operation is performed */
+	onOperation?: (operation: SQLOperation) => void;
+}
+
 export interface DrizzleCollectionConfig<
 	TDrizzle extends AnyDrizzleDatabase,
 	TTableName extends ValidTableNames<DrizzleSchema<TDrizzle>>,
@@ -80,6 +134,10 @@ export interface DrizzleCollectionConfig<
 	 * This ensures WAL is flushed to the main database file for OPFS persistence
 	 */
 	checkpoint?: () => Promise<void>;
+	/**
+	 * Optional interceptor for tracking SQLite operations (for testing/debugging)
+	 */
+	interceptor?: SQLInterceptor;
 }
 
 export type ValidTableNames<TSchema extends Record<string, unknown>> = {
@@ -88,6 +146,12 @@ export type ValidTableNames<TSchema extends Record<string, unknown>> = {
 
 /**
  * Converts TanStack DB IR BasicExpression to Drizzle SQL expression
+ *
+ * Supported operators that TanStack DB pushes down to backend (SUPPORTED_COLLECTION_FUNCS):
+ * - eq, gt, lt, gte, lte, and, or, in, isNull, isUndefined, not
+ *
+ * Additional operators handled for completeness (won't be pushed down in on-demand mode):
+ * - ne, isNotNull, like
  */
 function convertBasicExpressionToDrizzle<TTable extends Table>(
 	expression: IR.BasicExpression,
@@ -99,7 +163,13 @@ function convertBasicExpressionToDrizzle<TTable extends Table>(
 		const columnName = propRef.path[propRef.path.length - 1];
 		const column = table[columnName as keyof typeof table];
 
-		if (!column || typeof column !== "object" || !("_" in column)) {
+		if (!column || !(column instanceof SQLiteColumn)) {
+			console.error("[SQLite Collection] Column lookup failed:", {
+				columnName,
+				column,
+				tableKeys: Object.keys(table),
+				hasColumn: columnName in table,
+			});
 			throw new Error(`Column ${String(columnName)} not found in table`);
 		}
 
@@ -193,22 +263,11 @@ export function sqliteCollectionOptions<
 	const TTableName extends string & ValidTableNames<DrizzleSchema<TDrizzle>>,
 	TTable extends DrizzleSchema<TDrizzle>[TTableName] & TableWithRequiredFields,
 >(config: DrizzleCollectionConfig<TDrizzle, TTableName>) {
-	type CollectionType = CollectionConfig<
-		InferSchemaOutput<SelectSchema<TTable>>,
-		string,
-		// biome-ignore lint/suspicious/noExplicitAny: Schema type parameter needs to be flexible
-		any
-	>;
-
 	const tableName = config.tableName as string &
 		ValidTableNames<DrizzleSchema<TDrizzle>>;
 
 	const table = config.drizzle?._.fullSchema[tableName] as TTable;
 
-	let insertListener: CollectionType["onInsert"] | null = null;
-	let updateListener: CollectionType["onUpdate"] | null = null;
-	let deleteListener: CollectionType["onDelete"] | null = null;
-
 	// Transaction queue to serialize SQLite transactions (SQLite only supports one transaction at a time)
 	// The queue ensures transactions run sequentially and continues even if one fails
 	let transactionQueue = Promise.resolve();
@@ -225,52 +284,141 @@ export function sqliteCollectionOptions<
 		return result;
 	};
 
-	const sync: SyncConfig<
-		InferSchemaOutput<SelectSchema<TTable>>,
-		string
-	>["sync"] = (params) => {
-		const { begin, write, commit, markReady } = params;
+	// Create backend-specific implementation
+	const backend: SyncBackend<TTable> = {
+		initialLoad: async (write) => {
+			const items = (await config.drizzle
+				.select()
+				.from(table)) as unknown as InferSchemaOutput<SelectSchema<TTable>>[];
+
+			// Log SQL operation
+			if (config.interceptor?.onOperation) {
+				config.interceptor.onOperation({
+					type: "select-all",
+					tableName: config.tableName as string,
+					itemsReturned: items,
+					itemCount: items.length,
+					context: "Initial load (eager mode)",
+					timestamp: Date.now(),
+				});
+			}
 
-		const initialSync = async () => {
-			await config.readyPromise;
+			// Log write operation
+			if (config.interceptor?.onOperation) {
+				config.interceptor.onOperation({
+					type: "write",
+					tableName: config.tableName as string,
+					itemsWritten: items,
+					writeCount: items.length,
+					context: "Initial load (eager mode)",
+					timestamp: Date.now(),
+				});
+			}
 
-			try {
-				begin();
+			for (const item of items) {
+				write(item);
+			}
+		},
 
-				const items = (await config.drizzle
-					.select()
-					.from(table)) as unknown as InferSchemaOutput<SelectSchema<TTable>>[];
+		loadSubset: async (options, write) => {
+			// Build the query with optional where, orderBy, and limit
+			// Use $dynamic() to enable dynamic query building
+			let query = config.drizzle.select().from(table).$dynamic();
+
+			// Convert TanStack DB IR expressions to Drizzle expressions
+			let hasWhere = false;
+			if (options.where) {
+				const drizzleWhere = convertBasicExpressionToDrizzle(
+					options.where,
+					table,
+				);
+				query = query.where(drizzleWhere);
+				hasWhere = true;
+			}
 
-				for (const item of items) {
-					write({
-						type: "insert",
-						value: item,
+			if (options.orderBy) {
+				const drizzleOrderBy = convertOrderByToDrizzle(options.orderBy, table);
+				query = query.orderBy(...drizzleOrderBy);
+			}
+
+			if (options.limit !== undefined) {
+				query = query.limit(options.limit);
+			}
+
+			const items = (await query) as unknown as InferSchemaOutput<
+				SelectSchema<TTable>
+			>[];
+
+			// Log SQL operation
+			if (config.interceptor?.onOperation) {
+				const contextParts: string[] = ["On-demand load"];
+				if (options.orderBy) {
+					contextParts.push("with sorting");
+				}
+				if (options.limit !== undefined) {
+					contextParts.push(`limit ${options.limit}`);
+				}
+
+				if (hasWhere) {
+					config.interceptor.onOperation({
+						type: "select-where",
+						tableName: config.tableName as string,
+						whereClause: "WHERE clause applied",
+						itemsReturned: items,
+						itemCount: items.length,
+						context: contextParts.join(", "),
+						timestamp: Date.now(),
+					});
+				} else {
+					config.interceptor.onOperation({
+						type: "select-all",
+						tableName: config.tableName as string,
+						itemsReturned: items,
+						itemCount: items.length,
+						context: contextParts.join(", "),
+						timestamp: Date.now(),
 					});
 				}
+			}
 
-				commit();
-			} finally {
-				markReady();
+			// Log write operation
+			if (config.interceptor?.onOperation) {
+				const contextParts: string[] = ["On-demand load"];
+				if (hasWhere) {
+					contextParts.push("with WHERE clause");
+				}
+				if (options.orderBy) {
+					contextParts.push("with sorting");
+				}
+				if (options.limit !== undefined) {
+					contextParts.push(`limit ${options.limit}`);
+				}
+
+				config.interceptor.onOperation({
+					type: "write",
+					tableName: config.tableName as string,
+					itemsWritten: items,
+					writeCount: items.length,
+					context: contextParts.join(", "),
+					timestamp: Date.now(),
+				});
 			}
-		};
 
-		if (config.syncMode === "eager" || !config.syncMode) {
-			initialSync();
-		} else {
-			markReady();
-		}
+			for (const item of items) {
+				write(item);
+			}
+		},
 
-		insertListener = async (params) => {
-			// Store results to write after transaction succeeds
+		handleInsert: async (mutations) => {
 			const results: Array<InferSchemaOutput<SelectSchema<TTable>>> = [];
 
 			// Queue the transaction to serialize SQLite operations
 			await queueTransaction(async () => {
 				await config.drizzle.transaction(async (tx) => {
-					for (const item of params.transaction.mutations) {
+					for (const mutation of mutations) {
 						// TanStack DB applies schema transform (including ID default) before calling this listener
-						// So item.modified already has the ID from insertSchemaWithIdDefault
-						const itemToInsert = item.modified;
+						// So mutation.modified already has the ID from insertSchemaWithIdDefault
+						const itemToInsert = mutation.modified;
 
 						if (config.debug) {
 							console.log(
@@ -306,27 +454,20 @@ export function sqliteCollectionOptions<
 				}
 			});
 
-			// Only update reactive store after transaction succeeds
-			begin();
-			for (const result of results) {
-				write({
-					type: "insert",
-					value: result as unknown as InferSchemaOutput<SelectSchema<TTable>>,
-				});
-			}
-			commit();
-		};
+			return results;
+		},
 
-		updateListener = async (params) => {
-			// Queue the transaction to serialize SQLite operations
+		handleUpdate: async (mutations) => {
 			const results: Array<InferSchemaOutput<SelectSchema<TTable>>> = [];
+
+			// Queue the transaction to serialize SQLite operations
 			await queueTransaction(async () => {
 				await config.drizzle.transaction(async (tx) => {
-					for (const item of params.transaction.mutations) {
+					for (const mutation of mutations) {
 						if (config.debug) {
 							console.log(
 								`[${new Date().toISOString()}] updateListener updating`,
-								item,
+								mutation,
 							);
 						}
 
@@ -335,10 +476,11 @@ export function sqliteCollectionOptions<
 							(await tx
 								.update(table)
 								.set({
-									...item.changes,
+									...mutation.changes,
 									updatedAt: updateTime,
 								} as SQLiteUpdateSetSource<typeof table>)
-								.where(eq(table.id, item.key))
+								// biome-ignore lint/suspicious/noExplicitAny: Key is string but table.id is branded type
+								.where(eq(table.id, mutation.key as any))
 								.returning()) as Array<InferSchemaOutput<SelectSchema<TTable>>>;
 
 						if (config.debug) {
@@ -359,24 +501,16 @@ export function sqliteCollectionOptions<
 				}
 			});
 
-			// Update the reactive store with actual database results
-			// This happens after checkpoint completes
-			begin();
-			for (const result of results) {
-				write({
-					type: "update",
-					value: result,
-				});
-			}
-			commit();
-		};
+			return results;
+		},
 
-		deleteListener = async (params) => {
+		handleDelete: async (mutations) => {
 			// Queue the transaction to serialize SQLite operations
 			await queueTransaction(async () => {
 				await config.drizzle.transaction(async (tx) => {
-					for (const item of params.transaction.mutations) {
-						await tx.delete(table).where(eq(table.id, item.key));
+					for (const mutation of mutations) {
+						// biome-ignore lint/suspicious/noExplicitAny: Key is string but table.id is branded type
+						await tx.delete(table).where(eq(table.id, mutation.key as any));
 					}
 				});
 
@@ -384,149 +518,47 @@ export function sqliteCollectionOptions<
 				if (config.checkpoint) {
 					await config.checkpoint();
 				}
-
-				begin();
-				for (const item of params.transaction.mutations) {
-					if (config.debug) {
-						console.log(
-							`[${new Date().toISOString()}] deleteListener write`,
-							item,
-						);
-					}
-					write({
-						type: "delete",
-						value: item.modified,
-					});
-				}
-				commit();
 			});
-		};
-
-		const loadSubset = async (options: LoadSubsetOptions) => {
-			await config.readyPromise;
-
-			begin();
+		},
+	};
 
-			try {
-				// Build the query with optional where, orderBy, and limit
-				// Use $dynamic() to enable dynamic query building
-				let query = config.drizzle.select().from(table).$dynamic();
+	// Create sync function using shared utilities
+	const baseSyncConfig: BaseSyncConfig<TTable> = {
+		table,
+		readyPromise: config.readyPromise,
+		syncMode: config.syncMode,
+		debug: config.debug,
+	};
 
-				// Convert TanStack DB IR expressions to Drizzle expressions
-				if (options.where) {
-					const drizzleWhere = convertBasicExpressionToDrizzle(
-						options.where,
-						table,
-					);
-					query = query.where(drizzleWhere);
-				}
+	const syncResult = createSyncFunction(baseSyncConfig, backend);
 
-				if (options.orderBy) {
-					const drizzleOrderBy = convertOrderByToDrizzle(
-						options.orderBy,
-						table,
-					);
-					query = query.orderBy(...drizzleOrderBy);
+	// Create insert schema with ID default
+	// (Other defaults like createdAt/updatedAt are handled by SQLite)
+	const schema = createInsertSchemaWithIdDefault(table);
+
+	// Create collection config using shared utilities
+	const collectionConfig = createCollectionConfig({
+		schema,
+		getKey: createGetKeyFunction<TTable>(),
+		syncResult,
+		onInsert: config.debug
+			? async (params) => {
+					console.log("onInsert", params);
 				}
-
-				if (options.limit !== undefined) {
-					query = query.limit(options.limit);
+			: undefined,
+		onUpdate: config.debug
+			? async (params) => {
+					console.log("onUpdate", params);
 				}
-
-				const items = (await query) as unknown as InferSchemaOutput<
-					SelectSchema<TTable>
-				>[];
-
-				for (const item of items) {
-					write({
-						type: "insert",
-						value: item,
-					});
+			: undefined,
+		onDelete: config.debug
+			? async (params) => {
+					console.log("onDelete", params);
 				}
-
-				commit();
-			} catch (error) {
-				// If there's an error, we should still commit to maintain consistency
-				commit();
-				throw error;
-			}
-		};
-
-		// Create deduplicated loadSubset wrapper to avoid redundant queries
-		let loadSubsetDedupe: DeduplicatedLoadSubset | null = null;
-		if (useDedupe) {
-			loadSubsetDedupe = new DeduplicatedLoadSubset({
-				loadSubset,
-			});
-		}
-
-		return {
-			cleanup: () => {
-				insertListener = null;
-				updateListener = null;
-				deleteListener = null;
-				loadSubsetDedupe?.reset();
-			},
-			loadSubset: loadSubsetDedupe?.loadSubset ?? loadSubset,
-		} satisfies SyncConfigRes;
-	};
-
-	// Create insert schema and augment it to apply ID default
-	// (Other defaults like createdAt/updatedAt are handled by SQLite)
-	const insertSchema = createInsertSchema(table);
-	const columns = getTableColumns(table);
-	const idColumn = columns.id;
-
-	const insertSchemaWithIdDefault = v.pipe(
-		insertSchema,
-		v.transform((input) => {
-			const result = { ...input } as Record<string, unknown>;
-
-			// Apply ID default if missing
-			if (result.id === undefined && idColumn?.defaultFn) {
-				result.id = idColumn.defaultFn();
-			}
-
-			return result as typeof input;
-		}),
-	);
-
-	return {
-		schema: insertSchemaWithIdDefault,
-		getKey: (item: InferSchemaOutput<SelectSchema<TTable>>) => {
-			const id = (item as { id: string }).id;
-			return id;
-		},
-		sync: {
-			sync,
-		},
-		onInsert: async (
-			params: Parameters<NonNullable<CollectionType["onInsert"]>>[0],
-		) => {
-			if (config.debug) {
-				console.log("onInsert", params);
-			}
-
-			await insertListener?.(params);
-		},
-		onUpdate: async (
-			params: Parameters<NonNullable<CollectionType["onUpdate"]>>[0],
-		) => {
-			if (config.debug) {
-				console.log("onUpdate", params);
-			}
-
-			await updateListener?.(params);
-		},
-		onDelete: async (
-			params: Parameters<NonNullable<CollectionType["onDelete"]>>[0],
-		) => {
-			if (config.debug) {
-				console.log("onDelete", params);
-			}
-
-			await deleteListener?.(params);
-		},
+			: undefined,
 		syncMode: config.syncMode,
-	} satisfies CollectionType;
+	});
+
+	// biome-ignore lint/suspicious/noExplicitAny: Collection schema type needs to be flexible
+	return collectionConfig as any;
 }

package/src/context/useDrizzleSqlite.ts

@@ -8,6 +8,7 @@ import type { ValidTableNames } from "../collections/sqlite-collection";
 
 export type UseDrizzleSqliteReturn<TSchema extends Record<string, unknown>> = {
 	drizzle: DrizzleSqliteContextValue<TSchema>["drizzle"];
+	readyPromise: DrizzleSqliteContextValue<TSchema>["readyPromise"];
 	useCollection: <TTableName extends string & ValidTableNames<TSchema>>(
 		tableName: TTableName,
 	) => ReturnType<typeof useSqliteCollection<TSchema, TTableName>>;
@@ -28,6 +29,7 @@ export function useDrizzleSqlite<
 
 	return {
 		drizzle: context.drizzle,
+		readyPromise: context.readyPromise,
 		useCollection: <TTableName extends string & ValidTableNames<TSchema>>(
 			tableName: TTableName,
 		) => useSqliteCollection(context, tableName),

package/src/index.ts

@@ -1,5 +1,9 @@
 export { drizzleSqliteWasm } from "./drizzle/direct";
-export { sqliteCollectionOptions as drizzleCollectionOptions } from "./collections/sqlite-collection";
+export {
+	sqliteCollectionOptions as drizzleCollectionOptions,
+	type SQLOperation,
+	type SQLInterceptor,
+} from "./collections/sqlite-collection";
 export { syncableTable } from "@firtoz/drizzle-utils";
 export { makeId } from "@firtoz/drizzle-utils";
 export type {