@proseql/core 0.1.0

package/dist/factories/database-effect.js

@@ -0,0 +1,859 @@
+/**
+ * Effect-based database factory.
+ *
+ * Creates an in-memory database with typed collections, each backed by
+ * Ref<ReadonlyMap<string, T>> for O(1) ID lookup and atomic state updates.
+ *
+ * Query pipeline: Ref snapshot → Stream.fromIterable → filter → populate → sort → paginate → select
+ * CRUD: Effect-based operations with typed error channels
+ * Persistence: Optional debounced save after each CRUD mutation via Effect.fork
+ */
+import { Chunk, Effect, Layer, PubSub, Ref, Schema, Stream, } from "effect";
+import { NotFoundError, OperationError, ValidationError, } from "../errors/crud-errors.js";
+import { resolveWithIndex } from "../indexes/index-lookup.js";
+import { buildIndexes, normalizeIndexes } from "../indexes/index-manager.js";
+import { buildSearchIndex, resolveWithSearchIndex, } from "../indexes/search-index.js";
+import { dryRunMigrations, validateMigrationRegistry, } from "../migrations/migration-runner.js";
+import { create, createMany } from "../operations/crud/create.js";
+import { createWithRelationships } from "../operations/crud/create-with-relationships.js";
+import { del, deleteMany } from "../operations/crud/delete.js";
+import { deleteManyWithRelationships, deleteWithRelationships, } from "../operations/crud/delete-with-relationships.js";
+import { normalizeConstraints } from "../operations/crud/unique-check.js";
+import { update, updateMany } from "../operations/crud/update.js";
+import { updateWithRelationships } from "../operations/crud/update-with-relationships.js";
+import { upsert, upsertMany } from "../operations/crud/upsert.js";
+import { computeAggregates, computeGroupedAggregates, } from "../operations/query/aggregate.js";
+import { applyCursor } from "../operations/query/cursor-stream.js";
+import { applyFilter } from "../operations/query/filter-stream.js";
+import { applyPagination } from "../operations/query/paginate-stream.js";
+import { resolveComputedStreamWithLazySkip } from "../operations/query/resolve-computed.js";
+import { applySelect, applySelectToArray, } from "../operations/query/select-stream.js";
+import { applyRelevanceSort, applySort, attachSearchScores, extractSearchConfig, } from "../operations/query/sort-stream.js";
+import { applyPopulate } from "../operations/relationships/populate-stream.js";
+import { mergeGlobalHooks } from "../plugins/plugin-hooks.js";
+import { buildPluginRegistry } from "../plugins/plugin-registry.js";
+import { validateIdGeneratorReferences } from "../plugins/plugin-validation.js";
+import { watch } from "../reactive/watch.js";
+import { watchById } from "../reactive/watch-by-id.js";
+import { mergeSerializerWithPluginCodecs, } from "../serializers/format-codec.js";
+import { SerializerRegistry } from "../serializers/serializer-service.js";
+import { createFileWatcher, loadData, saveData, } from "../storage/persistence-effect.js";
+import { StorageAdapter } from "../storage/storage-service.js";
+import { $transaction as $transactionImpl } from "../transactions/transaction.js";
+import { isGroupedAggregateConfig, } from "../types/aggregate-types.js";
+/**
+ * Attach a lazy `runPromise` getter to an Effect value.
+ * The effect is only executed when `.runPromise` is accessed.
+ */
+const withRunPromise = (effect) => {
+    let cached;
+    Object.defineProperty(effect, "runPromise", {
+        get() {
+            if (cached === undefined) {
+                cached = Effect.runPromise(effect);
+            }
+            return cached;
+        },
+        enumerable: false,
+        configurable: true,
+    });
+    return effect;
+};
+/**
+ * Attach a lazy `runPromise` getter to a Stream value.
+ * The stream is collected into an array when `.runPromise` is accessed.
+ */
+const withStreamRunPromise = (stream) => {
+    let cached;
+    Object.defineProperty(stream, "runPromise", {
+        get() {
+            if (cached === undefined) {
+                cached = Effect.runPromise(Stream.runCollect(stream).pipe(Effect.map(Chunk.toReadonlyArray)));
+            }
+            return cached;
+        },
+        enumerable: false,
+        configurable: true,
+    });
+    return stream;
+};
+/**
+ * Attach a lazy `runPromise` getter to an Effect returning CursorPageResult.
+ * The effect is only executed when `.runPromise` is accessed.
+ */
+const withCursorRunPromise = (effect) => {
+    let cached;
+    Object.defineProperty(effect, "runPromise", {
+        get() {
+            if (cached === undefined) {
+                cached = Effect.runPromise(effect);
+            }
+            return cached;
+        },
+        enumerable: false,
+        configurable: true,
+    });
+    return effect;
+};
+const createPersistenceTrigger = (delayMs, makeSaveEffect) => {
+    const pendingTimers = new Map();
+    const executeSave = (key) => Effect.runPromise(makeSaveEffect(key).pipe(Effect.catchAll(() => Effect.void)));
+    const schedule = (key) => {
+        // Cancel existing timer for this key
+        const existing = pendingTimers.get(key);
+        if (existing !== undefined) {
+            clearTimeout(existing);
+        }
+        // Schedule new debounced write
+        const timer = setTimeout(() => {
+            pendingTimers.delete(key);
+            executeSave(key);
+        }, delayMs);
+        pendingTimers.set(key, timer);
+    };
+    const flush = async () => {
+        // Take all pending keys, clear timers, execute saves
+        const keys = Array.from(pendingTimers.keys());
+        for (const [, timer] of pendingTimers) {
+            clearTimeout(timer);
+        }
+        pendingTimers.clear();
+        // Execute all saves
+        await Promise.all(keys.map((key) => executeSave(key)));
+    };
+    const pendingCount = () => pendingTimers.size;
+    const shutdown = () => {
+        for (const [, timer] of pendingTimers) {
+            clearTimeout(timer);
+        }
+        pendingTimers.clear();
+    };
+    return { schedule, flush, pendingCount, shutdown };
+};
+// ============================================================================
+// Extract Populate Config from Object-based Select
+// ============================================================================
+/**
+ * Normalize select config for lazy skip optimization.
+ * Returns the select config as a Record if it's object-based, or undefined if it's array-based.
+ * The lazy skip optimization only works with object-based select.
+ */
+function normalizeSelectForLazySkip(select) {
+    if (select === undefined) {
+        // undefined means select all, which includes computed fields
+        return undefined;
+    }
+    // Array.isArray works for ReadonlyArray too but TypeScript needs help narrowing
+    if (Array.isArray(select)) {
+        // Array-based select is rare and we don't optimize for it
+        return undefined;
+    }
+    // TypeScript now knows select is Record<string, unknown>
+    return select;
+}
+function extractPopulateFromSelect(select, relationships) {
+    const populate = {};
+    let hasPopulate = false;
+    for (const [key, value] of Object.entries(select)) {
+        if (key in relationships) {
+            if (value === true) {
+                populate[key] = true;
+                hasPopulate = true;
+            }
+            else if (typeof value === "object" &&
+                value !== null &&
+                !Array.isArray(value)) {
+                populate[key] = {
+                    select: value,
+                    ...value,
+                };
+                hasPopulate = true;
+            }
+        }
+    }
+    return hasPopulate ? populate : undefined;
+}
+const buildCollection = (collectionName, collectionConfig, ref, stateRefs, dbConfig, afterMutation, indexes, searchIndexRef, searchIndexFields, customOperators, idGeneratorMap, globalHooks, changePubSub, appendOnlyConfig) => {
+    const schema = collectionConfig.schema;
+    const relationships = collectionConfig.relationships;
+    // Merge global plugin hooks with collection-specific hooks.
+    // Global hooks run first (cross-cutting concerns), then collection hooks.
+    // Type narrowing: globalHooks is HooksConfig<Record<string, unknown>>,
+    // collectionHooks is HooksConfig<T>. mergeGlobalHooks returns HooksConfig<T>.
+    const collectionHooks = collectionConfig.hooks;
+    const hooks = mergeGlobalHooks(globalHooks, collectionHooks);
+    // Normalize unique fields constraints (default to empty array if not configured)
+    const uniqueFields = normalizeConstraints(collectionConfig.uniqueFields);
+    // Get computed fields config (undefined means no computed fields)
+    const computed = collectionConfig.computed;
+    // Get ID generator name from collection config (used with idGeneratorMap)
+    const idGeneratorName = collectionConfig.idGenerator;
+    // Build allRelationships map for delete (needs all collections' relationships)
+    const allRelationships = {};
+    for (const [name, config] of Object.entries(dbConfig)) {
+        allRelationships[name] = config.relationships;
+    }
+    // Query function: read Ref snapshot → Stream pipeline
+    // Returns RunnableStream for standard queries, RunnableCursorPage for cursor pagination
+    const queryFn = (options) => {
+        // Determine populate config: explicit populate or extract from object-based select
+        let populateConfig = options?.populate;
+        if (!populateConfig && options?.select && !Array.isArray(options.select)) {
+            populateConfig = extractPopulateFromSelect(options.select, relationships);
+        }
+        // Handle cursor pagination: validate and inject implicit sort if needed
+        const cursorConfig = options?.cursor;
+        let effectiveSort = options?.sort;
+        if (cursorConfig) {
+            const cursorKey = cursorConfig.key;
+            if (options?.sort) {
+                // Explicit sort provided: validate cursor key matches primary sort field
+                const sortKeys = Object.keys(options.sort);
+                if (sortKeys.length === 0) {
+                    // Empty sort object: inject implicit ascending sort on cursor key
+                    effectiveSort = { [cursorKey]: "asc" };
+                }
+                else {
+                    const primarySortKey = sortKeys[0];
+                    if (primarySortKey !== cursorKey) {
+                        // Sort mismatch: return effect that immediately fails
+                        const errorEffect = Effect.fail(new ValidationError({
+                            message: "Invalid cursor configuration",
+                            issues: [
+                                {
+                                    field: "cursor.key",
+                                    message: `cursor key '${cursorKey}' must match primary sort field '${primarySortKey}'`,
+                                },
+                            ],
+                        }));
+                        return withCursorRunPromise(errorEffect);
+                    }
+                }
+            }
+            else {
+                // No explicit sort: inject implicit ascending sort on cursor key
+                effectiveSort = { [cursorKey]: "asc" };
+            }
+            // Cursor pagination branch: populate → resolve computed → filter → sort → applyCursor → select
+            const cursorEffect = Effect.gen(function* () {
+                const map = yield* Ref.get(ref);
+                // Try index-accelerated lookup first (equality index, then search index)
+                let narrowed = indexes
+                    ? yield* resolveWithIndex(options?.where, indexes, map)
+                    : undefined;
+                // If equality index didn't help, try search index
+                if (narrowed === undefined && searchIndexRef) {
+                    narrowed = yield* resolveWithSearchIndex(options?.where, searchIndexRef, searchIndexFields, map);
+                }
+                const items = (narrowed ?? Array.from(map.values()));
+                let s = Stream.fromIterable(items);
+                // Apply pipeline stages: populate → resolve computed (with lazy skip) → filter → sort
+                s = applyPopulate(populateConfig, stateRefs, dbConfig, collectionName)(s);
+                s = resolveComputedStreamWithLazySkip(computed, normalizeSelectForLazySkip(options?.select))(s);
+                s = applyFilter(options?.where, customOperators)(s);
+                // When $search is active, compute and attach relevance scores after filtering
+                // (even though cursor pagination uses explicit sort, scores are still computed)
+                const cursorSearchConfig = extractSearchConfig(options?.where);
+                if (cursorSearchConfig) {
+                    s = attachSearchScores(cursorSearchConfig)(s);
+                }
+                s = applySort(effectiveSort)(s);
+                // Collect via applyCursor (extracts cursor values from pre-select items)
+                const cursorResult = yield* applyCursor(cursorConfig)(s);
+                // Apply select to collected items (after cursor extraction)
+                const selectedItems = applySelectToArray(cursorResult.items, options?.select);
+                // Return CursorPageResult with projected items but original cursor metadata
+                return {
+                    items: selectedItems,
+                    pageInfo: cursorResult.pageInfo,
+                };
+            });
+            return withCursorRunPromise(cursorEffect);
+        }
+        // Standard stream branch: populate → resolve computed → filter → sort → paginate → select
+        const stream = Stream.unwrap(Effect.gen(function* () {
+            const map = yield* Ref.get(ref);
+            // Try index-accelerated lookup first (equality index, then search index)
+            let narrowed = indexes
+                ? yield* resolveWithIndex(options?.where, indexes, map)
+                : undefined;
+            // If equality index didn't help, try search index
+            if (narrowed === undefined && searchIndexRef) {
+                narrowed = yield* resolveWithSearchIndex(options?.where, searchIndexRef, searchIndexFields, map);
+            }
+            const items = (narrowed ?? Array.from(map.values()));
+            let s = Stream.fromIterable(items);
+            // Apply pipeline stages: populate → resolve computed (with lazy skip) → filter → sort → paginate → select
+            s = applyPopulate(populateConfig, stateRefs, dbConfig, collectionName)(s);
+            s = resolveComputedStreamWithLazySkip(computed, normalizeSelectForLazySkip(options?.select))(s);
+            s = applyFilter(options?.where, customOperators)(s);
+            // When $search is active, compute and attach relevance scores after filtering
+            const searchConfig = extractSearchConfig(options?.where);
+            if (searchConfig) {
+                s = attachSearchScores(searchConfig)(s);
+            }
+            // When $search is active and no explicit sort provided, use relevance sort
+            if (searchConfig && !options?.sort) {
+                s = applyRelevanceSort(searchConfig)(s);
+            }
+            else {
+                s = applySort(effectiveSort)(s);
+            }
+            s = applyPagination(options?.offset, options?.limit)(s);
+            s = applySelect(options?.select)(s);
+            return s;
+        }));
+        return withStreamRunPromise(stream);
+    };
+    // Helper to wrap a function so its return value gets .runPromise.
+    // When afterMutation is configured, each CRUD method triggers a
+    // persistence save schedule after the mutation succeeds (synchronous,
+    // non-blocking — the actual save runs in a debounced setTimeout).
+    const wrapEffect = (fn) => (...args) => {
+        const effect = afterMutation
+            ? fn(...args).pipe(Effect.tap(() => afterMutation()))
+            : fn(...args);
+        return withRunPromise(effect);
+    };
+    // Helper to create a forbidden operation for append-only collections
+    const forbiddenOp = (opName) => (..._args) => withRunPromise(Effect.fail(new OperationError({
+        operation: opName,
+        reason: "append-only",
+        message: `Operation '${opName}' is not allowed on append-only collection '${collectionName}'`,
+    })));
+    // Wire CRUD operations with runPromise convenience
+    const rawCreate = create(collectionName, schema, relationships, ref, stateRefs, indexes, hooks, uniqueFields, computed, searchIndexRef, searchIndexFields, idGeneratorName, idGeneratorMap, changePubSub);
+    const rawCreateMany = createMany(collectionName, schema, relationships, ref, stateRefs, indexes, hooks, uniqueFields, computed, searchIndexRef, searchIndexFields, idGeneratorName, idGeneratorMap, changePubSub);
+    // For append-only: wrap create to also append each entity to the file
+    const createFn = appendOnlyConfig
+        ? (...args) => {
+            const effect = rawCreate(...args).pipe(Effect.tap((entity) => appendOnlyConfig.onEntityCreated(entity)));
+            return withRunPromise(effect);
+        }
+        : wrapEffect(rawCreate);
+    const createManyFn = appendOnlyConfig
+        ? (...args) => {
+            const effect = rawCreateMany(...args).pipe(Effect.tap((result) => Effect.forEach(result.created, (entity) => appendOnlyConfig.onEntityCreated(entity))));
+            return withRunPromise(effect);
+        }
+        : wrapEffect(rawCreateMany);
+    // For append-only: update/updateMany/delete/deleteMany/upsert/upsertMany are forbidden
+    const updateFn = appendOnlyConfig
+        ? // biome-ignore lint/suspicious/noExplicitAny: forbidden ops return OperationError regardless of input type
+            forbiddenOp("update")
+        : wrapEffect(update(collectionName, schema, relationships, ref, stateRefs, indexes, hooks, uniqueFields, computed, searchIndexRef, searchIndexFields, changePubSub));
+    const updateManyFn = appendOnlyConfig
+        ? // biome-ignore lint/suspicious/noExplicitAny: forbidden ops return OperationError regardless of input type
+            forbiddenOp("updateMany")
+        : wrapEffect(updateMany(collectionName, schema, relationships, ref, stateRefs, indexes, hooks, uniqueFields, computed, searchIndexRef, searchIndexFields, changePubSub));
+    // Check if schema defines a deletedAt field for soft delete support
+    const supportsSoftDelete = "fields" in schema &&
+        "deletedAt" in
+            schema
+                .fields;
+    const deleteFn = appendOnlyConfig
+        ? // biome-ignore lint/suspicious/noExplicitAny: forbidden ops return OperationError regardless of input type
+            forbiddenOp("delete")
+        : wrapEffect(del(collectionName, allRelationships, ref, stateRefs, supportsSoftDelete, indexes, hooks, searchIndexRef, searchIndexFields, changePubSub));
+    const deleteManyFn = appendOnlyConfig
+        ? // biome-ignore lint/suspicious/noExplicitAny: forbidden ops return OperationError regardless of input type
+            forbiddenOp("deleteMany")
+        : wrapEffect(deleteMany(collectionName, allRelationships, ref, stateRefs, supportsSoftDelete, indexes, hooks, searchIndexRef, searchIndexFields, changePubSub));
+    const upsertFn = appendOnlyConfig
+        ? // biome-ignore lint/suspicious/noExplicitAny: forbidden ops return OperationError regardless of input type
+            forbiddenOp("upsert")
+        : wrapEffect(upsert(collectionName, schema, relationships, ref, stateRefs, indexes, hooks, uniqueFields, searchIndexRef, searchIndexFields, changePubSub));
+    const upsertManyFn = appendOnlyConfig
+        ? // biome-ignore lint/suspicious/noExplicitAny: forbidden ops return OperationError regardless of input type
+            forbiddenOp("upsertMany")
+        : wrapEffect(upsertMany(collectionName, schema, relationships, ref, stateRefs, indexes, hooks, uniqueFields, searchIndexRef, searchIndexFields, changePubSub));
+    const createWithRelsFn = wrapEffect(createWithRelationships(collectionName, schema, relationships, ref, stateRefs, dbConfig, computed, changePubSub));
+    const updateWithRelsFn = appendOnlyConfig
+        ? // biome-ignore lint/suspicious/noExplicitAny: forbidden ops return OperationError regardless of input type
+            forbiddenOp("updateWithRelationships")
+        : wrapEffect(updateWithRelationships(collectionName, schema, relationships, ref, stateRefs, dbConfig, computed, changePubSub));
+    const deleteWithRelsFn = appendOnlyConfig
+        ? // biome-ignore lint/suspicious/noExplicitAny: forbidden ops return OperationError regardless of input type
+            forbiddenOp("deleteWithRelationships")
+        : wrapEffect(deleteWithRelationships(collectionName, relationships, ref, stateRefs, dbConfig, changePubSub));
+    const deleteManyWithRelsFn = appendOnlyConfig
+        ? // biome-ignore lint/suspicious/noExplicitAny: forbidden ops return OperationError regardless of input type
+            forbiddenOp("deleteManyWithRelationships")
+        : wrapEffect(deleteManyWithRelationships(collectionName, relationships, ref, stateRefs, dbConfig, changePubSub));
+    // findById: O(1) lookup directly from the ReadonlyMap
+    const findByIdFn = (id) => {
+        const effect = Effect.gen(function* () {
+            const map = yield* Ref.get(ref);
+            const entity = map.get(id);
+            if (entity === undefined) {
+                return yield* new NotFoundError({
+                    collection: collectionName,
+                    id,
+                    message: `Entity with id "${id}" not found in collection "${collectionName}"`,
+                });
+            }
+            return entity;
+        });
+        return withRunPromise(effect);
+    };
+    // aggregate: read Ref → filter → collect → delegate to aggregate functions
+    const aggregateFn = (config) => {
+        const effect = Effect.gen(function* () {
+            // 1. Read Ref snapshot
+            const map = yield* Ref.get(ref);
+            const items = Array.from(map.values());
+            // 2. Create stream and apply filter
+            let s = Stream.fromIterable(items);
+            s = applyFilter(config.where, customOperators)(s);
+            // 3. Collect filtered entities
+            const chunk = yield* Stream.runCollect(s);
+            const entities = Chunk.toReadonlyArray(chunk);
+            // 4. Delegate to appropriate aggregate function based on groupBy presence
+            if (isGroupedAggregateConfig(config)) {
+                return computeGroupedAggregates(entities, config);
+            }
+            return computeAggregates(entities, config);
+        });
+        // Type assertion needed because TypeScript can't infer the conditional return type
+        return withRunPromise(effect);
+    };
+    // watch: create reactive subscription to query results
+    // Requires changePubSub to be available; throws if called within a transaction
+    const watchFn = (config) => {
+        if (changePubSub === undefined) {
+            // This happens when called within a transaction context
+            // Reactive queries aren't supported within transactions since transaction
+            // data is isolated and temporary (rolled back or committed atomically)
+            return Effect.die(new Error(`watch() is not supported within transactions. ` +
+                `Reactive queries can only be used on the main database collections.`));
+        }
+        return watch(changePubSub, ref, collectionName, config);
+    };
+    // watchById: create reactive subscription for a single entity
+    // Requires changePubSub to be available; throws if called within a transaction
+    const watchByIdFn = (id) => {
+        if (changePubSub === undefined) {
+            // This happens when called within a transaction context
+            // Reactive queries aren't supported within transactions since transaction
+            // data is isolated and temporary (rolled back or committed atomically)
+            return Effect.die(new Error(`watchById() is not supported within transactions. ` +
+                `Reactive queries can only be used on the main database collections.`));
+        }
+        return watchById(changePubSub, ref, collectionName, id);
+    };
+    return {
+        query: queryFn,
+        findById: findByIdFn,
+        create: createFn,
+        createMany: createManyFn,
+        update: updateFn,
+        updateMany: updateManyFn,
+        delete: deleteFn,
+        deleteMany: deleteManyFn,
+        upsert: upsertFn,
+        upsertMany: upsertManyFn,
+        createWithRelationships: createWithRelsFn,
+        updateWithRelationships: updateWithRelsFn,
+        deleteWithRelationships: deleteWithRelsFn,
+        deleteManyWithRelationships: deleteManyWithRelsFn,
+        aggregate: aggregateFn,
+        watch: watchFn,
+        watchById: watchByIdFn,
+    };
+};
+/**
+ * Create a `buildCollectionForTx` callback that mirrors `buildCollection` but
+ * accepts a transaction-aware `afterMutation`. The returned callback creates
+ * collection accessors that record mutations to the transaction's set instead
+ * of triggering persistence writes.
+ *
+ * Used by `createTransaction` and `$transaction` to provide collection accessors
+ * that participate in transaction semantics.
+ *
+ * @param config - The database configuration
+ * @param stateRefs - Shared state refs for cross-collection access
+ * @param typedRefs - Typed refs for each collection
+ * @param collectionIndexes - Pre-built indexes for each collection
+ * @param searchIndexRefs - Pre-built search indexes for each collection (optional)
+ * @param searchIndexFields - Fields covered by search index for each collection (optional)
+ * @param customOperators - Custom operators from plugins for query filtering (optional)
+ * @param idGeneratorMap - ID generators from plugins for custom ID generation (optional)
+ * @returns A callback matching the BuildCollectionForTx type
+ */
+const makeBuildCollectionForTx = (config, stateRefs, typedRefs, collectionIndexes, searchIndexRefs, searchIndexFields, customOperators, idGeneratorMap, globalHooks) => {
+    return (collectionName, addMutation) => {
+        // Transaction-aware afterMutation: records mutation instead of scheduling persistence
+        const afterMutation = () => Effect.sync(() => addMutation(collectionName));
+        // Explicitly pass undefined for changePubSub to suppress reactive events during transactions.
+        // Individual mutations within a transaction should not publish change events;
+        // events are only published after commit (see task 7.3).
+        return buildCollection(collectionName, config[collectionName], typedRefs[collectionName], stateRefs, config, afterMutation, collectionIndexes[collectionName], searchIndexRefs?.[collectionName], searchIndexFields?.[collectionName], customOperators, idGeneratorMap, globalHooks, undefined);
+    };
+};
+// ============================================================================
+// Database Factory
+// ============================================================================
+/**
+ * Create an Effect-based in-memory database.
+ *
+ * Accepts a DatabaseConfig and optional initial data (arrays keyed by collection name).
+ * Returns an Effect that initializes Ref state for each collection and wires up
+ * the query pipeline and CRUD methods.
+ *
+ * Optionally accepts plugins that provide custom codecs, operators, ID generators,
+ * and global lifecycle hooks.
+ *
+ * Usage:
+ * ```ts
+ * const db = yield* createEffectDatabase(config, {
+ *   users: [{ id: "1", name: "Alice", age: 30 }],
+ *   companies: [{ id: "c1", name: "TechCorp" }],
+ * })
+ *
+ * // Query
+ * const results = yield* Stream.runCollect(db.users.query({ where: { age: { $gt: 18 } } }))
+ *
+ * // CRUD
+ * const user = yield* db.users.create({ name: "Bob", age: 25 })
+ * ```
+ *
+ * With plugins:
+ * ```ts
+ * const db = yield* createEffectDatabase(config, initialData, {
+ *   plugins: [regexPlugin, snowflakeIdPlugin]
+ * })
+ * ```
+ */
+export const createEffectDatabase = (config, initialData, options) => Effect.gen(function* () {
+    // 0. Validate migration registries for all versioned collections at startup
+    for (const collectionName of Object.keys(config)) {
+        const collectionConfig = config[collectionName];
+        if (collectionConfig.version !== undefined) {
+            yield* validateMigrationRegistry(collectionName, collectionConfig.version, collectionConfig.migrations ?? []);
+        }
+    }
+    // 0b. Build plugin registry (validates plugins, merges contributions)
+    const pluginRegistry = yield* buildPluginRegistry(options?.plugins);
+    // 0c. Validate that all idGenerator references in collection configs exist
+    yield* validateIdGeneratorReferences(config, pluginRegistry.idGenerators);
+    // 0d. Run plugin initialize effects
+    for (const plugin of options?.plugins ?? []) {
+        if (plugin.initialize !== undefined) {
+            yield* plugin.initialize();
+        }
+    }
+    // 1. Create transaction lock for single-writer isolation
+    const transactionLock = yield* Ref.make(false);
+    // 2. Create Ref for each collection from initial data
+    const stateRefs = {};
+    const typedRefs = {};
+    for (const collectionName of Object.keys(config)) {
+        const items = (initialData?.[collectionName] ??
+            []);
+        const map = new Map(items.map((item) => [item.id, item]));
+        const ref = yield* Ref.make(map);
+        stateRefs[collectionName] = ref;
+        typedRefs[collectionName] = ref;
+    }
+    // 3. Build indexes for each collection from initial data
+    const collectionIndexes = {};
+    for (const collectionName of Object.keys(config)) {
+        const collectionConfig = config[collectionName];
+        const normalizedIndexes = normalizeIndexes(collectionConfig.indexes);
+        const items = (initialData?.[collectionName] ??
+            []);
+        const indexes = yield* buildIndexes(normalizedIndexes, items);
+        collectionIndexes[collectionName] = indexes;
+    }
+    // 3b. Build search indexes for collections that have searchIndex configured
+    const searchIndexRefs = {};
+    const searchIndexFields = {};
+    for (const collectionName of Object.keys(config)) {
+        const collectionConfig = config[collectionName];
+        if (collectionConfig.searchIndex &&
+            collectionConfig.searchIndex.length > 0) {
+            const items = (initialData?.[collectionName] ??
+                []);
+            const searchIdx = yield* buildSearchIndex(collectionConfig.searchIndex, items);
+            searchIndexRefs[collectionName] = searchIdx;
+            searchIndexFields[collectionName] = collectionConfig.searchIndex;
+        }
+    }
+    // 4. Create shared PubSub for reactive change notifications (one per database)
+    // This PubSub is shared by all collections and broadcasts ChangeEvents to reactive subscribers
+    const changePubSub = yield* PubSub.unbounded();
+    // 5. Build each collection with its Ref, indexes, and shared state refs
+    // Pass plugin registry data: custom operators, ID generators, and global hooks
+    // Pass the shared changePubSub so CRUD operations publish ChangeEvents
+    const collections = {};
+    for (const collectionName of Object.keys(config)) {
+        collections[collectionName] = buildCollection(collectionName, config[collectionName], typedRefs[collectionName], stateRefs, config, undefined, // afterMutation
+        collectionIndexes[collectionName], searchIndexRefs[collectionName], searchIndexFields[collectionName], pluginRegistry.operators, pluginRegistry.idGenerators, pluginRegistry.globalHooks, changePubSub);
+    }
+    // 5. Build transaction support
+    const buildCollectionForTx = makeBuildCollectionForTx(config, stateRefs, typedRefs, collectionIndexes, searchIndexRefs, searchIndexFields, pluginRegistry.operators, pluginRegistry.idGenerators, pluginRegistry.globalHooks);
+    // Create the $transaction method
+    const $transactionMethod = (fn) => $transactionImpl(stateRefs, transactionLock, buildCollectionForTx, undefined, // no persistence trigger for in-memory database
+    changePubSub, // shared PubSub for reactive change notifications
+    fn);
+    // Return database with $transaction method
+    return Object.assign(collections, {
+        $transaction: $transactionMethod,
+    });
+});
+/**
+ * Create an Effect-based in-memory database with persistence.
+ *
+ * Like `createEffectDatabase`, but additionally wires debounced persistence hooks
+ * so that each CRUD mutation triggers a fire-and-forget save to disk.
+ *
+ * Collections with a `file` field in their config are persisted. Collections
+ * without a `file` are in-memory only.
+ *
+ * Requires `StorageAdapter` and `SerializerRegistry` services in the environment.
+ *
+ * Optionally accepts plugins that provide custom codecs, operators, ID generators,
+ * and global lifecycle hooks.
+ *
+ * Usage:
+ * ```ts
+ * const db = yield* createPersistentEffectDatabase(config, initialData, { writeDebounce: 200 })
+ * // CRUD mutations now trigger debounced saves
+ * yield* db.users.create({ name: "Alice", age: 30 })
+ * // Flush all pending writes before shutdown
+ * yield* db.flush()
+ * ```
+ *
+ * With plugins:
+ * ```ts
+ * const db = yield* createPersistentEffectDatabase(config, initialData, persistenceConfig, {
+ *   plugins: [regexPlugin, snowflakeIdPlugin]
+ * })
+ * ```
+ */
+export const createPersistentEffectDatabase = (config, initialData, persistenceConfig, options) => Effect.gen(function* () {
+    // 0. Validate migration registries for all versioned collections at startup
+    for (const collectionName of Object.keys(config)) {
+        const collectionConfig = config[collectionName];
+        if (collectionConfig.version !== undefined) {
+            yield* validateMigrationRegistry(collectionName, collectionConfig.version, collectionConfig.migrations ?? []);
+        }
+    }
+    // 0b. Build plugin registry (validates plugins, merges contributions)
+    const pluginRegistry = yield* buildPluginRegistry(options?.plugins);
+    // 0c. Validate that all idGenerator references in collection configs exist
+    yield* validateIdGeneratorReferences(config, pluginRegistry.idGenerators);
+    // 0d. Run plugin initialize effects
+    for (const plugin of options?.plugins ?? []) {
+        if (plugin.initialize !== undefined) {
+            yield* plugin.initialize();
+        }
+    }
+    // 0e. Register plugin shutdown effects as scope finalizers.
+    // Register BEFORE the flush finalizer so they run AFTER flush (LIFO order).
+    // Iterate in registration order so that when finalizers run in LIFO order,
+    // plugins shut down in reverse registration order (last registered = first to shut down).
+    // This matches typical dependency patterns: if plugin A depends on plugin B,
+    // B is registered first, so A (registered later) shuts down first.
+    for (const plugin of options?.plugins ?? []) {
+        if (plugin.shutdown !== undefined) {
+            // Capture shutdown function to avoid optional chaining in the finalizer
+            const shutdownFn = plugin.shutdown;
+            yield* Effect.addFinalizer(() => shutdownFn().pipe(Effect.catchAll(() => Effect.void)));
+        }
+    }
+    // 1. Resolve services from the environment and capture as a Layer
+    // so save effects can be executed outside the creation runtime.
+    const storageAdapter = yield* StorageAdapter;
+    const baseSerializerRegistry = yield* SerializerRegistry;
+    // 1a. Merge plugin codecs with the base serializer registry
+    // Plugin codecs from the registry take precedence, followed by any codecs
+    // passed via _pluginCodecs (for backwards compatibility)
+    const allPluginCodecs = [
+        ...pluginRegistry.codecs,
+        ...(persistenceConfig?._pluginCodecs ?? []),
+    ];
+    const serializerRegistry = allPluginCodecs.length > 0
+        ? mergeSerializerWithPluginCodecs(baseSerializerRegistry, allPluginCodecs)
+        : baseSerializerRegistry;
+    const serviceLayer = Layer.merge(Layer.succeed(StorageAdapter, storageAdapter), Layer.succeed(SerializerRegistry, serializerRegistry));
+    // 2. Create transaction lock for single-writer isolation
+    const transactionLock = yield* Ref.make(false);
+    // 3. Load data from files for persistent collections, then merge with initialData.
+    // initialData takes precedence (allows overriding file data for testing/seeding).
+    const stateRefs = {};
+    const typedRefs = {};
+    for (const collectionName of Object.keys(config)) {
+        const collectionConfig = config[collectionName];
+        const filePath = collectionConfig.file;
+        // Load from file if configured, passing version and migrations for auto-migration
+        let loadedData = new Map();
+        if (filePath) {
+            // Only pass version options when collection is versioned
+            // Build options object conditionally to satisfy exactOptionalPropertyTypes
+            const loadOptions = collectionConfig.version !== undefined
+                ? collectionConfig.migrations !== undefined
+                    ? {
+                        version: collectionConfig.version,
+                        migrations: collectionConfig.migrations,
+                        collectionName,
+                    }
+                    : { version: collectionConfig.version, collectionName }
+                : undefined;
+            loadedData = yield* loadData(filePath, collectionConfig.schema, loadOptions);
+        }
+        // Merge with initialData (initialData takes precedence)
+        const providedItems = (initialData?.[collectionName] ??
+            []);
+        const mergedMap = new Map(loadedData);
+        for (const item of providedItems) {
+            mergedMap.set(item.id, item);
+        }
+        const ref = yield* Ref.make(mergedMap);
+        stateRefs[collectionName] = ref;
+        typedRefs[collectionName] = ref;
+    }
+    // 4. Build indexes for each collection from loaded/merged data
+    const collectionIndexes = {};
+    for (const collectionName of Object.keys(config)) {
+        const collectionConfig = config[collectionName];
+        const normalizedIndexes = normalizeIndexes(collectionConfig.indexes);
+        // Use actual data from the Ref (loaded from file + initialData)
+        const dataMap = yield* Ref.get(typedRefs[collectionName]);
+        const items = Array.from(dataMap.values());
+        const indexes = yield* buildIndexes(normalizedIndexes, items);
+        collectionIndexes[collectionName] = indexes;
+    }
+    // 4b. Build search indexes for collections that have searchIndex configured
+    const searchIndexRefs = {};
+    const searchIndexFields = {};
+    for (const collectionName of Object.keys(config)) {
+        const collectionConfig = config[collectionName];
+        if (collectionConfig.searchIndex &&
+            collectionConfig.searchIndex.length > 0) {
+            // Use actual data from the Ref (loaded from file + initialData)
+            const dataMap = yield* Ref.get(typedRefs[collectionName]);
+            const items = Array.from(dataMap.values());
+            const searchIdx = yield* buildSearchIndex(collectionConfig.searchIndex, items);
+            searchIndexRefs[collectionName] = searchIdx;
+            searchIndexFields[collectionName] = collectionConfig.searchIndex;
+        }
+    }
+    // 5. Build the save effect factory. Each save reads the Ref at execution
+    // time (capturing latest state) and writes through saveData with services.
+    const collectionFilePaths = {};
+    for (const collectionName of Object.keys(config)) {
+        const filePath = config[collectionName].file;
+        if (filePath) {
+            collectionFilePaths[collectionName] = filePath;
+        }
+    }
+    const makeSaveEffect = (collectionName) => {
+        const filePath = collectionFilePaths[collectionName];
+        if (!filePath)
+            return Effect.void;
+        const collectionConfig = config[collectionName];
+        return Effect.provide(Effect.gen(function* () {
+            const currentData = yield* Ref.get(typedRefs[collectionName]);
+            yield* saveData(filePath, collectionConfig.schema, currentData, 
+            // Pass version option to stamp _version in output for versioned collections
+            collectionConfig.version !== undefined
+                ? { version: collectionConfig.version }
+                : undefined);
+        }), serviceLayer);
+    };
+    // 6. Create the runtime-independent persistence trigger
+    const trigger = createPersistenceTrigger(persistenceConfig?.writeDebounce ?? 100, makeSaveEffect);
+    // 7. Register scope finalizer: flush pending writes and shut down timers
+    yield* Effect.addFinalizer(() => Effect.promise(() => trigger.flush()).pipe(Effect.catchAll(() => Effect.void), Effect.tap(() => Effect.sync(() => trigger.shutdown()))));
+    // 8. Create shared PubSub for reactive change notifications (one per database)
+    // This PubSub is shared by all collections and broadcasts ChangeEvents to reactive subscribers
+    const changePubSub = yield* PubSub.unbounded();
+    // 9. Build each collection with its Ref, indexes, state refs, and persistence hooks
+    // Pass the shared changePubSub so CRUD operations publish ChangeEvents
+    const collections = {};
+    for (const collectionName of Object.keys(config)) {
+        const collectionConfig = config[collectionName];
+        const filePath = collectionConfig.file;
+        const isAppendOnly = collectionConfig.appendOnly === true;
+        // For append-only collections, afterMutation is undefined (no debounced full-file save).
+        // Instead, each create appends a single JSONL line via appendOnlyConfig.
+        const afterMutation = filePath && !isAppendOnly
+            ? () => Ref.get(transactionLock).pipe(Effect.flatMap((isLocked) => isLocked
+                ? Effect.void // Skip persistence during transactions
+                : Effect.sync(() => trigger.schedule(collectionName))))
+            : undefined;
+        // Build appendOnlyConfig for append-only persistent collections
+        let appendOnlyConfig;
+        if (isAppendOnly && filePath) {
+            const appendSchema = collectionConfig.schema;
+            appendOnlyConfig = {
+                onEntityCreated: (entity) => Effect.provide(Effect.gen(function* () {
+                    const storage = yield* StorageAdapter;
+                    const encode = Schema.encode(appendSchema);
+                    const encoded = yield* encode(entity).pipe(Effect.catchAll(() => Effect.succeed(entity)));
+                    const line = `${JSON.stringify(encoded)}\n`;
+                    yield* storage.ensureDir(filePath);
+                    yield* storage.append(filePath, line);
+                }).pipe(
+                // Catch storage errors to avoid propagating them to the caller.
+                // The entity was already created in memory; storage failure is logged but not fatal.
+                Effect.catchAll(() => Effect.void)), serviceLayer),
+            };
+        }
+        collections[collectionName] = buildCollection(collectionName, collectionConfig, typedRefs[collectionName], stateRefs, config, afterMutation, collectionIndexes[collectionName], searchIndexRefs[collectionName], searchIndexFields[collectionName], pluginRegistry.operators, pluginRegistry.idGenerators, pluginRegistry.globalHooks, changePubSub, appendOnlyConfig);
+    }
+    const db = collections;
+    // 10. Create file watchers for persistent collections to detect external file changes
+    // Each watcher monitors its file and reloads data into the Ref on changes,
+    // publishing a reload event to the changePubSub for reactive query subscribers.
+    // File watching is best-effort: if the storage adapter doesn't support watching
+    // (e.g., browser adapters in test environments), the database still functions
+    // without reactive file change detection.
+    for (const collectionName of Object.keys(config)) {
+        const collectionConfig = config[collectionName];
+        const filePath = collectionConfig.file;
+        if (filePath) {
+            yield* createFileWatcher({
+                filePath,
+                schema: collectionConfig.schema,
+                ref: typedRefs[collectionName],
+                changePubSub,
+                collectionName,
+            }).pipe(Effect.catchAll(() => Effect.void));
+        }
+    }
+    // Build the $dryRunMigrations method
+    const dryRunMigrationsFn = () => {
+        const effect = Effect.provide(dryRunMigrations(config, stateRefs), serviceLayer);
+        return withRunPromise(effect);
+    };
+    // Build transaction support
+    const buildCollectionForTx = makeBuildCollectionForTx(config, stateRefs, typedRefs, collectionIndexes, searchIndexRefs, searchIndexFields, pluginRegistry.operators, pluginRegistry.idGenerators, pluginRegistry.globalHooks);
+    // Create the $transaction method with persistence trigger
+    const $transactionMethod = (fn) => $transactionImpl(stateRefs, transactionLock, buildCollectionForTx, trigger, // persistence trigger for debounced saves on commit
+    changePubSub, // shared PubSub for reactive change notifications
+    fn);
+    // Build the flush method: flushes debounced writes AND writes canonical files
+    // for append-only collections (which don't use the debounced trigger).
+    const flushAll = async () => {
+        // Flush debounced writes for non-append-only collections
+        await trigger.flush();
+        // Write canonical JSONL files for append-only collections
+        const appendOnlyFlushes = [];
+        for (const collectionName of Object.keys(config)) {
+            const cc = config[collectionName];
+            if (cc.appendOnly && cc.file) {
+                appendOnlyFlushes.push(Effect.runPromise(makeSaveEffect(collectionName).pipe(Effect.catchAll(() => Effect.void))));
+            }
+        }
+        await Promise.all(appendOnlyFlushes);
+    };
+    return Object.assign(db, {
+        flush: flushAll,
+        pendingCount: () => trigger.pendingCount(),
+        $dryRunMigrations: dryRunMigrationsFn,
+        $transaction: $transactionMethod,
+    });
+});
+//# sourceMappingURL=database-effect.js.map