@proseql/core 0.1.0

package/dist/indexes/search-index.d.ts

@@ -0,0 +1,179 @@
+/**
+ * Search index management for full-text search.
+ *
+ * Provides functions for building and maintaining an inverted index
+ * that maps tokens to entity IDs for fast text search queries.
+ *
+ * Follows the same Ref-based pattern as index-manager.ts for consistency.
+ */
+import { Effect, Ref } from "effect";
+import type { SearchIndexMap } from "../types/search-types.js";
+/**
+ * Entity constraint: must have a readonly string `id` field.
+ */
+type HasId = {
+    readonly id: string;
+};
+/**
+ * Build a search index from entities for the specified fields.
+ *
+ * Creates an inverted index mapping tokens to sets of entity IDs.
+ * For each entity, tokenizes the values of the specified fields
+ * and adds the entity's ID to each token's set.
+ *
+ * @param fields - The fields to index for full-text search
+ * @param entities - All entities in the collection
+ * @returns Effect producing a Ref containing the SearchIndexMap
+ *
+ * @example
+ * ```ts
+ * const books = [
+ *   { id: "1", title: "Dune", author: "Frank Herbert" },
+ *   { id: "2", title: "Neuromancer", author: "William Gibson" },
+ * ]
+ *
+ * const indexRef = yield* buildSearchIndex(["title", "author"], books)
+ * // Index structure:
+ * // "dune" -> Set(["1"])
+ * // "frank" -> Set(["1"])
+ * // "herbert" -> Set(["1"])
+ * // "neuromancer" -> Set(["2"])
+ * // "william" -> Set(["2"])
+ * // "gibson" -> Set(["2"])
+ * ```
+ */
+export declare const buildSearchIndex: <T extends HasId>(fields: ReadonlyArray<string>, entities: ReadonlyArray<T>) => Effect.Effect<Ref.Ref<SearchIndexMap>>;
+/**
+ * Lookup candidate entity IDs from the search index for a given query.
+ *
+ * For each query token:
+ * - Finds exact matches in the index
+ * - Finds prefix matches (index tokens that start with the query token)
+ *
+ * Returns the intersection of ID sets across all query tokens (AND semantics).
+ * If a query token has no matches, returns an empty set.
+ * If queryTokens is empty, returns an empty set.
+ *
+ * @param indexRef - Ref containing the SearchIndexMap
+ * @param queryTokens - Tokenized query terms to search for
+ * @returns Effect producing a Set of candidate entity IDs
+ *
+ * @example
+ * ```ts
+ * // Index: "dune" -> Set(["1"]), "frank" -> Set(["1"]), "neuromancer" -> Set(["2"])
+ *
+ * // Exact match
+ * const ids = yield* lookupSearchIndex(indexRef, ["dune"])
+ * // → Set(["1"])
+ *
+ * // Prefix match
+ * const ids = yield* lookupSearchIndex(indexRef, ["neuro"])
+ * // → Set(["2"]) (matches "neuromancer")
+ *
+ * // Multi-token (AND semantics)
+ * const ids = yield* lookupSearchIndex(indexRef, ["dune", "frank"])
+ * // → Set(["1"]) (intersection)
+ *
+ * // No match
+ * const ids = yield* lookupSearchIndex(indexRef, ["xyz"])
+ * // → Set([])
+ * ```
+ */
+export declare const lookupSearchIndex: (indexRef: Ref.Ref<SearchIndexMap>, queryTokens: ReadonlyArray<string>) => Effect.Effect<Set<string>>;
+/**
+ * Resolve candidate entities using the search index when a search query is present.
+ *
+ * Checks if the where clause contains a $search operator (field-level or top-level).
+ * If found and the search index covers the queried fields, uses the index to
+ * narrow the candidate set before full filtering.
+ *
+ * Returns undefined if:
+ * - No $search operator is present
+ * - The search index doesn't cover the queried fields
+ * - The search index is empty
+ *
+ * @param where - The where clause from the query
+ * @param searchIndexRef - The search index Ref (or undefined if not configured)
+ * @param searchIndexFields - The fields covered by the search index (or undefined)
+ * @param map - The entity data map (id -> entity)
+ * @returns Effect producing Array<T> if index was used, undefined if no usable index
+ */
+export declare const resolveWithSearchIndex: <T extends HasId>(where: Record<string, unknown> | undefined, searchIndexRef: Ref.Ref<SearchIndexMap> | undefined, searchIndexFields: ReadonlyArray<string> | undefined, map: ReadonlyMap<string, T>) => Effect.Effect<ReadonlyArray<T> | undefined>;
+/**
+ * Add an entity to the search index.
+ *
+ * Tokenizes the entity's indexed fields and adds the entity ID to each
+ * token's set in the inverted index. This should be called after creating
+ * a new entity to keep the search index up to date.
+ *
+ * @param indexRef - Ref containing the SearchIndexMap
+ * @param entity - The entity to add to the index
+ * @param fields - The fields to index for full-text search
+ * @returns Effect that completes when the entity is added
+ *
+ * @example
+ * ```ts
+ * const newBook = { id: "5", title: "Snow Crash", author: "Neal Stephenson" }
+ * yield* addToSearchIndex(indexRef, newBook, ["title", "author"])
+ * // Index now contains:
+ * // "snow" -> Set([..., "5"])
+ * // "crash" -> Set([..., "5"])
+ * // "neal" -> Set([..., "5"])
+ * // "stephenson" -> Set([..., "5"])
+ * ```
+ */
+export declare const addToSearchIndex: <T extends HasId>(indexRef: Ref.Ref<SearchIndexMap>, entity: T, fields: ReadonlyArray<string>) => Effect.Effect<void>;
+/**
+ * Remove an entity from the search index.
+ *
+ * Tokenizes the entity's indexed fields and removes the entity ID from each
+ * token's set in the inverted index. Cleans up empty sets to avoid memory
+ * leaks. This should be called after deleting an entity to keep the search
+ * index up to date.
+ *
+ * @param indexRef - Ref containing the SearchIndexMap
+ * @param entity - The entity to remove from the index
+ * @param fields - The fields that were indexed for full-text search
+ * @returns Effect that completes when the entity is removed
+ *
+ * @example
+ * ```ts
+ * const bookToDelete = { id: "5", title: "Snow Crash", author: "Neal Stephenson" }
+ * yield* removeFromSearchIndex(indexRef, bookToDelete, ["title", "author"])
+ * // Index now has entity "5" removed from:
+ * // "snow", "crash", "neal", "stephenson" sets
+ * // Empty sets are deleted from the index
+ * ```
+ */
+export declare const removeFromSearchIndex: <T extends HasId>(indexRef: Ref.Ref<SearchIndexMap>, entity: T, fields: ReadonlyArray<string>) => Effect.Effect<void>;
+/**
+ * Update an entity in the search index.
+ *
+ * Efficiently handles updates by only reindexing fields that have changed.
+ * For changed fields, removes old tokens and adds new tokens. This should
+ * be called after updating an entity to keep the search index up to date.
+ *
+ * Optimization: If a field's value hasn't changed, no index operations are
+ * performed for that field. This is more efficient than a full remove+add
+ * when only some indexed fields are modified.
+ *
+ * @param indexRef - Ref containing the SearchIndexMap
+ * @param oldEntity - The entity before the update
+ * @param newEntity - The entity after the update
+ * @param fields - The fields that are indexed for full-text search
+ * @returns Effect that completes when the index is updated
+ *
+ * @example
+ * ```ts
+ * const oldBook = { id: "5", title: "Snow Crash", author: "Neal Stephenson" }
+ * const newBook = { id: "5", title: "Snow Crash (Revised)", author: "Neal Stephenson" }
+ * yield* updateInSearchIndex(indexRef, oldBook, newBook, ["title", "author"])
+ * // Only "title" changed, so:
+ * // - Removes "5" from "snow", "crash"
+ * // - Adds "5" to "snow", "crash", "revised"
+ * // - "author" field is unchanged, no operations needed
+ * ```
+ */
+export declare const updateInSearchIndex: <T extends HasId>(indexRef: Ref.Ref<SearchIndexMap>, oldEntity: T, newEntity: T, fields: ReadonlyArray<string>) => Effect.Effect<void>;
+export {};
+//# sourceMappingURL=search-index.d.ts.map

package/dist/indexes/search-index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"search-index.d.ts","sourceRoot":"","sources":["../../src/indexes/search-index.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,QAAQ,CAAC;AAErC,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAC;AAG/D;;GAEG;AACH,KAAK,KAAK,GAAG;IAAE,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAA;CAAE,CAAC;AAErC;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,eAAO,MAAM,gBAAgB,GAAI,CAAC,SAAS,KAAK,EAC/C,QAAQ,aAAa,CAAC,MAAM,CAAC,EAC7B,UAAU,aAAa,CAAC,CAAC,CAAC,KACxB,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,cAAc,CAAC,CASrC,CAAC;AAEJ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmCG;AACH,eAAO,MAAM,iBAAiB,GAC7B,UAAU,GAAG,CAAC,GAAG,CAAC,cAAc,CAAC,EACjC,aAAa,aAAa,CAAC,MAAM,CAAC,KAChC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,CA+CzB,CAAC;AAEJ;;;;;;;;;;;;;;;;;GAiBG;AACH,eAAO,MAAM,sBAAsB,GAAI,CAAC,SAAS,KAAK,EACrD,OAAO,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,EAC1C,gBAAgB,GAAG,CAAC,GAAG,CAAC,cAAc,CAAC,GAAG,SAAS,EACnD,mBAAmB,aAAa,CAAC,MAAM,CAAC,GAAG,SAAS,EACpD,KAAK,WAAW,CAAC,MAAM,EAAE,CAAC,CAAC,KACzB,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC,GAAG,SAAS,CA8C1C,CAAC;AA0GJ;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,eAAO,MAAM,gBAAgB,GAAI,CAAC,SAAS,KAAK,EAC/C,UAAU,GAAG,CAAC,GAAG,CAAC,cAAc,CAAC,EACjC,QAAQ,CAAC,EACT,QAAQ,aAAa,CAAC,MAAM,CAAC,KAC3B,MAAM,CAAC,MAAM,CAAC,IAAI,CAMlB,CAAC;AA2CJ;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,eAAO,MAAM,qBAAqB,GAAI,CAAC,SAAS,KAAK,EACpD,UAAU,GAAG,CAAC,GAAG,CAAC,cAAc,CAAC,EACjC,QAAQ,CAAC,EACT,QAAQ,aAAa,CAAC,MAAM,CAAC,KAC3B,MAAM,CAAC,MAAM,CAAC,IAAI,CAUlB,CAAC;AAEJ;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,eAAO,MAAM,mBAAmB,GAAI,CAAC,SAAS,KAAK,EAClD,UAAU,GAAG,CAAC,GAAG,CAAC,cAAc,CAAC,EACjC,WAAW,CAAC,EACZ,WAAW,CAAC,EACZ,QAAQ,aAAa,CAAC,MAAM,CAAC,KAC3B,MAAM,CAAC,MAAM,CAAC,IAAI,CAgClB,CAAC"}

package/dist/indexes/search-index.js

@@ -0,0 +1,405 @@
+/**
+ * Search index management for full-text search.
+ *
+ * Provides functions for building and maintaining an inverted index
+ * that maps tokens to entity IDs for fast text search queries.
+ *
+ * Follows the same Ref-based pattern as index-manager.ts for consistency.
+ */
+import { Effect, Ref } from "effect";
+import { tokenize } from "../operations/query/search.js";
+import { getNestedValue } from "../utils/nested-path.js";
+/**
+ * Build a search index from entities for the specified fields.
+ *
+ * Creates an inverted index mapping tokens to sets of entity IDs.
+ * For each entity, tokenizes the values of the specified fields
+ * and adds the entity's ID to each token's set.
+ *
+ * @param fields - The fields to index for full-text search
+ * @param entities - All entities in the collection
+ * @returns Effect producing a Ref containing the SearchIndexMap
+ *
+ * @example
+ * ```ts
+ * const books = [
+ *   { id: "1", title: "Dune", author: "Frank Herbert" },
+ *   { id: "2", title: "Neuromancer", author: "William Gibson" },
+ * ]
+ *
+ * const indexRef = yield* buildSearchIndex(["title", "author"], books)
+ * // Index structure:
+ * // "dune" -> Set(["1"])
+ * // "frank" -> Set(["1"])
+ * // "herbert" -> Set(["1"])
+ * // "neuromancer" -> Set(["2"])
+ * // "william" -> Set(["2"])
+ * // "gibson" -> Set(["2"])
+ * ```
+ */
+export const buildSearchIndex = (fields, entities) => Effect.gen(function* () {
+    const searchIndex = new Map();
+    for (const entity of entities) {
+        addEntityToIndexMut(searchIndex, entity, fields);
+    }
+    return yield* Ref.make(searchIndex);
+});
+/**
+ * Lookup candidate entity IDs from the search index for a given query.
+ *
+ * For each query token:
+ * - Finds exact matches in the index
+ * - Finds prefix matches (index tokens that start with the query token)
+ *
+ * Returns the intersection of ID sets across all query tokens (AND semantics).
+ * If a query token has no matches, returns an empty set.
+ * If queryTokens is empty, returns an empty set.
+ *
+ * @param indexRef - Ref containing the SearchIndexMap
+ * @param queryTokens - Tokenized query terms to search for
+ * @returns Effect producing a Set of candidate entity IDs
+ *
+ * @example
+ * ```ts
+ * // Index: "dune" -> Set(["1"]), "frank" -> Set(["1"]), "neuromancer" -> Set(["2"])
+ *
+ * // Exact match
+ * const ids = yield* lookupSearchIndex(indexRef, ["dune"])
+ * // → Set(["1"])
+ *
+ * // Prefix match
+ * const ids = yield* lookupSearchIndex(indexRef, ["neuro"])
+ * // → Set(["2"]) (matches "neuromancer")
+ *
+ * // Multi-token (AND semantics)
+ * const ids = yield* lookupSearchIndex(indexRef, ["dune", "frank"])
+ * // → Set(["1"]) (intersection)
+ *
+ * // No match
+ * const ids = yield* lookupSearchIndex(indexRef, ["xyz"])
+ * // → Set([])
+ * ```
+ */
+export const lookupSearchIndex = (indexRef, queryTokens) => Effect.gen(function* () {
+    // Empty query returns empty set
+    if (queryTokens.length === 0) {
+        return new Set();
+    }
+    const index = yield* Ref.get(indexRef);
+    // For each query token, find all matching IDs (exact + prefix)
+    const tokenMatchSets = [];
+    for (const queryToken of queryTokens) {
+        const matchingIds = new Set();
+        // Check each token in the index for exact or prefix match
+        for (const [indexToken, entityIds] of index) {
+            if (indexToken === queryToken || indexToken.startsWith(queryToken)) {
+                // Union all matching entity IDs for this query token
+                for (const id of entityIds) {
+                    matchingIds.add(id);
+                }
+            }
+        }
+        tokenMatchSets.push(matchingIds);
+    }
+    // Intersect all token match sets (AND semantics)
+    // If any token has no matches, the result is empty
+    if (tokenMatchSets.length === 0) {
+        return new Set();
+    }
+    // Start with the first set and intersect with the rest
+    const result = new Set(tokenMatchSets[0]);
+    for (let i = 1; i < tokenMatchSets.length; i++) {
+        const currentSet = tokenMatchSets[i];
+        for (const id of result) {
+            if (!currentSet.has(id)) {
+                result.delete(id);
+            }
+        }
+    }
+    return result;
+});
+/**
+ * Resolve candidate entities using the search index when a search query is present.
+ *
+ * Checks if the where clause contains a $search operator (field-level or top-level).
+ * If found and the search index covers the queried fields, uses the index to
+ * narrow the candidate set before full filtering.
+ *
+ * Returns undefined if:
+ * - No $search operator is present
+ * - The search index doesn't cover the queried fields
+ * - The search index is empty
+ *
+ * @param where - The where clause from the query
+ * @param searchIndexRef - The search index Ref (or undefined if not configured)
+ * @param searchIndexFields - The fields covered by the search index (or undefined)
+ * @param map - The entity data map (id -> entity)
+ * @returns Effect producing Array<T> if index was used, undefined if no usable index
+ */
+export const resolveWithSearchIndex = (where, searchIndexRef, searchIndexFields, map) => Effect.gen(function* () {
+    // No where clause or no search index configured
+    if (!where ||
+        !searchIndexRef ||
+        !searchIndexFields ||
+        searchIndexFields.length === 0) {
+        return undefined;
+    }
+    // Extract search query from where clause
+    const searchInfo = extractSearchFromWhere(where, searchIndexFields);
+    if (!searchInfo) {
+        return undefined;
+    }
+    const { queryTokens, queriedFields } = searchInfo;
+    // Check if the search index covers all queried fields
+    const indexCovered = queriedFields.every((field) => searchIndexFields.includes(field));
+    if (!indexCovered) {
+        return undefined;
+    }
+    // Use the search index to get candidate entity IDs
+    const candidateIds = yield* lookupSearchIndex(searchIndexRef, queryTokens);
+    // Empty result set means no candidates
+    if (candidateIds.size === 0) {
+        return [];
+    }
+    // Load candidate entities from the map
+    const entities = [];
+    for (const id of candidateIds) {
+        const entity = map.get(id);
+        if (entity !== undefined) {
+            entities.push(entity);
+        }
+    }
+    return entities;
+});
+/**
+ * Extract search query info from a where clause.
+ *
+ * Looks for:
+ * 1. Top-level $search: { query: "...", fields?: [...] }
+ * 2. Field-level $search: { fieldName: { $search: "..." } }
+ *
+ * Returns the tokenized query and the fields being searched, or undefined if no search.
+ */
+const extractSearchFromWhere = (where, defaultFields) => {
+    // Check for top-level $search
+    if ("$search" in where) {
+        const searchValue = where.$search;
+        if (searchValue !== null && typeof searchValue === "object") {
+            const config = searchValue;
+            if (typeof config.query === "string") {
+                const queryTokens = tokenize(config.query);
+                if (queryTokens.length === 0) {
+                    return undefined; // Empty query matches everything, no index help
+                }
+                const queriedFields = config.fields && config.fields.length > 0
+                    ? config.fields
+                    : defaultFields;
+                return { queryTokens, queriedFields };
+            }
+        }
+    }
+    // Check for field-level $search on any field
+    for (const [field, value] of Object.entries(where)) {
+        // Skip logical operators
+        if (field === "$or" ||
+            field === "$and" ||
+            field === "$not" ||
+            field === "$search") {
+            continue;
+        }
+        if (value !== null && typeof value === "object" && !Array.isArray(value)) {
+            const obj = value;
+            if ("$search" in obj && typeof obj.$search === "string") {
+                const queryTokens = tokenize(obj.$search);
+                if (queryTokens.length === 0) {
+                    continue; // Empty query, skip
+                }
+                // Field-level search applies to just this field
+                return { queryTokens, queriedFields: [field] };
+            }
+        }
+    }
+    return undefined;
+};
+/**
+ * Helper function to add a single entity to a search index map.
+ *
+ * Tokenizes the values of the specified fields and adds the entity's ID
+ * to each token's set in the index.
+ *
+ * @param index - The search index map to mutate
+ * @param entity - The entity to add
+ * @param fields - The fields to index
+ */
+const addEntityToIndexMut = (index, entity, fields) => {
+    const entityRecord = entity;
+    const entityId = entity.id;
+    for (const field of fields) {
+        const fieldValue = getNestedValue(entityRecord, field);
+        // Only index string fields
+        if (typeof fieldValue !== "string") {
+            continue;
+        }
+        // Tokenize the field value and add to index
+        const tokens = tokenize(fieldValue);
+        for (const token of tokens) {
+            const existingSet = index.get(token);
+            if (existingSet) {
+                existingSet.add(entityId);
+            }
+            else {
+                index.set(token, new Set([entityId]));
+            }
+        }
+    }
+};
+/**
+ * Add an entity to the search index.
+ *
+ * Tokenizes the entity's indexed fields and adds the entity ID to each
+ * token's set in the inverted index. This should be called after creating
+ * a new entity to keep the search index up to date.
+ *
+ * @param indexRef - Ref containing the SearchIndexMap
+ * @param entity - The entity to add to the index
+ * @param fields - The fields to index for full-text search
+ * @returns Effect that completes when the entity is added
+ *
+ * @example
+ * ```ts
+ * const newBook = { id: "5", title: "Snow Crash", author: "Neal Stephenson" }
+ * yield* addToSearchIndex(indexRef, newBook, ["title", "author"])
+ * // Index now contains:
+ * // "snow" -> Set([..., "5"])
+ * // "crash" -> Set([..., "5"])
+ * // "neal" -> Set([..., "5"])
+ * // "stephenson" -> Set([..., "5"])
+ * ```
+ */
+export const addToSearchIndex = (indexRef, entity, fields) => Ref.update(indexRef, (index) => {
+    // Clone the index to avoid mutating the original
+    const newIndex = new Map(index);
+    addEntityToIndexMut(newIndex, entity, fields);
+    return newIndex;
+});
+/**
+ * Helper function to remove a single entity from a search index map.
+ *
+ * Tokenizes the values of the specified fields and removes the entity's ID
+ * from each token's set in the index. Cleans up empty sets.
+ *
+ * @param index - The search index map to mutate
+ * @param entity - The entity to remove
+ * @param fields - The fields that were indexed
+ */
+const removeEntityFromIndexMut = (index, entity, fields) => {
+    const entityRecord = entity;
+    const entityId = entity.id;
+    for (const field of fields) {
+        const fieldValue = getNestedValue(entityRecord, field);
+        // Only process string fields
+        if (typeof fieldValue !== "string") {
+            continue;
+        }
+        // Tokenize the field value and remove from index
+        const tokens = tokenize(fieldValue);
+        for (const token of tokens) {
+            const existingSet = index.get(token);
+            if (existingSet) {
+                existingSet.delete(entityId);
+                // Clean up empty sets
+                if (existingSet.size === 0) {
+                    index.delete(token);
+                }
+            }
+        }
+    }
+};
+/**
+ * Remove an entity from the search index.
+ *
+ * Tokenizes the entity's indexed fields and removes the entity ID from each
+ * token's set in the inverted index. Cleans up empty sets to avoid memory
+ * leaks. This should be called after deleting an entity to keep the search
+ * index up to date.
+ *
+ * @param indexRef - Ref containing the SearchIndexMap
+ * @param entity - The entity to remove from the index
+ * @param fields - The fields that were indexed for full-text search
+ * @returns Effect that completes when the entity is removed
+ *
+ * @example
+ * ```ts
+ * const bookToDelete = { id: "5", title: "Snow Crash", author: "Neal Stephenson" }
+ * yield* removeFromSearchIndex(indexRef, bookToDelete, ["title", "author"])
+ * // Index now has entity "5" removed from:
+ * // "snow", "crash", "neal", "stephenson" sets
+ * // Empty sets are deleted from the index
+ * ```
+ */
+export const removeFromSearchIndex = (indexRef, entity, fields) => Ref.update(indexRef, (index) => {
+    // Clone the index to avoid mutating the original
+    const newIndex = new Map(index);
+    // Also clone the sets that we'll modify to maintain immutability
+    for (const [token, idSet] of index) {
+        newIndex.set(token, new Set(idSet));
+    }
+    removeEntityFromIndexMut(newIndex, entity, fields);
+    return newIndex;
+});
+/**
+ * Update an entity in the search index.
+ *
+ * Efficiently handles updates by only reindexing fields that have changed.
+ * For changed fields, removes old tokens and adds new tokens. This should
+ * be called after updating an entity to keep the search index up to date.
+ *
+ * Optimization: If a field's value hasn't changed, no index operations are
+ * performed for that field. This is more efficient than a full remove+add
+ * when only some indexed fields are modified.
+ *
+ * @param indexRef - Ref containing the SearchIndexMap
+ * @param oldEntity - The entity before the update
+ * @param newEntity - The entity after the update
+ * @param fields - The fields that are indexed for full-text search
+ * @returns Effect that completes when the index is updated
+ *
+ * @example
+ * ```ts
+ * const oldBook = { id: "5", title: "Snow Crash", author: "Neal Stephenson" }
+ * const newBook = { id: "5", title: "Snow Crash (Revised)", author: "Neal Stephenson" }
+ * yield* updateInSearchIndex(indexRef, oldBook, newBook, ["title", "author"])
+ * // Only "title" changed, so:
+ * // - Removes "5" from "snow", "crash"
+ * // - Adds "5" to "snow", "crash", "revised"
+ * // - "author" field is unchanged, no operations needed
+ * ```
+ */
+export const updateInSearchIndex = (indexRef, oldEntity, newEntity, fields) => Ref.update(indexRef, (index) => {
+    const oldRecord = oldEntity;
+    const newRecord = newEntity;
+    // Find which fields have actually changed
+    const changedFields = [];
+    for (const field of fields) {
+        const oldValue = getNestedValue(oldRecord, field);
+        const newValue = getNestedValue(newRecord, field);
+        if (oldValue !== newValue) {
+            changedFields.push(field);
+        }
+    }
+    // If no indexed fields changed, return the original index unchanged
+    if (changedFields.length === 0) {
+        return index;
+    }
+    // Clone the index to avoid mutating the original
+    const newIndex = new Map(index);
+    // Also clone the sets that we'll modify to maintain immutability
+    for (const [token, idSet] of index) {
+        newIndex.set(token, new Set(idSet));
+    }
+    // Remove old tokens and add new tokens for changed fields only
+    removeEntityFromIndexMut(newIndex, oldEntity, changedFields);
+    addEntityToIndexMut(newIndex, newEntity, changedFields);
+    return newIndex;
+});
+//# sourceMappingURL=search-index.js.map

package/dist/indexes/search-index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"search-index.js","sourceRoot":"","sources":["../../src/indexes/search-index.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,QAAQ,CAAC;AACrC,OAAO,EAAE,QAAQ,EAAE,MAAM,+BAA+B,CAAC;AAEzD,OAAO,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAC;AAOzD;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAC/B,MAA6B,EAC7B,QAA0B,EACe,EAAE,CAC3C,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;IACnB,MAAM,WAAW,GAAmB,IAAI,GAAG,EAAE,CAAC;IAE9C,KAAK,MAAM,MAAM,IAAI,QAAQ,EAAE,CAAC;QAC/B,mBAAmB,CAAC,WAAW,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;IAClD,CAAC;IAED,OAAO,KAAK,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;AACrC,CAAC,CAAC,CAAC;AAEJ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmCG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAChC,QAAiC,EACjC,WAAkC,EACL,EAAE,CAC/B,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;IACnB,gCAAgC;IAChC,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC9B,OAAO,IAAI,GAAG,EAAU,CAAC;IAC1B,CAAC;IAED,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IAEvC,+DAA+D;IAC/D,MAAM,cAAc,GAAuB,EAAE,CAAC;IAE9C,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE,CAAC;QACtC,MAAM,WAAW,GAAG,IAAI,GAAG,EAAU,CAAC;QAEtC,0DAA0D;QAC1D,KAAK,MAAM,CAAC,UAAU,EAAE,SAAS,CAAC,IAAI,KAAK,EAAE,CAAC;YAC7C,IAAI,UAAU,KAAK,UAAU,IAAI,UAAU,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;gBACpE,qDAAqD;gBACrD,KAAK,MAAM,EAAE,IAAI,SAAS,EAAE,CAAC;oBAC5B,WAAW,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBACrB,CAAC;YACF,CAAC;QACF,CAAC;QAED,cAAc,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IAClC,CAAC;IAED,iDAAiD;IACjD,mDAAmD;IACnD,IAAI,cAAc,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACjC,OAAO,IAAI,GAAG,EAAU,CAAC;IAC1B,CAAC;IAED,uDAAuD;IACvD,MAAM,MAAM,GAAG,IAAI,GAAG,CAAS,cAAc,CAAC,CAAC,CAAC,CAAC,CAAC;IAElD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,cAAc,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAChD,MAAM,UAAU,GAAG,cAAc,CAAC,CAAC,CAAC,CAAC;QACrC,KAAK,MAAM,EAAE,IAAI,MAAM,EAAE,CAAC;YACzB,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC;gBACzB,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;YACnB,CAAC;QACF,CAAC;IACF,CAAC;IAED,OAAO,MAAM,CAAC;AACf,CAAC,CAAC,CAAC;AAEJ;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,CAAC,MAAM,sBAAsB,GAAG,CACrC,KAA0C,EAC1C,cAAmD,EACnD,iBAAoD,EACpD,GAA2B,EACmB,EAAE,CAChD,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;IACnB,gDAAgD;IAChD,IACC,CAAC,KAAK;QACN,CAAC,cAAc;QACf,CAAC,iBAAiB;QAClB,iBAAiB,CAAC,MAAM,KAAK,CAAC,EAC7B,CAAC;QACF,OAAO,SAAS,CAAC;IAClB,CAAC;IAED,yCAAyC;IACzC,MAAM,UAAU,GAAG,sBAAsB,CAAC,KAAK,EAAE,iBAAiB,CAAC,CAAC;IACpE,IAAI,CAAC,UAAU,EAAE,CAAC;QACjB,OAAO,SAAS,CAAC;IAClB,CAAC;IAED,MAAM,EAAE,WAAW,EAAE,aAAa,EAAE,GAAG,UAAU,CAAC;IAElD,sDAAsD;IACtD,MAAM,YAAY,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAClD,iBAAiB,CAAC,QAAQ,CAAC,KAAK,CAAC,CACjC,CAAC;IACF,IAAI,CAAC,YAAY,EAAE,CAAC;QACnB,OAAO,SAAS,CAAC;IAClB,CAAC;IAED,mDAAmD;IACnD,MAAM,YAAY,GAAG,KAAK,CAAC,CAAC,iBAAiB,CAAC,cAAc,EAAE,WAAW,CAAC,CAAC;IAE3E,uCAAuC;IACvC,IAAI,YAAY,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;QAC7B,OAAO,EAAE,CAAC;IACX,CAAC;IAED,uCAAuC;IACvC,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,KAAK,MAAM,EAAE,IAAI,YAAY,EAAE,CAAC;QAC/B,MAAM,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC3B,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;YAC1B,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACvB,CAAC;IACF,CAAC;IAED,OAAO,QAAQ,CAAC;AACjB,CAAC,CAAC,CAAC;AAEJ;;;;;;;;GAQG;AACH,MAAM,sBAAsB,GAAG,CAC9B,KAA8B,EAC9B,aAAoC,EAGxB,EAAE;IACd,8BAA8B;IAC9B,IAAI,SAAS,IAAI,KAAK,EAAE,CAAC;QACxB,MAAM,WAAW,GAAG,KAAK,CAAC,OAAO,CAAC;QAClC,IAAI,WAAW,KAAK,IAAI,IAAI,OAAO,WAAW,KAAK,QAAQ,EAAE,CAAC;YAC7D,MAAM,MAAM,GAAG,WAGd,CAAC;YACF,IAAI,OAAO,MAAM,CAAC,KAAK,KAAK,QAAQ,EAAE,CAAC;gBACtC,MAAM,WAAW,GAAG,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;gBAC3C,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oBAC9B,OAAO,SAAS,CAAC,CAAC,gDAAgD;gBACnE,CAAC;gBACD,MAAM,aAAa,GAClB,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC;oBACxC,CAAC,CAAC,MAAM,CAAC,MAAM;oBACf,CAAC,CAAC,aAAa,CAAC;gBAClB,OAAO,EAAE,WAAW,EAAE,aAAa,EAAE,CAAC;YACvC,CAAC;QACF,CAAC;IACF,CAAC;IAED,6CAA6C;IAC7C,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACpD,yBAAyB;QACzB,IACC,KAAK,KAAK,KAAK;YACf,KAAK,KAAK,MAAM;YAChB,KAAK,KAAK,MAAM;YAChB,KAAK,KAAK,SAAS,EAClB,CAAC;YACF,SAAS;QACV,CAAC;QAED,IAAI,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;YAC1E,MAAM,GAAG,GAAG,KAAgC,CAAC;YAC7C,IAAI,SAAS,IAAI,GAAG,IAAI,OAAO,GAAG,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;gBACzD,MAAM,WAAW,GAAG,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;gBAC1C,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oBAC9B,SAAS,CAAC,oBAAoB;gBAC/B,CAAC;gBACD,gDAAgD;gBAChD,OAAO,EAAE,WAAW,EAAE,aAAa,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC;YAChD,CAAC;QACF,CAAC;IACF,CAAC;IAED,OAAO,SAAS,CAAC;AAClB,CAAC,CAAC;AAEF;;;;;;;;;GASG;AACH,MAAM,mBAAmB,GAAG,CAC3B,KAAqB,EACrB,MAAS,EACT,MAA6B,EACtB,EAAE;IACT,MAAM,YAAY,GAAG,MAAiC,CAAC;IACvD,MAAM,QAAQ,GAAG,MAAM,CAAC,EAAE,CAAC;IAE3B,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC5B,MAAM,UAAU,GAAG,cAAc,CAAC,YAAY,EAAE,KAAK,CAAC,CAAC;QAEvD,2BAA2B;QAC3B,IAAI,OAAO,UAAU,KAAK,QAAQ,EAAE,CAAC;YACpC,SAAS;QACV,CAAC;QAED,4CAA4C;QAC5C,MAAM,MAAM,GAAG,QAAQ,CAAC,UAAU,CAAC,CAAC;QACpC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC5B,MAAM,WAAW,GAAG,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YACrC,IAAI,WAAW,EAAE,CAAC;gBACjB,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YAC3B,CAAC;iBAAM,CAAC;gBACP,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;YACvC,CAAC;QACF,CAAC;IACF,CAAC;AACF,CAAC,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAC/B,QAAiC,EACjC,MAAS,EACT,MAA6B,EACP,EAAE,CACxB,GAAG,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,KAAK,EAAE,EAAE;IAC9B,iDAAiD;IACjD,MAAM,QAAQ,GAAmB,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC;IAChD,mBAAmB,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;IAC9C,OAAO,QAAQ,CAAC;AACjB,CAAC,CAAC,CAAC;AAEJ;;;;;;;;;GASG;AACH,MAAM,wBAAwB,GAAG,CAChC,KAAqB,EACrB,MAAS,EACT,MAA6B,EACtB,EAAE;IACT,MAAM,YAAY,GAAG,MAAiC,CAAC;IACvD,MAAM,QAAQ,GAAG,MAAM,CAAC,EAAE,CAAC;IAE3B,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC5B,MAAM,UAAU,GAAG,cAAc,CAAC,YAAY,EAAE,KAAK,CAAC,CAAC;QAEvD,6BAA6B;QAC7B,IAAI,OAAO,UAAU,KAAK,QAAQ,EAAE,CAAC;YACpC,SAAS;QACV,CAAC;QAED,iDAAiD;QACjD,MAAM,MAAM,GAAG,QAAQ,CAAC,UAAU,CAAC,CAAC;QACpC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC5B,MAAM,WAAW,GAAG,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YACrC,IAAI,WAAW,EAAE,CAAC;gBACjB,WAAW,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;gBAC7B,sBAAsB;gBACtB,IAAI,WAAW,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;oBAC5B,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;gBACrB,CAAC;YACF,CAAC;QACF,CAAC;IACF,CAAC;AACF,CAAC,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,MAAM,CAAC,MAAM,qBAAqB,GAAG,CACpC,QAAiC,EACjC,MAAS,EACT,MAA6B,EACP,EAAE,CACxB,GAAG,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,KAAK,EAAE,EAAE;IAC9B,iDAAiD;IACjD,MAAM,QAAQ,GAAmB,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC;IAChD,iEAAiE;IACjE,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,KAAK,EAAE,CAAC;QACpC,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;IACrC,CAAC;IACD,wBAAwB,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;IACnD,OAAO,QAAQ,CAAC;AACjB,CAAC,CAAC,CAAC;AAEJ;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAClC,QAAiC,EACjC,SAAY,EACZ,SAAY,EACZ,MAA6B,EACP,EAAE,CACxB,GAAG,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,KAAK,EAAE,EAAE;IAC9B,MAAM,SAAS,GAAG,SAAoC,CAAC;IACvD,MAAM,SAAS,GAAG,SAAoC,CAAC;IAEvD,0CAA0C;IAC1C,MAAM,aAAa,GAAkB,EAAE,CAAC;IACxC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC5B,MAAM,QAAQ,GAAG,cAAc,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;QAClD,MAAM,QAAQ,GAAG,cAAc,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;QAClD,IAAI,QAAQ,KAAK,QAAQ,EAAE,CAAC;YAC3B,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC3B,CAAC;IACF,CAAC;IAED,oEAAoE;IACpE,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAChC,OAAO,KAAK,CAAC;IACd,CAAC;IAED,iDAAiD;IACjD,MAAM,QAAQ,GAAmB,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC;IAChD,iEAAiE;IACjE,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,KAAK,EAAE,CAAC;QACpC,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;IACrC,CAAC;IAED,+DAA+D;IAC/D,wBAAwB,CAAC,QAAQ,EAAE,SAAS,EAAE,aAAa,CAAC,CAAC;IAC7D,mBAAmB,CAAC,QAAQ,EAAE,SAAS,EAAE,aAAa,CAAC,CAAC;IAExD,OAAO,QAAQ,CAAC;AACjB,CAAC,CAAC,CAAC"}

package/dist/migrations/migration-runner.d.ts

@@ -0,0 +1,70 @@
+/**
+ * Migration Runner
+ *
+ * Core migration logic: validate migration registries, run migrations,
+ * and preview migrations via dry-run.
+ */
+import { Effect, type Ref } from "effect";
+import { MigrationError } from "../errors/migration-errors.js";
+import type { SerializationError, StorageError, UnsupportedFormatError } from "../errors/storage-errors.js";
+import { SerializerRegistry } from "../serializers/serializer-service.js";
+import { StorageAdapter } from "../storage/storage-service.js";
+import type { DatabaseConfig } from "../types/database-config-types.js";
+import type { DryRunResult, Migration } from "./migration-types.js";
+/**
+ * Run migrations to transform data from one version to another.
+ *
+ * Filters migrations to only those applicable for the version transition,
+ * then runs each transform in order, piping the output of one to the input
+ * of the next.
+ *
+ * @param data - The raw entity map to migrate
+ * @param fileVersion - The version of the data (from file's _version, or 0 if absent)
+ * @param targetVersion - The target schema version from collection config
+ * @param migrations - The full migration registry for this collection
+ * @param collectionName - Name of the collection (for error messages)
+ * @returns Effect<Record<string, unknown>, MigrationError> - the migrated data
+ */
+export declare const runMigrations: (data: Record<string, unknown>, fileVersion: number, targetVersion: number, migrations: ReadonlyArray<Migration>, collectionName: string) => Effect.Effect<Record<string, unknown>, MigrationError>;
+/**
+ * Validate that a migration registry forms a valid, contiguous chain.
+ *
+ * Validation rules:
+ * - Migrations must form a contiguous chain (no gaps in `from`/`to`)
+ * - Each migration's `to` must equal `from + 1`
+ * - No duplicate `from` values
+ * - The last migration's `to` must equal the collection's `version`
+ * - Version 0 with no migrations is valid
+ * - Version > 0 with empty migrations is invalid
+ *
+ * @param collectionName - Name of the collection (for error messages)
+ * @param version - Target schema version from collection config
+ * @param migrations - Array of migrations to validate
+ * @returns Effect<void, MigrationError> - succeeds if valid, fails with MigrationError if invalid
+ */
+export declare const validateMigrationRegistry: (collectionName: string, version: number, migrations: ReadonlyArray<Migration>) => Effect.Effect<void, MigrationError>;
+/**
+ * Internal Ref map type for cross-collection access.
+ */
+type HasId = {
+    readonly id: string;
+};
+type StateRefs = Record<string, Ref.Ref<ReadonlyMap<string, HasId>>>;
+/**
+ * Preview which files need migration and what transforms would apply.
+ *
+ * For each versioned collection with a file path:
+ * - Read the file and extract `_version`
+ * - Compare to config `version`
+ * - List which migrations would apply (without running transforms)
+ * - Report status
+ *
+ * No transforms are executed. No files are written.
+ *
+ * @param config - The database configuration
+ * @param _stateRefs - State refs (unused, but kept for API consistency)
+ * @returns Effect<DryRunResult, MigrationError | StorageError | SerializationError | UnsupportedFormatError>
+ */
+export declare const dryRunMigrations: (config: DatabaseConfig, _stateRefs: StateRefs) => Effect.Effect<DryRunResult, MigrationError | StorageError | SerializationError | UnsupportedFormatError, StorageAdapter | SerializerRegistry>;
+export {};
+//# sourceMappingURL=migration-runner.d.ts.map