@smartdoca/search-host 0.1.0

package/dist/search-host.js

@@ -0,0 +1,389 @@
+import { sameSearchSource, searchIndexNames, searchSourceKey, } from "./naming.js";
+import { SearchSourceRegistry } from "./registry.js";
+export class SearchSourceNotFoundError extends Error {
+    source;
+    constructor(source) {
+        super(`Search source is not registered: ${typeof source === "string"
+            ? source
+            : `${source.pluginId}/${source.sourceId}`}`);
+        this.source = source;
+        this.name = "SearchSourceNotFoundError";
+    }
+}
+function compareText(left, right) {
+    return left < right ? -1 : left > right ? 1 : 0;
+}
+function copyDescriptor(descriptor) {
+    return {
+        pluginId: descriptor.pluginId,
+        sourceId: descriptor.sourceId,
+        schemaVersion: descriptor.schemaVersion,
+        renderer: { ...descriptor.renderer },
+    };
+}
+function validatePagination(offset, limit) {
+    if (!Number.isSafeInteger(offset) || offset < 0)
+        throw new RangeError("offset must be a non-negative safe integer");
+    if (!Number.isSafeInteger(limit) || limit < 0)
+        throw new RangeError("limit must be a non-negative safe integer");
+}
+function compactIds(ids) {
+    const unique = new Set();
+    for (const id of ids) {
+        if (typeof id !== "string" || id.trim().length === 0)
+            throw new TypeError("Document IDs must be non-empty strings");
+        unique.add(id);
+    }
+    return [...unique].sort(compareText);
+}
+function validateProjection(projection) {
+    if (typeof projection.id !== "string" || projection.id.trim().length === 0)
+        throw new TypeError("Projection IDs must be non-empty strings");
+    if (typeof projection.text !== "string")
+        throw new TypeError("Projection text must be a string");
+}
+function hydrationMap(hydration) {
+    if (!Array.isArray(hydration))
+        return hydration;
+    const values = new Map();
+    for (const item of hydration)
+        values.set(item.id, item.value);
+    return values;
+}
+function abortError() {
+    const error = new Error("Search rebuild was aborted");
+    error.name = "AbortError";
+    return error;
+}
+class DefaultSearchHost {
+    registry;
+    #provider;
+    #namespace;
+    #now;
+    #rebuildBatchSize;
+    #locks = new Map();
+    #sequence = 0;
+    constructor(provider, options) {
+        this.#provider = provider;
+        this.registry = options.registry ?? new SearchSourceRegistry();
+        this.#namespace = options.namespace ?? "doca_search";
+        this.#now = options.now ?? (() => new Date());
+        this.#rebuildBatchSize = options.rebuildBatchSize ?? 100;
+        if (!Number.isSafeInteger(this.#rebuildBatchSize) ||
+            this.#rebuildBatchSize < 1)
+            throw new RangeError("rebuildBatchSize must be a positive safe integer");
+    }
+    indexNames(source) {
+        return searchIndexNames(this.#requireSource(source).descriptor, this.#namespace);
+    }
+    async upsertProjections(request) {
+        const source = this.#requireSource(request.source);
+        const key = searchSourceKey(source.descriptor);
+        const projections = [...request.projections];
+        for (const projection of projections)
+            validateProjection(projection);
+        await this.#exclusive(key, async () => {
+            const indexName = await this.#ensureIndex(source);
+            await this.#provider.upsertProjections(indexName, projections);
+            await this.#provider.deleteTombstones(key, projections.map((projection) => projection.id));
+        });
+    }
+    async deleteProjections(request) {
+        const source = this.#requireSource(request.source);
+        const key = searchSourceKey(source.descriptor);
+        const documentIds = compactIds(request.documentIds);
+        await this.#exclusive(key, async () => {
+            await this.#provider.putTombstones(documentIds.map((documentId) => ({
+                sourceKey: key,
+                documentId,
+                deletedAt: (request.deletedAt ?? this.#now()).toISOString(),
+            })));
+            const active = await this.#provider.resolveAlias(searchIndexNames(source.descriptor, this.#namespace).alias);
+            if (active)
+                await this.#provider.deleteProjections(active, documentIds);
+        });
+    }
+    async rebuild(request) {
+        const source = this.#requireSource(request.source);
+        if (!source.projections)
+            throw new TypeError(`Search source ${searchSourceKey(source.descriptor)} does not provide projections()`);
+        const key = searchSourceKey(source.descriptor);
+        return this.#exclusive(key, async () => {
+            if (request.signal?.aborted)
+                throw abortError();
+            const names = searchIndexNames(source.descriptor, this.#namespace);
+            const version = request.version ?? this.#nextVersion("rebuild");
+            const indexName = names.version(version);
+            let created = false;
+            let published = false;
+            try {
+                await this.#provider.createIndex(indexName, source.descriptor);
+                created = true;
+                const iterable = await source.projections({
+                    context: request.context,
+                    signal: request.signal,
+                });
+                let count = 0;
+                let batch = [];
+                const flush = async () => {
+                    if (!batch.length)
+                        return;
+                    await this.#provider.upsertProjections(indexName, batch);
+                    batch = [];
+                };
+                for await (const output of iterable) {
+                    if (request.signal?.aborted)
+                        throw abortError();
+                    const projections = Array.isArray(output) ? output : [output];
+                    for (const projection of projections) {
+                        validateProjection(projection);
+                        batch.push(projection);
+                        count++;
+                        if (batch.length >= this.#rebuildBatchSize)
+                            await flush();
+                    }
+                }
+                await flush();
+                if (request.signal?.aborted)
+                    throw abortError();
+                const tombstones = await this.#provider.listTombstones(key);
+                if (tombstones.length)
+                    await this.#provider.deleteProjections(indexName, tombstones.map((tombstone) => tombstone.documentId));
+                const switched = await this.#provider.swapAlias(names.alias, indexName);
+                published = true;
+                if (switched.previousIndex &&
+                    switched.previousIndex !== switched.activeIndex &&
+                    !request.retainPrevious)
+                    await this.#provider.deleteIndex(switched.previousIndex);
+                return {
+                    alias: names.alias,
+                    indexName: switched.activeIndex,
+                    previousIndex: switched.previousIndex,
+                    projectionCount: count,
+                };
+            }
+            catch (error) {
+                if (created && !published)
+                    await this.#provider.deleteIndex(indexName);
+                throw error;
+            }
+        });
+    }
+    async switchIndex(request) {
+        const source = this.#requireSource(request.source);
+        const key = searchSourceKey(source.descriptor);
+        return this.#exclusive(key, async () => {
+            const metadata = await this.#provider.inspectIndex(request.indexName);
+            if (!metadata)
+                throw new Error(`Search index does not exist: ${request.indexName}`);
+            if (!sameSearchSource(metadata.descriptor, source.descriptor))
+                throw new Error(`Search index ${request.indexName} belongs to a different source or schema`);
+            const alias = searchIndexNames(source.descriptor, this.#namespace).alias;
+            const switched = await this.#provider.swapAlias(alias, request.indexName);
+            if (request.deletePrevious &&
+                switched.previousIndex &&
+                switched.previousIndex !== switched.activeIndex)
+                await this.#provider.deleteIndex(switched.previousIndex);
+            return {
+                alias,
+                indexName: switched.activeIndex,
+                previousIndex: switched.previousIndex,
+            };
+        });
+    }
+    async query(request) {
+        const offset = request.offset ?? 0;
+        const limit = request.limit ?? 20;
+        validatePagination(offset, limit);
+        const sources = request.sources
+            ? [
+                ...new Map(request.sources.map((reference) => {
+                    const source = this.#requireSource(reference);
+                    return [searchSourceKey(source.descriptor), source];
+                })).values(),
+            ]
+            : this.registry.list();
+        const sourceResults = await Promise.all(sources.map((source) => this.#querySource(source, request)));
+        const failures = sourceResults
+            .flatMap((result) => (result.failure ? [result.failure] : []))
+            .sort((left, right) => {
+            const sourceOrder = compareText(searchSourceKey(left.source), searchSourceKey(right.source));
+            return sourceOrder || compareText(left.stage, right.stage);
+        });
+        const ranked = [];
+        for (const result of sourceResults) {
+            if (!result.values.length)
+                continue;
+            const scores = result.values.map((value) => value.rawScore);
+            const minimum = Math.min(...scores);
+            const maximum = Math.max(...scores);
+            for (const value of result.values) {
+                const score = maximum === minimum
+                    ? 1
+                    : (value.rawScore - minimum) / (maximum - minimum);
+                ranked.push({
+                    id: value.id,
+                    source: value.source,
+                    renderer: { ...value.source.renderer },
+                    providerScore: value.rawScore,
+                    score,
+                    value: value.value,
+                });
+            }
+        }
+        ranked.sort((left, right) => {
+            const scoreOrder = right.score - left.score;
+            if (scoreOrder)
+                return scoreOrder;
+            const sourceOrder = compareText(searchSourceKey(left.source), searchSourceKey(right.source));
+            return sourceOrder || compareText(left.id, right.id);
+        });
+        return {
+            items: ranked.slice(offset, offset + limit),
+            failures,
+            offset,
+            limit,
+            total: ranked.length,
+        };
+    }
+    async dispose() {
+        await this.registry.dispose();
+    }
+    async #querySource(source, request) {
+        const descriptor = copyDescriptor(source.descriptor);
+        const fail = (stage, error) => ({
+            values: [],
+            failure: { source: descriptor, stage, error },
+        });
+        const names = searchIndexNames(descriptor, this.#namespace);
+        let providerQuery;
+        try {
+            providerQuery = source.prepareQuery
+                ? await source.prepareQuery({
+                    context: request.context,
+                    query: request.query,
+                })
+                : { query: request.query };
+            if (providerQuery === null)
+                return { values: [] };
+            if (providerQuery.limit !== undefined &&
+                (!Number.isSafeInteger(providerQuery.limit) || providerQuery.limit < 0))
+                throw new RangeError("Search provider query limit must be a non-negative safe integer");
+        }
+        catch (error) {
+            return fail("prepare", error);
+        }
+        let indexName;
+        try {
+            indexName = await this.#provider.resolveAlias(names.alias);
+            if (!indexName)
+                return { values: [] };
+            const metadata = await this.#provider.inspectIndex(indexName);
+            if (!metadata || !sameSearchSource(metadata.descriptor, descriptor))
+                throw new Error(`Active index is missing or incompatible: ${indexName}`);
+        }
+        catch (error) {
+            return fail("index", error);
+        }
+        let hits;
+        try {
+            const raw = await this.#provider.queryIndex(indexName, providerQuery);
+            const scopedCandidates = providerQuery.candidateIds
+                ? new Set(compactIds(providerQuery.candidateIds))
+                : undefined;
+            const byId = new Map();
+            for (const hit of raw) {
+                if (typeof hit.id !== "string" ||
+                    hit.id.trim().length === 0 ||
+                    !Number.isFinite(hit.score))
+                    throw new TypeError("Search provider returned an invalid hit");
+                if (scopedCandidates && !scopedCandidates.has(hit.id))
+                    continue;
+                byId.set(hit.id, Math.max(byId.get(hit.id) ?? -Infinity, hit.score));
+            }
+            hits = [...byId.entries()]
+                .map(([id, score]) => ({ id, score }))
+                .sort((left, right) => right.score - left.score || compareText(left.id, right.id));
+        }
+        catch (error) {
+            return fail("query", error);
+        }
+        let authorized;
+        try {
+            const candidateIds = hits.map((hit) => hit.id);
+            const candidates = new Set(candidateIds);
+            const allowed = new Set(await source.authorize({
+                context: request.context,
+                query: request.query,
+                candidateIds,
+            }));
+            authorized = hits.filter((hit) => candidates.has(hit.id) && allowed.has(hit.id));
+        }
+        catch (error) {
+            return fail("authorize", error);
+        }
+        try {
+            const ids = authorized.map((hit) => hit.id);
+            const hydration = hydrationMap(await source.hydrate({
+                context: request.context,
+                query: request.query,
+                ids,
+            }));
+            const key = searchSourceKey(descriptor);
+            return {
+                values: authorized.flatMap((hit) => hydration.has(hit.id)
+                    ? [
+                        {
+                            id: hit.id,
+                            key,
+                            rawScore: hit.score,
+                            source: descriptor,
+                            value: hydration.get(hit.id),
+                        },
+                    ]
+                    : []),
+            };
+        }
+        catch (error) {
+            return fail("hydrate", error);
+        }
+    }
+    #requireSource(source) {
+        const registered = this.registry.get(source);
+        if (!registered)
+            throw new SearchSourceNotFoundError(source);
+        return registered;
+    }
+    async #ensureIndex(source) {
+        const names = searchIndexNames(source.descriptor, this.#namespace);
+        const active = await this.#provider.resolveAlias(names.alias);
+        if (active) {
+            const metadata = await this.#provider.inspectIndex(active);
+            if (metadata && sameSearchSource(metadata.descriptor, source.descriptor))
+                return active;
+        }
+        const indexName = names.version(this.#nextVersion("bootstrap"));
+        await this.#provider.createIndex(indexName, source.descriptor);
+        return (await this.#provider.swapAlias(names.alias, indexName)).activeIndex;
+    }
+    #nextVersion(prefix) {
+        this.#sequence++;
+        return `${prefix}-${this.#now().getTime().toString(36)}-${this.#sequence.toString(36)}`;
+    }
+    async #exclusive(key, action) {
+        const previous = this.#locks.get(key) ?? Promise.resolve();
+        const run = previous.then(action);
+        const tail = run.then(() => undefined, () => undefined);
+        this.#locks.set(key, tail);
+        try {
+            return await run;
+        }
+        finally {
+            if (this.#locks.get(key) === tail)
+                this.#locks.delete(key);
+        }
+    }
+}
+export function createSearchHost(provider, options = {}) {
+    return new DefaultSearchHost(provider, options);
+}

package/dist/types.d.ts

@@ -0,0 +1,141 @@
+export type MaybePromise<T> = T | Promise<T>;
+export interface SearchResultRenderer {
+    readonly kind: string;
+    readonly version?: number;
+}
+export interface SearchSourceDescriptor {
+    readonly pluginId: string;
+    readonly sourceId: string;
+    readonly schemaVersion: number;
+    readonly renderer: SearchResultRenderer;
+}
+export interface SearchProjection {
+    readonly id: string;
+    readonly text: string;
+    readonly metadata?: Readonly<Record<string, unknown>>;
+}
+export interface SearchAuthorizationRequest<TContext> {
+    readonly context: TContext;
+    readonly query: string;
+    readonly candidateIds: readonly string[];
+}
+export interface SearchHydrationRequest<TContext> {
+    readonly context: TContext;
+    readonly query: string;
+    readonly ids: readonly string[];
+}
+export interface SearchProjectionSnapshotRequest<TContext> {
+    readonly context: TContext;
+    readonly signal?: AbortSignal;
+}
+export interface SearchQueryPreparationRequest<TContext> {
+    readonly context: TContext;
+    readonly query: string;
+}
+export interface HydratedSearchValue<TValue = unknown> {
+    readonly id: string;
+    readonly value: TValue;
+}
+export type SearchHydration<TValue = unknown> = ReadonlyMap<string, TValue> | readonly HydratedSearchValue<TValue>[];
+export type SearchProjectionBatch = SearchProjection | readonly SearchProjection[];
+/**
+ * A source owns authorization and hydration. The host deliberately receives
+ * only opaque IDs from an index, so indexed fields can never bypass the source.
+ */
+export interface SearchSource<TContext = unknown, TValue = unknown> {
+    readonly descriptor: SearchSourceDescriptor;
+    /**
+     * Narrows provider work to the source-owned query scope. Returning null
+     * makes the source inapplicable to this request. Authorization still runs
+     * after the provider returns and is never replaced by candidateIds.
+     */
+    prepareQuery?(request: SearchQueryPreparationRequest<TContext>): MaybePromise<SearchProviderQuery | null>;
+    authorize(request: SearchAuthorizationRequest<TContext>): MaybePromise<Iterable<string>>;
+    hydrate(request: SearchHydrationRequest<TContext>): MaybePromise<SearchHydration<TValue>>;
+    projections?(request: SearchProjectionSnapshotRequest<TContext>): Iterable<SearchProjectionBatch> | AsyncIterable<SearchProjectionBatch> | Promise<Iterable<SearchProjectionBatch> | AsyncIterable<SearchProjectionBatch>>;
+}
+export interface SearchSourceEffectResource<TContext = unknown, TValue = unknown> {
+    readonly source: SearchSource<TContext, TValue>;
+    readonly dispose?: () => MaybePromise<void>;
+}
+export interface SearchSourceEffect<TContext = unknown, TValue = unknown> {
+    acquire(): MaybePromise<SearchSourceEffectResource<TContext, TValue>>;
+}
+export interface SearchIndexMetadata {
+    readonly name: string;
+    readonly descriptor: SearchSourceDescriptor;
+    readonly createdAt: string;
+}
+export interface SearchIndexLifecycle {
+    createIndex(name: string, descriptor: SearchSourceDescriptor): Promise<void>;
+    deleteIndex(name: string): Promise<void>;
+    inspectIndex(name: string): Promise<SearchIndexMetadata | undefined>;
+    resolveAlias(alias: string): Promise<string | undefined>;
+    swapAlias(alias: string, indexName: string): Promise<SearchAliasSwap>;
+}
+/**
+ * Providers may implement aliases as pointers or as stable physical indexes
+ * whose contents are atomically swapped. activeIndex makes both models
+ * explicit to the host.
+ */
+export interface SearchAliasSwap {
+    readonly activeIndex: string;
+    readonly previousIndex?: string;
+}
+export interface SearchProjectionIndex {
+    upsertProjections(indexName: string, projections: readonly SearchProjection[]): Promise<void>;
+    deleteProjections(indexName: string, documentIds: readonly string[]): Promise<void>;
+}
+export interface SearchProviderHit {
+    readonly id: string;
+    readonly score: number;
+}
+export interface SearchProviderQuery {
+    readonly query: string;
+    readonly candidateIds?: readonly string[];
+    readonly limit?: number;
+    readonly semantic?: boolean;
+    readonly rankingScoreThreshold?: number;
+}
+export interface SearchIndexQuery {
+    queryIndex(indexName: string, request: SearchProviderQuery): Promise<readonly SearchProviderHit[]>;
+}
+export interface SearchTombstone {
+    readonly sourceKey: string;
+    readonly documentId: string;
+    readonly deletedAt: string;
+}
+export interface SearchTombstoneStore {
+    putTombstones(tombstones: readonly SearchTombstone[]): Promise<void>;
+    deleteTombstones(sourceKey: string, documentIds: readonly string[]): Promise<void>;
+    listTombstones(sourceKey: string): Promise<readonly SearchTombstone[]>;
+}
+/**
+ * Concrete engines implement this boundary. SearchHost has no dependency on
+ * any engine SDK or wire format.
+ */
+export interface SearchProvider extends SearchIndexLifecycle, SearchProjectionIndex, SearchIndexQuery, SearchTombstoneStore {
+}
+export interface SearchResultItem<TValue = unknown> {
+    readonly id: string;
+    readonly source: SearchSourceDescriptor;
+    readonly renderer: SearchResultRenderer;
+    /** Provider-local score retained for source-specific reranking. */
+    readonly providerScore: number;
+    /** Per-source normalized score used for federation. */
+    readonly score: number;
+    readonly value: TValue;
+}
+export type SearchFailureStage = "index" | "prepare" | "query" | "authorize" | "hydrate";
+export interface SearchSourceFailure {
+    readonly source: SearchSourceDescriptor;
+    readonly stage: SearchFailureStage;
+    readonly error: unknown;
+}
+export interface FederatedSearchResult<TValue = unknown> {
+    readonly items: readonly SearchResultItem<TValue>[];
+    readonly failures: readonly SearchSourceFailure[];
+    readonly offset: number;
+    readonly limit: number;
+    readonly total: number;
+}

package/dist/types.js

@@ -0,0 +1 @@
+export {};

package/package.json

@@ -0,0 +1,23 @@
+{
+  "name": "@smartdoca/search-host",
+  "version": "0.1.0",
+  "license": "AGPL-3.0-only",
+  "type": "module",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    },
+    "./memory": {
+      "types": "./dist/memory-provider.d.ts",
+      "import": "./dist/memory-provider.js"
+    }
+  },
+  "files": [
+    "dist"
+  ],
+  "publishConfig": {
+    "access": "public"
+  }
+}