@lde/search-typesense 0.27.4 → 0.29.0

package/dist/in-place-rebuild.d.ts

@@ -5,27 +5,33 @@ import { type RebuildOptions } from './rebuild-support.js';
 /** {@link InPlaceRebuild} options: the shared rebuild options plus the
  *  membership-sweep ceiling. */
 export interface InPlaceRebuildOptions extends RebuildOptions {
-    /** Most distinct sources the membership sweep may enumerate before it throws
-     *  rather than risk missing departed sources (default 10 000). */
+    /**
+     * Most distinct datasets the membership sweep may enumerate before it throws
+     * rather than risk missing departed ones (default 10 000). The sweep needs
+     * the complete set to spot which datasets left, so a truncated facet would
+     * silently miss deletions. Raise it (up to the engine’s `max_facet_values`
+     * limit) before an index approaches it, so the ceiling is a tunable guard
+     * rather than a hard wall.
+     */
     readonly maxSweepableSources?: number;
 }
 /**
  * In-place Rebuild (update the live index directly – upsert changed sources,
  * sweep the rest – rather than swap in a fresh one) as a transactional
  * `Writer`: documents are upserted into one long-lived collection with
- * per-source atomicity – no swap, no staging. Every document is stamped with
- * its `source` (the dataset IRI) and `last_seen` (the run id); deletion is a
- * sweep, never special-cased:
+ * per-dataset atomicity – no swap, no staging. Deletion is a sweep, never
+ * special-cased, and {@link openDocuments} decides what a deletion physically
+ * is – for a canonically keyed type, dropping one dataset’s membership of a
+ * document several datasets share, and the document itself only with the last
+ * of them:
  *
- * - a **successful dataset flush** deletes the source’s documents the run did
- *   not rewrite (`source = dataset && last_seen != runId`). A failed dataset
- *   is not swept – its output is incomplete, and the next successful run
- *   reconciles;
- * - **reset** (the pipeline’s dump-fallback discard) deletes only *this run’s*
- *   writes for the dataset (`source = dataset && last_seen = runId`), so the
- *   dump re-run rebuilds it cleanly while the source’s prior-run documents are
- *   left for the success sweep to reconcile;
- * - **commit** deletes every document whose source left the run’s selection
+ * - a **successful dataset flush** drops what the dataset had and this run did
+ *   not rewrite. A failed dataset is not swept – its output is incomplete, and
+ *   the next successful run reconciles;
+ * - **reset** (the pipeline’s dump-fallback discard) drops only *this run’s*
+ *   writes for the dataset, so the dump re-run rebuilds it cleanly while the
+ *   dataset’s prior-run documents are left for the success sweep to reconcile;
+ * - **commit** drops what belongs to datasets that left the run’s selection
  *   (registry-membership sweep over {@link RunContext.selectedSources}, which
  *   includes datasets skipped as unchanged) and releases the lock;
  * - **abort** only releases the lock: upserts are idempotent, so whatever
@@ -33,15 +39,17 @@ export interface InPlaceRebuildOptions extends RebuildOptions {
  *
  * `openRun` takes the single-flight cross-pod lock (throwing
  * `RebuildAlreadyRunning` when another rebuild holds it) and creates
- * the collection on demand from the {@link SearchType} plus the two
- * bookkeeping fields. A collection that already exists is used as it is, with
- * one exception: it must carry every reference field the declaration asks for
- * and every facet companion its facets read, or the run fails with the
- * drop-and-rebuild that fixes it
- * ({@link assertReferencesAndFacetCompanionsPresent}).
+ * the collection on demand from the {@link SearchType} plus the bookkeeping
+ * fields. A collection that already exists is used as it is, with two
+ * exceptions: it must carry every reference field the declaration asks for and
+ * every facet companion its facets read
+ * ({@link assertReferencesAndFacetCompanionsPresent}), and the membership field
+ * a keyed type records in ({@link assertBookkeepingPresent}) – each failing
+ * with the rebuild that fixes it.
  *
- * Document ids must be unique per (source, entity) – the caller keys them –
- * or documents from different sources overwrite each other.
+ * Unless the type is canonically keyed, document ids must be unique per
+ * (dataset, entity) – the caller keys them – or documents from different
+ * datasets overwrite each other.
  *
  * The name is the NDE Stack’s pattern
  * ({@link https://docs.nde.nl/stack/patterns | Stack patterns}).
@@ -60,20 +68,7 @@ export declare class InPlaceRebuild<TDocument extends {
     readonly collectionName: string;
     private readonly maxSweepableSources;
     private readonly resolved;
-    /** The column this collection carries its documents’ dataset IRI in: the
-     *  type’s declared dataset field, or the private `source` when it declares
-     *  none. Everything below – the stamp, every sweep filter, the source
-     *  enumeration – reads this one name. */
-    private readonly sourceField;
     constructor(client: Client, searchType: SearchType, options?: InPlaceRebuildOptions);
     openRun(context: RunContext): Promise<RunWriter<TDocument>>;
-    /**
-     * The distinct sources present in the collection, via a single facet over its
-     * {@link provenanceField}. Requests one bucket beyond `maxSources` so genuine truncation is
-     * distinguishable from an exactly-full result: `maxSources` buckets are
-     * returned intact, `maxSources + 1` proves more exist and the sweep would
-     * miss some, so it throws rather than delete blind.
-     */
-    private indexedSources;
 }
 //# sourceMappingURL=in-place-rebuild.d.ts.map

package/dist/in-place-rebuild.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"in-place-rebuild.d.ts","sourceRoot":"","sources":["../src/in-place-rebuild.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,WAAW,CAAC;AAExC,OAAO,KAAK,EAAgB,UAAU,EAAE,MAAM,aAAa,CAAC;AAG5D,OAAO,KAAK,EAEV,UAAU,EACV,SAAS,EACT,MAAM,EACP,MAAM,eAAe,CAAC;AAavB,OAAO,EAML,KAAK,cAAc,EAEpB,MAAM,sBAAsB,CAAC;AAa9B;gCACgC;AAChC,MAAM,WAAW,qBAAsB,SAAQ,cAAc;IAC3D;sEACkE;IAClE,QAAQ,CAAC,mBAAmB,CAAC,EAAE,MAAM,CAAC;CACvC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoCG;AACH,qBAAa,cAAc,CACzB,SAAS,SAAS;IAAE,EAAE,EAAE,MAAM,CAAA;CAAE,CAChC,YAAW,MAAM,CAAC,SAAS,CAAC;IAiB1B,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,UAAU;IAjB7B;;;;;OAKG;IACH,SAAgB,cAAc,EAAE,MAAM,CAAC;IACvC,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAS;IAC7C,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAyB;IAClD;;;6CAGyC;IACzC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;gBAGlB,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,UAAU,EACvC,OAAO,GAAE,qBAA0B;IAkB/B,OAAO,CAAC,OAAO,EAAE,UAAU,GAAG,OAAO,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;IA6HjE;;;;;;OAMG;YACW,cAAc;CAsB7B"}
+{"version":3,"file":"in-place-rebuild.d.ts","sourceRoot":"","sources":["../src/in-place-rebuild.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,WAAW,CAAC;AAExC,OAAO,KAAK,EAAgB,UAAU,EAAE,MAAM,aAAa,CAAC;AAG5D,OAAO,KAAK,EAEV,UAAU,EACV,SAAS,EACT,MAAM,EACP,MAAM,eAAe,CAAC;AASvB,OAAO,EAEL,KAAK,cAAc,EAEpB,MAAM,sBAAsB,CAAC;AAE9B;gCACgC;AAChC,MAAM,WAAW,qBAAsB,SAAQ,cAAc;IAC3D;;;;;;;OAOG;IACH,QAAQ,CAAC,mBAAmB,CAAC,EAAE,MAAM,CAAC;CACvC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsCG;AACH,qBAAa,cAAc,CACzB,SAAS,SAAS;IAAE,EAAE,EAAE,MAAM,CAAA;CAAE,CAChC,YAAW,MAAM,CAAC,SAAS,CAAC;IAY1B,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,UAAU;IAZ7B;;;;;OAKG;IACH,SAAgB,cAAc,EAAE,MAAM,CAAC;IACvC,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAqB;IACzD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAyB;gBAG/B,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,UAAU,EACvC,OAAO,GAAE,qBAA0B;IAe/B,OAAO,CAAC,OAAO,EAAE,UAAU,GAAG,OAAO,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;CAuElE"}

package/dist/in-place-rebuild.js

@@ -1,36 +1,25 @@
 import { facetableFields, physicalFields } from '@lde/search/adapter';
 import { buildCollectionDefinition } from './collection-definition.js';
-import { BatchImporter } from './import.js';
 import { ensureCollectionExists, openLockedRun, releaseLock } from './lock.js';
-import { LAST_SEEN_FIELD, SOURCE_FIELD, departedSources, membershipSweepFilters, provenanceField, staleDocumentsFilter, thisRunDocumentsFilter, } from './sweep.js';
-import { assertNoReservedFields, assertSweepableProvenanceField, deleteByFilter, resolveRebuildOptions, stampDocuments, } from './rebuild-support.js';
-/**
- * Default ceiling on the distinct sources the membership sweep enumerates via
- * a single `source` facet. The sweep needs the complete source set to spot
- * departed ones, so a truncated facet would silently miss deletions;
- * {@link InPlaceRebuild} throws instead. Raise it (up to the engine’s
- * `max_facet_values` limit) with {@link InPlaceRebuildOptions.maxSweepableSources}
- * before an index approaches it, so the ceiling is a tunable guard rather than
- * a hard wall.
- */
-const DEFAULT_MAX_SWEEPABLE_SOURCES = 10_000;
+import { assertBookkeepingPresent, assertWritableType, openDocuments, withBookkeeping, } from './documents.js';
+import { resolveRebuildOptions, } from './rebuild-support.js';
 /**
  * In-place Rebuild (update the live index directly – upsert changed sources,
  * sweep the rest – rather than swap in a fresh one) as a transactional
  * `Writer`: documents are upserted into one long-lived collection with
- * per-source atomicity – no swap, no staging. Every document is stamped with
- * its `source` (the dataset IRI) and `last_seen` (the run id); deletion is a
- * sweep, never special-cased:
+ * per-dataset atomicity – no swap, no staging. Deletion is a sweep, never
+ * special-cased, and {@link openDocuments} decides what a deletion physically
+ * is – for a canonically keyed type, dropping one dataset’s membership of a
+ * document several datasets share, and the document itself only with the last
+ * of them:
  *
- * - a **successful dataset flush** deletes the source’s documents the run did
- *   not rewrite (`source = dataset && last_seen != runId`). A failed dataset
- *   is not swept – its output is incomplete, and the next successful run
- *   reconciles;
- * - **reset** (the pipeline’s dump-fallback discard) deletes only *this run’s*
- *   writes for the dataset (`source = dataset && last_seen = runId`), so the
- *   dump re-run rebuilds it cleanly while the source’s prior-run documents are
- *   left for the success sweep to reconcile;
- * - **commit** deletes every document whose source left the run’s selection
+ * - a **successful dataset flush** drops what the dataset had and this run did
+ *   not rewrite. A failed dataset is not swept – its output is incomplete, and
+ *   the next successful run reconciles;
+ * - **reset** (the pipeline’s dump-fallback discard) drops only *this run’s*
+ *   writes for the dataset, so the dump re-run rebuilds it cleanly while the
+ *   dataset’s prior-run documents are left for the success sweep to reconcile;
+ * - **commit** drops what belongs to datasets that left the run’s selection
  *   (registry-membership sweep over {@link RunContext.selectedSources}, which
  *   includes datasets skipped as unchanged) and releases the lock;
  * - **abort** only releases the lock: upserts are idempotent, so whatever
@@ -38,15 +27,17 @@ const DEFAULT_MAX_SWEEPABLE_SOURCES = 10_000;
  *
  * `openRun` takes the single-flight cross-pod lock (throwing
  * `RebuildAlreadyRunning` when another rebuild holds it) and creates
- * the collection on demand from the {@link SearchType} plus the two
- * bookkeeping fields. A collection that already exists is used as it is, with
- * one exception: it must carry every reference field the declaration asks for
- * and every facet companion its facets read, or the run fails with the
- * drop-and-rebuild that fixes it
- * ({@link assertReferencesAndFacetCompanionsPresent}).
+ * the collection on demand from the {@link SearchType} plus the bookkeeping
+ * fields. A collection that already exists is used as it is, with two
+ * exceptions: it must carry every reference field the declaration asks for and
+ * every facet companion its facets read
+ * ({@link assertReferencesAndFacetCompanionsPresent}), and the membership field
+ * a keyed type records in ({@link assertBookkeepingPresent}) – each failing
+ * with the rebuild that fixes it.
  *
- * Document ids must be unique per (source, entity) – the caller keys them –
- * or documents from different sources overwrite each other.
+ * Unless the type is canonically keyed, document ids must be unique per
+ * (dataset, entity) – the caller keys them – or documents from different
+ * datasets overwrite each other.
  *
  * The name is the NDE Stack’s pattern
  * ({@link https://docs.nde.nl/stack/patterns | Stack patterns}).
@@ -63,21 +54,16 @@ export class InPlaceRebuild {
     collectionName;
     maxSweepableSources;
     resolved;
-    /** The column this collection carries its documents’ dataset IRI in: the
-     *  type’s declared dataset field, or the private `source` when it declares
-     *  none. Everything below – the stamp, every sweep filter, the source
-     *  enumeration – reads this one name. */
-    sourceField;
     constructor(client, searchType, options = {}) {
         this.client = client;
         this.searchType = searchType;
-        assertNoReservedFields(searchType, [SOURCE_FIELD, LAST_SEEN_FIELD]);
-        assertSweepableProvenanceField(searchType, {
+        // The membership sweep enumerates the indexed datasets by faceting the
+        // column they are recorded in, so a declared one has to be facetable.
+        assertWritableType(searchType, {
             requireFacetable: true,
             schema: options.schema,
         });
-        this.sourceField = provenanceField(searchType);
-        const { maxSweepableSources = DEFAULT_MAX_SWEEPABLE_SOURCES, ...rebuildOptions } = options;
+        const { maxSweepableSources, ...rebuildOptions } = options;
         this.maxSweepableSources = maxSweepableSources;
         this.resolved = resolveRebuildOptions(searchType, rebuildOptions);
         this.collectionName =
@@ -86,78 +72,47 @@ export class InPlaceRebuild {
     async openRun(context) {
         const { batchSize, lockTtlMs, definitionOptions } = this.resolved;
         const name = this.collectionName;
-        const sourceField = this.sourceField;
         return openLockedRun(this.client, name, lockTtlMs, async () => {
-            // Create the collection on demand: SearchType schema + the bookkeeping
-            // fields. The private `source` – faceted so the membership sweep can
-            // enumerate the distinct sources – only when the type declares no dataset
-            // field of its own; when it does, that declared field already carries the
-            // IRI (and its own `facet: true`), and a second column would be the same
-            // value twice, free to drift.
+            // Create the collection on demand: the SearchType’s schema plus whatever
+            // bookkeeping its membership regime keeps.
             const definition = buildCollectionDefinition(this.searchType, definitionOptions);
             const declaredFields = definition.fields ?? [];
-            const created = await ensureCollectionExists(this.client, name, () => {
-                const bookkeeping = [
-                    ...(sourceField === SOURCE_FIELD
-                        ? [{ name: SOURCE_FIELD, type: 'string', facet: true }]
-                        : []),
-                    { name: LAST_SEEN_FIELD, type: 'string' },
-                ];
-                return {
-                    ...definition,
-                    fields: [...declaredFields, ...bookkeeping],
-                };
-            });
+            const created = await ensureCollectionExists(this.client, name, () => withBookkeeping(definition, this.searchType));
             // A collection this run did NOT create may predate a `joinable` being
-            // declared, or a facet policy on a type this one references – and
-            // neither a reference nor a facet companion can be added to it here.
+            // declared, a facet policy on a type this one references, or the type
+            // being keyed – and none of the three can be added to it here.
             if (!created) {
-                await assertReferencesAndFacetCompanionsPresent(this.client, name, declaredFields, facetCompanionsOf(this.searchType, definitionOptions.schema));
+                const existing = (await this.client.collections(name).retrieve())
+                    .fields;
+                assertReferencesAndFacetCompanionsPresent(name, declaredFields, facetCompanionsOf(this.searchType, definitionOptions.schema), existing);
+                assertBookkeepingPresent(this.searchType, existing, name);
             }
-            const importer = new BatchImporter(this.client, name, batchSize);
+            const documents = openDocuments(this.client, name, {
+                searchType: this.searchType,
+                runId: context.runId,
+                startedAt: context.startedAt,
+                batchSize,
+                maxSweepableSources: this.maxSweepableSources,
+            });
             return {
-                // Stamping the provenance field – rather than trusting the projection
-                // to have filled a declared one – is what keeps the sweep total: a
-                // caller projecting without a dataset context, or a document reaching
-                // the writer any other way, still lands in a swept collection. Where
-                // the projection did fill it, the value is the same dataset IRI, so the
-                // stamp only ever reasserts it.
-                //
-                // It reasserts the sweep’s column only, never the physical fanout a
-                // declared field also produces (the folded `${name}_search` companion
-                // of a `searchable` one). Folding is the projection’s convention, and
-                // restating it here would put it in two places – the split this change
-                // exists to close. So a document that reached the writer unprojected is
-                // sweepable but not full-text findable by dataset: absent, never wrong.
-                write: async (dataset, documents) => importer.add(stampDocuments(documents, {
-                    [sourceField]: dataset.iri.toString(),
-                    [LAST_SEEN_FIELD]: context.runId,
-                })),
+                write: async (dataset, items) => documents.add(dataset, items),
                 flush: async (dataset, outcome) => {
-                    // Land the buffered documents first, so the sweep below never
-                    // deletes what this run just rewrote.
-                    await importer.flush();
                     if (outcome !== 'success') {
                         // A failed dataset’s output is incomplete: sweeping against it
                         // would delete documents the run never got to rewrite. Leave the
-                        // stale ones for the next successful run to reconcile.
+                        // stale ones for the next successful run to reconcile. The buffered
+                        // documents stay buffered; the next flush lands them.
                         return;
                     }
-                    await deleteByFilter(this.client, name, staleDocumentsFilter(sourceField, dataset.iri.toString(), context.runId));
-                },
-                reset: async (dataset) => {
-                    // Discard only this run’s partial writes for the source (the failed
-                    // endpoint attempt) so the dump re-run rebuilds it cleanly; the
-                    // source’s prior-run documents stay for the success sweep.
-                    await importer.flush();
-                    await deleteByFilter(this.client, name, thisRunDocumentsFilter(sourceField, dataset.iri.toString(), context.runId));
+                    await documents.dropUnwritten(dataset);
                 },
+                // Discard only this run’s partial writes for the dataset (the failed
+                // endpoint attempt) so the dump re-run rebuilds it cleanly; the
+                // dataset’s prior-run documents stay for the success sweep.
+                reset: async (dataset) => documents.dropWrittenThisRun(dataset),
                 commit: async () => {
-                    await importer.flush();
-                    const departed = departedSources(await this.indexedSources(name, this.maxSweepableSources), context.selectedSources());
-                    for (const filter of membershipSweepFilters(sourceField, departed)) {
-                        await deleteByFilter(this.client, name, filter);
-                    }
+                    await documents.flush();
+                    await documents.dropUnselected(context.selectedSources());
                     await releaseLock(this.client, name);
                 },
                 abort: async () => {
@@ -166,30 +121,6 @@ export class InPlaceRebuild {
             };
         });
     }
-    /**
-     * The distinct sources present in the collection, via a single facet over its
-     * {@link provenanceField}. Requests one bucket beyond `maxSources` so genuine truncation is
-     * distinguishable from an exactly-full result: `maxSources` buckets are
-     * returned intact, `maxSources + 1` proves more exist and the sweep would
-     * miss some, so it throws rather than delete blind.
-     */
-    async indexedSources(name, maxSources) {
-        const response = await this.client
-            .collections(name)
-            .documents()
-            .search({
-            q: '*',
-            query_by: this.sourceField,
-            per_page: 0,
-            facet_by: this.sourceField,
-            max_facet_values: maxSources + 1,
-        });
-        const counts = response.facet_counts?.[0]?.counts ?? [];
-        if (counts.length > maxSources) {
-            throw new Error(`Membership sweep cannot see beyond ${maxSources} distinct sources in “${name}”; raise maxSweepableSources or departed sources might be missed`);
-        }
-        return counts.map((count) => count.value);
-    }
 }
 /**
  * Fail loudly when an EXISTING collection does not carry every reference field
@@ -218,15 +149,12 @@ export class InPlaceRebuild {
  * self-correcting (a new plain field simply starts being written) or
  * harmless, and general drift detection is a feature of its own.
  */
-async function assertReferencesAndFacetCompanionsPresent(client, name, fields, facetCompanions) {
+function assertReferencesAndFacetCompanionsPresent(name, fields, facetCompanions, existingFields) {
     const references = fields.filter((field) => field.reference !== undefined);
     if (references.length === 0 && facetCompanions.length === 0) {
         return;
     }
-    const existing = new Map((await client.collections(name).retrieve()).fields.map((field) => [
-        field.name,
-        field,
-    ]));
+    const existing = new Map(existingFields.map((field) => [field.name, field]));
     const missingReferences = references.filter((field) => existing.get(field.name)?.reference !== field.reference);
     if (missingReferences.length > 0) {
         throw new Error(`Typesense collection “${name}” exists without the reference field(s) ${missingReferences

package/dist/index.d.ts

@@ -3,7 +3,6 @@ export type { BlueGreenRebuildOptions } from './blue-green-rebuild.js';
 export { InPlaceRebuild } from './in-place-rebuild.js';
 export type { InPlaceRebuildOptions } from './in-place-rebuild.js';
 export { RebuildAlreadyRunning } from './lock.js';
-export { departedSources, membershipSweepFilters, sourceDocumentsFilter, staleDocumentsFilter, } from './sweep.js';
 export { buildCollectionDefinition } from './collection-definition.js';
 export type { CollectionDefinitionOptions } from './collection-definition.js';
 export { deriveCollectionName } from './collection-name.js';

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAC3D,YAAY,EAAE,uBAAuB,EAAE,MAAM,yBAAyB,CAAC;AACvE,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvD,YAAY,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AACnE,OAAO,EAAE,qBAAqB,EAAE,MAAM,WAAW,CAAC;AAClD,OAAO,EACL,eAAe,EACf,sBAAsB,EACtB,qBAAqB,EACrB,oBAAoB,GACrB,MAAM,YAAY,CAAC;AACpB,OAAO,EAAE,yBAAyB,EAAE,MAAM,4BAA4B,CAAC;AACvE,YAAY,EAAE,2BAA2B,EAAE,MAAM,4BAA4B,CAAC;AAC9E,OAAO,EAAE,oBAAoB,EAAE,MAAM,sBAAsB,CAAC;AAC5D,OAAO,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AACxD,YAAY,EAAE,wBAAwB,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAChF,OAAO,EAAE,2BAA2B,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AAC/E,YAAY,EACV,qBAAqB,EACrB,4BAA4B,EAC5B,uBAAuB,GACxB,MAAM,aAAa,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAC3D,YAAY,EAAE,uBAAuB,EAAE,MAAM,yBAAyB,CAAC;AACvE,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvD,YAAY,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AACnE,OAAO,EAAE,qBAAqB,EAAE,MAAM,WAAW,CAAC;AAClD,OAAO,EAAE,yBAAyB,EAAE,MAAM,4BAA4B,CAAC;AACvE,YAAY,EAAE,2BAA2B,EAAE,MAAM,4BAA4B,CAAC;AAC9E,OAAO,EAAE,oBAAoB,EAAE,MAAM,sBAAsB,CAAC;AAC5D,OAAO,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AACxD,YAAY,EAAE,wBAAwB,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAChF,OAAO,EAAE,2BAA2B,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AAC/E,YAAY,EACV,qBAAqB,EACrB,4BAA4B,EAC5B,uBAAuB,GACxB,MAAM,aAAa,CAAC"}

package/dist/index.js

@@ -1,7 +1,6 @@
 export { BlueGreenRebuild } from './blue-green-rebuild.js';
 export { InPlaceRebuild } from './in-place-rebuild.js';
 export { RebuildAlreadyRunning } from './lock.js';
-export { departedSources, membershipSweepFilters, sourceDocumentsFilter, staleDocumentsFilter, } from './sweep.js';
 export { buildCollectionDefinition } from './collection-definition.js';
 export { deriveCollectionName } from './collection-name.js';
 export { buildSearchParams } from './query-compiler.js';

package/dist/lookup.d.ts

@@ -1,22 +1,31 @@
 import type { Client } from 'typesense';
 import { type ReferenceProjection, type RootType, type SearchSchema, type SearchType } from '@lde/search';
 /**
- * One resolved level of a {@link ReferenceProjection}: the referents fetched
- * for it, keyed by IRI and still in the engine’s flat physical shape, plus the
- * levels resolved *from* them.
+ * One resolved level of a {@link ReferenceProjection}. A level is one of two
+ * things, discriminated by `via`, because the two cost different things:
+ *
+ * - **`lookup`** – referents fetched from the target’s own collection, keyed by
+ *   id and still in the engine’s flat physical shape. One batched round trip.
+ * - **`nested`** – an inline reference descended into. Its entries are already
+ *   in the hit, so nothing is fetched and there is no target; the level exists
+ *   only to carry the levels resolved *below* it.
  *
  * Kept flat rather than reconstructed here so one reconstruction path serves
  * hits and referents alike – the surface never learns that a nested document
  * arrived by a second round-trip rather than from the hit itself.
  */
-export interface ResolvedReferents {
-    /** The Root Type these referents are declared by – reconstruction reads them
-     *  through the target’s own declaration, never the referrer’s. */
+export type ResolvedReferents = {
+    readonly via: 'lookup';
+    /** The Root Type these referents are declared by – reconstruction reads
+     *  them through the target’s own declaration, never the referrer’s. */
     readonly target: RootType;
     readonly documents: ReadonlyMap<string, Record<string, unknown>>;
     /** Keyed by the reference field’s name on the type this level carries. */
     readonly children: ReadonlyMap<string, ResolvedReferents>;
-}
+} | {
+    readonly via: 'nested';
+    readonly children: ReadonlyMap<string, ResolvedReferents>;
+};
 /**
  * Resolve a projection against the documents that name its referents: one
  * batched `multi_search` **per level**, never per document, because the IRIs of

package/dist/lookup.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"lookup.d.ts","sourceRoot":"","sources":["../src/lookup.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,WAAW,CAAC;AACxC,OAAO,EACL,KAAK,mBAAmB,EACxB,KAAK,QAAQ,EACb,KAAK,YAAY,EACjB,KAAK,UAAU,EAChB,MAAM,aAAa,CAAC;AASrB;;;;;;;;GAQG;AACH,MAAM,WAAW,iBAAiB;IAChC;sEACkE;IAClE,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC;IAC1B,QAAQ,CAAC,SAAS,EAAE,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;IACjE,0EAA0E;IAC1E,QAAQ,CAAC,QAAQ,EAAE,WAAW,CAAC,MAAM,EAAE,iBAAiB,CAAC,CAAC;CAC3D;AAKD;;;;;;;;GAQG;AACH,wBAAsB,iBAAiB,CACrC,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,aAAa,CAAC,EACnC,UAAU,EAAE,mBAAmB,GAAG,SAAS,EAC3C,UAAU,EAAE,UAAU,EACtB,MAAM,EAAE,YAAY,EACpB,WAAW,EAAE,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,EACxC,OAAO,EAAE,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,EAC3C,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,OAAO,KAAK,IAAI,GACjC,OAAO,CAAC,WAAW,CAAC,MAAM,EAAE,iBAAiB,CAAC,CAAC,CAkEjD"}
+{"version":3,"file":"lookup.d.ts","sourceRoot":"","sources":["../src/lookup.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,WAAW,CAAC;AACxC,OAAO,EACL,KAAK,mBAAmB,EACxB,KAAK,QAAQ,EACb,KAAK,YAAY,EACjB,KAAK,UAAU,EAChB,MAAM,aAAa,CAAC;AAUrB;;;;;;;;;;;;;GAaG;AACH,MAAM,MAAM,iBAAiB,GACzB;IACE,QAAQ,CAAC,GAAG,EAAE,QAAQ,CAAC;IACvB;2EACuE;IACvE,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC;IAC1B,QAAQ,CAAC,SAAS,EAAE,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;IACjE,0EAA0E;IAC1E,QAAQ,CAAC,QAAQ,EAAE,WAAW,CAAC,MAAM,EAAE,iBAAiB,CAAC,CAAC;CAC3D,GACD;IACE,QAAQ,CAAC,GAAG,EAAE,QAAQ,CAAC;IACvB,QAAQ,CAAC,QAAQ,EAAE,WAAW,CAAC,MAAM,EAAE,iBAAiB,CAAC,CAAC;CAC3D,CAAC;AAKN;;;;;;;;GAQG;AACH,wBAAsB,iBAAiB,CACrC,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,aAAa,CAAC,EACnC,UAAU,EAAE,mBAAmB,GAAG,SAAS,EAC3C,UAAU,EAAE,UAAU,EACtB,MAAM,EAAE,YAAY,EACpB,WAAW,EAAE,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,EACxC,OAAO,EAAE,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,EAC3C,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,OAAO,KAAK,IAAI,GACjC,OAAO,CAAC,WAAW,CAAC,MAAM,EAAE,iBAAiB,CAAC,CAAC,CAwFjD"}

package/dist/lookup.js

@@ -1,4 +1,4 @@
-import { displayFieldName, fieldNamed, labelFieldOf, rootTypeNamed, } from '@lde/search/adapter';
+import { displayFieldName, fieldNamed, labelFieldOf, nestedReferenceType, rootTypeNamed, } from '@lde/search/adapter';
 import { escapeFilterValue } from './query-compiler.js';
 /** Typesense caps a filter list; the same batch size the label lookup uses. */
 const BATCH_SIZE = 200;
@@ -21,9 +21,24 @@ export async function resolveProjection(client, projection, searchType, schema,
     // stated bound – one round-trip per level – one per field instead.
     const levels = await Promise.all(Object.entries(projection).map(async ([name, level]) => {
         const field = fieldNamed(searchType, name);
-        if (field === undefined ||
-            field.kind !== 'reference' ||
-            field.ref?.strategy !== 'lookup') {
+        if (field === undefined) {
+            return undefined;
+        }
+        // An inline level costs no round trip: its entries are already in the
+        // parents. Flatten them out and hand them down, so the level BELOW reads
+        // its ids off the entries rather than off the document – which is the
+        // whole of what makes a lookup inside a qualified edge resolvable.
+        const nested = nestedReferenceType(schema, field);
+        if (nested !== undefined) {
+            return [
+                name,
+                {
+                    via: 'nested',
+                    children: await resolveProjection(client, level.resolve, nested, schema, collections, entriesOf(parents, name), onError),
+                },
+            ];
+        }
+        if (field.kind !== 'reference' || field.ref?.strategy !== 'lookup') {
             // `assertValidQuery` rejects this for every caller; skipping keeps a
             // hand-built query from throwing here rather than at the port’s guard.
             return undefined;
@@ -44,6 +59,7 @@ export async function resolveProjection(client, projection, searchType, schema,
         return [
             name,
             {
+                via: 'lookup',
                 target,
                 documents,
                 // The level below reads the IRIs off the documents this one just
@@ -59,14 +75,40 @@ export async function resolveProjection(client, projection, searchType, schema,
     }
     return resolved;
 }
-/** Every distinct IRI a page of documents carries under one reference field. */
+/**
+ * The nested entries a page of documents carries under one inline reference,
+ * flattened into one list – the “documents” the level below resolves against.
+ *
+ * Flattening rather than grouping is what keeps the round-trip count per
+ * *level* rather than per document: every entry on the page contributes its ids
+ * to one batch, and reconstruction re-associates them afterwards by id.
+ */
+function entriesOf(documents, field) {
+    return documents.flatMap((document) => {
+        const raw = document[field];
+        return (Array.isArray(raw) ? raw : [raw]).filter((entry) => typeof entry === 'object' && entry !== null);
+    });
+}
+/**
+ * Every distinct id a page of documents carries under one reference field.
+ *
+ * A reference stores its ids one of two ways, and which one is about how much
+ * of the referent the deployment wants stored, not about identity: a plain
+ * reference stores the id itself, while a
+ * {@link ReferenceStrategy.local local} lookup stores the endpoint’s own
+ * document, whose `id` is the same value. Both are read, so a level resolves
+ * the same either way.
+ */
 function distinctIris(documents, field) {
     const iris = new Set();
     for (const document of documents) {
         const raw = document[field];
         for (const value of Array.isArray(raw) ? raw : [raw]) {
-            if (typeof value === 'string' && value !== '') {
-                iris.add(value);
+            const id = typeof value === 'object' && value !== null
+                ? value.id
+                : value;
+            if (typeof id === 'string' && id !== '') {
+                iris.add(id);
             }
         }
     }

package/dist/query-compiler.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"query-compiler.d.ts","sourceRoot":"","sources":["../src/query-compiler.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sCAAsC,CAAC;AAEzE,OAAO,EAGL,KAAK,MAAM,EACX,KAAK,QAAQ,EAEb,KAAK,WAAW,EAChB,KAAK,YAAY,EACjB,KAAK,UAAU,EAEhB,MAAM,aAAa,CAAC;AAarB;mDACmD;AACnD,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,UAAU,EAAE,QAAQ,CAAC;IAC9B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;CAC7B;AAED;;;;GAIG;AACH,MAAM,WAAW,wBAAwB;IACvC;;;;;;;;;;;;;;;;OAgBG;IACH,QAAQ,CAAC,aAAa,CAAC,EAAE,CACvB,IAAI,EAAE,UAAU,EAChB,IAAI,EAAE,SAAS,MAAM,EAAE,KACpB,UAAU,GAAG,SAAS,CAAC;IAC5B;;;;;;OAMG;IACH,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM,CAAC;IACjC;;;;;;;;OAQG;IACH,QAAQ,CAAC,MAAM,CAAC,EAAE,YAAY,CAAC;IAC/B;;;;;;;;;;;OAWG;IACH,QAAQ,CAAC,eAAe,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,IAAI,CAAC;CACrD;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,iBAAiB,CAC/B,KAAK,EAAE,WAAW,EAClB,UAAU,EAAE,UAAU,EACtB,OAAO,GAAE,wBAA6B,GACrC,YAAY,CAAC,MAAM,CAAC,CA8BtB;AA6VD;;;;GAIG;AACH,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAEvD"}
+{"version":3,"file":"query-compiler.d.ts","sourceRoot":"","sources":["../src/query-compiler.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sCAAsC,CAAC;AAEzE,OAAO,EAGL,KAAK,MAAM,EACX,KAAK,QAAQ,EAEb,KAAK,WAAW,EAChB,KAAK,YAAY,EACjB,KAAK,UAAU,EAIhB,MAAM,aAAa,CAAC;AAmBrB;mDACmD;AACnD,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,UAAU,EAAE,QAAQ,CAAC;IAC9B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;CAC7B;AAED;;;;GAIG;AACH,MAAM,WAAW,wBAAwB;IACvC;;;;;;;;;;;;;;;;OAgBG;IACH,QAAQ,CAAC,aAAa,CAAC,EAAE,CACvB,IAAI,EAAE,UAAU,EAChB,IAAI,EAAE,SAAS,MAAM,EAAE,KACpB,UAAU,GAAG,SAAS,CAAC;IAC5B;;;;;;OAMG;IACH,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM,CAAC;IACjC;;;;;;;;OAQG;IACH,QAAQ,CAAC,MAAM,CAAC,EAAE,YAAY,CAAC;IAC/B;;;;;;;;;;;OAWG;IACH,QAAQ,CAAC,eAAe,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,IAAI,CAAC;CACrD;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,iBAAiB,CAC/B,KAAK,EAAE,WAAW,EAClB,UAAU,EAAE,UAAU,EACtB,OAAO,GAAE,wBAA6B,GACrC,YAAY,CAAC,MAAM,CAAC,CAkCtB;AAwfD;;;;GAIG;AACH,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAEvD"}