@lde/search-typesense 0.28.0 → 0.29.0

package/dist/documents.js

@@ -0,0 +1,529 @@
+// The documents a run writes into one collection, and every decision about
+// which of them leave it. Owns the private bookkeeping columns, the batching,
+// and the two regimes a collection can be in – a document belongs to one
+// dataset, or several reference it – so a writer states *what* should go and
+// never *how* it is stored.
+import { datasetField, isInternalField, physicalFields, } from '@lde/search/adapter';
+import { escapeFilterValue } from './query-compiler.js';
+/** The private field carrying the dataset IRI a document came from, in a
+ *  collection whose documents each belong to one dataset. */
+const SOURCE_FIELD = 'source';
+/** The private field carrying the id of the run that last wrote a document,
+ *  beside {@link SOURCE_FIELD}. */
+const LAST_SEEN_FIELD = 'last_seen';
+/** The private field carrying the datasets that reference a document, in a
+ *  collection keyed on a canonical identifier: one `{ dataset, run }` element
+ *  per referring dataset. */
+const REFERENCED_BY_FIELD = 'referenced_by';
+/** Documents read back per round-trip while retracting a dataset – the unit of
+ *  work the sweep’s memory is bounded by, and Typesense’s `per_page` ceiling. */
+const RETRACTION_PAGE = 250;
+/** Ids per `multi_search` entry when reading membership back before a write. */
+const LOOKUP_BATCH = 200;
+/** Entries per `multi_search` request: the engine’s own default ceiling
+ *  (`max-searches-in-multi-search`), which a large `batchSize` would exceed. */
+const MAX_SEARCHES_PER_REQUEST = 50;
+/** Stay well under Typesense’s ~4000-char URL query-string limit per delete. */
+const MAX_FILTER_VALUES_LENGTH = 3000;
+/**
+ * Default ceiling on the distinct datasets a collection may hold before the
+ * membership sweep refuses to guess. The sweep needs the complete set to spot
+ * the ones that left, so a truncated facet would silently miss deletions.
+ */
+export const DEFAULT_MAX_SWEEPABLE_SOURCES = 10_000;
+/**
+ * Open a run’s documents on a collection.
+ *
+ * Two regimes, chosen once from the type and never surfaced again:
+ *
+ * - a type with no {@link RootType.key} keys every document on its own node
+ *   IRI, so a document belongs to exactly one dataset. It is stamped with that
+ *   dataset and the run id, and a drop is a delete.
+ * - a **canonically keyed** type keys documents on an identifier several
+ *   datasets can share – two datasets referencing the same GeoNames place
+ *   produce one document. Membership is therefore a set: a write **adds** the
+ *   dataset to `referenced_by`, a drop **removes** it, and the document is
+ *   deleted when the last referring dataset goes. That is what stops one
+ *   dataset’s sweep from deleting what another still points at, without any
+ *   special case for a dataset skipped as unchanged – its membership simply
+ *   stays, untouched by anyone else’s sweep.
+ *
+ * The run is recorded on a keyed collection as `Date.parse(startedAt)` rather
+ * than as the run id, because *this dataset did not write it this run* has to
+ * be asked of one element of a nested array, and Typesense correlates a
+ * comparison within an element but **not** a negation – `run:<thisRun`
+ * correlates, `run:!=thisRun` silently answers at document level and sweeps
+ * nothing. See `test/membership-filters.integration.test.ts`, where both are
+ * pinned against the engine.
+ */
+export function openDocuments(client, collection, options) {
+    return keyOf(options.searchType) === undefined
+        ? ownedDocuments(client, collection, options)
+        : sharedDocuments(client, collection, options);
+}
+/**
+ * The collection definition plus the bookkeeping a writer keeps on it: the
+ * private membership columns, and the nesting a keyed collection’s
+ * `referenced_by` needs enabled.
+ *
+ * A single-dataset collection carries `source` only when the type declares no
+ * dataset field of its own; when it does, that field already holds the same
+ * IRI ({@link provenanceFieldOf}) and a second column would be the same value
+ * twice, free to drift.
+ */
+export function withBookkeeping(definition, searchType) {
+    const fields = [...(definition.fields ?? [])];
+    if (keyOf(searchType) !== undefined) {
+        return {
+            ...definition,
+            enable_nested_fields: true,
+            fields: [
+                ...fields,
+                { name: REFERENCED_BY_FIELD, type: 'object[]' },
+                {
+                    name: `${REFERENCED_BY_FIELD}.dataset`,
+                    type: 'string[]',
+                    facet: true,
+                },
+                { name: `${REFERENCED_BY_FIELD}.run`, type: 'int64[]' },
+            ],
+        };
+    }
+    return {
+        ...definition,
+        fields: [
+            ...fields,
+            ...(provenanceFieldOf(searchType) === SOURCE_FIELD
+                ? [{ name: SOURCE_FIELD, type: 'string', facet: true }]
+                : []),
+            { name: LAST_SEEN_FIELD, type: 'string' },
+        ],
+    };
+}
+/**
+ * Reject a {@link SearchType} whose declaration cannot carry this module’s
+ * bookkeeping, at writer construction – so it fails before a run touches the
+ * index rather than at the sweep, after the writes.
+ *
+ * A **canonically keyed** type may not declare a field over the dataset. Such a
+ * field asks *which dataset is this document’s*, and a document several
+ * datasets share has no answer: it would hold whichever wrote last, and a user
+ * filtering by one of the others would silently miss it.
+ *
+ * A single-dataset type declaring one makes it the collection’s provenance
+ * column, so it has to answer that question the way the private `source` did:
+ * not `array` (Typesense reads `field:=[a,b]` over a `string[]` as *contains
+ * any*, so a departed dataset would take every document merely mentioning it);
+ * no `transform` (the sweep matches the stored value against the run’s raw
+ * dataset IRIs); and `facetable` on its own name where the writer enumerates
+ * the indexed datasets by faceting it – a reference inheriting a facet policy
+ * facets a companion holding only the admitted keys, so enumerating through it
+ * would miss every dataset the policy excludes.
+ *
+ * @param requireFacetable Whether this writer enumerates the indexed datasets
+ *   ({@link RunDocuments.dropUnselected}); a writer that only ever filters by a
+ *   known IRI does not need the column facetable.
+ */
+export function assertWritableType(searchType, options) {
+    const keyed = keyOf(searchType) !== undefined;
+    const reserved = keyed
+        ? [REFERENCED_BY_FIELD]
+        : [SOURCE_FIELD, LAST_SEEN_FIELD];
+    const clashing = searchType.fields.filter((field) => reserved.includes(field.name));
+    if (clashing.length > 0) {
+        throw new Error(`SearchType “${searchType.name}” declares reserved bookkeeping field(s) ${clashing
+            .map((field) => `“${field.name}”`)
+            .join(', ')}`);
+    }
+    const declared = datasetField(searchType);
+    if (declared === undefined || isInternalField(declared)) {
+        return;
+    }
+    if (keyed) {
+        throw new Error(`SearchType “${searchType.name}” is keyed on “${searchType.key?.field}”, so several datasets can share one document, and “${declared.name}” over the indexed dataset would hold whichever of them wrote it last – a filter on any of the others would then miss the document. Drop the field: a keyed collection records every dataset that references a document.`);
+    }
+    const problems = [];
+    if (declared.array === true) {
+        problems.push('it is an array, and a membership sweep over one would delete every document merely mentioning a departed dataset');
+    }
+    if (declared.transform !== undefined) {
+        problems.push('it declares a transform, and the sweep matches the stored value against the run’s raw dataset IRIs');
+    }
+    if (options.requireFacetable && declared.facetable !== true) {
+        problems.push('it is not facetable, and the membership sweep enumerates the indexed datasets by faceting it');
+    }
+    else if (options.requireFacetable &&
+        physicalFields(declared, options.schema).facet !== declared.name) {
+        problems.push('it inherits a facet policy from the type it names, so the engine facets only the datasets the policy admits, and the membership sweep enumerates the indexed datasets by faceting it');
+    }
+    if (problems.length > 0) {
+        throw new Error(`SearchType “${searchType.name}” declares “${declared.name}” over the indexed dataset, which this writer keeps its provenance bookkeeping on, but ${problems.join('; and ')}.`);
+    }
+}
+/**
+ * Fail an EXISTING collection that predates its type being canonically keyed,
+ * naming the two steps that fix it.
+ *
+ * The field cannot be added: nesting is a create-time collection setting, and
+ * even where it could be it would be **empty** on every document already
+ * indexed. Membership is only written by a dataset that reprocesses, and
+ * skip-unchanged means most do not – so those documents would match no
+ * dataset, never be retracted and never be collected, while looking perfectly
+ * healthy. Dropping the collection alone has the same hole: a rebuilt
+ * collection is repopulated only by datasets that changed, so the pipeline
+ * version has to rotate with it to force every dataset through.
+ */
+export function assertBookkeepingPresent(searchType, existing, collection) {
+    if (keyOf(searchType) === undefined) {
+        return;
+    }
+    const names = new Set(existing.map((field) => field.name));
+    if (names.has(`${REFERENCED_BY_FIELD}.dataset`)) {
+        return;
+    }
+    throw new Error(`Typesense collection “${collection}” exists without the “${REFERENCED_BY_FIELD}” field its keyed SearchType “${searchType.name}” records membership in, so nothing in it can be swept. It cannot be added to a live collection, and would be empty on every document already indexed: drop “${collection}” (and the collections that join to it) and rotate the pipeline version, so the next run reprocesses every dataset and rebuilds them.`);
+}
+/** The dataset column a single-dataset collection carries: the type’s declared
+ *  dataset field, or the private `source` when it declares none. */
+function provenanceFieldOf(searchType) {
+    const declared = datasetField(searchType);
+    return declared === undefined || isInternalField(declared)
+        ? SOURCE_FIELD
+        : declared.name;
+}
+/** A type’s {@link RootType.key}, for the types that can have one. */
+function keyOf(searchType) {
+    return searchType.key;
+}
+/**
+ * A collection whose documents each belong to one dataset: stamped with it and
+ * the run, dropped by deleting them. What every collection did before keying
+ * existed, and still the regime for every type that declares no `key`.
+ */
+function ownedDocuments(client, collection, options) {
+    const { searchType, runId, batchSize } = options;
+    const field = provenanceFieldOf(searchType);
+    const buffer = new Batch(batchSize);
+    const run = escapeFilterValue(runId);
+    const of = (dataset) => `${field}:=${escapeFilterValue(dataset.iri.toString())}`;
+    const land = async () => {
+        const batch = buffer.take();
+        if (batch.length > 0) {
+            await importDocuments(client, collection, batch, 'upsert');
+        }
+    };
+    const drop = async (filter) => {
+        await land();
+        await deleteByFilter(client, collection, filter);
+    };
+    return {
+        // Stamping the dataset column here – rather than trusting the projection to
+        // have filled a declared one – is what keeps the sweep total: a caller
+        // projecting without a dataset context, or a document reaching the writer
+        // any other way, still lands sweepable. Where the projection did fill it,
+        // the value is the same IRI, so the stamp only reasserts it.
+        //
+        // It reasserts the sweep’s column only, never the physical fanout a
+        // declared field also produces (the folded `${name}_search` companion of a
+        // searchable one). Folding is the projection’s convention, and restating it
+        // here would put it in two places. So a document that reached the writer
+        // unprojected is sweepable but not full-text findable by dataset: absent,
+        // never wrong.
+        add: async (dataset, documents) => {
+            const stamp = {
+                [field]: dataset.iri.toString(),
+                [LAST_SEEN_FIELD]: runId,
+            };
+            for await (const document of documents) {
+                buffer.push({ ...document, ...stamp });
+                if (buffer.full()) {
+                    await land();
+                }
+            }
+        },
+        flush: land,
+        dropUnwritten: async (dataset) => drop(`${of(dataset)} && ${LAST_SEEN_FIELD}:!=${run}`),
+        dropWrittenThisRun: async (dataset) => drop(`${of(dataset)} && ${LAST_SEEN_FIELD}:=${run}`),
+        dropAll: async (dataset) => drop(of(dataset)),
+        dropUnselected: async (selectedDatasets) => {
+            await land();
+            const departed = await departedDatasets(client, collection, field, selectedDatasets, options.maxSweepableSources ?? DEFAULT_MAX_SWEEPABLE_SOURCES);
+            for (const chunk of chunked(departed.map(escapeFilterValue))) {
+                await deleteByFilter(client, collection, `${field}:=[${chunk.join(',')}]`);
+            }
+        },
+    };
+}
+/**
+ * A collection keyed on a canonical identifier, where one document can be
+ * reached from several datasets. Membership is the set of datasets referencing
+ * it; a drop retracts one dataset’s entry and deletes only what nobody is left
+ * referencing.
+ */
+function sharedDocuments(client, collection, options) {
+    const { batchSize } = options;
+    const run = Date.parse(options.startedAt);
+    const facet = `${REFERENCED_BY_FIELD}.dataset`;
+    const buffer = new Batch(batchSize);
+    const referencing = (datasets) => `${facet}:=[${datasets.map(escapeFilterValue).join(',')}]`;
+    /**
+     * Land a batch, adding each document’s dataset to the membership already
+     * stored. Typesense replaces a whole document on upsert and has no way to
+     * append to an array, so the set has to be read back first – one
+     * `multi_search` per batch, against the ids the batch carries.
+     */
+    const land = async () => {
+        const batch = buffer.take();
+        if (batch.length === 0) {
+            return;
+        }
+        // Two nodes of one dataset, or two datasets, can key onto one document
+        // within a single batch. Later content wins, as a merged document always
+        // does – but membership is the union, or the second would erase the first.
+        const merged = new Map();
+        for (const { document, dataset } of batch) {
+            const seen = merged.get(document.id);
+            merged.set(document.id, {
+                document,
+                datasets: seen === undefined
+                    ? [dataset]
+                    : seen.datasets.includes(dataset)
+                        ? seen.datasets
+                        : [...seen.datasets, dataset],
+            });
+        }
+        const stored = await storedMembership(client, collection, [
+            ...merged.keys(),
+        ]);
+        await importDocuments(client, collection, [...merged.values()].map(({ document, datasets }) => ({
+            ...document,
+            [REFERENCED_BY_FIELD]: [
+                ...(stored.get(document.id) ?? []).filter((referrer) => !datasets.includes(referrer.dataset)),
+                ...datasets.map((dataset) => ({ dataset, run })),
+            ],
+        })), 'upsert');
+    };
+    /**
+     * Take the datasets out of every document the filter selects, deleting the
+     * documents left with no referrer.
+     *
+     * The filter is re-asked from the first page each round rather than
+     * paginated: a document this pass touches stops matching it, so the loop
+     * drains, its memory is one page whatever the collection holds, and deep
+     * pagination never comes into it. A round that changes nothing would spin,
+     * so it throws instead.
+     */
+    const retract = async (filter, datasets) => {
+        await land();
+        for (;;) {
+            const page = await search(client, collection, {
+                filter_by: filter,
+                include_fields: `id,${REFERENCED_BY_FIELD}`,
+                per_page: RETRACTION_PAGE,
+            });
+            const hits = (page.hits ?? []).map((hit) => hit.document);
+            if (hits.length === 0) {
+                return;
+            }
+            const keeping = [];
+            const emptied = [];
+            for (const document of hits) {
+                const referrers = referrersOf(document);
+                const remaining = referrers.filter((referrer) => !datasets.includes(referrer.dataset));
+                if (remaining.length === referrers.length) {
+                    // Selected, yet carries none of the datasets being retracted: writing
+                    // it back would change nothing and it would match again next round.
+                    continue;
+                }
+                if (remaining.length === 0) {
+                    emptied.push(String(document.id));
+                }
+                else {
+                    keeping.push({
+                        id: document.id,
+                        [REFERENCED_BY_FIELD]: remaining,
+                    });
+                }
+            }
+            if (keeping.length === 0 && emptied.length === 0) {
+                throw new Error(`Retracting ${datasets.join(', ')} from “${collection}” selected ${hits.length} document(s) carrying none of them, so the sweep cannot make progress.`);
+            }
+            if (keeping.length > 0) {
+                await importDocuments(client, collection, keeping, 'update');
+            }
+            for (const chunk of chunked(emptied.map(escapeFilterValue))) {
+                await deleteByFilter(client, collection, `id:=[${chunk.join(',')}]`);
+            }
+        }
+    };
+    return {
+        add: async (dataset, documents) => {
+            const iri = dataset.iri.toString();
+            for await (const document of documents) {
+                buffer.push({ document, dataset: iri });
+                if (buffer.full()) {
+                    await land();
+                }
+            }
+        },
+        flush: land,
+        dropUnwritten: async (dataset) => {
+            const iri = dataset.iri.toString();
+            await retract(`${REFERENCED_BY_FIELD}.{dataset:=${escapeFilterValue(iri)} && run:<${run}}`, [iri]);
+        },
+        dropWrittenThisRun: async (dataset) => {
+            const iri = dataset.iri.toString();
+            await retract(`${REFERENCED_BY_FIELD}.{dataset:=${escapeFilterValue(iri)} && run:=${run}}`, [iri]);
+        },
+        dropAll: async (dataset) => {
+            const iri = dataset.iri.toString();
+            await retract(referencing([iri]), [iri]);
+        },
+        dropUnselected: async (selectedDatasets) => {
+            await land();
+            const departed = await departedDatasets(client, collection, facet, selectedDatasets, options.maxSweepableSources ?? DEFAULT_MAX_SWEEPABLE_SOURCES);
+            // Every departed dataset at once: one document can be referenced by
+            // several of them, and retracting them together spares it a second pass.
+            for (const chunk of chunked(departed.map(escapeFilterValue))) {
+                await retract(`${facet}:=[${chunk.join(',')}]`, departed);
+            }
+        },
+    };
+}
+/** The membership stored for these ids, read back before a write adds to it. */
+async function storedMembership(client, collection, ids) {
+    const searches = [];
+    for (let start = 0; start < ids.length; start += LOOKUP_BATCH) {
+        const batch = ids.slice(start, start + LOOKUP_BATCH);
+        searches.push({
+            collection,
+            q: '*',
+            filter_by: `id:[${batch.map(escapeFilterValue).join(',')}]`,
+            include_fields: `id,${REFERENCED_BY_FIELD}`,
+            per_page: batch.length,
+        });
+    }
+    const membership = new Map();
+    // A batch wide enough to need more entries than the engine accepts per
+    // request travels as several requests: `batchSize` is the caller's knob, and
+    // it must not decide whether a write succeeds.
+    for (let start = 0; start < searches.length; start += MAX_SEARCHES_PER_REQUEST) {
+        const { results } = (await client.multiSearch.perform({
+            searches: searches.slice(start, start + MAX_SEARCHES_PER_REQUEST),
+        }));
+        for (const result of results) {
+            // multi_search reports a failed entry inline instead of rejecting. A lost
+            // entry would silently drop the membership of everything it covered, so
+            // it is raised rather than skipped.
+            if (result.error !== undefined) {
+                throw new Error(`Reading membership from “${collection}” failed: ${result.error}`);
+            }
+            for (const hit of result.hits ?? []) {
+                membership.set(String(hit.document.id), referrersOf(hit.document));
+            }
+        }
+    }
+    return membership;
+}
+/** The referrers stored on a document, ignoring anything malformed. */
+function referrersOf(document) {
+    const stored = document[REFERENCED_BY_FIELD];
+    return (Array.isArray(stored) ? stored : []).filter((referrer) => typeof referrer === 'object' &&
+        referrer !== null &&
+        typeof referrer.dataset === 'string');
+}
+/**
+ * The datasets a collection holds that the run’s selection does not, via a
+ * single facet over its membership column.
+ *
+ * Asks for one bucket beyond the ceiling so genuine truncation is
+ * distinguishable from an exactly-full result: `maxDatasets` buckets are
+ * returned intact, one more proves the sweep would miss some, so it throws
+ * rather than delete blind.
+ */
+async function departedDatasets(client, collection, facetField, selectedDatasets, maxDatasets) {
+    const response = await search(client, collection, {
+        per_page: 0,
+        facet_by: facetField,
+        max_facet_values: maxDatasets + 1,
+    });
+    const counts = response.facet_counts?.[0]?.counts ?? [];
+    if (counts.length > maxDatasets) {
+        throw new Error(`Membership sweep cannot see beyond ${maxDatasets} distinct datasets in “${collection}”; raise maxSweepableSources or departed datasets might be missed`);
+    }
+    const selected = new Set(selectedDatasets);
+    return counts
+        .map((count) => count.value)
+        .filter((dataset) => !selected.has(dataset));
+}
+/** Escaped values grouped into filters that fit a URL query string. */
+function* chunked(values) {
+    let chunk = [];
+    let length = 0;
+    for (const value of values) {
+        if (length + value.length > MAX_FILTER_VALUES_LENGTH && chunk.length > 0) {
+            yield chunk;
+            chunk = [];
+            length = 0;
+        }
+        chunk.push(value);
+        length += value.length + 1;
+    }
+    if (chunk.length > 0) {
+        yield chunk;
+    }
+}
+/** Accumulates items until a batch is full. */
+class Batch {
+    size;
+    items = [];
+    constructor(size) {
+        this.size = size;
+    }
+    push(item) {
+        this.items.push(item);
+    }
+    full() {
+        return this.items.length >= this.size;
+    }
+    /** The accumulated items, emptying the batch. */
+    take() {
+        const items = this.items;
+        this.items = [];
+        return items;
+    }
+}
+/**
+ * Import one batch, throwing if any individual document fails – Typesense’s
+ * bulk import otherwise reports per-document failures without rejecting, which
+ * would turn a rejected write into silently missing data.
+ */
+async function importDocuments(client, collection, batch, action) {
+    const results = (await client
+        .collections(collection)
+        .documents()
+        .import(batch, {
+        action,
+        throwOnFail: false,
+    }));
+    const failures = results.filter((result) => !result.success);
+    if (failures.length > 0) {
+        throw new Error(`Typesense ${action} into “${collection}” failed for ${failures.length}/${results.length} documents: ${failures
+            .map((failure) => failure.error)
+            .join('; ')}`);
+    }
+}
+/** A whole-collection search; `q: '*'` needs no field to query by. */
+async function search(client, collection, parameters) {
+    return (await client
+        .collections(collection)
+        .documents()
+        .search({ q: '*', ...parameters }));
+}
+/** Delete a collection’s documents matching a Typesense filter. */
+async function deleteByFilter(client, collection, filterBy) {
+    await client
+        .collections(collection)
+        .documents()
+        .delete({ filter_by: filterBy });
+}

package/dist/in-place-rebuild.d.ts

@@ -5,27 +5,33 @@ import { type RebuildOptions } from './rebuild-support.js';
 /** {@link InPlaceRebuild} options: the shared rebuild options plus the
  *  membership-sweep ceiling. */
 export interface InPlaceRebuildOptions extends RebuildOptions {
-    /** Most distinct sources the membership sweep may enumerate before it throws
-     *  rather than risk missing departed sources (default 10 000). */
+    /**
+     * Most distinct datasets the membership sweep may enumerate before it throws
+     * rather than risk missing departed ones (default 10 000). The sweep needs
+     * the complete set to spot which datasets left, so a truncated facet would
+     * silently miss deletions. Raise it (up to the engine’s `max_facet_values`
+     * limit) before an index approaches it, so the ceiling is a tunable guard
+     * rather than a hard wall.
+     */
     readonly maxSweepableSources?: number;
 }
 /**
  * In-place Rebuild (update the live index directly – upsert changed sources,
  * sweep the rest – rather than swap in a fresh one) as a transactional
  * `Writer`: documents are upserted into one long-lived collection with
- * per-source atomicity – no swap, no staging. Every document is stamped with
- * its `source` (the dataset IRI) and `last_seen` (the run id); deletion is a
- * sweep, never special-cased:
+ * per-dataset atomicity – no swap, no staging. Deletion is a sweep, never
+ * special-cased, and {@link openDocuments} decides what a deletion physically
+ * is – for a canonically keyed type, dropping one dataset’s membership of a
+ * document several datasets share, and the document itself only with the last
+ * of them:
  *
- * - a **successful dataset flush** deletes the source’s documents the run did
- *   not rewrite (`source = dataset && last_seen != runId`). A failed dataset
- *   is not swept – its output is incomplete, and the next successful run
- *   reconciles;
- * - **reset** (the pipeline’s dump-fallback discard) deletes only *this run’s*
- *   writes for the dataset (`source = dataset && last_seen = runId`), so the
- *   dump re-run rebuilds it cleanly while the source’s prior-run documents are
- *   left for the success sweep to reconcile;
- * - **commit** deletes every document whose source left the run’s selection
+ * - a **successful dataset flush** drops what the dataset had and this run did
+ *   not rewrite. A failed dataset is not swept – its output is incomplete, and
+ *   the next successful run reconciles;
+ * - **reset** (the pipeline’s dump-fallback discard) drops only *this run’s*
+ *   writes for the dataset, so the dump re-run rebuilds it cleanly while the
+ *   dataset’s prior-run documents are left for the success sweep to reconcile;
+ * - **commit** drops what belongs to datasets that left the run’s selection
  *   (registry-membership sweep over {@link RunContext.selectedSources}, which
  *   includes datasets skipped as unchanged) and releases the lock;
  * - **abort** only releases the lock: upserts are idempotent, so whatever
@@ -33,15 +39,17 @@ export interface InPlaceRebuildOptions extends RebuildOptions {
  *
  * `openRun` takes the single-flight cross-pod lock (throwing
  * `RebuildAlreadyRunning` when another rebuild holds it) and creates
- * the collection on demand from the {@link SearchType} plus the two
- * bookkeeping fields. A collection that already exists is used as it is, with
- * one exception: it must carry every reference field the declaration asks for
- * and every facet companion its facets read, or the run fails with the
- * drop-and-rebuild that fixes it
- * ({@link assertReferencesAndFacetCompanionsPresent}).
+ * the collection on demand from the {@link SearchType} plus the bookkeeping
+ * fields. A collection that already exists is used as it is, with two
+ * exceptions: it must carry every reference field the declaration asks for and
+ * every facet companion its facets read
+ * ({@link assertReferencesAndFacetCompanionsPresent}), and the membership field
+ * a keyed type records in ({@link assertBookkeepingPresent}) – each failing
+ * with the rebuild that fixes it.
  *
- * Document ids must be unique per (source, entity) – the caller keys them –
- * or documents from different sources overwrite each other.
+ * Unless the type is canonically keyed, document ids must be unique per
+ * (dataset, entity) – the caller keys them – or documents from different
+ * datasets overwrite each other.
  *
  * The name is the NDE Stack’s pattern
  * ({@link https://docs.nde.nl/stack/patterns | Stack patterns}).
@@ -60,20 +68,7 @@ export declare class InPlaceRebuild<TDocument extends {
     readonly collectionName: string;
     private readonly maxSweepableSources;
     private readonly resolved;
-    /** The column this collection carries its documents’ dataset IRI in: the
-     *  type’s declared dataset field, or the private `source` when it declares
-     *  none. Everything below – the stamp, every sweep filter, the source
-     *  enumeration – reads this one name. */
-    private readonly sourceField;
     constructor(client: Client, searchType: SearchType, options?: InPlaceRebuildOptions);
     openRun(context: RunContext): Promise<RunWriter<TDocument>>;
-    /**
-     * The distinct sources present in the collection, via a single facet over its
-     * {@link provenanceField}. Requests one bucket beyond `maxSources` so genuine truncation is
-     * distinguishable from an exactly-full result: `maxSources` buckets are
-     * returned intact, `maxSources + 1` proves more exist and the sweep would
-     * miss some, so it throws rather than delete blind.
-     */
-    private indexedSources;
 }
 //# sourceMappingURL=in-place-rebuild.d.ts.map

package/dist/in-place-rebuild.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"in-place-rebuild.d.ts","sourceRoot":"","sources":["../src/in-place-rebuild.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,WAAW,CAAC;AAExC,OAAO,KAAK,EAAgB,UAAU,EAAE,MAAM,aAAa,CAAC;AAG5D,OAAO,KAAK,EAEV,UAAU,EACV,SAAS,EACT,MAAM,EACP,MAAM,eAAe,CAAC;AAavB,OAAO,EAML,KAAK,cAAc,EAEpB,MAAM,sBAAsB,CAAC;AAa9B;gCACgC;AAChC,MAAM,WAAW,qBAAsB,SAAQ,cAAc;IAC3D;sEACkE;IAClE,QAAQ,CAAC,mBAAmB,CAAC,EAAE,MAAM,CAAC;CACvC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoCG;AACH,qBAAa,cAAc,CACzB,SAAS,SAAS;IAAE,EAAE,EAAE,MAAM,CAAA;CAAE,CAChC,YAAW,MAAM,CAAC,SAAS,CAAC;IAiB1B,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,UAAU;IAjB7B;;;;;OAKG;IACH,SAAgB,cAAc,EAAE,MAAM,CAAC;IACvC,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAS;IAC7C,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAyB;IAClD;;;6CAGyC;IACzC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;gBAGlB,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,UAAU,EACvC,OAAO,GAAE,qBAA0B;IAkB/B,OAAO,CAAC,OAAO,EAAE,UAAU,GAAG,OAAO,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;IA6HjE;;;;;;OAMG;YACW,cAAc;CAsB7B"}
+{"version":3,"file":"in-place-rebuild.d.ts","sourceRoot":"","sources":["../src/in-place-rebuild.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,WAAW,CAAC;AAExC,OAAO,KAAK,EAAgB,UAAU,EAAE,MAAM,aAAa,CAAC;AAG5D,OAAO,KAAK,EAEV,UAAU,EACV,SAAS,EACT,MAAM,EACP,MAAM,eAAe,CAAC;AASvB,OAAO,EAEL,KAAK,cAAc,EAEpB,MAAM,sBAAsB,CAAC;AAE9B;gCACgC;AAChC,MAAM,WAAW,qBAAsB,SAAQ,cAAc;IAC3D;;;;;;;OAOG;IACH,QAAQ,CAAC,mBAAmB,CAAC,EAAE,MAAM,CAAC;CACvC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsCG;AACH,qBAAa,cAAc,CACzB,SAAS,SAAS;IAAE,EAAE,EAAE,MAAM,CAAA;CAAE,CAChC,YAAW,MAAM,CAAC,SAAS,CAAC;IAY1B,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,UAAU;IAZ7B;;;;;OAKG;IACH,SAAgB,cAAc,EAAE,MAAM,CAAC;IACvC,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAqB;IACzD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAyB;gBAG/B,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,UAAU,EACvC,OAAO,GAAE,qBAA0B;IAe/B,OAAO,CAAC,OAAO,EAAE,UAAU,GAAG,OAAO,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;CAuElE"}