@lde/search-typesense 0.18.1 → 0.19.0

package/dist/blue-green-rebuild.d.ts

@@ -50,6 +50,10 @@ export declare class BlueGreenRebuild<TDocument extends {
      */
     readonly collectionName: string;
     private readonly resolved;
+    /** The column this collection carries its documents’ dataset IRI in: the
+     *  type’s declared dataset field, or the private `source` when it declares
+     *  none. Both the stamp and the per-dataset rollback filter read it. */
+    private readonly sourceField;
     constructor(client: Client, searchType: SearchType, options?: BlueGreenRebuildOptions);
     openRun(context: RunContext): Promise<RunWriter<TDocument>>;
     /** The collection an alias currently points at, or `undefined` if unset. */

package/dist/blue-green-rebuild.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"blue-green-rebuild.d.ts","sourceRoot":"","sources":["../src/blue-green-rebuild.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,WAAW,CAAC;AACxC,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAC9C,OAAO,KAAK,EAEV,UAAU,EACV,SAAS,EACT,MAAM,EACP,MAAM,eAAe,CAAC;AAMvB,OAAO,EAKL,KAAK,cAAc,EAEpB,MAAM,sBAAsB,CAAC;AAE9B;;yDAEyD;AACzD,MAAM,MAAM,uBAAuB,GAAG,cAAc,CAAC;AAErD;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,qBAAa,gBAAgB,CAC3B,SAAS,SAAS;IAAE,EAAE,EAAE,MAAM,CAAA;CAAE,CAChC,YAAW,MAAM,CAAC,SAAS,CAAC;IAa1B,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,UAAU;IAb7B;;;;;;;OAOG;IACH,SAAgB,cAAc,EAAE,MAAM,CAAC;IACvC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAyB;gBAG/B,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,UAAU,EACvC,OAAO,GAAE,uBAA4B;IAQjC,OAAO,CAAC,OAAO,EAAE,UAAU,GAAG,OAAO,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;IA6FjE,4EAA4E;YAC9D,WAAW;CAW1B"}
+{"version":3,"file":"blue-green-rebuild.d.ts","sourceRoot":"","sources":["../src/blue-green-rebuild.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,WAAW,CAAC;AACxC,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAC9C,OAAO,KAAK,EAEV,UAAU,EACV,SAAS,EACT,MAAM,EACP,MAAM,eAAe,CAAC;AAUvB,OAAO,EAML,KAAK,cAAc,EAEpB,MAAM,sBAAsB,CAAC;AAE9B;;yDAEyD;AACzD,MAAM,MAAM,uBAAuB,GAAG,cAAc,CAAC;AAErD;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,qBAAa,gBAAgB,CAC3B,SAAS,SAAS;IAAE,EAAE,EAAE,MAAM,CAAA;CAAE,CAChC,YAAW,MAAM,CAAC,SAAS,CAAC;IAiB1B,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,UAAU;IAjB7B;;;;;;;OAOG;IACH,SAAgB,cAAc,EAAE,MAAM,CAAC;IACvC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAyB;IAClD;;4EAEwE;IACxE,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;gBAGlB,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,UAAU,EACvC,OAAO,GAAE,uBAA4B;IAYjC,OAAO,CAAC,OAAO,EAAE,UAAU,GAAG,OAAO,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;IAmGjE,4EAA4E;YAC9D,WAAW;CAW1B"}

package/dist/blue-green-rebuild.js

@@ -1,8 +1,8 @@
 import { buildCollectionDefinition } from './collection-definition.js';
 import { BatchImporter } from './import.js';
 import { httpStatus, openLockedRun, releaseLock } from './lock.js';
-import { SOURCE_FIELD, sourceDocumentsFilter } from './sweep.js';
-import { assertNoReservedFields, deleteByFilter, resolveRebuildOptions, stampDocuments, } from './rebuild-support.js';
+import { SOURCE_FIELD, provenanceField, sourceDocumentsFilter, } from './sweep.js';
+import { assertNoReservedFields, assertSweepableProvenanceField, deleteByFilter, resolveRebuildOptions, stampDocuments, } from './rebuild-support.js';
 /**
  * Blue/green Rebuild (build a fresh index alongside the live one, then swap to
  * it atomically) as a transactional `Writer`: each run builds the search
@@ -45,17 +45,26 @@ export class BlueGreenRebuild {
      */
     collectionName;
     resolved;
+    /** The column this collection carries its documents’ dataset IRI in: the
+     *  type’s declared dataset field, or the private `source` when it declares
+     *  none. Both the stamp and the per-dataset rollback filter read it. */
+    sourceField;
     constructor(client, searchType, options = {}) {
         this.client = client;
         this.searchType = searchType;
         // `source` is stamped on every document for per-dataset rollback.
         assertNoReservedFields(searchType, [SOURCE_FIELD]);
+        // Rollback filters by a known dataset IRI rather than enumerating the
+        // indexed ones, so a declared field need not be facetable here.
+        assertSweepableProvenanceField(searchType, { requireFacetable: false });
+        this.sourceField = provenanceField(searchType);
         this.resolved = resolveRebuildOptions(searchType, options);
         this.collectionName = this.resolved.definitionOptions.name;
     }
     async openRun(context) {
         const { batchSize, lockTtlMs, definitionOptions } = this.resolved;
         const name = this.collectionName;
+        const sourceField = this.sourceField;
         return openLockedRun(this.client, name, lockTtlMs, async () => {
             // Create the fresh (blue) collection up front, so a failure surfaces
             // before any dataset is processed. startedAt orders the versioned names;
@@ -66,9 +75,14 @@ export class BlueGreenRebuild {
             await this.client.collections().create({
                 ...definition,
                 name: collection,
+                // The private `source` only when the type declares no dataset field of
+                // its own; a declared one is already in the definition and carries the
+                // same IRI.
                 fields: [
                     ...(definition.fields ?? []),
-                    { name: SOURCE_FIELD, type: 'string' },
+                    ...(sourceField === SOURCE_FIELD
+                        ? [{ name: SOURCE_FIELD, type: 'string' }]
+                        : []),
                 ],
             });
             const importer = new BatchImporter(this.client, collection, batchSize);
@@ -77,11 +91,11 @@ export class BlueGreenRebuild {
             // still buffered must land first, so the delete filter sees them.
             const rollback = async (dataset) => {
                 await importer.flush();
-                await deleteByFilter(this.client, collection, sourceDocumentsFilter(dataset.iri.toString()));
+                await deleteByFilter(this.client, collection, sourceDocumentsFilter(sourceField, dataset.iri.toString()));
             };
             return {
                 write: async (dataset, documents) => importer.add(stampDocuments(documents, {
-                    [SOURCE_FIELD]: dataset.iri.toString(),
+                    [sourceField]: dataset.iri.toString(),
                 })),
                 flush: async (dataset, outcome) => {
                     // A successful dataset keeps its streamed documents (they go live at

package/dist/in-place-rebuild.d.ts

@@ -56,11 +56,16 @@ export declare class InPlaceRebuild<TDocument extends {
     readonly collectionName: string;
     private readonly maxSweepableSources;
     private readonly resolved;
+    /** The column this collection carries its documents’ dataset IRI in: the
+     *  type’s declared dataset field, or the private `source` when it declares
+     *  none. Everything below – the stamp, every sweep filter, the source
+     *  enumeration – reads this one name. */
+    private readonly sourceField;
     constructor(client: Client, searchType: SearchType, options?: InPlaceRebuildOptions);
     openRun(context: RunContext): Promise<RunWriter<TDocument>>;
     /**
-     * The distinct sources present in the collection, via a single `source`
-     * facet. Requests one bucket beyond `maxSources` so genuine truncation is
+     * The distinct sources present in the collection, via a single facet over its
+     * {@link provenanceField}. Requests one bucket beyond `maxSources` so genuine truncation is
      * distinguishable from an exactly-full result: `maxSources` buckets are
      * returned intact, `maxSources + 1` proves more exist and the sweep would
      * miss some, so it throws rather than delete blind.

package/dist/in-place-rebuild.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"in-place-rebuild.d.ts","sourceRoot":"","sources":["../src/in-place-rebuild.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,WAAW,CAAC;AAExC,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAE9C,OAAO,KAAK,EAEV,UAAU,EACV,SAAS,EACT,MAAM,EACP,MAAM,eAAe,CAAC;AAYvB,OAAO,EAKL,KAAK,cAAc,EAEpB,MAAM,sBAAsB,CAAC;AAa9B;gCACgC;AAChC,MAAM,WAAW,qBAAsB,SAAQ,cAAc;IAC3D;sEACkE;IAClE,QAAQ,CAAC,mBAAmB,CAAC,EAAE,MAAM,CAAC;CACvC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AACH,qBAAa,cAAc,CACzB,SAAS,SAAS;IAAE,EAAE,EAAE,MAAM,CAAA;CAAE,CAChC,YAAW,MAAM,CAAC,SAAS,CAAC;IAY1B,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,UAAU;IAZ7B;;;;;OAKG;IACH,SAAgB,cAAc,EAAE,MAAM,CAAC;IACvC,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAS;IAC7C,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAyB;gBAG/B,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,UAAU,EACvC,OAAO,GAAE,qBAA0B;IAY/B,OAAO,CAAC,OAAO,EAAE,UAAU,GAAG,OAAO,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;IAsFjE;;;;;;OAMG;YACW,cAAc;CAsB7B"}
+{"version":3,"file":"in-place-rebuild.d.ts","sourceRoot":"","sources":["../src/in-place-rebuild.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,WAAW,CAAC;AAExC,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAE9C,OAAO,KAAK,EAEV,UAAU,EACV,SAAS,EACT,MAAM,EACP,MAAM,eAAe,CAAC;AAavB,OAAO,EAML,KAAK,cAAc,EAEpB,MAAM,sBAAsB,CAAC;AAa9B;gCACgC;AAChC,MAAM,WAAW,qBAAsB,SAAQ,cAAc;IAC3D;sEACkE;IAClE,QAAQ,CAAC,mBAAmB,CAAC,EAAE,MAAM,CAAC;CACvC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AACH,qBAAa,cAAc,CACzB,SAAS,SAAS;IAAE,EAAE,EAAE,MAAM,CAAA;CAAE,CAChC,YAAW,MAAM,CAAC,SAAS,CAAC;IAiB1B,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,UAAU;IAjB7B;;;;;OAKG;IACH,SAAgB,cAAc,EAAE,MAAM,CAAC;IACvC,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAS;IAC7C,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAyB;IAClD;;;6CAGyC;IACzC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;gBAGlB,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,UAAU,EACvC,OAAO,GAAE,qBAA0B;IAc/B,OAAO,CAAC,OAAO,EAAE,UAAU,GAAG,OAAO,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;IAiHjE;;;;;;OAMG;YACW,cAAc;CAsB7B"}

package/dist/in-place-rebuild.js

@@ -1,8 +1,8 @@
 import { buildCollectionDefinition } from './collection-definition.js';
 import { BatchImporter } from './import.js';
 import { ensureCollectionExists, openLockedRun, releaseLock } from './lock.js';
-import { LAST_SEEN_FIELD, SOURCE_FIELD, departedSources, membershipSweepFilters, staleDocumentsFilter, thisRunDocumentsFilter, } from './sweep.js';
-import { assertNoReservedFields, deleteByFilter, resolveRebuildOptions, stampDocuments, } from './rebuild-support.js';
+import { LAST_SEEN_FIELD, SOURCE_FIELD, departedSources, membershipSweepFilters, provenanceField, staleDocumentsFilter, thisRunDocumentsFilter, } from './sweep.js';
+import { assertNoReservedFields, assertSweepableProvenanceField, deleteByFilter, resolveRebuildOptions, stampDocuments, } from './rebuild-support.js';
 /**
  * Default ceiling on the distinct sources the membership sweep enumerates via
  * a single `source` facet. The sweep needs the complete source set to spot
@@ -58,10 +58,17 @@ export class InPlaceRebuild {
     collectionName;
     maxSweepableSources;
     resolved;
+    /** The column this collection carries its documents’ dataset IRI in: the
+     *  type’s declared dataset field, or the private `source` when it declares
+     *  none. Everything below – the stamp, every sweep filter, the source
+     *  enumeration – reads this one name. */
+    sourceField;
     constructor(client, searchType, options = {}) {
         this.client = client;
         this.searchType = searchType;
         assertNoReservedFields(searchType, [SOURCE_FIELD, LAST_SEEN_FIELD]);
+        assertSweepableProvenanceField(searchType, { requireFacetable: true });
+        this.sourceField = provenanceField(searchType);
         const { maxSweepableSources = DEFAULT_MAX_SWEEPABLE_SOURCES, ...rebuildOptions } = options;
         this.maxSweepableSources = maxSweepableSources;
         this.resolved = resolveRebuildOptions(searchType, rebuildOptions);
@@ -70,14 +77,20 @@ export class InPlaceRebuild {
     async openRun(context) {
         const { batchSize, lockTtlMs, definitionOptions } = this.resolved;
         const name = this.collectionName;
+        const sourceField = this.sourceField;
         return openLockedRun(this.client, name, lockTtlMs, async () => {
             // Create the collection on demand: SearchType schema + the bookkeeping
-            // fields, `source` faceted so the membership sweep can enumerate the
-            // distinct sources.
+            // fields. The private `source` – faceted so the membership sweep can
+            // enumerate the distinct sources – only when the type declares no dataset
+            // field of its own; when it does, that declared field already carries the
+            // IRI (and its own `facet: true`), and a second column would be the same
+            // value twice, free to drift.
             await ensureCollectionExists(this.client, name, () => {
                 const definition = buildCollectionDefinition(this.searchType, definitionOptions);
                 const bookkeeping = [
-                    { name: SOURCE_FIELD, type: 'string', facet: true },
+                    ...(sourceField === SOURCE_FIELD
+                        ? [{ name: SOURCE_FIELD, type: 'string', facet: true }]
+                        : []),
                     { name: LAST_SEEN_FIELD, type: 'string' },
                 ];
                 return {
@@ -87,8 +100,21 @@ export class InPlaceRebuild {
             });
             const importer = new BatchImporter(this.client, name, batchSize);
             return {
+                // Stamping the provenance field – rather than trusting the projection
+                // to have filled a declared one – is what keeps the sweep total: a
+                // caller projecting without a dataset context, or a document reaching
+                // the writer any other way, still lands in a swept collection. Where
+                // the projection did fill it, the value is the same dataset IRI, so the
+                // stamp only ever reasserts it.
+                //
+                // It reasserts the sweep’s column only, never the physical fanout a
+                // declared field also produces (the folded `${name}_search` companion
+                // of a `searchable` one). Folding is the projection’s convention, and
+                // restating it here would put it in two places – the split this change
+                // exists to close. So a document that reached the writer unprojected is
+                // sweepable but not full-text findable by dataset: absent, never wrong.
                 write: async (dataset, documents) => importer.add(stampDocuments(documents, {
-                    [SOURCE_FIELD]: dataset.iri.toString(),
+                    [sourceField]: dataset.iri.toString(),
                     [LAST_SEEN_FIELD]: context.runId,
                 })),
                 flush: async (dataset, outcome) => {
@@ -101,19 +127,19 @@ export class InPlaceRebuild {
                         // stale ones for the next successful run to reconcile.
                         return;
                     }
-                    await deleteByFilter(this.client, name, staleDocumentsFilter(dataset.iri.toString(), context.runId));
+                    await deleteByFilter(this.client, name, staleDocumentsFilter(sourceField, dataset.iri.toString(), context.runId));
                 },
                 reset: async (dataset) => {
                     // Discard only this run’s partial writes for the source (the failed
                     // endpoint attempt) so the dump re-run rebuilds it cleanly; the
                     // source’s prior-run documents stay for the success sweep.
                     await importer.flush();
-                    await deleteByFilter(this.client, name, thisRunDocumentsFilter(dataset.iri.toString(), context.runId));
+                    await deleteByFilter(this.client, name, thisRunDocumentsFilter(sourceField, dataset.iri.toString(), context.runId));
                 },
                 commit: async () => {
                     await importer.flush();
                     const departed = departedSources(await this.indexedSources(name, this.maxSweepableSources), context.selectedSources());
-                    for (const filter of membershipSweepFilters(departed)) {
+                    for (const filter of membershipSweepFilters(sourceField, departed)) {
                         await deleteByFilter(this.client, name, filter);
                     }
                     await releaseLock(this.client, name);
@@ -125,8 +151,8 @@ export class InPlaceRebuild {
         });
     }
     /**
-     * The distinct sources present in the collection, via a single `source`
-     * facet. Requests one bucket beyond `maxSources` so genuine truncation is
+     * The distinct sources present in the collection, via a single facet over its
+     * {@link provenanceField}. Requests one bucket beyond `maxSources` so genuine truncation is
      * distinguishable from an exactly-full result: `maxSources` buckets are
      * returned intact, `maxSources + 1` proves more exist and the sweep would
      * miss some, so it throws rather than delete blind.
@@ -137,9 +163,9 @@ export class InPlaceRebuild {
             .documents()
             .search({
             q: '*',
-            query_by: SOURCE_FIELD,
+            query_by: this.sourceField,
             per_page: 0,
-            facet_by: SOURCE_FIELD,
+            facet_by: this.sourceField,
             max_facet_values: maxSources + 1,
         });
         const counts = response.facet_counts?.[0]?.counts ?? [];

package/dist/rebuild-support.d.ts

@@ -43,6 +43,29 @@ export declare function resolveRebuildOptions(searchType: SearchType, options: R
  * `last_seen`.
  */
 export declare function assertNoReservedFields(searchType: SearchType, reserved: readonly string[]): void;
+/**
+ * Reject a declared dataset field the provenance bookkeeping cannot be kept on.
+ * A type declaring one makes it the collection’s provenance field – the column
+ * the membership sweep enumerates and deletes by – so the declaration has to answer
+ * *which single dataset is this document’s* the way the private `source` field
+ * did:
+ *
+ * - **not `array`**: Typesense reads `field:=[a,b]` over a `string[]` as
+ *   *contains any*, so a departed source would take every document that merely
+ *   mentions it – including entities another selected dataset still
+ *   contributes;
+ * - **no `transform`**: the sweep compares the stored value against the run’s
+ *   selection, which carries raw dataset IRIs, so a transformed value would
+ *   match nothing and the sweep would silently stop deleting;
+ * - **`facetable`** where the writer enumerates the indexed sources by faceting
+ *   it (the In-place writer does; Blue/green only ever filters by a known IRI).
+ *
+ * Thrown at writer construction, so a declaration that cannot be swept fails
+ * before a run touches the index rather than at the sweep, after the writes.
+ */
+export declare function assertSweepableProvenanceField(searchType: SearchType, options: {
+    readonly requireFacetable: boolean;
+}): void;
 /** Stamp each document with fixed bookkeeping fields as it streams past. */
 export declare function stampDocuments<TDocument>(documents: AsyncIterable<TDocument>, stamp: Readonly<Record<string, string>>): AsyncIterable<TDocument & Record<string, string>>;
 /** Delete a collection’s documents matching a Typesense filter. */

package/dist/rebuild-support.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"rebuild-support.d.ts","sourceRoot":"","sources":["../src/rebuild-support.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,WAAW,CAAC;AACxC,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAC9C,OAAO,EAEL,KAAK,2BAA2B,EACjC,MAAM,4BAA4B,CAAC;AAKpC;;;GAGG;AACH,MAAM,WAAW,cAAe,SAAQ,2BAA2B;IACjE,+DAA+D;IAC/D,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B,0EAA0E;IAC1E,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;CAC7B;AAED;0FAC0F;AAC1F,MAAM,WAAW,sBAAsB;IACrC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B;;6DAEyD;IACzD,QAAQ,CAAC,iBAAiB,EAAE,2BAA2B,GAAG;QACxD,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;KACvB,CAAC;CACH;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,qBAAqB,CACnC,UAAU,EAAE,UAAU,EACtB,OAAO,EAAE,cAAc,GACtB,sBAAsB,CAgBxB;AAED;;;;GAIG;AACH,wBAAgB,sBAAsB,CACpC,UAAU,EAAE,UAAU,EACtB,QAAQ,EAAE,SAAS,MAAM,EAAE,GAC1B,IAAI,CAWN;AAED,4EAA4E;AAC5E,wBAAuB,cAAc,CAAC,SAAS,EAC7C,SAAS,EAAE,aAAa,CAAC,SAAS,CAAC,EACnC,KAAK,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,GACtC,aAAa,CAAC,SAAS,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAInD;AAED,mEAAmE;AACnE,wBAAsB,cAAc,CAClC,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,IAAI,CAAC,CAKf"}
+{"version":3,"file":"rebuild-support.d.ts","sourceRoot":"","sources":["../src/rebuild-support.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,WAAW,CAAC;AACxC,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAE9C,OAAO,EAEL,KAAK,2BAA2B,EACjC,MAAM,4BAA4B,CAAC;AAKpC;;;GAGG;AACH,MAAM,WAAW,cAAe,SAAQ,2BAA2B;IACjE,+DAA+D;IAC/D,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B,0EAA0E;IAC1E,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;CAC7B;AAED;0FAC0F;AAC1F,MAAM,WAAW,sBAAsB;IACrC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B;;6DAEyD;IACzD,QAAQ,CAAC,iBAAiB,EAAE,2BAA2B,GAAG;QACxD,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;KACvB,CAAC;CACH;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,qBAAqB,CACnC,UAAU,EAAE,UAAU,EACtB,OAAO,EAAE,cAAc,GACtB,sBAAsB,CAgBxB;AAED;;;;GAIG;AACH,wBAAgB,sBAAsB,CACpC,UAAU,EAAE,UAAU,EACtB,QAAQ,EAAE,SAAS,MAAM,EAAE,GAC1B,IAAI,CAWN;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,wBAAgB,8BAA8B,CAC5C,UAAU,EAAE,UAAU,EACtB,OAAO,EAAE;IAAE,QAAQ,CAAC,gBAAgB,EAAE,OAAO,CAAA;CAAE,GAC9C,IAAI,CA0BN;AAED,4EAA4E;AAC5E,wBAAuB,cAAc,CAAC,SAAS,EAC7C,SAAS,EAAE,aAAa,CAAC,SAAS,CAAC,EACnC,KAAK,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,GACtC,aAAa,CAAC,SAAS,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAInD;AAED,mEAAmE;AACnE,wBAAsB,cAAc,CAClC,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,IAAI,CAAC,CAKf"}

package/dist/rebuild-support.js

@@ -1,3 +1,4 @@
+import { datasetField, isInternalField } from '@lde/search/adapter';
 import { buildCollectionDefinition, } from './collection-definition.js';
 import { deriveCollectionName } from './collection-name.js';
 import { DEFAULT_LOCK_TTL_MS } from './lock.js';
@@ -41,6 +42,45 @@ export function assertNoReservedFields(searchType, reserved) {
             .join(', ')}`);
     }
 }
+/**
+ * Reject a declared dataset field the provenance bookkeeping cannot be kept on.
+ * A type declaring one makes it the collection’s provenance field – the column
+ * the membership sweep enumerates and deletes by – so the declaration has to answer
+ * *which single dataset is this document’s* the way the private `source` field
+ * did:
+ *
+ * - **not `array`**: Typesense reads `field:=[a,b]` over a `string[]` as
+ *   *contains any*, so a departed source would take every document that merely
+ *   mentions it – including entities another selected dataset still
+ *   contributes;
+ * - **no `transform`**: the sweep compares the stored value against the run’s
+ *   selection, which carries raw dataset IRIs, so a transformed value would
+ *   match nothing and the sweep would silently stop deleting;
+ * - **`facetable`** where the writer enumerates the indexed sources by faceting
+ *   it (the In-place writer does; Blue/green only ever filters by a known IRI).
+ *
+ * Thrown at writer construction, so a declaration that cannot be swept fails
+ * before a run touches the index rather than at the sweep, after the writes.
+ */
+export function assertSweepableProvenanceField(searchType, options) {
+    const field = datasetField(searchType);
+    if (field === undefined || isInternalField(field)) {
+        return;
+    }
+    const problems = [];
+    if (field.array === true) {
+        problems.push('it is an array, and a membership sweep over one would delete every document merely mentioning a departed dataset');
+    }
+    if (field.transform !== undefined) {
+        problems.push('it declares a transform, and the sweep matches the stored value against the run’s raw dataset IRIs');
+    }
+    if (options.requireFacetable && field.facetable !== true) {
+        problems.push('it is not facetable, and the membership sweep enumerates the indexed datasets by faceting it');
+    }
+    if (problems.length > 0) {
+        throw new Error(`SearchType “${searchType.name}” declares “${field.name}” over the indexed dataset, which this writer keeps its provenance bookkeeping on, but ${problems.join('; and ')}.`);
+    }
+}
 /** Stamp each document with fixed bookkeeping fields as it streams past. */
 export async function* stampDocuments(documents, stamp) {
     for await (const document of documents) {

package/dist/sweep.d.ts

@@ -1,7 +1,23 @@
-/** The document field carrying the dataset IRI a document came from. */
+import type { SearchType } from '@lde/search';
+/** The private field carrying the dataset IRI a document came from, for a type
+ *  that declares none of its own – see {@link provenanceField}. */
 export declare const SOURCE_FIELD = "source";
 /** The document field carrying the id of the run that last wrote a document. */
 export declare const LAST_SEEN_FIELD = "last_seen";
+/**
+ * The field a collection carries its documents’ dataset IRI in – the one the
+ * membership sweep enumerates, filters and deletes by.
+ *
+ * A type that declares a field over the dataset
+ * ({@link datasetField `from: 'dataset'`}) *is* its provenance: the writer keeps
+ * its bookkeeping on that field rather than stamping a private one beside it,
+ * so the facet, the label resolution, any `derive` and the sweep all read one
+ * column that cannot drift. A type declaring nothing – or declaring the dataset
+ * only as an *internal* field, which the projection prunes before the writer
+ * ever sees it – falls back to the private {@link SOURCE_FIELD}, and behaves
+ * exactly as it always has.
+ */
+export declare function provenanceField(searchType: SearchType): string;
 /**
  * Sources whose documents must leave the index: indexed, but no longer part
  * of the run’s selection. Selection is membership, not processing – a dataset
@@ -15,18 +31,20 @@ export declare function departedSources(indexedSources: Iterable<string>, select
  * Typesense filter matching a source’s documents that this run did not touch:
  * everything the source no longer contains, ready for a per-source sweep.
  *
- * @param sourceIri The dataset IRI stamped on the documents as `source`
+ * @param sourceField The collection’s {@link provenanceField}
+ * @param sourceIri The dataset IRI the documents carry in that field
  * @param runId The current run; documents it wrote carry it as `last_seen`
  */
-export declare function staleDocumentsFilter(sourceIri: string, runId: string): string;
+export declare function staleDocumentsFilter(sourceField: string, sourceIri: string, runId: string): string;
 /**
  * Typesense filter matching all of a source’s documents, ready to drop a whole
  * source (a departed source’s membership sweep, or a Blue/green writer rolling
  * a failed dataset out of its not-yet-live collection).
  *
- * @param sourceIri The dataset IRI stamped on the documents as `source`
+ * @param sourceField The collection’s {@link provenanceField}
+ * @param sourceIri The dataset IRI the documents carry in that field
  */
-export declare function sourceDocumentsFilter(sourceIri: string): string;
+export declare function sourceDocumentsFilter(sourceField: string, sourceIri: string): string;
 /**
  * Typesense filter matching the documents **this run** wrote for a source: the
  * inverse of {@link staleDocumentsFilter}. An In-place writer deletes these to
@@ -34,17 +52,19 @@ export declare function sourceDocumentsFilter(sourceIri: string): string;
  * touching its prior-run documents, which the success sweep or a failed run
  * still owns.
  *
- * @param sourceIri The dataset IRI stamped on the documents as `source`
+ * @param sourceField The collection’s {@link provenanceField}
+ * @param sourceIri The dataset IRI the documents carry in that field
  * @param runId The current run, stamped on its documents as `last_seen`
  */
-export declare function thisRunDocumentsFilter(sourceIri: string, runId: string): string;
+export declare function thisRunDocumentsFilter(sourceField: string, sourceIri: string, runId: string): string;
 /**
  * Typesense filters deleting every departed source’s documents, combined into
  * as few filters as fit: deletes travel in the URL query string, so each
  * filter stays under a conservative length budget rather than listing every
  * source in one string.
  *
+ * @param sourceField The collection’s {@link provenanceField}
  * @param departed The departed source IRIs ({@link departedSources})
  */
-export declare function membershipSweepFilters(departed: readonly string[]): string[];
+export declare function membershipSweepFilters(sourceField: string, departed: readonly string[]): string[];
 //# sourceMappingURL=sweep.d.ts.map

package/dist/sweep.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"sweep.d.ts","sourceRoot":"","sources":["../src/sweep.ts"],"names":[],"mappings":"AAQA,wEAAwE;AACxE,eAAO,MAAM,YAAY,WAAW,CAAC;AAErC,gFAAgF;AAChF,eAAO,MAAM,eAAe,cAAc,CAAC;AAE3C;;;;;;;GAOG;AACH,wBAAgB,eAAe,CAC7B,cAAc,EAAE,QAAQ,CAAC,MAAM,CAAC,EAChC,eAAe,EAAE,QAAQ,CAAC,MAAM,CAAC,GAChC,MAAM,EAAE,CAGV;AAED;;;;;;GAMG;AACH,wBAAgB,oBAAoB,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,CAE7E;AAED;;;;;;GAMG;AACH,wBAAgB,qBAAqB,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAE/D;AAED;;;;;;;;;GASG;AACH,wBAAgB,sBAAsB,CACpC,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,MAAM,GACZ,MAAM,CAER;AAED;;;;;;;GAOG;AACH,wBAAgB,sBAAsB,CAAC,QAAQ,EAAE,SAAS,MAAM,EAAE,GAAG,MAAM,EAAE,CAqB5E"}
+{"version":3,"file":"sweep.d.ts","sourceRoot":"","sources":["../src/sweep.ts"],"names":[],"mappings":"AAQA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAG9C;mEACmE;AACnE,eAAO,MAAM,YAAY,WAAW,CAAC;AAErC,gFAAgF;AAChF,eAAO,MAAM,eAAe,cAAc,CAAC;AAE3C;;;;;;;;;;;;GAYG;AACH,wBAAgB,eAAe,CAAC,UAAU,EAAE,UAAU,GAAG,MAAM,CAK9D;AAED;;;;;;;GAOG;AACH,wBAAgB,eAAe,CAC7B,cAAc,EAAE,QAAQ,CAAC,MAAM,CAAC,EAChC,eAAe,EAAE,QAAQ,CAAC,MAAM,CAAC,GAChC,MAAM,EAAE,CAGV;AAED;;;;;;;GAOG;AACH,wBAAgB,oBAAoB,CAClC,WAAW,EAAE,MAAM,EACnB,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,MAAM,GACZ,MAAM,CAER;AAED;;;;;;;GAOG;AACH,wBAAgB,qBAAqB,CACnC,WAAW,EAAE,MAAM,EACnB,SAAS,EAAE,MAAM,GAChB,MAAM,CAER;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,sBAAsB,CACpC,WAAW,EAAE,MAAM,EACnB,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,MAAM,GACZ,MAAM,CAER;AAED;;;;;;;;GAQG;AACH,wBAAgB,sBAAsB,CACpC,WAAW,EAAE,MAAM,EACnB,QAAQ,EAAE,SAAS,MAAM,EAAE,GAC1B,MAAM,EAAE,CAqBV"}

package/dist/sweep.js

@@ -1,13 +1,35 @@
 // Pure deletion planning shared by the rebuild writers: which documents leave
 // a collection, expressed as source sets and Typesense filter strings. Kept
 // free of the Typesense client so the logic is unit-testable. This module owns
-// the bookkeeping field names, so stamping and every filter that reads them
-// can never disagree.
+// the private bookkeeping field names and resolves which field a collection
+// carries its provenance in, so stamping and every filter that reads it can
+// never disagree.
+import { datasetField, isInternalField } from '@lde/search/adapter';
 import { escapeFilterValue } from './query-compiler.js';
-/** The document field carrying the dataset IRI a document came from. */
+/** The private field carrying the dataset IRI a document came from, for a type
+ *  that declares none of its own – see {@link provenanceField}. */
 export const SOURCE_FIELD = 'source';
 /** The document field carrying the id of the run that last wrote a document. */
 export const LAST_SEEN_FIELD = 'last_seen';
+/**
+ * The field a collection carries its documents’ dataset IRI in – the one the
+ * membership sweep enumerates, filters and deletes by.
+ *
+ * A type that declares a field over the dataset
+ * ({@link datasetField `from: 'dataset'`}) *is* its provenance: the writer keeps
+ * its bookkeeping on that field rather than stamping a private one beside it,
+ * so the facet, the label resolution, any `derive` and the sweep all read one
+ * column that cannot drift. A type declaring nothing – or declaring the dataset
+ * only as an *internal* field, which the projection prunes before the writer
+ * ever sees it – falls back to the private {@link SOURCE_FIELD}, and behaves
+ * exactly as it always has.
+ */
+export function provenanceField(searchType) {
+    const declared = datasetField(searchType);
+    return declared === undefined || isInternalField(declared)
+        ? SOURCE_FIELD
+        : declared.name;
+}
 /**
  * Sources whose documents must leave the index: indexed, but no longer part
  * of the run’s selection. Selection is membership, not processing – a dataset
@@ -24,21 +46,23 @@ export function departedSources(indexedSources, selectedSources) {
  * Typesense filter matching a source’s documents that this run did not touch:
  * everything the source no longer contains, ready for a per-source sweep.
  *
- * @param sourceIri The dataset IRI stamped on the documents as `source`
+ * @param sourceField The collection’s {@link provenanceField}
+ * @param sourceIri The dataset IRI the documents carry in that field
  * @param runId The current run; documents it wrote carry it as `last_seen`
  */
-export function staleDocumentsFilter(sourceIri, runId) {
-    return `${sourceDocumentsFilter(sourceIri)} && ${LAST_SEEN_FIELD}:!=${escapeFilterValue(runId)}`;
+export function staleDocumentsFilter(sourceField, sourceIri, runId) {
+    return `${sourceDocumentsFilter(sourceField, sourceIri)} && ${LAST_SEEN_FIELD}:!=${escapeFilterValue(runId)}`;
 }
 /**
  * Typesense filter matching all of a source’s documents, ready to drop a whole
  * source (a departed source’s membership sweep, or a Blue/green writer rolling
  * a failed dataset out of its not-yet-live collection).
  *
- * @param sourceIri The dataset IRI stamped on the documents as `source`
+ * @param sourceField The collection’s {@link provenanceField}
+ * @param sourceIri The dataset IRI the documents carry in that field
  */
-export function sourceDocumentsFilter(sourceIri) {
-    return `${SOURCE_FIELD}:=${escapeFilterValue(sourceIri)}`;
+export function sourceDocumentsFilter(sourceField, sourceIri) {
+    return `${sourceField}:=${escapeFilterValue(sourceIri)}`;
 }
 /**
  * Typesense filter matching the documents **this run** wrote for a source: the
@@ -47,11 +71,12 @@ export function sourceDocumentsFilter(sourceIri) {
  * touching its prior-run documents, which the success sweep or a failed run
  * still owns.
  *
- * @param sourceIri The dataset IRI stamped on the documents as `source`
+ * @param sourceField The collection’s {@link provenanceField}
+ * @param sourceIri The dataset IRI the documents carry in that field
  * @param runId The current run, stamped on its documents as `last_seen`
  */
-export function thisRunDocumentsFilter(sourceIri, runId) {
-    return `${sourceDocumentsFilter(sourceIri)} && ${LAST_SEEN_FIELD}:=${escapeFilterValue(runId)}`;
+export function thisRunDocumentsFilter(sourceField, sourceIri, runId) {
+    return `${sourceDocumentsFilter(sourceField, sourceIri)} && ${LAST_SEEN_FIELD}:=${escapeFilterValue(runId)}`;
 }
 /**
  * Typesense filters deleting every departed source’s documents, combined into
@@ -59,15 +84,16 @@ export function thisRunDocumentsFilter(sourceIri, runId) {
  * filter stays under a conservative length budget rather than listing every
  * source in one string.
  *
+ * @param sourceField The collection’s {@link provenanceField}
  * @param departed The departed source IRIs ({@link departedSources})
  */
-export function membershipSweepFilters(departed) {
+export function membershipSweepFilters(sourceField, departed) {
     const filters = [];
     let chunk = [];
     let chunkLength = 0;
     const flush = () => {
         if (chunk.length > 0) {
-            filters.push(`${SOURCE_FIELD}:=[${chunk.join(',')}]`);
+            filters.push(`${sourceField}:=[${chunk.join(',')}]`);
             chunk = [];
             chunkLength = 0;
         }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lde/search-typesense",
-  "version": "0.18.1",
+  "version": "0.19.0",
   "description": "Typesense implementation of the @lde/search SearchEngine port: collection-schema builder, query compiler, label-resolving result reconstruction, and blue/green index lifecycle. Engine-specific (Typesense) but domain-agnostic.",
   "repository": {
     "url": "git+https://github.com/ldelements/lde.git",
@@ -25,7 +25,7 @@
     "!**/*.tsbuildinfo"
   ],
   "dependencies": {
-    "@lde/search": "^0.15.0",
+    "@lde/search": "^0.16.0",
     "@lde/text-normalization": "^0.1.2",
     "tslib": "^2.3.0",
     "typesense": "^3.0.6"
@@ -35,6 +35,6 @@
   },
   "peerDependencies": {
     "@lde/dataset": "^0.7.9",
-    "@lde/pipeline": "^0.35.5"
+    "@lde/pipeline": "^0.35.6"
   }
 }