@lde/search-typesense 0.28.0 → 0.29.0

package/dist/in-place-rebuild.js

@@ -1,36 +1,25 @@
 import { facetableFields, physicalFields } from '@lde/search/adapter';
 import { buildCollectionDefinition } from './collection-definition.js';
-import { BatchImporter } from './import.js';
 import { ensureCollectionExists, openLockedRun, releaseLock } from './lock.js';
-import { LAST_SEEN_FIELD, SOURCE_FIELD, departedSources, membershipSweepFilters, provenanceField, staleDocumentsFilter, thisRunDocumentsFilter, } from './sweep.js';
-import { assertNoReservedFields, assertSweepableProvenanceField, deleteByFilter, resolveRebuildOptions, stampDocuments, } from './rebuild-support.js';
-/**
- * Default ceiling on the distinct sources the membership sweep enumerates via
- * a single `source` facet. The sweep needs the complete source set to spot
- * departed ones, so a truncated facet would silently miss deletions;
- * {@link InPlaceRebuild} throws instead. Raise it (up to the engine’s
- * `max_facet_values` limit) with {@link InPlaceRebuildOptions.maxSweepableSources}
- * before an index approaches it, so the ceiling is a tunable guard rather than
- * a hard wall.
- */
-const DEFAULT_MAX_SWEEPABLE_SOURCES = 10_000;
+import { assertBookkeepingPresent, assertWritableType, openDocuments, withBookkeeping, } from './documents.js';
+import { resolveRebuildOptions, } from './rebuild-support.js';
 /**
  * In-place Rebuild (update the live index directly – upsert changed sources,
  * sweep the rest – rather than swap in a fresh one) as a transactional
  * `Writer`: documents are upserted into one long-lived collection with
- * per-source atomicity – no swap, no staging. Every document is stamped with
- * its `source` (the dataset IRI) and `last_seen` (the run id); deletion is a
- * sweep, never special-cased:
+ * per-dataset atomicity – no swap, no staging. Deletion is a sweep, never
+ * special-cased, and {@link openDocuments} decides what a deletion physically
+ * is – for a canonically keyed type, dropping one dataset’s membership of a
+ * document several datasets share, and the document itself only with the last
+ * of them:
  *
- * - a **successful dataset flush** deletes the source’s documents the run did
- *   not rewrite (`source = dataset && last_seen != runId`). A failed dataset
- *   is not swept – its output is incomplete, and the next successful run
- *   reconciles;
- * - **reset** (the pipeline’s dump-fallback discard) deletes only *this run’s*
- *   writes for the dataset (`source = dataset && last_seen = runId`), so the
- *   dump re-run rebuilds it cleanly while the source’s prior-run documents are
- *   left for the success sweep to reconcile;
- * - **commit** deletes every document whose source left the run’s selection
+ * - a **successful dataset flush** drops what the dataset had and this run did
+ *   not rewrite. A failed dataset is not swept – its output is incomplete, and
+ *   the next successful run reconciles;
+ * - **reset** (the pipeline’s dump-fallback discard) drops only *this run’s*
+ *   writes for the dataset, so the dump re-run rebuilds it cleanly while the
+ *   dataset’s prior-run documents are left for the success sweep to reconcile;
+ * - **commit** drops what belongs to datasets that left the run’s selection
  *   (registry-membership sweep over {@link RunContext.selectedSources}, which
  *   includes datasets skipped as unchanged) and releases the lock;
  * - **abort** only releases the lock: upserts are idempotent, so whatever
@@ -38,15 +27,17 @@ const DEFAULT_MAX_SWEEPABLE_SOURCES = 10_000;
  *
  * `openRun` takes the single-flight cross-pod lock (throwing
  * `RebuildAlreadyRunning` when another rebuild holds it) and creates
- * the collection on demand from the {@link SearchType} plus the two
- * bookkeeping fields. A collection that already exists is used as it is, with
- * one exception: it must carry every reference field the declaration asks for
- * and every facet companion its facets read, or the run fails with the
- * drop-and-rebuild that fixes it
- * ({@link assertReferencesAndFacetCompanionsPresent}).
+ * the collection on demand from the {@link SearchType} plus the bookkeeping
+ * fields. A collection that already exists is used as it is, with two
+ * exceptions: it must carry every reference field the declaration asks for and
+ * every facet companion its facets read
+ * ({@link assertReferencesAndFacetCompanionsPresent}), and the membership field
+ * a keyed type records in ({@link assertBookkeepingPresent}) – each failing
+ * with the rebuild that fixes it.
  *
- * Document ids must be unique per (source, entity) – the caller keys them –
- * or documents from different sources overwrite each other.
+ * Unless the type is canonically keyed, document ids must be unique per
+ * (dataset, entity) – the caller keys them – or documents from different
+ * datasets overwrite each other.
  *
  * The name is the NDE Stack’s pattern
  * ({@link https://docs.nde.nl/stack/patterns | Stack patterns}).
@@ -63,21 +54,16 @@ export class InPlaceRebuild {
     collectionName;
     maxSweepableSources;
     resolved;
-    /** The column this collection carries its documents’ dataset IRI in: the
-     *  type’s declared dataset field, or the private `source` when it declares
-     *  none. Everything below – the stamp, every sweep filter, the source
-     *  enumeration – reads this one name. */
-    sourceField;
     constructor(client, searchType, options = {}) {
         this.client = client;
         this.searchType = searchType;
-        assertNoReservedFields(searchType, [SOURCE_FIELD, LAST_SEEN_FIELD]);
-        assertSweepableProvenanceField(searchType, {
+        // The membership sweep enumerates the indexed datasets by faceting the
+        // column they are recorded in, so a declared one has to be facetable.
+        assertWritableType(searchType, {
             requireFacetable: true,
             schema: options.schema,
         });
-        this.sourceField = provenanceField(searchType);
-        const { maxSweepableSources = DEFAULT_MAX_SWEEPABLE_SOURCES, ...rebuildOptions } = options;
+        const { maxSweepableSources, ...rebuildOptions } = options;
         this.maxSweepableSources = maxSweepableSources;
         this.resolved = resolveRebuildOptions(searchType, rebuildOptions);
         this.collectionName =
@@ -86,78 +72,47 @@ export class InPlaceRebuild {
     async openRun(context) {
         const { batchSize, lockTtlMs, definitionOptions } = this.resolved;
         const name = this.collectionName;
-        const sourceField = this.sourceField;
         return openLockedRun(this.client, name, lockTtlMs, async () => {
-            // Create the collection on demand: SearchType schema + the bookkeeping
-            // fields. The private `source` – faceted so the membership sweep can
-            // enumerate the distinct sources – only when the type declares no dataset
-            // field of its own; when it does, that declared field already carries the
-            // IRI (and its own `facet: true`), and a second column would be the same
-            // value twice, free to drift.
+            // Create the collection on demand: the SearchType’s schema plus whatever
+            // bookkeeping its membership regime keeps.
             const definition = buildCollectionDefinition(this.searchType, definitionOptions);
             const declaredFields = definition.fields ?? [];
-            const created = await ensureCollectionExists(this.client, name, () => {
-                const bookkeeping = [
-                    ...(sourceField === SOURCE_FIELD
-                        ? [{ name: SOURCE_FIELD, type: 'string', facet: true }]
-                        : []),
-                    { name: LAST_SEEN_FIELD, type: 'string' },
-                ];
-                return {
-                    ...definition,
-                    fields: [...declaredFields, ...bookkeeping],
-                };
-            });
+            const created = await ensureCollectionExists(this.client, name, () => withBookkeeping(definition, this.searchType));
             // A collection this run did NOT create may predate a `joinable` being
-            // declared, or a facet policy on a type this one references – and
-            // neither a reference nor a facet companion can be added to it here.
+            // declared, a facet policy on a type this one references, or the type
+            // being keyed – and none of the three can be added to it here.
             if (!created) {
-                await assertReferencesAndFacetCompanionsPresent(this.client, name, declaredFields, facetCompanionsOf(this.searchType, definitionOptions.schema));
+                const existing = (await this.client.collections(name).retrieve())
+                    .fields;
+                assertReferencesAndFacetCompanionsPresent(name, declaredFields, facetCompanionsOf(this.searchType, definitionOptions.schema), existing);
+                assertBookkeepingPresent(this.searchType, existing, name);
             }
-            const importer = new BatchImporter(this.client, name, batchSize);
+            const documents = openDocuments(this.client, name, {
+                searchType: this.searchType,
+                runId: context.runId,
+                startedAt: context.startedAt,
+                batchSize,
+                maxSweepableSources: this.maxSweepableSources,
+            });
             return {
-                // Stamping the provenance field – rather than trusting the projection
-                // to have filled a declared one – is what keeps the sweep total: a
-                // caller projecting without a dataset context, or a document reaching
-                // the writer any other way, still lands in a swept collection. Where
-                // the projection did fill it, the value is the same dataset IRI, so the
-                // stamp only ever reasserts it.
-                //
-                // It reasserts the sweep’s column only, never the physical fanout a
-                // declared field also produces (the folded `${name}_search` companion
-                // of a `searchable` one). Folding is the projection’s convention, and
-                // restating it here would put it in two places – the split this change
-                // exists to close. So a document that reached the writer unprojected is
-                // sweepable but not full-text findable by dataset: absent, never wrong.
-                write: async (dataset, documents) => importer.add(stampDocuments(documents, {
-                    [sourceField]: dataset.iri.toString(),
-                    [LAST_SEEN_FIELD]: context.runId,
-                })),
+                write: async (dataset, items) => documents.add(dataset, items),
                 flush: async (dataset, outcome) => {
-                    // Land the buffered documents first, so the sweep below never
-                    // deletes what this run just rewrote.
-                    await importer.flush();
                     if (outcome !== 'success') {
                         // A failed dataset’s output is incomplete: sweeping against it
                         // would delete documents the run never got to rewrite. Leave the
-                        // stale ones for the next successful run to reconcile.
+                        // stale ones for the next successful run to reconcile. The buffered
+                        // documents stay buffered; the next flush lands them.
                         return;
                     }
-                    await deleteByFilter(this.client, name, staleDocumentsFilter(sourceField, dataset.iri.toString(), context.runId));
-                },
-                reset: async (dataset) => {
-                    // Discard only this run’s partial writes for the source (the failed
-                    // endpoint attempt) so the dump re-run rebuilds it cleanly; the
-                    // source’s prior-run documents stay for the success sweep.
-                    await importer.flush();
-                    await deleteByFilter(this.client, name, thisRunDocumentsFilter(sourceField, dataset.iri.toString(), context.runId));
+                    await documents.dropUnwritten(dataset);
                 },
+                // Discard only this run’s partial writes for the dataset (the failed
+                // endpoint attempt) so the dump re-run rebuilds it cleanly; the
+                // dataset’s prior-run documents stay for the success sweep.
+                reset: async (dataset) => documents.dropWrittenThisRun(dataset),
                 commit: async () => {
-                    await importer.flush();
-                    const departed = departedSources(await this.indexedSources(name, this.maxSweepableSources), context.selectedSources());
-                    for (const filter of membershipSweepFilters(sourceField, departed)) {
-                        await deleteByFilter(this.client, name, filter);
-                    }
+                    await documents.flush();
+                    await documents.dropUnselected(context.selectedSources());
                     await releaseLock(this.client, name);
                 },
                 abort: async () => {
@@ -166,30 +121,6 @@ export class InPlaceRebuild {
             };
         });
     }
-    /**
-     * The distinct sources present in the collection, via a single facet over its
-     * {@link provenanceField}. Requests one bucket beyond `maxSources` so genuine truncation is
-     * distinguishable from an exactly-full result: `maxSources` buckets are
-     * returned intact, `maxSources + 1` proves more exist and the sweep would
-     * miss some, so it throws rather than delete blind.
-     */
-    async indexedSources(name, maxSources) {
-        const response = await this.client
-            .collections(name)
-            .documents()
-            .search({
-            q: '*',
-            query_by: this.sourceField,
-            per_page: 0,
-            facet_by: this.sourceField,
-            max_facet_values: maxSources + 1,
-        });
-        const counts = response.facet_counts?.[0]?.counts ?? [];
-        if (counts.length > maxSources) {
-            throw new Error(`Membership sweep cannot see beyond ${maxSources} distinct sources in “${name}”; raise maxSweepableSources or departed sources might be missed`);
-        }
-        return counts.map((count) => count.value);
-    }
 }
 /**
  * Fail loudly when an EXISTING collection does not carry every reference field
@@ -218,15 +149,12 @@ export class InPlaceRebuild {
  * self-correcting (a new plain field simply starts being written) or
  * harmless, and general drift detection is a feature of its own.
  */
-async function assertReferencesAndFacetCompanionsPresent(client, name, fields, facetCompanions) {
+function assertReferencesAndFacetCompanionsPresent(name, fields, facetCompanions, existingFields) {
     const references = fields.filter((field) => field.reference !== undefined);
     if (references.length === 0 && facetCompanions.length === 0) {
         return;
     }
-    const existing = new Map((await client.collections(name).retrieve()).fields.map((field) => [
-        field.name,
-        field,
-    ]));
+    const existing = new Map(existingFields.map((field) => [field.name, field]));
     const missingReferences = references.filter((field) => existing.get(field.name)?.reference !== field.reference);
     if (missingReferences.length > 0) {
         throw new Error(`Typesense collection “${name}” exists without the reference field(s) ${missingReferences

package/dist/index.d.ts

@@ -3,7 +3,6 @@ export type { BlueGreenRebuildOptions } from './blue-green-rebuild.js';
 export { InPlaceRebuild } from './in-place-rebuild.js';
 export type { InPlaceRebuildOptions } from './in-place-rebuild.js';
 export { RebuildAlreadyRunning } from './lock.js';
-export { departedSources, membershipSweepFilters, sourceDocumentsFilter, staleDocumentsFilter, } from './sweep.js';
 export { buildCollectionDefinition } from './collection-definition.js';
 export type { CollectionDefinitionOptions } from './collection-definition.js';
 export { deriveCollectionName } from './collection-name.js';

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAC3D,YAAY,EAAE,uBAAuB,EAAE,MAAM,yBAAyB,CAAC;AACvE,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvD,YAAY,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AACnE,OAAO,EAAE,qBAAqB,EAAE,MAAM,WAAW,CAAC;AAClD,OAAO,EACL,eAAe,EACf,sBAAsB,EACtB,qBAAqB,EACrB,oBAAoB,GACrB,MAAM,YAAY,CAAC;AACpB,OAAO,EAAE,yBAAyB,EAAE,MAAM,4BAA4B,CAAC;AACvE,YAAY,EAAE,2BAA2B,EAAE,MAAM,4BAA4B,CAAC;AAC9E,OAAO,EAAE,oBAAoB,EAAE,MAAM,sBAAsB,CAAC;AAC5D,OAAO,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AACxD,YAAY,EAAE,wBAAwB,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAChF,OAAO,EAAE,2BAA2B,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AAC/E,YAAY,EACV,qBAAqB,EACrB,4BAA4B,EAC5B,uBAAuB,GACxB,MAAM,aAAa,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAC3D,YAAY,EAAE,uBAAuB,EAAE,MAAM,yBAAyB,CAAC;AACvE,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvD,YAAY,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AACnE,OAAO,EAAE,qBAAqB,EAAE,MAAM,WAAW,CAAC;AAClD,OAAO,EAAE,yBAAyB,EAAE,MAAM,4BAA4B,CAAC;AACvE,YAAY,EAAE,2BAA2B,EAAE,MAAM,4BAA4B,CAAC;AAC9E,OAAO,EAAE,oBAAoB,EAAE,MAAM,sBAAsB,CAAC;AAC5D,OAAO,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AACxD,YAAY,EAAE,wBAAwB,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAChF,OAAO,EAAE,2BAA2B,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AAC/E,YAAY,EACV,qBAAqB,EACrB,4BAA4B,EAC5B,uBAAuB,GACxB,MAAM,aAAa,CAAC"}

package/dist/index.js

@@ -1,7 +1,6 @@
 export { BlueGreenRebuild } from './blue-green-rebuild.js';
 export { InPlaceRebuild } from './in-place-rebuild.js';
 export { RebuildAlreadyRunning } from './lock.js';
-export { departedSources, membershipSweepFilters, sourceDocumentsFilter, staleDocumentsFilter, } from './sweep.js';
 export { buildCollectionDefinition } from './collection-definition.js';
 export { deriveCollectionName } from './collection-name.js';
 export { buildSearchParams } from './query-compiler.js';

package/dist/rebuild-support.d.ts

@@ -1,5 +1,4 @@
-import type { Client } from 'typesense';
-import type { SearchSchema, SearchType } from '@lde/search';
+import type { SearchType } from '@lde/search';
 import { type CollectionDefinitionOptions } from './collection-definition.js';
 /**
  * The tuning knobs both rebuild writers share, on top of the collection-definition
@@ -39,43 +38,4 @@ export interface ResolvedRebuildOptions {
  * been paid for.
  */
 export declare function resolveRebuildOptions(searchType: SearchType, options: RebuildOptions): ResolvedRebuildOptions;
-/**
- * Reject a {@link SearchType} that declares any of the bookkeeping field names
- * a writer stamps, so a domain field can never collide with `source` /
- * `last_seen`.
- */
-export declare function assertNoReservedFields(searchType: SearchType, reserved: readonly string[]): void;
-/**
- * Reject a declared dataset field the provenance bookkeeping cannot be kept on.
- * A type declaring one makes it the collection’s provenance field – the column
- * the membership sweep enumerates and deletes by – so the declaration has to answer
- * *which single dataset is this document’s* the way the private `source` field
- * did:
- *
- * - **not `array`**: Typesense reads `field:=[a,b]` over a `string[]` as
- *   *contains any*, so a departed source would take every document that merely
- *   mentions it – including entities another selected dataset still
- *   contributes;
- * - **no `transform`**: the sweep compares the stored value against the run’s
- *   selection, which carries raw dataset IRIs, so a transformed value would
- *   match nothing and the sweep would silently stop deleting;
- * - **`facetable`** where the writer enumerates the indexed sources by faceting
- *   it (the In-place writer does; Blue/green only ever filters by a known IRI)
- *   – and faceted **on its own name**: a reference inheriting a facet policy
- *   from the type it names facets a companion holding only the admitted keys
- *   (`physicalFields(field, schema).facet`), so enumerating the sources
- *   through it would miss every dataset the policy excludes. `schema` is what
- *   resolves that; without one no policy is visible, as nowhere else.
- *
- * Thrown at writer construction, so a declaration that cannot be swept fails
- * before a run touches the index rather than at the sweep, after the writes.
- */
-export declare function assertSweepableProvenanceField(searchType: SearchType, options: {
-    readonly requireFacetable: boolean;
-    readonly schema?: SearchSchema;
-}): void;
-/** Stamp each document with fixed bookkeeping fields as it streams past. */
-export declare function stampDocuments<TDocument>(documents: AsyncIterable<TDocument>, stamp: Readonly<Record<string, string>>): AsyncIterable<TDocument & Record<string, string>>;
-/** Delete a collection’s documents matching a Typesense filter. */
-export declare function deleteByFilter(client: Client, collection: string, filterBy: string): Promise<void>;
 //# sourceMappingURL=rebuild-support.d.ts.map

package/dist/rebuild-support.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"rebuild-support.d.ts","sourceRoot":"","sources":["../src/rebuild-support.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,WAAW,CAAC;AACxC,OAAO,KAAK,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAQ5D,OAAO,EAEL,KAAK,2BAA2B,EACjC,MAAM,4BAA4B,CAAC;AAKpC;;;GAGG;AACH,MAAM,WAAW,cAAe,SAAQ,2BAA2B;IACjE,+DAA+D;IAC/D,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B,0EAA0E;IAC1E,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;CAC7B;AAED;0FAC0F;AAC1F,MAAM,WAAW,sBAAsB;IACrC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B;;;uEAGmE;IACnE,QAAQ,CAAC,iBAAiB,EAAE,2BAA2B,GAAG;QACxD,QAAQ,CAAC,iBAAiB,EAAE,CAAC,UAAU,EAAE,UAAU,KAAK,MAAM,CAAC;KAChE,CAAC;CACH;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAgB,qBAAqB,CACnC,UAAU,EAAE,UAAU,EACtB,OAAO,EAAE,cAAc,GACtB,sBAAsB,CAkBxB;AAsCD;;;;GAIG;AACH,wBAAgB,sBAAsB,CACpC,UAAU,EAAE,UAAU,EACtB,QAAQ,EAAE,SAAS,MAAM,EAAE,GAC1B,IAAI,CAWN;AAED;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,wBAAgB,8BAA8B,CAC5C,UAAU,EAAE,UAAU,EACtB,OAAO,EAAE;IACP,QAAQ,CAAC,gBAAgB,EAAE,OAAO,CAAC;IACnC,QAAQ,CAAC,MAAM,CAAC,EAAE,YAAY,CAAC;CAChC,GACA,IAAI,CAiCN;AAED,4EAA4E;AAC5E,wBAAuB,cAAc,CAAC,SAAS,EAC7C,SAAS,EAAE,aAAa,CAAC,SAAS,CAAC,EACnC,KAAK,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,GACtC,aAAa,CAAC,SAAS,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAInD;AAED,mEAAmE;AACnE,wBAAsB,cAAc,CAClC,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,IAAI,CAAC,CAKf"}
+{"version":3,"file":"rebuild-support.d.ts","sourceRoot":"","sources":["../src/rebuild-support.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAE9C,OAAO,EAEL,KAAK,2BAA2B,EACjC,MAAM,4BAA4B,CAAC;AAOpC;;;GAGG;AACH,MAAM,WAAW,cAAe,SAAQ,2BAA2B;IACjE,+DAA+D;IAC/D,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B,0EAA0E;IAC1E,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;CAC7B;AAED;0FAC0F;AAC1F,MAAM,WAAW,sBAAsB;IACrC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B;;;uEAGmE;IACnE,QAAQ,CAAC,iBAAiB,EAAE,2BAA2B,GAAG;QACxD,QAAQ,CAAC,iBAAiB,EAAE,CAAC,UAAU,EAAE,UAAU,KAAK,MAAM,CAAC;KAChE,CAAC;CACH;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAgB,qBAAqB,CACnC,UAAU,EAAE,UAAU,EACtB,OAAO,EAAE,cAAc,GACtB,sBAAsB,CAkBxB"}

package/dist/rebuild-support.js

@@ -1,8 +1,9 @@
-import { datasetField, isInternalField, joinGraph, physicalFields, referenceFields, } from '@lde/search/adapter';
+import { joinGraph, referenceFields } from '@lde/search/adapter';
 import { buildCollectionDefinition, } from './collection-definition.js';
 import { deriveCollectionName } from './collection-name.js';
 import { DEFAULT_LOCK_TTL_MS } from './lock.js';
-import { DEFAULT_BATCH_SIZE } from './import.js';
+/** Documents imported per Typesense request unless a writer is told otherwise. */
+const DEFAULT_BATCH_SIZE = 1000;
 /**
  * Apply the shared defaults, once, so neither writer restates them – including
  * the collection naming, derived from each type ({@link deriveCollectionName})
@@ -59,77 +60,3 @@ function assertDistinctJoinTargetNames(searchType, options) {
         }
     }
 }
-/**
- * Reject a {@link SearchType} that declares any of the bookkeeping field names
- * a writer stamps, so a domain field can never collide with `source` /
- * `last_seen`.
- */
-export function assertNoReservedFields(searchType, reserved) {
-    const clashing = searchType.fields.filter((field) => reserved.includes(field.name));
-    if (clashing.length > 0) {
-        throw new Error(`SearchType “${searchType.name}” declares reserved bookkeeping field(s) ${clashing
-            .map((field) => `“${field.name}”`)
-            .join(', ')}`);
-    }
-}
-/**
- * Reject a declared dataset field the provenance bookkeeping cannot be kept on.
- * A type declaring one makes it the collection’s provenance field – the column
- * the membership sweep enumerates and deletes by – so the declaration has to answer
- * *which single dataset is this document’s* the way the private `source` field
- * did:
- *
- * - **not `array`**: Typesense reads `field:=[a,b]` over a `string[]` as
- *   *contains any*, so a departed source would take every document that merely
- *   mentions it – including entities another selected dataset still
- *   contributes;
- * - **no `transform`**: the sweep compares the stored value against the run’s
- *   selection, which carries raw dataset IRIs, so a transformed value would
- *   match nothing and the sweep would silently stop deleting;
- * - **`facetable`** where the writer enumerates the indexed sources by faceting
- *   it (the In-place writer does; Blue/green only ever filters by a known IRI)
- *   – and faceted **on its own name**: a reference inheriting a facet policy
- *   from the type it names facets a companion holding only the admitted keys
- *   (`physicalFields(field, schema).facet`), so enumerating the sources
- *   through it would miss every dataset the policy excludes. `schema` is what
- *   resolves that; without one no policy is visible, as nowhere else.
- *
- * Thrown at writer construction, so a declaration that cannot be swept fails
- * before a run touches the index rather than at the sweep, after the writes.
- */
-export function assertSweepableProvenanceField(searchType, options) {
-    const field = datasetField(searchType);
-    if (field === undefined || isInternalField(field)) {
-        return;
-    }
-    const problems = [];
-    if (field.array === true) {
-        problems.push('it is an array, and a membership sweep over one would delete every document merely mentioning a departed dataset');
-    }
-    if (field.transform !== undefined) {
-        problems.push('it declares a transform, and the sweep matches the stored value against the run’s raw dataset IRIs');
-    }
-    if (options.requireFacetable && field.facetable !== true) {
-        problems.push('it is not facetable, and the membership sweep enumerates the indexed datasets by faceting it');
-    }
-    else if (options.requireFacetable &&
-        physicalFields(field, options.schema).facet !== field.name) {
-        problems.push('it inherits a facet policy from the type it names, so the engine facets only the datasets the policy admits, and the membership sweep enumerates the indexed datasets by faceting it');
-    }
-    if (problems.length > 0) {
-        throw new Error(`SearchType “${searchType.name}” declares “${field.name}” over the indexed dataset, which this writer keeps its provenance bookkeeping on, but ${problems.join('; and ')}.`);
-    }
-}
-/** Stamp each document with fixed bookkeeping fields as it streams past. */
-export async function* stampDocuments(documents, stamp) {
-    for await (const document of documents) {
-        yield { ...document, ...stamp };
-    }
-}
-/** Delete a collection’s documents matching a Typesense filter. */
-export async function deleteByFilter(client, collection, filterBy) {
-    await client
-        .collections(collection)
-        .documents()
-        .delete({ filter_by: filterBy });
-}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lde/search-typesense",
-  "version": "0.28.0",
+  "version": "0.29.0",
   "description": "Typesense implementation of the @lde/search SearchEngine port: collection-schema builder, query compiler, label-resolving result reconstruction, and blue/green index lifecycle. Engine-specific (Typesense) but domain-agnostic.",
   "repository": {
     "url": "git+https://github.com/ldelements/lde.git",

package/dist/import.d.ts

@@ -1,31 +0,0 @@
-import type { Client } from 'typesense';
-/** Documents imported per Typesense request unless a writer is told otherwise. */
-export declare const DEFAULT_BATCH_SIZE = 1000;
-/**
- * Accumulates documents across writes and upserts them into a collection in
- * fixed-size batches, so many small writes (one dataset at a time) still land
- * as few Typesense requests. Call {@link flush} to import the remainder.
- */
-export declare class BatchImporter<TDocument extends {
-    id: string;
-}> {
-    private readonly client;
-    private readonly collection;
-    private readonly batchSize;
-    private buffer;
-    constructor(client: Client, collection: string, batchSize: number);
-    /** Add a stream of documents, importing a batch whenever one fills up. */
-    add(documents: AsyncIterable<TDocument>): Promise<void>;
-    /** Import any remaining documents. */
-    flush(): Promise<void>;
-    private importBuffer;
-}
-/**
- * Create-or-replace one batch of whole documents, keyed on `id`, throwing if any
- * individual document fails (Typesense’s bulk import otherwise reports
- * per-document failures without rejecting).
- */
-export declare function importBatch(client: Client, collection: string, batch: readonly {
-    id: string;
-}[]): Promise<void>;
-//# sourceMappingURL=import.d.ts.map

package/dist/import.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"import.d.ts","sourceRoot":"","sources":["../src/import.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAkB,MAAM,WAAW,CAAC;AAExD,kFAAkF;AAClF,eAAO,MAAM,kBAAkB,OAAO,CAAC;AAEvC;;;;GAIG;AACH,qBAAa,aAAa,CAAC,SAAS,SAAS;IAAE,EAAE,EAAE,MAAM,CAAA;CAAE;IAIvD,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,UAAU;IAC3B,OAAO,CAAC,QAAQ,CAAC,SAAS;IAL5B,OAAO,CAAC,MAAM,CAAmB;gBAGd,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,SAAS,EAAE,MAAM;IAGpC,0EAA0E;IACpE,GAAG,CAAC,SAAS,EAAE,aAAa,CAAC,SAAS,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAS7D,sCAAsC;IAChC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;YAMd,YAAY;CAK3B;AAED;;;;GAIG;AACH,wBAAsB,WAAW,CAC/B,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,KAAK,EAAE,SAAS;IAAE,EAAE,EAAE,MAAM,CAAA;CAAE,EAAE,GAC/B,OAAO,CAAC,IAAI,CAAC,CAkBf"}

package/dist/import.js

@@ -1,60 +0,0 @@
-/** Documents imported per Typesense request unless a writer is told otherwise. */
-export const DEFAULT_BATCH_SIZE = 1000;
-/**
- * Accumulates documents across writes and upserts them into a collection in
- * fixed-size batches, so many small writes (one dataset at a time) still land
- * as few Typesense requests. Call {@link flush} to import the remainder.
- */
-export class BatchImporter {
-    client;
-    collection;
-    batchSize;
-    buffer = [];
-    constructor(client, collection, batchSize) {
-        this.client = client;
-        this.collection = collection;
-        this.batchSize = batchSize;
-    }
-    /** Add a stream of documents, importing a batch whenever one fills up. */
-    async add(documents) {
-        for await (const document of documents) {
-            this.buffer.push(document);
-            if (this.buffer.length >= this.batchSize) {
-                await this.importBuffer();
-            }
-        }
-    }
-    /** Import any remaining documents. */
-    async flush() {
-        if (this.buffer.length > 0) {
-            await this.importBuffer();
-        }
-    }
-    async importBuffer() {
-        const batch = this.buffer;
-        this.buffer = [];
-        await importBatch(this.client, this.collection, batch);
-    }
-}
-/**
- * Create-or-replace one batch of whole documents, keyed on `id`, throwing if any
- * individual document fails (Typesense’s bulk import otherwise reports
- * per-document failures without rejecting).
- */
-export async function importBatch(client, collection, batch) {
-    const results = (await client
-        .collections(collection)
-        .documents()
-        .import(batch, {
-        action: 'upsert',
-        // Collect per-document outcomes instead of throwing the client’s opaque
-        // ImportError, so we can report which documents failed and why.
-        throwOnFail: false,
-    }));
-    const failures = results.filter((result) => !result.success);
-    if (failures.length > 0) {
-        throw new Error(`Typesense upsert into “${collection}” failed for ${failures.length}/${results.length} documents: ${failures
-            .map((failure) => failure.error)
-            .join('; ')}`);
-    }
-}

package/dist/sweep.d.ts

@@ -1,70 +0,0 @@
-import type { SearchType } from '@lde/search';
-/** The private field carrying the dataset IRI a document came from, for a type
- *  that declares none of its own – see {@link provenanceField}. */
-export declare const SOURCE_FIELD = "source";
-/** The document field carrying the id of the run that last wrote a document. */
-export declare const LAST_SEEN_FIELD = "last_seen";
-/**
- * The field a collection carries its documents’ dataset IRI in – the one the
- * membership sweep enumerates, filters and deletes by.
- *
- * A type that declares a field over the dataset
- * ({@link datasetField `from: 'dataset'`}) *is* its provenance: the writer keeps
- * its bookkeeping on that field rather than stamping a private one beside it,
- * so the facet, the label resolution, any `derive` and the sweep all read one
- * column that cannot drift. A type declaring nothing – or declaring the dataset
- * only as an *internal* field, which the projection prunes before the writer
- * ever sees it – falls back to the private {@link SOURCE_FIELD}, and behaves
- * exactly as it always has.
- */
-export declare function provenanceField(searchType: SearchType): string;
-/**
- * Sources whose documents must leave the index: indexed, but no longer part
- * of the run’s selection. Selection is membership, not processing – a dataset
- * skipped as unchanged is still selected, so its documents survive.
- *
- * @param indexedSources Source IRIs present in the collection
- * @param selectedSources Source IRIs the run’s selector produced
- */
-export declare function departedSources(indexedSources: Iterable<string>, selectedSources: Iterable<string>): string[];
-/**
- * Typesense filter matching a source’s documents that this run did not touch:
- * everything the source no longer contains, ready for a per-source sweep.
- *
- * @param sourceField The collection’s {@link provenanceField}
- * @param sourceIri The dataset IRI the documents carry in that field
- * @param runId The current run; documents it wrote carry it as `last_seen`
- */
-export declare function staleDocumentsFilter(sourceField: string, sourceIri: string, runId: string): string;
-/**
- * Typesense filter matching all of a source’s documents, ready to drop a whole
- * source (a departed source’s membership sweep, or a Blue/green writer rolling
- * a failed dataset out of its not-yet-live collection).
- *
- * @param sourceField The collection’s {@link provenanceField}
- * @param sourceIri The dataset IRI the documents carry in that field
- */
-export declare function sourceDocumentsFilter(sourceField: string, sourceIri: string): string;
-/**
- * Typesense filter matching the documents **this run** wrote for a source: the
- * inverse of {@link staleDocumentsFilter}. An In-place writer deletes these to
- * discard a source’s in-progress writes (the dump-fallback reset) without
- * touching its prior-run documents, which the success sweep or a failed run
- * still owns.
- *
- * @param sourceField The collection’s {@link provenanceField}
- * @param sourceIri The dataset IRI the documents carry in that field
- * @param runId The current run, stamped on its documents as `last_seen`
- */
-export declare function thisRunDocumentsFilter(sourceField: string, sourceIri: string, runId: string): string;
-/**
- * Typesense filters deleting every departed source’s documents, combined into
- * as few filters as fit: deletes travel in the URL query string, so each
- * filter stays under a conservative length budget rather than listing every
- * source in one string.
- *
- * @param sourceField The collection’s {@link provenanceField}
- * @param departed The departed source IRIs ({@link departedSources})
- */
-export declare function membershipSweepFilters(sourceField: string, departed: readonly string[]): string[];
-//# sourceMappingURL=sweep.d.ts.map

package/dist/sweep.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"sweep.d.ts","sourceRoot":"","sources":["../src/sweep.ts"],"names":[],"mappings":"AAQA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAG9C;mEACmE;AACnE,eAAO,MAAM,YAAY,WAAW,CAAC;AAErC,gFAAgF;AAChF,eAAO,MAAM,eAAe,cAAc,CAAC;AAE3C;;;;;;;;;;;;GAYG;AACH,wBAAgB,eAAe,CAAC,UAAU,EAAE,UAAU,GAAG,MAAM,CAK9D;AAED;;;;;;;GAOG;AACH,wBAAgB,eAAe,CAC7B,cAAc,EAAE,QAAQ,CAAC,MAAM,CAAC,EAChC,eAAe,EAAE,QAAQ,CAAC,MAAM,CAAC,GAChC,MAAM,EAAE,CAGV;AAED;;;;;;;GAOG;AACH,wBAAgB,oBAAoB,CAClC,WAAW,EAAE,MAAM,EACnB,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,MAAM,GACZ,MAAM,CAER;AAED;;;;;;;GAOG;AACH,wBAAgB,qBAAqB,CACnC,WAAW,EAAE,MAAM,EACnB,SAAS,EAAE,MAAM,GAChB,MAAM,CAER;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,sBAAsB,CACpC,WAAW,EAAE,MAAM,EACnB,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,MAAM,GACZ,MAAM,CAER;AAED;;;;;;;;GAQG;AACH,wBAAgB,sBAAsB,CACpC,WAAW,EAAE,MAAM,EACnB,QAAQ,EAAE,SAAS,MAAM,EAAE,GAC1B,MAAM,EAAE,CAqBV"}